diff --git a/.github/workflows/backend-test.yml b/.github/workflows/backend-test.yml index 940eb34b1f..b6fd9066cf 100644 --- a/.github/workflows/backend-test.yml +++ b/.github/workflows/backend-test.yml @@ -83,6 +83,7 @@ jobs: RUST_LOG: info RUST_LOG_STYLE: never CARGO_NET_GIT_FETCH_WITH_CLI: true + WMDEBUG_FORCE_V0_WORKSPACE_DEPENDENCIES: 1 run: | deno --version && bun -v && go version && python3 --version cd windmill-duckdb-ffi-internal && ./build_dev.sh && cd .. diff --git a/Dockerfile b/Dockerfile index 7aeb2b218a..fe27c7a022 100644 --- a/Dockerfile +++ b/Dockerfile @@ -100,6 +100,7 @@ ARG POWERSHELL_VERSION=7.5.0 ARG POWERSHELL_DEB_VERSION=7.5.0-1 ARG KUBECTL_VERSION=1.28.7 ARG HELM_VERSION=3.14.3 +# NOTE: If changing, also change go version in workspace dependencies template at WorkspaceDependenciesEditor.svelte ARG GO_VERSION=1.25.0 ARG APP=/usr/src/app ARG WITH_POWERSHELL=true diff --git a/backend/.sqlx/query-04ce5c530c80ae6f911dfe0dc9ed7d1a2e10342bbbc7f8486df0b73f5657a493.json b/backend/.sqlx/query-04ce5c530c80ae6f911dfe0dc9ed7d1a2e10342bbbc7f8486df0b73f5657a493.json new file mode 100644 index 0000000000..2878e54920 --- /dev/null +++ b/backend/.sqlx/query-04ce5c530c80ae6f911dfe0dc9ed7d1a2e10342bbbc7f8486df0b73f5657a493.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COUNT(*) FROM v2_job", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null + ] + }, + "hash": "04ce5c530c80ae6f911dfe0dc9ed7d1a2e10342bbbc7f8486df0b73f5657a493" +} diff --git a/backend/.sqlx/query-05bbdf192c51cd75552674c7db209cad66016cc112b58eb943f038308090ec5c.json b/backend/.sqlx/query-05bbdf192c51cd75552674c7db209cad66016cc112b58eb943f038308090ec5c.json new file mode 100644 index 0000000000..3977f2c356 --- /dev/null +++ b/backend/.sqlx/query-05bbdf192c51cd75552674c7db209cad66016cc112b58eb943f038308090ec5c.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO workspace_dependencies (workspace_id, language, name, description, content, archived, created_at)\n SELECT $1, language, name, description, content, archived, created_at\n FROM workspace_dependencies \n WHERE workspace_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "05bbdf192c51cd75552674c7db209cad66016cc112b58eb943f038308090ec5c" +} diff --git a/backend/.sqlx/query-0bcbed8d2a7ad88b809a211a8c13a3d74b8e8141be95cbcd63e227d13091a8dd.json b/backend/.sqlx/query-0bcbed8d2a7ad88b809a211a8c13a3d74b8e8141be95cbcd63e227d13091a8dd.json deleted file mode 100644 index fa2b39ff0b..0000000000 --- a/backend/.sqlx/query-0bcbed8d2a7ad88b809a211a8c13a3d74b8e8141be95cbcd63e227d13091a8dd.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT path, hash FROM script WHERE workspace_id = $1 AND archived = false AND deleted = false", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "path", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "hash", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [ - false, - false - ] - }, - "hash": "0bcbed8d2a7ad88b809a211a8c13a3d74b8e8141be95cbcd63e227d13091a8dd" -} diff --git a/backend/.sqlx/query-0bfd22be1d6966c61c9a5fedc2522be8a17f06391b882337c74c1817c99b533d.json b/backend/.sqlx/query-0bfd22be1d6966c61c9a5fedc2522be8a17f06391b882337c74c1817c99b533d.json deleted file mode 100644 index 38bc687e7e..0000000000 --- a/backend/.sqlx/query-0bfd22be1d6966c61c9a5fedc2522be8a17f06391b882337c74c1817c99b533d.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT name FROM folder WHERE workspace_id = $1 ORDER BY name desc LIMIT $2 OFFSET $3", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "name", - "type_info": "Varchar" - } - ], - "parameters": { - "Left": [ - "Text", - "Int8", - "Int8" - ] - }, - "nullable": [ - false - ] - }, - "hash": "0bfd22be1d6966c61c9a5fedc2522be8a17f06391b882337c74c1817c99b533d" -} diff --git a/backend/.sqlx/query-0f58d4e7e6f3e962e8a86a2d9feb921c308b6a047530eccd8966513ffdc722d0.json b/backend/.sqlx/query-0f58d4e7e6f3e962e8a86a2d9feb921c308b6a047530eccd8966513ffdc722d0.json new file mode 100644 index 0000000000..f7bb120f53 --- /dev/null +++ b/backend/.sqlx/query-0f58d4e7e6f3e962e8a86a2d9feb921c308b6a047530eccd8966513ffdc722d0.json @@ -0,0 +1,47 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE workspace_dependencies\n SET archived = true\n WHERE name IS NOT DISTINCT FROM $1 AND workspace_id = $2 AND archived = false AND language = $3\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + { + "Custom": { + "name": "script_lang", + "kind": { + "Enum": [ + "python3", + "deno", + "go", + "bash", + "postgresql", + "nativets", + "bun", + "mysql", + "bigquery", + "snowflake", + "graphql", + "powershell", + "mssql", + "php", + "bunnative", + "rust", + "ansible", + "csharp", + "oracledb", + "nu", + "java", + "duckdb", + "ruby" + ] + } + } + } + ] + }, + "nullable": [] + }, + "hash": "0f58d4e7e6f3e962e8a86a2d9feb921c308b6a047530eccd8966513ffdc722d0" +} diff --git a/backend/.sqlx/query-1e285da98ac08999f0ad489f1b81885d682de0d7c6ff3e09a9fddef8bb682708.json b/backend/.sqlx/query-1e285da98ac08999f0ad489f1b81885d682de0d7c6ff3e09a9fddef8bb682708.json new file mode 100644 index 0000000000..ed3665b584 --- /dev/null +++ b/backend/.sqlx/query-1e285da98ac08999f0ad489f1b81885d682de0d7c6ff3e09a9fddef8bb682708.json @@ -0,0 +1,35 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT \n importer_path,\n importer_kind::text as \"importer_kind!\", -- sqlx thinks this is nullable somehow, so enfore with !\n array_agg(importer_node_id) as importer_node_ids\n FROM dependency_map \n WHERE workspace_id = $1 AND imported_path = $2\n GROUP BY importer_path, importer_kind\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "importer_path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "importer_kind!", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "importer_node_ids", + "type_info": "VarcharArray" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + null, + null + ] + }, + "hash": "1e285da98ac08999f0ad489f1b81885d682de0d7c6ff3e09a9fddef8bb682708" +} diff --git a/backend/.sqlx/query-1f1e477b27f38f410b7e6f436ced0751f77f9cb41055481bc06af6827d647041.json b/backend/.sqlx/query-1f1e477b27f38f410b7e6f436ced0751f77f9cb41055481bc06af6827d647041.json new file mode 100644 index 0000000000..cdcb277b24 --- /dev/null +++ b/backend/.sqlx/query-1f1e477b27f38f410b7e6f436ced0751f77f9cb41055481bc06af6827d647041.json @@ -0,0 +1,96 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT id, content, language AS \"language: ScriptLang\", name, archived, description, workspace_id, created_at\n FROM workspace_dependencies\n WHERE id = $1 AND workspace_id = $2\n LIMIT 1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "content", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "language: ScriptLang", + "type_info": { + "Custom": { + "name": "script_lang", + "kind": { + "Enum": [ + "python3", + "deno", + "go", + "bash", + "postgresql", + "nativets", + "bun", + "mysql", + "bigquery", + "snowflake", + "graphql", + "powershell", + "mssql", + "php", + "bunnative", + "rust", + "ansible", + "csharp", + "oracledb", + "nu", + "java", + "duckdb", + "ruby" + ] + } + } + } + }, + { + "ordinal": 3, + "name": "name", + "type_info": "Varchar" + }, + { + "ordinal": 4, + "name": "archived", + "type_info": "Bool" + }, + { + "ordinal": 5, + "name": "description", + "type_info": "Text" + }, + { + "ordinal": 6, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 7, + "name": "created_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Int8", + "Text" + ] + }, + "nullable": [ + false, + false, + false, + true, + false, + false, + false, + false + ] + }, + "hash": "1f1e477b27f38f410b7e6f436ced0751f77f9cb41055481bc06af6827d647041" +} diff --git a/backend/.sqlx/query-37409e147cf69c39dad848b117bdb77654c167a254053bfd3682c7d9add30b6b.json b/backend/.sqlx/query-37409e147cf69c39dad848b117bdb77654c167a254053bfd3682c7d9add30b6b.json new file mode 100644 index 0000000000..43d54b5a1c --- /dev/null +++ b/backend/.sqlx/query-37409e147cf69c39dad848b117bdb77654c167a254053bfd3682c7d9add30b6b.json @@ -0,0 +1,55 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE workspace_dependencies\n SET archived = true \n WHERE archived = false\n AND name IS NOT DISTINCT FROM $1\n AND workspace_id = $2\n AND language = $3\n RETURNING description\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "description", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + { + "Custom": { + "name": "script_lang", + "kind": { + "Enum": [ + "python3", + "deno", + "go", + "bash", + "postgresql", + "nativets", + "bun", + "mysql", + "bigquery", + "snowflake", + "graphql", + "powershell", + "mssql", + "php", + "bunnative", + "rust", + "ansible", + "csharp", + "oracledb", + "nu", + "java", + "duckdb", + "ruby" + ] + } + } + } + ] + }, + "nullable": [ + false + ] + }, + "hash": "37409e147cf69c39dad848b117bdb77654c167a254053bfd3682c7d9add30b6b" +} diff --git a/backend/.sqlx/query-5369258383098062e454539be369ea2f0a41771376720fb72c6b3c1fa4972eab.json b/backend/.sqlx/query-5369258383098062e454539be369ea2f0a41771376720fb72c6b3c1fa4972eab.json new file mode 100644 index 0000000000..6bf181d09b --- /dev/null +++ b/backend/.sqlx/query-5369258383098062e454539be369ea2f0a41771376720fb72c6b3c1fa4972eab.json @@ -0,0 +1,47 @@ +{ + "db_name": "PostgreSQL", + "query": "\n DELETE\n FROM workspace_dependencies\n WHERE name IS NOT DISTINCT FROM $1 AND workspace_id = $2 AND language = $3\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + { + "Custom": { + "name": "script_lang", + "kind": { + "Enum": [ + "python3", + "deno", + "go", + "bash", + "postgresql", + "nativets", + "bun", + "mysql", + "bigquery", + "snowflake", + "graphql", + "powershell", + "mssql", + "php", + "bunnative", + "rust", + "ansible", + "csharp", + "oracledb", + "nu", + "java", + "duckdb", + "ruby" + ] + } + } + } + ] + }, + "nullable": [] + }, + "hash": "5369258383098062e454539be369ea2f0a41771376720fb72c6b3c1fa4972eab" +} diff --git a/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json b/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json index 713ccb9dd3..36ddb8ab9f 100644 --- a/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json +++ b/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json @@ -15,7 +15,7 @@ ] }, "nullable": [ - null + true ] }, "hash": "5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55" diff --git a/backend/.sqlx/query-5bec60e207a5933aa301f87c0afaeaa8e9a3a2c64e23ab42100d48039ba422b0.json b/backend/.sqlx/query-5bec60e207a5933aa301f87c0afaeaa8e9a3a2c64e23ab42100d48039ba422b0.json new file mode 100644 index 0000000000..2ce0b8015a --- /dev/null +++ b/backend/.sqlx/query-5bec60e207a5933aa301f87c0afaeaa8e9a3a2c64e23ab42100d48039ba422b0.json @@ -0,0 +1,57 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO workspace_dependencies(name, workspace_id, content, language, description)\n VALUES ($1, $2, $3, $4, $5) \n RETURNING id\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Text", + { + "Custom": { + "name": "script_lang", + "kind": { + "Enum": [ + "python3", + "deno", + "go", + "bash", + "postgresql", + "nativets", + "bun", + "mysql", + "bigquery", + "snowflake", + "graphql", + "powershell", + "mssql", + "php", + "bunnative", + "rust", + "ansible", + "csharp", + "oracledb", + "nu", + "java", + "duckdb", + "ruby" + ] + } + } + }, + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "5bec60e207a5933aa301f87c0afaeaa8e9a3a2c64e23ab42100d48039ba422b0" +} diff --git a/backend/.sqlx/query-5ec262094e8ddf420b9098f2b3ef6d2c2caa94f39a0e57fe015e19fd1d0ab0ea.json b/backend/.sqlx/query-5ec262094e8ddf420b9098f2b3ef6d2c2caa94f39a0e57fe015e19fd1d0ab0ea.json new file mode 100644 index 0000000000..e48f00924b --- /dev/null +++ b/backend/.sqlx/query-5ec262094e8ddf420b9098f2b3ef6d2c2caa94f39a0e57fe015e19fd1d0ab0ea.json @@ -0,0 +1,65 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT path, hash, language AS \"language: ScriptLang\" FROM script WHERE workspace_id = $1 AND archived = false AND deleted = false", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "hash", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "language: ScriptLang", + "type_info": { + "Custom": { + "name": "script_lang", + "kind": { + "Enum": [ + "python3", + "deno", + "go", + "bash", + "postgresql", + "nativets", + "bun", + "mysql", + "bigquery", + "snowflake", + "graphql", + "powershell", + "mssql", + "php", + "bunnative", + "rust", + "ansible", + "csharp", + "oracledb", + "nu", + "java", + "duckdb", + "ruby" + ] + } + } + } + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false, + false + ] + }, + "hash": "5ec262094e8ddf420b9098f2b3ef6d2c2caa94f39a0e57fe015e19fd1d0ab0ea" +} diff --git a/backend/.sqlx/query-9c6d44ffae63b4050ef3a66cb05d5a4ee50dea4b30093df2e0ceb684201ca4b0.json b/backend/.sqlx/query-9c6d44ffae63b4050ef3a66cb05d5a4ee50dea4b30093df2e0ceb684201ca4b0.json deleted file mode 100644 index 8c2c014662..0000000000 --- a/backend/.sqlx/query-9c6d44ffae63b4050ef3a66cb05d5a4ee50dea4b30093df2e0ceb684201ca4b0.json +++ /dev/null @@ -1,66 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT workspace_id, name, display_name, owners, extra_perms, summary, created_by, edited_at FROM folder WHERE workspace_id = $1 ORDER BY name desc LIMIT $2 OFFSET $3", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "workspace_id", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "name", - "type_info": "Varchar" - }, - { - "ordinal": 2, - "name": "display_name", - "type_info": "Varchar" - }, - { - "ordinal": 3, - "name": "owners", - "type_info": "VarcharArray" - }, - { - "ordinal": 4, - "name": "extra_perms", - "type_info": "Jsonb" - }, - { - "ordinal": 5, - "name": "summary", - "type_info": "Text" - }, - { - "ordinal": 6, - "name": "created_by", - "type_info": "Varchar" - }, - { - "ordinal": 7, - "name": "edited_at", - "type_info": "Timestamptz" - } - ], - "parameters": { - "Left": [ - "Text", - "Int8", - "Int8" - ] - }, - "nullable": [ - false, - false, - false, - false, - false, - true, - true, - true - ] - }, - "hash": "9c6d44ffae63b4050ef3a66cb05d5a4ee50dea4b30093df2e0ceb684201ca4b0" -} diff --git a/backend/.sqlx/query-a264bbd8dbabb03854bd25350a7aeda0704770eb200bae635f1933eece90c9d6.json b/backend/.sqlx/query-a264bbd8dbabb03854bd25350a7aeda0704770eb200bae635f1933eece90c9d6.json new file mode 100644 index 0000000000..1030965135 --- /dev/null +++ b/backend/.sqlx/query-a264bbd8dbabb03854bd25350a7aeda0704770eb200bae635f1933eece90c9d6.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS(SELECT 1 FROM app WHERE path = 'g/all/setup_app')", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null + ] + }, + "hash": "a264bbd8dbabb03854bd25350a7aeda0704770eb200bae635f1933eece90c9d6" +} diff --git a/backend/.sqlx/query-abc9f034e62ac224894173356aa69e09f3647a45d176e253e4fc8f7206f6a18d.json b/backend/.sqlx/query-abc9f034e62ac224894173356aa69e09f3647a45d176e253e4fc8f7206f6a18d.json deleted file mode 100644 index 395816d916..0000000000 --- a/backend/.sqlx/query-abc9f034e62ac224894173356aa69e09f3647a45d176e253e4fc8f7206f6a18d.json +++ /dev/null @@ -1,42 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT * FROM group_ WHERE workspace_id = $1 ORDER BY name desc LIMIT $2 OFFSET $3", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "workspace_id", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "name", - "type_info": "Varchar" - }, - { - "ordinal": 2, - "name": "summary", - "type_info": "Text" - }, - { - "ordinal": 3, - "name": "extra_perms", - "type_info": "Jsonb" - } - ], - "parameters": { - "Left": [ - "Text", - "Int8", - "Int8" - ] - }, - "nullable": [ - false, - false, - true, - false - ] - }, - "hash": "abc9f034e62ac224894173356aa69e09f3647a45d176e253e4fc8f7206f6a18d" -} diff --git a/backend/.sqlx/query-bdb53068a223c12c2d317635993e8fda531e69a880d0ebc44fb5f13ac976b67d.json b/backend/.sqlx/query-bdb53068a223c12c2d317635993e8fda531e69a880d0ebc44fb5f13ac976b67d.json new file mode 100644 index 0000000000..c835fd2b7b --- /dev/null +++ b/backend/.sqlx/query-bdb53068a223c12c2d317635993e8fda531e69a880d0ebc44fb5f13ac976b67d.json @@ -0,0 +1,55 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT id FROM workspace_dependencies\n WHERE name IS NOT DISTINCT FROM $1 AND workspace_id = $2 AND language = $3\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + { + "Custom": { + "name": "script_lang", + "kind": { + "Enum": [ + "python3", + "deno", + "go", + "bash", + "postgresql", + "nativets", + "bun", + "mysql", + "bigquery", + "snowflake", + "graphql", + "powershell", + "mssql", + "php", + "bunnative", + "rust", + "ansible", + "csharp", + "oracledb", + "nu", + "java", + "duckdb", + "ruby" + ] + } + } + } + ] + }, + "nullable": [ + false + ] + }, + "hash": "bdb53068a223c12c2d317635993e8fda531e69a880d0ebc44fb5f13ac976b67d" +} diff --git a/backend/.sqlx/query-c6637102979d1acaf7fb76ff8e51badc6df20086769ffd9f9ee9e6bf1527810c.json b/backend/.sqlx/query-c6637102979d1acaf7fb76ff8e51badc6df20086769ffd9f9ee9e6bf1527810c.json new file mode 100644 index 0000000000..b24e503df9 --- /dev/null +++ b/backend/.sqlx/query-c6637102979d1acaf7fb76ff8e51badc6df20086769ffd9f9ee9e6bf1527810c.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT \n imported_path,\n COUNT(DISTINCT importer_path) as \"count!\"\n FROM dependency_map \n WHERE workspace_id = $1 AND imported_path = ANY($2)\n GROUP BY imported_path\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "imported_path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "count!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "TextArray" + ] + }, + "nullable": [ + false, + null + ] + }, + "hash": "c6637102979d1acaf7fb76ff8e51badc6df20086769ffd9f9ee9e6bf1527810c" +} diff --git a/backend/.sqlx/query-d22f81df67d55b2d8f1d15404194a7b4604171d12c6a4620514c6d030086936d.json b/backend/.sqlx/query-d22f81df67d55b2d8f1d15404194a7b4604171d12c6a4620514c6d030086936d.json new file mode 100644 index 0000000000..4ff6142871 --- /dev/null +++ b/backend/.sqlx/query-d22f81df67d55b2d8f1d15404194a7b4604171d12c6a4620514c6d030086936d.json @@ -0,0 +1,95 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT id, created_at, archived, name, description, workspace_id, content, language AS \"language: ScriptLang\"\n FROM workspace_dependencies\n WHERE archived = false AND workspace_id = $1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 2, + "name": "archived", + "type_info": "Bool" + }, + { + "ordinal": 3, + "name": "name", + "type_info": "Varchar" + }, + { + "ordinal": 4, + "name": "description", + "type_info": "Text" + }, + { + "ordinal": 5, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 6, + "name": "content", + "type_info": "Text" + }, + { + "ordinal": 7, + "name": "language: ScriptLang", + "type_info": { + "Custom": { + "name": "script_lang", + "kind": { + "Enum": [ + "python3", + "deno", + "go", + "bash", + "postgresql", + "nativets", + "bun", + "mysql", + "bigquery", + "snowflake", + "graphql", + "powershell", + "mssql", + "php", + "bunnative", + "rust", + "ansible", + "csharp", + "oracledb", + "nu", + "java", + "duckdb", + "ruby" + ] + } + } + } + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false, + false, + true, + false, + false, + false, + false + ] + }, + "hash": "d22f81df67d55b2d8f1d15404194a7b4604171d12c6a4620514c6d030086936d" +} diff --git a/backend/.sqlx/query-d814833e31b3b3657c57dde1c8cd21896d6cb8256fe05dd5b0fecb53782956ce.json b/backend/.sqlx/query-d814833e31b3b3657c57dde1c8cd21896d6cb8256fe05dd5b0fecb53782956ce.json deleted file mode 100644 index 50d30ab09a..0000000000 --- a/backend/.sqlx/query-d814833e31b3b3657c57dde1c8cd21896d6cb8256fe05dd5b0fecb53782956ce.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT name FROM group_ WHERE workspace_id = $1 UNION SELECT name FROM instance_group ORDER BY name desc", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "name", - "type_info": "Varchar" - } - ], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "d814833e31b3b3657c57dde1c8cd21896d6cb8256fe05dd5b0fecb53782956ce" -} diff --git a/backend/.sqlx/query-e7bc612ccdbb2532a321ed83e26a0854975ad4c3f6fe24557b87a197485dff39.json b/backend/.sqlx/query-e7bc612ccdbb2532a321ed83e26a0854975ad4c3f6fe24557b87a197485dff39.json new file mode 100644 index 0000000000..8e8a5c98e9 --- /dev/null +++ b/backend/.sqlx/query-e7bc612ccdbb2532a321ed83e26a0854975ad4c3f6fe24557b87a197485dff39.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COUNT(*) FROM v2_job_queue", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null + ] + }, + "hash": "e7bc612ccdbb2532a321ed83e26a0854975ad4c3f6fe24557b87a197485dff39" +} diff --git a/backend/.sqlx/query-ec3359bbc309c2b893e9f68c09c9f61501068f4b027f58279e0c8508839607f4.json b/backend/.sqlx/query-ec3359bbc309c2b893e9f68c09c9f61501068f4b027f58279e0c8508839607f4.json deleted file mode 100644 index b2fa44659b..0000000000 --- a/backend/.sqlx/query-ec3359bbc309c2b893e9f68c09c9f61501068f4b027f58279e0c8508839607f4.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT importer_path, importer_kind::text, array_agg(importer_node_id) as importer_node_ids FROM dependency_map\n WHERE imported_path = $1\n AND workspace_id = $2\n GROUP BY importer_path, importer_kind", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "importer_path", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "importer_kind", - "type_info": "Text" - }, - { - "ordinal": 2, - "name": "importer_node_ids", - "type_info": "VarcharArray" - } - ], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [ - false, - null, - null - ] - }, - "hash": "ec3359bbc309c2b893e9f68c09c9f61501068f4b027f58279e0c8508839607f4" -} diff --git a/backend/.sqlx/query-f0e7d28be69c4b922b34b76abb780766003d79884d33f94a79e61f0259807dbc.json b/backend/.sqlx/query-f0e7d28be69c4b922b34b76abb780766003d79884d33f94a79e61f0259807dbc.json new file mode 100644 index 0000000000..ab4421f9b4 --- /dev/null +++ b/backend/.sqlx/query-f0e7d28be69c4b922b34b76abb780766003d79884d33f94a79e61f0259807dbc.json @@ -0,0 +1,128 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT id, content, language AS \"language: ScriptLang\", name, description, archived, workspace_id, created_at\n FROM workspace_dependencies\n WHERE name IS NOT DISTINCT FROM $1 AND workspace_id = $2 AND archived = false AND language = $3\n LIMIT 1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "content", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "language: ScriptLang", + "type_info": { + "Custom": { + "name": "script_lang", + "kind": { + "Enum": [ + "python3", + "deno", + "go", + "bash", + "postgresql", + "nativets", + "bun", + "mysql", + "bigquery", + "snowflake", + "graphql", + "powershell", + "mssql", + "php", + "bunnative", + "rust", + "ansible", + "csharp", + "oracledb", + "nu", + "java", + "duckdb", + "ruby" + ] + } + } + } + }, + { + "ordinal": 3, + "name": "name", + "type_info": "Varchar" + }, + { + "ordinal": 4, + "name": "description", + "type_info": "Text" + }, + { + "ordinal": 5, + "name": "archived", + "type_info": "Bool" + }, + { + "ordinal": 6, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 7, + "name": "created_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + { + "Custom": { + "name": "script_lang", + "kind": { + "Enum": [ + "python3", + "deno", + "go", + "bash", + "postgresql", + "nativets", + "bun", + "mysql", + "bigquery", + "snowflake", + "graphql", + "powershell", + "mssql", + "php", + "bunnative", + "rust", + "ansible", + "csharp", + "oracledb", + "nu", + "java", + "duckdb", + "ruby" + ] + } + } + } + ] + }, + "nullable": [ + false, + false, + false, + true, + false, + false, + false, + false + ] + }, + "hash": "f0e7d28be69c4b922b34b76abb780766003d79884d33f94a79e61f0259807dbc" +} diff --git a/backend/.sqlx/query-f4af0affaed3b1d30f5c6f4ddf4114718468fbf75ef13c6087c2f57fbbe0b82f.json b/backend/.sqlx/query-f4af0affaed3b1d30f5c6f4ddf4114718468fbf75ef13c6087c2f57fbbe0b82f.json new file mode 100644 index 0000000000..45fea6bdda --- /dev/null +++ b/backend/.sqlx/query-f4af0affaed3b1d30f5c6f4ddf4114718468fbf75ef13c6087c2f57fbbe0b82f.json @@ -0,0 +1,51 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT language AS \"language: ScriptLang\" FROM script WHERE path = 'u/admin/hub_sync'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "language: ScriptLang", + "type_info": { + "Custom": { + "name": "script_lang", + "kind": { + "Enum": [ + "python3", + "deno", + "go", + "bash", + "postgresql", + "nativets", + "bun", + "mysql", + "bigquery", + "snowflake", + "graphql", + "powershell", + "mssql", + "php", + "bunnative", + "rust", + "ansible", + "csharp", + "oracledb", + "nu", + "java", + "duckdb", + "ruby" + ] + } + } + } + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false + ] + }, + "hash": "f4af0affaed3b1d30f5c6f4ddf4114718468fbf75ef13c6087c2f57fbbe0b82f" +} diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 41334d27f5..06f4a1023e 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -15193,6 +15193,7 @@ dependencies = [ "reqwest 0.12.24", "rustls 0.23.35", "serde", + "serde_derive", "serde_json", "serde_yml", "sha1", @@ -15438,6 +15439,8 @@ dependencies = [ "opentelemetry-otlp", "opentelemetry-semantic-conventions", "opentelemetry_sdk", + "pep440_rs", + "phf 0.11.3", "pin-project-lite", "prometheus", "quick_cache", @@ -15450,6 +15453,7 @@ dependencies = [ "semver 1.0.27", "serde", "serde_json", + "serde_yml", "sha2 0.10.9", "size", "sqlx", @@ -15525,9 +15529,13 @@ version = "1.586.0" dependencies = [ "itertools 0.14.0", "lazy_static", + "pep440_rs", "proc-macro2", "quote", "regex", + "serde", + "serde_derive", + "serde_yml", "syn 2.0.111", ] @@ -15653,6 +15661,7 @@ dependencies = [ "serde_json", "sqlx", "toml", + "tracing", "windmill-common", "windmill-parser", ] diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 38d478fc7a..fddebdfaad 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -137,6 +137,7 @@ prometheus = { workspace = true, optional = true } uuid.workspace = true gethostname.workspace = true serde_json.workspace = true +serde_derive.workspace = true serde_yml.workspace = true serde.workspace = true deno_core = { workspace = true, optional = true } diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 2a3de2d939..8dce612c3c 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -99828e4a3a27af428a45d1e6e65092ca15082b1e +88d5023df7f41bb86e948b35889f962b741c860e diff --git a/backend/migrations/20251106152104_raw_requirements.down.sql b/backend/migrations/20251106152104_raw_requirements.down.sql new file mode 100644 index 0000000000..0b900d1937 --- /dev/null +++ b/backend/migrations/20251106152104_raw_requirements.down.sql @@ -0,0 +1,10 @@ +-- Drop indexes first (though dropping table will cascade) +DROP INDEX IF EXISTS workspace_dependencies_id_workspace ; +DROP INDEX IF EXISTS workspace_dependencies_workspace_lang_name_archived_idx ; +DROP INDEX IF EXISTS workspace_dependencies_workspace_archived_idx; +DROP INDEX IF EXISTS one_non_archived_per_null_name_language_constraint; +DROP INDEX IF EXISTS one_non_archived_per_name_language_constraint; + +-- Drop table and sequence +DROP TABLE IF EXISTS workspace_dependencies; +DROP SEQUENCE IF EXISTS workspace_dependencies_id_seq CASCADE; diff --git a/backend/migrations/20251106152104_raw_requirements.up.sql b/backend/migrations/20251106152104_raw_requirements.up.sql new file mode 100644 index 0000000000..b390cce8e2 --- /dev/null +++ b/backend/migrations/20251106152104_raw_requirements.up.sql @@ -0,0 +1,23 @@ +CREATE SEQUENCE IF NOT EXISTS workspace_dependencies_id_seq; + +CREATE TABLE IF NOT EXISTS workspace_dependencies( + id BIGINT DEFAULT nextval('workspace_dependencies_id_seq') PRIMARY KEY, + name VARCHAR(255), -- If NULL - it's global + content TEXT NOT NULL, + language SCRIPT_LANG NOT NULL, + description text NOT NULL DEFAULT '', + archived BOOLEAN NOT NULL DEFAULT false, + workspace_id character varying(50) NOT NULL, + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now() +); + +-- Make any query that tries to create non-linear history fail +CREATE UNIQUE INDEX IF NOT EXISTS one_non_archived_per_name_language_constraint ON workspace_dependencies(name, language, workspace_id) WHERE archived = false AND name IS NOT NULL; +CREATE UNIQUE INDEX IF NOT EXISTS one_non_archived_per_null_name_language_constraint ON workspace_dependencies(language, workspace_id) WHERE archived = false AND name IS NULL; + +-- Performance indexes for common query patterns +-- For the list query (filtering by workspace_id and archived) +CREATE INDEX IF NOT EXISTS workspace_dependencies_workspace_archived_idx ON workspace_dependencies(workspace_id, archived) WHERE archived = false; + +CREATE INDEX IF NOT EXISTS workspace_dependencies_workspace_lang_name_archived_idx ON workspace_dependencies(workspace_id, language, name, archived); +CREATE INDEX IF NOT EXISTS workspace_dependencies_id_workspace ON workspace_dependencies(id, workspace_id); diff --git a/backend/parsers/windmill-parser-py-imports/Cargo.toml b/backend/parsers/windmill-parser-py-imports/Cargo.toml index abd363b42b..deea68c2ef 100644 --- a/backend/parsers/windmill-parser-py-imports/Cargo.toml +++ b/backend/parsers/windmill-parser-py-imports/Cargo.toml @@ -30,3 +30,4 @@ async-recursion.workspace = true toml.workspace = true serde.workspace = true pep440_rs.workspace = true +tracing.workspace = true diff --git a/backend/parsers/windmill-parser-py-imports/src/lib.rs b/backend/parsers/windmill-parser-py-imports/src/lib.rs index b024ba6244..06762dbc9e 100644 --- a/backend/parsers/windmill-parser-py-imports/src/lib.rs +++ b/backend/parsers/windmill-parser-py-imports/src/lib.rs @@ -27,7 +27,11 @@ use rustpython_parser::{ use sqlx::{Pool, Postgres}; use windmill_common::{ error::{self, to_anyhow}, - worker::PythonAnnotations, + worker::{ + split_python_requirements, try_parse_locked_python_version_from_requirements, + PythonAnnotations, + }, + workspace_dependencies::{RawWorkspaceDependencies, WorkspaceDependenciesPrefetched}, }; fn replace_import(x: String) -> String { @@ -170,7 +174,6 @@ fn parse_code_for_imports(code: &str, path: &str) -> error::Result> // This is needed because we've split off the real main function above let code_with_fake_main = format!("{}\n\ndef main(): pass", code); - let ast = Suite::parse(&code_with_fake_main, "main.py").map_err(|e| { error::Error::ExecutionErr(format!("Error parsing code for imports: {}", e.to_string())) })?; @@ -259,6 +262,8 @@ pub async fn parse_python_imports( path: &str, db: &Pool, version_specifiers: &mut Vec, + locked_v: &mut Option, + raw_workspace_dependencies_o: &Option, ) -> error::Result<(Vec, Option)> { let mut compile_error_hint: Option = None; let mut imports = parse_python_imports_inner( @@ -268,8 +273,9 @@ pub async fn parse_python_imports( db, &mut vec![], version_specifiers, - // &mut version_specifier.and_then(|_| Some(path.to_owned())), - &mut None + &mut None, + locked_v, + raw_workspace_dependencies_o, ) .await? .into_values() @@ -323,11 +329,22 @@ async fn parse_python_imports_inner( already_visited: &mut Vec, version_specifiers: &mut Vec, path_where_annotated_pyv: &mut Option, + locked_v: &mut Option, + raw_workspace_dependencies_o: &Option, ) -> error::Result> { + tracing::debug!("Parsing python imports for path: {}", path); let PythonAnnotations { py310, py311, py312, py313, .. } = PythonAnnotations::parse(&code); + tracing::debug!( + "Found python annotations - py310: {}, py311: {}, py312: {}, py313: {}", + py310, + py311, + py312, + py313 + ); let mut push_version_specifiers = |perform, unparsed: String| -> error::Result<()> { if perform { + tracing::debug!("Adding version specifier: {}", unparsed); pep440_rs::VersionSpecifiers::from_str(unparsed.as_str()) .ok() .map(|vs| version_specifiers.extend(vs.to_vec())); @@ -341,10 +358,9 @@ async fn parse_python_imports_inner( for x in code.lines() { if x.starts_with("# py:") || x.starts_with("#py:") { - push_version_specifiers( - true, - x.replace('#', "").replace("py:", "").trim().to_owned(), - )?; + let version_spec = x.replace('#', "").replace("py:", "").trim().to_owned(); + tracing::debug!("Found inline python version specifier: {}", version_spec); + push_version_specifiers(true, version_spec)?; } else if !x.starts_with('#') { break; } @@ -369,12 +385,40 @@ async fn parse_python_imports_inner( // dependencies: Vec, // } - let find_requirements = code.lines().find_position(|x| { - x.starts_with("#requirements:") - || x.starts_with("# requirements:") - || x.starts_with("# /// script") - }); + let mut final_imports = HashMap::new(); + tracing::debug!("Extracting workspace dependencies for workspace: {}", w_id); + let wdp = WorkspaceDependenciesPrefetched::extract( + code, + windmill_common::scripts::ScriptLang::Python3, + w_id, + raw_workspace_dependencies_o, + path, + db.into(), + ) + .await?; + + if let Some(c) = wdp.get_python()? { + *locked_v = extract_nimports_from_content(&c, &mut final_imports); + } + + if wdp.is_manual() { + tracing::debug!( + "Workspace dependencies mode is Manual, returning {} imports", + final_imports.len() + ); + return Ok(final_imports); + } + + let find_requirements = code + .lines() + .find_position(|x| x.starts_with("# /// script")); + tracing::debug!( + "Looking for script metadata block, found: {}", + find_requirements.is_some() + ); + if let Some((pos, item)) = find_requirements { + tracing::debug!("Found script metadata block at position {}: {}", pos, item); let mut requirements = HashMap::new(); if item.starts_with("# /// script") { let mut incorrect = false; @@ -384,7 +428,7 @@ async fn parse_python_imports_inner( .map_while(|x| { incorrect = !x.starts_with('#'); if incorrect || x.starts_with("# ///") { - None + Option::None } else { x.get(1..) } @@ -392,9 +436,11 @@ async fn parse_python_imports_inner( .join("\n") .parse::() .map_err(to_anyhow)?; + tracing::debug!("Parsed script metadata: {:?}", metadata); { if let Some(v) = metadata.get("requires-python").and_then(|v| v.as_str()) { + tracing::debug!("Found requires-python in metadata: {}", v); push_version_specifiers(true, v.to_owned())?; } }; @@ -403,9 +449,15 @@ async fn parse_python_imports_inner( .get("dependencies") .and_then(|dependencies| dependencies.as_array()) .inspect(|list| { + tracing::debug!("Found {} dependencies in script metadata", list.len()); for dependency_v in list.into_iter() { let requirement = dependency_v.as_str().unwrap_or("ERROR").to_owned(); let key = extract_pkg_name(&requirement); + tracing::debug!( + "Adding dependency from metadata: {} (key: {})", + requirement, + key + ); requirements.insert( key.clone(), NImportResolved::Pin { @@ -418,217 +470,199 @@ async fn parse_python_imports_inner( ); } }); - } else { - code.lines() - .skip(pos + 1) - .map_while(|x| { - RE.captures(x).and_then(|x| { - x.get(1).map(|m| { - let requirement = m.as_str().to_string(); - let key = extract_pkg_name(&requirement); - requirements.insert( - key.clone(), - NImportResolved::Pin { - pins: vec![ImportPin { - pkg: requirement.clone(), - path: Default::default(), - }], - key, - }, - ); - }) - }) - }) - .collect_vec(); - } - Ok(requirements) - } else { - let find_extra_requirements = code.lines().find_position(|x| { - x.starts_with("#extra_requirements:") || x.starts_with("# extra_requirements:") - }); - let mut imports: HashMap = HashMap::new(); - if let Some((pos, _)) = find_extra_requirements { - code.lines() - .skip(pos + 1) - .map_while(|x| { - RE.captures(x).and_then(|x| { - x.get(1).map(|m| { - let requirement = m.as_str().to_string(); - let key = extract_pkg_name(&requirement); - imports.insert( - key.clone(), - NImportResolved::Pin { - pins: vec![ImportPin { - pkg: requirement, - path: Default::default(), - }], - key, - }, - ); - }) - }) - }) - .collect_vec(); } + tracing::debug!( + "Returning {} requirements from script metadata", + requirements.len() + ); + return Ok(requirements); + } + // Will get unsorted vector of imports found in current script + let mut nimports = parse_code_for_imports(code, path)?; + tracing::debug!("Found {} imports in code", nimports.len()); - // Will get unsorted vector of imports found in current script - let mut nimports = parse_code_for_imports(code, path)?; + // It is important to note, that sorting is important and will always result in this pattern: + // 1. All Repins go first + // 2. All Pins go second + // 3. All Auto go third + // 4. All relative imports go the last + // + // This way we make sure all repins are resolved before (re)pins inside imported relative scripts. + nimports.sort(); + tracing::debug!("Processing imports in sorted order"); - // It is important to note, that sorting is important and will always result in this pattern: - // 1. All Repins go first - // 2. All Pins go second - // 3. All Auto go third - // 4. All relative imports go the last - // - // This way we make sure all repins are resolved before (re)pins inside imported relative scripts. - nimports.sort(); - - for n in nimports.into_iter() { - let mut nested = match n { - NImport::Relative(rpath) => { - let code = sqlx::query_scalar!( - r#" + for n in nimports.into_iter() { + let mut nested = match n { + NImport::Relative(rpath) => { + let code = sqlx::query_scalar!( + r#" SELECT content FROM script WHERE path = $1 AND workspace_id = $2 AND archived = false ORDER BY created_at DESC LIMIT 1 "#, - &rpath, - w_id - ) - .fetch_optional(db) - .await? - .unwrap_or_else(|| "".to_string()); + &rpath, + w_id + ) + .fetch_optional(db) + .await? + .unwrap_or_else(|| "".to_string()); - if already_visited.contains(&rpath) { - vec![] + if already_visited.contains(&rpath) { + vec![] + } else { + already_visited.push(rpath.clone()); + // Because the algo goes depth first, this function will never return relative import + // This why we can safely assume later, that there is no relative imports + parse_python_imports_inner( + &code, + w_id, + &rpath, + db, + already_visited, + version_specifiers, + path_where_annotated_pyv, + locked_v, + raw_workspace_dependencies_o, + ) + .await? + .into_values() + .collect_vec() + } + } + NImport::Repin { pin, key } => vec![NImportResolved::Repin { pin, key }], + NImport::Pin { pins, key } => vec![NImportResolved::Pin { pins, key }], + NImport::Auto { pkg, key } => vec![NImportResolved::Auto { pkg, key }], + }; + + // Nested should also be sorted for the same reason + nested.sort(); + tracing::debug!("Processing {} nested imports", nested.len()); + + // At this point there should be no NImport::Relative in `nested` + for imp in nested { + let key = match imp.clone() { + NImportResolved::Pin { key, .. } => key, + NImportResolved::Repin { key, .. } => key, + NImportResolved::Auto { key, pkg } => key.unwrap_or(pkg), + }; + tracing::debug!("Resolving import with key: {}", key); + // Handled cases: + // + // 1. + // Error: Imported windmill scripts have different pins + // + // auto + // ├── pin:2 + // └── pin:1 + // + // Fix 1: + // + // auto + // ├── pin:1 + // └── pin:1 + // + // Fix 2: + // + // repin:1 + // ├── pin:2 + // └── pin:1 + // + // 2. + // Error: Imported windmill scripts have different pins + // + // pin:2 + // └── pin:1 + // + // Fix 1: + // + // auto + // └── pin:1 + // + // Fix 2: + // + // repin:2 + // └── pin:1 + // + // 3. repins allowed to be repinned again + // + // repin:2 + // └── repin:1 + // + match imp.clone() { + NImportResolved::Repin { .. } => { + if let Some(existing_import) = final_imports.get(&key) { + match existing_import { + // replace + p if matches!( + p, + NImportResolved::Pin { .. } | NImportResolved::Auto { .. } + ) => + { + final_imports.insert(key, imp); + } + // do nothing (older repins have greater precedence) + NImportResolved::Repin { .. } => {} + // Should not be possible + _ => { + return Err(anyhow::anyhow!( + "Internal error: cannot resolve requirement pins", + ) + .into()); + } + } } else { - already_visited.push(rpath.clone()); - // Because the algo goes depth first, this function will never return relative import - // This why we can safely assume later, that there is no relative imports - parse_python_imports_inner( - &code, - w_id, - &rpath, - db, - already_visited, - version_specifiers, - path_where_annotated_pyv, - ) - .await? - .into_values() - .collect_vec() + final_imports.insert(key, imp.clone()); } } - NImport::Repin { pin, key } => vec![NImportResolved::Repin { pin, key }], - NImport::Pin { pins, key } => vec![NImportResolved::Pin { pins, key }], - NImport::Auto { pkg, key } => vec![NImportResolved::Auto { pkg, key }], - }; - - // Nested should also be sorted for the same reason - nested.sort(); - - // At this point there should be no NImport::Relative in `nested` - for imp in nested { - let key = match imp.clone() { - NImportResolved::Pin { key, .. } => key, - NImportResolved::Repin { key, .. } => key, - NImportResolved::Auto { key, pkg } => key.unwrap_or(pkg), - }; - // Handled cases: - // - // 1. - // Error: Imported windmill scripts have different pins - // - // auto - // ├── pin:2 - // └── pin:1 - // - // Fix 1: - // - // auto - // ├── pin:1 - // └── pin:1 - // - // Fix 2: - // - // repin:1 - // ├── pin:2 - // └── pin:1 - // - // 2. - // Error: Imported windmill scripts have different pins - // - // pin:2 - // └── pin:1 - // - // Fix 1: - // - // auto - // └── pin:1 - // - // Fix 2: - // - // repin:2 - // └── pin:1 - // - // 3. repins allowed to be repinned again - // - // repin:2 - // └── repin:1 - // - match imp.clone() { - NImportResolved::Repin { .. } => { - if let Some(existing_import) = imports.get(&key) { - match existing_import { - // replace - p if matches!( - p, - NImportResolved::Pin { .. } | NImportResolved::Auto { .. } - ) => - { - imports.insert(key, imp); - } - // do nothing (older repins have greater precedence) - NImportResolved::Repin { .. } => {} - // Should not be possible - _ => { - return Err(anyhow::anyhow!( - "Internal error: cannot resolve requirement pins", - ) - .into()); - } + NImportResolved::Pin { pins: new_pins, .. } => { + if let Some(existing_import) = final_imports.get_mut(&key) { + match existing_import { + // Check if pin is the same version, if same, do nothing, if not error + NImportResolved::Pin { pins: existing_pins, .. } => { + existing_pins.extend(new_pins) } - } else { - imports.insert(key, imp.clone()); - } - } - NImportResolved::Pin { pins: new_pins, .. } => { - if let Some(existing_import) = imports.get_mut(&key) { - match existing_import { - // Check if pin is the same version, if same, do nothing, if not error - NImportResolved::Pin { pins: existing_pins, .. } => { - existing_pins.extend(new_pins) - } - // do nothing - NImportResolved::Repin { .. } => {} - // Replace with new pin - NImportResolved::Auto { .. } => { - imports.insert(key, imp); - } + // do nothing + NImportResolved::Repin { .. } => {} + // Replace with new pin + NImportResolved::Auto { .. } => { + final_imports.insert(key, imp); } - } else { - imports.insert(key, imp.clone()); } + } else { + final_imports.insert(key, imp.clone()); } - NImportResolved::Auto { .. } => { - if !imports.contains_key(&key) { - imports.insert(key, imp); - } + } + NImportResolved::Auto { .. } => { + if !final_imports.contains_key(&key) { + final_imports.insert(key, imp); } } } } - Ok(imports) } + tracing::debug!( + "Finished processing imports, returning {} final imports", + final_imports.len() + ); + Ok(final_imports) +} + +fn extract_nimports_from_content( + content: &str, + hm: &mut HashMap, +) -> Option { + let lines = split_python_requirements(content); + let locked_version = try_parse_locked_python_version_from_requirements(&lines); + for requirement in lines { + let key = extract_pkg_name(&requirement); + hm.insert( + key.clone(), + NImportResolved::Pin { + pins: vec![ImportPin { pkg: requirement, path: Default::default() }], + key, + }, + ); + } + locked_version } const STDIMPORTS: [&str; 303] = [ diff --git a/backend/parsers/windmill-parser-py-imports/tests/fixtures/base.sql b/backend/parsers/windmill-parser-py-imports/tests/fixtures/base.sql index 590ce1bd5f..4b3df78534 100644 --- a/backend/parsers/windmill-parser-py-imports/tests/fixtures/base.sql +++ b/backend/parsers/windmill-parser-py-imports/tests/fixtures/base.sql @@ -2581,3 +2581,102 @@ import innerdifffolder '', '', 'f/foobar/bar', -28028598712388159, 'python3', ''); + + + +-- +-- Name: workspace_dependencies_id_seq; Type: SEQUENCE; Schema: public; Owner: postgres +-- + +CREATE SEQUENCE public.workspace_dependencies_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +ALTER SEQUENCE public.workspace_dependencies_id_seq OWNER TO postgres; + +-- +-- Name: workspace_dependencies_id_seq; Type: SEQUENCE SET; Schema: public; Owner: postgres +-- + +SELECT pg_catalog.setval('public.workspace_dependencies_id_seq', 7, true); + + +-- +-- Name: SEQUENCE workspace_dependencies_id_seq; Type: ACL; Schema: public; Owner: postgres +-- + +GRANT ALL ON SEQUENCE public.workspace_dependencies_id_seq TO windmill_user; +GRANT ALL ON SEQUENCE public.workspace_dependencies_id_seq TO windmill_admin; + + +-- +-- Name: workspace_dependencies; Type: TABLE; Schema: public; Owner: postgres +-- + +CREATE TABLE public.workspace_dependencies ( + id bigint DEFAULT nextval('public.workspace_dependencies_id_seq'::regclass) NOT NULL, + name character varying(255), + content text NOT NULL, + language public.script_lang NOT NULL, + description text DEFAULT ''::text NOT NULL, + archived boolean DEFAULT false NOT NULL, + workspace_id character varying(50) NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +); + + +ALTER TABLE public.workspace_dependencies OWNER TO postgres; + +-- +-- Name: workspace_dependencies workspace_dependencies_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.workspace_dependencies + ADD CONSTRAINT workspace_dependencies_pkey PRIMARY KEY (id); + + +-- +-- Name: one_non_archived_per_name_language_constraint; Type: INDEX; Schema: public; Owner: postgres +-- + +CREATE UNIQUE INDEX one_non_archived_per_name_language_constraint ON public.workspace_dependencies USING btree (name, language, workspace_id) WHERE ((archived = false) AND (name IS NOT NULL)); + + +-- +-- Name: one_non_archived_per_null_name_language_constraint; Type: INDEX; Schema: public; Owner: postgres +-- + +CREATE UNIQUE INDEX one_non_archived_per_null_name_language_constraint ON public.workspace_dependencies USING btree (language, workspace_id) WHERE ((archived = false) AND (name IS NULL)); + + +-- +-- Name: workspace_dependencies_workspace_archived_idx; Type: INDEX; Schema: public; Owner: postgres +-- + +CREATE INDEX workspace_dependencies_workspace_archived_idx ON public.workspace_dependencies USING btree (workspace_id, archived) WHERE (archived = false); + + +-- +-- Name: workspace_dependencies_workspace_lang_name_idx; Type: INDEX; Schema: public; Owner: postgres +-- + +CREATE INDEX workspace_dependencies_workspace_lang_name_idx ON public.workspace_dependencies USING btree (workspace_id, language, name); + + +-- +-- Name: TABLE workspace_dependencies; Type: ACL; Schema: public; Owner: postgres +-- + +GRANT SELECT,INSERT,REFERENCES,DELETE,TRIGGER,TRUNCATE,UPDATE ON TABLE public.workspace_dependencies TO windmill_user; +GRANT SELECT,INSERT,REFERENCES,DELETE,TRIGGER,TRUNCATE,UPDATE ON TABLE public.workspace_dependencies TO windmill_admin; + + +-- +-- PostgreSQL database dump complete +-- + + diff --git a/backend/parsers/windmill-parser-py-imports/tests/tests.rs b/backend/parsers/windmill-parser-py-imports/tests/tests.rs index b7fd2c0737..e61b4cc7db 100644 --- a/backend/parsers/windmill-parser-py-imports/tests/tests.rs +++ b/backend/parsers/windmill-parser-py-imports/tests/tests.rs @@ -18,8 +18,16 @@ def main(): pass "; - let (r, ..) = - parse_python_imports(code, "test-workspace", "f/foo/bar", &db, &mut vec![]).await?; + let (r, ..) = parse_python_imports( + code, + "test-workspace", + "f/foo/bar", + &db, + &mut vec![], + &mut None, + &None, + ) + .await?; // println!("{}", serde_json::to_string(&r)?); assert_eq!( r, @@ -51,8 +59,16 @@ def main(): pass "; - let (r, ..) = - parse_python_imports(code, "test-workspace", "f/foo/bar", &db, &mut vec![]).await?; + let (r, ..) = parse_python_imports( + code, + "test-workspace", + "f/foo/bar", + &db, + &mut vec![], + &mut None, + &None, + ) + .await?; println!("{}", serde_json::to_string(&r)?); assert_eq!(r, vec!["burkina=0.4", "nigeria"]); @@ -74,8 +90,16 @@ def main(): "; - let (r, ..) = - parse_python_imports(code, "test-workspace", "f/foo/bar", &db, &mut vec![]).await?; + let (r, ..) = parse_python_imports( + code, + "test-workspace", + "f/foo/bar", + &db, + &mut vec![], + &mut None, + &None, + ) + .await?; println!("{}", serde_json::to_string(&r)?); assert_eq!( r, diff --git a/backend/src/main.rs b/backend/src/main.rs index a619572f50..c4a5ca805c 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -271,8 +271,8 @@ async fn cache_hub_scripts(file_path: Option) -> anyhow::Result<()> { let job_id = Uuid::new_v4(); let job_dir = format!("{}/cache_init/{}", TMP_DIR, job_id); create_dir_all(&job_dir)?; - if let Some(lockfile) = res.lockfile { - let _ = windmill_worker::prepare_job_dir(&lockfile, &job_dir).await?; + if let Some(lock) = res.lockfile { + let _ = windmill_worker::prepare_job_dir(&lock, &job_dir).await?; let envs = windmill_worker::get_common_bun_proc_envs(None).await; let _ = windmill_worker::install_bun_lockfile( &mut 0, @@ -292,7 +292,7 @@ async fn cache_hub_scripts(file_path: Option) -> anyhow::Result<()> { if let Err(e) = windmill_worker::prebundle_bun_script( &res.content, - Some(&lockfile), + &lock, &path, &job_id, "admins", diff --git a/backend/tests/relative_imports.rs b/backend/tests/debouncing.rs similarity index 86% rename from backend/tests/relative_imports.rs rename to backend/tests/debouncing.rs index 052f5a3743..72fdc8a203 100644 --- a/backend/tests/relative_imports.rs +++ b/backend/tests/debouncing.rs @@ -1,7 +1,9 @@ -// TODO: move all related logic here (if anything left anywhere in codebase) mod common; + +#[cfg(feature = "test_job_debouncing")] use windmill_api_client::types::NewScript; +#[cfg(feature = "test_job_debouncing")] fn quick_ns( content: &str, language: windmill_api_client::types::ScriptLang, @@ -43,490 +45,6 @@ fn quick_ns( } } -mod dependency_map { - use super::quick_ns; - use sqlx::{Pool, Postgres}; - use tokio_stream::StreamExt; - - use crate::common::{in_test_worker, init_client, listen_for_completed_jobs, ApiServer}; - - async fn init(db: Pool) -> (windmill_api_client::Client, u16, ApiServer) { - init_client(db).await - } - - async fn _clear_dmap(db: &Pool) { - sqlx::query!("DELETE FROM dependency_map WHERE workspace_id = 'test-workspace'") - .execute(db) - .await - .unwrap(); - } - - /// Corrects map according to provided replacements. - /// Only changes importer_path and/or id - /// Does not affect imported_path nor kind! - fn corrected_dmap(replacements: Vec<(&str, &str)>) -> Vec<(String, String, String, String)> { - CORRECT_DMAP - .clone() - .into_iter() - .map(|e| { - let mut r = ( - e.0.to_owned(), - e.1.to_owned(), - e.2.to_owned(), - e.3.to_owned(), - ); - for (from, to) in &replacements { - r = ( - r.0.replace(from, to), - r.1, // Kind should be immutable - r.2, // Imported path should be immutable - // We do not modify script contents in test, so we can assume scripts always import the same path - // Modification of kind or imported path considered to be incorrect. - r.3.replace(from, to), - ); - } - r - }) - .collect() - } - - async fn assert_dmap( - db: &Pool, - importer: Option, - expected: Vec<( - impl Into, - impl Into, - impl Into, - impl Into, - )>, - ) { - let dmap = sqlx::query_as::<_, (String, String, String, String)>( - "SELECT importer_path, importer_kind::text, imported_path, importer_node_id FROM dependency_map WHERE workspace_id = 'test-workspace' AND ($1::text IS NULL OR importer_path = $1::text)", - ) - .bind(importer) - .fetch_all(db) - .await - .unwrap(); - - assert_eq!( - dmap, - expected - .into_iter() - .map(|(f, s, t, fo)| (f.into(), s.into(), t.into(), fo.into())) - .collect::>() - ); - } - - lazy_static::lazy_static! { - pub static ref CORRECT_DMAP: Vec<(&'static str, &'static str, &'static str, &'static str)> = vec![ - ("f/rel/branch", "script", "f/rel/leaf_1", ""), - ("f/rel/root_script", "script", "f/rel/branch", ""), - ("f/rel/root_script", "script", "f/rel/leaf_1", ""), - ("f/rel/root_script", "script", "f/rel/leaf_2", ""), - ("f/rel/root_app", "app", "f/rel/leaf_2", "dontpressmeplz"), - ("f/rel/root_flow", "flow", "f/rel/leaf_2", "failure"), - ("f/rel/root_flow", "flow", "f/rel/branch", "nstep1"), - ("f/rel/root_flow", "flow", "f/rel/leaf_1", "nstep1"), - ("f/rel/root_flow", "flow", "f/rel/leaf_2", "nstep1"), - ("f/rel/root_flow", "flow", "f/rel/leaf_2", "nstep2_2"), - ("f/rel/root_flow", "flow", "f/rel/branch", "nstep4_1"), - ("f/rel/root_flow", "flow", "f/rel/branch", "nstep5_1"), - ("f/rel/root_flow", "flow", "f/rel/leaf_1", "nstep5_1"), - ("f/rel/root_flow", "flow", "f/rel/leaf_2", "nstep5_1"), - ("f/rel/root_flow", "flow", "f/rel/branch", "preprocessor"), - ("f/rel/root_flow", "flow", "f/rel/leaf_1", "preprocessor"), - ("f/rel/root_flow", "flow", "f/rel/leaf_2", "preprocessor"), - ("f/rel/root_app", "app", "f/rel/branch", "pressmeplz"), - ("f/rel/root_app", "app", "f/rel/leaf_1", "pressmeplz"), - ("f/rel/root_app", "app", "f/rel/leaf_2", "pressmeplz"), - ("f/rel/root_flow", "flow", "f/rel/leaf_2", "qtool1"), - ("f/rel/root_app", "app", "f/rel/branch", "youcanpressme")]; - } - - // TODO: - // Test that checks that we can run rebuild_dmap multiple times in tests. - - #[cfg(feature = "python")] - #[sqlx::test(fixtures("base", "dependency_map"))] - async fn relative_imports_test_rebuild_correctness(db: Pool) -> anyhow::Result<()> { - let (client, _port, _s) = init(db.clone()).await; - assert_dmap(&db, None, CORRECT_DMAP.clone()).await; - // rebuild map - assert!(super::common::rebuild_dmap(&client).await); - assert_dmap(&db, None, CORRECT_DMAP.clone()).await; - Ok(()) - } - - #[cfg(feature = "python")] - #[sqlx::test(fixtures("base", "dependency_map"))] - async fn relative_imports_test_rebuild_lock(db: Pool) -> anyhow::Result<()> { - let (client, _port, _s) = init(db.clone()).await; - - // Spawn first rebuild - let handle = { - let client = client.clone(); - tokio::spawn(async move { super::common::rebuild_dmap(&client).await }) - }; - - // Immidiately spawn another - let res = client - .client() - .post(format!( - "{}/w/test-workspace/workspaces/rebuild_dependency_map", - client.baseurl() - )) - .send() - .await - .unwrap() - .text() - .await - .unwrap(); - - // Should tell us there is already rebuilt in progress - // Or if it is too fast we will be able to trigger it second time - assert!(&res == "There is already one task pending, try again later." || &res == "Success"); - - assert!(handle.await.unwrap()); - Ok(()) - } - - // If you deploy from cli and you use raw requirements you don't want the script be included in dmap - // Otherwise script will be overwritten once any relative import is updated - #[cfg(feature = "python")] - #[sqlx::test(fixtures("base", "dependency_map"))] - async fn relative_imports_test_with_requirements_txt(db: Pool) -> anyhow::Result<()> { - let (client, _port, _s) = init(db.clone()).await; - - client - .create_script( - "test-workspace", - &quick_ns( - " -from f.rel.branch import main as br; -from f.rel.leaf_1 import main as lf_1; -from f.rel.leaf_2 import main as lf_2; - -def main(): - return [br(), lf_1(), lf_2]; - ", - windmill_api_client::types::ScriptLang::Python3, - "f/rel/root_script", - Some("# from requirements.txt".to_string()), - Some("000000000005165B".into()), - ), - ) - .await - .unwrap(); - - assert_dmap( - &db, - Some("f/rel/root_script".into()), - vec![ - ("f/rel/root_script", "script", "f/rel/branch", ""), - ("f/rel/root_script", "script", "f/rel/leaf_1", ""), - ("f/rel/root_script", "script", "f/rel/leaf_2", ""), - ], - ) - .await; - - tokio::time::sleep(std::time::Duration::from_secs(13)).await; - - assert_dmap( - &db, - Some("f/rel/root_script".into()), - Vec::<(String, String, String, String)>::new(), - ) - .await; - - Ok(()) - } - - #[cfg(feature = "python")] - #[sqlx::test(fixtures("base", "dependency_map"))] - async fn relative_imports_test_without_requirements_txt( - db: Pool, - ) -> anyhow::Result<()> { - let (client, _port, _s) = init(db.clone()).await; - - client - .create_script( - "test-workspace", - &quick_ns( - " -from f.rel.branch import main as br; -from f.rel.leaf_1 import main as lf_1; -from f.rel.leaf_2 import main as lf_2; - -def main(): - return [br(), lf_1(), lf_2]; - ", - windmill_api_client::types::ScriptLang::Python3, - "f/rel/root_script", - // We still want to pass lock to it. - Some("# py311".to_string()), - Some("000000000005165B".into()), - ), - ) - .await - .unwrap(); - assert_dmap(&db, None, CORRECT_DMAP.clone()).await; - // tokio::time::sleep(std::time::Duration::from_secs(13)).await; - assert_dmap(&db, None, CORRECT_DMAP.clone()).await; - Ok(()) - } - // Consider simple one. Only referenced directly. No deep connections - #[cfg(feature = "python")] - #[sqlx::test(fixtures("base", "dependency_map"))] - async fn relative_imports_test_rename_leaf_2(db: Pool) -> anyhow::Result<()> { - let (client, port, _s) = init(db.clone()).await; - client - .create_script( - "test-workspace", - &quick_ns( - " -def main(): - return 'leaf3'; - ", - windmill_api_client::types::ScriptLang::Python3, - "f/rel/leaf_2_renamed", - None, - Some("0000000000051659".into()), - ), - ) - .await - .unwrap(); - - let mut completed = listen_for_completed_jobs(&db).await; - in_test_worker(&db, completed.next(), port).await; - - // Changing leafs should not change dependency map - assert_dmap(&db, None, CORRECT_DMAP.clone()).await; - Ok(()) - } - - // Consider hard one. Referenced deeply and exists in double references. - #[cfg(feature = "python")] - #[sqlx::test(fixtures("base", "dependency_map"))] - async fn relative_imports_test_rename_leaf_1(db: Pool) -> anyhow::Result<()> { - let (client, port, _s) = init(db.clone()).await; - client - .create_script( - "test-workspace", - &quick_ns( - " -def main(): - return 'leaf1'; - ", - windmill_api_client::types::ScriptLang::Python3, - "f/rel/leaf_1_renamed", - None, - Some("0000000000051658".into()), - ), - ) - .await - .unwrap(); - - let mut completed = listen_for_completed_jobs(&db).await; - in_test_worker(&db, completed.next(), port).await; - - // Changing leafs should not change dependency map - assert_dmap(&db, None, CORRECT_DMAP.clone()).await; - Ok(()) - } - - #[cfg(feature = "python")] - #[sqlx::test(fixtures("base", "dependency_map"))] - async fn relative_imports_test_rename_branch(db: Pool) -> anyhow::Result<()> { - let (client, port, _s) = init(db.clone()).await; - client - .create_script( - "test-workspace", - &quick_ns( - " -from f.rel.leaf_1 import main as lf_1; - -def main(): - return lf_1(); - ", - windmill_api_client::types::ScriptLang::Python3, - "f/rel/branch_renamed", - None, - Some("000000000005165A".into()), - ), - ) - .await - .unwrap(); - - let mut completed = listen_for_completed_jobs(&db).await; - in_test_worker(&db, completed.next(), port).await; - - // Changing branches SHOULD change dependency map - // Though it should only change branch item in dmap when it is importer. - // All entries when branch is imported should not change. - let mut corrected_dmap = CORRECT_DMAP.clone(); - // Corresponds to importer path of branch entry - corrected_dmap[0].0 = "f/rel/branch_renamed"; - assert_dmap(&db, None, corrected_dmap).await; - Ok(()) - } - - #[cfg(feature = "python")] - #[sqlx::test(fixtures("base", "dependency_map"))] - async fn relative_imports_test_rename_primary_script(db: Pool) -> anyhow::Result<()> { - let (client, port, _s) = init(db.clone()).await; - - client - .create_script( - "test-workspace", - &quick_ns( - " -from f.rel.branch import main as br; -from f.rel.leaf_1 import main as lf_1; -from f.rel.leaf_2 import main as lf_2; - -def main(): - return [br(), lf_1(), lf_2]; - ", - windmill_api_client::types::ScriptLang::Python3, - "f/rel/root_script_renamed", - None, - Some("000000000005165B".into()), - ), - ) - .await - .unwrap(); - - let corrected_dmap = corrected_dmap(vec![("root_script", "root_script_renamed")]); - let mut completed = listen_for_completed_jobs(&db).await; - in_test_worker(&db, completed.next(), port).await; - assert_dmap(&db, None, corrected_dmap.clone()).await; - Ok(()) - } - - #[cfg(feature = "python")] - #[sqlx::test(fixtures("base", "dependency_map"))] - async fn relative_imports_test_rename_primary_flow(db: Pool) -> anyhow::Result<()> { - use windmill_common::{cache::flow::fetch_version, flows::NewFlow, worker::to_raw_value}; - - let (client, port, _s) = init(db.clone()).await; - let flow = fetch_version(&db, 1443253234253454).await.unwrap(); - let res = client - .client() - .post(format!( - "{}/w/test-workspace/flows/update/{}", - client.baseurl(), - "f/rel/root_flow" // encode_path() - )) - .json(&NewFlow { - path: "f/rel/root_flow_renamed".into(), - summary: "".into(), - description: None, - value: to_raw_value( - &serde_json::from_str::( - &serde_json::to_string(flow.value()) - .unwrap() - .replace("nstep1", "Foxes") - .replace("nstep2_2", "like") - .replace("nstep_4_1", "Emeralds"), - ) - .unwrap(), - ), - schema: None, - draft_only: None, - tag: None, - dedicated_worker: None, - timeout: None, - deployment_message: None, - visible_to_runner_only: None, - on_behalf_of_email: None, - ws_error_handler_muted: None, - }) - .send() - .await - .unwrap(); - - assert_eq!(res.text().await.unwrap(), "f/rel/root_flow_renamed"); - - let mut completed = listen_for_completed_jobs(&db).await; - in_test_worker(&db, completed.next(), port).await; - - assert_dmap( - &db, - None, - corrected_dmap(vec![ - ("f/rel/root_flow", "f/rel/root_flow_renamed"), - ("nstep1", "Foxes"), - ("nstep2_2", "like"), - ("nstep_4_1", "Emeralds"), - ]), - ) - .await; - Ok(()) - } - - #[cfg(feature = "python")] - #[sqlx::test(fixtures("base", "dependency_map"))] - async fn relative_imports_test_rename_primary_app(db: Pool) -> anyhow::Result<()> { - let (client, port, _s) = init(db.clone()).await; - - let app_value: String = - sqlx::query_scalar!("SELECT value::text FROM app_version WHERE id = 0 AND app_id = 2") - .fetch_one(&db) - .await - .unwrap() - .unwrap(); - - // TODO: There is: - // 1. update app - // 2. create app - // 3. update app raw - // Ideally all of them should be handled - let res = client - .client() - .post(format!( - "{}/w/test-workspace/apps/update/{}", - client.baseurl(), - "f/rel/root_app" // encode_path() - )) - .json(&windmill_api::EditApp { - path: Some("f/rel/root_app_renamed".into()), - summary: None, - value: serde_json::from_str( - &app_value - .replace("dontpressmeplz", "Apps") - .replace("youcanpressme", "Work"), - ) - .unwrap(), - policy: None, - deployment_message: None, - custom_path: None, - }) - .send() - .await - .unwrap(); - - assert_eq!( - res.text().await.unwrap(), - "app f/rel/root_app updated (npath: \"f/rel/root_app_renamed\")" - ); - - let mut completed = listen_for_completed_jobs(&db).await; - in_test_worker(&db, completed.next(), port).await; - - assert_dmap( - &db, - None, - corrected_dmap(vec![ - ("f/rel/root_app", "f/rel/root_app_renamed"), - ("dontpressmeplz", "Apps"), - ("youcanpressme", "Work"), - ]), - ) - .await; - Ok(()) - } -} - #[cfg(feature = "test_job_debouncing")] mod dependency_job_debouncing { async fn trigger_djob_for( diff --git a/backend/tests/dependency_map.rs b/backend/tests/dependency_map.rs new file mode 100644 index 0000000000..d217b710d6 --- /dev/null +++ b/backend/tests/dependency_map.rs @@ -0,0 +1,535 @@ +use sqlx::{Pool, Postgres}; +use tokio_stream::StreamExt; + +use windmill_api_client::types::NewScript; +mod common; +use common::{in_test_worker, init_client, listen_for_completed_jobs, ApiServer}; + +mod dependency_map { + use super::*; + fn quick_ns( + content: &str, + language: windmill_api_client::types::ScriptLang, + path: &str, + lock: Option, + parent_hash: Option, + ) -> NewScript { + NewScript { + content: content.into(), + language, + lock, + parent_hash, + path: path.into(), + concurrent_limit: None, + concurrency_time_window_s: None, + cache_ttl: None, + dedicated_worker: None, + description: "".to_string(), + draft_only: None, + envs: vec![], + is_template: None, + kind: None, + summary: "".to_string(), + tag: None, + schema: std::collections::HashMap::new(), + ws_error_handler_muted: Some(false), + priority: None, + delete_after_use: None, + timeout: None, + restart_unless_cancelled: None, + deployment_message: None, + concurrency_key: None, + visible_to_runner_only: None, + no_main_func: None, + codebase: None, + has_preprocessor: None, + on_behalf_of_email: None, + assets: vec![], + } + } + async fn init(db: Pool) -> (windmill_api_client::Client, u16, ApiServer) { + init_client(db).await + } + + async fn _clear_dmap(db: &Pool) { + sqlx::query!("DELETE FROM dependency_map WHERE workspace_id = 'test-workspace'") + .execute(db) + .await + .unwrap(); + } + + /// Corrects map according to provided replacements. + /// Only changes importer_path and/or id + /// Does not affect imported_path nor kind! + fn corrected_dmap(replacements: Vec<(&str, &str)>) -> Vec<(String, String, String, String)> { + CORRECT_DMAP + .clone() + .into_iter() + .map(|e| { + let mut r = ( + e.0.to_owned(), + e.1.to_owned(), + e.2.to_owned(), + e.3.to_owned(), + ); + for (from, to) in &replacements { + r = ( + r.0.replace(from, to), + r.1, // Kind should be immutable + r.2, // Imported path should be immutable + // We do not modify script contents in test, so we can assume scripts always import the same path + // Modification of kind or imported path considered to be incorrect. + r.3.replace(from, to), + ); + } + r + }) + .collect() + } + + async fn assert_dmap( + db: &Pool, + importer: Option, + expected: Vec<( + impl Into, + impl Into, + impl Into, + impl Into, + )>, + ) { + let mut dmap = sqlx::query_as::<_, (String, String, String, String)>( + "SELECT importer_path, importer_kind::text, imported_path, importer_node_id FROM dependency_map WHERE workspace_id = 'test-workspace' AND ($1::text IS NULL OR importer_path = $1::text)", + ) + .bind(importer) + .fetch_all(db) + .await + .unwrap(); + + let mut expected = expected + .into_iter() + .map(|(f, s, t, fo)| (f.into(), s.into(), t.into(), fo.into())) + .collect::>(); + + dmap.sort(); + expected.sort(); + + assert_eq!(dmap, expected); + } + + lazy_static::lazy_static! { + pub static ref CORRECT_DMAP: Vec<(&'static str, &'static str, &'static str, &'static str)> = vec![ + ("f/rel/root_script", "script", "dependencies/test.requirements.in", ""), + ("f/rel/root_flow", "flow", "dependencies/test.requirements.in", "nstep1"), + ("f/rel/root_app", "app", "dependencies/test.requirements.in", "dontpressmeplz"), + ("f/rel/branch", "script", "f/rel/leaf_1", ""), + ("f/rel/root_script", "script", "f/rel/branch", ""), + ("f/rel/root_script", "script", "f/rel/leaf_1", ""), + ("f/rel/root_script", "script", "f/rel/leaf_2", ""), + ("f/rel/root_app", "app", "f/rel/leaf_2", "dontpressmeplz"), + ("f/rel/root_flow", "flow", "f/rel/leaf_2", "failure"), + ("f/rel/root_flow", "flow", "f/rel/branch", "nstep1"), + ("f/rel/root_flow", "flow", "f/rel/leaf_1", "nstep1"), + ("f/rel/root_flow", "flow", "f/rel/leaf_2", "nstep1"), + ("f/rel/root_flow", "flow", "f/rel/leaf_2", "nstep2_2"), + ("f/rel/root_flow", "flow", "f/rel/branch", "nstep4_1"), + ("f/rel/root_flow", "flow", "f/rel/branch", "nstep5_1"), + ("f/rel/root_flow", "flow", "f/rel/leaf_1", "nstep5_1"), + ("f/rel/root_flow", "flow", "f/rel/leaf_2", "nstep5_1"), + ("f/rel/root_flow", "flow", "f/rel/branch", "preprocessor"), + ("f/rel/root_flow", "flow", "f/rel/leaf_1", "preprocessor"), + ("f/rel/root_flow", "flow", "f/rel/leaf_2", "preprocessor"), + ("f/rel/root_app", "app", "f/rel/branch", "pressmeplz"), + ("f/rel/root_app", "app", "f/rel/leaf_1", "pressmeplz"), + ("f/rel/root_app", "app", "f/rel/leaf_2", "pressmeplz"), + ("f/rel/root_flow", "flow", "f/rel/leaf_2", "qtool1"), + ("f/rel/root_app", "app", "f/rel/branch", "youcanpressme"), + + // Default + ("f/rel/leaf_1", "script", "dependencies/requirements.in", ""), + ("f/rel/leaf_2", "script", "dependencies/requirements.in", ""), + ("f/rel/branch", "script", "dependencies/requirements.in", ""), + ("f/rel/root_flow", "flow", "dependencies/requirements.in", "failure"), + ("f/rel/root_flow", "flow", "dependencies/package.json", "nstep2_1"), + ("f/rel/root_flow", "flow", "dependencies/package.json", "nstep3_2"), + ("f/rel/root_flow", "flow", "dependencies/requirements.in", "nstep2_2"), + ("f/rel/root_flow", "flow", "dependencies/requirements.in", "nstep3_1"), + ("f/rel/root_flow", "flow", "dependencies/requirements.in", "nstep4_1"), + ("f/rel/root_flow", "flow", "dependencies/requirements.in", "nstep5_1"), + ("f/rel/root_flow", "flow", "dependencies/requirements.in", "preprocessor"), + ("f/rel/root_app", "app", "dependencies/requirements.in", "pressmeplz"), + ("f/rel/root_flow", "flow", "dependencies/requirements.in", "qtool1"), + ("f/rel/root_app", "app", "dependencies/requirements.in", "youcanpressme") + ]; + } + // TODO: + // Test that checks that we can run rebuild_dmap multiple times in tests. + + #[cfg(feature = "python")] + #[sqlx::test(fixtures("base", "dependency_map"))] + async fn relative_imports_test_rebuild_correctness(db: Pool) -> anyhow::Result<()> { + let (client, _port, _s) = init(db.clone()).await; + assert_dmap(&db, None, CORRECT_DMAP.clone()).await; + // rebuild map + assert!(common::rebuild_dmap(&client).await); + assert_dmap(&db, None, CORRECT_DMAP.clone()).await; + Ok(()) + } + + #[cfg(feature = "python")] + #[sqlx::test(fixtures("base", "dependency_map"))] + async fn relative_imports_test_rebuild_lock(db: Pool) -> anyhow::Result<()> { + let (client, _port, _s) = init(db.clone()).await; + + // Spawn first rebuild + let handle = { + let client = client.clone(); + tokio::spawn(async move { common::rebuild_dmap(&client).await }) + }; + + // Immidiately spawn another + let res = client + .client() + .post(format!( + "{}/w/test-workspace/workspaces/rebuild_dependency_map", + client.baseurl() + )) + .send() + .await + .unwrap() + .text() + .await + .unwrap(); + + // Should tell us there is already rebuilt in progress + // Or if it is too fast we will be able to trigger it second time + assert!(&res == "There is already one task pending, try again later." || &res == "Success"); + + assert!(handle.await.unwrap()); + Ok(()) + } + + // If you deploy from cli and you use raw requirements you don't want the script be included in dmap + // Otherwise script will be overwritten once any relative import is updated + // #[cfg(feature = "python")] + // #[sqlx::test(fixtures("base", "dependency_map"))] + // async fn relative_imports_test_with_legacy(db: Pool) -> anyhow::Result<()> { + // let (client, _port, _s) = init(db.clone()).await; + + // client + // .create_script( + // "test-workspace", + // &quick_ns( + // " + // from f.rel.branch import main as br; + // from f.rel.leaf_1 import main as lf_1; + // from f.rel.leaf_2 import main as lf_2; + + // def main(): + // return [br(), lf_1(), lf_2]; + // ", + // windmill_api_client::types::ScriptLang::Python3, + // "f/rel/root_script", + // Some("# from requirements.txt".to_string()), + // Some("000000000005165B".into()), + // ), + // ) + // .await + // .unwrap(); + + // assert_dmap( + // &db, + // Some("f/rel/root_script".into()), + // vec![ + // ("f/rel/root_script", "script", "f/rel/branch", ""), + // ("f/rel/root_script", "script", "f/rel/leaf_1", ""), + // ("f/rel/root_script", "script", "f/rel/leaf_2", ""), + // ], + // ) + // .await; + + // tokio::time::sleep(std::time::Duration::from_secs(13)).await; + + // assert_dmap( + // &db, + // Some("f/rel/root_script".into()), + // vec![ + // ("f/rel/root_script", "script", "f/rel/branch", ""), + // ("f/rel/root_script", "script", "f/rel/leaf_1", ""), + // ("f/rel/root_script", "script", "f/rel/leaf_2", ""), + // ], + // ) + // .await; + + // Ok(()) + // } + + // Consider simple one. Only referenced directly. No deep connections + #[cfg(feature = "python")] + #[sqlx::test(fixtures("base", "dependency_map"))] + async fn relative_imports_test_rename_leaf_2(db: Pool) -> anyhow::Result<()> { + let (client, port, _s) = init(db.clone()).await; + client + .create_script( + "test-workspace", + &quick_ns( + " +def main(): + return 'leaf3'; + ", + windmill_api_client::types::ScriptLang::Python3, + "f/rel/leaf_2_renamed", + None, + Some("0000000000051659".into()), + ), + ) + .await + .unwrap(); + + let mut completed = listen_for_completed_jobs(&db).await; + in_test_worker(&db, completed.next(), port).await; + + // Changing leafs should not change dependency map + assert_dmap( + &db, + None, + corrected_dmap(vec![("f/rel/leaf_2", "f/rel/leaf_2_renamed")]), + ) + .await; + Ok(()) + } + + // Consider hard one. Referenced deeply and exists in double references. + #[cfg(feature = "python")] + #[sqlx::test(fixtures("base", "dependency_map"))] + async fn relative_imports_test_rename_leaf_1(db: Pool) -> anyhow::Result<()> { + let (client, port, _s) = init(db.clone()).await; + client + .create_script( + "test-workspace", + &quick_ns( + " +def main(): + return 'leaf1'; + ", + windmill_api_client::types::ScriptLang::Python3, + "f/rel/leaf_1_renamed", + None, + Some("0000000000051658".into()), + ), + ) + .await + .unwrap(); + + let mut completed = listen_for_completed_jobs(&db).await; + in_test_worker(&db, completed.next(), port).await; + + // Changing leafs should not change dependency map + assert_dmap( + &db, + None, + corrected_dmap(vec![("f/rel/leaf_1", "f/rel/leaf_1_renamed")]), + ) + .await; + Ok(()) + } + + #[cfg(feature = "python")] + #[sqlx::test(fixtures("base", "dependency_map"))] + async fn relative_imports_test_rename_branch(db: Pool) -> anyhow::Result<()> { + let (client, port, _s) = init(db.clone()).await; + client + .create_script( + "test-workspace", + &quick_ns( + " +from f.rel.leaf_1 import main as lf_1; + +def main(): + return lf_1(); + ", + windmill_api_client::types::ScriptLang::Python3, + "f/rel/branch_renamed", + None, + Some("000000000005165A".into()), + ), + ) + .await + .unwrap(); + + let mut completed = listen_for_completed_jobs(&db).await; + in_test_worker(&db, completed.next(), port).await; + + // Changing branches SHOULD change dependency map + // Though it should only change branch item in dmap when it is importer. + // All entries when branch is imported should not change. + let corrected_dmap = CORRECT_DMAP + .clone() + .iter_mut() + .map(|el| { + if el.0 == "f/rel/branch" { + el.0 = "f/rel/branch_renamed"; + } + *el + }) + .collect::>(); + assert_dmap(&db, None, corrected_dmap).await; + Ok(()) + } + + #[cfg(feature = "python")] + #[sqlx::test(fixtures("base", "dependency_map"))] + async fn relative_imports_test_rename_primary_script(db: Pool) -> anyhow::Result<()> { + let (client, port, _s) = init(db.clone()).await; + + client + .create_script( + "test-workspace", + &quick_ns( + " +# requirements: test +from f.rel.branch import main as br; +from f.rel.leaf_1 import main as lf_1; +from f.rel.leaf_2 import main as lf_2; + +def main(): + return [br(), lf_1(), lf_2]; + ", + windmill_api_client::types::ScriptLang::Python3, + "f/rel/root_script_renamed", + None, + Some("000000000005165B".into()), + ), + ) + .await + .unwrap(); + + let corrected_dmap = corrected_dmap(vec![("root_script", "root_script_renamed")]); + let mut completed = listen_for_completed_jobs(&db).await; + in_test_worker(&db, completed.next(), port).await; + assert_dmap(&db, None, corrected_dmap.clone()).await; + Ok(()) + } + + #[cfg(feature = "python")] + #[sqlx::test(fixtures("base", "dependency_map"))] + async fn relative_imports_test_rename_primary_flow(db: Pool) -> anyhow::Result<()> { + use windmill_common::{cache::flow::fetch_version, flows::NewFlow, worker::to_raw_value}; + + let (client, port, _s) = init(db.clone()).await; + let flow = fetch_version(&db, 1443253234253454).await.unwrap(); + let res = client + .client() + .post(format!( + "{}/w/test-workspace/flows/update/{}", + client.baseurl(), + "f/rel/root_flow" // encode_path() + )) + .json(&NewFlow { + path: "f/rel/root_flow_renamed".into(), + summary: "".into(), + description: None, + value: to_raw_value( + &serde_json::from_str::( + &serde_json::to_string(flow.value()) + .unwrap() + .replace("nstep1", "Foxes") + .replace("nstep2_2", "like") + .replace("nstep_4_1", "Emeralds"), + ) + .unwrap(), + ), + schema: None, + draft_only: None, + tag: None, + dedicated_worker: None, + timeout: None, + deployment_message: None, + visible_to_runner_only: None, + on_behalf_of_email: None, + ws_error_handler_muted: None, + }) + .send() + .await + .unwrap(); + + assert_eq!(res.text().await.unwrap(), "f/rel/root_flow_renamed"); + + let mut completed = listen_for_completed_jobs(&db).await; + in_test_worker(&db, completed.next(), port).await; + + assert_dmap( + &db, + None, + corrected_dmap(vec![ + ("f/rel/root_flow", "f/rel/root_flow_renamed"), + ("nstep1", "Foxes"), + ("nstep2_2", "like"), + ("nstep_4_1", "Emeralds"), + ]), + ) + .await; + Ok(()) + } + + #[cfg(feature = "python")] + #[sqlx::test(fixtures("base", "dependency_map"))] + async fn relative_imports_test_rename_primary_app(db: Pool) -> anyhow::Result<()> { + let (client, port, _s) = init(db.clone()).await; + + let app_value: String = + sqlx::query_scalar!("SELECT value::text FROM app_version WHERE id = 0 AND app_id = 2") + .fetch_one(&db) + .await + .unwrap() + .unwrap(); + + // TODO: There is: + // 1. update app + // 2. create app + // 3. update app raw + // Ideally all of them should be handled + let res = client + .client() + .post(format!( + "{}/w/test-workspace/apps/update/{}", + client.baseurl(), + "f/rel/root_app" // encode_path() + )) + .json(&windmill_api::EditApp { + path: Some("f/rel/root_app_renamed".into()), + summary: None, + value: serde_json::from_str( + &app_value + .replace("dontpressmeplz", "Apps") + .replace("youcanpressme", "Work"), + ) + .unwrap(), + policy: None, + deployment_message: None, + custom_path: None, + }) + .send() + .await + .unwrap(); + + assert_eq!( + res.text().await.unwrap(), + "app f/rel/root_app updated (npath: \"f/rel/root_app_renamed\")" + ); + + let mut completed = listen_for_completed_jobs(&db).await; + in_test_worker(&db, completed.next(), port).await; + + assert_dmap( + &db, + None, + corrected_dmap(vec![ + ("f/rel/root_app", "f/rel/root_app_renamed"), + ("dontpressmeplz", "Apps"), + ("youcanpressme", "Work"), + ]), + ) + .await; + Ok(()) + } +} diff --git a/backend/tests/fixtures/dependency_map.sql b/backend/tests/fixtures/dependency_map.sql index c2caf110be..0b8c797155 100644 --- a/backend/tests/fixtures/dependency_map.sql +++ b/backend/tests/fixtures/dependency_map.sql @@ -43,6 +43,7 @@ INSERT INTO public.script(workspace_id, created_by, content, schema, summary, de 'test-workspace', 'test-user', ' +# requirements: test from f.rel.branch import main as br; from f.rel.leaf_1 import main as lf_1; from f.rel.leaf_2 import main as lf_2; @@ -63,7 +64,7 @@ INSERT INTO public.flow(workspace_id, summary, description, path, versions, sche 'f/rel/root_flow', '{1443253234253454}', '{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object","order":[]}', -$tag${"modules":[{"id":"nstep1","value":{"lock":"# py: 3.11\n","type":"rawscript","assets":[],"content":"from f.rel.branch import main as br;\nfrom f.rel.leaf_1 import main as lf_1;\nfrom ..leaf_1 import main as lf_12;\nfrom ...rel.leaf_2 import main as lf_2;\n\ndef check():\n return [br(), lf_1(), lf_2(), lf_12()];\n\ndef main(x: str):\n return x","language":"python3","input_transforms":{"x":{"type":"static","value":""}}}},{"id":"nstep2","value":{"type":"branchall","branches":[{"expr":"","modules":[{"id":"nstep2_1","value":{"lock":"{\n \"dependencies\": {}\n}\n//bun.lock\n","type":"rawscript","assets":[],"content":"// import * as wmill from \"windmill-client\"\n\nexport async function main(x: string) {\n return x\n}\n","language":"bun","input_transforms":{"x":{"type":"static","value":""}}}}],"parallel":true,"skip_failure":false},{"expr":"false","modules":[{"id":"nstep2_2","value":{"lock":"# py: 3.11\n","type":"rawscript","assets":[],"content":"from f.rel.leaf_2 import main as lf_2;\n\ndef check():\n return [lf_2()];\n\ndef main(x: str):\n return x","language":"python3","input_transforms":{"x":{"type":"static","value":""}}}}],"summary":"","parallel":true,"skip_failure":false}],"parallel":true},"summary":""},{"id":"nstep3","value":{"type":"branchone","default":[{"id":"nstep3_2","value":{"lock":"{\n \"dependencies\": {}\n}\n//bun.lock\n","type":"rawscript","assets":[],"content":"// import * as wmill from \"windmill-client\"\n\nexport async function main(x: string) {\n return x\n}\n","language":"bun","input_transforms":{"x":{"type":"static","value":""}}}}],"branches":[{"expr":"false","modules":[{"id":"nstep3_1","value":{"lock":"# py: 3.11\n","type":"rawscript","assets":[],"content":"def main(x: str):\n return x","language":"python3","input_transforms":{"x":{"type":"static","value":""}}}}],"summary":"","parallel":true,"skip_failure":true}]},"summary":""},{"id":"nstep4","value":{"type":"whileloopflow","modules":[{"id":"nstep4_1","value":{"lock":"# py: 3.11\n","type":"rawscript","assets":[],"content":"from f.rel.branch import main as br;\n\ndef check():\n return [br()];\n\ndef main(x: str):\n return x","language":"python3","input_transforms":{"x":{"type":"static","value":""}}}}],"skip_failures":false}},{"id":"nstep5","value":{"type":"forloopflow","modules":[{"id":"nstep5_1","value":{"lock":"# py: 3.11\n","type":"rawscript","assets":[],"content":"from f.rel.branch import main as br;\nfrom f.rel.leaf_1 import main as lf_1;\nfrom ..leaf_1 import main as lf_12;\nfrom ...rel.leaf_2 import main as lf_2;\n\ndef check():\n return [br(), lf_1(), lf_2(), lf_12()];\n\ndef main(x: str):\n return x","language":"python3","input_transforms":{"x":{"type":"static","value":""}}}}],"iterator":{"expr":"['dynamic or static array']","type":"javascript"},"parallel":false,"skip_failures":true}},{"id":"nstep_ai","value":{"type":"aiagent","tools":[{"id":"qtool1","value":{"type":"rawscript","content":"from f.rel.leaf_2 import main as lf_2;\n\ndef check():\n return [lf_2()];\n\ndef main(x):\n return x\n","language":"python3","input_transforms":{"x":{"type":"static"}}},"summary":"tool"}],"input_transforms":{"image":{"type":"static"},"provider":{"type":"static","value":{"kind":"openai"}},"output_type":{"type":"static","value":"text"},"temperature":{"type":"static"},"user_message":{"type":"static","value":""},"output_schema":{"type":"static"},"system_prompt":{"type":"static","value":""},"max_completion_tokens":{"type":"static"}}},"continue_on_error":false}],"failure_module":{"id":"failure","value":{"type":"rawscript","content":"from f.rel.leaf_2 import main as lf_2;\n\ndef check():\n return [lf_2()];\n\nimport os\n\ndef main(message: str, name: str, step_id: str):\n flow_id = os.environ.get(\"WM_ROOT_FLOW_JOB_ID\")\n print(\"message\", message)\n print(\"name\", name)\n print(\"step_id\", step_id)\n return { \"message\": message, \"flow_id\": flow_id, \"step_id\": step_id, \"recover\": False }","language":"python3","input_transforms":{"name":{"expr":"error.name","type":"javascript"},"message":{"expr":"error.message","type":"javascript"},"step_id":{"expr":"error.step_id","type":"javascript"}}}},"preprocessor_module":{"id":"preprocessor","value":{"type":"rawscript","content":"from f.rel.branch import main as br;\nfrom f.rel.leaf_1 import main as lf_1;\nfrom ..leaf_1 import main as lf_12;\nfrom ...rel.leaf_2 import main as lf_2;\n\ndef check():\n return [br(), lf_1(), lf_2(), lf_12()];\n\ndef preprocessor(event):\n return {\n # return the args to be passed to the runnable\n }\n","language":"python3","input_transforms":{"event":{"type":"static"}}}}}$tag$, +$tag${"modules":[{"id":"nstep1","value":{"lock":"# py: 3.11\n","type":"rawscript","assets":[],"content":"#requirements: test\nfrom f.rel.branch import main as br;\nfrom f.rel.leaf_1 import main as lf_1;\nfrom ..leaf_1 import main as lf_12;\nfrom ...rel.leaf_2 import main as lf_2;\n\ndef check():\n return [br(), lf_1(), lf_2(), lf_12()];\n\ndef main(x: str):\n return x","language":"python3","input_transforms":{"x":{"type":"static","value":""}}}},{"id":"nstep2","value":{"type":"branchall","branches":[{"expr":"","modules":[{"id":"nstep2_1","value":{"lock":"{\n \"dependencies\": {}\n}\n//bun.lock\n","type":"rawscript","assets":[],"content":"// import * as wmill from \"windmill-client\"\n\nexport async function main(x: string) {\n return x\n}\n","language":"bun","input_transforms":{"x":{"type":"static","value":""}}}}],"parallel":true,"skip_failure":false},{"expr":"false","modules":[{"id":"nstep2_2","value":{"lock":"# py: 3.11\n","type":"rawscript","assets":[],"content":"from f.rel.leaf_2 import main as lf_2;\n\ndef check():\n return [lf_2()];\n\ndef main(x: str):\n return x","language":"python3","input_transforms":{"x":{"type":"static","value":""}}}}],"summary":"","parallel":true,"skip_failure":false}],"parallel":true},"summary":""},{"id":"nstep3","value":{"type":"branchone","default":[{"id":"nstep3_2","value":{"lock":"{\n \"dependencies\": {}\n}\n//bun.lock\n","type":"rawscript","assets":[],"content":"// import * as wmill from \"windmill-client\"\n\nexport async function main(x: string) {\n return x\n}\n","language":"bun","input_transforms":{"x":{"type":"static","value":""}}}}],"branches":[{"expr":"false","modules":[{"id":"nstep3_1","value":{"lock":"# py: 3.11\n","type":"rawscript","assets":[],"content":"def main(x: str):\n return x","language":"python3","input_transforms":{"x":{"type":"static","value":""}}}}],"summary":"","parallel":true,"skip_failure":true}]},"summary":""},{"id":"nstep4","value":{"type":"whileloopflow","modules":[{"id":"nstep4_1","value":{"lock":"# py: 3.11\n","type":"rawscript","assets":[],"content":"from f.rel.branch import main as br;\n\ndef check():\n return [br()];\n\ndef main(x: str):\n return x","language":"python3","input_transforms":{"x":{"type":"static","value":""}}}}],"skip_failures":false}},{"id":"nstep5","value":{"type":"forloopflow","modules":[{"id":"nstep5_1","value":{"lock":"# py: 3.11\n","type":"rawscript","assets":[],"content":"from f.rel.branch import main as br;\nfrom f.rel.leaf_1 import main as lf_1;\nfrom ..leaf_1 import main as lf_12;\nfrom ...rel.leaf_2 import main as lf_2;\n\ndef check():\n return [br(), lf_1(), lf_2(), lf_12()];\n\ndef main(x: str):\n return x","language":"python3","input_transforms":{"x":{"type":"static","value":""}}}}],"iterator":{"expr":"['dynamic or static array']","type":"javascript"},"parallel":false,"skip_failures":true}},{"id":"nstep_ai","value":{"type":"aiagent","tools":[{"id":"qtool1","value":{"type":"rawscript","content":"from f.rel.leaf_2 import main as lf_2;\n\ndef check():\n return [lf_2()];\n\ndef main(x):\n return x\n","language":"python3","input_transforms":{"x":{"type":"static"}}},"summary":"tool"}],"input_transforms":{"image":{"type":"static"},"provider":{"type":"static","value":{"kind":"openai"}},"output_type":{"type":"static","value":"text"},"temperature":{"type":"static"},"user_message":{"type":"static","value":""},"output_schema":{"type":"static"},"system_prompt":{"type":"static","value":""},"max_completion_tokens":{"type":"static"}}},"continue_on_error":false}],"failure_module":{"id":"failure","value":{"type":"rawscript","content":"from f.rel.leaf_2 import main as lf_2;\n\ndef check():\n return [lf_2()];\n\nimport os\n\ndef main(message: str, name: str, step_id: str):\n flow_id = os.environ.get(\"WM_ROOT_FLOW_JOB_ID\")\n print(\"message\", message)\n print(\"name\", name)\n print(\"step_id\", step_id)\n return { \"message\": message, \"flow_id\": flow_id, \"step_id\": step_id, \"recover\": False }","language":"python3","input_transforms":{"name":{"expr":"error.name","type":"javascript"},"message":{"expr":"error.message","type":"javascript"},"step_id":{"expr":"error.step_id","type":"javascript"}}}},"preprocessor_module":{"id":"preprocessor","value":{"type":"rawscript","content":"from f.rel.branch import main as br;\nfrom f.rel.leaf_1 import main as lf_1;\nfrom ..leaf_1 import main as lf_12;\nfrom ...rel.leaf_2 import main as lf_2;\n\ndef check():\n return [br(), lf_1(), lf_2(), lf_12()];\n\ndef preprocessor(event):\n return {\n # return the args to be passed to the runnable\n }\n","language":"python3","input_transforms":{"event":{"type":"static"}}}}}$tag$, 'system' ); @@ -72,7 +73,7 @@ INSERT INTO public.flow_version(id, workspace_id, path, schema, value, created_b 'test-workspace', 'f/rel/root_flow', '{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object","order":[]}', -$tag${"modules":[{"id":"nstep1","value":{"lock":"# py: 3.11\n","type":"rawscript","assets":[],"content":"from f.rel.branch import main as br;\nfrom f.rel.leaf_1 import main as lf_1;\nfrom ..leaf_1 import main as lf_12;\nfrom ...rel.leaf_2 import main as lf_2;\n\ndef check():\n return [br(), lf_1(), lf_2(), lf_12()];\n\ndef main(x: str):\n return x","language":"python3","input_transforms":{"x":{"type":"static","value":""}}}},{"id":"nstep2","value":{"type":"branchall","branches":[{"expr":"","modules":[{"id":"nstep2_1","value":{"lock":"{\n \"dependencies\": {}\n}\n//bun.lock\n","type":"rawscript","assets":[],"content":"// import * as wmill from \"windmill-client\"\n\nexport async function main(x: string) {\n return x\n}\n","language":"bun","input_transforms":{"x":{"type":"static","value":""}}}}],"parallel":true,"skip_failure":false},{"expr":"false","modules":[{"id":"nstep2_2","value":{"lock":"# py: 3.11\n","type":"rawscript","assets":[],"content":"from f.rel.leaf_2 import main as lf_2;\n\ndef check():\n return [lf_2()];\n\ndef main(x: str):\n return x","language":"python3","input_transforms":{"x":{"type":"static","value":""}}}}],"summary":"","parallel":true,"skip_failure":false}],"parallel":true},"summary":""},{"id":"nstep3","value":{"type":"branchone","default":[{"id":"nstep3_2","value":{"lock":"{\n \"dependencies\": {}\n}\n//bun.lock\n","type":"rawscript","assets":[],"content":"// import * as wmill from \"windmill-client\"\n\nexport async function main(x: string) {\n return x\n}\n","language":"bun","input_transforms":{"x":{"type":"static","value":""}}}}],"branches":[{"expr":"false","modules":[{"id":"nstep3_1","value":{"lock":"# py: 3.11\n","type":"rawscript","assets":[],"content":"def main(x: str):\n return x","language":"python3","input_transforms":{"x":{"type":"static","value":""}}}}],"summary":"","parallel":true,"skip_failure":true}]},"summary":""},{"id":"nstep4","value":{"type":"whileloopflow","modules":[{"id":"nstep4_1","value":{"lock":"# py: 3.11\n","type":"rawscript","assets":[],"content":"from f.rel.branch import main as br;\n\ndef check():\n return [br()];\n\ndef main(x: str):\n return x","language":"python3","input_transforms":{"x":{"type":"static","value":""}}}}],"skip_failures":false}},{"id":"nstep5","value":{"type":"forloopflow","modules":[{"id":"nstep5_1","value":{"lock":"# py: 3.11\n","type":"rawscript","assets":[],"content":"from f.rel.branch import main as br;\nfrom f.rel.leaf_1 import main as lf_1;\nfrom ..leaf_1 import main as lf_12;\nfrom ...rel.leaf_2 import main as lf_2;\n\ndef check():\n return [br(), lf_1(), lf_2(), lf_12()];\n\ndef main(x: str):\n return x","language":"python3","input_transforms":{"x":{"type":"static","value":""}}}}],"iterator":{"expr":"['dynamic or static array']","type":"javascript"},"parallel":false,"skip_failures":true}},{"id":"nstep_ai","value":{"type":"aiagent","tools":[{"id":"qtool1","value":{"type":"rawscript","content":"from f.rel.leaf_2 import main as lf_2;\n\ndef check():\n return [lf_2()];\n\ndef main(x):\n return x\n","language":"python3","input_transforms":{"x":{"type":"static"}}},"summary":"tool"}],"input_transforms":{"image":{"type":"static"},"provider":{"type":"static","value":{"kind":"openai"}},"output_type":{"type":"static","value":"text"},"temperature":{"type":"static"},"user_message":{"type":"static","value":""},"output_schema":{"type":"static"},"system_prompt":{"type":"static","value":""},"max_completion_tokens":{"type":"static"}}},"continue_on_error":false}],"failure_module":{"id":"failure","value":{"type":"rawscript","content":"from f.rel.leaf_2 import main as lf_2;\n\ndef check():\n return [lf_2()];\n\nimport os\n\ndef main(message: str, name: str, step_id: str):\n flow_id = os.environ.get(\"WM_ROOT_FLOW_JOB_ID\")\n print(\"message\", message)\n print(\"name\", name)\n print(\"step_id\", step_id)\n return { \"message\": message, \"flow_id\": flow_id, \"step_id\": step_id, \"recover\": False }","language":"python3","input_transforms":{"name":{"expr":"error.name","type":"javascript"},"message":{"expr":"error.message","type":"javascript"},"step_id":{"expr":"error.step_id","type":"javascript"}}}},"preprocessor_module":{"id":"preprocessor","value":{"type":"rawscript","content":"from f.rel.branch import main as br;\nfrom f.rel.leaf_1 import main as lf_1;\nfrom ..leaf_1 import main as lf_12;\nfrom ...rel.leaf_2 import main as lf_2;\n\ndef check():\n return [br(), lf_1(), lf_2(), lf_12()];\n\ndef preprocessor(event):\n return {\n # return the args to be passed to the runnable\n }\n","language":"python3","input_transforms":{"event":{"type":"static"}}}}}$tag$, +$tag${"modules":[{"id":"nstep1","value":{"lock":"# py: 3.11\n","type":"rawscript","assets":[],"content":"#requirements: test\nfrom f.rel.branch import main as br;\nfrom f.rel.leaf_1 import main as lf_1;\nfrom ..leaf_1 import main as lf_12;\nfrom ...rel.leaf_2 import main as lf_2;\n\ndef check():\n return [br(), lf_1(), lf_2(), lf_12()];\n\ndef main(x: str):\n return x","language":"python3","input_transforms":{"x":{"type":"static","value":""}}}},{"id":"nstep2","value":{"type":"branchall","branches":[{"expr":"","modules":[{"id":"nstep2_1","value":{"lock":"{\n \"dependencies\": {}\n}\n//bun.lock\n","type":"rawscript","assets":[],"content":"// import * as wmill from \"windmill-client\"\n\nexport async function main(x: string) {\n return x\n}\n","language":"bun","input_transforms":{"x":{"type":"static","value":""}}}}],"parallel":true,"skip_failure":false},{"expr":"false","modules":[{"id":"nstep2_2","value":{"lock":"# py: 3.11\n","type":"rawscript","assets":[],"content":"from f.rel.leaf_2 import main as lf_2;\n\ndef check():\n return [lf_2()];\n\ndef main(x: str):\n return x","language":"python3","input_transforms":{"x":{"type":"static","value":""}}}}],"summary":"","parallel":true,"skip_failure":false}],"parallel":true},"summary":""},{"id":"nstep3","value":{"type":"branchone","default":[{"id":"nstep3_2","value":{"lock":"{\n \"dependencies\": {}\n}\n//bun.lock\n","type":"rawscript","assets":[],"content":"// import * as wmill from \"windmill-client\"\n\nexport async function main(x: string) {\n return x\n}\n","language":"bun","input_transforms":{"x":{"type":"static","value":""}}}}],"branches":[{"expr":"false","modules":[{"id":"nstep3_1","value":{"lock":"# py: 3.11\n","type":"rawscript","assets":[],"content":"def main(x: str):\n return x","language":"python3","input_transforms":{"x":{"type":"static","value":""}}}}],"summary":"","parallel":true,"skip_failure":true}]},"summary":""},{"id":"nstep4","value":{"type":"whileloopflow","modules":[{"id":"nstep4_1","value":{"lock":"# py: 3.11\n","type":"rawscript","assets":[],"content":"from f.rel.branch import main as br;\n\ndef check():\n return [br()];\n\ndef main(x: str):\n return x","language":"python3","input_transforms":{"x":{"type":"static","value":""}}}}],"skip_failures":false}},{"id":"nstep5","value":{"type":"forloopflow","modules":[{"id":"nstep5_1","value":{"lock":"# py: 3.11\n","type":"rawscript","assets":[],"content":"from f.rel.branch import main as br;\nfrom f.rel.leaf_1 import main as lf_1;\nfrom ..leaf_1 import main as lf_12;\nfrom ...rel.leaf_2 import main as lf_2;\n\ndef check():\n return [br(), lf_1(), lf_2(), lf_12()];\n\ndef main(x: str):\n return x","language":"python3","input_transforms":{"x":{"type":"static","value":""}}}}],"iterator":{"expr":"['dynamic or static array']","type":"javascript"},"parallel":false,"skip_failures":true}},{"id":"nstep_ai","value":{"type":"aiagent","tools":[{"id":"qtool1","value":{"type":"rawscript","content":"from f.rel.leaf_2 import main as lf_2;\n\ndef check():\n return [lf_2()];\n\ndef main(x):\n return x\n","language":"python3","input_transforms":{"x":{"type":"static"}}},"summary":"tool"}],"input_transforms":{"image":{"type":"static"},"provider":{"type":"static","value":{"kind":"openai"}},"output_type":{"type":"static","value":"text"},"temperature":{"type":"static"},"user_message":{"type":"static","value":""},"output_schema":{"type":"static"},"system_prompt":{"type":"static","value":""},"max_completion_tokens":{"type":"static"}}},"continue_on_error":false}],"failure_module":{"id":"failure","value":{"type":"rawscript","content":"from f.rel.leaf_2 import main as lf_2;\n\ndef check():\n return [lf_2()];\n\nimport os\n\ndef main(message: str, name: str, step_id: str):\n flow_id = os.environ.get(\"WM_ROOT_FLOW_JOB_ID\")\n print(\"message\", message)\n print(\"name\", name)\n print(\"step_id\", step_id)\n return { \"message\": message, \"flow_id\": flow_id, \"step_id\": step_id, \"recover\": False }","language":"python3","input_transforms":{"name":{"expr":"error.name","type":"javascript"},"message":{"expr":"error.message","type":"javascript"},"step_id":{"expr":"error.step_id","type":"javascript"}}}},"preprocessor_module":{"id":"preprocessor","value":{"type":"rawscript","content":"from f.rel.branch import main as br;\nfrom f.rel.leaf_1 import main as lf_1;\nfrom ..leaf_1 import main as lf_12;\nfrom ...rel.leaf_2 import main as lf_2;\n\ndef check():\n return [br(), lf_1(), lf_2(), lf_12()];\n\ndef preprocessor(event):\n return {\n # return the args to be passed to the runnable\n }\n","language":"python3","input_transforms":{"event":{"type":"static"}}}}}$tag$, 'system' ); @@ -87,19 +88,35 @@ INSERT INTO public.app(id, workspace_id, path, versions, policy) VALUES ( INSERT INTO public.app_version(id, app_id, value, created_by) VALUES ( 0, 2, -$tag${"grid":[{"3":{"h":2,"w":6,"x":0,"y":0,"fixed":true,"fullHeight":false},"12":{"h":2,"w":12,"x":0,"y":0,"fixed":true,"fullHeight":false},"id":"topbar","data":{"id":"topbar","type":"containercomponent","customCss":{"container":{"class":"!p-0","style":""}},"configuration":{},"numberOfSubgrids":1}},{"3":{"h":8,"w":2,"x":0,"y":2,"fixed":false,"fullHeight":false},"12":{"h":2,"w":6,"x":0,"y":2,"fixed":false,"fullHeight":false},"id":"a","data":{"id":"a","type":"containercomponent","customCss":{"container":{"class":"","style":""}},"configuration":{},"numberOfSubgrids":1}},{"3":{"h":1,"w":1,"x":2,"y":2,"fixed":false,"fullHeight":false},"12":{"h":1,"w":2,"x":6,"y":2,"fixed":false,"fullHeight":false},"id":"dontpressmeplz","data":{"id":"dontpressmeplz","type":"buttoncomponent","customCss":{"button":{"class":"","style":""},"container":{"class":"","style":""}},"recomputeIds":[],"configuration":{"size":{"type":"static","value":"xs"},"color":{"type":"static","value":"blue"},"label":{"type":"static","value":"Press me"},"onError":{"type":"oneOf","selected":"errorOverlay","configuration":{"open":{"id":{"type":"static","value":""}},"close":{"id":{"type":"static","value":""}},"setTab":{"setTab":{"type":"static","value":[]}},"gotoUrl":{"url":{"type":"static","value":""},"newTab":{"type":"static","value":true}},"errorOverlay":{},"sendErrorToast":{"message":{"type":"static","value":"An error occured"},"appendError":{"type":"static","value":true}}}},"disabled":{"type":"static","value":false},"afterIcon":{"type":"static"},"onSuccess":{"type":"oneOf","selected":"none","configuration":{"none":{},"open":{"id":{"type":"static","value":""}},"close":{"id":{"type":"static","value":""}},"setTab":{"setTab":{"type":"static","value":[]}},"gotoUrl":{"url":{"type":"static","value":""},"newTab":{"type":"static","value":true}},"openModal":{"modalId":{"type":"static","value":""}},"sendToast":{"message":{"type":"static","value":""}},"clearFiles":{"id":{"type":"static","value":""}},"closeModal":{"modalId":{"type":"static","value":""}}}},"beforeIcon":{"type":"static"},"fillContainer":{"type":"static","value":false},"triggerOnAppLoad":{"type":"static","value":false},"confirmationModal":{"type":"oneOf","selected":"none","configuration":{"none":{},"confirmationModal":{"title":{"type":"static","value":"Title"},"description":{"type":"static","value":"Are you sure?"},"confirmationText":{"type":"static","value":"Confirm"}}}}},"componentInput":{"type":"runnable","fields":{"x":{"type":"static","value":null,"fieldType":"string"}},"runnable":{"name":"Inline Script","type":"runnableByName","inlineScript":{"path":"u/admin@windmill.dev/newapp/Inline_Script","schema":{"type":"object","$schema":"https://json-schema.org/draft/2020-12/schema","required":["x"],"properties":{"x":{"default":null,"description":"","originalType":"string","type":"string"}}},"content":"from f.rel.leaf_2 import main as lf_2;\n\ndef check():\n return [lf_2()];\n \ndef main(x: str):\n return x","language":"python3"}},"fieldType":"any","autoRefresh":false,"recomputeOnInputChanged":false},"verticalAlignment":"center","horizontalAlignment":"center"}},{"3":{"h":1,"w":1,"x":2,"y":3,"fixed":false,"fullHeight":false},"12":{"h":1,"w":2,"x":8,"y":2,"fixed":false,"fullHeight":false},"id":"d","data":{"id":"d","type":"checkboxcomponent","customCss":{"text":{"class":"","style":""},"container":{"class":"","style":""}},"recomputeIds":[],"configuration":{"label":{"type":"static","value":"Label"},"disabled":{"type":"static","value":false},"defaultValue":{"type":"static","value":false}},"verticalAlignment":"center","horizontalAlignment":"center"}},{"3":{"h":1,"w":1,"x":2,"y":4,"fixed":false,"fullHeight":false},"12":{"h":1,"w":2,"x":6,"y":3,"fixed":false,"fullHeight":false},"id":"youcanpressme","data":{"id":"youcanpressme","type":"buttoncomponent","customCss":{"button":{"class":"","style":""},"container":{"class":"","style":""}},"recomputeIds":[],"configuration":{"size":{"type":"static","value":"xs"},"color":{"type":"static","value":"blue"},"label":{"type":"static","value":"Press me"},"onError":{"type":"oneOf","selected":"errorOverlay","configuration":{"open":{"id":{"type":"static","value":""}},"close":{"id":{"type":"static","value":""}},"setTab":{"setTab":{"type":"static","value":[]}},"gotoUrl":{"url":{"type":"static","value":""},"newTab":{"type":"static","value":true}},"errorOverlay":{},"sendErrorToast":{"message":{"type":"static","value":"An error occured"},"appendError":{"type":"static","value":true}}}},"disabled":{"type":"static","value":false},"afterIcon":{"type":"static"},"onSuccess":{"type":"oneOf","selected":"none","configuration":{"none":{},"open":{"id":{"type":"static","value":""}},"close":{"id":{"type":"static","value":""}},"setTab":{"setTab":{"type":"static","value":[]}},"gotoUrl":{"url":{"type":"static","value":""},"newTab":{"type":"static","value":true}},"openModal":{"modalId":{"type":"static","value":""}},"sendToast":{"message":{"type":"static","value":""}},"clearFiles":{"id":{"type":"static","value":""}},"closeModal":{"modalId":{"type":"static","value":""}}}},"beforeIcon":{"type":"static"},"fillContainer":{"type":"static","value":false},"triggerOnAppLoad":{"type":"static","value":false},"confirmationModal":{"type":"oneOf","selected":"none","configuration":{"none":{},"confirmationModal":{"title":{"type":"static","value":"Title"},"description":{"type":"static","value":"Are you sure?"},"confirmationText":{"type":"static","value":"Confirm"}}}}},"componentInput":{"type":"runnable","fields":{"x":{"type":"static","value":null,"fieldType":"string"}},"runnable":{"name":"Inline Script","type":"runnableByName","inlineScript":{"path":"u/admin/easy_to_use_app/Inline_Script","schema":{"type":"object","$schema":"https://json-schema.org/draft/2020-12/schema","required":["x"],"properties":{"x":{"default":null,"description":"","originalType":"string","type":"string"}}},"content":"from f.rel.branch import main as br;\n\ndef check():\n return [br()];\n\ndef main(x: str):\n return x","language":"python3"}},"fieldType":"any","autoRefresh":false,"recomputeOnInputChanged":false},"verticalAlignment":"center","horizontalAlignment":"center"}}],"theme":{"path":"f/app_themes/theme_0","type":"path"},"subgrids":{"a-0":[{"3":{"h":1,"w":1,"x":0,"y":0,"fixed":false,"fullHeight":false},"12":{"h":2,"w":5,"x":0,"y":0,"fixed":false,"fullHeight":false},"id":"pressmeplz","data":{"id":"pressmeplz","type":"buttoncomponent","customCss":{"button":{"class":"","style":""},"container":{"class":"","style":""}},"recomputeIds":[],"configuration":{"size":{"type":"static","value":"xs"},"color":{"type":"static","value":"blue"},"label":{"type":"static","value":"Press me"},"onError":{"type":"oneOf","selected":"errorOverlay","configuration":{"open":{"id":{"type":"static","value":""}},"close":{"id":{"type":"static","value":""}},"setTab":{"setTab":{"type":"static","value":[]}},"gotoUrl":{"url":{"type":"static","value":""},"newTab":{"type":"static","value":true}},"errorOverlay":{},"sendErrorToast":{"message":{"type":"static","value":"An error occured"},"appendError":{"type":"static","value":true}}}},"disabled":{"type":"static","value":false},"afterIcon":{"type":"static"},"onSuccess":{"type":"oneOf","selected":"none","configuration":{"none":{},"open":{"id":{"type":"static","value":""}},"close":{"id":{"type":"static","value":""}},"setTab":{"setTab":{"type":"static","value":[]}},"gotoUrl":{"url":{"type":"static","value":""},"newTab":{"type":"static","value":true}},"openModal":{"modalId":{"type":"static","value":""}},"sendToast":{"message":{"type":"static","value":""}},"clearFiles":{"id":{"type":"static","value":""}},"closeModal":{"modalId":{"type":"static","value":""}}}},"beforeIcon":{"type":"static"},"fillContainer":{"type":"static","value":false},"triggerOnAppLoad":{"type":"static","value":false},"confirmationModal":{"type":"oneOf","selected":"none","configuration":{"none":{},"confirmationModal":{"title":{"type":"static","value":"Title"},"description":{"type":"static","value":"Are you sure?"},"confirmationText":{"type":"static","value":"Confirm"}}}}},"componentInput":{"type":"runnable","fields":{"x":{"type":"static","value":null,"fieldType":"string"}},"runnable":{"name":"Inline Script","type":"runnableByName","inlineScript":{"path":"u/admin@windmill.dev/newapp/Inline_Script","schema":{"type":"object","$schema":"https://json-schema.org/draft/2020-12/schema","required":["x"],"properties":{"x":{"default":null,"description":"","originalType":"string","type":"string"}}},"content":"from f.rel.branch import main as br;\nfrom f.rel.leaf_1 import main as lf_1;\nfrom ..leaf_1 import main as lf_12;\nfrom ...rel.leaf_2 import main as lf_2;\n\ndef check():\n return [br(), lf_1(), lf_2(), lf_12()];\n\n\ndef main(x: str):\n return x","language":"python3"}},"fieldType":"any","autoRefresh":false,"recomputeOnInputChanged":false},"verticalAlignment":"center","horizontalAlignment":"center"}}],"topbar-0":[{"3":{"h":1,"w":6,"x":0,"y":0,"fixed":false,"fullHeight":false},"12":{"h":1,"w":6,"x":0,"y":0,"fixed":false,"fullHeight":false},"id":"title","data":{"id":"title","type":"textcomponent","customCss":{"text":{"class":"text-xl font-semibold whitespace-nowrap truncate","style":""},"container":{"class":"","style":""}},"configuration":{"style":{"type":"static","value":"Body"},"tooltip":{"expr":"`Author: ${ctx.author}`","type":"evalv2","value":"","fieldType":"text","connections":[{"id":"author","componentId":"ctx"}]},"copyButton":{"type":"static","value":false},"disableNoText":{"type":"static","value":true,"fieldType":"boolean"}},"componentInput":{"eval":"${ctx.summary}","type":"templatev2","fieldType":"template","connections":[{"id":"summary","componentId":"ctx"}]},"verticalAlignment":"center","horizontalAlignment":"left"}},{"3":{"h":1,"w":3,"x":0,"y":1,"fixed":false,"fullHeight":false},"12":{"h":1,"w":6,"x":6,"y":0,"fixed":false,"fullHeight":false},"id":"recomputeall","data":{"id":"recomputeall","type":"recomputeallcomponent","customCss":{"container":{"class":"","style":""}},"menuItems":[],"configuration":{"defaultRefreshInterval":{"type":"static","value":"0"}},"verticalAlignment":"center","horizontalAlignment":"right"}}]},"fullscreen":false,"norefreshbar":false,"hideLegacyTopBar":true,"hiddenInlineScripts":[],"unusedInlineScripts":[],"mobileViewOnSmallerScreens":false}$tag$, +$tag${"grid":[{"3":{"h":2,"w":6,"x":0,"y":0,"fixed":true,"fullHeight":false},"12":{"h":2,"w":12,"x":0,"y":0,"fixed":true,"fullHeight":false},"id":"topbar","data":{"id":"topbar","type":"containercomponent","customCss":{"container":{"class":"!p-0","style":""}},"configuration":{},"numberOfSubgrids":1}},{"3":{"h":8,"w":2,"x":0,"y":2,"fixed":false,"fullHeight":false},"12":{"h":2,"w":6,"x":0,"y":2,"fixed":false,"fullHeight":false},"id":"a","data":{"id":"a","type":"containercomponent","customCss":{"container":{"class":"","style":""}},"configuration":{},"numberOfSubgrids":1}},{"3":{"h":1,"w":1,"x":2,"y":2,"fixed":false,"fullHeight":false},"12":{"h":1,"w":2,"x":6,"y":2,"fixed":false,"fullHeight":false},"id":"dontpressmeplz","data":{"id":"dontpressmeplz","type":"buttoncomponent","customCss":{"button":{"class":"","style":""},"container":{"class":"","style":""}},"recomputeIds":[],"configuration":{"size":{"type":"static","value":"xs"},"color":{"type":"static","value":"blue"},"label":{"type":"static","value":"Press me"},"onError":{"type":"oneOf","selected":"errorOverlay","configuration":{"open":{"id":{"type":"static","value":""}},"close":{"id":{"type":"static","value":""}},"setTab":{"setTab":{"type":"static","value":[]}},"gotoUrl":{"url":{"type":"static","value":""},"newTab":{"type":"static","value":true}},"errorOverlay":{},"sendErrorToast":{"message":{"type":"static","value":"An error occured"},"appendError":{"type":"static","value":true}}}},"disabled":{"type":"static","value":false},"afterIcon":{"type":"static"},"onSuccess":{"type":"oneOf","selected":"none","configuration":{"none":{},"open":{"id":{"type":"static","value":""}},"close":{"id":{"type":"static","value":""}},"setTab":{"setTab":{"type":"static","value":[]}},"gotoUrl":{"url":{"type":"static","value":""},"newTab":{"type":"static","value":true}},"openModal":{"modalId":{"type":"static","value":""}},"sendToast":{"message":{"type":"static","value":""}},"clearFiles":{"id":{"type":"static","value":""}},"closeModal":{"modalId":{"type":"static","value":""}}}},"beforeIcon":{"type":"static"},"fillContainer":{"type":"static","value":false},"triggerOnAppLoad":{"type":"static","value":false},"confirmationModal":{"type":"oneOf","selected":"none","configuration":{"none":{},"confirmationModal":{"title":{"type":"static","value":"Title"},"description":{"type":"static","value":"Are you sure?"},"confirmationText":{"type":"static","value":"Confirm"}}}}},"componentInput":{"type":"runnable","fields":{"x":{"type":"static","value":null,"fieldType":"string"}},"runnable":{"name":"Inline Script","type":"runnableByName","inlineScript":{"path":"u/admin@windmill.dev/newapp/Inline_Script","schema":{"type":"object","$schema":"https://json-schema.org/draft/2020-12/schema","required":["x"],"properties":{"x":{"default":null,"description":"","originalType":"string","type":"string"}}},"content":"#requirements: test\nfrom f.rel.leaf_2 import main as lf_2;\n\ndef check():\n return [lf_2()];\n \ndef main(x: str):\n return x","language":"python3"}},"fieldType":"any","autoRefresh":false,"recomputeOnInputChanged":false},"verticalAlignment":"center","horizontalAlignment":"center"}},{"3":{"h":1,"w":1,"x":2,"y":3,"fixed":false,"fullHeight":false},"12":{"h":1,"w":2,"x":8,"y":2,"fixed":false,"fullHeight":false},"id":"d","data":{"id":"d","type":"checkboxcomponent","customCss":{"text":{"class":"","style":""},"container":{"class":"","style":""}},"recomputeIds":[],"configuration":{"label":{"type":"static","value":"Label"},"disabled":{"type":"static","value":false},"defaultValue":{"type":"static","value":false}},"verticalAlignment":"center","horizontalAlignment":"center"}},{"3":{"h":1,"w":1,"x":2,"y":4,"fixed":false,"fullHeight":false},"12":{"h":1,"w":2,"x":6,"y":3,"fixed":false,"fullHeight":false},"id":"youcanpressme","data":{"id":"youcanpressme","type":"buttoncomponent","customCss":{"button":{"class":"","style":""},"container":{"class":"","style":""}},"recomputeIds":[],"configuration":{"size":{"type":"static","value":"xs"},"color":{"type":"static","value":"blue"},"label":{"type":"static","value":"Press me"},"onError":{"type":"oneOf","selected":"errorOverlay","configuration":{"open":{"id":{"type":"static","value":""}},"close":{"id":{"type":"static","value":""}},"setTab":{"setTab":{"type":"static","value":[]}},"gotoUrl":{"url":{"type":"static","value":""},"newTab":{"type":"static","value":true}},"errorOverlay":{},"sendErrorToast":{"message":{"type":"static","value":"An error occured"},"appendError":{"type":"static","value":true}}}},"disabled":{"type":"static","value":false},"afterIcon":{"type":"static"},"onSuccess":{"type":"oneOf","selected":"none","configuration":{"none":{},"open":{"id":{"type":"static","value":""}},"close":{"id":{"type":"static","value":""}},"setTab":{"setTab":{"type":"static","value":[]}},"gotoUrl":{"url":{"type":"static","value":""},"newTab":{"type":"static","value":true}},"openModal":{"modalId":{"type":"static","value":""}},"sendToast":{"message":{"type":"static","value":""}},"clearFiles":{"id":{"type":"static","value":""}},"closeModal":{"modalId":{"type":"static","value":""}}}},"beforeIcon":{"type":"static"},"fillContainer":{"type":"static","value":false},"triggerOnAppLoad":{"type":"static","value":false},"confirmationModal":{"type":"oneOf","selected":"none","configuration":{"none":{},"confirmationModal":{"title":{"type":"static","value":"Title"},"description":{"type":"static","value":"Are you sure?"},"confirmationText":{"type":"static","value":"Confirm"}}}}},"componentInput":{"type":"runnable","fields":{"x":{"type":"static","value":null,"fieldType":"string"}},"runnable":{"name":"Inline Script","type":"runnableByName","inlineScript":{"path":"u/admin/easy_to_use_app/Inline_Script","schema":{"type":"object","$schema":"https://json-schema.org/draft/2020-12/schema","required":["x"],"properties":{"x":{"default":null,"description":"","originalType":"string","type":"string"}}},"content":"from f.rel.branch import main as br;\n\ndef check():\n return [br()];\n\ndef main(x: str):\n return x","language":"python3"}},"fieldType":"any","autoRefresh":false,"recomputeOnInputChanged":false},"verticalAlignment":"center","horizontalAlignment":"center"}}],"theme":{"path":"f/app_themes/theme_0","type":"path"},"subgrids":{"a-0":[{"3":{"h":1,"w":1,"x":0,"y":0,"fixed":false,"fullHeight":false},"12":{"h":2,"w":5,"x":0,"y":0,"fixed":false,"fullHeight":false},"id":"pressmeplz","data":{"id":"pressmeplz","type":"buttoncomponent","customCss":{"button":{"class":"","style":""},"container":{"class":"","style":""}},"recomputeIds":[],"configuration":{"size":{"type":"static","value":"xs"},"color":{"type":"static","value":"blue"},"label":{"type":"static","value":"Press me"},"onError":{"type":"oneOf","selected":"errorOverlay","configuration":{"open":{"id":{"type":"static","value":""}},"close":{"id":{"type":"static","value":""}},"setTab":{"setTab":{"type":"static","value":[]}},"gotoUrl":{"url":{"type":"static","value":""},"newTab":{"type":"static","value":true}},"errorOverlay":{},"sendErrorToast":{"message":{"type":"static","value":"An error occured"},"appendError":{"type":"static","value":true}}}},"disabled":{"type":"static","value":false},"afterIcon":{"type":"static"},"onSuccess":{"type":"oneOf","selected":"none","configuration":{"none":{},"open":{"id":{"type":"static","value":""}},"close":{"id":{"type":"static","value":""}},"setTab":{"setTab":{"type":"static","value":[]}},"gotoUrl":{"url":{"type":"static","value":""},"newTab":{"type":"static","value":true}},"openModal":{"modalId":{"type":"static","value":""}},"sendToast":{"message":{"type":"static","value":""}},"clearFiles":{"id":{"type":"static","value":""}},"closeModal":{"modalId":{"type":"static","value":""}}}},"beforeIcon":{"type":"static"},"fillContainer":{"type":"static","value":false},"triggerOnAppLoad":{"type":"static","value":false},"confirmationModal":{"type":"oneOf","selected":"none","configuration":{"none":{},"confirmationModal":{"title":{"type":"static","value":"Title"},"description":{"type":"static","value":"Are you sure?"},"confirmationText":{"type":"static","value":"Confirm"}}}}},"componentInput":{"type":"runnable","fields":{"x":{"type":"static","value":null,"fieldType":"string"}},"runnable":{"name":"Inline Script","type":"runnableByName","inlineScript":{"path":"u/admin@windmill.dev/newapp/Inline_Script","schema":{"type":"object","$schema":"https://json-schema.org/draft/2020-12/schema","required":["x"],"properties":{"x":{"default":null,"description":"","originalType":"string","type":"string"}}},"content":"from f.rel.branch import main as br;\nfrom f.rel.leaf_1 import main as lf_1;\nfrom ..leaf_1 import main as lf_12;\nfrom ...rel.leaf_2 import main as lf_2;\n\ndef check():\n return [br(), lf_1(), lf_2(), lf_12()];\n\n\ndef main(x: str):\n return x","language":"python3"}},"fieldType":"any","autoRefresh":false,"recomputeOnInputChanged":false},"verticalAlignment":"center","horizontalAlignment":"center"}}],"topbar-0":[{"3":{"h":1,"w":6,"x":0,"y":0,"fixed":false,"fullHeight":false},"12":{"h":1,"w":6,"x":0,"y":0,"fixed":false,"fullHeight":false},"id":"title","data":{"id":"title","type":"textcomponent","customCss":{"text":{"class":"text-xl font-semibold whitespace-nowrap truncate","style":""},"container":{"class":"","style":""}},"configuration":{"style":{"type":"static","value":"Body"},"tooltip":{"expr":"`Author: ${ctx.author}`","type":"evalv2","value":"","fieldType":"text","connections":[{"id":"author","componentId":"ctx"}]},"copyButton":{"type":"static","value":false},"disableNoText":{"type":"static","value":true,"fieldType":"boolean"}},"componentInput":{"eval":"${ctx.summary}","type":"templatev2","fieldType":"template","connections":[{"id":"summary","componentId":"ctx"}]},"verticalAlignment":"center","horizontalAlignment":"left"}},{"3":{"h":1,"w":3,"x":0,"y":1,"fixed":false,"fullHeight":false},"12":{"h":1,"w":6,"x":6,"y":0,"fixed":false,"fullHeight":false},"id":"recomputeall","data":{"id":"recomputeall","type":"recomputeallcomponent","customCss":{"container":{"class":"","style":""}},"menuItems":[],"configuration":{"defaultRefreshInterval":{"type":"static","value":"0"}},"verticalAlignment":"center","horizontalAlignment":"right"}}]},"fullscreen":false,"norefreshbar":false,"hideLegacyTopBar":true,"hiddenInlineScripts":[],"unusedInlineScripts":[],"mobileViewOnSmallerScreens":false}$tag$, 'system' ); +INSERT INTO public.workspace_dependencies(name, content, language, workspace_id) VALUES ( +'test', +'', +'python3', +'test-workspace' +); + +INSERT INTO public.workspace_dependencies(content, language, workspace_id) VALUES ( +'', +'python3', +'test-workspace' +); + -- Prebuild dependency_map -- It would be done by Windmill, but this one is static. INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path, importer_node_id) VALUES ('test-workspace', 'f/rel/branch', 'script', 'f/rel/leaf_1', ''); INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path, importer_node_id) VALUES ('test-workspace', 'f/rel/root_script', 'script', 'f/rel/branch', ''); INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path, importer_node_id) VALUES ('test-workspace', 'f/rel/root_script', 'script', 'f/rel/leaf_1', ''); INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path, importer_node_id) VALUES ('test-workspace', 'f/rel/root_script', 'script', 'f/rel/leaf_2', ''); +INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path, importer_node_id) VALUES ('test-workspace', 'f/rel/root_script', 'script', 'dependencies/test.requirements.in', ''); INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path, importer_node_id) VALUES ('test-workspace', 'f/rel/root_app', 'app', 'f/rel/leaf_2', 'dontpressmeplz'); +INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path, importer_node_id) VALUES ('test-workspace', 'f/rel/root_app', 'app', 'dependencies/test.requirements.in', 'dontpressmeplz'); INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path, importer_node_id) VALUES ('test-workspace', 'f/rel/root_flow', 'flow', 'f/rel/leaf_2', 'failure'); INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path, importer_node_id) VALUES ('test-workspace', 'f/rel/root_flow', 'flow', 'f/rel/branch', 'nstep1'); +INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path, importer_node_id) VALUES ('test-workspace', 'f/rel/root_flow', 'flow', 'dependencies/test.requirements.in', 'nstep1'); INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path, importer_node_id) VALUES ('test-workspace', 'f/rel/root_flow', 'flow', 'f/rel/leaf_1', 'nstep1'); INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path, importer_node_id) VALUES ('test-workspace', 'f/rel/root_flow', 'flow', 'f/rel/leaf_2', 'nstep1'); INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path, importer_node_id) VALUES ('test-workspace', 'f/rel/root_flow', 'flow', 'f/rel/leaf_2', 'nstep2_2'); @@ -116,3 +133,18 @@ INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path, importer_node_id) VALUES ('test-workspace', 'f/rel/root_flow', 'flow', 'f/rel/leaf_2', 'qtool1'); INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path, importer_node_id) VALUES ('test-workspace', 'f/rel/root_app', 'app', 'f/rel/branch', 'youcanpressme'); +-- Default +INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path, importer_node_id) VALUES ('test-workspace', 'f/rel/leaf_1', 'script', 'dependencies/requirements.in', ''); +INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path, importer_node_id) VALUES ('test-workspace', 'f/rel/leaf_2', 'script', 'dependencies/requirements.in', ''); +INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path, importer_node_id) VALUES ('test-workspace', 'f/rel/branch', 'script', 'dependencies/requirements.in', ''); +INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path, importer_node_id) VALUES ('test-workspace', 'f/rel/root_flow', 'flow', 'dependencies/requirements.in', 'failure'); +INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path, importer_node_id) VALUES ('test-workspace', 'f/rel/root_flow', 'flow', 'dependencies/package.json', 'nstep2_1'); +INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path, importer_node_id) VALUES ('test-workspace', 'f/rel/root_flow', 'flow', 'dependencies/package.json', 'nstep3_2'); +INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path, importer_node_id) VALUES ('test-workspace', 'f/rel/root_flow', 'flow', 'dependencies/requirements.in', 'nstep2_2'); +INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path, importer_node_id) VALUES ('test-workspace', 'f/rel/root_flow', 'flow', 'dependencies/requirements.in', 'nstep3_1'); +INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path, importer_node_id) VALUES ('test-workspace', 'f/rel/root_flow', 'flow', 'dependencies/requirements.in', 'nstep4_1'); +INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path, importer_node_id) VALUES ('test-workspace', 'f/rel/root_flow', 'flow', 'dependencies/requirements.in', 'nstep5_1'); +INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path, importer_node_id) VALUES ('test-workspace', 'f/rel/root_flow', 'flow', 'dependencies/requirements.in', 'preprocessor'); +INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path, importer_node_id) VALUES ('test-workspace', 'f/rel/root_app', 'app', 'dependencies/requirements.in', 'pressmeplz'); +INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path, importer_node_id) VALUES ('test-workspace', 'f/rel/root_flow', 'flow', 'dependencies/requirements.in', 'qtool1'); +INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path, importer_node_id) VALUES ('test-workspace', 'f/rel/root_app', 'app', 'dependencies/requirements.in', 'youcanpressme'); diff --git a/backend/tests/fixtures/hub_sync_blacklist.sql b/backend/tests/fixtures/hub_sync_blacklist.sql new file mode 100644 index 0000000000..b3c59b2edb --- /dev/null +++ b/backend/tests/fixtures/hub_sync_blacklist.sql @@ -0,0 +1,15 @@ +-- Fixture for testing hub_sync blacklist from workspace dependencies +-- hub_sync and apps are already built-in via migrations + +-- Insert a simple Bun script that should be affected by workspace dependencies +INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES ( +'admins', +'test-user', +' +export async function main() { + return "Simple bun script"; +}', +'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}', +'Simple bun script', +'', +'u/admin/simple_bun', 700001, 'bun', ''); diff --git a/backend/tests/fixtures/workspace_dependencies.sql b/backend/tests/fixtures/workspace_dependencies.sql new file mode 100644 index 0000000000..9491f62e7c --- /dev/null +++ b/backend/tests/fixtures/workspace_dependencies.sql @@ -0,0 +1,33 @@ +-- NOTE: Applied after workspace_dependencies_leafs.sql +-- Python script that uses relative imports +INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES ( +'test-workspace', +'test-user', +' +from f.leafs.python import main as python_leaf + +def main(): + return {"python_import": python_leaf()}', +'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}', +'', +'', +'f/python_importer', 500005, 'python3', ''); + +-- TypeScript script that uses relative imports +INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES ( +'test-workspace', +'test-user', +' +import { main as tsLeaf } from "./leafs/ts"; + +export async function main() { + return { ts_import: await tsLeaf() }; +}', +'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}', +'', +'', +'f/ts_importer', 500006, 'nativets', ''); + +-- Dependency map entries +INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path, importer_node_id) VALUES ('test-workspace', 'f/python_importer', 'script', 'f/leafs/python', ''); +INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path, importer_node_id) VALUES ('test-workspace', 'f/ts_importer', 'script', 'f/leafs/ts', ''); diff --git a/backend/tests/fixtures/workspace_dependencies_leafs.sql b/backend/tests/fixtures/workspace_dependencies_leafs.sql new file mode 100644 index 0000000000..3896cb4b00 --- /dev/null +++ b/backend/tests/fixtures/workspace_dependencies_leafs.sql @@ -0,0 +1,54 @@ +-- Basic leaf scripts in different languages +INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES ( +'test-workspace', +'test-user', +' +export async function main() { + return "TypeScript leaf"; +}', +'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}', +'', +'', +'f/leafs/ts', 500001, 'nativets', ''); + +INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES ( +'test-workspace', +'test-user', +' +package main + +import "fmt" + +func main() { + fmt.Println("Go leaf") +}', +'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}', +'', +'', +'f/leafs/go', 500002, 'go', ''); + +INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES ( +'test-workspace', +'test-user', +' +def main(): + return "Python leaf"', +'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}', +'', +'', +'f/leafs/python', 500003, 'python3', ''); + +INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES ( +'test-workspace', +'test-user', +' +', +'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}', +'', +'', +'f/leafs/php', 500004, 'php', ''); + diff --git a/backend/tests/python_jobs.rs b/backend/tests/python_jobs.rs index afa662f582..549e052b65 100644 --- a/backend/tests/python_jobs.rs +++ b/backend/tests/python_jobs.rs @@ -25,7 +25,7 @@ def main(): &db, content, ScriptLang::Python3, - vec!["# py: 3.11.11", "tiny==0.1.3"], + vec!["# workspace-dependencies-mode: manual\n# py: 3.11.11","tiny==0.1.3"], ) .await?; Ok(()) @@ -55,7 +55,11 @@ def main(): &db, content, ScriptLang::Python3, - vec!["# py: 3.11.11", "bottle==0.13.2", "tiny==0.1.2"], + vec![ + "# workspace-dependencies-mode: extra\n# py: 3.11.11", + "bottle==0.13.2", + "tiny==0.1.2", + ], ) .await?; } @@ -79,7 +83,11 @@ def main(): &db, content, ScriptLang::Python3, - vec!["# py: 3.11.11", "simplejson==3.20.1", "tiny==0.1.3"], + vec![ + "# workspace-dependencies-mode: extra\n# py: 3.11.11", + "simplejson==3.20.1", + "tiny==0.1.3", + ], ) .await?; Ok(()) @@ -108,7 +116,7 @@ def main(): content, ScriptLang::Python3, vec![ - "# py: 3.11.11", + "# workspace-dependencies-mode: extra\n# py: 3.11.11", "bottle==0.13.2", "microdot==2.2.0", "simplejson==3.19.3", diff --git a/backend/tests/workspace_dependencies.rs b/backend/tests/workspace_dependencies.rs new file mode 100644 index 0000000000..6875520125 --- /dev/null +++ b/backend/tests/workspace_dependencies.rs @@ -0,0 +1,194 @@ +mod common; +mod workspace_dependencies { + + use crate::common::in_test_worker; + use crate::common::init_client; + use crate::common::listen_for_completed_jobs; + use sqlx::{Pool, Postgres}; + use tokio_stream::StreamExt; + use windmill_common::scripts::ScriptLang; + use windmill_worker::workspace_dependencies::NewWorkspaceDependencies; + mod deps { + pub const REQUIREMENTS_IN: &'static str = "tiny==0.1.3"; + // pub const GO_MOD: &'static str = r##" + // module example.com/project + + // go 1.20 + + // require github.com/gin-gonic/gin v1.8.1 + // "##; + + pub const PACKAGE_JSON: &'static str = r##" + { + "name": "example-project", + "version": "1.0.0", + "dependencies": { + "express": "^4.17.1" + } + } + "##; + + pub const COMPOSER_JSON: &'static str = r##" + { + "name": "example/project", + "require": { + "monolog/monolog": "^2.3" + } + } + "##; + } + + #[cfg(feature = "python")] + #[sqlx::test(fixtures("base", "workspace_dependencies_leafs"))] + #[ignore] + async fn basic_manual_named(db: Pool) -> anyhow::Result<()> { + let ((_client, port, _s), db, mut completed) = ( + init_client(db.clone()).await, + &db, + listen_for_completed_jobs(&db).await, + ); + + for (idx, (l, c)) in [ + (ScriptLang::Python3, deps::REQUIREMENTS_IN), + (ScriptLang::Bun, deps::PACKAGE_JSON), + (ScriptLang::Php, deps::COMPOSER_JSON), + // (ScriptLang::Go, deps::GO_MOD), + ] + .iter() + .enumerate() + { + let id = NewWorkspaceDependencies { + workspace_id: "test-workspace".into(), + language: *l, + content: (*c).into(), + name: Some("test".to_owned()), + description: None, + } + .create("", "", "", db) + .await + .unwrap(); + + assert_eq!(idx + 1, id as usize); + } + + // Wait for 4 jobs. + // Creating those dependencies will trigger redeployment of all scripts in workspace_dependencies_leafs.sql + in_test_worker( + db, + async { + completed.next().await; + completed.next().await; + completed.next().await; + // completed.next().await; + }, + port, + ) + .await; + + // Verify all scripts have correct locks + // let mut langs = vec![]; + // for r in sqlx::query!( + // r#"SELECT language AS "language: ScriptLang",lock FROM script WHERE archived = false"# + // ) + // .fetch_all(db) + // .await + // .unwrap() + // { + // match r.language { + // ScriptLang::Python3 => assert_eq!("", &r.lock.unwrap()), + // ScriptLang::Go => todo!(), + // ScriptLang::Bun => todo!(), + // ScriptLang::Bunnative => todo!(), + // ScriptLang::Php => todo!(), + // _ => panic!("Unsupported language"), + // } + + // langs.push(r.language); + // } + + // langs.sort(); + // // Just tiny additional verification for peace of mind. + // assert_eq!(langs.as_slice(), &[]); + + Ok(()) + } + + #[sqlx::test(fixtures("base", "hub_sync_blacklist"))] + async fn hub_sync_blacklist_from_workspace_deps(db: Pool) -> anyhow::Result<()> { + let ((_client, port, _s), db, mut completed) = ( + init_client(db.clone()).await, + &db, + listen_for_completed_jobs(&db).await, + ); + + // Verify built-in fixtures exist + // Check that the setup_app exists + let app_exists = + sqlx::query_scalar!("SELECT EXISTS(SELECT 1 FROM app WHERE path = 'g/all/setup_app')") + .fetch_one(db) + .await + .unwrap(); + assert!(app_exists.unwrap(), "Expected g/all/setup_app to exist"); + + // Check that hub_sync script exists and is a Bun script + let hub_sync_lang = sqlx::query_scalar!( + r#"SELECT language AS "language: ScriptLang" FROM script WHERE path = 'u/admin/hub_sync'"# + ) + .fetch_one(db) + .await + .unwrap(); + assert_eq!( + hub_sync_lang, + ScriptLang::Bun, + "Expected hub_sync to be a Bun script" + ); + + // Create unnamed (default) workspace dependencies for Bun + let _id = NewWorkspaceDependencies { + workspace_id: "admins".into(), + language: ScriptLang::Bun, + content: deps::PACKAGE_JSON.into(), + name: None, // No name = default workspace dependencies + description: None, + } + .create("", "", "", db) + .await + .unwrap(); + + // Wait for exactly 1 job (only u/admin/simple_bun, not hub_sync) + let job_id = in_test_worker(db, async { completed.next().await }, port) + .await + .expect("Expected one job to complete"); + + // Query the job's runnable_path + let runnable_path = + sqlx::query_scalar!("SELECT runnable_path FROM v2_job WHERE id = $1", job_id) + .fetch_one(db) + .await + .unwrap(); + + assert_eq!( + runnable_path, + Some("u/admin/simple_bun".to_string()), + "Expected job runnable_path to be 'u/admin/simple_bun' (hub_sync should be blacklisted)" + ); + + // Assert total job count is 1 + let job_count = sqlx::query_scalar!("SELECT COUNT(*) FROM v2_job") + .fetch_one(db) + .await + .unwrap(); + + assert_eq!(job_count, Some(1), "Expected exactly one job total"); + + // Assert v2_job_queue is empty + let queue_count = sqlx::query_scalar!("SELECT COUNT(*) FROM v2_job_queue") + .fetch_one(db) + .await + .unwrap(); + + assert_eq!(queue_count, Some(0), "Expected job queue to be empty"); + + Ok(()) + } +} diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 7ee99aca7b..43c9a40053 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -2249,6 +2249,56 @@ paths: text/plain: schema: type: string + /w/{workspace}/workspaces/get_dependents/{imported_path}: + get: + summary: get dependents of an imported path + operationId: getDependents + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: imported_path + in: path + required: true + schema: + type: string + description: The imported path to get dependents for + responses: + "200": + description: list of dependents + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/DependencyDependent" + + /w/{workspace}/workspaces/get_dependents_amounts: + post: + summary: get dependents amounts for multiple imported paths + operationId: getDependentsAmounts + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + description: List of imported paths to get dependents counts for + required: true + content: + application/json: + schema: + type: array + items: + type: string + responses: + "200": + description: list of dependents amounts + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/DependentsAmount" /w/{workspace}/workspaces/get_dependency_map: get: @@ -5400,6 +5450,126 @@ paths: schema: type: boolean + /w/{workspace}/workspace_dependencies/create: + post: + summary: create workspace dependencies + operationId: createWorkspaceDependencies + tags: + - workspace_dependencies + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + description: New workspace dependencies + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/NewWorkspaceDependencies" + + responses: + "201": + description: workspace dependencies created + content: + text/plain: + schema: + type: string + + /w/{workspace}/workspace_dependencies/archive/{language}: + post: + summary: archive workspace dependencies (require admin) + operationId: archiveWorkspaceDependencies + tags: + - workspace_dependencies + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: language + in: path + required: true + schema: + $ref: "#/components/schemas/ScriptLang" + - name: name + in: query + required: false + schema: + type: string + + responses: + "200": + description: result + content: + application/json: + schema: {} + + /w/{workspace}/workspace_dependencies/delete/{language}: + post: + summary: delete workspace dependencies (require admin) + operationId: deleteWorkspaceDependencies + tags: + - workspace_dependencies + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: language + in: path + required: true + schema: + $ref: "#/components/schemas/ScriptLang" + - name: name + in: query + required: false + schema: + type: string + + responses: + "200": + description: result + content: + application/json: + schema: {} + + /w/{workspace}/workspace_dependencies/list: + get: + summary: list all workspace dependencies + operationId: listWorkspaceDependencies + tags: + - workspace_dependencies + parameters: + - $ref: "#/components/parameters/WorkspaceId" + responses: + "200": + description: All workspace dependencies + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/WorkspaceDependencies" + + /w/{workspace}/workspace_dependencies/get_latest/{language}: + get: + summary: get latest workspace dependencies by language and name + operationId: getLatestWorkspaceDependencies + tags: + - workspace_dependencies + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: language + in: path + required: true + schema: + $ref: "#/components/schemas/ScriptLang" + - name: name + in: query + required: false + schema: + type: string + responses: + "200": + description: Latest workspace dependencies + content: + application/json: + schema: + $ref: "#/components/schemas/WorkspaceDependencies" + /w/{workspace}/scripts/archive/p/{path}: post: summary: archive script by path @@ -15759,6 +15929,52 @@ components: items: $ref: "#/components/schemas/Alert" + WorkspaceDependencies: + type: object + properties: + id: + type: integer + archived: + type: boolean + name: + type: string + description: + type: string + content: + type: string + language: + $ref: "#/components/schemas/ScriptLang" + workspace_id: + type: string + created_at: + type: string + format: date-time + required: + - workspace_id + - language + - created_at + - content + - id + - archived + + NewWorkspaceDependencies: + type: object + properties: + workspace_id: + type: string + language: + $ref: "#/components/schemas/ScriptLang" + name: + type: string + description: + type: string + content: + type: string + required: + - workspace_id + - language + - content + Script: type: object properties: @@ -18957,6 +19173,38 @@ components: type: string nullable: true + DependencyDependent: + type: object + properties: + importer_path: + type: string + importer_kind: + type: string + enum: + - script + - flow + - app + importer_node_ids: + type: array + items: + type: string + nullable: true + required: + - importer_path + - importer_kind + + DependentsAmount: + type: object + properties: + imported_path: + type: string + count: + type: integer + format: int64 + required: + - imported_path + - count + WorkspaceInvite: type: object properties: diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index 86e16a3bb0..584ddad54c 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -25,6 +25,7 @@ use std::str::FromStr; use std::time::Instant; use tokio::io::AsyncReadExt; use tower::ServiceBuilder; +use url::Url; #[cfg(all(feature = "enterprise", feature = "smtp"))] use windmill_common::auth::is_super_admin_email; use windmill_common::auth::TOKEN_PREFIX_LEN; @@ -39,6 +40,9 @@ use windmill_common::jobs::{ use windmill_common::s3_helpers::{upload_artifact_to_store, BundleFormat}; use windmill_common::utils::{RunnableKind, WarnAfterExt}; use windmill_common::worker::{Connection, CLOUD_HOSTED, TMP_DIR}; +use windmill_common::workspace_dependencies::{ + RawWorkspaceDependencies, MIN_VERSION_WORKSPACE_DEPENDENCIES, +}; use windmill_common::DYNAMIC_INPUT_CACHE; #[cfg(all(feature = "enterprise", feature = "smtp"))] use windmill_common::{email_oss::send_email_html, server::load_smtp_config}; @@ -6116,14 +6120,17 @@ async fn run_bundle_preview_script( Ok((StatusCode::CREATED, job_id.unwrap().to_string())) } -#[derive(Deserialize)] +#[derive(Deserialize, Debug)] pub struct RunDependenciesRequest { pub raw_scripts: Vec, pub entrypoint: String, + #[serde(default)] + pub raw_workspace_dependencies: Option, + #[serde(default)] pub raw_deps: Option, } -#[derive(Deserialize, Clone)] +#[derive(Deserialize, Clone, Debug)] pub struct RawScriptForDependencies { pub script_path: String, pub raw_code: Option, @@ -6148,41 +6155,49 @@ async fn run_dependencies_job( )); } + if req.raw_deps.is_some() { + return Err(error::Error::MigrationNeeded { + feature: "cli is outdated".into(), + version: MIN_VERSION_WORKSPACE_DEPENDENCIES.to_owned(), + guide_url: Url::from_str( + "https://www.windmill.dev/docs/core_concepts/workspace_dependencies/migration", + )?, + }); + } + + // Check if workers support workspace dependencies feature + if req.raw_workspace_dependencies.is_some() { + windmill_common::workspace_dependencies::min_version_supports_v0_workspace_dependencies() + .await?; + } + if req.raw_scripts.len() != 1 || req.raw_scripts[0].script_path != req.entrypoint { return Err(error::Error::internal_err( "For now only a single raw script can be passed to this endpoint, and the entrypoint should be set to the script path".to_string(), )); } - let raw_script = req.raw_scripts[0].clone(); - let script_path = raw_script.script_path; - let ehm = HashMap::new(); - let raw_code = raw_script.raw_code.unwrap_or_else(|| "".to_string()); - let language = raw_script.language; - let (args, raw_code) = if let Some(deps) = req.raw_deps { - let mut hm = HashMap::new(); - hm.insert( - "raw_deps".to_string(), - JsonRawValue::from_string("true".to_string()).unwrap(), - ); - if language == ScriptLang::Bun { - let annotation = windmill_common::worker::TypeScriptAnnotations::parse(&raw_code); - hm.insert( - "npm_mode".to_string(), - JsonRawValue::from_string(annotation.npm.to_string()).unwrap(), - ); - } - (PushArgs { extra: Some(hm), args: &ehm }, deps) - } else { - (PushArgs::from(&ehm), raw_code) - }; + let RawScriptForDependencies { + // unwrap + script_path, + raw_code, + language, + } = req.raw_scripts[0].clone(); + + let mut hm = HashMap::new(); + req.raw_workspace_dependencies + .map(|v| hm.insert("raw_workspace_dependencies".to_owned(), to_raw_value(&v))); let (uuid, tx) = push( &db, PushIsolationLevel::IsolatedRoot(db.clone()), &w_id, - JobPayload::RawScriptDependencies { script_path, content: raw_code, language }, - args, + JobPayload::RawScriptDependencies { + script_path, + content: raw_code.unwrap_or_default(), + language, + }, + PushArgs { extra: Some(hm), args: &HashMap::new() }, authed.display_username(), &authed.email, username_to_permissioned_as(&authed.username), @@ -6218,6 +6233,9 @@ async fn run_dependencies_job( pub struct RunFlowDependenciesRequest { pub path: String, pub flow_value: FlowValue, + #[serde(default)] + pub raw_workspace_dependencies: Option, + #[serde(default)] pub raw_deps: Option>, } @@ -6239,13 +6257,28 @@ async fn run_flow_dependencies_job( )); } - // Create args HashMap with skip_flow_update and raw_deps if present + if req.raw_deps.is_some() { + return Err(error::Error::MigrationNeeded { + feature: "cli is outdated".into(), + version: MIN_VERSION_WORKSPACE_DEPENDENCIES.to_owned(), + guide_url: Url::from_str( + "https://www.windmill.dev/docs/core_concepts/workspace_dependencies/migration", + )?, + }); + } + + // Check if workers support workspace dependencies feature + if req.raw_workspace_dependencies.is_some() { + windmill_common::workspace_dependencies::min_version_supports_v0_workspace_dependencies() + .await?; + } + + // Create args HashMap with skip_flow_update and raw_workspace_dependencies if present let mut args_map = HashMap::from([("skip_flow_update".to_string(), to_raw_value(&true))]); - // Add raw_deps to args if present - if let Some(ref raw_deps) = req.raw_deps { - args_map.insert("raw_deps".to_string(), to_raw_value(raw_deps)); - } + // Add raw_workspace_dependencies to args if present + req.raw_workspace_dependencies + .map(|v| args_map.insert("raw_workspace_dependencies".to_string(), to_raw_value(&v))); let (uuid, tx) = push( &db, diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index 96bf0ed085..ac39df0ee2 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -110,6 +110,7 @@ mod openapi; #[cfg(all(feature = "private", feature = "parquet"))] pub mod s3_proxy_ee; mod s3_proxy_oss; +mod workspace_dependencies; mod approvals; #[cfg(all(feature = "enterprise", feature = "private"))] @@ -455,6 +456,10 @@ pub async fn run_server( .nest("/drafts", drafts::workspaced_service()) .nest("/favorites", favorite::workspaced_service()) .nest("/flows", flows::workspaced_service()) + .nest( + "/workspace_dependencies", + workspace_dependencies::workspaced_service(), + ) .nest( "/flow_conversations", flow_conversations::workspaced_service(), diff --git a/backend/windmill-api/src/scripts.rs b/backend/windmill-api/src/scripts.rs index 01200f1674..7cbcf8cc03 100644 --- a/backend/windmill-api/src/scripts.rs +++ b/backend/windmill-api/src/scripts.rs @@ -41,11 +41,11 @@ use windmill_audit::ActionKind; use windmill_worker::{process_relative_imports, scoped_dependency_map::ScopedDependencyMap}; use windmill_common::{ - assets::{AssetUsageKind, AssetWithAltAccessType, clear_asset_usage, insert_asset_usage}, + assets::{clear_asset_usage, insert_asset_usage, AssetUsageKind, AssetWithAltAccessType}, error::to_anyhow, s3_helpers::upload_artifact_to_store, scripts::hash_script, - utils::{WarnAfterExt, paginate_without_limits}, + utils::{paginate_without_limits, WarnAfterExt}, worker::{CLOUD_HOSTED, MIN_VERSION_SUPPORTS_DEBOUNCING}, }; @@ -60,9 +60,7 @@ use windmill_common::{ ScriptHistory, ScriptHistoryUpdate, ScriptKind, ScriptLang, ScriptWithStarred, }, users::username_to_permissioned_as, - utils::{ - not_found_if_none, query_elems_from_hub, require_admin, Pagination, StripPath, - }, + utils::{not_found_if_none, query_elems_from_hub, require_admin, Pagination, StripPath}, worker::to_raw_value, HUB_BASE_URL, }; @@ -1045,14 +1043,10 @@ async fn create_script_internal<'c>( let permissioned_as2 = permissioned_as.clone(); let script_path2 = script_path.clone(); let parent_path = p_path_opt.clone(); - let lock = ns.lock.clone(); let deployment_message = ns.deployment_message.clone(); let content = ns.content.clone(); let language = ns.language.clone(); tokio::spawn(async move { - // TODO: I don't think we want this. We might want to send dependency job. But skip any calculations if lock is already present. - // It will allow us to make code more consistent and predictable. - // wait for 10 seconds to make sure the script is deployed and that the CLI sync that pushed it (f one) is complete tokio::time::sleep(std::time::Duration::from_secs(10)).await; if let Err(e) = process_relative_imports( @@ -1068,7 +1062,6 @@ async fn create_script_internal<'c>( &authed2.email, &authed2.username, &permissioned_as2, - lock, ) .await { diff --git a/backend/windmill-api/src/workspace_dependencies.rs b/backend/windmill-api/src/workspace_dependencies.rs new file mode 100644 index 0000000000..bb511518d2 --- /dev/null +++ b/backend/windmill-api/src/workspace_dependencies.rs @@ -0,0 +1,145 @@ +use axum::{ + extract::{Path, Query}, + routing::{get, post}, + Extension, Json, Router, +}; +use http::StatusCode; +use serde::Deserialize; +use windmill_common::{ + error::{self, JsonResult}, + scripts::ScriptLang, + users::username_to_permissioned_as, + utils::require_admin, + workspace_dependencies::WorkspaceDependencies, + DB, +}; +use windmill_worker::{ + scoped_dependency_map, trigger_dependents_to_recompute_dependencies, + workspace_dependencies::NewWorkspaceDependencies, +}; + +use crate::db::ApiAuthed; + +pub fn workspaced_service() -> Router { + Router::new() + .route("/create", post(create)) + .route("/list", get(list)) + .route("/archive/:language", post(archive)) + .route("/get_latest/:language", get(get_latest)) + .route("/delete/:language", post(delete)) +} + +#[axum::debug_handler] +async fn create( + authed: ApiAuthed, + // Extension(user_db): Extension, + Extension(db): Extension, + Json(nwd): Json, +) -> error::Result<(StatusCode, String)> { + tracing::info!(workspace_id = %nwd.workspace_id, name = ?nwd.name, language = ?nwd.language, "create workspace dependencies"); + require_admin(authed.is_admin, &authed.username)?; + Ok(( + StatusCode::CREATED, + format!( + "{}", + nwd.create( + &authed.email, + &authed.username, + &username_to_permissioned_as(&authed.username), + &db + ) + .await? + ), + )) +} + +#[axum::debug_handler] +async fn list( + // Extension(user_db): Extension, + Extension(db): Extension, + Path(w_id): Path, +) -> JsonResult> { + tracing::info!(workspace_id = %w_id, "list workspace dependencies"); + Ok(Json(WorkspaceDependencies::list(&w_id, &db).await?)) +} + +#[derive(Deserialize)] +pub(super) struct NameQuery { + name: Option, +} + +#[axum::debug_handler] +pub(super) async fn get_latest( + Extension(db): Extension, + Path((w_id, language)): Path<(String, ScriptLang)>, + Query(params): Query, +) -> JsonResult> { + tracing::info!(workspace_id = %w_id, language = ?language, name = ?params.name, "get latest workspace dependencies"); + Ok(Json( + WorkspaceDependencies::get_latest(params.name, language, &w_id, db.into()).await?, + )) +} + +#[axum::debug_handler] +async fn archive( + authed: ApiAuthed, + // Extension(user_db): Extension, + Extension(db): Extension, + Path((w_id, language)): Path<(String, ScriptLang)>, + Query(params): Query, +) -> error::Result<()> { + tracing::info!(workspace_id = %w_id, language = ?language, name = ?params.name, "archive workspace dependencies"); + require_admin(authed.is_admin, &authed.username)?; + let db = &db; + WorkspaceDependencies::archive(params.name.clone(), language, &w_id, db).await?; + + trigger_dependents_to_recompute_dependencies( + &w_id, + scoped_dependency_map::ScopedDependencyMap::get_dependents( + WorkspaceDependencies::to_path(¶ms.name, language)?.as_str(), + &w_id, + db, + ) + .await?, + None, + None, + &authed.email, + &authed.username, + &username_to_permissioned_as(&authed.username), + db, + vec![], + ) + .await +} + +#[axum::debug_handler] +async fn delete( + authed: ApiAuthed, + // Extension(user_db): Extension, + Extension(db): Extension, + Path((w_id, language)): Path<(String, ScriptLang)>, + Query(params): Query, +) -> error::Result<()> { + tracing::info!(workspace_id = %w_id, language = ?language, name = ?params.name, "delete workspace dependencies"); + require_admin(authed.is_admin, &authed.username)?; + let db = &db; + WorkspaceDependencies::delete(params.name.clone(), language, &w_id, db).await?; + + trigger_dependents_to_recompute_dependencies( + &w_id, + scoped_dependency_map::ScopedDependencyMap::get_dependents( + WorkspaceDependencies::to_path(¶ms.name, language)?.as_str(), + &w_id, + db, + ) + .await?, + None, + None, + &authed.email, + &authed.username, + &username_to_permissioned_as(&authed.username), + db, + vec![], + ) + .await +} diff --git a/backend/windmill-api/src/workspaces.rs b/backend/windmill-api/src/workspaces.rs index 9fbba5cc48..effe576d55 100644 --- a/backend/windmill-api/src/workspaces.rs +++ b/backend/windmill-api/src/workspaces.rs @@ -51,7 +51,9 @@ use windmill_common::{ utils::{paginate, rd_string, require_admin, Pagination}, }; use windmill_git_sync::{handle_deployment_metadata, handle_fork_branch_creation, DeployedObject}; -use windmill_worker::scoped_dependency_map::{DependencyMap, ScopedDependencyMap}; +use windmill_worker::scoped_dependency_map::{ + DependencyDependent, DependencyMap, ScopedDependencyMap, +}; #[cfg(feature = "enterprise")] use windmill_common::utils::require_admin_or_devops; @@ -82,6 +84,8 @@ pub fn workspaced_service() -> Router { .route("/delete_invite", post(delete_invite)) .route("/rebuild_dependency_map", post(rebuild_dependency_map)) .route("/get_dependency_map", get(get_dependency_map)) + .route("/get_dependents/*imported_path", get(get_dependents)) + .route("/get_dependents_amounts", post(get_dependents_amounts)) .route("/get_settings", get(get_settings)) .route("/get_deploy_to", get(get_deploy_to)) .route("/edit_slack_command", post(edit_slack_command)) @@ -2553,8 +2557,11 @@ async fn clone_workspace_data( clone_raw_apps(tx, source_workspace_id, target_workspace_id).await?; // Clone workspace runnable dependencies and dependency map - clone_workspace_dependencies(tx, source_workspace_id, target_workspace_id).await?; + clone_workspace_runnable_dependencies(tx, source_workspace_id, target_workspace_id).await?; + // TODO: Enable when git sync is implemented for workspace dependencies. + // // Clone workspace dependencies + // clone_workspace_dependencies(tx, source_workspace_id, target_workspace_id).await?; Ok(()) } @@ -3017,7 +3024,7 @@ async fn clone_raw_apps( Ok(()) } -async fn clone_workspace_dependencies( +async fn clone_workspace_runnable_dependencies( tx: &mut Transaction<'_, Postgres>, source_workspace_id: &str, target_workspace_id: &str, @@ -3049,6 +3056,27 @@ async fn clone_workspace_dependencies( Ok(()) } +#[allow(dead_code)] +async fn clone_workspace_dependencies( + tx: &mut Transaction<'_, Postgres>, + source_workspace_id: &str, + target_workspace_id: &str, +) -> Result<()> { + // Clone workspace_runnable_dependencies + sqlx::query!( + "INSERT INTO workspace_dependencies (workspace_id, language, name, description, content, archived, created_at) + SELECT $1, language, name, description, content, archived, created_at + FROM workspace_dependencies + WHERE workspace_id = $2", + target_workspace_id, + source_workspace_id + ) + .execute(&mut **tx) + .await?; + + Ok(()) +} + async fn deprecated_create_workspace_fork(_authed: ApiAuthed) -> Result { return Err(Error::BadRequest("This API endpoint has been relocated. Your Windmill CLI version is outdated and needs to be updated.".to_string())); } @@ -3583,6 +3611,75 @@ async fn rebuild_dependency_map( ScopedDependencyMap::rebuild_map(&w_id, &db).await } +#[axum::debug_handler] +async fn get_dependents( + Extension(db): Extension, + Path((w_id, imported_path)): Path<(String, String)>, + _authed: ApiAuthed, +) -> JsonResult> { + tracing::debug!( + workspace_id = %w_id, + imported_path = %imported_path, + "API: Getting dependents for imported path" + ); + + let dependents = ScopedDependencyMap::get_dependents(&imported_path, &w_id, &db).await?; + + tracing::debug!( + workspace_id = %w_id, + imported_path = %imported_path, + dependents_count = dependents.len(), + "API: Found dependents: {:?}", + dependents + ); + + Ok(Json(dependents)) +} + +#[derive(Serialize, Debug)] +struct DependentsAmount { + imported_path: String, + count: i64, +} + +#[axum::debug_handler] +async fn get_dependents_amounts( + Extension(db): Extension, + Path(w_id): Path, + Json(imported_paths): Json>, +) -> JsonResult> { + tracing::debug!( + workspace_id = %w_id, + imported_paths = ?imported_paths, + "API: Getting dependents amounts for imported paths" + ); + + let results = sqlx::query_as!( + DependentsAmount, + r#" + SELECT + imported_path, + COUNT(DISTINCT importer_path) as "count!" + FROM dependency_map + WHERE workspace_id = $1 AND imported_path = ANY($2) + GROUP BY imported_path + "#, + w_id, + &imported_paths + ) + .fetch_all(&db) + .await?; + + tracing::debug!( + workspace_id = %w_id, + results_count = results.len(), + "API: Found dependents amounts: {:?}", + results + ); + + Ok(Json(results)) +} + #[derive(Deserialize)] struct ChangeWorkspaceName { new_name: String, diff --git a/backend/windmill-api/src/workspaces_export.rs b/backend/windmill-api/src/workspaces_export.rs index 74732c9f22..3af2f5d44a 100644 --- a/backend/windmill-api/src/workspaces_export.rs +++ b/backend/windmill-api/src/workspaces_export.rs @@ -44,6 +44,7 @@ use axum::{ use http::HeaderName; use itertools::Itertools; +use windmill_common::utils::require_admin; use windmill_common::variables::decrypt; use windmill_common::{ db::UserDB, @@ -52,6 +53,7 @@ use windmill_common::{ schedule::Schedule, scripts::{Schema, Script, ScriptLang}, variables::{build_crypt, ExportableListableVariable}, + workspace_dependencies::WorkspaceDependencies, }; use hyper::header; @@ -177,6 +179,7 @@ pub(crate) struct ArchiveQueryParams { include_groups: Option, include_settings: Option, include_key: Option, + include_workspace_dependencies: Option, default_ts: Option, } @@ -309,11 +312,20 @@ pub(crate) async fn tarball_workspace( include_groups, include_settings, include_key, + include_workspace_dependencies, default_ts, }): Query, ) -> Result<([(HeaderName, String); 2], impl IntoResponse)> { // require_admin(authed.is_admin, &authed.username)?; + tracing::info!( + "tarball_workspace called for workspace {}: include_workspace_dependencies={:?}, skip_variables={:?}, skip_resources={:?}", + w_id, + include_workspace_dependencies, + skip_variables, + skip_resources + ); + let mut tx = user_db.begin(&authed).await?; let tmp_dir = TempDir::new_in("/tmp/windmill/")?; @@ -545,6 +557,33 @@ pub(crate) async fn tarball_workspace( } } + if include_workspace_dependencies.unwrap_or(false) + && require_admin(authed.is_admin, &authed.username).is_ok() + { + tracing::info!("Including workspace dependencies in tarball export"); + let workspace_dependencies = WorkspaceDependencies::list(&w_id, &db).await?; + tracing::info!( + "Found {} workspace dependencies", + workspace_dependencies.len() + ); + for dep in workspace_dependencies { + // let dep_str = &to_string_without_metadata(&dep, false, None).unwrap(); + let filename = WorkspaceDependencies::to_path(&dep.name, dep.language)?; + tracing::info!( + "Adding workspace dependency: name={:?}, language={:?}, filename={}", + dep.name, + dep.language, + filename + ); + archive.write_to_archive(&dep.content, &filename).await?; + } + } else { + tracing::info!( + "Skipping workspace dependencies: include_workspace_dependencies={:?}", + include_workspace_dependencies + ); + } + if include_schedules.unwrap_or(false) { let schedules = sqlx::query_as::<_, Schedule>( "SELECT * FROM schedule diff --git a/backend/windmill-common/Cargo.toml b/backend/windmill-common/Cargo.toml index c0a52c6677..0ba041e1ec 100644 --- a/backend/windmill-common/Cargo.toml +++ b/backend/windmill-common/Cargo.toml @@ -33,6 +33,7 @@ thiserror.workspace = true anyhow.workspace = true serde.workspace = true serde_json.workspace = true +serde_yml.workspace = true chrono.workspace = true chrono-tz.workspace = true hex.workspace = true @@ -68,6 +69,7 @@ aws-credential-types.workspace = true aws-smithy-types.workspace = true base64.workspace = true bitflags.workspace = true +phf.workspace = true aws-smithy-types-convert = { workspace = true, optional = true } aws-sdk-rds = { workspace = true, optional = true } @@ -91,6 +93,7 @@ strum_macros.workspace = true url.workspace = true urlencoding.workspace = true async-recursion.workspace = true +pep440_rs.workspace = true semver.workspace = true croner = "2.2.0" diff --git a/backend/windmill-common/src/error.rs b/backend/windmill-common/src/error.rs index 8048b59586..1e051f064e 100644 --- a/backend/windmill-common/src/error.rs +++ b/backend/windmill-common/src/error.rs @@ -86,6 +86,12 @@ pub enum Error { Generic(StatusCode, String), #[error("{feature} is unavailable due to some workers being behind. Do not use the feature or make sure all workers run at least {min_version}")] WorkersAreBehind { feature: String, min_version: String }, + #[error( + "Breaking change was introduced in v{version} ({feature}). Follow this migration guide: {guide_url}" + )] + MigrationNeeded { version: String, feature: String, guide_url: url::Url }, + #[error("{0} is unavailable. It is possible for this worker to be behind.")] + FeatureUnavailable(String), } impl Error { diff --git a/backend/windmill-common/src/flows.rs b/backend/windmill-common/src/flows.rs index 104f99b301..9134da3c41 100644 --- a/backend/windmill-common/src/flows.rs +++ b/backend/windmill-common/src/flows.rs @@ -246,7 +246,13 @@ impl FlowValue { } } } - BranchOne { branches, .. } | BranchAll { branches, .. } => { + BranchOne { default, branches, .. } => { + Self::traverse_leafs(default.iter().collect(), cb)?; + for branch in branches { + Self::traverse_leafs(branch.modules.iter().collect(), cb)?; + } + } + BranchAll { branches, .. } => { for branch in branches { Self::traverse_leafs(branch.modules.iter().collect(), cb)?; } diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs index 13be9ff02f..e6beb964df 100644 --- a/backend/windmill-common/src/global_settings.rs +++ b/backend/windmill-common/src/global_settings.rs @@ -133,3 +133,18 @@ pub async fn load_value_from_global_settings( .map(|x| x.value); Ok(r) } + +pub async fn set_value_in_global_settings( + db: &Pool, + setting_name: &str, + value: serde_json::Value, +) -> error::Result<()> { + sqlx::query!( + "INSERT INTO global_settings (name, value) VALUES ($1, $2) ON CONFLICT (name) DO UPDATE SET value = EXCLUDED.value, updated_at = now()", + setting_name, + value + ) + .execute(db) + .await?; + Ok(()) +} diff --git a/backend/windmill-common/src/jobs.rs b/backend/windmill-common/src/jobs.rs index f32b3447c8..d4589dd5fd 100644 --- a/backend/windmill-common/src/jobs.rs +++ b/backend/windmill-common/src/jobs.rs @@ -412,7 +412,7 @@ pub enum JobPayload { /// Dependency Job, exposed with API. Requirements can be predefined RawScriptDependencies { script_path: String, - /// Will reflect raw requirements content (e.g. requirements.txt) + /// Will reflect raw requirements content (e.g. requirements.in) content: String, language: ScriptLang, }, diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index 04e59cfac9..1f3dbd414b 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -55,7 +55,7 @@ pub mod job_metrics; pub mod job_s3_helpers_ee; #[cfg(feature = "parquet")] pub mod job_s3_helpers_oss; -pub mod lockfiles; +pub mod workspace_dependencies; #[cfg(feature = "private")] pub mod git_sync_ee; diff --git a/backend/windmill-common/src/schema.rs b/backend/windmill-common/src/schema.rs index d16b86d507..6c5d3cedd2 100644 --- a/backend/windmill-common/src/schema.rs +++ b/backend/windmill-common/src/schema.rs @@ -345,15 +345,7 @@ fn find_annotation(comm_lit: &str, annotation: &str, code: &str) -> bool { pub fn should_validate_schema(code: &str, lang: &ScriptLang) -> bool { let annotation = "schema_validation"; - use ScriptLang::*; - let comment = match lang { - Nativets | Bun | Bunnative | Deno | Php | CSharp | Java => "//", - Python3 | Go | Bash | Powershell | Graphql | Ansible | Nu | Ruby => "#", - Postgresql | Mysql | Bigquery | Snowflake | Mssql | OracleDB | DuckDb => "--", - Rust => "//!", - // for related places search: ADD_NEW_LANG - }; - find_annotation(comment, annotation, code) + find_annotation(&lang.as_comment_lit(), annotation, code) } #[derive(Serialize, Deserialize, Debug, Clone)] diff --git a/backend/windmill-common/src/scripts.rs b/backend/windmill-common/src/scripts.rs index a5ba95f93b..ca34b5eb5e 100644 --- a/backend/windmill-common/src/scripts.rs +++ b/backend/windmill-common/src/scripts.rs @@ -17,6 +17,7 @@ use crate::{ assets::AssetWithAltAccessType, error::{to_anyhow, Error}, utils::http_get_from_hub, + workspace_dependencies::WorkspaceDependenciesAnnotatedRefs, DB, DEFAULT_HUB_BASE_URL, HUB_BASE_URL, PRIVATE_HUB_MIN_VERSION, }; @@ -25,12 +26,26 @@ use anyhow::Context; use backon::ConstantBuilder; use backon::{BackoffBuilder, Retryable}; use itertools::Itertools; +use regex::Regex; use serde::de::Error as _; use serde::{ser::SerializeSeq, Deserialize, Deserializer, Serialize}; use crate::utils::StripPath; -#[derive(Serialize, Deserialize, Debug, PartialEq, Copy, Clone, Hash, Eq, sqlx::Type, Default)] +#[derive( + Serialize, + Deserialize, + Debug, + PartialEq, + Copy, + Clone, + Hash, + Eq, + sqlx::Type, + Default, + Ord, + PartialOrd, +)] #[sqlx(type_name = "SCRIPT_LANG", rename_all = "lowercase")] #[serde(rename_all(serialize = "lowercase", deserialize = "lowercase"))] pub enum ScriptLang { @@ -90,6 +105,71 @@ impl ScriptLang { // for related places search: ADD_NEW_LANG } } + + pub fn as_dependencies_filename(&self) -> Option { + use ScriptLang::*; + Some( + match self { + Bun | Bunnative => "package.json", + Python3 => "requirements.in", + // Go => "go.mod", + Php => "composer.json", + _ => return None, + } + .to_owned(), + ) + } + + pub fn as_comment_lit(&self) -> String { + use ScriptLang::*; + match self { + Nativets | Bun | Bunnative | Deno | Go | Php | CSharp | Java => "//", + Python3 | Bash | Powershell | Graphql | Ansible | Nu | Ruby => "#", + Postgresql | Mysql | Bigquery | Snowflake | Mssql | OracleDB | DuckDb => "--", + Rust => "//!", + // for related places search: ADD_NEW_LANG + } + .to_owned() + } + + pub fn extract_workspace_dependencies_annotated_refs( + &self, + code: &str, + runnable_path: &str, + ) -> Option> { + use ScriptLang::*; + lazy_static::lazy_static! { + static ref RE_PYTHON: Regex = Regex::new(r"^\#\s?(\S+)\s*$").unwrap(); + } + match self { + // TODO: Maybe use regex + Bun | Bunnative => WorkspaceDependenciesAnnotatedRefs::parse( + "//", + "package_json", + code, + None, + runnable_path, + ), + Python3 => WorkspaceDependenciesAnnotatedRefs::parse( + "#", + "requirements", + code, + Some(&RE_PYTHON), + runnable_path, + ), + Go => { + WorkspaceDependenciesAnnotatedRefs::parse("//", "go_mod", code, None, runnable_path) + } + Php => WorkspaceDependenciesAnnotatedRefs::parse( + "//", + "composer_json", + code, + None, + runnable_path, + ), + _ => return None, + } + } } impl FromStr for ScriptLang { @@ -739,12 +819,14 @@ pub struct ClonedScript { pub old_script: NewScript, pub new_hash: i64, } +// TODO: What if dependency job fails, there is script with NULL in the lock pub async fn clone_script<'c>( base_hash: ScriptHash, w_id: &str, deployment_message: Option, tx: &mut sqlx::Transaction<'c, sqlx::Postgres>, ) -> crate::error::Result { + // TODO:! let s = sqlx::query_as::<_, Script>( "SELECT * FROM script WHERE hash = $1 AND workspace_id = $2 AND archived = false FOR UPDATE", ) diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index 2918d40b0c..76c04cbe42 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -264,6 +264,10 @@ lazy_static::lazy_static! { .unwrap_or(false); pub static ref MIN_VERSION: Arc> = Arc::new(RwLock::new(Version::new(0, 0, 0))); + /// Global flag indicating if all workers support workspace dependencies feature (>= 1.583.0) + /// This flag is updated during worker initialization by checking the minimum version across all workers + /// When false, creation of workspace dependencies is forbidden and extraction of external workspace dependencies will error + pub static ref MIN_VERSION_SUPPORTS_V0_WORKSPACE_DEPENDENCIES: Arc> = Arc::new(RwLock::new(false)); /// Global flag indicating if all workers support the debouncing feature (>= 1.566.0) /// Debouncing consolidates multiple dependency job requests within a time window to avoid redundant work /// This flag is updated during worker initialization by checking the minimum version across all workers @@ -505,7 +509,10 @@ pub const ROOT_CACHE_DIR: &str = concatcp!(TMP_DIR, "/cache/"); pub fn write_file(dir: &str, path: &str, content: &str) -> error::Result { let path = format!("{}/{}", dir, path); - let mut file = File::create(&path)?; + let mut file = File::create(&path).map_err(|e| { + tracing::error!("Failed to create file at {path}: {:?}", &e); + e + })?; file.write_all(content.as_bytes())?; file.flush()?; Ok(file) @@ -1264,6 +1271,10 @@ pub async fn update_min_version(conn: &Connection) -> bool { tracing::info!("Minimal worker version: {min_version}"); } + // Workspace dependencies feature requires minimum version across all workers + *MIN_VERSION_SUPPORTS_V0_WORKSPACE_DEPENDENCIES.write().await = min_version + >= Version::parse(crate::workspace_dependencies::MIN_VERSION_WORKSPACE_DEPENDENCIES) + .unwrap(); // Debouncing feature requires minimum version 1.566.0 across all workers // This ensures all workers can handle debounce keys and stale data accumulation *MIN_VERSION_SUPPORTS_DEBOUNCING.write().await = min_version >= Version::new(1, 566, 0); @@ -1949,6 +1960,115 @@ pub fn to_raw_value_owned(result: serde_json::Value) -> Box { .unwrap_or_else(|_| RawValue::from_string("{}".to_string()).unwrap()) } +pub fn split_python_requirements>(requirements: T) -> Vec { + requirements + .as_ref() + .lines() + .filter(|x| !x.trim_start().starts_with("--") && !x.trim().is_empty()) + .map(String::from) + .collect() +} + +#[derive(Eq, PartialEq, Clone, Copy, Default, Debug)] +#[repr(u32)] +pub enum PyVAlias { + Py310 = 10, + #[default] + Py311, + Py312, + Py313, +} + +impl Into for PyVAlias { + fn into(self) -> pep440_rs::Version { + pep440_rs::Version::new([self.major() as u64, self as u64]) + } +} + +impl Into for PyVAlias { + fn into(self) -> u32 { + self.major() * 100 + self as u32 + } +} + +impl PyVAlias { + pub fn all>() -> Vec { + use PyVAlias::*; + vec![Py310.into(), Py311.into(), Py312.into(), Py313.into()] + } + // Get MAJOR part of alias. (semver: MAJOR.MINOR.PATCH) + fn major(&self) -> u32 { + use PyVAlias::*; + match self { + Py310 | Py311 | Py312 | Py313 => 3, + // Py400 | Py401 => 4 + } + } + + /// Converts numeric format to alias + /// Example: + /// 310u32 (in) -> PyVAlias::Py310 (out) + pub fn try_from_v1(numeric: T) -> Option { + use PyVAlias::*; + match numeric.to_string().as_str() { + "310" => Some(Py310), + "311" => Some(Py311), + "312" => Some(Py312), + "313" => Some(Py313), + _ => None, + } + } +} + +/// Parse lockfile for assigned python version. +/// If not found returns None +pub fn try_parse_locked_python_version_from_requirements>( + requirements_lines: &[S], +) -> Option { + let parse_version = |s: &str| -> Option { + // Possible inputs: + // V2: + // # py: 3.11.0 or #py:3.11.0 or #py: 3.11.0 + // + // V1: + // # py311 or #py311 + let version_unparsed = s + .to_owned() + // Remove whitespaces. That leaves us with: + // V2: #py:3.11.0 + // V1: #py311 + // + // Remove # + // V2: py:3.11.0 + // V1: py311 + // + // Remove : + // V2: py3.11.0 + // V1: py311 + .replace([' ', '#', ':'], "") + // Remove "py" + // V2: 3.11.0 + // V1: 311 + .replace("py", ""); + + // We will support reading V1 syntax, but it will be overwritten next deploy + PyVAlias::try_from_v1(&version_unparsed) + .map(PyVAlias::into) + .or(pep440_rs::Version::from_str(&version_unparsed) + .ok() + .map(pep440_rs::Version::into)) + }; + + requirements_lines + .iter() + .find(|s| { + let s = s.as_ref(); + s.starts_with("#py") || s.starts_with("# py") + }) + .map(S::as_ref) + .and_then(parse_version) +} + #[cfg(test)] mod tests { use super::*; diff --git a/backend/windmill-common/src/workspace_dependencies.rs b/backend/windmill-common/src/workspace_dependencies.rs new file mode 100644 index 0000000000..e93f6686f7 --- /dev/null +++ b/backend/windmill-common/src/workspace_dependencies.rs @@ -0,0 +1,1186 @@ +use itertools::Itertools; +use regex::Regex; +use serde::{Deserialize, Serialize}; +use sqlx::PgExecutor; +use std::time::{Duration, Instant}; + +use crate::{error, scripts::ScriptLang, utils::calculate_hash, worker::Connection}; +use phf::phf_set; + +pub static BLACKLIST: phf::Set<&'static str> = phf_set! { + "u/admin/hub_sync", + "g/all/setup_app/app", + "g/all/setup_app" +}; + +lazy_static::lazy_static! { + static ref WMDEBUG_FORCE_V0_WORKSPACE_DEPENDENCIES: bool = std::env::var("WMDEBUG_FORCE_V0_WORKSPACE_DEPENDENCIES").is_ok(); + + /// Simple in-memory cache for workspace dependencies get_latest with 10-second timeout. + /// Cache key: (workspace_id, language, name) + /// Cache value: (Option, cached_at timestamp) + static ref WORKSPACE_DEPENDENCIES_CACHE: quick_cache::sync::Cache<(String, ScriptLang, Option), (Option, Instant)> = quick_cache::sync::Cache::new(1000); +} + +/// Cache timeout for workspace dependencies +const CACHE_TIMEOUT: Duration = Duration::from_secs(10); + +/// Minimum Windmill version required for workspace dependencies feature +pub const MIN_VERSION_WORKSPACE_DEPENDENCIES: &str = "1.587.0"; + +pub async fn min_version_supports_v0_workspace_dependencies() -> error::Result<()> { + // Check if workers support workspace dependencies feature + if !*WMDEBUG_FORCE_V0_WORKSPACE_DEPENDENCIES + && !*crate::worker::MIN_VERSION_SUPPORTS_V0_WORKSPACE_DEPENDENCIES + .read() + .await + { + tracing::warn!( + "Workspace dependencies feature will be disabled because not all workers support it (minimum version {} required)", + MIN_VERSION_WORKSPACE_DEPENDENCIES + ); + return Err(error::Error::WorkersAreBehind { + feature: "Workspace dependencies".to_string(), + min_version: MIN_VERSION_WORKSPACE_DEPENDENCIES.to_string(), + }); + } else { + Ok(()) + } +} + +pub type RawWorkspaceDependencies = std::collections::HashMap; + +/// Removes workspace dependencies annotation comments from lock files. +/// This is used when passing locks to resolver expect no comments (looking at you, json). +/// IMPORTANT: lock is expected to start with annotations +pub fn clean_lock_from_annotations(lock: &str, language: ScriptLang) -> String { + let mat = format!("{} workspace-dependencies", language.as_comment_lit()); + lock.lines().filter(|l| !l.starts_with(&mat)).collect() +} + +pub fn get_raw_workspace_dependencies( + raw_workspace_dependencies_o: &Option, + name: Option, + language: ScriptLang, + workspace_id: String, +) -> Option { + raw_workspace_dependencies_o + .as_ref() + .zip(WorkspaceDependencies::to_path(&name, language).ok()) + .and_then(|(hm, path)| hm.get(&path)) + .map(|raw_content| WorkspaceDependencies { + name, + workspace_id, + created_at: chrono::Utc::now(), + language, + content: raw_content.to_owned(), + ..Default::default() + }) +} + +fn map_err(e: String) -> error::Error { + error::Error::FeatureUnavailable(e) +} +#[derive(sqlx::FromRow, Debug, Clone, Serialize, Deserialize, Default)] +pub struct WorkspaceDependencies { + /// Global id (across all workspaces) + id: i64, + archived: bool, + /// If not set becomes default for given language + pub name: Option, + pub description: Option, + pub workspace_id: String, + pub created_at: chrono::DateTime, + pub language: ScriptLang, + pub content: String, +} + +impl WorkspaceDependencies { + pub fn hash(&self) -> String { + // non-raw workspace dependencies will start with index 1. + // so if we see index 0, it is either default or default and raw deps + // if so we will use it's content as baseline + if self.id == 0 { + calculate_hash(&self.content) + } else { + self.id.to_string() + } + } + /// Marks workspace dependencies as archived. + pub async fn archive<'c>( + name: Option, + language: ScriptLang, + workspace_id: &str, + e: impl PgExecutor<'c>, + ) -> error::Result<()> { + if language.as_dependencies_filename().is_none() { + return Ok(()); + } + + sqlx::query!( + " + UPDATE workspace_dependencies + SET archived = true + WHERE name IS NOT DISTINCT FROM $1 AND workspace_id = $2 AND archived = false AND language = $3 + ", + name, + workspace_id, + language as ScriptLang + ) + .execute(e) + .await?; + Ok(()) + } + + /// Permanently deletes workspace dependencies from the database. + pub async fn delete<'c>( + name: Option, + language: ScriptLang, + workspace_id: &str, + e: impl PgExecutor<'c>, + ) -> error::Result<()> { + if language.as_dependencies_filename().is_none() { + return Ok(()); + } + + sqlx::query!( + " + DELETE + FROM workspace_dependencies + WHERE name IS NOT DISTINCT FROM $1 AND workspace_id = $2 AND language = $3 + ", + name, + workspace_id, + language as ScriptLang + ) + .execute(e) + .await?; + Ok(()) + } + + /// Lists all active workspace dependencies for a workspace. + pub async fn list<'c>(workspace_id: &str, e: impl PgExecutor<'c>) -> error::Result> { + sqlx::query_as!( + Self, + r##" + SELECT id, created_at, archived, name, description, workspace_id, content, language AS "language: ScriptLang" + FROM workspace_dependencies + WHERE archived = false AND workspace_id = $1 + "##, + workspace_id, + ) + .fetch_all(e) + .await + .map_err(error::Error::from) + } + + /// Gets the latest version of workspace dependencies by name and language. + pub async fn get_latest( + name: Option, + language: ScriptLang, + workspace_id: &str, + conn: Connection, + ) -> error::Result> { + if language.as_dependencies_filename().is_none() { + return Ok(None); + } + + let cache_key = (workspace_id.to_string(), language, name.clone()); + + // Check if cached value is still valid + if let Some((cached_value, cached_at)) = WORKSPACE_DEPENDENCIES_CACHE.get(&cache_key) { + if cached_at.elapsed() < CACHE_TIMEOUT { + return Ok(cached_value); + } + // Expired, remove it + WORKSPACE_DEPENDENCIES_CACHE.remove(&cache_key); + } + + // Fetch and cache + let fetch = Box::pin(async { + match &conn { + Connection::Sql(db) => sqlx::query_as!( + Self, + r#" + SELECT id, content, language AS "language: ScriptLang", name, description, archived, workspace_id, created_at + FROM workspace_dependencies + WHERE name IS NOT DISTINCT FROM $1 AND workspace_id = $2 AND archived = false AND language = $3 + LIMIT 1 + "#, + name, + workspace_id, + language as ScriptLang + ) + .fetch_optional(db) + .await + .map_err(error::Error::from), + + Connection::Http(http_client) => http_client + .get::>(&format!( + "/api/w/{workspace_id}/agent_workers/workspace_dependencies/get_latest/{}{}", + language.as_str(), + if let Some(ref name_val) = name { + format!("?name={name_val}") + } else { + "".to_owned() + } + )) + .await + .map_err(error::Error::from), + } + }); + + let (workspace_dependencies_o, ..) = WORKSPACE_DEPENDENCIES_CACHE + .get_or_insert_async(&cache_key, async { + Ok::<_, error::Error>((fetch.await?, Instant::now())) + }) + .await?; + + Ok(workspace_dependencies_o) + } + + /// Gets workspace dependencies by their unique ID. + pub async fn get<'c>( + id: i64, + workspace_id: &str, + e: impl PgExecutor<'c>, + ) -> error::Result> { + sqlx::query_as!( + Self, + r#" + SELECT id, content, language AS "language: ScriptLang", name, archived, description, workspace_id, created_at + FROM workspace_dependencies + WHERE id = $1 AND workspace_id = $2 + LIMIT 1 + "#, + id, + workspace_id + ) + .fetch_optional(e) + .await + .map_err(error::Error::from) + } + + /// Gets the version history for workspace dependencies. + pub async fn get_history<'c>( + name: Option, + language: ScriptLang, + workspace_id: &str, + e: impl PgExecutor<'c>, + ) -> error::Result> { + if language.as_dependencies_filename().is_none() { + return Ok(vec![]); + } + sqlx::query_scalar!( + r#" + SELECT id FROM workspace_dependencies + WHERE name IS NOT DISTINCT FROM $1 AND workspace_id = $2 AND language = $3 + "#, + name, + workspace_id, + language as ScriptLang + ) + .fetch_all(e) + .await + .map_err(error::Error::from) + } + + pub fn to_path(name: &Option, language: ScriptLang) -> error::Result { + let requirements_filename = + language + .as_dependencies_filename() + .ok_or(error::Error::BadConfig(format!( + "workspace dependencies are not supported for: {}", + language.as_str() + )))?; + + Ok(if let Some(name) = name { + format!("dependencies/{name}.{requirements_filename}") + } else { + format!("dependencies/{requirements_filename}") + }) + } +} + +#[derive(Debug, Clone)] +pub struct WorkspaceDependenciesPrefetched { + language: ScriptLang, + runnable_path: String, + #[allow(dead_code)] + workspace_id: String, + internal: WorkspaceDependenciesPrefetchedInternal, +} + +#[derive(Debug, Clone)] +enum WorkspaceDependenciesPrefetchedInternal { + Explicit(WorkspaceDependenciesAnnotatedRefs), + Implicit { workspace_dependencies: WorkspaceDependencies, mode: Mode }, + None, +} + +impl WorkspaceDependenciesPrefetched { + pub async fn extract<'c>( + code: &str, + language: ScriptLang, + workspace_id: &str, + raw_workspace_dependencies_o: &Option, + runnable_path: &str, + conn: Connection, + ) -> error::Result { + use WorkspaceDependenciesPrefetchedInternal::*; + + tracing::debug!(workspace_id, ?language, "extracting workspace dependencies"); + + Box::pin(async { + let r = if let Some(wdar) = + language.extract_workspace_dependencies_annotated_refs(code, runnable_path) + { + tracing::debug!(workspace_id, ?language, "found explicit annotations"); + + let expanded = wdar + .expand(language, workspace_id, raw_workspace_dependencies_o, conn) + .await?; + + Explicit(expanded) + // First try in raw dependencies + } else if let Some(workspace_dependencies) = get_raw_workspace_dependencies( + raw_workspace_dependencies_o, + Option::None, + language, + workspace_id.to_owned(), + ) { + tracing::debug!( + workspace_id, + ?language, + dep_id = workspace_dependencies.id, + "using implicit raw" + ); + + // Hardcode to manual for now. + Implicit { workspace_dependencies, mode: Mode::manual } + } else if let Some(workspace_dependencies) = + // If not found, fetch from db + WorkspaceDependencies::get_latest( + Option::None, + language, + workspace_id, + conn, + ) + .await? + { + tracing::debug!( + workspace_id, + ?language, + dep_id = workspace_dependencies.id, + "using implicit default" + ); + + // Hardcode to manual for now. + Implicit { workspace_dependencies, mode: Mode::manual } + } else { + tracing::debug!(workspace_id, ?language, "no dependencies found"); + + None + }; + + // Crucial part. It will drop all blacklisted runnables + WorkspaceDependenciesPrefetched { + internal: r, + language, + runnable_path: runnable_path.to_owned(), + workspace_id: workspace_id.to_owned(), + } + .preprocess() + .await + }) + .await + } + + pub fn get_python(&self) -> error::Result> { + use WorkspaceDependenciesPrefetchedInternal::*; + Ok(match &self.internal { + Explicit(wdar @ WorkspaceDependenciesAnnotatedRefs { inline, external, .. }) => { + tracing::debug!( + "Processing explicit workspace dependencies with inline: {}, external count: {}", + inline.is_some(), + external.len() + ); + + wdar.assert_inline_or_external_or_none().map_err(map_err)?; + wdar.assert_external_less_than(2).map_err(map_err)?; + wdar.assert_no_extra_mode_for_external().map_err(map_err)?; + external + .get(0) + .map(|wd| wd.content.clone()) + .or(inline.to_owned()) + } + Implicit { workspace_dependencies, .. } => { + self.internal.assert_no_extra_mode().map_err(map_err)?; + Some(workspace_dependencies.content.to_owned()) + } + None => Option::None, + }) + } + + pub fn get_bun(&self) -> error::Result> { + use WorkspaceDependenciesPrefetchedInternal::*; + self.internal.assert_no_extra_mode().map_err(map_err)?; + Ok(match &self.internal { + Explicit(wdar @ WorkspaceDependenciesAnnotatedRefs { external, .. }) => { + wdar.assert_no_inline().map_err(map_err)?; + wdar.assert_external_less_than(2).map_err(map_err)?; + external + .get(0) + .map(|wd| wd.content.clone()) + .or(Some("".to_owned())) + } + Implicit { workspace_dependencies, .. } => Some(workspace_dependencies.content.clone()), + None => Option::None, + }) + } + + pub fn get_go(&self) -> error::Result> { + // NOTE: go is disabled for now: + // https://discord.com/channels/930051556043276338/1031563866641018910/1443541229349634189 + + self.internal + .assert_no_workspace_dependencies() + .map_err(map_err)?; + Ok(None) + // use WorkspaceDependenciesPrefetchedInternal::*; + // self.internal.assert_no_manual_mode().map_err(map_err)?; + // Ok(match &self.internal { + // Explicit(wdar @ WorkspaceDependenciesAnnotatedRefs { external, .. }) => { + // wdar.assert_no_inline().map_err(map_err)?; + // wdar.assert_external_less_than(2).map_err(map_err)?; + // external.get(0).map(|wd| dbg!(wd.content.clone())).or(Some( + // " + // module mymod + // go 1.25 + // require () + // " + // .to_owned(), + // )) + // } + // Implicit { workspace_dependencies, .. } => Some(workspace_dependencies.content.clone()), + // None => Option::None, + // } + // .map(|go_mod_content| { + // if let Some(module) = go_mod_content + // .lines() + // .find(|l| l.trim_start().starts_with("module ")) + // { + // go_mod_content.replace(module, "module mymod") + // } else { + // format!("module mymod\n{go_mod_content}") + // } + // })) + } + + pub fn get_php(&self) -> error::Result> { + use WorkspaceDependenciesPrefetchedInternal::*; + self.internal.assert_no_extra_mode().map_err(map_err)?; + Ok(match &self.internal { + Explicit(wdar @ WorkspaceDependenciesAnnotatedRefs { external, .. }) => { + wdar.assert_no_inline().map_err(map_err)?; + wdar.assert_external_less_than(2).map_err(map_err)?; + external + .get(0) + .map(|wd| wd.content.clone()) + .or(Some(r#"{"require": {}}"#.to_owned())) + } + Implicit { workspace_dependencies, .. } => Some(workspace_dependencies.content.clone()), + None => Option::None, + }) + } + + /// Is the runnable permitted to have external references + pub fn is_external_references_permitted(runnable_path: &str) -> bool { + !BLACKLIST.contains(runnable_path) && !runnable_path.starts_with("hub/") + } + + async fn preprocess(mut self) -> error::Result { + // NOTE: we should error if it is not compatible. User should either update workers or do not use incompatible feature. + // Check if compatible with legacy and if not check that all workers run compatible versions. + if let Err(feature) = self.check_legacy_compat() { + min_version_supports_v0_workspace_dependencies() + .await + .map_err(|_| { + // NOTE: this error is flakey, sometimes it will error, somethimes it will just ignore. + error::Error::WorkersAreBehind { + feature, + min_version: MIN_VERSION_WORKSPACE_DEPENDENCIES.to_owned(), + } + })?; + } + + // NOTE: if you update or add new language, add compatibility checks here. + // for example if you were to update lang do: + // if let Err(incompatible_e) = self.check_v0_python_compat() { + // min_version_supports_v1_workspace_dependencies_python() + // ... + // } + // + // Where `check_v0_python_compat` describes all features of previous python + + if !Self::is_external_references_permitted(&self.runnable_path) { + self.remove_external_references(); + } + Ok(self) + } + + fn check_legacy_compat(&self) -> Result<(), String> { + use ScriptLang::*; + use WorkspaceDependenciesPrefetchedInternal::*; + match (self.language, &self.internal) { + // These languages except for python had none of this functionality + (Php | Bun | Bunnative | Go, wdp) => wdp.assert_no_workspace_dependencies()?, + + // Python, had #(extra_)requirements: + // but it had no external requirements. + // that's why we check if it is using only inline syntax + (Python3, Explicit(wdar)) => wdar.assert_no_external()?, + (Python3, wdp) => wdp.assert_no_implicit()?, + + _ => return Err(format!("language is unsupported")), + } + Ok(()) + } + + // TODO: + // pub fn check_v0_compat(&self) -> bool {} + // pub fn check_v1_compat(&self) -> bool {} + // pub fn check_v1_python_compat + + fn remove_external_references(&mut self) { + use WorkspaceDependenciesPrefetchedInternal::*; + match self.internal { + Explicit(WorkspaceDependenciesAnnotatedRefs { ref mut external, .. }) + if !external.is_empty() => + { + external.clear(); + } + // Implicit is an external reference to the default. So we just replace it to none + ref mut wdp @ Implicit { .. } => drop(std::mem::replace(wdp, None)), + // Return early not to show warning + _ => return, + } + tracing::warn!( + self.runnable_path, + "skipping external workspace dependencies for runnable" + ); + } + + pub async fn to_lock_header(&self) -> Option { + use WorkspaceDependenciesPrefetchedInternal::*; + + if min_version_supports_v0_workspace_dependencies() + .await + .is_err() + { + return Option::None; + } + + let mut header = vec![]; + let prepend_mode = |mode| { + format!( + "{} workspace-dependencies-mode: {}", + self.language.as_comment_lit(), + mode + ) + }; + + let insert_line = |hash, name: Option| { + format!( + "{} workspace-dependencies: {}:{}", + self.language.as_comment_lit(), + name.unwrap_or("default".to_owned()), + hash + ) + }; + match &self.internal { + Explicit(workspace_dependencies_annotated_refs) => { + header.push(prepend_mode(workspace_dependencies_annotated_refs.mode)); + for wd in &workspace_dependencies_annotated_refs.external { + header.push(insert_line(wd.hash(), wd.name.clone())); + } + } + Implicit { workspace_dependencies: wd, mode } => { + header.push(prepend_mode(*mode)); + header.push(insert_line(wd.hash(), Option::None)); + } + None => return Option::None, + } + Some(header.join("\n")) + } + + pub fn is_manual(&self) -> bool { + self.internal.get_mode() == Some(Mode::manual) + } +} + +#[derive(Debug, Clone)] +pub struct WorkspaceDependenciesAnnotatedRefs { + /// ```python + /// # requirements: + /// # rich==x.y.z << + /// # pandas==x.y.z << + /// ``` + pub inline: Option, + /// ```python + /// # requirements: default, base, prod + /// ^^^^^^^ ^^^^ ^^^^ + /// ``` + /// + /// Can either be a [[String]] or [[WorkspaceDependencies]] + /// + /// The workflow is following: + /// 1. You create Self with <[[String]]> - this will fetch a minimal amount of info (just the name). + /// 2. You [[Self::expand]] to replace all external names with <[[WorkspaceDependencies]]> + pub external: Vec, + pub mode: Mode, +} + +/// `# extra_requirements:` - Extra +/// `# requirements:` - Manual +#[allow(non_camel_case_types)] +#[derive(PartialEq, Eq, strum_macros::Display, Clone, Copy, Debug)] +pub enum Mode { + manual, + extra, +} + +impl WorkspaceDependenciesPrefetchedInternal { + fn get_mode(&self) -> Option { + use WorkspaceDependenciesPrefetchedInternal::*; + match &self { + Explicit(WorkspaceDependenciesAnnotatedRefs { mode, .. }) | Implicit { mode, .. } => { + Some(*mode) + } + None => Option::None, + } + } + + fn assert_no_implicit(&self) -> Result<(), String> { + if matches!(self, Self::Implicit { .. }) { + Err(format!("'default workspace dependencies'")) + } else { + Ok(()) + } + } + + fn assert_no_workspace_dependencies(&self) -> Result<(), String> { + if !matches!(self, Self::None { .. }) { + Err(format!("'workspace dependencies'")) + } else { + Ok(()) + } + } + + #[allow(dead_code)] + fn assert_no_explicit(&self) -> Result<(), String> { + if matches!(self, Self::Explicit { .. }) { + Err(format!("'external workspace dependencies'")) + } else { + Ok(()) + } + } + + fn assert_no_extra_mode(&self) -> Result<(), String> { + if self.get_mode() == Some(Mode::extra) { + Err(format!("'workspace dependencies in extra mode'")) + } else { + Ok(()) + } + } + + #[allow(dead_code)] + fn assert_no_manual_mode(&self) -> Result<(), String> { + if self.get_mode() == Some(Mode::manual) { + Err(format!("'workspace dependencies in manual mode'")) + } else { + Ok(()) + } + } +} + +impl WorkspaceDependenciesAnnotatedRefs { + fn assert_no_inline(&self) -> Result<(), String> { + if self.inline.is_none() { + Ok(()) + } else { + Err(format!("'inline workspace dependencies'")) + } + } + + fn assert_no_extra_mode_for_external(&self) -> Result<(), String> { + if self.mode == Mode::extra && !self.external.is_empty() { + Err(format!("'external workspace dependencies in extra mode'")) + } else { + Ok(()) + } + } + + #[allow(dead_code)] + fn assert_no_extra_mode_for_inline(&self) -> Result<(), String> { + if self.mode == Mode::extra && self.inline.is_some() { + Err(format!("'inline workspace dependencies in extra mode'")) + } else { + Ok(()) + } + } + + fn assert_no_external(&self) -> Result<(), String> { + self.assert_external_less_than(1) + .map_err(|_e| format!("'external workspace dependencies'")) + } + + fn assert_inline_or_external_or_none(&self) -> Result<(), String> { + if self.inline.is_some() && !self.external.is_empty() { + Err(format!( + "'inline and externally referenced workspace dependencies at the same time'" + )) + } else { + Ok(()) + } + } + fn assert_external_less_than(&self, amount: usize) -> Result<(), String> { + if self.external.len() < amount { + Ok(()) + } else { + Err(format!( + "'multiple external workspace dependencies referenced'", + )) + } + } +} + +impl WorkspaceDependenciesAnnotatedRefs { + pub(super) async fn expand( + self, + language: ScriptLang, + workspace_id: &str, + raw_workspace_dependencies_o: &Option, + conn: Connection, + ) -> error::Result> { + let mut res = WorkspaceDependenciesAnnotatedRefs { + inline: self.inline, + external: vec![], + mode: self.mode, + }; + + for name in self.external { + // "default" maps to unnamed workspace dependencies. + let name = if name == "default" { None } else { Some(name) }; + // First try in raw dependencies + if let Some(wd) = get_raw_workspace_dependencies( + raw_workspace_dependencies_o, + name.clone(), + language, + workspace_id.to_owned(), + ) { + res.external.push(wd); + + // If not found, fetch from db + } else if let Some(wd) = WorkspaceDependencies::get_latest( + name.clone(), + language, + workspace_id, + conn.clone(), + ) + .await? + { + res.external.push(wd.clone()); + } else { + tracing::warn!( + workspace_id, + ?language, + dependency_name = name, + "workspace dependencies not found" + ); + } + } + Ok(res) + } + // TODO: Maybe implemented by our Annotations macro + // TODO: Add sep config ':' or '='? + pub fn parse( + comment: &str, + keyword: &str, + code: &str, + validity_re_o: Option<&Regex>, + runnable_path: &str, + ) -> Option { + let (extra_deps, manual_deps) = (format!("extra_{keyword}:"), format!("{keyword}:")); + + let Some((pos, mat)) = code.lines().find_position(|l| { + l.starts_with(&comment) && (l.contains(&extra_deps) || l.contains(&manual_deps)) + }) else { + return None; + }; + let mut lines_it = code.lines().skip(pos); + + let mode = if mat.contains(&extra_deps) { + Mode::extra + } else { + Mode::manual + }; + + let external = { + let next_line = lines_it.next(); + if !WorkspaceDependenciesPrefetched::is_external_references_permitted(runnable_path) { + tracing::warn!( + runnable_path, + "skipping external workspace dependencies for runnable" + ); + + Default::default() + // return Err(error::Error::BadConfig(format!( + // "{runnable_path} should not include external Workspace Dependencies" + // ))); + } else { + next_line + .map(|s| { + match mode { + Mode::manual => s.replace(&manual_deps, ""), + Mode::extra => s.replace(&extra_deps, ""), + } + .replace(comment, "") + }) + .map(|unparsed| { + unparsed + .split(',') + // TODO: do we want to sort it? + .map(str::trim) + .filter(|s| !s.is_empty()) + // .map(FromName::from_name) + .map(str::to_owned) + .collect_vec() + }) + .unwrap_or_default() + } + }; + + let inline_deps = lines_it + .map_while(|l| { + match validity_re_o { + Some(re) => re.captures(l).and_then(|c| c.get(1).map(|m| m.as_str())), + None => { + if !l.starts_with(comment) { + None + } else { + // Skip comment + // If it fails (None) iteration is just finished. + l.get(comment.len()..) + } + } + } + }) + .join("\n"); + + let inline = if inline_deps.trim().is_empty() { + None + } else { + Some(inline_deps) + }; + + Some(WorkspaceDependenciesAnnotatedRefs { + inline, // TODO: Parse + external, + mode, + }) + } +} + +#[cfg(test)] +mod workspace_dependencies_tests { + use super::*; + + #[test] + fn test_parse_annotation_python_requirements_manual_mode() { + let code = r#" +# requirements: default, base +#requests==2.31.0 +#pandas>=1.5.0 + +def main(): + pass +"#; + + let result = WorkspaceDependenciesAnnotatedRefs::::parse( + "#", + "requirements", + code, + None, + "", + ) + .unwrap(); + assert!(matches!(result.mode, Mode::manual)); + assert_eq!( + result.external, + vec!["default".to_owned(), "base".to_owned()] + ); + assert_eq!( + result.inline.as_ref().unwrap(), + "requests==2.31.0\npandas>=1.5.0" + ); + } + + #[test] + fn test_parse_annotation_python_extra_requirements_mode() { + let code = r#" +# extra_requirements: utils +#numpy>=1.24.0 + +def main(): + pass +"#; + + let result = WorkspaceDependenciesAnnotatedRefs::::parse( + "#", + "requirements", + code, + None, + "", + ) + .unwrap(); + assert!(matches!(result.mode, Mode::extra)); + assert_eq!(result.external, vec!["utils".to_owned()]); + assert_eq!(result.inline.as_ref().unwrap(), "numpy>=1.24.0"); + } + + #[test] + fn test_parse_annotation_typescript_requirements() { + let code = r#" +// requirements: utils, base +//{ +// "dependencies": { +// "axios": "^1.6.0" +// } +//} + +export function main() {} +"#; + + let result = WorkspaceDependenciesAnnotatedRefs::::parse( + "//", + "requirements", + code, + None, + "", + ) + .unwrap(); + assert!(matches!(result.mode, Mode::manual)); + assert_eq!(result.external, vec!["utils".to_owned(), "base".to_owned()]); + let expected_inline = r#"{ + "dependencies": { + "axios": "^1.6.0" + } +}"#; + assert_eq!(result.inline.as_ref().unwrap(), expected_inline); + } + + #[test] + fn test_parse_annotation_with_spacing_variations() { + let code = r#" +#requirements: no_space +def main(): + pass +"#; + + let result = WorkspaceDependenciesAnnotatedRefs::::parse( + "#", + "requirements", + code, + None, + "", + ) + .unwrap(); + assert!(matches!(result.mode, Mode::manual)); + assert_eq!(result.external, vec!["no_space".to_owned()]); + assert!(result.inline.is_none()); + } + + #[test] + fn test_parse_annotation_with_spacing_variations_spaced() { + let code = r#" +# requirements: with_space +def main(): + pass +"#; + + let result = WorkspaceDependenciesAnnotatedRefs::::parse( + "#", + "requirements", + code, + None, + "", + ) + .unwrap(); + assert!(matches!(result.mode, Mode::manual)); + assert_eq!(result.external, vec!["with_space".to_owned()]); + assert!(result.inline.is_none()); + } + + #[test] + fn test_parse_annotation_go_style() { + let code = r#" +// go_mod: base, +//github.com/gin-gonic/gin v1.9.1 + +package main +func main() {} +"#; + + let result = + WorkspaceDependenciesAnnotatedRefs::::parse("//", "go_mod", code, None, "") + .unwrap(); + assert!(matches!(result.mode, Mode::manual)); + assert_eq!(result.external, vec!["base".to_owned()]); + assert_eq!( + result.inline.as_ref().unwrap(), + "github.com/gin-gonic/gin v1.9.1" + ); + } + + #[test] + fn test_parse_annotation_no_inline_deps() { + let code = r#" +# requirements: default + +def main(): + pass +"#; + + let result = WorkspaceDependenciesAnnotatedRefs::::parse( + "#", + "requirements", + code, + None, + "", + ) + .unwrap(); + assert!(matches!(result.mode, Mode::manual)); + assert_eq!(result.external, vec!["default".to_owned()]); + assert!(result.inline.is_none()); + } + + #[test] + fn test_parse_annotation_inline_only() { + let code = r#" +# requirements: +#requests==2.31.0 +# pandas>=1.5.0 + +def main(): + pass +"#; + + let result = WorkspaceDependenciesAnnotatedRefs::::parse( + "#", + "requirements", + code, + None, + "", + ) + .unwrap(); + assert!(matches!(result.mode, Mode::manual)); + assert!(result.external.is_empty()); + assert_eq!( + result.inline.as_ref().unwrap(), + "requests==2.31.0\n pandas>=1.5.0" + ); + } + + #[test] + fn test_parse_annotation_no_match() { + let code = r#" +def main(): + pass +"#; + + let result = WorkspaceDependenciesAnnotatedRefs::::parse( + "#", + "requirements", + code, + None, + "", + ); + assert!(result.is_none()); + } + + #[test] + fn test_parse_annotation_php_style() { + let code = r#" +::parse( + "//", + "requirements", + code, + None, + "", + ) + .unwrap(); + assert!(matches!(result.mode, Mode::manual)); + assert_eq!(result.external, vec!["composer".to_owned()]); + let expected_inline = r#"{ + "require": { + "guzzlehttp/guzzle": "^7.0" + } +}"#; + assert_eq!(result.inline.as_ref().unwrap(), expected_inline); + } + + #[test] + fn test_parse_annotation_just_requirements_colon() { + let code = r#" +#requirements: + +def main(): + pass +"#; + + let result = WorkspaceDependenciesAnnotatedRefs::::parse( + "#", + "requirements", + code, + None, + "", + ) + .unwrap(); + assert!(matches!(result.mode, Mode::manual)); + assert!(result.external.is_empty()); + assert!(result.inline.is_none()); + } + #[test] + fn test_parse_annotation_blacklisted() { + let code = r#" +#requirements: hello, world + +def main(): + pass +"#; + + let result = WorkspaceDependenciesAnnotatedRefs::::parse( + "#", + "requirements", + code, + None, + "u/admin/hub_sync", + ) + .unwrap(); + assert!(matches!(result.mode, Mode::manual)); + assert!(result.external.is_empty()); + assert!(result.inline.is_none()); + } +} diff --git a/backend/windmill-macros/Cargo.toml b/backend/windmill-macros/Cargo.toml index 100b4678d3..5d7d12cc6a 100644 --- a/backend/windmill-macros/Cargo.toml +++ b/backend/windmill-macros/Cargo.toml @@ -11,6 +11,9 @@ proc-macro = true proc-macro2.workspace = true quote.workspace = true syn.workspace = true +serde.workspace = true +serde_yml.workspace = true +serde_derive.workspace = true # Dependencies for tests [dev-dependencies] @@ -18,3 +21,7 @@ syn.workspace = true lazy_static.workspace = true itertools.workspace = true regex.workspace = true +serde.workspace = true +serde_yml.workspace = true +serde_derive.workspace = true +pep440_rs.workspace = true diff --git a/backend/windmill-macros/tests/annotations.rs b/backend/windmill-macros/tests/annotations.rs index d906b72887..395643364d 100644 --- a/backend/windmill-macros/tests/annotations.rs +++ b/backend/windmill-macros/tests/annotations.rs @@ -3,6 +3,7 @@ mod annotations_tests { extern crate windmill_macros; use itertools::Itertools; + // use pep440_rs::Version; use windmill_macros::annotations; // Previous implementation. @@ -166,4 +167,99 @@ mod annotations_tests { assert_eq!(expected, Annotations::parse(cont)); } } + + // // #[derive(serde_derive::Serialize, serde_derive::Deserialize, Eq, PartialEq)] + // // #[annotations("#")] + // // pub struct SerAnnotations { + // // pub ann1: bool, + // // pub stt: String, + // // } + + // #[test] + // fn non_bool_1() { + // let cont = r#"#ann1, stt: "hey""#; + // // non-bool take entire line, so you can't have one normal and than parsed. + + // let a = SerAnnotations { ann1: false, stt: "".to_owned() }; + // assert_eq!(a, SerAnnotations::parse(cont)); + // } + + // #[test] + // fn non_bool_2() { + // let cont = "#ann1, \n#stt: hey"; + // let a = SerAnnotations { ann1: true, stt: "hey".to_owned() }; + // assert_eq!(a, SerAnnotations::parse(cont)); + // } + + // #[test] + // fn non_bool_different_idents() { + // #[derive(serde_derive::Serialize, serde_derive::Deserialize, Eq, PartialEq)] + // #[annotations("#")] + // pub struct A { + // pub s: String, + // } + // assert_eq!(A { s: "hey".to_owned() }, A::parse("#s:hey")); + // assert_eq!(A { s: "hey".to_owned() }, A::parse("#s : hey")); + // assert_eq!(A { s: "hey".to_owned() }, A::parse("#s :hey")); + // assert_eq!(A { s: "hey".to_owned() }, A::parse("#s : hey ")); + // assert_eq!(A { s: "hey".to_owned() }, A::parse("# s : hey ")); + // assert_eq!(A { s: "".to_owned() }, A::parse(" # s : hey ")); + // } + // #[test] + // fn non_bool_unparseable_last() { + // #[derive(serde_derive::Serialize, serde_derive::Deserialize, Eq, PartialEq)] + // #[annotations("#")] + // pub struct A { + // pub v: i32, + // } + // let cont = "#v: 1\n#v: non int"; + // let a = A { + // v: 1, // Should still be first, second unparsable one should have no affect on existing values + // }; + // assert_eq!(a, A::parse(cont)); + // } + + // #[test] + // fn non_bool_different_types() { + // #[derive(serde_derive::Serialize, serde_derive::Deserialize, Eq, PartialEq)] + // #[annotations("#")] + // pub struct A { + // pub s: String, + // pub i: i32, + // pub o: Option, + // pub a: Vec, + // pub v: Option, // Custom deser + // pub e: E, + // } + + // #[derive( + // serde_derive::Serialize, serde_derive::Deserialize, Eq, PartialEq, Clone, Default, Debug, + // )] + // pub enum E { + // #[default] + // One, + // Two(String), + // Three, + // } + // assert_eq!( + // A { + // s: "foo".to_owned(), + // i: 33, + // o: None, + // a: vec![1, 2, 3], + // v: Some(Version::new([1, 0, 0])), + // e: E::Three + // }, + // A::parse( + // "# + // #s: foo + // #i: 33 + // #o: + // #a: [1, 2, 3] + // #v: 1.0.0 + // #e: Three + // " + // ) + // ); + // } } diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index cc29b94f1b..25d10784bb 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -38,7 +38,6 @@ use windmill_common::add_time; use windmill_common::auth::JobPerms; #[cfg(feature = "benchmark")] use windmill_common::bench::BenchmarkIter; -use windmill_common::lockfiles::is_generated_from_raw_requirements; use windmill_common::jobs::{JobTriggerKind, EMAIL_ERROR_HANDLER_USER_EMAIL}; use windmill_common::utils::{configure_client, now_from_db}; use windmill_common::worker::{Connection, MIN_VERSION_SUPPORTS_DEBOUNCING, SCRIPT_TOKEN_EXPIRY}; @@ -2659,13 +2658,6 @@ impl PulledJobResult { ) .await?; - if is_generated_from_raw_requirements(&Some(cloned_script.old_script.language), &cloned_script.old_script.lock.map(|v| v.to_string())) { - return Err(Error::BadRequest(format!( - "Script at path {} is generated from raw requirements, not overriding", - pulled_job.runnable_path() - ))); - } - cloned_script.new_hash } JobKind::FlowDependencies => { @@ -3632,7 +3624,7 @@ pub enum PushIsolationLevel<'c> { } impl<'c> PushIsolationLevel<'c> { - async fn into_tx(self) -> error::Result> { + pub async fn into_tx(self) -> error::Result> { match self { PushIsolationLevel::Isolated(db, authed) => Ok((db.begin(&authed).await?).into()), PushIsolationLevel::IsolatedRoot(db) => Ok(db.begin().await?), @@ -5867,3 +5859,4 @@ pub async fn get_same_worker_job( )) }) } + diff --git a/backend/windmill-worker/src/ansible_executor.rs b/backend/windmill-worker/src/ansible_executor.rs index 9aed0e18ad..3da3fa619d 100644 --- a/backend/windmill-worker/src/ansible_executor.rs +++ b/backend/windmill-worker/src/ansible_executor.rs @@ -13,10 +13,10 @@ use tokio::process::Command; use uuid::Uuid; use windmill_common::{ error, - git_sync_oss::{prepend_token_to_github_url}, + git_sync_oss::prepend_token_to_github_url, worker::{ - is_allowed_file_location, to_raw_value, write_file, write_file_at_user_defined_location, - Connection, WORKER_CONFIG, + is_allowed_file_location, split_python_requirements, to_raw_value, write_file, + write_file_at_user_defined_location, Connection, PyVAlias, WORKER_CONFIG, }, }; use windmill_queue::MiniPulledJob; @@ -34,8 +34,8 @@ use crate::{ }, handle_child::handle_child, python_executor::{create_dependencies_dir, handle_python_reqs, uv_pip_compile}, - DISABLE_NSJAIL, DISABLE_NUSER, GIT_PATH, HOME_ENV, NSJAIL_PATH, PATH_ENV, - PROXY_ENVS, PY_INSTALL_DIR, PyVAlias, TZ_ENV, + DISABLE_NSJAIL, DISABLE_NUSER, GIT_PATH, HOME_ENV, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, + PY_INSTALL_DIR, TZ_ENV, }; use windmill_common::client::AuthedClient; @@ -399,7 +399,7 @@ async fn handle_ansible_python_deps( if requirements.len() > 0 { let mut venv_path = handle_python_reqs( - crate::python_executor::split_requirements(requirements), + split_python_requirements(requirements), job_id, w_id, mem_peak, @@ -958,7 +958,11 @@ pub async fn handle_ansible_job( #[cfg(feature = "enterprise")] if is_github_app { if let Connection::Sql(db) = conn { - let token = windmill_common::git_sync_oss::get_github_app_token_internal(db, &client.token).await?; + let token = windmill_common::git_sync_oss::get_github_app_token_internal( + db, + &client.token, + ) + .await?; secret_url = prepend_token_to_github_url(&secret_url, &token)?; } else { return Err(windmill_common::error::Error::BadRequest("Github App authentication is currently unavailable for agent workers. Contact the windmill team to request this feature".to_string())); @@ -970,8 +974,12 @@ pub async fn handle_ansible_job( let target_path = "delegate_git_repository".to_string(); - let repo = - GitRepo { url: secret_url, commit: delegated_git_repo.commit.clone(), branch, target_path }; + let repo = GitRepo { + url: secret_url, + commit: delegated_git_repo.commit.clone(), + branch, + target_path, + }; append_logs( &job.id, &job.workspace_id, @@ -1241,10 +1249,8 @@ fi start_child_process(nsjail_cmd, NSJAIL_PATH.as_str(), false).await? } else { let ansible_args: Vec<&str> = cmd_args.iter().map(|s| s.as_ref()).collect(); - let mut ansible_cmd = build_command_with_isolation( - ANSIBLE_PLAYBOOK_PATH.as_str(), - &ansible_args, - ); + let mut ansible_cmd = + build_command_with_isolation(ANSIBLE_PLAYBOOK_PATH.as_str(), &ansible_args); ansible_cmd .current_dir(job_dir) .env_clear() diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index d1c50b9024..e76974d319 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -16,7 +16,7 @@ use crate::{ common::{ build_command_with_isolation, create_args_and_out_file, get_reserved_variables, parse_npm_config, read_file, read_file_content, read_result, start_child_process, - write_file_binary, OccupancyMetrics, StreamNotifier, + write_file_binary, MaybeLock, OccupancyMetrics, StreamNotifier, }, handle_child::handle_child, BUNFIG_INSTALL_SCOPES, BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR, BUN_NO_CACHE, BUN_PATH, @@ -27,6 +27,7 @@ use windmill_common::{ client::AuthedClient, s3_helpers::BundleFormat, scripts::{id_to_codebase_info, CodebaseInfo}, + workspace_dependencies::WorkspaceDependenciesPrefetched, }; #[cfg(windows)] @@ -40,7 +41,7 @@ use windmill_common::{ error::{self, Result}, get_latest_hash_for_path, scripts::ScriptLang, - worker::{exists_in_cache, save_cache, to_raw_value, write_file, Connection, DISABLE_BUNDLING}, + worker::{exists_in_cache, save_cache, write_file, Connection, DISABLE_BUNDLING}, DB, }; @@ -101,7 +102,7 @@ pub async fn gen_bun_lockfile( base_internal_url: &str, worker_name: &str, export_pkg: bool, - raw_deps: Option, + workspace_dependencies: &WorkspaceDependenciesPrefetched, npm_mode: bool, occupancy_metrics: &mut Option<&mut OccupancyMetrics>, ) -> Result> { @@ -109,11 +110,11 @@ pub async fn gen_bun_lockfile( let mut empty_deps = false; - if let Some(raw_deps) = raw_deps.as_ref() { + if let Some(package_json_content) = workspace_dependencies.get_bun()? { gen_bunfig(job_dir).await?; - write_file(job_dir, "package.json", raw_deps.as_str())?; + write_file(job_dir, "package.json", package_json_content.as_str())?; } else { - let _ = write_file( + write_file( &job_dir, "build.js", &format!( @@ -205,16 +206,7 @@ pub async fn gen_bun_lockfile( let mut file = File::open(format!("{job_dir}/package.json")).await?; let mut buf = String::default(); file.read_to_string(&mut buf).await?; - if raw_deps.is_some() { - let mut json_map: HashMap> = serde_json::from_str(&buf)?; - json_map.insert( - "generatedFromPackageJson".to_string(), - to_raw_value(&"true".to_string()), - ); - content = serde_json::to_string_pretty(&json_map)?; - } else { - content = buf; - } + content = buf; } if !npm_mode { #[cfg(any(target_os = "linux", target_os = "macos"))] @@ -703,7 +695,7 @@ fn extract_saved_codebase( pub async fn prebundle_bun_script( inner_content: &str, - lockfile: Option<&String>, + lock: &str, script_path: &str, job_id: &Uuid, w_id: &str, @@ -715,7 +707,7 @@ pub async fn prebundle_bun_script( occupancy_metrics: &mut Option<&mut OccupancyMetrics>, ) -> Result<()> { let (local_path, remote_path) = - compute_bundle_local_and_remote_path(inner_content, lockfile, script_path, db, w_id).await; + compute_bundle_local_and_remote_path(inner_content, lock, script_path, db, w_id).await; if exists_in_cache(&local_path, &remote_path).await { return Ok(()); } @@ -779,16 +771,12 @@ async fn get_script_import_updated_at(db: &DB, w_id: &str, script_path: &str) -> pub async fn compute_bundle_local_and_remote_path( inner_content: &str, - requirements_o: Option<&String>, + lock: &str, script_path: &str, db: Option<&DB>, w_id: &str, ) -> (String, String) { - let mut input_src = format!( - "{}{}", - inner_content, - requirements_o.as_ref().map(|x| x.as_str()).unwrap_or("") - ); + let mut input_src = format!("{inner_content}{lock}",); if let Some(db) = db { let relative_imports = crate::worker_lockfiles::extract_relative_imports( @@ -844,7 +832,7 @@ async fn write_lock(splitted_lockb_2: &str, job_dir: &str, is_binary: bool) -> R #[tracing::instrument(level = "trace", skip_all)] pub async fn handle_bun_job( - requirements_o: Option<&String>, + maybe_lock: MaybeLock, codebase: Option<&String>, mem_peak: &mut i32, canceled_by: &mut Option, @@ -865,16 +853,15 @@ pub async fn handle_bun_job( ) -> error::Result> { let mut annotation = windmill_common::worker::TypeScriptAnnotations::parse(inner_content); - let (mut has_bundle_cache, cache_logs, local_path, remote_path) = if requirements_o.is_some() - && !annotation.nobundling - && !*DISABLE_BUNDLING - && codebase.is_none() - { + let (mut has_bundle_cache, cache_logs, local_path, remote_path) = if let (Some(lock), true) = ( + maybe_lock.get_lock(), + !annotation.nobundling && !*DISABLE_BUNDLING && codebase.is_none(), + ) { let (local_path, remote_path) = match conn { Connection::Sql(db) => { compute_bundle_local_and_remote_path( inner_content, - requirements_o, + lock, job.runnable_path(), Some(db), &job.workspace_id, @@ -940,59 +927,70 @@ pub async fn handle_bun_job( if pulled_codebase.is_esm { format = BundleFormat::Esm; } - } else if let Some(reqs) = requirements_o.as_ref() { - let (pkg, lock, empty, is_binary) = split_lockfile(reqs); - - if lock.is_none() && !annotation.npm { - return Err(error::Error::ExecutionErr( - format!("Invalid requirements, expected to find //bun.lock{} split pattern in reqs. Found: |{reqs}|", if is_binary {"b"} else {""}) - )); - } - - let _ = write_file(job_dir, "package.json", pkg)?; - let lock = if annotation.npm { "" } else { lock.unwrap() }; - if !empty { - if !annotation.npm { - let _ = write_lock(lock, job_dir, is_binary).await?; - } - - install_bun_lockfile( - mem_peak, - canceled_by, - &job.id, - &job.workspace_id, - Some(conn), - job_dir, - worker_name, - common_bun_proc_envs.clone(), - annotation.npm, - &mut Some(occupancy_metrics), - ) - .await?; - } } else { - // if !*DISABLE_NSJAIL || !empty_trusted_deps || has_custom_config_registry { - let logs1 = "\n\n--- BUN INSTALL ---\n".to_string(); - append_logs(&job.id, &job.workspace_id, logs1, conn).await; - let _ = gen_bun_lockfile( - mem_peak, - canceled_by, - &job.id, - &job.workspace_id, - Some(conn), - &client.token, - job.runnable_path(), - job_dir, - base_internal_url, - worker_name, - false, - None, - annotation.npm, - &mut Some(occupancy_metrics), - ) - .await?; + match &maybe_lock { + MaybeLock::Resolved { lock } => { + let (package_json, bun_lock, empty, is_binary) = split_lockfile(lock); - // } + if bun_lock.is_none() && !annotation.npm { + return Err(error::Error::ExecutionErr( + format!("Invalid requirements, expected to find //bun.lock{} split pattern in reqs. Found: |{lock}|", if is_binary {"b"} else {""}) + )); + } + + write_file(job_dir, "package.json", package_json)?; + + let bun_lock = if annotation.npm { + "" + } else { + bun_lock.unwrap() + }; + + if !empty { + if !annotation.npm { + write_lock(bun_lock, job_dir, is_binary).await?; + } + + install_bun_lockfile( + mem_peak, + canceled_by, + &job.id, + &job.workspace_id, + Some(conn), + job_dir, + worker_name, + common_bun_proc_envs.clone(), + annotation.npm, + &mut Some(occupancy_metrics), + ) + .await?; + } + } + MaybeLock::Unresolved { ref workspace_dependencies } => { + // if !*DISABLE_NSJAIL || !empty_trusted_deps || has_custom_config_registry { + let logs1 = "\n\n--- BUN INSTALL ---\n".to_string(); + append_logs(&job.id, &job.workspace_id, logs1, conn).await; + gen_bun_lockfile( + mem_peak, + canceled_by, + &job.id, + &job.workspace_id, + Some(conn), + &client.token, + job.runnable_path(), + job_dir, + base_internal_url, + worker_name, + false, + workspace_dependencies, + annotation.npm, + &mut Some(occupancy_metrics), + ) + .await?; + + // } + } + } } if codebase.is_some() && format == BundleFormat::Cjs { @@ -1184,7 +1182,7 @@ try {{ && !annotation.nobundling && !*DISABLE_BUNDLING && !codebase.is_some() - && (requirements_o.is_some() || annotation.native); + && (maybe_lock.get_lock().is_some() || annotation.native); let write_loader_f = async { if build_cache { @@ -1689,7 +1687,15 @@ pub async fn start_worker( base_internal_url, worker_name, false, - None, + &WorkspaceDependenciesPrefetched::extract( + inner_content, + ScriptLang::Bun, + w_id, + &None, + &script_path, + db.into(), + ) + .await?, annotation.npm, &mut None, ) diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs index 2d995ea9bb..4856e030f7 100644 --- a/backend/windmill-worker/src/common.rs +++ b/backend/windmill-worker/src/common.rs @@ -24,6 +24,7 @@ use windmill_common::worker::{ to_raw_value, update_ping_for_failed_init_script_query, write_file, Connection, Ping, PingType, CLOUD_HOSTED, ROOT_CACHE_DIR, WORKER_CONFIG, }; +use windmill_common::workspace_dependencies::WorkspaceDependenciesPrefetched; use windmill_common::{ cache::{Cache, RawData}, error::{self, Error}, @@ -629,10 +630,7 @@ lazy_static! { static ref DISABLE_PROCESS_GROUP: bool = std::env::var("DISABLE_PROCESS_GROUP").is_ok(); } -pub fn build_command_with_isolation( - program: &str, - args: &[&str], -) -> Command { +pub fn build_command_with_isolation(program: &str, args: &[&str]) -> Command { use tokio::process::Command; if *crate::ENABLE_UNSHARE_PID { @@ -1371,3 +1369,35 @@ pub fn s3_mode_args_to_worker_data( workspace_id: job.workspace_id.clone(), } } + +#[derive(Debug)] +pub enum MaybeLock { + /// Deployed Scripts + Resolved { lock: String }, + /// Previews + Unresolved { workspace_dependencies: WorkspaceDependenciesPrefetched }, +} + +impl MaybeLock { + pub fn map_unresolved(&self, mut f: F) -> Option + where + Self: Sized, + F: FnMut(&WorkspaceDependenciesPrefetched) -> B, + { + self.get_workspace_dependencies().map(|wd| f(wd)) + } + + pub fn get_workspace_dependencies(&self) -> Option<&WorkspaceDependenciesPrefetched> { + match self { + MaybeLock::Resolved { .. } => None, + MaybeLock::Unresolved { ref workspace_dependencies } => Some(workspace_dependencies), + } + } + + pub fn get_lock(&self) -> Option<&String> { + match self { + MaybeLock::Resolved { ref lock } => Some(lock), + MaybeLock::Unresolved { .. } => None, + } + } +} diff --git a/backend/windmill-worker/src/go_executor.rs b/backend/windmill-worker/src/go_executor.rs index bf3957c430..d99adff7e3 100644 --- a/backend/windmill-worker/src/go_executor.rs +++ b/backend/windmill-worker/src/go_executor.rs @@ -1,4 +1,4 @@ -use crate::PROXY_ENVS; +use crate::{common::MaybeLock, PROXY_ENVS}; use std::{collections::HashMap, fs::DirBuilder, process::Stdio}; use itertools::Itertools; @@ -19,8 +19,8 @@ use windmill_queue::{append_logs, CanceledBy, MiniPulledJob}; use crate::{ common::{ - build_command_with_isolation, capitalize, create_args_and_out_file, get_reserved_variables, read_result, - start_child_process, OccupancyMetrics, + build_command_with_isolation, capitalize, create_args_and_out_file, get_reserved_variables, + read_result, start_child_process, OccupancyMetrics, }, handle_child::handle_child, DISABLE_NSJAIL, DISABLE_NUSER, GOPRIVATE, GOPROXY, GO_BIN_CACHE_DIR, GO_CACHE_DIR, HOME_ENV, @@ -91,12 +91,12 @@ pub async fn handle_go_job( parent_runnable_path: Option, inner_content: &str, job_dir: &str, - requirements_o: Option<&String>, shared_mount: &str, base_internal_url: &str, worker_name: &str, envs: HashMap, occupation_metrics: &mut OccupancyMetrics, + maybe_lock: MaybeLock, ) -> Result, Error> { //go does not like executing modules at temp root let job_dir = &format!("{job_dir}/go"); @@ -105,14 +105,7 @@ pub async fn handle_go_job( .create(&job_dir) .expect("could not create go job dir"); - let hash = calculate_hash(&format!( - "{}{}v2", - inner_content, - requirements_o - .as_ref() - .map(|x| x.to_string()) - .unwrap_or_default() - )); + let hash = calculate_hash(&format!("{}{:?}v2", inner_content, &maybe_lock)); let bin_path = format!("{}/{hash}", GO_BIN_CACHE_DIR); let remote_path = format!("{GO_OBJECT_STORE_PREFIX}{hash}"); let (cache, cache_logs) = @@ -120,8 +113,8 @@ pub async fn handle_go_job( let (skip_go_mod, skip_tidy) = if cache { (true, true) - } else if let Some(requirements) = requirements_o { - gen_go_mod(inner_content, job_dir, &requirements).await? + } else if let Some(lock) = maybe_lock.get_lock() { + gen_go_mod(inner_content, job_dir, &lock).await? } else { (false, false) }; @@ -133,6 +126,7 @@ pub async fn handle_go_job( install_go_dependencies( &job.id, inner_content, + maybe_lock, mem_peak, canceled_by, job_dir, @@ -140,7 +134,6 @@ pub async fn handle_go_job( true, skip_go_mod, skip_tidy, - false, worker_name, &job.workspace_id, occupation_metrics, @@ -430,20 +423,16 @@ func Run(req Req) (interface{{}}, error){{ read_result(job_dir, handle_result.result_stream).await } -async fn gen_go_mod( - inner_content: &str, - job_dir: &str, - requirements: &str, -) -> error::Result<(bool, bool)> { +async fn gen_go_mod(inner_content: &str, job_dir: &str, lock: &str) -> error::Result<(bool, bool)> { gen_go_mymod(inner_content, job_dir).await?; - let md = requirements.split_once(GO_REQ_SPLITTER); + let md = lock.split_once(GO_REQ_SPLITTER); if let Some((req, sum)) = md { write_file(job_dir, "go.mod", &req)?; write_file(job_dir, "go.sum", &sum)?; Ok((true, true)) } else { - write_file(job_dir, "go.mod", &requirements)?; + write_file(job_dir, "go.mod", &lock)?; Ok((true, false)) } } @@ -454,92 +443,99 @@ use std::io::prelude::*; pub async fn install_go_dependencies( job_id: &Uuid, code: &str, + maybe_lock: MaybeLock, mem_peak: &mut i32, canceled_by: &mut Option, job_dir: &str, conn: &Connection, non_dep_job: bool, + // NOTE: this is impossible for skip_go_mod be `false` and maybe_lock be `Resolved`. + // TODO: make it comptime gurantee skip_go_mod: bool, has_sum: bool, - raw_deps: bool, worker_name: &str, w_id: &str, occupation_metrics: &mut OccupancyMetrics, ) -> error::Result { let anns = GoAnnotations::parse(code); - if raw_deps { - let go_mod = - if let Some(module) = code.lines().find(|l| l.trim_start().starts_with("module ")) { - code.replace(module, "module mymod") + + let hash_input = match maybe_lock { + MaybeLock::Resolved { ref lock } => lock.clone(), + MaybeLock::Unresolved { ref workspace_dependencies } => { + // NOTE: This will always be none, go workspace dependencies are disabled for now. + // read more on discord (internal): + // https://discord.com/channels/930051556043276338/1031563866641018910/1443541229349634189 + if let Some(go_mod) = workspace_dependencies.get_go()? { + if !skip_go_mod { + gen_go_mymod(code, job_dir).await?; + fs::write(format!("{job_dir}/go.mod"), &go_mod).await?; + } + go_mod } else { - format!("module mymod\n{code}") - }; - fs::write(format!("{job_dir}/go.mod"), go_mod).await?; - } - if !raw_deps && !skip_go_mod { - gen_go_mymod(code, job_dir).await?; - let mut child_cmd = Command::new(GO_PATH.as_str()); - child_cmd - .current_dir(job_dir) - .env_clear() - .args(vec!["mod", "init", "mymod"]) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); + if !skip_go_mod { + gen_go_mymod(code, job_dir).await?; + let mut child_cmd = Command::new(GO_PATH.as_str()); + child_cmd + .current_dir(job_dir) + .env_clear() + .args(vec!["mod", "init", "mymod"]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); - #[cfg(windows)] - child_cmd.env("GOPATH", windows_gopath()); - #[cfg(unix)] - child_cmd.env("GOPATH", GO_CACHE_DIR); + #[cfg(windows)] + child_cmd.env("GOPATH", windows_gopath()); + #[cfg(unix)] + child_cmd.env("GOPATH", GO_CACHE_DIR); - #[cfg(windows)] - set_windows_env_vars(&mut child_cmd); - let child_process = start_child_process(child_cmd, GO_PATH.as_str(), false).await?; + #[cfg(windows)] + set_windows_env_vars(&mut child_cmd); + let child_process = + start_child_process(child_cmd, GO_PATH.as_str(), false).await?; - handle_child( - job_id, - conn, - mem_peak, - canceled_by, - child_process, - false, - worker_name, - w_id, - "go init", - None, - false, - &mut Some(occupation_metrics), - None, - None, - ) - .await?; + handle_child( + job_id, + conn, + mem_peak, + canceled_by, + child_process, + false, + worker_name, + w_id, + "go init", + None, + false, + &mut Some(occupation_metrics), + None, + None, + ) + .await?; - for x in REQUIRE_PARSE.captures_iter(code) { - let mut file = OpenOptions::new() - .write(true) - .append(true) - .open(format!("{job_dir}/go.mod")) - .unwrap(); + for x in REQUIRE_PARSE.captures_iter(code) { + let mut file = OpenOptions::new() + .write(true) + .append(true) + .open(format!("{job_dir}/go.mod")) + .unwrap(); - writeln!(file, "require {}\n", &x[1])?; + writeln!(file, "require {}\n", &x[1])?; + } + } + if !has_sum { + calculate_hash(parse_go_imports(&code)?.iter().join("\n").as_str()) + } else { + "".to_owned() + } + } } - } - - let mut new_lockfile = false; - - let hash = if raw_deps { - calculate_hash(code) - } else if !has_sum { - calculate_hash(parse_go_imports(&code)?.iter().join("\n").as_str()) - } else { - "".to_string() }; + let hash = format!( "go{}-{}", if anns.go1_22_compat { "1.22" } else { "" }, - hash + calculate_hash(&hash_input) ); - let mut skip_tidy = has_sum; + let (mut new_lockfile, mut skip_tidy) = (false, has_sum); if !has_sum { if let Some(db) = conn.as_sql() { @@ -561,15 +557,7 @@ pub async fn install_go_dependencies( } } - let mod_command = if skip_tidy || - // If there is go.mod provided we want to use `download` only. - // Unlike `tidy` it does not modify local go.mod - raw_deps - { - "download" - } else { - "tidy" - }; + let mod_command = if skip_tidy { "download" } else { "tidy" }; let mut child_cmd = Command::new(GO_PATH.as_str()); child_cmd .current_dir(job_dir) diff --git a/backend/windmill-worker/src/lib.rs b/backend/windmill-worker/src/lib.rs index c5db591c42..6c9663d2d2 100644 --- a/backend/windmill-worker/src/lib.rs +++ b/backend/windmill-worker/src/lib.rs @@ -70,6 +70,7 @@ mod worker; mod worker_flow; mod worker_lockfiles; mod worker_utils; +pub mod workspace_dependencies; pub use worker::*; pub use worker_lockfiles::{ @@ -85,4 +86,4 @@ pub use bun_executor::{ pub use deno_executor::generate_deno_lock; #[cfg(feature = "python")] -pub use python_versions::{PyV, PyVAlias}; +pub use python_versions::PyV; diff --git a/backend/windmill-worker/src/php_executor.rs b/backend/windmill-worker/src/php_executor.rs index 4ead742f63..dd6f3b202e 100644 --- a/backend/windmill-worker/src/php_executor.rs +++ b/backend/windmill-worker/src/php_executor.rs @@ -7,7 +7,9 @@ use tokio::{fs::File, io::AsyncReadExt, process::Command}; use uuid::Uuid; use windmill_common::{ error::{self, to_anyhow, Result}, + scripts::ScriptLang, worker::{write_file, Connection}, + workspace_dependencies::clean_lock_from_annotations, }; use windmill_queue::MiniPulledJob; @@ -17,11 +19,10 @@ use windmill_queue::{append_logs, CanceledBy}; use crate::{ common::{ build_command_with_isolation, check_executor_binary_exists, create_args_and_out_file, - get_reserved_variables, read_result, start_child_process, OccupancyMetrics, + get_reserved_variables, read_result, start_child_process, MaybeLock, OccupancyMetrics, }, handle_child::handle_child, - COMPOSER_CACHE_DIR, COMPOSER_PATH, DISABLE_NSJAIL, DISABLE_NUSER, - NSJAIL_PATH, PHP_PATH, + COMPOSER_CACHE_DIR, COMPOSER_PATH, DISABLE_NSJAIL, DISABLE_NUSER, NSJAIL_PATH, PHP_PATH, }; use windmill_common::client::AuthedClient; @@ -135,9 +136,23 @@ $args->{arg_name} = new {rt_name}($args->{arg_name});" ) } +fn split_reqs_and_lock(content: &String) -> error::Result<(Option, Option)> { + let splitted = content.split(COMPOSER_LOCK_SPLIT).collect_vec(); + if splitted.len() != 2 { + return Err(error::Error::ExecutionErr(format!( + "Invalid requirements, expected to find LOCK split pattern in reqs. Found: |{content}|" + ))); + } + + Ok(( + Some(clean_lock_from_annotations(splitted[0], ScriptLang::Php)), + Some(splitted[1].to_string()), + )) +} + #[tracing::instrument(level = "trace", skip_all)] pub async fn handle_php_job( - requirements_o: Option<&String>, + maybe_lock: MaybeLock, mem_peak: &mut i32, canceled_by: &mut Option, job: &MiniPulledJob, @@ -154,16 +169,14 @@ pub async fn handle_php_job( ) -> error::Result> { check_executor_binary_exists("php", PHP_PATH.as_str(), "php")?; - let (composer_json, composer_lock) = match requirements_o { - Some(reqs_and_lock) if !reqs_and_lock.is_empty() => { - let splitted = reqs_and_lock.split(COMPOSER_LOCK_SPLIT).collect_vec(); - if splitted.len() != 2 { - return Err(error::Error::ExecutionErr( - format!("Invalid requirements, expected to find LOCK split pattern in reqs. Found: |{reqs_and_lock}|") - )); - } - (Some(splitted[0].to_string()), Some(splitted[1].to_string())) - } + let (composer_json, composer_lock) = match &maybe_lock { + MaybeLock::Resolved { lock } if !lock.is_empty() => split_reqs_and_lock(lock)?, + MaybeLock::Unresolved { workspace_dependencies } => ( + workspace_dependencies + .get_php()? + .or(parse_php_imports(inner_content)?), + None, + ), _ => (parse_php_imports(inner_content)?, None), }; diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index b417014d85..3f64641813 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -30,7 +30,8 @@ use windmill_common::{ }, utils::calculate_hash, worker::{ - copy_dir_recursively, pad_string, write_file, Connection, PythonAnnotations, WORKER_CONFIG, + copy_dir_recursively, pad_string, split_python_requirements, write_file, Connection, + PyVAlias, PythonAnnotations, WORKER_CONFIG, }, }; @@ -122,13 +123,13 @@ use windmill_common::s3_helpers::OBJECT_STORE_SETTINGS; use crate::{ common::{ - build_command_with_isolation, create_args_and_out_file, get_reserved_variables, read_file, read_result, - start_child_process, OccupancyMetrics, StreamNotifier, + build_command_with_isolation, create_args_and_out_file, get_reserved_variables, read_file, + read_result, start_child_process, OccupancyMetrics, StreamNotifier, }, handle_child::handle_child, worker_utils::ping_job_status, - PyV, PyVAlias, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV, - PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, PROXY_ENVS, PY_INSTALL_DIR, TZ_ENV, UV_CACHE_DIR, + PyV, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV, PIP_EXTRA_INDEX_URL, + PIP_INDEX_URL, PROXY_ENVS, PY_INSTALL_DIR, TZ_ENV, UV_CACHE_DIR, }; use windmill_common::client::AuthedClient; @@ -566,7 +567,7 @@ pub async fn handle_python_job( canceled_by, &mut Some(occupancy_metrics), precomputed_agent_info, - annotations, + annotations.clone(), ) .await?; @@ -838,10 +839,7 @@ mount {{ } else { let args = vec!["-u", "-m", "wrapper"]; - let mut python_cmd = build_command_with_isolation( - &python_path, - &args, - ); + let mut python_cmd = build_command_with_isolation(&python_path, &args); python_cmd .current_dir(job_dir) .env_clear() @@ -1177,36 +1175,43 @@ async fn handle_python_deps( let (pyv, resolved_lines) = match requirements_o { // Deployed Some(r) => { - let rl = split_requirements(r); + let rl = split_python_requirements(r); (PyV::parse_from_requirements(&rl), rl) } // Preview None => { let (v, requirements_lines, error_hint) = match conn { Connection::Sql(db) => { - let mut version_specifiers = vec![]; + let (mut version_specifiers, mut locked_v) = (vec![], None); let (r, h) = Box::pin(windmill_parser_py_imports::parse_python_imports( inner_content, w_id, script_path, db, &mut version_specifiers, + &mut locked_v, + &None, )) .await?; - let v = PyV::resolve( - version_specifiers, - job_id, - w_id, - annotations.py_select_latest, - Some(conn.clone()), - None, - None, - ) - .await?; + let v = if let Some(v) = locked_v { + v.into() + } else { + PyV::resolve( + version_specifiers, + job_id, + w_id, + annotations.py_select_latest, + Some(conn.clone()), + None, + None, + ) + .await? + }; (v, r, h) } + Connection::Http(_) => match precomputed_agent_info { Some(PrecomputedAgentInfo::Python { requirements, @@ -1237,7 +1242,7 @@ Returned from server: py_version - {:?}, py_version_v2 - {:?} } }; - let r = split_requirements(requirements.unwrap_or_default()); + let r = split_python_requirements(requirements.unwrap_or_default()); let h = None; (v, r, h) @@ -2113,15 +2118,6 @@ pub async fn handle_python_reqs( }; } -pub fn split_requirements>(requirements: T) -> Vec { - requirements - .as_ref() - .lines() - .filter(|x| !x.trim_start().starts_with("--") && !x.trim().is_empty()) - .map(String::from) - .collect() -} - // Returns code snippet that needs to be injected into wrapper to post-process results or leave unprocessed fn get_result_postprocessor<'a>(skip: bool) -> &'a str { if skip { @@ -2157,7 +2153,7 @@ pub async fn start_worker( killpill_rx: tokio::sync::broadcast::Receiver<()>, client: windmill_common::client::AuthedClient, ) -> error::Result<()> { - use crate::{PyV, PyVAlias}; + use crate::PyV; tracing::info!("script path: {}", script_path); let mut mem_peak: i32 = 0; @@ -2317,7 +2313,7 @@ for line in sys.stdin: proc_envs.insert("BASE_URL".to_string(), base_internal_url.to_string()); let py_version = if let Some(requirements) = requirements_o { - PyV::parse_from_requirements(&split_requirements(requirements.as_str())) + PyV::parse_from_requirements(&split_python_requirements(requirements.as_str())) } else { tracing::warn!(workspace_id = %w_id, "lockfile is empty for dedicated worker, thus python version cannot be inferred. Fallback to 3.11"); PyVAlias::Py311.into() diff --git a/backend/windmill-worker/src/python_versions.rs b/backend/windmill-worker/src/python_versions.rs index b4b39a0c46..9cf74e9e0e 100644 --- a/backend/windmill-worker/src/python_versions.rs +++ b/backend/windmill-worker/src/python_versions.rs @@ -12,8 +12,7 @@ use tokio::{fs::DirBuilder, process::Command, sync::RwLock}; use uuid::Uuid; use windmill_common::{ error::{self, Error}, - lockfiles::LOCKFILE_GENERATED_FROM_REQUIREMENTS_TXT, - worker::Connection, + worker::{try_parse_locked_python_version_from_requirements, Connection, PyVAlias}, }; use anyhow::{anyhow, bail}; @@ -26,28 +25,6 @@ use crate::{ HOME_ENV, INSTANCE_PYTHON_VERSION, PATH_ENV, PROXY_ENVS, PY_INSTALL_DIR, WIN_ENVS, }; -#[derive(Eq, PartialEq, Clone, Copy, Default, Debug)] -#[repr(u32)] -pub enum PyVAlias { - Py310 = 10, - #[default] - Py311, - Py312, - Py313, -} - -impl Into for PyVAlias { - fn into(self) -> pep440_rs::Version { - pep440_rs::Version::new([self.major() as u64, self as u64]) - } -} - -impl Into for PyVAlias { - fn into(self) -> u32 { - self.major() * 100 + self as u32 - } -} - impl From for PyVAlias { fn from(value: PyV) -> Self { match value.release() { @@ -66,34 +43,6 @@ impl From for PyVAlias { Self::default() } } -impl PyVAlias { - fn all>() -> Vec { - use PyVAlias::*; - vec![Py310.into(), Py311.into(), Py312.into(), Py313.into()] - } - // Get MAJOR part of alias. (semver: MAJOR.MINOR.PATCH) - fn major(&self) -> u32 { - use PyVAlias::*; - match self { - Py310 | Py311 | Py312 | Py313 => 3, - // Py400 | Py401 => 4 - } - } - - /// Converts numeric format to alias - /// Example: - /// 310u32 (in) -> PyVAlias::Py310 (out) - pub(crate) fn try_from_v1(numeric: T) -> Option { - use PyVAlias::*; - match numeric.to_string().as_str() { - "310" => Some(Py310), - "311" => Some(Py311), - "312" => Some(Py312), - "313" => Some(Py313), - _ => None, - } - } -} // To change latest stable version: // 1. Change placeholder in instanceSettings.ts @@ -431,51 +380,7 @@ impl PyV { /// Parse lockfile for assigned python version. /// If not found returns None pub fn try_parse_from_requirements>(requirements_lines: &[S]) -> Option { - let parse_version = |s: &str| -> Option { - // Possible inputs: - // V2: - // # py: 3.11.0 or #py:3.11.0 or #py: 3.11.0 - // - // V1: - // # py311 or #py311 - let version_unparsed = s - .to_owned() - // Remove whitespaces. That leaves us with: - // V2: #py:3.11.0 - // V1: #py311 - // - // Remove # - // V2: py:3.11.0 - // V1: py311 - // - // Remove : - // V2: py3.11.0 - // V1: py311 - .replace([' ', '#', ':'], "") - // Remove "py" - // V2: 3.11.0 - // V1: 311 - .replace("py", ""); - - // We will support reading V1 syntax, but it will be overwritten next deploy - PyVAlias::try_from_v1(&version_unparsed) - .map(PyVAlias::into) - .or(pep440_rs::Version::from_str(&version_unparsed) - .ok() - .map(pep440_rs::Version::into)) - }; - let index = if requirements_lines.get(0).map_or(false, |line| { - line.as_ref() - .starts_with(LOCKFILE_GENERATED_FROM_REQUIREMENTS_TXT) - }) { - 1 - } else { - 0 - }; - requirements_lines - .get(index) - .map(S::as_ref) - .and_then(parse_version) + try_parse_locked_python_version_from_requirements(requirements_lines).map(PyV::from) } pub async fn get_python( diff --git a/backend/windmill-worker/src/scoped_dependency_map.rs b/backend/windmill-worker/src/scoped_dependency_map.rs index 072fedc8a6..9e9884e538 100644 --- a/backend/windmill-worker/src/scoped_dependency_map.rs +++ b/backend/windmill-worker/src/scoped_dependency_map.rs @@ -1,22 +1,24 @@ use serde::Serialize; +use sqlx::PgExecutor; use tokio::sync::RwLock; use windmill_common::{ apps::traverse_app_inline_scripts, cache, error::{Error, Result}, flows::{FlowModuleValue, FlowValue}, + scripts::ScriptLang, }; use std::collections::HashSet; -use crate::worker_lockfiles::extract_relative_imports; -use windmill_common::lockfiles::is_generated_from_raw_requirements; +use crate::worker_lockfiles::extract_referenced_paths; // TODO: To be removed in future versions lazy_static::lazy_static! { pub static ref WMDEBUG_NO_DMAP_DISSOLVE: bool = std::env::var("WMDEBUG_NO_DMAP_DISSOLVE").is_ok(); } +// TODO: Rename to DependencyRelation #[derive(Serialize)] pub struct DependencyMap { pub workspace_id: Option, @@ -26,9 +28,17 @@ pub struct DependencyMap { pub importer_node_id: Option, } +#[derive(Debug, Clone, Serialize, sqlx::FromRow)] +pub struct DependencyDependent { + pub importer_path: String, + pub importer_kind: String, + pub importer_node_ids: Option>, +} + #[derive(Debug)] pub struct ScopedDependencyMap { - dmap: HashSet<(String, String)>, + /// (importer_node_id, imported_path) + to_delete: HashSet<(String, String)>, w_id: String, importer_path: String, importer_kind: String, @@ -72,7 +82,7 @@ RETURNING importer_node_id, imported_path .fetch_all(executor) .await?; Ok(Self { - dmap: HashSet::from_iter(dmap.into_iter()), + to_delete: HashSet::from_iter(dmap.into_iter()), w_id: w_id.to_owned(), importer_path: importer_path.to_owned(), importer_kind: importer_kind.to_owned(), @@ -104,7 +114,7 @@ SELECT importer_node_id, imported_path .await?; Ok(Self { - dmap: HashSet::from_iter(dmap.into_iter()), + to_delete: HashSet::from_iter(dmap.into_iter()), w_id: w_id.to_owned(), importer_path: importer_path.to_owned(), importer_kind: importer_kind.to_owned(), @@ -115,22 +125,23 @@ SELECT importer_node_id, imported_path /// Remove matching entries pub(crate) async fn patch<'c>( &mut self, - relative_imports: Option>, + referenced_paths: Option>, node_id: String, // Flow Step/Node ID mut tx: sqlx::Transaction<'c, sqlx::Postgres>, ) -> Result> { - self.patch_tx_ref(relative_imports, &node_id, &mut tx) + self.patch_tx_ref(referenced_paths, &node_id, &mut tx) .await?; Ok(tx) } pub(crate) async fn patch_tx_ref<'c>( &mut self, - relative_imports: Option>, + // NOTE: Referenced_paths should include all of the paths. + referenced_paths: Option>, node_id: &str, // Flow Step/Node ID tx: &mut sqlx::Transaction<'c, sqlx::Postgres>, ) -> Result<()> { - let Some(mut relative_imports) = relative_imports else { + let Some(mut referenced_paths) = referenced_paths else { tracing::info!("relative imports are not found for: importer - {}, importer_node_id - {}, importer_kind - {}", &self.importer_path, &node_id, @@ -149,9 +160,9 @@ SELECT importer_node_id, imported_path // After all `reduce`'s called ScopedDependencyMap has only extra/orphan imports // these are going to be clean up by calling [dissolve] // NOTE: `retain` iterates over vec and remove the ones whose closures returned false. - relative_imports.retain(|imported_path| { + referenced_paths.retain(|imported_path| { !self - .dmap + .to_delete // As dmap is HashSet, removing is O(1) operation // thus making entire process very efficient // NOTE: `remove` returns true if item was removed and false if wasn't. @@ -159,15 +170,15 @@ SELECT importer_node_id, imported_path }); // As mentioned above, usually this will always be empty. - if !relative_imports.is_empty() { + if !referenced_paths.is_empty() { tracing::info!("adding missing entries to dependency_map: importer_node_id - {}, importer_kind - {}, new_imported_paths - {:?}", &node_id, &self.importer_kind, - &relative_imports, + &referenced_paths, ); } - for import in relative_imports { + for import in referenced_paths { sqlx::query!( "INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path, importer_node_id) VALUES ($1, $2, $3::text::IMPORTER_KIND, $4, $5) ON CONFLICT DO NOTHING", @@ -200,7 +211,7 @@ SELECT importer_node_id, imported_path tracing::info!("dissolving dependency_map: {:?}", &self); // We _could_ shove it into single query, but this query is rarely called AND let's keep it simple for redability. - for (importer_node_id, imported_path) in self.dmap.into_iter() { + for (importer_node_id, imported_path) in self.to_delete.into_iter() { tracing::info!("cleaning orphan entry from dependency_map: importer_kind - {}, imported_path - {}, importer_node_id - {}", &self.importer_kind, &imported_path, @@ -275,14 +286,14 @@ SELECT importer_node_id, imported_path tx } - /// Run if you want to rebuild maps on specific workspace. - /// Potentially takes much time - pub async fn rebuild_map(w_id: &str, db: &sqlx::Pool) -> Result { - async fn inner<'c>(w_id: &str, db: &sqlx::Pool) -> Result { - // Scripts - tracing::info!(workspace_id = w_id, "Rebuilding dependency map for scripts"); - for r in sqlx::query!( - "SELECT path, hash FROM script WHERE workspace_id = $1 AND archived = false AND deleted = false", + pub(crate) async fn rebuild_map_unchecked<'c>( + w_id: &str, + db: &sqlx::Pool, + ) -> Result { + // Scripts + tracing::info!(workspace_id = w_id, "Rebuilding dependency map for scripts"); + for r in sqlx::query!( + r#"SELECT path, hash, language AS "language: ScriptLang" FROM script WHERE workspace_id = $1 AND archived = false AND deleted = false"#, w_id ) .fetch_all(db) @@ -292,29 +303,24 @@ SELECT importer_node_id, imported_path let mut dmap = ScopedDependencyMap::fetch(w_id, &r.path, "script", db).await?; let mut tx = db.begin().await?; - if is_generated_from_raw_requirements(&smd.language, &sd.lock) { - // if the lock file is generated from a package.json/requirements.txt, we need to clear the dependency map - // because we do not want to have dependencies be recomputed automatically. Empty relative imports passed - // to update_script_dependency_map will clear the dependency map. - } else { - tx = dmap - .patch( - extract_relative_imports(&sd.code, &r.path, &smd.language), - "".into(), - tx, - ) - .await?; - } + tx = dmap + .patch( + extract_referenced_paths(&sd.code, &r.path, smd.language), + "".into(), + tx, + ) + .await?; + if !*WMDEBUG_NO_DMAP_DISSOLVE { dmap.dissolve(tx).await.commit().await?; } tracing::info!(workspace_id = w_id, "Rebuilt for script {}", &r.path); } - // Fetch only top level versions and paths - // It is not fetching value - tracing::info!(workspace_id = w_id, "Rebuilding dependency map for flows"); - for r in sqlx::query!("SELECT path, versions[array_upper(versions, 1)] as version FROM flow WHERE workspace_id = $1 AND archived = false", w_id).fetch_all(db).await? { + // Fetch only top level versions and paths + // It is not fetching value + tracing::info!(workspace_id = w_id, "Rebuilding dependency map for flows"); + for r in sqlx::query!("SELECT path, versions[array_upper(versions, 1)] as version FROM flow WHERE workspace_id = $1 AND archived = false", w_id).fetch_all(db).await? { if let Some(version) = r.version { // To reduce stress on db try to fetch from cache // Since our flow versions are immutable it is safe to assume if we have cache for specific version/id it is up to date. @@ -337,17 +343,15 @@ SELECT importer_node_id, imported_path FlowValue::traverse_leafs(modules_to_check, &mut |fmv, id| { match fmv { // Since we fetched from flow_version it is safe to assume all inline scripts are in form of RawScript. - FlowModuleValue::RawScript { content, language, lock ,.. } => { - if !is_generated_from_raw_requirements(&Some(*language), lock) { - to_process.push(( - extract_relative_imports( - content, - &(r.path.clone() + "/flow"), - &Some(language.clone()), - ), - id.clone(), - )); - } + FlowModuleValue::RawScript { content, language, .. } => { + to_process.push(( + extract_referenced_paths( + content, + &(r.path.clone() + "/flow"), + Some(*language), + ), + id.clone(), + )); } // But just in case we will also handle other cases. FlowModuleValue::FlowScript { .. } => { @@ -359,8 +363,8 @@ SELECT importer_node_id, imported_path Ok(()) })?; - for (ri, id) in to_process { - tx = dmap.patch(ri, id, tx).await?; + for (rp, id) in to_process { + tx = dmap.patch(rp, id, tx).await?; } if !*WMDEBUG_NO_DMAP_DISSOLVE { @@ -374,9 +378,9 @@ SELECT importer_node_id, imported_path } } - // Apps - tracing::info!(workspace_id = w_id, "Rebuilding dependency map for apps"); - for r in sqlx::query!("SELECT path, versions[array_upper(versions, 1)] as version FROM app WHERE workspace_id = $1", w_id).fetch_all(db).await? { + // Apps + tracing::info!(workspace_id = w_id, "Rebuilding dependency map for apps"); + for r in sqlx::query!("SELECT path, versions[array_upper(versions, 1)] as version FROM app WHERE workspace_id = $1", w_id).fetch_all(db).await? { if let Some(version) = r.version { // TODO: Use cache when implemented. let value = sqlx::query_scalar!( @@ -391,10 +395,10 @@ SELECT importer_node_id, imported_path let mut to_process = vec![]; traverse_app_inline_scripts(&value, None, &mut |ais, id| { to_process.push(( - extract_relative_imports( + extract_referenced_paths( &ais.content, &(r.path.clone() + "/app"), - &ais.language, + ais.language, ), id, )); @@ -417,9 +421,11 @@ SELECT importer_node_id, imported_path } } - Ok("Success".into()) - } - + Ok("Success".into()) + } + /// Run if you want to rebuild maps on specific workspace. + /// Potentially takes much time + pub async fn rebuild_map(w_id: &str, db: &sqlx::Pool) -> Result { lazy_static::lazy_static! { pub static ref LOCKED: RwLock = RwLock::new(false); } @@ -438,9 +444,34 @@ SELECT importer_node_id, imported_path } *LOCKED.write().await = true; - let r = inner(w_id, db).await; + let r = Self::rebuild_map_unchecked(w_id, db).await; *LOCKED.write().await = false; r } } + + /// Get dependents of any imported path - returns scripts/flows/apps that depend on it + pub async fn get_dependents<'c>( + imported_path: &str, + workspace_id: &str, + e: impl PgExecutor<'c>, + ) -> Result> { + sqlx::query_as!( + DependencyDependent, + r#" + SELECT + importer_path, + importer_kind::text as "importer_kind!", -- sqlx thinks this is nullable somehow, so enfore with ! + array_agg(importer_node_id) as importer_node_ids + FROM dependency_map + WHERE workspace_id = $1 AND imported_path = $2 + GROUP BY importer_path, importer_kind + "#, + workspace_id, + imported_path + ) + .fetch_all(e) + .await + .map_err(Error::from) + } } diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 2919243dbc..9dd8e89219 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -20,6 +20,8 @@ use windmill_common::scripts::is_special_codebase_hash; use windmill_common::utils::report_critical_error; use windmill_common::utils::retrieve_common_worker_prefix; use windmill_common::worker::error_to_value; +use windmill_common::workspace_dependencies::RawWorkspaceDependencies; +use windmill_common::workspace_dependencies::WorkspaceDependenciesPrefetched; use windmill_common::{ agent_workers::DECODED_AGENT_TOKEN, apps::AppScriptId, @@ -109,6 +111,7 @@ use tokio::{ use rand::Rng; use crate::ai_executor::handle_ai_agent_job; +use crate::common::MaybeLock; use crate::common::StreamNotifier; use crate::{ agent_workers::{queue_init_job, queue_periodic_job}, @@ -153,10 +156,9 @@ use crate::ruby_executor::{handle_ruby_job, JobHandlerInput as JobHandlerInputRu use crate::php_executor::handle_php_job; #[cfg(feature = "python")] -use crate::{ - python_executor::handle_python_job, - python_versions::{PyV, PyVAlias}, -}; +use crate::{python_executor::handle_python_job, python_versions::PyV}; +#[cfg(feature = "python")] +use windmill_common::worker::PyVAlias; #[cfg(feature = "python")] use crate::ansible_executor::handle_ansible_job; @@ -2857,6 +2859,16 @@ pub async fn handle_queued_job( let mut column_order: Option> = None; let mut new_args: Option>> = None; let mut has_stream = false; + + let raw_workspace_dependencies_o = if job.kind.is_dependency() { + job.args + .as_ref() + .and_then(|x| x.get("raw_workspace_dependencies")) + .map(|v| v.get()) + .and_then(|v| serde_json::from_str::(v).ok()) + } else { + None + }; // Box::pin all async branches to prevent large match enum on stack let result = match job.kind { JobKind::Dependencies => match conn { @@ -2873,6 +2885,7 @@ pub async fn handle_queued_job( base_internal_url, &client.token, occupancy_metrics, + raw_workspace_dependencies_o, )) .await } @@ -2896,6 +2909,7 @@ pub async fn handle_queued_job( base_internal_url, &client.token, occupancy_metrics, + raw_workspace_dependencies_o, )) .await } @@ -2917,6 +2931,7 @@ pub async fn handle_queued_job( base_internal_url, &client.token, occupancy_metrics, + raw_workspace_dependencies_o, )) .await .map(|()| serde_json::from_str("{}").unwrap()), @@ -3614,14 +3629,32 @@ mount {{ let envs = build_envs(envs.as_ref())?; + let Some(language) = language else { + return Err(Error::ExecutionErr( + "Require language to be not null".to_string(), + ))?; + }; + + let maybe_lock = if let Some(lock) = lock.clone() { + MaybeLock::Resolved { lock } + } else { + MaybeLock::Unresolved { + workspace_dependencies: WorkspaceDependenciesPrefetched::extract( + code, + language, + &job.workspace_id, + // TODO: implement + &None, + job.runnable_path(), + conn.clone(), + ) + .await?, + } + }; + // Box::pin all language handlers to prevent large match enum on stack let result: error::Result> = match language { - None => { - return Err(Error::ExecutionErr( - "Require language to be not null".to_string(), - ))?; - } - Some(ScriptLang::Python3) => { + ScriptLang::Python3 => { #[cfg(not(feature = "python"))] return Err(Error::internal_err( "Python requires the python feature to be enabled".to_string(), @@ -3650,7 +3683,7 @@ mount {{ )) .await } - Some(ScriptLang::Deno) => { + ScriptLang::Deno => { Box::pin(handle_deno_job( lock.as_ref(), mem_peak, @@ -3670,9 +3703,9 @@ mount {{ )) .await } - Some(ScriptLang::Bun) | Some(ScriptLang::Bunnative) => { + ScriptLang::Bun | ScriptLang::Bunnative => { Box::pin(handle_bun_job( - lock.as_ref(), + maybe_lock, codebase.as_ref(), mem_peak, canceled_by, @@ -3693,7 +3726,7 @@ mount {{ )) .await } - Some(ScriptLang::Go) => { + ScriptLang::Go => { Box::pin(handle_go_job( mem_peak, canceled_by, @@ -3703,16 +3736,16 @@ mount {{ parent_runnable_path, &code, job_dir, - lock.as_ref(), &shared_mount, base_internal_url, worker_name, envs, occupancy_metrics, + maybe_lock, )) .await } - Some(ScriptLang::Bash) => { + ScriptLang::Bash => { Box::pin(handle_bash_job( mem_peak, canceled_by, @@ -3731,7 +3764,7 @@ mount {{ )) .await } - Some(ScriptLang::Powershell) => { + ScriptLang::Powershell => { Box::pin(handle_powershell_job( mem_peak, canceled_by, @@ -3749,7 +3782,7 @@ mount {{ )) .await } - Some(ScriptLang::Php) => { + ScriptLang::Php => { #[cfg(not(feature = "php"))] return Err(Error::internal_err( "PHP requires the php feature to be enabled".to_string(), @@ -3757,7 +3790,7 @@ mount {{ #[cfg(feature = "php")] Box::pin(handle_php_job( - lock.as_ref(), + maybe_lock, mem_peak, canceled_by, job, @@ -3774,7 +3807,7 @@ mount {{ )) .await } - Some(ScriptLang::Rust) => { + ScriptLang::Rust => { #[cfg(not(feature = "rust"))] return Err(Error::internal_err( "Rust requires the rust feature to be enabled".to_string(), @@ -3799,7 +3832,7 @@ mount {{ )) .await } - Some(ScriptLang::Ansible) => { + ScriptLang::Ansible => { #[cfg(not(feature = "python"))] return Err(Error::internal_err( "Ansible requires the python feature to be enabled".to_string(), @@ -3825,7 +3858,7 @@ mount {{ )) .await } - Some(ScriptLang::CSharp) => { + ScriptLang::CSharp => { Box::pin(handle_csharp_job( mem_peak, canceled_by, @@ -3844,7 +3877,7 @@ mount {{ )) .await } - Some(ScriptLang::Nu) => { + ScriptLang::Nu => { #[cfg(not(feature = "nu"))] return Err( anyhow::anyhow!("Nu is not available because the feature is not enabled").into(), @@ -3869,7 +3902,7 @@ mount {{ })) .await } - Some(ScriptLang::Java) => { + ScriptLang::Java => { #[cfg(not(feature = "java"))] return Err(anyhow::anyhow!( "Java is not available because the feature is not enabled" @@ -3895,7 +3928,7 @@ mount {{ })) .await } - Some(ScriptLang::Ruby) => { + ScriptLang::Ruby => { #[cfg(not(feature = "ruby"))] return Err(anyhow::anyhow!( "Ruby is not available because the feature is not enabled" diff --git a/backend/windmill-worker/src/worker_lockfiles.rs b/backend/windmill-worker/src/worker_lockfiles.rs index 4ec0fa648e..b3b6333f05 100644 --- a/backend/windmill-worker/src/worker_lockfiles.rs +++ b/backend/windmill-worker/src/worker_lockfiles.rs @@ -5,7 +5,7 @@ use std::path::{Component, Path, PathBuf}; #[cfg(feature = "python")] use crate::ansible_executor::{get_git_repos_lock, AnsibleDependencyLocks}; -use crate::scoped_dependency_map::ScopedDependencyMap; +use crate::scoped_dependency_map::{DependencyDependent, ScopedDependencyMap}; use async_recursion::async_recursion; use chrono::{Duration, Utc}; use itertools::Itertools; @@ -21,12 +21,14 @@ use windmill_common::error::Error; use windmill_common::error::Result; use windmill_common::flows::{FlowModule, FlowModuleValue, FlowNodeId}; use windmill_common::jobs::JobPayload; -use windmill_common::lockfiles::is_generated_from_raw_requirements; use windmill_common::scripts::ScriptHash; use windmill_common::utils::WarnAfterExt; #[cfg(feature = "python")] use windmill_common::worker::PythonAnnotations; use windmill_common::worker::{to_raw_value, to_raw_value_owned, write_file, Connection}; +use windmill_common::workspace_dependencies::{ + RawWorkspaceDependencies, WorkspaceDependencies, WorkspaceDependenciesPrefetched, +}; #[cfg(feature = "python")] use windmill_parser_yaml::AnsibleRequirements; @@ -55,7 +57,7 @@ lazy_static::lazy_static! { ); } -use crate::common::OccupancyMetrics; +use crate::common::{MaybeLock, OccupancyMetrics}; use crate::csharp_executor::generate_nuget_lockfile; #[cfg(feature = "java")] @@ -67,9 +69,7 @@ use crate::ruby_executor; #[cfg(feature = "php")] use crate::php_executor::{composer_install, parse_php_imports}; #[cfg(feature = "python")] -use crate::python_executor::{ - create_dependencies_dir, handle_python_reqs, split_requirements, uv_pip_compile, -}; +use crate::python_executor::{create_dependencies_dir, handle_python_reqs, uv_pip_compile}; #[cfg(feature = "rust")] use crate::rust_executor::generate_cargo_lockfile; use crate::{ @@ -137,6 +137,43 @@ pub fn extract_relative_imports( } } +pub fn extract_referenced_paths( + raw_code: &str, + script_path: &str, + language: Option, +) -> Option> { + let mut referenced_paths = vec![]; + if let Some(wk_deps_refs) = language + .and_then(|l| l.extract_workspace_dependencies_annotated_refs(raw_code, script_path)) + .map(|r| r.external) + { + let l = language.expect("should be some"); + for wk_deps_ref in wk_deps_refs { + if let Some(path) = WorkspaceDependencies::to_path(&Some(wk_deps_ref), l).ok() { + referenced_paths.push(path); + }; + } + } else if let (Some(l), true /* Only if it is not blacklisted */) = ( + language, + WorkspaceDependenciesPrefetched::is_external_references_permitted(script_path), + ) { + // we assume all runnables without annotated dependencies reference default dependencies file. + WorkspaceDependencies::to_path(&None, l) + .ok() + .inspect(|p| referenced_paths.push(p.to_owned())); + } + + if let Some(relative_imports) = extract_relative_imports(raw_code, script_path, &language) { + referenced_paths.extend(relative_imports); + } + + if referenced_paths.is_empty() { + None + } else { + Some(referenced_paths) + } +} + #[tracing::instrument(level = "trace", skip_all)] pub async fn handle_dependency_job( job: &MiniPulledJob, @@ -150,6 +187,7 @@ pub async fn handle_dependency_job( base_internal_url: &str, token: &str, occupancy_metrics: &mut OccupancyMetrics, + raw_workspace_dependencies_o: Option, ) -> error::Result> { // Processing a dependency job - these jobs handle lockfile generation and dependency updates // for scripts, flows, and apps when their dependencies or imported scripts change @@ -158,33 +196,6 @@ pub async fn handle_dependency_job( job.runnable_path() ); let script_path = job.runnable_path(); - let raw_deps = job - .args - .as_ref() - .map(|x| { - x.get("raw_deps") - .is_some_and(|y| y.to_string().as_str() == "true") - }) - .unwrap_or(false); - - let npm_mode = if job - .script_lang - .as_ref() - .map(|v| v == &ScriptLang::Bun) - .unwrap_or(false) - { - Some( - job.args - .as_ref() - .map(|x| { - x.get("npm_mode") - .is_some_and(|y| y.to_string().as_str() == "true") - }) - .unwrap_or(false), - ) - } else { - None - }; // `JobKind::Dependencies` job store either: // - A saved script `hash` in the `script_hash` column. @@ -239,9 +250,8 @@ pub async fn handle_dependency_job( base_internal_url, token, script_path, - raw_deps, - npm_mode, occupancy_metrics, + &raw_workspace_dependencies_o, ) .await; @@ -307,7 +317,6 @@ pub async fn handle_dependency_job( &job.permissioned_as_email, &job.created_by, &job.permissioned_as, - None, ) .await?; @@ -360,45 +369,33 @@ pub async fn process_relative_imports( permissioned_as_email: &str, created_by: &str, permissioned_as: &str, - lock: Option, ) -> error::Result<()> { // TODO: Should be moved into handle_dependency_job body to be more consistent with how flows and apps are handled { - let relative_imports = extract_relative_imports(&code, script_path, script_lang); - if let Some(relative_imports) = relative_imports { - let mut tx = db.begin().await?; - let mut dependency_map = ScopedDependencyMap::fetch_maybe_rearranged( - &w_id, - script_path, - "script", - &parent_path, - db, + let mut tx = db.begin().await?; + let mut dependency_map = ScopedDependencyMap::fetch_maybe_rearranged( + &w_id, + script_path, + "script", + &parent_path, + db, + ) + .await?; + + tx = dependency_map + .patch( + extract_referenced_paths(&code, script_path, *script_lang), + // Ideally should be None, but due to current implementation will use empty string to represent None. + "".into(), + tx, ) .await?; - if is_generated_from_raw_requirements(script_lang, &lock) { - // if the lock file is generated from a package.json/requirements.txt, we need to clear the dependency map - // because we do not want to have dependencies be recomputed automatically. Empty relative imports passed - // to update_script_dependency_map will clear the dependency map. - // TODO: Rework the logic for synchronized raw requirements PR. - // For now we will just do nothing and let dissolve clear every item related to this script. - } else { - tx = dependency_map - .patch( - Some(relative_imports), - // Ideally should be None, but due to current implementation will use empty string to represent None. - "".into(), - tx, - ) - .await?; - } - // If felt into first branch which did not call .patch(, this operation will clean dependency_map for this script. - dependency_map.dissolve(tx).await.commit().await?; - } + dependency_map.dissolve(tx).await.commit().await?; } { - let already_visited = args + let mut already_visited = args .map(|x| { x.get("already_visited") .map(|v| serde_json::from_str::>(v.get()).ok()) @@ -407,13 +404,30 @@ pub async fn process_relative_imports( .flatten() .unwrap_or_default(); + // TODO: There is a race-condition. + // This can be old version. + + // Check lines of code below, you will find that we get the latest version of the script/app/flow + + // However the latest version does not necessarily mean that it is finalized. + // Instead we assume that this would be the version we would base on. + + // So the script_importers might be behind. Thus some information like nodes_to_relock might be lost. + let importers = crate::scoped_dependency_map::ScopedDependencyMap::get_dependents( + script_path, + w_id, + db, + ) + .await?; + + already_visited.push(script_path.to_string()); // But currently we will do this extra db call for every script regardless of whether they have relative imports or not // Script might have no relative imports but still be referenced by someone else. match timeout( core::time::Duration::from_secs(60), Box::pin(trigger_dependents_to_recompute_dependencies( w_id, - script_path, + importers, deployment_message, parent_path, permissioned_as_email, @@ -441,44 +455,24 @@ pub async fn process_relative_imports( pub async fn trigger_dependents_to_recompute_dependencies( w_id: &str, - script_path: &str, + importers: Vec, + // imported_path: &str, deployment_message: Option, parent_path: Option, email: &str, created_by: &str, permissioned_as: &str, db: &sqlx::Pool, - mut already_visited: Vec, + already_visited: Vec, ) -> error::Result<()> { - // TODO: There is a race-condition. - // This can be old version. - // - // Check lines of code below, you will find that we get the latest version of the script/app/flow - // - // However the latest version does not necessarily mean that it is finalized. - // Instead we assume that this would be the version we would base on. - // - // So the script_importers might be behind. Thus some information like nodes_to_relock might be lost. - let script_importers = sqlx::query!( - "SELECT importer_path, importer_kind::text, array_agg(importer_node_id) as importer_node_ids FROM dependency_map - WHERE imported_path = $1 - AND workspace_id = $2 - GROUP BY importer_path, importer_kind", - script_path, - w_id - ) - .fetch_all(db) - .await?; - tracing::debug!( - "Triggering dependents to recompute dependencies for: {}", - &script_path + "Triggering dependents to recompute dependencies: {}", + importers.iter().map(|dd| &dd.importer_path).join(",") ); - - already_visited.push(script_path.to_string()); - for s in script_importers.iter() { - tracing::trace!("Processing dependency: {:?}", &s); - if already_visited.contains(&s.importer_path) { + for DependencyDependent { importer_path, importer_kind, importer_node_ids } in importers.iter() + { + tracing::trace!("Processing dependency: {:?}", importer_path); + if already_visited.contains(importer_path) { tracing::trace!("Skipping already visited dependency"); continue; } @@ -517,33 +511,33 @@ pub async fn trigger_dependents_to_recompute_dependencies( // After our transaction commits, any pending push/pull requests can proceed with // their debounce logic. let debounce_job_id_o = - windmill_common::jobs::lock_debounce_key(w_id, &s.importer_path, &mut tx).await?; + windmill_common::jobs::lock_debounce_key(w_id, &importer_path, &mut tx).await?; tracing::debug!( debounce_job_id = ?debounce_job_id_o, - importer_path = %s.importer_path, + importer_path = %importer_path, "Retrieved debounce job ID (if exists)" ); - let kind = s.importer_kind.clone().unwrap_or_default(); - let job_payload = if kind == "script" { + let job_payload = match importer_kind.as_str() { // TODO: Make it query only non-archived - match sqlx::query_scalar!( + // Scripts + "script" => match sqlx::query_scalar!( "SELECT hash FROM script WHERE path = $1 AND workspace_id = $2 AND deleted = false ORDER BY created_at DESC LIMIT 1", - s.importer_path.clone(), + importer_path, w_id ) .fetch_optional(&mut *tx) .await? { Some(hash) => { - tracing::debug!("newest hash for {} is: {hash}", &s.importer_path); + tracing::debug!("newest hash for {} is: {hash}", importer_path); let info = windmill_common::get_script_info_for_hash(None, db, w_id, hash).await?; JobPayload::Dependencies { - path: s.importer_path.clone(), + path: importer_path.clone(), hash: ScriptHash(hash), language: info.language, dedicated_worker: info.dedicated_worker, @@ -551,7 +545,7 @@ pub async fn trigger_dependents_to_recompute_dependencies( } None => { ScopedDependencyMap::clear_map_for_item( - &s.importer_path, + importer_path, w_id, "script", tx, @@ -562,86 +556,80 @@ pub async fn trigger_dependents_to_recompute_dependencies( .await?; continue; } - } - } else if kind == "flow" { - tracing::debug!("Handling flow dependency update for: {}", s.importer_path); + }, - args.insert( - "nodes_to_relock".to_string(), - to_raw_value(&s.importer_node_ids), - ); - - match sqlx::query_scalar!( + // Flows + "flow" => match sqlx::query_scalar!( "SELECT id FROM flow_version WHERE path = $1 AND workspace_id = $2 ORDER BY created_at DESC LIMIT 1", - s.importer_path.clone(), - w_id - ) - .fetch_optional(&mut *tx) - .await? - { - Some(version) => JobPayload::FlowDependencies { - path: s.importer_path.clone(), - version, - dedicated_worker: None, - }, - None => { - ScopedDependencyMap::clear_map_for_item( - &s.importer_path, - w_id, - "flow", - tx, - &None, - ) - .await - .commit() - .await?; - continue; - } - } - } else if kind == "app" && !*WMDEBUG_NO_NEW_APP_VERSION_ON_DJ { - tracing::debug!("Handling flow dependency update for: {}", s.importer_path); - - args.insert( - "components_to_relock".to_string(), - // TODO: unsafe. Importer Node Ids are not checked. They can simply be array of empty strings! - to_raw_value(&s.importer_node_ids), - ); - - match sqlx::query_scalar!( - "SELECT id FROM app_version WHERE app_id = (SELECT id FROM app WHERE path = $1 AND workspace_id = $2) ORDER BY created_at DESC LIMIT 1", - s.importer_path.clone(), + importer_path, w_id ) .fetch_optional(&mut *tx) .await? { Some(version) => { - JobPayload::AppDependencies { path: s.importer_path.clone(), version } + tracing::debug!("Handling flow dependency update for: {}", importer_path); + + args.insert( + "nodes_to_relock".to_string(), + to_raw_value(&importer_node_ids), + ); + + JobPayload::FlowDependencies { + path: importer_path.clone(), + version, + dedicated_worker: None, + } } None => { - ScopedDependencyMap::clear_map_for_item( - &s.importer_path, - w_id, - "app", - tx, - &None, - ) - .await - .commit() - .await?; + ScopedDependencyMap::clear_map_for_item(importer_path, w_id, "flow", tx, &None) + .await + .commit() + .await?; continue; } + }, + + // Apps + "app" => match sqlx::query_scalar!( + "SELECT id FROM app_version WHERE app_id = (SELECT id FROM app WHERE path = $1 AND workspace_id = $2) ORDER BY created_at DESC LIMIT 1", + importer_path, + w_id + ) + .fetch_optional(&mut *tx) + .await? + { + Some(version) => { + tracing::debug!("Handling app dependency update for: {}", importer_path); + + args.insert( + "components_to_relock".to_string(), + // TODO: unsafe. Importer Node Ids are not checked. They can simply be array of empty strings! + to_raw_value(importer_node_ids), + ); + + JobPayload::AppDependencies { path: importer_path.clone(), version } + } + None => { + ScopedDependencyMap::clear_map_for_item(importer_path, w_id, "app", tx, &None) + .await + .commit() + .await?; + continue; + } + }, + + _ => { + tracing::error!( + "unexpected importer kind: {kind:?} for path {path}", + kind = importer_kind, + path = importer_path + ); + continue; } - } else { - tracing::error!( - "unexpected importer kind: {kind} for path {path}", - kind = kind, - path = s.importer_path - ); - continue; }; - tracing::debug!("Pushing dependency job for: {}", s.importer_path); + tracing::debug!("Pushing dependency job for: {}", importer_path); let (job_uuid, new_tx) = windmill_queue::push( db, PushIsolationLevel::Transaction(tx), @@ -677,7 +665,7 @@ pub async fn trigger_dependents_to_recompute_dependencies( tracing::info!( "pushed dependency job due to common python path: {job_uuid} for path {path}", - path = s.importer_path, + path = importer_path, ); new_tx.commit().await?; } @@ -696,6 +684,7 @@ pub async fn handle_flow_dependency_job( base_internal_url: &str, token: &str, occupancy_metrics: &mut OccupancyMetrics, + raw_workspace_dependencies_o: Option, ) -> error::Result> { tracing::debug!("Processing flow dependency job"); tracing::trace!("Job details: {:?}", &job); @@ -820,6 +809,7 @@ pub async fn handle_flow_dependency_job( skip_flow_update, &raw_deps, &mut dependency_map, + &raw_workspace_dependencies_o, ) .await?; @@ -1031,6 +1021,7 @@ async fn lock_flow_value<'c>( skip_flow_update: bool, raw_deps: &Option>, dependency_map: &mut ScopedDependencyMap, + raw_workspace_dependencies_o: &Option, ) -> Result<( FlowValue, sqlx::Transaction<'c, sqlx::Postgres>, @@ -1059,6 +1050,7 @@ async fn lock_flow_value<'c>( skip_flow_update, &raw_deps, dependency_map, + &raw_workspace_dependencies_o, ) .await?; @@ -1088,6 +1080,7 @@ async fn lock_flow_value<'c>( skip_flow_update, &raw_deps, dependency_map, + &raw_workspace_dependencies_o, ) .await?; @@ -1123,6 +1116,7 @@ async fn lock_flow_value<'c>( skip_flow_update, &raw_deps, dependency_map, + raw_workspace_dependencies_o, ) .await?; @@ -1159,6 +1153,7 @@ async fn lock_modules<'c>( skip_flow_update: bool, raw_deps: &Option>, dependency_map: &mut ScopedDependencyMap, // (modules to replace old seq (even unmmodified ones), new transaction, modified ids) ) + raw_workspace_dependencies_o: &Option, ) -> Result<( Vec, sqlx::Transaction<'c, sqlx::Postgres>, @@ -1214,6 +1209,7 @@ async fn lock_modules<'c>( skip_flow_update, &raw_deps, dependency_map, + &raw_workspace_dependencies_o, )) .await?; e.value = FlowModuleValue::ForloopFlow { @@ -1251,6 +1247,7 @@ async fn lock_modules<'c>( skip_flow_update, &raw_deps, dependency_map, + raw_workspace_dependencies_o, )) .await?; nmodified_ids.extend(inner_modified_ids); @@ -1280,6 +1277,7 @@ async fn lock_modules<'c>( skip_flow_update, &raw_deps, dependency_map, + raw_workspace_dependencies_o, )) .await?; e.value = FlowModuleValue::WhileloopFlow { @@ -1314,6 +1312,7 @@ async fn lock_modules<'c>( skip_flow_update, &raw_deps, dependency_map, + raw_workspace_dependencies_o, )) .await?; nmodified_ids.extend(inner_modified_ids); @@ -1342,6 +1341,7 @@ async fn lock_modules<'c>( skip_flow_update, &raw_deps, dependency_map, + raw_workspace_dependencies_o, )) .await?; errors.extend(ninner_errors); @@ -1410,6 +1410,7 @@ async fn lock_modules<'c>( skip_flow_update, &raw_deps, dependency_map, + raw_workspace_dependencies_o, )) .await?; @@ -1442,23 +1443,16 @@ async fn lock_modules<'c>( .await?; } - let get_imports = || { + let get_references = || { let dep_path = path.clone().unwrap_or_else(|| job_path.to_string()); - extract_relative_imports( - &content, - &format!("{dep_path}/flow"), - &Some(language.clone()), - ) + extract_referenced_paths(&content, &format!("{dep_path}/flow"), Some(language)) }; if let Some(locks_to_reload) = locks_to_reload { if !locks_to_reload.contains(&e.id) { - if !is_generated_from_raw_requirements(&Some(language), &lock) { - let relative_imports = get_imports(); - tx = dependency_map - .patch(relative_imports.clone(), e.id.clone(), tx) - .await?; - } + tx = dependency_map + .patch(get_references(), e.id.clone(), tx) + .await?; new_flow_modules.push(e); continue; } @@ -1466,12 +1460,9 @@ async fn lock_modules<'c>( if lock.as_ref().is_some_and(|x| !x.trim().is_empty()) { let skip_creating_new_lock = skip_creating_new_lock(&language, &content); if skip_creating_new_lock { - if !is_generated_from_raw_requirements(&Some(language), &lock) { - let relative_imports = get_imports(); - tx = dependency_map - .patch(relative_imports.clone(), e.id.clone(), tx) - .await?; - } + tx = dependency_map + .patch(get_references(), e.id.clone(), tx) + .await?; new_flow_modules.push(e); continue; @@ -1512,16 +1503,15 @@ async fn lock_modules<'c>( "{}/flow", &path.clone().unwrap_or_else(|| job_path.to_string()) ), - raw_deps, - None, occupancy_metrics, + raw_workspace_dependencies_o, ) .await; // let lock = match new_lock { Ok(new_lock) => { if !raw_deps && !skip_flow_update { - let relative_imports = get_imports(); + let relative_imports = get_references(); tx = dependency_map .patch(relative_imports.clone(), e.id.clone(), tx) .await?; @@ -1926,6 +1916,7 @@ async fn lock_modules_app( // Represents the closest container id container_id: Option, dependency_map: &mut ScopedDependencyMap, + raw_workspace_dependencies_o: &Option, ) -> Result { match value { Value::Object(mut m) => { @@ -1960,10 +1951,10 @@ async fn lock_modules_app( .to_string(); let mut logs = "".to_string(); - let relative_imports = extract_relative_imports( + let referenced_paths = extract_referenced_paths( &content, &format!("{job_path}/app"), - &Some(language.clone()), + Some(language), ); if let Some((l, id)) = locks_to_reload @@ -1981,7 +1972,7 @@ async fn lock_modules_app( if !l.contains(id) { dependency_map .patch( - relative_imports.clone(), + referenced_paths.clone(), container_id.unwrap_or_default(), db.begin().await?, ) @@ -1997,7 +1988,7 @@ async fn lock_modules_app( if skip_creating_new_lock(&language, &content) { dependency_map .patch( - relative_imports.clone(), + referenced_paths.clone(), container_id.unwrap_or_default(), db.begin().await?, ) @@ -2026,9 +2017,9 @@ async fn lock_modules_app( base_internal_url, token, &format!("{}/app", job.runnable_path()), - false, - None, occupancy_metrics, + // TODO: + &None, ) .await; match new_lock { @@ -2037,7 +2028,7 @@ async fn lock_modules_app( dependency_map .patch( - relative_imports.clone(), + referenced_paths.clone(), container_id.unwrap_or_default(), db.begin().await?, ) @@ -2104,6 +2095,7 @@ async fn lock_modules_app( .map(str::to_owned) .or(container_id.clone()), dependency_map, + raw_workspace_dependencies_o, ) .await?, ); @@ -2130,6 +2122,7 @@ async fn lock_modules_app( locks_to_reload, container_id.clone(), dependency_map, + raw_workspace_dependencies_o, ) .await?, ); @@ -2151,6 +2144,7 @@ pub async fn handle_app_dependency_job( base_internal_url: &str, token: &str, occupancy_metrics: &mut OccupancyMetrics, + raw_workspace_dependencies_o: Option, ) -> error::Result<()> { let job_path = job.runnable_path.clone().ok_or_else(|| { error::Error::internal_err( @@ -2222,6 +2216,7 @@ pub async fn handle_app_dependency_job( &components_to_relock, None, &mut dependency_map, + &raw_workspace_dependencies_o, ) .await?; @@ -2422,7 +2417,7 @@ async fn python_dep( py_version: crate::PyV, annotations: PythonAnnotations, ) -> std::result::Result { - use crate::python_executor::split_requirements; + use windmill_common::worker::{split_python_requirements, PyVAlias}; create_dependencies_dir(job_dir).await; @@ -2443,8 +2438,7 @@ async fn python_dep( // install the dependencies to pre-fill the cache if let Ok(req) = req.as_ref() { let r = handle_python_reqs( - split_requirements(req), - // req.split("\n").filter(|x| !x.starts_with("--")).collect(), + split_python_requirements(req), job_id, w_id, mem_peak, @@ -2455,7 +2449,7 @@ async fn python_dep( worker_dir, occupancy_metrics, // final_version, - crate::PyVAlias::default().into(), + PyVAlias::default().into(), ) .await; @@ -2599,11 +2593,20 @@ async fn capture_dependency_job( base_internal_url: &str, token: &str, script_path: &str, - raw_deps: bool, - npm_mode: Option, occupancy_metrics: &mut OccupancyMetrics, + raw_workspace_dependencies_o: &Option, ) -> error::Result { - match job_language { + let workspace_dependencies = WorkspaceDependenciesPrefetched::extract( + job_raw_code, + *job_language, + w_id, + raw_workspace_dependencies_o, + script_path, + db.into(), + ) + .await?; + + let lock = match job_language { ScriptLang::Python3 => { #[cfg(not(feature = "python"))] return Err(Error::internal_err( @@ -2611,49 +2614,41 @@ async fn capture_dependency_job( )); #[cfg(feature = "python")] { - // Manually assigned version from requirements.txt - // let assigned_py_version; - let (reqs, py_version) = if raw_deps { - // `wmill script generate-metadata` - // should also respect annotated pyversion - // can be annotated in script itself - // or in requirements.txt if present + let annotations = PythonAnnotations::parse(job_raw_code); - ( - job_raw_code.to_owned(), - match crate::PyV::try_parse_from_requirements(&split_requirements( - job_raw_code, - )) { - Some(pyv) => pyv, - None => crate::PyV::gravitational_version(job_id, w_id, None).await, - }, + let (pyv, reqs) = { + let (mut version_specifiers, mut locked_v) = (vec![], None); + let reqs = windmill_parser_py_imports::parse_python_imports( + job_raw_code, + &w_id, + script_path, + &db, + &mut version_specifiers, + &mut locked_v, + raw_workspace_dependencies_o, ) - } else { - let mut version_specifiers = vec![]; - let PythonAnnotations { py_select_latest, .. } = - PythonAnnotations::parse(job_raw_code); - ( - windmill_parser_py_imports::parse_python_imports( - job_raw_code, - &w_id, - script_path, - &db, - &mut version_specifiers, - ) - .await? - .0 - .join("\n"), + .await? + .0 + .join("\n"); + + // Resolve python version + // It is based on version specifiers + let pyv = if let Some(v) = locked_v { + v.into() + } else { crate::PyV::resolve( version_specifiers, job_id, w_id, - py_select_latest, + annotations.py_select_latest, Some(db.clone().into()), None, None, ) - .await?, - ) + .await? + }; + + (pyv, reqs) }; python_dep( @@ -2667,21 +2662,10 @@ async fn capture_dependency_job( w_id, worker_dir, &mut Some(occupancy_metrics), - py_version, - PythonAnnotations::parse(job_raw_code), + pyv, + annotations, ) - .await - .map(|res| { - if raw_deps { - format!( - "{}\n{}", - windmill_common::lockfiles::LOCKFILE_GENERATED_FROM_REQUIREMENTS_TXT, - res - ) - } else { - res - } - }) + .await? } } ScriptLang::Ansible => { @@ -2692,11 +2676,6 @@ async fn capture_dependency_job( #[cfg(feature = "python")] { - if raw_deps { - return Err(Error::ExecutionErr( - "Raw dependencies not supported for ansible".to_string(), - )); - } let (_logs, reqs, _) = windmill_parser_yaml::parse_ansible_reqs(job_raw_code)?; ansible_dep( @@ -2713,13 +2692,14 @@ async fn capture_dependency_job( token, base_internal_url, ) - .await + .await? } } ScriptLang::Go => { install_go_dependencies( job_id, job_raw_code, + MaybeLock::Unresolved { workspace_dependencies: workspace_dependencies.clone() }, mem_peak, canceled_by, job_dir, @@ -2727,19 +2707,13 @@ async fn capture_dependency_job( false, false, false, - raw_deps, worker_name, w_id, occupancy_metrics, ) - .await + .await? } ScriptLang::Deno => { - if raw_deps { - return Err(Error::ExecutionErr( - "Raw dependencies not supported for deno".to_string(), - )); - } generate_deno_lock( job_id, job_raw_code, @@ -2752,16 +2726,12 @@ async fn capture_dependency_job( base_internal_url, &mut Some(occupancy_metrics), ) - .await + .await? } ScriptLang::Bun | ScriptLang::Bunnative => { - let npm_mode = npm_mode.unwrap_or_else(|| { - windmill_common::worker::TypeScriptAnnotations::parse(job_raw_code).npm - }); - if !raw_deps { - let _ = write_file(job_dir, "main.ts", job_raw_code)?; - } - let req = gen_bun_lockfile( + // TODO: move inside gen_bun_lockfile + write_file(job_dir, "main.ts", job_raw_code)?; + if let Some(lock) = gen_bun_lockfile( mem_peak, canceled_by, job_id, @@ -2773,19 +2743,15 @@ async fn capture_dependency_job( base_internal_url, worker_name, true, - if raw_deps { - Some(job_raw_code.to_string()) - } else { - None - }, - npm_mode, + &workspace_dependencies, + windmill_common::worker::TypeScriptAnnotations::parse(job_raw_code).npm, &mut Some(occupancy_metrics), ) - .await?; - if req.is_some() && !raw_deps { + .await? + { crate::bun_executor::prebundle_bun_script( job_raw_code, - req.as_ref(), + &lock, script_path, job_id, w_id, @@ -2797,8 +2763,11 @@ async fn capture_dependency_job( &mut Some(occupancy_metrics), ) .await?; + + lock + } else { + Default::default() } - Ok(req.unwrap_or_else(String::new)) } ScriptLang::Php => { #[cfg(not(feature = "php"))] @@ -2808,19 +2777,15 @@ async fn capture_dependency_job( #[cfg(feature = "php")] { - let reqs = if raw_deps { - if job_raw_code.is_empty() { - return Ok("".to_string()); - } - job_raw_code.to_string() + let composer_content = if let Some(c) = workspace_dependencies.get_php()? { + c } else { match parse_php_imports(job_raw_code)? { Some(reqs) => reqs, - None => { - return Ok("".to_string()); - } + None => return Ok("".to_string()), } }; + composer_install( mem_peak, canceled_by, @@ -2829,20 +2794,14 @@ async fn capture_dependency_job( &Connection::Sql(db.clone()), job_dir, worker_name, - reqs, + composer_content, None, occupancy_metrics, ) - .await + .await? } } ScriptLang::Rust => { - if raw_deps { - return Err(Error::ExecutionErr( - "Raw dependencies not supported for rust".to_string(), - )); - } - #[cfg(not(feature = "rust"))] return Err(Error::internal_err( "Rust requires the rust feature to be enabled".to_string(), @@ -2863,15 +2822,9 @@ async fn capture_dependency_job( .await?; #[cfg(feature = "rust")] - Ok(lockfile) + lockfile } ScriptLang::CSharp => { - if raw_deps { - return Err(Error::ExecutionErr( - "Raw dependencies not supported for C#".to_string(), - )); - } - generate_nuget_lockfile( job_id, job_raw_code, @@ -2883,16 +2836,10 @@ async fn capture_dependency_job( w_id, occupancy_metrics, ) - .await + .await? } #[cfg(feature = "java")] ScriptLang::Java => { - if raw_deps { - return Err(Error::ExecutionErr( - "Raw dependencies not supported for Java".to_string(), - )); - } - java_executor::resolve( job_id, job_raw_code, @@ -2900,16 +2847,10 @@ async fn capture_dependency_job( &Connection::Sql(db.clone()), w_id, ) - .await + .await? } #[cfg(feature = "ruby")] ScriptLang::Ruby => { - if raw_deps { - return Err(Error::ExecutionErr( - "Raw dependencies not supported for Ruby".to_string(), - )); - } - ruby_executor::resolve( job_id, job_raw_code, @@ -2920,9 +2861,32 @@ async fn capture_dependency_job( worker_name, w_id, ) - .await + .await? } // for related places search: ADD_NEW_LANG - _ => Ok("".to_owned()), + _ => "".to_owned(), + }; + { + let mut lines = vec![]; + add_lock_header(&mut lines, workspace_dependencies, *job_language, w_id, db).await?; + Ok(if lines.is_empty() { + lock + } else { + format!("{}\n{lock}", lines.join("\n")) + }) } } + +async fn add_lock_header( + lines: &mut Vec, + wd: WorkspaceDependenciesPrefetched, + _language: ScriptLang, + _workspace_id: &str, + _db: &sqlx::Pool, +) -> error::Result<()> { + if let Some(header) = wd.to_lock_header().await { + lines.push(header); + } + + Ok(()) +} diff --git a/backend/windmill-worker/src/workspace_dependencies.rs b/backend/windmill-worker/src/workspace_dependencies.rs new file mode 100644 index 0000000000..dd060122a7 --- /dev/null +++ b/backend/windmill-worker/src/workspace_dependencies.rs @@ -0,0 +1,268 @@ +use serde::{Deserialize, Serialize}; +use windmill_common::{error, scripts::ScriptLang, workspace_dependencies::WorkspaceDependencies}; + +use crate::{ + scoped_dependency_map::ScopedDependencyMap, trigger_dependents_to_recompute_dependencies, +}; + +#[derive(sqlx::FromRow, Clone, Serialize, Deserialize, Hash, Debug)] +pub struct NewWorkspaceDependencies { + pub workspace_id: String, + pub language: ScriptLang, + pub name: Option, + /// If None, will use description of previous version + /// If there is no older versions, will set to default + pub description: Option, + // TODO: Make Option, or optimize it in any other way. + pub content: String, +} + +impl NewWorkspaceDependencies { + /// Creates a new workspace dependencies entry in the database. + /// + /// Archives any existing dependencies with the same name/language/workspace, + /// then inserts the new dependencies. Triggers recomputation of dependent scripts + /// and rebuilds the dependency map if this is the first unnamed dependency for the workspace. + pub async fn create<'c>( + self, + email: &str, + created_by: &str, + permissioned_as: &str, + db: &sqlx::Pool, + ) -> error::Result { + // Check if all workers support workspace dependencies feature + windmill_common::workspace_dependencies::min_version_supports_v0_workspace_dependencies() + .await?; + + let path = WorkspaceDependencies::to_path(&self.name, self.language)?; + + // If it is unnamed then we want to rebuild dependency map. Otherwise trigger dependents to recompute locks will not work + // NOTE: We rebuild first, even before creating new w deps. We want to make sure that if rebuild failed, then no new default workspace dependencies were created. + if self.name.is_none() { + // Check if we already rebuilt the map for this workspace by checking if the setting exists + let setting_name = format!("workspace_dependencies_map_rebuilt:{}", self.workspace_id); + let already_rebuilt = + windmill_common::global_settings::load_value_from_global_settings( + db, + &setting_name, + ) + .await? + .is_some(); + + if !already_rebuilt { + tracing::info!( + workspace_id = %self.workspace_id, + "Rebuilding workspace dependencies map for first unnamed workspace dependencies" + ); + ScopedDependencyMap::rebuild_map_unchecked(&self.workspace_id, db).await?; + + // Mark as rebuilt by creating the setting + windmill_common::global_settings::set_value_in_global_settings( + db, + &setting_name, + serde_json::json!({}), + ) + .await?; + tracing::info!( + workspace_id = %self.workspace_id, + "Marked workspace dependencies map as rebuilt" + ); + } else { + tracing::info!( + workspace_id = %self.workspace_id, + "Skipping workspace dependencies map rebuild - already rebuilt for this workspace" + ); + } + }; + + let mut tx = db.begin().await?; + let prev_description = sqlx::query_scalar!( + " + UPDATE workspace_dependencies + SET archived = true + WHERE archived = false + AND name IS NOT DISTINCT FROM $1 + AND workspace_id = $2 + AND language = $3 + RETURNING description + ", + self.name, + self.workspace_id, + self.language as ScriptLang + ) + .fetch_optional(&mut *tx) + .await?; + + let new_id = sqlx::query_scalar!( + " + INSERT INTO workspace_dependencies(name, workspace_id, content, language, description) + VALUES ($1, $2, $3, $4, $5) + RETURNING id + ", + self.name.clone(), + self.workspace_id, + self.content, + self.language as ScriptLang, + self.description + .or(prev_description.clone()) + .unwrap_or("Default Workspace Dependencies".to_owned()) + ) + .fetch_one(&mut *tx) + .await?; + tx.commit().await?; + + // Make sure trigger dependents will have latest view. + // NOTE: Uncomment for tests + // #[cfg(test)] + // assert_eq!( + // sqlx::query_scalar!( + // " + // SELECT id FROM workspace_dependencies + // WHERE archived = false + // AND name IS NOT DISTINCT FROM $1 + // AND workspace_id = $2 + // AND language = $3 + // ", + // self.name, + // self.workspace_id, + // self.language as ScriptLang, + // ) + // .fetch_one(db) // Use db + // .await?, + // new_id + // ); + + // It's ok to fail, it will return an error and user will get notified that they should redeploy workspace dependencies + trigger_dependents_to_recompute_dependencies( + &self.workspace_id, + crate::scoped_dependency_map::ScopedDependencyMap::get_dependents( + path.as_str(), + &self.workspace_id, + db, + ) + .await?, + None, + None, + email, + created_by, + permissioned_as, + db, + vec![], + ) + .await?; + + Ok(new_id) + } +} + +// Type aliases for backward compatibility +pub type RawRequirements = WorkspaceDependencies; +pub type NewRawRequirements = NewWorkspaceDependencies; + +#[cfg(test)] +mod workspace_dependencies_tests { + + // // TODO: test all cases when it should reject. + // #[cfg(feature = "python")] + // mod new_workspace_dependencies { + // use windmill_common::scripts::ScriptLang; + + // use crate::workspace_dependencies::NewWorkspaceDependencies; + + // #[sqlx::test( + // fixtures("../../tests/fixtures/base.sql",), + // migrations = "../migrations" + // )] + // async fn test_create(db: sqlx::Pool) -> anyhow::Result<()> { + // assert_eq!( + // NewWorkspaceDependencies { + // workspace_id: "test-workspace".into(), + // language: ScriptLang::Python3, + // name: None, + // description: None, + // content: "global:rev1".to_owned(), + // } + // .create("", "", "", &db) + // .await + // .unwrap(), + // 1 + // ); + + // assert_eq!( + // NewWorkspaceDependencies { + // workspace_id: "test-workspace".into(), + // language: ScriptLang::Python3, + // name: Some("rrs1".to_owned()), + // description: None, + // content: "rrs1:rev1".to_owned(), + // } + // .create("", "", "", &db) + // .await + // .unwrap(), + // 2 + // ); + + // assert!(NewWorkspaceDependencies { + // workspace_id: "test-workspace".into(), + // language: ScriptLang::DuckDb, + // description: None, + // name: None, + // content: "".to_owned(), + // } + // .create("", "", "", &db) + // .await + // .is_err()); + + // // Will act as redeployment + // assert_eq!( + // NewWorkspaceDependencies { + // workspace_id: "test-workspace".into(), + // language: ScriptLang::Python3, + // description: None, + // name: Some("rrs1".to_owned()), + // content: "rrs1:rev2".to_owned(), + // } + // .create("", "", "", &db) + // .await + // .unwrap(), + // // It will just increment id + // 3 + // ); + // Ok(()) + // } + + // #[sqlx::test( + // fixtures("../../tests/fixtures/base.sql",), + // migrations = "../migrations" + // )] + // async fn violate_constraints(db: sqlx::Pool) -> anyhow::Result<()> { + // let db = &db; + // let create = |name| { + // sqlx::query_scalar!( + // " + // INSERT INTO workspace_dependencies(name, workspace_id, content, language) + // VALUES ($1, 'test-workspace', 'test', 'python3') + // RETURNING id + // ", + // name + // ) + // .fetch_one(db) + // }; + + // assert_eq!(create(Some("test".to_owned())).await.unwrap(), 1); + // assert_eq!(create(None).await.unwrap(), 2); + + // assert!(create(Some("test".to_owned())).await.is_err()); + // assert!(create(None).await.is_err()); + // assert_eq!( + // sqlx::query_scalar!("SELECT COUNT(*) FROM workspace_dependencies",) + // .fetch_one(db) + // .await + // .unwrap() + // .unwrap(), + // 2 + // ); + // Ok(()) + // } + // } +} diff --git a/cli/src/commands/dependencies/dependencies.ts b/cli/src/commands/dependencies/dependencies.ts new file mode 100644 index 0000000000..5c1d4aa385 --- /dev/null +++ b/cli/src/commands/dependencies/dependencies.ts @@ -0,0 +1,105 @@ +// deno-lint-ignore-file no-explicit-any +import { requireLogin } from "../../core/auth.ts"; +import { resolveWorkspace } from "../../core/context.ts"; +import { GlobalOptions } from "../../types.ts"; +import { colors, Command, log } from "../../../deps.ts"; +import * as wmill from "../../../gen/services.gen.ts"; +import type { ScriptLang } from "../../../gen/types.gen.ts"; +import fs from "node:fs"; +import { generateHash } from "../../utils/utils.ts"; +import { checkifMetadataUptodate, updateMetadataGlobalLock, workspaceDependenciesPathToLanguageAndFilename } from "../../utils/metadata.ts"; + +async function push( + opts: GlobalOptions, + filePath: string, + language?: ScriptLang, + name?: string +): Promise { + const workspace = await resolveWorkspace(opts); + await requireLogin(opts); + + if (!fs.existsSync(filePath)) { + throw new Error(`File not found: ${filePath}`); + } + + const content = fs.readFileSync(filePath, "utf8"); + + // Use the existing pushWorkspaceDependencies function + await pushWorkspaceDependencies(workspace.workspaceId, filePath, null, content); +} + +const command = new Command() + .alias("deps") + .description("workspace dependencies related commands") + .command( + "push", + "Push workspace dependencies from a local file" + ) + .arguments("") + .option( + "--language ", + "Programming language (python3, typescript, go, php). If not specified, will be inferred from file extension." + ) + .option( + "--name ", + "Name for the dependencies. If not specified, creates workspace default dependencies." + ) + .action(push as any); + +export async function pushWorkspaceDependencies( + workspace: string, + path: string, + _befObj: any, + newDependenciesContent: string +): Promise { + try { + + let res = workspaceDependenciesPathToLanguageAndFilename(path); + if (!res) { + throw new Error(`Unknown workspace dependencies file format: ${path}`); + } + + let { + language, + name + } = res; + + // TODO: include workspace? + // Generate hash for workspace dependencies content and metadata + const contentHash = await generateHash(newDependenciesContent + path); + + // Check if dependencies are up-to-date using wmill-lock.yaml tracking + const isUpToDate = await checkifMetadataUptodate(path, contentHash, undefined); + + if (isUpToDate) { + const displayName = name ? `named dependencies "${name}"` : `workspace default dependencies`; + log.info(colors.green(`${displayName} for ${language} are up-to-date, skipping push`)); + return; + } + + log.info(colors.yellow(`Pushing ${name ? 'named' : 'workspace default'} dependencies for ${language}...`)); + + await wmill.createWorkspaceDependencies({ + workspace, + requestBody: { + name, + content: newDependenciesContent, + language, + workspace_id: workspace, + // Description is not supported in cli, it will use old description + description: undefined + } + }); + + // Update wmill-lock.yaml with new hash after successful push + await updateMetadataGlobalLock(path, contentHash); + + const displayName = name ? `named dependencies "${name}"` : `workspace default dependencies`; + log.info(colors.green(`Successfully pushed ${displayName} for ${language}`)); + } catch (error: any) { + log.error(colors.red(`Failed to push workspace dependencies: ${error.message}`)); + throw error; + } +} + +export default command; diff --git a/cli/src/commands/flow/flow.ts b/cli/src/commands/flow/flow.ts index e16f2fbcb3..4be9e53867 100644 --- a/cli/src/commands/flow/flow.ts +++ b/cli/src/commands/flow/flow.ts @@ -200,9 +200,6 @@ async function generateLocks( } & SyncOptions, folder: string | undefined ) { - const useRawReqs = - opts.useRawRequirements || Deno.env.get("USE_RAW_REQUIREMENTS") === "true"; - const workspace = await resolveWorkspace(opts); await requireLogin(opts); opts = await mergeConfigWithConfigFile(opts); @@ -212,10 +209,7 @@ async function generateLocks( folder, false, workspace, - opts, - undefined, - undefined, - useRawReqs + opts ); } else { const ignore = await ignoreF(opts); @@ -241,10 +235,7 @@ async function generateLocks( folder, true, workspace, - opts, - undefined, - undefined, - useRawReqs + opts ); if (candidate) { hasAny = true; @@ -271,10 +262,7 @@ async function generateLocks( folder, false, workspace, - opts, - undefined, - undefined, - useRawReqs + opts ); } } diff --git a/cli/src/commands/script/script.ts b/cli/src/commands/script/script.ts index de1ed3289b..ff513336a9 100644 --- a/cli/src/commands/script/script.ts +++ b/cli/src/commands/script/script.ts @@ -26,13 +26,14 @@ import { import { Workspace } from "../workspace/workspace.ts"; import { generateScriptMetadataInternal, + getRawWorkspaceDependencies, parseMetadataFile, } from "../../utils/metadata.ts"; import { - LanguageWithRawReqsSupport, + WorkspaceDependenciesLanguage, ScriptLanguage, inferContentTypeFromFilePath, - languagesWithRawReqsSupport, + workspaceDependenciesLanguages, } from "../../utils/script_common.ts"; import { elementsToMap, @@ -88,14 +89,13 @@ async function push(opts: PushOptions, filePath: string) { await requireLogin(opts); const codebases = await listSyncCodebases(opts as SyncOptions); - const globalDeps = await findGlobalDeps(); await handleFile( filePath, workspace, [], undefined, opts, - globalDeps, + await getRawWorkspaceDependencies(), codebases ); log.info(colors.bold.underline.green(`Script ${filePath} pushed`)); @@ -156,7 +156,7 @@ export async function handleScriptMetadata( workspace: Workspace, alreadySynced: string[], message: string | undefined, - globalDeps: GlobalDeps, + rawWorkspaceDependencies: Record, codebases: SyncCodebase[], opts: GlobalOptions ): Promise { @@ -172,7 +172,7 @@ export async function handleScriptMetadata( alreadySynced, message, opts, - globalDeps, + rawWorkspaceDependencies, codebases ); } else { @@ -194,7 +194,7 @@ export async function handleFile( alreadySynced: string[], message: string | undefined, opts: (GlobalOptions & { defaultTs?: "bun" | "deno" } & Skips) | undefined, - globalDeps: GlobalDeps, + rawWorkspaceDependencies: Record, codebases: SyncCodebase[] ): Promise { if ( @@ -326,7 +326,7 @@ export async function handleFile( path, workspaceRemote: workspace, schemaOnly: codebase ? true : undefined, - globalDeps, + rawWorkspaceDependencies, codebases, } : undefined @@ -943,39 +943,10 @@ async function bootstrap( } export type GlobalDeps = Map< - LanguageWithRawReqsSupport, + WorkspaceDependenciesLanguage, Record >; -export async function findGlobalDeps(): Promise { - var globalDeps: GlobalDeps = new Map(); - const els = await FSFSElement(Deno.cwd(), [], false); - for await (const entry of readDirRecursiveWithIgnore((p, isDir) => { - p = SEP + p; - return ( - !isDir && - // Skip if the filename is not one of lockfile names - !languagesWithRawReqsSupport.some((lockfile) => - p.endsWith(SEP + lockfile.rrFilename) - ) - ); - }, els)) { - if (entry.isDirectory || entry.ignored) continue; - const content = await entry.getContentText(); - - // Iterate over available languages to find which lockfile - languagesWithRawReqsSupport.map((lock) => { - if (entry.path.endsWith(lock.rrFilename)) { - const current = globalDeps.get(lock) ?? {}; - current[ - entry.path.substring(0, entry.path.length - lock.rrFilename.length) - ] = content; - globalDeps.set(lock, current); - } - }); - } - return globalDeps; -} async function generateMetadata( opts: GlobalOptions & { lockOnly?: boolean; @@ -999,7 +970,7 @@ async function generateMetadata( opts = await mergeConfigWithConfigFile(opts); const codebases = await listSyncCodebases(opts); - const globalDeps = await findGlobalDeps(); + const rawWorkspaceDependencies = await getRawWorkspaceDependencies(); if (scriptPath) { // read script metadata file await generateScriptMetadataInternal( @@ -1008,11 +979,12 @@ async function generateMetadata( opts, false, false, - globalDeps, + rawWorkspaceDependencies, codebases, false ); } else { + // TODO: test this as well. const ignore = await ignoreF(opts); const elems = await elementsToMap( await FSFSElement(Deno.cwd(), codebases, false), @@ -1036,7 +1008,7 @@ async function generateMetadata( opts, true, true, - globalDeps, + rawWorkspaceDependencies, codebases, false ); @@ -1063,6 +1035,7 @@ async function generateMetadata( log.info(colors.green.bold("No metadata to update")); return; } + // TODO: test this for (const e of Object.keys(elems)) { await generateScriptMetadataInternal( e, @@ -1070,7 +1043,7 @@ async function generateMetadata( opts, false, true, - globalDeps, + rawWorkspaceDependencies, codebases, false ); diff --git a/cli/src/commands/sync/pull.ts b/cli/src/commands/sync/pull.ts index 18e0e350fb..27bb04c3ad 100644 --- a/cli/src/commands/sync/pull.ts +++ b/cli/src/commands/sync/pull.ts @@ -17,6 +17,7 @@ export async function downloadZip( includeGroups?: boolean, includeSettings?: boolean, includeKey?: boolean, + skipWorkspaceDependencies?: boolean, defaultTs?: "bun" | "deno" ): Promise { const requestHeaders = new Headers(); @@ -30,8 +31,8 @@ export async function downloadZip( } } - const zipResponse = await fetch( - workspace.remote + + const includeWorkspaceDependenciesValue = !(skipWorkspaceDependencies ?? false); + const url = workspace.remote + "api/w/" + workspace.workspaceId + `/workspaces/tarball?archive_type=zip&plain_secret=${plainSecrets ?? false @@ -39,8 +40,9 @@ export async function downloadZip( }&skip_secrets=${skipSecrets ?? false}&include_schedules=${includeSchedules ?? false }&include_triggers=${includeTriggers ?? false}&include_users=${includeUsers ?? false }&include_groups=${includeGroups ?? false}&include_settings=${includeSettings ?? false - }&include_key=${includeKey ?? false}&default_ts=${defaultTs ?? "bun"}&skip_resource_types=${skipResourceTypes ?? false}`, - { + }&include_key=${includeKey ?? false}&include_workspace_dependencies=${includeWorkspaceDependenciesValue}&default_ts=${defaultTs ?? "bun"}&skip_resource_types=${skipResourceTypes ?? false}`; + + const zipResponse = await fetch(url, { headers: requestHeaders, method: "GET", } diff --git a/cli/src/commands/sync/sync.ts b/cli/src/commands/sync/sync.ts index 3ea6ced18c..4a1befb82f 100644 --- a/cli/src/commands/sync/sync.ts +++ b/cli/src/commands/sync/sync.ts @@ -28,14 +28,13 @@ import { downloadZip } from "./pull.ts"; import { exts, findContentFile, - findGlobalDeps, findResourceFile, handleScriptMetadata, removeExtensionToPath, } from "../script/script.ts"; import { handleFile } from "../script/script.ts"; -import { deepEqual, isFileResource } from "../../utils/utils.ts"; +import { deepEqual, isFileResource, isWorkspaceDependencies } from "../../utils/utils.ts"; import { SyncOptions, getEffectiveSettings, @@ -58,7 +57,9 @@ import { SyncCodebase, listSyncCodebases } from "../../utils/codebase.ts"; import { generateFlowLockInternal, generateScriptMetadataInternal, + getRawWorkspaceDependencies, readLockfile, + workspaceDependenciesPathToLanguageAndFilename, } from "../../utils/metadata.ts"; import { OpenFlow } from "../../../gen/types.gen.ts"; import { pushResource } from "../resource/resource.ts"; @@ -311,7 +312,7 @@ function ZipFSElement( p: string, f: JSZip.JSZipObject ): Promise { - const kind: "flow" | "app" | "script" | "resource" | "other" = p.endsWith( + const kind: "flow" | "app" | "script" | "resource" | "dependencies" | "other" = p.endsWith( "flow.json" ) ? "flow" @@ -321,6 +322,8 @@ function ZipFSElement( ? "script" : p.endsWith("resource.json") ? "resource" + : p.startsWith("dependencies/") + ? "dependencies" : "other"; const isJson = p.endsWith(".json"); @@ -330,6 +333,8 @@ function ZipFSElement( return p.replace("flow.json", "flow"); } else if (kind == "app") { return p.replace("app.json", "app"); + } else if (kind == "dependencies") { + return p; } else { return useYaml && isJson ? p.replaceAll(".json", ".yaml") : p; } @@ -489,7 +494,7 @@ function ZipFSElement( : JSON.stringify(parsed, null, 2); } - return useYaml && isJson + return useYaml && isJson && kind != "dependencies" ? (() => { try { return yamlStringify(JSON.parse(content), yamlOptions); @@ -622,7 +627,6 @@ export async function* readDirRecursiveWithIgnore( while (stack.length > 0) { const e = stack.pop()!; - // console.log(e.path); yield e; for await (const e2 of e.c()) { if (e2.isDirectory) { @@ -631,10 +635,9 @@ export async function* readDirRecursiveWithIgnore( continue; } } - // console.log(e2.path); stack.push({ path: e2.path, - ignored: e.ignored || ignore(e2.path, e2.isDirectory), + ignored: e.ignored || e2.isDirectory && e2.path == "dependencies" ? false : ignore(e2.path, e2.isDirectory), isDirectory: e2.isDirectory, // getContentBytes: e2.getContentBytes, getContentText: e2.getContentText, @@ -661,16 +664,20 @@ export async function elementsToMap( ignore: (path: string, isDirectory: boolean) => boolean, json: boolean, skips: Skips, - specificItems?: SpecificItemsConfig + specificItems?: SpecificItemsConfig, ): Promise<{ [key: string]: string }> { const map: { [key: string]: string } = {}; const processedBasePaths = new Set(); - for await (const entry of readDirRecursiveWithIgnore(ignore, els)) { - if (entry.isDirectory || entry.ignored) continue; + if (entry.isDirectory || entry.ignored) { + if (entry.path.includes("dependencies/")) { + log.info(`Ignoring dependencies-related path: ${entry.path} (isDirectory: ${entry.isDirectory}, ignored: ${entry.ignored})`); + } + continue; + } const path = entry.path; - if (json && path.endsWith(".yaml") && !isFileResource(path)) continue; - if (!json && path.endsWith(".json") && !isFileResource(path)) continue; + if (json && path.endsWith(".yaml") && !isFileResource(path) && !isWorkspaceDependencies(path)) continue; + if (!json && path.endsWith(".json") && !isFileResource(path) && !isWorkspaceDependencies(path)) continue; const ext = json ? ".json" : ".yaml"; if (!skips.includeSchedules && path.endsWith(".schedule" + ext)) continue; if ( @@ -694,6 +701,7 @@ export async function elementsToMap( if (skips.skipResourceTypes && path.endsWith(".resource-type" + ext)) continue; + // Use getTypeStrFromPath for consistent type detection try { const fileType = getTypeStrFromPath(path); @@ -702,6 +710,7 @@ export async function elementsToMap( if (skips.skipFlows && fileType === "flow") continue; if (skips.skipApps && fileType === "app") continue; if (skips.skipFolders && fileType === "folder") continue; + if (skips.skipWorkspaceDependencies && fileType === "workspace_dependencies") continue; } catch { // If getTypeStrFromPath can't determine the type, continue processing the file } @@ -728,6 +737,8 @@ export async function elementsToMap( "nu", "java", "rb", + "in", // Python requirements.in files + "mod", // Go go.mod files // for related places search: ADD_NEW_LANG ].includes(path.split(".").pop() ?? "") && !isFileResource(path) @@ -797,7 +808,7 @@ export async function elementsToMap( // No specific items configuration, use regular path map[entry.path] = content; } - } + } return map; } @@ -810,6 +821,7 @@ export interface Skips { skipFlows?: boolean | undefined; skipApps?: boolean | undefined; skipFolders?: boolean | undefined; + skipWorkspaceDependencies?: boolean | undefined; skipScriptsMetadata?: boolean | undefined; includeSchedules?: boolean | undefined; includeTriggers?: boolean | undefined; @@ -878,11 +890,14 @@ async function compareDynFSElement( if (skipMetadata) { continue; } + if (k.startsWith("dependencies/")) { + log.info(`Adding workspace dependencies file: ${k}`); + } changes.push({ name: "added", path: k, content: v }); } else { if (m2[k] == v) { continue; - } else if (k.endsWith(".json")) { + } else if (k.endsWith(".json") && !isWorkspaceDependencies(k)) { let parsedV, parsedM2; try { parsedV = JSON.parse(v); @@ -1093,6 +1108,7 @@ export async function ignoreF(wmillconf: { excludes?: string[]; extraIncludes?: string[]; skipResourceTypes?: boolean; + skipWorkspaceDependencies?: boolean; json?: boolean; includeUsers?: boolean; includeGroups?: boolean; @@ -1151,6 +1167,9 @@ export async function ignoreF(wmillconf: { if (wmillconf.includeKey && fileType === "encryption_key") { return false; // Don't ignore, always include } + if (!wmillconf.skipWorkspaceDependencies && fileType === "workspace_dependencies") { + return false; // Don't ignore workspace dependencies (they are always included unless explicitly skipped) + } } catch { // If getTypeStrFromPath can't determine the type, fall through to normal logic } @@ -1269,8 +1288,7 @@ export async function pull( } catch { // ignore } - const remote = ZipFSElement( - (await downloadZip( + const zipFile = await downloadZip( workspace, opts.plainSecrets, opts.skipVariables, @@ -1283,8 +1301,12 @@ export async function pull( opts.includeGroups, opts.includeSettings, opts.includeKey, + opts.skipWorkspaceDependencies, opts.defaultTs - ))!, + ); + + const remote = ZipFSElement( + zipFile!, !opts.json, opts.defaultTs ?? "bun", resourceTypeToFormatExtension, @@ -1308,6 +1330,19 @@ export async function pull( log.info( `remote (${workspace.name}) -> local: ${changes.length} changes to apply` ); + + + // Debug: show all changes for push operation + if (changes.length > 0) { + log.info("All changes:"); + changes.forEach(change => { + if (change.path.startsWith("dependencies/")) { + log.info(` ${change.name}: ${change.path} [WORKSPACE DEPS]`); + } else { + log.info(` ${change.name}: ${change.path}`); + } + }); + } // Handle JSON output for dry-run if (opts.dryRun && opts.jsonOutput) { @@ -1495,9 +1530,9 @@ export async function pull( } log.info("All local changes pulled, now updating wmill-lock.yaml"); await readLockfile(); // ensure wmill-lock.yaml exists - const globalDeps = await findGlobalDeps(); const tracker: ChangeTracker = await buildTracker(changes); + const rawWorkspaceDependencies: Record = await getRawWorkspaceDependencies(); for (const change of tracker.scripts) { await generateScriptMetadataInternal( @@ -1506,14 +1541,14 @@ export async function pull( opts, false, true, - globalDeps, + rawWorkspaceDependencies, codebases, true ); } for (const change of tracker.flows) { log.info(`Updating lock for flow ${change}`); - await generateFlowLockInternal(change, false, workspace, opts, true); + await generateFlowLockInternal(change, false, workspace, opts, true, false); } if (tracker.apps.length > 0) { log.info( @@ -1733,6 +1768,7 @@ export async function push( opts.includeGroups, opts.includeSettings, opts.includeKey, + opts.skipWorkspaceDependencies, opts.defaultTs ))!, !opts.json, @@ -1754,7 +1790,8 @@ export async function push( specificItems ); - const globalDeps = await findGlobalDeps(); + + const rawWorkspaceDependencies = await getRawWorkspaceDependencies(); const tracker: ChangeTracker = await buildTracker(changes); @@ -1767,7 +1804,7 @@ export async function push( opts, true, true, - globalDeps, + rawWorkspaceDependencies, codebases, false ); @@ -1932,7 +1969,7 @@ export async function push( workspace, alreadySynced, opts.message, - globalDeps, + rawWorkspaceDependencies, codebases, opts ) @@ -1948,7 +1985,7 @@ export async function push( alreadySynced, opts.message, opts, - globalDeps, + rawWorkspaceDependencies, codebases ) ) { @@ -2040,7 +2077,7 @@ export async function push( alreadySynced, opts.message, opts, - globalDeps, + rawWorkspaceDependencies, codebases ) ) { @@ -2225,6 +2262,22 @@ export async function push( ".group.json" ), }); + break; + case "workspace_dependencies": + const relativePath = removePathPrefix(change.path, "dependencies"); + + const res = workspaceDependenciesPathToLanguageAndFilename(change.path); + if (!res) { + throw new Error(`Unknown workspace dependencies file format: ${change.path}`); + } + const { name, language } = res; + + await wmill.deleteWorkspaceDependencies({ + workspace: workspaceId, + language, + name + }); + break; default: break; @@ -2323,6 +2376,7 @@ const command = new Command() .option("--skip-flows", "Skip syncing flows") .option("--skip-apps", "Skip syncing apps") .option("--skip-folders", "Skip syncing folders") + .option("--skip-workspace-dependencies", "Skip syncing workspace dependencies") // .option("--skip-scripts-metadata", "Skip syncing scripts metadata, focus solely on logic") .option("--include-schedules", "Include syncing schedules") .option("--include-triggers", "Include syncing triggers") @@ -2371,6 +2425,7 @@ const command = new Command() .option("--skip-flows", "Skip syncing flows") .option("--skip-apps", "Skip syncing apps") .option("--skip-folders", "Skip syncing folders") + .option("--skip-workspace-dependencies", "Skip syncing workspace dependencies") // .option("--skip-scripts-metadata", "Skip syncing scripts metadata, focus solely on logic") .option("--include-schedules", "Include syncing schedules") .option("--include-triggers", "Include syncing triggers") diff --git a/cli/src/core/conf.ts b/cli/src/core/conf.ts index 0a6dbe8167..aa9a7a83be 100644 --- a/cli/src/core/conf.ts +++ b/cli/src/core/conf.ts @@ -26,6 +26,7 @@ export interface SyncOptions { skipResources?: boolean; skipResourceTypes?: boolean; skipSecrets?: boolean; + skipWorkspaceDependencies?: boolean; skipScripts?: boolean; skipFlows?: boolean; skipApps?: boolean; @@ -317,6 +318,7 @@ export const DEFAULT_SYNC_OPTIONS: Readonly< | "skipSecrets" | "includeSchedules" | "includeTriggers" + | "skipWorkspaceDependencies" | "skipScripts" | "skipFlows" | "skipApps" @@ -346,6 +348,7 @@ export const DEFAULT_SYNC_OPTIONS: Readonly< includeGroups: false, includeSettings: false, includeKey: false, + skipWorkspaceDependencies: false, } as const; export async function mergeConfigWithConfigFile( diff --git a/cli/src/main.ts b/cli/src/main.ts index df61ec8c81..ce1137f369 100644 --- a/cli/src/main.ts +++ b/cli/src/main.ts @@ -36,6 +36,7 @@ import { pull, push } from "./commands/sync/sync.ts"; import { add as workspaceAdd } from "./commands/workspace/workspace.ts"; import workers from "./commands/workers/workers.ts"; import queues from "./commands/queues/queues.ts"; +import dependencies from "./commands/dependencies/dependencies.ts"; import init from "./commands/init/init.ts"; export { @@ -126,6 +127,7 @@ const command = new Command() .command("worker-groups", workerGroups) .command("workers", workers) .command("queues", queues) + .command("dependencies", dependencies) .command("version --version", "Show version information") .action(async (opts: any) => { console.log("CLI version: " + VERSION); diff --git a/cli/src/types.ts b/cli/src/types.ts index a7b821882a..1e8fd6a9eb 100644 --- a/cli/src/types.ts +++ b/cli/src/types.ts @@ -17,10 +17,11 @@ import { pushResourceType } from "./commands/resource-type/resource-type.ts"; import { pushVariable } from "./commands/variable/variable.ts"; import { yamlOptions } from "./commands/sync/sync.ts"; import { showDiffs } from "./core/conf.ts"; -import { deepEqual, isFileResource } from "./utils/utils.ts"; +import { deepEqual, isFileResource, isWorkspaceDependencies } from "./utils/utils.ts"; import { pushSchedule } from "./commands/schedule/schedule.ts"; import { pushWorkspaceUser } from "./commands/user/user.ts"; import { pushGroup } from "./commands/user/user.ts"; +import { pushWorkspaceDependencies } from "./commands/dependencies/dependencies.ts"; import { pushWorkspaceSettings, pushWorkspaceKey } from "./core/settings.ts"; import { pushTrigger } from "./commands/trigger/trigger.ts"; @@ -187,6 +188,8 @@ export async function pushObj( await pushWorkspaceUser(workspace, p, befObj, newObj); } else if (typeEnding === "group") { await pushGroup(workspace, p, befObj, newObj); + } else if (typeEnding === "workspace_dependencies") { + await pushWorkspaceDependencies(workspace, p, befObj, newObj); } else if (typeEnding === "settings") { await pushWorkspaceSettings(workspace, p, befObj, newObj); } else if (typeEnding === "encryption_key") { @@ -199,7 +202,9 @@ export async function pushObj( } export function parseFromPath(p: string, content: string): any { - return p.endsWith(".yaml") + return isWorkspaceDependencies(p) + ? content + : p.endsWith(".yaml") ? yamlParseContent(p, content) : p.endsWith(".json") ? JSON.parse(content) @@ -237,13 +242,17 @@ export function getTypeStrFromPath( | "user" | "group" | "settings" - | "encryption_key" { + | "encryption_key" + | "workspace_dependencies" { if (p.includes(".flow" + SEP)) { return "flow"; } if (p.includes(".app" + SEP)) { return "app"; } + if (p.startsWith("dependencies" + SEP)) { + return "workspace_dependencies"; + } const parsed = path.parse(p); if ( parsed.ext == ".go" || diff --git a/cli/src/utils/metadata.ts b/cli/src/utils/metadata.ts index 00e2206933..97cc76e504 100644 --- a/cli/src/utils/metadata.ts +++ b/cli/src/utils/metadata.ts @@ -14,12 +14,12 @@ import { } from "../../bootstrap/script_bootstrap.ts"; import { Workspace } from "../commands/workspace/workspace.ts"; import { - languagesWithRawReqsSupport, - LanguageWithRawReqsSupport, + workspaceDependenciesLanguages, + WorkspaceDependenciesLanguage, ScriptLanguage, } from "./script_common.ts"; import { inferContentTypeFromFilePath } from "./script_common.ts"; -import { GlobalDeps, exts, findGlobalDeps } from "../commands/script/script.ts"; +import { exts } from "../commands/script/script.ts"; import { FSFSElement, findCodebase, @@ -48,29 +48,51 @@ export class LockfileGenerationError extends Error { export async function generateAllMetadata() {} -function findClosestRawReqs( - lang: LanguageWithRawReqsSupport | undefined, - remotePath: string, - globalDeps: GlobalDeps -): string | undefined { - let bestCandidate: { k: string; v: string } | undefined = undefined; - if (lang) { - Object.entries(globalDeps.get(lang) ?? {}).forEach(([k, v]) => { - if ( - remotePath.startsWith(k) && - k.length >= (bestCandidate?.k ?? "").length - ) { - bestCandidate = { k, v }; +export async function getRawWorkspaceDependencies(): Promise> { + const rawWorkspaceDeps: Record = {}; + + try { + for await (const entry of Deno.readDir("dependencies")) { + if (entry.isDirectory) continue; + + const filePath = `dependencies/${entry.name}`; + const content = await Deno.readTextFile(filePath); + + // Find matching language + for (const lang of workspaceDependenciesLanguages) { + if (entry.name.endsWith(lang.filename)) { + // Check if out of sync + const contentHash = await generateHash(content + filePath); + const isUpToDate = await checkifMetadataUptodate(filePath, contentHash, undefined); + + if (!isUpToDate) { + rawWorkspaceDeps[filePath] = content; + } + break; + } } - }); + } + } catch { + // dependencies directory doesn't exist } - // @ts-ignore - return bestCandidate?.v; + return rawWorkspaceDeps; +} + +export function workspaceDependenciesPathToLanguageAndFilename(path: string): { name: string | undefined, language: ScriptLanguage } | undefined { + const relativePath = path.replace("dependencies/", ""); + for (const { filename, language } of workspaceDependenciesLanguages) { + if (relativePath.endsWith(filename)) { + return { + name: relativePath === filename ? undefined : relativePath.replace("." + filename, ""), + language + }; + } + } } const TOP_HASH = "__flow_hash"; async function generateFlowHash( - rawReqs: Record | undefined, + rawWorkspaceDependencies: Record, folder: string, defaultTs: "bun" | "deno" | undefined ) { @@ -78,17 +100,9 @@ async function generateFlowHash( const hashes: Record = {}; for await (const f of elems.getChildren()) { if (exts.some((e) => f.path.endsWith(e))) { - let reqs: string | undefined; - if (rawReqs) { - // Get language name from path - const lang = inferContentTypeFromFilePath(f.path, defaultTs); - // Get lock for that language - [, reqs] = - Object.entries(rawReqs).find(([lang2, _]) => lang == lang2) ?? []; - } - // Embed lock into hash + // Embed workspace dependencies into hash hashes[f.path] = await generateHash( - (await f.getContentText()) + (reqs ?? "") + (await f.getContentText()) + JSON.stringify(rawWorkspaceDependencies) ); } } @@ -102,8 +116,7 @@ export async function generateFlowLockInternal( defaultTs?: "bun" | "deno"; }, justUpdateMetadataLock?: boolean, - noStaleMessage?: boolean, - useRawReqs?: boolean + noStaleMessage?: boolean ): Promise { if (folder.endsWith(SEP)) { folder = folder.substring(0, folder.length - 1); @@ -115,24 +128,9 @@ export async function generateFlowLockInternal( log.info(`Generating lock for flow ${folder} at ${remote_path}`); } - let rawReqs: Record | undefined = undefined; - if (useRawReqs) { - // Find all dependency files in the workspace - const globalDeps = await findGlobalDeps(); - - // Find closest dependency files for this flow - rawReqs = {}; - - // TODO: PERF: Only include raw reqs for the languages that are in the flow - languagesWithRawReqsSupport.map((lang) => { - const dep = findClosestRawReqs(lang, folder, globalDeps); - if (dep) { - // @ts-ignore - rawReqs[lang.language] = dep; - } - }); - } - let hashes = await generateFlowHash(rawReqs, folder, opts.defaultTs); + // Always get out-of-sync workspace dependencies + let rawWorkspaceDependencies: Record = await getRawWorkspaceDependencies(); + let hashes = await generateFlowHash(rawWorkspaceDependencies, folder, opts.defaultTs); const conf = await readLockfile(); if (await checkifMetadataUptodate(folder, hashes[TOP_HASH], conf, TOP_HASH)) { @@ -146,15 +144,12 @@ export async function generateFlowLockInternal( return remote_path; } - if (useRawReqs) { - log.warn( - "If using local lockfiles, following redeployments from Web App will inevitably override generated lockfiles by CLI. To maintain your script's lockfiles you will need to redeploy only from CLI. (Behavior is subject to change)" - ); + if (Object.keys(rawWorkspaceDependencies).length > 0) { log.info( (await blueColor())( - `Found raw requirements (${languagesWithRawReqsSupport - .map((l) => l.rrFilename) - .join("/")}) for ${folder}, using it` + `Found workspace dependencies (${workspaceDependenciesLanguages + .map((l) => l.filename) + .join("/")}) for ${folder}, using them` ) ); } @@ -192,7 +187,7 @@ export async function generateFlowLockInternal( workspace, flowValue.value, remote_path, - rawReqs + rawWorkspaceDependencies ); const inlineScripts = extractInlineScriptsForFlows( @@ -212,7 +207,7 @@ export async function generateFlowLockInternal( ); } - hashes = await generateFlowHash(rawReqs, folder, opts.defaultTs); + hashes = await generateFlowHash(rawWorkspaceDependencies, folder, opts.defaultTs); await clearGlobalLock(folder); for (const [path, hash] of Object.entries(hashes)) { await updateMetadataGlobalLock(folder, hash, path); @@ -236,7 +231,7 @@ export async function generateScriptMetadataInternal( }, dryRun: boolean, noStaleMessage: boolean, - globalDeps: GlobalDeps, + rawWorkspaceDependencies: Record, codebases: SyncCodebase[], justUpdateMetadataLock?: boolean ): Promise { @@ -246,6 +241,14 @@ export async function generateScriptMetadataInternal( const language = inferContentTypeFromFilePath(scriptPath, opts.defaultTs); + // Filter workspace dependencies to only include those matching the script's language + const filteredRawWorkspaceDependencies: Record = {}; + for (const [depPath, depContent] of Object.entries(rawWorkspaceDependencies)) { + const depInfo = workspaceDependenciesPathToLanguageAndFilename(depPath); + if (depInfo && depInfo.language === language) { + filteredRawWorkspaceDependencies[depPath] = depContent; + } + } const metadataWithType = await parseMetadataFile( remotePath, @@ -256,14 +259,8 @@ export async function generateScriptMetadataInternal( const scriptContent = await Deno.readTextFile(scriptPath); const metadataContent = await Deno.readTextFile(metadataWithType.path); - const rrLang = languagesWithRawReqsSupport.find( - (l) => language == l.language - ); - - const rawReqs = findClosestRawReqs(rrLang, scriptPath, globalDeps); - - - let hash = await generateScriptHash(rawReqs, scriptContent, metadataContent); + // Note: rawWorkspaceDependencies are now passed in as parameter instead of being searched hierarchically + let hash = await generateScriptHash(filteredRawWorkspaceDependencies, scriptContent, metadataContent); if (await checkifMetadataUptodate(remotePath, hash, undefined)) { if (!noStaleMessage) { @@ -304,7 +301,7 @@ export async function generateScriptMetadataInternal( language, remotePath, metadataParsedContent, - rawReqs + filteredRawWorkspaceDependencies ); } else { metadataParsedContent.lock = ""; @@ -323,7 +320,7 @@ export async function generateScriptMetadataInternal( const metadataContentUsedForHash = newMetadataContent; hash = await generateScriptHash( - rawReqs, + filteredRawWorkspaceDependencies, scriptContent, metadataContentUsedForHash ); @@ -366,11 +363,11 @@ async function updateScriptLock( language: ScriptLanguage, remotePath: string, metadataContent: Record, - rawDeps: string | undefined + rawWorkspaceDependencies: Record ): Promise { if ( !( - languagesWithRawReqsSupport.some((l) => l.language == language) || + workspaceDependenciesLanguages.some((l) => l.language == language) || language == "deno" || language == "rust" || language == "ansible" @@ -379,9 +376,12 @@ async function updateScriptLock( return; } - if (rawDeps) { - log.info(`Generating script lock for ${remotePath} with raw deps`); + + if (Object.keys(rawWorkspaceDependencies).length > 0) { + const dependencyPaths = Object.keys(rawWorkspaceDependencies).join(', '); + log.info(`Generating script lock for ${remotePath} with raw workspace dependencies: ${dependencyPaths}`); } + // generate the script lock running a dependency job in Windmill and update it inplace // TODO: update this once the client is released const extraHeaders = getHeaders(); @@ -402,7 +402,8 @@ async function updateScriptLock( script_path: remotePath, }, ], - raw_deps: rawDeps, + raw_workspace_dependencies: Object.keys(rawWorkspaceDependencies).length > 0 + ? rawWorkspaceDependencies : null, entrypoint: remotePath, }), } @@ -451,12 +452,12 @@ export async function updateFlow( workspace: Workspace, flow_value: FlowValue, remotePath: string, - rawDeps?: Record + rawWorkspaceDependencies: Record ): Promise { let rawResponse; - if (rawDeps != undefined) { - log.info(colors.blue("Using raw requirements for flow dependencies")); + if (Object.keys(rawWorkspaceDependencies).length > 0) { + log.info(colors.blue("Using raw workspace dependencies for flow dependencies")); // generate the script lock running a dependency job in Windmill and update it inplace const extraHeaders = getHeaders(); @@ -473,7 +474,9 @@ export async function updateFlow( flow_value, path: remotePath, use_local_lockfiles: true, - raw_deps: rawDeps, + raw_workspace_dependencies: Object.keys(rawWorkspaceDependencies).length > 0 + ? rawWorkspaceDependencies + : null, }), } ); @@ -767,7 +770,7 @@ export async function parseMetadataFile( path: string; workspaceRemote: Workspace; schemaOnly?: boolean; - globalDeps: GlobalDeps; + rawWorkspaceDependencies: Record; codebases: SyncCodebase[] }) | undefined, @@ -829,7 +832,7 @@ export async function parseMetadataFile( generateMetadataIfMissing, false, false, - generateMetadataIfMissing.globalDeps, + generateMetadataIfMissing.rawWorkspaceDependencies, generateMetadataIfMissing.codebases, false ); @@ -911,12 +914,12 @@ export async function checkifMetadataUptodate( } export async function generateScriptHash( - rawReqs: string | undefined, + rawWorkspaceDependencies: Record, scriptContent: string, newMetadataContent: string ) { return await generateHash( - (rawReqs ?? "") + scriptContent + newMetadataContent + JSON.stringify(rawWorkspaceDependencies) + scriptContent + newMetadataContent ); } diff --git a/cli/src/utils/script_common.ts b/cli/src/utils/script_common.ts index df661d25fe..a5aba792c8 100644 --- a/cli/src/utils/script_common.ts +++ b/cli/src/utils/script_common.ts @@ -26,18 +26,17 @@ export type ScriptLanguage = // To make language support raw requirements: // 1. Add value here // 2. Modify backend to allow raw deps -export type LanguageWithRawReqsSupport = - | { language: "bun", rrFilename /** (raw requirements filename) */: "package.json" } - // TODO: Add `requirements.in` - more intuitive and reflects better what actually happens - | { language: "python3", rrFilename: "requirements.txt" } - | { language: "php", rrFilename: "composer.json" } - | { language: "go", rrFilename: "go.mod" }; +export type WorkspaceDependenciesLanguage = + | { language: "bun", filename /** (raw requirements filename) */: "package.json" } + | { language: "python3", filename: "requirements.in" } + | { language: "php", filename: "composer.json" } + | { language: "go", filename: "go.mod" }; -export const languagesWithRawReqsSupport: LanguageWithRawReqsSupport[] = [ - { language: "bun", rrFilename: "package.json" }, - { language: "python3", rrFilename: "requirements.txt" }, - { language: "php", rrFilename: "composer.json" }, - { language: "go", rrFilename: "go.mod" }, +export const workspaceDependenciesLanguages: WorkspaceDependenciesLanguage[] = [ + { language: "bun", filename: "package.json" }, + { language: "python3", filename: "requirements.in" }, + { language: "php", filename: "composer.json" }, + { language: "go", filename: "go.mod" }, ] as const; export function inferContentTypeFromFilePath( diff --git a/cli/src/utils/utils.ts b/cli/src/utils/utils.ts index d11b751a5b..3d33a53405 100644 --- a/cli/src/utils/utils.ts +++ b/cli/src/utils/utils.ts @@ -145,6 +145,10 @@ export function isFileResource(path: string): boolean { ); } +export function isWorkspaceDependencies(path: string): boolean { + return path.startsWith("dependencies/") +} + export function printSync(input: string | Uint8Array, to = Deno.stdout) { let bytesWritten = 0 const bytes = typeof input === 'string' ? new TextEncoder().encode(input) : input @@ -235,4 +239,4 @@ export function writeIfChanged(path: string, content: string): boolean { // console.log(`Writing content to ${path}`); Deno.writeTextFileSync(path, content); return true; // File was written -} \ No newline at end of file +} diff --git a/flake.nix b/flake.nix index 102c79e8dd..b818e0245a 100644 --- a/flake.nix +++ b/flake.nix @@ -50,7 +50,7 @@ xmlsec.dev libxslt.dev libclang.dev - libffi # For deno_ffi + libffi # For deno_ffi libtool nodejs postgresql @@ -118,20 +118,17 @@ fi wm-cli-deps ''; - buildInputs = buildInputs ++ [ - pkgs.deno - ]; + buildInputs = buildInputs ++ [ pkgs.deno ]; packages = [ (pkgs.writeScriptBin "wm-cli" '' deno run -A --no-check $FLAKE_ROOT/cli/src/main.ts $* '') (pkgs.writeScriptBin "wm-cli-deps" '' pushd $FLAKE_ROOT/cli/ - ${ - if pkgs.stdenv.isDarwin - then "./gen_wm_client_mac.sh && ./windmill-utils-internal/gen_wm_client_mac.sh" - else "./gen_wm_client.sh && ./windmill-utils-internal/gen_wm_client.sh" - } + ${if pkgs.stdenv.isDarwin then + "./gen_wm_client_mac.sh && ./windmill-utils-internal/gen_wm_client_mac.sh" + else + "./gen_wm_client.sh && ./windmill-utils-internal/gen_wm_client.sh"} popd '') ]; @@ -278,6 +275,8 @@ PHP_PATH = "${pkgs.php}/bin/php"; COMPOSER_PATH = "${pkgs.php84Packages.composer}/bin/composer"; BUN_PATH = "${pkgs.bun}/bin/bun"; + NODE_PATH = "${pkgs.nodejs}/bin/node"; + NODE_BIN_PATH = "${pkgs.nodejs}/bin/node"; UV_PATH = "${pkgs.uv}/bin/uv"; NU_PATH = "${pkgs.nushell}/bin/nu"; JAVA_PATH = "${pkgs.jdk21}/bin/java"; @@ -314,9 +313,11 @@ # See https://web.archive.org/web/20220523141208/https://hoverbear.org/blog/rust-bindgen-in-nix/ BINDGEN_EXTRA_CLANG_ARGS = # Prevent clang from using system headers - only use Nix headers - "-nostdinc ${builtins.readFile "${stdenv.cc}/nix-support/libc-crt1-cflags"} ${ - builtins.readFile "${stdenv.cc}/nix-support/libc-cflags" - } ${builtins.readFile "${stdenv.cc}/nix-support/cc-cflags"} ${ + "-nostdinc ${ + builtins.readFile "${stdenv.cc}/nix-support/libc-crt1-cflags" + } ${builtins.readFile "${stdenv.cc}/nix-support/libc-cflags"} ${ + builtins.readFile "${stdenv.cc}/nix-support/cc-cflags" + } ${ builtins.readFile "${stdenv.cc}/nix-support/libcxx-cxxflags" } -idirafter ${pkgs.libiconv}/include ${ lib.optionalString stdenv.cc.isClang diff --git a/frontend/src/lib/components/DependenciesDeploymentWarning.svelte b/frontend/src/lib/components/DependenciesDeploymentWarning.svelte new file mode 100644 index 0000000000..305fc93512 --- /dev/null +++ b/frontend/src/lib/components/DependenciesDeploymentWarning.svelte @@ -0,0 +1,295 @@ + + + +
+ {#if loading} +
+
+ Loading dependencies... +
+ {:else} + +
+
+ + Workspace Dependencies +
+
+ {importedPath} +
+
+ + {#if dependencies.length === 0} + + {#snippet children()} +

No dependent runnables were found for these workspace dependencies, but the action will still proceed.

+ {/snippet} + + {:else} + +
+ +
+ + +
+

+ This action will trigger redeployment of {getTotalDependentsCount()} + {getTotalDependentsCount() === 1 ? 'dependent runnable' : 'dependent runnables'}: +

+
+ + +
+ {#each dependencies as dependency} + {@render DependencyNode({ node: dependency, level: 0 })} + {/each} +
+ {/if} + + {/if} +
+ + + + + + +{#snippet DependencyNode({ node, level }: { node: DependencyNode, level: number })} + {@const Icon = getIcon(node.kind)} +
+
+ + {#if (node.childrenCount ?? 0) > 0} + + {:else} +
+ {/if} + + +
+ + + {getKindLabel(node.kind)} + +
+ + +
+ + {node.path} + + {#if node.nodeIds && node.nodeIds.length > 0} + + {node.nodeIds.length} node{node.nodeIds.length !== 1 ? 's' : ''} + + {/if} +
+ + + {#if (node.childrenCount ?? 0) > 0} + + {node.childrenCount} dep{node.childrenCount !== 1 ? 's' : ''} + + {/if} +
+ + + {#if node.expanded && node.children} +
+ {#each node.children as child} + {@render DependencyNode({ node: child, level: level + 1 })} + {/each} +
+ {/if} +
+{/snippet} diff --git a/frontend/src/lib/components/HighlightCode.svelte b/frontend/src/lib/components/HighlightCode.svelte index 63e1d8a999..be6e6b6ef0 100644 --- a/frontend/src/lib/components/HighlightCode.svelte +++ b/frontend/src/lib/components/HighlightCode.svelte @@ -19,11 +19,11 @@ import { copyToClipboard } from '$lib/utils' import { ClipboardCopy } from 'lucide-svelte' import HighlightTheme from './HighlightTheme.svelte' - import type { LanguageType } from 'svelte-highlight/languages' + import { json, type LanguageType } from 'svelte-highlight/languages' interface Props { code?: string - language: Script['language'] | 'bunnative' | 'frontend' | undefined + language: Script['language'] | 'bunnative' | 'frontend' | 'json' | undefined highlightLanguage?: LanguageType | undefined lines?: boolean className?: string @@ -43,7 +43,7 @@ applyButtonIcon = undefined }: Props = $props() - function getLang(lang: Script['language'] | 'bunnative' | 'frontend' | undefined) { + function getLang(lang: Script['language'] | 'bunnative' | 'frontend' | 'json' | undefined) { switch (lang) { case 'python3': return python @@ -91,6 +91,8 @@ return java case 'ruby': return ruby + case 'json': + return json // for related places search: ADD_NEW_LANG default: return typescript diff --git a/frontend/src/lib/components/ScriptBuilder.svelte b/frontend/src/lib/components/ScriptBuilder.svelte index 4b386aa363..fd2326cb23 100644 --- a/frontend/src/lib/components/ScriptBuilder.svelte +++ b/frontend/src/lib/components/ScriptBuilder.svelte @@ -12,7 +12,7 @@ PostgresTriggerService, CaptureService, type ScriptLang, - WorkerService + WorkerService, } from '$lib/gen' import { inferArgs } from '$lib/infer' import { @@ -189,7 +189,7 @@ : undefined ) const simplifiedPoll = writable(false) - + export function setPrimarySchedule(schedule: ScheduleTrigger | undefined | false) { primaryScheduleStore.set(schedule) loadTriggers() @@ -1759,6 +1759,7 @@ /> {/if} + {#if $enterpriseLicense && initialPath != ''} diff --git a/frontend/src/lib/components/WorkspaceDependenciesEditor.svelte b/frontend/src/lib/components/WorkspaceDependenciesEditor.svelte new file mode 100644 index 0000000000..86e1bba5d6 --- /dev/null +++ b/frontend/src/lib/components/WorkspaceDependenciesEditor.svelte @@ -0,0 +1,512 @@ + + + + +
+ {#if !can_write} + + You only have read access to this resource and cannot edit it + + {/if} + + {#if showWarning && currentImportedPath} + + {/if} + +
+
+ {#if hasWorkspaceDefault(workspaceDependencies.language) && !edit} + + {#snippet children({ item })} + + {/snippet} + +
+ + + Workspace default already exists for {workspaceDependencies.language} + + +
+ {:else} + + {#snippet children({ item })} + + + {/snippet} + + {/if} + {#if workspaceDependenciesType === 'named'} + + {/if} +
+ + Default Enforced Dependencies are used when no specific Dependencies are referenced from runnables. + Named dependencies can be referenced by scripts using + + annotations + . +
+
+
+ +
+
+ +
+ Provide a brief description to help others understand the purpose of these enforced dependencies. +
+
+ +
+ {#snippet header()} + + ({workspaceDependencies.content.length}/{MAX_WORKSPACE_DEPENDENCIES_LENGTH} characters) + + {/snippet} +
+ {#await import('$lib/components/SimpleEditor.svelte')} + + {:then Module} + handleEditorChange(e.detail)} + fixedOverflowWidgets={false} + disabled={!can_write} + /> + {/await} +
+
+ +
+ + {#snippet actions()} + + {/snippet} + + diff --git a/frontend/src/lib/components/WorkspaceDependenciesViewer.svelte b/frontend/src/lib/components/WorkspaceDependenciesViewer.svelte new file mode 100644 index 0000000000..fc3f246fc1 --- /dev/null +++ b/frontend/src/lib/components/WorkspaceDependenciesViewer.svelte @@ -0,0 +1,100 @@ + + + + + + + {#snippet actions()} +
+
+ + {viewLanguage} +
+ {#if canWriteDeps} + + {/if} +
+ {/snippet} + +
+ {#if viewDescription} +
+

{viewDescription}

+
+ {/if} + + {#if viewContent} + + {:else} +
+ +

No workspace dependencies found for this path

+

Create workspace dependencies to define dependencies for scripts in this directory

+
+ {/if} +
+ + \ No newline at end of file diff --git a/frontend/src/lib/components/workspaceSettings/WorkspaceDependenciesSettings.svelte b/frontend/src/lib/components/workspaceSettings/WorkspaceDependenciesSettings.svelte new file mode 100644 index 0000000000..34cd534ae2 --- /dev/null +++ b/frontend/src/lib/components/workspaceSettings/WorkspaceDependenciesSettings.svelte @@ -0,0 +1,393 @@ + + + + + (x.name || 'Default') + ' ' + (x.language || '') + ' ' + (x.content || '')} +/> + +
+
+
Enforced Dependencies
+ + Enforced Dependencies define dependency specifications for scripts by language. Unnamed dependencies serve as workspace defaults, while named dependencies can be referenced by scripts using #raw_reqs annotations. + +
+ +
+ +
+
+ +
+
+ + +
+
+ +
+ +
+ +
+ {#if !filteredItems} + + {#each new Array(3) as _} + + {/each} + {:else if filteredItems.length == 0} +
+ +
No enforced dependencies found
+
+ Try changing the filters or creating new enforced dependencies +
+ +
+ {:else} + + + + Name + Language + Description + Type + Edited + Actions + + + + {#each filteredItems as deps} + + +
+ +
+ + + {workspaceDependenciesEditor?.getFullFilename(deps.language, deps.name ?? null)} • {deps.language} + +
+
+ + +
+ + + {deps.language || 'python3'} + +
+ + + + {deps.description || '-'} + + + + + {deps.name === null ? 'Default' : 'Named'} + + + + + + + + +
+ + + + + + +
+ + + {/each} + + + {/if} +
+ + + + {#snippet actions()} +
+ + {viewLanguage} +
+ {/snippet} + +
+ {#if viewContent} + + {:else} +
+ +

No content available for this requirement

+
+ {/if} +
+ + + +{#if showDependencyWarning && currentImportedPath} + +{/if} diff --git a/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte b/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte index 132b626a21..9294ddbf63 100644 --- a/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte @@ -59,6 +59,7 @@ import UnsavedConfirmationModal from '$lib/components/common/confirmationModal/UnsavedConfirmationModal.svelte' import TextInput from '$lib/components/text_input/TextInput.svelte' import CollapseLink from '$lib/components/CollapseLink.svelte' + import WorkspaceDependenciesSettings from '$lib/components/workspaceSettings/WorkspaceDependenciesSettings.svelte' let slackInitialPath: string = $state('') let slackScriptPath: string = $state('') @@ -134,7 +135,8 @@ | 'windmill_lfs' | 'git_sync' | 'default_app' - | 'encryption') ?? 'users' + | 'encryption' + | 'dependencies') ?? 'users' ) let usingOpenaiClientCredentialsOauth = $state(false) @@ -692,6 +694,13 @@ aiDescription="General workspace settings" label="General" /> + {#if !loadedSettings} @@ -1124,6 +1133,8 @@
Loading workspace...
{/if} + {:else if tab == 'dependencies'} + {:else if tab == 'default_app'}