From 3d79187f31f63ab8771390116df67085f175a0ee Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sat, 26 Aug 2023 09:12:19 +0200 Subject: [PATCH] feat: lock inline scripts for apps on deploy --- ...5a4f20ff806315a9ed02b372977e2d9479338.json | 4 +- ...17cbcf1013ae311c36b42d423bf6a02fa016c.json | 4 +- ...273203aab7344abd03df4d86e4c5f5b9a394a.json | 22 +++ ...0273b942202cdf9017ce6b5ba612e66ce3875.json | 1 + ...46d313f6ef05062ac55f7397caf8c02ccec5e.json | 3 +- ...1eb737da453e9fcac6cfa098296d6c211d2a9.json | 15 ++ ...e7183380c43d0feeb615c651aa486eef3bd53.json | 1 + ...f1eac35a6fdbc82a634b7eb9da4d7931f3582.json | 6 +- .../20230826001320_appdependencies.down.sql | 1 + .../20230826001320_appdependencies.up.sql | 2 + backend/windmill-api/openapi.yaml | 59 ++++--- backend/windmill-api/src/apps.rs | 106 ++++++++++--- backend/windmill-api/src/flows.rs | 10 +- backend/windmill-audit/src/lib.rs | 3 + backend/windmill-common/src/jobs.rs | 5 + backend/windmill-queue/src/jobs.rs | 12 ++ backend/windmill-worker/src/worker.rs | 148 +++++++++++++++++- .../helpers/RunnableComponent.svelte | 3 +- .../InlineScriptEditor.svelte | 3 + .../settingsPanel/ComponentPanel.svelte | 30 ++-- .../editor/settingsPanel/TableActions.svelte | 2 +- frontend/src/lib/components/apps/types.ts | 1 + .../auditLogs/AuditLogsFilters.svelte | 1 + .../(logged)/runs/[...path]/+page.svelte | 4 +- 24 files changed, 373 insertions(+), 73 deletions(-) create mode 100644 backend/.sqlx/query-29fce7e56bb4a874b4c1e3a6454273203aab7344abd03df4d86e4c5f5b9a394a.json create mode 100644 backend/.sqlx/query-6fb904692513c453989ffedb14b1eb737da453e9fcac6cfa098296d6c211d2a9.json create mode 100644 backend/migrations/20230826001320_appdependencies.down.sql create mode 100644 backend/migrations/20230826001320_appdependencies.up.sql diff --git a/backend/.sqlx/query-01d5e4e371a507602fca01190965a4f20ff806315a9ed02b372977e2d9479338.json b/backend/.sqlx/query-01d5e4e371a507602fca01190965a4f20ff806315a9ed02b372977e2d9479338.json index bd29c4cd61..14bba19013 100644 --- a/backend/.sqlx/query-01d5e4e371a507602fca01190965a4f20ff806315a9ed02b372977e2d9479338.json +++ b/backend/.sqlx/query-01d5e4e371a507602fca01190965a4f20ff806315a9ed02b372977e2d9479338.json @@ -39,7 +39,8 @@ "http", "graphql", "postgresql", - "noop" + "noop", + "appdependencies" ] } } @@ -59,6 +60,7 @@ "bash", "postgresql", "nativets", + "Nativets", "bun", "mysql", "bigquery", diff --git a/backend/.sqlx/query-25bef6a248f3ee0ea2cbcc376c217cbcf1013ae311c36b42d423bf6a02fa016c.json b/backend/.sqlx/query-25bef6a248f3ee0ea2cbcc376c217cbcf1013ae311c36b42d423bf6a02fa016c.json index ae4abdc440..7287cc5bb3 100644 --- a/backend/.sqlx/query-25bef6a248f3ee0ea2cbcc376c217cbcf1013ae311c36b42d423bf6a02fa016c.json +++ b/backend/.sqlx/query-25bef6a248f3ee0ea2cbcc376c217cbcf1013ae311c36b42d423bf6a02fa016c.json @@ -44,7 +44,8 @@ "http", "graphql", "postgresql", - "noop" + "noop", + "appdependencies" ] } } @@ -66,6 +67,7 @@ "bash", "postgresql", "nativets", + "Nativets", "bun", "mysql", "bigquery", diff --git a/backend/.sqlx/query-29fce7e56bb4a874b4c1e3a6454273203aab7344abd03df4d86e4c5f5b9a394a.json b/backend/.sqlx/query-29fce7e56bb4a874b4c1e3a6454273203aab7344abd03df4d86e4c5f5b9a394a.json new file mode 100644 index 0000000000..a761d91385 --- /dev/null +++ b/backend/.sqlx/query-29fce7e56bb4a874b4c1e3a6454273203aab7344abd03df4d86e4c5f5b9a394a.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT value FROM app_version WHERE id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "value", + "type_info": "Json" + } + ], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [ + false + ] + }, + "hash": "29fce7e56bb4a874b4c1e3a6454273203aab7344abd03df4d86e4c5f5b9a394a" +} diff --git a/backend/.sqlx/query-637ead7701775609c772c9985450273b942202cdf9017ce6b5ba612e66ce3875.json b/backend/.sqlx/query-637ead7701775609c772c9985450273b942202cdf9017ce6b5ba612e66ce3875.json index d79dd20110..9820f0e0f4 100644 --- a/backend/.sqlx/query-637ead7701775609c772c9985450273b942202cdf9017ce6b5ba612e66ce3875.json +++ b/backend/.sqlx/query-637ead7701775609c772c9985450273b942202cdf9017ce6b5ba612e66ce3875.json @@ -17,6 +17,7 @@ "bash", "postgresql", "nativets", + "Nativets", "bun", "mysql", "bigquery", diff --git a/backend/.sqlx/query-6dbf0275e52d937a114ba305c6646d313f6ef05062ac55f7397caf8c02ccec5e.json b/backend/.sqlx/query-6dbf0275e52d937a114ba305c6646d313f6ef05062ac55f7397caf8c02ccec5e.json index 5eeddb80bb..9344caf4c3 100644 --- a/backend/.sqlx/query-6dbf0275e52d937a114ba305c6646d313f6ef05062ac55f7397caf8c02ccec5e.json +++ b/backend/.sqlx/query-6dbf0275e52d937a114ba305c6646d313f6ef05062ac55f7397caf8c02ccec5e.json @@ -22,7 +22,8 @@ "http", "graphql", "postgresql", - "noop" + "noop", + "appdependencies" ] } } diff --git a/backend/.sqlx/query-6fb904692513c453989ffedb14b1eb737da453e9fcac6cfa098296d6c211d2a9.json b/backend/.sqlx/query-6fb904692513c453989ffedb14b1eb737da453e9fcac6cfa098296d6c211d2a9.json new file mode 100644 index 0000000000..1f15905eda --- /dev/null +++ b/backend/.sqlx/query-6fb904692513c453989ffedb14b1eb737da453e9fcac6cfa098296d6c211d2a9.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE app_version SET value = $1 WHERE id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Json", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "6fb904692513c453989ffedb14b1eb737da453e9fcac6cfa098296d6c211d2a9" +} diff --git a/backend/.sqlx/query-9ee1423945740c17dedc2fc7ea7e7183380c43d0feeb615c651aa486eef3bd53.json b/backend/.sqlx/query-9ee1423945740c17dedc2fc7ea7e7183380c43d0feeb615c651aa486eef3bd53.json index db3d7ee568..cd3ae03d9f 100644 --- a/backend/.sqlx/query-9ee1423945740c17dedc2fc7ea7e7183380c43d0feeb615c651aa486eef3bd53.json +++ b/backend/.sqlx/query-9ee1423945740c17dedc2fc7ea7e7183380c43d0feeb615c651aa486eef3bd53.json @@ -28,6 +28,7 @@ "bash", "postgresql", "nativets", + "Nativets", "bun", "mysql", "bigquery", diff --git a/backend/.sqlx/query-fc2ec6fc4e22e46cc35b00503eff1eac35a6fdbc82a634b7eb9da4d7931f3582.json b/backend/.sqlx/query-fc2ec6fc4e22e46cc35b00503eff1eac35a6fdbc82a634b7eb9da4d7931f3582.json index e4a88df6c1..145dd54e3d 100644 --- a/backend/.sqlx/query-fc2ec6fc4e22e46cc35b00503eff1eac35a6fdbc82a634b7eb9da4d7931f3582.json +++ b/backend/.sqlx/query-fc2ec6fc4e22e46cc35b00503eff1eac35a6fdbc82a634b7eb9da4d7931f3582.json @@ -38,7 +38,8 @@ "http", "graphql", "postgresql", - "noop" + "noop", + "appdependencies" ] } } @@ -59,7 +60,8 @@ "http", "graphql", "postgresql", - "noop" + "noop", + "appdependencies" ] } } diff --git a/backend/migrations/20230826001320_appdependencies.down.sql b/backend/migrations/20230826001320_appdependencies.down.sql new file mode 100644 index 0000000000..d2f607c5b8 --- /dev/null +++ b/backend/migrations/20230826001320_appdependencies.down.sql @@ -0,0 +1 @@ +-- Add down migration script here diff --git a/backend/migrations/20230826001320_appdependencies.up.sql b/backend/migrations/20230826001320_appdependencies.up.sql new file mode 100644 index 0000000000..510994806a --- /dev/null +++ b/backend/migrations/20230826001320_appdependencies.up.sql @@ -0,0 +1,2 @@ +-- Add up migration script here +ALTER TYPE JOB_KIND ADD VALUE IF NOT EXISTS 'appdependencies'; \ No newline at end of file diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index f414ddb0c6..457ac816bf 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1873,7 +1873,8 @@ paths: /w/{workspace}/resources/get_value_interpolated/{path}: get: - summary: get resource interpolated (variables and resources are fully unrolled) + summary: + get resource interpolated (variables and resources are fully unrolled) operationId: getResourceValueInterpolated tags: - resource @@ -2328,7 +2329,8 @@ paths: schema: type: string - name: first_parent_hash - description: mask to filter scripts whom first direct parent has exact hash + description: + mask to filter scripts whom first direct parent has exact hash in: query schema: type: string @@ -2496,7 +2498,8 @@ paths: /workers/custom_tags: get: - summary: get all instance custom tags (tags are used to dispatch jobs to + summary: + get all instance custom tags (tags are used to dispatch jobs to different worker groups) operationId: getCustomTags tags: @@ -2547,7 +2550,8 @@ paths: /w/{workspace}/scripts/delete/h/{hash}: post: - summary: delete script by hash (erase content but keep hash, require admin) + summary: + delete script by hash (erase content but keep hash, require admin) operationId: deleteScriptByHash tags: - script @@ -2739,14 +2743,16 @@ paths: type: string format: date-time - name: scheduled_in_secs - description: schedule the script to execute in the number of seconds starting now + description: + schedule the script to execute in the number of seconds starting now in: query schema: type: integer - $ref: "#/components/parameters/ParentJob" - $ref: "#/components/parameters/NewJobId" - name: invisible_to_owner - description: make the run invisible to the the script owner (default false) + description: + make the run invisible to the the script owner (default false) in: query schema: type: boolean @@ -3624,7 +3630,8 @@ paths: type: string format: date-time - name: scheduled_in_secs - description: schedule the script to execute in the number of seconds starting now + description: + schedule the script to execute in the number of seconds starting now in: query schema: type: integer @@ -3632,7 +3639,8 @@ paths: - $ref: "#/components/parameters/NewJobId" - $ref: "#/components/parameters/IncludeHeader" - name: invisible_to_owner - description: make the run invisible to the the flow owner (default false) + description: + make the run invisible to the the flow owner (default false) in: query schema: type: boolean @@ -3670,7 +3678,8 @@ paths: type: string format: date-time - name: scheduled_in_secs - description: schedule the script to execute in the number of seconds starting now + description: + schedule the script to execute in the number of seconds starting now in: query schema: type: integer @@ -3678,7 +3687,8 @@ paths: - $ref: "#/components/parameters/NewJobId" - $ref: "#/components/parameters/IncludeHeader" - name: invisible_to_owner - description: make the run invisible to the the script owner (default false) + description: + make the run invisible to the the script owner (default false) in: query schema: type: boolean @@ -3709,7 +3719,8 @@ paths: - $ref: "#/components/parameters/WorkspaceId" - $ref: "#/components/parameters/IncludeHeader" - name: invisible_to_owner - description: make the run invisible to the the script owner (default false) + description: + make the run invisible to the the script owner (default false) in: query schema: type: boolean @@ -3742,7 +3753,8 @@ paths: - $ref: "#/components/parameters/WorkspaceId" - $ref: "#/components/parameters/IncludeHeader" - name: invisible_to_owner - description: make the run invisible to the the script owner (default false) + description: + make the run invisible to the the script owner (default false) in: query schema: type: boolean @@ -3935,7 +3947,8 @@ paths: - job responses: "200": - description: the timestamp of the db that can be used to compute the drift + description: + the timestamp of the db that can be used to compute the drift content: application/json: schema: @@ -4168,7 +4181,8 @@ paths: /w/{workspace}/jobs/resume_urls/{id}/{resume_id}: get: - summary: get resume urls given a job_id, resume_id and a nonce to resume a flow + summary: + get resume urls given a job_id, resume_id and a nonce to resume a flow operationId: getResumeUrls tags: - job @@ -4681,7 +4695,8 @@ paths: - $ref: "#/components/parameters/WorkspaceId" - name: only_member_of in: query - description: only list the groups the user is member of (default false) + description: + only list the groups the user is member of (default false) schema: type: boolean responses: @@ -4869,7 +4884,8 @@ paths: - $ref: "#/components/parameters/WorkspaceId" - name: only_member_of in: query - description: only list the folders the user is member of (default false) + description: + only list the folders the user is member of (default false) schema: type: boolean responses: @@ -5507,7 +5523,8 @@ components: type: integer PerPage: name: per_page - description: number of items to return for a given page (default 30, max 100) + description: + number of items to return for a given page (default 30, max 100) in: query schema: type: integer @@ -5641,7 +5658,8 @@ components: type: boolean ArgsFilter: name: args - description: filter on jobs containing those args as a json subset (@> in postgres) + description: + filter on jobs containing those args as a json subset (@> in postgres) in: query schema: type: string @@ -5653,7 +5671,8 @@ components: type: string ResultFilter: name: result - description: filter on jobs containing those result as a json subset (@> in postgres) + description: + filter on jobs containing those result as a json subset (@> in postgres) in: query schema: type: string @@ -6016,6 +6035,7 @@ components: "preview", "dependencies", "flowdependencies", + "appdependencies", "flow", "flowpreview", "script_hub", @@ -6121,6 +6141,7 @@ components: "dependencies", "flow", "flowdependencies", + "appdependencies", "flowpreview", "script_hub", "identity", diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs index 75811f039f..7fec95571a 100644 --- a/backend/windmill-api/src/apps.rs +++ b/backend/windmill-api/src/apps.rs @@ -38,7 +38,7 @@ use windmill_common::{ http_get_from_hub, list_elems_from_hub, not_found_if_none, paginate, Pagination, StripPath, }, }; -use windmill_queue::push; +use windmill_queue::{push, PushIsolationLevel, QueueTransaction}; pub fn workspaced_service() -> Router { Router::new() @@ -372,14 +372,16 @@ async fn get_secret_id( async fn create_app( authed: ApiAuthed, Extension(user_db): Extension, + Extension(db): Extension, + Extension(rsmq): Extension>, Extension(webhook): Extension, Path(w_id): Path, Json(mut app): Json, ) -> Result<(StatusCode, String)> { - let mut tx = user_db.begin(&authed).await?; + let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.clone().begin(&authed).await?).into(); app.policy.on_behalf_of = Some(username_to_permissioned_as(&authed.username)); - app.policy.on_behalf_of_email = Some(authed.email); + app.policy.on_behalf_of_email = Some(authed.email.clone()); if &app.path == "" { return Err(Error::BadRequest("App path cannot be empty".to_string())); @@ -390,7 +392,7 @@ async fn create_app( &app.path, w_id ) - .fetch_one(&mut *tx) + .fetch_one(&mut tx) .await? .unwrap_or(false); @@ -406,7 +408,7 @@ async fn create_app( &app.path, &w_id ) - .execute(&mut *tx) + .execute(&mut tx) .await?; let id = sqlx::query_scalar!( @@ -419,7 +421,7 @@ async fn create_app( json!(app.policy), app.draft_only, ) - .fetch_one(&mut *tx) + .fetch_one(&mut tx) .await?; let v_id = sqlx::query_scalar!( @@ -431,7 +433,7 @@ async fn create_app( serde_json::to_string(&app.value).unwrap(), authed.username, ) - .fetch_one(&mut *tx) + .fetch_one(&mut tx) .await?; sqlx::query!( @@ -439,11 +441,11 @@ async fn create_app( v_id, id ) - .execute(&mut *tx) + .execute(&mut tx) .await?; audit_log( - &mut *tx, + &mut tx, &authed.username, "apps.create", ActionKind::Create, @@ -453,6 +455,31 @@ async fn create_app( ) .await?; + let tx = PushIsolationLevel::Transaction(tx); + let (dependency_job_uuid, tx) = push( + &db, + tx, + &w_id, + JobPayload::AppDependencies { path: app.path.clone(), version: v_id }, + serde_json::Map::new(), + &authed.username, + &authed.email, + windmill_common::users::username_to_permissioned_as(&authed.username), + None, + None, + None, + None, + None, + false, + false, + None, + true, + None, + None, + None, + ) + .await?; + tracing::info!("Pushed app dependency job {}", dependency_job_uuid); tx.commit().await?; webhook.send_message( w_id.clone(), @@ -541,8 +568,10 @@ async fn delete_app( async fn update_app( authed: ApiAuthed, + Extension(db): Extension, Extension(user_db): Extension, Extension(webhook): Extension, + Extension(rsmq): Extension>, Path((w_id, path)): Path<(String, StripPath)>, Json(ns): Json, ) -> Result { @@ -550,7 +579,7 @@ async fn update_app( let path = path.to_path(); - let mut tx = user_db.begin(&authed).await?; + let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.clone().begin(&authed).await?).into(); let npath = if ns.policy.is_some() || ns.path.is_some() || ns.summary.is_some() { let mut sqlb = SqlBuilder::update_table("app"); @@ -567,7 +596,7 @@ async fn update_app( npath, w_id ) - .fetch_one(&mut *tx) + .fetch_one(&mut tx) .await? .unwrap_or(false); @@ -587,7 +616,7 @@ async fn update_app( if let Some(mut npolicy) = ns.policy { npolicy.on_behalf_of = Some(username_to_permissioned_as(&authed.username)); - npolicy.on_behalf_of_email = Some(authed.email); + npolicy.on_behalf_of_email = Some(authed.email.clone()); sqlb.set( "policy", &format!( @@ -602,18 +631,18 @@ async fn update_app( sqlb.returning("path"); let sql = sqlb.sql().map_err(|e| Error::InternalErr(e.to_string()))?; - let npath_o: Option = sqlx::query_scalar(&sql).fetch_optional(&mut *tx).await?; + let npath_o: Option = sqlx::query_scalar(&sql).fetch_optional(&mut tx).await?; not_found_if_none(npath_o, "App", path)? } else { "".to_string() }; - if let Some(nvalue) = &ns.value { + let v_id = if let Some(nvalue) = &ns.value { let app_id = sqlx::query_scalar!( "SELECT id FROM app WHERE path = $1 AND workspace_id = $2", npath, w_id ) - .fetch_one(&mut *tx) + .fetch_one(&mut tx) .await?; let v_id = sqlx::query_scalar!( @@ -625,7 +654,7 @@ async fn update_app( serde_json::to_string(&nvalue).unwrap(), authed.username, ) - .fetch_one(&mut *tx) + .fetch_one(&mut tx) .await?; sqlx::query!( @@ -634,20 +663,23 @@ async fn update_app( npath, w_id ) - .execute(&mut *tx) + .execute(&mut tx) .await?; - } + Some(v_id) + } else { + None + }; sqlx::query!( "DELETE FROM draft WHERE path = $1 AND workspace_id = $2 AND typ = 'app'", path, &w_id ) - .execute(&mut *tx) + .execute(&mut tx) .await?; audit_log( - &mut *tx, + &mut tx, &authed.username, "apps.update", ActionKind::Update, @@ -656,7 +688,39 @@ async fn update_app( None, ) .await?; - tx.commit().await?; + + if let Some(v_id) = v_id { + let tx: PushIsolationLevel<'_, rsmq_async::MultiplexedRsmq> = + PushIsolationLevel::Transaction(tx); + + let (dependency_job_uuid, tx) = push( + &db, + tx, + &w_id, + JobPayload::AppDependencies { path: npath.clone(), version: v_id }, + serde_json::Map::new(), + &authed.username, + &authed.email, + windmill_common::users::username_to_permissioned_as(&authed.username), + None, + None, + None, + None, + None, + false, + false, + None, + true, + None, + None, + None, + ) + .await?; + tracing::info!("Pushed app dependency job {}", dependency_job_uuid); + tx.commit().await?; + } else { + tx.commit().await?; + } webhook.send_message( w_id.clone(), WebhookMessage::UpdateApp { diff --git a/backend/windmill-api/src/flows.rs b/backend/windmill-api/src/flows.rs index af71698a44..8db9b17796 100644 --- a/backend/windmill-api/src/flows.rs +++ b/backend/windmill-api/src/flows.rs @@ -244,11 +244,6 @@ async fn create_flow( ) .await?; - webhook.send_message( - w_id.clone(), - WebhookMessage::CreateFlow { workspace: w_id.clone(), path: nf.path.clone() }, - ); - let tx = PushIsolationLevel::Transaction(tx); let (dependency_job_uuid, mut tx) = push( &db, @@ -284,6 +279,11 @@ async fn create_flow( .await?; tx.commit().await?; + webhook.send_message( + w_id.clone(), + WebhookMessage::CreateFlow { workspace: w_id.clone(), path: nf.path.clone() }, + ); + Ok((StatusCode::CREATED, nf.path.to_string())) } diff --git a/backend/windmill-audit/src/lib.rs b/backend/windmill-audit/src/lib.rs index ac98439eb8..978f7cc731 100644 --- a/backend/windmill-audit/src/lib.rs +++ b/backend/windmill-audit/src/lib.rs @@ -59,6 +59,9 @@ pub async fn audit_log<'c, E: sqlx::Executor<'c, Database = Postgres>>( #[cfg(not(feature = "enterprise"))] let _resource: Option<&str> = Some("EE only"); + #[cfg(not(feature = "enterprise"))] + let username: &str = "redacted"; + tracing::info!( operation = operation, action_kind = ?action_kind, diff --git a/backend/windmill-common/src/jobs.rs b/backend/windmill-common/src/jobs.rs index 9f9c531781..3206e241c0 100644 --- a/backend/windmill-common/src/jobs.rs +++ b/backend/windmill-common/src/jobs.rs @@ -23,6 +23,7 @@ pub enum JobKind { FlowPreview, Identity, FlowDependencies, + AppDependencies, Noop, } @@ -180,6 +181,10 @@ pub enum JobPayload { FlowDependencies { path: String, }, + AppDependencies { + path: String, + version: i64, + }, Flow(String), RawFlow { value: FlowValue, diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index 3b4c256aa2..1cb197248b 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -1359,6 +1359,7 @@ pub async fn push<'c, R: rsmq_async::RsmqConnection + Send + 'c>( if usage > MAX_FREE_EXECS && !matches!(job_payload, JobPayload::Dependencies { .. }) && !matches!(job_payload, JobPayload::FlowDependencies { .. }) + && !matches!(job_payload, JobPayload::AppDependencies { .. }) { return Err(error::Error::BadRequest(format!( "User {email} has exceeded the free usage limit of {MAX_FREE_EXECS} that applies outside of premium workspaces." @@ -1492,6 +1493,16 @@ pub async fn push<'c, R: rsmq_async::RsmqConnection + Send + 'c>( None, ) } + JobPayload::AppDependencies { path, version } => ( + Some(version), + Some(path), + None, + JobKind::AppDependencies, + None, + None, + None, + None, + ), JobPayload::RawFlow { value, path } => ( None, path, @@ -1700,6 +1711,7 @@ pub async fn push<'c, R: rsmq_async::RsmqConnection + Send + 'c>( JobKind::Identity => "jobs.run.identity", JobKind::Noop => "jobs.run.noop", JobKind::FlowDependencies => "jobs.run.flow_dependencies", + JobKind::AppDependencies => "jobs.run.app_dependencies", }; audit_log( diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 27d6f40aca..fe7b50a05d 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -1079,6 +1079,11 @@ async fn handle_queued_job( handle_flow_dependency_job(&job, &mut logs, job_dir, db, worker_name, worker_dir, base_internal_url, &client.get_token().await) .await .map(|()| Value::Null) + }, + JobKind::AppDependencies => { + handle_app_dependency_job(&job, &mut logs, job_dir, db, worker_name, worker_dir, base_internal_url, &client.get_token().await) + .await + .map(|()| Value::Null) } JobKind::Identity => match job.args.clone() { Some(Value::Object(args)) @@ -1540,7 +1545,6 @@ async fn handle_dependency_job( } } -#[tracing::instrument(level = "trace", skip_all)] async fn handle_flow_dependency_job( job: &QueuedJob, logs: &mut String, @@ -1563,7 +1567,7 @@ async fn handle_flow_dependency_job( })?; let mut flow = serde_json::from_value::(raw_flow).map_err(to_anyhow)?; - flow.modules = lock_modules(flow.modules, job, logs, job_dir, db, worker_name, worker_dir, job_path.clone(), base_internal_url, token).await?; + flow.modules = lock_modules(flow.modules, job, logs, job_dir, db, worker_name, worker_dir, &job_path, base_internal_url, token).await?; let new_flow_value = serde_json::to_value(flow).map_err(to_anyhow)?; // Re-check cancelation to ensure we don't accidentially override a flow. @@ -1599,7 +1603,7 @@ async fn lock_modules( db: &sqlx::Pool, worker_name: &str, worker_dir: &str, - job_path: String, + job_path: &str, base_internal_url: &str, token: &str) -> Result> { let mut new_flow_modules = Vec::new(); @@ -1635,7 +1639,7 @@ async fn lock_modules( }; // sync with windmill-api/scripts let dependencies = match language { - ScriptLang::Python3 => windmill_parser_py_imports::parse_python_imports(&content, &job.workspace_id, &path.clone().unwrap_or_else(|| job_path.clone()), &db).await?.join("\n"), + ScriptLang::Python3 => windmill_parser_py_imports::parse_python_imports(&content, &job.workspace_id, &path.clone().unwrap_or_else(|| job_path.to_string()), &db).await?.join("\n"), _ => content.clone(), }; let new_lock = capture_dependency_job( @@ -1694,6 +1698,142 @@ async fn lock_modules( } Ok(new_flow_modules) } + +#[async_recursion] +async fn lock_modules_app( + value: Value, + job: &QueuedJob, + logs: &mut String, + job_dir: &str, + db: &sqlx::Pool, + worker_name: &str, + worker_dir: &str, + job_path: &str, + base_internal_url: &str, + token: &str) -> Result { + match value { + Value::Object(mut m) => { + if m.contains_key("inlineScript") { + let v = m.get_mut("inlineScript").unwrap(); + if let Some(v) = v.as_object_mut() { + if v.contains_key("content") && v.contains_key("language") { + if let Ok(language) = serde_json::from_value::(v.get("language").unwrap().clone()) { + let content = v.get("content").unwrap().as_str().unwrap_or_default().to_string(); + let dependencies = match language { + ScriptLang::Python3 => windmill_parser_py_imports::parse_python_imports(&content, &job.workspace_id, job_path, &db).await?.join("\n"), + _ => content.clone(), + }; + logs.push_str("Found lockable inline script. Generating lock...\n"); + let new_lock = capture_dependency_job( + &job.id, + &language, + &dependencies, + logs, + job_dir, + db, + worker_name, + &job.workspace_id, + worker_dir, + base_internal_url, + token, + job.script_path() + ) + .await; + match new_lock { + Ok(new_lock) => { + v.insert("lock".to_string(), serde_json::Value::String(new_lock)); + return Ok(Value::Object(m.clone())) + } + Err(e) => { + tracing::warn!( + language = ?language, + error = ?e, + logs = ?logs, + "Failed to generate flow lock for inline script" + ); + () + } + } + } + } + } + } + for (a, b) in m.clone().into_iter() { + m.insert( + a.clone(), + lock_modules_app(b, job, logs, job_dir, db, worker_name, worker_dir, job_path, base_internal_url, token).await?, + ); + } + Ok(Value::Object(m)) + }, + Value::Array(a) => { + let mut nv = vec![]; + for b in a.clone().into_iter() { + nv.push(lock_modules_app(b, job, logs, job_dir, db, worker_name, worker_dir, job_path, base_internal_url, token).await?); + } + Ok(Value::Array(nv)) + }, + a @ _ => Ok(a), + } +} + + +async fn handle_app_dependency_job( + job: &QueuedJob, + logs: &mut String, + job_dir: &str, + db: &sqlx::Pool, + worker_name: &str, + worker_dir: &str, + base_internal_url: &str, + token: &str, +) -> error::Result<()> { + + let job_path = job.script_path.clone().ok_or_else(|| { + error::Error::InternalErr( + "Cannot resolve flow dependencies for flow without path".to_string(), + ) + })?; + + let id = job.script_hash.clone().ok_or_else(|| { + Error::InternalErr( + "Flow Dependency requires script hash".to_owned(), + )})?.0; + let value = sqlx::query_scalar!("SELECT value FROM app_version WHERE id = $1", id) + .fetch_optional(db) + .await?; + + if let Some(value) = value { + let value = lock_modules_app(value, job, logs, job_dir, db, worker_name, worker_dir, &job_path, base_internal_url, token).await?; + + + // Re-check cancelation to ensure we don't accidentially override a flow. + if sqlx::query_scalar!("SELECT canceled FROM queue WHERE id = $1", job.id) + .fetch_optional(db) + .await + .map(|v| Some(true) == v) + .unwrap_or_else(|err| { + tracing::error!(%job.id, %err, "error checking cancelation for job {0}: {err}", job.id); + false + }) + { + return Ok(()); + } + + sqlx::query!( + "UPDATE app_version SET value = $1 WHERE id = $2", + value, + id, + ) + .execute(db) + .await?; + Ok(()) + } else { + Ok(()) + } + } + + async fn capture_dependency_job( job_id: &Uuid, job_language: &ScriptLang, diff --git a/frontend/src/lib/components/apps/components/helpers/RunnableComponent.svelte b/frontend/src/lib/components/apps/components/helpers/RunnableComponent.svelte index 6a01d872bd..0607399238 100644 --- a/frontend/src/lib/components/apps/components/helpers/RunnableComponent.svelte +++ b/frontend/src/lib/components/apps/components/helpers/RunnableComponent.svelte @@ -246,7 +246,8 @@ requestBody['raw_code'] = { content: inlineScript.content, language: inlineScript.language, - path: inlineScript.path + path: inlineScript.path, + lock: inlineScript.lock } } } else if (runnable?.type === 'runnableByPath') { diff --git a/frontend/src/lib/components/apps/editor/inlineScriptsPanel/InlineScriptEditor.svelte b/frontend/src/lib/components/apps/editor/inlineScriptsPanel/InlineScriptEditor.svelte index 6ad8848d10..5aab3a9294 100644 --- a/frontend/src/lib/components/apps/editor/inlineScriptsPanel/InlineScriptEditor.svelte +++ b/frontend/src/lib/components/apps/editor/inlineScriptsPanel/InlineScriptEditor.svelte @@ -262,6 +262,9 @@ }} on:change={async (e) => { if (inlineScript && inlineScript.language != 'frontend') { + if (inlineScript.lock) { + inlineScript.lock = undefined + } const oldSchema = JSON.stringify(inlineScript.schema) if (inlineScript.schema == undefined) { inlineScript.schema = emptySchema() diff --git a/frontend/src/lib/components/apps/editor/settingsPanel/ComponentPanel.svelte b/frontend/src/lib/components/apps/editor/settingsPanel/ComponentPanel.svelte index fade5ed593..a662696d64 100644 --- a/frontend/src/lib/components/apps/editor/settingsPanel/ComponentPanel.svelte +++ b/frontend/src/lib/components/apps/editor/settingsPanel/ComponentPanel.svelte @@ -297,21 +297,6 @@ {/key} {/if} - {#if Object.values(initialConfiguration).length > 0} - - - - {:else} -
{ccomponents[component.type].name} has no configuration
- {/if} - {#if componentSettings.item.data.type === 'tabscomponent'} {/if} + {#if Object.values(initialConfiguration).length > 0} + + + + {:else} +
{ccomponents[component.type].name} has no configuration
+ {/if} + {#if (`recomputeIds` in componentSettings.item.data && Array.isArray(componentSettings.item.data.recomputeIds)) || componentSettings.item.data.type === 'buttoncomponent' || componentSettings.item.data.type === 'formcomponent' || componentSettings.item.data.type === 'formbuttoncomponent' || componentSettings.item.data.type === 'checkboxcomponent'} - + {#if components.length == 0} No action buttons {/if} diff --git a/frontend/src/lib/components/apps/types.ts b/frontend/src/lib/components/apps/types.ts index 89f87dfa27..180b074bd3 100644 --- a/frontend/src/lib/components/apps/types.ts +++ b/frontend/src/lib/components/apps/types.ts @@ -106,6 +106,7 @@ export type InlineScript = { language: Preview.language | 'frontend' path?: string schema?: Schema + lock?: String refreshOn?: { id: string; key: string }[] suggestedRefreshOn?: { id: string; key: string }[] } diff --git a/frontend/src/lib/components/auditLogs/AuditLogsFilters.svelte b/frontend/src/lib/components/auditLogs/AuditLogsFilters.svelte index 57b08cdc88..4bdc44f3e6 100644 --- a/frontend/src/lib/components/auditLogs/AuditLogsFilters.svelte +++ b/frontend/src/lib/components/auditLogs/AuditLogsFilters.svelte @@ -177,6 +177,7 @@ JOBS_RUN_IDENTITY: 'jobs.run.identity', JOBS_RUN_NOOP: 'jobs.run.noop', JOBS_FLOW_DEPENDENCIES: 'jobs.flow_dependencies', + JOBS_APP_DEPENDENCIES: 'jobs.app_dependencies', JOBS: 'jobs', JOBS_CANCEL: 'jobs.cancel', JOBS_FORCE_CANCEL: 'jobs.force_cancel', diff --git a/frontend/src/routes/(root)/(logged)/runs/[...path]/+page.svelte b/frontend/src/routes/(root)/(logged)/runs/[...path]/+page.svelte index f99a93db45..7ddcfb8047 100644 --- a/frontend/src/routes/(root)/(logged)/runs/[...path]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/runs/[...path]/+page.svelte @@ -53,9 +53,9 @@ function computeJobKinds(jobKindsCat: string | undefined): string { if (jobKindsCat == 'all') { - return `${CompletedJob.job_kind.SCRIPT},${CompletedJob.job_kind.FLOW},${CompletedJob.job_kind.DEPENDENCIES},${CompletedJob.job_kind.FLOWDEPENDENCIES},${CompletedJob.job_kind.PREVIEW},${CompletedJob.job_kind.FLOWPREVIEW},${CompletedJob.job_kind.SCRIPT_HUB}` + return `${CompletedJob.job_kind.SCRIPT},${CompletedJob.job_kind.FLOW},${CompletedJob.job_kind.DEPENDENCIES},${CompletedJob.job_kind.FLOWDEPENDENCIES},${CompletedJob.job_kind.APPDEPENDENCIES},${CompletedJob.job_kind.PREVIEW},${CompletedJob.job_kind.FLOWPREVIEW},${CompletedJob.job_kind.SCRIPT_HUB}` } else if (jobKindsCat == 'dependencies') { - return `${CompletedJob.job_kind.DEPENDENCIES},${CompletedJob.job_kind.FLOWDEPENDENCIES}` + return `${CompletedJob.job_kind.DEPENDENCIES},${CompletedJob.job_kind.FLOWDEPENDENCIES},${CompletedJob.job_kind.APPDEPENDENCIES}` } else if (jobKindsCat == 'previews') { return `${CompletedJob.job_kind.PREVIEW},${CompletedJob.job_kind.FLOWPREVIEW}` } else {