kalugin66/windmill
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Add possibility to delete flow step results when the flow is complete (#2806)

Browse Source 
* feat: Add possibility to delete flow step results when the flow is complete

* Add third layer of tabs and gate feature to EE
This commit is contained in:
Guillaume Bouvignies
2023-12-08 21:30:44 +01:00
committed by GitHub
parent b0894492f1
commit b65657d0f8
44 changed files with 733 additions and 420 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
backend/windmill-api/src/flows.rs
View File

@@ -863,6 +863,7 @@ mod tests {
mock: None,
timeout: None,
priority: None,
delete_after_use: None,
},
FlowModule {
id: "b".to_string(),
@@ -888,6 +889,7 @@ mod tests {
mock: None,
timeout: None,
priority: None,
delete_after_use: None,
},
FlowModule {
id: "c".to_string(),
@@ -910,6 +912,7 @@ mod tests {
mock: None,
timeout: None,
priority: None,
delete_after_use: None,
},
],
failure_module: Some(FlowModule {
@@ -931,6 +934,7 @@ mod tests {
mock: None,
timeout: None,
priority: None,
delete_after_use: None,
}),
same_worker: false,
concurrent_limit: None,

46
backend/windmill-api/src/jobs.rs
View File

@@ -329,9 +329,10 @@ pub async fn get_path_tag_limits_cache_for_hash(
ScriptLang,
Option<bool>,
Option<i16>,
Option<bool>,
)> {
let script = sqlx::query!(
"select path, tag, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority from script where hash = $1 AND workspace_id = $2",
"select path, tag, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use from script where hash = $1 AND workspace_id = $2",
hash,
w_id
)
@@ -351,6 +352,7 @@ pub async fn get_path_tag_limits_cache_for_hash(
script.language,
script.dedicated_worker,
script.priority,
script.delete_after_use,
))
}
@@ -1812,7 +1814,8 @@ pub async fn run_job_by_path(
check_scopes(&authed, || format!("run:script/{script_path}"))?;
let (job_payload, tag) = script_path_to_payload(script_path, &db, &w_id).await?;
let (job_payload, tag, _delete_after_use) =
script_path_to_payload(script_path, &db, &w_id).await?;
let scheduled_for = run_query.get_scheduled_for(&db).await?;
check_tag_available_for_workspace(&w_id, &tag).await?;
@@ -1997,6 +2000,18 @@ async fn run_wait_result<T>(
}
}
async fn delete_job_metadata_after_use(db: &DB, job_uuid: Uuid) -> Result<(), Error> {
sqlx::query!(
"UPDATE completed_job
SET logs = '##DELETED##', args = '{}'::jsonb, result = '{}'::jsonb
WHERE id = $1",
job_uuid,
)
.execute(db)
.await?;
Ok(())
}
pub async fn check_queue_too_long(db: &DB, queue_limit: Option<i64>) -> error::Result<()> {
if let Some(limit) = queue_limit {
let count = sqlx::query_scalar!(
@@ -2073,7 +2088,8 @@ pub async fn run_wait_result_job_by_path_get(
let script_path = script_path.to_path();
check_scopes(&authed, || format!("run:script/{script_path}"))?;
let (job_payload, tag) = script_path_to_payload(script_path, &db, &w_id).await?;
let (job_payload, tag, delete_after_use) =
script_path_to_payload(script_path, &db, &w_id).await?;
check_tag_available_for_workspace(&w_id, &tag).await?;
let tx = PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into(), rsmq);
@@ -2103,7 +2119,11 @@ pub async fn run_wait_result_job_by_path_get(
.await?;
tx.commit().await?;
run_wait_result(&db, uuid, Path((w_id, script_path)), None).await
let wait_result = run_wait_result(&db, uuid, Path((w_id, script_path)), None).await;
if delete_after_use.unwrap_or(false) {
delete_job_metadata_after_use(&db, uuid).await?;
}
return wait_result;
}
pub async fn run_wait_result_flow_by_path_get(
@@ -2185,7 +2205,8 @@ async fn run_wait_result_script_by_path_internal(
let script_path = script_path.to_path();
check_scopes(&authed, || format!("run:script/{script_path}"))?;
let (job_payload, tag) = script_path_to_payload(script_path, &db, &w_id).await?;
let (job_payload, tag, delete_after_use) =
script_path_to_payload(script_path, &db, &w_id).await?;
check_tag_available_for_workspace(&w_id, &tag).await?;
let tx = PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into(), rsmq);
@@ -2216,7 +2237,11 @@ async fn run_wait_result_script_by_path_internal(
.await?;
tx.commit().await?;
run_wait_result(&db, uuid, Path((w_id, script_path)), None).await
let wait_result = run_wait_result(&db, uuid, Path((w_id, script_path)), None).await;
if delete_after_use.unwrap_or(false) {
delete_job_metadata_after_use(&db, uuid).await?;
}
return wait_result;
}
pub async fn run_wait_result_script_by_hash(
@@ -2243,6 +2268,7 @@ pub async fn run_wait_result_script_by_hash(
language,
dedicated_worker,
priority,
delete_after_use,
) = get_path_tag_limits_cache_for_hash(&db, &w_id, hash).await?;
check_scopes(&authed, || format!("run:script/{path}"))?;
@@ -2284,7 +2310,11 @@ pub async fn run_wait_result_script_by_hash(
.await?;
tx.commit().await?;
run_wait_result(&db, uuid, Path((w_id, script_hash)), None).await
let wait_result = run_wait_result(&db, uuid, Path((w_id, script_hash)), None).await;
if delete_after_use.unwrap_or(false) {
delete_job_metadata_after_use(&db, uuid).await?;
}
return wait_result;
}
pub async fn run_wait_result_flow_by_path(
@@ -2465,6 +2495,7 @@ async fn add_batch_jobs(
language,
dedicated_worker,
_priority,
_delete_after_use,
) = get_latest_deployed_hash_for_path(&db, &w_id, &path).await?;
(
Some(script_hash),
@@ -2656,6 +2687,7 @@ pub async fn run_job_by_hash(
language,
dedicated_worker,
priority,
_delete_after_use, // not taken into account in async endpoints
) = get_path_tag_limits_cache_for_hash(&db, &w_id, hash).await?;
check_scopes(&authed, || format!("run:script/{path}"))?;

1
backend/windmill-api/src/oauth2.rs
View File

@@ -873,6 +873,7 @@ async fn slack_command(
language,
dedicated_worker,
priority,
_delete_after_use,
) = windmill_common::get_latest_deployed_hash_for_path(
&db,
&settings.workspace_id,

11
backend/windmill-api/src/scripts.rs
View File

@@ -84,6 +84,8 @@ pub struct ScriptWDraft {
pub priority: Option<i16>,
#[serde(skip_serializing_if = "Option::is_none")]
pub restart_unless_cancelled: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub delete_after_use: Option<bool>,
}
pub fn global_service() -> Router {
@@ -472,8 +474,8 @@ async fn create_script(
"INSERT INTO script (workspace_id, hash, path, parent_hashes, summary, description, \
content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, \
draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, \
dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled) \
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24)",
dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, delete_after_use) \
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25)",
&w_id,
&hash.0,
ns.path,
@@ -497,7 +499,8 @@ async fn create_script(
ns.dedicated_worker,
ns.ws_error_handler_muted.unwrap_or(false),
ns.priority,
ns.restart_unless_cancelled
ns.restart_unless_cancelled,
ns.delete_after_use,
)
.execute(&mut tx)
.await?;
@@ -716,7 +719,7 @@ async fn get_script_by_path_w_draft(
let mut tx = user_db.begin(&authed).await?;
let script_o = sqlx::query_as::<_, ScriptWDraft>(
"SELECT hash, script.path, summary, description, content, language, kind, tag, schema, draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, ws_error_handler_muted, draft.value as draft, dedicated_worker, priority, restart_unless_cancelled FROM script LEFT JOIN draft ON
"SELECT hash, script.path, summary, description, content, language, kind, tag, schema, draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, ws_error_handler_muted, draft.value as draft, dedicated_worker, priority, restart_unless_cancelled, delete_after_use FROM script LEFT JOIN draft ON
script.path = draft.path AND script.workspace_id = draft.workspace_id AND draft.typ = 'script'
WHERE script.path = $1 AND script.workspace_id = $2 \
AND script.created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND \