/* * Author: Ruben Fiszel * Copyright: Windmill Labs, Inc 2022 * This file and its contents are licensed under the AGPLv3 License. * Please see the included NOTICE for copyright information and * LICENSE-AGPL for a copy of the license. */ use crate::{ db::{ApiAuthed, DB}, schedule::clear_schedule, users::{maybe_refresh_folders, require_owner_of_path, AuthCache}, webhook_util::{WebhookMessage, WebhookShared}, HTTP_CLIENT, }; use axum::{ extract::{Extension, Path, Query}, response::IntoResponse, routing::{get, post}, Json, Router, }; use hyper::StatusCode; use serde::{Deserialize, Serialize}; use serde_json::json; use sql_builder::prelude::*; use sqlx::{FromRow, Postgres, Transaction}; use std::{ collections::{hash_map::DefaultHasher, HashMap}, hash::{Hash, Hasher}, sync::Arc, }; use windmill_audit::audit_ee::audit_log; use windmill_audit::ActionKind; use windmill_common::{ db::UserDB, error::{Error, JsonResult, Result}, jobs::JobPayload, schedule::Schedule, scripts::{ to_i64, HubScript, ListScriptQuery, ListableScript, NewScript, Schema, Script, ScriptHash, ScriptHistory, ScriptHistoryUpdate, ScriptKind, ScriptLang, }, users::username_to_permissioned_as, utils::{ not_found_if_none, paginate, query_elems_from_hub, require_admin, Pagination, StripPath, }, HUB_BASE_URL, }; use windmill_git_sync::{handle_deployment_metadata, DeployedObject}; use windmill_parser_ts::remove_pinned_imports; use windmill_queue::{schedule::push_scheduled_job, PushIsolationLevel, QueueTransaction}; const MAX_HASH_HISTORY_LENGTH_STORED: usize = 20; #[derive(Serialize, sqlx::FromRow)] pub struct ScriptWDraft { pub hash: ScriptHash, pub path: String, pub summary: String, pub description: String, pub content: String, pub language: ScriptLang, pub kind: ScriptKind, pub tag: Option, #[serde(skip_serializing_if = "Option::is_none")] pub draft: Option, pub schema: Option, #[serde(skip_serializing_if = "Option::is_none")] pub draft_only: Option, #[serde(skip_serializing_if = "Option::is_none")] pub envs: Option>, #[serde(skip_serializing_if = "Option::is_none")] pub concurrent_limit: Option, #[serde(skip_serializing_if = "Option::is_none")] pub concurrency_time_window_s: Option, #[serde(skip_serializing_if = "Option::is_none")] pub cache_ttl: Option, #[serde(skip_serializing_if = "Option::is_none")] pub dedicated_worker: Option, #[serde(skip_serializing_if = "Option::is_none")] pub ws_error_handler_muted: Option, #[serde(skip_serializing_if = "Option::is_none")] pub priority: Option, #[serde(skip_serializing_if = "Option::is_none")] pub restart_unless_cancelled: Option, #[serde(skip_serializing_if = "Option::is_none")] pub delete_after_use: Option, #[serde(skip_serializing_if = "Option::is_none")] pub timeout: Option, #[serde(skip_serializing_if = "Option::is_none")] pub concurrency_key: Option, #[serde(skip_serializing_if = "Option::is_none")] pub visible_to_runner_only: Option, } pub fn global_service() -> Router { Router::new() .route("/hub/top", get(get_top_hub_scripts)) .route("/hub/get/*path", get(get_hub_script_by_path)) .route("/hub/get_full/*path", get(get_full_hub_script_by_path)) } pub fn global_unauthed_service() -> Router { Router::new() .route( "/tokened_raw/:workspace/:token/*path", get(get_tokened_raw_script_by_path), ) .route("/empty_ts/*path", get(get_empty_ts_script_by_path)) } pub fn workspaced_service() -> Router { Router::new() .route("/list", get(list_scripts)) .route("/list_search", get(list_search_scripts)) .route("/create", post(create_script)) .route("/archive/p/*path", post(archive_script_by_path)) .route("/get/draft/*path", get(get_script_by_path_w_draft)) .route("/get/p/*path", get(get_script_by_path)) .route("/raw/p/*path", get(raw_script_by_path)) .route("/raw_unpinned/p/*path", get(raw_script_by_path_unpinned)) .route("/exists/p/*path", get(exists_script_by_path)) .route("/archive/h/:hash", post(archive_script_by_hash)) .route("/delete/h/:hash", post(delete_script_by_hash)) .route("/delete/p/*path", post(delete_script_by_path)) .route("/get/h/:hash", get(get_script_by_hash)) .route("/raw/h/:hash", get(raw_script_by_hash)) .route("/deployment_status/h/:hash", get(get_deployment_status)) .route("/list_paths", get(list_paths)) .route( "/toggle_workspace_error_handler/p/*path", post(toggle_workspace_error_handler), ) .route("/history/p/*path", get(get_script_history)) .route( "/history_update/h/:hash/p/*path", post(update_script_history), ) } #[derive(Serialize, FromRow)] pub struct SearchScript { path: String, content: String, } async fn list_search_scripts( authed: ApiAuthed, Path(w_id): Path, Extension(user_db): Extension, ) -> JsonResult> { let mut tx = user_db.begin(&authed).await?; #[cfg(feature = "enterprise")] let n = 1000; #[cfg(not(feature = "enterprise"))] let n = 10; let rows = sqlx::query_as!( SearchScript, "SELECT path, content from script WHERE workspace_id = $1 AND archived = false LIMIT $2", &w_id, n ) .fetch_all(&mut *tx) .await? .into_iter() .collect::>(); tx.commit().await?; Ok(Json(rows)) } async fn list_scripts( authed: ApiAuthed, Extension(user_db): Extension, Path(w_id): Path, Query(pagination): Query, Query(lq): Query, ) -> JsonResult> { let (per_page, offset) = paginate(pagination); let mut sqlb = SqlBuilder::select_from("script as o") .fields(&[ "hash", "o.path", "summary", "COALESCE(draft.created_at, o.created_at) as created_at", "archived", "extra_perms", "CASE WHEN lock_error_logs IS NOT NULL THEN true ELSE false END as has_deploy_errors", "language", "favorite.path IS NOT NULL as starred", "tag", "draft.path IS NOT NULL as has_draft", "draft_only", "ws_error_handler_muted" ]) .left() .join("favorite") .on( "favorite.favorite_kind = 'script' AND favorite.workspace_id = o.workspace_id AND favorite.path = o.path AND favorite.usr = ?" .bind(&authed.username), ) .left() .join("draft") .on( "draft.path = o.path AND draft.workspace_id = o.workspace_id AND draft.typ = 'script'" ) .order_desc("favorite.path IS NOT NULL") .order_by("created_at", lq.order_desc.unwrap_or(true)) .and_where("o.workspace_id = ?".bind(&w_id)) .offset(offset) .limit(per_page) .clone(); if lq.show_archived.unwrap_or(false) { sqlb.and_where_eq( "o.created_at", "(select max(created_at) from script where o.path = path AND workspace_id = ?)" .bind(&w_id), ); sqlb.and_where_eq("archived", true); } else { sqlb.and_where_eq("archived", false); } if let Some(ps) = &lq.path_start { sqlb.and_where_like_left("path", "?".bind(ps)); } if let Some(p) = &lq.path_exact { sqlb.and_where_eq("path", "?".bind(p)); } if let Some(cb) = &lq.created_by { sqlb.and_where_eq("created_by", "?".bind(cb)); } if let Some(ph) = &lq.first_parent_hash { sqlb.and_where_eq("parent_hashes[1]", &ph.0); } if let Some(ph) = &lq.last_parent_hash { sqlb.and_where_eq("parent_hashes[array_upper(parent_hashes, 1)]", &ph.0); } if let Some(ph) = &lq.parent_hash { sqlb.and_where_eq("any(parent_hashes)", &ph.0); } if let Some(it) = &lq.is_template { sqlb.and_where_eq("is_template", it); } if let Some(kinds_val) = &lq.kinds { let lowercased_kinds: Vec = kinds_val .split(",") .map(&str::to_lowercase) .map(sql_builder::quote) .collect(); if lowercased_kinds.len() > 0 { sqlb.and_where_in("kind", lowercased_kinds.as_slice()); } } if lq.starred_only.unwrap_or(false) { sqlb.and_where_is_not_null("favorite.path"); } let sql = sqlb.sql().map_err(|e| Error::InternalErr(e.to_string()))?; let mut tx = user_db.begin(&authed).await?; let rows = sqlx::query_as::<_, ListableScript>(&sql) .fetch_all(&mut *tx) .await?; tx.commit().await?; Ok(Json(rows)) } #[derive(Deserialize)] struct TopHubScriptsQuery { limit: Option, app: Option, kind: Option, } async fn get_top_hub_scripts( Query(query): Query, Extension(db): Extension, ) -> impl IntoResponse { let mut query_params = vec![]; if let Some(query_limit) = query.limit { query_params.push(("limit", query_limit.to_string().clone())); } if let Some(query_app) = query.app { query_params.push(("app", query_app.to_string().clone())); } if let Some(query_kind) = query.kind { query_params.push(("kind", query_kind.to_string().clone())); } let (status_code, headers, response) = query_elems_from_hub( &HTTP_CLIENT, &format!("{}/scripts/top", *HUB_BASE_URL.read().await), Some(query_params), &db, ) .await?; Ok::<_, Error>((status_code, headers, response)) } fn hash_script(ns: &NewScript) -> i64 { let mut dh = DefaultHasher::new(); ns.hash(&mut dh); dh.finish() as i64 } async fn create_script( authed: ApiAuthed, Extension(user_db): Extension, Extension(rsmq): Extension>, Extension(webhook): Extension, Extension(db): Extension, Path(w_id): Path, Json(ns): Json, ) -> Result<(StatusCode, String)> { #[cfg(not(feature = "enterprise"))] if ns.ws_error_handler_muted.is_some_and(|val| val) { return Err(Error::BadRequest( "Muting the error handler for certain script is only available in enterprise version" .to_string(), )); } let script_path = ns.path.clone(); let hash = ScriptHash(hash_script(&ns)); let authed = maybe_refresh_folders(&ns.path, &w_id, authed, &db).await; let mut tx: QueueTransaction<'_, _> = (rsmq.clone(), user_db.begin(&authed).await?).into(); if sqlx::query_scalar!( "SELECT 1 FROM script WHERE hash = $1 AND workspace_id = $2", hash.0, &w_id ) .fetch_optional(&mut tx) .await? .is_some() { return Err(Error::BadRequest( "A script with same hash (hence same path, description, summary, content) already \ exists!" .to_owned(), )); }; let clashing_script = sqlx::query_as::<_, Script>( "SELECT * FROM script WHERE path = $1 AND archived = false AND workspace_id = $2", ) .bind(&ns.path) .bind(&w_id) .fetch_optional(&mut tx) .await?; struct ParentInfo { p_hashes: Vec, perms: serde_json::Value, p_path: String, } let parent_hashes_and_perms: Option = match (&ns.parent_hash, clashing_script) { (None, None) => Ok(None), (None, Some(s)) if !s.draft_only.unwrap_or(false) => Err(Error::BadRequest(format!( "Path conflict for {} with non-archived hash {}", &ns.path, &s.hash ))), (None, Some(s)) => { sqlx::query!( "DELETE FROM script WHERE hash = $1 AND workspace_id = $2", s.hash.0, &w_id ) .execute(&mut tx) .await?; Ok(None) } (Some(p_hash), o) => { if sqlx::query_scalar!( "SELECT 1 FROM script WHERE hash = $1 AND workspace_id = $2", p_hash.0, &w_id ) .fetch_optional(&mut tx) .await? .is_none() { return Err(Error::BadRequest( "The parent hash does not seem to exist".to_owned(), )); }; let clashing_hash_o = sqlx::query_scalar!( "SELECT hash FROM script WHERE parent_hashes[1] = $1 AND workspace_id = $2", p_hash.0, &w_id ) .fetch_optional(&mut tx) .await?; if let Some(clashing_hash) = clashing_hash_o { return Err(Error::BadRequest(format!( "A script with hash {} with same parent_hash has been found. However, the \ lineage must be linear: no 2 scripts can have the same parent", ScriptHash(clashing_hash) ))); }; let ps = get_script_by_hash_internal(tx.transaction_mut(), &w_id, p_hash).await?; if ps.path != ns.path { require_owner_of_path(&authed, &ps.path)?; } let ph = { let v = ps.parent_hashes.map(|x| x.0).unwrap_or_default(); let mut v: Vec = v .into_iter() .take(MAX_HASH_HISTORY_LENGTH_STORED - 1) .collect(); v.insert(0, p_hash.0); v }; let r: Result> = match o { Some(clashing_script) if clashing_script.path == ns.path && clashing_script.hash.0 != p_hash.0 => { Err(Error::BadRequest(format!( "Path conflict for {} with non-archived hash {}", &ns.path, &clashing_script.hash ))) } Some(_) | None => Ok(Some(ParentInfo { p_hashes: ph, perms: ps.extra_perms, p_path: ps.path, })), }; sqlx::query!( "UPDATE script SET archived = true WHERE hash = $1 AND workspace_id = $2", p_hash.0, &w_id ) .execute(&mut tx) .await?; r } }?; let p_hashes = parent_hashes_and_perms.as_ref().map(|v| &v.p_hashes[..]); let extra_perms = parent_hashes_and_perms .as_ref() .map(|v| v.perms.clone()) .unwrap_or(json!({})); let lock = if !(ns.language == ScriptLang::Python3 || ns.language == ScriptLang::Go || ns.language == ScriptLang::Bun || ns.language == ScriptLang::Deno) { Some(String::new()) } else { ns.lock .and_then(|e| if e.is_empty() { None } else { Some(e) }) }; let needs_lock_gen = lock.is_none(); let envs = ns.envs.as_ref().map(|x| x.as_slice()); let envs = if ns.envs.is_none() || ns.envs.as_ref().unwrap().is_empty() { None } else { envs }; //::text::json is to ensure we use serde_json with preserve order sqlx::query!( "INSERT INTO script (workspace_id, hash, path, parent_hashes, summary, description, \ content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, \ draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, \ dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, \ delete_after_use, timeout, concurrency_key, visible_to_runner_only) \ VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28)", &w_id, &hash.0, ns.path, p_hashes, ns.summary, ns.description, &ns.content, &authed.username, ns.schema.and_then(|x| serde_json::to_string(&x.0).ok()), ns.is_template.unwrap_or(false), extra_perms, lock, ns.language.clone() as ScriptLang, ns.kind.unwrap_or(ScriptKind::Script) as ScriptKind, ns.tag, ns.draft_only, envs, ns.concurrent_limit, ns.concurrency_time_window_s, ns.cache_ttl, ns.dedicated_worker, ns.ws_error_handler_muted.unwrap_or(false), ns.priority, ns.restart_unless_cancelled, ns.delete_after_use, ns.timeout, ns.concurrency_key, ns.visible_to_runner_only, ) .execute(&mut tx) .await?; let p_path_opt = parent_hashes_and_perms.as_ref().map(|x| x.p_path.clone()); if let Some(ref p_path) = p_path_opt { sqlx::query!( "DELETE FROM draft WHERE path = $1 AND workspace_id = $2 AND typ = 'script'", p_path, &w_id ) .execute(&mut tx) .await?; let mut schedulables = sqlx::query_as!( Schedule, "UPDATE schedule SET script_path = $1 WHERE script_path = $2 AND path != $2 AND workspace_id = $3 AND is_flow IS false RETURNING *", ns.path, p_path, w_id, ) .fetch_all(&mut tx) .await?; let schedule = sqlx::query_as!(Schedule, "UPDATE schedule SET path = $1, script_path = $1 WHERE path = $2 AND workspace_id = $3 AND is_flow IS false RETURNING *", ns.path, p_path, w_id, ) .fetch_optional(&mut tx) .await?; if let Some(schedule) = schedule { schedulables.push(schedule); } for schedule in schedulables { clear_schedule(tx.transaction_mut(), &schedule.path, &w_id).await?; if schedule.enabled { tx = push_scheduled_job(&db, tx, schedule).await?; } } } else { sqlx::query!( "DELETE FROM draft WHERE path = $1 AND workspace_id = $2 AND typ = 'script'", ns.path, &w_id ) .execute(&mut tx) .await?; } if p_hashes.is_some() && !p_hashes.unwrap().is_empty() { audit_log( &mut tx, &authed.username, "scripts.update", ActionKind::Update, &w_id, Some(&ns.path), Some([("hash", hash.to_string().as_str())].into()), ) .await?; webhook.send_message( w_id.clone(), WebhookMessage::UpdateScript { workspace: w_id.clone(), path: ns.path.clone(), hash: hash.to_string(), }, ); } else { audit_log( &mut tx, &authed.username, "scripts.create", ActionKind::Create, &w_id, Some(&ns.path), Some( [ ("workspace", w_id.as_str()), ("hash", hash.to_string().as_str()), ] .into(), ), ) .await?; webhook.send_message( w_id.clone(), WebhookMessage::CreateScript { workspace: w_id.clone(), path: ns.path.clone(), hash: hash.to_string(), }, ); } let permissioned_as = username_to_permissioned_as(&authed.username); if needs_lock_gen { let tag = if ns.dedicated_worker.is_some_and(|x| x) { Some(format!("{}:{}", &w_id, &ns.path,)) } else if ns.tag.as_ref().is_some_and(|x| x.contains("$args[")) { None } else { ns.tag }; let mut args: HashMap = HashMap::new(); if let Some(dm) = ns.deployment_message { args.insert("deployment_message".to_string(), json!(dm)); } if let Some(ref p_path) = p_path_opt { args.insert("parent_path".to_string(), json!(p_path)); } let tx = PushIsolationLevel::Transaction(tx); let (_, new_tx) = windmill_queue::push( &db, tx, &w_id, JobPayload::Dependencies { hash, language: ns.language, path: ns.path, dedicated_worker: ns.dedicated_worker, }, args, &authed.username, &authed.email, permissioned_as, None, None, None, None, None, false, false, None, true, tag, None, None, None, ) .await?; new_tx.commit().await?; } else { handle_deployment_metadata( &authed.email, &authed.username, &db, &w_id, DeployedObject::Script { hash: hash.clone(), path: script_path.clone(), parent_path: p_path_opt, }, ns.deployment_message, rsmq, false, ) .await?; tx.commit().await?; } Ok((StatusCode::CREATED, format!("{}", hash))) } pub async fn get_hub_script_by_path( Path(path): Path, Extension(db): Extension, ) -> Result { windmill_common::scripts::get_hub_script_by_path(path, &HTTP_CLIENT, &db).await } pub async fn get_full_hub_script_by_path( Path(path): Path, Extension(db): Extension, ) -> JsonResult { Ok(Json( windmill_common::scripts::get_full_hub_script_by_path(path, &HTTP_CLIENT, &db).await?, )) } async fn get_script_by_path( authed: ApiAuthed, Extension(user_db): Extension, Path((w_id, path)): Path<(String, StripPath)>, ) -> JsonResult