From 6d4d8e2d64c61ef3b0a97d6ed4b9386c88ebae5e Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Thu, 20 Oct 2022 20:16:08 +0200 Subject: [PATCH] feat: support running and publishing go, python scripts to the hub (#779) --- .../20221020164416_approval_script.down.sql | 1 + .../20221020164416_approval_script.up.sql | 2 + backend/openapi.yaml | 36 +++++++++++-- backend/src/jobs.rs | 51 ++++++++++--------- backend/src/scripts.rs | 36 +++++++++++++ backend/src/worker.rs | 18 ++++++- .../src/lib/components/ScriptBuilder.svelte | 8 +-- .../components/common/button/Button.svelte | 6 ++- .../flows/content/FlowInputs.svelte | 40 +++++++++++---- .../flows/content/FlowModule.svelte | 10 +++- .../lib/components/flows/flowStateUtils.ts | 7 +-- .../flows/pickers/PickHubScript.svelte | 7 +-- .../flows/pickers/PickScript.svelte | 7 +-- frontend/src/lib/scripts.ts | 12 +++-- frontend/src/lib/utils.ts | 7 +-- .../src/routes/scripts/get/[...hash].svelte | 1 + 16 files changed, 189 insertions(+), 60 deletions(-) create mode 100644 backend/migrations/20221020164416_approval_script.down.sql create mode 100644 backend/migrations/20221020164416_approval_script.up.sql diff --git a/backend/migrations/20221020164416_approval_script.down.sql b/backend/migrations/20221020164416_approval_script.down.sql new file mode 100644 index 0000000000..d2f607c5b8 --- /dev/null +++ b/backend/migrations/20221020164416_approval_script.down.sql @@ -0,0 +1 @@ +-- Add down migration script here diff --git a/backend/migrations/20221020164416_approval_script.up.sql b/backend/migrations/20221020164416_approval_script.up.sql new file mode 100644 index 0000000000..5239ed3a5e --- /dev/null +++ b/backend/migrations/20221020164416_approval_script.up.sql @@ -0,0 +1,2 @@ +-- Add up migration script here +ALTER TYPE SCRIPT_KIND ADD VALUE 'approval'; \ No newline at end of file diff --git a/backend/openapi.yaml b/backend/openapi.yaml index d8c56c90c9..33f92c1eaf 100644 --- a/backend/openapi.yaml +++ b/backend/openapi.yaml @@ -1534,7 +1534,7 @@ paths: type: boolean kind: type: string - enum: [script, failure, trigger, command] + enum: [script, failure, trigger, command, approval] votes: type: number views: @@ -1626,6 +1626,35 @@ paths: schema: type: string + /scripts/hub/get_full/{path}: + get: + summary: get full hub script by path + operationId: getHubScriptByPath + tags: + - script + parameters: + - $ref: "#/components/parameters/ScriptPath" + responses: + "200": + description: script details + content: + application/json: + schema: + type: object + properties: + content: + type: string + lockfile: + type: string + schema: + type: string + language: + type: string + enum: [deno, python3, go] + required: + - content + - language + /w/{workspace}/scripts/list: get: summary: list all available scripts @@ -1745,7 +1774,7 @@ paths: enum: [python3, deno, go] kind: type: string - enum: [script, failure, trigger, command] + enum: [script, failure, trigger, command, approval] required: - path - summary @@ -1908,6 +1937,7 @@ paths: schema: type: string + /w/{workspace}/scripts/exists/p/{path}: get: summary: exists script by path @@ -3404,7 +3434,7 @@ components: enum: [python3, deno, go] kind: type: string - enum: [script, failure, trigger, command] + enum: [script, failure, trigger, command, approval] required: - hash - path diff --git a/backend/src/jobs.rs b/backend/src/jobs.rs index 847732a7c9..b241e8b01a 100644 --- a/backend/src/jobs.rs +++ b/backend/src/jobs.rs @@ -22,7 +22,7 @@ use crate::{ flows::FlowValue, oauth2::HmacSha256, schedule::get_schedule_opt, - scripts::{get_hub_script_by_path, ScriptHash, ScriptLang}, + scripts::{get_full_hub_script_by_path, HubScript, ScriptHash, ScriptLang}, users::{owner_to_token_owner, Authed}, utils::{now_from_db, require_admin, Pagination, StripPath}, variables::get_workspace_key, @@ -1435,31 +1435,14 @@ pub async fn push<'c>( ) .fetch_optional(&mut tx) .await?; + let script = get_hub_script(path.clone(), email, user).await?; ( None, - Some(path.clone()), - Some( - get_hub_script_by_path( - Authed { - email, - username: user.to_string(), - is_admin: false, - groups: vec![], - }, - Path(StripPath(path)), - Extension( - reqwest::ClientBuilder::new() - .user_agent("windmill/beta") - .build() - .map_err(to_anyhow)?, - ), - Host(std::env::var("BASE_URL").unwrap_or_else(|_| "".to_string())), - ) - .await?, - ), + Some(path), + Some(script.content.clone()), JobKind::Script_Hub, None, - Some(ScriptLang::Deno), + Some(script.language.clone()), ) } JobPayload::Code(RawCode { content, path, language }) => ( @@ -1588,6 +1571,26 @@ pub async fn push<'c>( Ok((uuid, tx)) } +pub async fn get_hub_script( + path: String, + email: Option, + user: &str, +) -> error::Result { + get_full_hub_script_by_path( + Authed { email, username: user.to_string(), is_admin: false, groups: vec![] }, + Path(StripPath(path)), + Extension( + reqwest::ClientBuilder::new() + .user_agent("windmill/beta") + .build() + .map_err(to_anyhow)?, + ), + Host(std::env::var("BASE_URL").unwrap_or_else(|_| "".to_string())), + ) + .await + .map(|e| e.0) +} + #[instrument(level = "trace", skip_all)] pub async fn add_completed_job_error( db: &DB, @@ -1755,11 +1758,11 @@ pub async fn pull(db: &DB) -> Result, crate::Error> { ) .fetch_optional(db) .await?; - + if job.is_some() { QUEUE_PULL_COUNT.inc(); } - + Ok(job) } diff --git a/backend/src/scripts.rs b/backend/src/scripts.rs index eb1a3c52d3..106285732c 100644 --- a/backend/src/scripts.rs +++ b/backend/src/scripts.rs @@ -46,6 +46,7 @@ pub fn global_service() -> Router { .route("/go/tojsonschema", post(parse_go_code_to_jsonschema)) .route("/hub/list", get(list_hub_scripts)) .route("/hub/get/*path", get(get_hub_script_by_path)) + .route("/hub/get_full/*path", get(get_full_hub_script_by_path)) } pub fn workspaced_service() -> Router { @@ -134,6 +135,7 @@ pub enum ScriptKind { Trigger, Failure, Script, + Approval, } #[derive(FromRow, Serialize)] @@ -531,6 +533,40 @@ pub async fn get_hub_script_by_path( Ok(content) } +#[derive(Deserialize, Serialize)] +pub struct HubScript { + pub content: String, + pub lockfile: Option, + pub language: ScriptLang, + pub schema: Option, +} + +pub async fn get_full_hub_script_by_path( + Authed { email, username, .. }: Authed, + Path(path): Path, + Extension(http_client): Extension, + Host(host): Host, +) -> JsonResult { + let path = path + .to_path() + .strip_prefix("hub/") + .ok_or_else(|| Error::BadRequest("Impossible to remove prefix hex".to_string()))?; + + let value = http_get_from_hub( + http_client, + &format!("https://hub.windmill.dev/raw2/{path}"), + email, + username, + host, + true, + ) + .await? + .json::() + .await + .map_err(to_anyhow)?; + Ok(Json(value)) +} + async fn get_script_by_path( authed: Authed, Extension(user_db): Extension, diff --git a/backend/src/worker.rs b/backend/src/worker.rs index 0cc6802552..3598ae868b 100644 --- a/backend/src/worker.rs +++ b/backend/src/worker.rs @@ -14,7 +14,10 @@ use uuid::Uuid; use crate::{ db::DB, error::{self, Error}, - jobs::{add_completed_job, add_completed_job_error, get_queued_job, pull, JobKind, QueuedJob}, + jobs::{ + add_completed_job, add_completed_job_error, get_hub_script, get_queued_job, pull, JobKind, + QueuedJob, + }, parser::Typ, parser_go::otyp_to_string, parser_py, @@ -657,10 +660,21 @@ async fn handle_code_execution_job( envs: &Envs, ) -> error::Result { let (inner_content, requirements_o, language) = if matches!(job.job_kind, JobKind::Preview) - || matches!(job.job_kind, JobKind::Script_Hub) + || (matches!(job.job_kind, JobKind::Script_Hub) && job.language == Some(ScriptLang::Deno)) { let code = (job.raw_code.as_ref().unwrap_or(&"no raw code".to_owned())).to_owned(); (code, None, job.language.to_owned()) + } else if matches!(job.job_kind, JobKind::Script_Hub) { + let code = (job.raw_code.as_ref().unwrap_or(&"no raw code".to_owned())).to_owned(); + let script = get_hub_script( + job.script_path + .clone() + .unwrap_or_else(|| "missing script path".to_string()), + None, + &job.created_by, + ) + .await?; + (code, script.lockfile, job.language.to_owned()) } else { sqlx::query_as::<_, (String, Option, Option)>( "SELECT content, lock, language FROM script WHERE hash = $1 AND (workspace_id = $2 OR \ diff --git a/frontend/src/lib/components/ScriptBuilder.svelte b/frontend/src/lib/components/ScriptBuilder.svelte index a133099f35..26c4af0900 100644 --- a/frontend/src/lib/components/ScriptBuilder.svelte +++ b/frontend/src/lib/components/ScriptBuilder.svelte @@ -197,9 +197,11 @@ { diff --git a/frontend/src/lib/components/common/button/Button.svelte b/frontend/src/lib/components/common/button/Button.svelte index 74eb05abe8..2a7b64886d 100644 --- a/frontend/src/lib/components/common/button/Button.svelte +++ b/frontend/src/lib/components/common/button/Button.svelte @@ -66,7 +66,11 @@ event.preventDefault() dispatch('click', event) if (href) { - goto(href) + if (href.startsWith('http')) { + window.open(href, target) + } else { + goto(href) + } } } diff --git a/frontend/src/lib/components/flows/content/FlowInputs.svelte b/frontend/src/lib/components/flows/content/FlowInputs.svelte index 8b5a77d5ea..811ab3e98f 100644 --- a/frontend/src/lib/components/flows/content/FlowInputs.svelte +++ b/frontend/src/lib/components/flows/content/FlowInputs.svelte @@ -20,16 +20,38 @@ {/if}
- - - - dispatch('loop')} + + + {#if !failureModule} + + + {/if} + + {#if !shouldDisableLoopCreation} + dispatch('loop')} + /> + {/if} {#if !failureModule} apply(pickScript, e.detail.path)} + on:pick={async (e) => { + await apply(pickScript, { path: e.detail.path, summary: e.detail.summary }) + if (e.detail.kind == Script.kind.APPROVAL) { + flowModule.suspend = { required_events: 1, timeout: 1800 } + flowModule = flowModule + } + }} on:new={(e) => apply(createInlineScriptModule, { language: e.detail.language, diff --git a/frontend/src/lib/components/flows/flowStateUtils.ts b/frontend/src/lib/components/flows/flowStateUtils.ts index 8f704c90bd..fc41e19197 100644 --- a/frontend/src/lib/components/flows/flowStateUtils.ts +++ b/frontend/src/lib/components/flows/flowStateUtils.ts @@ -72,10 +72,11 @@ export function nextId(): string { const len = computeLength(flowState.modules) return numberToChars(len); } -export async function pickScript(path: string): Promise<[FlowModule, FlowModuleState]> { +export async function pickScript({ path, summary }: { path: string, summary?: string }): Promise<[FlowModule, FlowModuleState]> { const flowModule: FlowModule = { id: nextId(), value: { type: 'script', path }, + summary, input_transforms: {} } @@ -191,7 +192,7 @@ export async function createScriptFromInlineScript({ workspace: get(workspaceStore)!, requestBody: { path: availablePath, - summary: '', + summary: flowModule.summary ?? '', description, content: flowModule.value.content, parent_hash: undefined, @@ -201,7 +202,7 @@ export async function createScriptFromInlineScript({ } }) - return pickScript(availablePath) + return pickScript({ path: availablePath, summary: flowModule.summary }) } async function findNextAvailablePath(path: string): Promise { diff --git a/frontend/src/lib/components/flows/pickers/PickHubScript.svelte b/frontend/src/lib/components/flows/pickers/PickHubScript.svelte index 9f733976b8..969fac52d5 100644 --- a/frontend/src/lib/components/flows/pickers/PickHubScript.svelte +++ b/frontend/src/lib/components/flows/pickers/PickHubScript.svelte @@ -9,6 +9,7 @@ import type { HubItem } from './model' export let kind: Script.kind + export let customText: string | undefined = undefined let items: HubItem[] $: items = $hubScripts?.filter((x) => x.kind == kind) ?? [] @@ -19,8 +20,8 @@ { - dispatch('pick', { path }) + pickCallback={(path, summary) => { + dispatch('pick', { path, summary, kind }) }} itemName={'Script'} extraField="summary" @@ -31,7 +32,7 @@ /> itemPicker.openModal()} diff --git a/frontend/src/lib/components/flows/pickers/PickScript.svelte b/frontend/src/lib/components/flows/pickers/PickScript.svelte index a639d6742b..a4512c6633 100644 --- a/frontend/src/lib/components/flows/pickers/PickScript.svelte +++ b/frontend/src/lib/components/flows/pickers/PickScript.svelte @@ -8,6 +8,7 @@ import FlowScriptPicker from './FlowScriptPicker.svelte' export let kind: string + export let customText: string | undefined = undefined type Item = { summary: String; path: String; version?: String } @@ -22,8 +23,8 @@ { - dispatch('pick', { path }) + pickCallback={(path, summary) => { + dispatch('pick', { path, summary }) }} itemName={'Script'} extraField="summary" @@ -31,7 +32,7 @@ /> itemPicker.openModal()} diff --git a/frontend/src/lib/scripts.ts b/frontend/src/lib/scripts.ts index 48c4366d9a..7c69329709 100644 --- a/frontend/src/lib/scripts.ts +++ b/frontend/src/lib/scripts.ts @@ -7,10 +7,14 @@ import { emptySchema } from './utils' export async function loadSchema(path: string): Promise { if (path.startsWith('hub/')) { - const code = await ScriptService.getHubScriptContentByPath({ path }) - const schema = emptySchema() - await inferArgs('deno', code, schema) - return schema + const { content, language, schema } = await ScriptService.getHubScriptByPath({ path }) + if (language == 'deno') { + const newSchema = emptySchema() + await inferArgs('deno', content ?? '', newSchema) + return newSchema + } else { + return JSON.parse(schema ?? "{}") + } } else { const script = await ScriptService.getScriptByPath({ workspace: get(workspaceStore)!, diff --git a/frontend/src/lib/utils.ts b/frontend/src/lib/utils.ts index 64e5f47092..53cd9ba5d5 100644 --- a/frontend/src/lib/utils.ts +++ b/frontend/src/lib/utils.ts @@ -473,10 +473,11 @@ export async function getScriptByPath(path: string): Promise<{ language: 'deno' | 'python3' | 'go' }> { if (path.startsWith('hub/')) { - const content = await ScriptService.getHubScriptContentByPath({ path }) + const { content, language, schema } = await ScriptService.getHubScriptByPath({ path }) + return { content, - language: 'deno' + language, } } else { const script = await ScriptService.getScriptByPath({ @@ -496,7 +497,7 @@ export async function loadHubScripts() { const processed = scripts .map((x) => ({ path: `hub/${x.id}/${x.app}/${x.summary.toLowerCase().replaceAll(/\s+/g, '_')}`, - summary: `${x.summary} (${x.app}) ${x.views} uses`, + summary: `${x.summary} (${x.app})`, approved: x.approved, kind: x.kind, app: x.app, diff --git a/frontend/src/routes/scripts/get/[...hash].svelte b/frontend/src/routes/scripts/get/[...hash].svelte index 04864da0f7..391db4c3b9 100644 --- a/frontend/src/routes/scripts/get/[...hash].svelte +++ b/frontend/src/routes/scripts/get/[...hash].svelte @@ -171,6 +171,7 @@ View runs