kalugin66/windmill
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: support running and publishing go, python scripts to the hub (#779)

Browse Source
This commit is contained in:
Ruben Fiszel
2022-10-20 20:16:08 +02:00
committed by GitHub
parent 71a75a57b6
commit 6d4d8e2d64
16 changed files with 189 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
backend/migrations/20221020164416_approval_script.down.sql Normal file
View File

@@ -0,0 +1 @@
-- Add down migration script here

2
backend/migrations/20221020164416_approval_script.up.sql Normal file
View File

@@ -0,0 +1,2 @@
-- Add up migration script here
ALTER TYPE SCRIPT_KIND ADD VALUE 'approval';

36
backend/openapi.yaml
View File

@@ -1534,7 +1534,7 @@ paths:
type: boolean
kind:
type: string
enum: [script, failure, trigger, command]
enum: [script, failure, trigger, command, approval]
votes:
type: number
views:
@@ -1626,6 +1626,35 @@ paths:
schema:
type: string
/scripts/hub/get_full/{path}:
get:
summary: get full hub script by path
operationId: getHubScriptByPath
tags:
- script
parameters:
- $ref: "#/components/parameters/ScriptPath"
responses:
"200":
description: script details
content:
application/json:
schema:
type: object
properties:
content:
type: string
lockfile:
type: string
schema:
type: string
language:
type: string
enum: [deno, python3, go]
required:
- content
- language
/w/{workspace}/scripts/list:
get:
summary: list all available scripts
@@ -1745,7 +1774,7 @@ paths:
enum: [python3, deno, go]
kind:
type: string
enum: [script, failure, trigger, command]
enum: [script, failure, trigger, command, approval]
required:
- path
- summary
@@ -1908,6 +1937,7 @@ paths:
schema:
type: string
/w/{workspace}/scripts/exists/p/{path}:
get:
summary: exists script by path
@@ -3404,7 +3434,7 @@ components:
enum: [python3, deno, go]
kind:
type: string
enum: [script, failure, trigger, command]
enum: [script, failure, trigger, command, approval]
required:
- hash
- path

51
backend/src/jobs.rs
View File

@@ -22,7 +22,7 @@ use crate::{
flows::FlowValue,
oauth2::HmacSha256,
schedule::get_schedule_opt,
scripts::{get_hub_script_by_path, ScriptHash, ScriptLang},
scripts::{get_full_hub_script_by_path, HubScript, ScriptHash, ScriptLang},
users::{owner_to_token_owner, Authed},
utils::{now_from_db, require_admin, Pagination, StripPath},
variables::get_workspace_key,
@@ -1435,31 +1435,14 @@ pub async fn push<'c>(
)
.fetch_optional(&mut tx)
.await?;
let script = get_hub_script(path.clone(), email, user).await?;
(
None,
Some(path.clone()),
Some(
get_hub_script_by_path(
Authed {
email,
username: user.to_string(),
is_admin: false,
groups: vec![],
},
Path(StripPath(path)),
Extension(
reqwest::ClientBuilder::new()
.user_agent("windmill/beta")
.build()
.map_err(to_anyhow)?,
),
Host(std::env::var("BASE_URL").unwrap_or_else(|_| "".to_string())),
)
.await?,
),
Some(path),
Some(script.content.clone()),
JobKind::Script_Hub,
None,
Some(ScriptLang::Deno),
Some(script.language.clone()),
)
}
JobPayload::Code(RawCode { content, path, language }) => (
@@ -1588,6 +1571,26 @@ pub async fn push<'c>(
Ok((uuid, tx))
}
pub async fn get_hub_script(
path: String,
email: Option<String>,
user: &str,
) -> error::Result<HubScript> {
get_full_hub_script_by_path(
Authed { email, username: user.to_string(), is_admin: false, groups: vec![] },
Path(StripPath(path)),
Extension(
reqwest::ClientBuilder::new()
.user_agent("windmill/beta")
.build()
.map_err(to_anyhow)?,
),
Host(std::env::var("BASE_URL").unwrap_or_else(|_| "".to_string())),
)
.await
.map(|e| e.0)
}
#[instrument(level = "trace", skip_all)]
pub async fn add_completed_job_error<E: ToString + std::fmt::Debug>(
db: &DB,
@@ -1755,11 +1758,11 @@ pub async fn pull(db: &DB) -> Result<Option<QueuedJob>, crate::Error> {
)
.fetch_optional(db)
.await?;
if job.is_some() {
QUEUE_PULL_COUNT.inc();
}
Ok(job)
}

36
backend/src/scripts.rs
View File

@@ -46,6 +46,7 @@ pub fn global_service() -> Router {
.route("/go/tojsonschema", post(parse_go_code_to_jsonschema))
.route("/hub/list", get(list_hub_scripts))
.route("/hub/get/*path", get(get_hub_script_by_path))
.route("/hub/get_full/*path", get(get_full_hub_script_by_path))
}
pub fn workspaced_service() -> Router {
@@ -134,6 +135,7 @@ pub enum ScriptKind {
Trigger,
Failure,
Script,
Approval,
}
#[derive(FromRow, Serialize)]
@@ -531,6 +533,40 @@ pub async fn get_hub_script_by_path(
Ok(content)
}
#[derive(Deserialize, Serialize)]
pub struct HubScript {
pub content: String,
pub lockfile: Option<String>,
pub language: ScriptLang,
pub schema: Option<String>,
}
pub async fn get_full_hub_script_by_path(
Authed { email, username, .. }: Authed,
Path(path): Path<StripPath>,
Extension(http_client): Extension<Client>,
Host(host): Host,
) -> JsonResult<HubScript> {
let path = path
.to_path()
.strip_prefix("hub/")
.ok_or_else(|| Error::BadRequest("Impossible to remove prefix hex".to_string()))?;
let value = http_get_from_hub(
http_client,
&format!("https://hub.windmill.dev/raw2/{path}"),
email,
username,
host,
true,
)
.await?
.json::<HubScript>()
.await
.map_err(to_anyhow)?;
Ok(Json(value))
}
async fn get_script_by_path(
authed: Authed,
Extension(user_db): Extension<UserDB>,

18
backend/src/worker.rs
View File

@@ -14,7 +14,10 @@ use uuid::Uuid;
use crate::{
db::DB,
error::{self, Error},
jobs::{add_completed_job, add_completed_job_error, get_queued_job, pull, JobKind, QueuedJob},
jobs::{
add_completed_job, add_completed_job_error, get_hub_script, get_queued_job, pull, JobKind,
QueuedJob,
},
parser::Typ,
parser_go::otyp_to_string,
parser_py,
@@ -657,10 +660,21 @@ async fn handle_code_execution_job(
envs: &Envs,
) -> error::Result<serde_json::Value> {
let (inner_content, requirements_o, language) = if matches!(job.job_kind, JobKind::Preview)
|| matches!(job.job_kind, JobKind::Script_Hub)
|| (matches!(job.job_kind, JobKind::Script_Hub) && job.language == Some(ScriptLang::Deno))
{
let code = (job.raw_code.as_ref().unwrap_or(&"no raw code".to_owned())).to_owned();
(code, None, job.language.to_owned())
} else if matches!(job.job_kind, JobKind::Script_Hub) {
let code = (job.raw_code.as_ref().unwrap_or(&"no raw code".to_owned())).to_owned();
let script = get_hub_script(
job.script_path
.clone()
.unwrap_or_else(|| "missing script path".to_string()),
None,
&job.created_by,
)
.await?;
(code, script.lockfile, job.language.to_owned())
} else {
sqlx::query_as::<_, (String, Option<String>, Option<ScriptLang>)>(
"SELECT content, lock, language FROM script WHERE hash = $1 AND (workspace_id = $2 OR \

8
frontend/src/lib/components/ScriptBuilder.svelte
View File

@@ -197,9 +197,11 @@
<RadioButton
label="Script Type"
options={[
['General Script', Script.kind.SCRIPT],
['Trigger Script', Script.kind.TRIGGER]
// ['Failure Handler', Script.kind.FAILURE],
['Common Script', Script.kind.SCRIPT],
['Trigger Script', Script.kind.TRIGGER],
['Error Handler', Script.kind.FAILURE],
['Approval Script', Script.kind.APPROVAL]
// ['Command Handler', Script.kind.COMMAND]
]}
on:change={(e) => {

6
frontend/src/lib/components/common/button/Button.svelte
View File

@@ -66,7 +66,11 @@
event.preventDefault()
dispatch('click', event)
if (href) {
goto(href)
if (href.startsWith('http')) {
window.open(href, target)
} else {
goto(href)
}
}
}

40
frontend/src/lib/components/flows/content/FlowInputs.svelte
View File

@@ -20,16 +20,38 @@
{/if}
<div class="grid sm:grid-col-2 lg:grid-cols-3 gap-4">
<PickScript kind={failureModule ? Script.kind.FAILURE : Script.kind.SCRIPT} on:pick />
<PickHubScript kind={failureModule ? Script.kind.FAILURE : Script.kind.SCRIPT} on:pick />
<FlowScriptPicker
label={`Create a for-loop here`}
disabled={shouldDisableLoopCreation}
icon={faRepeat}
iconColor="text-blue-500"
on:click={() => dispatch('loop')}
<PickScript
customText={failureModule ? 'Pick an error handler from your workspace' : undefined}
kind={failureModule ? Script.kind.FAILURE : Script.kind.SCRIPT}
on:pick
/>
<PickHubScript
customText={failureModule ? 'Pick an error handler from your workspace' : undefined}
kind={failureModule ? Script.kind.FAILURE : Script.kind.SCRIPT}
on:pick
/>
{#if !failureModule}
<PickScript
customText={failureModule ? 'Pick an approval script from your workspace' : undefined}
kind={failureModule ? Script.kind.FAILURE : Script.kind.APPROVAL}
on:pick
/>
<PickHubScript
customText={'Pick an approval script from the Hub'}
kind={Script.kind.APPROVAL}
on:pick
/>
{/if}
{#if !shouldDisableLoopCreation}
<FlowScriptPicker
label={`Create a for-loop here`}
disabled={shouldDisableLoopCreation}
icon={faRepeat}
iconColor="text-blue-500"
on:click={() => dispatch('loop')}
/>
{/if}
{#if !failureModule}
<FlowScriptPicker

10
frontend/src/lib/components/flows/content/FlowModule.svelte
View File

@@ -24,7 +24,7 @@
} from '$lib/components/flows/flowStateUtils'
import { flowStore } from '$lib/components/flows/flowStore'
import SchemaForm from '$lib/components/SchemaForm.svelte'
import { RawScript, type FlowModule } from '$lib/gen'
import { RawScript, Script, type FlowModule } from '$lib/gen'
import FlowCard from '../common/FlowCard.svelte'
import FlowModuleHeader from './FlowModuleHeader.svelte'
import { flowStateStore, type FlowModuleState } from '../flowState'
@@ -144,7 +144,13 @@
applyCreateLoop()
select(['loop', $selectedId].join('-'))
}}
on:pick={(e) => apply(pickScript, e.detail.path)}
on:pick={async (e) => {
await apply(pickScript, { path: e.detail.path, summary: e.detail.summary })
if (e.detail.kind == Script.kind.APPROVAL) {
flowModule.suspend = { required_events: 1, timeout: 1800 }
flowModule = flowModule
}
}}
on:new={(e) =>
apply(createInlineScriptModule, {
language: e.detail.language,

7
frontend/src/lib/components/flows/flowStateUtils.ts
View File

@@ -72,10 +72,11 @@ export function nextId(): string {
const len = computeLength(flowState.modules)
return numberToChars(len);
}
export async function pickScript(path: string): Promise<[FlowModule, FlowModuleState]> {
export async function pickScript({ path, summary }: { path: string, summary?: string }): Promise<[FlowModule, FlowModuleState]> {
const flowModule: FlowModule = {
id: nextId(),
value: { type: 'script', path },
summary,
input_transforms: {}
}
@@ -191,7 +192,7 @@ export async function createScriptFromInlineScript({
workspace: get(workspaceStore)!,
requestBody: {
path: availablePath,
summary: '',
summary: flowModule.summary ?? '',
description,
content: flowModule.value.content,
parent_hash: undefined,
@@ -201,7 +202,7 @@ export async function createScriptFromInlineScript({
}
})
return pickScript(availablePath)
return pickScript({ path: availablePath, summary: flowModule.summary })
}
async function findNextAvailablePath(path: string): Promise<string> {

7
frontend/src/lib/components/flows/pickers/PickHubScript.svelte
View File

@@ -9,6 +9,7 @@
import type { HubItem } from './model'
export let kind: Script.kind
export let customText: string | undefined = undefined
let items: HubItem[]
$: items = $hubScripts?.filter((x) => x.kind == kind) ?? []
@@ -19,8 +20,8 @@
<ItemPicker
bind:this={itemPicker}
pickCallback={(path) => {
dispatch('pick', { path })
pickCallback={(path, summary) => {
dispatch('pick', { path, summary, kind })
}}
itemName={'Script'}
extraField="summary"
@@ -31,7 +32,7 @@
/>
<FlowScriptPicker
label={`Pick a ${kind == Script.kind.SCRIPT ? '' : kind} script from the Hub`}
label={customText ?? `Pick a ${kind == Script.kind.SCRIPT ? '' : kind} script from the Hub`}
icon={faUserGroup}
iconColor="text-blue-500"
on:click={() => itemPicker.openModal()}

7
frontend/src/lib/components/flows/pickers/PickScript.svelte
View File

@@ -8,6 +8,7 @@
import FlowScriptPicker from './FlowScriptPicker.svelte'
export let kind: string
export let customText: string | undefined = undefined
type Item = { summary: String; path: String; version?: String }
@@ -22,8 +23,8 @@
<ItemPicker
bind:this={itemPicker}
pickCallback={(path) => {
dispatch('pick', { path })
pickCallback={(path, summary) => {
dispatch('pick', { path, summary })
}}
itemName={'Script'}
extraField="summary"
@@ -31,7 +32,7 @@
/>
<FlowScriptPicker
label={`Pick a ${kind == 'script' ? '' : kind} script from your workspace`}
label={customText ?? `Pick a ${kind == 'script' ? '' : kind} script from your workspace`}
icon={faUserGroup}
iconColor="text-blue-500"
on:click={() => itemPicker.openModal()}

12
frontend/src/lib/scripts.ts
View File

@@ -7,10 +7,14 @@ import { emptySchema } from './utils'
export async function loadSchema(path: string): Promise<Schema> {
if (path.startsWith('hub/')) {
const code = await ScriptService.getHubScriptContentByPath({ path })
const schema = emptySchema()
await inferArgs('deno', code, schema)
return schema
const { content, language, schema } = await ScriptService.getHubScriptByPath({ path })
if (language == 'deno') {
const newSchema = emptySchema()
await inferArgs('deno', content ?? '', newSchema)
return newSchema
} else {
return JSON.parse(schema ?? "{}")
}
} else {
const script = await ScriptService.getScriptByPath({
workspace: get(workspaceStore)!,

7
frontend/src/lib/utils.ts
View File

@@ -473,10 +473,11 @@ export async function getScriptByPath(path: string): Promise<{
language: 'deno' | 'python3' | 'go'
}> {
if (path.startsWith('hub/')) {
const content = await ScriptService.getHubScriptContentByPath({ path })
const { content, language, schema } = await ScriptService.getHubScriptByPath({ path })
return {
content,
language: 'deno'
language,
}
} else {
const script = await ScriptService.getScriptByPath({
@@ -496,7 +497,7 @@ export async function loadHubScripts() {
const processed = scripts
.map((x) => ({
path: `hub/${x.id}/${x.app}/${x.summary.toLowerCase().replaceAll(/\s+/g, '_')}`,
summary: `${x.summary} (${x.app}) ${x.views} uses`,
summary: `${x.summary} (${x.app})`,
approved: x.approved,
kind: x.kind,
app: x.app,

1
frontend/src/routes/scripts/get/[...hash].svelte
View File

@@ -171,6 +171,7 @@
View runs
</Button>
<Button
disabled={deploymentInProgress}
target="_blank"
href={scriptToHubUrl(
script.content,