kalugin66/windmill
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: support custom env variables (#1675)

Browse Source 
* custom envs

* feat: custom envs

* only on non cloud

* remove unecessary imports
This commit is contained in:
Ruben Fiszel
2023-06-05 17:42:57 +02:00
committed by GitHub
parent 2273fa8be5
commit 8c47957a02
14 changed files with 199 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
backend/migrations/20230605043929_add_envs.down.sql Normal file
View File

@@ -0,0 +1,2 @@
-- Add down migration script here
ALTER TABLE script DROP COLUMN envs;

2
backend/migrations/20230605043929_add_envs.up.sql Normal file
View File

@@ -0,0 +1,2 @@
-- Add up migration script here
ALTER TABLE script ADD COLUMN envs VARCHAR(1000)[];

105
backend/sqlx-data.json
View File

@@ -3656,6 +3656,59 @@
},
"query": "SELECT EXISTS(SELECT 1 FROM variable WHERE path = $1 AND workspace_id = $2)"
},
"88a5f7d43e2775cdbc121e4f39f7d881af75c24740446ca86f26ba8b2e9e5405": {
"describe": {
"columns": [],
"nullable": [],
"parameters": {
"Left": [
"Varchar",
"Int8",
"Varchar",
"Int8Array",
"Text",
"Text",
"Text",
"Varchar",
"Text",
"Bool",
"Jsonb",
"Text",
{
"Custom": {
"kind": {
"Enum": [
"python3",
"deno",
"go",
"bash"
]
},
"name": "script_lang"
}
},
{
"Custom": {
"kind": {
"Enum": [
"script",
"trigger",
"failure",
"command",
"approval"
]
},
"name": "script_kind"
}
},
"Varchar",
"Bool",
"VarcharArray"
]
}
},
"query": "INSERT INTO script (workspace_id, hash, path, parent_hashes, summary, description, content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, draft_only, envs) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17)"
},
"88b7589a6416da8be4b26af3bf30fcfcd6aeae7bc5a37e9a735cabbe2691c570": {
"describe": {
"columns": [
@@ -4010,58 +4063,6 @@
},
"query": "SELECT email, login_type::TEXT, super_admin, verified, name, company FROM password WHERE email = $1"
},
"90c5f1eea5d2cceb157ce0667a625f0fa4506961e2ad181765022a20b11d314d": {
"describe": {
"columns": [],
"nullable": [],
"parameters": {
"Left": [
"Varchar",
"Int8",
"Varchar",
"Int8Array",
"Text",
"Text",
"Text",
"Varchar",
"Text",
"Bool",
"Jsonb",
"Text",
{
"Custom": {
"kind": {
"Enum": [
"python3",
"deno",
"go",
"bash"
]
},
"name": "script_lang"
}
},
{
"Custom": {
"kind": {
"Enum": [
"script",
"trigger",
"failure",
"command",
"approval"
]
},
"name": "script_kind"
}
},
"Varchar",
"Bool"
]
}
},
"query": "INSERT INTO script (workspace_id, hash, path, parent_hashes, summary, description, content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, draft_only) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16)"
},
"911b1e1f2a5ba6d5159916e5598020e680c45043b0736ad0153ee261a151dd90": {
"describe": {
"columns": [

8
backend/windmill-api/openapi.yaml
View File

@@ -5493,6 +5493,10 @@ components:
type: boolean
draft_only:
type: boolean
envs:
type: array
items:
type: string
required:
- hash
- path
@@ -5540,6 +5544,10 @@ components:
type: string
draft_only:
type: boolean
envs:
type: array
items:
type: string
required:
- path
- summary

18
backend/windmill-api/src/scripts.rs
View File

@@ -70,6 +70,8 @@ pub struct ScriptWDraft {
pub schema: Option<Schema>,
#[serde(skip_serializing_if = "Option::is_none")]
pub draft_only: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub envs: Option<Vec<String>>,
}
pub fn global_service() -> Router {
@@ -363,11 +365,18 @@ async fn create_script(
};
let needs_lock_gen = lock.is_none();
let envs = ns.envs.as_ref().map(|x| x.as_slice());
let envs = if ns.envs.is_none() || ns.envs.as_ref().unwrap().is_empty() {
None
} else {
envs
};
//::text::json is to ensure we use serde_json with preserve order
sqlx::query!(
"INSERT INTO script (workspace_id, hash, path, parent_hashes, summary, description, \
content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, draft_only) \
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16)",
content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, draft_only, envs) \
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17)",
&w_id,
&hash.0,
ns.path,
@@ -383,7 +392,8 @@ async fn create_script(
ns.language: ScriptLang,
ns.kind.unwrap_or(ScriptKind::Script): ScriptKind,
ns.tag,
ns.draft_only
ns.draft_only,
envs
)
.execute(&mut tx)
.await?;
@@ -550,7 +560,7 @@ async fn get_script_by_path_w_draft(
let mut tx = user_db.begin(&authed).await?;
let script_o = sqlx::query_as::<_, ScriptWDraft>(
"SELECT hash, script.path, summary, description, content, language, kind, tag, schema, draft_only, draft.value as draft FROM script LEFT JOIN draft ON
"SELECT hash, script.path, summary, description, content, language, kind, tag, schema, draft_only, envs, draft.value as draft FROM script LEFT JOIN draft ON
script.path = draft.path AND script.workspace_id = draft.workspace_id AND draft.typ = 'script'
WHERE script.path = $1 AND script.workspace_id = $2 \
AND script.created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND \

3
backend/windmill-common/src/scripts.rs
View File

@@ -139,6 +139,8 @@ pub struct Script {
pub tag: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub draft_only: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub envs: Option<Vec<String>>,
}
#[derive(Serialize)]
@@ -189,6 +191,7 @@ pub struct NewScript {
pub kind: Option<ScriptKind>,
pub tag: Option<String>,
pub draft_only: Option<bool>,
pub envs: Option<Vec<String>>,
}
#[derive(Deserialize)]

4
backend/windmill-worker/src/go_executor.rs
View File

@@ -1,4 +1,4 @@
use std::process::Stdio;
use std::{collections::HashMap, process::Stdio};
use itertools::Itertools;
use tokio::{
@@ -40,6 +40,7 @@ pub async fn handle_go_job(
shared_mount: &str,
base_internal_url: &str,
worker_name: &str,
envs: HashMap<String, String>,
) -> Result<serde_json::Value, Error> {
//go does not like executing modules at temp root
let job_dir = &format!("{job_dir}/go");
@@ -210,6 +211,7 @@ func Run(req Req) (interface{{}}, error){{
let mut cmd = Command::new(GO_PATH.as_str());
cmd.current_dir(job_dir)
.env_clear()
.envs(envs)
.envs(reserved_variables)
.env("PATH", PATH_ENV.as_str())
.env("BASE_INTERNAL_URL", base_internal_url)

4
backend/windmill-worker/src/python_executor.rs
View File

@@ -1,4 +1,4 @@
use std::process::Stdio;
use std::{collections::HashMap, process::Stdio};
use itertools::Itertools;
use regex::Regex;
@@ -165,6 +165,7 @@ pub async fn handle_python_job(
inner_content: &String,
shared_mount: &str,
base_internal_url: &str,
envs: HashMap<String, String>,
) -> windmill_common::error::Result<serde_json::Value> {
create_dependencies_dir(job_dir).await;
@@ -435,6 +436,7 @@ mount {{
Command::new(PYTHON_PATH.as_str())
.current_dir(job_dir)
.env_clear()
.envs(envs)
.envs(reserved_variables)
.env("PATH", PATH_ENV.as_str())
.env("BASE_INTERNAL_URL", base_internal_url)

41
backend/windmill-worker/src/worker.rs
View File

@@ -1204,13 +1204,14 @@ async fn handle_code_execution_job(
worker_name: &str
) -> error::Result<serde_json::Value> {
let (inner_content, requirements_o, language) = match job.job_kind {
let (inner_content, requirements_o, language, envs) = match job.job_kind {
JobKind::Preview => (
job.raw_code
.clone()
.unwrap_or_else(|| "no raw code".to_owned()),
job.raw_lock.clone(),
job.language.to_owned(),
None,
),
JobKind::Script_Hub => {
let script_path = job.script_path.clone().ok_or_else(|| Error::InternalErr(format!("expected script path for hub script")))?;
@@ -1231,10 +1232,11 @@ async fn handle_code_execution_job(
(
script.content,
script.lockfile,
Some(script.language)
Some(script.language),
None
)},
JobKind::Script => sqlx::query_as::<_, (String, Option<String>, Option<ScriptLang>)>(
"SELECT content, lock, language FROM script WHERE hash = $1 AND workspace_id = $2",
JobKind::Script => sqlx::query_as::<_, (String, Option<String>, Option<ScriptLang>, Option<Vec<String>>)>(
"SELECT content, lock, language, envs FROM script WHERE hash = $1 AND workspace_id = $2",
)
.bind(&job.script_hash.unwrap_or(ScriptHash(0)).0)
.bind(&job.workspace_id)
@@ -1276,6 +1278,21 @@ mount {{
};
// println!("handle lang job {:?}", SystemTime::now());
let envs = if *CLOUD_HOSTED || envs.is_none() {
HashMap::new()
} else {
let mut hm = HashMap::new();
for s in envs.unwrap() {
let (k, v) = s.split_once('=').ok_or_else(|| {
Error::BadRequest(format!(
"Invalid env var: {}. Must be in the form of KEY=VALUE",
s
))
})?;
hm.insert(k.to_string(), v.to_string());
}
hm
};
let result: error::Result<serde_json::Value> = match language {
None => {
@@ -1296,6 +1313,7 @@ mount {{
&inner_content,
&shared_mount,
base_internal_url,
envs
)
.await
}
@@ -1308,7 +1326,8 @@ mount {{
job_dir,
&inner_content,
base_internal_url,
worker_name
worker_name,
envs
)
.await
}
@@ -1323,7 +1342,8 @@ mount {{
requirements_o,
&shared_mount,
base_internal_url,
worker_name
worker_name,
envs
)
.await
}
@@ -1337,7 +1357,8 @@ mount {{
job_dir,
&shared_mount,
base_internal_url,
worker_name
worker_name,
envs
)
.await
}
@@ -1368,6 +1389,7 @@ async fn handle_bash_job(
shared_mount: &str,
base_internal_url: &str,
worker_name: &str,
envs: HashMap<String, String>,
) -> Result<serde_json::Value, Error> {
logs.push_str("\n\n--- BASH CODE EXECUTION ---\n");
set_logs(logs, &job.id, db).await;
@@ -1422,6 +1444,7 @@ async fn handle_bash_job(
Command::new("/bin/bash")
.current_dir(job_dir)
.env_clear()
.envs(envs)
.envs(reserved_variables)
.env("PATH", PATH_ENV.as_str())
.env("BASE_INTERNAL_URL", base_internal_url)
@@ -1469,7 +1492,8 @@ async fn handle_deno_job(
job_dir: &str,
inner_content: &String,
base_internal_url: &str,
worker_name: &str
worker_name: &str,
envs: HashMap<String, String>,
) -> error::Result<serde_json::Value> {
// let mut start = Instant::now();
@@ -1608,6 +1632,7 @@ run().catch(async (e) => {{
Command::new(DENO_PATH.as_str())
.current_dir(job_dir)
.env_clear()
.envs(envs)
.envs(reserved_variables)
.envs(common_deno_proc_envs)
.env("DENO_DIR", DENO_CACHE_DIR)

65
frontend/src/lib/components/ScriptBuilder.svelte
View File

@@ -10,13 +10,13 @@
import ScriptEditor from './ScriptEditor.svelte'
import ScriptSchema from './ScriptSchema.svelte'
import { dirtyStore } from './common/confirmationModal/dirtyStore'
import { Badge, Button, Drawer, Kbd } from './common'
import { faSave } from '@fortawesome/free-solid-svg-icons'
import { Alert, Badge, Button, Drawer, Kbd } from './common'
import { faPlus, faSave } from '@fortawesome/free-solid-svg-icons'
import LanguageIcon from './common/languageIcons/LanguageIcon.svelte'
import type { SupportedLanguage } from '$lib/common'
import Tooltip from './Tooltip.svelte'
import DrawerContent from './common/drawer/DrawerContent.svelte'
import { Pen } from 'lucide-svelte'
import { Pen, X } from 'lucide-svelte'
import autosize from 'svelte-autosize'
import type Editor from './Editor.svelte'
import {
@@ -29,6 +29,8 @@
import { sendUserToast } from '$lib/toast'
import { isCloudHosted } from '$lib/cloud'
import Awareness from './Awareness.svelte'
import { Icon } from 'svelte-awesome'
import { fade } from 'svelte/transition'
export let script: NewScript
export let initialPath: string = ''
@@ -157,7 +159,8 @@
is_template: script.is_template,
language: script.language,
kind: script.kind,
tag: script.tag
tag: script.tag,
envs: script.envs
}
})
history.replaceState(history.state, '', `/scripts/edit/${script.path}`)
@@ -196,7 +199,8 @@
language: script.language,
kind: script.kind,
tag: script.tag,
draft_only: true
draft_only: true,
envs: script.envs
}
})
}
@@ -402,6 +406,57 @@
{/if}
{/if}
</div>
{#if !isCloudHosted()}
<h2 class="border-b pb-1 mt-10 mb-4"
>Custom env variables<Tooltip
>Additional static custom env variables to pass to the script.</Tooltip
></h2
>
<div class="w-full">
<span class="text-gray-600 text-xs pb-2">Format is: `{'<KEY>=<VALUE>'}`</span>
{#if Array.isArray(script.envs ?? [])}
{#each script.envs ?? [] as v, i}
<div class="flex max-w-md mt-1 w-full items-center">
<input type="text" bind:value={v} placeholder="<KEY>=<VALUE>" />
<button
transition:fade|local={{ duration: 50 }}
class="rounded-full p-1 bg-white/60 duration-200 hover:bg-gray-200"
aria-label="Clear"
on:click={() => {
script.envs && script.envs.splice(i, 1)
script.envs = script.envs
}}
>
<X size={14} />
</button>
</div>
{/each}
{#if script.envs && script.envs.length > 0}
<div class="pt-2" />
<Alert type="warning" title="Not passed in previews"
>Static envs variables are not passed in preview but solely on deployed scripts.</Alert
>
{/if}
{/if}
</div>
{/if}
<div class="flex mt-2">
<Button
variant="border"
color="dark"
size="xs"
btnClasses="mt-1"
on:click={() => {
if (script.envs == undefined || !Array.isArray(script.envs)) {
script.envs = []
}
script.envs = script.envs.concat('')
}}
>
<Icon data={faPlus} class="mr-2" />
Add item
</Button>
</div>
</DrawerContent>
</Drawer>

5
frontend/src/lib/components/apps/components/display/AppFlowStatusComponent.svelte
View File

@@ -10,7 +10,7 @@
export let id: string
export let componentInput: AppInput | undefined
export let initializing: boolean | undefined = undefined
export let initializing: boolean | undefined = false
export let customCss: ComponentCustomCSS<'flowstatuscomponent'> | undefined = undefined
export let render: boolean
@@ -21,6 +21,8 @@
loading: false
})
initializing = false
$: css = concatCustomCss($app.css?.flowstatuscomponent, customCss)
let jobId: string | undefined
@@ -34,7 +36,6 @@
{render}
{componentInput}
{id}
bind:initializing
>
<div class="flex flex-col w-full h-full">
<div

5
frontend/src/lib/components/apps/components/display/AppLogsComponent.svelte
View File

@@ -12,7 +12,7 @@
export let id: string
export let componentInput: AppInput | undefined
export let initializing: boolean | undefined = undefined
export let initializing: boolean | undefined = false
export let customCss: ComponentCustomCSS<'logcomponent'> | undefined = undefined
export let render: boolean
@@ -23,6 +23,8 @@
loading: false
})
initializing = false
$: css = concatCustomCss($app.css?.logcomponent, customCss)
let testJobLoader: TestJobLoader | undefined = undefined
@@ -40,7 +42,6 @@
{render}
{componentInput}
{id}
bind:initializing
>
<div class="flex flex-col w-full h-full">
<div

4
frontend/src/lib/components/flows/content/ScriptEditorDrawer.svelte
View File

@@ -35,6 +35,7 @@
content: string
schema?: any
kind: 'script' | 'failure' | 'trigger' | 'command' | 'approval' | undefined
envs?: string[]
}
| undefined = undefined
@@ -62,7 +63,8 @@
schema: script.schema,
is_template: false,
language: script.language,
kind: script.kind as Script.kind | undefined
kind: script.kind as Script.kind | undefined,
envs: script.envs
}
})
callback?.()

9
frontend/src/routes/(root)/(logged)/scripts/get/[...hash]/+page.svelte
View File

@@ -461,6 +461,15 @@
</Tabs>
</div>
{#if script.envs && script.envs.length > 0}
<h3>Static Env Variables</h3>
<ul>
{#each script.envs as e}
<li>{e}</li>
{/each}
</ul>
{/if}
<div class="max-w-2xl mt-12">
<h3 class="mb-4" bind:this={webhookElem} id="webhooks">
Webhooks