From 350c8e843f8facb154bb6b8223744e1c5ba65a28 Mon Sep 17 00:00:00 2001 From: Guillaume Bouvignies Date: Tue, 12 Dec 2023 07:04:55 +0100 Subject: [PATCH] feat: EnvVar allow list is configurable from the UI (#2829) * feat: EnvVar allow list is configurable from the UI * variable renaming --- backend/windmill-common/src/worker.rs | 36 +++++- backend/windmill-worker/src/bun_executor.rs | 4 +- backend/windmill-worker/src/common.rs | 15 ++- backend/windmill-worker/src/deno_executor.rs | 2 +- .../windmill-worker/src/python_executor.rs | 2 +- backend/windmill-worker/src/worker.rs | 6 - .../src/lib/components/WorkspaceGroup.svelte | 112 ++++++++++++++++++ 7 files changed, 158 insertions(+), 19 deletions(-) diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index 2e062b3a74..7a62e93452 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -42,7 +42,8 @@ lazy_static::lazy_static! { cache_clear: Default::default(), init_bash: Default::default(), additional_python_paths: Default::default(), - pip_local_dependencies: Default::default() + pip_local_dependencies: Default::default(), + env_vars: Default::default(), })); pub static ref SERVER_CONFIG: Arc> = Arc::new(RwLock::new(ServerConfig { smtp: Default::default(), timeout_wait_result: 20 })); @@ -273,6 +274,33 @@ pub async fn load_worker_config( } tracing::debug!("Custom tags priority set: {:?}", priority_tags_sorted); + let env_vars_static = config.env_vars_static.unwrap_or_default().clone(); + let resolved_env_vars: HashMap = env_vars_static + .keys() + .map(|x| x.to_string()) + .chain(config.env_vars_allowlist.unwrap_or_default()) + .chain( + std::env::var("WHITELIST_ENVS") + .ok() + .map(|x| x.split(',').map(|x| x.to_string()).collect_vec()) + .unwrap_or_default() + .into_iter(), + ) + .sorted() + .unique() + .map(|envvar_name| { + ( + envvar_name.clone(), + env_vars_static + .get::(&envvar_name) + .map(|v| v.to_owned()) + .unwrap_or_else(|| { + std::env::var(envvar_name.clone()).unwrap_or("".to_string()) + }), + ) + }) + .collect(); + Ok(WorkerConfig { worker_tags, priority_tags_sorted, @@ -297,6 +325,7 @@ pub async fn load_worker_config( .ok() .map(|x| x.split(':').map(|x| x.to_string()).collect()) }), + env_vars: resolved_env_vars, }) } @@ -315,6 +344,8 @@ pub struct WorkerConfigOpt { pub cache_clear: Option, pub additional_python_paths: Option>, pub pip_local_dependencies: Option>, + pub env_vars_static: Option>, + pub env_vars_allowlist: Option>, } impl Default for WorkerConfigOpt { @@ -327,6 +358,8 @@ impl Default for WorkerConfigOpt { cache_clear: Default::default(), additional_python_paths: Default::default(), pip_local_dependencies: Default::default(), + env_vars_static: Default::default(), + env_vars_allowlist: Default::default(), } } } @@ -340,6 +373,7 @@ pub struct WorkerConfig { pub cache_clear: Option, pub additional_python_paths: Option>, pub pip_local_dependencies: Option>, + pub env_vars: HashMap, } #[derive(PartialEq, Debug, Clone)] diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index 1793a78595..22e762459f 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -573,12 +573,12 @@ pub async fn start_worker( None, ) .await; - let context_envs = build_envs_map(context.to_vec()); + let context_envs = build_envs_map(context.to_vec()).await; if let Some(reqs) = requirements_o { let splitted = reqs.split(BUN_LOCKB_SPLIT).collect::>(); if splitted.len() != 2 { return Err(error::Error::ExecutionErr( - format!("Invalid requirements, expectd to find //bun.lockb split pattern in reqs. Found: |{reqs}|") + format!("Invalid requirements, expected to find //bun.lockb split pattern in reqs. Found: |{reqs}|") )); } let _ = write_file(job_dir, "package.json", &splitted[0]).await?; diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs index 598c89def1..6aad40165c 100644 --- a/backend/windmill-worker/src/common.rs +++ b/backend/windmill-worker/src/common.rs @@ -9,7 +9,7 @@ use serde_json::{json, Value}; use sqlx::{Pool, Postgres}; use tokio::process::Command; use tokio::{fs::File, io::AsyncReadExt}; -use windmill_common::worker::CLOUD_HOSTED; +use windmill_common::worker::{CLOUD_HOSTED, WORKER_CONFIG}; use windmill_common::{ error::{self, Error}, jobs::QueuedJob, @@ -47,7 +47,7 @@ use futures::{ use crate::{ AuthedClient, AuthedClientBackgroundTask, MAX_RESULT_SIZE, MAX_WAIT_FOR_SIGTERM, - ROOT_CACHE_DIR, TIMEOUT_DURATION, WHITELIST_ENVS, + ROOT_CACHE_DIR, TIMEOUT_DURATION, }; pub async fn build_args_map<'a>( @@ -368,17 +368,16 @@ pub async fn get_reserved_variables( .await .to_vec(); - Ok(build_envs_map(variables)) + Ok(build_envs_map(variables).await) } -pub fn build_envs_map(context: Vec) -> HashMap { +pub async fn build_envs_map(context: Vec) -> HashMap { let mut r: HashMap = context.into_iter().map(|rv| (rv.name, rv.value)).collect(); - if let Some(ref envs) = *WHITELIST_ENVS { - for e in envs { - r.insert(e.0.clone(), e.1.clone()); - } + let envs = WORKER_CONFIG.read().await.clone().env_vars; + for env in envs { + r.insert(env.0.clone(), env.1.clone()); } r diff --git a/backend/windmill-worker/src/deno_executor.rs b/backend/windmill-worker/src/deno_executor.rs index 6c44815654..fd0ae1b3b5 100644 --- a/backend/windmill-worker/src/deno_executor.rs +++ b/backend/windmill-worker/src/deno_executor.rs @@ -418,7 +418,7 @@ pub async fn start_worker( None, ) .await; - let context_envs = build_envs_map(context.to_vec()); + let context_envs = build_envs_map(context.to_vec()).await; { // let mut start = Instant::now(); diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index b758c8d35a..f73224660b 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -835,7 +835,7 @@ pub async fn start_worker( .await .to_vec(); - let context_envs = build_envs_map(context); + let context_envs = build_envs_map(context).await; let additional_python_paths = handle_python_deps( job_dir, requirements_o, diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 8d478c266d..cf1b79c9d4 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -243,12 +243,6 @@ lazy_static::lazy_static! { pub static ref NPM_CONFIG_REGISTRY: Arc>> = Arc::new(RwLock::new(None)); pub static ref PIP_EXTRA_INDEX_URL: Arc>> = Arc::new(RwLock::new(None)); - - - pub static ref WHITELIST_ENVS: Option> = std::env::var("WHITELIST_ENVS") - .ok() - .map(|x| x.split(',').map(|x| (x.to_string(), std::env::var(x).unwrap_or("".to_string()))).collect()); - pub static ref TAR_CACHE_RATE: i32 = std::env::var("TAR_CACHE_RATE") .ok() .and_then(|x| x.parse::().ok()) diff --git a/frontend/src/lib/components/WorkspaceGroup.svelte b/frontend/src/lib/components/WorkspaceGroup.svelte index 03df4dfa02..1dc200298d 100644 --- a/frontend/src/lib/components/WorkspaceGroup.svelte +++ b/frontend/src/lib/components/WorkspaceGroup.svelte @@ -8,6 +8,7 @@ import ConfirmationModal from './common/confirmationModal/ConfirmationModal.svelte' import { createEventDispatcher } from 'svelte' import { sendUserToast } from '$lib/toast' + import { emptyString } from '$lib/utils' import { enterpriseLicense, superadmin } from '$lib/stores' import Tooltip from './Tooltip.svelte' import Editor from './Editor.svelte' @@ -33,6 +34,8 @@ priority_tags?: Map cache_clear?: number init_bash?: string + env_vars_static?: Map + env_vars_allowlist?: string[] } = {} function loadNConfig() { @@ -48,9 +51,24 @@ if (nconfig.priority_tags === undefined) { nconfig.priority_tags = new Map() } + if (nconfig.env_vars_allowlist === undefined) { + nconfig.env_vars_allowlist = [] + } + if (nconfig.env_vars_static === undefined) { + nconfig.env_vars_static = new Map() + } else { + addedEnvVars = [] + for (const [key, value] of Object.entries(nconfig.env_vars_static)) { + addedEnvVars.push({ key, value }) + } + } } let selectedPriorityTags: string[] = [] + let addedEnvVars: { + key: string + value: string + }[] = [] const defaultTags = [ 'deno', @@ -75,6 +93,7 @@ ] let newTag: string = '' + let newEnvVar: string = '' $: selected = nconfig?.dedicated_worker != undefined ? 'dedicated' : 'normal' $: { selectedPriorityTags = [] @@ -338,6 +357,94 @@ {/if}
+
+
+ {#each addedEnvVars as envvar, i} +
+
+ + + +
+
+ {/each} + +
+ + {#if nconfig?.env_vars_allowlist !== undefined} +
+ {#each nconfig.env_vars_allowlist as envvar} +
{envvar}
+
+ {/each} +
+
+ +
+ +
+ {/if} +
+
+
{ + addedEnvVars.forEach((envvar) => { + if (nconfig.env_vars_static !== undefined && !emptyString(envvar.key)) { + nconfig.env_vars_static[envvar.key] = envvar.value + } + }) await ConfigService.updateConfig({ name: 'worker__' + name, requestBody: nconfig }) sendUserToast('Configuration set') dispatch('reload')