diff --git a/backend/src/main.rs b/backend/src/main.rs index bafd387205..4616156b78 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -26,8 +26,9 @@ use windmill_common::{ DEFAULT_TAGS_PER_WORKSPACE_SETTING, DISABLE_STATS_SETTING, ENV_SETTINGS, EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, - NPM_CONFIG_REGISTRY_SETTING, OAUTH_SETTING, REQUEST_SIZE_LIMIT_SETTING, - REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING, + NPM_CONFIG_REGISTRY_SETTING, OAUTH_SETTING, PIP_INDEX_URL_SETTING, + REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, + RETENTION_PERIOD_SECS_SETTING, }, stats::schedule_stats, utils::{rd_string, Mode}, @@ -45,8 +46,8 @@ use crate::monitor::{ initial_load, load_keep_job_dir, load_require_preexisting_user, load_tag_per_workspace_enabled, monitor_db, monitor_pool, reload_base_url_setting, reload_bunfig_install_scopes_setting, reload_extra_pip_index_url_setting, reload_job_default_timeout_setting, reload_license_key, - reload_npm_config_registry_setting, reload_retention_period_setting, reload_server_config, - reload_worker_config, + reload_npm_config_registry_setting, reload_pip_index_url_setting, + reload_retention_period_setting, reload_server_config, reload_worker_config, }; const GIT_VERSION: &str = git_version!(args = ["--tag", "--always"], fallback = "unknown-version"); @@ -401,6 +402,9 @@ Windmill Community Edition {GIT_VERSION} EXTRA_PIP_INDEX_URL_SETTING => { reload_extra_pip_index_url_setting(&db).await }, + PIP_INDEX_URL_SETTING => { + reload_pip_index_url_setting(&db).await + }, NPM_CONFIG_REGISTRY_SETTING => { reload_npm_config_registry_setting(&db).await }, diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index 7b00e235f5..61bf89ea3f 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -24,8 +24,9 @@ use windmill_common::{ BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING, EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, - NPM_CONFIG_REGISTRY_SETTING, OAUTH_SETTING, REQUEST_SIZE_LIMIT_SETTING, - REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING, + NPM_CONFIG_REGISTRY_SETTING, OAUTH_SETTING, PIP_INDEX_URL_SETTING, + REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, + RETENTION_PERIOD_SECS_SETTING, }, jobs::{JobKind, QueuedJob}, oauth2::REQUIRE_PREEXISTING_USER_FOR_OAUTH, @@ -39,7 +40,7 @@ use windmill_common::{ }; use windmill_worker::{ create_token_for_owner, handle_job_error, AuthedClient, SendResult, BUNFIG_INSTALL_SCOPES, - JOB_DEFAULT_TIMEOUT, KEEP_JOB_DIR, NPM_CONFIG_REGISTRY, PIP_EXTRA_INDEX_URL, + JOB_DEFAULT_TIMEOUT, KEEP_JOB_DIR, NPM_CONFIG_REGISTRY, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, SCRIPT_TOKEN_EXPIRY, }; @@ -107,9 +108,6 @@ pub async fn initial_load( if worker_mode { load_keep_job_dir(db).await; - } - - if worker_mode { reload_worker_config(&db, tx, false).await; } @@ -125,12 +123,7 @@ pub async fn initial_load( if server_mode { reload_server_config(&db).await; - } - - if server_mode { reload_retention_period_setting(&db).await; - } - if server_mode { reload_request_size(&db).await; } @@ -141,12 +134,8 @@ pub async fn initial_load( if worker_mode { reload_extra_pip_index_url_setting(&db).await; - } - - if worker_mode { + reload_pip_index_url_setting(&db).await; reload_npm_config_registry_setting(&db).await; - } - if worker_mode { reload_bunfig_install_scopes_setting(&db).await; } } @@ -338,6 +327,16 @@ pub async fn reload_extra_pip_index_url_setting(db: &DB) { .await; } +pub async fn reload_pip_index_url_setting(db: &DB) { + reload_option_setting_with_tracing( + db, + PIP_INDEX_URL_SETTING, + "PIP_INDEX_URL", + PIP_INDEX_URL.clone(), + ) + .await; +} + pub async fn reload_npm_config_registry_setting(db: &DB) { reload_option_setting_with_tracing( db, diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs index 277d19f6ec..d5bd88d0ed 100644 --- a/backend/windmill-common/src/global_settings.rs +++ b/backend/windmill-common/src/global_settings.rs @@ -11,6 +11,10 @@ pub const NPM_CONFIG_REGISTRY_SETTING: &str = "npm_config_registry"; pub const BUNFIG_INSTALL_SCOPES_SETTING: &str = "bunfig_install_scopes"; pub const EXTRA_PIP_INDEX_URL_SETTING: &str = "pip_extra_index_url"; +pub const PIP_INDEX_URL_SETTING: &str = "pip_index_url"; +pub const SCIM_TOKEN_SETTING: &str = "scim_token"; +pub const SAML_METADATA_SETTING: &str = "saml_token"; + pub const UNIQUE_ID_SETTING: &str = "uid"; pub const DISABLE_STATS_SETTING: &str = "disable_stats"; pub const EXPOSE_METRICS_SETTING: &str = "expose_metrics"; diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index dcaa1dff28..4e720e7734 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -30,7 +30,6 @@ lazy_static::lazy_static! { std::env::var("FLOCK_PATH").unwrap_or_else(|_| "/usr/bin/flock".to_string()); static ref NON_ALPHANUM_CHAR: Regex = regex::Regex::new(r"[^0-9A-Za-z=.-]").unwrap(); - static ref PIP_INDEX_URL: Option = std::env::var("PIP_INDEX_URL").ok(); static ref PIP_TRUSTED_HOST: Option = std::env::var("PIP_TRUSTED_HOST").ok(); static ref PIP_INDEX_CERT: Option = std::env::var("PIP_INDEX_CERT").ok(); @@ -57,7 +56,8 @@ use crate::{ start_child_process, write_file, }, AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, HTTPS_PROXY, HTTP_PROXY, - LOCK_CACHE_DIR, NO_PROXY, NSJAIL_PATH, PATH_ENV, PIP_CACHE_DIR, PIP_EXTRA_INDEX_URL, TZ_ENV, + LOCK_CACHE_DIR, NO_PROXY, NSJAIL_PATH, PATH_ENV, PIP_CACHE_DIR, PIP_EXTRA_INDEX_URL, + PIP_INDEX_URL, TZ_ENV, }; pub async fn create_dependencies_dir(job_dir: &str) { @@ -161,7 +161,11 @@ pub async fn pip_compile( args.extend(["--extra-index-url", url, "--no-emit-index-url"]); pip_args.push(format!("--extra-index-url {}", url)); } - let pip_index_url = PIP_INDEX_URL.clone().map(handle_ephemeral_token); + let pip_index_url = PIP_INDEX_URL + .read() + .await + .clone() + .map(handle_ephemeral_token); if let Some(url) = pip_index_url.as_ref() { args.extend(["--index-url", url, "--no-emit-index-url"]); pip_args.push(format!("--index-url {}", url)); @@ -707,7 +711,12 @@ pub async fn handle_python_reqs( vars.push(("EXTRA_INDEX_URL", url)); } - pip_index_url = PIP_INDEX_URL.clone().map(handle_ephemeral_token); + pip_index_url = PIP_INDEX_URL + .read() + .await + .clone() + .map(handle_ephemeral_token); + if let Some(url) = pip_index_url.as_ref() { vars.push(("INDEX_URL", url)); } @@ -821,7 +830,11 @@ pub async fn handle_python_reqs( if let Some(url) = pip_extra_index_url.as_ref() { command_args.extend(["--extra-index-url", url]); } - let pip_index_url = PIP_INDEX_URL.clone().map(handle_ephemeral_token); + let pip_index_url = PIP_INDEX_URL + .read() + .await + .clone() + .map(handle_ephemeral_token); if let Some(url) = pip_index_url.as_ref() { command_args.extend(["--index-url", url]); diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 86a2e41a01..26cf657595 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -261,6 +261,7 @@ lazy_static::lazy_static! { pub static ref BUNFIG_INSTALL_SCOPES: Arc>> = Arc::new(RwLock::new(None)); pub static ref PIP_EXTRA_INDEX_URL: Arc>> = Arc::new(RwLock::new(None)); + pub static ref PIP_INDEX_URL: Arc>> = Arc::new(RwLock::new(None)); pub static ref JOB_DEFAULT_TIMEOUT: Arc>> = Arc::new(RwLock::new(None)); pub static ref TAR_CACHE_RATE: i32 = std::env::var("TAR_CACHE_RATE") @@ -1148,10 +1149,7 @@ pub async fn run_worker { + FlowModuleValue::RawScript { content, lock, path: spath, language, .. } => { if let Some(dedi_w) = spawn_dedicated_worker( SpawnWorker::RawScript { path: spath.clone().unwrap_or(path.to_string()), @@ -1837,16 +1823,8 @@ async fn spawn_dedicated_workers_for_flow( } enum SpawnWorker { - Script { - path: String, - hash: Option, - }, - RawScript { - path: String, - content: String, - lock: Option, - lang: ScriptLang, - }, + Script { path: String, hash: Option }, + RawScript { path: String, content: String, lock: Option, lang: ScriptLang }, } // spawn one dedicated worker and return the key, the channel sender and the join handle @@ -1927,12 +1905,7 @@ async fn spawn_dedicated_worker( return None; } } - SpawnWorker::RawScript { - content, - lock, - lang, - .. - } => (content, lock, Some(lang), None), + SpawnWorker::RawScript { content, lock, lang, .. } => (content, lock, Some(lang), None), }; match language { @@ -2843,12 +2816,7 @@ async fn get_script_content_by_hash( .fetch_optional(db) .await? .ok_or_else(|| Error::InternalErr(format!("expected content and lock")))?; - Ok(ContentReqLangEnvs { - content: r.0, - lockfile: r.1, - language: r.2, - envs: r.3, - }) + Ok(ContentReqLangEnvs { content: r.0, lockfile: r.1, language: r.2, envs: r.3 }) } #[tracing::instrument(level = "trace", skip_all)] @@ -2864,39 +2832,35 @@ async fn handle_code_execution_job( base_internal_url: &str, worker_name: &str, ) -> error::Result> { - let ContentReqLangEnvs { - content: inner_content, - lockfile: requirements_o, - language, - envs, - } = match job.job_kind { - JobKind::Preview => ContentReqLangEnvs { - content: job - .raw_code - .clone() - .unwrap_or_else(|| "no raw code".to_owned()), - lockfile: job.raw_lock.clone(), - language: job.language.to_owned(), - envs: None, - }, - JobKind::Script_Hub => { - get_hub_script_content_and_requirements(job.script_path.clone(), db).await? - } - JobKind::Script => { - get_script_content_by_hash( - &job.script_hash.unwrap_or(ScriptHash(0)), - &job.workspace_id, - db, - ) - .await? - } - JobKind::DeploymentCallback => { - get_script_content_by_path(job.script_path.clone(), &job.workspace_id, db).await? - } - _ => unreachable!( - "handle_code_execution_job should never be reachable with a non-code execution job" - ), - }; + let ContentReqLangEnvs { content: inner_content, lockfile: requirements_o, language, envs } = + match job.job_kind { + JobKind::Preview => ContentReqLangEnvs { + content: job + .raw_code + .clone() + .unwrap_or_else(|| "no raw code".to_owned()), + lockfile: job.raw_lock.clone(), + language: job.language.to_owned(), + envs: None, + }, + JobKind::Script_Hub => { + get_hub_script_content_and_requirements(job.script_path.clone(), db).await? + } + JobKind::Script => { + get_script_content_by_hash( + &job.script_hash.unwrap_or(ScriptHash(0)), + &job.workspace_id, + db, + ) + .await? + } + JobKind::DeploymentCallback => { + get_script_content_by_path(job.script_path.clone(), &job.workspace_id, db).await? + } + _ => unreachable!( + "handle_code_execution_job should never be reachable with a non-code execution job" + ), + }; if language == Some(ScriptLang::Postgresql) { return do_postgresql(job, &client, &inner_content, db).await; @@ -3430,10 +3394,7 @@ async fn handle_flow_dependency_job { let mut nbranches = vec![]; @@ -3589,10 +3547,7 @@ async fn lock_modules( token, ) .await?; - e.value = FlowModuleValue::BranchOne { - branches: nbranches, - default, - }; + e.value = FlowModuleValue::BranchOne { branches: nbranches, default }; } _ => (), }; @@ -3853,11 +3808,7 @@ async fn handle_app_dependency_job = { 'SSO/OAuth': [], Registries: [ { - label: 'Pip Extra Index Url', + label: 'Pip Index Url', description: 'Add private PIP registry', + key: 'pip_index_url', + fieldType: 'text', + placeholder: 'https://username:password@pypi.company.com/simple', + storage: 'setting', + ee_only: '' + }, + { + label: 'Pip Extra Index Url', + description: 'Add private extra PIP registry', key: 'pip_extra_index_url', fieldType: 'text', placeholder: 'https://username:password@pypi.company.com/simple',