kalugin66/windmill
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add aws oidc support for instance s3 storage (#5810)

Browse Source 
* backend

* iterate

* all

* all

* all

* iterate

* revert

* all

* add tracing to get of authed client

* all

* all

* lal

* all

* update

* fix

* push

* all

* all

* revert

* frontend

* fix checks

* avoid deadlock

* safer

* fix

* fix
This commit is contained in:
Ruben Fiszel
2025-05-25 14:03:38 +02:00
committed by GitHub
parent 2833d6da3f
commit e15b94692d
44 changed files with 806 additions and 424 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
backend/windmill-worker/src/python_executor.rs
View File

@@ -70,7 +70,7 @@ const RELATIVE_PYTHON_LOADER: &str = include_str!("../loader.py");
use crate::global_cache::{build_tar_and_push, pull_from_tar};
#[cfg(all(feature = "enterprise", feature = "parquet", unix))]
use windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS;
use windmill_common::s3_helpers::OBJECT_STORE_SETTINGS;
use crate::{
common::{
@@ -79,9 +79,10 @@ use crate::{
},
handle_child::handle_child,
worker_utils::ping_job_status,
AuthedClient, PyV, PyVAlias, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV,
PyV, PyVAlias, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV,
PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, PROXY_ENVS, PY_INSTALL_DIR, TZ_ENV, UV_CACHE_DIR,
};
use windmill_common::client::AuthedClient;
#[cfg(windows)]
use crate::SYSTEM_ROOT;
@@ -1424,7 +1425,7 @@ pub async fn handle_python_reqs(
}
#[cfg(all(feature = "enterprise", feature = "parquet", unix))]
if OBJECT_STORE_CACHE_SETTINGS.read().await.is_none() {
if OBJECT_STORE_SETTINGS.read().await.is_none() {
(s3_pull, s3_push) = (false, false);
}
@@ -1735,7 +1736,7 @@ pub async fn handle_python_reqs(
let start = std::time::Instant::now();
#[cfg(all(feature = "enterprise", feature = "parquet", unix))]
if is_not_pro {
if let Some(os) = OBJECT_STORE_CACHE_SETTINGS.read().await.clone() {
if let Some(os) = windmill_common::s3_helpers::get_object_store().await {
tokio::select! {
// Cancel was called on the job
_ = kill_rx.recv() => return Err(anyhow::anyhow!("S3 pull was canceled")),
@@ -1889,7 +1890,7 @@ pub async fn handle_python_reqs(
#[cfg(all(feature = "enterprise", feature = "parquet", unix))]
if s3_push {
if let Some(os) = OBJECT_STORE_CACHE_SETTINGS.read().await.clone() {
if let Some(os) = windmill_common::s3_helpers::get_object_store().await {
tokio::spawn(build_tar_and_push(os, venv_p.clone(), py_version.to_cache_dir_top_level(), None, false));
}
}