From 0e80775d6d2bd1931fed6d5a1d63fc2e70980d55 Mon Sep 17 00:00:00 2001
From: pyranota <92104930+pyranota@users.noreply.github.com>
Date: Mon, 3 Feb 2025 21:31:55 +0300
Subject: [PATCH] feat(python): make S3 cache arch specific (#5196)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* add .minio-data to .gitignore

* feat(python): make S3 cache arch specific

Current schema:

S3-Bucket
├── python_311
│   ├── wheel==1.0
│   └── wheel2==1.0
└── python_312
    ├── wheel==1.0
    └── wheel2==1.0

New schema:

S3-Bucket
├── linux_aarch64
│   └── ...
└── linux_x86_64
    ├── python_311
    │   ├── wheel==1.0
    │   └── wheel2==1.0
    └── python_312
        ├── wheel==1.0
        └── wheel2==1.0

* remove .minio-data from .gitignore

* remove unneeded tracing::error
---
 .gitignore                                     | 2 +-
 backend/windmill-worker/src/global_cache.rs    | 7 +++++--
 backend/windmill-worker/src/python_executor.rs | 2 +-
 3 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/.gitignore b/.gitignore
index 27885185f1..3e6c2f3a75 100644
--- a/.gitignore
+++ b/.gitignore
@@ -7,4 +7,4 @@ CaddyfileRemoteMalo
 *.swp
 **/.idea/
 .direnv
-.vscode
\ No newline at end of file
+.vscode
diff --git a/backend/windmill-worker/src/global_cache.rs b/backend/windmill-worker/src/global_cache.rs
index 57ded13ee7..8239ab77d4 100644
--- a/backend/windmill-worker/src/global_cache.rs
+++ b/backend/windmill-worker/src/global_cache.rs
@@ -13,6 +13,9 @@ use windmill_common::error;
 #[cfg(all(feature = "enterprise", feature = "parquet", unix))]
 use std::sync::Arc;
 
+#[cfg(all(feature = "enterprise", feature = "parquet"))]
+pub const TARGET: &str = const_format::concatcp!(std::env::consts::OS, "_", std::env::consts::ARCH);
+
 #[cfg(all(feature = "enterprise", feature = "parquet", unix))]
 pub async fn build_tar_and_push(
     s3_client: Arc<dyn ObjectStore>,
@@ -57,7 +60,7 @@ pub async fn build_tar_and_push(
     if let Err(e) = s3_client
         .put(
             &Path::from(format!(
-                "/tar/{}/{folder_name}.tar",
+                "/tar/{TARGET}/{}/{folder_name}.tar",
                 if no_uv { "pip" } else { &python_xyz }
             )),
             std::fs::read(&tar_path)?.into(),
@@ -100,7 +103,7 @@ pub async fn pull_from_tar(
     let start = Instant::now();
 
     let tar_path = format!(
-        "tar/{}/{folder_name}.tar",
+        "tar/{TARGET}/{}/{folder_name}.tar",
         if no_uv { "pip".to_owned() } else { python_xyz }
     );
     let bytes = attempt_fetch_bytes(client, &tar_path).await?;
diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs
index 8c134166a0..a48e1cb396 100644
--- a/backend/windmill-worker/src/python_executor.rs
+++ b/backend/windmill-worker/src/python_executor.rs
@@ -632,7 +632,7 @@ pub async fn uv_pip_compile(
         if *NATIVE_CERT {
             args.extend(["--native-tls"]);
         }
-        tracing::error!("uv args: {:?}", args);
+        tracing::debug!("uv args: {:?}", args);
 
         #[cfg(windows)]
         let uv_cmd = "uv";