fix: minor fixes to private NPM and python registries to get everything working (#2728)

backend/windmill-worker/src/bun_executor.rs

@@ -16,7 +16,7 @@ use crate::{
         start_child_process, write_file, write_file_binary,
     },
     AuthedClientBackgroundTask, BUN_CACHE_DIR, BUN_PATH, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV,
-    NPM_CONFIG_REGISTRY, NSJAIL_PATH, PATH_ENV, TZ_ENV,
+    NSJAIL_PATH, PATH_ENV, TZ_ENV,
 };
 
 use tokio::{
@@ -49,8 +49,6 @@ pub const EMPTY_FILE: &str = "<empty>";
 
 lazy_static::lazy_static! {
     pub static ref TRUSTED_DEP: Regex = Regex::new(r"//\s?trustedDependencies:(.*)\n").unwrap();
-
-    static ref BUN_TLS_REJECT_UNAUTHORIZED: String = std::env::var("NODE_TLS_REJECT_UNAUTHORIZED").ok().unwrap_or_else(|| String::new());
 }
 
 pub async fn gen_lockfile(
@@ -514,7 +512,7 @@ plugin(p)
 }
 
 pub async fn get_common_bun_proc_envs(base_internal_url: &str) -> HashMap<String, String> {
-    let mut bun_envs: HashMap<String, String> = HashMap::from([
+    let bun_envs: HashMap<String, String> = HashMap::from([
         (String::from("PATH"), PATH_ENV.clone()),
         (String::from("HOME"), HOME_ENV.clone()),
         (String::from("TZ"), TZ_ENV.clone()),
@@ -531,16 +529,6 @@ pub async fn get_common_bun_proc_envs(base_internal_url: &str) -> HashMap<String
             BUN_CACHE_DIR.to_string(),
         ),
     ]);
-
-    if let Some(ref s) = NPM_CONFIG_REGISTRY.read().await.clone() {
-        bun_envs.insert(String::from("NPM_CONFIG_REGISTRY"), s.clone());
-    }
-    if BUN_TLS_REJECT_UNAUTHORIZED.len() > 0 {
-        bun_envs.insert(
-            String::from("NODE_TLS_REJECT_UNAUTHORIZED"),
-            BUN_TLS_REJECT_UNAUTHORIZED.clone(),
-        );
-    }
     return bun_envs;
 }
 

backend/windmill-worker/src/python_executor.rs

@@ -122,8 +122,8 @@ pub async fn pip_compile(
     if let Some(host) = PIP_TRUSTED_HOST.as_ref() {
         args.extend(["--trusted-host", host]);
     }
-    if let Some(host) = PIP_INDEX_CERT.as_ref() {
-        args.extend(["--cert", host]);
+    if let Some(cert_path) = PIP_INDEX_CERT.as_ref() {
+        args.extend(["--cert", cert_path]);
     }
 
     let mut child_cmd = Command::new("pip-compile");
@@ -609,6 +609,9 @@ pub async fn handle_python_reqs(
         if let Some(url) = PIP_INDEX_URL.as_ref() {
             vars.push(("INDEX_URL", url));
         }
+        if let Some(cert_path) = PIP_INDEX_CERT.as_ref() {
+            vars.push(("PIP_INDEX_CERT", cert_path));
+        }
         if let Some(host) = PIP_TRUSTED_HOST.as_ref() {
             vars.push(("TRUSTED_HOST", host));
         }
@@ -708,6 +711,9 @@ pub async fn handle_python_reqs(
             if let Some(url) = PIP_INDEX_URL.as_ref() {
                 command_args.extend(["--index-url", url]);
             }
+            if let Some(cert_path) = PIP_INDEX_CERT.as_ref() {
+                command_args.extend(["--cert", cert_path]);
+            }
             if let Some(host) = PIP_TRUSTED_HOST.as_ref() {
                 command_args.extend(["--trusted-host", &host]);
             }

examples/deploy/private-package-registry-tls/README.md

@@ -80,14 +80,16 @@ python setup.py sdist
 twine upload --repository-url https://localhost/ dist/* --cert ../certs/windmill-root.crt
 # no username and password, just press enter. For the purpose of the demo we're running pypiserver completely unauthenticated
 ```
-You can check that the package is uploaded by visiting [https://localhost/simple](https://localhost/pypi/simple).
+You can check that the package is uploaded by visiting [https://localhost/simple](https://localhost/simple).
 
 Go to Windmill at `http://localhost:8000`. Create a simple Python script:
 ```python
+#requirements:
+#windmill-helloworld==0.0.1
 import windmill_helloworld
 
 def main():
-  print(windmill_helloworld.say_hello("Windmill"))
+    print(windmill_helloworld.say_hello("Windmill"))
 ```
 and execute it. It should return successfully with:
 ```

examples/deploy/private-package-registry-tls/docker-compose.yml

@@ -48,11 +48,15 @@ services:
     environment:
       - DATABASE_URL=postgres://postgres:changeme@db/windmill?sslmode=disable
       - WORKER_TAGS=deno,go,python3,bash,flow,hub,dependency,nativets
-      - NPM_CONFIG_REGISTRY=https://caddy/npm/
+
+      # For DENO and REST scripts:
+      - NPM_CONFIG_REGISTRY=http://caddy/npm/
       - DENO_CERT=/custom-certs/windmill-root.crt # this will make deno trust this RootCA for all sessions
       # - DENO_TLS_CA_STORE=system # alternatively, you can use this but you'll need to manually trust the RootCA at the host level, see README.md
+
+      # For Python scripts:
+      - PIP_INDEX_URL=https://caddy/simple/
       - PIP_INDEX_CERT=/custom-certs/windmill-root.crt # this will make pip trust this RootCA for all sessions
-      - BUN_TLS_REJECT_UNAUTHORIZED=0 # this will make bun ignore TLS errors. Bun does not support trusting an additional RootCA yet
     volumes:
       - ./certs:/custom-certs
     depends_on:

examples/deploy/private-package-registry-tls/helloworld_python_module/setup.py

@@ -7,12 +7,12 @@ module_path = os.path.join(os.path.dirname(__file__), "windmill_helloworld.py")
 setuptools.setup(
     name="windmill-helloworld",
     version="0.0.1",
-    url="https://github.com/windmill-labs/windmill/blob/exit()/examples/deploy/private-npm-registry-tls/README.md",
+    url="https://github.com/windmill-labs/windmill/blob//examples/deploy/private-package-registry-tls/README.md",
     author="WindmillLabs",
     author_email="contact@windmill.dev",
     description="Simple hello world python module to host on a private Pypi server",
     long_description=open("README.md").read(),
-    py_modules=["helloworld_python_module"],
+    py_modules=["windmill_helloworld"],
     zip_safe=False,
     platforms="any",
     install_requires=[],

examples/deploy/private-package-registry-tls/verdaccio_conf/config.yaml

@@ -11,6 +11,7 @@ packages:
     access: $all
     publish: $authenticated
   "**":
+    access: $all
     proxy: npmjs
 listen:
   - http://localhost:4873