feat: replace polars with datafusion for data preview

backend/Cargo.toml

@@ -65,7 +65,6 @@ windmill-worker.workspace = true
 futures.workspace = true
 tracing.workspace = true
 sqlx.workspace = true
-tokio-metrics.workspace = true
 rand.workspace = true
 chrono.workspace = true
 git-version.workspace = true
@@ -191,7 +190,6 @@ dotenv = "^0"
 ulid = { version = "^1", features = ["uuid"] }
 futures = "^0"
 futures-core = "^0"
-tokio-metrics = "0.1.0"
 lazy_static = "1.4.0"
 serde_derive = "1.0.147"
 const_format = { version = "0.2", features = ["rust_1_64", "rust_1_51"] }
@@ -236,8 +234,7 @@ tiberius = { git = "https://github.com/prisma/tiberius", rev = "8f66a699dfa041e7
 pin-project = "1"
 indexmap = { version = "2.2.5", features = ["serde"]}
 
-polars = { version = "0.38.3", features = ["lazy", "parquet", "aws", "azure", "csv", "dtype-full", "serde", "strings", "extract_groups"] }
-polars-io = { version = "0.38.3", features = ["csv"] }
+datafusion = "38.0.0"
 object_store = { version = "0.9.1", features = ["aws", "azure"] }
 openidconnect = { version = "3.4.0" }
 

backend/ee-repo-ref.txt

@@ -1 +1 @@
-982774ea62ab95914bd0502aac8c5c9b5a91b5c2
+50475eab460efad2aa149e11437dbe98acea52a5

backend/src/main.rs

@@ -687,8 +687,8 @@ pub async fn run_workers<R: rsmq_async::RsmqConnection + Send + Sync + Clone + '
         })
         .unwrap_or_else(|| rd_string(5));
 
-    #[cfg(tokio_unstable)]
-    let monitor = tokio_metrics::TaskMonitor::new();
+    // #[cfg(tokio_unstable)]
+    // let monitor = tokio_metrics::TaskMonitor::new();
 
     let ip = windmill_common::external_ip::get_ip()
         .await
@@ -747,15 +747,15 @@ pub async fn run_workers<R: rsmq_async::RsmqConnection + Send + Sync + Clone + '
                 agent_mode,
             );
 
-            #[cfg(tokio_unstable)]
-            {
-                monitor.monitor(f, "worker").await
-            }
+            // #[cfg(tokio_unstable)]
+            // {
+            //     monitor.monitor(f, "worker").await
+            // }
 
-            #[cfg(not(tokio_unstable))]
-            {
-                f.await
-            }
+            // #[cfg(not(tokio_unstable))]
+            // {
+            f.await
+            // }
         }));
     }
 

backend/windmill-api/Cargo.toml

@@ -15,7 +15,7 @@ stripe = ["dep:async-stripe"]
 enterprise_saml = ["dep:samael"]
 benchmark = []
 embedding = ["dep:tinyvector", "dep:hf-hub", "dep:tokenizers", "dep:candle-core", "dep:candle-transformers", "dep:candle-nn"]
-parquet = ["dep:polars", "dep:polars-io", "dep:object_store", "windmill-common/parquet"]
+parquet = ["dep:datafusion", "dep:object_store", "dep:url", "windmill-common/parquet"]
 prometheus = ["windmill-common/prometheus", "windmill-queue/prometheus", "dep:prometheus"]
 openidconnect = ["dep:openidconnect"]
 
@@ -82,10 +82,11 @@ tokenizers = { workspace = true, optional = true}
 candle-core = { workspace = true, optional = true}
 candle-transformers = { workspace = true, optional = true}
 candle-nn = { workspace = true, optional = true}
-polars = { workspace = true, optional = true}
-polars-io = { workspace = true, optional = true}
+datafusion = { workspace = true, optional = true}
 object_store = { workspace = true, optional = true}
 openidconnect = { workspace = true, optional = true}
+url = { workspace = true, optional = true}
+
 pin-project.workspace = true
 crc.workspace = true
 http.workspace = true

backend/windmill-common/src/ee.rs

@@ -26,12 +26,6 @@ pub enum CriticalErrorChannel {}
 
 pub async fn trigger_critical_error_channels(_error_message: String) {}
 
-#[cfg(feature = "enterprise")]
-pub async fn renew_license_key(_http_client: &reqwest::Client, _db: &crate::db::DB) -> String {
-    // Implementation is not open source
-    "".to_string()
-}
-
 #[cfg(feature = "enterprise")]
 pub async fn schedule_key_renewal(_http_client: &reqwest::Client, _db: &crate::db::DB) -> () {
     // Implementation is not open source

backend/windmill-common/src/ee.rs~Updated upstream

@@ -0,0 +1,38 @@
+use crate::ee::LicensePlan::Community;
+use serde::Deserialize;
+use std::sync::Arc;
+use tokio::sync::RwLock;
+
+lazy_static::lazy_static! {
+  pub static ref LICENSE_KEY_VALID: Arc<RwLock<bool>> = Arc::new(RwLock::new(true));
+  pub static ref LICENSE_KEY_ID: Arc<RwLock<String>> = Arc::new(RwLock::new("".to_string()));
+  pub static ref LICENSE_KEY: Arc<RwLock<String>> = Arc::new(RwLock::new("".to_string()));
+}
+
+pub enum LicensePlan {
+    Community,
+    Pro,
+    Enterprise,
+}
+
+pub async fn get_license_plan() -> LicensePlan {
+    // Implementation is not open source
+    return Community;
+}
+
+#[derive(Deserialize)]
+#[serde(untagged)]
+pub enum CriticalErrorChannel {}
+
+pub async fn trigger_critical_error_channels(_error_message: String) {}
+
+#[cfg(feature = "enterprise")]
+pub async fn renew_license_key(_http_client: &reqwest::Client, _db: &crate::db::DB) -> String {
+    // Implementation is not open source
+    "".to_string()
+}
+
+#[cfg(feature = "enterprise")]
+pub async fn schedule_key_renewal(_http_client: &reqwest::Client, _db: &crate::db::DB) -> () {
+    // Implementation is not open source
+}

cli/main.ts

@@ -10,7 +10,7 @@ import {
 import flow from "./flow.ts";
 import app from "./apps.ts";
 import script from "./script.ts";
-import workspace from "./workspace.ts";
+import workspace, { getActiveWorkspace } from "./workspace.ts";
 import resource from "./resource.ts";
 import user from "./user.ts";
 import variable from "./variable.ts";
@@ -21,7 +21,7 @@ import folder from "./folder.ts";
 import schedule from "./schedule.ts";
 import sync from "./sync.ts";
 import dev from "./dev.ts";
-import { tryResolveVersion } from "./context.ts";
+import { fetchVersion, tryResolveVersion } from "./context.ts";
 import { GlobalOptions } from "./types.ts";
 import { OpenAPI } from "./deps.ts";
 import { getHeaders } from "./utils.ts";
@@ -93,11 +93,14 @@ let command: any = new Command()
   .command("version", "Show version information")
   .action(async (opts) => {
     console.log("CLI build against " + VERSION);
-    const backendVersion = await tryResolveVersion(opts as GlobalOptions);
-    if (backendVersion) {
+    const workspace = await getActiveWorkspace(opts as GlobalOptions);
+    if (workspace) {
+      const backendVersion = await fetchVersion(workspace.remote);
       console.log("Backend Version: " + backendVersion);
     } else {
-      console.log("Cannot resolve Backend Version");
+      console.log(
+        "Cannot fetch backend version: no active workspace selected, choose one to pick a remote to fetch version of"
+      );
     }
   })
   .command(

frontend/src/lib/components/ParqetTableRenderer.svelte

@@ -28,23 +28,18 @@
 					searchCol: searchCol,
 					searchTerm: searchCol ? params.filterModel?.[searchCol]?.filter : undefined
 				})) as any
+				for (let i = 0; i < res.rows.length; i++) {
+					res.rows[i]['__index'] = i + params.startRow
+					if (!$enterpriseLicense) {
+						Object.keys(res.rows[i]).forEach((key) => {
+							if (key != '__index') {
+								res.rows[i][key] = 'Require EE'
+							}
+						})
+					}
+				}
 
-				const data: any[] = []
-
-				res?.columns?.forEach((c) => {
-					c.values.forEach((v, i) => {
-						if (data[i] == undefined) {
-							data.push({ __index: params.startRow + i })
-						}
-						if (!$enterpriseLicense && params.startRow + i > 3) {
-							data[i][c.name] = 'Require EE'
-						} else {
-							data[i][c.name] = v
-						}
-					})
-				})
-
-				params.successCallback(data)
+				params.successCallback(res.rows)
 			} catch (e) {
 				console.error(e)
 				params.failCallback()
@@ -87,7 +82,7 @@
 					// @ts-ignore
 					columnDefs: res.columns.map((c) => {
 						return {
-							field: c.name,
+							field: c,
 							sortable: true,
 							filter: true,
 							filterParams: {