repro mem

improve history from the script editor
fix: remove result and args from list completed and list queue jobs
2023-09-10 00:24:36 +02:00 · 2023-09-09 09:44:24 +02:00 · 2023-09-09 09:31:45 +02:00 · 2023-09-09 00:42:18 +02:00 · 2023-09-09 00:37:15 +02:00 · 2023-09-08 19:00:46 +02:00
86 changed files with 2219 additions and 472 deletions
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,36 @@
 # Changelog


+## [1.170.0](https://github.com/windmill-labs/windmill/compare/v1.169.0...v1.170.0) (2023-09-08)
+
+
+### Features
+
+* display jobs currently waiting for a worker ([3c950c0](https://github.com/windmill-labs/windmill/commit/3c950c03de0bc71974eb29985381adba8c098660))
+* snowflake schema explorer + refactoring ([#2260](https://github.com/windmill-labs/windmill/issues/2260)) ([5cca583](https://github.com/windmill-labs/windmill/commit/5cca5833e94fc4c8a80e210164da09f2a1ceb677))
+
+
+### Bug Fixes
+
+* fix get_result for python-client ([fe41f4f](https://github.com/windmill-labs/windmill/commit/fe41f4ff4ce596cf394bd69a0ba48e88db8d2328))
+
+## [1.169.0](https://github.com/windmill-labs/windmill/compare/v1.168.3...v1.169.0) (2023-09-08)
+
+
+### Features
+
+* benchmarks graph ([#2244](https://github.com/windmill-labs/windmill/issues/2244)) ([c496602](https://github.com/windmill-labs/windmill/commit/c496602e9e2e0dfecaaffe731e58e551d039d02f))
+* big query schema explorer ([#2247](https://github.com/windmill-labs/windmill/issues/2247)) ([ec7d923](https://github.com/windmill-labs/windmill/commit/ec7d923cca0f6050855473ababd1bb27d668711b))
+* flow copilot ([#2219](https://github.com/windmill-labs/windmill/issues/2219)) ([2f3138c](https://github.com/windmill-labs/windmill/commit/2f3138c65d9d3f0161bf3e069c6eec0c32ac3b86))
+* **frontend:** fix runs page when the row has a parent ([#2255](https://github.com/windmill-labs/windmill/issues/2255)) ([2271263](https://github.com/windmill-labs/windmill/commit/22712632f683fb63ad6d4b475a01c63800a9559d))
+* introduce container groups ([49c5553](https://github.com/windmill-labs/windmill/commit/49c5553f3b496c2aaf03376689ee0fd42ecbd2bf))
+
+
+### Bug Fixes
+
+* benchmark svg ([#2249](https://github.com/windmill-labs/windmill/issues/2249)) ([24c5802](https://github.com/windmill-labs/windmill/commit/24c580211572d6447ca502db141e90c5e084d790))
+* pass TZ from env to runtimes ([75a1490](https://github.com/windmill-labs/windmill/commit/75a149009a5a13230b4d6de6eac8bba0618629d6))
+
 ## [1.168.3](https://github.com/windmill-labs/windmill/compare/v1.168.2...v1.168.3) (2023-09-07)


--- a/2
+++ b/2
@@ -180,7 +180,7 @@ RUN chmod 755 /usr/bin/deno

 COPY --from=nsjail /nsjail/nsjail /bin/nsjail

-COPY --from=oven/bun:0.8.0 /usr/local/bin/bun /usr/bin/bun
+COPY --from=oven/bun:1.0.0 /usr/local/bin/bun /usr/bin/bun

 # add the docker client to call docker from a worker if enabled
 COPY --from=docker:dind /usr/local/bin/docker /usr/local/bin/
--- a/backend/.sqlx/query-f5681df5935ec753f4a6d9ab8b927bbf890f0b4fe2de16dc764b24f24d7bbdca.json
+++ b/backend/.sqlx/query-f5681df5935ec753f4a6d9ab8b927bbf890f0b4fe2de16dc764b24f24d7bbdca.json
@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT coalesce(COUNT(*), 0) as \"database_length!\" FROM completed_job WHERE workspace_id = $1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "database_length!",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "f5681df5935ec753f4a6d9ab8b927bbf890f0b4fe2de16dc764b24f24d7bbdca"
+}
--- a/backend/Cargo.lock
+++ b/backend/Cargo.lock
@@ -604,7 +604,7 @@ dependencies = [
 "lazycell",
 "log",
 "peeking_take_while",
- "prettyplease 0.2.14",
+ "prettyplease 0.2.15",
 "proc-macro2",
 "quote",
 "regex",
@@ -858,9 +858,9 @@ checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"

 [[package]]
 name = "bytes"
-version = "1.4.0"
+version = "1.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be"
+checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223"

 [[package]]
 name = "bzip2"
@@ -891,7 +891,7 @@ checksum = "e11c675378efb449ed3ce8de78d75d0d80542fc98487c26aba28eb3b82feac72"
 dependencies = [
 "semver 1.0.18",
 "serde",
- "toml 0.7.6",
+ "toml 0.7.7",
 "url",
 ]

@@ -922,16 +922,15 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"

 [[package]]
 name = "chrono"
-version = "0.4.29"
+version = "0.4.30"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d87d9d13be47a5b7c3907137f1290b0459a7f80efb26be8c52afb11963bccb02"
+checksum = "defd4e7873dbddba6c7c91e199c7fcb946abc4a6a4ac3195400bcfb01b5de877"
 dependencies = [
 "android-tzdata",
 "iana-time-zone",
 "js-sys",
 "num-traits",
 "serde",
- "time 0.1.45",
 "wasm-bindgen",
 "windows-targets",
 ]
@@ -1115,7 +1114,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7efb37c3e1ccb1ff97164ad95ac1606e8ccd35b3fa0a7d99a304c7f4a428cc24"
 dependencies = [
 "percent-encoding",
- "time 0.3.28",
+ "time",
 "version_check",
 ]

@@ -2141,7 +2140,7 @@ dependencies = [
 "serde",
 "serde_json",
 "thiserror",
- "time 0.3.28",
+ "time",
 "tokio",
 "tracing",
 "tracing-futures",
@@ -3222,7 +3221,7 @@ dependencies = [
 "smallvec",
 "subprocess",
 "thiserror",
- "time 0.3.28",
+ "time",
 "uuid 1.4.1",
 ]

@@ -3879,9 +3878,9 @@ dependencies = [

 [[package]]
 name = "prettyplease"
-version = "0.2.14"
+version = "0.2.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8832c0f9be7e3cae60727e6256cfd2cd3c3e2b6cd5dad4190ecb2fd658c9030b"
+checksum = "ae005bd773ab59b4725093fd7df83fd7892f7d8eafb48dbd7de6e024e4215f9d"
 dependencies = [
 "proc-macro2",
 "syn 2.0.31",
@@ -5140,7 +5139,7 @@ dependencies = [
 "num-bigint",
 "num-traits",
 "thiserror",
- "time 0.3.28",
+ "time",
 ]

 [[package]]
@@ -6073,17 +6072,6 @@ dependencies = [
 "digest 0.9.0",
 ]

-[[package]]
-name = "time"
-version = "0.1.45"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1b797afad3f312d1c66a56d11d0316f916356d11bd158fbc6ca6389ff6bf805a"
-dependencies = [
- "libc",
- "wasi 0.10.0+wasi-snapshot-preview1",
- "winapi",
-]
-
 [[package]]
 name = "time"
 version = "0.3.28"
@@ -6287,9 +6275,9 @@ dependencies = [

 [[package]]
 name = "toml"
-version = "0.7.6"
+version = "0.7.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c17e963a819c331dcacd7ab957d80bc2b9a9c1e71c804826d2f283dd65306542"
+checksum = "de0a3ab2091e52d7299a39d098e200114a972df0a7724add02a273aa9aada592"
 dependencies = [
 "serde",
 "serde_spanned",
@@ -6308,9 +6296,9 @@ dependencies = [

 [[package]]
 name = "toml_edit"
-version = "0.19.14"
+version = "0.19.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f8123f27e969974a3dfba720fdb560be359f57b44302d280ba72e76a74480e8a"
+checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421"
 dependencies = [
 "indexmap 2.0.0",
 "serde",
@@ -6863,12 +6851,6 @@ version = "0.9.0+wasi-snapshot-preview1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519"

-[[package]]
-name = "wasi"
-version = "0.10.0+wasi-snapshot-preview1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f"
-
 [[package]]
 name = "wasi"
 version = "0.11.0+wasi-snapshot-preview1"
@@ -7086,7 +7068,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"

 [[package]]
 name = "windmill"
-version = "1.168.3"
+version = "1.170.0"
 dependencies = [
 "anyhow",
 "axum",
@@ -7121,7 +7103,7 @@ dependencies = [

 [[package]]
 name = "windmill-api"
-version = "1.168.3"
+version = "1.170.0"
 dependencies = [
 "anyhow",
 "argon2",
@@ -7161,7 +7143,7 @@ dependencies = [
 "sql-builder",
 "sqlx",
 "tempfile",
- "time 0.3.28",
+ "time",
 "tokio",
 "tokio-tar",
 "tokio-util",
@@ -7180,7 +7162,7 @@ dependencies = [

 [[package]]
 name = "windmill-api-client"
-version = "1.168.3"
+version = "1.170.0"
 dependencies = [
 "base64 0.21.3",
 "chrono",
@@ -7198,7 +7180,7 @@ dependencies = [

 [[package]]
 name = "windmill-audit"
-version = "1.168.3"
+version = "1.170.0"
 dependencies = [
 "chrono",
 "serde",
@@ -7211,7 +7193,7 @@ dependencies = [

 [[package]]
 name = "windmill-common"
-version = "1.168.3"
+version = "1.170.0"
 dependencies = [
 "anyhow",
 "axum",
@@ -7237,7 +7219,7 @@ dependencies = [

 [[package]]
 name = "windmill-parser"
-version = "1.168.3"
+version = "1.170.0"
 dependencies = [
 "serde",
 "serde_json",
@@ -7245,7 +7227,7 @@ dependencies = [

 [[package]]
 name = "windmill-parser-bash"
-version = "1.168.3"
+version = "1.170.0"
 dependencies = [
 "anyhow",
 "lazy_static",
@@ -7256,7 +7238,7 @@ dependencies = [

 [[package]]
 name = "windmill-parser-go"
-version = "1.168.3"
+version = "1.170.0"
 dependencies = [
 "anyhow",
 "gosyn",
@@ -7268,7 +7250,7 @@ dependencies = [

 [[package]]
 name = "windmill-parser-graphql"
-version = "1.168.3"
+version = "1.170.0"
 dependencies = [
 "anyhow",
 "lazy_static",
@@ -7279,7 +7261,7 @@ dependencies = [

 [[package]]
 name = "windmill-parser-py"
-version = "1.168.3"
+version = "1.170.0"
 dependencies = [
 "anyhow",
 "itertools 0.11.0",
@@ -7290,7 +7272,7 @@ dependencies = [

 [[package]]
 name = "windmill-parser-py-imports"
-version = "1.168.3"
+version = "1.170.0"
 dependencies = [
 "anyhow",
 "async-recursion",
@@ -7307,7 +7289,7 @@ dependencies = [

 [[package]]
 name = "windmill-parser-sql"
-version = "1.168.3"
+version = "1.170.0"
 dependencies = [
 "anyhow",
 "lazy_static",
@@ -7318,7 +7300,7 @@ dependencies = [

 [[package]]
 name = "windmill-parser-ts"
-version = "1.168.3"
+version = "1.170.0"
 dependencies = [
 "anyhow",
 "convert_case 0.6.0",
@@ -7335,7 +7317,7 @@ dependencies = [

 [[package]]
 name = "windmill-parser-wasm"
-version = "1.168.3"
+version = "1.170.0"
 dependencies = [
 "anyhow",
 "getrandom 0.2.10",
@@ -7353,7 +7335,7 @@ dependencies = [

 [[package]]
 name = "windmill-queue"
-version = "1.168.3"
+version = "1.170.0"
 dependencies = [
 "anyhow",
 "async-recursion",
@@ -7382,7 +7364,7 @@ dependencies = [

 [[package]]
 name = "windmill-worker"
-version = "1.168.3"
+version = "1.170.0"
 dependencies = [
 "anyhow",
 "async-recursion",
--- a/backend/Cargo.toml
+++ b/backend/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "windmill"
-version = "1.168.3"
+version = "1.170.0"
 authors.workspace = true
 edition.workspace = true

@@ -22,7 +22,7 @@ members = [
 ]

 [workspace.package]
-version = "1.168.3"
+version = "1.170.0"
 authors = ["Ruben Fiszel <ruben@windmill.dev>"]
 edition = "2021"

--- a/backend/windmill-api/openapi.yaml
+++ b/backend/windmill-api/openapi.yaml
@@ -1,7 +1,7 @@
 openapi: "3.0.3"

 info:
-  version: 1.168.3
+  version: 1.170.0
  title: Windmill API

  contact:
@@ -2386,6 +2386,46 @@ paths:
                  - content
                  - language

+  /scripts/hub/query:
+    get:
+      summary: query hub scripts by similarity
+      operationId: queryHubScripts
+      tags:
+        - script
+      parameters:
+        - name: text
+          description: query text
+          in: query
+          required: true
+          schema:
+            type: string
+        - name: kind
+          description: query scripts kind
+          in: query
+          required: false
+          schema:
+            type: string
+        - name: limit
+          description: query limit
+          in: query
+          required: false
+          schema:
+            type: number
+      responses:
+        "200":
+          description: script details
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  type: object
+                  properties:
+                    id:
+                      type: string
+                  required:
+                    - id
+
  /w/{workspace}/scripts/list:
    get:
      summary: list all available scripts
@@ -3914,6 +3954,27 @@ paths:
                required:
                  - database_length

+  /w/{workspace}/jobs/completed/count:
+    get:
+      summary: get completed count
+      operationId: getCompletedCount
+      tags:
+        - job
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+      responses:
+        "200":
+          description: completed count
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  database_length:
+                    type: integer
+                required:
+                  - database_length
+
  /w/{workspace}/jobs/queue/cancel_all:
    post:
      summary: cancel all jobs
--- a/backend/windmill-api/src/apps.rs
+++ b/backend/windmill-api/src/apps.rs
@@ -508,6 +508,7 @@ pub async fn get_hub_app_by_id(
        &format!("https://hub.windmill.dev/apps/{id}/json"),
        &email,
        false,
+        None,
    )
    .await?
    .json()
--- a/backend/windmill-api/src/flows.rs
+++ b/backend/windmill-api/src/flows.rs
@@ -160,6 +160,7 @@ pub async fn get_hub_flow_by_id(
        &format!("https://hub.windmill.dev/flows/{id}/json"),
        &email,
        false,
+        None,
    )
    .await?
    .json()
--- a/backend/windmill-api/src/jobs.rs
+++ b/backend/windmill-api/src/jobs.rs
@@ -110,6 +110,7 @@ pub fn workspaced_service() -> Router {
        .route("/queue/list", get(list_queue_jobs))
        .route("/queue/count", get(count_queue_jobs))
        .route("/queue/cancel_all", post(cancel_all))
+        .route("/completed/count", get(count_completed_jobs))
        .route(
            "/completed/list",
            get(list_completed_jobs).layer(cors.clone()),
@@ -631,6 +632,7 @@ struct ListableQueuedJob {
    pub language: Option<ScriptLang>,
    pub email: String,
    pub suspend: Option<i32>,
+    pub tag: String,
 }

 async fn list_queue_jobs(
@@ -649,7 +651,7 @@ async fn list_queue_jobs(
            "scheduled_for",
            "script_hash",
            "script_path",
-            "args",
+            "null as args",
            "job_kind",
            "schedule_path",
            "permissioned_as",
@@ -658,6 +660,7 @@ async fn list_queue_jobs(
            "same_worker",
            "email",
            "suspend",
+            "tag",
        ],
    )
    .sql()?;
@@ -704,6 +707,21 @@ async fn count_queue_jobs(
    ))
 }

+async fn count_completed_jobs(
+    Extension(db): Extension<DB>,
+    Path(w_id): Path<String>,
+) -> error::JsonResult<QueueStats> {
+    Ok(Json(
+        sqlx::query_as!(
+            QueueStats,
+            "SELECT coalesce(COUNT(*), 0) as \"database_length!\" FROM completed_job WHERE workspace_id = $1",
+            w_id
+        )
+        .fetch_one(&db)
+        .await?,
+    ))
+}
+
 async fn list_jobs(
    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
@@ -712,64 +730,10 @@ async fn list_jobs(
    Query(lq): Query<ListCompletedQuery>,
 ) -> error::JsonResult<Vec<Job>> {
    check_scopes(&authed, || format!("listjobs"))?;
-    // TODO: todo!("rewrite this to just run list_queue_jobs and list_completed_jobs separately and return as one");
+
    let (per_page, offset) = paginate(pagination);
    let lqc = lq.clone();
-    let sqlq = list_queue_jobs_query(
-        &w_id,
-        &ListQueueQuery {
-            script_path_start: lq.script_path_start,
-            script_path_exact: lq.script_path_exact,
-            script_hash: lq.script_hash,
-            created_by: lq.created_by,
-            started_before: lq.started_before,
-            started_after: lq.started_after,
-            created_before: lq.created_before,
-            created_after: lq.created_after,
-            created_or_started_before: lq.created_or_started_before,
-            created_or_started_after: lq.created_or_started_after,
-            running: None,
-            parent_job: lq.parent_job,
-            order_desc: Some(true),
-            job_kinds: lq.job_kinds,
-            suspended: lq.suspended,
-            args: lq.args,
-            tag: lq.tag,
-            schedule_path: lq.schedule_path,
-        },
-        &[
-            "'QueuedJob' as typ",
-            "id",
-            "workspace_id",
-            "parent_job",
-            "created_by",
-            "created_at",
-            "started_at",
-            "scheduled_for",
-            "running",
-            "script_hash",
-            "script_path",
-            "null as args",
-            "null as duration_ms",
-            "null as success",
-            "false as deleted",
-            "canceled",
-            "canceled_by",
-            "job_kind",
-            "schedule_path",
-            "permissioned_as",
-            "is_flow_step",
-            "language",
-            "false as is_skipped",
-            "email",
-            "visible_to_owner",
-            "suspend",
-            "mem_peak",
-            "tag",
-            "concurrent_limit",
-            "concurrency_time_window_s",
-        ],
-    );
+
    let sqlc = list_completed_jobs_query(
        &w_id,
        per_page + offset,
@@ -808,13 +772,74 @@ async fn list_jobs(
            "null as concurrency_time_window_s",
        ],
    );
-    let sql = format!(
-        "{} UNION ALL {} ORDER BY created_at DESC LIMIT {} OFFSET {};",
-        &sqlq.subquery()?,
-        &sqlc.subquery()?,
-        per_page,
-        offset
-    );
+
+    let sql = if lq.success.is_none() {
+        let sqlq = list_queue_jobs_query(
+            &w_id,
+            &ListQueueQuery {
+                script_path_start: lq.script_path_start,
+                script_path_exact: lq.script_path_exact,
+                script_hash: lq.script_hash,
+                created_by: lq.created_by,
+                started_before: lq.started_before,
+                started_after: lq.started_after,
+                created_before: lq.created_before,
+                created_after: lq.created_after,
+                created_or_started_before: lq.created_or_started_before,
+                created_or_started_after: lq.created_or_started_after,
+                running: None,
+                parent_job: lq.parent_job,
+                order_desc: Some(true),
+                job_kinds: lq.job_kinds,
+                suspended: lq.suspended,
+                args: lq.args,
+                tag: lq.tag,
+                schedule_path: lq.schedule_path,
+            },
+            &[
+                "'QueuedJob' as typ",
+                "id",
+                "workspace_id",
+                "parent_job",
+                "created_by",
+                "created_at",
+                "started_at",
+                "scheduled_for",
+                "running",
+                "script_hash",
+                "script_path",
+                "null as args",
+                "null as duration_ms",
+                "null as success",
+                "false as deleted",
+                "canceled",
+                "canceled_by",
+                "job_kind",
+                "schedule_path",
+                "permissioned_as",
+                "is_flow_step",
+                "language",
+                "false as is_skipped",
+                "email",
+                "visible_to_owner",
+                "suspend",
+                "mem_peak",
+                "tag",
+                "concurrent_limit",
+                "concurrency_time_window_s",
+            ],
+        );
+
+        format!(
+            "{} UNION ALL {} LIMIT {} OFFSET {};",
+            &sqlq.subquery()?,
+            &sqlc.subquery()?,
+            per_page,
+            offset
+        )
+    } else {
+        sqlc.query()?
+    };
    let mut tx = user_db.begin(&authed).await?;
    let jobs: Vec<UnifiedJob> = sqlx::query_as(&sql).fetch_all(&mut *tx).await?;
    tx.commit().await?;
@@ -2617,8 +2642,8 @@ async fn list_completed_jobs(
            "success",
            "script_hash",
            "script_path",
-            "args",
-            "result",
+            "null as args",
+            "null as result",
            "null as logs",
            "deleted",
            "canceled",
@@ -2665,7 +2690,18 @@ async fn get_completed_job(
 async fn get_completed_job_result(
    Extension(db): Extension<DB>,
    Path((w_id, id)): Path<(String, Uuid)>,
-) -> error::JsonResult<Option<serde_json::Value>> {
+) -> error::Result<String> {
+    // let result_o = sqlx::query_scalar!(
+    //     "SELECT result::text FROM completed_job WHERE id = $1 AND workspace_id = $2",
+    //     id,
+    //     w_id,
+    // )
+    // .fetch_optional(&db)
+    // .await?;
+
+    // let result = not_found_if_none(result_o, "Completed Job", id.to_string())?;
+    // Ok(Json(result))
+
    let result_o = sqlx::query_scalar!(
        "SELECT result FROM completed_job WHERE id = $1 AND workspace_id = $2",
        id,
@@ -2674,6 +2710,32 @@ async fn get_completed_job_result(
    .fetch_optional(&db)
    .await?;

+    let result = serde_json::to_string(&result_o).expect("FOO");
+    Ok(result)
+    // let mut v = Vec::new();
+    // let mut i = 0;
+    // while i < 1000000 {
+    //     v.push(serde_json::json!({"a": format!("b{}", i)}));
+    //     i += 1;
+    // }
+    // let result_o = Some(Some(serde_json::json! {{"test": v}}));
+
+    // let result = not_found_if_none(result_o, "Completed Job", id.to_string())?;
+    // Ok(Json(result))
+}
+
+pub async fn bench(
+    Extension(db): Extension<DB>,
+    Path((w_id, id)): Path<(String, Uuid)>,
+) -> error::JsonResult<Option<serde_json::Value>> {
+    let mut v = Vec::new();
+    let mut i = 0;
+    while i < 1000000 {
+        v.push(serde_json::json!({"a": format!("b{}", i)}));
+        i += 1;
+    }
+    let result_o = Some(Some(serde_json::json! {{"test": v}}));
+
    let result = not_found_if_none(result_o, "Completed Job", id.to_string())?;
    Ok(Json(result))
 }
--- a/backend/windmill-api/src/lib.rs
+++ b/backend/windmill-api/src/lib.rs
@@ -278,6 +278,7 @@ pub async fn run_server(
                    oauth2::global_service().layer(Extension(Arc::new(sp_extension.1))),
                )
                .route("/version", get(git_v))
+                .route("/bench/:workspace/:uuid", get(jobs::bench))
                .route("/uptodate", get(is_up_to_date))
                .route("/ee_license", get(ee_license))
                .route("/openapi.yaml", get(openapi)),
--- a/backend/windmill-api/src/scripts.rs
+++ b/backend/windmill-api/src/scripts.rs
@@ -19,7 +19,7 @@ use axum::{
    Json, Router,
 };
 use hyper::StatusCode;
-use serde::Serialize;
+use serde::{Deserialize, Serialize};
 use serde_json::json;
 use sql_builder::prelude::*;
 use sql_builder::SqlBuilder;
@@ -41,7 +41,8 @@ use windmill_common::{
    },
    users::username_to_permissioned_as,
    utils::{
-        list_elems_from_hub, not_found_if_none, paginate, require_admin, Pagination, StripPath,
+        list_elems_from_hub, not_found_if_none, paginate, query_elems_from_hub, require_admin,
+        Pagination, StripPath,
    },
 };
 use windmill_queue::{self, schedule::push_scheduled_job, PushIsolationLevel, QueueTransaction};
@@ -80,6 +81,7 @@ pub fn global_service() -> Router {
        .route("/hub/list", get(list_hub_scripts))
        .route("/hub/get/*path", get(get_hub_script_by_path))
        .route("/hub/get_full/*path", get(get_full_hub_script_by_path))
+        .route("/hub/query", get(query_hub_scripts))
 }

 pub fn global_unauthed_service() -> Router {
@@ -209,6 +211,28 @@ async fn list_hub_scripts(ApiAuthed { email, .. }: ApiAuthed) -> JsonResult<serd
    Ok(Json(asks))
 }

+#[derive(Deserialize)]
+struct HubScriptsQuery {
+    text: String,
+    kind: Option<String>,
+    limit: Option<i64>,
+}
+async fn query_hub_scripts(
+    ApiAuthed { email, .. }: ApiAuthed,
+    Query(query): Query<HubScriptsQuery>,
+) -> JsonResult<serde_json::Value> {
+    let asks = query_elems_from_hub(
+        &HTTP_CLIENT,
+        "https://hub.windmill.dev/scripts/query",
+        &email,
+        &query.text,
+        &query.kind,
+        &query.limit,
+    )
+    .await?;
+    Ok(Json(asks))
+}
+
 fn hash_script(ns: &NewScript) -> i64 {
    let mut dh = DefaultHasher::new();
    ns.hash(&mut dh);
--- a/backend/windmill-common/src/scripts.rs
+++ b/backend/windmill-common/src/scripts.rs
@@ -274,6 +274,7 @@ pub async fn get_hub_script_by_path(
        &format!("https://hub.windmill.dev/raw/{path}.ts"),
        email,
        true,
+        None,
    )
    .await?
    .text()
@@ -303,6 +304,7 @@ pub async fn get_full_hub_script_by_path(
        &format!("https://hub.windmill.dev/raw2/{path}"),
        email,
        true,
+        None,
    )
    .await?
    .json::<HubScript>()
--- a/backend/windmill-common/src/utils.rs
+++ b/backend/windmill-common/src/utils.rs
@@ -79,7 +79,32 @@ pub async fn list_elems_from_hub(
    url: &str,
    email: &str,
 ) -> Result<serde_json::Value> {
-    let rows = http_get_from_hub(http_client, url, email, false)
+    let rows = http_get_from_hub(http_client, url, email, false, None)
+        .await?
+        .json::<serde_json::Value>()
+        .await
+        .map_err(crate::error::to_anyhow)?;
+    Ok(rows)
+}
+
+#[cfg(feature = "reqwest")]
+pub async fn query_elems_from_hub(
+    http_client: &reqwest::Client,
+    url: &str,
+    email: &str,
+    query_text: &str,
+    query_kind: &Option<String>,
+    query_limit: &Option<i64>,
+) -> Result<serde_json::Value> {
+    let mut query_params = vec![("text", query_text)];
+    if let Some(query_kind) = query_kind {
+        query_params.push(("kind", query_kind.as_str()));
+    }
+    let query_limit = query_limit.unwrap_or(0).to_string();
+    if query_limit.parse::<i64>().unwrap() > 0 {
+        query_params.push(("limit", query_limit.as_str()));
+    }
+    let rows = http_get_from_hub(http_client, url, email, false, Some(query_params))
        .await?
        .json::<serde_json::Value>()
        .await
@@ -93,8 +118,9 @@ pub async fn http_get_from_hub(
    url: &str,
    email: &str,
    plain: bool,
+    query_params: Option<Vec<(&str, &str)>>,
 ) -> Result<reqwest::Response> {
-    let response = http_client
+    let mut request = http_client
        .get(url)
        .header(
            "Accept",
@@ -104,10 +130,15 @@ pub async fn http_get_from_hub(
                "application/json"
            },
        )
-        .header("X-email", email)
-        .send()
-        .await
-        .map_err(crate::error::to_anyhow)?;
+        .header("X-email", email);
+
+    if let Some(query_params) = query_params {
+        for (key, value) in query_params {
+            request = request.query(&[(key, value)]);
+        }
+    }
+
+    let response = request.send().await.map_err(crate::error::to_anyhow)?;

    Ok(response)
 }
--- a/backend/windmill-queue/src/jobs.rs
+++ b/backend/windmill-queue/src/jobs.rs
@@ -1353,12 +1353,21 @@ pub async fn delete_job<'c, R: rsmq_async::RsmqConnection + Clone + Send>(
        w_id,
        job_id
    )
-    .fetch_one(&mut tx)
-    .await
-    .map_err(|e| Error::InternalErr(format!("Error during deletion of job {job_id}: {e}")))?
-    .unwrap_or(0)
-        == 1;
-    tracing::debug!("Job {job_id} deleted: {job_removed}");
+    .fetch_optional(&mut tx)
+    .await;
+
+    if let Err(job_removed) = job_removed {
+        tracing::error!(
+            "Job {job_id} could not be deleted: {job_removed}. This is not necessarily an error, as the job might have been deleted by another process such as in the case of cancelling"
+        );
+    } else {
+        let job_removed = job_removed.unwrap().flatten().unwrap_or(0);
+        if job_removed != 1 {
+            tracing::error!("Job {job_id} could not be deleted, returned not 1: {job_removed}. This is not necessarily an error, as the job might have been deleted by another process such as in the case of cancelling");
+        }
+    }
+
+    tracing::debug!("Job {job_id} deleted");
    Ok(tx)
 }

--- a/benchmarks/main.ts
+++ b/benchmarks/main.ts
@@ -22,7 +22,7 @@ async function login(email: string, password: string): Promise<string> {
  });
 }

-export const VERSION = "v1.168.3";
+export const VERSION = "v1.170.0";

 export async function main({
  host,
@@ -242,20 +242,28 @@ export async function main({
  const jobsSent = Array(num_workers).fill(0);
  const enc = (s: string) => new TextEncoder().encode(s);

+  async function getQueueCount() {
+    return (
+      await (
+        await fetch(
+          config.server + "/api/w/" + config.workspace_id + "/jobs/queue/count",
+          { headers: { ["Authorization"]: "Bearer " + config.token } }
+        )
+      ).json()
+    ).database_length;
+  }
+
+  const initial_queue_length = await getQueueCount();
+
+  console.log("Initial queue length:", initial_queue_length);
+
  const updateState = setInterval(async () => {
    const elapsed = start ? Math.ceil((Date.now() - start) / 1000) : 0;
    const sum = jobsSent.reduce((a, b) => a + b, 0);
    let queue_length = -1;
    while (queue_length === -1) {
      try {
-        queue_length = (
-          await (
-            await fetch(
-              host + "/api/w/" + config.workspace_id + "/jobs/queue/count",
-              { headers: { ["Authorization"]: "Bearer " + config.token } }
-            )
-          ).json()
-        ).database_length;
+        queue_length = await getQueueCount();
      } catch (e) {
        console.log(
          `queue count not reachable. waiting...                                                           `
@@ -268,7 +276,9 @@ export async function main({
      enc(
        `elapsed: ${elapsed}/${seconds} | jobs sent: ${JSON.stringify(
          jobsSent
-        )} (sum: ${sum} thr: ${(sum / elapsed).toFixed(
+        )} (sum: ${sum} thr: ${(sum / elapsed).toFixed(2)}) - processed (sum: ${
+          sum - queue_length
+        } thr: ${((sum - queue_length) / elapsed).toFixed(
          2
        )}) | queue: ${queue_length}                          \r`
      )
--- a/cli/main.ts
+++ b/cli/main.ts
@@ -31,7 +31,7 @@ addEventListener("error", (event) => {
  }
 });

-export const VERSION = "v1.168.3";
+export const VERSION = "v1.170.0";

 let command: any = new Command()
  .name("wmill")
--- a/frontend/package-lock.json
+++ b/frontend/package-lock.json
@@ -1,12 +1,12 @@
 {
 	"name": "windmill-components",
-	"version": "1.168.3",
+	"version": "1.170.0",
 	"lockfileVersion": 2,
 	"requires": true,
 	"packages": {
 		"": {
 			"name": "windmill-components",
-			"version": "1.168.3",
+			"version": "1.170.0",
 			"license": "AGPL-3.0",
 			"dependencies": {
 				"@aws-crypto/sha256-js": "^4.0.0",
@@ -32,7 +32,7 @@
 				"lucide-svelte": "^0.246.0",
 				"monaco-graphql": "^1.3.0",
 				"monaco-languageclient": "~6.0.3",
-				"openai": "^4.0.0-beta.12",
+				"openai": "^4.3.0",
 				"quill": "^1.3.7",
 				"svelte-autosize": "^1.0.1",
 				"svelte-carousel": "^1.0.25",
@@ -52,7 +52,6 @@
 			},
 			"devDependencies": {
 				"@floating-ui/core": "^1.3.1",
-				"@modyfi/vite-plugin-yaml": "^1.0.4",
 				"@playwright/test": "^1.34.3",
 				"@rgossiaux/svelte-headlessui": "^1.0.2",
 				"@sveltejs/adapter-static": "^2.0.2",
@@ -1048,20 +1047,6 @@
 			"integrity": "sha512-HPnRdYO0WjFjRTSwO3frz1wKaU649OBFPX3Zo/2WZvuRi6zMiRGui8SnPQiQABgqCf8YikDe5t3HViTVw1WUzA==",
 			"dev": true
 		},
-		"node_modules/@modyfi/vite-plugin-yaml": {
-			"version": "1.0.4",
-			"resolved": "https://registry.npmjs.org/@modyfi/vite-plugin-yaml/-/vite-plugin-yaml-1.0.4.tgz",
-			"integrity": "sha512-qkT0KiR3AQQRfUvDzLv4+1rYAzXj+QmGhAbyUd0Ordf9xynK76i758lk5GiEfxuQxbvdqDaJ9oXkH/KacbSjQQ==",
-			"dev": true,
-			"dependencies": {
-				"@rollup/pluginutils": "5.0.2",
-				"js-yaml": "4.1.0",
-				"tosource": "2.0.0-alpha.3"
-			},
-			"peerDependencies": {
-				"vite": "^2.6.0 || ^3.0.0 || ^4.0.0"
-			}
-		},
 		"node_modules/@nodelib/fs.scandir": {
 			"version": "2.1.5",
 			"resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz",
@@ -1151,28 +1136,6 @@
 				"svelte": "^3.44.0"
 			}
 		},
-		"node_modules/@rollup/pluginutils": {
-			"version": "5.0.2",
-			"resolved": "https://registry.npmjs.org/@rollup/pluginutils/-/pluginutils-5.0.2.tgz",
-			"integrity": "sha512-pTd9rIsP92h+B6wWwFbW8RkZv4hiR/xKsqre4SIuAOaOEQRxi0lqLke9k2/7WegC85GgUs9pjmOjCUi3In4vwA==",
-			"dev": true,
-			"dependencies": {
-				"@types/estree": "^1.0.0",
-				"estree-walker": "^2.0.2",
-				"picomatch": "^2.3.1"
-			},
-			"engines": {
-				"node": ">=14.0.0"
-			},
-			"peerDependencies": {
-				"rollup": "^1.20.0||^2.0.0||^3.0.0"
-			},
-			"peerDependenciesMeta": {
-				"rollup": {
-					"optional": true
-				}
-			}
-		},
 		"node_modules/@sveltejs/adapter-static": {
 			"version": "2.0.2",
 			"resolved": "https://registry.npmjs.org/@sveltejs/adapter-static/-/adapter-static-2.0.2.tgz",
@@ -1608,12 +1571,6 @@
 				"@types/ms": "*"
 			}
 		},
-		"node_modules/@types/estree": {
-			"version": "1.0.1",
-			"resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.1.tgz",
-			"integrity": "sha512-LG4opVs2ANWZ1TJoKc937iMmNstM/d0ae1vNbnBvBhqCSezgVUOzcLCqbI5elV8Vy6WKwKjaqR+zO9VKirBBCA==",
-			"dev": true
-		},
 		"node_modules/@types/geojson": {
 			"version": "7946.0.10",
 			"resolved": "https://registry.npmjs.org/@types/geojson/-/geojson-7946.0.10.tgz",
@@ -3874,12 +3831,6 @@
 				"node": ">=4.0"
 			}
 		},
-		"node_modules/estree-walker": {
-			"version": "2.0.2",
-			"resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-2.0.2.tgz",
-			"integrity": "sha512-Rfkk/Mp/DL7JVje3u18FxFujQlTNR2q6QfMSMB7AvCBx91NGj/ba3kCfza0f6dVDbw7YlRf/nDrn7pQrCCyQ/w==",
-			"dev": true
-		},
 		"node_modules/esutils": {
 			"version": "2.0.3",
 			"resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz",
@@ -6707,9 +6658,9 @@
 			}
 		},
 		"node_modules/openai": {
-			"version": "4.0.0-beta.12",
-			"resolved": "https://registry.npmjs.org/openai/-/openai-4.0.0-beta.12.tgz",
-			"integrity": "sha512-K7PrIyF+Rv2kI8/FKcb1ZiM4L9ORR5J8DnoSgOT+mHrR9d3eBkAdxqVJ4uCXLFTGh7rDzU4pbuD+DpfdgzL1IQ==",
+			"version": "4.3.0",
+			"resolved": "https://registry.npmjs.org/openai/-/openai-4.3.0.tgz",
+			"integrity": "sha512-ObF5jxvZoQbCNAI6FiiNkzFDinBRbu4KPm73/PKCy9UvjI24kPpnN9kK56rtPDVYxfk78C0A2SK0bJAE633BuQ==",
 			"dependencies": {
 				"@types/node": "^18.11.18",
 				"@types/node-fetch": "^2.6.4",
@@ -9158,15 +9109,6 @@
 				"node": ">=8.0"
 			}
 		},
-		"node_modules/tosource": {
-			"version": "2.0.0-alpha.3",
-			"resolved": "https://registry.npmjs.org/tosource/-/tosource-2.0.0-alpha.3.tgz",
-			"integrity": "sha512-KAB2lrSS48y91MzFPFuDg4hLbvDiyTjOVgaK7Erw+5AmZXNq4sFRVn8r6yxSLuNs15PaokrDRpS61ERY9uZOug==",
-			"dev": true,
-			"engines": {
-				"node": ">=10"
-			}
-		},
 		"node_modules/totalist": {
 			"version": "3.0.0",
 			"resolved": "https://registry.npmjs.org/totalist/-/totalist-3.0.0.tgz",
@@ -10574,17 +10516,6 @@
 			"integrity": "sha512-HPnRdYO0WjFjRTSwO3frz1wKaU649OBFPX3Zo/2WZvuRi6zMiRGui8SnPQiQABgqCf8YikDe5t3HViTVw1WUzA==",
 			"dev": true
 		},
-		"@modyfi/vite-plugin-yaml": {
-			"version": "1.0.4",
-			"resolved": "https://registry.npmjs.org/@modyfi/vite-plugin-yaml/-/vite-plugin-yaml-1.0.4.tgz",
-			"integrity": "sha512-qkT0KiR3AQQRfUvDzLv4+1rYAzXj+QmGhAbyUd0Ordf9xynK76i758lk5GiEfxuQxbvdqDaJ9oXkH/KacbSjQQ==",
-			"dev": true,
-			"requires": {
-				"@rollup/pluginutils": "5.0.2",
-				"js-yaml": "4.1.0",
-				"tosource": "2.0.0-alpha.3"
-			}
-		},
 		"@nodelib/fs.scandir": {
 			"version": "2.1.5",
 			"resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz",
@@ -10651,17 +10582,6 @@
 			"dev": true,
 			"requires": {}
 		},
-		"@rollup/pluginutils": {
-			"version": "5.0.2",
-			"resolved": "https://registry.npmjs.org/@rollup/pluginutils/-/pluginutils-5.0.2.tgz",
-			"integrity": "sha512-pTd9rIsP92h+B6wWwFbW8RkZv4hiR/xKsqre4SIuAOaOEQRxi0lqLke9k2/7WegC85GgUs9pjmOjCUi3In4vwA==",
-			"dev": true,
-			"requires": {
-				"@types/estree": "^1.0.0",
-				"estree-walker": "^2.0.2",
-				"picomatch": "^2.3.1"
-			}
-		},
 		"@sveltejs/adapter-static": {
 			"version": "2.0.2",
 			"resolved": "https://registry.npmjs.org/@sveltejs/adapter-static/-/adapter-static-2.0.2.tgz",
@@ -11034,12 +10954,6 @@
 				"@types/ms": "*"
 			}
 		},
-		"@types/estree": {
-			"version": "1.0.1",
-			"resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.1.tgz",
-			"integrity": "sha512-LG4opVs2ANWZ1TJoKc937iMmNstM/d0ae1vNbnBvBhqCSezgVUOzcLCqbI5elV8Vy6WKwKjaqR+zO9VKirBBCA==",
-			"dev": true
-		},
 		"@types/geojson": {
 			"version": "7946.0.10",
 			"resolved": "https://registry.npmjs.org/@types/geojson/-/geojson-7946.0.10.tgz",
@@ -12626,12 +12540,6 @@
 			"integrity": "sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw==",
 			"dev": true
 		},
-		"estree-walker": {
-			"version": "2.0.2",
-			"resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-2.0.2.tgz",
-			"integrity": "sha512-Rfkk/Mp/DL7JVje3u18FxFujQlTNR2q6QfMSMB7AvCBx91NGj/ba3kCfza0f6dVDbw7YlRf/nDrn7pQrCCyQ/w==",
-			"dev": true
-		},
 		"esutils": {
 			"version": "2.0.3",
 			"resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz",
@@ -14623,9 +14531,9 @@
 			}
 		},
 		"openai": {
-			"version": "4.0.0-beta.12",
-			"resolved": "https://registry.npmjs.org/openai/-/openai-4.0.0-beta.12.tgz",
-			"integrity": "sha512-K7PrIyF+Rv2kI8/FKcb1ZiM4L9ORR5J8DnoSgOT+mHrR9d3eBkAdxqVJ4uCXLFTGh7rDzU4pbuD+DpfdgzL1IQ==",
+			"version": "4.3.0",
+			"resolved": "https://registry.npmjs.org/openai/-/openai-4.3.0.tgz",
+			"integrity": "sha512-ObF5jxvZoQbCNAI6FiiNkzFDinBRbu4KPm73/PKCy9UvjI24kPpnN9kK56rtPDVYxfk78C0A2SK0bJAE633BuQ==",
 			"requires": {
 				"@types/node": "^18.11.18",
 				"@types/node-fetch": "^2.6.4",
@@ -16334,12 +16242,6 @@
 				"is-number": "^7.0.0"
 			}
 		},
-		"tosource": {
-			"version": "2.0.0-alpha.3",
-			"resolved": "https://registry.npmjs.org/tosource/-/tosource-2.0.0-alpha.3.tgz",
-			"integrity": "sha512-KAB2lrSS48y91MzFPFuDg4hLbvDiyTjOVgaK7Erw+5AmZXNq4sFRVn8r6yxSLuNs15PaokrDRpS61ERY9uZOug==",
-			"dev": true
-		},
 		"totalist": {
 			"version": "3.0.0",
 			"resolved": "https://registry.npmjs.org/totalist/-/totalist-3.0.0.tgz",
--- a/frontend/package.json
+++ b/frontend/package.json
@@ -1,6 +1,6 @@
 {
 	"name": "windmill-components",
-	"version": "1.168.3",
+	"version": "1.170.0",
 	"scripts": {
 		"dev": "vite dev",
 		"build": "vite build",
@@ -17,7 +17,6 @@
 	},
 	"devDependencies": {
 		"@floating-ui/core": "^1.3.1",
-		"@modyfi/vite-plugin-yaml": "^1.0.4",
 		"@playwright/test": "^1.34.3",
 		"@rgossiaux/svelte-headlessui": "^1.0.2",
 		"@sveltejs/adapter-static": "^2.0.2",
@@ -96,7 +95,7 @@
 		"lucide-svelte": "^0.246.0",
 		"monaco-graphql": "^1.3.0",
 		"monaco-languageclient": "~6.0.3",
-		"openai": "^4.0.0-beta.12",
+		"openai": "^4.3.0",
 		"quill": "^1.3.7",
 		"svelte-autosize": "^1.0.1",
 		"svelte-carousel": "^1.0.25",
--- a/frontend/src/lib/components/DBSchemaExplorer.svelte
+++ b/frontend/src/lib/components/DBSchemaExplorer.svelte
@@ -1,6 +1,12 @@
 <script lang="ts">
 	import { JobService, Preview } from '$lib/gen'
-	import { dbSchemas, workspaceStore, type DBSchema, type GraphqlSchema } from '$lib/stores'
+	import {
+		dbSchemas,
+		workspaceStore,
+		type DBSchema,
+		type GraphqlSchema,
+		type SQLSchema
+	} from '$lib/stores'
 	import Button from './common/button/Button.svelte'
 	import Drawer from './common/drawer/Drawer.svelte'
 	import DrawerContent from './common/drawer/DrawerContent.svelte'
@@ -180,6 +186,51 @@ GROUP BY
                cols.append(col)
            schema[dataset.dataset_id][row[0]] = cols
    return schema
+`,
+			lang: 'python3'
+		},
+		snowflake: {
+			code: `# requirements:
+# snowflake-connector-python==3.2.0
+from typing import Any
+import snowflake.connector as sf
+from cryptography.hazmat.backends import default_backend
+from cryptography.hazmat.primitives import serialization
+def main(args):
+    if not args["database"]:
+        raise Exception("a selected database is required for the schema explorer")
+    p_key = serialization.load_pem_private_key(
+        args["private_key"].encode(), password=None, backend=default_backend()
+    )
+    pkb = p_key.private_bytes(
+        encoding=serialization.Encoding.DER,
+        format=serialization.PrivateFormat.PKCS8,
+        encryption_algorithm=serialization.NoEncryption(),
+    )
+    ctx = sf.connect(
+        user=args["username"],
+        account=args["account_identifier"],
+        private_key=pkb,
+        warehouse=args["warehouse"],
+        database=args["database"],
+        schema=args["schema"],
+        role=args["role"],
+    )
+    cs = ctx.cursor()
+    rows = cs.execute("select TABLE_SCHEMA, TABLE_NAME, DATA_TYPE, COLUMN_NAME, COLUMN_DEFAULT, IS_NULLABLE from information_schema.columns where table_schema != 'INFORMATION_SCHEMA'")
+    schema = dict()
+    for row in rows:
+        if row[0] not in schema:
+            schema[row[0]] = dict()
+        if row[1] not in schema[row[0]]:
+            schema[row[0]][row[1]] = dict()
+        schema[row[0]][row[1]][row[3]] = {
+            "type": row[2],
+            "required": row[5] == "YES",
+        }
+        if row[4] is not None:
+            schema[row[0]][row[1]][row[3]]["default"] = row[4]
+    return schema
 `,
 			lang: 'python3'
 		}
@@ -210,19 +261,18 @@ GROUP BY
 					if (!testResult.success) {
 						console.error(testResult.result?.['error']?.['message'])
 					} else {
-						if (resourceType === 'postgresql') {
-							$dbSchemas[resourcePath] = {
-								lang: 'postgresql',
-								schema: testResult.result,
-								publicOnly: true
-							}
-						} else if (
-							resourceType !== undefined &&
-							['mysql', 'graphql', 'bigquery'].includes(resourceType)
-						) {
-							$dbSchemas[resourcePath] = {
-								lang: resourceType as 'mysql' | 'graphql' | 'bigquery',
-								schema: testResult.result
+						if (resourceType !== undefined) {
+							if (resourceType !== 'graphql') {
+								$dbSchemas[resourcePath] = {
+									lang: resourceType as SQLSchema['lang'],
+									schema: testResult.result,
+									publicOnly: !!testResult.result.public || !!testResult.result.PUBLIC
+								}
+							} else {
+								$dbSchemas[resourcePath] = {
+									lang: 'graphql',
+									schema: testResult.result
+								}
 							}
 						}
 					}
@@ -253,8 +303,8 @@ GROUP BY
 	}

 	function formatSchema(dbSchema: DBSchema) {
-		if (dbSchema.lang === 'postgresql' && dbSchema.publicOnly) {
-			return dbSchema.schema.public || dbSchema
+		if (dbSchema.lang !== 'graphql' && dbSchema.publicOnly) {
+			return dbSchema.schema.public || dbSchema.schema.PUBLIC || dbSchema
 		} else if (dbSchema.lang === 'mysql' && Object.keys(dbSchema.schema).length === 1) {
 			return dbSchema.schema[Object.keys(dbSchema.schema)[0]]
 		} else {
@@ -299,7 +349,7 @@ GROUP BY
 					>Refresh
 				</Button>
 			</svelte:fragment>
-			{#if dbSchema.lang === 'postgresql'}
+			{#if dbSchema.lang !== 'graphql' && (dbSchema.schema?.public || dbSchema.schema?.PUBLIC)}
 				<ToggleButtonGroup class="mb-4" bind:selected={dbSchema.publicOnly}>
 					<ToggleButton value={true} label="Public" />
 					<ToggleButton value={false} label="All" />
--- a/frontend/src/lib/components/Editor.svelte
+++ b/frontend/src/lib/components/Editor.svelte
@@ -70,6 +70,7 @@
 	export let yContent: Text | undefined = undefined
 	export let awareness: any | undefined = undefined
 	export let folding = false
+	export let args: Record<string, any> | undefined = undefined

 	languages.typescript.typescriptDefaults.setModeConfiguration({
 		completionItems: false,
@@ -188,6 +189,27 @@
 		}
 	}

+	export function append(code): void {
+		if (editor) {
+			const lineCount = editor.getModel()?.getLineCount() || 0
+			const lastLineLength = editor.getModel()?.getLineLength(lineCount) || 0
+			const range: IRange = {
+				startLineNumber: lineCount,
+				startColumn: lastLineLength + 1,
+				endLineNumber: lineCount,
+				endColumn: lastLineLength + 1
+			}
+			editor.executeEdits('append', [
+				{
+					range,
+					text: code,
+					forceMoveMarkers: true
+				}
+			])
+			editor.revealLine(lineCount)
+		}
+	}
+
 	export function format() {
 		if (editor) {
 			code = getCode()
@@ -203,7 +225,8 @@
 	let command: Disposable | undefined = undefined

 	let sqlSchemaCompletor: Disposable | undefined = undefined
-	$: dbSchema = $dbSchemas[Object.keys($dbSchemas)[0]]
+	$: args &&
+		(dbSchema = $dbSchemas[(lang === 'graphql' ? args.api : args.database)?.replace('$res:', '')])
 	$: dbSchema && ['sql', 'graphql'].includes(lang) && addDBSchemaCompletions()
 	$: (!dbSchema || lang !== 'sql') && sqlSchemaCompletor && sqlSchemaCompletor.dispose()
 	$: (!dbSchema || lang !== 'graphql') && graphqlService && graphqlService.setSchemaConfig([])
@@ -221,7 +244,7 @@
 					introspectionJSON: schema
 				}
 			])
-		} else if (schemaLang === 'mysql' || schemaLang === 'postgresql') {
+		} else {
 			if (sqlSchemaCompletor) {
 				sqlSchemaCompletor.dispose()
 			}
--- a/frontend/src/lib/components/EditorBar.svelte
+++ b/frontend/src/lib/components/EditorBar.svelte
@@ -37,7 +37,7 @@
 	import { capitalize, toCamel } from '$lib/utils'
 	import type { Schema, SchemaProperty, SupportedLanguage } from '$lib/common'
 	import ScriptVersionHistory from './ScriptVersionHistory.svelte'
-	import { ScriptGen } from './codeGen'
+	import ScriptGen from './copilot/ScriptGen.svelte'
 	import type DiffEditor from './DiffEditor.svelte'
 	import { getResetCode } from '$lib/script_helpers'
 	import type { Script } from '$lib/gen'
@@ -62,6 +62,7 @@
 	export let collabUsers: { name: string }[] = []
 	export let scriptPath: string | undefined = undefined
 	export let diffEditor: DiffEditor | undefined = undefined
+	export let args: Record<string, any>

 	let contextualVariablePicker: ItemPicker
 	let variablePicker: ItemPicker
@@ -530,7 +531,7 @@
 				</div>
 			{/if}

-			<ScriptGen {editor} {diffEditor} {lang} {iconOnly} />
+			<ScriptGen {editor} {diffEditor} {lang} {iconOnly} {args} />

 			<!-- <Popover
 				notClickable
--- a/frontend/src/lib/components/FlowBuilder.svelte
+++ b/frontend/src/lib/components/FlowBuilder.svelte
@@ -5,12 +5,15 @@
 		type Flow,
 		type FlowModule,
 		DraftService,
-		type PathScript
+		type PathScript,
+		RawScript,
+		ScriptService
 	} from '$lib/gen'
 	import { initHistory, redo, undo } from '$lib/history'
-	import { enterpriseLicense, userStore, workspaceStore } from '$lib/stores'
-	import { encodeState, formatCron } from '$lib/utils'
+	import { enterpriseLicense, hubScripts, userStore, workspaceStore } from '$lib/stores'
+	import { encodeState, formatCron, sleep } from '$lib/utils'
 	import { sendUserToast } from '$lib/toast'
+	import type { Drawer } from '$lib/components/common'

 	import { faCalendarAlt, faSave } from '@fortawesome/free-solid-svg-icons'
 	import { setContext } from 'svelte'
@@ -32,6 +35,17 @@
 	import { createEventDispatcher } from 'svelte'
 	import Awareness from './Awareness.svelte'
 	import { getAllModules } from './flows/flowExplorer'
+	import {
+		stepCopilot,
+		type FlowCopilotModule,
+		glueCopilot,
+		type FlowCopilotContext
+	} from './copilot/flow'
+	import { numberToChars } from './flows/idUtils'
+	import type { Schema, SchemaProperty } from '$lib/common'
+	import FlowCopilotDrawer from './copilot/FlowCopilotDrawer.svelte'
+	import FlowCopilotStatus from './copilot/FlowCopilotStatus.svelte'
+	import { fade } from 'svelte/transition'

 	export let initialPath: string = ''
 	export let selectedId: string | undefined
@@ -348,18 +362,396 @@
 			onClick: () => window.open(`/flows/add?template=${initialPath}`)
 		})
 	}
+
+	let flowCopilotContext: FlowCopilotContext = {
+		drawerStore: writable<Drawer | undefined>(undefined),
+		modulesStore: writable<FlowCopilotModule[]>([]),
+		currentStepStore: writable<string | undefined>(undefined)
+	}
+
+	setContext('FlowCopilotContext', flowCopilotContext)
+
+	const {
+		drawerStore: copilotDrawerStore,
+		modulesStore: copilotModulesStore,
+		currentStepStore: copilotCurrentStepStore
+	} = flowCopilotContext
+
+	let doneTs = 0
+	async function hubCompletions(text: string, idx: number, type: 'trigger' | 'script') {
+		try {
+			// make sure we display the results of the last request last
+			const ts = Date.now()
+			const scriptIds = await ScriptService.queryHubScripts({
+				text: `${text}`,
+				limit: 3,
+				kind: type
+			})
+			if (ts < doneTs) return
+			doneTs = ts
+
+			const scripts = scriptIds
+				.map((qs) => {
+					const s = $hubScripts?.find((hs) => hs.ask_id === Number(qs.id))
+					return s
+				})
+				.filter((s) => !!s)
+
+			$copilotModulesStore[idx].hubCompletions = scripts as {
+				path: string
+				summary: string
+				approved: boolean
+				kind: string
+				app: string
+				ask_id: number
+			}[]
+		} catch (err) {
+			if (err.name !== 'CancelError') throw err
+		}
+	}
+
+	let abortController: AbortController | undefined = undefined
+	let copilotLoading = false
+	let flowCopilotMode: 'trigger' | 'sequence' = 'trigger'
+	let copilotStatus: string = ''
+
+	function getInitCopilotModules(mode: typeof flowCopilotMode): FlowCopilotModule[] {
+		return [
+			{
+				id: 'a',
+				type: mode === 'trigger' ? 'trigger' : 'script',
+				description: '',
+				code: '',
+				hubCompletions: [],
+				selectedCompletion: undefined,
+				source: undefined
+			},
+			{
+				id: 'b',
+				type: 'script',
+				description: '',
+				code: '',
+				hubCompletions: [],
+				selectedCompletion: undefined,
+				source: undefined
+			}
+		]
+	}
+
+	$: {
+		copilotModulesStore.set(getInitCopilotModules(flowCopilotMode))
+	}
+
+	async function genFlow(i: number) {
+		copilotLoading = true
+		copilotStatus = "Generating code for step '" + numberToChars(i) + "'..."
+		$copilotCurrentStepStore = numberToChars(i)
+		try {
+			abortController = new AbortController()
+
+			$flowStore.value.modules = $flowStore.value.modules.slice(0, i)
+			let prevCode = ''
+			if (i === 0) {
+				prevCode = ''
+				$flowStore.schema = {
+					$schema: 'https://json-schema.org/draft/2020-12/schema',
+					properties: {},
+					required: [],
+					type: 'object'
+				}
+			} else {
+				prevCode = ($flowStore.value.modules[i - 1].value as RawScript).content
+			}
+
+			let module = $copilotModulesStore[i]
+
+			if (module.type === 'trigger') {
+				if (!$scheduleStore.cron) {
+					$scheduleStore.cron = '0 */15 * * *'
+				}
+				$scheduleStore.enabled = true
+			}
+
+			const flowModule = {
+				id: numberToChars(i),
+				stop_after_if:
+					module.type === 'trigger'
+						? {
+								expr: 'result == undefined || Array.isArray(result) && result.length == 0',
+								skip_if_stopped: true
+						  }
+						: undefined,
+				value: {
+					input_transforms: {},
+					content: '',
+					language: RawScript.language.BUN,
+					type: 'rawscript' as const
+				},
+				summary:
+					$copilotModulesStore[i].selectedCompletion?.summary ?? $copilotModulesStore[i].description
+			}
+
+			if (i === 1 && $copilotModulesStore[i - 1].type === 'trigger') {
+				const loopModule: FlowModule = {
+					id: numberToChars(i) + '_loop',
+					value: {
+						type: 'forloopflow',
+						iterator: {
+							type: 'javascript',
+							expr: 'results.a'
+						},
+						skip_failures: true,
+						modules: [flowModule]
+					}
+				}
+
+				$flowStore.value.modules.push(loopModule)
+			} else {
+				$flowStore.value.modules.push(flowModule)
+			}
+
+			$copilotDrawerStore?.closeDrawer()
+			select(numberToChars(i))
+			await sleep(200)
+
+			$copilotModulesStore[i].editor?.setCode('')
+			const deltaStore = writable<string>('')
+			const unsubscribe = deltaStore.subscribe(async (delta) => {
+				$copilotModulesStore[i].editor?.append(delta)
+			})
+			await stepCopilot(module, deltaStore, prevCode, abortController)
+			unsubscribe()
+
+			copilotStatus = "Generating inputs for step '" + numberToChars(i) + "'..."
+			await sleep(500) // make sure code was parsed
+
+			try {
+				let currentFlowModule = $flowStore.value.modules[i]
+				if (currentFlowModule.value.type === 'forloopflow') {
+					currentFlowModule = currentFlowModule.value.modules[0]
+				}
+
+				if (currentFlowModule.value.type === 'rawscript') {
+					const stepSchema: Schema = JSON.parse(JSON.stringify($flowStateStore[module.id].schema)) // deep copy
+					if (module.source === 'hub' && i >= 1) {
+						// ask AI to set step inputs
+						const pastModule = $flowStore.value.modules[i - 1]
+						const inputs = await glueCopilot(
+							Object.keys(currentFlowModule.value.input_transforms),
+							pastModule.value.type === 'rawscript' ? pastModule.value.content : '',
+							i === 1 && $copilotModulesStore[i - 1].type === 'trigger',
+							abortController
+						)
+
+						// create flow inputs used by AI for autocompletion
+						Object.entries(inputs)
+							.filter(
+								([key, expr]) =>
+									key in stepSchema.properties &&
+									expr.startsWith('flow_inputs.') &&
+									!expr.startsWith('flow_inputs.iter')
+							)
+							.map(([key, _]) => {
+								const inputSchemaProperty = stepSchema.properties[key]
+								const isRequired = stepSchema.required.includes(key)
+
+								if ($flowStore.schema) {
+									$flowStore.schema.properties[key] = inputSchemaProperty
+									if (isRequired) {
+										$flowStore.schema.required.push(key)
+									}
+								} else {
+									$flowStore.schema = {
+										$schema: 'https://json-schema.org/draft/2020-12/schema',
+										properties: {
+											[key]: inputSchemaProperty
+										},
+										required: isRequired ? [key] : [],
+										type: 'object'
+									}
+								}
+								$flowStore.schema
+							})
+
+						flowModule.value.input_transforms = Object.entries(inputs).reduce(
+							(acc, [key, expr]) => {
+								acc[key] = {
+									type: 'javascript',
+									expr
+								}
+								return acc
+							},
+							{}
+						)
+					} else {
+						// create possible flow inputs for autocompletion
+						delete stepSchema.properties.prev_output
+						$flowStore.schema = {
+							$schema: 'https://json-schema.org/draft/2020-12/schema',
+							properties: {
+								...$flowStore.schema?.properties,
+								...stepSchema.properties
+							},
+							required: Array.from(
+								new Set([...$flowStore.schema?.required, ...stepSchema.required])
+							),
+							type: 'object'
+						}
+
+						// programatically set step inputs
+						for (const key of Object.keys(currentFlowModule.value.input_transforms)) {
+							if (key !== 'prev_output') {
+								const schema = $flowStateStore[module.id].schema
+								const schemaProperty = Object.entries(schema.properties).find(
+									(x) => x[0] === key
+								)?.[1]
+								if (schemaProperty) {
+									$flowStore.schema = {
+										$schema: 'https://json-schema.org/draft/2020-12/schema',
+										properties: {
+											...$flowStore.schema?.properties,
+											[key]: schemaProperty
+										},
+										required: schemaProperty.required
+											? Array.from(new Set([...$flowStore.schema?.required, key]))
+											: $flowStore.schema?.required,
+										type: 'object'
+									}
+								}
+							}
+
+							flowModule.value.input_transforms[key] = {
+								type: 'javascript',
+								expr:
+									key === 'prev_output'
+										? $copilotModulesStore[i - 1].type === 'trigger'
+											? 'flow_input.iter.value'
+											: 'results.' + $copilotModulesStore[i - 1].id
+										: 'flow_input.' + key
+							}
+						}
+					}
+
+					const wrappingFlowModule = $flowStore.value.modules[i]
+					if (wrappingFlowModule.value.type === 'forloopflow') {
+						wrappingFlowModule.value = {
+							...wrappingFlowModule.value,
+							modules: [flowModule]
+						}
+						$flowStore.value.modules[i] = wrappingFlowModule
+					} else {
+						$flowStore.value.modules[i] = flowModule
+					}
+				}
+			} catch (err) {
+				console.error(err)
+			}
+
+			copilotStatus =
+				"Waiting for the user to validate code and inputs of step '" + numberToChars(i) + "'"
+		} catch (err) {
+			if (err?.message) {
+				sendUserToast('Failed to generate code: ' + err.message, true)
+			} else {
+				sendUserToast('Failed to generate code', true)
+				console.error(err)
+			}
+		} finally {
+			copilotLoading = false
+		}
+	}
+
+	async function handleFlowGenInputs() {
+		copilotLoading = true
+		select('Input')
+		$copilotCurrentStepStore = 'Input'
+		copilotStatus = 'Setting flow inputs...'
+
+		// filter out unused flow inputs
+		const flowInputs: Record<string, SchemaProperty> = {}
+		const required = new Set<string>()
+		function getFlowInputs(modules: FlowModule[]) {
+			for (const module of modules) {
+				if (module.value.type === 'rawscript') {
+					for (const moduleAttr of Object.keys(module.value.input_transforms)) {
+						const input = module.value.input_transforms[moduleAttr]
+						if (
+							input.type === 'javascript' &&
+							input.expr.startsWith('flow_input.') &&
+							!input.expr.startsWith('flow_input.iter')
+						) {
+							const flowAttr = input.expr.split('.')[1]
+							const schema = $flowStateStore[module.id].schema
+							const schemaProperty = Object.entries(schema.properties).find(
+								(x) => x[0] === moduleAttr
+							)?.[1]
+							if (schemaProperty) {
+								flowInputs[flowAttr] = schemaProperty
+								required.add(flowAttr)
+							}
+						}
+					}
+				} else if (module.value.type === 'forloopflow') {
+					getFlowInputs(module.value.modules)
+				}
+			}
+		}
+		getFlowInputs($flowStore.value.modules)
+
+		$flowStore.schema = {
+			$schema: 'https://json-schema.org/draft/2020-12/schema',
+			properties: flowInputs,
+			required: Array.from(required),
+			type: 'object'
+		}
+
+		copilotStatus = "Done! Just check the flow's inputs and you're good to go!"
+		$copilotCurrentStepStore = undefined
+		copilotLoading = false
+		await sleep(3000)
+		copilotStatus = ''
+	}
+
+	function focusCopilot() {
+		document.querySelectorAll('.splitpanes__splitter').forEach((el) => {
+			el.classList.add('hidden')
+		})
+		document.querySelectorAll('#flow-graph *').forEach((el) => {
+			if (el instanceof HTMLElement) {
+				el.style.pointerEvents = 'none'
+			}
+		})
+	}
+
+	function blurCopilot() {
+		document.querySelectorAll('.splitpanes__splitter').forEach((el) => {
+			el.classList.remove('hidden')
+		})
+		document.querySelectorAll('#flow-graph *').forEach((el) => {
+			if (el instanceof HTMLElement) {
+				el.style.pointerEvents = ''
+			}
+		})
+	}
+
+	$: $copilotCurrentStepStore !== undefined ? focusCopilot() : blurCopilot()
 </script>

 <svelte:window on:keydown={onKeyDown} />

+<FlowCopilotDrawer {hubCompletions} {genFlow} bind:flowCopilotMode />
+
 {#if !$userStore?.operator}
 	<ScriptEditorDrawer bind:this={$scriptEditorDrawer} />

 	<div class="flex flex-col flex-1 h-screen">
 		<!-- Nav between steps-->
 		<div
-			class="justify-between flex flex-row items-center pl-2.5 pr-6 space-x-4 scrollbar-hidden max-h-12 h-full"
+			class="justify-between flex flex-row items-center pl-2.5 pr-6 space-x-4 scrollbar-hidden max-h-12 h-full relative"
 		>
+			{#if $copilotCurrentStepStore !== undefined}
+				<div transition:fade class="absolute inset-0 bg-gray-500 bg-opacity-75 z-[900] !m-0" />
+			{/if}
 			<div class="flex w-full max-w-md gap-4 items-center">
 				<div class="min-w-64 w-full">
 					<input
@@ -429,6 +821,14 @@
 					<Awareness />
 				{/if}

+				<FlowCopilotStatus
+					{copilotLoading}
+					bind:copilotStatus
+					{genFlow}
+					{handleFlowGenInputs}
+					{abortController}
+				/>
+
 				<FlowImportExportMenu />

 				<FlowPreviewButtons />
--- a/frontend/src/lib/components/InputTransformForm.svelte
+++ b/frontend/src/lib/components/InputTransformForm.svelte
@@ -47,6 +47,12 @@

 	let propertyType = getPropertyType(arg)

+	function updatePropertyType() {
+		propertyType = arg.type
+	}
+
+	$: arg?.type && arg.type !== propertyType && updatePropertyType()
+
 	function getPropertyType(arg: InputTransform | any): 'static' | 'javascript' {
 		let type: 'static' | 'javascript' = arg?.type ?? 'static'

--- a/frontend/src/lib/components/ManualPopover.svelte
+++ b/frontend/src/lib/components/ManualPopover.svelte
@@ -0,0 +1,33 @@
+<script lang="ts">
+	import { createPopperActions } from 'svelte-popperjs'
+	import type { PopoverPlacement } from './Popover.model'
+	import Portal from 'svelte-portal'
+	import { fade } from 'svelte/transition'
+
+	export let placement: PopoverPlacement = 'bottom'
+
+	const [popperRef, popperContent] = createPopperActions({ placement })
+
+	let showTooltip = false
+	export function open() {
+		showTooltip = true
+	}
+	export function close() {
+		showTooltip = false
+	}
+</script>
+
+<fragment use:popperRef>
+	<slot />
+</fragment>
+{#if showTooltip}
+	<Portal>
+		<div
+			use:popperContent
+			class="z-[901] rounded-lg shadow-md border p-4 bg-surface"
+			transition:fade={{ duration: 200 }}
+		>
+			<slot name="content" />
+		</div>
+	</Portal>
+{/if}
--- a/frontend/src/lib/components/ModulePreview.svelte
+++ b/frontend/src/lib/components/ModulePreview.svelte
@@ -19,7 +19,7 @@
 	import type { PickableProperties } from './flows/previousResults'
 	import type DiffEditor from './DiffEditor.svelte'
 	import type Editor from './Editor.svelte'
-	import ScriptFix from './codeGen/ScriptFix.svelte'
+	import ScriptFix from './copilot/ScriptFix.svelte'

 	export let mod: FlowModule
 	export let schema: Schema
@@ -135,12 +135,13 @@
 							jobId={testJob?.id}
 							result={testJob.result}>
 							<svelte:fragment slot="copilot-fix">
-								{#if lang && editor && diffEditor && testJob?.result?.error}
+								{#if lang && editor && diffEditor && stepArgs && testJob?.result?.error}
 									<ScriptFix
 										error={JSON.stringify(testJob.result.error)}
 										{lang}
 										{editor}
 										{diffEditor}
+										args={stepArgs}
 									/>
 								{/if}
 							</svelte:fragment>
--- a/frontend/src/lib/components/ScriptEditor.svelte
+++ b/frontend/src/lib/components/ScriptEditor.svelte
@@ -249,6 +249,7 @@
 			kind={asKind(kind)}
 			{template}
 			{diffEditor}
+			{args}
 		/>
 		{#if !noSyncFromGithub}
 			<div class="py-1">
@@ -302,6 +303,7 @@
 						deno={lang == 'deno'}
 						automaticLayout={true}
 						{fixedOverflowWidgets}
+						{args}
 					/>
 					<DiffEditor
 						bind:this={diffEditor}
@@ -364,6 +366,7 @@
 							previewIsLoading={testIsLoading}
 							{editor}
 							{diffEditor}
+							{args}
 						/>
 					</Pane>
 				</Splitpanes>
--- a/frontend/src/lib/components/apps/components/GroupWrapper.svelte
+++ b/frontend/src/lib/components/apps/components/GroupWrapper.svelte
@@ -0,0 +1,11 @@
+<script lang="ts">
+	import { setContext } from 'svelte'
+	import type { Writable } from 'svelte/store'
+	import type { GroupContext } from '../types'
+
+	export let groupContext: Writable<Record<string, any>>
+
+	setContext<GroupContext>('GroupContext', groupContext)
+</script>
+
+<slot />
--- a/frontend/src/lib/components/apps/components/helpers/InputValue.svelte
+++ b/frontend/src/lib/components/apps/components/helpers/InputValue.svelte
@@ -7,7 +7,7 @@
 		TemplateV2Input,
 		UploadAppInput
 	} from '../../inputType'
-	import type { AppViewerContext, ListContext, RichConfiguration } from '../../types'
+	import type { AppViewerContext, GroupContext, ListContext, RichConfiguration } from '../../types'
 	import { accessPropertyByPath } from '../../utils'
 	import { computeGlobalContext, eval_like } from './eval'
 	import deepEqualWithOrderedArray from './deepEqualWithOrderedArray'
@@ -26,6 +26,7 @@

 	const iterContext = getContext<ListContext>('ListWrapperContext')
 	const rowContext = getContext<ListContext>('RowWrapperContext')
+	const groupContext = getContext<GroupContext>('GroupContext')

 	let previousConnectedValue: any | undefined = undefined

@@ -33,19 +34,28 @@

 	$: fullContext = {
 		iter: iterContext ? $iterContext : undefined,
-		row: rowContext ? $rowContext : undefined
+		row: rowContext ? $rowContext : undefined,
+		group: groupContext ? $groupContext : undefined
 	}

 	$: lastInput?.type == 'evalv2' &&
-		(fullContext.iter != undefined || fullContext.row != undefined) &&
-		lastInput.connections.some((x) => x.componentId == 'row' || x.componentId == 'iter') &&
+		(fullContext.iter != undefined ||
+			fullContext.row != undefined ||
+			fullContext.group != undefined) &&
+		lastInput.connections.some(
+			(x) => x.componentId == 'row' || x.componentId == 'iter' || x.componentId == 'group'
+		) &&
 		debounceEval()

 	$: lastInput &&
 		lastInput.type == 'templatev2' &&
 		isCodeInjection(lastInput.eval) &&
-		(fullContext.iter != undefined || fullContext.row != undefined) &&
-		lastInput.connections.some((x) => x.componentId == 'row' || x.componentId == 'iter') &&
+		(fullContext.iter != undefined ||
+			fullContext.row != undefined ||
+			fullContext.group != undefined) &&
+		lastInput.connections.some(
+			(x) => x.componentId == 'row' || x.componentId == 'iter' || x.componentId == 'group'
+		) &&
 		debounceTemplate()

 	const dispatch = createEventDispatcher()
--- a/frontend/src/lib/components/apps/components/helpers/RunnableComponent.svelte
+++ b/frontend/src/lib/components/apps/components/helpers/RunnableComponent.svelte
@@ -10,7 +10,13 @@
 	import { createEventDispatcher, getContext, onDestroy, onMount } from 'svelte'
 	import type { AppInputs, Runnable } from '../../inputType'
 	import type { Output } from '../../rx'
-	import type { AppViewerContext, CancelablePromise, InlineScript, ListContext } from '../../types'
+	import type {
+		AppViewerContext,
+		CancelablePromise,
+		GroupContext,
+		InlineScript,
+		ListContext
+	} from '../../types'
 	import { computeGlobalContext, eval_like } from './eval'
 	import InputValue from './InputValue.svelte'
 	import RefreshButton from './RefreshButton.svelte'
@@ -65,6 +71,7 @@
 	} = getContext<AppViewerContext>('AppViewerContext')
 	const iterContext = getContext<ListContext>('ListWrapperContext')
 	const rowContext = getContext<ListContext>('RowWrapperContext')
+	const groupContext = getContext<GroupContext>('GroupContext')

 	const dispatch = createEventDispatcher()

@@ -179,7 +186,8 @@
 					runnable.inlineScript?.content,
 					computeGlobalContext($worldStore, {
 						iter: iterContext ? $iterContext : undefined,
-						row: rowContext ? $rowContext : undefined
+						row: rowContext ? $rowContext : undefined,
+						group: groupContext ? $groupContext : undefined
 					}),
 					false,
 					$state,
--- a/frontend/src/lib/components/apps/components/layout/AppContainer.svelte
+++ b/frontend/src/lib/components/apps/components/layout/AppContainer.svelte
@@ -2,20 +2,28 @@
 	import { getContext } from 'svelte'
 	import { initOutput } from '../../editor/appUtils'
 	import SubGridEditor from '../../editor/SubGridEditor.svelte'
-	import type { AppViewerContext, ComponentCustomCSS } from '../../types'
+	import type { AppViewerContext, ComponentCustomCSS, RichConfigurations } from '../../types'
 	import { concatCustomCss } from '../../utils'
 	import InitializeComponent from '../helpers/InitializeComponent.svelte'
+	// import type { EvalV2AppInput, StaticAppInput } from '../../inputType'
+	import { writable } from 'svelte/store'
+	import { InputValue } from '../helpers'
+	import GroupWrapper from '../GroupWrapper.svelte'

 	export let id: string
 	export let componentContainerHeight: number
 	export let customCss: ComponentCustomCSS<'containercomponent'> | undefined = undefined
 	export let render: boolean
+	export let groupFields: RichConfigurations | undefined = undefined

 	const { app, focusedGrid, selectedComponent, worldStore, connectingInput } =
 		getContext<AppViewerContext>('AppViewerContext')

-	//used so that we can count number of outputs setup for first refresh
-	initOutput($worldStore, id, {})
+	let groupContext = writable({})
+
+	let outputs = initOutput($worldStore, id, { group: $groupContext })
+
+	$: outputs.group.set($groupContext, true)

 	function onFocus() {
 		$focusedGrid = {
@@ -29,21 +37,29 @@

 <InitializeComponent {id} />

+{#each Object.keys(groupFields ?? {}) as field}
+	{#if groupFields && field in groupFields}
+		<InputValue key={field} {id} input={groupFields[field]} bind:value={$groupContext[field]} />
+	{/if}
+{/each}
+
 <div class="w-full h-full">
 	{#if $app.subgrids?.[`${id}-0`]}
-		<SubGridEditor
-			visible={render}
-			{id}
-			class={css?.container?.class}
-			style={css?.container?.style}
-			subGridId={`${id}-0`}
-			containerHeight={componentContainerHeight}
-			on:focus={() => {
-				if (!$connectingInput.opened) {
-					$selectedComponent = [id]
-				}
-				onFocus()
-			}}
-		/>
+		<GroupWrapper {groupContext}>
+			<SubGridEditor
+				visible={render}
+				{id}
+				class={css?.container?.class}
+				style={css?.container?.style}
+				subGridId={`${id}-0`}
+				containerHeight={componentContainerHeight}
+				on:focus={() => {
+					if (!$connectingInput.opened) {
+						$selectedComponent = [id]
+					}
+					onFocus()
+				}}
+			/>
+		</GroupWrapper>
 	{/if}
 </div>
--- a/frontend/src/lib/components/apps/editor/appUtils.ts
+++ b/frontend/src/lib/components/apps/editor/appUtils.ts
@@ -13,7 +13,8 @@ import {
 	getRecommendedDimensionsByComponent,
 	type AppComponent,
 	type BaseComponent,
-	type InitialAppComponent
+	type InitialAppComponent,
+	type TypedComponent
 } from './component'
 import { gridColumns } from '../gridUtils'
 import { allItems } from '../utils'
@@ -598,7 +599,8 @@ export function sortGridItemsPosition<T>(
 export function connectInput(
 	connectingInput: ConnectingInput,
 	componentId: string,
-	path: string
+	path: string,
+	componentType?: TypedComponent['type']
 ): ConnectingInput {
 	if (connectingInput) {
 		if (connectingInput.onConnect) {
@@ -610,7 +612,8 @@ export function connectInput(
 			input: {
 				connection: {
 					componentId,
-					path
+					path,
+					componentType
 				},
 				type: 'connected'
 			},
--- a/frontend/src/lib/components/apps/editor/appUtilsInfer.ts
+++ b/frontend/src/lib/components/apps/editor/appUtilsInfer.ts
@@ -15,7 +15,10 @@ export async function inferDeps(
 		const noutputs = outputs
 			.filter(
 				([componentId, id]) =>
-					componentId == 'row' || componentId == 'iter' || id in (worldOutputs[componentId] ?? {})
+					componentId == 'row' ||
+					componentId == 'iter' ||
+					componentId == 'group' ||
+					id in (worldOutputs[componentId] ?? {})
 			)
 			.map(([componentId, id]) => ({
 				componentId: componentId,
--- a/frontend/src/lib/components/apps/editor/component/Component.svelte
+++ b/frontend/src/lib/components/apps/editor/component/Component.svelte
@@ -480,6 +480,7 @@
 			/>
 		{:else if component.type === 'containercomponent'}
 			<AppContainer
+				groupFields={component.groupFields}
 				id={component.id}
 				customCss={component.customCss}
 				{componentContainerHeight}
--- a/frontend/src/lib/components/apps/editor/component/components.ts
+++ b/frontend/src/lib/components/apps/editor/component/components.ts
@@ -48,6 +48,7 @@ import type {
 	ComponentCustomCSS,
 	GridItem,
 	RichConfiguration,
+	RichConfigurations,
 	StaticRichConfigurations
 } from '../../types'
 import type { Size } from '../../svelte-grid/types'
@@ -115,7 +116,9 @@ export type TabsComponent = BaseComponent<'tabscomponent'> & {
 	disabledTabs: RichConfiguration[]
 }
 export type ListComponent = BaseComponent<'listcomponent'>
-export type ContainerComponent = BaseComponent<'containercomponent'>
+export type ContainerComponent = BaseComponent<'containercomponent'> & {
+	groupFields: RichConfigurations
+}
 export type DrawerComponent = BaseComponent<'drawercomponent'>
 export type MapComponent = BaseComponent<'mapcomponent'>
 export type VerticalSplitPanesComponent = BaseComponent<'verticalsplitpanescomponent'> & {
--- a/frontend/src/lib/components/apps/editor/contextPanel/ComponentOutput.svelte
+++ b/frontend/src/lib/components/apps/editor/contextPanel/ComponentOutput.svelte
@@ -39,7 +39,11 @@
 		componentId={gridItem.id}
 		on:select={({ detail }) => {
 			if ($connectingInput.opened) {
-				$connectingInput = connectInput($connectingInput, gridItem.id, detail)
+				let typ = gridItem?.data?.type
+				let splitted = detail?.split('.')
+				let componentId = typ == 'containercomponent' ? splitted?.[0] : gridItem.id
+				let path = typ == 'containercomponent' ? splitted?.[1] : detail
+				$connectingInput = connectInput($connectingInput, componentId, path, typ)
 			}
 		}}
 	/>
--- a/frontend/src/lib/components/apps/editor/contextPanel/ComponentOutputViewer.svelte
+++ b/frontend/src/lib/components/apps/editor/contextPanel/ComponentOutputViewer.svelte
@@ -17,6 +17,7 @@
 		if (observableOutputs) {
 			Object.entries(observableOutputs).forEach(([k, output]) => {
 				object[k] = undefined
+				console.log('subscribing to', k, output)
 				output?.subscribe(
 					{
 						id: 'alloutputs' + componentId + '-' + k,
--- a/frontend/src/lib/components/apps/editor/inlineScriptsPanel/InlineScriptEditor.svelte
+++ b/frontend/src/lib/components/apps/editor/inlineScriptsPanel/InlineScriptEditor.svelte
@@ -18,7 +18,7 @@
 	import { buildExtraLib } from '../../utils'
 	import RunButton from './RunButton.svelte'
 	import { scriptLangToEditorLang } from '$lib/scripts'
-	import ScriptGen from '$lib/components/codeGen/ScriptGen.svelte'
+	import ScriptGen from '$lib/components/copilot/ScriptGen.svelte'
 	import DiffEditor from '$lib/components/DiffEditor.svelte'
 	import { userStore } from '$lib/stores'
 	import CacheTtlPopup from './CacheTtlPopup.svelte'
@@ -195,6 +195,10 @@
 					editor={inlineScript?.language === 'frontend' ? simpleEditor : editor}
 					{diffEditor}
 					inlineScript
+					args={Object.entries(fields).reduce((acc, [key, obj]) => {
+						acc[key] = obj.type === 'static' ? obj.value : undefined
+						return acc
+					}, {})}
 				/>

 				<Button
@@ -281,6 +285,10 @@
 							}
 							$app = $app
 						}}
+						args={Object.entries(fields).reduce((acc, [key, obj]) => {
+							acc[key] = obj.type === 'static' ? obj.value : undefined
+							return acc
+						}, {})}
 					/>
 				{:else}
 					<SimpleEditor
--- a/frontend/src/lib/components/apps/editor/settingsPanel/ComponentPanel.svelte
+++ b/frontend/src/lib/components/apps/editor/settingsPanel/ComponentPanel.svelte
@@ -32,6 +32,7 @@
 	import { inferDeps } from '../appUtilsInfer'
 	import EvalV2InputEditor from './inputEditor/EvalV2InputEditor.svelte'
 	import type { ResultAppInput } from '../../inputType'
+	import GridGroup from './GridGroup.svelte'

 	export let componentSettings: { item: GridItem; parent: string | undefined } | undefined =
 		undefined
@@ -310,6 +311,11 @@
 				bind:component={componentSettings.item.data}
 				word="Step"
 			/>
+		{:else if componentSettings.item.data.type === 'containercomponent'}
+			<GridGroup
+				bind:groupFields={componentSettings.item.data.groupFields}
+				bind:component={componentSettings.item.data}
+			/>
 		{:else if componentSettings.item.data.type === 'conditionalwrapper'}
 			<GridCondition
 				bind:conditions={componentSettings.item.data.conditions}
@@ -333,7 +339,7 @@
 					userInputEnabled={false}
 				/>
 			</PanelSection>
-		{:else}
+		{:else if componentSettings.item.data.type != 'containercomponent'}
 			<div class="h-full w-full text-sm text-tertiary text-center py-8 px-2"
 				>{ccomponents[component.type].name} has no configuration</div
 			>
--- a/frontend/src/lib/components/apps/editor/settingsPanel/GridGroup.svelte
+++ b/frontend/src/lib/components/apps/editor/settingsPanel/GridGroup.svelte
@@ -0,0 +1,104 @@
+<script lang="ts">
+	import Button from '$lib/components/common/button/Button.svelte'
+	import { faPlus } from '@fortawesome/free-solid-svg-icons'
+	import type { RichConfigurations } from '../../types'
+	import type { AppComponent } from '../component'
+	import PanelSection from './common/PanelSection.svelte'
+	import Toggle from '$lib/components/Toggle.svelte'
+	import InputsSpecsEditor from './InputsSpecsEditor.svelte'
+	import Tooltip from '$lib/components/Tooltip.svelte'
+
+	export let groupFields: RichConfigurations | undefined
+
+	export let component: AppComponent
+
+	// const { app, runnableComponents } = getContext<AppViewerContext>('AppViewerContext')
+
+	let fieldName: string = ''
+	function addField(name: string) {
+		if (name == '') return
+		groupFields = {
+			...(groupFields ?? {}),
+			[name]: {
+				type: 'static',
+				value: '',
+				fieldType: 'object'
+			}
+		}
+	}
+</script>
+
+<div class="flex p-2 gap-1 items-center">
+	<Toggle
+		size="xs"
+		checked={groupFields != undefined}
+		on:change={(e) => {
+			if (e.detail) {
+				groupFields = {}
+			} else {
+				groupFields = undefined
+			}
+			console.log(groupFields)
+		}}
+		options={{ right: 'container is a component group' }}
+	/>
+	<Tooltip
+		>Group fields allow inner components to depend on the group fields which make the container a
+		group of component that is encapsulated. Inside the group, it is possible to retrieve the values
+		using `group.<x />` where x is the group field name</Tooltip
+	>
+</div>
+{#if groupFields != undefined}
+	<PanelSection
+		title={`Group Fields ${
+			Object.keys(groupFields ?? {}).length > 0
+				? `(${Object.keys(groupFields ?? {}).length ?? 0})`
+				: ''
+		}`}
+	>
+		{#if Object.keys(groupFields ?? {}).length == 0}
+			<span class="text-xs text-tertiary">No group fields</span>
+		{/if}
+		<div class="w-full flex gap-2 flex-col mt-2">
+			<InputsSpecsEditor
+				on:delete={(e) => {
+					if (!groupFields) {
+						return
+					}
+					delete groupFields[e.detail]
+					groupFields = groupFields
+				}}
+				id={component.id}
+				shouldCapitalize={false}
+				displayType
+				deletable
+				bind:inputSpecs={groupFields}
+			/>
+			<div class="flex flex-row gap-2 items-center relative">
+				<input
+					type="text"
+					on:keydown|stopPropagation={(event) => {
+						switch (event.key) {
+							case 'Enter':
+								event.preventDefault()
+								addField(fieldName)
+								break
+						}
+					}}
+					placeholder="Group Field Name"
+					bind:value={fieldName}
+				/>
+
+				<Button
+					disabled={fieldName == ''}
+					size="sm"
+					color="light"
+					variant="border"
+					startIcon={{ icon: faPlus }}
+					on:click={() => addField(fieldName)}
+					iconOnly
+				/>
+			</div>
+		</div>
+	</PanelSection>
+{/if}
--- a/frontend/src/lib/components/apps/editor/settingsPanel/GridTab.svelte
+++ b/frontend/src/lib/components/apps/editor/settingsPanel/GridTab.svelte
@@ -17,15 +17,6 @@

 	export let canDisableTabs: boolean = false

-	// Migration code
-	$: if (tabs?.length !== disabledTabs?.length && canDisableTabs) {
-		disabledTabs = Array(tabs.length).fill({
-			type: 'static',
-			value: false,
-			fieldType: 'boolean'
-		})
-	}
-
 	export let word: string = 'Tab'

 	export let component: AppComponent
--- a/frontend/src/lib/components/apps/editor/settingsPanel/InputsSpecEditor.svelte
+++ b/frontend/src/lib/components/apps/editor/settingsPanel/InputsSpecEditor.svelte
@@ -75,7 +75,7 @@
 					{/if}
 				</span>
 				{#if displayType}
-					<div class="text-xs text-tertiary">
+					<div class="text-xs text-tertiary mr-1">
 						{fieldType === 'array' && subFieldType
 							? `${fieldTypeToTsType(subFieldType)}[]`
 							: fieldTypeToTsType(fieldType)}
--- a/frontend/src/lib/components/apps/editor/settingsPanel/InputsSpecsEditor.svelte
+++ b/frontend/src/lib/components/apps/editor/settingsPanel/InputsSpecsEditor.svelte
@@ -1,4 +1,6 @@
 <script lang="ts">
+	import CloseButton from '$lib/components/common/CloseButton.svelte'
+	import { createEventDispatcher } from 'svelte'
 	import type { RichConfigurations } from '../../types'

 	import InputsSpecEditor from './InputsSpecEditor.svelte'
@@ -11,8 +13,11 @@
 	export let shouldCapitalize: boolean = true
 	export let resourceOnly = false
 	export let displayType = false
+	export let deletable = false

 	$: finalInputSpecsConfiguration = inputSpecsConfiguration ?? inputSpecs
+
+	const dispatch = createEventDispatcher()
 </script>

 {#if inputSpecs}
@@ -48,6 +53,10 @@
 					customTitle={meta?.['customTitle']}
 					{displayType}
 				/>
+				{#if deletable}
+					<div class="flex flex-row-reverse -mt-4">
+						<CloseButton noBg on:close={() => dispatch('delete', k)} />
+					</div>{/if}
 			{/if}
 		{/each}
 	</div>
--- a/frontend/src/lib/components/apps/inputType.ts
+++ b/frontend/src/lib/components/apps/inputType.ts
@@ -1,4 +1,5 @@
 import type { ReadFileAs } from '../common/fileInput/model'
+import type { TypedComponent } from './editor/component'
 import type { InlineScript } from './types'

 export type InputType =
@@ -25,6 +26,7 @@ export type InputType =
 // Connection to an output of another component
 // defined by the id of the component and the path of the output
 export type InputConnection = {
+	componentType?: TypedComponent['type']
 	componentId: string
 	path: string
 }
--- a/frontend/src/lib/components/apps/types.ts
+++ b/frontend/src/lib/components/apps/types.ts
@@ -163,6 +163,8 @@ export type ListContext = Writable<{

 export type ListInputs = (id: string, value: any) => void

+export type GroupContext = Writable<Record<string, any>>
+
 export type AppViewerContext = {
 	worldStore: Writable<World>
 	app: Writable<App>
--- a/frontend/src/lib/components/apps/utils.ts
+++ b/frontend/src/lib/components/apps/utils.ts
@@ -185,7 +185,9 @@ declare function closeModal(id: string): void;
 }
 declare const state: ${JSON.stringify(state)};
 declare const iter: {index: number, value: any};
-declare const row: {index: number, value: any};
+declare const row: Record<string, any>;
+declare const group: Record<string, any>;
+
 `
 }

--- a/frontend/src/lib/components/codeGen/index.ts
+++ b/frontend/src/lib/components/codeGen/index.ts
@@ -1 +0,0 @@
-export { default as ScriptGen } from './ScriptGen.svelte'
--- a/frontend/src/lib/components/copilot/FlowCopilotDrawer.svelte
+++ b/frontend/src/lib/components/copilot/FlowCopilotDrawer.svelte
@@ -0,0 +1,232 @@
+<script lang="ts">
+	import { Button, Drawer, DrawerContent, Badge } from '$lib/components/common'
+	import ToggleButtonGroup from '$lib/components/common/toggleButton-v2/ToggleButtonGroup.svelte'
+	import ToggleButton from '$lib/components/common/toggleButton-v2/ToggleButton.svelte'
+	import { getContext } from 'svelte'
+	import { Icon } from 'svelte-awesome'
+	import { faAdd, faClose, faMagicWandSparkles } from '@fortawesome/free-solid-svg-icons'
+	import { capitalize, classNames } from '$lib/utils'
+	import { APP_TO_ICON_COMPONENT } from '../icons'
+	import { charsToNumber, numberToChars } from '../flows/idUtils'
+	import type { FlowCopilotContext } from './flow'
+
+	export let hubCompletions: (text: string, idx: number, type: 'trigger' | 'script') => void
+	export let genFlow: (index: number) => void
+	export let flowCopilotMode: 'trigger' | 'sequence'
+
+	const { drawerStore, modulesStore, currentStepStore } =
+		getContext<FlowCopilotContext>('FlowCopilotContext')
+</script>
+
+<Drawer bind:this={$drawerStore}>
+	<DrawerContent on:close={$drawerStore.closeDrawer} title="AI Flow Builder">
+		<div class="flex flex-col gap-6">
+			<ToggleButtonGroup bind:selected={flowCopilotMode}>
+				<ToggleButton value="trigger" label="Trigger" />
+				<ToggleButton value="sequence" label="Sequence" />
+			</ToggleButtonGroup>
+			{#each $modulesStore as copilotModule, i}
+				<div>
+					{#if i === 1 && $modulesStore[i - 1].type === 'trigger'}
+						<div class="flex flex-row items-center mb-4 gap-1">
+							<p class="text-sm font-semibold">For loop</p>
+							<Badge color="indigo">{numberToChars(i)}_loop</Badge>
+						</div>
+					{/if}
+					<div class={i === 1 && $modulesStore[i - 1].type === 'trigger' ? 'pl-4' : ''}>
+						<div class="flex flex-row items-center justify-between">
+							<div class="flex flex-row justify-between items-center w-full mb-2">
+								<div class="flex flex-row items-center gap-1">
+									<p class="text-sm font-semibold"
+										>{copilotModule.type === 'trigger' ? 'Trigger' : 'Action'}</p
+									>
+									<Badge color="indigo">{numberToChars(i)}</Badge>
+								</div>
+								{#if flowCopilotMode === 'sequence' && i >= 1}
+									<button
+										on:click={() => {
+											modulesStore.update((prev) => {
+												prev.splice(i, 1)
+												return prev
+											})
+										}}
+									>
+										<Icon data={faClose} />
+									</button>
+								{/if}
+							</div>
+						</div>
+						{#if copilotModule.source !== undefined}
+							<div
+								class={classNames(
+									'p-4 gap-4 flex flex-row grow  transition-all items-center rounded-md justify-between border',
+									$currentStepStore !== undefined &&
+										$currentStepStore !== 'Input' &&
+										i < charsToNumber($currentStepStore)
+										? 'bg-gray-700/10'
+										: 'bg-surface'
+								)}
+							>
+								<div class="flex items-center gap-4">
+									<div
+										class="rounded-md p-1 flex justify-center items-center bg-surface border h-6 w-6"
+									>
+										{#if copilotModule.source === 'hub' && copilotModule.selectedCompletion}
+											<svelte:component
+												this={APP_TO_ICON_COMPONENT[copilotModule.selectedCompletion['app']]}
+											/>
+										{:else}
+											<Icon data={faMagicWandSparkles} />
+										{/if}
+									</div>
+
+									<div class="w-full text-left font-normal">
+										<div class="text-primary flex-wrap text-sm font-medium">
+											{copilotModule.source === 'hub' && copilotModule.selectedCompletion
+												? copilotModule.selectedCompletion.summary
+												: copilotModule.description}
+										</div>
+									</div>
+								</div>
+								{#if copilotModule.source === 'hub' && copilotModule.selectedCompletion && copilotModule.selectedCompletion?.kind !== 'script'}
+									<Badge color="gray" baseClass="border"
+										>{capitalize(copilotModule.selectedCompletion.kind)}</Badge
+									>
+								{/if}
+
+								<button
+									on:click={() => {
+										copilotModule.selectedCompletion = undefined
+										copilotModule.source = undefined
+										if (
+											$currentStepStore !== undefined &&
+											$currentStepStore !== 'Input' &&
+											i < charsToNumber($currentStepStore)
+										) {
+											$currentStepStore = numberToChars(i)
+										}
+									}}
+								>
+									<Icon data={faClose} />
+								</button>
+							</div>
+							{#if $currentStepStore !== undefined && $currentStepStore !== 'Input' && i < charsToNumber($currentStepStore)}
+								<p class="font-semibold text-sm text-green-600"
+									>Already generated, edit step to regenerate from this point</p
+								>
+							{/if}
+						{:else}
+							<input
+								name="description"
+								type="text"
+								placeholder={copilotModule.type === 'trigger'
+									? 'describe what should trigger your flow'
+									: 'describe what this step should do'}
+								bind:value={copilotModule.description}
+								on:input={() => {
+									if (copilotModule.description.length > 2) {
+										hubCompletions(copilotModule.description, i, copilotModule.type)
+									} else {
+										copilotModule.hubCompletions = []
+									}
+								}}
+							/>
+						{/if}
+						{#if copilotModule.description.length > 2 && copilotModule.source === undefined}
+							<button
+								class="mt-2 p-4 gap-4 flex flex-row hover:bg-surface-hover bg-surface transition-all items-center rounded-md justify-between w-full border"
+								on:click={() => {
+									copilotModule.source = 'custom'
+									copilotModule.selectedCompletion = undefined
+								}}
+							>
+								<div class="flex items-center gap-4">
+									<div
+										class="rounded-md p-1 flex justify-center items-center bg-surface border w-6 h-6"
+									>
+										<Icon data={faMagicWandSparkles} />
+									</div>
+
+									<div class="w-full text-left text-sm">
+										<div class="text-primary flex-wrap font-medium">
+											Generate step from scratch using AI
+										</div>
+									</div>
+								</div>
+							</button>
+							{#if copilotModule.hubCompletions.length > 0}
+								<p class="mt-2 font-semibold text-sm">Hub scripts</p>
+								<ul class="divide-y border rounded-md transition-all mt-1">
+									{#each copilotModule.hubCompletions as item (item.path)}
+										<li>
+											<button
+												class="p-4 gap-4 flex flex-row hover:bg-surface-hover bg-surface transition-all items-center rounded-md justify-between w-full"
+												on:click={() => {
+													copilotModule.source = 'hub'
+													copilotModule.selectedCompletion = item
+												}}
+											>
+												<div class="flex items-center gap-4">
+													<div
+														class="rounded-md p-1 flex justify-center items-center bg-surface border w-6 h-6"
+													>
+														<svelte:component this={APP_TO_ICON_COMPONENT[item['app']]} />
+													</div>
+
+													<div class="text-left font-normal text-sm">
+														<div class="text-primary font-medium">
+															{item.summary ?? ''}
+														</div>
+													</div>
+												</div>
+												{#if item.kind !== 'script'}
+													<Badge color="gray" baseClass="border">{capitalize(item.kind)}</Badge>
+												{/if}
+											</button>
+										</li>
+									{/each}
+								</ul>
+							{/if}
+						{/if}
+					</div>
+				</div>
+			{/each}
+			{#if flowCopilotMode !== 'trigger'}
+				<div class="flex justify-start">
+					<Button
+						startIcon={{ icon: faAdd }}
+						size="xs"
+						variant="border"
+						on:click={() =>
+							modulesStore.update((prev) => [
+								...prev,
+								{
+									id: numberToChars(prev.length),
+									type: 'script',
+									description: '',
+									code: '',
+									source: undefined,
+									hubCompletions: [],
+									selectedCompletion: undefined
+								}
+							])}>Add step</Button
+					>
+				</div>
+			{/if}
+
+			<Button
+				on:click={() =>
+					$currentStepStore !== undefined && $currentStepStore !== 'Input'
+						? genFlow(charsToNumber($currentStepStore))
+						: genFlow(0)}
+				spacingSize="md"
+				startIcon={{ icon: faMagicWandSparkles }}
+				disabled={$modulesStore.find((m) => m.source === undefined) !== undefined}
+			>
+				{$currentStepStore !== undefined && $currentStepStore !== 'Input'
+					? `Regenerate flow from step '${$currentStepStore}'`
+					: 'Build flow'}
+			</Button>
+		</div>
+	</DrawerContent>
+</Drawer>
--- a/frontend/src/lib/components/copilot/FlowCopilotStatus.svelte
+++ b/frontend/src/lib/components/copilot/FlowCopilotStatus.svelte
@@ -0,0 +1,124 @@
+<script lang="ts">
+	import { getContext } from 'svelte'
+	import ManualPopover from '../ManualPopover.svelte'
+	import Button from '../common/button/Button.svelte'
+	import { WindmillIcon } from '../icons'
+	import type { FlowCopilotContext } from './flow'
+	import { faMagicWandSparkles } from '@fortawesome/free-solid-svg-icons'
+	import { charsToNumber } from '../flows/idUtils'
+	import { existsOpenaiResourcePath } from '$lib/stores'
+	import Popup from '../common/popup/Popup.svelte'
+
+	export let copilotLoading: boolean
+	export let copilotStatus: string
+	export let abortController: AbortController | undefined
+	export let genFlow: (index: number) => void
+	export let handleFlowGenInputs: () => void
+
+	let copilotPopover: ManualPopover | undefined = undefined
+
+	const { modulesStore, drawerStore, currentStepStore } =
+		getContext<FlowCopilotContext>('FlowCopilotContext')
+
+	$: copilotStatus.length > 0 ? copilotPopover?.open() : copilotPopover?.close()
+</script>
+
+{#if $existsOpenaiResourcePath}
+	<ManualPopover bind:this={copilotPopover}>
+		<Button
+			size="xs"
+			btnClasses={'mr-2 ' + ($currentStepStore !== undefined ? 'z-[901]' : '')}
+			on:click={() => {
+				if (copilotLoading || ($currentStepStore !== undefined && $currentStepStore !== 'Input')) {
+					abortController?.abort()
+					copilotStatus = ''
+					if (!copilotLoading) {
+						$currentStepStore = undefined
+					}
+				} else {
+					$drawerStore?.openDrawer()
+				}
+			}}
+			startIcon={copilotLoading ||
+			($currentStepStore !== undefined && $currentStepStore !== 'Input')
+				? undefined
+				: {
+						icon: faMagicWandSparkles
+				  }}
+			color={copilotLoading || ($currentStepStore !== undefined && $currentStepStore !== 'Input')
+				? 'red'
+				: 'light'}
+			variant={copilotLoading || ($currentStepStore !== undefined && $currentStepStore !== 'Input')
+				? 'contained'
+				: 'border'}
+		>
+			{#if copilotLoading}
+				<WindmillIcon white class="mr-1 text-white" height="16px" width="20px" spin="veryfast" />
+			{/if}
+
+			{copilotLoading
+				? 'Stop'
+				: $currentStepStore !== undefined && $currentStepStore !== 'Input'
+				? 'Exit'
+				: 'AI Flow Builder'}
+		</Button>
+		<div slot="content" class="text-sm flex flex-row items-center z-[901]"
+			><span class="font-semibold">
+				{copilotStatus}
+			</span>
+			{#if !copilotLoading && $currentStepStore !== undefined && $currentStepStore !== 'Input'}
+				<Button
+					size="xs"
+					btnClasses="ml-2"
+					color="red"
+					on:click={() => {
+						$drawerStore?.openDrawer()
+					}}>Edit prompts</Button
+				>
+				<Button
+					btnClasses="ml-2"
+					color="green"
+					size="xs"
+					on:click={() => {
+						if ($currentStepStore === undefined) {
+							return
+						}
+						const stepNb = charsToNumber($currentStepStore)
+						if (stepNb >= $modulesStore.length - 1) {
+							handleFlowGenInputs()
+						} else {
+							genFlow(stepNb + 1)
+						}
+					}}
+				>
+					{charsToNumber($currentStepStore) >= $modulesStore.length - 1
+						? 'Flow inputs'
+						: 'Next step'}
+				</Button>
+			{/if}</div
+		>
+	</ManualPopover>
+{:else}
+	<Popup>
+		<svelte:fragment slot="button">
+			<Button
+				size="xs"
+				btnClasses="mr-2"
+				startIcon={{
+					icon: faMagicWandSparkles
+				}}
+				color={'light'}
+				variant={'border'}
+				nonCaptureEvent
+			>
+				AI Flow Builder
+			</Button>
+		</svelte:fragment>
+		<div class="block text-primary">
+			<p class="text-sm"
+				>Enable Windmill AI in the <a href="/workspace_settings?tab=openai">workspace settings.</a
+				></p
+			>
+		</div>
+	</Popup>
+{/if}
--- a/frontend/src/lib/components/copilot/ScriptFix.svelte
+++ b/frontend/src/lib/components/copilot/ScriptFix.svelte
@@ -23,6 +23,7 @@
 	export let editor: Editor | undefined
 	export let diffEditor: DiffEditor | undefined
 	export let error: string
+	export let args: Record<string, any>

 	// state
 	let genLoading: boolean = false
@@ -103,7 +104,7 @@

 	$: !$generatedCode && hideDiff()

-	$: dbSchema = $dbSchemas[Object.keys($dbSchemas)[0]]
+	$: dbSchema = $dbSchemas[(lang === 'graphql' ? args.api : args.database)?.replace('$res:', '')]
 </script>

 {#if SUPPORTED_LANGUAGES.has(lang)}
@@ -173,7 +174,7 @@
 								width="20px"
 								spin="veryfast"
 							/>
-							Cancel
+							Stop
 						{:else}
 							AI Fix
 						{/if}
--- a/frontend/src/lib/components/copilot/ScriptGen.svelte
+++ b/frontend/src/lib/components/copilot/ScriptGen.svelte
@@ -34,6 +34,7 @@
 	export let editor: Editor | SimpleEditor | undefined
 	export let diffEditor: DiffEditor | undefined
 	export let inlineScript = false
+	export let args: Record<string, any>

 	// state
 	let funcDesc: string = ''
@@ -143,7 +144,8 @@
 	$: !$generatedCode && hideDiff()
 	$: editor && setSelectionHandler()
 	$: selection && (isEdit = !selection.isEmpty())
-	$: dbSchema = $dbSchemas[Object.keys($dbSchemas)[0]]
+
+	$: dbSchema = $dbSchemas[(lang === 'graphql' ? args.api : args.database)?.replace('$res:', '')]
 </script>

 {#if $generatedCode.length > 0 && !genLoading}
@@ -242,7 +244,7 @@
 							width="20px"
 							spin="veryfast"
 						/>
-						Cancel
+						Stop
 					{:else}
 						{isEdit ? 'AI Edit' : 'AI Gen'}
 					{/if}
@@ -297,7 +299,7 @@
 								In order to better generate the script, we pass the selected DB schema to GPT-4.
 							</Tooltip>
 						</p>
-						{#if dbSchema.lang === 'postgresql'}
+						{#if dbSchema.lang !== 'graphql' && (dbSchema.schema?.public || dbSchema.schema?.PUBLIC)}
 							<ToggleButtonGroup class="w-auto shrink-0" bind:selected={dbSchema.publicOnly}>
 								<ToggleButton value={true} label="Public schema" />
 								<ToggleButton value={false} label="All schemas" />
--- a/frontend/src/lib/components/copilot/flow.ts
+++ b/frontend/src/lib/components/copilot/flow.ts
@@ -0,0 +1,165 @@
+import { ScriptService, type Script } from '$lib/gen'
+import { addResourceTypes, deltaCodeCompletion, getNonStreamingCompletion } from './lib'
+import type { Writable } from 'svelte/store'
+import type Editor from '../Editor.svelte'
+import type { Drawer } from '../common'
+
+export type FlowCopilotModule = {
+	id: string
+	type: 'trigger' | 'script'
+	description: string
+	code: string
+	source: 'hub' | 'custom' | undefined
+	hubCompletions: {
+		path: string
+		summary: string
+		approved: boolean
+		kind: string
+		app: string
+		ask_id: number
+	}[]
+	selectedCompletion:
+		| {
+				path: string
+				summary: string
+				approved: boolean
+				kind: string
+				app: string
+				ask_id: number
+		  }
+		| undefined
+	editor?: Editor
+}
+
+export type FlowCopilotContext = {
+	drawerStore: Writable<Drawer | undefined>
+	modulesStore: Writable<FlowCopilotModule[]>
+	currentStepStore: Writable<string | undefined>
+}
+
+const systemPrompt = `You write code as instructed by the user. Only output code. Wrap the code in a code block. 
+Put explanations directly in the code as comments.
+
+Here's how interactions have to look like:
+user: {sample_question}
+assistant: \`\`\`typescript
+{code}
+\`\`\``
+
+const additionalInformation = `Additional information: We have to export a "main" function like this: "export async function main(...)" and specify the parameter types but do not call it.
+You have access to the following resource types, if you need them, you have to define the type exactly as specified and add them as parameters: {resourceTypes}
+Only use the ones you need. If the type name conflicts with the imported object, rename the imported object NOT THE TYPE.`
+
+const triggerPrompt = `I'm building a workflow which is a sequence of script steps. Write the first script in typescript which should check for {description} and return an array.
+You can use "const {state_name}: {state_type} = getState(...)" and "setState(...)" from "npm:windmill-client@1" to maintain state across runs.
+
+${additionalInformation}`
+
+const firstActionPrompt = `I'm building a workflow which is a sequence of script steps. Write a script in typescript which should {description}.
+Return the script's output.
+
+${additionalInformation}`
+
+const actionPrompt = `I'm building a workflow which is a sequence of script steps. Write a script in typescript which should {description} using as a parameter called "prev_output" the output of the previous script.
+Infer the type of "prev_output" from the previous script: \`\`\`typescript\n{prevCode}\n\`\`\`.
+Return the script's output.
+
+${additionalInformation}`
+
+const loopGluePrompt = `I'm building a workflow which is a sequence of script steps. 
+My current step code has the following inputs: {inputs}. 
+Determine what to pass as inputs. You can only use the following:
+- \`flow_input\` (javascript object): general inputs that are passed to the workflow, you can assume any object properties.
+- \`flow_input.iter.value\` (javascript object): it is ONE ELEMENT of the output of the previous step. Infer its type from the previous's step code: \`\`\`typescript\n{prevCode}\n\`\`\`
+
+Reply in the following format:
+input_name: expr`
+
+const gluePrompt = `I'm building a workflow which is a sequence of script steps. 
+My current step code has the following inputs: {inputs}. 
+Determine what to pass as inputs. You can only use the following:
+- \`flow_input\` (javascript object): general inputs that are passed to the workflow, you can assume any object properties.
+- \`prev_output\` (javascript object): previous output is the output of the previous step. Infer its type from the previous's step code: \`\`\`typescript\n{prevCode}\n\`\`\`
+
+Reply in the following format:
+input_name: expr`
+
+export async function stepCopilot(
+	module: FlowCopilotModule,
+	deltaCodeStore: Writable<string>,
+	prevCode: string,
+	abortController: AbortController
+) {
+	if (module.source === undefined) {
+		throw new Error('Module not configured')
+	}
+	if (module.source === 'hub' && module.selectedCompletion) {
+		const hubScript = await ScriptService.getHubScriptByPath({
+			path: module.selectedCompletion.path
+		})
+		deltaCodeStore.set(hubScript.content)
+		return hubScript.content
+	} else {
+		let prompt =
+			module.type === 'trigger'
+				? triggerPrompt
+				: prevCode.length > 0
+				? actionPrompt
+				: firstActionPrompt
+		prompt = prompt.replace('{description}', module.description).replace('{prevCode}', prevCode)
+		prompt = await addResourceTypes(
+			{
+				type: 'gen',
+				language: 'bun' as Script.language,
+				description: module.description,
+				dbSchema: undefined
+			},
+			prompt
+		)
+		const code = await deltaCodeCompletion(
+			[
+				{
+					role: 'system',
+					content: systemPrompt
+				},
+				{
+					role: 'user',
+					content: prompt
+				}
+			],
+			deltaCodeStore,
+			abortController
+		)
+		return code
+	}
+}
+
+export async function glueCopilot(
+	inputs: string[],
+	prevCode: string,
+	isLoop: boolean,
+	abortController: AbortController
+) {
+	let response = await getNonStreamingCompletion(
+		[
+			{
+				role: 'user',
+				content: (isLoop ? loopGluePrompt : gluePrompt)
+					.replace('{inputs}', inputs.join(', '))
+					.replace('{prevCode}', prevCode)
+			}
+		],
+		abortController
+	)
+
+	const matches = response.matchAll(/(.+?): (.+)/g)
+
+	const result: Record<string, string> = {}
+	for (const match of matches) {
+		const inputName = match[1]
+		const inputExpr = match[2].replace(',', '')
+		result[inputName] = inputExpr
+	}
+
+	return result
+}
--- a/frontend/src/lib/components/copilot/lib.ts
+++ b/frontend/src/lib/components/copilot/lib.ts
@@ -7,12 +7,15 @@ import { existsOpenaiResourcePath, workspaceStore, type DBSchema } from '$lib/st
 import { formatResourceTypes } from './utils'

 import { EDIT_CONFIG, FIX_CONFIG, GEN_CONFIG } from './prompts'
-import type { CompletionCreateParamsStreaming } from 'openai/resources/chat'
+import type {
+	CompletionCreateParamsStreaming,
+	CreateChatCompletionRequestMessage
+} from 'openai/resources/chat'

 export const SUPPORTED_LANGUAGES = new Set(Object.keys(GEN_CONFIG.prompts))

 const openaiConfig: CompletionCreateParamsStreaming = {
-	temperature: 0.5,
+	temperature: 0.3,
 	max_tokens: 2048,
 	model: 'gpt-4',
 	stream: true,
@@ -20,12 +23,24 @@ const openaiConfig: CompletionCreateParamsStreaming = {
 }

 let workspace: string | undefined = undefined
+let openai: OpenAI | undefined = undefined

 workspaceStore.subscribe(async (value) => {
 	workspace = value
-	if (workspace) {
+	const baseURL = `${location.origin}${OpenAPI.BASE}/w/${workspace}/openai/proxy`
+	openai = new OpenAI({
+		baseURL,
+		apiKey: 'fakekey',
+		defaultHeaders: {
+			Authorization: ''
+		},
+		dangerouslyAllowBrowser: true
+	})
+	if (value) {
 		try {
-			existsOpenaiResourcePath.set(await WorkspaceService.existsOpenaiResourcePath({ workspace }))
+			existsOpenaiResourcePath.set(
+				await WorkspaceService.existsOpenaiResourcePath({ workspace: value })
+			)
 		} catch (err) {
 			existsOpenaiResourcePath.set(false)
 			console.error('Could not get if OpenAI resource exists')
@@ -57,7 +72,11 @@ interface FixScriptOpions extends BaseOptions {

 type CopilotOptions = ScriptGenerationOptions | EditScriptOptions | FixScriptOpions

-async function addResourceTypes(scriptOptions: CopilotOptions, workspace: string, prompt: string) {
+export async function addResourceTypes(scriptOptions: CopilotOptions, prompt: string) {
+	if (!workspace) {
+		throw new Error('Workspace not initialized')
+	}
+
 	if (['deno', 'bun', 'nativets'].includes(scriptOptions.language)) {
 		const resourceTypes = await ResourceService.listResourceType({ workspace })
 		const resourceTypesText = formatResourceTypes(resourceTypes, 'typescript')
@@ -116,7 +135,7 @@ function addDBSChema(scriptOptions: CopilotOptions, prompt: string) {
 	return prompt
 }

-async function getPrompts(scriptOptions: CopilotOptions, workspace: string) {
+async function getPrompts(scriptOptions: CopilotOptions) {
 	const promptsConfig = PROMPTS_CONFIGS[scriptOptions.type]
 	let prompt = promptsConfig.prompts[scriptOptions.language].prompt
 	if (scriptOptions.type !== 'fix') {
@@ -134,7 +153,7 @@ async function getPrompts(scriptOptions: CopilotOptions, workspace: string) {
 		prompt = prompt.replace('{error}', scriptOptions.error)
 	}

-	prompt = await addResourceTypes(scriptOptions, workspace, prompt)
+	prompt = await addResourceTypes(scriptOptions, prompt)

 	prompt = addDBSChema(scriptOptions, prompt)

@@ -165,55 +184,79 @@ const PROMPTS_CONFIGS = {
 	gen: GEN_CONFIG
 }

+export async function getNonStreamingCompletion(
+	messages: CreateChatCompletionRequestMessage[],
+	abortController: AbortController
+) {
+	if (!openai) {
+		throw new Error('OpenAI not initialized')
+	}
+
+	const completion = await openai.chat.completions.create(
+		{
+			...openaiConfig,
+			messages,
+			stream: false
+		},
+		{
+			signal: abortController.signal
+		}
+	)
+
+	return completion.choices[0]?.message.content || ''
+}
+
+export async function getCompletion(
+	messages: CreateChatCompletionRequestMessage[],
+	abortController: AbortController
+) {
+	if (!openai) {
+		throw new Error('OpenAI not initialized')
+	}
+
+	const completion = await openai.chat.completions.create(
+		{
+			...openaiConfig,
+			messages
+		},
+		{
+			signal: abortController.signal
+		}
+	)
+
+	return completion
+}
+
 export async function copilot(
 	scriptOptions: CopilotOptions,
 	generatedCode: Writable<string>,
 	abortController: AbortController,
 	generatedExplanation?: Writable<string>
 ) {
-	if (!workspace) {
-		throw new Error('No workspace selected')
-	}
-
-	const baseURL = `${location.origin}${OpenAPI.BASE}/w/${workspace}/openai/proxy`
-	const openai = new OpenAI({
-		baseURL,
-		apiKey: 'fakekey',
-		defaultHeaders: {
-			Authorization: ''
-		},
-		dangerouslyAllowBrowser: true
-	})
-
-	const { prompt, systemPrompt } = await getPrompts(scriptOptions, workspace)
+	const { prompt, systemPrompt } = await getPrompts(scriptOptions)

 	const { samplePrompt, sampleAnswer } = getSampleInteraction(scriptOptions)

-	const completion = await openai.chat.completions.create(
-		{
-			...openaiConfig,
-			messages: [
-				{
-					role: 'system',
-					content: systemPrompt
-				},
-				{
-					role: 'user',
-					content: samplePrompt
-				},
-				{
-					role: 'assistant',
-					content: sampleAnswer
-				},
-				{
-					role: 'user',
-					content: prompt
-				}
-			]
-		},
-		{
-			signal: abortController.signal
-		}
+	const completion = await getCompletion(
+		[
+			{
+				role: 'system',
+				content: systemPrompt
+			},
+			{
+				role: 'user',
+				content: samplePrompt
+			},
+			{
+				role: 'assistant',
+				content: sampleAnswer
+			},
+			{
+				role: 'user',
+				content: prompt
+			}
+		],
+		abortController
 	)

 	let response = ''
@@ -274,4 +317,55 @@ export async function copilot(
 	if (code.length === 0) {
 		throw new Error('No code block found')
 	}
+
+	return code
+}
+
+function getStringEndDelta(prev: string, now: string) {
+	return now.slice(prev.length)
+}
+
+export async function deltaCodeCompletion(
+	messages: CreateChatCompletionRequestMessage[],
+	generatedCodeDelta: Writable<string>,
+	abortController: AbortController
+) {
+	const completion = await getCompletion(messages, abortController)
+
+	let response = ''
+	let code = ''
+	let delta = ''
+	for await (const part of completion) {
+		response += part.choices[0]?.delta?.content || ''
+		let match = response.match(/```[a-zA-Z]+\n([\s\S]*?)\n```/)
+
+		if (match) {
+			// if we have a full code block
+			delta = getStringEndDelta(code, match[1])
+			code = match[1]
+			generatedCodeDelta.set(delta)
+
+			break
+		}
+
+		// partial code block, keep going
+		match = response.match(/```[a-zA-Z]+\n([\s\S]*)/)
+
+		if (!match) {
+			continue
+		}
+
+		if (!match[1].endsWith('`')) {
+			// skip udpating if possible that part of three ticks (end of code block)s
+			delta = getStringEndDelta(code, match[1])
+			generatedCodeDelta.set(delta)
+			code = match[1]
+		}
+	}
+
+	if (code.length === 0) {
+		throw new Error('No code block found')
+	}
+
+	return code
 }
--- a/frontend/src/lib/components/copilot/prompts/edit.yaml
+++ b/frontend/src/lib/components/copilot/prompts/edit.yaml
--- a/frontend/src/lib/components/copilot/prompts/editPrompt.ts
+++ b/frontend/src/lib/components/copilot/prompts/editPrompt.ts
--- a/frontend/src/lib/components/copilot/prompts/fix.yaml
+++ b/frontend/src/lib/components/copilot/prompts/fix.yaml
--- a/frontend/src/lib/components/copilot/prompts/fixPrompt.ts
+++ b/frontend/src/lib/components/copilot/prompts/fixPrompt.ts
--- a/frontend/src/lib/components/copilot/prompts/gen.yaml
+++ b/frontend/src/lib/components/copilot/prompts/gen.yaml
--- a/frontend/src/lib/components/copilot/prompts/genPrompt.ts
+++ b/frontend/src/lib/components/copilot/prompts/genPrompt.ts
--- a/frontend/src/lib/components/copilot/prompts/index.ts
+++ b/frontend/src/lib/components/copilot/prompts/index.ts
@@ -1,7 +1,3 @@
-// import editObj from './edit.yaml'
-// import fixObj from './fix.yaml'
-// import genObj from './gen.yaml'
-
 import { EDIT_PROMPT } from './editPrompt'
 import { FIX_PROMPT } from './fixPrompt'
 import { GEN_PROMPT } from './genPrompt'
--- a/frontend/src/lib/components/copilot/utils.ts
+++ b/frontend/src/lib/components/copilot/utils.ts
--- a/frontend/src/lib/components/flows/FlowEditor.svelte
+++ b/frontend/src/lib/components/flows/FlowEditor.svelte
@@ -6,15 +6,25 @@
 	import { Skeleton } from '../common'
 	import { getContext } from 'svelte'
 	import type { FlowEditorContext } from './types'
+	import type { FlowCopilotContext } from '../copilot/flow'
+	import { classNames } from '$lib/utils'

 	export let loading: boolean

 	const { flowStore } = getContext<FlowEditorContext>('FlowEditorContext')

 	let size = 40
+
+	const { currentStepStore: copilotCurrentStepStore } =
+		getContext<FlowCopilotContext>('FlowCopilotContext')
 </script>

-<div class="h-full overflow-hidden border-t">
+<div
+	class={classNames(
+		'h-full overflow-hidden transition-colors duration-[400ms] ease-linear border-t',
+		$copilotCurrentStepStore !== undefined ? 'border-gray-500/75' : ''
+	)}
+>
 	<Splitpanes>
 		<Pane {size} minSize={15} class="h-full relative z-0">
 			<div class="grow overflow-hidden bg-gray h-full bg-surface-secondary relative">
--- a/frontend/src/lib/components/flows/content/FlowModuleComponent.svelte
+++ b/frontend/src/lib/components/flows/content/FlowModuleComponent.svelte
@@ -38,6 +38,7 @@
 	import ToggleButtonGroup from '$lib/components/common/toggleButton-v2/ToggleButtonGroup.svelte'
 	import ToggleButton from '$lib/components/common/toggleButton-v2/ToggleButton.svelte'
 	import s3Scripts from './s3Scripts/lib'
+	import type { FlowCopilotContext } from '$lib/components/copilot/flow'

 	const { selectedId, previewArgs, flowStateStore, flowStore, saveDraft } =
 		getContext<FlowEditorContext>('FlowEditorContext')
@@ -71,6 +72,20 @@
 	let validCode = true
 	let width = 1200

+	const { modulesStore: copilotModulesStore } =
+		getContext<FlowCopilotContext | undefined>('FlowCopilotContext') || {}
+
+	function setCopilotModuleEditor() {
+		copilotModulesStore?.update((modules) => {
+			const module = modules.find((m) => m.id === flowModule.id)
+			if (module) {
+				module.editor = editor
+			}
+			return modules
+		})
+	}
+	$: editor !== undefined && setCopilotModuleEditor()
+
 	$: stepPropPicker = failureModule
 		? {
 				pickableProperties: {
@@ -195,6 +210,10 @@
 						iconOnly={width < 850}
 						kind={scriptKind}
 						template={scriptTemplate}
+						args={Object.entries(flowModule.value.input_transforms).reduce((acc, [key, obj]) => {
+							acc[key] = obj.type === 'static' ? obj.value : undefined
+							return acc
+						}, {})}
 					/>
 				</div>
 			{/if}
@@ -239,6 +258,13 @@
 										saveDraft()
 									}}
 									fixedOverflowWidgets={true}
+									args={Object.entries(flowModule.value.input_transforms).reduce(
+										(acc, [key, obj]) => {
+											acc[key] = obj.type === 'static' ? obj.value : undefined
+											return acc
+										},
+										{}
+									)}
 								/>
 								<DiffEditor
 									bind:this={diffEditor}
--- a/frontend/src/lib/components/flows/map/FlowErrorHandlerItem.svelte
+++ b/frontend/src/lib/components/flows/map/FlowErrorHandlerItem.svelte
@@ -7,6 +7,8 @@
 	import type { FlowModuleState } from '../flowState'
 	import Toggle from '$lib/components/Toggle.svelte'
 	import { NEVER_TESTED_THIS_FAR } from '../models'
+	import type { FlowCopilotContext } from '$lib/components/copilot/flow'
+	import { fade } from 'svelte/transition'

 	const { selectedId, flowStateStore, flowStore } =
 		getContext<FlowEditorContext>('FlowEditorContext')
@@ -32,11 +34,15 @@
 			$flowStore = $flowStore
 		}
 	}
+
+	const { currentStepStore: copilotCurrentStepStore } =
+		getContext<FlowCopilotContext | undefined>('FlowCopilotContext') || {}
 </script>

 <!-- svelte-ignore a11y-click-events-have-key-events -->
 <div
 	on:click={() => {
+		if ($copilotCurrentStepStore !== undefined) return
 		if ($flowStore?.value?.failure_module) {
 			$selectedId = 'failure'
 		} else {
@@ -44,11 +50,15 @@
 		}
 	}}
 	class={classNames(
-		'border mx-auto rounded-sm px-2 py-1 bg-surface text-sm cursor-pointer flex justify-between items-center flex-row overflow-x-hidden ',
+		$copilotCurrentStepStore !== undefined ? 'border-gray-500/75' : 'cursor-pointer',
+		'border transition-colors duration-[400ms] ease-linear mx-auto rounded-sm px-2 py-1 bg-surface text-sm flex justify-between items-center flex-row overflow-x-hidden relative',
 		$selectedId?.includes('failure') ? 'outline outline-offset-1 outline-2 outline-slate-900' : ''
 	)}
 	style="min-width: 275px"
 >
+	{#if $copilotCurrentStepStore !== undefined}
+		<div transition:fade class="absolute inset-0 bg-gray-500 bg-opacity-75 z-[900]" />
+	{/if}
 	<div class=" flex justify-between items-center flex-wrap">
 		<div>
 			<Icon data={faBug} class="mr-2" />
--- a/frontend/src/lib/components/flows/map/FlowModuleSchemaItem.svelte
+++ b/frontend/src/lib/components/flows/map/FlowModuleSchemaItem.svelte
@@ -1,5 +1,6 @@
 <script lang="ts">
 	import Badge from '$lib/components/common/badge/Badge.svelte'
+	import type { FlowCopilotContext } from '$lib/components/copilot/flow'
 	import Popover from '$lib/components/Popover.svelte'
 	import { classNames } from '$lib/utils'
 	import {
@@ -13,7 +14,7 @@
 		Voicemail,
 		X
 	} from 'lucide-svelte'
-	import { createEventDispatcher } from 'svelte'
+	import { createEventDispatcher, getContext } from 'svelte'
 	import { fade } from 'svelte/transition'

 	export let selected: boolean = false
@@ -32,6 +33,9 @@
 	export let concurrency: boolean = false

 	const dispatch = createEventDispatcher()
+
+	const { currentStepStore: copilotCurrentStepStore } =
+		getContext<FlowCopilotContext | undefined>('FlowCopilotContext') || {}
 </script>

 <!-- svelte-ignore a11y-click-events-have-key-events -->
@@ -39,7 +43,8 @@
 	class={classNames(
 		'w-full module flex rounded-sm cursor-pointer',
 		selected ? 'outline outline-offset-1 outline-2  outline-gray-600' : '',
-		'flex relative'
+		'flex relative',
+		$copilotCurrentStepStore === id ? 'z-[901]' : ''
 	)}
 	style="width: 275px; height: 34px; background-color: {bgColor};"
 	on:click
--- a/frontend/src/lib/components/flows/map/FlowModuleSchemaMap.svelte
+++ b/frontend/src/lib/components/flows/map/FlowModuleSchemaMap.svelte
@@ -20,6 +20,8 @@
 	import ConfirmationModal from '$lib/components/common/confirmationModal/ConfirmationModal.svelte'
 	import Portal from 'svelte-portal'
 	import { getDependentComponents } from '../flowExplorer'
+	import type { FlowCopilotContext } from '$lib/components/copilot/flow'
+	import { fade } from 'svelte/transition'

 	export let modules: FlowModule[] | undefined
 	export let sidebarSize: number | undefined = undefined
@@ -129,6 +131,9 @@

 	let deleteCallback: (() => void) | undefined = undefined
 	let dependents: Record<string, string[]> = {}
+
+	const { currentStepStore: copilotCurrentStepStore } =
+		getContext<FlowCopilotContext | undefined>('FlowCopilotContext') || {}
 </script>

 <Portal>
@@ -163,8 +168,13 @@
 </Portal>
 <div class="flex flex-col h-full relative -pt-1">
 	<div
-		class="z-10 sticky inline-flex flex-col gap-2 top-0 bg-surface-secondary flex-initial p-2 items-center border-b"
+		class={`z-10 sticky inline-flex flex-col gap-2 top-0 bg-surface-secondary flex-initial p-2 items-center transition-colors duration-[400ms] ease-linear border-b ${
+			$copilotCurrentStepStore !== undefined ? 'border-gray-500/75' : ''
+		}`}
 	>
+		{#if $copilotCurrentStepStore !== undefined}
+			<div transition:fade class="absolute inset-0 bg-gray-500 bg-opacity-75 z-[900] !m-0" />
+		{/if}
 		<FlowSettingsItem />
 		<FlowConstantsItem />
 	</div>
--- a/frontend/src/lib/components/flows/map/VirtualItem.svelte
+++ b/frontend/src/lib/components/flows/map/VirtualItem.svelte
@@ -2,11 +2,14 @@
 	import { Badge } from '$lib/components/common'
 	import type { FlowModule } from '$lib/gen'
 	import { classNames } from '$lib/utils'
-	import { faBolt } from '@fortawesome/free-solid-svg-icons'
+	import { faBolt, faMagicWandSparkles } from '@fortawesome/free-solid-svg-icons'
 	import { ClipboardCopy, X } from 'lucide-svelte'
-	import { createEventDispatcher } from 'svelte'
+	import { createEventDispatcher, getContext } from 'svelte'
 	import { Icon } from 'svelte-awesome'
 	import InsertModuleButton from './InsertModuleButton.svelte'
+	import type { FlowCopilotContext } from '$lib/components/copilot/flow'
+	import { existsOpenaiResourcePath } from '$lib/stores'
+	import Popup from '$lib/components/common/popup/Popup.svelte'

 	export let label: string
 	export let modules: FlowModule[] | undefined
@@ -31,6 +34,9 @@
 		deleteBranch: { module: FlowModule; index: number }
 	}>()
 	let openMenu = false
+
+	const { drawerStore: copilotDrawerStore, currentStepStore: copilotCurrentStepStore } =
+		getContext<FlowCopilotContext | undefined>('FlowCopilotContext') || {}
 </script>

 {#if insertable && deleteBranch}
@@ -52,7 +58,8 @@
 	class={classNames(
 		'w-full flex relative overflow-hidden rounded-sm',
 		selectable ? 'cursor-pointer' : '',
-		selected ? 'outline outline-offset-1 outline-2  outline-gray-600' : ''
+		selected ? 'outline outline-offset-1 outline-2  outline-gray-600' : '',
+		label === 'Input' && $copilotCurrentStepStore === 'Input' ? 'z-[901]' : ''
 	)}
 	style="min-width: 275px; max-height: 80px; background-color: {bgColor};"
 	on:click={() => {
@@ -140,4 +147,35 @@
 			<Icon data={faBolt} scale={0.8} />
 		</button>
 	</div>
+
+	<div class="w-7 absolute top-12 left-[80%] -translate-x-1/2">
+		<Popup let:close>
+			<svelte:fragment slot="button">
+				<button
+					title="AI Flow Builder"
+					on:click={$existsOpenaiResourcePath
+						? (ev) => {
+								ev.preventDefault()
+								ev.stopPropagation()
+								$copilotDrawerStore?.openDrawer()
+						  }
+						: undefined}
+					type="button"
+					class="text-primary bg-surface border mx-0.5 focus:outline-none hover:bg-surface-hover focus:ring-4 focus:ring-gray-200 font-medium rounded-full text-sm w-6 h-6 flex items-center justify-center"
+				>
+					<Icon data={faMagicWandSparkles} scale={0.8} />
+				</button>
+			</svelte:fragment>
+			<div class="block text-primary">
+				<p class="text-sm"
+					>Enable Windmill AI in the <a
+						href="/workspace_settings?tab=openai"
+						on:click={() => {
+							close(null)
+						}}>workspace settings.</a
+					></p
+				>
+			</div>
+		</Popup>
+	</div>
 {/if}
--- a/frontend/src/lib/components/graph/FlowGraph.svelte
+++ b/frontend/src/lib/components/graph/FlowGraph.svelte
@@ -766,7 +766,11 @@

 <DarkModeObserver on:change={onThemeChange} />

-<div bind:clientWidth={width} class={fullSize ? '' : 'w-full h-full overflow-hidden relative'}>
+<div
+	bind:clientWidth={width}
+	class={fullSize ? '' : 'w-full h-full overflow-hidden relative'}
+	id="flow-graph"
+>
 	{#if !error}
 		{#if width && height}
 			{#key renderCount}
--- a/frontend/src/lib/components/graph/svelvet/container/views/GraphView.svelte
+++ b/frontend/src/lib/components/graph/svelvet/container/views/GraphView.svelte
@@ -1,7 +1,7 @@
 <script lang="ts">
 	import { pointer, select, selectAll } from 'd3-selection'
 	import { zoom, zoomIdentity, zoomTransform } from 'd3-zoom'
-	import { createEventDispatcher, onMount } from 'svelte'
+	import { createEventDispatcher, getContext, onMount } from 'svelte'
 	import SimpleBezierEdge from '../../edges/views/Edges/SimpleBezierEdge.svelte'
 	import SmoothStepEdge from '../../edges/views/Edges/SmoothStepEdge.svelte'
 	import StepEdge from '../../edges/views/Edges/StepEdge.svelte'
@@ -13,6 +13,8 @@

 	import { Expand, Minus, Plus } from 'lucide-svelte'
 	import Toggle from '$lib/components/Toggle.svelte'
+	import type { FlowCopilotContext } from '$lib/components/copilot/flow'
+	import { fade } from 'svelte/transition'

 	//these are typscripted as any, however they have been transformed inside of store.ts
 	export let canvasId: string
@@ -124,6 +126,16 @@
 			)
 			.style('transform-origin', '0 0')
 	}
+
+	const { currentStepStore: copilotCurrentStepStore } =
+		getContext<FlowCopilotContext | undefined>('FlowCopilotContext') || {}
+
+	$: {
+		if ($copilotCurrentStepStore !== undefined) {
+			// reset zoom
+			d3.select(`#zoomable-${canvasId}`).call(d3Zoom.transform, d3.zoomIdentity)
+		}
+	}
 </script>

 <div class="zoomable" id={`zoomable-${canvasId}`}>
@@ -132,6 +144,9 @@
 		<div class={`Nodes Nodes-${canvasId}`} on:contextmenu|preventDefault>
 			<!-- This container is transformed by d3zoom -->
 			<div class={`Node Node-${canvasId}`}>
+				{#if $copilotCurrentStepStore !== undefined}
+					<div transition:fade class="fixed inset-0 bg-gray-500 bg-opacity-75 z-[900] !m-0" />
+				{/if}
 				{#each nodes as node}
 					{#if node.data.html}
 						<Node {node} {canvasId}>{@html node.data.html}</Node>
@@ -203,6 +218,7 @@
 {#if showDataflowToggle}
 	<div id="dataflow_toggle">
 		<Toggle
+			disabled={$copilotCurrentStepStore !== undefined}
 			textClass="!text-secondary"
 			size="xs"
 			bind:checked={dataflow}
@@ -214,10 +230,24 @@
 {/if}

 <div id="buttons">
-	<button title="Zoom In" id={`zoom_in_${canvasId}`}>
+	<button
+		title="Zoom In"
+		id={`zoom_in_${canvasId}`}
+		class={'relative overflow-hidden ' +
+			($copilotCurrentStepStore !== undefined
+				? '!bg-gray-400/20 text-black/20 border-gray-400/20'
+				: '')}
+	>
 		<Plus size="14" class="flex justify-start m-1" />
 	</button>
-	<button title="Zoom Out" id={`zoom_out_${canvasId}`}>
+	<button
+		title="Zoom Out"
+		id={`zoom_out_${canvasId}`}
+		class={'relative overflow-hidden ' +
+			($copilotCurrentStepStore !== undefined
+				? '!bg-gray-400/20 text-black/20 border-gray-400/20'
+				: '')}
+	>
 		<Minus size="14" class="flex justify-start m-1" />
 	</button>

--- a/frontend/src/lib/components/runs/QueuePopover.svelte
+++ b/frontend/src/lib/components/runs/QueuePopover.svelte
@@ -0,0 +1,42 @@
+<script lang="ts">
+	import { JobService, QueuedJob } from '$lib/gen'
+	import { workspaceStore } from '$lib/stores'
+	import { ExternalLink } from 'lucide-svelte'
+	import Skeleton from '../common/skeleton/Skeleton.svelte'
+	import { displayDate } from '$lib/utils'
+
+	let jobs: QueuedJob[] | undefined = undefined
+
+	getQueuedJobs()
+	async function getQueuedJobs() {
+		jobs = await JobService.listQueue({ workspace: $workspaceStore ?? '' })
+	}
+</script>
+
+{#if jobs == undefined}
+	<Skeleton
+		layout={[
+			[2, 2],
+			[2, 2]
+		]}
+	/>
+{:else}
+	<div class="flex flex-col gap-2 text-sm">
+		{#if jobs.length > 100}
+			<div class="text-secondary text-xs">Only showing the first 100 jobs</div>
+		{/if}
+
+		{#each jobs.slice(0, 100) as job}
+			<div class="flex">
+				<a
+					target="_blank"
+					href={`/run/${job.id}?workspace=${$workspaceStore}`}
+					class="flex flex-row gap-2 items-center font-mono mr-8"
+					>{job.id} <ExternalLink size={10} />
+				</a>
+				<div class="w-32">{displayDate(job.created_at)}</div>
+				<div class="text-2xs text-tertiary">tag: {job.tag}</div>
+			</div>
+		{/each}
+	</div>
+{/if}
--- a/frontend/src/lib/components/runs/RunRow.svelte
+++ b/frontend/src/lib/components/runs/RunRow.svelte
@@ -115,7 +115,7 @@
 		</div>
 	</div>

-	<div class="w-4/12 flex justify-star">
+	<div class="w-4/12 flex justify-start flex-col">
 		<div class="flex flex-row text-sm">
 			{#if job === undefined}
 				No job found
@@ -167,19 +167,23 @@

 		{#if job && job.parent_job}
 			{#if job.is_flow_step}
-				<Icon class="text-secondary" data={faBarsStaggered} scale={SMALL_ICON_SCALE} />
-				<span class="mx-1">
-					Step of flow <a href={`/run/${job.parent_job}?workspace=${job.workspace_id}`}
-						>{truncateRev(job.parent_job, 6)}
-					</a>
-				</span>
+				<div class="flex flex-row gap-1 items-center">
+					<Icon class="text-secondary" data={faBarsStaggered} scale={SMALL_ICON_SCALE} />
+					<span class="mx-1 text-xs">
+						Step of flow <a href={`/run/${job.parent_job}?workspace=${job.workspace_id}`}>
+							{truncateRev(job.parent_job, 6)}
+						</a>
+					</span>
+				</div>
 			{:else}
-				<Icon class="text-secondary" data={faRobot} scale={SMALL_ICON_SCALE} />
-				<span class="mx-1">
-					Parent <a href={`/run/${job.parent_job}?workspace=${job.workspace_id}`}
-						>{job.parent_job}</a
-					>
-				</span>
+				<div class="flex flex-row gap-1 items-center">
+					<Icon class="text-secondary" data={faRobot} scale={SMALL_ICON_SCALE} />
+					<span class="mx-1 text-xs">
+						Parent <a href={`/run/${job.parent_job}?workspace=${job.workspace_id}`}>
+							{job.parent_job}
+						</a>
+					</span>
+				</div>
 			{/if}
 		{/if}
 	</div>
--- a/frontend/src/lib/components/runs/RunsQueue.svelte
+++ b/frontend/src/lib/components/runs/RunsQueue.svelte
@@ -0,0 +1,22 @@
+<script lang="ts">
+	import type { Tweened } from 'svelte/motion'
+	import QueuePopover from './QueuePopover.svelte'
+	import Popup from '../common/popup/Popup.svelte'
+
+	export let queue_count: Tweened<number> | undefined = undefined
+</script>
+
+<div class="flex gap-1 relative max-w-36 min-w-[50px] items-baseline">
+	<div class="text-xs absolute -top-4 truncate">Jobs waiting for a worker</div>
+	<div class="mt-1">{queue_count ? ($queue_count ?? 0).toFixed(0) : '...'}</div>
+	<div class="truncate text-2xs text-blue-400">
+		{#if queue_count && ($queue_count ?? 0) > 0}
+			<Popup>
+				<svelte:fragment slot="button">
+					<span class="text-2xs truncate">jobs</span>
+				</svelte:fragment>
+				<QueuePopover />
+			</Popup>
+		{/if}
+	</div>
+</div>
--- a/frontend/src/lib/components/runs/RunsTable.svelte
+++ b/frontend/src/lib/components/runs/RunsTable.svelte
@@ -3,12 +3,11 @@
 	import RunRow from './RunRow.svelte'
 	import VirtualList from 'svelte-tiny-virtual-list'
 	import { onMount } from 'svelte'
-	import InfiniteLoading from 'svelte-infinite-loading'
+	//import InfiniteLoading from 'svelte-infinite-loading'

 	export let jobs: Job[] = []
 	export let selectedId: string | undefined = undefined
-	export let nbOfJobs: number = 30
-	const loadMoreQuantity: number = 100
+	// const loadMoreQuantity: number = 100

 	function getTime(job: Job): string | undefined {
 		return job['started_at'] ?? job['scheduled_for'] ?? job['created_at']
@@ -57,7 +56,7 @@
 		return sortedLogs
 	}

-	$: groupedJobs = groupJobsByDay(jobs.slice(0, nbOfJobs))
+	$: groupedJobs = groupJobsByDay(jobs)

 	type FlatJobs =
 		| {
@@ -96,11 +95,13 @@
 			index++
 		}
 	}
+
 	let tableHeight: number = 0
 	let header: number = 0
 	let containerWidth: number = 0
-	const MAX_ITEMS = 1000
+	// const MAX_ITEMS = 1000

+	/*
 	function infiniteHandler({ detail: { loaded, error, complete } }) {
 		try {
 			nbOfJobs += loadMoreQuantity
@@ -114,6 +115,7 @@
 			error()
 		}
 	}
+	*/

 	onMount(() => {
 		tableHeight = document.querySelector('#runs-table-wrapper')!.parentElement?.clientHeight ?? 0
@@ -131,13 +133,19 @@
 		<div class="w-3/12 text-xs font-semibold">Triggered by</div>
 	</div>

-	<VirtualList width="100%" height={tableHeight - header} itemCount={flatJobs.length} itemSize={42}>
+	<VirtualList
+		width="100%"
+		height={tableHeight - header}
+		itemCount={flatJobs.length}
+		itemSize={42}
+		{stickyIndices}
+	>
 		<div slot="item" let:index let:style {style} class="w-full">
 			{@const jobOrDate = flatJobs[index]}

 			{#if jobOrDate}
 				{#if jobOrDate?.type === 'date'}
-					<div class="bg-surface-secondary/30 py-2 border-b font-semibold text-xs pl-5">
+					<div class="bg-surface-secondary py-2 border-b font-semibold text-xs pl-5">
 						{jobOrDate.date}
 					</div>
 				{:else}
@@ -157,7 +165,7 @@
 				{JSON.stringify(jobOrDate)}
 			{/if}
 		</div>
-		<div slot="footer">
+		<!-- <div slot="footer">
 			<InfiniteLoading on:infinite={infiniteHandler}>
 				<div slot="noMore">
 					<div class="text-center text-xs text-secondary p-2">
@@ -165,10 +173,10 @@
 					</div>
 				</div>
 			</InfiniteLoading>
-		</div>
+		</div> -->
 	</VirtualList>
 </div>
-{#if jobs.length == 0}
+{#if jobs?.length == 0}
 	<tr>
 		<td colspan="4" class="text-center py-8">
 			<div class="text-xs text-secondary"> No jobs found for the selected filters. </div>
--- a/frontend/src/lib/components/scriptEditor/LogPanel.svelte
+++ b/frontend/src/lib/components/scriptEditor/LogPanel.svelte
@@ -11,8 +11,6 @@
 	import DisplayResult from '../DisplayResult.svelte'
 	import TableCustom from '../TableCustom.svelte'
 	import Drawer from '../common/drawer/Drawer.svelte'
-	import { Highlight } from 'svelte-highlight'
-	import { json } from 'svelte-highlight/languages'
 	import DrawerContent from '../common/drawer/DrawerContent.svelte'
 	import HighlightCode from '../HighlightCode.svelte'
 	import LogViewer from '../LogViewer.svelte'
@@ -21,7 +19,7 @@
 	import { Loader2 } from 'lucide-svelte'
 	import type Editor from '../Editor.svelte'
 	import type DiffEditor from '../DiffEditor.svelte'
-	import ScriptFix from '../codeGen/ScriptFix.svelte'
+	import ScriptFix from '../copilot/ScriptFix.svelte'

 	export let lang: Preview.language | undefined
 	export let previewIsLoading = false
@@ -29,11 +27,12 @@
 	export let pastPreviews: CompletedJob[] = []
 	export let editor: Editor | undefined = undefined
 	export let diffEditor: DiffEditor | undefined = undefined
+	export let args: Record<string, any> | undefined = undefined

 	type DrawerContent = {
 		mode: 'json' | Preview.language | 'plain'
 		title: string
-		content: string
+		content: any
 	}

 	let selectedTab = 'logs'
@@ -56,12 +55,18 @@

 <Drawer bind:open={drawerOpen} size="800px">
 	<DrawerContent title={drawerContent?.title} on:close={() => closeDrawer()}>
-		{#if drawerContent?.mode === 'json'}
-			<Highlight language={json} code={JSON.stringify(drawerContent.content, null, 4)} />
+		{#if drawerContent?.content == undefined}
+			<div class="p-2"> <Loader2 class="animate-spin" /> </div>
+		{:else if drawerContent?.mode === 'json'}
+			<DisplayResult
+				workspaceId={previewJob?.workspace_id}
+				jobId={previewJob?.id}
+				result={drawerContent.content}
+			/>
 		{:else if drawerContent?.mode === 'plain'}
 			<pre
-				class="overflow-x-auto break-words relative h-full m-2 text-xs bg-surface shadow-inner p-2">
-				{drawerContent?.content}
+				class="overflow-x-auto break-words relative h-full m-2 text-xs bg-surface shadow-inner p-2"
+				>{drawerContent?.content}
 			</pre>
 		{:else if drawerContent?.mode === 'deno' || drawerContent?.mode === 'python3' || drawerContent?.mode === 'go' || drawerContent?.mode === 'bash' || drawerContent?.mode === 'nativets'}
 			<HighlightCode language={drawerContent?.mode} code={drawerContent?.content} />
@@ -99,12 +104,13 @@
 									result={previewJob.result}
 								>
 									<svelte:fragment slot="copilot-fix">
-										{#if lang && editor && diffEditor && previewJob?.result?.error}
+										{#if lang && editor && diffEditor && args && previewJob?.result?.error}
 											<ScriptFix
 												error={JSON.stringify(previewJob.result.error)}
 												{lang}
 												{editor}
 												{diffEditor}
+												{args}
 											/>
 										{/if}
 									</svelte:fragment>
@@ -134,10 +140,12 @@
 					<th class="text-xs">Logs</th>
 				</tr>
 				<tbody slot="body">
-					{#each pastPreviews as { id, created_at, success, result }}
+					{#each pastPreviews as { id, created_at, success }}
 						<tr class="">
 							<td class="text-xs">
-								<a class="pr-3" href="/run/{id}?workspace={$workspaceStore}" target="_blank">{id.substring(30)}</a>
+								<a class="pr-3" href="/run/{id}?workspace={$workspaceStore}" target="_blank"
+									>{id.substring(30)}</a
+								>
 							</td>
 							<td class="text-xs">{displayDate(created_at)}</td>
 							<td class="text-xs">
@@ -149,20 +157,26 @@
 							</td>
 							<td class="text-xs">
 								<a
-									href="#result"
+									href=""
 									class="text-xs"
-									on:click={() => {
-										openDrawer({ mode: 'json', content: result, title: 'Result' })
+									on:click|preventDefault={() => {
+										openDrawer({ mode: 'json', content: undefined, title: 'Result' })
+										JobService.getCompletedJobResult({
+											workspace: $workspaceStore ?? 'NO_W',
+											id
+										}).then((res) => {
+											drawerContent && (drawerContent.content = res)
+										})
 									}}
 								>
-									{JSON.stringify(result).substring(0, 30)}...
+									See Result
 								</a>
 							</td>
 							<td class="text-xs">
 								<a
-									href="#code"
+									href=""
 									class="text-xs"
-									on:click={async () => {
+									on:click|preventDefault={async () => {
 										const code = (
 											await JobService.getCompletedJob({
 												workspace: $workspaceStore ?? 'NO_W',
@@ -182,16 +196,16 @@
 							</td>
 							<td>
 								<a
-									href="#logs"
+									href=""
 									class="text-xs"
-									on:click={async () => {
+									on:click|preventDefault={async () => {
 										const logs = (
 											await JobService.getCompletedJob({
 												workspace: $workspaceStore ?? 'NO_W',
 												id
 											})
 										).logs
-										openDrawer({ mode: 'plain', content: String(logs), title: `Code ${lang}` })
+										openDrawer({ mode: 'plain', content: String(logs), title: `Logs for ${id}` })
 									}}
 								>
 									View logs
--- a/frontend/src/lib/stores.ts
+++ b/frontend/src/lib/stores.ts
@@ -82,14 +82,9 @@ type SQLBaseSchema = {
 }

 export interface SQLSchema {
-	lang: 'mysql' | 'bigquery'
+	lang: 'mysql' | 'bigquery' | 'postgresql' | 'snowflake'
 	schema: SQLBaseSchema
-}
-
-export interface PostgresqlSchema {
-	lang: 'postgresql'
-	schema: SQLBaseSchema
-	publicOnly: boolean
+	publicOnly: boolean | undefined
 }

 export interface GraphqlSchema {
@@ -97,7 +92,7 @@ export interface GraphqlSchema {
 	schema: IntrospectionQuery
 }

-export type DBSchema = SQLSchema | PostgresqlSchema | GraphqlSchema
+export type DBSchema = SQLSchema | GraphqlSchema

 interface DBSchemas {
 	[resourcePath: string]: DBSchema
--- a/frontend/src/lib/utils.ts
+++ b/frontend/src/lib/utils.ts
@@ -170,7 +170,12 @@ export function encodeState(state: any): string {
 }

 export function decodeState(query: string): any {
-	return JSON.parse(decodeURIComponent(atob(query)))
+	try {
+		return JSON.parse(decodeURIComponent(atob(query)))
+	} catch (e) {
+		sendUserToast('Impossible to parse state', true)
+		return {}
+	}
 }

 export function itemsExists<T>(arr: T[] | undefined, item: T): boolean {
--- a/frontend/src/routes/(root)/(logged)/runs/[...path]/+page.svelte
+++ b/frontend/src/routes/(root)/(logged)/runs/[...path]/+page.svelte
@@ -30,6 +30,7 @@
 	import ConfirmationModal from '$lib/components/common/confirmationModal/ConfirmationModal.svelte'
 	import { tweened, type Tweened } from 'svelte/motion'
 	import { goto } from '$app/navigation'
+	import RunsQueue from '$lib/components/runs/RunsQueue.svelte'

 	let jobs: Job[] | undefined
 	let intervalId: NodeJS.Timer | undefined
@@ -95,7 +96,6 @@
 		goto(newUrl)
 	}

-	let nbOfJobs = 30
 	let queue_count: Tweened<number> | undefined = undefined

 	$: jobKinds = computeJobKinds(jobKindsCat)
@@ -463,10 +463,7 @@
 	</div>
 	<div class="flex flex-col gap-1 md:flex-row w-full p-4">
 		<div class="flex gap-2 grow mb-2">
-			<div class="flex gap-1 relative max-w-36 min-w-[50px]">
-				<div class="text-xs absolute -top-4 truncate">Jobs waiting for a worker</div>
-				<div class="mt-1">{queue_count ? ($queue_count ?? 0).toFixed(0) : '...'}</div>
-			</div>
+			<RunsQueue {queue_count} />
 			<div class="flex"
 				><Button
 					size="xs"
@@ -568,7 +565,6 @@
 					<RunsTable
 						{jobs}
 						bind:selectedId
-						bind:nbOfJobs
 						on:filterByPath={(e) => {
 							user = null
 							folder = null
@@ -680,10 +676,9 @@
 				</div>
 				<div class="flex flex-col gap-1 md:flex-row w-full p-4">
 					<div class="flex gap-2 grow mb-2">
-						<div class="flex gap-1 relative max-w-36 min-w-[50px]">
-							<div class="text-xs absolute -top-4 truncate">Jobs waiting for a worker</div>
-							<div class="mt-1">{queue_count ? ($queue_count ?? 0).toFixed(0) : '...'}</div>
-						</div>
+						{#if queue_count}
+							<RunsQueue {queue_count} />
+						{/if}
 						<div class="flex"
 							><Button
 								size="xs"
@@ -787,7 +782,6 @@
 					<RunsTable
 						{jobs}
 						bind:selectedId
-						bind:nbOfJobs
 						on:select={() => {
 							runDrawer.openDrawer()
 						}}
--- a/frontend/vite.config.js
+++ b/frontend/vite.config.js
@@ -1,7 +1,6 @@
 import { sveltekit } from '@sveltejs/kit/vite'
 import { readFileSync } from 'fs'
 import { fileURLToPath } from 'url'
-import ViteYaml from '@modyfi/vite-plugin-yaml'
 import monacoEditorPlugin from 'vite-plugin-monaco-editor'

 const file = fileURLToPath(new URL('package.json', import.meta.url))
@@ -35,7 +34,6 @@ const config = {
 	},
 	plugins: [
 		sveltekit(),
-		ViteYaml(),
 		monacoEditorPlugin.default({
 			publicPath: 'workers',
 			languageWorkers: [],
--- a/llm/embeddings_benchmark.ts
+++ b/llm/embeddings_benchmark.ts
@@ -0,0 +1,81 @@
+const getWords = async () => {
+  const response = await fetch("http://localhost:3001/searchData");
+  const data: {
+    asks: {
+      summary: string;
+      app: string;
+    }[];
+  } = await response.json();
+  const words = data.asks
+    .map((a) => [
+      ...a.summary.split(" ").map((w: string) => w.toLowerCase()),
+      ...a.app.split(" ").map((w: string) => w.toLowerCase()),
+    ])
+    .flat();
+  return words;
+};
+
+const generateQuery = async (words: string[]) => {
+  const word1 = words[Math.floor(Math.random() * words.length)];
+  const word2 = words[Math.floor(Math.random() * words.length)];
+  const word3 = words[Math.floor(Math.random() * words.length)];
+  const query = `${word1} ${word2} ${word3}`;
+  return query.substring(0, 3 + Math.random() * (query.length - 3));
+};
+
+async function sendRequest(q: string) {
+  const time = Date.now();
+
+  try {
+    await fetch(
+      "http://localhost:3001/scripts/query?" +
+        new URLSearchParams({
+          text: q,
+        })
+    );
+    return {
+      time: Date.now() - time,
+      error: false,
+    };
+  } catch (err) {
+    return {
+      time: Date.now() - time,
+      error: true,
+    };
+  }
+}
+
+async function benchmark() {
+  // first fetch to mitigate cold start
+  await fetch(
+    "http://localhost:3001/scripts/query?" +
+      new URLSearchParams({
+        text: "init",
+      })
+  );
+  const words = await getWords();
+  const tryouts = [1, 10, 100, 1000, 10000];
+
+  for (const tryout of tryouts) {
+    const requests: Promise<{
+      time: number;
+      error: boolean;
+    }>[] = [];
+    for (let i = 0; i < tryout; i++) {
+      const q = await generateQuery(words);
+      requests.push(sendRequest(q));
+    }
+    const startTime = Date.now();
+    const times = await Promise.all(requests);
+    const duration = Date.now() - startTime;
+    const avg = times.reduce((a, b) => a + b.time, 0) / times.length;
+    const errors = times.filter((t) => t.error).length;
+    console.log(`Average time for ${tryout} simultaneous requests: ${avg}ms`);
+    console.log(
+      `Total time for ${tryout} simultaneous requests: ${duration}ms`
+    );
+    console.log(`Number of errors: ${errors}`);
+  }
+}
+
+benchmark();
--- a/llm/src/gen_samples.py
+++ b/llm/src/gen_samples.py
@@ -178,7 +178,7 @@ if __name__ == "__main__":
    parser.add_argument(
        "--prompts_path",
        type=str,
-        default="../frontend/src/lib/components/codeGen/prompts",
+        default="../frontend/src/lib/components/copilot/prompts",
    )
    args = parser.parse_args()
    gen_samples(args.queries_path, args.answers_path, args.prompts_path)
--- a/lsp/Pipfile
+++ b/lsp/Pipfile
@@ -4,8 +4,8 @@ verify_ssl = true
 name = "pypi"

 [packages]
-wmill = ">=1.168.3"
-wmill_pg = ">=1.168.3"
+wmill = ">=1.170.0"
+wmill_pg = ">=1.170.0"
 sendgrid = "*"
 mysql-connector-python = "*"
 pymongo = "*"
--- a/openflow.openapi.yaml
+++ b/openflow.openapi.yaml
@@ -1,7 +1,7 @@
 openapi: "3.0.3"

 info:
-  version: 1.168.3
+  version: 1.170.0
  title: OpenFlow Spec
  contact:
    name: Ruben Fiszel
--- a/python-client/wmill/pyproject.toml
+++ b/python-client/wmill/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "wmill"
-version = "1.168.3"
+version = "1.170.0"
 description = "A client library for accessing Windmill server wrapping the Windmill client API"
 license = "Apache-2.0"
 homepage = "https://windmill.dev"
@@ -16,7 +16,7 @@ include = ["wmill/py.typed"]

 [tool.poetry.dependencies]
 python = "^3.7"
-windmill-api = "^1.168.3"
+windmill-api = "^1.170.0"

 [build-system]
 requires = ["poetry>=1.0.2", "poetry-dynamic-versioning"]
--- a/python-client/wmill/wmill/client.py
+++ b/python-client/wmill/wmill/client.py
@@ -191,7 +191,7 @@ def get_result(job_id: str) -> Dict[str, Any]:
    if not res.result:
        raise Exception(f"Unexpected result not found for completed job {job_id}")
    else:
-        return res.result.to_dict()  # type: ignore
+        return res.result


 def get_resource(path: str | None = None, none_if_undefined: bool = False) -> Any:
--- a/python-client/wmill_pg/pyproject.toml
+++ b/python-client/wmill_pg/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "wmill-pg"
-version = "1.168.3"
+version = "1.170.0"
 description = "An extension client for the wmill client library focused on pg"
 license = "Apache-2.0"
 homepage = "https://windmill.dev"
--- a/typescript-client/package.json
+++ b/typescript-client/package.json
@@ -1,7 +1,7 @@
 {
  "name": "windmill-client",
  "description": "Windmill SDK client for browsers and Node.js",
-  "version": "1.168.3",
+  "version": "1.170.0",
  "author": "Ruben Fiszel",
  "license": "Apache 2.0",
  "devDependencies": {
--- a/version.txt
+++ b/version.txt
@@ -1 +1 @@
-1.168.3
+1.170.0
Author	SHA1	Message	Date
Ruben Fiszel	9629534087	repro mem	2023-09-10 00:24:36 +02:00
Ruben Fiszel	39cab72951	improve history from the script editor	2023-09-09 09:44:24 +02:00
Ruben Fiszel	e7e63e111a	fix: remove result and args from list completed and list queue jobs	2023-09-09 09:31:45 +02:00
Ruben Fiszel	d7716579da	chore(main): release 1.170.0 (#2258 ) * chore(main): release 1.170.0 * Apply automatic changes --------- Co-authored-by: rubenfiszel <rubenfiszel@users.noreply.github.com>	2023-09-09 00:42:18 +02:00
Ruben Fiszel	fe41f4ff4c	fix: fix get_result for python-client	2023-09-09 00:37:15 +02:00
Ruben Fiszel	c669776e86	sqlx	2023-09-08 19:00:46 +02:00
Ruben Fiszel	3ecc969649	update to bun 1.0.0	2023-09-08 18:35:17 +02:00
Ruben Fiszel	e220dc2869	improve benchmarks	2023-09-08 17:32:26 +02:00
HugoCasa	5cca5833e9	feat: snowflake schema explorer + refactoring (#2260 )	2023-09-08 16:49:47 +02:00
Ruben Fiszel	67496abc7e	improve displaying jobs currently waiting for a worker	2023-09-08 14:35:44 +02:00
Ruben Fiszel	3c950c03de	feat: display jobs currently waiting for a worker	2023-09-08 14:24:01 +02:00
Ruben Fiszel	58026e75b3	chore(main): release 1.169.0 (#2246 ) * chore(main): release 1.169.0 * Apply automatic changes --------- Co-authored-by: rubenfiszel <rubenfiszel@users.noreply.github.com>	2023-09-08 10:53:33 +02:00
Faton Ramadani	22712632f6	feat(frontend): fix runs page when the row has a parent (#2255 )	2023-09-08 10:44:36 +02:00
Ruben Fiszel	49c5553f3b	feat: introduce container groups	2023-09-08 09:48:28 +02:00
Ruben Fiszel	973d11882e	fix flow builder input transforms	2023-09-08 00:52:05 +02:00
Ruben Fiszel	c6cc8769ae	fix flow builder input transforms	2023-09-08 00:51:09 +02:00
Ruben Fiszel	a2c0bede49	fix flow builder input transforms	2023-09-08 00:47:05 +02:00
HugoCasa	2f3138c65d	feat: flow copilot (#2219 ) * chore: refactoring * feat: hub embeddings search benchmark * feat: flow copilot v0 * feat: as trigger schedule + refactoring * feat: improvements * fix: remove unecessary filter * feat: UX improvements * fix: frontend check * feat: switch to bun + other improvements * fix: improve reactivity	2023-09-07 18:03:48 +02:00
				`@@ -1 +0,0 @@`
				`export { default as ScriptGen } from './ScriptGen.svelte'`
@@ -1 +1 @@
 .168.3
 .170.0