kalugin66/windmill
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: hub flows integration

Browse Source
This commit is contained in:
Ruben Fiszel
2022-07-25 10:03:52 +02:00
parent f402cc67c1
commit 05ba0a06d0
24 changed files with 648 additions and 362 deletions
Download Patch File Download Diff File Expand all files Collapse all files

128
backend/openapi.yaml
View File

@@ -1509,26 +1509,99 @@ paths:
content:
application/json:
schema:
type: array
items:
type: object
properties:
id:
type: number
summary:
type: string
app:
type: string
approved:
type: boolean
is_trigger:
type: boolean
required:
- id
- summary
- app
- approved
- is_trigger
type: object
properties:
asks:
type: array
items:
type: object
properties:
id:
type: number
ask_id:
type: number
summary:
type: string
app:
type: string
approved:
type: boolean
is_trigger:
type: boolean
votes:
type: number
views:
type: number
required:
- id
- ask_id
- summary
- app
- approved
- is_trigger
- views
- votes
/flows/hub/list:
get:
summary: list all available hub flows
operationId: listHubFlows
tags:
- flow
responses:
"200":
description: hub flows list
content:
application/json:
schema:
type: object
properties:
flows:
type: array
items:
type: object
properties:
id:
type: number
flow_id:
type: number
summary:
type: string
apps:
type: array
items:
type: string
approved:
type: boolean
votes:
type: number
required:
- id
- flow_id
- summary
- apps
- approved
- votes
/flows/hub/get/{id}:
get:
summary: get hub flow by id
operationId: getHubFlowById
tags:
- flow
parameters:
- $ref: "#/components/parameters/PathId"
responses:
"200":
description: flow
content:
application/json:
schema:
type: object
properties:
flow:
$ref: "#/components/schemas/OpenFlow"
/scripts/hub/get/{path}:
get:
@@ -3933,6 +4006,21 @@ components:
- archived
- extra_perms
OpenFlow:
type: object
properties:
summary:
type: string
description:
type: string
value:
$ref: "#/components/schemas/FlowValue"
schema:
type: object
required:
- summary
- value
FlowValue:
type: object
properties:

54
backend/src/flow.rs → backend/src/flows.rs
View File

@@ -7,10 +7,11 @@
use std::collections::HashMap;
use reqwest::Client;
use sql_builder::prelude::*;
use axum::{
extract::{Extension, Path, Query},
extract::{Extension, Host, Path, Query},
routing::{get, post},
Json, Router,
};
@@ -21,11 +22,11 @@ use sqlx::{FromRow, Postgres, Transaction};
use crate::{
audit::{audit_log, ActionKind},
db::{UserDB, DB},
error::{self, Error, JsonResult, Result},
error::{self, to_anyhow, Error, JsonResult, Result},
jobs::RawCode,
scripts::Schema,
users::Authed,
utils::{Pagination, StripPath},
utils::{http_get_from_hub, list_elems_from_hub, Pagination, StripPath},
};
pub fn workspaced_service() -> Router {
@@ -38,6 +39,12 @@ pub fn workspaced_service() -> Router {
.route("/exists/*path", get(exists_flow_by_path))
}
pub fn global_service() -> Router {
Router::new()
.route("/hub/list", get(list_hub_flows))
.route("/hub/get/:id", get(get_hub_flow_by_id))
}
#[derive(FromRow, Serialize)]
pub struct Flow {
pub workspace_id: String,
@@ -162,6 +169,47 @@ async fn list_flows(
Ok(Json(rows))
}
async fn list_hub_flows(
Authed {
email, username, ..
}: Authed,
Extension(http_client): Extension<Client>,
Host(host): Host,
) -> JsonResult<serde_json::Value> {
let flows = list_elems_from_hub(
http_client,
"https://hub.windmill.dev/searchFlowData?approved=true",
email,
username,
host,
)
.await?;
Ok(Json(flows))
}
pub async fn get_hub_flow_by_id(
Authed {
email, username, ..
}: Authed,
Path(id): Path<i32>,
Extension(http_client): Extension<Client>,
Host(host): Host,
) -> JsonResult<serde_json::Value> {
let value = http_get_from_hub(
http_client,
&format!("https://hub.windmill.dev/flows/{id}/json"),
email,
username,
host,
false,
)
.await?
.json()
.await
.map_err(to_anyhow)?;
Ok(Json(value))
}
async fn create_flow(
authed: Authed,
Extension(user_db): Extension<UserDB>,

18
backend/src/jobs.rs
View File

@@ -5,6 +5,7 @@
* LICENSE-AGPL for a copy of the license.
*/
use axum::extract::Host;
use chrono::Duration;
use sql_builder::prelude::*;
@@ -12,6 +13,7 @@ use sqlx::{query_scalar, Postgres, Transaction};
use std::collections::HashMap;
use tracing::instrument;
use crate::error::to_anyhow;
use crate::scripts::{get_hub_script_by_path, ScriptLang};
use crate::worker_flow::init_flow_status;
use crate::{
@@ -19,7 +21,7 @@ use crate::{
db::{UserDB, DB},
error,
error::Error,
flow::FlowValue,
flows::FlowValue,
schedule::get_schedule_opt,
scripts::ScriptHash,
users::{owner_to_token_owner, Authed},
@@ -1127,6 +1129,13 @@ pub async fn push<'c>(
groups: vec![],
},
Path(StripPath(path)),
Extension(
reqwest::ClientBuilder::new()
.user_agent("windmill/beta")
.build()
.map_err(to_anyhow)?,
),
Host(std::env::var("BASE_URL").unwrap_or_else(|_| "".to_string())),
)
.await?,
),
@@ -1337,7 +1346,12 @@ pub async fn schedule_again_if_scheduled(
let mut tx = db.begin().await?;
let schedule = get_schedule_opt(&mut tx, &w_id, &schedule_path)
.await?
.unwrap();
.ok_or_else(|| {
Error::InternalErr(format!(
"Could not find schedule {:?} for workspace {}",
schedule_path, w_id
))
})?;
if schedule.enabled && script_path.is_some() && script_path.unwrap() == schedule.script_path
{
tx = crate::schedule::push_scheduled_job(tx, schedule).await?;

5
backend/src/lib.rs
View File

@@ -25,7 +25,7 @@ mod client;
mod db;
mod email;
mod error;
mod flow;
mod flows;
mod granular_acls;
mod groups;
mod jobs;
@@ -146,7 +146,7 @@ pub async fn run_server(
.nest("/audit", audit::workspaced_service())
.nest("/acls", granular_acls::workspaced_service())
.nest("/workspaces", workspaces::workspaced_service())
.nest("/flows", flow::workspaced_service()),
.nest("/flows", flows::workspaced_service()),
)
.nest("/workspaces", workspaces::global_service())
.nest(
@@ -155,6 +155,7 @@ pub async fn run_server(
)
.nest("/workers", worker_ping::global_service())
.nest("/scripts", scripts::global_service())
.nest("/flows", flows::global_service())
.nest("/schedules", schedule::global_service())
.route_layer(from_extractor::<users::Authed>())
.route_layer(from_extractor::<users::Tokened>())

119
backend/src/scripts.rs
View File

@@ -5,6 +5,7 @@
* LICENSE-AGPL for a copy of the license.
*/
use reqwest::Client;
use serde::Deserializer;
use sql_builder::prelude::*;
@@ -14,7 +15,7 @@ use crate::{
error::{to_anyhow, Error, JsonResult, Result},
jobs, parser,
users::{owner_to_token_owner, truncate_token, Authed, Tokened},
utils::{require_admin, Pagination, StripPath},
utils::{http_get_from_hub, list_elems_from_hub, require_admin, Pagination, StripPath},
};
use axum::{
extract::{Extension, Host, Path, Query},
@@ -251,88 +252,22 @@ async fn list_scripts(
Ok(Json(rows))
}
#[derive(Deserialize, Serialize)]
struct SearchScriptData {
asks: Vec<ScriptSearch>,
}
#[derive(Deserialize, Serialize)]
struct ScriptSearch {
id: i32,
ask_id: i32,
summary: String,
app: String,
approved: bool,
is_trigger: bool,
views: i32,
votes: i32,
}
async fn list_hub_scripts(
Authed {
email, username, ..
}: Authed,
Extension(http_client): Extension<Client>,
Host(host): Host,
) -> JsonResult<Vec<ScriptSearch>> {
let http_client = reqwest::ClientBuilder::new()
.user_agent("windmill/beta")
.build()
.map_err(to_anyhow)?;
let rows = http_client
.get("https://hub.windmill.dev/searchData?approved=true")
.header("X-email", email.unwrap_or_else(|| "".to_string()))
.header("X-username", username)
.header("X-hostname", host)
.send()
.await
.map_err(to_anyhow)?
.json::<SearchScriptData>()
.await
.map_err(to_anyhow)?
.asks;
Ok(Json(rows))
}
#[derive(Deserialize, Serialize)]
struct SearchFlowData {
asks: Vec<FlowSearch>,
}
#[derive(Deserialize, Serialize)]
struct FlowSearch {
id: i32,
ask_id: i32,
summary: String,
app: String,
approved: bool,
is_trigger: bool,
views: i32,
votes: i32,
}
async fn list_hub_flows(
Authed {
email, username, ..
}: Authed,
Host(host): Host,
) -> JsonResult<Vec<ScriptSearch>> {
let http_client = reqwest::ClientBuilder::new()
.user_agent("windmill/beta")
.build()
.map_err(to_anyhow)?;
let rows = http_client
.get("https://hub.windmill.dev/searchFlowData?approved=true")
.header("X-email", email.unwrap_or_else(|| "".to_string()))
.header("X-username", username)
.header("X-hostname", host)
.send()
.await
.map_err(to_anyhow)?
.json::<SearchScriptData>()
.await
.map_err(to_anyhow)?
.asks;
Ok(Json(rows))
) -> JsonResult<serde_json::Value> {
let asks = list_elems_from_hub(
http_client,
"https://hub.windmill.dev/searchData?approved=true",
email,
username,
host,
)
.await?;
Ok(Json(asks))
}
fn hash_script(ns: &NewScript) -> i64 {
@@ -548,26 +483,26 @@ pub async fn get_hub_script_by_path(
email, username, ..
}: Authed,
Path(path): Path<StripPath>,
Extension(http_client): Extension<Client>,
Host(host): Host,
) -> Result<String> {
let path = path
.to_path()
.strip_prefix("hub/")
.ok_or_else(|| Error::BadRequest("Impossible to remove prefix hex".to_string()))?;
let http_client = reqwest::ClientBuilder::new()
.user_agent("windmill/beta")
.build()
.map_err(to_anyhow)?;
let content = http_client
.get(format!("https://hub.windmill.dev/raw/{path}.ts"))
.header("X-email", email.unwrap_or_else(|| "".to_string()))
.header("X-username", username)
.send()
.await
.map_err(to_anyhow)?
.text()
.await
.map_err(to_anyhow)?;
let content = http_get_from_hub(
http_client,
&format!("https://hub.windmill.dev/raw/{path}.ts"),
email,
username,
host,
true,
)
.await?
.text()
.await
.map_err(to_anyhow)?;
Ok(content)
}

46
backend/src/utils.rs
View File

@@ -6,10 +6,11 @@
*/
use rand::{distributions::Alphanumeric, thread_rng, Rng};
use reqwest::Response;
use serde::Deserialize;
use sqlx::{Postgres, Transaction};
use crate::error::{Error, Result};
use crate::error::{to_anyhow, Error, Result};
pub const MAX_PER_PAGE: usize = 1000;
pub const DEFAULT_PER_PAGE: usize = 100;
@@ -95,3 +96,46 @@ pub fn not_found_if_none<T, U: AsRef<str>>(opt: Option<T>, kind: &str, name: U)
pub fn get_owner_from_path(path: &str) -> String {
path.split('/').take(2).collect::<Vec<_>>().join("/")
}
pub async fn list_elems_from_hub(
http_client: reqwest::Client,
url: &str,
email: Option<String>,
username: String,
host: String,
) -> Result<serde_json::Value> {
let rows = http_get_from_hub(http_client, url, email, username, host, false)
.await?
.json::<serde_json::Value>()
.await
.map_err(to_anyhow)?;
Ok(rows)
}
pub async fn http_get_from_hub(
http_client: reqwest::Client,
url: &str,
email: Option<String>,
username: String,
host: String,
plain: bool,
) -> Result<Response> {
let response = http_client
.get(url)
.header(
"Accept",
if plain {
"text/plain"
} else {
"application/json"
},
)
.header("X-email", email.unwrap_or_else(|| "".to_string()))
.header("X-username", username)
.header("X-hostname", host)
.send()
.await
.map_err(to_anyhow)?;
Ok(response)
}

8
backend/src/worker_flow.rs
View File

@@ -1,6 +1,6 @@
use std::collections::HashMap;
use crate::flow::{FlowModuleValue, FlowValue, InputTransform};
use crate::flows::{FlowModuleValue, FlowValue, InputTransform};
use crate::jobs::{
add_completed_job, add_completed_job_error, get_queued_job, postprocess_queued_job, push,
script_path_to_payload, JobPayload,
@@ -394,9 +394,9 @@ async fn push_next_flow_job(
let module = &flow.modules[i];
let mut tx = db.begin().await?;
let job_payload = match &module.value {
FlowModuleValue::Script {
path: script_path
} => script_path_to_payload(script_path, &mut tx, &flow_job.workspace_id).await?,
FlowModuleValue::Script { path: script_path } => {
script_path_to_payload(script_path, &mut tx, &flow_job.workspace_id).await?
}
FlowModuleValue::RawScript(raw_code) => {
let mut raw_code = raw_code.clone();
if raw_code.path.is_none() {

2
backend/src/workspaces.rs
View File

@@ -8,7 +8,7 @@
use crate::{
db::{UserDB, DB},
error::{Error, JsonResult, Result},
users::{Authed, WorkspaceInvite}, utils::{require_admin, require_super_admin, Pagination}, audit::{audit_log, ActionKind}, scripts::{Script, Schema}, resources::{Resource, ResourceType}, flow::Flow, variables::ListableVariable,
users::{Authed, WorkspaceInvite}, utils::{require_admin, require_super_admin, Pagination}, audit::{audit_log, ActionKind}, scripts::{Script, Schema}, resources::{Resource, ResourceType}, flows::Flow, variables::ListableVariable,
};
use axum::{extract::{Extension, Path, Query}, routing::{get, post, delete}, Json, Router, response::{IntoResponse}, body::StreamBody};

2
frontend/package.json
View File

@@ -1,6 +1,6 @@
{
"name": "windmill",
"version": "1.22.0",
"version": "1.22.42",
"scripts": {
"dev": "vite dev",
"build": "vite build",

32
frontend/src/lib/components/Editor.svelte
View File

@@ -25,6 +25,8 @@
let websockets: WebSocket[] = []
let websocketInterval: NodeJS.Timer | undefined
let lastWsAttempt: Date | undefined
let nbWsAttempt = 0
let uri: string = ''
let disposeMethod: () => void | undefined
const dispatch = createEventDispatcher()
@@ -154,6 +156,8 @@
const writer = new WebSocketMessageWriter(socket)
const languageClient = createLanguageClient({ reader, writer }, name, options)
languageClient.start()
lastWsAttempt = undefined
nbWsAttempt = 0
reader.onClose(() => {
try {
languageClient.stop()
@@ -239,14 +243,27 @@
}
})
}
websocketInterval && clearInterval(websocketInterval)
websocketInterval = setInterval(() => {
if (document.visibilityState == 'visible') {
if (!websocketAlive.black && !websocketAlive.deno && !websocketAlive.pyright) {
sendUserToast(
'Smart assistant got disconnected. Reconnecting to windmill language server for smart assistance'
)
reloadWebsocket()
if (
!lastWsAttempt ||
(lastWsAttempt.getTime() - new Date().getTime() > 60000 && nbWsAttempt < 2)
) {
if (!websocketAlive.black && !websocketAlive.deno && !websocketAlive.pyright) {
sendUserToast(
'Smart assistant got disconnected. Reconnecting to windmill language server for smart assistance'
)
lastWsAttempt = new Date()
nbWsAttempt++
reloadWebsocket()
} else {
if (nbWsAttempt >= 2) {
sendUserToast('Giving up on establishing smart assistant connection', true)
clearInterval(websocketInterval)
}
}
}
}
}, 5000)
@@ -380,9 +397,8 @@
})
onDestroy(() => {
if (disposeMethod) {
disposeMethod()
}
disposeMethod && disposeMethod()
websocketInterval && clearInterval(websocketInterval)
})
</script>

8
frontend/src/lib/components/FlowBuilder.svelte
View File

@@ -1,12 +1,10 @@
<script lang="ts">
import { goto } from '$app/navigation'
import { page } from '$app/stores'
import { FlowService, ScriptService, type Flow } from '$lib/gen'
import { FlowService, type Flow } from '$lib/gen'
import { clearPreviewResults, hubScripts, workspaceStore } from '$lib/stores'
import { loadHubScripts, sendUserToast, setQueryWithoutLoad } from '$lib/utils'
import { faFileExport, faFileImport } from '@fortawesome/free-solid-svg-icons'
import { onMount } from 'svelte'
import Icon from 'svelte-awesome'
import FlowEditor from './FlowEditor.svelte'
import { flowStore, type FlowMode } from './flows/flowStore'
import { flowToMode } from './flows/utils'
@@ -114,8 +112,8 @@
</div>
<div class="flex flex-row-reverse">
<span class="my-1 text-sm text-gray-500 italic">
{#if initialPath && initialPath != $flowStore.path} {initialPath} &rightarrow; {/if}
{$flowStore.path}
{#if initialPath && initialPath != $flowStore?.path} {initialPath} &rightarrow; {/if}
{$flowStore?.path}
</span>
</div>
</div>

265
frontend/src/lib/components/FlowEditor.svelte
View File

@@ -61,72 +61,76 @@
</div>
</Modal>
<div class="flow-root bg-gray-50 rounded-xl border border-gray-200">
<ul class="relative -mt-10">
<span class="absolute top-0 left-1/2 h-full w-1 bg-gray-400" aria-hidden="true" />
<div class="relative">
<li class="flex flex-row flex-shrink max-w-full mx-auto mt-20">
<div
class="bg-white border border-gray xl-rounded shadow-lg w-full max-w-4xl mx-4 md:mx-auto p-4"
>
<div class="flex flex-row-reverse mr-4">
<Dropdown
dropdownItems={[
{
displayName: 'Import from JSON',
icon: faFileImport,
action: () => {
jsonSetter.openModal()
{#if $flowStore}
<div class="flow-root bg-gray-50 rounded-xl border border-gray-200">
<ul class="relative -mt-10">
<span class="absolute top-0 left-1/2 h-full w-1 bg-gray-400" aria-hidden="true" />
<div class="relative">
<li class="flex flex-row flex-shrink max-w-full mx-auto mt-20">
<div
class="bg-white border border-gray xl-rounded shadow-lg w-full max-w-4xl mx-4 md:mx-auto p-4"
>
<div class="flex flex-row-reverse mr-4">
<Dropdown
dropdownItems={[
{
displayName: 'Import from JSON',
icon: faFileImport,
action: () => {
jsonSetter.openModal()
}
},
{
displayName: 'Export to JSON',
icon: faFileExport,
action: () => {
jsonViewer.openModal()
}
},
{
displayName: 'Publish to Hub',
icon: faGlobe,
action: () => {
const url = new URL('https://hub.windmill.dev/flows/add')
const openFlow = {
value: $flowStore.value,
summary: $flowStore.summary,
description: $flowStore.description,
schema: $flowStore.schema
}
url.searchParams.append(
'flow',
btoa(JSON.stringify(flowToMode(openFlow, mode)))
)
window.open(url, '_blank')?.focus()
}
}
},
{
displayName: 'Export to JSON',
icon: faFileExport,
action: () => {
jsonViewer.openModal()
}
},
{
displayName: 'Publish to Hub',
icon: faGlobe,
action: () => {
const url = new URL('https://hub.windmill.dev/flows/add')
url.searchParams.append(
'flow',
btoa(JSON.stringify(flowToMode($flowStore, mode).value))
)
url.searchParams.append('schema', btoa(JSON.stringify($flowStore.schema)))
url.searchParams.append('description', $flowStore.description ?? '')
url.searchParams.append('summary', $flowStore.summary)
window.open(url, '_blank')?.focus()
}
}
]}
relative={false}
/>
</div>
<div class="mb-8 p-4">
<Path
bind:error={pathError}
bind:path={$flowStore.path}
{initialPath}
namePlaceholder="my_flow"
kind="flow"
>
<div slot="ownerToolkit">
Flow permissions depend on their path. Select the group <span class="font-mono"
>all</span
>
to share your flow, and <span class="font-mono">user</span> to keep it private.
<a href="https://docs.windmill.dev/docs/reference/namespaces">docs</a>
</div>
</Path>
]}
relative={false}
/>
</div>
<div class="mb-8 p-4">
<Path
bind:error={pathError}
bind:path={$flowStore.path}
{initialPath}
namePlaceholder="my_flow"
kind="flow"
>
<div slot="ownerToolkit">
Flow permissions depend on their path. Select the group <span class="font-mono"
>all</span
>
to share your flow, and <span class="font-mono">user</span> to keep it private.
<a href="https://docs.windmill.dev/docs/reference/namespaces">docs</a>
</div>
</Path>
<label class="block mt-4">
<span class="text-gray-700">Summary <Required required={false} /></span>
<textarea
bind:value={$flowStore.summary}
class="
<label class="block mt-4">
<span class="text-gray-700">Summary <Required required={false} /></span>
<textarea
bind:value={$flowStore.summary}
class="
mt-1
block
w-full
@@ -135,82 +139,85 @@
shadow-sm
focus:border-indigo-300 focus:ring focus:ring-indigo-200 focus:ring-opacity-50
"
placeholder="A very short summary of the flow displayed when the flow is listed"
rows="1"
/>
</label>
placeholder="A very short summary of the flow displayed when the flow is listed"
rows="1"
/>
</label>
</div>
<RadioButtonV2
options={[
[
{
title: 'Push',
desc: 'Trigger this flow through the generated UI, a manual schedule or by calling the associated webhook'
},
'push'
],
[
{
title: 'Pull',
desc: 'The first module of this flow is a trigger script whose purpose is to pull data from an external source and return all new items since last run. This flow is meant to be scheduled very regularly to reduce latency to react to new events. It will trigger the rest of the flow once per item. If no new items, the flow will be skipped.'
},
'pull'
]
]}
bind:value={mode}
/>
</div>
<RadioButtonV2
options={[
[
{
title: 'Push',
desc: 'Trigger this flow through the generated UI, a manual schedule or by calling the associated webhook'
},
'push'
],
[
{
title: 'Pull',
desc: 'The first module of this flow is a trigger script whose purpose is to pull data from an external source and return all new items since last run. This flow is meant to be scheduled very regularly to reduce latency to react to new events. It will trigger the rest of the flow once per item. If no new items, the flow will be skipped.'
},
'pull'
]
]}
bind:value={mode}
/>
</div>
</li>
<li class="flex flex-row flex-shrink max-w-full mx-auto mt-20">
<div class="bg-white border border-gray xl-rounded shadow-lg w-full mx-4 xl:mx-20">
<div
class="flex items-center justify-between flex-wra px-4 py-5 border-b border-gray-200 sm:px-6"
>
<h3 class="text-lg leading-6 font-medium text-gray-900">Flow Input</h3>
<CopyFirstStepSchema />
</li>
<li class="flex flex-row flex-shrink max-w-full mx-auto mt-20">
<div class="bg-white border border-gray xl-rounded shadow-lg w-full mx-4 xl:mx-20">
<div
class="flex items-center justify-between flex-wra px-4 py-5 border-b border-gray-200 sm:px-6"
>
<h3 class="text-lg leading-6 font-medium text-gray-900">Flow Input</h3>
<CopyFirstStepSchema />
</div>
<div class="p-4">
<SchemaEditor schema={$flowStore.schema} />
<div class="my-4" />
<FlowPreview {mode} flow={$flowStore} i={numberOfSteps} bind:args />
</div>
</div>
<div class="p-4">
<SchemaEditor schema={$flowStore.schema} />
<div class="my-4" />
<FlowPreview {mode} flow={$flowStore} i={numberOfSteps} bind:args />
</div>
</div>
</li>
{#each $flowStore?.value.modules as mod, i}
<li class="relative mt-16">
</li>
{#each $flowStore?.value.modules as mod, i}
<li class="relative mt-16">
<div class="relative flex justify-center">
<button
class="default-button h-10 w-10 shadow-blue-600/40 border-blue-600 shadow"
on:click={() => {
addModule(i)
open = i
}}
>
<Icon class="text-white mb-1" data={faPlus} />
</button>
</div>
</li>
<ModuleStep bind:open bind:mod bind:args {i} {mode} />
{/each}
<li class="relative m-20 ">
<div class="relative flex justify-center">
<button
class="default-button h-10 w-10 shadow-blue-600/40 border-blue-600 shadow"
disabled={pathIsEmpty($flowStore.path)}
class="default-button h-10 w-10 shadow"
on:click={() => {
addModule(i)
open = i
addModule()
open = $flowStore?.value.modules.length - 1
}}
>
<Icon class="text-white mb-1" data={faPlus} />
Add step {pathIsEmpty($flowStore?.path) ? '(pick a name first!)' : ''}
</button>
</div>
</li>
<ModuleStep bind:open bind:mod bind:args {i} {mode} />
{/each}
<li class="relative m-20 ">
<div class="relative flex justify-center">
<button
disabled={pathIsEmpty($flowStore.path)}
class="default-button h-10 w-10 shadow"
on:click={() => {
addModule()
open = $flowStore?.value.modules.length - 1
}}
>
<Icon class="text-white mb-1" data={faPlus} />
Add step {pathIsEmpty($flowStore.path) ? '(pick a name first!)' : ''}
</button>
</div>
</li>
</div>
</ul>
</div>
<div class="py-10 bg-white" />
</div>
</ul>
</div>
<div class="py-10 bg-white" />
{:else}
<h3>Loading flow</h3>
{/if}
<style>
.shadow:not([disabled]) {

49
frontend/src/lib/components/FlowViewer.svelte
View File

@@ -10,6 +10,8 @@
import github from 'svelte-highlight/styles/github'
import { slide } from 'svelte/transition'
import Tabs from './Tabs.svelte'
import SchemaViewer from './SchemaViewer.svelte'
import FieldHeader from './FieldHeader.svelte'
export let flow: {
summary: string
@@ -27,7 +29,7 @@
export let embedded = false
export let tab: 'ui' | 'json' = 'ui'
export let tab: 'ui' | 'json' | 'schema' = 'ui'
let open: { [id: number]: boolean } = {}
</script>
@@ -39,14 +41,40 @@
<Tabs
tabs={[
['ui', 'Flow rendered'],
['json', 'JSON']
['json', 'JSON'],
['schema', 'Input schema of the flow']
]}
bind:tab
/>
{/if}
{#if tab == 'ui'}
<div class="flow-root w-full p-4">
<p class="font-black text-lg mb-6 w-full ml-2">
<p class="font-black text-lg w-full ml-2">
<span>Inputs</span>
</p>
{#if flow.schema && flow.schema.properties && Object.keys(flow.schema.properties).length > 0 && flow.schema}
<ul class="my-4 ml-6">
{#each Object.entries(flow.schema.properties) as [inp, v]}
<li class="list-disc flex flex-row">
<FieldHeader
label={inp}
required={flow.schema.required?.includes(inp)}
type={v?.type}
contentEncoding={v?.contentEncoding}
format={v?.format}
itemsType={v?.itemsType}
/><span class="ml-4 mt-2 text-xs"
>{v.default != undefined ? 'default: ' + JSON.stringify(v.default) : ''}</span
>
</li>
{/each}
</ul>
{:else}
<div class="text-gray-700 text-xs italic mb-4">
This script has no argument or is ill-defined
</div>
{/if}
<p class="font-black text-lg my-6 w-full ml-2">
<span>{flow?.value?.modules?.length} Steps </span>
<span class="mt-4" />
</p>
@@ -107,6 +135,17 @@
{/each}
</ul>
</div>
{:else}
<Highlight language={json} code={JSON.stringify(flowFiltered, null, 4)} />
{:else if tab == 'json'}
<div class="relative">
<button
on:click={async () => {
await navigator.clipboard.writeText(JSON.stringify(flowFiltered, null, 4))
}}
class="absolute default-secondary-button-v2 bg-white/30 right-0 my-2 ml-4"
>copy content</button
>
<Highlight language={json} code={JSON.stringify(flowFiltered, null, 4)} />
</div>
{:else if tab == 'schema'}
<SchemaViewer schema={flow.schema} />
{/if}

82
frontend/src/lib/components/Modal.svelte
View File

@@ -32,45 +32,18 @@
<svelte:window on:keyup={handleKeyUp} />
<div class="blurred-background {open ? '' : 'hidden'}" />
{#if open}
<div class="blurred-background" />
<div class="fixed top-0 w-screen h-screen {open ? '' : 'hidden'} {z}">
<div
class="fixed right-0 flex flex-col w-3/4 sm:w-2/3 lg:w-1/2 h-screen border border-gray-300 shadow-xl"
>
{#if open}
<div class="flex flex-row justify-between p-2 bg-white border-b border-gray-200">
<button
on:click={() => {
open = false
closeModal()
}}
>
<svg
class="w-6 h-6"
fill="none"
stroke="currentColor"
viewBox="0 0 24 24"
xmlns="http://www.w3.org/2000/svg"
>
<path
stroke-linecap="round"
stroke-linejoin="round"
stroke-width="2"
d="M6 18L18 6M6 6l12 12"
/>
</svg>
</button>
<p class="font-semibold text-gray-800"><slot name="title" /></p>
<div />
</div>
<div class="flex flex-col bg-gray-50 pt-3 px-6 grow overflow-y-auto">
<slot name="content" />
</div>
<div class="flex flex-col bg-white border-gray-200 p-2">
<div class="flex flex-row justify-between p-2 ">
<div class="fixed top-0 w-screen h-screen {z}">
<div
class="fixed right-0 flex flex-col w-3/4 sm:w-2/3 lg:w-1/2 h-screen border border-gray-300 shadow-xl"
>
{#if open}
<div class="flex flex-row justify-between p-2 bg-white border-b border-gray-200">
<button
on:click={() => {
open = false
closeModal()
}}
>
@@ -89,12 +62,41 @@
/>
</svg>
</button>
<span class="mr-4"><slot name="submission">&nbsp;</slot></span>
<p class="font-semibold text-gray-800"><slot name="title" /></p>
<div />
</div>
</div>
{/if}
<div class="flex flex-col bg-gray-50 pt-3 px-6 grow overflow-y-auto">
<slot name="content" />
</div>
<div class="flex flex-col bg-white border-gray-200 p-2">
<div class="flex flex-row justify-between p-2 ">
<button
on:click={() => {
closeModal()
}}
>
<svg
class="w-6 h-6"
fill="none"
stroke="currentColor"
viewBox="0 0 24 24"
xmlns="http://www.w3.org/2000/svg"
>
<path
stroke-linecap="round"
stroke-linejoin="round"
stroke-width="2"
d="M6 18L18 6M6 6l12 12"
/>
</svg>
</button>
<span class="mr-4"><slot name="submission">&nbsp;</slot></span>
</div>
</div>
{/if}
</div>
</div>
</div>
{/if}
<style>
.blurred-background {

1
frontend/src/lib/components/Path.svelte
View File

@@ -176,6 +176,7 @@
<label class="block col-span-2">
<span class="text-gray-700 text-sm">Name<span class="text-red-600 text-sm">*</span></span>
<input
autofocus
bind:value={meta.name}
placeholder={namePlaceholder}
class={error === ''

2
frontend/src/lib/components/SchemaViewer.svelte
View File

@@ -72,7 +72,7 @@
</TableCustom>
</div>
{:else}
<div class="text-gray-700 text-xs italic">This script has no argument</div>
<div class="text-gray-700 text-xs italic">This script has no arguments</div>
{/if}
</div>
<div class={viewJsonSchema ? '' : 'hidden'}>

2
frontend/src/lib/components/flows/utils.ts
View File

@@ -11,7 +11,7 @@ import { loadSchema } from '$lib/scripts'
import { emptySchema, getScriptByPath } from '$lib/utils'
import type { FlowMode } from './flowStore'
export function flowToMode(flow: Flow, mode: FlowMode): Flow {
export function flowToMode(flow: Flow | any, mode: FlowMode): Flow {
if (mode == 'pull') {
const newFlow: Flow = JSON.parse(JSON.stringify(flow))
const triggerModule = newFlow.value.modules[0]

23
frontend/src/lib/utils.ts
View File

@@ -1,6 +1,6 @@
/* eslint-disable @typescript-eslint/explicit-module-boundary-types */
import { goto } from '$app/navigation'
import { Script, ScriptService, type User } from '$lib/gen'
import { FlowService, Script, ScriptService, type User } from '$lib/gen'
import { toast } from '@zerodevx/svelte-toast'
import { get } from 'svelte/store'
import type { Schema } from './common'
@@ -437,12 +437,23 @@ export async function getScriptByPath(path: string): Promise<{
export async function loadHubScripts() {
const scripts = await ScriptService.listHubScripts()
hubScripts.set(scripts.map((x) => ({
const scripts = (await ScriptService.listHubScripts()).asks ?? []
const processed = scripts.map((x) => ({
path: `hub/${x.id}/${x.summary.toLowerCase().replaceAll(/\s+/g, '_')}`,
summary: `${x.summary} (${x.app})`,
summary: `${x.summary} (${x.app}) ${x.views} uses`,
approved: x.approved,
is_trigger: x.is_trigger,
app: x.app
})))
app: x.app,
views: x.views,
votes: x.votes,
ask_id: x.ask_id,
})).sort((a, b) => b.views - a.views)
hubScripts.set(processed)
}
export async function loadHubFlows() {
const flows = (await FlowService.listHubFlows()).flows ?? []
const processed = flows.sort((a, b) => b.votes - a.votes)
return processed
}

102
frontend/src/routes/flows.svelte
View File

@@ -1,9 +1,9 @@
<script lang="ts">
import Fuse from 'fuse.js'
import { FlowService } from '$lib/gen'
import { FlowService, type OpenFlow } from '$lib/gen'
import type { Flow } from '$lib/gen'
import { sendUserToast, groupBy, canWrite } from '$lib/utils'
import { sendUserToast, groupBy, canWrite, loadHubFlows } from '$lib/utils'
import Icon from 'svelte-awesome'
import {
faArchive,
@@ -24,27 +24,47 @@
import { superadmin, userStore, workspaceStore } from '$lib/stores'
import CenteredPage from '$lib/components/CenteredPage.svelte'
import Tabs from '$lib/components/Tabs.svelte'
import TableCustom from '$lib/components/TableCustom.svelte'
import Modal from '$lib/components/Modal.svelte'
import FlowViewer from '$lib/components/FlowViewer.svelte'
type Tab = 'all' | 'personal' | 'groups' | 'shared'
type Tab = 'all' | 'personal' | 'groups' | 'shared' | 'hub'
type Section = [string, FlowW[]]
type FlowW = Flow & { canWrite: boolean; tab: Tab }
let flows: FlowW[] = []
let filteredFlows: FlowW[]
let hubFlows: any[] = []
let filteredHubFlows: any[]
let flowFilter = ''
let groupedFlows: Section[] = []
let hubFilter = ''
let tab: Tab = 'all'
let shareModal: ShareModal
const fuseOptions = {
const flowFuseOptions = {
includeScore: false,
keys: ['description', 'path', 'content', 'hash', 'summary']
}
const fuse: Fuse<FlowW> = new Fuse(flows, fuseOptions)
const flowFuse: Fuse<FlowW> = new Fuse(flows, flowFuseOptions)
const flowHubFuse: Fuse<FlowW> = new Fuse(flows, {
includeScore: false,
keys: ['summary']
})
$: filteredFlows =
flowFilter.length > 0 ? fuse.search(flowFilter).map((value) => value.item) : flows
flowFilter.length > 0 ? flowFuse.search(flowFilter).map((value) => value.item) : flows
$: filteredHubFlows =
hubFilter.length > 0 ? flowHubFuse.search(hubFilter).map((value) => value.item) : hubFlows
let flowViewer: Modal
let flowViewerFlow: OpenFlow | undefined
$: {
let defaults: string[] = []
@@ -86,7 +106,12 @@
}
)
flows = tab == 'all' ? allFlows : allFlows.filter((x) => x.tab == tab)
fuse.setCollection(flows)
flowFuse.setCollection(flows)
}
async function loadHubFlowsWFuse(): Promise<void> {
hubFlows = await loadHubFlows()
flowFuse.setCollection(flows)
}
async function archiveFlow(path: string): Promise<void> {
@@ -99,6 +124,14 @@
}
}
async function viewFlow(id: number): Promise<void> {
const hub = (await FlowService.getHubFlowById({ id: Number(id) })).flow
flowViewerFlow = hub
flowViewer.openModal()
}
loadHubFlowsWFuse()
$: {
if ($workspaceStore && ($userStore || $superadmin)) {
loadFlows()
@@ -106,6 +139,15 @@
}
</script>
<Modal bind:this={flowViewer}>
<div slot="title">Hub flow '{flowViewerFlow?.summary ?? ''}'</div>
<div slot="content">
{#if flowViewerFlow}
<FlowViewer flow={flowViewerFlow} />
{/if}
</div></Modal
>
<CenteredPage>
<PageHeader title="Flows" tooltip="Flows can compose and chain scripts together">
<div class="flex flex-row">
@@ -126,9 +168,11 @@
bind:tab
on:update={loadFlows}
/>
<input placeholder="Search flows" bind:value={flowFilter} class="search-bar mt-2" />
{#if tab != 'hub'}
<input placeholder="Search flows" bind:value={flowFilter} class="search-bar mt-2" />
{/if}
<div class="grid grid-cols-1 divide-y">
{#each tab == 'all' ? ['personal', 'groups', 'shared'] : [tab] as sectionTab}
{#each tab == 'all' ? ['personal', 'groups', 'shared', 'hub'] : [tab] as sectionTab}
<div class="shadow p-4 my-2">
{#if sectionTab == 'personal'}
<h2 class="">
@@ -136,18 +180,52 @@
</h2>
<p class="italic text-xs text-gray-600 mb-4">
All flows owned by you (and visible only to you if you do not explicitely share them)
will be displayed below
</p>
{:else if sectionTab == 'groups'}
<h2 class="">Groups that I am member of</h2>
<p class="italic text-xs text-gray-600">
All flows being owned by groups that you are member of will be displayed below
All flows being owned by groups that you are member of
</p>
{:else if sectionTab == 'shared'}
<h2 class="">Shared with me</h2>
<p class="italic text-xs text-gray-600">
All flows visible to you because they have been shared to you will be displayed below
All flows visible to you because they have been shared to you
</p>
{:else if sectionTab == 'hub'}
<h2 class="">Approved flows from the WindmillHub</h2>
<p class="italic text-xs text-gray-600 mb-8">
All approved Flow from the <a href="https://hub.windmill.dev">WindmillHub</a>. Approved
flows have been potentially contributed by the community but reviewed and selected
carefully by the Windmill team.
</p>
<input placeholder="Search hub flows" bind:value={hubFilter} class="search-bar mt-2" />
<div class="relative">
<TableCustom>
<tr slot="header-row">
<th>Apps</th>
<th>Summary</th>
<th />
</tr>
<tbody slot="body">
{#each filteredHubFlows ?? [] as { summary, apps, id, flow_id }}
<tr>
<td class="font-black">{apps.join(', ')}</td>
<td><button on:click={() => viewFlow(flow_id)}>{summary}</button></td>
<td
><button class="text-blue-500" on:click={() => viewFlow(flow_id)}
>view flow</button
>
|
<a target="_blank" href={`https://hub.windmill.dev/flows/${flow_id}`}
>hub's page
</a>
| <a href={`/flows/add?hub=${flow_id}`}>fork</a>
</td>
</tr>
{/each}
</tbody>
</TableCustom>
</div>
{/if}
{#each groupedFlows.filter((x) => tabFromPath(x[0]) == sectionTab) as [section, flows]}
{#if sectionTab != 'personal'}

21
frontend/src/routes/flows/add.svelte
View File

@@ -3,10 +3,11 @@
import FlowBuilder from '$lib/components/FlowBuilder.svelte'
import { initFlow } from '$lib/components/flows/flowStore'
import type { Flow } from '$lib/gen'
import { emptySchema } from '$lib/utils'
import { FlowService, type Flow } from '$lib/gen'
import { emptySchema, sendUserToast } from '$lib/utils'
const initialState = $page.url.searchParams.get('state')
const hubId = $page.url.searchParams.get('hub')
let flow: Flow =
initialState != undefined
@@ -22,7 +23,21 @@
schema: emptySchema()
}
initFlow(flow)
async function loadFlow() {
if (hubId) {
const hub = (await FlowService.getHubFlowById({ id: Number(hubId) })).flow
flow.summary = hub?.summary ?? ''
flow.value = hub?.value ?? { modules: [] }
flow.description = hub?.description
flow.schema = hub?.schema ?? emptySchema()
flow = flow
$page.url.searchParams.delete('hub')
sendUserToast(`Flow has been loaded from hub flow id ${hubId}.`)
}
initFlow(flow)
}
loadFlow()
</script>
<FlowBuilder />

14
frontend/src/routes/flows/get/[...path].svelte
View File

@@ -168,20 +168,6 @@
></code
></pre>
</div>
<div>
<div class="grid grid-cols-2 gap-4 pb-1 mb-3 border-b">
<h3 class="text-gray-700 ">
Arguments JSON schema <Tooltip
>The jsonschema defines the constraints that the payload must respect to be compatible
with the input parameters of this flow. The UI form is generated automatically from
the flow jsonschema. See <a href="https://json-schema.org/"
>jsonschema documentation</a
></Tooltip
>
</h3>
</div>
<SchemaViewer schema={flow.schema} />
</div>
<div>
<h3 class="text-gray-700 pb-1 mb-3 border-b">Flow</h3>
<FlowViewer {flow} />

18
frontend/src/routes/scripts.svelte
View File

@@ -147,7 +147,13 @@
codeViewerPath = path
codeViewer.openModal()
}
loadHubScripts()
async function loadHubScriptsWFuse(): Promise<void> {
await loadHubScripts()
hubScriptsFuse.setCollection($hubScripts ?? [])
}
loadHubScriptsWFuse()
$: {
if ($workspaceStore && ($userStore || $superadmin)) {
@@ -250,7 +256,7 @@
<th />
</tr>
<tbody slot="body">
{#each filteredHub ?? [] as { path, summary, app }}
{#each filteredHub ?? [] as { path, summary, app, ask_id }}
<tr>
<td class="font-black">{app}</td>
<td><button on:click={() => viewCode(path)}>{summary}</button></td>
@@ -259,12 +265,8 @@
>view code</button
>
|
<a
target="_blank"
href={`https://hub.windmill.dev/from_version/${path
.split('/')
.slice(1, 3)
.join('/')}`}>hub's page</a
<a target="_blank" href={`https://hub.windmill.dev/scripts/${app}/${ask_id}`}
>hub's page</a
>
| <a href={`/scripts/add?hub=${encodeURIComponent(path)}`}>fork</a>
</td>

3
frontend/src/routes/scripts/add.svelte
View File

@@ -58,10 +58,11 @@
}
}
loadHub()
$: {
if ($workspaceStore) {
loadTemplate()
loadHub()
}
}
</script>

6
frontend/src/routes/user/login@user.svelte
View File

@@ -114,9 +114,9 @@
{#if showPassword}
<div transition:slide>
<p class="text-xs text-gray-400 italic my-2">
Signup without Github is not supported currently but if you do not want to use the github
login flow, you can send us an email at contact@windmill.dev and you will receive
credentials that you can use below.
To get credentials without the OAuth providers above, you can send us an email at
contact@windmill.dev or your admin owners if this instance is self-hosted and you will
receive credentials that you can use below.
</p>
<label class="block pb-2">
<span class="text-gray-700">email</span>