kalugin66/windmill
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Add workspace settings to sync scripts/flows/apps to git repo on deployment (#2740)

Browse Source 
* feat: Add workspace settings to sync script to git repo on deployment

* track deployment callback jobs in separate table

* sqlx prepare

* update deployment_metadata table

* Point to hub script

* Add details to the settings page and restrict to EE only

* cleanup unused impots

* Flows and Apps are now pushed to GH as well

* Ignore scripts/flows/apps located in private user folders
This commit is contained in:
Guillaume Bouvignies
2023-12-04 14:26:27 +01:00
committed by GitHub
parent 5cb3c34e44
commit 194ee830cc
33 changed files with 1359 additions and 690 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
backend/.sqlx/query-08ca7828d9f10bb7e4ee1c1df4bafd70b39ecae360773bf10d65f5db7b1821c5.json generated Normal file
View File

@@ -0,0 +1,16 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO deployment_metadata (workspace_id, path, callback_job_ids) VALUES ($1, $2, $3) ON CONFLICT (workspace_id, path) WHERE script_hash IS NULL AND app_version IS NULL DO UPDATE SET callback_job_ids = $3",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"UuidArray"
]
},
"nullable": []
},
"hash": "08ca7828d9f10bb7e4ee1c1df4bafd70b39ecae360773bf10d65f5db7b1821c5"
}

1
backend/.sqlx/query-11ae5e1aade2bc9a78af2d956ece73766ae3330319619a68383b508a76f2d19c.json generated
View File

@@ -42,7 +42,6 @@
"bash",
"postgresql",
"nativets",
"Nativets",
"bun",
"mysql",
"bigquery",

6
backend/.sqlx/query-1730f39fd1793d45fbb41b21389c61296a3ff7489ae12f52a19f9543173ac597.json generated
View File

@@ -87,6 +87,11 @@
"ordinal": 16,
"name": "large_file_storage",
"type_info": "Jsonb"
},
{
"ordinal": 17,
"name": "git_sync",
"type_info": "Jsonb"
}
],
"parameters": {
@@ -111,6 +116,7 @@
false,
true,
false,
true,
true
]
},

17
backend/.sqlx/query-238ab39af085912580b92ae7a482dae44b565a5f205fa33e9b5b35a34744b61d.json generated Normal file
View File

@@ -0,0 +1,17 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO deployment_metadata (workspace_id, path, app_version, callback_job_ids) VALUES ($1, $2, $3, $4)",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"Int8",
"UuidArray"
]
},
"nullable": []
},
"hash": "238ab39af085912580b92ae7a482dae44b565a5f205fa33e9b5b35a34744b61d"
}

4
backend/.sqlx/query-254776959f60455a00a2c29a1783ee6b27c836197df6454d3bb5c59441c34f44.json generated
View File

@@ -45,7 +45,8 @@
"graphql",
"postgresql",
"noop",
"appdependencies"
"appdependencies",
"deploymentcallback"
]
}
}
@@ -67,7 +68,6 @@
"bash",
"postgresql",
"nativets",
"Nativets",
"bun",
"mysql",
"bigquery",

17
backend/.sqlx/query-2dd55a9987fce9187fcaf00e4d8bcbe42f0d233f6f973ec77b44518a5c361b2c.json generated Normal file
View File

@@ -0,0 +1,17 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO deployment_metadata (workspace_id, path, script_hash, callback_job_ids) VALUES ($1, $2, $3, $4)",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"Int8",
"UuidArray"
]
},
"nullable": []
},
"hash": "2dd55a9987fce9187fcaf00e4d8bcbe42f0d233f6f973ec77b44518a5c361b2c"
}

14
backend/.sqlx/query-51d253d3cba9e2a36c291f6f9cf7023834484e89d91f0267d9dcaaafe406c64c.json generated Normal file
View File

@@ -0,0 +1,14 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE workspace_settings SET git_sync = NULL WHERE workspace_id = $1",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text"
]
},
"nullable": []
},
"hash": "51d253d3cba9e2a36c291f6f9cf7023834484e89d91f0267d9dcaaafe406c64c"
}

1
backend/.sqlx/query-52ad1773a9f5a1b6be1bb16af408f051f27d3f21b615c468b85aa75b64d943db.json generated
View File

@@ -28,7 +28,6 @@
"bash",
"postgresql",
"nativets",
"Nativets",
"bun",
"mysql",
"bigquery",

6
backend/.sqlx/query-5445083864b2b092b012e894bff7630a1d7b9deb8d33e9f909061f351f96844e.json generated
View File

@@ -87,6 +87,11 @@
"ordinal": 16,
"name": "large_file_storage",
"type_info": "Jsonb"
},
{
"ordinal": 17,
"name": "git_sync",
"type_info": "Jsonb"
}
],
"parameters": {
@@ -111,6 +116,7 @@
false,
true,
false,
true,
true
]
},

4
backend/.sqlx/query-620ddf29c5e867079df4c2aa6e80bccb19beeb9ddfa308ca97f254cd5ba8157e.json generated
View File

@@ -40,7 +40,8 @@
"graphql",
"postgresql",
"noop",
"appdependencies"
"appdependencies",
"deploymentcallback"
]
}
}
@@ -60,7 +61,6 @@
"bash",
"postgresql",
"nativets",
"Nativets",
"bun",
"mysql",
"bigquery",

4
backend/.sqlx/query-9d3556319411a27a875bf6cf0e5eda837cc63e4d8be912c0b5bfeea4a0c8db2e.json generated
View File

@@ -24,7 +24,8 @@
"graphql",
"postgresql",
"noop",
"appdependencies"
"appdependencies",
"deploymentcallback"
]
}
}
@@ -40,7 +41,6 @@
"bash",
"postgresql",
"nativets",
"Nativets",
"bun",
"mysql",
"bigquery",

4
backend/.sqlx/query-a9915d3c72a278ff08fbe6571c2a4fc993507e255004107345b7151322e063de.json generated
View File

@@ -30,7 +30,8 @@
"graphql",
"postgresql",
"noop",
"appdependencies"
"appdependencies",
"deploymentcallback"
]
}
}
@@ -46,7 +47,6 @@
"bash",
"postgresql",
"nativets",
"Nativets",
"bun",
"mysql",
"bigquery",

1
backend/.sqlx/query-b6074eb9369de071f79165bf506fbc44f1688035e1928b4c3649d875187b1e1d.json generated
View File

@@ -42,7 +42,6 @@
"bash",
"postgresql",
"nativets",
"Nativets",
"bun",
"mysql",
"bigquery",

1
backend/.sqlx/query-c7febee7146a28481ce36f3570bceb38751a6527827525a582d9f8804981f552.json generated
View File

@@ -42,7 +42,6 @@
"bash",
"postgresql",
"nativets",
"Nativets",
"bun",
"mysql",
"bigquery",

1
backend/.sqlx/query-d48070ef889723f408ed6c1a97b5ab9f0e98340cd33c6e29824f6030f0089071.json generated
View File

@@ -37,7 +37,6 @@
"bash",
"postgresql",
"nativets",
"Nativets",
"bun",
"mysql",
"bigquery",

15
backend/.sqlx/query-ec4f70d9e6392ade016245de73e7a2831d430838307b086c96511f91e76a214a.json generated Normal file
View File

@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE workspace_settings SET git_sync = $1 WHERE workspace_id = $2",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Jsonb",
"Text"
]
},
"nullable": []
},
"hash": "ec4f70d9e6392ade016245de73e7a2831d430838307b086c96511f91e76a214a"
}

6
backend/.sqlx/query-fc2ec6fc4e22e46cc35b00503eff1eac35a6fdbc82a634b7eb9da4d7931f3582.json generated
View File

@@ -39,7 +39,8 @@
"graphql",
"postgresql",
"noop",
"appdependencies"
"appdependencies",
"deploymentcallback"
]
}
}
@@ -61,7 +62,8 @@
"graphql",
"postgresql",
"noop",
"appdependencies"
"appdependencies",
"deploymentcallback"
]
}
}

4
backend/migrations/20231129164511_git_repo_workspace_setting.down.sql Normal file
View File

@@ -0,0 +1,4 @@
-- Add down migration script here
ALTER TABLE workspace_settings DROP COLUMN git_sync;
DROP TABLE IF EXISTS deployment_metadata;

16
backend/migrations/20231129164511_git_repo_workspace_setting.up.sql Normal file
View File

@@ -0,0 +1,16 @@
-- Add up migration script here
ALTER TYPE JOB_KIND ADD VALUE IF NOT EXISTS 'deploymentcallback';
ALTER TABLE workspace_settings ADD COLUMN IF NOT EXISTS git_sync JSONB;
CREATE TABLE deployment_metadata(
workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id),
path VARCHAR(255) NOT NULL,
script_hash BIGINT,
app_version BIGSERIAL,
callback_job_ids UUID[],
deployment_msg TEXT
);
CREATE UNIQUE INDEX IF NOT EXISTS deployment_metadata_script ON deployment_metadata (workspace_id, script_hash) WHERE script_hash IS NOT NULL;
CREATE UNIQUE INDEX IF NOT EXISTS deployment_metadata_flow ON deployment_metadata (workspace_id, path) WHERE script_hash IS NULL AND app_version IS NULL;
CREATE UNIQUE INDEX IF NOT EXISTS deployment_metadata_app ON deployment_metadata (workspace_id, path, app_version) WHERE app_version IS NOT NULL;

1355
backend/windmill-api/openapi-deref.yaml
View File

File diff suppressed because it is too large Load Diff

41
backend/windmill-api/openapi.yaml
View File

@@ -1192,6 +1192,8 @@ paths:
type: boolean
large_file_storage:
$ref: "#/components/schemas/LargeFileStorage"
git_sync:
$ref: "#/components/schemas/WorkspaceGitSync"
required:
- code_completion_enabled
@@ -1513,6 +1515,32 @@ paths:
application/json:
schema: {}
/w/{workspace}/workspaces/edit_git_sync_config:
post:
summary: edit workspace git sync settings
operationId: editWorkspaceGitSyncConfig
tags:
- workspace
parameters:
- $ref: "#/components/parameters/WorkspaceId"
requestBody:
description: Workspace Git sync settings
required: true
content:
application/json:
schema:
type: object
properties:
git_sync_settings:
$ref: "#/components/schemas/WorkspaceGitSync"
responses:
"200":
description: status
content:
application/json:
schema: {}
/w/{workspace}/workspaces/get_large_file_storage_config:
get:
summary: get large file storage config
@@ -7221,6 +7249,7 @@ components:
"flowpreview",
"script_hub",
"identity",
"deploymentcallback",
]
schedule_path:
type: string
@@ -7329,6 +7358,7 @@ components:
"flowpreview",
"script_hub",
"identity",
"deploymentcallback",
]
schedule_path:
type: string
@@ -8662,3 +8692,14 @@ components:
- endPoint
- useSSL
- pathStyle
WorkspaceGitSync:
type: object
properties:
script_path:
type: string
git_repo_resource_path:
type: string
required:
- script_path
- git_repo_resource_path

56
backend/windmill-api/src/apps.rs
View File

@@ -9,6 +9,7 @@ use std::collections::HashMap;
*/
use crate::{
db::{ApiAuthed, DB},
git_sync_helpers,
users::{require_owner_of_path, OptAuthed},
variables::build_crypt,
webhook_util::{WebhookMessage, WebhookShared},
@@ -512,8 +513,8 @@ async fn create_app(
)
.await?;
let tx = PushIsolationLevel::Transaction(tx);
let (dependency_job_uuid, tx) = push(
let mut tx = PushIsolationLevel::Transaction(tx);
let (dependency_job_uuid, new_tx) = push(
&db,
tx,
&w_id,
@@ -538,7 +539,26 @@ async fn create_app(
)
.await?;
tracing::info!("Pushed app dependency job {}", dependency_job_uuid);
tx.commit().await?;
tx = PushIsolationLevel::Transaction(new_tx);
tx = git_sync_helpers::run_workspace_repo_git_callback(
tx,
&authed,
&db,
&w_id,
git_sync_helpers::DeployedObject::App { path: app.path.clone(), version: v_id },
)
.await?;
match tx {
PushIsolationLevel::Transaction(tx) => tx.commit().await?,
_ => {
return Err(Error::InternalErr(
"Expected a transaction here".to_string(),
));
}
}
webhook.send_message(
w_id.clone(),
WebhookMessage::CreateApp { workspace: w_id, path: app.path.clone() },
@@ -753,11 +773,10 @@ async fn update_app(
)
.await?;
let mut tx: PushIsolationLevel<'_, rsmq_async::MultiplexedRsmq> =
PushIsolationLevel::Transaction(tx);
if let Some(v_id) = v_id {
let tx: PushIsolationLevel<'_, rsmq_async::MultiplexedRsmq> =
PushIsolationLevel::Transaction(tx);
let (dependency_job_uuid, tx) = push(
let (dependency_job_uuid, new_tx) = push(
&db,
tx,
&w_id,
@@ -782,10 +801,27 @@ async fn update_app(
)
.await?;
tracing::info!("Pushed app dependency job {}", dependency_job_uuid);
tx.commit().await?;
} else {
tx.commit().await?;
tx = PushIsolationLevel::Transaction(new_tx);
tx = git_sync_helpers::run_workspace_repo_git_callback(
tx,
&authed,
&db,
&w_id,
git_sync_helpers::DeployedObject::App { path: npath.clone(), version: v_id },
)
.await?;
}
match tx {
PushIsolationLevel::Transaction(tx) => tx.commit().await?,
_ => {
return Err(Error::InternalErr(
"Expected a transaction here".to_string(),
));
}
}
webhook.send_message(
w_id.clone(),
WebhookMessage::UpdateApp {

56
backend/windmill-api/src/flows.rs
View File

@@ -7,6 +7,7 @@
*/
use crate::db::ApiAuthed;
use crate::git_sync_helpers;
use crate::{
db::DB,
schedule::clear_schedule,
@@ -354,8 +355,8 @@ async fn create_flow(
)
.await?;
let tx = PushIsolationLevel::Transaction(tx);
let (dependency_job_uuid, mut tx) = push(
let mut tx = PushIsolationLevel::Transaction(tx);
let (dependency_job_uuid, mut new_tx) = push(
&db,
tx,
&w_id,
@@ -389,9 +390,27 @@ async fn create_flow(
nf.path,
w_id
)
.execute(&mut tx)
.execute(&mut new_tx)
.await?;
tx.commit().await?;
tx = PushIsolationLevel::Transaction(new_tx);
tx = git_sync_helpers::run_workspace_repo_git_callback(
tx,
&authed,
&db,
&w_id,
git_sync_helpers::DeployedObject::Flow { path: nf.path.clone() },
)
.await?;
match tx {
PushIsolationLevel::Transaction(tx) => tx.commit().await?,
_ => {
return Err(Error::InternalErr(
"Expected a transaction here".to_string(),
));
}
}
webhook.send_message(
w_id.clone(),
@@ -565,9 +584,9 @@ async fn update_flow(
},
);
let tx = PushIsolationLevel::Transaction(tx);
let mut tx = PushIsolationLevel::Transaction(tx);
let (dependency_job_uuid, mut tx) = push(
let (dependency_job_uuid, mut new_tx) = push(
&db,
tx,
&w_id,
@@ -600,17 +619,36 @@ async fn update_flow(
nf.path,
w_id
)
.execute(&mut tx)
.execute(&mut new_tx)
.await?;
if let Some(old_dep_job) = old_dep_job {
sqlx::query!(
"UPDATE queue SET canceled = true WHERE id = $1",
old_dep_job
)
.execute(&mut tx)
.execute(&mut new_tx)
.await?;
}
tx.commit().await?;
tx = PushIsolationLevel::Transaction(new_tx);
tx = git_sync_helpers::run_workspace_repo_git_callback(
tx,
&authed,
&db,
&w_id,
git_sync_helpers::DeployedObject::Flow { path: nf.path.clone() },
)
.await?;
match tx {
PushIsolationLevel::Transaction(tx) => tx.commit().await?,
_ => {
return Err(Error::InternalErr(
"Expected a transaction here".to_string(),
));
}
}
Ok(nf.path.to_string())
}

153
backend/windmill-api/src/git_sync_helpers.rs Normal file
View File

@@ -0,0 +1,153 @@
/*
* Author: Ruben Fiszel
* Copyright: Windmill Labs, Inc 2022
* This file and its contents are licensed under the AGPLv3 License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
use std::collections::HashMap;
use crate::db::ApiAuthed;
use crate::db::DB;
use crate::workspaces::{WorkspaceGitRepo, WorkspaceSettings};
use serde_json::json;
use windmill_common::error::{Error, Result};
use windmill_common::jobs::JobPayload;
use windmill_common::scripts::ScriptHash;
use windmill_common::users::username_to_permissioned_as;
use windmill_queue::PushIsolationLevel;
#[derive(Clone)]
pub enum DeployedObject {
Script { hash: ScriptHash, path: String },
Flow { path: String },
App { path: String, version: i64 },
}
impl DeployedObject {
pub fn get_path(&self) -> &str {
match self {
DeployedObject::Script { path, .. } => path,
DeployedObject::Flow { path } => path,
DeployedObject::App { path, .. } => path,
}
}
}
pub async fn run_workspace_repo_git_callback<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
mut tx: PushIsolationLevel<'c, R>,
authed: &ApiAuthed,
db: &DB,
w_id: &str,
obj: DeployedObject,
) -> Result<PushIsolationLevel<'c, R>> {
if obj.get_path().starts_with("u/") {
tracing::debug!(
"Ignoring {} from git sync as it's in a private user folder",
obj.get_path()
);
return Ok(tx);
}
let workspace_git_repo_setting = sqlx::query_as::<_, WorkspaceSettings>(
"SELECT * FROM workspace_settings WHERE workspace_id = $1",
)
.bind(&w_id)
.fetch_optional(db)
.await?;
if workspace_git_repo_setting.is_none() {
return Err(Error::InternalErr(
"No workspace settings found for workspace ID".to_string(),
));
}
let workspace_git_repo = workspace_git_repo_setting
.unwrap()
.git_sync
.map(|conf| serde_json::from_value::<WorkspaceGitRepo>(conf).ok())
.flatten();
if workspace_git_repo.is_none() {
return Ok(tx);
}
let workspace_git_repo = workspace_git_repo.unwrap();
let mut args: HashMap<String, serde_json::Value> = HashMap::new();
args.insert(
"repo_url_resource_path".to_string(),
json!(workspace_git_repo
.git_repo_resource_path
.strip_prefix("$res:")),
);
let commit_msg: String; // for now auto-generate a commit message
match obj.clone() {
DeployedObject::Script { path, .. } => {
args.insert("path".to_string(), json!(path.to_string()));
commit_msg = format!("Script '{}' deployed", path); // for now auto-generate a commit message
}
DeployedObject::Flow { path } => {
args.insert("path".to_string(), json!(path.to_string()));
commit_msg = format!("Flow '{}' deployed", path);
}
DeployedObject::App { path, .. } => {
args.insert("path".to_string(), json!(path.to_string()));
commit_msg = format!("App '{}' deployed", path);
}
}
args.insert("commit_msg".to_string(), json!(commit_msg));
let (job_uuid, mut new_tx) = windmill_queue::push(
&db,
tx,
&w_id,
JobPayload::DeploymentCallback { path: workspace_git_repo.script_path.clone() },
args,
&authed.username,
&authed.email,
username_to_permissioned_as(&authed.username),
None,
None,
None,
None,
None,
false,
false,
None,
true,
None,
None,
None,
None,
)
.await?;
// We're not persisting the default commit msg as it's pretty useless. We will persist the ones manually set by users
match obj.clone() {
DeployedObject::Script { path, hash, .. } => {
sqlx::query!(
"INSERT INTO deployment_metadata (workspace_id, path, script_hash, callback_job_ids) VALUES ($1, $2, $3, $4)",
w_id, path, hash.0, &[job_uuid]
)
},
DeployedObject::Flow { path } => {
sqlx::query!(
"INSERT INTO deployment_metadata (workspace_id, path, callback_job_ids) VALUES ($1, $2, $3) ON CONFLICT (workspace_id, path) WHERE script_hash IS NULL AND app_version IS NULL DO UPDATE SET callback_job_ids = $3",
w_id, path, &[job_uuid]
)
}
DeployedObject::App { path, version } => {
sqlx::query!(
"INSERT INTO deployment_metadata (workspace_id, path, app_version, callback_job_ids) VALUES ($1, $2, $3, $4)",
w_id, path, version, &[job_uuid]
)
}
}.execute(&mut new_tx)
.await?;
tx = PushIsolationLevel::Transaction(new_tx);
return Ok(tx);
}

1
backend/windmill-api/src/lib.rs
View File

@@ -53,6 +53,7 @@ pub mod embeddings;
mod favorite;
mod flows;
mod folders;
mod git_sync_helpers;
mod granular_acls;
mod groups;
mod inputs;

11
backend/windmill-api/src/scripts.rs
View File

@@ -8,6 +8,7 @@
use crate::{
db::{ApiAuthed, DB},
git_sync_helpers,
schedule::clear_schedule,
users::{maybe_refresh_folders, require_owner_of_path, AuthCache},
webhook_util::{WebhookMessage, WebhookShared},
@@ -310,6 +311,7 @@ async fn create_script(
));
}
let script_path = ns.path.clone();
let hash = ScriptHash(hash_script(&ns));
let authed = maybe_refresh_folders(&ns.path, &w_id, authed, &db).await;
let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.begin(&authed).await?).into();
@@ -636,6 +638,15 @@ async fn create_script(
tx = PushIsolationLevel::Transaction(new_tx);
}
tx = git_sync_helpers::run_workspace_repo_git_callback(
tx,
&authed,
&db,
&w_id,
git_sync_helpers::DeployedObject::Script { hash: hash, path: script_path },
)
.await?;
match tx {
PushIsolationLevel::Transaction(tx) => tx.commit().await?,
_ => {

71
backend/windmill-api/src/workspaces.rs
View File

@@ -84,6 +84,7 @@ pub fn workspaced_service() -> Router {
.route("/get_copilot_info", get(get_copilot_info) )
.route("/edit_error_handler", post(edit_error_handler))
.route("/edit_large_file_storage_config", post(edit_large_file_storage_config))
.route("/edit_git_sync_config", post(edit_git_sync_config))
.route("/leave", post(leave_workspace));
#[cfg(feature = "enterprise")]
@@ -151,6 +152,7 @@ pub struct WorkspaceSettings {
pub error_handler_extra_args: Option<serde_json::Value>,
pub error_handler_muted_on_cancel: Option<bool>,
pub large_file_storage: Option<serde_json::Value>, // effectively: DatasetsStorage
pub git_sync: Option<serde_json::Value>, // effectively: WorkspaceGitRepo
}
#[derive(FromRow, Serialize, Debug)]
@@ -979,7 +981,72 @@ async fn edit_large_file_storage_config(
}
tx.commit().await?;
Ok(format!("Edit copilot config for workspace {}", &w_id))
Ok(format!("Edit large file storage config for workspace {}", &w_id))
}
#[derive(Deserialize)]
struct EditGitSyncConfig {
git_sync_settings: Option<WorkspaceGitRepo>,
}
#[derive(Serialize, Deserialize, Debug)]
pub struct WorkspaceGitRepo {
pub script_path: String,
pub git_repo_resource_path: String,
}
async fn edit_git_sync_config(
authed: ApiAuthed,
Extension(db): Extension<DB>,
Path(w_id): Path<String>,
ApiAuthed { is_admin, username, .. }: ApiAuthed,
Json(new_config): Json<EditGitSyncConfig>,
) -> Result<String> {
#[cfg(not(feature = "enterprise"))]
{
return Err(Error::BadRequest(
"Git sync is only available on Windmill Enterprise Edition".to_string(),
));
}
require_admin(is_admin, &username)?;
let mut tx = db.begin().await?;
let args_for_audit = format!("{:?}", new_config.git_sync_settings);
audit_log(
&mut *tx,
&authed.username,
"workspaces.edit_git_sync_config",
ActionKind::Update,
&w_id,
Some(&authed.email),
Some([("git_sync_settings", args_for_audit.as_str())].into()),
)
.await?;
if let Some(git_sync_settings) = new_config.git_sync_settings {
let serialized_config = serde_json::to_value::<WorkspaceGitRepo>(git_sync_settings)
.map_err(|err| Error::InternalErr(err.to_string()))?;
sqlx::query!(
"UPDATE workspace_settings SET git_sync = $1 WHERE workspace_id = $2",
serialized_config,
&w_id
)
.execute(&mut *tx)
.await?;
} else {
sqlx::query!(
"UPDATE workspace_settings SET git_sync = NULL WHERE workspace_id = $1",
&w_id,
)
.execute(&mut *tx)
.await?;
}
tx.commit().await?;
Ok(format!("Edit git sync config for workspace {}", &w_id))
}
async fn edit_error_handler(
@@ -1114,7 +1181,7 @@ async fn _check_nb_of_workspaces(db: &DB) -> Result<()> {
.await?;
if nb_workspaces.unwrap_or(0) >= 2 {
return Err(Error::BadRequest(
"You have reached the maximum number of workspaces (2 outside of default worskapce 'admins') without an enterprise license. Archive/delete another workspace to create a new one"
"You have reached the maximum number of workspaces (2 outside of default workspace 'admins') without an enterprise license. Archive/delete another workspace to create a new one"
.to_string(),
));
}

4
backend/windmill-common/src/jobs.rs
View File

@@ -28,6 +28,7 @@ pub enum JobKind {
FlowDependencies,
AppDependencies,
Noop,
DeploymentCallback,
}
#[derive(sqlx::FromRow, Debug, Serialize, Clone)]
@@ -307,6 +308,9 @@ pub enum JobPayload {
path: Option<String>,
restarted_from: Option<RestartedFrom>,
},
DeploymentCallback {
path: String,
},
Identity,
Noop,
}

21
backend/windmill-queue/src/jobs.rs
View File

@@ -2502,6 +2502,20 @@ pub async fn push<'c, T: Serialize + Send + Sync, R: rsmq_async::RsmqConnection
priority,
)
}
JobPayload::DeploymentCallback { path } => (
None,
Some(path),
None,
JobKind::DeploymentCallback,
None,
None,
None,
None,
None,
None,
None,
None,
),
JobPayload::Identity => (
None,
None,
@@ -2600,7 +2614,11 @@ pub async fn push<'c, T: Serialize + Send + Sync, R: rsmq_async::RsmqConnection
|| job_kind == JobKind::Identity
{
"flow".to_string()
} else if job_kind == JobKind::Dependencies || job_kind == JobKind::FlowDependencies {
} else if job_kind == JobKind::Dependencies
|| job_kind == JobKind::FlowDependencies
|| job_kind == JobKind::DeploymentCallback
{
// using the dependency tag for deployment callback for now. We can create a separate tag when we need
"dependency".to_string()
} else {
"deno".to_string()
@@ -2712,6 +2730,7 @@ pub async fn push<'c, T: Serialize + Send + Sync, R: rsmq_async::RsmqConnection
JobKind::Noop => "jobs.run.noop",
JobKind::FlowDependencies => "jobs.run.flow_dependencies",
JobKind::AppDependencies => "jobs.run.app_dependencies",
JobKind::DeploymentCallback => "jobs.run.deployment_callback",
};
audit_log(

34
backend/windmill-worker/src/worker.rs
View File

@@ -30,6 +30,7 @@ use uuid::Uuid;
use windmill_common::{
error::{self, to_anyhow, Error},
flows::{FlowModule, FlowModuleValue, FlowValue},
get_latest_deployed_hash_for_path,
jobs::{JobKind, QueuedJob},
scripts::{get_full_hub_script_by_path, ScriptHash, ScriptLang},
users::{SUPERADMIN_NOTIFICATION_EMAIL, SUPERADMIN_SECRET_EMAIL},
@@ -2701,11 +2702,10 @@ struct ContentReqLangEnvs {
}
async fn get_hub_script_content_and_requirements(
job: &QueuedJob,
script_path: Option<String>,
db: &DB,
) -> error::Result<ContentReqLangEnvs> {
let script_path = job
.script_path
let script_path = script_path
.clone()
.ok_or_else(|| Error::InternalErr(format!("expected script path for hub script")))?;
let mut script_path_iterator = script_path.split("/");
@@ -2716,8 +2716,8 @@ async fn get_hub_script_content_and_requirements(
let cache_path = format!("{HUB_CACHE_DIR}/{version}");
let script;
if tokio::fs::metadata(&cache_path).await.is_err() {
script =
get_full_hub_script_by_path(StripPath(script_path.clone()), &HTTP_CLIENT, db).await?;
script = get_full_hub_script_by_path(StripPath(script_path.to_string()), &HTTP_CLIENT, db)
.await?;
write_file(
HUB_CACHE_DIR,
&version,
@@ -2738,6 +2738,23 @@ async fn get_hub_script_content_and_requirements(
})
}
async fn get_script_content_by_path(
script_path: Option<String>,
w_id: &str,
db: &DB,
) -> error::Result<ContentReqLangEnvs> {
let script_path = script_path
.clone()
.ok_or_else(|| Error::InternalErr(format!("expected script path")))?;
return if script_path.starts_with("hub/") {
get_hub_script_content_and_requirements(Some(script_path), db).await
} else {
let (script_hash, ..) =
get_latest_deployed_hash_for_path(db, w_id, script_path.as_str()).await?;
get_script_content_by_hash(&script_hash, w_id, db).await
};
}
async fn get_script_content_by_hash(
script_hash: &ScriptHash,
w_id: &str,
@@ -2786,7 +2803,9 @@ async fn handle_code_execution_job(
language: job.language.to_owned(),
envs: None,
},
JobKind::Script_Hub => get_hub_script_content_and_requirements(job, db).await?,
JobKind::Script_Hub => {
get_hub_script_content_and_requirements(job.script_path.clone(), db).await?
}
JobKind::Script => {
get_script_content_by_hash(
&job.script_hash.unwrap_or(ScriptHash(0)),
@@ -2795,6 +2814,9 @@ async fn handle_code_execution_job(
)
.await?
}
JobKind::DeploymentCallback => {
get_script_content_by_path(job.script_path.clone(), &job.workspace_id, db).await?
}
_ => unreachable!(
"handle_code_execution_job should never be reachable with a non-code execution job"
),

12
frontend/src/lib/components/runs/RunsFilter.svelte
View File

@@ -184,6 +184,12 @@
showTooltipIcon
tooltip="Deploying a script, flow or an app launch a dependency job that create and then attach the lockfile to the deployed item. This mechanism ensure that logic is always executed with the exact same direct and indirect dependencies."
/>
<ToggleButton
value="deploymentcallbacks"
label="Sync"
showTooltipIcon
tooltip="Sync jobs that are triggered on every script deployment to sync the workspace with the Git repository configured in the the workspace settings"
/>
</ToggleButtonGroup>
</div>
<div class="relative">
@@ -362,6 +368,12 @@
showTooltipIcon
tooltip="Deploying a script, flow or an app launch a dependency job that create and then attach the lockfile to the deployed item. This mechanism ensure that logic is always executed with the exact same direct and indirect dependencies."
/>
<ToggleButton
value="deploymentcallbacks"
label="Sync"
showTooltipIcon
tooltip="Sync jobs that are triggered on every script deployment to sync the workspace with the Git repository configured in the the workspace settings"
/>
</ToggleButtonGroup>
</Label>

4
frontend/src/routes/(root)/(logged)/runs/[...path]/+page.svelte
View File

@@ -110,11 +110,13 @@
function computeJobKinds(jobKindsCat: string | undefined): string {
if (jobKindsCat == 'all') {
return `${CompletedJob.job_kind.SCRIPT},${CompletedJob.job_kind.FLOW},${CompletedJob.job_kind.DEPENDENCIES},${CompletedJob.job_kind.FLOWDEPENDENCIES},${CompletedJob.job_kind.APPDEPENDENCIES},${CompletedJob.job_kind.PREVIEW},${CompletedJob.job_kind.FLOWPREVIEW},${CompletedJob.job_kind.SCRIPT_HUB}`
return `${CompletedJob.job_kind.SCRIPT},${CompletedJob.job_kind.FLOW},${CompletedJob.job_kind.DEPENDENCIES},${CompletedJob.job_kind.FLOWDEPENDENCIES},${CompletedJob.job_kind.APPDEPENDENCIES},${CompletedJob.job_kind.PREVIEW},${CompletedJob.job_kind.FLOWPREVIEW},${CompletedJob.job_kind.SCRIPT_HUB},${CompletedJob.job_kind.DEPLOYMENTCALLBACK}`
} else if (jobKindsCat == 'dependencies') {
return `${CompletedJob.job_kind.DEPENDENCIES},${CompletedJob.job_kind.FLOWDEPENDENCIES},${CompletedJob.job_kind.APPDEPENDENCIES}`
} else if (jobKindsCat == 'previews') {
return `${CompletedJob.job_kind.PREVIEW},${CompletedJob.job_kind.FLOWPREVIEW}`
} else if (jobKindsCat == 'deploymentcallbacks') {
return `${CompletedJob.job_kind.DEPLOYMENTCALLBACK}`
} else {
return `${CompletedJob.job_kind.SCRIPT},${CompletedJob.job_kind.FLOW}`
}

92
frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte
View File

@@ -54,6 +54,7 @@
let errorHandlerMutedOnCancel: boolean | undefined = undefined
let openaiResourceInitialPath: string | undefined = undefined
let s3ResourceInitialPath: string | undefined = undefined
let gitSyncResourceInitialPath: string | undefined = undefined
let codeCompletionEnabled: boolean = false
let tab =
($page.url.searchParams.get('tab') as
@@ -180,6 +181,31 @@
}
}
async function editWindmillGitSyncSettings(gitRepoResourcePath: string): Promise<void> {
gitSyncResourceInitialPath = gitRepoResourcePath
if (gitRepoResourcePath) {
let resourcePathWithPrefix = `$res:${gitRepoResourcePath}`
await WorkspaceService.editWorkspaceGitSyncConfig({
workspace: $workspaceStore!,
requestBody: {
git_sync_settings: {
script_path: 'hub/7835/sync-script-to-git-repo-windmill',
git_repo_resource_path: resourcePathWithPrefix
}
}
})
sendUserToast(`Workspace Git sync settings updated`)
} else {
await WorkspaceService.editWorkspaceGitSyncConfig({
workspace: $workspaceStore!,
requestBody: {
git_sync_settings: undefined
}
})
sendUserToast(`Workspace Git sync settings reset`)
}
}
async function loadSettings(): Promise<void> {
const settings = await WorkspaceService.getSettings({ workspace: $workspaceStore! })
team_name = settings.slack_name
@@ -214,6 +240,7 @@
settings.large_file_storage?.type === LargeFileStorage.type.S3STORAGE
? settings.large_file_storage?.s3_resource_path?.replace('$res:', '')
: undefined
gitSyncResourceInitialPath = settings.git_sync?.git_repo_resource_path?.replace('$res:', '')
}
$: {
@@ -309,6 +336,9 @@
<Tab size="xs" value="windmill_lfs">
<div class="flex gap-2 items-center my-1"> S3 Storage </div>
</Tab>
<Tab size="xs" value="git_sync">
<div class="flex gap-2 items-center my-1"> Git sync </div>
</Tab>
<Tab size="xs" value="export_delete">
<div class="flex gap-2 items-center my-1"> Delete Workspace </div>
</Tab>
@@ -625,6 +655,68 @@
}}>Test Connection</Button
>
</div>
{:else if tab == 'git_sync'}
<PageHeader
title="Git sync"
primary={false}
tooltip="Connect the Windmill workspace to a Git repository to automatically commit and push scripts, flows and apps to the repository on each deploy."
/>
{#if !$enterpriseLicense}
<Alert type="warning" title="Syncing workspace to Git is an EE feature">
Automatically saving scripts to a Git repository on each deploy is a Windmill EE feature.
</Alert>
{/if}
<Alert type="info" title="Script, flows and apps in the user private folders will be ignored">
All scripts, flows and apps located in the workspace will be pushed to the Git repository,
except the ones that are saved in private user folders (i.e. where the path starts with
`u/`).
</Alert>
<div class="mt-5 mb-5 flex gap-1">
{#key s3ResourceInitialPath}
<ResourcePicker
resourceType="git_repository"
initialValue={gitSyncResourceInitialPath}
on:change={(ev) => {
editWindmillGitSyncSettings(ev.detail)
}}
/>
{/key}
</div>
<div class="bg-surface-disabled p-4 rounded-md flex flex-col gap-1">
<div class="text-primary font-md font-semibold"> Git repository initial setup </div>
<div class="prose max-w-none text-2xs text-tertiary">
Every time a script is deployed, only the updated script will be pushed to the remote Git
repository.
<br />
For the git repo to be representative of the entire workspace, it is recommended to set it
up using the Windmill CLI before turning this option on.
<br /><br />
Not familiar with Windmill CLI?
<a href="https://www.windmill.dev/docs/advanced/cli">Check out the docs</a>
<br /><br />
Run the following commands from the git repo folder to push the initial workspace content
to the remote:
<br />
<pre class="overflow-auto max-h-screen"
><code
>> wmill workspace add WORKSPACE_NAME WORKSPACE_ID WINDMILL_URL
> wmill sync pull --raw --skip-variables --skip-secrets --skip-resources
> git add -A git commit -m 'Initial commit'
> git push</code
></pre
>
</div>
</div>
{/if}
{:else}
<div class="bg-red-100 border-l-4 border-red-600 text-orange-700 p-4 m-4" role="alert">