chore: SAML EE (#3176)

* Extract SAML logic into its own file

* Remove saml.rs core logic

* hello

* Add substitute_ee_code.sh and check_no_symlink.sh scripts

* dry-run docker image build

* test hook

* add setup-hooks.sh script

* Update pre-commit hook

* Update substitution script

* revert docker-image action yaml

* revert Cargo.lock

* publish custom image

* swap for ce build as well

* empty

* revert temp action override

* fix docker-image.yml
This commit is contained in:
Guillaume Bouvignies
2024-02-08 16:09:11 +01:00
committed by GitHub
parent de858f3cc6
commit ec6f533419
11 changed files with 242 additions and 209 deletions

14
.githooks/pre-commit Executable file
View File

@@ -0,0 +1,14 @@
#!/bin/sh
#
# This file is symlinked to local .git/hooks/pre-commit by the setup-hooks.sh script
# It wil run before every commit, so it needs to be quick and efficient. If it returns
# a non-zero exit code, the commit will be aborted.
echo "Running pre-commit hook"
# This checks that there is no symlinks in the backend directory among the EE files
./backend/check_no_symlink.sh > /dev/null
if [ $? -ne 0 ]; then
echo "/!\ Symlinks detected in the backend directory. Please run './backend/substitute_ee_code.sh --revert' before committing."
exit 1
fi

View File

@@ -26,6 +26,13 @@ jobs:
with:
fetch-depth: 0
- uses: actions/checkout@v3
with:
repository: windmill-labs/windmill-ee-private
path: ./windmill-ee-private
token: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }}
fetch-depth: 0
# - name: Set up Docker Buildx
# uses: docker/setup-buildx-action@v2
- uses: depot/setup-action@v1
@@ -37,6 +44,10 @@ jobs:
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Substitute EE code (EE logic is behind feature flag)
run: |
./backend/substitute_ee_code.sh --copy --dir ./windmill-ee-private
- name: Docker meta
id: meta-public
uses: docker/metadata-action@v4
@@ -69,9 +80,16 @@ jobs:
- uses: actions/checkout@v3
with:
fetch-depth: 0
- uses: actions/checkout@v3
with:
repository: windmill-labs/windmill-ee-private
path: ./windmill-ee-private
token: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }}
fetch-depth: 0
# - name: Set up Docker Buildx
# uses: docker/setup-buildx-action@v2
- uses: depot/setup-action@v1
- name: Docker meta
@@ -94,6 +112,10 @@ jobs:
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Substitute EE code
run: |
./backend/substitute_ee_code.sh --copy --dir ./windmill-ee-private
- name: Build and push publicly ee
uses: depot/build-push-action@v1
with:
@@ -143,7 +165,7 @@ jobs:
username: ${{ secrets.AWS_ACCESS_KEY_ID }}
password: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
- name: Build and push publicly ee
- name: Build and push publicly ee reports
uses: depot/build-push-action@v1
with:
context: .
@@ -393,6 +415,7 @@ jobs:
- uses: actions/checkout@v3
with:
fetch-depth: 0
# - name: Set up Docker Buildx
# uses: docker/setup-buildx-action@v2

49
backend/Cargo.lock generated
View File

@@ -2684,9 +2684,9 @@ dependencies = [
[[package]]
name = "gemm"
version = "0.17.0"
version = "0.17.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e97d506c68f4fb12325b52a638e7d54cc87e3593a4ded0de60218b6dfd65f645"
checksum = "6ab24cc62135b40090e31a76a9b2766a501979f3070fa27f689c27ec04377d32"
dependencies = [
"dyn-stack",
"gemm-c32",
@@ -2704,9 +2704,9 @@ dependencies = [
[[package]]
name = "gemm-c32"
version = "0.17.0"
version = "0.17.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0dd16f26e8f34661edc906d8c9522b59ec1655c865a98a58950d0246eeaca9da"
checksum = "b9c030d0b983d1e34a546b86e08f600c11696fde16199f971cd46c12e67512c0"
dependencies = [
"dyn-stack",
"gemm-common",
@@ -2719,9 +2719,9 @@ dependencies = [
[[package]]
name = "gemm-c64"
version = "0.17.0"
version = "0.17.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a8e34381bc060b47fbd25522a281799ef763cd27f43bbd1783d935774659242a"
checksum = "fbb5f2e79fefb9693d18e1066a557b4546cd334b226beadc68b11a8f9431852a"
dependencies = [
"dyn-stack",
"gemm-common",
@@ -2734,9 +2734,9 @@ dependencies = [
[[package]]
name = "gemm-common"
version = "0.17.0"
version = "0.17.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "22518a76339b09276f77c3166c44262e55f633712fe8a44fd0573505887feeab"
checksum = "a2e7ea062c987abcd8db95db917b4ffb4ecdfd0668471d8dc54734fdff2354e8"
dependencies = [
"bytemuck",
"dyn-stack",
@@ -2754,9 +2754,9 @@ dependencies = [
[[package]]
name = "gemm-f16"
version = "0.17.0"
version = "0.17.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "70409bbf3ef83b38cbe4a58cd4b797c1c27902505bdd926a588ea61b6c550a84"
checksum = "7ca4c06b9b11952071d317604acb332e924e817bd891bec8dfb494168c7cedd4"
dependencies = [
"dyn-stack",
"gemm-common",
@@ -2772,9 +2772,9 @@ dependencies = [
[[package]]
name = "gemm-f32"
version = "0.17.0"
version = "0.17.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5ea3068edca27f100964157211782eba19e961aa4d0d2bdac3e1775a51aa7680"
checksum = "e9a69f51aaefbd9cf12d18faf273d3e982d9d711f60775645ed5c8047b4ae113"
dependencies = [
"dyn-stack",
"gemm-common",
@@ -2787,9 +2787,9 @@ dependencies = [
[[package]]
name = "gemm-f64"
version = "0.17.0"
version = "0.17.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5fd41e8f5a60dce8d8acd852a3f4b22f8e18be957e1937731be692c037652510"
checksum = "aa397a48544fadf0b81ec8741e5c0fba0043008113f71f2034def1935645d2b0"
dependencies = [
"dyn-stack",
"gemm-common",
@@ -3409,9 +3409,9 @@ checksum = "9028f49264629065d057f340a86acb84867925865f73bbf8d47b4d149a7e88b8"
[[package]]
name = "jobserver"
version = "0.1.27"
version = "0.1.28"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8c37f63953c4c63420ed5fd3d6d398c719489b9f872b9fa683262f8edd363c7d"
checksum = "ab46a6e9526ddef3ae7f787c06f0f2600639ba80ea3eade3d8e670a2230f51d6"
dependencies = [
"libc",
]
@@ -4178,19 +4178,18 @@ checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9"
[[package]]
name = "num-integer"
version = "0.1.45"
version = "0.1.46"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9"
checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f"
dependencies = [
"autocfg",
"num-traits",
]
[[package]]
name = "num-iter"
version = "0.1.43"
version = "0.1.44"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7d03e6c028c5dc5cac6e2dec0efda81fc887605bb3d884578bb6d6bf7514e252"
checksum = "d869c01cc0c455284163fd0092f1f93835385ccab5a98a0dcc497b2f8bf055a9"
dependencies = [
"autocfg",
"num-integer",
@@ -4199,9 +4198,9 @@ dependencies = [
[[package]]
name = "num-traits"
version = "0.2.17"
version = "0.2.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c"
checksum = "da0df0e5185db44f69b44f26786fe401b6c293d1907744beaa7fa62b2e5a517a"
dependencies = [
"autocfg",
"libm",
@@ -5521,9 +5520,9 @@ dependencies = [
[[package]]
name = "pulp"
version = "0.18.6"
version = "0.18.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "16785ee69419641c75affff7c9fdbdb7c0ab26dc9a5fb5218c2a2e9e4ef2087d"
checksum = "b1618ee89537c2b388d62ac260e124be07c20c2d9f531787a62c4528c485d46c"
dependencies = [
"bytemuck",
"libm",

View File

@@ -198,6 +198,8 @@ tokio-postgres = {version = "^0.7", features = ["array-impls", "with-serde_json-
mysql_async = { version = "*", default-features = false, features = ["minimal", "default", "native-tls-tls"]}
postgres-native-tls = "^0"
native-tls = "^0"
# samael will break compilation on MacOS. Use this fork instead to make it work
# samael = { git="https://github.com/gbouv/samael", rev="2344211ed0ac041a86222b38b928adfc1030cb94", features = ["xmlsec"] }
samael = { version="0.0.14", features = ["xmlsec"] }
gcp_auth = "0.9.0"
rust_decimal = { version = "^1", features = ["db-postgres"]}

47
backend/check_no_symlink.sh Executable file
View File

@@ -0,0 +1,47 @@
#!/bin/bash
set -euo pipefail
script_dirpath="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
root_dirpath="$(cd "${script_dirpath}/.." && pwd)"
EE_CODE_DIR="../windmill-ee-private/"
while [[ $# -gt 0 ]]; do
case $1 in
-d|--dir)
EE_CODE_DIR="$2"
shift # past argument
shift # past value
;;
-*|--*)
echo "Unknown option $1"
exit 1
;;
*)
POSITIONAL_ARGS+=("$1") # save positional arg
shift # past argument
;;
esac
done
if [[ $EE_CODE_DIR == /* ]]; then
EE_CODE_DIR="${EE_CODE_DIR}"
else
EE_CODE_DIR="${root_dirpath}/${EE_CODE_DIR}"
fi
echo "EE code directory = ${EE_CODE_DIR}"
if [ ! -d "${EE_CODE_DIR}" ]; then
echo "Windmill EE repo not found, nothing to do"
exit 0
fi
for ee_file in $(find "${EE_CODE_DIR}" -name "*.rs"); do
ce_file="${ee_file/${EE_CODE_DIR}/.}"
ce_file="${root_dirpath}/backend/${ce_file}"
echo "Checking if '${ce_file}' is a symlink"
if [[ -L "${ce_file}" ]]; then
echo "File ${ce_file} is a symlink, cannot commit symlinks"
exit 1
fi
done
echo "All good!"

86
backend/substitute_ee_code.sh Executable file
View File

@@ -0,0 +1,86 @@
#!/bin/bash
set -euo pipefail
script_dirpath="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
root_dirpath="$(cd "${script_dirpath}/.." && pwd)"
REVERT="NO"
COPY="NO"
EE_CODE_DIR="../windmill-ee-private/"
while [[ $# -gt 0 ]]; do
case $1 in
-r|--revert)
# If EE files have been substituted, this will revert them to their initial content.
# This relies on `git restore` so the EE files must not be committed to the repo for
# this to work (commit hooks should prevent this from happening, as well as the fact
# that we're using symlinks by default).
REVERT="YES"
shift
;;
-c|--copy)
# By default, EE files are symlinked. Pass this option to do a real copy instead.
# This might be necessary if you want to build the Docker Image as Docker COPY seems
# to not follow symlinks. For local development, symlinks should be preferred as they
# adds a safeguards EE files can't be commited to the OSS repo.
COPY="YES"
shift # past argument
;;
-d|--dir)
# Path to the local directory of the windmill-ee-private repository. By defaults, it
# assumes it is cloned next to the Windmill OSS repo.
EE_CODE_DIR="$2"
shift # past argument
shift # past value
;;
-*|--*)
echo "Unknown option $1"
exit 1
;;
*)
POSITIONAL_ARGS+=("$1") # save positional arg
shift # past argument
;;
esac
done
if [[ $EE_CODE_DIR == /* ]]; then
EE_CODE_DIR="${EE_CODE_DIR}"
else
EE_CODE_DIR="${root_dirpath}/${EE_CODE_DIR}"
fi
echo "EE code directory = ${EE_CODE_DIR} | Revert = ${REVERT}"
if [ ! -d "${EE_CODE_DIR}" ]; then
echo "Windmill EE repo not found, please clone it next to this repository (or use the --dir option) and try again"
echo "> git clone git@github.com:windmill-labs/windmill-ee-private.git"
echo ""
exit 0
fi
if [ "$REVERT" == "YES" ]; then
for ee_file in $(find ${EE_CODE_DIR} -name "*.rs"); do
ce_file="${ee_file/${EE_CODE_DIR}/.}"
ce_file="${root_dirpath}/backend/${ce_file}"
git restore --staged ${ce_file} || true
git restore ${ce_file} || true
done
else
# This replaces all files in current repo with alternative EE files in windmill-ee-private
for ee_file in $(find "${EE_CODE_DIR}" -name "*.rs"); do
ce_file="${ee_file/${EE_CODE_DIR}/.}"
ce_file="${root_dirpath}/backend/${ce_file}"
if [[ -f "${ce_file}" ]]; then
rm "${ce_file}"
if [ "$COPY" == "YES" ]; then
cp "${ee_file}" "${ce_file}"
echo "File copied '${ee_file}' -->> '${ce_file}'"
else
ln -s "${ee_file}" "${ce_file}"
echo "Symlink created '${ee_file}' -->> '${ce_file}'"
fi
else
echo "File ${ce_file} is not a file, ignoring"
fi
done
fi

View File

@@ -9,7 +9,6 @@
use crate::db::ApiAuthed;
use crate::embeddings::load_embeddings_db;
use crate::oauth2::AllClients;
use crate::saml::ServiceProviderExt;
use crate::scim::has_scim_token;
use crate::tracing_init::MyOnFailure;
use crate::{
@@ -36,8 +35,6 @@ use tower_http::{
trace::TraceLayer,
};
use windmill_common::db::UserDB;
#[cfg(feature = "enterprise_saml")]
use windmill_common::ee::{get_license_plan, LicensePlan};
use windmill_common::utils::rd_string;
use windmill_common::worker::ALL_TAGS;
use windmill_common::BASE_URL;
@@ -69,7 +66,7 @@ mod oidc;
mod openai;
mod raw_apps;
mod resources;
mod saml;
mod saml_ee;
mod schedule;
mod scim;
mod scripts;
@@ -163,16 +160,7 @@ pub async fn run_server(
.allow_headers([http::header::CONTENT_TYPE, http::header::AUTHORIZATION])
.allow_origin(Any);
#[cfg(feature = "enterprise_saml")]
let sp_extension: ServiceProviderExt = match get_license_plan().await {
LicensePlan::Enterprise => saml::build_sp_extension().await?,
LicensePlan::Pro => ServiceProviderExt(None),
};
#[cfg(not(feature = "enterprise_saml"))]
let sp_extension = ServiceProviderExt();
let sp_extension_arc = Arc::new(sp_extension);
let sp_extension = Arc::new(saml_ee::build_sp_extension().await?);
let embeddings_db = if server_mode {
Some(load_embeddings_db(&db))
@@ -244,7 +232,7 @@ pub async fn run_server(
.nest("/oidc", oidc::global_service())
.nest(
"/saml",
saml::global_service().layer(Extension(Arc::clone(&sp_extension_arc))),
saml_ee::global_service().layer(Extension(Arc::clone(&sp_extension))),
)
.nest(
"/scim",
@@ -276,7 +264,7 @@ pub async fn run_server(
)
.nest(
"/oauth",
oauth2::global_service().layer(Extension(Arc::clone(&sp_extension_arc))),
oauth2::global_service().layer(Extension(Arc::clone(&sp_extension))),
)
.route("/version", get(git_v))
.route("/uptodate", get(is_up_to_date))

View File

@@ -41,7 +41,7 @@ use windmill_common::utils::{not_found_if_none, now_from_db};
use windmill_common::variables::build_crypt;
use crate::db::ApiAuthed;
use crate::saml::{generate_redirect_url, ServiceProviderExt};
use crate::saml_ee::{generate_redirect_url, ServiceProviderExt};
use crate::users::{login_externally, LoginUserInfo};
use crate::webhook_util::{InstanceEvent, WebhookShared};
use crate::{db::DB, variables::encrypt, workspaces::WorkspaceSettings};

View File

@@ -1,164 +0,0 @@
/*
* Author: Ruben Fiszel
* Copyright: Windmill Labs, Inc 2023
* This file and its contents are licensed under the AGPLv3 License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
#![allow(non_snake_case)]
#[cfg(feature = "enterprise_saml")]
use axum::response::Redirect;
use axum::{routing::post, Router};
#[cfg(feature = "enterprise_saml")]
use axum::{Extension, Form};
use std::sync::Arc;
#[cfg(feature = "enterprise_saml")]
use samael::metadata::{ContactPerson, ContactType, EntityDescriptor};
#[cfg(feature = "enterprise_saml")]
use samael::service_provider::{ServiceProvider, ServiceProviderBuilder};
use serde::Deserialize;
#[cfg(feature = "enterprise_saml")]
use tower_cookies::Cookies;
#[cfg(feature = "enterprise_saml")]
use windmill_common::error::{Error, Result};
#[cfg(feature = "enterprise_saml")]
use crate::db::DB;
#[cfg(feature = "enterprise_saml")]
use crate::users::login_externally;
#[cfg(feature = "enterprise_saml")]
use crate::BASE_URL;
#[cfg(feature = "enterprise_saml")]
#[derive(Clone)]
pub struct ServiceProviderExt(pub Option<ServiceProvider>);
#[cfg(not(feature = "enterprise_saml"))]
pub struct ServiceProviderExt();
#[cfg(feature = "enterprise_saml")]
use windmill_common::ee::{get_license_plan, LicensePlan};
#[cfg(feature = "enterprise_saml")]
pub async fn build_sp_extension() -> anyhow::Result<ServiceProviderExt> {
if let Some(url_metadata) = std::env::var("SAML_METADATA").ok() {
//todo restrict for non ee
let resp = reqwest::get(url_metadata).await?.text().await?;
let idp_metadata: EntityDescriptor = samael::metadata::de::from_str(&resp)?;
// let pub_key = openssl::x509::X509::from_pem("")?;
// let private_key = openssl::rsa::Rsa::private_key_from_pem("")?;
let acs_url = format!("{}/api/saml/acs", BASE_URL.read().await.clone());
let sp = ServiceProviderBuilder::default()
.entity_id("windmill".to_string())
// .key(private_key)
// .certificate(pub_key)
.allow_idp_initiated(true)
.contact_person(ContactPerson {
sur_name: Some("Ruben Fiszel <ruben@windmill.dev>".to_string()),
contact_type: Some(ContactType::Technical.value().to_string()),
..ContactPerson::default()
})
.idp_metadata(idp_metadata)
.acs_url(acs_url.clone())
.build()?;
tracing::info!("SAML Configured - ACS url is {}", acs_url);
Ok(ServiceProviderExt(Some(sp)))
} else {
Ok(ServiceProviderExt(None))
}
}
#[cfg(not(feature = "enterprise_saml"))]
pub async fn generate_redirect_url(
_service_provider: Arc<ServiceProviderExt>,
) -> anyhow::Result<Option<String>> {
return Ok(None);
}
#[cfg(feature = "enterprise_saml")]
pub async fn generate_redirect_url(
service_provider: Arc<ServiceProviderExt>,
) -> anyhow::Result<Option<String>> {
if let Some(sp) = &service_provider.0 {
let url = sp
.idp_metadata
.idp_sso_descriptors
.clone()
.unwrap_or_default()
.get(0)
.and_then(|x| x.single_sign_on_services.get(0).map(|x| x.location.clone()));
let authn_req = sp
.make_authentication_request(url.unwrap_or_default().as_str())
.map_err(|e| anyhow::anyhow!(e.to_string()))?;
let redirect_url = authn_req
.redirect(BASE_URL.read().await.clone().as_str())
.map_err(|e| anyhow::anyhow!(e.to_string()))?
.map(|u| u.to_string());
tracing::debug!(
"SAML Configured, sso login link at: {:?}",
redirect_url.clone()
);
Ok(redirect_url)
} else {
Ok(None)
}
}
pub fn global_service() -> Router {
Router::new().route("/acs", post(acs))
}
#[derive(Deserialize)]
pub struct SamlForm {
pub SAMLResponse: Option<String>,
}
#[cfg(feature = "enterprise_saml")]
pub async fn acs(
Extension(db): Extension<DB>,
cookies: Cookies,
Extension(se): Extension<Arc<ServiceProviderExt>>,
Form(s): Form<SamlForm>,
) -> Result<Redirect> {
if matches!(get_license_plan().await, LicensePlan::Pro) {
return Err(Error::BadRequest(
"SAML not available in the pro plan".to_string(),
));
};
if let Some(sp_m) = &se.0 {
let sp = sp_m.clone();
if let Some(encoded_resp) = s.SAMLResponse {
tracing::info!("{:?}", encoded_resp);
let t = sp.parse_base64_response(&encoded_resp, None).map_err(|e| {
Error::BadRequest(format!("Error parsing acs request as base64: {e:?}"))
})?;
if let Some(email) = t.subject.and_then(|x| x.name_id.map(|x| x.value)) {
login_externally(db, &email, "saml".to_string(), cookies, None, None).await?;
Ok(Redirect::to("/"))
} else {
Err(Error::BadRequest(
"email not found in saml response".to_string(),
))
}
} else {
Err(Error::BadRequest("SAMLResponse not found".to_string()))
}
} else {
Err(Error::BadConfig("SAML not configured".to_string()))
}
}
#[cfg(not(feature = "enterprise_saml"))]
pub async fn acs() -> String {
"SAML available only in enterprise version".to_string()
}

View File

@@ -0,0 +1,33 @@
/*
* Author: Ruben Fiszel
* Copyright: Windmill Labs, Inc 2023
* This file and its contents are licensed under the AGPLv3 License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
#![allow(non_snake_case)]
use axum::{routing::post, Router};
use std::sync::Arc;
pub struct ServiceProviderExt();
pub async fn build_sp_extension() -> anyhow::Result<ServiceProviderExt> {
return Ok(ServiceProviderExt());
}
pub async fn generate_redirect_url(
_service_provider: Arc<ServiceProviderExt>,
) -> anyhow::Result<Option<String>> {
// Implementation is not open source as it is a Windmill Enterprise Edition feature
return Ok(None);
}
pub fn global_service() -> Router {
Router::new().route("/acs", post(acs))
}
pub async fn acs() -> String {
// Implementation is not open source as it is a Windmill Enterprise Edition feature
"SAML available only in enterprise version".to_string()
}

5
setup-hooks.sh Executable file
View File

@@ -0,0 +1,5 @@
#!/bin/bash
set -euo pipefail
root_dirpath="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ln -s -f ../../.githooks/pre-commit ./.git/hooks