feat: oidc support for sqs trigger (#5614)

* feat: oidc auth with sqs

* rafctor

* update: add id token struct

* update has_expired function

* fix duration

* fix typo

* Update backend/windmill-common/src/auth.rs

Co-authored-by: graphite-app[bot] <96075541+graphite-app[bot]@users.noreply.github.com>

* Update backend/windmill-common/src/auth.rs

Co-authored-by: ellipsis-dev[bot] <65095814+ellipsis-dev[bot]@users.noreply.github.com>

* update ref

* update sqlx

* update resource type

* update repo ref

* Update settings.json

---------

Co-authored-by: graphite-app[bot] <96075541+graphite-app[bot]@users.noreply.github.com>
Co-authored-by: ellipsis-dev[bot] <65095814+ellipsis-dev[bot]@users.noreply.github.com>
This commit is contained in:
dieriba
2025-04-17 15:36:17 +02:00
committed by GitHub
parent 6a9e4b77d9
commit b1db3c7dc6
25 changed files with 160 additions and 528 deletions

View File

@@ -1,22 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT suspend > 0 AS \"r!\" FROM v2_job_queue WHERE id = $1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "r!",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Uuid"
]
},
"nullable": [
null
]
},
"hash": "12828c9b2964f2b484a68de1e01b65cdcd277257192ee0a6d18a00f41bce49d4"
}

View File

@@ -1,22 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT script_path FROM v2_as_completed_job WHERE id = $1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "script_path",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Uuid"
]
},
"nullable": [
true
]
},
"hash": "280a361076d1c6317610765960f543252891c53351bdc98da66cc30ffc895866"
}

View File

@@ -1,79 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT \n j.id,\n j.kind AS \"kind: _\",\n COALESCE(s.path, f.path) AS \"script_path!\",\n COALESCE(s.hash, f.id) AS \"script_hash!: _\",\n COALESCE(jc.started_at, jq.scheduled_for, make_date(1970, 1, 1)) AS \"scheduled_for!: _\",\n args AS input\n FROM v2_job j\n LEFT JOIN script s ON j.runnable_id = s.hash AND j.kind = 'script'\n LEFT JOIN flow_version f ON j.runnable_id = f.id AND j.runnable_path = f.path AND j.kind = 'flow'\n LEFT JOIN v2_job_completed jc ON jc.id = j.id\n LEFT JOIN v2_job_queue jq ON jq.id = j.id\n WHERE j.id = ANY($1)\n AND j.workspace_id = $2\n AND COALESCE(s.hash, f.id) IS NOT NULL\n AND COALESCE(s.path, f.path) IS NOT NULL",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Uuid"
},
{
"ordinal": 1,
"name": "kind: _",
"type_info": {
"Custom": {
"name": "job_kind",
"kind": {
"Enum": [
"script",
"preview",
"flow",
"dependencies",
"flowpreview",
"script_hub",
"identity",
"flowdependencies",
"http",
"graphql",
"postgresql",
"noop",
"appdependencies",
"deploymentcallback",
"singlescriptflow",
"flowscript",
"flownode",
"appscript"
]
}
}
}
},
{
"ordinal": 2,
"name": "script_path!",
"type_info": "Varchar"
},
{
"ordinal": 3,
"name": "script_hash!: _",
"type_info": "Int8"
},
{
"ordinal": 4,
"name": "scheduled_for!: _",
"type_info": "Timestamptz"
},
{
"ordinal": 5,
"name": "input",
"type_info": "Jsonb"
}
],
"parameters": {
"Left": [
"UuidArray",
"Text"
]
},
"nullable": [
false,
false,
null,
null,
null,
true
]
},
"hash": "3376b42d273c2499f3517c7754f4cdb6eae2e71e56bc8e5215559af967c81bd8"
}

View File

@@ -0,0 +1,14 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO global_settings (name, value)\n VALUES ('rsa_keys', $1)\n ON CONFLICT (name) DO UPDATE\n SET value = EXCLUDED.value\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Jsonb"
]
},
"nullable": []
},
"hash": "65e6aa50db7cb878ce456d6665a4809e7b7f2c1b0d53b10da3b5578d337e28ec"
}

View File

@@ -1,14 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO global_settings (name, value) VALUES ('rsa_keys', $1)",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Jsonb"
]
},
"nullable": []
},
"hash": "85c116da4a43a5ade37bb9ab2660d206b722ec4192368599c3a5027f50a89c80"
}

View File

@@ -1,22 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT workflow_as_code_status FROM v2_job_completed WHERE id = $1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "workflow_as_code_status",
"type_info": "Jsonb"
}
],
"parameters": {
"Left": [
"Uuid"
]
},
"nullable": [
true
]
},
"hash": "867d5c75ddc6c5d20136880c7294844b4c1a38701190795a801fa43c74a0beeb"
}

View File

@@ -1,22 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT id FROM v2_job WHERE parent_job = $1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Uuid"
}
],
"parameters": {
"Left": [
"Uuid"
]
},
"nullable": [
false
]
},
"hash": "99f74bf675120daf965e063e5eaff808ba646f4f99d0c8837097e747b481f03a"
}

View File

@@ -1,22 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT labels FROM v2_job WHERE id = $1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "labels",
"type_info": "TextArray"
}
],
"parameters": {
"Left": [
"Uuid"
]
},
"nullable": [
true
]
},
"hash": "9d518842a9ad90ff9c28dc39690deb0ee6b62cf1d8ae1a02b28c23255d377b3d"
}

View File

@@ -1,20 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT count(*) AS \"count!\" FROM resume_job",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "count!",
"type_info": "Int8"
}
],
"parameters": {
"Left": []
},
"nullable": [
null
]
},
"hash": "a0c35cb515a842067b294343c90f1bfbe4e2db85da9a478a07460733999e9beb"
}

View File

@@ -1,28 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT preprocessed, script_entrypoint_override FROM v2_job WHERE id = $1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "preprocessed",
"type_info": "Bool"
},
{
"ordinal": 1,
"name": "script_entrypoint_override",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Uuid"
]
},
"nullable": [
true,
true
]
},
"hash": "a56eef5f5ecbe1a8d309ff65d9a8c456a3c165f7f2a107cf7fa6a4cdd30d55c0"
}

View File

@@ -1,73 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT\n j.kind AS \"kind!: JobKind\",\n j.runnable_id AS \"script_hash: _\",\n j.runnable_path AS script_path,\n COUNT(*) AS \"count!\",\n ANY_VALUE(COALESCE(f.schema, s.schema)) AS schema\n FROM v2_job j\n LEFT JOIN script s ON s.hash = j.runnable_id AND j.kind = 'script'\n LEFT JOIN flow_version f ON f.id = j.runnable_id AND f.path = j.runnable_path AND j.kind = 'flow'\n WHERE COALESCE(s.hash, f.id) IS NOT NULL\n AND COALESCE(s.path, f.path) IS NOT NULL\n AND j.workspace_id = $1 AND j.id = ANY($2)\n GROUP BY j.runnable_id, j.runnable_path, j.kind",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "kind!: JobKind",
"type_info": {
"Custom": {
"name": "job_kind",
"kind": {
"Enum": [
"script",
"preview",
"flow",
"dependencies",
"flowpreview",
"script_hub",
"identity",
"flowdependencies",
"http",
"graphql",
"postgresql",
"noop",
"appdependencies",
"deploymentcallback",
"singlescriptflow",
"flowscript",
"flownode",
"appscript"
]
}
}
}
},
{
"ordinal": 1,
"name": "script_hash: _",
"type_info": "Int8"
},
{
"ordinal": 2,
"name": "script_path",
"type_info": "Varchar"
},
{
"ordinal": 3,
"name": "count!",
"type_info": "Int8"
},
{
"ordinal": 4,
"name": "schema",
"type_info": "Json"
}
],
"parameters": {
"Left": [
"Text",
"UuidArray"
]
},
"nullable": [
false,
true,
true,
null,
null
]
},
"hash": "b61a00e6a8ca0a1d24e64fdc9223b077c957da8455ea747b0680923d8282425b"
}

View File

@@ -1,15 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE v2_job SET labels = $2 WHERE id = $1 AND $2::TEXT[] IS NOT NULL",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Uuid",
"TextArray"
]
},
"nullable": []
},
"hash": "bd5a0c06e2f2361c9fc670eb0b975b58d65ca93d68b29124d04bd526239b9df2"
}

View File

@@ -1,73 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT DISTINCT ON (j.runnable_path, j.kind) \n j.kind AS \"kind!: JobKind\",\n j.runnable_path AS script_path,\n NULL as \"script_hash: _\",\n -1::bigint as \"count!: _\",\n COALESCE(f.schema, s.schema) AS schema\n FROM v2_job j\n LEFT JOIN script s ON s.path = j.runnable_path AND j.kind = 'script'\n LEFT JOIN flow_version f ON f.path = j.runnable_path AND j.kind = 'flow'\n WHERE COALESCE(s.hash, f.id) IS NOT NULL\n AND j.workspace_id = $1 AND j.id = ANY($2)\n ORDER BY j.runnable_path, j.kind, COALESCE(f.created_at, s.created_at) DESC",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "kind!: JobKind",
"type_info": {
"Custom": {
"name": "job_kind",
"kind": {
"Enum": [
"script",
"preview",
"flow",
"dependencies",
"flowpreview",
"script_hub",
"identity",
"flowdependencies",
"http",
"graphql",
"postgresql",
"noop",
"appdependencies",
"deploymentcallback",
"singlescriptflow",
"flowscript",
"flownode",
"appscript"
]
}
}
}
},
{
"ordinal": 1,
"name": "script_path",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "script_hash: _",
"type_info": "Text"
},
{
"ordinal": 3,
"name": "count!: _",
"type_info": "Int8"
},
{
"ordinal": 4,
"name": "schema",
"type_info": "Json"
}
],
"parameters": {
"Left": [
"Text",
"UuidArray"
]
},
"nullable": [
false,
true,
null,
null,
null
]
},
"hash": "c4f382045e5c47986e02f1e57667e6b3cb1d60f15fed9a24e3f05b1bd60c0fc8"
}

View File

@@ -1,67 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT j.kind AS \"kind: _\", COALESCE(s.path, f.path) AS \"script_path!\", COALESCE(s.hash, f.id) AS \"script_hash!: _\", args\n FROM v2_job j\n LEFT JOIN script s ON j.runnable_id = s.hash AND j.kind = 'script'\n LEFT JOIN flow_version f ON j.runnable_id = f.id AND j.runnable_path = f.path AND j.kind = 'flow'\n WHERE j.id = ANY($1)\n AND j.workspace_id = $2\n AND COALESCE(s.hash, f.id) IS NOT NULL\n AND COALESCE(s.path, f.path) IS NOT NULL",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "kind: _",
"type_info": {
"Custom": {
"name": "job_kind",
"kind": {
"Enum": [
"script",
"preview",
"flow",
"dependencies",
"flowpreview",
"script_hub",
"identity",
"flowdependencies",
"http",
"graphql",
"postgresql",
"noop",
"appdependencies",
"deploymentcallback",
"singlescriptflow",
"flowscript",
"flownode",
"appscript"
]
}
}
}
},
{
"ordinal": 1,
"name": "script_path!",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "script_hash!: _",
"type_info": "Int8"
},
{
"ordinal": 3,
"name": "args",
"type_info": "Jsonb"
}
],
"parameters": {
"Left": [
"UuidArray",
"Text"
]
},
"nullable": [
false,
null,
null,
true
]
},
"hash": "ccd4c7fe5fbdf1ab4bbbbd7d2a9b1ec3fc844ed286ff563f0c9a5c4778cfb718"
}

View File

@@ -1,22 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT preprocessed FROM v2_job WHERE id = $1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "preprocessed",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Uuid"
]
},
"nullable": [
true
]
},
"hash": "cd5f02cf10cbf92dd1df53a54f2110efa11a7731ad0f0e5509f55efabdf535cd"
}

View File

@@ -15,7 +15,7 @@
]
},
"nullable": [
null
true
]
},
"hash": "ddf2eccb78a310ed00c7d8b9c3f05d394a7cbcf0038c72a78add5c7b02ef5927"

View File

@@ -1,22 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT hash FROM script WHERE path = $1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "hash",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
false
]
},
"hash": "ef6795d93423f98eea82eb18e6332580dc7f7a9e5a67026f8c0b3077f371fc62"
}

View File

@@ -11,5 +11,5 @@
"remote.autoForwardPorts": true,
"conventionalCommits.scopes": [
"restructring triggers, decoding trigger message on work"
],
]
}

1
backend/Cargo.lock generated
View File

@@ -13970,6 +13970,7 @@ dependencies = [
"async_zip",
"aws-config",
"aws-sdk-sqs",
"aws-sdk-sts",
"axum",
"base32",
"base64 0.22.1",

View File

@@ -73,7 +73,7 @@ websocket = ["windmill-api/websocket"]
http_trigger = ["windmill-api/http_trigger"]
postgres_trigger = ["windmill-api/postgres_trigger"]
mqtt_trigger = ["windmill-api/mqtt_trigger"]
sqs_trigger = ["windmill-api/sqs_trigger"]
sqs_trigger = ["windmill-api/sqs_trigger", "windmill-common/aws_auth", "windmill-api/openidconnect"]
gcp_trigger = ["windmill-api/gcp_trigger"]
smtp = ["windmill-api/smtp", "windmill-common/smtp"]
license = ["windmill-api/license"]

View File

@@ -1 +1 @@
1475a504133766e005ec551004c2af2283663352
29a22d49c710bf21abcfe0e4c081e7eace4ac21e

View File

@@ -133,6 +133,7 @@ rust-postgres-native-tls = { workspace = true, optional = true}
rumqttc = { workspace = true, optional = true }
aws-sdk-sqs = { workspace = true, optional = true }
aws-config = { workspace = true, optional = true }
aws-sdk-sts = { workspace = true, optional = true }
google-cloud-pubsub = { workspace = true, optional = true }
google-cloud-googleapis = { workspace = true , optional = true }
tonic = { workspace = true, optional = true }

View File

@@ -13,6 +13,7 @@ prometheus = ["dep:prometheus"]
loki = ["dep:tracing-loki"]
benchmark = []
parquet = ["dep:object_store", "dep:aws-config", "dep:aws-sdk-sts"]
aws_auth = ["dep:aws-sdk-sts", "dep:aws-config"]
otel = ["dep:opentelemetry-semantic-conventions", "dep:opentelemetry-otlp", "dep:opentelemetry_sdk",
"dep:opentelemetry", "dep:tracing-opentelemetry", "dep:opentelemetry-appender-tracing", "dep:tonic"]
smtp = ["dep:mail-send"]

View File

@@ -1,4 +1,5 @@
use anyhow::Context;
use chrono::{DateTime, Duration, Utc};
use serde::{Deserialize, Serialize};
use uuid::Uuid;
@@ -10,6 +11,48 @@ use crate::{
DB,
};
#[derive(Debug)]
pub struct IdToken {
token: String,
expiration: DateTime<Utc>,
}
pub fn has_expired(expiration_time: DateTime<Utc>, take: Option<Duration>) -> bool {
let now = Utc::now();
let expiration = match take {
Some(duration) => expiration_time - duration,
None => expiration_time,
};
now > expiration
}
impl From<IdToken> for String {
fn from(value: IdToken) -> Self {
value.token
}
}
impl ToString for IdToken {
fn to_string(&self) -> String {
self.token.clone()
}
}
impl IdToken {
pub fn new(token: String, expiration: DateTime<Utc>) -> Self {
Self { token, expiration }
}
pub fn token(&self) -> &str {
&self.token
}
pub fn expiration(&self) -> &DateTime<Utc> {
&self.expiration
}
}
#[derive(Deserialize, Serialize)]
pub struct JWTAuthClaims {
pub email: String,
@@ -267,3 +310,98 @@ pub async fn create_token_for_owner(
Ok(format!("jwt_{}", token))
}
#[cfg(feature = "aws_auth")]
pub mod aws {
use crate::error::to_anyhow;
use super::*;
use crate::utils::empty_string_as_none;
use aws_config::{BehaviorVersion, Region};
use aws_sdk_sts::{
config::Credentials as AwsCredentials,
operation::{
assume_role_with_saml::AssumeRoleWithSamlOutput,
assume_role_with_web_identity::AssumeRoleWithWebIdentityOutput,
},
types::Credentials,
Client,
};
pub const AWS_OIDC_AUDIENCE: &'static str = "sts.amazonaws.com";
pub trait GetAuthenticationOutput {
fn get_credentials(&self) -> Result<&Credentials>;
}
impl GetAuthenticationOutput for AssumeRoleWithSamlOutput {
fn get_credentials(&self) -> Result<&Credentials> {
let credentials = self.credentials.as_ref().ok_or(Error::BadGateway(
"Error fetching credentials from AWS STS".to_string(),
))?;
Ok(credentials)
}
}
impl GetAuthenticationOutput for AssumeRoleWithWebIdentityOutput {
fn get_credentials(&self) -> Result<&Credentials> {
let credentials = self.credentials.as_ref().ok_or(Error::BadGateway(
"Error fetching credentials from AWS STS".to_string(),
))?;
Ok(credentials)
}
}
#[derive(Debug, Deserialize)]
pub struct CredentialsAuth {
#[serde(deserialize_with = "empty_string_as_none")]
pub region: Option<String>,
pub access_key_id: String,
pub secret_access_key: String,
}
#[derive(Clone, Debug, Deserialize)]
pub struct OidcAuth {
#[serde(deserialize_with = "empty_string_as_none")]
pub region: Option<String>,
pub role_arn: String,
}
#[derive(Debug, Deserialize)]
#[serde(untagged)]
pub enum AWSAuthConfig {
Credentials(CredentialsAuth),
Oidc(OidcAuth),
}
pub async fn get_oidc_authentication_data(
oidc_auth: OidcAuth,
role_session_name: Option<impl ToString>,
token: String,
) -> Result<AssumeRoleWithWebIdentityOutput> {
let region = oidc_auth.region.unwrap_or_else(|| "us-east-1".to_string());
let credentials = AwsCredentials::new("", "", None, None, "UserInput");
let config = aws_config::defaults(BehaviorVersion::latest())
.credentials_provider(credentials)
.region(Region::new(region.clone()))
.load()
.await;
let assume_role_with_web_identity_fluent_builder = Client::new(&config)
.assume_role_with_web_identity()
.set_role_arn(Some(oidc_auth.role_arn))
.set_role_session_name(role_session_name.map(|str| str.to_string()))
.set_web_identity_token(Some(token));
let resp = assume_role_with_web_identity_fluent_builder
.clone()
.send()
.await
.map_err(to_anyhow)?;
Ok(resp)
}
}

View File

@@ -64,7 +64,7 @@
required={true}
/>
</p>
<ResourcePicker resourceType="aws" bind:value={aws_resource_path} />
<ResourcePicker resourceType="aws_auth" bind:value={aws_resource_path} />
{#if isValid}
<TestTriggerConnection kind="sqs" args={{ aws_resource_path, queue_url }} />
{/if}