kalugin66/windmill
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add new stats (#2568)

Browse Source 
* feat: add new stats

* feat: use license key id + manually send usage
This commit is contained in:
HugoCasa
2023-11-06 18:23:25 +01:00
committed by GitHub
parent f01760cac6
commit 0a6b37ffab
13 changed files with 168 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
backend/.sqlx/query-08e4a2dc49c75aa356f3cc75a4abd8fc61409776d641ddb592a4c731e61a0468.json generated Normal file
View File

@@ -0,0 +1,26 @@
{
"db_name": "PostgreSQL",
"query": "SELECT login_type, COUNT(*) FROM password GROUP BY login_type",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "login_type",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "count",
"type_info": "Int8"
}
],
"parameters": {
"Left": []
},
"nullable": [
false,
null
]
},
"hash": "08e4a2dc49c75aa356f3cc75a4abd8fc61409776d641ddb592a4c731e61a0468"
}

20
backend/.sqlx/query-1c67cce8cdf6e56752643d6382ebaab76e24581009ca5e8d8b5ce3f199b689ab.json generated Normal file
View File

@@ -0,0 +1,20 @@
{
"db_name": "PostgreSQL",
"query": "SELECT COUNT(*) FROM worker_ping WHERE ping_at > NOW() - INTERVAL '5 minutes'",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "count",
"type_info": "Int8"
}
],
"parameters": {
"Left": []
},
"nullable": [
null
]
},
"hash": "1c67cce8cdf6e56752643d6382ebaab76e24581009ca5e8d8b5ce3f199b689ab"
}

5
backend/src/ee.rs
View File

@@ -2,7 +2,8 @@
use windmill_common::error;
pub async fn set_license_key(license_key: String) -> anyhow::Result<()> {
use windmill_api::{ee::validate_license_key, LICENSE_KEY, LICENSE_KEY_ID, LICENSE_KEY_VALID};
use windmill_api::ee::validate_license_key;
use windmill_common::ee::{LICENSE_KEY, LICENSE_KEY_ID, LICENSE_KEY_VALID};
let id = validate_license_key(license_key.clone()).await?;
{
@@ -24,7 +25,7 @@ pub async fn set_license_key(license_key: String) -> anyhow::Result<()> {
#[cfg(feature = "enterprise")]
pub async fn verify_license_key() -> error::Result<()> {
use windmill_api::{LICENSE_KEY, LICENSE_KEY_VALID};
use windmill_common::ee::{LICENSE_KEY, LICENSE_KEY_VALID};
use windmill_common::error::to_anyhow;
let expiry_nb = LICENSE_KEY

2
backend/src/monitor.rs
View File

@@ -41,7 +41,7 @@ use windmill_worker::{
use crate::ee::verify_license_key;
#[cfg(feature = "enterprise")]
use windmill_api::LICENSE_KEY_VALID;
use windmill_common::ee::LICENSE_KEY_VALID;
use crate::ee::set_license_key;

14
backend/windmill-api/openapi.yaml
View File

@@ -650,6 +650,20 @@ paths:
schema:
type: string
/settings/send_stats:
post:
summary: send stats
operationId: sendStats
tags:
- setting
responses:
"200":
description: status
content:
text/plain::
schema:
type: string
/users/email:
get:
summary: get current user email (if logged in)

2
backend/windmill-api/src/jobs.rs
View File

@@ -1569,7 +1569,7 @@ async fn check_tag_available_for_workspace(w_id: &str, tag: &Option<String>) ->
#[cfg(feature = "enterprise")]
pub async fn check_license_key_valid() -> error::Result<()> {
use crate::LICENSE_KEY_VALID;
use windmill_common::ee::LICENSE_KEY_VALID;
let valid = *LICENSE_KEY_VALID.read().await;
if !valid {

6
backend/windmill-api/src/lib.rs
View File

@@ -104,10 +104,6 @@ lazy_static::lazy_static! {
connects: HashMap::new(),
slack: None
}));
pub static ref LICENSE_KEY_VALID: Arc<RwLock<bool>> = Arc::new(RwLock::new(true));
pub static ref LICENSE_KEY_ID: Arc<RwLock<String>> = Arc::new(RwLock::new("".to_string()));
pub static ref LICENSE_KEY: Arc<RwLock<String>> = Arc::new(RwLock::new("".to_string()));
}
pub async fn run_server(
@@ -337,6 +333,8 @@ async fn ee_license() -> &'static str {
#[cfg(feature = "enterprise")]
async fn ee_license() -> String {
use windmill_common::ee::LICENSE_KEY_ID;
LICENSE_KEY_ID.read().await.clone()
}

11
backend/windmill-api/src/settings.rs
View File

@@ -12,6 +12,7 @@ use crate::{
db::{ApiAuthed, DB},
ee::validate_license_key,
utils::require_super_admin,
HTTP_CLIENT,
};
use axum::{
@@ -24,7 +25,7 @@ use mail_send::{mail_builder::MessageBuilder, SmtpClientBuilder};
use serde::Deserialize;
use tokio::time::timeout;
use windmill_common::{
error::{self, to_anyhow, JsonResult},
error::{self, to_anyhow, JsonResult, Result},
global_settings::ENV_SETTINGS,
server::Smtp,
};
@@ -38,6 +39,7 @@ pub fn global_service() -> Router {
)
.route("/test_smtp", post(test_email))
.route("/test_license_key", post(test_license_key))
.route("/send_stats", post(send_stats))
}
#[derive(Deserialize)]
@@ -158,3 +160,10 @@ pub async fn get_global_setting(
Ok(Json(value.unwrap_or_else(|| serde_json::Value::Null)))
}
pub async fn send_stats(Extension(db): Extension<DB>, authed: ApiAuthed) -> Result<String> {
require_super_admin(&db, &authed.email).await?;
windmill_common::stats::send_stats(&"manual".to_string(), &HTTP_CLIENT, &db).await?;
Ok("Sent stats".to_string())
}

8
backend/windmill-common/src/ee.rs Normal file
View File

@@ -0,0 +1,8 @@
use std::sync::Arc;
use tokio::sync::RwLock;
lazy_static::lazy_static! {
pub static ref LICENSE_KEY_VALID: Arc<RwLock<bool>> = Arc::new(RwLock::new(true));
pub static ref LICENSE_KEY_ID: Arc<RwLock<String>> = Arc::new(RwLock::new("".to_string()));
pub static ref LICENSE_KEY: Arc<RwLock<String>> = Arc::new(RwLock::new("".to_string()));
}

1
backend/windmill-common/src/lib.rs
View File

@@ -17,6 +17,7 @@ use sqlx::{Pool, Postgres};
pub mod apps;
pub mod db;
pub mod ee;
pub mod error;
pub mod external_ip;
pub mod flow_status;

46
backend/windmill-common/src/stats.rs
View File

@@ -2,11 +2,11 @@ use std::str::FromStr;
use crate::{
error::{to_anyhow, Result},
global_settings::{DISABLE_STATS_SETTING, UNIQUE_ID_SETTING},
utils::GIT_VERSION,
global_settings::DISABLE_STATS_SETTING,
scripts::ScriptLang,
utils::{get_uid, GIT_VERSION},
DB,
};
use chrono::Utc;
use cron::Schedule;
@@ -74,24 +74,54 @@ pub async fn schedule_stats(db: &DB, instance_name: String, http_client: &reqwes
});
}
#[derive(Debug, sqlx::FromRow, serde::Serialize)]
struct JobsUsage {
language: Option<ScriptLang>,
total_duration: i64,
count: i64,
}
pub async fn send_stats(
instance_name: &String,
http_client: &reqwest::Client,
db: &DB,
) -> Result<()> {
let uid = sqlx::query_scalar!(
"SELECT value FROM global_settings WHERE name = $1",
UNIQUE_ID_SETTING
let uid = get_uid(db).await?;
let jobs_usage = sqlx::query_as::<_, JobsUsage>(
"SELECT language, COUNT(*) as count, SUM(duration_ms)::BIGINT as total_duration FROM completed_job GROUP BY language",
)
.fetch_one(db)
.fetch_all(db)
.await?;
let uid = serde_json::from_value::<String>(uid).map_err(to_anyhow)?;
let login_type_usage =
sqlx::query!("SELECT login_type, COUNT(*) FROM password GROUP BY login_type")
.fetch_all(db)
.await?
.into_iter()
.map(|r| {
serde_json::json!({
"login_type": r.login_type,
"count": r.count.unwrap_or(0),
})
})
.collect::<Vec<serde_json::Value>>();
let workers_usage = sqlx::query!(
"SELECT COUNT(*) FROM worker_ping WHERE ping_at > NOW() - INTERVAL '5 minutes'"
)
.fetch_one(db)
.await?
.count
.unwrap_or(0);
let payload = serde_json::json!({
"uid": uid,
"version": GIT_VERSION,
"instance_name": instance_name,
"jobs_usage": jobs_usage,
"login_type_usage": login_type_usage,
"workers_usage": workers_usage,
});
let request = http_client

36
backend/windmill-common/src/utils.rs
View File

@@ -6,6 +6,7 @@
* LICENSE-AGPL for a copy of the license.
*/
use crate::ee::LICENSE_KEY_ID;
use crate::error::{to_anyhow, Error, Result};
use crate::global_settings::UNIQUE_ID_SETTING;
use crate::DB;
@@ -103,13 +104,7 @@ pub async fn http_get_from_hub(
query_params: Option<Vec<(&str, String)>>,
db: &Pool<Postgres>,
) -> Result<reqwest::Response> {
let uid = sqlx::query_scalar!(
"SELECT value FROM global_settings WHERE name = $1",
UNIQUE_ID_SETTING
)
.fetch_optional(db)
.await?
.map(|v| serde_json::from_value::<String>(v));
let uid = get_uid(db).await;
let mut request = http_client.get(url).header(
"Accept",
@@ -120,12 +115,10 @@ pub async fn http_get_from_hub(
},
);
if let Some(uid) = uid {
if let Ok(uid) = uid {
request = request.header("X-uid", uid);
} else {
tracing::info!("Invalid uid in global settings: {}", uid.err().unwrap())
}
if let Ok(uid) = uid {
request = request.header("X-uid", uid);
} else {
tracing::info!("No valid uid found: {}", uid.err().unwrap())
}
if let Some(query_params) = query_params {
@@ -152,3 +145,20 @@ pub fn calculate_hash(s: &str) -> String {
hasher.update(s);
format!("{:x}", hasher.finalize())
}
pub async fn get_uid(db: &DB) -> Result<String> {
let mut uid = LICENSE_KEY_ID.read().await.clone();
if uid == "" {
let uid_value = sqlx::query_scalar!(
"SELECT value FROM global_settings WHERE name = $1",
UNIQUE_ID_SETTING
)
.fetch_one(db)
.await?;
uid = serde_json::from_value::<String>(uid_value).map_err(to_anyhow)?;
}
Ok(uid)
}

22
frontend/src/lib/components/InstanceSettings.svelte
View File

@@ -143,6 +143,11 @@
]
let oauth_name = 'custom'
async function sendStats() {
await SettingService.sendStats()
sendUserToast('Usage sent')
}
</script>
<div class="pb-8">
@@ -164,8 +169,23 @@
<div class="text-secondary pb-4 text-xs">
Anonymous usage data is collected to help improve Windmill.
<br />The following information is collected:
<ul class="list-disc list-inside pl-2"><li>version</li></ul>
<ul class="list-disc list-inside pl-2">
<li>version</li>
<li>number and total duration of jobs</li>
<li>login type usage</li>
<li>workers usage</li>
</ul>
</div>
{#if $enterpriseLicense}
<Button
on:click={sendStats}
variant="border"
color="light"
btnClasses="w-auto"
wrapperClasses="mb-4"
size="xs">Send usage</Button
>
{/if}
{/if}
{#if category == 'SSO/OAuth'}
<div>