Compare commits
5 Commits
v1.629.1
...
plan-3-scr
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8d3b18f523 | ||
|
|
8e19f9652d | ||
|
|
45980f0220 | ||
|
|
8363ff1eee | ||
|
|
cf596f370a |
12
.github/workflows/build_windows_worker_.yml
vendored
12
.github/workflows/build_windows_worker_.yml
vendored
@@ -40,6 +40,15 @@ jobs:
|
||||
run: |
|
||||
./backend/substitute_ee_code.sh --copy --dir ./windmill-ee-private
|
||||
|
||||
- name: Cargo check (fail fast on warnings)
|
||||
timeout-minutes: 60
|
||||
env:
|
||||
RUSTFLAGS: "-D warnings"
|
||||
run: |
|
||||
mkdir frontend/build && cd backend
|
||||
New-Item -Path . -Name "windmill-api/openapi-deref.yaml" -ItemType "File" -Force
|
||||
cargo check --features=ee_windows
|
||||
|
||||
- name: Cargo build dynamic libraries windows
|
||||
timeout-minutes: 180
|
||||
run: |
|
||||
@@ -54,8 +63,7 @@ jobs:
|
||||
vcpkg.exe integrate install
|
||||
$env:VCPKGRS_DYNAMIC=1
|
||||
$env:OPENSSL_DIR="${Env:VCPKG_INSTALLATION_ROOT}\installed\x64-windows-static"
|
||||
mkdir frontend/build && cd backend
|
||||
New-Item -Path . -Name "windmill-api/openapi-deref.yaml" -ItemType "File" -Force
|
||||
cd backend
|
||||
cargo build --release --features=ee_windows
|
||||
- name: Rename binary with corresponding architecture
|
||||
run: |
|
||||
|
||||
@@ -419,7 +419,7 @@ const_format = { version = "0.2.35", features = ["rust_1_64", "rust_1_51"] }
|
||||
const-str = "0.5"
|
||||
constant_time_eq = "0.3.1"
|
||||
rsa = "^0"
|
||||
aes-gcm = "^0"
|
||||
aes-gcm = "0.10.3"
|
||||
async_zip = { version = "0.0.17", features = ["tokio", "tokio-fs", "deflate", "chrono"] }
|
||||
once_cell = "1.17.1"
|
||||
dashmap = "6.1.0"
|
||||
|
||||
@@ -1 +1 @@
|
||||
7596cefdba81482c0b0c0b61be26369f112d8009
|
||||
7596cefdba81482c0b0c0b61be26369f112d8009
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
-- No-op: cannot restore deleted telemetry data
|
||||
@@ -0,0 +1,2 @@
|
||||
-- Delete all saved telemetry data from metrics table
|
||||
DELETE FROM metrics WHERE id = 'telemetry';
|
||||
@@ -381,5 +381,6 @@ pub static SHORT_IMPORTS_MAP: PyMap = phf_map! {
|
||||
"docx" => "python-docx",
|
||||
"vt" => "vt-py",
|
||||
"grpc" => "grpcio",
|
||||
"serpapi" => "google-search-results",
|
||||
// Add new entry here ^
|
||||
};
|
||||
|
||||
@@ -1310,6 +1310,20 @@ paths:
|
||||
schema:
|
||||
type: string
|
||||
|
||||
/settings/get_stats:
|
||||
get:
|
||||
summary: get encrypted telemetry stats (EE only)
|
||||
operationId: getStats
|
||||
tags:
|
||||
- setting
|
||||
responses:
|
||||
"200":
|
||||
description: base64-encoded encrypted telemetry blob
|
||||
content:
|
||||
text/plain:
|
||||
schema:
|
||||
type: string
|
||||
|
||||
/settings/latest_key_renewal_attempt:
|
||||
get:
|
||||
summary: get latest key renewal attempt
|
||||
|
||||
@@ -58,6 +58,7 @@ pub fn global_service() -> Router {
|
||||
.route("/test_smtp", post(test_email))
|
||||
.route("/test_license_key", post(test_license_key))
|
||||
.route("/send_stats", post(send_stats))
|
||||
.route("/get_stats", get(get_stats))
|
||||
.route(
|
||||
"/latest_key_renewal_attempt",
|
||||
get(get_latest_key_renewal_attempt),
|
||||
@@ -434,6 +435,25 @@ pub async fn send_stats(Extension(db): Extension<DB>, authed: ApiAuthed) -> Resu
|
||||
Ok("Sent stats".to_string())
|
||||
}
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
pub async fn get_stats(Extension(db): Extension<DB>, authed: ApiAuthed) -> Result<String> {
|
||||
require_super_admin(&db, &authed.email).await?;
|
||||
let stats = windmill_common::stats_oss::get_stats_payload(
|
||||
&db,
|
||||
&windmill_common::stats_oss::SendStatsReason::Manual,
|
||||
)
|
||||
.await?;
|
||||
let encrypted = windmill_common::stats_oss::encrypt_stats(&stats)?;
|
||||
Ok(encrypted)
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "enterprise"))]
|
||||
pub async fn get_stats() -> Result<String> {
|
||||
Err(error::Error::BadRequest(
|
||||
"Downloading telemetry is only available on enterprise edition".to_string(),
|
||||
))
|
||||
}
|
||||
|
||||
#[derive(serde::Serialize)]
|
||||
pub struct KeyRenewalAttempt {
|
||||
result: String,
|
||||
|
||||
@@ -50,3 +50,19 @@ pub async fn get_user_usage<'c, E: sqlx::Executor<'c, Database = Postgres>>(
|
||||
let usage = ActiveUserUsage { author_count: None, operator_count: None };
|
||||
Ok(usage)
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "private"))]
|
||||
#[derive(serde::Serialize)]
|
||||
pub struct Stats {}
|
||||
|
||||
#[cfg(not(feature = "private"))]
|
||||
pub async fn get_stats_payload(_db: &DB, _reason: &SendStatsReason) -> Result<Stats> {
|
||||
// stats details are closed source
|
||||
Ok(Stats {})
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "private"))]
|
||||
pub fn encrypt_stats(_stats: &Stats) -> Result<String> {
|
||||
// stats details are closed source
|
||||
Ok(String::new())
|
||||
}
|
||||
|
||||
@@ -1045,16 +1045,17 @@ pub async fn extract_tar(tar: bytes::Bytes, folder: &str) -> error::Result<()> {
|
||||
}
|
||||
#[cfg(all(feature = "enterprise", feature = "parquet"))]
|
||||
fn write_binary_file(main_path: &str, byts: &mut bytes::Bytes) -> error::Result<()> {
|
||||
use std::fs::{File, Permissions};
|
||||
use std::fs::File;
|
||||
use std::io::Write;
|
||||
|
||||
#[cfg(unix)]
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
|
||||
let mut file = File::create(main_path)?;
|
||||
file.write_all(byts)?;
|
||||
#[cfg(unix)]
|
||||
file.set_permissions(Permissions::from_mode(0o755))?;
|
||||
{
|
||||
use std::fs::Permissions;
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
file.set_permissions(Permissions::from_mode(0o755))?;
|
||||
}
|
||||
file.flush()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -752,10 +752,12 @@ pub async fn get_git_ssh_cmd(
|
||||
})?;
|
||||
content.push_str("\n");
|
||||
|
||||
let file = write_file(job_dir, &id_file_name, &content)?;
|
||||
#[cfg(not(unix))]
|
||||
let _ = write_file(job_dir, &id_file_name, &content)?;
|
||||
|
||||
#[cfg(unix)]
|
||||
{
|
||||
let file = write_file(job_dir, &id_file_name, &content)?;
|
||||
let perm = std::os::unix::fs::PermissionsExt::from_mode(0o600);
|
||||
file.set_permissions(perm)?;
|
||||
}
|
||||
@@ -1218,10 +1220,10 @@ fi
|
||||
ANSIBLE_PLAYBOOK_PATH.as_str()
|
||||
);
|
||||
|
||||
let file = write_file(job_dir, "wrapper.sh", &wrapper)?;
|
||||
let _file = write_file(job_dir, "wrapper.sh", &wrapper)?;
|
||||
|
||||
#[cfg(unix)]
|
||||
file.metadata()?.permissions().set_mode(0o777);
|
||||
_file.metadata()?.permissions().set_mode(0o777);
|
||||
// let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str());
|
||||
let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str());
|
||||
nsjail_cmd
|
||||
|
||||
@@ -21,10 +21,12 @@ use windmill_queue::append_logs;
|
||||
use crate::{
|
||||
common::{start_child_process, OccupancyMetrics},
|
||||
handle_child::handle_child,
|
||||
python_executor::{INDEX_CERT, NATIVE_CERT, PYTHON_PATH, UV_PATH},
|
||||
python_executor::{INDEX_CERT, NATIVE_CERT, PYTHON_PATH},
|
||||
HOME_ENV, INSTANCE_PYTHON_VERSION, PATH_ENV, PROXY_ENVS, PY_INSTALL_DIR, UV_CACHE_DIR,
|
||||
WIN_ENVS,
|
||||
};
|
||||
#[cfg(unix)]
|
||||
use crate::python_executor::UV_PATH;
|
||||
|
||||
impl From<PyV> for PyVAlias {
|
||||
fn from(value: PyV) -> Self {
|
||||
|
||||
@@ -12,7 +12,6 @@
|
||||
use anyhow::anyhow;
|
||||
use futures::TryFutureExt;
|
||||
use tokio::sync::Mutex;
|
||||
use tokio::time::sleep;
|
||||
use tokio::time::timeout;
|
||||
use windmill_common::client::AuthedClient;
|
||||
use windmill_common::jobs::WorkerInternalServerInlineUtils;
|
||||
@@ -3064,7 +3063,7 @@ pub async fn handle_queued_job(
|
||||
.flatten()
|
||||
{
|
||||
tracing::debug!("Debug: {} going to sleep for {}", job.id, dbg_djob_sleep);
|
||||
sleep(std::time::Duration::from_secs(dbg_djob_sleep as u64)).await;
|
||||
tokio::time::sleep(std::time::Duration::from_secs(dbg_djob_sleep as u64)).await;
|
||||
}
|
||||
|
||||
tracing::debug!(
|
||||
|
||||
@@ -177,7 +177,7 @@
|
||||
{/snippet}
|
||||
|
||||
<!-- {JSON.stringify($values, null, 2)} -->
|
||||
{#if (!setting.cloudonly || isCloudHosted()) && showSetting(setting.key, $values) && !(setting.hiddenIfNull && $values[setting.key] == null) && !(setting.hiddenIfEmpty && !$values[setting.key])}
|
||||
{#if (!setting.cloudonly || isCloudHosted()) && showSetting(setting.key, $values) && !(setting.hiddenIfNull && $values[setting.key] == null) && !(setting.hiddenIfEmpty && !$values[setting.key]) && !(setting.hiddenInEe && $enterpriseLicense)}
|
||||
{#if setting.fieldType == 'select'}
|
||||
<div>
|
||||
{@render LabelSnippet()}
|
||||
|
||||
@@ -231,6 +231,28 @@
|
||||
}
|
||||
}
|
||||
|
||||
let downloadingStats = $state(false)
|
||||
async function downloadStats() {
|
||||
try {
|
||||
downloadingStats = true
|
||||
const encryptedData = await SettingService.getStats()
|
||||
const blob = new Blob([encryptedData], { type: 'application/octet-stream' })
|
||||
const url = URL.createObjectURL(blob)
|
||||
const a = document.createElement('a')
|
||||
a.href = url
|
||||
a.download = `windmill-telemetry-${new Date().toISOString().split('T')[0]}.enc`
|
||||
document.body.appendChild(a)
|
||||
a.click()
|
||||
document.body.removeChild(a)
|
||||
URL.revokeObjectURL(url)
|
||||
sendUserToast('Telemetry data downloaded')
|
||||
} catch (err) {
|
||||
throw err
|
||||
} finally {
|
||||
downloadingStats = false
|
||||
}
|
||||
}
|
||||
|
||||
function isValidTeamsChannel(value: any): value is TeamsChannel {
|
||||
return (
|
||||
typeof value === 'object' &&
|
||||
@@ -341,18 +363,29 @@
|
||||
<div class="text-primary pb-4 text-xs">
|
||||
On Enterprise Edition, you must send data to check that usage is in line with the
|
||||
terms of the subscription. You can either enable telemetry or regularly send usage
|
||||
data by clicking the button below.
|
||||
data by clicking the button below. For air-gapped instances, you can download the
|
||||
telemetry data and send it manually.
|
||||
</div>
|
||||
<div class="flex gap-2 mb-4">
|
||||
<Button
|
||||
on:click={sendStats}
|
||||
variant="default"
|
||||
btnClasses="w-auto"
|
||||
loading={sendingStats}
|
||||
size="xs"
|
||||
>
|
||||
Send usage
|
||||
</Button>
|
||||
<Button
|
||||
on:click={downloadStats}
|
||||
variant="default"
|
||||
btnClasses="w-auto"
|
||||
loading={downloadingStats}
|
||||
size="xs"
|
||||
>
|
||||
Download usage
|
||||
</Button>
|
||||
</div>
|
||||
<Button
|
||||
on:click={sendStats}
|
||||
variant="default"
|
||||
btnClasses="w-auto"
|
||||
wrapperClasses="mb-4"
|
||||
loading={sendingStats}
|
||||
size="xs"
|
||||
>
|
||||
Send usage
|
||||
</Button>
|
||||
{/if}
|
||||
{:else if category == 'Auth/OAuth/SAML'}
|
||||
<AuthSettings
|
||||
|
||||
@@ -1,6 +1,12 @@
|
||||
import { ResourceService, JobService } from '$lib/gen/services.gen'
|
||||
import type { AIProvider, AIProviderModel, ResourceType, ScriptLang } from '$lib/gen/types.gen'
|
||||
import { capitalize, isObject, toCamel } from '$lib/utils'
|
||||
import { ResourceService, JobService, FlowService } from '$lib/gen/services.gen'
|
||||
import type {
|
||||
AIProvider,
|
||||
AIProviderModel,
|
||||
Flow,
|
||||
ResourceType,
|
||||
ScriptLang
|
||||
} from '$lib/gen/types.gen'
|
||||
import { capitalize, emptyString, isObject, toCamel } from '$lib/utils'
|
||||
import { get } from 'svelte/store'
|
||||
import { compile, phpCompile, pythonCompile } from '../../utils'
|
||||
import type {
|
||||
@@ -16,9 +22,11 @@ import {
|
||||
executeTestRun,
|
||||
buildTestRunArgs,
|
||||
buildContextString,
|
||||
extractAllModules,
|
||||
type ScriptLintResult,
|
||||
formatScriptLintResult
|
||||
} from '../shared'
|
||||
import uFuzzy from '@leeoniya/ufuzzy'
|
||||
import { setupTypeAcquisition, type DepsToGet } from '$lib/ata'
|
||||
import { getModelContextWindow } from '../../lib'
|
||||
import type { ReviewChangesOpts } from '../monaco-adapter'
|
||||
@@ -178,6 +186,7 @@ function buildChatSystemPrompt(currentModel: AIProviderModel) {
|
||||
- You can also receive a \`DIFF\` of the changes that have been made to the code. You should use this diff to give better answers.
|
||||
- Before giving your answer, check again that you carefully followed these instructions.
|
||||
- When asked to create a script that communicates with an external service, you can use the \`search_hub_scripts\` tool to search for relevant scripts in the hub. Make sure the language is the same as what the user is coding in. If you do not find any relevant scripts, you can use the \`search_npm_packages\` tool to search for relevant packages and their documentation. Always give a link to the documentation in your answer if possible.
|
||||
- If the user mentions a flow, you can use the \`search_flows\` tool to find it, and then \`get_flow_details\` to read its details and possibly reuse any inline script module inside it.
|
||||
- After applying code changes with the \`${editToolName}\` tool, ALWAYS use the \`get_lint_errors\` tool to check for lint errors. If there are errors, fix them before proceeding. Then use the \`test_run_script\` tool to test the code, and iterate on the code until it works as expected (MAX 3 times). If the user cancels the test run, do not try again and wait for the next user instruction.
|
||||
|
||||
Important:
|
||||
@@ -329,6 +338,8 @@ export function prepareScriptTools(
|
||||
}
|
||||
tools.push(testRunScriptTool)
|
||||
tools.push(getLintErrorsTool)
|
||||
tools.push(searchFlowsTool)
|
||||
tools.push(getFlowDetailsTool)
|
||||
return tools
|
||||
}
|
||||
|
||||
@@ -909,3 +920,140 @@ export const getLintErrorsTool: Tool<ScriptChatHelpers> = {
|
||||
return formatScriptLintResult(lintResult)
|
||||
}
|
||||
}
|
||||
|
||||
// ============= Flow Search Tools =============
|
||||
|
||||
class WorkspaceFlowsSearch {
|
||||
private uf: uFuzzy
|
||||
private workspace: string | undefined = undefined
|
||||
private flows: Flow[] | undefined = undefined
|
||||
|
||||
constructor() {
|
||||
this.uf = new uFuzzy()
|
||||
}
|
||||
|
||||
private async init(workspace: string) {
|
||||
if (this.flows === undefined || this.workspace !== workspace) {
|
||||
this.flows = await FlowService.listFlows({ workspace })
|
||||
this.workspace = workspace
|
||||
}
|
||||
}
|
||||
|
||||
async search(query: string, workspace: string) {
|
||||
await this.init(workspace)
|
||||
const flows = this.flows
|
||||
if (!flows) return []
|
||||
|
||||
const results = this.uf.search(
|
||||
flows.map((f) => (emptyString(f.summary) ? f.path : f.summary + ' (' + f.path + ')')),
|
||||
query.trim()
|
||||
)
|
||||
return (
|
||||
results[2]?.map((id) => ({
|
||||
path: flows[id].path,
|
||||
summary: flows[id].summary,
|
||||
description: flows[id].description
|
||||
})) ?? []
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
const workspaceFlowsSearch = new WorkspaceFlowsSearch()
|
||||
|
||||
const SEARCH_FLOWS_TOOL: ChatCompletionFunctionTool = {
|
||||
type: 'function',
|
||||
function: {
|
||||
name: 'search_flows',
|
||||
description:
|
||||
'Search for flows in the workspace. Use this when the user mentions a flow, wants to find existing flows, or wants to reuse inline script code from a flow.',
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
query: {
|
||||
type: 'string',
|
||||
description: 'The search query (e.g. "invoice processing", "stripe webhook")'
|
||||
}
|
||||
},
|
||||
required: ['query'],
|
||||
additionalProperties: false
|
||||
},
|
||||
strict: true
|
||||
}
|
||||
}
|
||||
|
||||
export const searchFlowsTool: Tool<ScriptChatHelpers> = {
|
||||
def: SEARCH_FLOWS_TOOL,
|
||||
fn: async ({ args, workspace, toolId, toolCallbacks }) => {
|
||||
toolCallbacks.setToolStatus(toolId, {
|
||||
content: 'Searching for flows related to "' + args.query + '"...'
|
||||
})
|
||||
const flowResults = await workspaceFlowsSearch.search(args.query, workspace)
|
||||
toolCallbacks.setToolStatus(toolId, {
|
||||
content: 'Found ' + flowResults.length + ' flow(s) related to "' + args.query + '"'
|
||||
})
|
||||
return JSON.stringify(flowResults)
|
||||
}
|
||||
}
|
||||
|
||||
const MAX_INLINE_SCRIPT_LENGTH = 2000
|
||||
|
||||
const GET_FLOW_DETAILS_TOOL: ChatCompletionFunctionTool = {
|
||||
type: 'function',
|
||||
function: {
|
||||
name: 'get_flow_details',
|
||||
description:
|
||||
'Get the details of a flow including its modules and inline script code. Use after search_flows to inspect a specific flow and potentially reuse its inline scripts.',
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
path: {
|
||||
type: 'string',
|
||||
description: 'The path of the flow (e.g. "f/ops/process_invoices")'
|
||||
}
|
||||
},
|
||||
required: ['path'],
|
||||
additionalProperties: false
|
||||
},
|
||||
strict: true
|
||||
}
|
||||
}
|
||||
|
||||
export const getFlowDetailsTool: Tool<ScriptChatHelpers> = {
|
||||
def: GET_FLOW_DETAILS_TOOL,
|
||||
fn: async ({ args, workspace, toolId, toolCallbacks }) => {
|
||||
toolCallbacks.setToolStatus(toolId, {
|
||||
content: 'Fetching flow details for "' + args.path + '"...'
|
||||
})
|
||||
const flow = await FlowService.getFlowByPath({ workspace, path: args.path })
|
||||
const modules = extractAllModules(flow.value.modules)
|
||||
const moduleDetails = modules.map((m) => {
|
||||
const base: Record<string, unknown> = {
|
||||
id: m.id,
|
||||
summary: m.summary,
|
||||
type: m.value.type
|
||||
}
|
||||
if (m.value.type === 'rawscript') {
|
||||
base.language = m.value.language
|
||||
const content = m.value.content ?? ''
|
||||
base.content =
|
||||
content.length > MAX_INLINE_SCRIPT_LENGTH
|
||||
? content.slice(0, MAX_INLINE_SCRIPT_LENGTH) + '...(truncated)'
|
||||
: content
|
||||
} else if (m.value.type === 'script') {
|
||||
base.path = m.value.path
|
||||
}
|
||||
return base
|
||||
})
|
||||
const result = {
|
||||
path: flow.path,
|
||||
summary: flow.summary,
|
||||
description: flow.description,
|
||||
schema: flow.schema,
|
||||
modules: moduleDetails
|
||||
}
|
||||
toolCallbacks.setToolStatus(toolId, {
|
||||
content: 'Retrieved flow details for "' + args.path + '"'
|
||||
})
|
||||
return JSON.stringify(result)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -58,6 +58,7 @@ export interface Setting {
|
||||
}
|
||||
hiddenIfNull?: boolean
|
||||
hiddenIfEmpty?: boolean
|
||||
hiddenInEe?: boolean
|
||||
requiresReloadOnChange?: boolean
|
||||
isValid?: (value: any) => boolean
|
||||
error?: string
|
||||
@@ -497,7 +498,8 @@ export const settings: Record<string, Setting[]> = {
|
||||
label: 'Disable telemetry',
|
||||
key: 'disable_stats',
|
||||
fieldType: 'boolean',
|
||||
storage: 'setting'
|
||||
storage: 'setting',
|
||||
hiddenInEe: true
|
||||
}
|
||||
],
|
||||
'Secret Storage': [
|
||||
|
||||
Reference in New Issue
Block a user