kalugin66/windmill
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(aiagent): add prompt caching for Anthropic models (#7878)

Browse Source 
* feat: add prompt caching support for Anthropic API

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

* exclude vertex

---------

Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
centdix
2026-02-10 18:42:49 +01:00
committed by GitHub
parent cb2957f02e
commit b5aa9c85c1
1 changed files with 67 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

74
backend/windmill-worker/src/ai/providers/anthropic.rs
View File

@@ -16,6 +16,17 @@ const ANTHROPIC_VERSION_STANDARD: &str = "2023-06-01";
/// Anthropic API version for Google Vertex AI
const ANTHROPIC_VERSION_VERTEX: &str = "vertex-2023-10-16";
#[derive(Serialize, Debug, Clone)]
pub struct CacheControl {
pub r#type: String,
}
impl CacheControl {
pub fn ephemeral() -> Self {
Self { r#type: "ephemeral".to_string() }
}
}
/// Custom tool for Anthropic native API (flat structure with type: "custom")
#[derive(Serialize, Debug)]
pub struct AnthropicCustomTool {
@@ -24,6 +35,8 @@ pub struct AnthropicCustomTool {
#[serde(skip_serializing_if = "Option::is_none")]
pub description: Option<String>,
pub input_schema: Box<RawValue>,
#[serde(skip_serializing_if = "Option::is_none")]
pub cache_control: Option<CacheControl>,
}
impl From<&ToolDef> for AnthropicCustomTool {
@@ -33,6 +46,7 @@ impl From<&ToolDef> for AnthropicCustomTool {
name: tool.function.name.clone(),
description: tool.function.description.clone(),
input_schema: tool.function.parameters.clone(),
cache_control: None,
}
}
}
@@ -70,13 +84,22 @@ impl AnthropicToolChoice {
#[serde(tag = "type")]
pub enum AnthropicRequestContent {
#[serde(rename = "text")]
Text { text: String },
Text {
text: String,
#[serde(skip_serializing_if = "Option::is_none")]
cache_control: Option<CacheControl>,
},
#[serde(rename = "image")]
Image { source: AnthropicImageSource },
#[serde(rename = "tool_use")]
ToolUse { id: String, name: String, input: Box<RawValue> },
#[serde(rename = "tool_result")]
ToolResult { tool_use_id: String, content: String },
ToolResult {
tool_use_id: String,
content: String,
#[serde(skip_serializing_if = "Option::is_none")]
cache_control: Option<CacheControl>,
},
}
/// Image source for Anthropic API
@@ -92,6 +115,8 @@ pub struct AnthropicImageSource {
pub struct AnthropicSystemContent {
pub r#type: String,
pub text: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub cache_control: Option<CacheControl>,
}
/// Message format for Anthropic API requests
@@ -163,7 +188,7 @@ fn convert_messages_to_anthropic(messages: &[OpenAIMessage]) -> Vec<AnthropicMes
if let Some(ref c) = msg.content {
let text = extract_text_content(c);
if !text.is_empty() {
content.push(AnthropicRequestContent::Text { text });
content.push(AnthropicRequestContent::Text { text, cache_control: None });
}
}
@@ -202,6 +227,7 @@ fn convert_messages_to_anthropic(messages: &[OpenAIMessage]) -> Vec<AnthropicMes
content: vec![AnthropicRequestContent::ToolResult {
tool_use_id: tool_call_id.clone(),
content: content_text,
cache_control: None,
}],
});
}
@@ -226,7 +252,7 @@ fn convert_content_to_anthropic(content: &Option<OpenAIContent>) -> Vec<Anthropi
if text.is_empty() {
Vec::new()
} else {
vec![AnthropicRequestContent::Text { text: text.clone() }]
vec![AnthropicRequestContent::Text { text: text.clone(), cache_control: None }]
}
}
OpenAIContent::Parts(parts) => {
@@ -235,7 +261,10 @@ fn convert_content_to_anthropic(content: &Option<OpenAIContent>) -> Vec<Anthropi
match part {
ContentPart::Text { text } => {
if !text.is_empty() {
result.push(AnthropicRequestContent::Text { text: text.clone() });
result.push(AnthropicRequestContent::Text {
text: text.clone(),
cache_control: None,
});
}
}
ContentPart::ImageUrl { image_url } => {
@@ -347,7 +376,7 @@ impl AnthropicQueryBuilder {
prepare_messages_for_api(args.messages, client, workspace_id).await?;
// Convert to Anthropic native message format
let anthropic_messages = convert_messages_to_anthropic(&prepared_messages);
let mut anthropic_messages = convert_messages_to_anthropic(&prepared_messages);
// Build tools array using typed structs
let mut tools: Vec<AnthropicTool> = Vec::new();
@@ -372,6 +401,11 @@ impl AnthropicQueryBuilder {
Some(s) if !s.is_empty() => Some(vec![AnthropicSystemContent {
r#type: "text".to_string(),
text: s.to_string(),
cache_control: if self.is_vertex() {
None
} else {
Some(CacheControl::ephemeral())
},
}]),
_ => None,
};
@@ -393,9 +427,35 @@ impl AnthropicQueryBuilder {
None
};
let tools_option = if tools.is_empty() { None } else { Some(tools) };
let mut tools_option = if tools.is_empty() { None } else { Some(tools) };
let max_tokens = Some(args.max_tokens.unwrap_or(64000));
// Apply cache_control on the last custom tool
if !self.is_vertex() {
if let Some(ref mut tools_vec) = tools_option {
if let Some(AnthropicTool::Custom(ref mut custom)) = tools_vec.last_mut() {
custom.cache_control = Some(CacheControl::ephemeral());
}
}
}
// Apply cache_control on the last content block of the last message
if !self.is_vertex() {
if let Some(last_msg) = anthropic_messages.last_mut() {
if let Some(last_block) = last_msg.content.last_mut() {
match last_block {
AnthropicRequestContent::Text { cache_control, .. } => {
*cache_control = Some(CacheControl::ephemeral());
}
AnthropicRequestContent::ToolResult { cache_control, .. } => {
*cache_control = Some(CacheControl::ephemeral());
}
_ => {}
}
}
}
}
// Build request based on platform
if self.is_vertex() {
// For Vertex AI: no model field, anthropic_version in body