From e63300887fe0a62f16759527df0291d4b73dc058 Mon Sep 17 00:00:00 2001 From: faces <108703161+faces-of-eth@users.noreply.github.com> Date: Tue, 13 May 2025 15:52:44 -0700 Subject: [PATCH] goose/providers: add Venice.ai private open-source LLM (#2252) --- crates/goose-cli/src/commands/configure.rs | 39 +- crates/goose/src/config/base.rs | 36 +- crates/goose/src/providers/anthropic.rs | 35 +- crates/goose/src/providers/base.rs | 5 + crates/goose/src/providers/factory.rs | 3 + crates/goose/src/providers/google.rs | 20 + crates/goose/src/providers/mod.rs | 1 + crates/goose/src/providers/openai.rs | 40 ++ crates/goose/src/providers/venice.rs | 587 ++++++++++++++++++ .../settings/models/hardcoded_stuff.tsx | 2 + 10 files changed, 760 insertions(+), 8 deletions(-) create mode 100644 crates/goose/src/providers/venice.rs diff --git a/crates/goose-cli/src/commands/configure.rs b/crates/goose-cli/src/commands/configure.rs index 51be37425d..bd3ca18117 100644 --- a/crates/goose-cli/src/commands/configure.rs +++ b/crates/goose-cli/src/commands/configure.rs @@ -327,11 +327,40 @@ pub async fn configure_provider_dialog() -> Result> { } } - // Select model, defaulting to the provider's recommended model UNLESS there is an env override - let default_model = std::env::var("GOOSE_MODEL").unwrap_or(provider_meta.default_model.clone()); - let model: String = cliclack::input("Enter a model from that provider:") - .default_input(&default_model) - .interact()?; + // Attempt to fetch supported models for this provider + let spin = spinner(); + spin.start("Attempting to fetch supported models..."); + let models_res = { + let temp_model_config = goose::model::ModelConfig::new(provider_meta.default_model.clone()); + let temp_provider = create(provider_name, temp_model_config)?; + temp_provider.fetch_supported_models_async().await + }; + spin.stop(style("Model fetch complete").green()); + + // Select a model: on fetch error show styled error and abort; if Some(models), show list; if None, free-text input + let model: String = match models_res { + Err(e) => { + // Provider hook error + cliclack::outro(style(e.to_string()).on_red().white())?; + return Ok(false); + } + Ok(Some(models)) => cliclack::select("Select a model:") + .items( + &models + .iter() + .map(|m| (m, m.as_str(), "")) + .collect::>(), + ) + .interact()? + .to_string(), + Ok(None) => { + let default_model = + std::env::var("GOOSE_MODEL").unwrap_or(provider_meta.default_model.clone()); + cliclack::input("Enter a model from that provider:") + .default_input(&default_model) + .interact()? + } + }; // Test the configuration let spin = spinner(); diff --git a/crates/goose/src/config/base.rs b/crates/goose/src/config/base.rs index 65887a9a4b..2863608282 100644 --- a/crates/goose/src/config/base.rs +++ b/crates/goose/src/config/base.rs @@ -342,10 +342,42 @@ impl Config { /// - There is an error reading or writing the config file /// - There is an error serializing the value pub fn set_param(&self, key: &str, value: Value) -> Result<(), ConfigError> { - let mut values = self.load_values()?; + // Open the file with write permissions, create if it doesn't exist + let mut file = OpenOptions::new() + .write(true) + .create(true) + .open(&self.config_path)?; + + // Acquire an exclusive lock for the entire operation + file.lock_exclusive() + .map_err(|e| ConfigError::LockError(e.to_string()))?; + + // Load current values while holding the lock + let mut values = if self.config_path.exists() { + let file_content = std::fs::read_to_string(&self.config_path)?; + let yaml_value: serde_yaml::Value = serde_yaml::from_str(&file_content)?; + let json_value: Value = serde_json::to_value(yaml_value)?; + match json_value { + Value::Object(map) => map.into_iter().collect(), + _ => HashMap::new(), + } + } else { + HashMap::new() + }; + + // Modify values values.insert(key.to_string(), value); - self.save_values(values) + // Convert to YAML for storage + let yaml_value = serde_yaml::to_string(&values)?; + + // Write the contents using the same file handle + file.set_len(0)?; // Clear the file + file.write_all(yaml_value.as_bytes())?; + file.sync_all()?; + + // Unlock is handled automatically when file is dropped + Ok(()) } /// Delete a configuration value in the config file. diff --git a/crates/goose/src/providers/anthropic.rs b/crates/goose/src/providers/anthropic.rs index 3272f43edf..c0cb0fb6e8 100644 --- a/crates/goose/src/providers/anthropic.rs +++ b/crates/goose/src/providers/anthropic.rs @@ -23,6 +23,7 @@ pub const ANTHROPIC_KNOWN_MODELS: &[&str] = &[ ]; pub const ANTHROPIC_DOC_URL: &str = "https://docs.anthropic.com/en/docs/about-claude/models"; +pub const ANTHROPIC_API_VERSION: &str = "2023-06-01"; #[derive(serde::Serialize)] pub struct AnthropicProvider { @@ -156,7 +157,7 @@ impl Provider for AnthropicProvider { let mut headers = reqwest::header::HeaderMap::new(); headers.insert("x-api-key", self.api_key.parse().unwrap()); - headers.insert("anthropic-version", "2023-06-01".parse().unwrap()); + headers.insert("anthropic-version", ANTHROPIC_API_VERSION.parse().unwrap()); let is_thinking_enabled = std::env::var("CLAUDE_THINKING_ENABLED").is_ok(); if self.model.model_name.starts_with("claude-3-7-sonnet-") && is_thinking_enabled { @@ -183,4 +184,36 @@ impl Provider for AnthropicProvider { emit_debug_trace(&self.model, &payload, &response, &usage); Ok((message, ProviderUsage::new(model, usage))) } + + /// Fetch supported models from Anthropic; returns Err on failure, Ok(None) if not present + async fn fetch_supported_models_async(&self) -> Result>, ProviderError> { + let url = format!("{}/v1/models", self.host); + let response = self + .client + .get(&url) + .header("anthropic-version", ANTHROPIC_API_VERSION) + .header("x-api-key", self.api_key.clone()) + .send() + .await?; + let json: serde_json::Value = response.json().await?; + // if 'models' key missing, return None + let arr = match json.get("models").and_then(|v| v.as_array()) { + Some(arr) => arr, + None => return Ok(None), + }; + let mut models: Vec = arr + .iter() + .filter_map(|m| { + if let Some(s) = m.as_str() { + Some(s.to_string()) + } else if let Some(obj) = m.as_object() { + obj.get("id").and_then(|v| v.as_str()).map(str::to_string) + } else { + None + } + }) + .collect(); + models.sort(); + Ok(Some(models)) + } } diff --git a/crates/goose/src/providers/base.rs b/crates/goose/src/providers/base.rs index 6ab1bbe507..aa4bea4e12 100644 --- a/crates/goose/src/providers/base.rs +++ b/crates/goose/src/providers/base.rs @@ -178,6 +178,11 @@ pub trait Provider: Send + Sync { /// Get the model config from the provider fn get_model_config(&self) -> ModelConfig; + + /// Optional hook to fetch supported models asynchronously. + async fn fetch_supported_models_async(&self) -> Result>, ProviderError> { + Ok(None) + } } #[cfg(test)] diff --git a/crates/goose/src/providers/factory.rs b/crates/goose/src/providers/factory.rs index 45e0b0a0e5..da65c9ef91 100644 --- a/crates/goose/src/providers/factory.rs +++ b/crates/goose/src/providers/factory.rs @@ -13,6 +13,7 @@ use super::{ ollama::OllamaProvider, openai::OpenAiProvider, openrouter::OpenRouterProvider, + venice::VeniceProvider, }; use crate::model::ModelConfig; use anyhow::Result; @@ -30,6 +31,7 @@ pub fn providers() -> Vec { OllamaProvider::metadata(), OpenAiProvider::metadata(), OpenRouterProvider::metadata(), + VeniceProvider::metadata(), ] } @@ -46,6 +48,7 @@ pub fn create(name: &str, model: ModelConfig) -> Result> { "openrouter" => Ok(Arc::new(OpenRouterProvider::from_env(model)?)), "gcp_vertex_ai" => Ok(Arc::new(GcpVertexAIProvider::from_env(model)?)), "google" => Ok(Arc::new(GoogleProvider::from_env(model)?)), + "venice" => Ok(Arc::new(VeniceProvider::from_env(model)?)), "github_copilot" => Ok(Arc::new(GithubCopilotProvider::from_env(model)?)), _ => Err(anyhow::anyhow!("Unknown provider: {}", name)), } diff --git a/crates/goose/src/providers/google.rs b/crates/goose/src/providers/google.rs index be327537f4..ac35e98d79 100644 --- a/crates/goose/src/providers/google.rs +++ b/crates/goose/src/providers/google.rs @@ -169,4 +169,24 @@ impl Provider for GoogleProvider { let provider_usage = ProviderUsage::new(model, usage); Ok((message, provider_usage)) } + + /// Fetch supported models from Google Generative Language API; returns Err on failure, Ok(None) if not present + async fn fetch_supported_models_async(&self) -> Result>, ProviderError> { + // List models via the v1beta/models endpoint + let url = format!("{}/v1beta/models?key={}", self.host, self.api_key); + let response = self.client.get(&url).send().await?; + let json: serde_json::Value = response.json().await?; + // If 'models' field missing, return None + let arr = match json.get("models").and_then(|v| v.as_array()) { + Some(arr) => arr, + None => return Ok(None), + }; + let mut models: Vec = arr + .iter() + .filter_map(|m| m.get("name").and_then(|v| v.as_str())) + .map(|name| name.split('/').last().unwrap_or(name).to_string()) + .collect(); + models.sort(); + Ok(Some(models)) + } } diff --git a/crates/goose/src/providers/mod.rs b/crates/goose/src/providers/mod.rs index b5299d1cef..98da6341cd 100644 --- a/crates/goose/src/providers/mod.rs +++ b/crates/goose/src/providers/mod.rs @@ -18,5 +18,6 @@ pub mod openai; pub mod openrouter; pub mod toolshim; pub mod utils; +pub mod venice; pub use factory::{create, providers}; diff --git a/crates/goose/src/providers/openai.rs b/crates/goose/src/providers/openai.rs index 2ecb45ba66..22c516d940 100644 --- a/crates/goose/src/providers/openai.rs +++ b/crates/goose/src/providers/openai.rs @@ -169,6 +169,46 @@ impl Provider for OpenAiProvider { emit_debug_trace(&self.model, &payload, &response, &usage); Ok((message, ProviderUsage::new(model, usage))) } + + /// Fetch supported models from OpenAI; returns Err on any failure, Ok(None) if no data + async fn fetch_supported_models_async(&self) -> Result>, ProviderError> { + // List available models via OpenAI API + let base_url = + url::Url::parse(&self.host).map_err(|e| ProviderError::RequestFailed(e.to_string()))?; + let url = base_url + .join("v1/models") + .map_err(|e| ProviderError::RequestFailed(e.to_string()))?; + let mut request = self.client.get(url).bearer_auth(&self.api_key); + if let Some(org) = &self.organization { + request = request.header("OpenAI-Organization", org); + } + if let Some(project) = &self.project { + request = request.header("OpenAI-Project", project); + } + if let Some(headers) = &self.custom_headers { + for (key, value) in headers { + request = request.header(key, value); + } + } + let response = request.send().await?; + let json: serde_json::Value = response.json().await?; + if let Some(err_obj) = json.get("error") { + let msg = err_obj + .get("message") + .and_then(|v| v.as_str()) + .unwrap_or("unknown error"); + return Err(ProviderError::Authentication(msg.to_string())); + } + let data = json.get("data").and_then(|v| v.as_array()).ok_or_else(|| { + ProviderError::UsageError("Missing data field in JSON response".into()) + })?; + let mut models: Vec = data + .iter() + .filter_map(|m| m.get("id").and_then(|v| v.as_str()).map(str::to_string)) + .collect(); + models.sort(); + Ok(Some(models)) + } } fn parse_custom_headers(s: String) -> HashMap { diff --git a/crates/goose/src/providers/venice.rs b/crates/goose/src/providers/venice.rs new file mode 100644 index 0000000000..81ee9c0b85 --- /dev/null +++ b/crates/goose/src/providers/venice.rs @@ -0,0 +1,587 @@ +use anyhow::Result; +use async_trait::async_trait; +use chrono::Utc; +use reqwest::{Client, Response}; +use serde::{Deserialize, Serialize}; +use serde_json::{json, Value}; +use std::time::Duration; + +use super::base::{ConfigKey, Provider, ProviderMetadata, ProviderUsage, Usage}; +use super::errors::ProviderError; +use crate::message::{Message, MessageContent}; +use crate::model::ModelConfig; +use mcp_core::{tool::Tool, Role, ToolCall, ToolResult}; + +// ---------- Capability Flags ---------- +#[derive(Debug)] +struct CapabilityFlags(String); + +impl CapabilityFlags { + fn from_json(value: &serde_json::Value) -> Self { + let caps = &value["model_spec"]["capabilities"]; + let mut s = String::with_capacity(6); + macro_rules! flag { + ($json_key:literal, $letter:literal) => { + if caps + .get($json_key) + .and_then(|v| v.as_bool()) + .unwrap_or(false) + { + s.push($letter); + } + }; + } + flag!("optimizedForCode", 'c'); // code + flag!("supportsVision", 'v'); // vision + flag!("supportsFunctionCalling", 'f'); + flag!("supportsResponseSchema", 's'); + flag!("supportsWebSearch", 'w'); + flag!("supportsReasoning", 'r'); + CapabilityFlags(s) + } +} + +impl std::fmt::Display for CapabilityFlags { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "[{}]", self.0) // e.g. "[cvfsw]" + } +} +// ---------- END Capability Flags ---------- + +// ---------- Helpers ---------- +/// Return the raw model id (everything before the first space). +fn strip_flags(model: &str) -> &str { + model.split_whitespace().next().unwrap_or(model) +} +// ---------- END Helpers ---------- + +pub const VENICE_DOC_URL: &str = "https://docs.venice.ai/"; +pub const VENICE_DEFAULT_MODEL: &str = "llama-3.3-70b"; +pub const VENICE_DEFAULT_HOST: &str = "https://api.venice.ai"; +pub const VENICE_DEFAULT_BASE_PATH: &str = "api/v1/chat/completions"; +pub const VENICE_DEFAULT_MODELS_PATH: &str = "api/v1/models"; + +// Fallback models to use when API is unavailable +const FALLBACK_MODELS: [&str; 3] = [ + "llama-3.2-3b", // Small model with function calling + "llama-3.3-70b", // Default model with function calling + "mistral-31-24b", // Another model with function calling +]; + +#[derive(Debug, Serialize, Deserialize)] +pub struct VeniceProvider { + #[serde(skip)] + client: Client, + host: String, + base_path: String, + models_path: String, + api_key: String, + model: ModelConfig, +} + +impl Default for VeniceProvider { + fn default() -> Self { + let model = ModelConfig::new(VENICE_DEFAULT_MODEL.to_string()); + VeniceProvider::from_env(model).expect("Failed to initialize Venice provider") + } +} + +impl VeniceProvider { + pub fn from_env(mut model: ModelConfig) -> Result { + let config = crate::config::Config::global(); + let api_key: String = config.get_secret("VENICE_API_KEY")?; + let host: String = config + .get_param("VENICE_HOST") + .unwrap_or_else(|_| VENICE_DEFAULT_HOST.to_string()); + let base_path: String = config + .get_param("VENICE_BASE_PATH") + .unwrap_or_else(|_| VENICE_DEFAULT_BASE_PATH.to_string()); + let models_path: String = config + .get_param("VENICE_MODELS_PATH") + .unwrap_or_else(|_| VENICE_DEFAULT_MODELS_PATH.to_string()); + + // Ensure we only keep the bare model id internally + model.model_name = strip_flags(&model.model_name).to_string(); + + let client = Client::builder() + .timeout(Duration::from_secs(600)) + .build()?; + + let instance = Self { + client, + host, + base_path, + models_path, + api_key, + model, + }; + + Ok(instance) + } + + async fn post(&self, path: &str, body: &str) -> Result { + let base_url = url::Url::parse(&self.host) + .map_err(|e| ProviderError::RequestFailed(format!("Invalid base URL: {e}")))?; + let url = base_url + .join(path) + .map_err(|e| ProviderError::RequestFailed(format!("Failed to construct URL: {e}")))?; + // Choose GET for models endpoint, POST otherwise + let method = if path.contains("models") { + tracing::debug!("Using GET method for models endpoint"); + self.client.get(url.clone()) + } else { + tracing::debug!("Using POST method for completions endpoint"); + self.client.post(url.clone()) + }; + + // Log the request details + tracing::debug!("Venice request URL: {}", url); + tracing::debug!("Venice request body: {}", body); + + let response = method + .header("Authorization", format!("Bearer {}", self.api_key)) + .header("Content-Type", "application/json") + .body(body.to_string()) + .send() + .await?; + + let status = response.status(); + tracing::debug!("Venice response status: {}", status); + + if !status.is_success() { + // Read response body for more details on error + let error_body = response.text().await.unwrap_or_default(); + + // Log full error response for debugging + tracing::debug!("Full Venice error response: {}", error_body); + + // Try to parse the error response + if let Ok(json) = serde_json::from_str::(&error_body) { + // Print the full JSON error for better debugging + println!( + "Venice API error response: {}", + serde_json::to_string_pretty(&json).unwrap_or_else(|_| json.to_string()) + ); + + // Check for tool support errors + if let Some(details) = json.get("details") { + // Specifically look for tool support issues + if let Some(tools) = details.get("tools") { + if let Some(errors) = tools.get("_errors") { + if errors.to_string().contains("not supported by this model") { + let model_name = self.model.model_name.clone(); + return Err(ProviderError::RequestFailed( + format!("The selected model '{}' does not support tool calls. Please select a model that supports tools, such as 'llama-3.3-70b' or 'mistral-31-24b'.", model_name) + )); + } + } + } + } + + // Check for specific error message in context.issues + if let Some(context) = json.get("context") { + if let Some(issues) = context.get("issues") { + if let Some(issues_array) = issues.as_array() { + for issue in issues_array { + if let Some(message) = issue.get("message").and_then(|m| m.as_str()) + { + if message.contains("tools is not supported by this model") { + let model_name = self.model.model_name.clone(); + return Err(ProviderError::RequestFailed( + format!("The selected model '{}' does not support tool calls. Please select a model that supports tools, such as 'llama-3.3-70b' or 'mistral-31-24b'.", model_name) + )); + } + } + } + } + } + } + + // General error extraction + if let Some(error_msg) = json.get("error").and_then(|e| e.as_str()) { + return Err(ProviderError::RequestFailed(format!( + "Venice API error: {}", + error_msg + ))); + } + } + + // Fallback for unparseable errors + return Err(ProviderError::RequestFailed(format!( + "Venice API request failed with status code {}", + status + ))); + } + + Ok(response) + } +} + +#[async_trait] +impl Provider for VeniceProvider { + fn metadata() -> ProviderMetadata { + ProviderMetadata::new( + "venice", + "Venice.ai", + "Venice.ai models (Llama, DeepSeek, Mistral) with function calling", + VENICE_DEFAULT_MODEL, + FALLBACK_MODELS.to_vec(), + VENICE_DOC_URL, + vec![ + ConfigKey::new("VENICE_API_KEY", true, true, None), + ConfigKey::new("VENICE_HOST", true, false, Some(VENICE_DEFAULT_HOST)), + ConfigKey::new( + "VENICE_BASE_PATH", + true, + false, + Some(VENICE_DEFAULT_BASE_PATH), + ), + ConfigKey::new( + "VENICE_MODELS_PATH", + true, + false, + Some(VENICE_DEFAULT_MODELS_PATH), + ), + ], + ) + } + + fn get_model_config(&self) -> ModelConfig { + self.model.clone() + } + + async fn fetch_supported_models_async(&self) -> Result>, ProviderError> { + // Fetch supported models via Venice API + let base_url = url::Url::parse(&self.host) + .map_err(|e| ProviderError::RequestFailed(format!("Invalid base URL: {}", e)))?; + let models_url = base_url.join(&self.models_path).map_err(|e| { + ProviderError::RequestFailed(format!("Failed to construct models URL: {}", e)) + })?; + let response = self + .client + .get(models_url) + .header("Authorization", format!("Bearer {}", self.api_key)) + .send() + .await?; + if !response.status().is_success() { + return Err(ProviderError::RequestFailed(format!( + "Venice API request failed with status {}", + response.status() + ))); + } + let body = response.text().await?; + let json: serde_json::Value = serde_json::from_str(&body) + .map_err(|e| ProviderError::RequestFailed(format!("Failed to parse JSON: {}", e)))?; + + // Print legend once so users know what flags mean + println!( + "Capabilities:\n c=code\n f=function calls (goose supported models)\n s=schema\n v=vision\n w=web search\n r=reasoning" + ); + + let mut models = json["data"] + .as_array() + .ok_or_else(|| ProviderError::RequestFailed("No data field in JSON".to_string()))? + .iter() + .filter_map(|model| { + let id = model["id"].as_str()?.to_owned(); + // Build flags from capabilities + let flags = CapabilityFlags::from_json(model); + // Only include models that support function calling (have 'f' flag) + if flags.0.contains('f') { + Some(format!("{id} {flags}")) + } else { + None + } + }) + .collect::>(); + models.sort(); + Ok(Some(models)) + } + + #[tracing::instrument( + skip(_system, messages, tools), + fields(model_config, input, output, input_tokens, output_tokens, total_tokens) + )] + async fn complete( + &self, + _system: &str, + messages: &[Message], + tools: &[Tool], + ) -> Result<(Message, ProviderUsage), ProviderError> { + // Create properly formatted messages for Venice API + let mut formatted_messages = Vec::new(); + + // Add the system message if present + if !_system.is_empty() { + formatted_messages.push(json!({ + "role": "system", + "content": _system + })); + } + + // Format regular messages according to Venice API requirements + for msg in messages { + // Venice API expects 'content' to be a string, not an array of MessageContent + let content = match msg.role { + Role::User => { + // For user messages, concatenate all text content + let text_content: String = msg + .content + .iter() + .filter_map(|c| c.as_text()) + .collect::>() + .join("\n"); + + // If we have text content, use it directly + if !text_content.is_empty() { + text_content + } else { + // Otherwise, try to get a reasonable string representation + msg.as_concat_text() + } + } + _ => { + // For assistant messages, handle possible tool calls + let has_tool_calls = msg + .content + .iter() + .any(|c| matches!(c, MessageContent::ToolRequest(_))); + + if has_tool_calls { + // If there are tool calls, we'll handle them separately + // Just use an empty string for content + "".to_string() + } else { + // Otherwise use text content + msg.as_concat_text() + } + } + }; + + // Create basic message with content as string + let mut venice_msg = json!({ + "role": match msg.role { + Role::User => "user", + Role::Assistant => "assistant", + }, + "content": content + }); + + // Add debug information to tracing + tracing::debug!( + "Venice message format: role={:?}, content_len={}, has_tool_calls={}", + msg.role, + content.len(), + msg.content + .iter() + .any(|c| matches!(c, MessageContent::ToolRequest(_))) + ); + + // For assistant messages with tool calls, add them in Venice format + if msg.role == Role::Assistant { + let tool_calls: Vec<_> = msg + .content + .iter() + .filter_map(|c| c.as_tool_request()) + .collect(); + + if !tool_calls.is_empty() { + // Transform our tool calls to Venice format + let venice_tool_calls: Vec = tool_calls + .iter() + .filter_map(|tr| { + if let ToolResult::Ok(tool_call) = &tr.tool_call { + // Log tool call details for debugging + tracing::debug!( + "Tool call conversion: id={}, name={}, args_len={}", + tr.id, + tool_call.name, + tool_call.arguments.to_string().len() + ); + + // Convert to Venice format + Some(json!({ + "id": tr.id, + "type": "function", + "function": { + "name": tool_call.name, + "arguments": tool_call.arguments.to_string() + } + })) + } else { + tracing::warn!("Skipping tool call with error: id={}", tr.id); + None + } + }) + .collect(); + + if !venice_tool_calls.is_empty() { + tracing::debug!("Adding {} tool calls to message", venice_tool_calls.len()); + venice_msg["tool_calls"] = json!(venice_tool_calls); + } + } + } + + // For tool messages with tool responses, add required tool_call_id + // Check for tool responses regardless of role - they should have an ID + // that corresponds to the tool call they're responding to + { + let tool_responses: Vec<_> = msg + .content + .iter() + .filter_map(|c| c.as_tool_response()) + .collect(); + + if !tool_responses.is_empty() && !tool_responses[0].id.is_empty() { + venice_msg["tool_call_id"] = json!(tool_responses[0].id); + // Venice expects tool messages to have 'role' = 'tool' + venice_msg["role"] = json!("tool"); + } + } + + formatted_messages.push(venice_msg); + } + + // Build Venice-specific payload + let mut payload = json!({ + "model": strip_flags(&self.model.model_name), + "messages": formatted_messages, + "stream": false, + "temperature": 0.7, + "max_tokens": 2048, + }); + + if !tools.is_empty() { + // Format tools specifically for Venice API + let formatted_tools: Vec = tools + .iter() + .map(|tool| { + // Format each tool in the expected Venice format + json!({ + "type": "function", + "function": { + "name": tool.name, + "description": tool.description, + "parameters": tool.input_schema + } + }) + }) + .collect(); + + payload["tools"] = json!(formatted_tools); + } + + tracing::debug!("Sending request to Venice API"); + tracing::debug!("Venice request payload: {}", payload.to_string()); + + // Send request + let response = self.post(&self.base_path, &payload.to_string()).await?; + + // Parse the response + let response_text = response.text().await?; + let response_json: Value = serde_json::from_str(&response_text).map_err(|e| { + ProviderError::RequestFailed(format!( + "Failed to parse JSON: {}\nResponse: {}", + e, response_text + )) + })?; + + // Handle tool calls from the response if present + let tool_calls = response_json["choices"] + .get(0) + .and_then(|choice| choice["message"]["tool_calls"].as_array()); + + if let Some(tool_calls) = tool_calls { + if !tool_calls.is_empty() { + // Extract tool calls and format for our internal model + let mut content = Vec::new(); + + for tool_call in tool_calls { + let id = tool_call["id"].as_str().unwrap_or("unknown").to_string(); + let function = tool_call["function"].clone(); + let name = function["name"].as_str().unwrap_or("unknown").to_string(); + + // Parse arguments string to Value if it's a string + let arguments = if let Some(args_str) = function["arguments"].as_str() { + serde_json::from_str::(args_str) + .unwrap_or(function["arguments"].clone()) + } else { + function["arguments"].clone() + }; + + // Create a ToolCall using the function name and arguments + let tool_call = ToolCall { name, arguments }; + + // Create a ToolRequest MessageContent + let tool_request = MessageContent::tool_request(id, ToolResult::Ok(tool_call)); + + content.push(tool_request); + } + + // Create message and add each content item + let mut message = Message::assistant(); + for item in content { + message = message.with_content(item); + } + + return Ok(( + message, + ProviderUsage::new( + strip_flags(&self.model.model_name).to_string(), + Usage::default(), + ), + )); + } + } + + // If we get here, it's a regular text response + // Extract content + let content = response_json["choices"] + .get(0) + .and_then(|choice| choice["message"]["content"].as_str()) + .ok_or_else(|| { + tracing::error!("Invalid response format: {:?}", response_json); + ProviderError::RequestFailed("Invalid response format: missing content".to_string()) + })? + .to_string(); + + // Create a vector with a single text content item + let content = vec![MessageContent::text(content)]; + + // Extract usage + let usage_data = &response_json["usage"]; + let usage = Usage { + input_tokens: usage_data["prompt_tokens"].as_i64().map(|v| v as i32), + output_tokens: usage_data["completion_tokens"].as_i64().map(|v| v as i32), + total_tokens: usage_data["total_tokens"].as_i64().map(|v| v as i32), + }; + + Ok(( + Message { + role: Role::Assistant, + created: Utc::now().timestamp(), + content, + }, + ProviderUsage::new(strip_flags(&self.model.model_name).to_string(), usage), + )) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_metadata_structure() { + let metadata = VeniceProvider::metadata(); + + assert_eq!(metadata.default_model, "llama-3.3-70b"); + assert!(!metadata.known_models.is_empty()); + + assert_eq!(metadata.config_keys.len(), 4); + assert_eq!(metadata.config_keys[0].name, "VENICE_API_KEY"); + assert_eq!(metadata.config_keys[1].name, "VENICE_HOST"); + assert_eq!(metadata.config_keys[2].name, "VENICE_BASE_PATH"); + assert_eq!(metadata.config_keys[3].name, "VENICE_MODELS_PATH"); + } +} diff --git a/ui/desktop/src/components/settings/models/hardcoded_stuff.tsx b/ui/desktop/src/components/settings/models/hardcoded_stuff.tsx index d8168e6e87..41b0d1f446 100644 --- a/ui/desktop/src/components/settings/models/hardcoded_stuff.tsx +++ b/ui/desktop/src/components/settings/models/hardcoded_stuff.tsx @@ -47,6 +47,7 @@ export const default_models = { azure_openai: 'gpt-4o', gcp_vertex_ai: 'gemini-2.0-flash-001', aws_bedrock: 'us.anthropic.claude-3-7-sonnet-20250219-v1:0', + venice: 'llama-3.3-70b', }; export function getDefaultModel(key: string): string | undefined { @@ -120,4 +121,5 @@ export const provider_aliases = [ { provider: 'Azure OpenAI', alias: 'azure_openai' }, { provider: 'GCP Vertex AI', alias: 'gcp_vertex_ai' }, { provider: 'AWS Bedrock', alias: 'aws_bedrock' }, + { provider: 'Venice', alias: 'venice' }, ];