From c438eb71af11241650bb3940fdf3cd6a7ec1409d Mon Sep 17 00:00:00 2001 From: Douwe Osinga Date: Sat, 28 Feb 2026 12:06:35 -0500 Subject: [PATCH] WIP --- crates/goose-cli/src/session/export.rs | 1 + crates/goose-server/src/openapi.rs | 1 + crates/goose-server/src/routes/reply.rs | 34 ++- crates/goose/src/action_required_manager.rs | 1 + crates/goose/src/agents/agent.rs | 50 +++- .../src/agents/platform_extensions/browser.rs | 260 ++++++++++++++++++ .../src/agents/platform_extensions/mod.rs | 14 + ui/desktop/openapi.json | 29 ++ ui/desktop/src/api/index.ts | 2 +- ui/desktop/src/api/types.gen.ts | 16 ++ ui/desktop/src/components/BaseChat.tsx | 4 + ui/desktop/src/components/BrowserContext.tsx | 182 ++++++++++++ ui/desktop/src/components/BrowserPanel.tsx | 155 +++++++++++ .../src/components/ChatSessionsContainer.tsx | 26 +- ui/desktop/src/hooks/useChatStream.ts | 46 ++++ .../src/hooks/useForwardedToolHandler.ts | 83 ++++++ ui/desktop/src/main.ts | 3 +- ui/desktop/src/preload.ts | 1 + 18 files changed, 888 insertions(+), 20 deletions(-) create mode 100644 crates/goose/src/agents/platform_extensions/browser.rs create mode 100644 ui/desktop/src/components/BrowserContext.tsx create mode 100644 ui/desktop/src/components/BrowserPanel.tsx create mode 100644 ui/desktop/src/hooks/useForwardedToolHandler.ts diff --git a/crates/goose-cli/src/session/export.rs b/crates/goose-cli/src/session/export.rs index 9b525a2730..0cab28c52b 100644 --- a/crates/goose-cli/src/session/export.rs +++ b/crates/goose-cli/src/session/export.rs @@ -361,6 +361,7 @@ pub fn message_to_markdown(message: &Message, export_all_content: bool) -> Strin .unwrap_or_else(|_| "{}".to_string()) )); } + }, MessageContent::Text(text) => { md.push_str(&text.text); diff --git a/crates/goose-server/src/openapi.rs b/crates/goose-server/src/openapi.rs index 374d08939f..953723d0c8 100644 --- a/crates/goose-server/src/openapi.rs +++ b/crates/goose-server/src/openapi.rs @@ -467,6 +467,7 @@ derive_utoipa!(Icon as IconSchema); goose::prompt_template::Template, super::routes::action_required::ConfirmToolActionRequest, super::routes::reply::ChatRequest, + super::routes::reply::ToolResultPayload, super::routes::session::ImportSessionRequest, super::routes::session::SessionListResponse, super::routes::session::UpdateSessionNameRequest, diff --git a/crates/goose-server/src/routes/reply.rs b/crates/goose-server/src/routes/reply.rs index 0cc40535eb..6eea2fa5a9 100644 --- a/crates/goose-server/src/routes/reply.rs +++ b/crates/goose-server/src/routes/reply.rs @@ -13,6 +13,7 @@ use bytes::Bytes; use futures::{stream::StreamExt, Stream}; use goose::agents::{AgentEvent, SessionConfig}; use goose::conversation::message::{Message, MessageContent, TokenState}; +use rmcp::model::{CallToolResult, Content}; use goose::conversation::Conversation; use goose::session::SessionManager; use rmcp::model::ServerNotification; @@ -76,6 +77,17 @@ fn track_tool_telemetry(content: &MessageContent, all_messages: &[Message]) { } } +#[derive(Debug, Deserialize, Serialize, utoipa::ToSchema)] +pub struct ToolResultPayload { + /// The tool_call_id that this result corresponds to + pub tool_call_id: String, + /// The result content from the client-side tool execution + pub result: String, + /// Whether the tool execution was an error + #[serde(default)] + pub is_error: bool, +} + #[derive(Debug, Deserialize, Serialize, utoipa::ToSchema)] pub struct ChatRequest { user_message: Message, @@ -87,6 +99,11 @@ pub struct ChatRequest { session_id: String, recipe_name: Option, recipe_version: Option, + /// When present, this reply contains a tool result from client-side execution + /// (e.g., browser commands). The server will prepend the tool response to the + /// conversation before continuing the agent loop. + #[serde(default)] + tool_result: Option, } pub struct SseResponse { @@ -242,7 +259,21 @@ pub async fn reply( let stream = ReceiverStream::new(rx); let cancel_token = CancellationToken::new(); - let user_message = request.user_message; + let user_message = if let Some(tr) = request.tool_result { + let content = vec![Content::text(&tr.result)]; + let call_result = if tr.is_error { + Err(rmcp::model::ErrorData::new( + rmcp::model::ErrorCode::INTERNAL_ERROR, + tr.result, + None, + )) + } else { + Ok(CallToolResult::success(content)) + }; + Message::user().with_tool_response(tr.tool_call_id, call_result) + } else { + request.user_message + }; let override_conversation = request.override_conversation; let task_cancel = cancel_token.clone(); @@ -496,6 +527,7 @@ mod tests { session_id: "test-session".to_string(), recipe_name: None, recipe_version: None, + tool_result: None, }) .unwrap(), )) diff --git a/crates/goose/src/action_required_manager.rs b/crates/goose/src/action_required_manager.rs index a35b52d733..0b76525b47 100644 --- a/crates/goose/src/action_required_manager.rs +++ b/crates/goose/src/action_required_manager.rs @@ -96,4 +96,5 @@ impl ActionRequiredManager { Ok(()) } + } diff --git a/crates/goose/src/agents/agent.rs b/crates/goose/src/agents/agent.rs index e4134dd79d..7faa0be2bd 100644 --- a/crates/goose/src/agents/agent.rs +++ b/crates/goose/src/agents/agent.rs @@ -1174,6 +1174,7 @@ impl Agent { let mut messages_to_add = Conversation::default(); let mut tools_updated = false; let mut did_recovery_compact_this_iteration = false; + let mut has_forward_to_client = false; while let Some(next) = stream.next().await { if is_token_cancelled(&cancel_token) { @@ -1250,6 +1251,7 @@ impl Agent { yield AgentEvent::Message(msg); } } + let mut forward_to_client_ids: std::collections::HashSet = std::collections::HashSet::new(); if goose_mode == GooseMode::Chat { // Skip all remaining tool calls in chat mode for request in remaining_requests.iter() { @@ -1372,9 +1374,17 @@ impl Agent { yield AgentEvent::McpNotification((request_id.clone(), server_notification)); } } + + if meta.0.get("forward_to_client").and_then(|v| v.as_bool()).unwrap_or(false) { + forward_to_client_ids.insert(request_id.clone()); + } } } + if forward_to_client_ids.contains(&request_id) { + continue; + } + if enable_extension_request_ids.contains(&request_id) && output.is_err() { @@ -1438,6 +1448,8 @@ impl Agent { for (idx, request) in frontend_requests.iter().chain(remaining_requests.iter()).enumerate() { if request.tool_call.is_ok() { + let is_forwarded = forward_to_client_ids.contains(&request.id); + let mut request_msg = Message::assistant() .with_id(format!("msg_{}", Uuid::new_v4())); @@ -1455,11 +1467,37 @@ impl Agent { request.metadata.as_ref(), request.tool_meta.clone(), ); - messages_to_add.push(request_msg); - let final_response = tool_response_messages[idx] - .lock().await.clone(); - yield AgentEvent::Message(final_response.clone()); - messages_to_add.push(final_response); + + if is_forwarded { + // For forwarded requests, annotate the tool request with + // forward_to_client so the client knows to execute it locally + // and call back with the result via /reply. + let mut forwarded_msg = Message::assistant() + .with_id(format!("msg_{}", Uuid::new_v4())); + for rc in &reasoning_content { + forwarded_msg = forwarded_msg.with_content(rc.clone()); + } + let mut meta = request.tool_meta.clone() + .unwrap_or_else(|| serde_json::json!({})); + meta.as_object_mut().unwrap() + .insert("forward_to_client".to_string(), serde_json::json!(true)); + forwarded_msg = forwarded_msg + .with_tool_request_with_metadata( + request.id.clone(), + request.tool_call.clone(), + request.metadata.as_ref(), + Some(meta), + ); + yield AgentEvent::Message(forwarded_msg.clone()); + messages_to_add.push(forwarded_msg); + has_forward_to_client = true; + } else { + messages_to_add.push(request_msg); + let final_response = tool_response_messages[idx] + .lock().await.clone(); + yield AgentEvent::Message(final_response.clone()); + messages_to_add.push(final_response); + } } else { let error_msg = format!( "[system: Tool call could not be parsed: {}. The response may have been truncated. Try breaking the task into smaller steps.]", @@ -1566,7 +1604,7 @@ impl Agent { (tools, toolshim_tools, system_prompt) = self.prepare_tools_and_prompt(&session_config.id, &session.working_dir).await?; } - let mut exit_chat = false; + let mut exit_chat = has_forward_to_client; if no_tools_called { if let Some(final_output_tool) = self.final_output_tool.lock().await.as_ref() { if final_output_tool.final_output.is_none() { diff --git a/crates/goose/src/agents/platform_extensions/browser.rs b/crates/goose/src/agents/platform_extensions/browser.rs new file mode 100644 index 0000000000..77c28bc8e6 --- /dev/null +++ b/crates/goose/src/agents/platform_extensions/browser.rs @@ -0,0 +1,260 @@ +use crate::agents::extension::PlatformExtensionContext; +use crate::agents::mcp_client::{Error, McpClientTrait}; +use async_trait::async_trait; +use rmcp::model::{ + CallToolResult, Content, Implementation, InitializeResult, JsonObject, ListResourcesResult, + ListToolsResult, Meta, ProtocolVersion, ReadResourceResult, ServerCapabilities, + Tool as McpTool, ToolsCapability, +}; +use schemars::{schema_for, JsonSchema}; +use serde::{Deserialize, Serialize}; +use serde_json::json; +use tokio_util::sync::CancellationToken; + +pub static EXTENSION_NAME: &str = "browser"; + +#[derive(Debug, Serialize, Deserialize, JsonSchema)] +struct BrowserOpenParams { + /// URL to open in the browser + url: String, +} + +#[derive(Debug, Serialize, Deserialize, JsonSchema)] +struct BrowserNavigateParams { + /// URL to navigate to + url: String, +} + +#[derive(Debug, Serialize, Deserialize, JsonSchema)] +struct BrowserClickParams { + /// CSS selector for the element to click + selector: String, +} + +#[derive(Debug, Serialize, Deserialize, JsonSchema)] +struct BrowserTypeParams { + /// CSS selector for the input element + selector: String, + /// Text to type into the element + text: String, +} + +#[derive(Debug, Serialize, Deserialize, JsonSchema)] +struct BrowserGetTextParams { + /// CSS selector (defaults to "body" if not provided) + selector: Option, +} + +#[derive(Debug, Serialize, Deserialize, JsonSchema)] +struct BrowserGetHtmlParams { + /// CSS selector (defaults to "body" if not provided) + selector: Option, +} + +#[derive(Debug, Serialize, Deserialize, JsonSchema)] +struct BrowserEvaluateParams { + /// JavaScript code to execute in the page context + script: String, +} + +#[derive(Debug, Serialize, Deserialize, JsonSchema)] +struct BrowserWaitParams { + /// CSS selector to wait for + selector: String, + /// Timeout in milliseconds (default: 5000) + timeout_ms: Option, +} + +#[derive(Debug, Serialize, Deserialize, JsonSchema)] +struct BrowserScrollParams { + /// Direction to scroll: "up", "down", "top", or "bottom" + direction: String, + /// Number of pixels to scroll (optional, defaults to one viewport) + amount: Option, +} + +#[derive(Debug, Serialize, Deserialize, JsonSchema)] +struct BrowserScreenshotParams {} + +#[derive(Debug, Serialize, Deserialize, JsonSchema)] +struct BrowserCloseParams {} + +pub struct BrowserClient { + #[allow(dead_code)] + context: PlatformExtensionContext, + info: InitializeResult, +} + +impl BrowserClient { + pub fn new(context: PlatformExtensionContext) -> Self { + let info = InitializeResult { + protocol_version: ProtocolVersion::V_2025_03_26, + capabilities: ServerCapabilities { + tools: Some(ToolsCapability { + list_changed: Some(false), + }), + tasks: None, + resources: None, + extensions: None, + prompts: None, + completions: None, + experimental: None, + logging: None, + }, + server_info: Implementation { + name: EXTENSION_NAME.to_string(), + description: None, + title: Some("Browser".to_string()), + version: "1.0.0".to_string(), + icons: None, + website_url: None, + }, + instructions: Some( + "Control an embedded browser window for web navigation and content extraction." + .to_string(), + ), + }; + Self { context, info } + } + + fn schema() -> JsonObject { + let schema = schema_for!(T); + let schema_value = + serde_json::to_value(schema).expect("Failed to serialize schema"); + schema_value.as_object().unwrap().clone() + } + + fn forward_to_client(command: &str, arguments: Option) -> CallToolResult { + let params = arguments + .map(serde_json::Value::Object) + .unwrap_or(json!({})); + + let content = json!({ + "command": command, + "params": params, + }); + + let mut meta_map = serde_json::Map::new(); + meta_map.insert("forward_to_client".to_string(), json!(true)); + + CallToolResult { + content: vec![Content::text(content.to_string())], + structured_content: None, + is_error: Some(false), + meta: Some(Meta(meta_map)), + } + } +} + +#[async_trait] +impl McpClientTrait for BrowserClient { + async fn list_tools( + &self, + _session_id: &str, + _next_cursor: Option, + _cancel_token: CancellationToken, + ) -> Result { + let tools = vec![ + McpTool::new( + "browser_open".to_string(), + "Open a URL in the embedded browser. This will display a browser panel in the UI." + .to_string(), + Self::schema::(), + ), + McpTool::new( + "browser_close".to_string(), + "Close the embedded browser panel.".to_string(), + Self::schema::(), + ), + McpTool::new( + "browser_navigate".to_string(), + "Navigate to a different URL in the browser.".to_string(), + Self::schema::(), + ), + McpTool::new( + "browser_screenshot".to_string(), + "Take a screenshot of the current browser view.".to_string(), + Self::schema::(), + ), + McpTool::new( + "browser_click".to_string(), + "Click an element in the page by CSS selector.".to_string(), + Self::schema::(), + ), + McpTool::new( + "browser_type".to_string(), + "Type text into an input element.".to_string(), + Self::schema::(), + ), + McpTool::new( + "browser_get_text".to_string(), + "Get the text content of an element (default: body).".to_string(), + Self::schema::(), + ), + McpTool::new( + "browser_get_html".to_string(), + "Get the HTML content of an element (default: body).".to_string(), + Self::schema::(), + ), + McpTool::new( + "browser_evaluate".to_string(), + "Execute JavaScript in the page context and return the result.".to_string(), + Self::schema::(), + ), + McpTool::new( + "browser_wait".to_string(), + "Wait for an element to appear in the page.".to_string(), + Self::schema::(), + ), + McpTool::new( + "browser_scroll".to_string(), + "Scroll the page. Direction can be 'up', 'down', 'top', or 'bottom'.".to_string(), + Self::schema::(), + ), + ]; + + Ok(ListToolsResult { + tools, + next_cursor: None, + meta: None, + }) + } + + async fn call_tool( + &self, + _session_id: &str, + name: &str, + arguments: Option, + _working_dir: Option<&str>, + _cancel_token: CancellationToken, + ) -> Result { + let command = name.strip_prefix("browser_").unwrap_or(name); + Ok(Self::forward_to_client(command, arguments)) + } + + async fn list_resources( + &self, + _session_id: &str, + _next_cursor: Option, + _cancel_token: CancellationToken, + ) -> Result { + Ok(ListResourcesResult { + resources: vec![], + next_cursor: None, + meta: None, + }) + } + + async fn read_resource( + &self, + _session_id: &str, + _uri: &str, + _cancel_token: CancellationToken, + ) -> Result { + Err(Error::TransportClosed) + } + + fn get_info(&self) -> Option<&InitializeResult> { + Some(&self.info) + } +} diff --git a/crates/goose/src/agents/platform_extensions/mod.rs b/crates/goose/src/agents/platform_extensions/mod.rs index 1b8ff10f0f..92ad5490f3 100644 --- a/crates/goose/src/agents/platform_extensions/mod.rs +++ b/crates/goose/src/agents/platform_extensions/mod.rs @@ -1,4 +1,5 @@ pub mod apps; +pub mod browser; pub mod chatrecall; #[cfg(feature = "code-mode")] pub mod code_execution; @@ -130,6 +131,19 @@ pub static PLATFORM_EXTENSIONS: Lazy }, ); + map.insert( + browser::EXTENSION_NAME, + PlatformExtensionDef { + name: browser::EXTENSION_NAME, + display_name: "Browser", + description: + "Control an embedded browser window to navigate, interact with, and extract content from web pages", + default_enabled: false, + unprefixed_tools: false, + client_factory: |ctx| Box::new(browser::BrowserClient::new(ctx)), + }, + ); + map }, ); diff --git a/ui/desktop/openapi.json b/ui/desktop/openapi.json index 35921d8cc9..58f1d5fce8 100644 --- a/ui/desktop/openapi.json +++ b/ui/desktop/openapi.json @@ -3917,6 +3917,14 @@ "session_id": { "type": "string" }, + "tool_result": { + "allOf": [ + { + "$ref": "#/components/schemas/ToolResultPayload" + } + ], + "nullable": true + }, "user_message": { "$ref": "#/components/schemas/Message" } @@ -8050,6 +8058,27 @@ } } }, + "ToolResultPayload": { + "type": "object", + "required": [ + "tool_call_id", + "result" + ], + "properties": { + "is_error": { + "type": "boolean", + "description": "Whether the tool execution was an error" + }, + "result": { + "type": "string", + "description": "The result content from the client-side tool execution" + }, + "tool_call_id": { + "type": "string", + "description": "The tool_call_id that this result corresponds to" + } + } + }, "TranscribeRequest": { "type": "object", "required": [ diff --git a/ui/desktop/src/api/index.ts b/ui/desktop/src/api/index.ts index 8af4598d0e..10b06582bd 100644 --- a/ui/desktop/src/api/index.ts +++ b/ui/desktop/src/api/index.ts @@ -1,4 +1,4 @@ // This file is auto-generated by @hey-api/openapi-ts export { addExtension, agentAddExtension, agentRemoveExtension, backupConfig, callTool, cancelDownload, cancelLocalModelDownload, checkProvider, configureProviderOauth, confirmToolAction, createCustomProvider, createRecipe, createSchedule, decodeRecipe, deleteLocalModel, deleteModel, deleteRecipe, deleteSchedule, deleteSession, detectProvider, diagnostics, downloadHfModel, downloadModel, encodeRecipe, exportApp, exportSession, forkSession, getCanonicalModelInfo, getCustomProvider, getDictationConfig, getDownloadProgress, getExtensions, getLocalModelDownloadProgress, getModelSettings, getPrompt, getPrompts, getProviderCatalog, getProviderCatalogTemplate, getProviderModels, getRepoFiles, getSession, getSessionExtensions, getSessionInsights, getSlashCommands, getTools, getTunnelStatus, importApp, importSession, initConfig, inspectRunningJob, killRunningJob, listApps, listLocalModels, listModels, listRecipes, listSchedules, listSessions, mcpUiProxy, type Options, parseRecipe, pauseSchedule, providers, readAllConfig, readConfig, readResource, recipeToYaml, recoverConfig, removeConfig, removeCustomProvider, removeExtension, reply, resetPrompt, restartAgent, resumeAgent, runNowHandler, savePrompt, saveRecipe, scanRecipe, scheduleRecipe, searchHfModels, searchSessions, sendTelemetryEvent, sessionsHandler, setConfigProvider, setRecipeSlashCommand, startAgent, startOpenrouterSetup, startTetrateSetup, startTunnel, status, stopAgent, stopTunnel, systemInfo, transcribeDictation, unpauseSchedule, updateAgentProvider, updateCustomProvider, updateFromSession, updateModelSettings, updateSchedule, updateSessionName, updateSessionUserRecipeValues, updateWorkingDir, upsertConfig, upsertPermissions, validateConfig } from './sdk.gen'; -export type { ActionRequired, ActionRequiredData, AddExtensionData, AddExtensionErrors, AddExtensionRequest, AddExtensionResponse, AddExtensionResponses, AgentAddExtensionData, AgentAddExtensionErrors, AgentAddExtensionResponse, AgentAddExtensionResponses, AgentRemoveExtensionData, AgentRemoveExtensionErrors, AgentRemoveExtensionResponse, AgentRemoveExtensionResponses, Annotations, Author, AuthorRequest, BackupConfigData, BackupConfigErrors, BackupConfigResponse, BackupConfigResponses, CallToolData, CallToolErrors, CallToolRequest, CallToolResponse, CallToolResponse2, CallToolResponses, CancelDownloadData, CancelDownloadErrors, CancelDownloadResponses, CancelLocalModelDownloadData, CancelLocalModelDownloadErrors, CancelLocalModelDownloadResponses, ChatRequest, CheckProviderData, CheckProviderRequest, ClientOptions, CommandType, ConfigKey, ConfigKeyQuery, ConfigResponse, ConfigureProviderOauthData, ConfigureProviderOauthErrors, ConfigureProviderOauthResponses, ConfirmToolActionData, ConfirmToolActionErrors, ConfirmToolActionRequest, ConfirmToolActionResponses, Content, Conversation, CreateCustomProviderData, CreateCustomProviderErrors, CreateCustomProviderResponse, CreateCustomProviderResponses, CreateRecipeData, CreateRecipeErrors, CreateRecipeRequest, CreateRecipeResponse, CreateRecipeResponse2, CreateRecipeResponses, CreateScheduleData, CreateScheduleErrors, CreateScheduleRequest, CreateScheduleResponse, CreateScheduleResponses, CspMetadata, DeclarativeProviderConfig, DecodeRecipeData, DecodeRecipeErrors, DecodeRecipeRequest, DecodeRecipeResponse, DecodeRecipeResponse2, DecodeRecipeResponses, DeleteLocalModelData, DeleteLocalModelErrors, DeleteLocalModelResponses, DeleteModelData, DeleteModelErrors, DeleteModelResponses, DeleteRecipeData, DeleteRecipeErrors, DeleteRecipeRequest, DeleteRecipeResponse, DeleteRecipeResponses, DeleteScheduleData, DeleteScheduleErrors, DeleteScheduleResponse, DeleteScheduleResponses, DeleteSessionData, DeleteSessionErrors, DeleteSessionResponses, DetectProviderData, DetectProviderErrors, DetectProviderRequest, DetectProviderResponse, DetectProviderResponse2, DetectProviderResponses, DiagnosticsData, DiagnosticsErrors, DiagnosticsResponse, DiagnosticsResponses, DictationProvider, DictationProviderStatus, DownloadHfModelData, DownloadHfModelErrors, DownloadHfModelResponse, DownloadHfModelResponses, DownloadModelData, DownloadModelErrors, DownloadModelRequest, DownloadModelResponses, DownloadProgress, DownloadStatus, EmbeddedResource, EncodeRecipeData, EncodeRecipeErrors, EncodeRecipeRequest, EncodeRecipeResponse, EncodeRecipeResponse2, EncodeRecipeResponses, Envs, ErrorResponse, ExportAppData, ExportAppError, ExportAppErrors, ExportAppResponse, ExportAppResponses, ExportSessionData, ExportSessionErrors, ExportSessionResponse, ExportSessionResponses, ExtensionConfig, ExtensionData, ExtensionEntry, ExtensionLoadResult, ExtensionQuery, ExtensionResponse, ForkRequest, ForkResponse, ForkSessionData, ForkSessionErrors, ForkSessionResponse, ForkSessionResponses, FrontendToolRequest, GetCanonicalModelInfoData, GetCanonicalModelInfoResponse, GetCanonicalModelInfoResponses, GetCustomProviderData, GetCustomProviderErrors, GetCustomProviderResponse, GetCustomProviderResponses, GetDictationConfigData, GetDictationConfigResponse, GetDictationConfigResponses, GetDownloadProgressData, GetDownloadProgressErrors, GetDownloadProgressResponse, GetDownloadProgressResponses, GetExtensionsData, GetExtensionsErrors, GetExtensionsResponse, GetExtensionsResponses, GetLocalModelDownloadProgressData, GetLocalModelDownloadProgressErrors, GetLocalModelDownloadProgressResponse, GetLocalModelDownloadProgressResponses, GetModelSettingsData, GetModelSettingsErrors, GetModelSettingsResponse, GetModelSettingsResponses, GetPromptData, GetPromptErrors, GetPromptResponse, GetPromptResponses, GetPromptsData, GetPromptsResponse, GetPromptsResponses, GetProviderCatalogData, GetProviderCatalogErrors, GetProviderCatalogResponse, GetProviderCatalogResponses, GetProviderCatalogTemplateData, GetProviderCatalogTemplateErrors, GetProviderCatalogTemplateResponse, GetProviderCatalogTemplateResponses, GetProviderModelsData, GetProviderModelsErrors, GetProviderModelsResponse, GetProviderModelsResponses, GetRepoFilesData, GetRepoFilesResponse, GetRepoFilesResponses, GetSessionData, GetSessionErrors, GetSessionExtensionsData, GetSessionExtensionsErrors, GetSessionExtensionsResponse, GetSessionExtensionsResponses, GetSessionInsightsData, GetSessionInsightsErrors, GetSessionInsightsResponse, GetSessionInsightsResponses, GetSessionResponse, GetSessionResponses, GetSlashCommandsData, GetSlashCommandsResponse, GetSlashCommandsResponses, GetToolsData, GetToolsErrors, GetToolsQuery, GetToolsResponse, GetToolsResponses, GetTunnelStatusData, GetTunnelStatusResponse, GetTunnelStatusResponses, GooseApp, HfGgufFile, HfModelInfo, HfQuantVariant, Icon, ImageContent, ImportAppData, ImportAppError, ImportAppErrors, ImportAppRequest, ImportAppResponse, ImportAppResponse2, ImportAppResponses, ImportSessionData, ImportSessionErrors, ImportSessionRequest, ImportSessionResponse, ImportSessionResponses, InitConfigData, InitConfigErrors, InitConfigResponse, InitConfigResponses, InspectJobResponse, InspectRunningJobData, InspectRunningJobErrors, InspectRunningJobResponse, InspectRunningJobResponses, JsonObject, KillJobResponse, KillRunningJobData, KillRunningJobResponses, ListAppsData, ListAppsError, ListAppsErrors, ListAppsRequest, ListAppsResponse, ListAppsResponse2, ListAppsResponses, ListLocalModelsData, ListLocalModelsResponse, ListLocalModelsResponses, ListModelsData, ListModelsResponse, ListModelsResponses, ListRecipeResponse, ListRecipesData, ListRecipesErrors, ListRecipesResponse, ListRecipesResponses, ListSchedulesData, ListSchedulesErrors, ListSchedulesResponse, ListSchedulesResponse2, ListSchedulesResponses, ListSessionsData, ListSessionsErrors, ListSessionsResponse, ListSessionsResponses, LoadedProvider, LocalModelResponse, McpAppResource, McpUiProxyData, McpUiProxyErrors, McpUiProxyResponses, Message, MessageContent, MessageEvent, MessageMetadata, ModelCapabilities, ModelConfig, ModelDownloadStatus, ModelInfo, ModelInfoData, ModelInfoQuery, ModelInfoResponse, ModelSettings, ModelTemplate, ParseRecipeData, ParseRecipeError, ParseRecipeErrors, ParseRecipeRequest, ParseRecipeResponse, ParseRecipeResponse2, ParseRecipeResponses, PauseScheduleData, PauseScheduleErrors, PauseScheduleResponse, PauseScheduleResponses, Permission, PermissionLevel, PermissionsMetadata, PrincipalType, PromptContentResponse, PromptsListResponse, ProviderCatalogEntry, ProviderDetails, ProviderEngine, ProviderMetadata, ProvidersData, ProvidersResponse, ProvidersResponse2, ProvidersResponses, ProviderTemplate, ProviderType, RawAudioContent, RawEmbeddedResource, RawImageContent, RawResource, RawTextContent, ReadAllConfigData, ReadAllConfigResponse, ReadAllConfigResponses, ReadConfigData, ReadConfigErrors, ReadConfigResponses, ReadResourceData, ReadResourceErrors, ReadResourceRequest, ReadResourceResponse, ReadResourceResponse2, ReadResourceResponses, ReasoningContent, Recipe, RecipeManifest, RecipeParameter, RecipeParameterInputType, RecipeParameterRequirement, RecipeToYamlData, RecipeToYamlError, RecipeToYamlErrors, RecipeToYamlRequest, RecipeToYamlResponse, RecipeToYamlResponse2, RecipeToYamlResponses, RecoverConfigData, RecoverConfigErrors, RecoverConfigResponse, RecoverConfigResponses, RedactedThinkingContent, RemoveConfigData, RemoveConfigErrors, RemoveConfigResponse, RemoveConfigResponses, RemoveCustomProviderData, RemoveCustomProviderErrors, RemoveCustomProviderResponse, RemoveCustomProviderResponses, RemoveExtensionData, RemoveExtensionErrors, RemoveExtensionRequest, RemoveExtensionResponse, RemoveExtensionResponses, ReplyData, ReplyErrors, ReplyResponse, ReplyResponses, RepoVariantsResponse, ResetPromptData, ResetPromptErrors, ResetPromptResponse, ResetPromptResponses, ResourceContents, ResourceMetadata, Response, RestartAgentData, RestartAgentErrors, RestartAgentRequest, RestartAgentResponse, RestartAgentResponse2, RestartAgentResponses, ResumeAgentData, ResumeAgentErrors, ResumeAgentRequest, ResumeAgentResponse, ResumeAgentResponse2, ResumeAgentResponses, RetryConfig, Role, RunNowHandlerData, RunNowHandlerErrors, RunNowHandlerResponse, RunNowHandlerResponses, RunNowResponse, SamplingConfig, SavePromptData, SavePromptErrors, SavePromptRequest, SavePromptResponse, SavePromptResponses, SaveRecipeData, SaveRecipeError, SaveRecipeErrors, SaveRecipeRequest, SaveRecipeResponse, SaveRecipeResponse2, SaveRecipeResponses, ScanRecipeData, ScanRecipeRequest, ScanRecipeResponse, ScanRecipeResponse2, ScanRecipeResponses, ScheduledJob, ScheduleRecipeData, ScheduleRecipeErrors, ScheduleRecipeRequest, ScheduleRecipeResponses, SearchHfModelsData, SearchHfModelsErrors, SearchHfModelsResponse, SearchHfModelsResponses, SearchSessionsData, SearchSessionsErrors, SearchSessionsResponse, SearchSessionsResponses, SendTelemetryEventData, SendTelemetryEventResponses, Session, SessionDisplayInfo, SessionExtensionsResponse, SessionInsights, SessionListResponse, SessionsHandlerData, SessionsHandlerErrors, SessionsHandlerResponse, SessionsHandlerResponses, SessionsQuery, SessionType, SetConfigProviderData, SetProviderRequest, SetRecipeSlashCommandData, SetRecipeSlashCommandErrors, SetRecipeSlashCommandResponses, SetSlashCommandRequest, Settings, SetupResponse, SlashCommand, SlashCommandsResponse, StartAgentData, StartAgentError, StartAgentErrors, StartAgentRequest, StartAgentResponse, StartAgentResponses, StartOpenrouterSetupData, StartOpenrouterSetupResponse, StartOpenrouterSetupResponses, StartTetrateSetupData, StartTetrateSetupResponse, StartTetrateSetupResponses, StartTunnelData, StartTunnelError, StartTunnelErrors, StartTunnelResponse, StartTunnelResponses, StatusData, StatusResponse, StatusResponses, StopAgentData, StopAgentErrors, StopAgentRequest, StopAgentResponse, StopAgentResponses, StopTunnelData, StopTunnelError, StopTunnelErrors, StopTunnelResponses, SubRecipe, SuccessCheck, SystemInfo, SystemInfoData, SystemInfoResponse, SystemInfoResponses, SystemNotificationContent, SystemNotificationType, TaskSupport, TelemetryEventRequest, Template, TextContent, ThinkingContent, TokenState, Tool, ToolAnnotations, ToolConfirmationRequest, ToolExecution, ToolInfo, ToolPermission, ToolRequest, ToolResponse, TranscribeDictationData, TranscribeDictationErrors, TranscribeDictationResponse, TranscribeDictationResponses, TranscribeRequest, TranscribeResponse, TunnelInfo, TunnelState, UiMetadata, UnpauseScheduleData, UnpauseScheduleErrors, UnpauseScheduleResponse, UnpauseScheduleResponses, UpdateAgentProviderData, UpdateAgentProviderErrors, UpdateAgentProviderResponses, UpdateCustomProviderData, UpdateCustomProviderErrors, UpdateCustomProviderRequest, UpdateCustomProviderResponse, UpdateCustomProviderResponses, UpdateFromSessionData, UpdateFromSessionErrors, UpdateFromSessionRequest, UpdateFromSessionResponses, UpdateModelSettingsData, UpdateModelSettingsErrors, UpdateModelSettingsResponse, UpdateModelSettingsResponses, UpdateProviderRequest, UpdateScheduleData, UpdateScheduleErrors, UpdateScheduleRequest, UpdateScheduleResponse, UpdateScheduleResponses, UpdateSessionNameData, UpdateSessionNameErrors, UpdateSessionNameRequest, UpdateSessionNameResponses, UpdateSessionUserRecipeValuesData, UpdateSessionUserRecipeValuesError, UpdateSessionUserRecipeValuesErrors, UpdateSessionUserRecipeValuesRequest, UpdateSessionUserRecipeValuesResponse, UpdateSessionUserRecipeValuesResponse2, UpdateSessionUserRecipeValuesResponses, UpdateWorkingDirData, UpdateWorkingDirErrors, UpdateWorkingDirRequest, UpdateWorkingDirResponses, UpsertConfigData, UpsertConfigErrors, UpsertConfigQuery, UpsertConfigResponse, UpsertConfigResponses, UpsertPermissionsData, UpsertPermissionsErrors, UpsertPermissionsQuery, UpsertPermissionsResponse, UpsertPermissionsResponses, ValidateConfigData, ValidateConfigErrors, ValidateConfigResponse, ValidateConfigResponses, WhisperModelResponse, WindowProps } from './types.gen'; +export type { ActionRequired, ActionRequiredData, AddExtensionData, AddExtensionErrors, AddExtensionRequest, AddExtensionResponse, AddExtensionResponses, AgentAddExtensionData, AgentAddExtensionErrors, AgentAddExtensionResponse, AgentAddExtensionResponses, AgentRemoveExtensionData, AgentRemoveExtensionErrors, AgentRemoveExtensionResponse, AgentRemoveExtensionResponses, Annotations, Author, AuthorRequest, BackupConfigData, BackupConfigErrors, BackupConfigResponse, BackupConfigResponses, CallToolData, CallToolErrors, CallToolRequest, CallToolResponse, CallToolResponse2, CallToolResponses, CancelDownloadData, CancelDownloadErrors, CancelDownloadResponses, CancelLocalModelDownloadData, CancelLocalModelDownloadErrors, CancelLocalModelDownloadResponses, ChatRequest, CheckProviderData, CheckProviderRequest, ClientOptions, CommandType, ConfigKey, ConfigKeyQuery, ConfigResponse, ConfigureProviderOauthData, ConfigureProviderOauthErrors, ConfigureProviderOauthResponses, ConfirmToolActionData, ConfirmToolActionErrors, ConfirmToolActionRequest, ConfirmToolActionResponses, Content, Conversation, CreateCustomProviderData, CreateCustomProviderErrors, CreateCustomProviderResponse, CreateCustomProviderResponses, CreateRecipeData, CreateRecipeErrors, CreateRecipeRequest, CreateRecipeResponse, CreateRecipeResponse2, CreateRecipeResponses, CreateScheduleData, CreateScheduleErrors, CreateScheduleRequest, CreateScheduleResponse, CreateScheduleResponses, CspMetadata, DeclarativeProviderConfig, DecodeRecipeData, DecodeRecipeErrors, DecodeRecipeRequest, DecodeRecipeResponse, DecodeRecipeResponse2, DecodeRecipeResponses, DeleteLocalModelData, DeleteLocalModelErrors, DeleteLocalModelResponses, DeleteModelData, DeleteModelErrors, DeleteModelResponses, DeleteRecipeData, DeleteRecipeErrors, DeleteRecipeRequest, DeleteRecipeResponse, DeleteRecipeResponses, DeleteScheduleData, DeleteScheduleErrors, DeleteScheduleResponse, DeleteScheduleResponses, DeleteSessionData, DeleteSessionErrors, DeleteSessionResponses, DetectProviderData, DetectProviderErrors, DetectProviderRequest, DetectProviderResponse, DetectProviderResponse2, DetectProviderResponses, DiagnosticsData, DiagnosticsErrors, DiagnosticsResponse, DiagnosticsResponses, DictationProvider, DictationProviderStatus, DownloadHfModelData, DownloadHfModelErrors, DownloadHfModelResponse, DownloadHfModelResponses, DownloadModelData, DownloadModelErrors, DownloadModelRequest, DownloadModelResponses, DownloadProgress, DownloadStatus, EmbeddedResource, EncodeRecipeData, EncodeRecipeErrors, EncodeRecipeRequest, EncodeRecipeResponse, EncodeRecipeResponse2, EncodeRecipeResponses, Envs, ErrorResponse, ExportAppData, ExportAppError, ExportAppErrors, ExportAppResponse, ExportAppResponses, ExportSessionData, ExportSessionErrors, ExportSessionResponse, ExportSessionResponses, ExtensionConfig, ExtensionData, ExtensionEntry, ExtensionLoadResult, ExtensionQuery, ExtensionResponse, ForkRequest, ForkResponse, ForkSessionData, ForkSessionErrors, ForkSessionResponse, ForkSessionResponses, FrontendToolRequest, GetCanonicalModelInfoData, GetCanonicalModelInfoResponse, GetCanonicalModelInfoResponses, GetCustomProviderData, GetCustomProviderErrors, GetCustomProviderResponse, GetCustomProviderResponses, GetDictationConfigData, GetDictationConfigResponse, GetDictationConfigResponses, GetDownloadProgressData, GetDownloadProgressErrors, GetDownloadProgressResponse, GetDownloadProgressResponses, GetExtensionsData, GetExtensionsErrors, GetExtensionsResponse, GetExtensionsResponses, GetLocalModelDownloadProgressData, GetLocalModelDownloadProgressErrors, GetLocalModelDownloadProgressResponse, GetLocalModelDownloadProgressResponses, GetModelSettingsData, GetModelSettingsErrors, GetModelSettingsResponse, GetModelSettingsResponses, GetPromptData, GetPromptErrors, GetPromptResponse, GetPromptResponses, GetPromptsData, GetPromptsResponse, GetPromptsResponses, GetProviderCatalogData, GetProviderCatalogErrors, GetProviderCatalogResponse, GetProviderCatalogResponses, GetProviderCatalogTemplateData, GetProviderCatalogTemplateErrors, GetProviderCatalogTemplateResponse, GetProviderCatalogTemplateResponses, GetProviderModelsData, GetProviderModelsErrors, GetProviderModelsResponse, GetProviderModelsResponses, GetRepoFilesData, GetRepoFilesResponse, GetRepoFilesResponses, GetSessionData, GetSessionErrors, GetSessionExtensionsData, GetSessionExtensionsErrors, GetSessionExtensionsResponse, GetSessionExtensionsResponses, GetSessionInsightsData, GetSessionInsightsErrors, GetSessionInsightsResponse, GetSessionInsightsResponses, GetSessionResponse, GetSessionResponses, GetSlashCommandsData, GetSlashCommandsResponse, GetSlashCommandsResponses, GetToolsData, GetToolsErrors, GetToolsQuery, GetToolsResponse, GetToolsResponses, GetTunnelStatusData, GetTunnelStatusResponse, GetTunnelStatusResponses, GooseApp, HfGgufFile, HfModelInfo, HfQuantVariant, Icon, ImageContent, ImportAppData, ImportAppError, ImportAppErrors, ImportAppRequest, ImportAppResponse, ImportAppResponse2, ImportAppResponses, ImportSessionData, ImportSessionErrors, ImportSessionRequest, ImportSessionResponse, ImportSessionResponses, InitConfigData, InitConfigErrors, InitConfigResponse, InitConfigResponses, InspectJobResponse, InspectRunningJobData, InspectRunningJobErrors, InspectRunningJobResponse, InspectRunningJobResponses, JsonObject, KillJobResponse, KillRunningJobData, KillRunningJobResponses, ListAppsData, ListAppsError, ListAppsErrors, ListAppsRequest, ListAppsResponse, ListAppsResponse2, ListAppsResponses, ListLocalModelsData, ListLocalModelsResponse, ListLocalModelsResponses, ListModelsData, ListModelsResponse, ListModelsResponses, ListRecipeResponse, ListRecipesData, ListRecipesErrors, ListRecipesResponse, ListRecipesResponses, ListSchedulesData, ListSchedulesErrors, ListSchedulesResponse, ListSchedulesResponse2, ListSchedulesResponses, ListSessionsData, ListSessionsErrors, ListSessionsResponse, ListSessionsResponses, LoadedProvider, LocalModelResponse, McpAppResource, McpUiProxyData, McpUiProxyErrors, McpUiProxyResponses, Message, MessageContent, MessageEvent, MessageMetadata, ModelCapabilities, ModelConfig, ModelDownloadStatus, ModelInfo, ModelInfoData, ModelInfoQuery, ModelInfoResponse, ModelSettings, ModelTemplate, ParseRecipeData, ParseRecipeError, ParseRecipeErrors, ParseRecipeRequest, ParseRecipeResponse, ParseRecipeResponse2, ParseRecipeResponses, PauseScheduleData, PauseScheduleErrors, PauseScheduleResponse, PauseScheduleResponses, Permission, PermissionLevel, PermissionsMetadata, PrincipalType, PromptContentResponse, PromptsListResponse, ProviderCatalogEntry, ProviderDetails, ProviderEngine, ProviderMetadata, ProvidersData, ProvidersResponse, ProvidersResponse2, ProvidersResponses, ProviderTemplate, ProviderType, RawAudioContent, RawEmbeddedResource, RawImageContent, RawResource, RawTextContent, ReadAllConfigData, ReadAllConfigResponse, ReadAllConfigResponses, ReadConfigData, ReadConfigErrors, ReadConfigResponses, ReadResourceData, ReadResourceErrors, ReadResourceRequest, ReadResourceResponse, ReadResourceResponse2, ReadResourceResponses, ReasoningContent, Recipe, RecipeManifest, RecipeParameter, RecipeParameterInputType, RecipeParameterRequirement, RecipeToYamlData, RecipeToYamlError, RecipeToYamlErrors, RecipeToYamlRequest, RecipeToYamlResponse, RecipeToYamlResponse2, RecipeToYamlResponses, RecoverConfigData, RecoverConfigErrors, RecoverConfigResponse, RecoverConfigResponses, RedactedThinkingContent, RemoveConfigData, RemoveConfigErrors, RemoveConfigResponse, RemoveConfigResponses, RemoveCustomProviderData, RemoveCustomProviderErrors, RemoveCustomProviderResponse, RemoveCustomProviderResponses, RemoveExtensionData, RemoveExtensionErrors, RemoveExtensionRequest, RemoveExtensionResponse, RemoveExtensionResponses, ReplyData, ReplyErrors, ReplyResponse, ReplyResponses, RepoVariantsResponse, ResetPromptData, ResetPromptErrors, ResetPromptResponse, ResetPromptResponses, ResourceContents, ResourceMetadata, Response, RestartAgentData, RestartAgentErrors, RestartAgentRequest, RestartAgentResponse, RestartAgentResponse2, RestartAgentResponses, ResumeAgentData, ResumeAgentErrors, ResumeAgentRequest, ResumeAgentResponse, ResumeAgentResponse2, ResumeAgentResponses, RetryConfig, Role, RunNowHandlerData, RunNowHandlerErrors, RunNowHandlerResponse, RunNowHandlerResponses, RunNowResponse, SamplingConfig, SavePromptData, SavePromptErrors, SavePromptRequest, SavePromptResponse, SavePromptResponses, SaveRecipeData, SaveRecipeError, SaveRecipeErrors, SaveRecipeRequest, SaveRecipeResponse, SaveRecipeResponse2, SaveRecipeResponses, ScanRecipeData, ScanRecipeRequest, ScanRecipeResponse, ScanRecipeResponse2, ScanRecipeResponses, ScheduledJob, ScheduleRecipeData, ScheduleRecipeErrors, ScheduleRecipeRequest, ScheduleRecipeResponses, SearchHfModelsData, SearchHfModelsErrors, SearchHfModelsResponse, SearchHfModelsResponses, SearchSessionsData, SearchSessionsErrors, SearchSessionsResponse, SearchSessionsResponses, SendTelemetryEventData, SendTelemetryEventResponses, Session, SessionDisplayInfo, SessionExtensionsResponse, SessionInsights, SessionListResponse, SessionsHandlerData, SessionsHandlerErrors, SessionsHandlerResponse, SessionsHandlerResponses, SessionsQuery, SessionType, SetConfigProviderData, SetProviderRequest, SetRecipeSlashCommandData, SetRecipeSlashCommandErrors, SetRecipeSlashCommandResponses, SetSlashCommandRequest, Settings, SetupResponse, SlashCommand, SlashCommandsResponse, StartAgentData, StartAgentError, StartAgentErrors, StartAgentRequest, StartAgentResponse, StartAgentResponses, StartOpenrouterSetupData, StartOpenrouterSetupResponse, StartOpenrouterSetupResponses, StartTetrateSetupData, StartTetrateSetupResponse, StartTetrateSetupResponses, StartTunnelData, StartTunnelError, StartTunnelErrors, StartTunnelResponse, StartTunnelResponses, StatusData, StatusResponse, StatusResponses, StopAgentData, StopAgentErrors, StopAgentRequest, StopAgentResponse, StopAgentResponses, StopTunnelData, StopTunnelError, StopTunnelErrors, StopTunnelResponses, SubRecipe, SuccessCheck, SystemInfo, SystemInfoData, SystemInfoResponse, SystemInfoResponses, SystemNotificationContent, SystemNotificationType, TaskSupport, TelemetryEventRequest, Template, TextContent, ThinkingContent, TokenState, Tool, ToolAnnotations, ToolConfirmationRequest, ToolExecution, ToolInfo, ToolPermission, ToolRequest, ToolResponse, ToolResultPayload, TranscribeDictationData, TranscribeDictationErrors, TranscribeDictationResponse, TranscribeDictationResponses, TranscribeRequest, TranscribeResponse, TunnelInfo, TunnelState, UiMetadata, UnpauseScheduleData, UnpauseScheduleErrors, UnpauseScheduleResponse, UnpauseScheduleResponses, UpdateAgentProviderData, UpdateAgentProviderErrors, UpdateAgentProviderResponses, UpdateCustomProviderData, UpdateCustomProviderErrors, UpdateCustomProviderRequest, UpdateCustomProviderResponse, UpdateCustomProviderResponses, UpdateFromSessionData, UpdateFromSessionErrors, UpdateFromSessionRequest, UpdateFromSessionResponses, UpdateModelSettingsData, UpdateModelSettingsErrors, UpdateModelSettingsResponse, UpdateModelSettingsResponses, UpdateProviderRequest, UpdateScheduleData, UpdateScheduleErrors, UpdateScheduleRequest, UpdateScheduleResponse, UpdateScheduleResponses, UpdateSessionNameData, UpdateSessionNameErrors, UpdateSessionNameRequest, UpdateSessionNameResponses, UpdateSessionUserRecipeValuesData, UpdateSessionUserRecipeValuesError, UpdateSessionUserRecipeValuesErrors, UpdateSessionUserRecipeValuesRequest, UpdateSessionUserRecipeValuesResponse, UpdateSessionUserRecipeValuesResponse2, UpdateSessionUserRecipeValuesResponses, UpdateWorkingDirData, UpdateWorkingDirErrors, UpdateWorkingDirRequest, UpdateWorkingDirResponses, UpsertConfigData, UpsertConfigErrors, UpsertConfigQuery, UpsertConfigResponse, UpsertConfigResponses, UpsertPermissionsData, UpsertPermissionsErrors, UpsertPermissionsQuery, UpsertPermissionsResponse, UpsertPermissionsResponses, ValidateConfigData, ValidateConfigErrors, ValidateConfigResponse, ValidateConfigResponses, WhisperModelResponse, WindowProps } from './types.gen'; diff --git a/ui/desktop/src/api/types.gen.ts b/ui/desktop/src/api/types.gen.ts index 2ccd506d8b..ce2498147b 100644 --- a/ui/desktop/src/api/types.gen.ts +++ b/ui/desktop/src/api/types.gen.ts @@ -69,6 +69,7 @@ export type ChatRequest = { recipe_name?: string | null; recipe_version?: string | null; session_id: string; + tool_result?: ToolResultPayload | null; user_message: Message; }; @@ -1430,6 +1431,21 @@ export type ToolResponse = { }; }; +export type ToolResultPayload = { + /** + * Whether the tool execution was an error + */ + is_error?: boolean; + /** + * The result content from the client-side tool execution + */ + result: string; + /** + * The tool_call_id that this result corresponds to + */ + tool_call_id: string; +}; + export type TranscribeRequest = { /** * Base64 encoded audio data diff --git a/ui/desktop/src/components/BaseChat.tsx b/ui/desktop/src/components/BaseChat.tsx index c813ed204f..aac0734402 100644 --- a/ui/desktop/src/components/BaseChat.tsx +++ b/ui/desktop/src/components/BaseChat.tsx @@ -40,6 +40,7 @@ import CreateRecipeFromSessionModal from './recipes/CreateRecipeFromSessionModal import { toastSuccess } from '../toasts'; import { Recipe } from '../recipe'; import { useAutoSubmit } from '../hooks/useAutoSubmit'; +import { useForwardedToolHandler } from '../hooks/useForwardedToolHandler'; import { Goose } from './icons'; import EnvironmentBadge from './GooseSidebar/EnvironmentBadge'; @@ -94,6 +95,7 @@ export default function BaseChat({ setChatState, handleSubmit, submitElicitationResponse, + submitToolResult, stopStreaming, sessionLoadError, setRecipeUserParams, @@ -116,6 +118,8 @@ export default function BaseChat({ handleSubmit, }); + useForwardedToolHandler(messages, chatState, submitToolResult); + useEffect(() => { let streamState: 'idle' | 'loading' | 'streaming' | 'error' = 'idle'; if (chatState === ChatState.LoadingConversation) { diff --git a/ui/desktop/src/components/BrowserContext.tsx b/ui/desktop/src/components/BrowserContext.tsx new file mode 100644 index 0000000000..ab7ee1c2eb --- /dev/null +++ b/ui/desktop/src/components/BrowserContext.tsx @@ -0,0 +1,182 @@ +import { createContext, useContext, useRef, useCallback, useState } from 'react'; + +interface BrowserState { + isOpen: boolean; + url: string; +} + +interface BrowserContextType { + state: BrowserState; + webviewRef: React.RefObject; + openUrl: (url: string) => void; + close: () => void; + navigate: (url: string) => void; + executeCommand: (command: string, params: Record) => Promise<{ success: boolean; data?: unknown }>; +} + +const BrowserContext = createContext(null); + +export function useBrowser() { + const ctx = useContext(BrowserContext); + if (!ctx) { + throw new Error('useBrowser must be used within a BrowserProvider'); + } + return ctx; +} + +export function useBrowserOptional() { + return useContext(BrowserContext); +} + +export function BrowserProvider({ children }: { children: React.ReactNode }) { + const webviewRef = useRef(null); + const [state, setState] = useState({ isOpen: false, url: '' }); + + const openUrl = useCallback((url: string) => { + setState({ isOpen: true, url }); + }, []); + + const close = useCallback(() => { + setState({ isOpen: false, url: '' }); + }, []); + + const navigate = useCallback((url: string) => { + const wv = webviewRef.current; + if (wv) { + wv.loadURL(url).catch((err: Error) => { + // ERR_ABORTED happens on redirects — not a real error + if (err.message?.includes('ERR_ABORTED')) return; + console.error('Navigation failed:', err); + }); + setState((prev) => ({ ...prev, url })); + } + }, []); + + const executeCommand = useCallback( + async (command: string, params: Record): Promise<{ success: boolean; data?: unknown }> => { + if (command === 'open') { + const url = params.url as string; + openUrl(url); + return { success: true }; + } + + if (command === 'close') { + close(); + return { success: true }; + } + + if (command === 'navigate') { + const url = params.url as string; + navigate(url); + return { success: true }; + } + + const wv = webviewRef.current; + if (!wv) { + return { success: false, data: 'Browser not open' }; + } + + switch (command) { + case 'screenshot': { + // webview.capturePage returns a NativeImage + const image = await wv.capturePage(); + return { success: true, data: image.toDataURL() }; + } + + case 'click': { + const selector = params.selector as string; + await wv.executeJavaScript(` + (function() { + const el = document.querySelector(${JSON.stringify(selector)}); + if (!el) throw new Error('Element not found: ${selector}'); + el.click(); + return true; + })() + `); + return { success: true }; + } + + case 'type': { + const selector = params.selector as string; + const text = params.text as string; + await wv.executeJavaScript(` + (function() { + const el = document.querySelector(${JSON.stringify(selector)}); + if (!el) throw new Error('Element not found: ${selector}'); + el.focus(); + el.value = ${JSON.stringify(text)}; + el.dispatchEvent(new Event('input', { bubbles: true })); + el.dispatchEvent(new Event('change', { bubbles: true })); + return true; + })() + `); + return { success: true }; + } + + case 'get_text': { + const selector = params.selector as string; + const text = await wv.executeJavaScript(` + (function() { + const el = document.querySelector(${JSON.stringify(selector)}); + if (!el) throw new Error('Element not found: ${selector}'); + return el.innerText || el.textContent || ''; + })() + `); + return { success: true, data: text }; + } + + case 'get_html': { + const selector = params.selector as string; + const html = await wv.executeJavaScript(` + (function() { + const el = document.querySelector(${JSON.stringify(selector)}); + if (!el) throw new Error('Element not found: ${selector}'); + return el.innerHTML; + })() + `); + return { success: true, data: html }; + } + + case 'evaluate': { + const script = params.script as string; + const result = await wv.executeJavaScript(script); + return { success: true, data: result }; + } + + case 'wait': { + const selector = params.selector as string; + const timeout = (params.timeout as number) || 5000; + const startTime = Date.now(); + + while (Date.now() - startTime < timeout) { + const found = await wv.executeJavaScript( + `!!document.querySelector(${JSON.stringify(selector)})` + ); + if (found) { + return { success: true }; + } + await new Promise((resolve) => setTimeout(resolve, 100)); + } + return { success: false, data: `Timeout waiting for element: ${selector}` }; + } + + case 'scroll': { + const x = (params.x as number) || 0; + const y = (params.y as number) || 0; + await wv.executeJavaScript(`window.scrollBy(${x}, ${y})`); + return { success: true }; + } + + default: + return { success: false, data: `Unknown command: ${command}` }; + } + }, + [openUrl, close, navigate] + ); + + return ( + + {children} + + ); +} diff --git a/ui/desktop/src/components/BrowserPanel.tsx b/ui/desktop/src/components/BrowserPanel.tsx new file mode 100644 index 0000000000..5a711f23cd --- /dev/null +++ b/ui/desktop/src/components/BrowserPanel.tsx @@ -0,0 +1,155 @@ +import { useEffect, useRef, useState, useCallback } from 'react'; +import { useBrowser } from './BrowserContext'; + +export default function BrowserPanel() { + const { state, webviewRef, close } = useBrowser(); + const containerRef = useRef(null); + const [displayUrl, setDisplayUrl] = useState(state.url); + const [isLoading, setIsLoading] = useState(false); + const [panelWidth, setPanelWidth] = useState(500); + const isDragging = useRef(false); + + useEffect(() => { + setDisplayUrl(state.url); + }, [state.url]); + + // Attach webview event listeners once the element mounts + useEffect(() => { + if (!state.isOpen) return; + + const container = containerRef.current; + if (!container) return; + + // The webview element is rendered declaratively but we need to grab it + const wv = container.querySelector('webview') as Electron.WebviewTag | null; + if (!wv) return; + + webviewRef.current = wv; + + const onLoadStart = () => setIsLoading(true); + const onLoadStop = () => setIsLoading(false); + const onNavigate = (e: Electron.DidNavigateEvent) => setDisplayUrl(e.url); + const onFailLoad = (e: Electron.DidFailLoadEvent) => { + // ERR_ABORTED (-3) happens on redirects — not a real error + if (e.errorCode === -3) return; + console.error(`Webview load failed: ${e.errorDescription} (${e.errorCode})`); + setIsLoading(false); + }; + + wv.addEventListener('did-start-loading', onLoadStart); + wv.addEventListener('did-stop-loading', onLoadStop); + wv.addEventListener('did-navigate', onNavigate as EventListener); + wv.addEventListener('did-navigate-in-page', onNavigate as EventListener); + wv.addEventListener('did-fail-load', onFailLoad as EventListener); + + return () => { + wv.removeEventListener('did-start-loading', onLoadStart); + wv.removeEventListener('did-stop-loading', onLoadStop); + wv.removeEventListener('did-navigate', onNavigate as EventListener); + wv.removeEventListener('did-navigate-in-page', onNavigate as EventListener); + wv.removeEventListener('did-fail-load', onFailLoad as EventListener); + if (webviewRef.current === wv) { + webviewRef.current = null; + } + }; + }, [state.isOpen, state.url, webviewRef]); + + // Resize drag handler + const onResizeMouseDown = useCallback( + (e: React.MouseEvent) => { + e.preventDefault(); + isDragging.current = true; + const startX = e.clientX; + const startWidth = panelWidth; + + const onMouseMove = (moveEvent: MouseEvent) => { + if (!isDragging.current) return; + const delta = startX - moveEvent.clientX; + const newWidth = Math.max(300, Math.min(1200, startWidth + delta)); + setPanelWidth(newWidth); + }; + + const onMouseUp = () => { + isDragging.current = false; + document.removeEventListener('mousemove', onMouseMove); + document.removeEventListener('mouseup', onMouseUp); + }; + + document.addEventListener('mousemove', onMouseMove); + document.addEventListener('mouseup', onMouseUp); + }, + [panelWidth] + ); + + const handleUrlSubmit = (e: React.FormEvent) => { + e.preventDefault(); + const wv = webviewRef.current; + if (wv && displayUrl) { + let url = displayUrl; + if (!url.startsWith('http://') && !url.startsWith('https://')) { + url = 'https://' + url; + } + wv.loadURL(url).catch((err: Error) => { + if (err.message?.includes('ERR_ABORTED')) return; + console.error('URL load failed:', err); + }); + } + }; + + if (!state.isOpen) return null; + + return ( +
+ {/* Resize handle */} +
+ + {/* Toolbar */} +
+
+ setDisplayUrl(e.target.value)} + className="flex-1 text-sm px-2 py-1 rounded bg-surface-default border border-border-default text-text-default placeholder:text-text-muted focus:outline-none focus:border-accent-default" + placeholder="Enter URL..." + /> +
+ {isLoading && ( +
+ )} + +
+ + {/* Webview */} +
+ {/* eslint-disable-next-line react/no-unknown-property */} + +
+
+ ); +} diff --git a/ui/desktop/src/components/ChatSessionsContainer.tsx b/ui/desktop/src/components/ChatSessionsContainer.tsx index 833e1beeed..9d2e82c809 100644 --- a/ui/desktop/src/components/ChatSessionsContainer.tsx +++ b/ui/desktop/src/components/ChatSessionsContainer.tsx @@ -1,5 +1,7 @@ import { useSearchParams } from 'react-router-dom'; import BaseChat from './BaseChat'; +import BrowserPanel from './BrowserPanel'; +import { BrowserProvider } from './BrowserContext'; import { ChatType } from '../types/chat'; import { UserInput } from '../types/message'; @@ -23,15 +25,12 @@ export default function ChatSessionsContainer({ const [searchParams] = useSearchParams(); const currentSessionId = searchParams.get('resumeSessionId') ?? undefined; - // Always render active sessions to keep SSE connections alive, even when not on /pair route if (!currentSessionId && activeSessions.length === 0) { return null; } - // Build the list of sessions to render let sessionsToRender = activeSessions; - // If we have a currentSessionId that's not in activeSessions, add it (handles page refresh) if (currentSessionId && !activeSessions.some((s) => s.sessionId === currentSessionId)) { sessionsToRender = [...activeSessions, { sessionId: currentSessionId }]; } @@ -44,16 +43,21 @@ export default function ChatSessionsContainer({ return (
- + +
+ +
+ +
); })} diff --git a/ui/desktop/src/hooks/useChatStream.ts b/ui/desktop/src/hooks/useChatStream.ts index c4732fbec0..399c4b3240 100644 --- a/ui/desktop/src/hooks/useChatStream.ts +++ b/ui/desktop/src/hooks/useChatStream.ts @@ -45,6 +45,7 @@ interface UseChatStreamReturn { elicitationId: string, userData: Record ) => Promise; + submitToolResult: (toolCallId: string, result: string, isError?: boolean) => Promise; setRecipeUserParams: (values: Record) => Promise; stopStreaming: () => void; sessionLoadError?: string; @@ -648,6 +649,50 @@ export function useChatStream({ [sessionId, onFinish] ); + const submitToolResult = useCallback( + async (toolCallId: string, result: string, isError: boolean = false) => { + const currentState = stateRef.current; + + if (!currentState.session || currentState.chatState === ChatState.LoadingConversation) { + return; + } + + dispatch({ type: 'START_STREAMING' }); + abortControllerRef.current = new AbortController(); + + try { + const { stream } = await reply({ + body: { + session_id: sessionId, + user_message: createUserMessage(''), + tool_result: { + tool_call_id: toolCallId, + result, + is_error: isError, + }, + }, + throwOnError: true, + signal: abortControllerRef.current.signal, + }); + + await streamFromResponse( + stream, + currentState.messages, + dispatch, + onFinish, + sessionId + ); + } catch (error) { + if (error instanceof Error && error.name === 'AbortError') { + // Silently handle abort + } else { + onFinish('Tool result submit error: ' + errorMessage(error)); + } + } + }, + [sessionId, onFinish] + ); + const setRecipeUserParams = useCallback( async (user_recipe_values: Record) => { const currentState = stateRef.current; @@ -828,6 +873,7 @@ export function useChatStream({ setChatState, handleSubmit, submitElicitationResponse, + submitToolResult, stopStreaming, setRecipeUserParams, tokenState: state.tokenState, diff --git a/ui/desktop/src/hooks/useForwardedToolHandler.ts b/ui/desktop/src/hooks/useForwardedToolHandler.ts new file mode 100644 index 0000000000..d1d401f3ad --- /dev/null +++ b/ui/desktop/src/hooks/useForwardedToolHandler.ts @@ -0,0 +1,83 @@ +import { useEffect, useRef } from 'react'; +import { Message } from '../api'; +import { ChatState } from '../types/chatState'; +import { getToolRequests } from '../types/message'; +import { useBrowserOptional } from '../components/BrowserContext'; + +function toolNameToCommand(toolName: string): string | null { + const mapping: Record = { + browser_open: 'open', + browser_close: 'close', + browser_navigate: 'navigate', + browser_screenshot: 'screenshot', + browser_click: 'click', + browser_type: 'type', + browser_get_text: 'get_text', + browser_get_html: 'get_html', + browser_evaluate: 'evaluate', + browser_wait: 'wait', + browser_scroll: 'scroll', + }; + return mapping[toolName] ?? null; +} + +/** + * Watches for forwarded tool requests (tool calls the agent delegated to the client). + * When the stream finishes and the last assistant message contains a tool request + * with _meta.forward_to_client, this hook executes it on the browser webview + * and submits the result back via submitToolResult. + */ +export function useForwardedToolHandler( + messages: Message[], + chatState: ChatState, + submitToolResult: (toolCallId: string, result: string, isError?: boolean) => Promise +) { + const browser = useBrowserOptional(); + const handledRef = useRef>(new Set()); + + useEffect(() => { + if (chatState !== ChatState.Idle) return; + if (!browser) return; + + const lastAssistant = [...messages].reverse().find((m) => m.role === 'assistant'); + if (!lastAssistant) return; + + const toolRequests = getToolRequests(lastAssistant); + if (!toolRequests.length) return; + + for (const toolRequest of toolRequests) { + const meta = toolRequest._meta as Record | undefined; + if (!meta?.forward_to_client) continue; + + const toolCallId = toolRequest.id; + if (handledRef.current.has(toolCallId)) continue; + handledRef.current.add(toolCallId); + + const toolName = toolRequest.toolCall?.name as string | undefined; + const args = toolRequest.toolCall?.arguments as Record | undefined; + const command = toolName ? toolNameToCommand(toolName) : null; + + if (!command || !args) { + submitToolResult( + toolCallId, + JSON.stringify({ error: `Unknown forwarded tool: ${toolName}` }), + true + ); + continue; + } + + (async () => { + try { + const result = await browser.executeCommand(command, args); + await submitToolResult(toolCallId, JSON.stringify(result)); + } catch (err) { + await submitToolResult( + toolCallId, + JSON.stringify({ error: String(err) }), + true + ); + } + })(); + } + }, [chatState, messages, browser, submitToolResult]); +} diff --git a/ui/desktop/src/main.ts b/ui/desktop/src/main.ts index bfba32cb8d..137e63bc7b 100644 --- a/ui/desktop/src/main.ts +++ b/ui/desktop/src/main.ts @@ -521,7 +521,6 @@ let appConfig = { const windowMap = new Map(); const goosedClients = new Map(); const appWindows = new Map(); - const windowPowerSaveBlockers = new Map(); // windowId -> blockerId // Track pending initial messages per window const pendingInitialMessages = new Map(); // windowId -> initialMessage @@ -598,6 +597,7 @@ const createChat = async (app: App, options: CreateChatOptions = {}) => { webSecurity: true, nodeIntegration: false, contextIsolation: true, + webviewTag: true, additionalArguments: [ JSON.stringify({ ...appConfig, @@ -2436,6 +2436,7 @@ async function appMain() { throw error; } }); + } app.whenReady().then(async () => { diff --git a/ui/desktop/src/preload.ts b/ui/desktop/src/preload.ts index bf1a620c3b..25ac9d5782 100644 --- a/ui/desktop/src/preload.ts +++ b/ui/desktop/src/preload.ts @@ -170,6 +170,7 @@ type ElectronAPI = { launchApp: (app: GooseApp) => Promise; refreshApp: (app: GooseApp) => Promise; closeApp: (appName: string) => Promise; + addRecentDir: (dir: string) => Promise; };