From 06d414196cbe8cb1141f7ddef50a99f7d2819f61 Mon Sep 17 00:00:00 2001 From: Dan Good <49254594+dan4thewin@users.noreply.github.com> Date: Tue, 12 May 2026 19:22:36 -0400 Subject: [PATCH] github_copilot /responses API (#9043) Signed-off-by: Douwe Osinga Co-authored-by: Douwe Osinga --- crates/goose/src/providers/databricks.rs | 27 +- .../src/providers/formats/openai_responses.rs | 684 +++++++++++++++++- crates/goose/src/providers/githubcopilot.rs | 363 +++++++--- crates/goose/src/providers/openai.rs | 26 +- .../goose/src/providers/openai_compatible.rs | 24 + 5 files changed, 946 insertions(+), 178 deletions(-) diff --git a/crates/goose/src/providers/databricks.rs b/crates/goose/src/providers/databricks.rs index 115bfcdf35..2695871beb 100644 --- a/crates/goose/src/providers/databricks.rs +++ b/crates/goose/src/providers/databricks.rs @@ -1,16 +1,10 @@ use anyhow::Result; -use async_stream::try_stream; use async_trait::async_trait; use futures::future::BoxFuture; -use futures::{StreamExt, TryStreamExt}; use serde::{Deserialize, Serialize}; use serde_json::Value; -use std::io; use std::sync::{Arc, Mutex}; use std::time::Duration; -use tokio::pin; -use tokio_util::codec::{FramedRead, LinesCodec}; -use tokio_util::io::StreamReader; use super::api_client::{ApiClient, AuthMethod, AuthProvider}; use super::base::{ @@ -20,13 +14,11 @@ use super::base::{ use super::embedding::EmbeddingCapable; use super::errors::ProviderError; use super::formats::databricks::create_request; -use super::formats::openai_responses::{ - create_responses_request, responses_api_to_streaming_message, -}; +use super::formats::openai_responses::create_responses_request; use super::oauth; use super::openai_compatible::{ handle_response_openai_compat, handle_status, map_http_error_to_provider_error, - stream_openai_compat, + stream_openai_compat, stream_responses_compat, }; use super::retry::ProviderRetry; use super::utils::{ImageFormat, RequestLog}; @@ -412,20 +404,7 @@ impl Provider for DatabricksProvider { let _ = log.error(e); })?; - let stream = response.bytes_stream().map_err(io::Error::other); - - Ok(Box::pin(try_stream! { - let stream_reader = StreamReader::new(stream); - let framed = FramedRead::new(stream_reader, LinesCodec::new()).map_err(anyhow::Error::from); - - let message_stream = responses_api_to_streaming_message(framed); - pin!(message_stream); - while let Some(message) = message_stream.next().await { - let (message, usage) = message.map_err(|e| ProviderError::RequestFailed(format!("Stream decode error: {}", e)))?; - log.write(&message, usage.as_ref().map(|f| f.usage).as_ref())?; - yield (message, usage); - } - })) + stream_responses_compat(response, log) } else { let mut payload = create_request(model_config, system, messages, tools, &self.image_format)?; diff --git a/crates/goose/src/providers/formats/openai_responses.rs b/crates/goose/src/providers/formats/openai_responses.rs index 1023805953..b328b0c9df 100644 --- a/crates/goose/src/providers/formats/openai_responses.rs +++ b/crates/goose/src/providers/formats/openai_responses.rs @@ -79,6 +79,9 @@ pub enum ResponseContentBlock { #[serde(skip_serializing_if = "Option::is_none")] annotations: Option>, }, + Refusal { + refusal: String, + }, ToolCall { id: String, name: String, @@ -187,6 +190,22 @@ pub enum ResponsesStreamEvent { output_index: i32, arguments: String, }, + #[serde(rename = "response.refusal.delta")] + RefusalDelta { + sequence_number: i32, + item_id: String, + output_index: i32, + content_index: i32, + delta: String, + }, + #[serde(rename = "response.refusal.done")] + RefusalDone { + sequence_number: i32, + item_id: String, + output_index: i32, + content_index: i32, + refusal: String, + }, #[serde(rename = "error")] Error { error: Value }, #[serde(rename = "keepalive")] @@ -211,6 +230,8 @@ fn is_known_responses_stream_event_type(event_type: &str) -> bool { | "response.failed" | "response.function_call_arguments.delta" | "response.function_call_arguments.done" + | "response.refusal.delta" + | "response.refusal.done" | "error" | "keepalive" ) @@ -293,6 +314,9 @@ pub enum ContentPart { #[serde(skip_serializing_if = "Option::is_none")] logprobs: Option>, }, + Refusal { + refusal: String, + }, ToolCall { id: String, name: String, @@ -331,28 +355,43 @@ fn add_message_items(input_items: &mut Vec, messages: &[Message]) { text_items = Vec::new(); } - if let Ok(tool_call) = &request.tool_call { - let arguments_str = tool_call - .arguments - .as_ref() - .map(|args| { - serde_json::to_string(args).unwrap_or_else(|_| "{}".to_string()) - }) - .unwrap_or_else(|| "{}".to_string()); + match &request.tool_call { + Ok(tool_call) => { + let arguments_str = tool_call + .arguments + .as_ref() + .map(|args| { + serde_json::to_string(args).unwrap_or_else(|_| "{}".to_string()) + }) + .unwrap_or_else(|| "{}".to_string()); - tracing::debug!( - "Replaying function_call with call_id: {}, name: {}", - request.id, - tool_call.name - ); - input_items.push(json!({ - "type": "function_call", - "call_id": request.id, - "name": tool_call.name, - "arguments": arguments_str - })); + tracing::debug!( + "Replaying function_call with call_id: {}, name: {}", + request.id, + tool_call.name + ); + input_items.push(json!({ + "type": "function_call", + "call_id": request.id, + "name": tool_call.name, + "arguments": arguments_str + })); + } + Err(e) => { + input_items.push(json!({ + "type": "function_call_output", + "call_id": request.id, + "output": format!("Error: {}", e.message) + })); + } } } + MessageContent::Image(image) => { + text_items.push(json!({ + "type": "input_image", + "image_url": format!("data:{};base64,{}", image.mime_type, image.data) + })); + } MessageContent::ToolResponse(response) => { if !text_items.is_empty() { input_items.push(json!({ @@ -434,6 +473,41 @@ fn add_message_items(input_items: &mut Vec, messages: &[Message]) { } } } + MessageContent::FrontendToolRequest(request) => { + if !text_items.is_empty() { + input_items.push(json!({ + "role": role, + "content": text_items + })); + text_items = Vec::new(); + } + + match &request.tool_call { + Ok(tool_call) => { + let arguments_str = tool_call + .arguments + .as_ref() + .map(|args| { + serde_json::to_string(args).unwrap_or_else(|_| "{}".to_string()) + }) + .unwrap_or_else(|| "{}".to_string()); + + input_items.push(json!({ + "type": "function_call", + "call_id": request.id, + "name": tool_call.name, + "arguments": arguments_str + })); + } + Err(e) => { + input_items.push(json!({ + "type": "function_call_output", + "call_id": request.id, + "output": format!("Error: {}", e.message) + })); + } + } + } _ => {} } } @@ -545,6 +619,11 @@ pub fn responses_api_to_message(response: &ResponsesApiResponse) -> anyhow::Resu content.push(MessageContent::text(text)); } } + ResponseContentBlock::Refusal { refusal } => { + if !refusal.is_empty() { + content.push(MessageContent::text(refusal)); + } + } ResponseContentBlock::ToolCall { id, name, input } => { content.push(MessageContent::tool_request( id.clone(), @@ -614,6 +693,11 @@ fn process_streaming_output_items( content.push(MessageContent::text(&text)); } } + ContentPart::Refusal { refusal } => { + if !refusal.is_empty() && !is_text_response { + content.push(MessageContent::text(&refusal)); + } + } ContentPart::ToolCall { id, name, @@ -777,6 +861,29 @@ where // Arguments are complete, will be in the OutputItemDone event } + ResponsesStreamEvent::RefusalDelta { delta, .. } => { + is_text_response = true; + if !delta.is_empty() { + accumulated_text.push_str(&delta); + + let mut msg = Message::new( + Role::Assistant, + chrono::Utc::now().timestamp(), + vec![MessageContent::text(&delta)], + ); + + if let Some(id) = &response_id { + msg = msg.with_id(id.clone()); + } + + yield (Some(msg), None); + } + } + + ResponsesStreamEvent::RefusalDone { .. } => { + // Refusal text already streamed via deltas + } + ResponsesStreamEvent::ResponseFailed { error, .. } => { Err(anyhow!("Responses API failed: {:?}", error))?; } @@ -1186,4 +1293,543 @@ mod tests { ); } } + + #[test] + fn test_user_image_serialized_in_responses_request() { + use crate::conversation::message::Message; + + let messages = vec![Message::user() + .with_text("describe this image") + .with_image("aW1hZ2VkYXRh", "image/png")]; + + let model_config = ModelConfig { + model_name: "gpt-5.5".to_string(), + context_limit: None, + temperature: None, + max_tokens: None, + toolshim: false, + toolshim_model: None, + fast_model_config: None, + request_params: None, + reasoning: None, + }; + + let result = + create_responses_request(&model_config, "You are helpful.", &messages, &[]).unwrap(); + + let input = result["input"].as_array().unwrap(); + assert_eq!(input.len(), 2); + + assert_eq!(input[0]["role"], "system"); + + assert_eq!(input[1]["role"], "user"); + let content = input[1]["content"].as_array().unwrap(); + assert_eq!(content.len(), 2); + + assert_eq!(content[0]["type"], "input_text"); + assert_eq!(content[0]["text"], "describe this image"); + + assert_eq!(content[1]["type"], "input_image"); + assert_eq!( + content[1]["image_url"], + "data:image/png;base64,aW1hZ2VkYXRh" + ); + } + + #[test] + fn test_tool_response_with_image_serializes_as_typed_array() { + use crate::conversation::message::Message; + use rmcp::model::{CallToolResult, Content}; + + let messages = vec![Message::user().with_content(MessageContent::tool_response( + "call_1", + Ok(CallToolResult::success(vec![ + Content::text("caption"), + Content::image("a+/=".to_string(), "image/png".to_string()), + ])), + ))]; + + let model_config = ModelConfig { + model_name: "gpt-5.5".to_string(), + context_limit: None, + temperature: None, + max_tokens: None, + toolshim: false, + toolshim_model: None, + fast_model_config: None, + request_params: None, + reasoning: None, + }; + + let result = create_responses_request(&model_config, "", &messages, &[]).unwrap(); + let input = result["input"].as_array().unwrap(); + + assert_eq!(input[0]["type"], "function_call_output"); + assert_eq!(input[0]["call_id"], "call_1"); + + let output = input[0]["output"].as_array().unwrap(); + assert_eq!(output.len(), 2); + assert_eq!(output[0], json!({"type": "input_text", "text": "caption"})); + assert_eq!( + output[1], + json!({"type": "input_image", "image_url": "data:image/png;base64,a+/="}) + ); + } + + #[test] + fn test_tool_request_serializes_function_call_with_arguments() { + use crate::conversation::message::Message; + + let messages = vec![Message::assistant().with_tool_request( + "call_1", + Ok(CallToolRequestParams::new("search") + .with_arguments(object!({"q": "rust", "limit": 2}))), + )]; + + let model_config = ModelConfig { + model_name: "gpt-5.5".to_string(), + context_limit: None, + temperature: None, + max_tokens: None, + toolshim: false, + toolshim_model: None, + fast_model_config: None, + request_params: None, + reasoning: None, + }; + + let result = create_responses_request(&model_config, "", &messages, &[]).unwrap(); + let input = result["input"].as_array().unwrap(); + + assert_eq!(input[0]["type"], "function_call"); + assert_eq!(input[0]["call_id"], "call_1"); + assert_eq!(input[0]["name"], "search"); + + let args: serde_json::Value = + serde_json::from_str(input[0]["arguments"].as_str().unwrap()).unwrap(); + assert_eq!(args["q"], "rust"); + assert_eq!(args["limit"], 2); + } + + #[test] + fn test_tool_request_none_arguments_serializes_empty_object() { + use crate::conversation::message::Message; + + let messages = vec![Message::assistant() + .with_tool_request("call_1", Ok(CallToolRequestParams::new("noop")))]; + + let model_config = ModelConfig { + model_name: "gpt-5.5".to_string(), + context_limit: None, + temperature: None, + max_tokens: None, + toolshim: false, + toolshim_model: None, + fast_model_config: None, + request_params: None, + reasoning: None, + }; + + let result = create_responses_request(&model_config, "", &messages, &[]).unwrap(); + let input = result["input"].as_array().unwrap(); + + assert_eq!(input[0]["type"], "function_call"); + assert_eq!(input[0]["name"], "noop"); + assert_eq!(input[0]["arguments"], "{}"); + } + + #[test] + fn test_text_flushed_before_tool_request() { + use crate::conversation::message::Message; + + let messages = vec![Message::assistant() + .with_text("planning") + .with_tool_request( + "call_1", + Ok(CallToolRequestParams::new("shell").with_arguments(object!({"command": "ls"}))), + )]; + + let model_config = ModelConfig { + model_name: "gpt-5.5".to_string(), + context_limit: None, + temperature: None, + max_tokens: None, + toolshim: false, + toolshim_model: None, + fast_model_config: None, + request_params: None, + reasoning: None, + }; + + let result = create_responses_request(&model_config, "", &messages, &[]).unwrap(); + let input = result["input"].as_array().unwrap(); + + assert_eq!(input.len(), 2); + assert_eq!(input[0]["role"], "assistant"); + assert_eq!(input[0]["content"][0]["type"], "output_text"); + assert_eq!(input[0]["content"][0]["text"], "planning"); + assert_eq!(input[1]["type"], "function_call"); + } + + #[test] + fn test_text_flushed_before_tool_response() { + use crate::conversation::message::Message; + use rmcp::model::{CallToolResult, Content}; + + let messages = + vec![Message::user() + .with_text("context") + .with_content(MessageContent::tool_response( + "call_1", + Ok(CallToolResult::success(vec![Content::text("done")])), + ))]; + + let model_config = ModelConfig { + model_name: "gpt-5.5".to_string(), + context_limit: None, + temperature: None, + max_tokens: None, + toolshim: false, + toolshim_model: None, + fast_model_config: None, + request_params: None, + reasoning: None, + }; + + let result = create_responses_request(&model_config, "", &messages, &[]).unwrap(); + let input = result["input"].as_array().unwrap(); + + assert_eq!(input.len(), 2); + assert_eq!(input[0]["role"], "user"); + assert_eq!(input[0]["content"][0]["type"], "input_text"); + assert_eq!(input[0]["content"][0]["text"], "context"); + assert_eq!(input[1]["type"], "function_call_output"); + assert_eq!(input[1]["output"], "done"); + } + + #[test] + fn test_tool_response_error_serializes_with_error_prefix() { + use crate::conversation::message::Message; + use rmcp::model::{ErrorCode, ErrorData}; + + let messages = vec![Message::user().with_content(MessageContent::tool_response( + "call_err", + Err(ErrorData { + code: ErrorCode::INTERNAL_ERROR, + message: "file not found".into(), + data: None, + }), + ))]; + + let model_config = ModelConfig { + model_name: "gpt-5.5".to_string(), + context_limit: None, + temperature: None, + max_tokens: None, + toolshim: false, + toolshim_model: None, + fast_model_config: None, + request_params: None, + reasoning: None, + }; + + let result = create_responses_request(&model_config, "", &messages, &[]).unwrap(); + let input = result["input"].as_array().unwrap(); + + assert_eq!(input[0]["type"], "function_call_output"); + assert_eq!(input[0]["call_id"], "call_err"); + assert_eq!(input[0]["output"], "Error: file not found"); + } + + #[test] + fn test_image_only_message_serializes() { + use crate::conversation::message::Message; + + let messages = vec![Message::user().with_image("aW1n", "image/png")]; + + let model_config = ModelConfig { + model_name: "gpt-5.5".to_string(), + context_limit: None, + temperature: None, + max_tokens: None, + toolshim: false, + toolshim_model: None, + fast_model_config: None, + request_params: None, + reasoning: None, + }; + + let result = create_responses_request(&model_config, "", &messages, &[]).unwrap(); + let input = result["input"].as_array().unwrap(); + + assert_eq!(input.len(), 1); + assert_eq!(input[0]["role"], "user"); + let content = input[0]["content"].as_array().unwrap(); + assert_eq!(content.len(), 1); + assert_eq!(content[0]["type"], "input_image"); + assert_eq!(content[0]["image_url"], "data:image/png;base64,aW1n"); + } + + #[test] + fn test_multiple_images_preserved_in_order() { + use crate::conversation::message::Message; + + let messages = vec![Message::user() + .with_text("compare") + .with_image("img1", "image/png") + .with_image("img2", "image/jpeg")]; + + let model_config = ModelConfig { + model_name: "gpt-5.5".to_string(), + context_limit: None, + temperature: None, + max_tokens: None, + toolshim: false, + toolshim_model: None, + fast_model_config: None, + request_params: None, + reasoning: None, + }; + + let result = create_responses_request(&model_config, "", &messages, &[]).unwrap(); + let input = result["input"].as_array().unwrap(); + + assert_eq!(input[0]["role"], "user"); + let content = input[0]["content"].as_array().unwrap(); + assert_eq!(content.len(), 3); + assert_eq!(content[0]["type"], "input_text"); + assert_eq!(content[0]["text"], "compare"); + assert_eq!(content[1]["type"], "input_image"); + assert_eq!(content[1]["image_url"], "data:image/png;base64,img1"); + assert_eq!(content[2]["type"], "input_image"); + assert_eq!(content[2]["image_url"], "data:image/jpeg;base64,img2"); + } + + #[test] + fn test_assistant_text_uses_output_text_type() { + use crate::conversation::message::Message; + + let messages = vec![Message::assistant().with_text("hello")]; + + let model_config = ModelConfig { + model_name: "gpt-5.5".to_string(), + context_limit: None, + temperature: None, + max_tokens: None, + toolshim: false, + toolshim_model: None, + fast_model_config: None, + request_params: None, + reasoning: None, + }; + + let result = create_responses_request(&model_config, "", &messages, &[]).unwrap(); + let input = result["input"].as_array().unwrap(); + + assert_eq!(input[0]["role"], "assistant"); + assert_eq!(input[0]["content"][0]["type"], "output_text"); + assert_eq!(input[0]["content"][0]["text"], "hello"); + } + + #[test] + fn test_refusal_content_block_deserializes_in_non_streaming_response() { + let json = r#"{ + "id": "resp_1", + "object": "response", + "created_at": 0, + "status": "completed", + "model": "gpt-5.5", + "output": [{ + "type": "message", + "id": "msg_1", + "status": "completed", + "role": "assistant", + "content": [{"type": "refusal", "refusal": "I cannot help with that request."}] + }] + }"#; + + let response: ResponsesApiResponse = serde_json::from_str(json).unwrap(); + let message = responses_api_to_message(&response).unwrap(); + assert_eq!(message.content.len(), 1); + if let MessageContent::Text(t) = &message.content[0] { + assert_eq!(t.text, "I cannot help with that request."); + } else { + panic!("expected text content from refusal"); + } + } + + #[test] + fn test_refusal_content_part_deserializes_in_streaming_output() { + let json = r#"{ + "type": "message", + "id": "msg_1", + "status": "completed", + "role": "assistant", + "content": [{"type": "refusal", "refusal": "I'm unable to assist."}] + }"#; + + let item: ResponseOutputItemInfo = serde_json::from_str(json).unwrap(); + let content = process_streaming_output_items(vec![item], false); + assert_eq!(content.len(), 1); + if let MessageContent::Text(t) = &content[0] { + assert_eq!(t.text, "I'm unable to assist."); + } else { + panic!("expected text content from refusal"); + } + } + + #[test] + fn test_refusal_delta_stream_event_deserializes() { + let json = r#"{"type":"response.refusal.delta","sequence_number":5,"item_id":"msg_1","output_index":0,"content_index":0,"delta":"I cannot"}"#; + + let event: ResponsesStreamEvent = serde_json::from_str(json).unwrap(); + match event { + ResponsesStreamEvent::RefusalDelta { delta, .. } => { + assert_eq!(delta, "I cannot"); + } + _ => panic!("expected RefusalDelta event"), + } + } + + #[test] + fn test_streamed_refusal_not_duplicated_in_output_items() { + let output_items = vec![ResponseOutputItemInfo::Message { + id: "msg_1".to_string(), + status: "completed".to_string(), + role: "assistant".to_string(), + content: vec![ContentPart::Refusal { + refusal: "I cannot help with that.".to_string(), + }], + }]; + + let content = process_streaming_output_items(output_items.clone(), true); + assert!( + content.is_empty(), + "refusal should be suppressed when already streamed" + ); + + let content = process_streaming_output_items(output_items, false); + assert_eq!( + content.len(), + 1, + "refusal should appear in non-streaming path" + ); + } + + #[test] + fn test_frontend_tool_request_serialized_in_responses_request() { + use crate::conversation::message::Message; + use rmcp::model::{CallToolResult, Content}; + + let messages = vec![ + Message::assistant().with_frontend_tool_request( + "call_ft1", + Ok(CallToolRequestParams::new("browser_click") + .with_arguments(object!({"selector": "#btn"}))), + ), + Message::user().with_content(MessageContent::tool_response( + "call_ft1", + Ok(CallToolResult::success(vec![Content::text("clicked")])), + )), + ]; + + let model_config = ModelConfig { + model_name: "gpt-5.5".to_string(), + context_limit: None, + temperature: None, + max_tokens: None, + toolshim: false, + toolshim_model: None, + fast_model_config: None, + request_params: None, + reasoning: None, + }; + + let result = create_responses_request(&model_config, "", &messages, &[]).unwrap(); + let input = result["input"].as_array().unwrap(); + + assert_eq!(input[0]["type"], "function_call"); + assert_eq!(input[0]["call_id"], "call_ft1"); + assert_eq!(input[0]["name"], "browser_click"); + + assert_eq!(input[1]["type"], "function_call_output"); + assert_eq!(input[1]["call_id"], "call_ft1"); + assert_eq!(input[1]["output"], "clicked"); + } + + #[test] + fn test_tool_request_error_emits_function_call_output() { + use crate::conversation::message::Message; + use rmcp::model::{ErrorCode, ErrorData}; + + let messages = vec![Message::assistant().with_tool_request( + "call_err1", + Err(ErrorData { + code: ErrorCode::INTERNAL_ERROR, + message: "invalid arguments".into(), + data: None, + }), + )]; + + let model_config = ModelConfig { + model_name: "gpt-5.5".to_string(), + context_limit: None, + temperature: None, + max_tokens: None, + toolshim: false, + toolshim_model: None, + fast_model_config: None, + request_params: None, + reasoning: None, + }; + + let result = create_responses_request(&model_config, "", &messages, &[]).unwrap(); + let input = result["input"].as_array().unwrap(); + + assert_eq!(input.len(), 1); + assert_eq!(input[0]["type"], "function_call_output"); + assert_eq!(input[0]["call_id"], "call_err1"); + assert!(input[0]["output"] + .as_str() + .unwrap() + .contains("invalid arguments")); + } + + #[test] + fn test_frontend_tool_request_error_emits_function_call_output() { + use crate::conversation::message::Message; + use rmcp::model::{ErrorCode, ErrorData}; + + let messages = vec![Message::assistant().with_frontend_tool_request( + "call_ft_err", + Err(ErrorData { + code: ErrorCode::INTERNAL_ERROR, + message: "malformed arguments".into(), + data: None, + }), + )]; + + let model_config = ModelConfig { + model_name: "gpt-5.5".to_string(), + context_limit: None, + temperature: None, + max_tokens: None, + toolshim: false, + toolshim_model: None, + fast_model_config: None, + request_params: None, + reasoning: None, + }; + + let result = create_responses_request(&model_config, "", &messages, &[]).unwrap(); + let input = result["input"].as_array().unwrap(); + + assert_eq!(input.len(), 1); + assert_eq!(input[0]["type"], "function_call_output"); + assert_eq!(input[0]["call_id"], "call_ft_err"); + assert!(input[0]["output"] + .as_str() + .unwrap() + .contains("malformed arguments")); + } } diff --git a/crates/goose/src/providers/githubcopilot.rs b/crates/goose/src/providers/githubcopilot.rs index 04a5406da0..8005db1715 100644 --- a/crates/goose/src/providers/githubcopilot.rs +++ b/crates/goose/src/providers/githubcopilot.rs @@ -1,7 +1,9 @@ use crate::config::paths::Paths; use crate::providers::api_client::{ApiClient, AuthMethod}; use crate::providers::oauth_device_flow::{run_device_flow, DeviceFlowConfig, RequestEncoding}; -use crate::providers::openai_compatible::{handle_status, stream_openai_compat}; +use crate::providers::openai_compatible::{ + handle_status, stream_openai_compat, stream_responses_compat, +}; use anyhow::{anyhow, Context, Result}; use async_trait::async_trait; use axum::http; @@ -25,9 +27,10 @@ use super::base::{ }; use super::errors::ProviderError; use super::formats::openai::{create_request, get_usage, response_to_message}; +use super::formats::openai_responses::create_responses_request; use super::openai_compatible::handle_response_openai_compat; use super::retry::ProviderRetry; -use super::utils::{get_model, ImageFormat, RequestLog}; +use super::utils::{get_model, is_openai_responses_model, ImageFormat, RequestLog}; use crate::config::{Config, ConfigError}; use crate::conversation::message::{Message, MessageContent}; @@ -35,30 +38,43 @@ use crate::conversation::message::{Message, MessageContent}; use crate::model::ModelConfig; use crate::providers::base::{ConfigKey, MessageStream}; use futures::future::BoxFuture; -use rmcp::model::Tool; +use rmcp::model::{RawContent, Tool}; +use std::ops::Deref; const GITHUB_COPILOT_PROVIDER_NAME: &str = "github_copilot"; pub const GITHUB_COPILOT_DEFAULT_MODEL: &str = "gpt-4.1"; pub const GITHUB_COPILOT_KNOWN_MODELS: &[&str] = &[ - "gpt-4.1", - "gpt-5-mini", - "gpt-5", - "gpt-4o", - "grok-code-fast-1", - "gpt-5-codex", + "claude-haiku-4.5", + "claude-opus-4.5", + "claude-opus-4.6", + "claude-opus-4.7", "claude-sonnet-4", "claude-sonnet-4.5", - "claude-haiku-4.5", + "claude-sonnet-4.6", "gemini-2.5-pro", + "gemini-3-flash-preview", + "gemini-3.1-pro-preview", + "gpt-4.1", + "gpt-4o", + "grok-code-fast-1", + "gpt-5-mini", + "gpt-5.2", + "gpt-5.2-codex", + "gpt-5.3-codex", + "gpt-5.4", + "gpt-5.4-mini", + "gpt-5.5", ]; +// Models that support streaming on the /chat/completions path. +// Models routed to /responses always stream and don't need to be listed here. pub const GITHUB_COPILOT_STREAM_MODELS: &[&str] = &[ "gpt-4.1", - "gpt-5", - "gpt-5-mini", - "gpt-5-codex", - "gemini-2.5-pro", + "gpt-4o", "grok-code-fast-1", + "gemini-2.5-pro", + "gemini-3-flash-preview", + "gemini-3.1-pro-preview", ]; const GITHUB_COPILOT_DOC_URL: &str = @@ -196,27 +212,18 @@ impl GithubCopilotProvider { DiskCache::new(&host).clear().await } - fn payload_contains_image(payload: &Value) -> bool { - payload - .get("messages") - .and_then(|m| m.as_array()) - .is_some_and(|messages| { - messages.iter().any(|msg| { - msg.get("content").is_some_and(|content| { - content - .as_array() - .map(|arr| arr.iter().collect::>()) - .unwrap_or_else(|| vec![content]) - .iter() - .any(|item| { - matches!( - item.get("type").and_then(|v| v.as_str()), - Some("image_url") | Some("image") - ) - }) - }) - }) + fn messages_contain_image(messages: &[Message]) -> bool { + messages.iter().any(|m| { + m.content.iter().any(|c| match c { + MessageContent::Image(_) => true, + MessageContent::ToolResponse(resp) => resp.tool_result.as_ref().is_ok_and(|r| { + r.content + .iter() + .any(|item| matches!(item.deref(), RawContent::Image(_))) + }), + _ => false, }) + }) } pub async fn from_env(model: ModelConfig) -> Result { @@ -250,13 +257,15 @@ impl GithubCopilotProvider { async fn post( &self, session_id: Option<&str>, + path: &str, is_user_initiated: bool, payload: &mut Value, + has_images: bool, ) -> Result { let (endpoint, token) = self.get_api_info().await?; let auth = AuthMethod::BearerToken(token); let mut headers = self.get_github_headers(); - if Self::payload_contains_image(payload) { + if has_images { headers.insert("Copilot-Vision-Request", "true".parse().unwrap()); } let initiator = if is_user_initiated { "user" } else { "agent" }; @@ -264,7 +273,7 @@ impl GithubCopilotProvider { let api_client = ApiClient::new(endpoint.clone(), auth)?.with_headers(headers)?; api_client - .response_post(session_id, "chat/completions", payload) + .response_post(session_id, path, payload) .await .map_err(|e| e.into()) } @@ -376,6 +385,139 @@ impl GithubCopilotProvider { headers.insert("editor-plugin-version", "copilot/1.155.0".parse().unwrap()); headers } + + #[allow(clippy::too_many_arguments)] + async fn stream_responses( + &self, + model_config: &ModelConfig, + session_id: &str, + is_user_initiated: bool, + system: &str, + messages: &[Message], + tools: &[Tool], + has_images: bool, + ) -> Result { + let mut payload = create_responses_request(model_config, system, messages, tools) + .map_err(|e| ProviderError::RequestFailed(e.to_string()))?; + payload["stream"] = serde_json::Value::Bool(true); + + let mut log = RequestLog::start(model_config, &payload)?; + + let response = self + .with_retry(|| async { + let mut payload_clone = payload.clone(); + let resp = self + .post( + Some(session_id), + "responses", + is_user_initiated, + &mut payload_clone, + has_images, + ) + .await?; + handle_status(resp).await + }) + .await + .inspect_err(|e| { + let _ = log.error(e); + })?; + + stream_responses_compat(response, log) + } + + #[allow(clippy::too_many_arguments)] + async fn stream_chat_completions( + &self, + model_config: &ModelConfig, + session_id: &str, + is_user_initiated: bool, + system: &str, + messages: &[Message], + tools: &[Tool], + has_images: bool, + ) -> Result { + let supports_streaming = GITHUB_COPILOT_STREAM_MODELS + .iter() + .any(|prefix| model_config.model_name.starts_with(prefix)); + + if supports_streaming { + let payload = create_request( + model_config, + system, + messages, + tools, + &ImageFormat::OpenAi, + true, + )?; + let mut log = RequestLog::start(model_config, &payload)?; + + let response = self + .with_retry(|| async { + let mut payload_clone = payload.clone(); + let resp = self + .post( + Some(session_id), + "chat/completions", + is_user_initiated, + &mut payload_clone, + has_images, + ) + .await?; + handle_status(resp).await + }) + .await + .inspect_err(|e| { + let _ = log.error(e); + })?; + + stream_openai_compat(response, log) + } else { + let session_id_opt = if session_id.is_empty() { + None + } else { + Some(session_id) + }; + let payload = create_request( + model_config, + system, + messages, + tools, + &ImageFormat::OpenAi, + false, + )?; + let mut log = RequestLog::start(model_config, &payload)?; + + let response = self + .with_retry(|| async { + let mut payload_clone = payload.clone(); + self.post( + session_id_opt, + "chat/completions", + is_user_initiated, + &mut payload_clone, + has_images, + ) + .await + }) + .await?; + let response = handle_response_openai_compat(response).await?; + + let response = promote_tool_choice(response); + + let message = response_to_message(&response)?; + let usage = response.get("usage").map(get_usage).unwrap_or_else(|| { + tracing::debug!("Failed to get usage data"); + Usage::default() + }); + let response_model = get_model(&response); + log.write(&response, Some(&usage))?; + + Ok(super::base::stream_from_single_message( + message, + ProviderUsage::new(response_model, usage), + )) + } + } } impl ProviderDef for GithubCopilotProvider { @@ -416,7 +558,6 @@ impl Provider for GithubCopilotProvider { self.model.clone() } - // complete_fast() (compaction, title generation) calls this — always agent-initiated. #[tracing::instrument( skip(self, model_config, session_id, system, messages, tools), fields(session.id = %session_id, gen_ai.request.model = %model_config.model_name) @@ -455,80 +596,30 @@ impl Provider for GithubCopilotProvider { .any(|c| matches!(c, MessageContent::ToolResponse(_))) }); let is_user_initiated = !is_agent_call && !last_is_tool_response; + let has_images = Self::messages_contain_image(messages); - // Check if this model supports streaming - let supports_streaming = GITHUB_COPILOT_STREAM_MODELS - .iter() - .any(|prefix| model_config.model_name.starts_with(prefix)); - - if supports_streaming { - // Use streaming API - let payload = create_request( + if is_openai_responses_model(&model_config.model_name) { + self.stream_responses( model_config, + session_id, + is_user_initiated, system, messages, tools, - &ImageFormat::OpenAi, - true, - )?; - let mut log = RequestLog::start(model_config, &payload)?; - - let response = self - .with_retry(|| async { - let mut payload_clone = payload.clone(); - let resp = self - .post(Some(session_id), is_user_initiated, &mut payload_clone) - .await?; - handle_status(resp).await - }) - .await - .inspect_err(|e| { - let _ = log.error(e); - })?; - - stream_openai_compat(response, log) + has_images, + ) + .await } else { - // Use non-streaming API and wrap result - let session_id_opt = if session_id.is_empty() { - None - } else { - Some(session_id) - }; - let payload = create_request( + self.stream_chat_completions( model_config, + session_id, + is_user_initiated, system, messages, tools, - &ImageFormat::OpenAi, - false, - )?; - let mut log = RequestLog::start(model_config, &payload)?; - - // Make request with retry - let response = self - .with_retry(|| async { - let mut payload_clone = payload.clone(); - self.post(session_id_opt, is_user_initiated, &mut payload_clone) - .await - }) - .await?; - let response = handle_response_openai_compat(response).await?; - - let response = promote_tool_choice(response); - - // Parse response - let message = response_to_message(&response)?; - let usage = response.get("usage").map(get_usage).unwrap_or_else(|| { - tracing::debug!("Failed to get usage data"); - Usage::default() - }); - let response_model = get_model(&response); - log.write(&response, Some(&usage))?; - - Ok(super::base::stream_from_single_message( - message, - ProviderUsage::new(response_model, usage), - )) + has_images, + ) + .await } } @@ -576,25 +667,20 @@ impl Provider for GithubCopilotProvider { async fn configure_oauth(&self) -> Result<(), ProviderError> { let config = Config::global(); - // Check if token already exists and is valid if config.get_secret::("GITHUB_COPILOT_TOKEN").is_ok() { - // Try to refresh API info to validate the token match self.refresh_api_info().await { - Ok(_) => return Ok(()), // Token is valid + Ok(_) => return Ok(()), Err(_) => { - // Token is invalid, continue with OAuth flow tracing::debug!("Existing token is invalid, starting OAuth flow"); } } } - // Start OAuth device code flow let token = self .get_access_token() .await .map_err(|e| ProviderError::Authentication(format!("OAuth flow failed: {}", e)))?; - // Save the token config .set_secret("GITHUB_COPILOT_TOKEN", &token) .map_err(|e| ProviderError::ExecutionError(format!("Failed to save token: {}", e)))?; @@ -605,10 +691,6 @@ impl Provider for GithubCopilotProvider { // Copilot sometimes returns multiple choices in a completion response for // Claude models and places the `tool_calls` payload in a non-zero index choice. -// Example: -// - Choice 0: {"finish_reason":"stop","message":{"content":"I'll check the Desktop directory…"}} -// - Choice 1: {"finish_reason":"tool_calls","message":{"tool_calls":[{"function":{"arguments":"{\"command\": -// \"ls -1 ~/Desktop | wc -l\"}","name":"developer__shell"},…}]}} // This function ensures the first choice contains tool metadata so the shared formatter emits a // `ToolRequest` instead of returning only the plain-text choice. fn promote_tool_choice(response: Value) -> Value { @@ -644,9 +726,66 @@ fn promote_tool_choice(response: Value) -> Value { #[cfg(test)] mod tests { - use super::{normalize_host, promote_tool_choice, GithubCopilotUrls}; + use super::{normalize_host, promote_tool_choice, GithubCopilotProvider, GithubCopilotUrls}; + use crate::providers::utils::is_openai_responses_model; use serde_json::json; + #[test] + fn responses_models_routed_correctly() { + assert!(is_openai_responses_model("gpt-5.5")); + assert!(is_openai_responses_model("gpt-5.4")); + assert!(is_openai_responses_model("gpt-5")); + assert!(is_openai_responses_model("gpt-5-mini")); + assert!(is_openai_responses_model("gpt-5-codex")); + assert!(is_openai_responses_model("o3")); + assert!(is_openai_responses_model("o3-mini")); + + assert!(!is_openai_responses_model("gpt-4.1")); + assert!(!is_openai_responses_model("gpt-4o")); + assert!(!is_openai_responses_model("claude-sonnet-4")); + assert!(!is_openai_responses_model("claude-haiku-4.5")); + assert!(!is_openai_responses_model("gemini-2.5-pro")); + } + + #[test] + fn detects_images_in_messages() { + use crate::conversation::message::Message; + + let messages_with_image = vec![Message::user() + .with_text("describe this") + .with_image("base64data", "image/png")]; + assert!(GithubCopilotProvider::messages_contain_image( + &messages_with_image + )); + + let messages_without_image = vec![Message::user().with_text("plain text")]; + assert!(!GithubCopilotProvider::messages_contain_image( + &messages_without_image + )); + } + + #[test] + fn detects_images_in_tool_responses() { + use crate::conversation::message::{Message, MessageContent}; + use rmcp::model::{CallToolResult, Content}; + + let image_content = Content::image("aW1hZ2VkYXRh".to_string(), "image/png".to_string()); + let tool_result = Ok(CallToolResult::success(vec![image_content])); + + let messages = + vec![Message::user() + .with_content(MessageContent::tool_response("call_123", tool_result))]; + assert!(GithubCopilotProvider::messages_contain_image(&messages)); + + let text_result = Ok(CallToolResult::success(vec![Content::text("no images")])); + let messages_text_only = + vec![Message::user() + .with_content(MessageContent::tool_response("call_456", text_result))]; + assert!(!GithubCopilotProvider::messages_contain_image( + &messages_text_only + )); + } + #[test] fn promotes_choice_with_tool_call() { let response = json!({ diff --git a/crates/goose/src/providers/openai.rs b/crates/goose/src/providers/openai.rs index bf4d975192..379b9594ff 100644 --- a/crates/goose/src/providers/openai.rs +++ b/crates/goose/src/providers/openai.rs @@ -6,28 +6,21 @@ use super::embedding::{EmbeddingCapable, EmbeddingRequest, EmbeddingResponse}; use super::errors::ProviderError; use super::formats::openai::{create_request, get_usage, response_to_message}; use super::formats::openai_responses::{ - create_responses_request, get_responses_usage, responses_api_to_message, - responses_api_to_streaming_message, ResponsesApiResponse, + create_responses_request, get_responses_usage, responses_api_to_message, ResponsesApiResponse, }; use super::inventory::{config_secret_value, InventoryIdentityInput}; use super::openai_compatible::{ - handle_response_openai_compat, handle_status, stream_openai_compat, + handle_response_openai_compat, handle_status, stream_openai_compat, stream_responses_compat, }; use super::retry::ProviderRetry; use super::utils::ImageFormat; use crate::config::declarative_providers::DeclarativeProviderConfig; use crate::conversation::message::Message; use anyhow::Result; -use async_stream::try_stream; use async_trait::async_trait; use futures::future::BoxFuture; -use futures::{StreamExt, TryStreamExt}; use reqwest::StatusCode; use std::collections::HashMap; -use std::io; -use tokio::pin; -use tokio_util::codec::{FramedRead, LinesCodec}; -use tokio_util::io::StreamReader; use crate::model::ModelConfig; use crate::providers::base::MessageStream; @@ -746,20 +739,7 @@ impl Provider for OpenAiProvider { })?; if self.supports_streaming { - let stream = response.bytes_stream().map_err(io::Error::other); - - Ok(Box::pin(try_stream! { - let stream_reader = StreamReader::new(stream); - let framed = FramedRead::new(stream_reader, LinesCodec::new()).map_err(anyhow::Error::from); - - let message_stream = responses_api_to_streaming_message(framed); - pin!(message_stream); - while let Some(message) = message_stream.next().await { - let (message, usage) = message.map_err(|e| ProviderError::RequestFailed(format!("Stream decode error: {}", e)))?; - log.write(&message, usage.as_ref().map(|f| f.usage).as_ref())?; - yield (message, usage); - } - })) + stream_responses_compat(response, log) } else { let json: serde_json::Value = response.json().await.map_err(|e| { ProviderError::RequestFailed(format!("Failed to parse JSON: {}", e)) diff --git a/crates/goose/src/providers/openai_compatible.rs b/crates/goose/src/providers/openai_compatible.rs index 1028e5079e..b993c36c68 100644 --- a/crates/goose/src/providers/openai_compatible.rs +++ b/crates/goose/src/providers/openai_compatible.rs @@ -18,6 +18,7 @@ use super::utils::{ImageFormat, RequestLog}; use crate::conversation::message::Message; use crate::model::ModelConfig; use crate::providers::formats::openai::{create_request, response_to_streaming_message}; +use crate::providers::formats::openai_responses::responses_api_to_streaming_message; use rmcp::model::Tool; pub struct OpenAiCompatibleProvider { @@ -161,6 +162,29 @@ pub fn stream_openai_compat( })) } +pub fn stream_responses_compat( + response: Response, + mut log: RequestLog, +) -> Result { + let stream = response.bytes_stream().map_err(std::io::Error::other); + + Ok(Box::pin(try_stream! { + let stream_reader = StreamReader::new(stream); + let framed = FramedRead::new(stream_reader, LinesCodec::new()) + .map_err(Error::from); + + let message_stream = responses_api_to_streaming_message(framed); + pin!(message_stream); + while let Some(message) = message_stream.next().await { + let (message, usage) = message.map_err(|e| + ProviderError::RequestFailed(format!("Stream decode error: {e}")) + )?; + log.write(&message, usage.as_ref().map(|f| f.usage).as_ref())?; + yield (message, usage); + } + })) +} + #[cfg(test)] mod tests { use super::*;