awesome-bookmarks/goose
3
0
Fork 0
mirror of https://github.com/block/goose.git synced 2026-07-17 12:56:20 +02:00
Code Issues Packages Projects Releases Wiki Activity

fix: prevent abort in local inference (#7633)

Browse Source 
Signed-off-by: Kube Cat <cat@kubecat.io>
Co-authored-by: Kube Cat <cat@kubecat.io>
This commit is contained in:
KubeCat
2026-03-04 15:07:28 +01:00
committed by GitHub
parent 1285b31ce6
commit dafc4db7b4
4 changed files with 95 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/goose-cli/Cargo.toml
+1
View File
@@ -72,6 +72,7 @@ winapi = { version = "0.3", features = ["wincred"] }
[features]
default = ["code-mode"]
code-mode = ["goose/code-mode", "goose-acp/code-mode"]
cuda = ["goose/cuda"]
# disables the update command
disable-update = []
crates/goose/src/providers/local_inference/inference_emulated_tools.rs
+10 -1
View File
@@ -359,10 +359,19 @@ pub(super) fn generate_with_emulated_tools(
ctx: &mut GenerationContext<'_>,
code_mode_enabled: bool,
) -> Result<(), ProviderError> {
// Use oaicompat variant — its C++ wrapper catches exceptions that would
// otherwise abort the process when other native libs disturb the C++ ABI.
let prompt = ctx
.loaded
.model
.apply_chat_template(&ctx.loaded.template, ctx.chat_messages, true)
.apply_chat_template_with_tools_oaicompat(
&ctx.loaded.template,
ctx.chat_messages,
None, // no tools for emulated path
None, // no json_schema
true, // add_generation_prompt
)
.map(|r| r.prompt)
.map_err(|e| {
ProviderError::ExecutionError(format!("Failed to apply chat template: {}", e))
})?;
crates/goose/tests/local_inference_integration.rs
+18 -51
View File
@@ -1,20 +1,30 @@
//! Integration tests for LocalInferenceProvider.
//!
//! These tests require a downloaded GGUF model and are ignored by default.
//! Run with: cargo test -p goose --test local_inference_integration -- --ignored
//! Download a model first:
//! goose local-models download bartowski/Llama-3.2-1B-Instruct-GGUF:Q4_K_M
//!
//! Run with the default model:
//! cargo test -p goose --test local_inference_integration -- --ignored
//!
//! Run with a specific model:
//! TEST_MODEL="bartowski/Qwen_Qwen3-32B-GGUF:Q4_K_M" cargo test -p goose --test local_inference_integration -- --ignored
use futures::StreamExt;
use goose::conversation::message::Message;
use goose::model::ModelConfig;
use goose::providers::create;
use std::time::Instant;
const TEST_MODEL: &str = "llama-3.2-1b";
const DEFAULT_TEST_MODEL: &str = "bartowski/Llama-3.2-1B-Instruct-GGUF:Q4_K_M";
fn test_model() -> String {
std::env::var("TEST_MODEL").unwrap_or_else(|_| DEFAULT_TEST_MODEL.to_string())
}
#[tokio::test]
#[ignore]
async fn test_local_inference_stream_produces_output() {
let model_config = ModelConfig::new(TEST_MODEL).expect("valid model config");
let model_config = ModelConfig::new(&test_model()).expect("valid model config");
let provider = create("local", model_config.clone(), Vec::new())
.await
.expect("provider creation should succeed");
@@ -53,55 +63,12 @@ async fn test_local_inference_stream_produces_output() {
assert!(got_usage, "stream should produce usage info");
}
#[tokio::test]
#[ignore]
async fn test_local_inference_cold_and_warm_performance() {
let model_config = ModelConfig::new(TEST_MODEL).expect("valid model config");
let provider = create("local", model_config.clone(), Vec::new())
.await
.expect("provider creation should succeed");
// Cold start (includes model loading)
let messages = vec![Message::user().with_text("what is the capital of Moldova?")];
let start = Instant::now();
let (response, _usage) = provider
.complete(&model_config, "test-session", "", &messages, &[])
.await
.expect("cold completion should succeed");
let cold_elapsed = start.elapsed();
let text = response.as_concat_text();
assert!(!text.is_empty(), "cold start should produce a response");
println!(
"Cold start: {cold_elapsed:.2?}, response length: {}",
text.len()
);
// Warm run (model already loaded)
let messages2 = vec![Message::user().with_text("what is the capital of France?")];
let start2 = Instant::now();
let (response2, _usage2) = provider
.complete(&model_config, "test-session", "", &messages2, &[])
.await
.expect("warm completion should succeed");
let warm_elapsed = start2.elapsed();
let text2 = response2.as_concat_text();
assert!(!text2.is_empty(), "warm run should produce a response");
println!(
"Warm run: {warm_elapsed:.2?}, response length: {}",
text2.len()
);
assert!(
warm_elapsed < cold_elapsed,
"warm run ({warm_elapsed:.2?}) should be faster than cold start ({cold_elapsed:.2?})"
);
}
#[tokio::test]
#[ignore]
async fn test_local_inference_large_prompt() {
let model_config = ModelConfig::new(TEST_MODEL).expect("valid model config");
let model_config = ModelConfig::new(&test_model())
.expect("valid model config")
.with_max_tokens(Some(20));
let provider = create("local", model_config.clone(), Vec::new())
.await
.expect("provider creation should succeed");
@@ -111,7 +78,7 @@ async fn test_local_inference_large_prompt() {
let prompt = format!("{padding}\nNow answer this: what is the capital of Moldova?");
let messages = vec![Message::user().with_text(&prompt)];
let start = Instant::now();
let start = std::time::Instant::now();
let (response, _usage) = provider
.complete(&model_config, "test-session", "", &messages, &[])
.await
crates/goose/tests/local_inference_perf.rs
+66
View File
@@ -0,0 +1,66 @@
//! Performance benchmarks for LocalInferenceProvider.
//!
//! These tests require a downloaded GGUF model and are ignored by default.
//! Download a model first:
//! goose local-models download bartowski/Llama-3.2-1B-Instruct-GGUF:Q4_K_M
//!
//! Run with the default model:
//! cargo test -p goose --test local_inference_perf -- --ignored --nocapture
//!
//! Run with a specific model:
//! TEST_MODEL="bartowski/Qwen_Qwen3-32B-GGUF:Q4_K_M" cargo test -p goose --test local_inference_perf -- --ignored --nocapture
use goose::conversation::message::Message;
use goose::model::ModelConfig;
use goose::providers::create;
use std::time::Instant;
const DEFAULT_TEST_MODEL: &str = "bartowski/Llama-3.2-1B-Instruct-GGUF:Q4_K_M";
fn test_model() -> String {
std::env::var("TEST_MODEL").unwrap_or_else(|_| DEFAULT_TEST_MODEL.to_string())
}
#[tokio::test]
#[ignore]
async fn test_local_inference_cold_vs_warm() {
let model_config = ModelConfig::new(&test_model())
.expect("valid model config")
.with_max_tokens(Some(20));
let provider = create("local", model_config.clone(), Vec::new())
.await
.expect("provider creation should succeed");
// Cold start — includes model loading from disk.
let messages = vec![Message::user().with_text("What is 2+2?")];
let start = Instant::now();
let (response, _) = provider
.complete(&model_config, "perf-session", "", &messages, &[])
.await
.expect("cold completion should succeed");
let cold_elapsed = start.elapsed();
let text = response.as_concat_text();
assert!(!text.is_empty(), "cold start should produce a response");
println!("Cold start: {cold_elapsed:.2?}, response: {}", text.len());
// Warm run — model already loaded, only inference.
let messages2 = vec![Message::user().with_text("What is 3+3?")];
let start2 = Instant::now();
let (response2, _) = provider
.complete(&model_config, "perf-session", "", &messages2, &[])
.await
.expect("warm completion should succeed");
let warm_elapsed = start2.elapsed();
let text2 = response2.as_concat_text();
assert!(!text2.is_empty(), "warm run should produce a response");
println!("Warm run: {warm_elapsed:.2?}, response: {}", text2.len());
if warm_elapsed < cold_elapsed {
let speedup = cold_elapsed.as_secs_f64() / warm_elapsed.as_secs_f64();
println!("Warm is {speedup:.1}x faster than cold");
} else {
println!("Warning: warm was not faster (model may have been pre-loaded by another test)");
}
}