awesome-bookmarks/goose
3
0
Fork 0
mirror of https://github.com/block/goose.git synced 2026-07-17 12:56:20 +02:00
Code Issues Packages Projects Releases Wiki Activity

Switched local inference to use llama.cpp and added HF model download

Browse Source
This commit is contained in:
jh-block
2026-02-10 21:14:19 +01:00
parent 2477a18be3
commit bf3193d890
25 changed files with 5218 additions and 1760 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+231 -22
View File
@@ -140,7 +140,7 @@ version = "1.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc"
dependencies = [
"windows-sys 0.60.2",
"windows-sys 0.61.2",
]
[[package]]
@@ -151,7 +151,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d"
dependencies = [
"anstyle",
"once_cell_polyfill",
"windows-sys 0.60.2",
"windows-sys 0.61.2",
]
[[package]]
@@ -1568,7 +1568,7 @@ dependencies = [
"candle-kernels",
"candle-metal-kernels",
"candle-ug",
"cudarc 0.19.1",
"cudarc 0.19.2",
"float8 0.6.1",
"gemm 0.19.0",
"half",
@@ -2285,9 +2285,9 @@ dependencies = [
[[package]]
name = "cudarc"
version = "0.19.1"
version = "0.19.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "60d4882b3e023670c25b8a3e1ac97349070f8e5400807c091b70e0bdad6e9b93"
checksum = "aed81f178e780f3d5d354d12b4c5c5a484c4a9c329ecd037ac57f2a0e0648397"
dependencies = [
"float8 0.7.0",
"half",
@@ -3020,7 +3020,7 @@ dependencies = [
"libc",
"option-ext",
"redox_users 0.5.2",
"windows-sys 0.59.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -3233,6 +3233,26 @@ dependencies = [
"syn 2.0.114",
]
[[package]]
name = "enumflags2"
version = "0.7.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1027f7680c853e056ebcec683615fb6fbbc07dbaa13b4d5d9442b146ded4ecef"
dependencies = [
"enumflags2_derive",
]
[[package]]
name = "enumflags2_derive"
version = "0.7.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "67c78a4d8fdf9953a5c9d458f9efe940fd97a0cab0941c075a813ac594733827"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.114",
]
[[package]]
name = "env-lock"
version = "1.0.2"
@@ -3292,7 +3312,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
dependencies = [
"libc",
"windows-sys 0.52.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -3459,6 +3479,15 @@ version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582"
[[package]]
name = "find_cuda_helper"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f9f9e65c593dd01ac77daad909ea4ad17f0d6d1776193fc8ea766356177abdad"
dependencies = [
"glob",
]
[[package]]
name = "fixedbitset"
version = "0.5.7"
@@ -3481,7 +3510,7 @@ version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "719a903cc23e4a89e87962c2a80fdb45cdaad0983a89bd150bb57b4c8571a7d5"
dependencies = [
"cudarc 0.19.1",
"cudarc 0.19.2",
"half",
"num-traits",
"rand 0.9.2",
@@ -4065,6 +4094,19 @@ dependencies = [
"wasm-bindgen",
]
[[package]]
name = "getrandom"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "139ef39800118c7683f2fd3c98c1b23c09ae076556b435f8e9064ae108aaeeec"
dependencies = [
"cfg-if",
"libc",
"r-efi",
"wasip2",
"wasip3",
]
[[package]]
name = "gif"
version = "0.13.3"
@@ -4135,6 +4177,7 @@ dependencies = [
"dashmap 6.1.0",
"dirs 5.0.1",
"dotenvy",
"encoding_rs",
"env-lock",
"etcetera 0.11.0",
"fs2",
@@ -4151,6 +4194,7 @@ dependencies = [
"jsonwebtoken",
"keyring",
"lazy_static",
"llama-cpp-2",
"lru",
"minijinja",
"mockall",
@@ -4973,6 +5017,12 @@ dependencies = [
"zerovec",
]
[[package]]
name = "id-arena"
version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954"
[[package]]
name = "ident_case"
version = "1.0.1"
@@ -5275,7 +5325,7 @@ dependencies = [
"portable-atomic",
"portable-atomic-util",
"serde_core",
"windows-sys 0.52.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -5476,6 +5526,12 @@ dependencies = [
"spin",
]
[[package]]
name = "leb128fmt"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2"
[[package]]
name = "lebe"
version = "0.5.3"
@@ -5484,9 +5540,9 @@ checksum = "7a79a3332a6609480d7d0c9eab957bca6b455b91bb84e66d19f5ff66294b85b8"
[[package]]
name = "libc"
version = "0.2.180"
version = "0.2.181"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bcc35a38544a891a5f7c865aca548a982ccb3b8650a5b06d0fd33a10283c56fc"
checksum = "459427e2af2b9c839b132acb702a1c654d95e10f8c326bfc2ad11310e458b1c5"
[[package]]
name = "libdbus-sys"
@@ -5580,6 +5636,34 @@ version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092"
[[package]]
name = "llama-cpp-2"
version = "0.1.133"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "888c8805527f4c35ec16f26003d54a318cde1629e7439da8e9ef2d6d3883e106"
dependencies = [
"encoding_rs",
"enumflags2",
"llama-cpp-sys-2",
"thiserror 1.0.69",
"tracing",
"tracing-core",
]
[[package]]
name = "llama-cpp-sys-2"
version = "0.1.133"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a180dfa6d6f9d1df1e031bcdf0464bbad4f9b326395bfd28f2fa539d8cbc9c2b"
dependencies = [
"bindgen",
"cc",
"cmake",
"find_cuda_helper",
"glob",
"walkdir",
]
[[package]]
name = "lock_api"
version = "0.4.14"
@@ -5996,7 +6080,7 @@ version = "0.50.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5"
dependencies = [
"windows-sys 0.59.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -8146,7 +8230,7 @@ dependencies = [
"errno",
"libc",
"linux-raw-sys 0.11.0",
"windows-sys 0.52.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -9971,15 +10055,15 @@ dependencies = [
[[package]]
name = "tempfile"
version = "3.24.0"
version = "3.25.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "655da9c7eb6305c55742045d5a8d2037996d61d8de95806335c7c86ce0f82e9c"
checksum = "0136791f7c95b1f6dd99f9cc786b91bb81c3800b639b3478e561ddb7be95e5f1"
dependencies = [
"fastrand",
"getrandom 0.3.4",
"getrandom 0.4.1",
"once_cell",
"rustix 1.1.3",
"windows-sys 0.52.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -10430,9 +10514,9 @@ dependencies = [
[[package]]
name = "toml"
version = "0.9.11+spec-1.1.0"
version = "0.9.12+spec-1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f3afc9a848309fe1aaffaed6e1546a7a14de1f935dc9d89d32afd9a44bab7c46"
checksum = "cf92845e79fc2e2def6a5d828f0801e29a2f8acc037becc5ab08595c7d5e9863"
dependencies = [
"indexmap 2.13.0",
"serde_core",
@@ -10454,9 +10538,9 @@ dependencies = [
[[package]]
name = "toml_parser"
version = "1.0.6+spec-1.1.0"
version = "1.0.7+spec-1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a3198b4b0a8e11f09dd03e133c0280504d0801269e9afa46362ffde1cbeebf44"
checksum = "247eaa3197818b831697600aadf81514e577e0cba5eab10f7e064e78ae154df1"
dependencies = [
"winnow",
]
@@ -11285,6 +11369,15 @@ dependencies = [
"wit-bindgen",
]
[[package]]
name = "wasip3"
version = "0.4.0+wasi-0.3.0-rc-2026-01-06"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5"
dependencies = [
"wit-bindgen",
]
[[package]]
name = "wasite"
version = "0.1.0"
@@ -11350,6 +11443,28 @@ dependencies = [
"unicode-ident",
]
[[package]]
name = "wasm-encoder"
version = "0.244.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319"
dependencies = [
"leb128fmt",
"wasmparser",
]
[[package]]
name = "wasm-metadata"
version = "0.244.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909"
dependencies = [
"anyhow",
"indexmap 2.13.0",
"wasm-encoder",
"wasmparser",
]
[[package]]
name = "wasm-streams"
version = "0.4.2"
@@ -11373,6 +11488,18 @@ dependencies = [
"thiserror 2.0.18",
]
[[package]]
name = "wasmparser"
version = "0.244.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe"
dependencies = [
"bitflags 2.10.0",
"hashbrown 0.15.5",
"indexmap 2.13.0",
"semver",
]
[[package]]
name = "web-sys"
version = "0.3.85"
@@ -11500,7 +11627,7 @@ version = "0.1.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
dependencies = [
"windows-sys 0.48.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -12101,6 +12228,88 @@ name = "wit-bindgen"
version = "0.51.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5"
dependencies = [
"wit-bindgen-rust-macro",
]
[[package]]
name = "wit-bindgen-core"
version = "0.51.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc"
dependencies = [
"anyhow",
"heck",
"wit-parser",
]
[[package]]
name = "wit-bindgen-rust"
version = "0.51.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21"
dependencies = [
"anyhow",
"heck",
"indexmap 2.13.0",
"prettyplease",
"syn 2.0.114",
"wasm-metadata",
"wit-bindgen-core",
"wit-component",
]
[[package]]
name = "wit-bindgen-rust-macro"
version = "0.51.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a"
dependencies = [
"anyhow",
"prettyplease",
"proc-macro2",
"quote",
"syn 2.0.114",
"wit-bindgen-core",
"wit-bindgen-rust",
]
[[package]]
name = "wit-component"
version = "0.244.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2"
dependencies = [
"anyhow",
"bitflags 2.10.0",
"indexmap 2.13.0",
"log",
"serde",
"serde_derive",
"serde_json",
"wasm-encoder",
"wasm-metadata",
"wasmparser",
"wit-parser",
]
[[package]]
name = "wit-parser"
version = "0.244.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736"
dependencies = [
"anyhow",
"id-arena",
"indexmap 2.13.0",
"log",
"semver",
"serde",
"serde_derive",
"serde_json",
"unicode-xid",
"wasmparser",
]
[[package]]
name = "writeable"
crates/goose-cli/src/cli.rs
+205
View File
@@ -854,6 +854,13 @@ enum Command {
#[command(subcommand)]
command: TermCommand,
},
/// Manage local inference models
#[command(about = "Manage local inference models", visible_alias = "lm")]
LocalModels {
#[command(subcommand)]
command: LocalModelsCommand,
},
/// Generate completions for various shells
#[command(about = "Generate the autocompletion script for the specified shell")]
Completion {
@@ -865,6 +872,38 @@ enum Command {
},
}
#[derive(Subcommand)]
enum LocalModelsCommand {
/// Search HuggingFace for GGUF models
#[command(about = "Search HuggingFace for GGUF models")]
Search {
/// Search query
query: String,
/// Maximum number of results
#[arg(short, long, default_value = "10")]
limit: usize,
},
/// Download a model from HuggingFace
#[command(about = "Download a GGUF model (e.g. bartowski/Llama-3.2-1B-Instruct-GGUF:Q4_K_M)")]
Download {
/// Model spec in user/repo:quantization format
spec: String,
},
/// List downloaded local models
#[command(about = "List downloaded local models")]
List,
/// Delete a downloaded model
#[command(about = "Delete a downloaded local model")]
Delete {
/// Model ID to delete
id: String,
},
}
#[derive(Subcommand)]
enum TermCommand {
/// Print shell initialization script
@@ -954,6 +993,7 @@ fn get_command_name(command: &Option<Command>) -> &'static str {
Some(Command::Recipe { .. }) => "recipe",
Some(Command::Web { .. }) => "web",
Some(Command::Term { .. }) => "term",
Some(Command::LocalModels { .. }) => "local-models",
Some(Command::Completion { .. }) => "completion",
None => "default_session",
}
@@ -1389,6 +1429,170 @@ async fn handle_term_subcommand(command: TermCommand) -> Result<()> {
}
}
async fn handle_local_models_command(command: LocalModelsCommand) -> Result<()> {
use goose::providers::local_inference::hf_models;
use goose::providers::local_inference::local_model_registry::{
display_name_from_repo, get_registry, model_id_from_repo, LocalModelEntry,
};
match command {
LocalModelsCommand::Search { query, limit } => {
println!("Searching HuggingFace for '{}'...", query);
let results = hf_models::search_gguf_models(&query, limit).await?;
if results.is_empty() {
println!("No GGUF models found.");
return Ok(());
}
for model in &results {
println!(
"\n{} (by {}) — {} downloads",
model.model_name, model.author, model.downloads
);
for file in &model.gguf_files {
let size = if file.size_bytes > 0 {
format!(
"{:.1}GB",
file.size_bytes as f64 / (1024.0 * 1024.0 * 1024.0)
)
} else {
"unknown".to_string()
};
println!(" {} — {}", file.quantization, size);
}
println!(
" Download: goose local-models download {}:<quantization>",
model.repo_id
);
}
}
LocalModelsCommand::Download { spec } => {
println!("Resolving {}...", spec);
let (repo_id, file) = hf_models::resolve_model_spec(&spec).await?;
let model_id = model_id_from_repo(&repo_id, &file.quantization);
let display_name = display_name_from_repo(&repo_id, &file.quantization);
let local_path =
goose::config::paths::Paths::in_data_dir("models").join(&file.filename);
println!(
"Downloading {} ({})...",
display_name,
if file.size_bytes > 0 {
format!(
"{:.1}GB",
file.size_bytes as f64 / (1024.0 * 1024.0 * 1024.0)
)
} else {
"unknown size".to_string()
}
);
// Register
let entry = LocalModelEntry {
id: model_id.clone(),
display_name: display_name.clone(),
repo_id: repo_id.clone(),
filename: file.filename.clone(),
quantization: file.quantization.clone(),
local_path: local_path.clone(),
source_url: file.download_url.clone(),
settings: Default::default(),
};
{
let mut registry = get_registry()
.lock()
.map_err(|_| anyhow::anyhow!("Failed to acquire registry lock"))?;
registry.add_model(entry)?;
}
// Download
let manager = goose::dictation::download_manager::get_download_manager();
manager
.download_model(
format!("{}-model", model_id),
file.download_url,
local_path,
None,
None,
)
.await?;
// Poll progress
loop {
if let Some(progress) = manager.get_progress(&format!("{}-model", model_id)) {
match progress.status {
goose::dictation::download_manager::DownloadStatus::Downloading => {
print!(
"\r {:.1}% ({:.0}MB / {:.0}MB)",
progress.progress_percent,
progress.bytes_downloaded as f64 / (1024.0 * 1024.0),
progress.total_bytes as f64 / (1024.0 * 1024.0),
);
use std::io::Write;
std::io::stdout().flush().ok();
}
goose::dictation::download_manager::DownloadStatus::Completed => {
println!("\nDownloaded: {} (id: {})", display_name, model_id);
break;
}
goose::dictation::download_manager::DownloadStatus::Failed => {
let err = progress.error.unwrap_or_default();
anyhow::bail!("Download failed: {}", err);
}
goose::dictation::download_manager::DownloadStatus::Cancelled => {
println!("\nDownload cancelled.");
break;
}
}
}
tokio::time::sleep(std::time::Duration::from_millis(500)).await;
}
}
LocalModelsCommand::List => {
let registry = get_registry()
.lock()
.map_err(|_| anyhow::anyhow!("Failed to acquire registry lock"))?;
let models = registry.list_models();
if models.is_empty() {
println!("No local models downloaded.");
return Ok(());
}
println!("{:<40} {:<20} {:<10} Downloaded", "ID", "Name", "Quant");
println!("{}", "-".repeat(80));
for m in models {
println!(
"{:<40} {:<20} {:<10} {}",
m.id,
m.display_name,
m.quantization,
if m.is_downloaded() { "✓" } else { "✗" }
);
}
}
LocalModelsCommand::Delete { id } => {
let mut registry = get_registry()
.lock()
.map_err(|_| anyhow::anyhow!("Failed to acquire registry lock"))?;
if let Some(entry) = registry.get_model(&id) {
if entry.local_path.exists() {
std::fs::remove_file(&entry.local_path)?;
}
registry.remove_model(&id)?;
println!("Deleted model: {}", id);
} else {
println!("Model not found: {}", id);
}
}
}
Ok(())
}
async fn handle_default_session() -> Result<()> {
if !Config::global().exists() {
return handle_configure().await;
@@ -1519,6 +1723,7 @@ pub async fn cli() -> anyhow::Result<()> {
no_auth,
}) => crate::commands::web::handle_web(port, host, open, auth_token, no_auth).await,
Some(Command::Term { command }) => handle_term_subcommand(command).await,
Some(Command::LocalModels { command }) => handle_local_models_command(command).await,
None => handle_default_session().await,
}
}
crates/goose-server/src/openapi.rs
+15
View File
@@ -422,10 +422,15 @@ derive_utoipa!(Icon as IconSchema);
super::routes::dictation::cancel_download,
super::routes::dictation::delete_model,
super::routes::local_inference::list_local_models,
super::routes::local_inference::search_hf_models,
super::routes::local_inference::get_repo_files,
super::routes::local_inference::download_hf_model,
super::routes::local_inference::download_local_model,
super::routes::local_inference::get_local_model_download_progress,
super::routes::local_inference::cancel_local_model_download,
super::routes::local_inference::delete_local_model,
super::routes::local_inference::get_model_settings,
super::routes::local_inference::update_model_settings,
),
components(schemas(
super::routes::config_management::UpsertConfigQuery,
@@ -592,8 +597,18 @@ derive_utoipa!(Icon as IconSchema);
super::routes::dictation::DictationProviderStatus,
super::routes::dictation::WhisperModelResponse,
super::routes::local_inference::LocalModelResponse,
super::routes::local_inference::RegistryModelResponse,
super::routes::local_inference::ModelListItem,
super::routes::local_inference::HfDownloadRequest,
super::routes::local_inference::HfDownloadResponse,
goose::providers::local_inference::LocalLlmModel,
goose::providers::local_inference::ModelTier,
goose::providers::local_inference::hf_models::HfModelInfo,
goose::providers::local_inference::hf_models::HfGgufFile,
goose::providers::local_inference::hf_models::HfQuantVariant,
super::routes::local_inference::RepoVariantsResponse,
goose::providers::local_inference::local_model_registry::ModelSettings,
goose::providers::local_inference::local_model_registry::SamplingConfig,
DownloadProgress,
DownloadStatus,
))
crates/goose-server/src/routes/local_inference.rs
+305 -62
View File
@@ -1,17 +1,22 @@
use crate::routes::errors::ErrorResponse;
use crate::state::AppState;
use axum::{
extract::Path,
extract::{Path, Query},
http::StatusCode,
routing::{delete, get, post},
Json, Router,
};
use goose::config::paths::Paths;
use goose::dictation::download_manager::{get_download_manager, DownloadProgress};
use goose::providers::local_inference::{
available_local_models, get_local_model, recommend_local_model, LocalLlmModel,
LOCAL_LLM_MODEL_CONFIG_KEY,
use goose::providers::local_inference::hf_models::{self, HfModelInfo, HfQuantVariant};
use goose::providers::local_inference::local_model_registry::{
display_name_from_repo, get_registry, model_id_from_repo, LocalModelEntry, ModelSettings,
};
use serde::Serialize;
use goose::providers::local_inference::{
available_inference_memory_bytes, available_local_models, get_local_model,
recommend_local_model, LocalLlmModel, LOCAL_LLM_MODEL_CONFIG_KEY,
};
use serde::{Deserialize, Serialize};
use std::sync::Arc;
use utoipa::ToSchema;
@@ -22,6 +27,27 @@ pub struct LocalModelResponse {
pub model: &'static LocalLlmModel,
pub downloaded: bool,
pub recommended: bool,
pub featured: bool,
}
#[derive(Debug, Serialize, ToSchema)]
pub struct RegistryModelResponse {
pub id: String,
pub display_name: String,
pub repo_id: String,
pub filename: String,
pub quantization: String,
pub size_bytes: u64,
pub downloaded: bool,
pub featured: bool,
pub settings: ModelSettings,
}
#[derive(Debug, Serialize, ToSchema)]
#[serde(untagged)]
pub enum ModelListItem {
Featured(LocalModelResponse),
Registry(Box<RegistryModelResponse>),
}
fn convert_error(e: anyhow::Error) -> ErrorResponse {
@@ -45,21 +71,194 @@ fn convert_error(e: anyhow::Error) -> ErrorResponse {
get,
path = "/local-inference/models",
responses(
(status = 200, description = "List of available local LLM models", body = Vec<LocalModelResponse>)
(status = 200, description = "List of available local LLM models", body = Vec<ModelListItem>)
)
)]
pub async fn list_local_models() -> Result<Json<Vec<LocalModelResponse>>, ErrorResponse> {
let recommended_id = recommend_local_model();
let models = available_local_models()
pub async fn list_local_models(
axum::extract::State(state): axum::extract::State<Arc<AppState>>,
) -> Result<Json<Vec<ModelListItem>>, ErrorResponse> {
let recommended_id = recommend_local_model(&state.inference_runtime);
let featured_ids: Vec<&str> = available_local_models().iter().map(|m| m.id).collect();
let mut items: Vec<ModelListItem> = available_local_models()
.iter()
.map(|m| LocalModelResponse {
model: m,
downloaded: m.is_downloaded(),
recommended: m.id == recommended_id,
.map(|m| {
ModelListItem::Featured(LocalModelResponse {
model: m,
downloaded: m.is_downloaded(),
recommended: m.id == recommended_id,
featured: true,
})
})
.collect();
Ok(Json(models))
// Add registry models that aren't in the featured list
if let Ok(registry) = get_registry().lock() {
for entry in registry.list_models() {
if !featured_ids.contains(&entry.id.as_str()) {
items.push(ModelListItem::Registry(Box::new(RegistryModelResponse {
id: entry.id.clone(),
display_name: entry.display_name.clone(),
repo_id: entry.repo_id.clone(),
filename: entry.filename.clone(),
quantization: entry.quantization.clone(),
size_bytes: entry.file_size(),
downloaded: entry.is_downloaded(),
featured: false,
settings: entry.settings.clone(),
})));
}
}
}
Ok(Json(items))
}
#[derive(Debug, Deserialize)]
pub struct SearchQuery {
pub q: String,
pub limit: Option<usize>,
}
#[utoipa::path(
get,
path = "/local-inference/search",
params(
("q" = String, Query, description = "Search query"),
("limit" = Option<usize>, Query, description = "Max results")
),
responses(
(status = 200, description = "Search results", body = Vec<HfModelInfo>),
(status = 500, description = "Search failed")
)
)]
pub async fn search_hf_models(
Query(params): Query<SearchQuery>,
) -> Result<Json<Vec<HfModelInfo>>, ErrorResponse> {
let limit = params.limit.unwrap_or(20).min(50);
let results = hf_models::search_gguf_models(&params.q, limit)
.await
.map_err(|e| ErrorResponse::internal(format!("Search failed: {}", e)))?;
Ok(Json(results))
}
#[derive(Debug, Serialize, ToSchema)]
pub struct RepoVariantsResponse {
pub variants: Vec<HfQuantVariant>,
pub recommended_index: Option<usize>,
}
#[utoipa::path(
get,
path = "/local-inference/repo/{author}/{repo}/files",
responses(
(status = 200, description = "GGUF quantization variants in repo", body = RepoVariantsResponse),
(status = 500, description = "Failed to fetch repo files")
)
)]
pub async fn get_repo_files(
axum::extract::State(state): axum::extract::State<Arc<AppState>>,
Path((author, repo)): Path<(String, String)>,
) -> Result<Json<RepoVariantsResponse>, ErrorResponse> {
let repo_id = format!("{}/{}", author, repo);
let variants = hf_models::get_repo_gguf_variants(&repo_id)
.await
.map_err(|e| ErrorResponse::internal(format!("Failed to fetch repo files: {}", e)))?;
let available_memory = available_inference_memory_bytes(&state.inference_runtime);
let recommended_index = hf_models::recommend_variant(&variants, available_memory);
Ok(Json(RepoVariantsResponse {
variants,
recommended_index,
}))
}
#[derive(Debug, Deserialize, ToSchema)]
pub struct HfDownloadRequest {
pub repo_id: Option<String>,
pub filename: Option<String>,
pub spec: Option<String>,
}
#[derive(Debug, Serialize, ToSchema)]
pub struct HfDownloadResponse {
pub model_id: String,
}
#[utoipa::path(
post,
path = "/local-inference/download",
request_body = HfDownloadRequest,
responses(
(status = 202, description = "Download started", body = HfDownloadResponse),
(status = 400, description = "Invalid request"),
(status = 500, description = "Internal server error")
)
)]
pub async fn download_hf_model(
Json(req): Json<HfDownloadRequest>,
) -> Result<(StatusCode, Json<HfDownloadResponse>), ErrorResponse> {
let (repo_id, filename, quantization, download_url) = if let Some(spec) = &req.spec {
let (repo_id, file) = hf_models::resolve_model_spec(spec)
.await
.map_err(|e| ErrorResponse::bad_request(format!("{}", e)))?;
(repo_id, file.filename, file.quantization, file.download_url)
} else if let (Some(repo_id), Some(filename)) = (&req.repo_id, &req.filename) {
let quantization = hf_models::parse_quantization_from_filename(filename);
let download_url = format!(
"https://huggingface.co/{}/resolve/main/{}",
repo_id, filename
);
(
repo_id.clone(),
filename.clone(),
quantization,
download_url,
)
} else {
return Err(ErrorResponse::bad_request(
"Provide either 'spec' or both 'repo_id' and 'filename'",
));
};
let model_id = model_id_from_repo(&repo_id, &quantization);
let display_name = display_name_from_repo(&repo_id, &quantization);
let local_path = Paths::in_data_dir("models").join(&filename);
let entry = LocalModelEntry {
id: model_id.clone(),
display_name,
repo_id,
filename: filename.clone(),
quantization,
local_path: local_path.clone(),
source_url: download_url.clone(),
settings: ModelSettings::default(),
};
{
let mut registry = get_registry()
.lock()
.map_err(|_| ErrorResponse::internal("Failed to acquire registry lock"))?;
registry
.add_model(entry)
.map_err(|e| ErrorResponse::internal(format!("{}", e)))?;
}
let manager = get_download_manager();
manager
.download_model(
format!("{}-model", model_id),
download_url,
local_path,
Some(LOCAL_LLM_MODEL_CONFIG_KEY.to_string()),
Some(model_id.clone()),
)
.await
.map_err(convert_error)?;
Ok((StatusCode::ACCEPTED, Json(HfDownloadResponse { model_id })))
}
#[utoipa::path(
@@ -79,26 +278,11 @@ pub async fn download_local_model(
let manager = get_download_manager();
// Download model file (don't set config yet - wait for tokenizer)
manager
.download_model(
format!("{}-model", model.id),
model.url.to_string(),
model.local_path(),
None,
None,
)
.await
.map_err(convert_error)?;
// Download tokenizer file (set config and provider when this completes)
// We'll set GOOSE_PROVIDER to "local" after the tokenizer download completes
// This is handled in the download_manager callback
manager
.download_model(
format!("{}-tokenizer", model.id),
model.tokenizer_url.to_string(),
model.tokenizer_path(),
Some(LOCAL_LLM_MODEL_CONFIG_KEY.to_string()),
Some(model.id.to_string()),
)
@@ -121,26 +305,10 @@ pub async fn get_local_model_download_progress(
) -> Result<Json<DownloadProgress>, ErrorResponse> {
let manager = get_download_manager();
// Check both model and tokenizer progress
let model_progress = manager
.get_progress(&format!("{}-model", model_id))
.ok_or_else(|| ErrorResponse::not_found("Download not found"))?;
let tokenizer_progress = manager.get_progress(&format!("{}-tokenizer", model_id));
// If tokenizer failed, return that error
if let Some(tok_prog) = tokenizer_progress {
if tok_prog.status == goose::dictation::download_manager::DownloadStatus::Failed {
return Ok(Json(tok_prog));
}
}
// If model failed, return that error
if model_progress.status == goose::dictation::download_manager::DownloadStatus::Failed {
return Ok(Json(model_progress));
}
// Otherwise return model progress (which shows overall download progress)
Ok(Json(model_progress))
}
@@ -159,9 +327,6 @@ pub async fn cancel_local_model_download(
manager
.cancel_download(&format!("{}-model", model_id))
.map_err(convert_error)?;
manager
.cancel_download(&format!("{}-tokenizer", model_id))
.map_err(convert_error)?;
Ok(StatusCode::OK)
}
@@ -176,34 +341,104 @@ pub async fn cancel_local_model_download(
)
)]
pub async fn delete_local_model(Path(model_id): Path<String>) -> Result<StatusCode, ErrorResponse> {
let model =
get_local_model(&model_id).ok_or_else(|| ErrorResponse::not_found("Model not found"))?;
let model_path = model.local_path();
let tokenizer_path = model.tokenizer_path();
if !model_path.exists() && !tokenizer_path.exists() {
return Err(ErrorResponse::not_found("Model not downloaded"));
// Try featured model first
if let Some(model) = get_local_model(&model_id) {
let model_path = model.local_path();
if !model_path.exists() {
return Err(ErrorResponse::not_found("Model not downloaded"));
}
tokio::fs::remove_file(&model_path)
.await
.map_err(|e| ErrorResponse::internal(format!("Failed to delete model: {}", e)))?;
return Ok(StatusCode::OK);
}
// Delete both files
// Try registry model
let model_path = {
let registry = get_registry()
.lock()
.map_err(|_| ErrorResponse::internal("Failed to acquire registry lock"))?;
let entry = registry
.get_model(&model_id)
.ok_or_else(|| ErrorResponse::not_found("Model not found"))?;
entry.local_path.clone()
};
if model_path.exists() {
tokio::fs::remove_file(&model_path)
.await
.map_err(|e| ErrorResponse::internal(format!("Failed to delete model: {}", e)))?;
}
if tokenizer_path.exists() {
tokio::fs::remove_file(&tokenizer_path)
.await
.map_err(|e| ErrorResponse::internal(format!("Failed to delete tokenizer: {}", e)))?;
}
// Remove from registry
let mut registry = get_registry()
.lock()
.map_err(|_| ErrorResponse::internal("Failed to acquire registry lock"))?;
registry
.remove_model(&model_id)
.map_err(|e| ErrorResponse::internal(format!("{}", e)))?;
Ok(StatusCode::OK)
}
#[utoipa::path(
get,
path = "/local-inference/models/{model_id}/settings",
responses(
(status = 200, description = "Model settings", body = ModelSettings),
(status = 404, description = "Model not found")
)
)]
pub async fn get_model_settings(
Path(model_id): Path<String>,
) -> Result<Json<ModelSettings>, ErrorResponse> {
let registry = get_registry()
.lock()
.map_err(|_| ErrorResponse::internal("Failed to acquire registry lock"))?;
if let Some(settings) = registry.get_model_settings(&model_id) {
return Ok(Json(settings.clone()));
}
Err(ErrorResponse::not_found("Model not found"))
}
#[utoipa::path(
put,
path = "/local-inference/models/{model_id}/settings",
request_body = ModelSettings,
responses(
(status = 200, description = "Settings updated", body = ModelSettings),
(status = 404, description = "Model not found"),
(status = 500, description = "Failed to save settings")
)
)]
pub async fn update_model_settings(
Path(model_id): Path<String>,
Json(settings): Json<ModelSettings>,
) -> Result<Json<ModelSettings>, ErrorResponse> {
let mut registry = get_registry()
.lock()
.map_err(|_| ErrorResponse::internal("Failed to acquire registry lock"))?;
registry
.update_model_settings(&model_id, settings.clone())
.map_err(|e| ErrorResponse::not_found(format!("{}", e)))?;
Ok(Json(settings))
}
pub fn routes(state: Arc<AppState>) -> Router {
goose::dictation::download_manager::cleanup_partial_downloads(&Paths::in_data_dir("models"));
Router::new()
.route("/local-inference/models", get(list_local_models))
.route("/local-inference/search", get(search_hf_models))
.route(
"/local-inference/repo/{author}/{repo}/files",
get(get_repo_files),
)
.route("/local-inference/download", post(download_hf_model))
.route(
"/local-inference/models/{model_id}/download",
post(download_local_model),
@@ -220,5 +455,13 @@ pub fn routes(state: Arc<AppState>) -> Router {
"/local-inference/models/{model_id}",
delete(delete_local_model),
)
.route(
"/local-inference/models/{model_id}/settings",
get(get_model_settings),
)
.route(
"/local-inference/models/{model_id}/settings",
axum::routing::put(update_model_settings),
)
.with_state(state)
}
crates/goose-server/src/state.rs
+3
View File
@@ -11,6 +11,7 @@ use tokio::task::JoinHandle;
use crate::tunnel::TunnelManager;
use goose::agents::ExtensionLoadResult;
use goose::providers::local_inference::InferenceRuntime;
type ExtensionLoadingTasks =
Arc<Mutex<HashMap<String, Arc<Mutex<Option<JoinHandle<Vec<ExtensionLoadResult>>>>>>>>;
@@ -23,6 +24,7 @@ pub struct AppState {
recipe_session_tracker: Arc<Mutex<HashSet<String>>>,
pub tunnel_manager: Arc<TunnelManager>,
pub extension_loading_tasks: ExtensionLoadingTasks,
pub inference_runtime: Arc<InferenceRuntime>,
}
impl AppState {
@@ -38,6 +40,7 @@ impl AppState {
recipe_session_tracker: Arc::new(Mutex::new(HashSet::new())),
tunnel_manager,
extension_loading_tasks: Arc::new(Mutex::new(HashMap::new())),
inference_runtime: InferenceRuntime::get_or_init(),
}))
}
crates/goose/Cargo.toml
+5 -6
View File
@@ -9,7 +9,7 @@ description.workspace = true
[features]
default = []
cuda = ["candle-core/cuda", "candle-nn/cuda"]
cuda = ["candle-core/cuda", "candle-nn/cuda", "llama-cpp-2/cuda"]
[lints]
workspace = true
@@ -128,14 +128,17 @@ ignore = { workspace = true }
which = { workspace = true }
pctx_code_mode = "^0.2.3"
unbinder = "0.1.7"
llama-cpp-2 = { version = "0.1.133", features = ["sampler"] }
encoding_rs = "0.8.35"
[target.'cfg(target_os = "windows")'.dependencies]
winapi = { version = "0.3", features = ["wincred"] }
# Platform-specific GPU acceleration for Whisper
# Platform-specific GPU acceleration for Whisper and local inference
[target.'cfg(target_os = "macos")'.dependencies]
candle-core = { version = "0.9", default-features = false, features = ["metal"] }
candle-nn = { version = "0.9", default-features = false, features = ["metal"] }
llama-cpp-2 = { version = "0.1.133", features = ["sampler", "metal"] }
[dev-dependencies]
serial_test = { workspace = true }
@@ -158,10 +161,6 @@ path = "examples/agent.rs"
name = "databricks_oauth"
path = "examples/databricks_oauth.rs"
[[example]]
name = "candle_quantized"
path = "examples/candle_quantized.rs"
[[example]]
name = "test_local_provider"
path = "examples/test_local_provider.rs"
crates/goose/examples/candle_quantized.rs
-702
View File
@@ -1,702 +0,0 @@
#[cfg(feature = "mkl")]
extern crate intel_mkl_src;
#[cfg(feature = "accelerate")]
extern crate accelerate_src;
use clap::{Parser, ValueEnum};
use std::io::Write;
use tokenizers::Tokenizer;
use candle::quantized::{ggml_file, gguf_file};
use candle::Tensor;
use candle_transformers::generation::{LogitsProcessor, Sampling};
use candle_examples::token_output_stream::TokenOutputStream;
use candle_transformers::models::quantized_llama as model;
use model::ModelWeights;
const DEFAULT_PROMPT: &str = "My favorite theorem is ";
#[derive(Debug)]
enum Prompt {
Interactive,
Chat,
One(String),
}
#[derive(Clone, Debug, Copy, PartialEq, Eq, ValueEnum)]
enum Which {
#[value(name = "7b")]
L7b,
#[value(name = "13b")]
L13b,
#[value(name = "70b")]
L70b,
#[value(name = "7b-chat")]
L7bChat,
#[value(name = "13b-chat")]
L13bChat,
#[value(name = "70b-chat")]
L70bChat,
#[value(name = "7b-code")]
L7bCode,
#[value(name = "13b-code")]
L13bCode,
#[value(name = "32b-code")]
L34bCode,
#[value(name = "7b-leo")]
Leo7b,
#[value(name = "13b-leo")]
Leo13b,
#[value(name = "7b-mistral")]
Mistral7b,
#[value(name = "7b-mistral-instruct")]
Mistral7bInstruct,
#[value(name = "7b-mistral-instruct-v0.2")]
Mistral7bInstructV02,
#[value(name = "7b-zephyr-a")]
Zephyr7bAlpha,
#[value(name = "7b-zephyr-b")]
Zephyr7bBeta,
#[value(name = "7b-open-chat-3.5")]
OpenChat35,
#[value(name = "7b-starling-a")]
Starling7bAlpha,
#[value(name = "mixtral")]
Mixtral,
#[value(name = "mixtral-instruct")]
MixtralInstruct,
#[value(name = "llama3-8b")]
L8b,
#[value(name = "phi3")]
Phi3,
#[value(name = "SmoLM2-360M-Instruct")]
SmolLM2_360MInstruct,
#[value(name = "SmoLM2-1.7B-Instruct")]
SmolLM2_1BInstruct,
#[value(name = "deepseekr1-llama8b")]
DeepseekR1Llama8b,
}
impl Which {
fn is_mistral(&self) -> bool {
match self {
Self::L7b
| Self::L13b
| Self::L70b
| Self::L7bChat
| Self::L13bChat
| Self::L70bChat
| Self::L7bCode
| Self::L13bCode
| Self::L34bCode
| Self::Leo7b
| Self::Leo13b
| Self::L8b
| Self::Phi3
| Self::SmolLM2_1BInstruct
| Self::SmolLM2_360MInstruct
| Self::DeepseekR1Llama8b => false,
// Zephyr and OpenChat are fine tuned versions of mistral and should be treated in the
// same way. Starling is a fine tuned version of OpenChat.
Self::OpenChat35
| Self::Starling7bAlpha
| Self::Zephyr7bAlpha
| Self::Zephyr7bBeta
| Self::Mixtral
| Self::MixtralInstruct
| Self::Mistral7b
| Self::Mistral7bInstruct
| Self::Mistral7bInstructV02 => true,
}
}
fn is_zephyr(&self) -> bool {
match self {
Self::L7b
| Self::L13b
| Self::L70b
| Self::L7bChat
| Self::L13bChat
| Self::L70bChat
| Self::L7bCode
| Self::L13bCode
| Self::L34bCode
| Self::Leo7b
| Self::Leo13b
| Self::Mixtral
| Self::MixtralInstruct
| Self::Mistral7b
| Self::Mistral7bInstruct
| Self::Mistral7bInstructV02
| Self::OpenChat35
| Self::Starling7bAlpha
| Self::L8b
| Self::SmolLM2_1BInstruct
| Self::SmolLM2_360MInstruct
| Self::Phi3
| Self::DeepseekR1Llama8b => false,
Self::Zephyr7bAlpha | Self::Zephyr7bBeta => true,
}
}
fn is_open_chat(&self) -> bool {
match self {
Self::L7b
| Self::L13b
| Self::L70b
| Self::L7bChat
| Self::L13bChat
| Self::L70bChat
| Self::L7bCode
| Self::L13bCode
| Self::L34bCode
| Self::Leo7b
| Self::Leo13b
| Self::Mixtral
| Self::MixtralInstruct
| Self::Mistral7b
| Self::Mistral7bInstruct
| Self::Mistral7bInstructV02
| Self::Zephyr7bAlpha
| Self::Zephyr7bBeta
| Self::L8b
| Self::SmolLM2_1BInstruct
| Self::SmolLM2_360MInstruct
| Self::Phi3
| Self::DeepseekR1Llama8b => false,
Self::OpenChat35 | Self::Starling7bAlpha => true,
}
}
fn is_deepseek(&self) -> bool {
match self {
Self::L7b
| Self::L13b
| Self::L70b
| Self::L7bChat
| Self::L13bChat
| Self::L70bChat
| Self::L7bCode
| Self::L13bCode
| Self::L34bCode
| Self::Leo7b
| Self::Leo13b
| Self::Mixtral
| Self::MixtralInstruct
| Self::Mistral7b
| Self::Mistral7bInstruct
| Self::Mistral7bInstructV02
| Self::Zephyr7bAlpha
| Self::Zephyr7bBeta
| Self::L8b
| Self::SmolLM2_1BInstruct
| Self::SmolLM2_360MInstruct
| Self::Phi3
| Self::OpenChat35
| Self::Starling7bAlpha => false,
Self::DeepseekR1Llama8b => true,
}
}
fn tokenizer_repo(&self) -> &'static str {
match self {
Self::L7b
| Self::L13b
| Self::L70b
| Self::L7bChat
| Self::L13bChat
| Self::L70bChat
| Self::L7bCode
| Self::L13bCode
| Self::L34bCode => "hf-internal-testing/llama-tokenizer",
Self::Leo7b => "LeoLM/leo-hessianai-7b",
Self::Leo13b => "LeoLM/leo-hessianai-13b",
Self::Mixtral => "mistralai/Mixtral-8x7B-v0.1",
Self::MixtralInstruct => "mistralai/Mixtral-8x7B-Instruct-v0.1",
Self::Mistral7b
| Self::Mistral7bInstruct
| Self::Mistral7bInstructV02
| Self::Zephyr7bAlpha
| Self::Zephyr7bBeta => "mistralai/Mistral-7B-v0.1",
Self::OpenChat35 => "openchat/openchat_3.5",
Self::Starling7bAlpha => "berkeley-nest/Starling-LM-7B-alpha",
Self::L8b => "meta-llama/Meta-Llama-3-8B",
Self::Phi3 => "microsoft/Phi-3-mini-4k-instruct",
Self::SmolLM2_360MInstruct => "HuggingFaceTB/SmolLM2-360M-Instruct",
Self::SmolLM2_1BInstruct => "HuggingFaceTB/SmolLM2-1.7B-Instruct",
Self::DeepseekR1Llama8b => "deepseek-ai/DeepSeek-R1-Distill-Llama-8B",
}
}
}
#[derive(Parser, Debug)]
#[command(author, version, about, long_about = None)]
struct Args {
/// GGML/GGUF file to load, typically a .bin/.gguf file generated by the quantize command from llama.cpp
#[arg(long)]
model: Option<String>,
/// The initial prompt, use 'interactive' for entering multiple prompts in an interactive way
/// and 'chat' for an interactive model where history of previous prompts and generated tokens
/// is preserved.
#[arg(long)]
prompt: Option<String>,
/// The length of the sample to generate (in tokens).
#[arg(short = 'n', long, default_value_t = 1000)]
sample_len: usize,
/// The tokenizer config in json format.
#[arg(long)]
tokenizer: Option<String>,
/// The temperature used to generate samples, use 0 for greedy sampling.
#[arg(long, default_value_t = 0.8)]
temperature: f64,
/// Nucleus sampling probability cutoff.
#[arg(long)]
top_p: Option<f64>,
/// Only sample among the top K samples.
#[arg(long)]
top_k: Option<usize>,
/// The seed to use when generating random samples.
#[arg(long, default_value_t = 299792458)]
seed: u64,
/// Enable tracing (generates a trace-timestamp.json file).
#[arg(long)]
tracing: bool,
/// Display the token for the specified prompt.
#[arg(long)]
verbose_prompt: bool,
/// Process prompt elements separately.
#[arg(long)]
split_prompt: bool,
/// Run on CPU rather than GPU even if a GPU is available.
#[arg(long)]
cpu: bool,
/// Penalty to be applied for repeating tokens, 1. means no penalty.
#[arg(long, default_value_t = 1.1)]
repeat_penalty: f32,
/// The context size to consider for the repeat penalty.
#[arg(long, default_value_t = 64)]
repeat_last_n: usize,
/// The model size to use.
#[arg(long, default_value = "7b")]
which: Which,
/// Group-Query Attention, use 8 for the 70B version of LLaMAv2.
#[arg(long)]
gqa: Option<usize>,
/// Use the slower dmmv cuda kernel.
#[arg(long)]
force_dmmv: bool,
}
impl Args {
fn tokenizer(&self) -> anyhow::Result<Tokenizer> {
let tokenizer_path = match &self.tokenizer {
Some(config) => std::path::PathBuf::from(config),
None => {
let api = hf_hub::api::sync::Api::new()?;
let repo = self.which.tokenizer_repo();
let api = api.model(repo.to_string());
api.get("tokenizer.json")?
}
};
Tokenizer::from_file(tokenizer_path).map_err(anyhow::Error::msg)
}
fn model(&self) -> anyhow::Result<std::path::PathBuf> {
let model_path = match &self.model {
Some(config) => std::path::PathBuf::from(config),
None => {
let (repo, filename) = match self.which {
Which::L7b => ("TheBloke/Llama-2-7B-GGML", "llama-2-7b.ggmlv3.q4_0.bin"),
Which::L13b => ("TheBloke/Llama-2-13B-GGML", "llama-2-13b.ggmlv3.q4_0.bin"),
Which::L70b => ("TheBloke/Llama-2-70B-GGML", "llama-2-70b.ggmlv3.q4_0.bin"),
Which::L7bChat => (
"TheBloke/Llama-2-7B-Chat-GGML",
"llama-2-7b-chat.ggmlv3.q4_0.bin",
),
Which::L13bChat => (
"TheBloke/Llama-2-13B-Chat-GGML",
"llama-2-13b-chat.ggmlv3.q4_0.bin",
),
Which::L70bChat => (
"TheBloke/Llama-2-70B-Chat-GGML",
"llama-2-70b-chat.ggmlv3.q4_0.bin",
),
Which::L7bCode => ("TheBloke/CodeLlama-7B-GGUF", "codellama-7b.Q8_0.gguf"),
Which::L13bCode => ("TheBloke/CodeLlama-13B-GGUF", "codellama-13b.Q8_0.gguf"),
Which::L34bCode => ("TheBloke/CodeLlama-34B-GGUF", "codellama-34b.Q8_0.gguf"),
Which::Leo7b => (
"TheBloke/leo-hessianai-7B-GGUF",
"leo-hessianai-7b.Q4_K_M.gguf",
),
Which::Leo13b => (
"TheBloke/leo-hessianai-13B-GGUF",
"leo-hessianai-13b.Q4_K_M.gguf",
),
Which::Mixtral => (
"TheBloke/Mixtral-8x7B-v0.1-GGUF",
"mixtral-8x7b-v0.1.Q4_K_M.gguf",
),
Which::MixtralInstruct => (
"TheBloke/Mixtral-8x7B-Instruct-v0.1-GGUF",
"mixtral-8x7b-instruct-v0.1.Q4_K_M.gguf",
),
Which::Mistral7b => (
"TheBloke/Mistral-7B-v0.1-GGUF",
"mistral-7b-v0.1.Q4_K_S.gguf",
),
Which::Mistral7bInstruct => (
"TheBloke/Mistral-7B-Instruct-v0.1-GGUF",
"mistral-7b-instruct-v0.1.Q4_K_S.gguf",
),
Which::Mistral7bInstructV02 => (
"TheBloke/Mistral-7B-Instruct-v0.2-GGUF",
"mistral-7b-instruct-v0.2.Q4_K_S.gguf",
),
Which::Zephyr7bAlpha => (
"TheBloke/zephyr-7B-alpha-GGUF",
"zephyr-7b-alpha.Q4_K_M.gguf",
),
Which::Zephyr7bBeta => {
("TheBloke/zephyr-7B-beta-GGUF", "zephyr-7b-beta.Q4_K_M.gguf")
}
Which::OpenChat35 => ("TheBloke/openchat_3.5-GGUF", "openchat_3.5.Q4_K_M.gguf"),
Which::Starling7bAlpha => (
"TheBloke/Starling-LM-7B-alpha-GGUF",
"starling-lm-7b-alpha.Q4_K_M.gguf",
),
// TODO: swap to TheBloke model when available
Which::L8b => (
"QuantFactory/Meta-Llama-3-8B-GGUF",
"Meta-Llama-3-8B.Q4_K_S.gguf",
),
Which::Phi3 => (
"microsoft/Phi-3-mini-4k-instruct-gguf",
"Phi-3-mini-4k-instruct-q4.gguf",
),
Which::SmolLM2_360MInstruct => (
"HuggingFaceTB/SmolLM2-360M-Instruct-GGUF",
"smollm2-360m-instruct-q8_0.gguf",
),
Which::SmolLM2_1BInstruct => (
"HuggingFaceTB/SmolLM2-1.7B-Instruct-GGUF",
"smollm2-1.7b-instruct-q4_k_m.gguf",
),
Which::DeepseekR1Llama8b => (
"unsloth/DeepSeek-R1-Distill-Llama-8B-GGUF",
"DeepSeek-R1-Distill-Llama-8B-Q4_K_M.gguf",
),
};
let revision = if self.which == Which::Phi3 {
"5eef2ce24766d31909c0b269fe90c817a8f263fb"
} else {
"main"
};
let api = hf_hub::api::sync::Api::new()?;
api.repo(hf_hub::Repo::with_revision(
repo.to_string(),
hf_hub::RepoType::Model,
revision.to_string(),
))
.get(filename)?
}
};
Ok(model_path)
}
}
fn format_size(size_in_bytes: usize) -> String {
if size_in_bytes < 1_000 {
format!("{size_in_bytes}B")
} else if size_in_bytes < 1_000_000 {
format!("{:.2}KB", size_in_bytes as f64 / 1e3)
} else if size_in_bytes < 1_000_000_000 {
format!("{:.2}MB", size_in_bytes as f64 / 1e6)
} else {
format!("{:.2}GB", size_in_bytes as f64 / 1e9)
}
}
fn main() -> anyhow::Result<()> {
use tracing_chrome::ChromeLayerBuilder;
use tracing_subscriber::prelude::*;
let args = Args::parse();
#[cfg(feature = "cuda")]
candle::quantized::cuda::set_force_dmmv(args.force_dmmv);
candle::cuda::set_gemm_reduced_precision_f16(true);
candle::cuda::set_gemm_reduced_precision_bf16(true);
let _guard = if args.tracing {
let (chrome_layer, guard) = ChromeLayerBuilder::new().build();
tracing_subscriber::registry().with(chrome_layer).init();
Some(guard)
} else {
None
};
println!(
"avx: {}, neon: {}, simd128: {}, f16c: {}",
candle::utils::with_avx(),
candle::utils::with_neon(),
candle::utils::with_simd128(),
candle::utils::with_f16c()
);
println!(
"temp: {:.2} repeat-penalty: {:.2} repeat-last-n: {}",
args.temperature, args.repeat_penalty, args.repeat_last_n
);
let model_path = args.model()?;
let mut file = std::fs::File::open(&model_path)?;
let start = std::time::Instant::now();
let device = candle_examples::device(args.cpu)?;
let mut model = match model_path.extension().and_then(|v| v.to_str()) {
Some("gguf") => {
let model = gguf_file::Content::read(&mut file).map_err(|e| e.with_path(model_path))?;
let mut total_size_in_bytes = 0;
for (_, tensor) in model.tensor_infos.iter() {
let elem_count = tensor.shape.elem_count();
total_size_in_bytes +=
elem_count * tensor.ggml_dtype.type_size() / tensor.ggml_dtype.block_size();
}
println!(
"loaded {:?} tensors ({}) in {:.2}s",
model.tensor_infos.len(),
&format_size(total_size_in_bytes),
start.elapsed().as_secs_f32(),
);
ModelWeights::from_gguf(model, &mut file, &device)?
}
Some("ggml" | "bin") | Some(_) | None => {
let model = ggml_file::Content::read(&mut file, &device)
.map_err(|e| e.with_path(model_path))?;
let mut total_size_in_bytes = 0;
for (_, tensor) in model.tensors.iter() {
let elem_count = tensor.shape().elem_count();
total_size_in_bytes +=
elem_count * tensor.dtype().type_size() / tensor.dtype().block_size();
}
println!(
"loaded {:?} tensors ({}) in {:.2}s",
model.tensors.len(),
&format_size(total_size_in_bytes),
start.elapsed().as_secs_f32(),
);
println!("params: {:?}", model.hparams);
let default_gqa = match args.which {
Which::L7b
| Which::L13b
| Which::L7bChat
| Which::L13bChat
| Which::L7bCode
| Which::L13bCode
| Which::L34bCode
| Which::Leo7b
| Which::Leo13b
| Which::L8b
| Which::SmolLM2_1BInstruct
| Which::SmolLM2_360MInstruct
| Which::DeepseekR1Llama8b
| Which::Phi3 => 1,
Which::Mixtral
| Which::MixtralInstruct
| Which::Mistral7b
| Which::Mistral7bInstruct
| Which::Mistral7bInstructV02
| Which::Zephyr7bAlpha
| Which::Zephyr7bBeta
| Which::L70b
| Which::L70bChat
| Which::OpenChat35
| Which::Starling7bAlpha => 8,
};
ModelWeights::from_ggml(model, args.gqa.unwrap_or(default_gqa))?
}
};
println!("model built");
let tokenizer = args.tokenizer()?;
let mut tos = TokenOutputStream::new(tokenizer);
let prompt = match args.prompt.as_deref() {
Some("chat") => Prompt::Chat,
Some("interactive") => Prompt::Interactive,
Some(s) => Prompt::One(s.to_string()),
None => Prompt::One(DEFAULT_PROMPT.to_string()),
};
let mut pre_prompt_tokens = vec![];
for prompt_index in 0.. {
let prompt_str = match &prompt {
Prompt::One(prompt) => prompt.clone(),
Prompt::Interactive | Prompt::Chat => {
let is_interactive = matches!(prompt, Prompt::Interactive);
print!("> ");
std::io::stdout().flush()?;
let mut prompt = String::new();
std::io::stdin().read_line(&mut prompt)?;
if prompt.ends_with('\n') {
prompt.pop();
if prompt.ends_with('\r') {
prompt.pop();
}
}
if args.which.is_open_chat() {
format!("GPT4 Correct User: {prompt}<|end_of_turn|>GPT4 Correct Assistant:")
} else if args.which.is_zephyr() {
if prompt_index == 0 || is_interactive {
format!("<|system|>\n</s>\n<|user|>\n{prompt}</s>\n<|assistant|>",)
} else {
format!("<|user|>\n{prompt}</s>\n<|assistant|>")
}
} else if args.which.is_mistral() {
format!("[INST] {prompt} [/INST]")
} else if args.which.is_deepseek() {
format!("<｜User｜>{prompt}<｜Assistant｜>")
} else {
prompt
}
}
};
print!("{}", &prompt_str);
let tokens = tos
.tokenizer()
.encode(prompt_str, true)
.map_err(anyhow::Error::msg)?;
if args.verbose_prompt {
for (token, id) in tokens.get_tokens().iter().zip(tokens.get_ids().iter()) {
let token = token.replace('▁', " ").replace("<0x0A>", "\n");
println!("{id:7} -> '{token}'");
}
}
let prompt_tokens = [&pre_prompt_tokens, tokens.get_ids()].concat();
let to_sample = args.sample_len.saturating_sub(1);
let prompt_tokens = if prompt_tokens.len() + to_sample > model::MAX_SEQ_LEN - 10 {
let to_remove = prompt_tokens.len() + to_sample + 10 - model::MAX_SEQ_LEN;
prompt_tokens[prompt_tokens.len().saturating_sub(to_remove)..].to_vec()
} else {
prompt_tokens
};
let mut all_tokens = vec![];
let mut logits_processor = {
let temperature = args.temperature;
let sampling = if temperature <= 0. {
Sampling::ArgMax
} else {
match (args.top_k, args.top_p) {
(None, None) => Sampling::All { temperature },
(Some(k), None) => Sampling::TopK { k, temperature },
(None, Some(p)) => Sampling::TopP { p, temperature },
(Some(k), Some(p)) => Sampling::TopKThenTopP { k, p, temperature },
}
};
LogitsProcessor::from_sampling(args.seed, sampling)
};
let start_prompt_processing = std::time::Instant::now();
let mut next_token = if !args.split_prompt {
let input = Tensor::new(prompt_tokens.as_slice(), &device)?.unsqueeze(0)?;
let logits = model.forward(&input, 0)?;
let logits = logits.squeeze(0)?;
logits_processor.sample(&logits)?
} else {
let mut next_token = 0;
for (pos, token) in prompt_tokens.iter().enumerate() {
let input = Tensor::new(&[*token], &device)?.unsqueeze(0)?;
let logits = model.forward(&input, pos)?;
let logits = logits.squeeze(0)?;
next_token = logits_processor.sample(&logits)?
}
next_token
};
let prompt_dt = start_prompt_processing.elapsed();
all_tokens.push(next_token);
if let Some(t) = tos.next_token(next_token)? {
print!("{t}");
std::io::stdout().flush()?;
}
let eos_token = match args.which {
Which::SmolLM2_360MInstruct | Which::SmolLM2_1BInstruct => "<|endoftext|>",
Which::L8b => "<|end_of_text|>",
Which::DeepseekR1Llama8b => "<｜end▁of▁sentence｜>",
_ => match args.which.is_open_chat() {
true => "<|end_of_turn|>",
false => "</s>",
},
};
let eos_token = *tos.tokenizer().get_vocab(true).get(eos_token).unwrap();
let start_post_prompt = std::time::Instant::now();
let mut sampled = 0;
for index in 0..to_sample {
let input = Tensor::new(&[next_token], &device)?.unsqueeze(0)?;
let logits = model.forward(&input, prompt_tokens.len() + index)?;
let logits = logits.squeeze(0)?;
let logits = if args.repeat_penalty == 1. {
logits
} else {
let start_at = all_tokens.len().saturating_sub(args.repeat_last_n);
candle_transformers::utils::apply_repeat_penalty(
&logits,
args.repeat_penalty,
&all_tokens[start_at..],
)?
};
next_token = logits_processor.sample(&logits)?;
all_tokens.push(next_token);
if let Some(t) = tos.next_token(next_token)? {
print!("{t}");
std::io::stdout().flush()?;
}
sampled += 1;
if next_token == eos_token {
break;
};
}
if let Some(rest) = tos.decode_rest().map_err(candle::Error::msg)? {
print!("{rest}");
}
std::io::stdout().flush()?;
let dt = start_post_prompt.elapsed();
println!(
"\n\n{:4} prompt tokens processed: {:.2} token/s",
prompt_tokens.len(),
prompt_tokens.len() as f64 / prompt_dt.as_secs_f64(),
);
println!(
"{sampled:4} tokens generated: {:.2} token/s",
sampled as f64 / dt.as_secs_f64(),
);
match prompt {
Prompt::One(_) => break,
Prompt::Interactive => {}
Prompt::Chat => {
pre_prompt_tokens = [prompt_tokens.as_slice(), all_tokens.as_slice()].concat()
}
}
}
Ok(())
}
crates/goose/examples/test_candle_minimal.rs
-54
View File
@@ -1,54 +0,0 @@
// Minimal test to see if candle works the same in goose repo as in candle repo
use anyhow::Result;
use candle_core::{Device, Tensor};
use candle_transformers::models::quantized_llama::ModelWeights;
use candle_core::quantized::gguf_file;
use tokenizers::Tokenizer;
fn main() -> Result<()> {
let home = std::env::var("HOME").map_err(anyhow::Error::msg)?;
let model_path = std::path::PathBuf::from(format!("{}/.local/share/goose/models/llama-3.2-3b.gguf", home));
let tokenizer_path = std::path::PathBuf::from(format!("{}/.local/share/goose/models/llama-3.2-3b_tokenizer.json", home));
let prompt = std::fs::read_to_string("/tmp/goose_prompt_stream.txt")?;
// Device
let device = if let Ok(device) = Device::new_metal(0) {
device
} else {
Device::Cpu
};
// Load model
let mut file = std::fs::File::open(&model_path)?;
let content = gguf_file::Content::read(&mut file)?;
let mut model = ModelWeights::from_gguf(content, &mut file, &device)?;
// Load tokenizer
let tokenizer = Tokenizer::from_file(tokenizer_path).map_err(anyhow::Error::msg)?;
// Tokenize - use false since prompt already has <|begin_of_text|>
let tokens = tokenizer.encode(prompt.as_str(), false).map_err(anyhow::Error::msg)?;
let prompt_tokens = tokens.get_ids().to_vec();
println!("Prompt tokens: {}", prompt_tokens.len());
// Split-prompt prefill
let mut next_token = 0u32;
for (pos, &token) in prompt_tokens.iter().enumerate() {
let input = Tensor::new(&[token], &device)?.unsqueeze(0)?;
let logits = model.forward(&input, pos)?;
let logits = logits.squeeze(0)?;
next_token = logits.argmax(0)?.to_scalar::<u32>()?;
if pos >= prompt_tokens.len().saturating_sub(5) {
println!("pos={}, input_token={}, next_token={}", pos, token, next_token);
}
}
// Decode first token
let decoded = tokenizer.decode(&[next_token], false).map_err(anyhow::Error::msg)?;
println!("\nFirst generated token: ID={}, text='{}'", next_token, decoded);
Ok(())
}
crates/goose/src/agents/extension_manager.rs
+3 -3
View File
@@ -1495,9 +1495,9 @@ impl ExtensionManager {
) -> Option<String> {
if let Ok(provider_guard) = self.provider.try_lock() {
if let Some(provider) = provider_guard.as_ref() {
if provider.get_model_config().context_limit() < 9 * 1024 * 1024 {
return None;
}
if provider.get_model_config().context_limit() < 9 * 1024 * 1024 {
return None;
}
}
}
crates/goose/src/dictation/download_manager.rs
+33 -4
View File
@@ -1,11 +1,32 @@
use anyhow::Result;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::path::PathBuf;
use std::path::{Path, PathBuf};
use std::sync::{Arc, Mutex};
use tokio::io::AsyncWriteExt;
use utoipa::ToSchema;
fn partial_path_for(destination: &Path) -> PathBuf {
destination.with_extension(
destination
.extension()
.map(|e| format!("{}.part", e.to_string_lossy()))
.unwrap_or_else(|| "part".to_string()),
)
}
/// Remove any leftover `.part` files in the given directory.
pub fn cleanup_partial_downloads(dir: &Path) {
if let Ok(entries) = std::fs::read_dir(dir) {
for entry in entries.flatten() {
let path = entry.path();
if path.extension().is_some_and(|e| e == "part") {
let _ = std::fs::remove_file(&path);
}
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
pub struct DownloadProgress {
/// Model ID being downloaded
@@ -116,6 +137,8 @@ impl DownloadManager {
let downloads = self.downloads.clone();
let model_id_clone = model_id.clone();
let destination_for_cleanup = destination.clone();
// Download in background task
tokio::spawn(async move {
match Self::download_file(&url, &destination, &downloads, &model_id_clone).await {
@@ -133,6 +156,10 @@ impl DownloadManager {
}
}
Err(e) => {
// Clean up partial file on failure
let partial = partial_path_for(&destination_for_cleanup);
let _ = tokio::fs::remove_file(&partial).await;
if let Ok(mut downloads) = downloads.lock() {
if let Some(progress) = downloads.get_mut(&model_id_clone) {
progress.status = DownloadStatus::Failed;
@@ -169,7 +196,8 @@ impl DownloadManager {
}
}
let mut file = tokio::fs::File::create(destination).await?;
let partial_path = partial_path_for(destination);
let mut file = tokio::fs::File::create(&partial_path).await?;
let mut bytes_downloaded = 0u64;
let start_time = std::time::Instant::now();
@@ -188,8 +216,7 @@ impl DownloadManager {
};
if should_cancel {
// Clean up partial download
let _ = tokio::fs::remove_file(destination).await;
let _ = tokio::fs::remove_file(&partial_path).await;
return Ok(());
}
@@ -229,6 +256,8 @@ impl DownloadManager {
}
file.flush().await?;
drop(file);
tokio::fs::rename(&partial_path, destination).await?;
Ok(())
}
crates/goose/src/prompts/tiny_model_system.md
+14 -13
View File
@@ -1,21 +1,22 @@
You are goose (lowercase), an AI assistant created by Block
You are goose, an autonomous AI agent created by Block. You act on the user's
behalf — you do not explain how to do things, you DO them directly.
Help the user using your knowledge, your reasoning and by executing commands
in the {{shell}} shell.
The OS is {{os}}, the shell is {{shell}}, and the working directory is {{working_directory}}
The OS is {{os}} and the current directory is {{working_directory}}
When the user asks you to do something, take action immediately. Do not describe
what you would do or give instructions — execute the commands yourself.
If you need to execute a shell command, you can do so by starting a new line with $, for example
to look at the files in the current folder, just end your message on
To run a shell command, start a new line with $:
$ ls
Other useful commands are: `rg` to search for text, `cat` to read or write files
or `head` to just see part of it. use `echo "content" > file` for small files,
`cat` for longer.
Keep your responses brief. State what you are doing, then do it. For example:
# Important
User: how many files are in /tmp?
You: Let me check.
$ ls -1 /tmp | wc -l
Only execute shell commands when you need to read a file you know exists or when
you need to create a file or execute a command. Do not use shell commands if you
know the answer. Do not assume files or folders exists until you check.
After a command runs, you will see its output. Use the output to answer the user
or take the next step. Do not repeat commands you have already run.
Do not use shell commands if you already know the answer.
crates/goose/src/providers/local_inference.rs
+1603 -658
View File
File diff suppressed because it is too large Load Diff
crates/goose/src/providers/local_inference/hf_models.rs
+565
View File
@@ -0,0 +1,565 @@
use anyhow::{bail, Result};
use serde::{Deserialize, Serialize};
use utoipa::ToSchema;
const HF_API_BASE: &str = "https://huggingface.co/api/models";
const HF_DOWNLOAD_BASE: &str = "https://huggingface.co";
#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
pub struct HfModelInfo {
pub repo_id: String,
pub author: String,
pub model_name: String,
pub downloads: u64,
pub gguf_files: Vec<HfGgufFile>,
}
/// A single downloadable GGUF file (used internally and for downloads).
#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
pub struct HfGgufFile {
pub filename: String,
pub size_bytes: u64,
pub quantization: String,
pub download_url: String,
}
/// A quantization variant — groups sharded files into one logical entry.
#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
pub struct HfQuantVariant {
pub quantization: String,
pub size_bytes: u64,
pub filename: String,
pub download_url: String,
pub description: &'static str,
pub quality_rank: u8,
}
#[derive(Debug, Deserialize)]
struct HfApiModel {
id: Option<String>,
author: Option<String>,
downloads: Option<u64>,
siblings: Option<Vec<HfApiSibling>>,
}
#[derive(Debug, Deserialize)]
struct HfApiSibling {
rfilename: String,
#[serde(default)]
size: Option<u64>,
}
struct QuantInfo {
description: &'static str,
quality_rank: u8,
}
fn quant_info(quant: &str) -> QuantInfo {
match quant {
"IQ1_S" => QuantInfo {
description: "Extremely small, very low quality",
quality_rank: 1,
},
"IQ1_M" => QuantInfo {
description: "Extremely small, very low quality",
quality_rank: 2,
},
"IQ2_XXS" => QuantInfo {
description: "Very small, low quality",
quality_rank: 3,
},
"IQ2_XS" => QuantInfo {
description: "Very small, low quality",
quality_rank: 4,
},
"IQ2_S" => QuantInfo {
description: "Very small, low quality",
quality_rank: 5,
},
"IQ2_M" => QuantInfo {
description: "Very small, low quality",
quality_rank: 6,
},
"Q2_K" | "Q2_K_S" => QuantInfo {
description: "Small, low quality",
quality_rank: 7,
},
"IQ3_XXS" => QuantInfo {
description: "Very small, moderate quality loss",
quality_rank: 8,
},
"IQ3_XS" | "IQ3_S" => QuantInfo {
description: "Small, moderate quality loss",
quality_rank: 9,
},
"Q3_K_S" => QuantInfo {
description: "Small, moderate quality loss",
quality_rank: 10,
},
"IQ3_M" => QuantInfo {
description: "Small, moderate quality loss",
quality_rank: 11,
},
"Q3_K_M" => QuantInfo {
description: "Small, balanced quality/size",
quality_rank: 12,
},
"Q3_K_L" => QuantInfo {
description: "Medium-small, decent quality",
quality_rank: 13,
},
"IQ4_XS" => QuantInfo {
description: "Medium, good quality",
quality_rank: 14,
},
"IQ4_NL" => QuantInfo {
description: "Medium, good quality",
quality_rank: 15,
},
"Q4_0" => QuantInfo {
description: "Medium, good quality",
quality_rank: 16,
},
"Q4_1" => QuantInfo {
description: "Medium, good quality",
quality_rank: 17,
},
"Q4_K_S" => QuantInfo {
description: "Medium, good quality/size balance",
quality_rank: 18,
},
"Q4_K_M" => QuantInfo {
description: "Medium, recommended balance of quality and size",
quality_rank: 19,
},
"Q5_0" => QuantInfo {
description: "Medium-large, high quality",
quality_rank: 20,
},
"Q5_1" => QuantInfo {
description: "Medium-large, high quality",
quality_rank: 21,
},
"Q5_K_S" => QuantInfo {
description: "Medium-large, high quality",
quality_rank: 22,
},
"Q5_K_M" => QuantInfo {
description: "Medium-large, very high quality",
quality_rank: 23,
},
"Q6_K" => QuantInfo {
description: "Large, near-lossless quality",
quality_rank: 24,
},
"Q8_0" => QuantInfo {
description: "Large, near-lossless quality",
quality_rank: 25,
},
"F16" => QuantInfo {
description: "Full size, original quality (16-bit)",
quality_rank: 26,
},
"BF16" => QuantInfo {
description: "Full size, original quality (bfloat16)",
quality_rank: 27,
},
"F32" => QuantInfo {
description: "Full size, original quality (32-bit)",
quality_rank: 28,
},
"MXFP4_MOE" => QuantInfo {
description: "Medium, mixed-precision 4-bit for MoE models",
quality_rank: 18,
},
"TQ1_0" => QuantInfo {
description: "Tiny, ternary quantization",
quality_rank: 1,
},
"Q2_K_XL" | "Q3_K_XL" | "Q4_K_XL" => QuantInfo {
description: "Extended-layer variant",
quality_rank: 15,
},
"Q2_K_L" => QuantInfo {
description: "Small, low quality (large variant)",
quality_rank: 8,
},
"Q4_K_L" => QuantInfo {
description: "Medium, good quality (large variant)",
quality_rank: 20,
},
_ => QuantInfo {
description: "",
quality_rank: 15,
},
}
}
pub fn parse_quantization_from_filename(filename: &str) -> String {
parse_quantization(filename)
}
fn parse_quantization(filename: &str) -> String {
// Strip directory prefix (e.g. "Q5_K_M/Model-Q5_K_M-00001-of-00002.gguf")
let basename = filename.rsplit('/').next().unwrap_or(filename);
let stem = basename.trim_end_matches(".gguf");
// Strip shard suffix like "-00001-of-00004"
let stem = if let Some(pos) = stem.rfind("-of-") {
stem.get(..pos)
.and_then(|s| s.rsplit_once('-').map(|(prefix, _)| prefix))
.unwrap_or(stem)
} else {
stem
};
// The quantization tag is typically the last hyphen-separated component
// that looks like a quant identifier (starts with Q, IQ, F, BF, TQ, MXFP, etc.)
// e.g. "Qwen3-Coder-Next-Q4_K_M" -> "Q4_K_M"
// "Model-UD-IQ1_M" -> "IQ1_M"
if let Some((_, candidate)) = stem.rsplit_once('-') {
if looks_like_quant(candidate) {
return candidate.to_string();
}
}
// Fallback: try dot separator (e.g. "model.Q4_K_M")
if let Some((_, candidate)) = stem.rsplit_once('.') {
if looks_like_quant(candidate) {
return candidate.to_string();
}
}
"unknown".to_string()
}
fn looks_like_quant(s: &str) -> bool {
let upper = s.to_uppercase();
upper.starts_with("Q")
|| upper.starts_with("IQ")
|| upper.starts_with("TQ")
|| upper.starts_with("MXFP")
|| upper == "F16"
|| upper == "F32"
|| upper == "BF16"
}
fn is_shard_file(filename: &str) -> bool {
// Matches patterns like "-00001-of-00003.gguf"
let basename = filename.rsplit('/').next().unwrap_or(filename);
let stem = basename.trim_end_matches(".gguf");
if let Some(pos) = stem.rfind("-of-") {
stem.get(..pos)
.and_then(|before| before.rsplit('-').next())
.map(|s| !s.is_empty() && s.chars().all(|c| c.is_ascii_digit()))
.unwrap_or(false)
} else {
false
}
}
fn build_download_url(repo_id: &str, filename: &str) -> String {
format!("{}/{}/resolve/main/{}", HF_DOWNLOAD_BASE, repo_id, filename)
}
/// Collect single-file GGUFs into quantization variants (sharded files are excluded).
fn group_into_variants(repo_id: &str, files: Vec<HfApiSibling>) -> Vec<HfQuantVariant> {
let mut variants: Vec<HfQuantVariant> = files
.into_iter()
.filter(|s| {
s.rfilename.ends_with(".gguf")
&& !is_shard_file(&s.rfilename)
&& parse_quantization(&s.rfilename) != "unknown"
})
.map(|s| {
let quant = parse_quantization(&s.rfilename);
let info = quant_info(&quant);
let download_url = build_download_url(repo_id, &s.rfilename);
HfQuantVariant {
quantization: quant,
size_bytes: s.size.unwrap_or(0),
filename: s.rfilename,
download_url,
description: info.description,
quality_rank: info.quality_rank,
}
})
.collect();
variants.sort_by_key(|v| v.quality_rank);
variants
}
pub async fn search_gguf_models(query: &str, limit: usize) -> Result<Vec<HfModelInfo>> {
let client = reqwest::Client::new();
let url = format!(
"{}?search={}&filter=gguf&sort=downloads&direction=-1&limit={}",
HF_API_BASE, query, limit
);
let response = client
.get(&url)
.header("User-Agent", "goose-ai-agent")
.send()
.await?;
if !response.status().is_success() {
bail!("HuggingFace API returned status {}", response.status());
}
let models: Vec<HfApiModel> = response.json().await?;
let results = models
.into_iter()
.filter_map(|m| {
let repo_id = m.id?;
let siblings = m.siblings.unwrap_or_default();
// The search endpoint may not include `siblings`; parse whatever
// is available. Files are fetched on-demand via `get_repo_gguf_variants`.
let gguf_files: Vec<HfGgufFile> = siblings
.into_iter()
.filter(|s| s.rfilename.ends_with(".gguf"))
.map(|s| {
let quantization = parse_quantization(&s.rfilename);
let download_url = build_download_url(&repo_id, &s.rfilename);
HfGgufFile {
filename: s.rfilename,
size_bytes: s.size.unwrap_or(0),
quantization,
download_url,
}
})
.collect();
let author = m
.author
.unwrap_or_else(|| repo_id.split('/').next().unwrap_or_default().to_string());
let model_name = repo_id
.split('/')
.next_back()
.unwrap_or(&repo_id)
.to_string();
Some(HfModelInfo {
repo_id,
author,
model_name,
downloads: m.downloads.unwrap_or(0),
gguf_files,
})
})
.collect();
Ok(results)
}
/// Fetch GGUF files for a repo and return them grouped by quantization.
pub async fn get_repo_gguf_variants(repo_id: &str) -> Result<Vec<HfQuantVariant>> {
let client = reqwest::Client::new();
let url = format!("{}/{}?blobs=true", HF_API_BASE, repo_id);
let response = client
.get(&url)
.header("User-Agent", "goose-ai-agent")
.send()
.await?;
if !response.status().is_success() {
bail!(
"HuggingFace API returned status {} for repo {}",
response.status(),
repo_id
);
}
let model: HfApiModel = response.json().await?;
let siblings = model.siblings.unwrap_or_default();
Ok(group_into_variants(repo_id, siblings))
}
/// Fetch raw GGUF files (kept for resolve_model_spec).
pub async fn get_repo_gguf_files(repo_id: &str) -> Result<Vec<HfGgufFile>> {
let client = reqwest::Client::new();
let url = format!("{}/{}?blobs=true", HF_API_BASE, repo_id);
let response = client
.get(&url)
.header("User-Agent", "goose-ai-agent")
.send()
.await?;
if !response.status().is_success() {
bail!(
"HuggingFace API returned status {} for repo {}",
response.status(),
repo_id
);
}
let model: HfApiModel = response.json().await?;
let siblings = model.siblings.unwrap_or_default();
let files = siblings
.into_iter()
.filter(|s| s.rfilename.ends_with(".gguf"))
.filter(|s| !is_shard_file(&s.rfilename))
.map(|s| {
let quantization = parse_quantization(&s.rfilename);
let download_url = build_download_url(repo_id, &s.rfilename);
HfGgufFile {
filename: s.rfilename,
size_bytes: s.size.unwrap_or(0),
quantization,
download_url,
}
})
.collect();
Ok(files)
}
/// Parse a model spec like "bartowski/Llama-3.2-1B-Instruct-GGUF:Q4_K_M" into (repo_id, quantization).
pub fn parse_model_spec(spec: &str) -> Result<(String, String)> {
let (repo_id, quant) = spec.rsplit_once(':').ok_or_else(|| {
anyhow::anyhow!(
"Invalid model spec '{}': expected format 'user/repo:quantization'",
spec
)
})?;
if !repo_id.contains('/') {
bail!("Invalid repo_id '{}': expected format 'user/repo'", repo_id);
}
Ok((repo_id.to_string(), quant.to_string()))
}
/// Resolve a model spec to a specific GGUF file from the repo.
pub async fn resolve_model_spec(spec: &str) -> Result<(String, HfGgufFile)> {
let (repo_id, quant) = parse_model_spec(spec)?;
let files = get_repo_gguf_files(&repo_id).await?;
let file = files
.into_iter()
.find(|f| f.quantization.eq_ignore_ascii_case(&quant))
.ok_or_else(|| {
anyhow::anyhow!(
"No GGUF file with quantization '{}' found in {}",
quant,
repo_id
)
})?;
Ok((repo_id, file))
}
/// Recommend which quantization variant to use based on available memory.
pub fn recommend_variant(
variants: &[HfQuantVariant],
available_memory_bytes: u64,
) -> Option<usize> {
// We need ~10-20% overhead beyond model size for inference context.
// Pick the highest-quality variant that fits.
let usable = (available_memory_bytes as f64 * 0.85) as u64;
let mut best: Option<usize> = None;
for (i, v) in variants.iter().enumerate() {
if v.size_bytes <= usable {
match best {
Some(bi) if variants[bi].quality_rank < v.quality_rank => best = Some(i),
None => best = Some(i),
_ => {}
}
}
}
best
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_parse_quantization() {
assert_eq!(parse_quantization("Model-Q4_K_M.gguf"), "Q4_K_M");
assert_eq!(parse_quantization("Model-Q8_0.gguf"), "Q8_0");
assert_eq!(parse_quantization("Model-IQ4_NL.gguf"), "IQ4_NL");
assert_eq!(parse_quantization("Model-F16.gguf"), "F16");
assert_eq!(parse_quantization("random-name.gguf"), "unknown");
}
#[test]
fn test_parse_quantization_with_directory() {
assert_eq!(
parse_quantization("Q5_K_M/Model-Q5_K_M-00001-of-00002.gguf"),
"Q5_K_M"
);
}
#[test]
fn test_parse_quantization_extended_tags() {
assert_eq!(parse_quantization("Model-MXFP4_MOE.gguf"), "MXFP4_MOE");
assert_eq!(parse_quantization("Model-UD-TQ1_0.gguf"), "TQ1_0");
assert_eq!(parse_quantization("Model-Q2_K_L.gguf"), "Q2_K_L");
assert_eq!(parse_quantization("Model-UD-Q4_K_XL.gguf"), "Q4_K_XL");
assert_eq!(parse_quantization("Model-UD-IQ1_M.gguf"), "IQ1_M");
}
#[test]
fn test_is_shard_file() {
assert!(is_shard_file("Q5_K_M/Model-Q5_K_M-00001-of-00002.gguf"));
assert!(is_shard_file("Model-BF16-00003-of-00004.gguf"));
assert!(!is_shard_file("Model-Q4_K_M.gguf"));
}
#[test]
fn test_parse_model_spec() {
let (repo, quant) =
parse_model_spec("bartowski/Llama-3.2-1B-Instruct-GGUF:Q4_K_M").unwrap();
assert_eq!(repo, "bartowski/Llama-3.2-1B-Instruct-GGUF");
assert_eq!(quant, "Q4_K_M");
}
#[test]
fn test_parse_model_spec_invalid() {
assert!(parse_model_spec("no-colon").is_err());
assert!(parse_model_spec("noslash:Q4_K_M").is_err());
}
#[test]
fn test_recommend_variant() {
let variants = vec![
HfQuantVariant {
quantization: "Q2_K".into(),
size_bytes: 2_000_000_000,
filename: "m-Q2_K.gguf".into(),
download_url: String::new(),
description: "Small",
quality_rank: 7,
},
HfQuantVariant {
quantization: "Q4_K_M".into(),
size_bytes: 4_000_000_000,
filename: "m-Q4_K_M.gguf".into(),
download_url: String::new(),
description: "Medium",
quality_rank: 19,
},
HfQuantVariant {
quantization: "Q8_0".into(),
size_bytes: 8_000_000_000,
filename: "m-Q8_0.gguf".into(),
download_url: String::new(),
description: "Large",
quality_rank: 25,
},
];
assert_eq!(recommend_variant(&variants, 5_000_000_000), Some(1));
assert_eq!(recommend_variant(&variants, 10_000_000_000), Some(2));
assert_eq!(recommend_variant(&variants, 1_000_000_000), None);
}
}
crates/goose/src/providers/local_inference/local_model_registry.rs
+297
View File
@@ -0,0 +1,297 @@
use crate::config::paths::Paths;
use anyhow::Result;
use serde::{Deserialize, Serialize};
use std::path::PathBuf;
use std::sync::{Mutex, OnceLock};
use utoipa::ToSchema;
#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
#[serde(tag = "type")]
pub enum SamplingConfig {
Greedy,
Temperature {
temperature: f32,
top_k: i32,
top_p: f32,
min_p: f32,
seed: Option<u32>,
},
MirostatV2 {
tau: f32,
eta: f32,
seed: Option<u32>,
},
}
impl Default for SamplingConfig {
fn default() -> Self {
SamplingConfig::Temperature {
temperature: 0.8,
top_k: 40,
top_p: 0.95,
min_p: 0.05,
seed: None,
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
pub struct ModelSettings {
pub context_size: Option<u32>,
pub max_output_tokens: Option<usize>,
#[serde(default)]
pub sampling: SamplingConfig,
#[serde(default = "default_repeat_penalty")]
pub repeat_penalty: f32,
#[serde(default = "default_repeat_last_n")]
pub repeat_last_n: i32,
#[serde(default)]
pub frequency_penalty: f32,
#[serde(default)]
pub presence_penalty: f32,
pub n_batch: Option<u32>,
pub n_gpu_layers: Option<u32>,
#[serde(default)]
pub use_mlock: bool,
pub flash_attention: Option<bool>,
pub n_threads: Option<i32>,
#[serde(default)]
pub native_tool_calling: bool,
}
fn default_repeat_penalty() -> f32 {
1.0
}
fn default_repeat_last_n() -> i32 {
64
}
impl Default for ModelSettings {
fn default() -> Self {
Self {
context_size: None,
max_output_tokens: None,
sampling: SamplingConfig::default(),
repeat_penalty: 1.0,
repeat_last_n: 64,
frequency_penalty: 0.0,
presence_penalty: 0.0,
n_batch: None,
n_gpu_layers: None,
use_mlock: false,
flash_attention: None,
n_threads: None,
native_tool_calling: false,
}
}
}
static REGISTRY: OnceLock<Mutex<LocalModelRegistry>> = OnceLock::new();
pub fn get_registry() -> &'static Mutex<LocalModelRegistry> {
REGISTRY.get_or_init(|| {
let mut registry = LocalModelRegistry::load().unwrap_or_default();
registry.migrate_legacy_models();
registry.migrate_model_ids();
Mutex::new(registry)
})
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct LocalModelEntry {
pub id: String,
pub display_name: String,
pub repo_id: String,
pub filename: String,
pub quantization: String,
pub local_path: PathBuf,
pub source_url: String,
#[serde(default)]
pub settings: ModelSettings,
}
impl LocalModelEntry {
pub fn is_downloaded(&self) -> bool {
self.local_path.exists()
}
pub fn file_size(&self) -> u64 {
std::fs::metadata(&self.local_path)
.map(|m| m.len())
.unwrap_or(0)
}
}
#[derive(Debug, Default, Serialize, Deserialize)]
pub struct LocalModelRegistry {
pub models: Vec<LocalModelEntry>,
}
fn registry_path() -> PathBuf {
Paths::in_data_dir("models/registry.json")
}
/// The 4 legacy hardcoded model definitions for migration.
struct LegacyModel {
id: &'static str,
display_name: &'static str,
repo_id: &'static str,
filename: &'static str,
quantization: &'static str,
source_url: &'static str,
}
const LEGACY_MODELS: &[LegacyModel] = &[
LegacyModel {
id: "llama-3.2-1b",
display_name: "Llama 3.2 1B Instruct",
repo_id: "bartowski/Llama-3.2-1B-Instruct-GGUF",
filename: "Llama-3.2-1B-Instruct-Q4_K_M.gguf",
quantization: "Q4_K_M",
source_url: "https://huggingface.co/bartowski/Llama-3.2-1B-Instruct-GGUF/resolve/main/Llama-3.2-1B-Instruct-Q4_K_M.gguf",
},
LegacyModel {
id: "llama-3.2-3b",
display_name: "Llama 3.2 3B Instruct",
repo_id: "bartowski/Llama-3.2-3B-Instruct-GGUF",
filename: "Llama-3.2-3B-Instruct-Q4_K_M.gguf",
quantization: "Q4_K_M",
source_url: "https://huggingface.co/bartowski/Llama-3.2-3B-Instruct-GGUF/resolve/main/Llama-3.2-3B-Instruct-Q4_K_M.gguf",
},
LegacyModel {
id: "hermes-2-pro-7b",
display_name: "Hermes 2 Pro Llama-3 7B",
repo_id: "NousResearch/Hermes-2-Pro-Llama-3-8B-GGUF",
filename: "Hermes-2-Pro-Llama-3-8B-Q4_K_M.gguf",
quantization: "Q4_K_M",
source_url: "https://huggingface.co/NousResearch/Hermes-2-Pro-Llama-3-8B-GGUF/resolve/main/Hermes-2-Pro-Llama-3-8B-Q4_K_M.gguf",
},
LegacyModel {
id: "mistral-small-22b",
display_name: "Mistral Small 22B Instruct",
repo_id: "bartowski/Mistral-Small-Instruct-2409-GGUF",
filename: "Mistral-Small-Instruct-2409-Q4_K_M.gguf",
quantization: "Q4_K_M",
source_url: "https://huggingface.co/bartowski/Mistral-Small-Instruct-2409-GGUF/resolve/main/Mistral-Small-Instruct-2409-Q4_K_M.gguf",
},
];
impl LocalModelRegistry {
pub fn load() -> Result<Self> {
let path = registry_path();
if !path.exists() {
return Ok(Self::default());
}
let data = std::fs::read_to_string(&path)?;
let registry: Self = serde_json::from_str(&data)?;
Ok(registry)
}
pub fn save(&self) -> Result<()> {
let path = registry_path();
if let Some(parent) = path.parent() {
std::fs::create_dir_all(parent)?;
}
let data = serde_json::to_string_pretty(self)?;
std::fs::write(&path, data)?;
Ok(())
}
/// Migrate model IDs from the old `author--repo--variant` format to the
/// HuggingFace-style `author/repo:variant` format. Only touches IDs that
/// contain `--` (the old separator); legacy featured models with short IDs
/// like `llama-3.2-1b` are left alone.
pub fn migrate_model_ids(&mut self) {
let mut changed = false;
for entry in &mut self.models {
if entry.id.contains("--") {
let new_id = model_id_from_repo(&entry.repo_id, &entry.quantization);
if entry.id != new_id {
entry.id = new_id;
changed = true;
}
}
}
if changed {
let _ = self.save();
}
}
/// Scan for legacy model files and add them to the registry if not already present.
pub fn migrate_legacy_models(&mut self) {
let mut changed = false;
for legacy in LEGACY_MODELS {
let legacy_path = Paths::in_data_dir("models").join(format!("{}.gguf", legacy.id));
if legacy_path.exists() && !self.models.iter().any(|m| m.id == legacy.id) {
self.models.push(LocalModelEntry {
id: legacy.id.to_string(),
display_name: legacy.display_name.to_string(),
repo_id: legacy.repo_id.to_string(),
filename: legacy.filename.to_string(),
quantization: legacy.quantization.to_string(),
local_path: legacy_path,
source_url: legacy.source_url.to_string(),
settings: ModelSettings::default(),
});
changed = true;
}
}
if changed {
let _ = self.save();
}
}
pub fn add_model(&mut self, entry: LocalModelEntry) -> Result<()> {
if let Some(existing) = self.models.iter_mut().find(|m| m.id == entry.id) {
*existing = entry;
} else {
self.models.push(entry);
}
self.save()
}
pub fn remove_model(&mut self, id: &str) -> Result<()> {
self.models.retain(|m| m.id != id);
self.save()
}
pub fn get_model(&self, id: &str) -> Option<&LocalModelEntry> {
self.models.iter().find(|m| m.id == id)
}
pub fn get_model_settings(&self, id: &str) -> Option<&ModelSettings> {
self.models.iter().find(|m| m.id == id).map(|m| &m.settings)
}
pub fn update_model_settings(&mut self, id: &str, settings: ModelSettings) -> Result<()> {
let entry = self
.models
.iter_mut()
.find(|m| m.id == id)
.ok_or_else(|| anyhow::anyhow!("Model not found: {}", id))?;
entry.settings = settings;
self.save()
}
pub fn list_models(&self) -> &[LocalModelEntry] {
&self.models
}
}
/// Generate a unique ID for a model from its repo_id and quantization.
/// Uses the HuggingFace convention: `author/model:variant`.
pub fn model_id_from_repo(repo_id: &str, quantization: &str) -> String {
format!("{}:{}", repo_id, quantization)
}
/// Generate a display name from repo_id and quantization.
pub fn display_name_from_repo(repo_id: &str, quantization: &str) -> String {
let model_name = repo_id
.split('/')
.next_back()
.unwrap_or(repo_id)
.trim_end_matches("-GGUF")
.trim_end_matches("-gguf");
format!("{} ({})", model_name, quantization)
}
ui/desktop/openapi.json
+565 -32
View File
@@ -1827,6 +1827,42 @@
}
}
},
"/local-inference/download": {
"post": {
"tags": [
"super::routes::local_inference"
],
"operationId": "download_hf_model",
"requestBody": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/HfDownloadRequest"
}
}
},
"required": true
},
"responses": {
"202": {
"description": "Download started",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/HfDownloadResponse"
}
}
}
},
"400": {
"description": "Invalid request"
},
"500": {
"description": "Internal server error"
}
}
}
},
"/local-inference/models": {
"get": {
"tags": [
@@ -1841,7 +1877,7 @@
"schema": {
"type": "array",
"items": {
"$ref": "#/components/schemas/LocalModelResponse"
"$ref": "#/components/schemas/ModelListItem"
}
}
}
@@ -1963,6 +1999,172 @@
}
}
},
"/local-inference/models/{model_id}/settings": {
"get": {
"tags": [
"super::routes::local_inference"
],
"operationId": "get_model_settings",
"parameters": [
{
"name": "model_id",
"in": "path",
"required": true,
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"description": "Model settings",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ModelSettings"
}
}
}
},
"404": {
"description": "Model not found"
}
}
},
"put": {
"tags": [
"super::routes::local_inference"
],
"operationId": "update_model_settings",
"parameters": [
{
"name": "model_id",
"in": "path",
"required": true,
"schema": {
"type": "string"
}
}
],
"requestBody": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ModelSettings"
}
}
},
"required": true
},
"responses": {
"200": {
"description": "Settings updated",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ModelSettings"
}
}
}
},
"404": {
"description": "Model not found"
},
"500": {
"description": "Failed to save settings"
}
}
}
},
"/local-inference/repo/{author}/{repo}/files": {
"get": {
"tags": [
"super::routes::local_inference"
],
"operationId": "get_repo_files",
"parameters": [
{
"name": "author",
"in": "path",
"required": true,
"schema": {
"type": "string"
}
},
{
"name": "repo",
"in": "path",
"required": true,
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"description": "GGUF quantization variants in repo",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/RepoVariantsResponse"
}
}
}
},
"500": {
"description": "Failed to fetch repo files"
}
}
}
},
"/local-inference/search": {
"get": {
"tags": [
"super::routes::local_inference"
],
"operationId": "search_hf_models",
"parameters": [
{
"name": "q",
"in": "query",
"description": "Search query",
"required": true,
"schema": {
"type": "string"
}
},
{
"name": "limit",
"in": "query",
"description": "Max results",
"required": false,
"schema": {
"type": "integer",
"nullable": true,
"minimum": 0
}
}
],
"responses": {
"200": {
"description": "Search results",
"content": {
"application/json": {
"schema": {
"type": "array",
"items": {
"$ref": "#/components/schemas/HfModelInfo"
}
}
}
}
},
"500": {
"description": "Search failed"
}
}
}
},
"/mcp-ui-proxy": {
"get": {
"tags": [
@@ -4704,6 +4906,128 @@
}
]
},
"HfDownloadRequest": {
"type": "object",
"properties": {
"filename": {
"type": "string",
"nullable": true
},
"repo_id": {
"type": "string",
"nullable": true
},
"spec": {
"type": "string",
"nullable": true
}
}
},
"HfDownloadResponse": {
"type": "object",
"required": [
"model_id"
],
"properties": {
"model_id": {
"type": "string"
}
}
},
"HfGgufFile": {
"type": "object",
"description": "A single downloadable GGUF file (used internally and for downloads).",
"required": [
"filename",
"size_bytes",
"quantization",
"download_url"
],
"properties": {
"download_url": {
"type": "string"
},
"filename": {
"type": "string"
},
"quantization": {
"type": "string"
},
"size_bytes": {
"type": "integer",
"format": "int64",
"minimum": 0
}
}
},
"HfModelInfo": {
"type": "object",
"required": [
"repo_id",
"author",
"model_name",
"downloads",
"gguf_files"
],
"properties": {
"author": {
"type": "string"
},
"downloads": {
"type": "integer",
"format": "int64",
"minimum": 0
},
"gguf_files": {
"type": "array",
"items": {
"$ref": "#/components/schemas/HfGgufFile"
}
},
"model_name": {
"type": "string"
},
"repo_id": {
"type": "string"
}
}
},
"HfQuantVariant": {
"type": "object",
"description": "A quantization variant — groups sharded files into one logical entry.",
"required": [
"quantization",
"size_bytes",
"filename",
"download_url",
"description",
"quality_rank"
],
"properties": {
"description": {
"type": "string"
},
"download_url": {
"type": "string"
},
"filename": {
"type": "string"
},
"quality_rank": {
"type": "integer",
"format": "int32",
"minimum": 0
},
"quantization": {
"type": "string"
},
"size_bytes": {
"type": "integer",
"format": "int64",
"minimum": 0
}
}
},
"Icon": {
"type": "object",
"required": [
@@ -4897,44 +5221,33 @@
"size_mb",
"context_limit",
"url",
"tokenizer_url",
"description",
"tier"
],
"properties": {
"context_limit": {
"type": "integer",
"description": "Maximum context window in tokens",
"minimum": 0
},
"description": {
"type": "string",
"description": "Description and use case"
"type": "string"
},
"id": {
"type": "string",
"description": "Model identifier (e.g., \"llama-3.2-1b\")"
"type": "string"
},
"name": {
"type": "string",
"description": "Display name"
"type": "string"
},
"size_mb": {
"type": "integer",
"format": "int32",
"description": "Model file size in MB",
"minimum": 0
},
"tier": {
"$ref": "#/components/schemas/ModelTier"
},
"tokenizer_url": {
"type": "string",
"description": "Download URL for the tokenizer JSON"
},
"url": {
"type": "string",
"description": "Download URL for the model GGUF file"
"type": "string"
}
}
},
@@ -4948,44 +5261,33 @@
"size_mb",
"context_limit",
"url",
"tokenizer_url",
"description",
"tier"
],
"properties": {
"context_limit": {
"type": "integer",
"description": "Maximum context window in tokens",
"minimum": 0
},
"description": {
"type": "string",
"description": "Description and use case"
"type": "string"
},
"id": {
"type": "string",
"description": "Model identifier (e.g., \"llama-3.2-1b\")"
"type": "string"
},
"name": {
"type": "string",
"description": "Display name"
"type": "string"
},
"size_mb": {
"type": "integer",
"format": "int32",
"description": "Model file size in MB",
"minimum": 0
},
"tier": {
"$ref": "#/components/schemas/ModelTier"
},
"tokenizer_url": {
"type": "string",
"description": "Download URL for the tokenizer JSON"
},
"url": {
"type": "string",
"description": "Download URL for the model GGUF file"
"type": "string"
}
}
},
@@ -4993,12 +5295,16 @@
"type": "object",
"required": [
"downloaded",
"recommended"
"recommended",
"featured"
],
"properties": {
"downloaded": {
"type": "boolean"
},
"featured": {
"type": "boolean"
},
"recommended": {
"type": "boolean"
}
@@ -5550,6 +5856,78 @@
}
}
},
"ModelListItem": {
"oneOf": [
{
"$ref": "#/components/schemas/LocalModelResponse"
},
{
"$ref": "#/components/schemas/RegistryModelResponse"
}
]
},
"ModelSettings": {
"type": "object",
"properties": {
"context_size": {
"type": "integer",
"format": "int32",
"nullable": true,
"minimum": 0
},
"flash_attention": {
"type": "boolean",
"nullable": true
},
"frequency_penalty": {
"type": "number",
"format": "float"
},
"max_output_tokens": {
"type": "integer",
"nullable": true,
"minimum": 0
},
"n_batch": {
"type": "integer",
"format": "int32",
"nullable": true,
"minimum": 0
},
"n_gpu_layers": {
"type": "integer",
"format": "int32",
"nullable": true,
"minimum": 0
},
"n_threads": {
"type": "integer",
"format": "int32",
"nullable": true
},
"native_tool_calling": {
"type": "boolean"
},
"presence_penalty": {
"type": "number",
"format": "float"
},
"repeat_last_n": {
"type": "integer",
"format": "int32"
},
"repeat_penalty": {
"type": "number",
"format": "float"
},
"sampling": {
"$ref": "#/components/schemas/SamplingConfig"
},
"use_mlock": {
"type": "boolean"
}
}
},
"ModelTier": {
"type": "string",
"enum": [
@@ -6189,6 +6567,51 @@
}
}
},
"RegistryModelResponse": {
"type": "object",
"required": [
"id",
"display_name",
"repo_id",
"filename",
"quantization",
"size_bytes",
"downloaded",
"featured",
"settings"
],
"properties": {
"display_name": {
"type": "string"
},
"downloaded": {
"type": "boolean"
},
"featured": {
"type": "boolean"
},
"filename": {
"type": "string"
},
"id": {
"type": "string"
},
"quantization": {
"type": "string"
},
"repo_id": {
"type": "string"
},
"settings": {
"$ref": "#/components/schemas/ModelSettings"
},
"size_bytes": {
"type": "integer",
"format": "int64",
"minimum": 0
}
}
},
"RemoveExtensionRequest": {
"type": "object",
"required": [
@@ -6204,6 +6627,25 @@
}
}
},
"RepoVariantsResponse": {
"type": "object",
"required": [
"variants"
],
"properties": {
"recommended_index": {
"type": "integer",
"nullable": true,
"minimum": 0
},
"variants": {
"type": "array",
"items": {
"$ref": "#/components/schemas/HfQuantVariant"
}
}
}
},
"ResourceContents": {
"anyOf": [
{
@@ -6395,6 +6837,97 @@
}
}
},
"SamplingConfig": {
"oneOf": [
{
"type": "object",
"required": [
"type"
],
"properties": {
"type": {
"type": "string",
"enum": [
"Greedy"
]
}
}
},
{
"type": "object",
"required": [
"temperature",
"top_k",
"top_p",
"min_p",
"type"
],
"properties": {
"min_p": {
"type": "number",
"format": "float"
},
"seed": {
"type": "integer",
"format": "int32",
"nullable": true,
"minimum": 0
},
"temperature": {
"type": "number",
"format": "float"
},
"top_k": {
"type": "integer",
"format": "int32"
},
"top_p": {
"type": "number",
"format": "float"
},
"type": {
"type": "string",
"enum": [
"Temperature"
]
}
}
},
{
"type": "object",
"required": [
"tau",
"eta",
"type"
],
"properties": {
"eta": {
"type": "number",
"format": "float"
},
"seed": {
"type": "integer",
"format": "int32",
"nullable": true,
"minimum": 0
},
"tau": {
"type": "number",
"format": "float"
},
"type": {
"type": "string",
"enum": [
"MirostatV2"
]
}
}
}
],
"discriminator": {
"propertyName": "type"
}
},
"SavePromptRequest": {
"type": "object",
"required": [
ui/desktop/src/api/index.ts
+2 -2
View File
File diff suppressed because one or more lines are too long
ui/desktop/src/api/sdk.gen.ts
+25 -1
View File
File diff suppressed because one or more lines are too long
ui/desktop/src/api/types.gen.ts
+232 -45
View File
@@ -433,6 +433,46 @@ export type GooseApp = McpAppResource & (WindowProps | null) & {
prd?: string | null;
};
export type HfDownloadRequest = {
filename?: string | null;
repo_id?: string | null;
spec?: string | null;
};
export type HfDownloadResponse = {
model_id: string;
};
/**
* A single downloadable GGUF file (used internally and for downloads).
*/
export type HfGgufFile = {
download_url: string;
filename: string;
quantization: string;
size_bytes: number;
};
export type HfModelInfo = {
author: string;
downloads: number;
gguf_files: Array<HfGgufFile>;
model_name: string;
repo_id: string;
};
/**
* A quantization variant — groups sharded files into one logical entry.
*/
export type HfQuantVariant = {
description: string;
download_url: string;
filename: string;
quality_rank: number;
quantization: string;
size_bytes: number;
};
export type Icon = {
mimeType?: string;
sizes?: Array<string>;
@@ -499,69 +539,26 @@ export type LoadedProvider = {
};
export type LocalLlmModel = {
/**
* Maximum context window in tokens
*/
context_limit: number;
/**
* Description and use case
*/
description: string;
/**
* Model identifier (e.g., "llama-3.2-1b")
*/
id: string;
/**
* Display name
*/
name: string;
/**
* Model file size in MB
*/
size_mb: number;
tier: ModelTier;
/**
* Download URL for the tokenizer JSON
*/
tokenizer_url: string;
/**
* Download URL for the model GGUF file
*/
url: string;
};
export type LocalModelResponse = {
/**
* Maximum context window in tokens
*/
context_limit: number;
/**
* Description and use case
*/
description: string;
/**
* Model identifier (e.g., "llama-3.2-1b")
*/
id: string;
/**
* Display name
*/
name: string;
/**
* Model file size in MB
*/
size_mb: number;
tier: ModelTier;
/**
* Download URL for the tokenizer JSON
*/
tokenizer_url: string;
/**
* Download URL for the model GGUF file
*/
url: string;
} & {
downloaded: boolean;
featured: boolean;
recommended: boolean;
};
@@ -721,6 +718,24 @@ export type ModelInfo = {
supports_cache_control?: boolean | null;
};
export type ModelListItem = LocalModelResponse | RegistryModelResponse;
export type ModelSettings = {
context_size?: number | null;
flash_attention?: boolean | null;
frequency_penalty?: number;
max_output_tokens?: number | null;
n_batch?: number | null;
n_gpu_layers?: number | null;
n_threads?: number | null;
native_tool_calling?: boolean;
presence_penalty?: number;
repeat_last_n?: number;
repeat_penalty?: number;
sampling?: SamplingConfig;
use_mlock?: boolean;
};
export type ModelTier = 'tiny' | 'small' | 'medium' | 'large';
export type ParseRecipeRequest = {
@@ -952,11 +967,28 @@ export type RedactedThinkingContent = {
data: string;
};
export type RegistryModelResponse = {
display_name: string;
downloaded: boolean;
featured: boolean;
filename: string;
id: string;
quantization: string;
repo_id: string;
settings: ModelSettings;
size_bytes: number;
};
export type RemoveExtensionRequest = {
name: string;
session_id: string;
};
export type RepoVariantsResponse = {
recommended_index?: number | null;
variants: Array<HfQuantVariant>;
};
export type ResourceContents = {
_meta?: {
[key: string]: unknown;
@@ -1034,6 +1066,22 @@ export type RunNowResponse = {
session_id: string;
};
export type SamplingConfig = {
type: 'Greedy';
} | {
min_p: number;
seed?: number | null;
temperature: number;
top_k: number;
top_p: number;
type: 'Temperature';
} | {
eta: number;
seed?: number | null;
tau: number;
type: 'MirostatV2';
};
export type SavePromptRequest = {
content: string;
};
@@ -2873,6 +2921,33 @@ export type StartTetrateSetupResponses = {
export type StartTetrateSetupResponse = StartTetrateSetupResponses[keyof StartTetrateSetupResponses];
export type DownloadHfModelData = {
body: HfDownloadRequest;
path?: never;
query?: never;
url: '/local-inference/download';
};
export type DownloadHfModelErrors = {
/**
* Invalid request
*/
400: unknown;
/**
* Internal server error
*/
500: unknown;
};
export type DownloadHfModelResponses = {
/**
* Download started
*/
202: HfDownloadResponse;
};
export type DownloadHfModelResponse = DownloadHfModelResponses[keyof DownloadHfModelResponses];
export type ListLocalModelsData = {
body?: never;
path?: never;
@@ -2884,7 +2959,7 @@ export type ListLocalModelsResponses = {
/**
* List of available local LLM models
*/
200: Array<LocalModelResponse>;
200: Array<ModelListItem>;
};
export type ListLocalModelsResponse = ListLocalModelsResponses[keyof ListLocalModelsResponses];
@@ -2991,6 +3066,118 @@ export type DownloadLocalModelResponses = {
202: unknown;
};
export type GetModelSettingsData = {
body?: never;
path: {
model_id: string;
};
query?: never;
url: '/local-inference/models/{model_id}/settings';
};
export type GetModelSettingsErrors = {
/**
* Model not found
*/
404: unknown;
};
export type GetModelSettingsResponses = {
/**
* Model settings
*/
200: ModelSettings;
};
export type GetModelSettingsResponse = GetModelSettingsResponses[keyof GetModelSettingsResponses];
export type UpdateModelSettingsData = {
body: ModelSettings;
path: {
model_id: string;
};
query?: never;
url: '/local-inference/models/{model_id}/settings';
};
export type UpdateModelSettingsErrors = {
/**
* Model not found
*/
404: unknown;
/**
* Failed to save settings
*/
500: unknown;
};
export type UpdateModelSettingsResponses = {
/**
* Settings updated
*/
200: ModelSettings;
};
export type UpdateModelSettingsResponse = UpdateModelSettingsResponses[keyof UpdateModelSettingsResponses];
export type GetRepoFilesData = {
body?: never;
path: {
author: string;
repo: string;
};
query?: never;
url: '/local-inference/repo/{author}/{repo}/files';
};
export type GetRepoFilesErrors = {
/**
* Failed to fetch repo files
*/
500: unknown;
};
export type GetRepoFilesResponses = {
/**
* GGUF quantization variants in repo
*/
200: RepoVariantsResponse;
};
export type GetRepoFilesResponse = GetRepoFilesResponses[keyof GetRepoFilesResponses];
export type SearchHfModelsData = {
body?: never;
path?: never;
query: {
/**
* Search query
*/
q: string;
/**
* Max results
*/
limit?: number | null;
};
url: '/local-inference/search';
};
export type SearchHfModelsErrors = {
/**
* Search failed
*/
500: unknown;
};
export type SearchHfModelsResponses = {
/**
* Search results
*/
200: Array<HfModelInfo>;
};
export type SearchHfModelsResponse = SearchHfModelsResponses[keyof SearchHfModelsResponses];
export type McpUiProxyData = {
body?: never;
path?: never;
ui/desktop/src/components/ToolCallWithResponse.tsx
+1 -1
View File
@@ -527,7 +527,7 @@ function ToolCallView({
// Function to create a descriptive representation of what the tool is doing
const getToolDescription = (): string | null => {
const args = toolCall.arguments as Record<string, ToolCallArgumentValue>;
const args = (toolCall.arguments ?? {}) as Record<string, ToolCallArgumentValue>;
const toolName = getToolName(toolCall.name);
const getStringValue = (value: ToolCallArgumentValue): string => {
ui/desktop/src/components/settings/SettingsView.tsx
+18 -1
View File
@@ -9,10 +9,11 @@ import ConfigSettings from './config/ConfigSettings';
import PromptsSettingsSection from './PromptsSettingsSection';
import { ExtensionConfig } from '../../api';
import { MainPanelLayout } from '../Layout/MainPanelLayout';
import { Bot, Share2, Monitor, MessageSquare, FileText, Keyboard } from 'lucide-react';
import { Bot, Share2, Monitor, MessageSquare, FileText, Keyboard, HardDrive } from 'lucide-react';
import { useState, useEffect, useRef } from 'react';
import ChatSettingsSection from './chat/ChatSettingsSection';
import KeyboardShortcutsSection from './keyboard/KeyboardShortcutsSection';
import LocalInferenceSection from './localInference/LocalInferenceSection';
import { CONFIGURATION_ENABLED } from '../../updates';
import { trackSettingsTabViewed } from '../../utils/analytics';
@@ -54,6 +55,7 @@ export default function SettingsView({
chat: 'chat',
prompts: 'prompts',
keyboard: 'keyboard',
'local-inference': 'local-inference',
};
const targetTab = sectionToTab[viewOptions.section];
@@ -112,6 +114,14 @@ export default function SettingsView({
<Bot className="h-4 w-4" />
Models
</TabsTrigger>
<TabsTrigger
value="local-inference"
className="flex gap-2"
data-testid="settings-local-inference-tab"
>
<HardDrive className="h-4 w-4" />
Local Inference
</TabsTrigger>
<TabsTrigger value="chat" className="flex gap-2" data-testid="settings-chat-tab">
<MessageSquare className="h-4 w-4" />
Chat
@@ -155,6 +165,13 @@ export default function SettingsView({
<ModelsSection setView={setView} />
</TabsContent>
<TabsContent
value="local-inference"
className="mt-0 focus-visible:outline-none focus-visible:ring-0"
>
<LocalInferenceSection />
</TabsContent>
<TabsContent
value="chat"
className="mt-0 focus-visible:outline-none focus-visible:ring-0"
ui/desktop/src/components/settings/localInference/HuggingFaceModelSearch.tsx
+331
View File
@@ -0,0 +1,331 @@
import { useState, useCallback, useRef } from 'react';
import { Search, Download, ChevronDown, ChevronUp, Loader2, Star } from 'lucide-react';
import { Button } from '../../ui/button';
import {
searchHfModels,
getRepoFiles,
downloadHfModel,
type HfModelInfo,
type HfQuantVariant,
} from '../../../api';
const formatBytes = (bytes: number): string => {
if (bytes === 0) return 'unknown';
if (bytes < 1024) return `${bytes}B`;
if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(0)}KB`;
if (bytes < 1024 * 1024 * 1024) return `${(bytes / (1024 * 1024)).toFixed(0)}MB`;
return `${(bytes / (1024 * 1024 * 1024)).toFixed(1)}GB`;
};
const formatDownloads = (n: number): string => {
if (n >= 1_000_000) return `${(n / 1_000_000).toFixed(1)}M`;
if (n >= 1_000) return `${(n / 1_000).toFixed(1)}K`;
return `${n}`;
};
interface RepoData {
variants: HfQuantVariant[];
recommendedIndex: number | null;
}
interface Props {
onDownloadStarted: (modelId: string) => void;
}
export const HuggingFaceModelSearch = ({ onDownloadStarted }: Props) => {
const [query, setQuery] = useState('');
const [results, setResults] = useState<HfModelInfo[]>([]);
const [expandedRepo, setExpandedRepo] = useState<string | null>(null);
const [repoData, setRepoData] = useState<Record<string, RepoData>>({});
const [searching, setSearching] = useState(false);
const [downloading, setDownloading] = useState<Set<string>>(new Set());
const [loadingFiles, setLoadingFiles] = useState<Set<string>>(new Set());
const [directSpec, setDirectSpec] = useState('');
const [error, setError] = useState<string | null>(null);
const debounceRef = useRef<ReturnType<typeof setTimeout> | null>(null);
const doSearch = useCallback(async (q: string) => {
if (!q.trim()) {
setResults([]);
setError(null);
return;
}
setSearching(true);
setError(null);
try {
const response = await searchHfModels({
query: { q, limit: 20 },
});
if (response.data) {
setResults(response.data);
if (response.data.length === 0) {
setError('No GGUF models found for this query.');
}
} else {
console.error('Search response:', response);
const errMsg = response.error
? `Search error: ${JSON.stringify(response.error)}`
: 'Search returned no data.';
setError(errMsg);
}
} catch (e) {
console.error('Search failed:', e);
setError('Search failed. Please try again.');
} finally {
setSearching(false);
}
}, []);
const handleQueryChange = (value: string) => {
setQuery(value);
if (debounceRef.current) clearTimeout(debounceRef.current);
debounceRef.current = setTimeout(() => doSearch(value), 300);
};
const toggleRepo = async (repoId: string) => {
if (expandedRepo === repoId) {
setExpandedRepo(null);
return;
}
setExpandedRepo(repoId);
if (!repoData[repoId]?.variants.length) {
setLoadingFiles((prev) => new Set(prev).add(repoId));
try {
const [author, repo] = repoId.split('/');
const response = await getRepoFiles({
path: { author, repo },
});
if (response.data) {
const variants = response.data.variants;
setRepoData((prev) => ({
...prev,
[repoId]: {
variants,
recommendedIndex: response.data!.recommended_index ?? null,
},
}));
if (variants.length === 0) {
setExpandedRepo(null);
setResults((prev) => prev.filter((m) => m.repo_id !== repoId));
}
}
} catch (e) {
console.error('Failed to fetch repo files:', e);
} finally {
setLoadingFiles((prev) => {
const next = new Set(prev);
next.delete(repoId);
return next;
});
}
}
};
const startDownload = async (repoId: string, filename: string) => {
const key = `${repoId}/${filename}`;
setDownloading((prev) => new Set(prev).add(key));
try {
const response = await downloadHfModel({
body: { repo_id: repoId, filename },
});
if (response.data) {
onDownloadStarted(response.data.model_id);
} else {
console.error('Download error:', response.error);
}
} catch (e) {
console.error('Download failed:', e);
} finally {
setDownloading((prev) => {
const next = new Set(prev);
next.delete(key);
return next;
});
}
};
const startDirectDownload = async () => {
if (!directSpec.trim()) return;
const key = `direct:${directSpec}`;
setDownloading((prev) => new Set(prev).add(key));
try {
const response = await downloadHfModel({
body: { spec: directSpec.trim() },
});
if (response.data) {
onDownloadStarted(response.data.model_id);
setDirectSpec('');
}
} catch (e) {
console.error('Direct download failed:', e);
} finally {
setDownloading((prev) => {
const next = new Set(prev);
next.delete(key);
return next;
});
}
};
return (
<div className="space-y-4">
<div>
<h4 className="text-sm font-medium text-text-default mb-2">Search HuggingFace</h4>
<div className="relative">
<Search className="absolute left-3 top-1/2 -translate-y-1/2 w-4 h-4 text-text-muted" />
<input
type="text"
value={query}
onChange={(e) => handleQueryChange(e.target.value)}
placeholder="Search for GGUF models..."
className="w-full pl-9 pr-4 py-2 text-sm border border-border-subtle rounded-lg bg-background-default text-text-default placeholder:text-text-muted focus:outline-none focus:border-accent-primary"
/>
{searching && (
<Loader2 className="absolute right-3 top-1/2 -translate-y-1/2 w-4 h-4 text-text-muted animate-spin" />
)}
</div>
</div>
{error && !searching && (
<p className="text-xs text-text-muted">{error}</p>
)}
{results.length > 0 && (
<div className="space-y-1 max-h-96 overflow-y-auto">
{results.map((model) => {
const isExpanded = expandedRepo === model.repo_id;
const data = repoData[model.repo_id];
const variants = data?.variants || [];
const recommendedIndex = data?.recommendedIndex ?? null;
return (
<div key={model.repo_id} className="border border-border-subtle rounded-lg">
<button
onClick={() => toggleRepo(model.repo_id)}
className="w-full flex items-center justify-between p-3 text-left hover:bg-background-subtle rounded-lg"
>
<div className="flex-1 min-w-0">
<div className="flex items-center gap-2">
<span className="text-sm font-medium text-text-default truncate">
{model.repo_id}
</span>
</div>
<div className="flex items-center gap-3 mt-0.5">
<span className="text-xs text-text-muted">
↓ {formatDownloads(model.downloads)}
</span>
</div>
</div>
{isExpanded ? (
<ChevronUp className="w-4 h-4 text-text-muted flex-shrink-0" />
) : (
<ChevronDown className="w-4 h-4 text-text-muted flex-shrink-0" />
)}
</button>
{isExpanded && (
<div className="border-t border-border-subtle px-3 pb-3 space-y-1">
{loadingFiles.has(model.repo_id) && (
<div className="flex items-center gap-2 py-2 text-xs text-text-muted">
<Loader2 className="w-3 h-3 animate-spin" />
Loading variants...
</div>
)}
{variants.map((variant, idx) => {
const dlKey = `${model.repo_id}/${variant.filename}`;
const isStarting = downloading.has(dlKey);
const isRecommended = idx === recommendedIndex;
return (
<div
key={variant.quantization}
className={`flex items-center justify-between py-2 px-2 rounded ${
isRecommended
? 'bg-blue-500/5 border border-blue-500/20'
: 'hover:bg-background-subtle'
}`}
>
<div className="flex flex-col gap-0.5 min-w-0 flex-1 mr-3">
<div className="flex items-center gap-2">
<span className="text-xs font-mono font-medium text-text-default">
{variant.quantization}
</span>
<span className="text-xs text-text-muted">
{formatBytes(variant.size_bytes)}
</span>
{isRecommended && (
<span className="inline-flex items-center gap-1 text-xs bg-blue-500 text-white px-1.5 py-0.5 rounded">
<Star className="w-3 h-3" />
Recommended
</span>
)}
</div>
{variant.description && (
<span className="text-xs text-text-muted">
{variant.description}
</span>
)}
</div>
<Button
variant="outline"
size="sm"
disabled={isStarting}
onClick={() => startDownload(model.repo_id, variant.filename)}
>
{isStarting ? (
<Loader2 className="w-3 h-3 animate-spin" />
) : (
<>
<Download className="w-3 h-3 mr-1" />
Download
</>
)}
</Button>
</div>
);
})}
</div>
)}
</div>
);
})}
</div>
)}
<div>
<h4 className="text-sm font-medium text-text-default mb-2">Direct Download</h4>
<p className="text-xs text-text-muted mb-2">
Specify a model directly: <code className="bg-background-subtle px-1 rounded">user/repo:quantization</code>
</p>
<div className="flex gap-2">
<input
type="text"
value={directSpec}
onChange={(e) => setDirectSpec(e.target.value)}
placeholder="bartowski/Llama-3.2-1B-Instruct-GGUF:Q4_K_M"
className="flex-1 px-3 py-2 text-sm border border-border-subtle rounded-lg bg-background-default text-text-default placeholder:text-text-muted focus:outline-none focus:border-accent-primary"
onKeyDown={(e) => {
if (e.key === 'Enter') startDirectDownload();
}}
/>
<Button
variant="outline"
size="sm"
disabled={!directSpec.trim() || downloading.has(`direct:${directSpec}`)}
onClick={startDirectDownload}
>
{downloading.has(`direct:${directSpec}`) ? (
<Loader2 className="w-4 h-4 animate-spin" />
) : (
<>
<Download className="w-4 h-4 mr-1" />
Download
</>
)}
</Button>
</div>
</div>
</div>
);
};
ui/desktop/src/components/settings/localInference/LocalInferenceSection.tsx
+9
View File
@@ -0,0 +1,9 @@
import { LocalInferenceSettings } from './LocalInferenceSettings';
export default function LocalInferenceSection() {
return (
<section id="local-inference" className="space-y-4 pr-4 pb-8">
<LocalInferenceSettings />
</section>
);
}
ui/desktop/src/components/settings/localInference/LocalInferenceSettings.tsx
+339 -148
View File
@@ -1,5 +1,5 @@
import { useState, useEffect } from 'react';
import { Download, Trash2, X, Check, ChevronDown, ChevronUp } from 'lucide-react';
import { useState, useEffect, useCallback, useRef } from 'react';
import { Download, Trash2, X, Check, ChevronDown, ChevronUp, Settings2 } from 'lucide-react';
import { Button } from '../../ui/button';
import { useConfig } from '../../ConfigContext';
import {
@@ -8,9 +8,13 @@ import {
getLocalModelDownloadProgress,
cancelLocalModelDownload,
deleteLocalModel,
type LocalModelResponse,
type DownloadProgress,
type LocalModelResponse,
type RegistryModelResponse,
type ModelListItem,
} from '../../../api';
import { HuggingFaceModelSearch } from './HuggingFaceModelSearch';
import { ModelSettingsPanel } from './ModelSettingsPanel';
const LOCAL_LLM_MODEL_CONFIG_KEY = 'LOCAL_LLM_MODEL';
@@ -21,12 +25,46 @@ const formatBytes = (bytes: number): string => {
return `${(bytes / (1024 * 1024 * 1024)).toFixed(1)}GB`;
};
function isFeaturedModel(item: ModelListItem): item is LocalModelResponse & { featured: boolean } {
return 'tier' in item;
}
function isRegistryModel(item: ModelListItem): item is RegistryModelResponse {
return 'display_name' in item && !('tier' in item);
}
export const LocalInferenceSettings = () => {
const [models, setModels] = useState<LocalModelResponse[]>([]);
const [featuredModels, setFeaturedModels] = useState<(LocalModelResponse & { featured?: boolean })[]>([]);
const [registryModels, setRegistryModels] = useState<RegistryModelResponse[]>([]);
const [downloads, setDownloads] = useState<Map<string, DownloadProgress>>(new Map());
const [selectedModelId, setSelectedModelId] = useState<string | null>(null);
const [showAllModels, setShowAllModels] = useState(false);
const [showAllFeatured, setShowAllFeatured] = useState(false);
const [settingsOpenFor, setSettingsOpenFor] = useState<string | null>(null);
const { read, upsert } = useConfig();
const downloadSectionRef = useRef<HTMLDivElement>(null);
const loadModels = useCallback(async () => {
try {
const response = await listLocalModels();
if (response.data) {
const featured: (LocalModelResponse & { featured?: boolean })[] = [];
const registry: RegistryModelResponse[] = [];
for (const item of response.data) {
if (isFeaturedModel(item)) {
featured.push(item);
} else if (isRegistryModel(item)) {
registry.push(item);
}
}
setFeaturedModels(featured);
setRegistryModels(registry);
}
} catch (error) {
console.error('Failed to load models:', error);
}
}, []);
useEffect(() => {
loadModels();
@@ -55,26 +93,23 @@ export const LocalInferenceSettings = () => {
setSelectedModelId(modelId);
};
const loadModels = async () => {
try {
const response = await listLocalModels();
if (response.data) {
setModels(response.data);
}
} catch (error) {
console.error('Failed to load models:', error);
}
};
const startDownload = async (modelId: string) => {
const startFeaturedDownload = async (modelId: string) => {
try {
await downloadLocalModel({ path: { model_id: modelId } });
pollDownloadProgress(modelId);
scrollToDownloads();
} catch (error) {
console.error('Failed to start download:', error);
}
};
const scrollToDownloads = useCallback(() => {
// Wait a tick for the download section to render before scrolling.
requestAnimationFrame(() => {
downloadSectionRef.current?.scrollIntoView({ behavior: 'smooth', block: 'nearest' });
});
}, []);
const pollDownloadProgress = (modelId: string) => {
const interval = setInterval(async () => {
try {
@@ -85,8 +120,7 @@ export const LocalInferenceSettings = () => {
if (progress.status === 'completed') {
clearInterval(interval);
await loadModels(); // Refresh model list
// Auto-select the model that was just downloaded
await loadModels();
await selectModel(modelId);
} else if (progress.status === 'failed') {
clearInterval(interval);
@@ -115,9 +149,8 @@ export const LocalInferenceSettings = () => {
}
};
const deleteModel = async (modelId: string) => {
const handleDeleteModel = async (modelId: string) => {
if (!window.confirm('Delete this model? You can re-download it later.')) return;
try {
await deleteLocalModel({ path: { model_id: modelId } });
if (selectedModelId === modelId) {
@@ -130,172 +163,330 @@ export const LocalInferenceSettings = () => {
}
};
const hasDownloadedNonRecommended = models.some(
const handleHfDownloadStarted = (modelId: string) => {
pollDownloadProgress(modelId);
scrollToDownloads();
};
// Featured models display logic
const hasDownloadedNonRecommended = featuredModels.some(
(model) => model.downloaded && !model.recommended
);
const displayedModels = showAllModels || hasDownloadedNonRecommended
? models
: models.filter((m) => m.recommended);
const hasNonRecommendedModels = models.some((m) => !m.recommended);
const showToggleButton = hasNonRecommendedModels && !hasDownloadedNonRecommended;
const displayedFeatured = showAllFeatured || hasDownloadedNonRecommended
? featuredModels
: featuredModels.filter((m) => m.recommended);
const hasNonRecommendedFeatured = featuredModels.some((m) => !m.recommended);
const showFeaturedToggle = hasNonRecommendedFeatured && !hasDownloadedNonRecommended;
// Downloaded models from both featured and registry
const downloadedFeatured = featuredModels.filter((m) => m.downloaded);
const downloadedRegistry = registryModels.filter((m) => m.downloaded);
const hasDownloaded = downloadedFeatured.length > 0 || downloadedRegistry.length > 0;
return (
<div className="space-y-4">
<div className="space-y-6">
<div>
<h3 className="text-text-default font-medium">Local Inference Models</h3>
<p className="text-xs text-text-muted max-w-2xl mt-1">
Download and manage local LLM models for inference without API keys. Supports GPU acceleration (Metal for Apple Silicon).
Download and manage local LLM models for inference without API keys. Search HuggingFace for any GGUF model or use the featured picks below.
</p>
</div>
<div className="space-y-2">
{displayedModels.map((model) => {
const progress = downloads.get(model.id);
const isDownloading = progress?.status === 'downloading';
const isSelected = selectedModelId === model.id;
const canSelect = model.downloaded && !isDownloading;
{/* Active Downloads */}
{downloads.size > 0 && (
<div ref={downloadSectionRef}>
<h4 className="text-sm font-medium text-text-default mb-2">Downloading</h4>
<div className="space-y-2">
{Array.from(downloads.entries()).map(([modelId, progress]) => {
if (progress.status === 'completed') return null;
return (
<div
key={modelId}
className="border rounded-lg p-3 border-border-subtle bg-background-default"
>
<div className="flex items-center justify-between mb-2">
<span className="text-sm font-medium text-text-default truncate">{modelId}</span>
{progress.status === 'downloading' && (
<Button
variant="ghost"
size="sm"
onClick={() => cancelDownload(modelId)}
className="text-destructive hover:text-destructive"
>
<X className="w-4 h-4" />
</Button>
)}
</div>
{progress.status === 'downloading' && (
<div className="space-y-1">
<div className="w-full bg-background-subtle rounded-full h-2">
<div
className="bg-accent-primary h-2 rounded-full transition-all duration-300"
style={{ width: `${progress.progress_percent}%` }}
/>
</div>
<div className="flex justify-between text-xs text-text-muted">
<span>{formatBytes(progress.bytes_downloaded)} / {formatBytes(progress.total_bytes)}</span>
<span>{progress.progress_percent.toFixed(0)}%</span>
</div>
</div>
)}
{progress.status === 'failed' && (
<p className="text-xs text-destructive">{progress.error || 'Download failed'}</p>
)}
</div>
);
})}
</div>
</div>
)}
return (
<div
key={model.id}
className={`border rounded-lg p-3 transition-colors ${
isSelected
? 'border-accent-primary bg-accent-primary/5'
: 'border-border-subtle bg-background-default hover:border-border-default'
}`}
>
<div className="flex items-start justify-between gap-3">
<div className="flex-1 min-w-0">
<div className="flex items-center gap-2 flex-wrap">
{canSelect && (
{/* Downloaded Models */}
{hasDownloaded && (
<div>
<h4 className="text-sm font-medium text-text-default mb-2">Downloaded Models</h4>
<div className="space-y-2">
{downloadedFeatured.map((model) => {
const isSelected = selectedModelId === model.id;
const showSettings = settingsOpenFor === model.id;
return (
<div
key={model.id}
className={`border rounded-lg p-3 transition-colors ${
isSelected
? 'border-accent-primary bg-accent-primary/5'
: 'border-border-subtle bg-background-default hover:border-border-default'
}`}
>
<div className="flex items-center justify-between">
<div className="flex items-center gap-2">
<input
type="radio"
checked={isSelected}
onChange={() => selectModel(model.id)}
className="cursor-pointer"
/>
)}
<h4 className="text-sm font-medium text-text-default">
{model.name}
</h4>
<span className="text-xs text-text-muted">
{model.size_mb}MB
</span>
<span className="text-xs text-text-muted">
{model.context_limit.toLocaleString()} tokens
</span>
{model.recommended && (
<span className="text-xs bg-blue-500 text-white px-2 py-0.5 rounded">
Recommended
</span>
)}
{isSelected && (
<span className="text-xs bg-accent-primary text-white px-2 py-0.5 rounded">
Active
</span>
)}
</div>
<p className="text-xs text-text-muted mt-1">
{model.description}
</p>
{model.recommended && (
<p className="text-xs text-blue-600 mt-1 font-medium">
Recommended for your hardware
</p>
)}
</div>
<div className="flex items-center gap-2">
{model.downloaded ? (
<>
<div className="flex items-center gap-1 text-xs text-green-600">
<Check className="w-4 h-4" />
<span>Downloaded</span>
</div>
<span className="text-sm font-medium text-text-default">{model.name}</span>
<span className="text-xs text-text-muted">{model.size_mb}MB</span>
{model.recommended && (
<span className="text-xs bg-blue-500 text-white px-2 py-0.5 rounded">Recommended</span>
)}
</div>
<div className="flex items-center gap-1">
<Button
variant="ghost"
size="sm"
onClick={() => deleteModel(model.id)}
onClick={() => setSettingsOpenFor(showSettings ? null : model.id)}
title="Model settings"
>
<Settings2 className="w-4 h-4" />
</Button>
<Button
variant="ghost"
size="sm"
onClick={() => handleDeleteModel(model.id)}
className="text-destructive hover:text-destructive"
>
<Trash2 className="w-4 h-4" />
</Button>
</>
) : isDownloading ? (
<>
<div className="text-xs text-text-muted min-w-[60px]">
{progress.progress_percent.toFixed(0)}%
</div>
</div>
</div>
{showSettings && <ModelSettingsPanel modelId={model.id} />}
</div>
);
})}
{downloadedRegistry.map((model) => {
const isSelected = selectedModelId === model.id;
const showSettings = settingsOpenFor === model.id;
return (
<div
key={model.id}
className={`border rounded-lg p-3 transition-colors ${
isSelected
? 'border-accent-primary bg-accent-primary/5'
: 'border-border-subtle bg-background-default hover:border-border-default'
}`}
>
<div className="flex items-center justify-between">
<div className="flex items-center gap-2">
<input
type="radio"
checked={isSelected}
onChange={() => selectModel(model.id)}
className="cursor-pointer"
/>
<span className="text-sm font-medium text-text-default">{model.display_name}</span>
<span className="text-xs text-text-muted">{formatBytes(model.size_bytes)}</span>
</div>
<div className="flex items-center gap-1">
<Button
variant="ghost"
size="sm"
onClick={() => cancelDownload(model.id)}
onClick={() => setSettingsOpenFor(showSettings ? null : model.id)}
title="Model settings"
>
<X className="w-4 h-4" />
<Settings2 className="w-4 h-4" />
</Button>
</>
) : (
<Button variant="outline" size="sm" onClick={() => startDownload(model.id)}>
<Download className="w-4 h-4 mr-1" />
Download
</Button>
)}
</div>
</div>
{isDownloading && progress && (
<div className="mt-2 space-y-1">
<div className="w-full bg-background-subtle rounded-full h-1.5">
<div
className="bg-accent-primary h-1.5 rounded-full transition-all"
style={{ width: `${progress.progress_percent}%` }}
/>
<Button
variant="ghost"
size="sm"
onClick={() => handleDeleteModel(model.id)}
className="text-destructive hover:text-destructive"
>
<Trash2 className="w-4 h-4" />
</Button>
</div>
</div>
<div className="flex justify-between text-xs text-text-muted">
<span>
{formatBytes(progress.bytes_downloaded)} / {formatBytes(progress.total_bytes)}
</span>
{progress.speed_bps && (
<span>{formatBytes(progress.speed_bps)}/s</span>
{showSettings && <ModelSettingsPanel modelId={model.id} />}
</div>
);
})}
</div>
</div>
)}
{/* Featured Models */}
<div>
<h4 className="text-sm font-medium text-text-default mb-2">Featured Models</h4>
<div className="space-y-2">
{displayedFeatured.map((model) => {
const progress = downloads.get(model.id);
const isDownloading = progress?.status === 'downloading';
return (
<div
key={model.id}
className="border rounded-lg p-3 border-border-subtle bg-background-default hover:border-border-default"
>
<div className="flex items-start justify-between gap-3">
<div className="flex-1 min-w-0">
<div className="flex items-center gap-2 flex-wrap">
<h4 className="text-sm font-medium text-text-default">{model.name}</h4>
<span className="text-xs text-text-muted">{model.size_mb}MB</span>
<span className="text-xs text-text-muted">
{model.context_limit.toLocaleString()} tokens
</span>
{model.recommended && (
<span className="text-xs bg-blue-500 text-white px-2 py-0.5 rounded">
Recommended
</span>
)}
</div>
<p className="text-xs text-text-muted mt-1">{model.description}</p>
</div>
<div className="flex items-center gap-2">
{model.downloaded ? (
<div className="flex items-center gap-1 text-xs text-green-600">
<Check className="w-4 h-4" />
<span>Downloaded</span>
</div>
) : isDownloading ? (
<>
<div className="text-xs text-text-muted min-w-[60px]">
{progress.progress_percent.toFixed(0)}%
</div>
<Button variant="ghost" size="sm" onClick={() => cancelDownload(model.id)}>
<X className="w-4 h-4" />
</Button>
</>
) : (
<Button
variant="outline"
size="sm"
onClick={() => startFeaturedDownload(model.id)}
>
<Download className="w-4 h-4 mr-1" />
Download
</Button>
)}
</div>
</div>
)}
{progress?.status === 'failed' && progress.error && (
<div className="mt-2 text-xs text-destructive">{progress.error}</div>
)}
{isDownloading && progress && (
<div className="mt-2 space-y-1">
<div className="w-full bg-background-subtle rounded-full h-1.5">
<div
className="bg-accent-primary h-1.5 rounded-full transition-all"
style={{ width: `${progress.progress_percent}%` }}
/>
</div>
<div className="flex justify-between text-xs text-text-muted">
<span>
{formatBytes(progress.bytes_downloaded)} / {formatBytes(progress.total_bytes)}
</span>
{progress.speed_bps && <span>{formatBytes(progress.speed_bps)}/s</span>}
</div>
</div>
)}
{progress?.status === 'failed' && progress.error && (
<div className="mt-2 text-xs text-destructive">{progress.error}</div>
)}
</div>
);
})}
</div>
{showFeaturedToggle && (
<Button
variant="ghost"
size="sm"
onClick={() => setShowAllFeatured(!showAllFeatured)}
className="w-full text-text-muted hover:text-text-default mt-2"
>
{showAllFeatured ? (
<>
<ChevronUp className="w-4 h-4 mr-1" />
Show recommended only
</>
) : (
<>
<ChevronDown className="w-4 h-4 mr-1" />
Show all featured ({featuredModels.length - displayedFeatured.length} more)
</>
)}
</Button>
)}
</div>
{/* Non-downloaded registry models being downloaded */}
{registryModels
.filter((m) => !m.downloaded && downloads.has(m.id))
.map((model) => {
const progress = downloads.get(model.id);
if (!progress || progress.status !== 'downloading') return null;
return (
<div key={model.id} className="border rounded-lg p-3 border-border-subtle bg-background-default">
<div className="flex items-center justify-between">
<div className="flex items-center gap-2">
<span className="text-sm font-medium text-text-default">{model.display_name}</span>
<span className="text-xs text-text-muted">{progress.progress_percent.toFixed(0)}%</span>
</div>
<Button variant="ghost" size="sm" onClick={() => cancelDownload(model.id)}>
<X className="w-4 h-4" />
</Button>
</div>
<div className="mt-2">
<div className="w-full bg-background-subtle rounded-full h-1.5">
<div
className="bg-accent-primary h-1.5 rounded-full transition-all"
style={{ width: `${progress.progress_percent}%` }}
/>
</div>
</div>
</div>
);
})}
{/* HuggingFace Search */}
<div className="border-t border-border-subtle pt-4">
<HuggingFaceModelSearch onDownloadStarted={handleHfDownloadStarted} />
</div>
{showToggleButton && (
<Button
variant="ghost"
size="sm"
onClick={() => setShowAllModels(!showAllModels)}
className="w-full text-text-muted hover:text-text-default"
>
{showAllModels ? (
<>
<ChevronUp className="w-4 h-4 mr-1" />
Show recommended only
</>
) : (
<>
<ChevronDown className="w-4 h-4 mr-1" />
Show all models ({models.length - displayedModels.length} more)
</>
)}
</Button>
)}
{models.length === 0 && (
<div className="text-center py-6 text-text-muted text-sm">
No models available
</div>
{featuredModels.length === 0 && registryModels.length === 0 && (
<div className="text-center py-6 text-text-muted text-sm">No models available</div>
)}
</div>
);
ui/desktop/src/components/settings/localInference/ModelSettingsPanel.tsx
+417
View File
@@ -0,0 +1,417 @@
import { useState, useEffect, useCallback } from 'react';
import { RotateCcw } from 'lucide-react';
import { Button } from '../../ui/button';
import { Switch } from '../../ui/switch';
import {
getModelSettings,
updateModelSettings,
type ModelSettings,
type SamplingConfig,
} from '../../../api';
const DEFAULT_SETTINGS: ModelSettings = {
context_size: null,
max_output_tokens: null,
sampling: { type: 'Temperature', temperature: 0.8, top_k: 40, top_p: 0.95, min_p: 0.05, seed: null },
repeat_penalty: 1.0,
repeat_last_n: 64,
frequency_penalty: 0.0,
presence_penalty: 0.0,
n_batch: null,
n_gpu_layers: null,
use_mlock: false,
flash_attention: null,
n_threads: null,
native_tool_calling: false,
};
type SamplingType = SamplingConfig['type'];
function NumberField({
label,
description,
value,
onChange,
placeholder,
min,
max,
step,
allowNull,
}: {
label: string;
description?: string;
value: number | null | undefined;
onChange: (v: number | null) => void;
placeholder?: string;
min?: number;
max?: number;
step?: number;
allowNull?: boolean;
}) {
return (
<div className="flex flex-col gap-1">
<label className="text-xs font-medium text-text-default">{label}</label>
{description && <span className="text-xs text-text-muted">{description}</span>}
<input
type="number"
className="w-full rounded border border-border-subtle bg-background-default px-2 py-1 text-sm text-text-default"
value={value ?? ''}
onChange={(e) => {
const raw = e.target.value;
if (raw === '' && allowNull) {
onChange(null);
} else {
const n = step && step < 1 ? parseFloat(raw) : parseInt(raw, 10);
if (!isNaN(n)) onChange(n);
}
}}
placeholder={placeholder ?? 'Auto'}
min={min}
max={max}
step={step}
/>
</div>
);
}
function ToggleField({
label,
description,
value,
onChange,
}: {
label: string;
description?: string;
value: boolean;
onChange: (v: boolean) => void;
}) {
return (
<div className="flex items-center justify-between gap-2">
<div>
<div className="text-xs font-medium text-text-default">{label}</div>
{description && <span className="text-xs text-text-muted">{description}</span>}
</div>
<Switch checked={value} onCheckedChange={onChange} variant="mono" />
</div>
);
}
function SelectField<T extends string>({
label,
description,
value,
options,
onChange,
}: {
label: string;
description?: string;
value: T;
options: { value: T; label: string }[];
onChange: (v: T) => void;
}) {
return (
<div className="flex items-center justify-between gap-2">
<div>
<div className="text-xs font-medium text-text-default">{label}</div>
{description && <span className="text-xs text-text-muted">{description}</span>}
</div>
<select
value={value}
onChange={(e) => onChange(e.target.value as T)}
className="rounded border border-border-subtle bg-background-default px-2 py-1 text-xs text-text-default"
>
{options.map((opt) => (
<option key={opt.value} value={opt.value}>
{opt.label}
</option>
))}
</select>
</div>
);
}
export const ModelSettingsPanel = ({ modelId }: { modelId: string }) => {
const [settings, setSettings] = useState<ModelSettings>(DEFAULT_SETTINGS);
const [loading, setLoading] = useState(true);
const [saving, setSaving] = useState(false);
const load = useCallback(async () => {
try {
const res = await getModelSettings({ path: { model_id: modelId } });
if (res.data) setSettings(res.data);
} catch {
// use defaults
} finally {
setLoading(false);
}
}, [modelId]);
useEffect(() => {
load();
}, [load]);
const save = async (updated: ModelSettings) => {
setSettings(updated);
setSaving(true);
try {
await updateModelSettings({ path: { model_id: modelId }, body: updated });
} catch (e) {
console.error('Failed to save settings:', e);
} finally {
setSaving(false);
}
};
const resetDefaults = () => save(DEFAULT_SETTINGS);
const updateField = <K extends keyof ModelSettings>(key: K, value: ModelSettings[K]) => {
save({ ...settings, [key]: value });
};
const samplingType: SamplingType = settings.sampling?.type ?? 'Temperature';
const setSamplingType = (type: SamplingType) => {
let sampling: SamplingConfig;
if (type === 'Greedy') {
sampling = { type: 'Greedy' };
} else if (type === 'MirostatV2') {
sampling = { type: 'MirostatV2', tau: 5.0, eta: 0.1, seed: null };
} else {
sampling = { type: 'Temperature', temperature: 0.8, top_k: 40, top_p: 0.95, min_p: 0.05, seed: null };
}
save({ ...settings, sampling });
};
const updateSampling = (partial: Partial<SamplingConfig>) => {
save({ ...settings, sampling: { ...settings.sampling!, ...partial } as SamplingConfig });
};
if (loading) {
return <div className="py-2 text-xs text-text-muted">Loading settings...</div>;
}
return (
<div className="space-y-4 border-t border-border-subtle pt-3 pb-2">
<div className="flex items-center justify-between">
<span className="text-xs font-medium text-text-muted uppercase tracking-wide">
Model Settings {saving && '(saving...)'}
</span>
<Button variant="ghost" size="sm" onClick={resetDefaults} title="Reset to defaults">
<RotateCcw className="w-3.5 h-3.5 mr-1" />
<span className="text-xs">Reset</span>
</Button>
</div>
{/* Context & Generation */}
<div className="space-y-2">
<h5 className="text-xs font-medium text-text-default">Context & Generation</h5>
<div className="grid grid-cols-2 gap-3">
<NumberField
label="Context size"
description="Max context window (0 = model default)"
value={settings.context_size}
onChange={(v) => updateField('context_size', v)}
placeholder="Auto"
min={0}
allowNull
/>
<NumberField
label="Max output tokens"
description="Cap on generated tokens"
value={settings.max_output_tokens}
onChange={(v) => updateField('max_output_tokens', v)}
placeholder="No limit"
min={1}
allowNull
/>
</div>
</div>
{/* Sampling */}
<div className="space-y-2">
<SelectField
label="Sampling Strategy"
value={samplingType}
options={[
{ value: 'Greedy' as SamplingType, label: 'Greedy' },
{ value: 'Temperature' as SamplingType, label: 'Temperature' },
{ value: 'MirostatV2' as SamplingType, label: 'Mirostat v2' },
]}
onChange={(v) => setSamplingType(v)}
/>
{samplingType === 'Temperature' && settings.sampling?.type === 'Temperature' && (
<div className="grid grid-cols-2 gap-3">
<NumberField
label="Temperature"
value={settings.sampling.temperature}
onChange={(v) => updateSampling({ temperature: v ?? 0.8 })}
min={0}
max={2}
step={0.05}
/>
<NumberField
label="Top K"
value={settings.sampling.top_k}
onChange={(v) => updateSampling({ top_k: v ?? 40 })}
min={0}
/>
<NumberField
label="Top P"
value={settings.sampling.top_p}
onChange={(v) => updateSampling({ top_p: v ?? 0.95 })}
min={0}
max={1}
step={0.01}
/>
<NumberField
label="Min P"
value={settings.sampling.min_p}
onChange={(v) => updateSampling({ min_p: v ?? 0.05 })}
min={0}
max={1}
step={0.01}
/>
<NumberField
label="Seed"
value={settings.sampling.seed}
onChange={(v) => updateSampling({ seed: v })}
placeholder="Random"
min={0}
allowNull
/>
</div>
)}
{samplingType === 'MirostatV2' && settings.sampling?.type === 'MirostatV2' && (
<div className="grid grid-cols-2 gap-3">
<NumberField
label="Tau (target entropy)"
value={settings.sampling.tau}
onChange={(v) => updateSampling({ tau: v ?? 5.0 })}
min={0}
step={0.1}
/>
<NumberField
label="Eta (learning rate)"
value={settings.sampling.eta}
onChange={(v) => updateSampling({ eta: v ?? 0.1 })}
min={0}
max={1}
step={0.01}
/>
<NumberField
label="Seed"
value={settings.sampling.seed}
onChange={(v) => updateSampling({ seed: v })}
placeholder="Random"
min={0}
allowNull
/>
</div>
)}
</div>
{/* Repetition Penalty */}
<div className="space-y-2">
<h5 className="text-xs font-medium text-text-default">Repetition Penalty</h5>
<div className="grid grid-cols-2 gap-3">
<NumberField
label="Repeat penalty"
description="1.0 = off"
value={settings.repeat_penalty}
onChange={(v) => updateField('repeat_penalty', v ?? 1.0)}
min={0}
step={0.05}
/>
<NumberField
label="Repeat window"
description="Tokens to look back"
value={settings.repeat_last_n}
onChange={(v) => updateField('repeat_last_n', v ?? 64)}
min={0}
/>
<NumberField
label="Frequency penalty"
description="0.0 = off"
value={settings.frequency_penalty}
onChange={(v) => updateField('frequency_penalty', v ?? 0.0)}
min={0}
max={2}
step={0.05}
/>
<NumberField
label="Presence penalty"
description="0.0 = off"
value={settings.presence_penalty}
onChange={(v) => updateField('presence_penalty', v ?? 0.0)}
min={0}
max={2}
step={0.05}
/>
</div>
</div>
{/* Performance */}
<div className="space-y-2">
<h5 className="text-xs font-medium text-text-default">Performance</h5>
<div className="grid grid-cols-2 gap-3">
<NumberField
label="Batch size"
description="Prompt processing batch"
value={settings.n_batch}
onChange={(v) => updateField('n_batch', v)}
placeholder="Auto"
min={1}
allowNull
/>
<NumberField
label="GPU layers"
description="Layers to offload to GPU"
value={settings.n_gpu_layers}
onChange={(v) => updateField('n_gpu_layers', v)}
placeholder="All"
min={0}
allowNull
/>
<NumberField
label="Threads"
description="CPU threads for generation"
value={settings.n_threads}
onChange={(v) => updateField('n_threads', v)}
placeholder="Auto"
min={1}
allowNull
/>
</div>
<ToggleField
label="Lock model in RAM (mlock)"
description="Prevent model from being swapped to disk"
value={settings.use_mlock ?? false}
onChange={(v) => updateField('use_mlock', v)}
/>
<SelectField
label="Flash attention"
description="Enable flash attention optimization"
value={settings.flash_attention === null || settings.flash_attention === undefined ? 'auto' : settings.flash_attention ? 'on' : 'off'}
options={[
{ value: 'auto', label: 'Auto' },
{ value: 'on', label: 'On' },
{ value: 'off', label: 'Off' },
]}
onChange={(v) => updateField('flash_attention', v === 'auto' ? null : v === 'on')}
/>
</div>
{/* Tool Calling */}
<div className="space-y-2">
<h5 className="text-xs font-medium text-text-default">Tool Calling</h5>
<ToggleField
label="Native tool calling"
description="Use the model's built-in tool-call format instead of the shell-command emulator. Enable for large models that reliably support tool calling."
value={settings.native_tool_calling ?? false}
onChange={(v) => updateField('native_tool_calling', v)}
/>
</div>
</div>
);
};
ui/desktop/src/components/settings/models/ModelsSection.tsx
-6
View File
@@ -11,7 +11,6 @@ import { toastError } from '../../../toasts';
import { Card, CardContent, CardDescription, CardHeader, CardTitle } from '../../ui/card';
import ResetProviderSection from '../reset_provider/ResetProviderSection';
import { LocalInferenceSettings } from '../localInference/LocalInferenceSettings';
interface ModelsSectionProps {
setView: (view: View) => void;
@@ -103,11 +102,6 @@ export default function ModelsSection({ setView }: ModelsSectionProps) {
<ModelSettingsButtons setView={setView} />
</CardContent>
</Card>
<Card className="pb-2 rounded-lg">
<CardContent className="px-2 pt-4">
<LocalInferenceSettings />
</CardContent>
</Card>
<Card className="pb-2 rounded-lg">
<CardHeader className="pb-0">
<CardTitle className="">Reset Provider and Model</CardTitle>