ai-space/goose
3
0
Fork 0
mirror of https://github.com/aaif-goose/goose.git synced 2026-07-03 14:10:03 +02:00
Code Issues Packages Projects Releases Wiki Activity

fix: replace panic with proper error handling in get_tokenizer (#7175)

Browse Source 
Signed-off-by: hobostay <110hqc@gmail.com>
Co-authored-by: Claude Sonnet 4.5 <noreply@anthropic.com>
This commit is contained in:
Chujiang
2026-02-12 19:05:03 +08:00
committed by GitHub
parent 0e11d18478
commit a82a1d7def
1 changed files with 6 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/goose/src/token_counter.rs
+6 -6
View File
@@ -8,7 +8,7 @@ use tokio::sync::OnceCell;
use crate::conversation::message::Message;
static TOKENIZER: OnceCell<Arc<CoreBPE>> = OnceCell::const_new();
static TOKENIZER: OnceCell<Result<Arc<CoreBPE>, String>> = OnceCell::const_new();
const MAX_TOKEN_CACHE_SIZE: usize = 10_000;
@@ -182,15 +182,15 @@ impl TokenCounter {
}
async fn get_tokenizer() -> Result<Arc<CoreBPE>, String> {
let tokenizer = TOKENIZER
TOKENIZER
.get_or_init(|| async {
match tiktoken_rs::o200k_base() {
Ok(bpe) => Arc::new(bpe),
Err(e) => panic!("Failed to initialize o200k_base tokenizer: {}", e),
Ok(bpe) => Ok(Arc::new(bpe)),
Err(e) => Err(format!("Failed to initialize o200k_base tokenizer: {}", e)),
}
})
.await;
Ok(tokenizer.clone())
.await
.clone()
}
pub async fn create_token_counter() -> Result<TokenCounter, String> {