awesome-bookmarks/goose
3
0
Fork 0
mirror of https://github.com/block/goose.git synced 2026-07-17 12:56:20 +02:00
Code Issues Packages Projects Releases Wiki Activity

refactor: simplify tokenizer initialization — remove unnecessary Result wrapper (#7744)

Browse Source 
Signed-off-by: Jonathan MERCIER <bioinfornatics@gmail.com>
This commit is contained in:
jonathan MERCIER
2026-03-11 19:22:51 +01:00
committed by GitHub
parent 6fed3e392c
commit 7247d61648
1 changed files with 5 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/goose/src/token_counter.rs
+5 -7
View File
@@ -8,7 +8,7 @@ use tokio::sync::OnceCell;
use crate::conversation::message::Message;
static TOKENIZER: OnceCell<Result<Arc<CoreBPE>, String>> = OnceCell::const_new();
static TOKENIZER: OnceCell<Arc<CoreBPE>> = OnceCell::const_new();
const MAX_TOKEN_CACHE_SIZE: usize = 10_000;
@@ -182,15 +182,13 @@ impl TokenCounter {
}
async fn get_tokenizer() -> Result<Arc<CoreBPE>, String> {
TOKENIZER
Ok(TOKENIZER
.get_or_init(|| async {
match tiktoken_rs::o200k_base() {
Ok(bpe) => Ok(Arc::new(bpe)),
Err(e) => Err(format!("Failed to initialize o200k_base tokenizer: {}", e)),
}
let bpe = tiktoken_rs::o200k_base().expect("Failed to initialize o200k_base tokenizer");
Arc::new(bpe)
})
.await
.clone()
.clone())
}
pub async fn create_token_counter() -> Result<TokenCounter, String> {