From 7247d616486fb785cd5dfc24ff6136642fd74c33 Mon Sep 17 00:00:00 2001 From: jonathan MERCIER Date: Wed, 11 Mar 2026 19:22:51 +0100 Subject: [PATCH] =?UTF-8?q?refactor:=20simplify=20tokenizer=20initializati?= =?UTF-8?q?on=20=E2=80=94=20remove=20unnecessary=20Result=20wrapper=20(#77?= =?UTF-8?q?44)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jonathan MERCIER --- crates/goose/src/token_counter.rs | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/crates/goose/src/token_counter.rs b/crates/goose/src/token_counter.rs index 64efa48ac0..cb41621427 100644 --- a/crates/goose/src/token_counter.rs +++ b/crates/goose/src/token_counter.rs @@ -8,7 +8,7 @@ use tokio::sync::OnceCell; use crate::conversation::message::Message; -static TOKENIZER: OnceCell, String>> = OnceCell::const_new(); +static TOKENIZER: OnceCell> = OnceCell::const_new(); const MAX_TOKEN_CACHE_SIZE: usize = 10_000; @@ -182,15 +182,13 @@ impl TokenCounter { } async fn get_tokenizer() -> Result, String> { - TOKENIZER + Ok(TOKENIZER .get_or_init(|| async { - match tiktoken_rs::o200k_base() { - Ok(bpe) => Ok(Arc::new(bpe)), - Err(e) => Err(format!("Failed to initialize o200k_base tokenizer: {}", e)), - } + let bpe = tiktoken_rs::o200k_base().expect("Failed to initialize o200k_base tokenizer"); + Arc::new(bpe) }) .await - .clone() + .clone()) } pub async fn create_token_counter() -> Result {