tkmind/tkmind_go
1
0
Fork 0
Code Issues Pull Requests Actions 4 Packages Projects Releases Wiki Activity

refactor: simplify tokenizer initialization — remove unnecessary Result wrapper (#7744)

Browse Source 
Signed-off-by: Jonathan MERCIER <bioinfornatics@gmail.com>
This commit is contained in:
jonathan MERCIER
2026-03-11 19:22:51 +01:00
committed by GitHub
parent 6fed3e392c
commit 7247d61648
1 changed files with 5 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/goose/src/token_counter.rs
+5 -7
View File
@@ -8,7 +8,7 @@ use tokio::sync::OnceCell;
use crate::conversation::message::Message; use crate::conversation::message::Message;
static TOKENIZER: OnceCell<Result<Arc<CoreBPE>, String>> = OnceCell::const_new(); static TOKENIZER: OnceCell<Arc<CoreBPE>> = OnceCell::const_new();
const MAX_TOKEN_CACHE_SIZE: usize = 10_000; const MAX_TOKEN_CACHE_SIZE: usize = 10_000;
@@ -182,15 +182,13 @@ impl TokenCounter {
} }
async fn get_tokenizer() -> Result<Arc<CoreBPE>, String> { async fn get_tokenizer() -> Result<Arc<CoreBPE>, String> {
TOKENIZER Ok(TOKENIZER
.get_or_init(|| async { .get_or_init(|| async {
match tiktoken_rs::o200k_base() { let bpe = tiktoken_rs::o200k_base().expect("Failed to initialize o200k_base tokenizer");
Ok(bpe) => Ok(Arc::new(bpe)), Arc::new(bpe)
Err(e) => Err(format!("Failed to initialize o200k_base tokenizer: {}", e)),
}
}) })
.await .await
.clone() .clone())
} }
pub async fn create_token_counter() -> Result<TokenCounter, String> { pub async fn create_token_counter() -> Result<TokenCounter, String> {