tkmind/tkmind_go
1
0
Fork 0
Code Issues Pull Requests Actions 8 Packages Projects Releases Wiki Activity

fix: replace panic with proper error handling in get_tokenizer (#7175)

Browse Source 
Signed-off-by: hobostay <110hqc@gmail.com>
Co-authored-by: Claude Sonnet 4.5 <noreply@anthropic.com>
This commit is contained in:
Chujiang
2026-02-12 19:05:03 +08:00
committed by GitHub
parent 0e11d18478
commit a82a1d7def
1 changed files with 6 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/goose/src/token_counter.rs
+6 -6
View File
@@ -8,7 +8,7 @@ use tokio::sync::OnceCell;
use crate::conversation::message::Message; use crate::conversation::message::Message;
static TOKENIZER: OnceCell<Arc<CoreBPE>> = OnceCell::const_new(); static TOKENIZER: OnceCell<Result<Arc<CoreBPE>, String>> = OnceCell::const_new();
const MAX_TOKEN_CACHE_SIZE: usize = 10_000; const MAX_TOKEN_CACHE_SIZE: usize = 10_000;
@@ -182,15 +182,15 @@ impl TokenCounter {
} }
async fn get_tokenizer() -> Result<Arc<CoreBPE>, String> { async fn get_tokenizer() -> Result<Arc<CoreBPE>, String> {
let tokenizer = TOKENIZER TOKENIZER
.get_or_init(|| async { .get_or_init(|| async {
match tiktoken_rs::o200k_base() { match tiktoken_rs::o200k_base() {
Ok(bpe) => Arc::new(bpe), Ok(bpe) => Ok(Arc::new(bpe)),
Err(e) => panic!("Failed to initialize o200k_base tokenizer: {}", e), Err(e) => Err(format!("Failed to initialize o200k_base tokenizer: {}", e)),
} }
}) })
.await; .await
Ok(tokenizer.clone()) .clone()
} }
pub async fn create_token_counter() -> Result<TokenCounter, String> { pub async fn create_token_counter() -> Result<TokenCounter, String> {