tkmind/tkmind_go
1
0
Fork 0
Code Issues Pull Requests Actions 4 Packages Projects Releases Wiki Activity

local inference: stricter GGUF requirements, auto detection of tool calling support, fixed thinking output parsing (#9442)

Browse Source 
Signed-off-by: jh-block <jhugo@block.xyz>
This commit is contained in:
jh-block
2026-05-27 20:00:30 +02:00
committed by GitHub
parent d90b349a69
commit 27b41d93f5
23 changed files with 1844 additions and 492 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/goose-server/src/openapi.rs
+3
View File
@@ -683,6 +683,7 @@ pub struct ApiDoc;
super::routes::local_inference::list_local_models,
super::routes::local_inference::sync_featured_models,
super::routes::local_inference::search_hf_models,
super::routes::local_inference::list_builtin_chat_templates,
super::routes::local_inference::get_repo_files,
super::routes::local_inference::download_hf_model,
super::routes::local_inference::get_local_model_download_progress,
@@ -701,7 +702,9 @@ pub struct ApiDoc;
goose::providers::local_inference::hf_models::HfQuantVariant,
super::routes::local_inference::RepoVariantsResponse,
goose::providers::local_inference::local_model_registry::ModelSettings,
goose::providers::local_inference::local_model_registry::ChatTemplate,
goose::providers::local_inference::local_model_registry::SamplingConfig,
goose::providers::local_inference::local_model_registry::ToolCallingMode,
))
)]
pub struct LocalInferenceApiDoc;
crates/goose-server/src/routes/local_inference.rs
+142 -51
View File
@@ -13,10 +13,10 @@ use goose::config::paths::Paths;
use goose::download_manager::{get_download_manager, DownloadProgress};
use goose::providers::local_inference::hf_models::{self, HfModelInfo, HfQuantVariant};
use goose::providers::local_inference::{
available_inference_memory_bytes,
hf_models::{resolve_model_spec, resolve_model_spec_full, HfGgufFile},
available_inference_memory_bytes, builtin_chat_template_names,
hf_models::{resolve_model_spec_full, HfGgufFile},
local_model_registry::{
default_settings_for_model, featured_mmproj_spec, get_registry, is_featured_model,
default_settings_for_model, get_registry, is_featured_model, mmproj_local_path,
model_id_from_repo, LocalModelEntry, ModelDownloadStatus as RegistryDownloadStatus,
ModelSettings, ShardFile, FEATURED_MODELS,
},
@@ -79,26 +79,18 @@ async fn ensure_featured_models_in_registry() -> Result<(), ErrorResponse> {
.lock()
.map_err(|_| ErrorResponse::internal("Failed to acquire registry lock"))?;
if let Some(existing) = registry.get_model(&model_id) {
let needs_backfill = existing.mmproj_path.is_none() && featured.mmproj.is_some();
let needs_download = existing.is_downloaded()
&& featured.mmproj.is_some()
&& !existing.mmproj_path.as_ref().is_some_and(|p| p.exists());
if needs_download {
if let Some(mmproj) = featured.mmproj.as_ref() {
let path = mmproj.local_path();
let url = format!(
"https://huggingface.co/{}/resolve/main/{}",
mmproj.repo, mmproj.filename
);
mmproj_downloads_needed.push((model_id.clone(), url, path));
if let Some(path) = &existing.mmproj_path {
if existing.is_downloaded() && !path.exists() {
if let Some(url) = &existing.mmproj_source_url {
mmproj_downloads_needed.push((
model_id.clone(),
url.clone(),
path.clone(),
));
}
}
}
if !needs_backfill {
continue;
}
// Fall through to resolve for backfill
}
}
@@ -110,36 +102,45 @@ async fn ensure_featured_models_in_registry() -> Result<(), ErrorResponse> {
});
}
let resolved: Vec<(PendingResolve, HfGgufFile)> =
let resolved: Vec<(PendingResolve, HfGgufFile, Option<HfGgufFile>)> =
join_all(to_resolve.into_iter().map(|pending| async move {
let hf_file = match resolve_model_spec(pending.spec).await {
Ok((_repo, file)) => file,
let (hf_file, mmproj) = match resolve_model_spec_full(pending.spec).await {
Ok((_repo, resolved)) => (resolved.files[0].clone(), resolved.mmproj),
Err(_) => {
let filename = format!(
"{}-{}.gguf",
pending.repo_id.split('/').next_back().unwrap_or("model"),
pending.quantization
);
HfGgufFile {
filename: filename.clone(),
size_bytes: 0,
quantization: pending.quantization.to_string(),
download_url: format!(
"https://huggingface.co/{}/resolve/main/{}",
pending.repo_id, filename
),
}
(
HfGgufFile {
filename: filename.clone(),
size_bytes: 0,
quantization: pending.quantization.to_string(),
download_url: format!(
"https://huggingface.co/{}/resolve/main/{}",
pending.repo_id, filename
),
},
None,
)
}
};
(pending, hf_file)
(pending, hf_file, mmproj)
}))
.await;
let entries_to_add: Vec<LocalModelEntry> = resolved
.into_iter()
.map(|(pending, hf_file)| {
.map(|(pending, hf_file, mmproj)| {
let local_path = Paths::in_data_dir("models").join(&hf_file.filename);
let settings = default_settings_for_model(&pending.model_id);
let mmproj_path = mmproj
.as_ref()
.map(|mmproj| mmproj_local_path(&pending.repo_id, &mmproj.filename));
let mmproj_source_url = mmproj.as_ref().map(|mmproj| mmproj.download_url.clone());
let mmproj_size_bytes = mmproj.as_ref().map_or(0, |mmproj| mmproj.size_bytes);
let mmproj_checked = mmproj.is_some();
LocalModelEntry {
id: pending.model_id,
repo_id: pending.repo_id,
@@ -149,9 +150,10 @@ async fn ensure_featured_models_in_registry() -> Result<(), ErrorResponse> {
source_url: hf_file.download_url,
settings,
size_bytes: hf_file.size_bytes,
mmproj_path: None,
mmproj_source_url: None,
mmproj_size_bytes: 0,
mmproj_path,
mmproj_source_url,
mmproj_size_bytes,
mmproj_checked,
shard_files: vec![],
}
})
@@ -165,20 +167,80 @@ async fn ensure_featured_models_in_registry() -> Result<(), ErrorResponse> {
if !entries_to_add.is_empty() {
registry.sync_with_featured(entries_to_add);
}
}
let to_backfill: Vec<(String, String, String)> = {
let registry = get_registry()
.lock()
.map_err(|_| ErrorResponse::internal("Failed to acquire registry lock"))?;
registry
.list_models()
.iter()
.filter(|model| model.is_downloaded())
.filter(|model| model.mmproj_path.is_none())
.filter(|model| !model.mmproj_checked)
.map(|model| {
(
model.id.clone(),
model.repo_id.clone(),
model.quantization.clone(),
)
})
.collect()
};
let mmproj_backfills: Vec<(String, String, Option<Option<HfGgufFile>>)> = join_all(
to_backfill
.into_iter()
.map(|(id, repo_id, quantization)| async move {
let spec = format!("{repo_id}:{quantization}");
let mmproj = resolve_model_spec_full(&spec)
.await
.ok()
.map(|(_, resolved)| resolved.mmproj);
(id, repo_id, mmproj)
}),
)
.await;
{
let mut registry = get_registry()
.lock()
.map_err(|_| ErrorResponse::internal("Failed to acquire registry lock"))?;
for (model_id, repo_id, mmproj_result) in mmproj_backfills {
if let Some(model) = registry
.list_models_mut()
.iter_mut()
.find(|model| model.id == model_id)
{
let Some(mmproj) = mmproj_result else {
continue;
};
model.mmproj_checked = true;
if let Some(mmproj) = mmproj {
model.mmproj_path = Some(mmproj_local_path(&repo_id, &mmproj.filename));
model.mmproj_source_url = Some(mmproj.download_url);
model.mmproj_size_bytes = mmproj.size_bytes;
}
model.refresh_mmproj_metadata();
}
}
// Backfill mmproj data for all registry models and collect any
// needed mmproj downloads for models already on disk.
for model in registry.list_models_mut() {
model.enrich_with_featured_mmproj();
model.refresh_mmproj_metadata();
if model.is_downloaded() {
if let Some(mmproj) = featured_mmproj_spec(&model.id) {
let path = mmproj.local_path();
if let Some(path) = &model.mmproj_path {
if !path.exists() {
let url = format!(
"https://huggingface.co/{}/resolve/main/{}",
mmproj.repo, mmproj.filename
);
mmproj_downloads_needed.push((model.id.clone(), url, path));
if let Some(url) = &model.mmproj_source_url {
mmproj_downloads_needed.push((
model.id.clone(),
url.clone(),
path.clone(),
));
}
}
}
}
@@ -431,6 +493,20 @@ pub async fn download_hf_model(
vec![]
};
let mmproj_path = resolved
.mmproj
.as_ref()
.map(|mmproj| mmproj_local_path(&repo_id, &mmproj.filename));
let mmproj_source_url = resolved
.mmproj
.as_ref()
.map(|mmproj| mmproj.download_url.clone());
let mmproj_size_bytes = resolved
.mmproj
.as_ref()
.map_or(0, |mmproj| mmproj.size_bytes);
let mmproj_checked = true;
let entry = LocalModelEntry {
id: model_id.clone(),
repo_id,
@@ -440,13 +516,13 @@ pub async fn download_hf_model(
source_url: first_file.download_url.clone(),
settings: default_settings_for_model(&model_id),
size_bytes: resolved.total_size,
mmproj_path: None,
mmproj_source_url: None,
mmproj_size_bytes: 0,
mmproj_path,
mmproj_source_url,
mmproj_size_bytes,
mmproj_checked,
shard_files: shard_files.clone(),
};
// add_model enriches the entry with mmproj metadata from the featured table
let mmproj_path = {
let mut registry = get_registry()
.lock()
@@ -649,6 +725,17 @@ pub async fn update_model_settings(
Ok(Json(settings))
}
#[utoipa::path(
get,
path = "/local-inference/chat-templates/builtin",
responses(
(status = 200, description = "llama.cpp built-in chat template names", body = Vec<String>)
)
)]
pub async fn list_builtin_chat_templates() -> Json<Vec<String>> {
Json(builtin_chat_template_names())
}
pub fn routes(state: Arc<AppState>) -> Router {
let registered_paths: std::collections::HashSet<std::path::PathBuf> = get_registry()
.lock()
@@ -672,6 +759,10 @@ pub fn routes(state: Arc<AppState>) -> Router {
.route("/local-inference/models", get(list_local_models))
.route("/local-inference/sync-featured", post(sync_featured_models))
.route("/local-inference/search", get(search_hf_models))
.route(
"/local-inference/chat-templates/builtin",
get(list_builtin_chat_templates),
)
.route(
"/local-inference/repo/{author}/{repo}/files",
get(get_repo_files),