From 471904312eae908a941a1e92d46401dd8bb5f089 Mon Sep 17 00:00:00 2001 From: Aaron Alaniz Date: Wed, 26 Aug 2026 09:24:34 +0000 Subject: [PATCH] fix(local-inference): discover GGUF repos with non-standard filenames (#11005) Co-authored-by: Claude Opus 5 Co-authored-by: Jasper Hugo --- crates/goose-local-inference/src/hf_models.rs | 568 +++++++++++++++--- .../tests/gguf_filename_parsing.rs | 132 ++++ 2 files changed, 631 insertions(+), 69 deletions(-) create mode 100644 crates/goose-local-inference/tests/gguf_filename_parsing.rs diff --git a/crates/goose-local-inference/src/hf_models.rs b/crates/goose-local-inference/src/hf_models.rs index 970fdbb05..1c9482df3 100644 --- a/crates/goose-local-inference/src/hf_models.rs +++ b/crates/goose-local-inference/src/hf_models.rs @@ -292,26 +292,49 @@ fn parse_quantization(filename: &str) -> String { stem }; - // The quantization tag is typically the last hyphen-separated component - // that looks like a quant identifier (starts with Q, IQ, F, BF, TQ, MXFP, etc.) - // e.g. "Qwen3-Coder-Next-Q4_K_M" -> "Q4_K_M" - // "Model-UD-IQ1_M" -> "IQ1_M" - if let Some((_, candidate)) = stem.rsplit_once('-') { - if looks_like_quant(candidate) { - return candidate.to_string(); + // Publishers do not agree on where the quantization tag goes. Most append it + // ("Model-Q4_K_M"), but some bury it mid-name ("gemma-4-26B_q4_0-it"). Scan + // name components right-to-left and, within each, peel `_`-separated prefixes + // so a tag fused to a neighbouring token still resolves. Whole components are + // tested before their suffixes so tags absent from QUANT_TABLE survive intact + // ("Q6_K_L" must not degrade to the "Q6_K" inside it). + for component in stem.rsplit(['-', '.']) { + let mut candidate = component; + loop { + if let Some(canonical) = canonical_quant(candidate) { + return canonical.to_string(); + } + if looks_like_quant(candidate) { + return candidate.to_string(); + } + match candidate.split_once('_') { + Some((_, rest)) => candidate = rest, + None => break, + } } } - // Fallback: try dot separator (e.g. "model.Q4_K_M") - if let Some((_, candidate)) = stem.rsplit_once('.') { - if looks_like_quant(candidate) { - return candidate.to_string(); + // Some publishers put a named preset rather than a quantization in the tag + // position ("...-APEX-I-Quality"). Keep accepting a trailing Q-word so those + // repos still expose the single variant they ship. + if let Some((_, tail)) = stem.rsplit_once(['-', '.']) { + if tail.starts_with(['Q', 'q']) { + return tail.to_string(); } } "unknown".to_string() } +/// Resolve a tag to its QUANT_TABLE spelling, so casing differences between +/// publishers still hit the description and quality-rank lookup. +fn canonical_quant(candidate: &str) -> Option<&'static str> { + QUANT_TABLE + .iter() + .map(|(name, _, _)| *name) + .find(|name| name.eq_ignore_ascii_case(candidate)) +} + fn quant_bits(quantization: &str) -> u8 { let digits: String = quantization .chars() @@ -330,12 +353,22 @@ fn mmproj_precision_preference(quantization: &str) -> u8 { } } +/// Shape check for quantization tags missing from QUANT_TABLE (e.g. "Q6_K_L", +/// "Q4_0_4_4"). The family prefix must be followed by a digit, so model names +/// like "Qwen3" are not mistaken for quantizations. fn looks_like_quant(s: &str) -> bool { let upper = s.to_uppercase(); - upper.starts_with("Q") - || upper.starts_with("IQ") - || upper.starts_with("TQ") - || upper.starts_with("MXFP") + let digit_follows = |prefix: &str| { + upper + .strip_prefix(prefix) + .and_then(|rest| rest.chars().next()) + .is_some_and(|c| c.is_ascii_digit()) + }; + + digit_follows("Q") + || digit_follows("IQ") + || digit_follows("TQ") + || digit_follows("MXFP") || upper == "F16" || upper == "F32" || upper == "BF16" @@ -346,6 +379,60 @@ fn is_shard_file(filename: &str) -> bool { parse_shard_index(filename).is_some() } +fn shard_set_key(filename: &str) -> Option<(&str, &str)> { + let stem = filename.trim_end_matches(".gguf"); + let pos = stem.rfind("-of-")?; + let before = stem.get(..pos)?; + let total = stem.get(pos + 4..)?; + let (prefix, index) = before.rsplit_once('-')?; + if !index.is_empty() && index.chars().all(|c| c.is_ascii_digit()) { + Some((prefix, total)) + } else { + None + } +} + +fn is_complete_shard_set(files: &[&HfApiSibling]) -> bool { + let Some(expected_total) = files + .first() + .and_then(|file| parse_shard_total(&file.rfilename)) + else { + return false; + }; + if files.len() != expected_total as usize { + return false; + } + + let mut indices = std::collections::HashSet::with_capacity(files.len()); + files.iter().all(|file| { + parse_shard_total(&file.rfilename) == Some(expected_total) + && parse_shard_index(&file.rfilename) + .is_some_and(|index| index > 0 && index <= expected_total && indices.insert(index)) + }) +} + +fn select_preferred_shard_set(files: Vec<&HfApiSibling>) -> Vec<&HfApiSibling> { + let mut shard_sets: std::collections::HashMap<(&str, &str), Vec<&HfApiSibling>> = + std::collections::HashMap::new(); + + for file in files { + let key = shard_set_key(&file.rfilename) + .expect("files passed to select_preferred_shard_set must be shards"); + shard_sets.entry(key).or_default().push(file); + } + + shard_sets + .into_iter() + .min_by(|(key_a, files_a), (key_b, files_b)| { + is_complete_shard_set(files_b) + .cmp(&is_complete_shard_set(files_a)) + .then_with(|| (key_a.0.len() + key_a.1.len()).cmp(&(key_b.0.len() + key_b.1.len()))) + .then_with(|| key_a.cmp(key_b)) + }) + .map(|(_, files)| files) + .unwrap_or_default() +} + /// Parse the shard index (1-based) from a filename like "model-BF16-00001-of-00002.gguf". fn parse_shard_index(filename: &str) -> Option { let basename = filename.rsplit('/').next().unwrap_or(filename); @@ -403,6 +490,94 @@ fn is_prefix(prefix: &[&str], parts: &[&str]) -> bool { prefix.len() <= parts.len() && prefix.iter().zip(parts).all(|(a, b)| a == b) } +fn normalize_family_fragment(value: &str, quantization: &str, projector: bool) -> String { + let mut family = value.to_ascii_lowercase(); + if quantization != "unknown" { + if let Some(position) = family.rfind(quantization) { + family.replace_range(position..position + quantization.len(), ""); + } + } + if projector { + if let Some(position) = family.rfind("mmproj") { + family.replace_range(position..position + "mmproj".len(), ""); + } + } + + family.retain(|character| character.is_ascii_alphanumeric()); + family +} + +fn gguf_family_key(filename: &str, projector: bool) -> String { + let basename = filename.rsplit('/').next().unwrap_or(filename); + let mut stem = basename.trim_end_matches(".gguf"); + if let Some(pos) = stem.rfind("-of-") { + stem = stem + .get(..pos) + .and_then(|prefix| prefix.rsplit_once('-').map(|(prefix, _)| prefix)) + .unwrap_or(stem); + } + + let quantization = parse_quantization(filename).to_ascii_lowercase(); + normalize_family_fragment(stem, &quantization, projector) +} + +fn gguf_family_identity(filename: &str) -> String { + let quantization = parse_quantization(filename).to_ascii_lowercase(); + parent_components(filename) + .into_iter() + .map(|component| normalize_family_fragment(component, &quantization, false)) + .chain(std::iter::once(gguf_family_key(filename, false))) + .filter(|component| !component.is_empty()) + .collect::>() + .join("/") +} + +fn ensure_unambiguous_model_family( + repo_id: &str, + quantization: &str, + files: &[&HfApiSibling], +) -> Result<()> { + let families: std::collections::HashSet = files + .iter() + .map(|file| gguf_family_identity(&file.rfilename)) + .collect(); + if families.len() > 1 { + let mut filenames: Vec<&str> = files.iter().map(|file| file.rfilename.as_str()).collect(); + filenames.sort_unstable(); + bail!( + "Quantization '{}' is ambiguous in {} because it belongs to multiple GGUF model families: {}", + quantization, + repo_id, + filenames.join(", ") + ); + } + Ok(()) +} + +fn mmproj_matches_model_family( + siblings: &[HfApiSibling], + mmproj_filename: &str, + model_filename: &str, +) -> bool { + let mmproj_dir = parent_components(mmproj_filename); + let model_families: std::collections::HashSet = siblings + .iter() + .filter(|sibling| { + sibling.rfilename.ends_with(".gguf") + && !is_auxiliary_gguf_file(&sibling.rfilename) + && is_prefix(&mmproj_dir, &parent_components(&sibling.rfilename)) + }) + .map(|sibling| gguf_family_identity(&sibling.rfilename)) + .collect(); + + if model_families.len() <= 1 { + return true; + } + + let projector_family = gguf_family_key(mmproj_filename, true); + !projector_family.is_empty() && projector_family == gguf_family_key(model_filename, false) +} + fn select_best_mmproj( repo_id: &str, siblings: &[HfApiSibling], @@ -420,7 +595,9 @@ fn select_best_mmproj( }) .filter_map(|s| { let mmproj_dir = parent_components(&s.rfilename); - if !is_prefix(&mmproj_dir, &model_dir) { + if !is_prefix(&mmproj_dir, &model_dir) + || !mmproj_matches_model_family(siblings, &s.rfilename, model_filename) + { return None; } @@ -451,42 +628,71 @@ fn select_best_mmproj( }) } -/// Derive the expected model filename stem from a repo_id. -/// e.g. "unsloth/gemma-4-26B-A4B-it-GGUF" → "gemma-4-26b-a4b-it" (lowercased) -fn model_stem_from_repo(repo_id: &str) -> String { - let repo_name = repo_id.rsplit('/').next().unwrap_or(repo_id); - let stem = repo_name - .strip_suffix("-GGUF") - .or_else(|| repo_name.strip_suffix("-gguf")) - .unwrap_or(repo_name); - stem.to_lowercase() +const AUXILIARY_TOKENS: &[&str] = &["encoder", "draft", "drafter", "adapter", "lora"]; + +fn contains_auxiliary_token(value: &str) -> bool { + value + .split(['-', '_', '.']) + .any(|token| AUXILIARY_TOKENS.contains(&token)) } -/// Check whether a GGUF file belongs to the main model (vs auxiliary files like mmproj). -/// Matches files whose basename starts with the model stem derived from the repo name. -fn is_model_file(filename: &str, model_stem_lower: &str) -> bool { - let basename = filename.rsplit('/').next().unwrap_or(filename); - basename.to_lowercase().starts_with(model_stem_lower) +/// Check whether a GGUF file ships alongside the weights rather than being a +/// downloadable variant itself (projectors, vision encoders, speculative drafters). +/// +/// Filenames are matched by exclusion rather than by a repo-name prefix, because +/// publishers routinely rename files relative to the repo — Google drops the `qat` +/// segment, unsloth drops `MTP`, Qwen writes `Qwen3VL` for `Qwen3-VL`. +/// +/// MTP drafters are published either under an `MTP/` directory or with a leading +/// `mtp-` on the basename; models whose *name* contains MTP carry it mid-name and +/// are real weights, so only the leading/directory forms count as auxiliary. +pub fn is_auxiliary_gguf_file(filename: &str) -> bool { + let lowercase = filename.to_lowercase(); + + if lowercase.contains("mmproj") { + return true; + } + + if parent_components(&lowercase) + .into_iter() + .any(|component| component == "mtp" || contains_auxiliary_token(component)) + { + return true; + } + + let basename = lowercase.rsplit('/').next().unwrap_or(&lowercase); + let stem = basename.trim_end_matches(".gguf"); + + stem.split(['-', '_', '.']).next() == Some("mtp") || contains_auxiliary_token(stem) } /// Collect GGUF files into quantization variants. /// Single-file quants use the file directly. /// Sharded quants (multiple files for one quantization) aggregate sizes and use the /// first shard filename as the representative — the download path must handle all shards. -fn group_into_variants(repo_id: &str, files: Vec) -> Vec { +fn group_into_variants(repo_id: &str, files: Vec) -> Result> { use std::collections::HashMap; - let stem = model_stem_from_repo(repo_id); - let gguf_files: Vec<_> = files .into_iter() .filter(|s| { s.rfilename.ends_with(".gguf") - && is_model_file(&s.rfilename, &stem) + && !is_auxiliary_gguf_file(&s.rfilename) && parse_quantization(&s.rfilename) != "unknown" }) .collect(); + let mut files_by_quant: HashMap> = HashMap::new(); + for file in &gguf_files { + files_by_quant + .entry(parse_quantization(&file.rfilename)) + .or_default() + .push(file); + } + for (quantization, files) in files_by_quant { + ensure_unambiguous_model_family(repo_id, &quantization, &files)?; + } + // Separate single files from shards let mut single_files: Vec<&HfApiSibling> = Vec::new(); let mut shard_groups: HashMap> = HashMap::new(); @@ -500,6 +706,17 @@ fn group_into_variants(repo_id: &str, files: Vec) -> Vec = Vec::new(); let mut seen_quants: std::collections::HashSet = std::collections::HashSet::new(); @@ -521,10 +738,11 @@ fn group_into_variants(repo_id: &str, files: Vec) -> Vec) -> Vec Result> { @@ -686,7 +904,7 @@ pub async fn get_repo_gguf_variants(repo_id: &str) -> Result let model: HfApiModel = response.json().await?; let siblings = model.siblings.unwrap_or_default(); - Ok(group_into_variants(repo_id, siblings)) + group_into_variants(repo_id, siblings) } /// Fetch raw GGUF files (kept for resolve_model_spec). @@ -711,13 +929,11 @@ pub async fn get_repo_gguf_files(repo_id: &str) -> Result> { let model: HfApiModel = response.json().await?; let siblings = model.siblings.unwrap_or_default(); - let stem = model_stem_from_repo(repo_id); - let files = siblings .into_iter() .filter(|s| s.rfilename.ends_with(".gguf")) .filter(|s| !is_shard_file(&s.rfilename)) - .filter(|s| is_model_file(&s.rfilename, &stem)) + .filter(|s| !is_auxiliary_gguf_file(&s.rfilename)) .map(|s| { let quantization = parse_quantization(&s.rfilename); let download_url = build_download_url(repo_id, &s.rfilename); @@ -778,14 +994,13 @@ pub async fn resolve_model_spec_full(spec: &str) -> Result<(String, ResolvedMode let model: HfApiModel = response.json().await?; let siblings = model.siblings.unwrap_or_default(); - let stem = model_stem_from_repo(&repo_id); // Collect all GGUF files matching the quantization let matching: Vec<_> = siblings .iter() .filter(|s| { s.rfilename.ends_with(".gguf") - && is_model_file(&s.rfilename, &stem) + && !is_auxiliary_gguf_file(&s.rfilename) && parse_quantization(&s.rfilename).eq_ignore_ascii_case(&quant) }) .collect(); @@ -797,6 +1012,7 @@ pub async fn resolve_model_spec_full(spec: &str) -> Result<(String, ResolvedMode repo_id ); } + ensure_unambiguous_model_family(&repo_id, &quant, &matching)?; // Separate single files from shards let mut single_files: Vec<&HfApiSibling> = Vec::new(); @@ -809,7 +1025,9 @@ pub async fn resolve_model_spec_full(spec: &str) -> Result<(String, ResolvedMode } } - // Prefer single file if available + // Prefer single file if available, picking the same plainest-name candidate + // that `group_into_variants` advertised for this quantization. + single_files.sort_by_key(|s| (s.rfilename.len(), s.rfilename.clone())); if let Some(single) = single_files.first() { let mmproj = select_best_mmproj(&repo_id, &siblings, &single.rfilename, &quant); let file = HfGgufFile { @@ -829,6 +1047,8 @@ pub async fn resolve_model_spec_full(spec: &str) -> Result<(String, ResolvedMode )); } + let mut shard_files = select_preferred_shard_set(shard_files); + // Use shards, sorted by filename so shard 1 is first shard_files.sort_by(|a, b| a.rfilename.cmp(&b.rfilename)); @@ -1264,19 +1484,6 @@ mod tests { assert_eq!(recommend_variant(&variants, 1_000_000_000), None); } - #[test] - fn test_model_stem_from_repo() { - assert_eq!( - model_stem_from_repo("unsloth/gemma-4-26B-A4B-it-GGUF"), - "gemma-4-26b-a4b-it" - ); - assert_eq!( - model_stem_from_repo("bartowski/Llama-3.2-3B-Instruct-GGUF"), - "llama-3.2-3b-instruct" - ); - assert_eq!(model_stem_from_repo("someone/SomeModel"), "somemodel"); - } - #[test] fn hf_download_progress_init_preserves_cancelled_reservation() { let model_id = "test-cancelled-hf-progress-init"; @@ -1310,15 +1517,62 @@ mod tests { } #[test] - fn test_is_model_file() { - let stem = "gemma-3-27b-it"; - assert!(is_model_file("gemma-3-27b-it-Q4_K_M.gguf", stem)); - assert!(is_model_file( - "BF16/gemma-3-27b-it-BF16-00001-of-00002.gguf", - stem + fn test_is_auxiliary_gguf_file() { + assert!(!is_auxiliary_gguf_file("gemma-3-27b-it-Q4_K_M.gguf")); + assert!(!is_auxiliary_gguf_file( + "BF16/gemma-3-27b-it-BF16-00001-of-00002.gguf" )); - assert!(!is_model_file("mmproj-BF16.gguf", stem)); - assert!(!is_model_file("vision-encoder-Q4_K_M.gguf", stem)); + assert!(is_auxiliary_gguf_file("mmproj-BF16.gguf")); + assert!(is_auxiliary_gguf_file("gemma-4-26B-it-mmproj.gguf")); + assert!(is_auxiliary_gguf_file("vision-encoder-Q4_K_M.gguf")); + } + + #[test] + fn test_is_auxiliary_gguf_file_distinguishes_mtp_drafters_from_mtp_models() { + assert!(is_auxiliary_gguf_file( + "MTP/mtp-gemma-4-26B-A4B-it-BF16.gguf" + )); + assert!(is_auxiliary_gguf_file("mtp-Qwen3.6-35B-A3B-BF16.gguf")); + // "MTP" mid-name is part of the model name, not a drafter marker. + assert!(!is_auxiliary_gguf_file( + "Qwopus3.6-27B-Coder-MTP-Q3_K_M.gguf" + )); + assert!(!is_auxiliary_gguf_file("Ornith-1.0-9B-MTP-BF16.gguf")); + } + + #[test] + fn test_parse_quantization_tag_not_in_final_position() { + // google/gemma-4-26B-A4B-it-qat-q4_0-gguf + assert_eq!(parse_quantization("gemma-4-26B_q4_0-it.gguf"), "Q4_0"); + // google/gemma-3-27b-it-qat-q4_0-gguf + assert_eq!(parse_quantization("gemma-3-27b-it-q4_0.gguf"), "Q4_0"); + // Qwen/Qwen3-VL-30B-A3B-Instruct-GGUF + assert_eq!( + parse_quantization("Qwen3VL-30B-A3B-Instruct-F16-split-00001-of-00002.gguf"), + "F16" + ); + assert_eq!( + parse_quantization("Wan2_1-InfiniteTalk_Multi_Q4_K_M.gguf"), + "Q4_K_M" + ); + } + + #[test] + fn test_parse_quantization_prefers_longest_match() { + // "Q2_K" is a delimiter-bounded substring of "Q2_K_L"; the longer tag wins. + assert_eq!(parse_quantization("Model-Q2_K_L.gguf"), "Q2_K_L"); + assert_eq!(parse_quantization("Model-Q4_K_M.gguf"), "Q4_K_M"); + // "F16" must not match inside "BF16". + assert_eq!(parse_quantization("mmproj-BF16.gguf"), "BF16"); + } + + #[test] + fn test_parse_quantization_canonicalizes_case() { + assert_eq!(parse_quantization("Model-q4_k_m.gguf"), "Q4_K_M"); + assert_eq!( + quant_info(&parse_quantization("gemma-4-26B_q4_0-it.gguf")).quality_rank, + 42 + ); } #[test] @@ -1333,7 +1587,7 @@ mod tests { size: Some(800_000_000), }, ]; - let variants = group_into_variants("unsloth/gemma-3-27b-it-GGUF", files); + let variants = group_into_variants("unsloth/gemma-3-27b-it-GGUF", files).unwrap(); assert_eq!(variants.len(), 1); assert_eq!(variants[0].quantization, "Q4_K_M"); } @@ -1354,7 +1608,7 @@ mod tests { size: Some(4_000_000_000), }, ]; - let variants = group_into_variants("unsloth/gemma-3-27b-it-GGUF", files); + let variants = group_into_variants("unsloth/gemma-3-27b-it-GGUF", files).unwrap(); assert_eq!(variants.len(), 2); // Sorted descending by quality_rank: BF16 (91) > Q4_K_M (45) assert_eq!(variants[0].quantization, "BF16"); @@ -1364,6 +1618,111 @@ mod tests { assert!(!variants[1].sharded); } + #[test] + fn test_group_into_variants_selects_plainest_duplicate_shard_set() { + let files = vec![ + HfApiSibling { + rfilename: "Model-Q4_K_M-00001-of-00002.gguf".into(), + size: Some(2_000_000_000), + }, + HfApiSibling { + rfilename: "Model-Q4_K_M-00002-of-00002.gguf".into(), + size: Some(1_000_000_000), + }, + HfApiSibling { + rfilename: "Q4_K_M/Model-Q4_K_M-00001-of-00002.gguf".into(), + size: Some(2_500_000_000), + }, + HfApiSibling { + rfilename: "Q4_K_M/Model-Q4_K_M-00002-of-00002.gguf".into(), + size: Some(1_500_000_000), + }, + ]; + + let variants = group_into_variants("someone/Model-GGUF", files).unwrap(); + + assert_eq!(variants.len(), 1); + assert_eq!(variants[0].quantization, "Q4_K_M"); + assert_eq!(variants[0].size_bytes, 3_000_000_000); + assert_eq!(variants[0].filename, "Model-Q4_K_M-00001-of-00002.gguf"); + } + + #[test] + fn test_group_into_variants_prefers_complete_duplicate_shard_set() { + let files = vec![ + HfApiSibling { + rfilename: "Q4_K_M/Model-Q4_K_M-00001-of-00002.gguf".into(), + size: Some(2_000_000_000), + }, + HfApiSibling { + rfilename: "Model-Q4_K_M-00001-of-00002.gguf".into(), + size: Some(2_500_000_000), + }, + HfApiSibling { + rfilename: "Model-Q4_K_M-00002-of-00002.gguf".into(), + size: Some(1_500_000_000), + }, + ]; + + let variants = group_into_variants("someone/Model-GGUF", files).unwrap(); + + assert_eq!(variants.len(), 1); + assert_eq!(variants[0].size_bytes, 4_000_000_000); + assert_eq!(variants[0].filename, "Model-Q4_K_M-00001-of-00002.gguf"); + } + + #[test] + fn test_group_into_variants_rejects_same_quant_different_families() { + let files = vec![ + HfApiSibling { + rfilename: "ModelA-Q4_K_M.gguf".into(), + size: Some(4_000_000_000), + }, + HfApiSibling { + rfilename: "ModelB-Q4_K_M.gguf".into(), + size: Some(5_000_000_000), + }, + ]; + + let error = group_into_variants("someone/models-GGUF", files).unwrap_err(); + + assert!(error + .to_string() + .contains("Quantization 'Q4_K_M' is ambiguous")); + } + + #[test] + fn test_group_into_variants_keeps_different_shard_totals_separate() { + let files = vec![ + HfApiSibling { + rfilename: "Model-Q4_K_M-00001-of-00002.gguf".into(), + size: Some(2_000_000_000), + }, + HfApiSibling { + rfilename: "Model-Q4_K_M-00002-of-00002.gguf".into(), + size: Some(1_000_000_000), + }, + HfApiSibling { + rfilename: "Model-Q4_K_M-00001-of-00003.gguf".into(), + size: Some(2_500_000_000), + }, + HfApiSibling { + rfilename: "Model-Q4_K_M-00002-of-00003.gguf".into(), + size: Some(1_500_000_000), + }, + HfApiSibling { + rfilename: "Model-Q4_K_M-00003-of-00003.gguf".into(), + size: Some(500_000_000), + }, + ]; + + let variants = group_into_variants("someone/Model-GGUF", files).unwrap(); + + assert_eq!(variants.len(), 1); + assert_eq!(variants[0].size_bytes, 3_000_000_000); + assert_eq!(variants[0].filename, "Model-Q4_K_M-00001-of-00002.gguf"); + } + #[test] fn test_group_into_variants_sorted_descending() { let files = vec![ @@ -1380,7 +1739,7 @@ mod tests { size: Some(8_000_000_000), }, ]; - let variants = group_into_variants("someone/Model-GGUF", files); + let variants = group_into_variants("someone/Model-GGUF", files).unwrap(); assert_eq!(variants.len(), 3); assert_eq!(variants[0].quantization, "Q8_0"); assert_eq!(variants[1].quantization, "Q4_K_M"); @@ -1489,6 +1848,59 @@ mod tests { assert_eq!(mmproj.filename, "mmproj-F32.gguf"); } + + #[test] + fn test_select_best_mmproj_ignores_ambiguous_projector() { + let files = vec![ + HfApiSibling { + rfilename: "ModelA-Q4_K_M.gguf".into(), + size: Some(4_000), + }, + HfApiSibling { + rfilename: "ModelB-Q8_0.gguf".into(), + size: Some(8_000), + }, + HfApiSibling { + rfilename: "mmproj-BF16.gguf".into(), + size: Some(2_000), + }, + ]; + + assert!( + select_best_mmproj("someone/models-GGUF", &files, "ModelB-Q8_0.gguf", "Q8_0").is_none() + ); + } + + #[test] + fn test_select_best_mmproj_matches_named_model_family() { + let files = vec![ + HfApiSibling { + rfilename: "ModelA-Q4_K_M.gguf".into(), + size: Some(4_000), + }, + HfApiSibling { + rfilename: "ModelB-Q8_0.gguf".into(), + size: Some(8_000), + }, + HfApiSibling { + rfilename: "ModelA-mmproj-BF16.gguf".into(), + size: Some(2_000), + }, + ]; + + let mmproj = select_best_mmproj( + "someone/models-GGUF", + &files, + "ModelA-Q4_K_M.gguf", + "Q4_K_M", + ) + .unwrap(); + + assert_eq!(mmproj.filename, "ModelA-mmproj-BF16.gguf"); + assert!( + select_best_mmproj("someone/models-GGUF", &files, "ModelB-Q8_0.gguf", "Q8_0").is_none() + ); + } } async fn hf_client() -> Result { @@ -1598,8 +2010,9 @@ async fn model_info_to_local_model_info( downloads_hint: Option, ) -> Result> { let repo_id = info.id.clone(); - let mut variants: Vec = get_repo_gguf_variants(&repo_id) - .await + let gguf_variants = get_repo_gguf_variants(&repo_id).await; + let mut variants: Vec = gguf_variants + .as_deref() .unwrap_or_default() .iter() .map(|variant| variant.to_model_variant(&repo_id)) @@ -1613,6 +2026,23 @@ async fn model_info_to_local_model_info( } if variants.is_empty() { + drop(gguf_variants?); + + let unusable: Vec<&str> = info + .siblings + .as_deref() + .unwrap_or(&[]) + .iter() + .map(|s| s.rfilename.as_str()) + .filter(|name| name.ends_with(".gguf")) + .collect(); + if !unusable.is_empty() { + tracing::warn!( + repo_id, + files = ?unusable, + "Dropping repo from results: no GGUF file yielded a recognizable quantization" + ); + } return Ok(None); } diff --git a/crates/goose-local-inference/tests/gguf_filename_parsing.rs b/crates/goose-local-inference/tests/gguf_filename_parsing.rs new file mode 100644 index 000000000..27ae071d7 --- /dev/null +++ b/crates/goose-local-inference/tests/gguf_filename_parsing.rs @@ -0,0 +1,132 @@ +//! Filename shapes captured from real HuggingFace repos, to keep GGUF discovery +//! working across publishers that disagree about where the quantization tag goes. + +use goose_local_inference::hf_models::{is_auxiliary_gguf_file, parse_quantization_from_filename}; + +fn quant(filename: &str) -> String { + parse_quantization_from_filename(filename) +} + +#[test] +fn parses_conventional_trailing_quant_tags() { + assert_eq!(quant("Meta-Llama-3.1-8B-Instruct-Q4_K_M.gguf"), "Q4_K_M"); + assert_eq!(quant("Qwen3.6-27B-IQ4_NL.gguf"), "IQ4_NL"); + assert_eq!(quant("Qwen3.6-35B-A3B-MXFP4_MOE.gguf"), "MXFP4_MOE"); + assert_eq!(quant("Model-UD-IQ1_M.gguf"), "IQ1_M"); + assert_eq!(quant("Model-UD-Q4_K_XL.gguf"), "Q4_K_XL"); +} + +#[test] +fn parses_quant_tags_buried_mid_filename() { + // google/gemma-4-*-it-qat-q4_0-gguf publish the tag before the "-it" suffix. + assert_eq!(quant("gemma-4-26B_q4_0-it.gguf"), "Q4_0"); + assert_eq!(quant("gemma-4-31B_q4_0-it.gguf"), "Q4_0"); + assert_eq!(quant("gemma-4-E4B_q4_0-it.gguf"), "Q4_0"); + // google/gemma-3-*-it-qat-q4_0-gguf drop the "qat" segment the repo name carries. + assert_eq!(quant("gemma-3-27b-it-q4_0.gguf"), "Q4_0"); + // Qwen/Qwen3-VL-30B-A3B-Instruct-GGUF interleaves a "split" marker. + assert_eq!( + quant("Qwen3VL-30B-A3B-Instruct-F16-split-00001-of-00002.gguf"), + "F16" + ); +} + +#[test] +fn canonicalizes_quant_case_so_lookups_match_the_quant_table() { + assert_eq!(quant("Model-q4_k_m.gguf"), "Q4_K_M"); + assert_eq!(quant("Gemma-3-1B-Heretic_Q3_k_m.gguf"), "Q3_K_M"); +} + +#[test] +fn strips_shard_and_directory_prefixes() { + assert_eq!(quant("Q5_K_M/Model-Q5_K_M-00001-of-00002.gguf"), "Q5_K_M"); + assert_eq!(quant("BF16/Qwen3.6-27B-BF16-00001-of-00002.gguf"), "BF16"); +} + +#[test] +fn does_not_match_a_quant_tag_inside_a_longer_token() { + // "F16" sits inside "BF16"; "Q2_K" sits inside "Q2_K_L". + assert_eq!(quant("mmproj-BF16.gguf"), "BF16"); + assert_eq!(quant("Model-Q2_K_L.gguf"), "Q2_K_L"); + assert_eq!(quant("Model-Q3_K_XL.gguf"), "Q3_K_XL"); +} + +#[test] +fn preserves_quant_tags_that_are_absent_from_the_quant_table() { + // Matching must not degrade these to the shorter table entry inside them. + assert_eq!(quant("Model-Q6_K_L.gguf"), "Q6_K_L"); + assert_eq!(quant("Model-Q5_K_L.gguf"), "Q5_K_L"); + assert_eq!(quant("Model-Q4_0_4_4.gguf"), "Q4_0_4_4"); + assert_eq!(quant("Model-Q2_K_P.gguf"), "Q2_K_P"); + assert_eq!(quant("ggml-model-Q3_K.gguf"), "Q3_K"); +} + +#[test] +fn reports_unknown_when_no_quant_tag_is_present() { + assert_eq!(quant("random-name.gguf"), "unknown"); + assert_eq!(quant("imatrix.gguf"), "unknown"); + assert_eq!(quant("surya-2.gguf"), "unknown"); + // A quant tag's family prefix must be followed by a digit, so a model name + // is never mistaken for a quantization by the component scan. + assert_eq!(quant("Qwen3.6-27B-Instruct.gguf"), "unknown"); +} + +#[test] +fn keeps_named_presets_in_the_tag_position_as_variant_labels() { + // "APEX"/"OPAL" builds ship a preset name where the quant tag normally sits. + // These are the only variant these repos expose, so they stay listed. + assert_eq!(quant("Qwen3.6-35B-A3B-APEX-I-Quality.gguf"), "Quality"); + assert_eq!(quant("SIQ-1-35B-OPAL-quality.gguf"), "quality"); + assert_eq!(quant("Model.Quality.gguf"), "Quality"); + // A real quant tag elsewhere in the name still wins over the preset word. + assert_eq!(quant("Model-Q4_0-quality.gguf"), "Q4_0"); +} + +#[test] +fn parses_dot_separated_quant_tags() { + assert_eq!( + quant("DeepSeek-V3-0324.IQ1_M.gguf-00001-of-00009.gguf"), + "IQ1_M" + ); + assert_eq!(quant("model.Q4_K_M.gguf"), "Q4_K_M"); +} + +#[test] +fn treats_projectors_and_encoders_as_auxiliary() { + assert!(is_auxiliary_gguf_file("mmproj-BF16.gguf")); + assert!(is_auxiliary_gguf_file("gemma-4-26B-it-mmproj.gguf")); + assert!(is_auxiliary_gguf_file("mmproj-model-f16-27B.gguf")); + assert!(is_auxiliary_gguf_file("vision-encoder-Q4_K_M.gguf")); + assert!(is_auxiliary_gguf_file("Q4_K_M/mmproj-F32.gguf")); + assert!(is_auxiliary_gguf_file("draft/Model-Q4_K_M.gguf")); + assert!(is_auxiliary_gguf_file("adapter/Model-Q4_K_M.gguf")); + assert!(is_auxiliary_gguf_file("lora/Model-Q4_K_M.gguf")); +} + +#[test] +fn treats_mtp_drafters_as_auxiliary_but_keeps_mtp_named_models() { + // Drafters: leading "mtp-" basename, or an "MTP/" directory. + assert!(is_auxiliary_gguf_file( + "MTP/mtp-gemma-4-26B-A4B-it-BF16.gguf" + )); + assert!(is_auxiliary_gguf_file("mtp-Qwen3.6-35B-A3B-BF16.gguf")); + assert!(is_auxiliary_gguf_file("mtp-gemma-4-26B-A4B-it.gguf")); + + // Real weights whose model name happens to contain "MTP". + assert!(!is_auxiliary_gguf_file( + "Qwopus3.6-27B-Coder-MTP-Q3_K_M.gguf" + )); + assert!(!is_auxiliary_gguf_file("Ornith-1.0-9B-MTP-BF16.gguf")); + assert!(!is_auxiliary_gguf_file( + "Qwen3.6-27B-Fable-Fus-711-NEO-MAX-MTP-IQ4_XS.gguf" + )); +} + +#[test] +fn keeps_ordinary_model_files() { + assert!(!is_auxiliary_gguf_file("gemma-4-26B_q4_0-it.gguf")); + assert!(!is_auxiliary_gguf_file("gemma-3-27b-it-Q4_K_M.gguf")); + assert!(!is_auxiliary_gguf_file( + "BF16/gemma-3-27b-it-BF16-00001-of-00002.gguf" + )); +}