tkmind/tkmind_go
1
0
Fork 0
Code Issues Pull Requests Actions 8 Packages Projects Releases Wiki Activity

fix(local-inference): discover GGUF repos with non-standard filenames (#11005)

Browse Source 
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
Co-authored-by: Jasper Hugo <jasper@jasperhugo.com>
This commit is contained in:
Aaron Alaniz
2026-08-26 09:24:34 +00:00
committed by GitHub
parent d9d08f0e05
commit 471904312e
2 changed files with 631 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/goose-local-inference/src/hf_models.rs
+499 -69
View File
@@ -292,26 +292,49 @@ fn parse_quantization(filename: &str) -> String {
stem
};
// The quantization tag is typically the last hyphen-separated component
// that looks like a quant identifier (starts with Q, IQ, F, BF, TQ, MXFP, etc.)
// e.g. "Qwen3-Coder-Next-Q4_K_M" -> "Q4_K_M"
// "Model-UD-IQ1_M" -> "IQ1_M"
if let Some((_, candidate)) = stem.rsplit_once('-') {
if looks_like_quant(candidate) {
return candidate.to_string();
// Publishers do not agree on where the quantization tag goes. Most append it
// ("Model-Q4_K_M"), but some bury it mid-name ("gemma-4-26B_q4_0-it"). Scan
// name components right-to-left and, within each, peel `_`-separated prefixes
// so a tag fused to a neighbouring token still resolves. Whole components are
// tested before their suffixes so tags absent from QUANT_TABLE survive intact
// ("Q6_K_L" must not degrade to the "Q6_K" inside it).
for component in stem.rsplit(['-', '.']) {
let mut candidate = component;
loop {
if let Some(canonical) = canonical_quant(candidate) {
return canonical.to_string();
}
if looks_like_quant(candidate) {
return candidate.to_string();
}
match candidate.split_once('_') {
Some((_, rest)) => candidate = rest,
None => break,
}
}
}
// Fallback: try dot separator (e.g. "model.Q4_K_M")
if let Some((_, candidate)) = stem.rsplit_once('.') {
if looks_like_quant(candidate) {
return candidate.to_string();
// Some publishers put a named preset rather than a quantization in the tag
// position ("...-APEX-I-Quality"). Keep accepting a trailing Q-word so those
// repos still expose the single variant they ship.
if let Some((_, tail)) = stem.rsplit_once(['-', '.']) {
if tail.starts_with(['Q', 'q']) {
return tail.to_string();
}
}
"unknown".to_string()
}
/// Resolve a tag to its QUANT_TABLE spelling, so casing differences between
/// publishers still hit the description and quality-rank lookup.
fn canonical_quant(candidate: &str) -> Option<&'static str> {
QUANT_TABLE
.iter()
.map(|(name, _, _)| *name)
.find(|name| name.eq_ignore_ascii_case(candidate))
}
fn quant_bits(quantization: &str) -> u8 {
let digits: String = quantization
.chars()
@@ -330,12 +353,22 @@ fn mmproj_precision_preference(quantization: &str) -> u8 {
}
}
/// Shape check for quantization tags missing from QUANT_TABLE (e.g. "Q6_K_L",
/// "Q4_0_4_4"). The family prefix must be followed by a digit, so model names
/// like "Qwen3" are not mistaken for quantizations.
fn looks_like_quant(s: &str) -> bool {
let upper = s.to_uppercase();
upper.starts_with("Q")
|| upper.starts_with("IQ")
|| upper.starts_with("TQ")
|| upper.starts_with("MXFP")
let digit_follows = |prefix: &str| {
upper
.strip_prefix(prefix)
.and_then(|rest| rest.chars().next())
.is_some_and(|c| c.is_ascii_digit())
};
digit_follows("Q")
|| digit_follows("IQ")
|| digit_follows("TQ")
|| digit_follows("MXFP")
|| upper == "F16"
|| upper == "F32"
|| upper == "BF16"
@@ -346,6 +379,60 @@ fn is_shard_file(filename: &str) -> bool {
parse_shard_index(filename).is_some()
}
fn shard_set_key(filename: &str) -> Option<(&str, &str)> {
let stem = filename.trim_end_matches(".gguf");
let pos = stem.rfind("-of-")?;
let before = stem.get(..pos)?;
let total = stem.get(pos + 4..)?;
let (prefix, index) = before.rsplit_once('-')?;
if !index.is_empty() && index.chars().all(|c| c.is_ascii_digit()) {
Some((prefix, total))
} else {
None
}
}
fn is_complete_shard_set(files: &[&HfApiSibling]) -> bool {
let Some(expected_total) = files
.first()
.and_then(|file| parse_shard_total(&file.rfilename))
else {
return false;
};
if files.len() != expected_total as usize {
return false;
}
let mut indices = std::collections::HashSet::with_capacity(files.len());
files.iter().all(|file| {
parse_shard_total(&file.rfilename) == Some(expected_total)
&& parse_shard_index(&file.rfilename)
.is_some_and(|index| index > 0 && index <= expected_total && indices.insert(index))
})
}
fn select_preferred_shard_set(files: Vec<&HfApiSibling>) -> Vec<&HfApiSibling> {
let mut shard_sets: std::collections::HashMap<(&str, &str), Vec<&HfApiSibling>> =
std::collections::HashMap::new();
for file in files {
let key = shard_set_key(&file.rfilename)
.expect("files passed to select_preferred_shard_set must be shards");
shard_sets.entry(key).or_default().push(file);
}
shard_sets
.into_iter()
.min_by(|(key_a, files_a), (key_b, files_b)| {
is_complete_shard_set(files_b)
.cmp(&is_complete_shard_set(files_a))
.then_with(|| (key_a.0.len() + key_a.1.len()).cmp(&(key_b.0.len() + key_b.1.len())))
.then_with(|| key_a.cmp(key_b))
})
.map(|(_, files)| files)
.unwrap_or_default()
}
/// Parse the shard index (1-based) from a filename like "model-BF16-00001-of-00002.gguf".
fn parse_shard_index(filename: &str) -> Option<u32> {
let basename = filename.rsplit('/').next().unwrap_or(filename);
@@ -403,6 +490,94 @@ fn is_prefix(prefix: &[&str], parts: &[&str]) -> bool {
prefix.len() <= parts.len() && prefix.iter().zip(parts).all(|(a, b)| a == b)
}
fn normalize_family_fragment(value: &str, quantization: &str, projector: bool) -> String {
let mut family = value.to_ascii_lowercase();
if quantization != "unknown" {
if let Some(position) = family.rfind(quantization) {
family.replace_range(position..position + quantization.len(), "");
}
}
if projector {
if let Some(position) = family.rfind("mmproj") {
family.replace_range(position..position + "mmproj".len(), "");
}
}
family.retain(|character| character.is_ascii_alphanumeric());
family
}
fn gguf_family_key(filename: &str, projector: bool) -> String {
let basename = filename.rsplit('/').next().unwrap_or(filename);
let mut stem = basename.trim_end_matches(".gguf");
if let Some(pos) = stem.rfind("-of-") {
stem = stem
.get(..pos)
.and_then(|prefix| prefix.rsplit_once('-').map(|(prefix, _)| prefix))
.unwrap_or(stem);
}
let quantization = parse_quantization(filename).to_ascii_lowercase();
normalize_family_fragment(stem, &quantization, projector)
}
fn gguf_family_identity(filename: &str) -> String {
let quantization = parse_quantization(filename).to_ascii_lowercase();
parent_components(filename)
.into_iter()
.map(|component| normalize_family_fragment(component, &quantization, false))
.chain(std::iter::once(gguf_family_key(filename, false)))
.filter(|component| !component.is_empty())
.collect::<Vec<_>>()
.join("/")
}
fn ensure_unambiguous_model_family(
repo_id: &str,
quantization: &str,
files: &[&HfApiSibling],
) -> Result<()> {
let families: std::collections::HashSet<String> = files
.iter()
.map(|file| gguf_family_identity(&file.rfilename))
.collect();
if families.len() > 1 {
let mut filenames: Vec<&str> = files.iter().map(|file| file.rfilename.as_str()).collect();
filenames.sort_unstable();
bail!(
"Quantization '{}' is ambiguous in {} because it belongs to multiple GGUF model families: {}",
quantization,
repo_id,
filenames.join(", ")
);
}
Ok(())
}
fn mmproj_matches_model_family(
siblings: &[HfApiSibling],
mmproj_filename: &str,
model_filename: &str,
) -> bool {
let mmproj_dir = parent_components(mmproj_filename);
let model_families: std::collections::HashSet<String> = siblings
.iter()
.filter(|sibling| {
sibling.rfilename.ends_with(".gguf")
&& !is_auxiliary_gguf_file(&sibling.rfilename)
&& is_prefix(&mmproj_dir, &parent_components(&sibling.rfilename))
})
.map(|sibling| gguf_family_identity(&sibling.rfilename))
.collect();
if model_families.len() <= 1 {
return true;
}
let projector_family = gguf_family_key(mmproj_filename, true);
!projector_family.is_empty() && projector_family == gguf_family_key(model_filename, false)
}
fn select_best_mmproj(
repo_id: &str,
siblings: &[HfApiSibling],
@@ -420,7 +595,9 @@ fn select_best_mmproj(
})
.filter_map(|s| {
let mmproj_dir = parent_components(&s.rfilename);
if !is_prefix(&mmproj_dir, &model_dir) {
if !is_prefix(&mmproj_dir, &model_dir)
|| !mmproj_matches_model_family(siblings, &s.rfilename, model_filename)
{
return None;
}
@@ -451,42 +628,71 @@ fn select_best_mmproj(
})
}
/// Derive the expected model filename stem from a repo_id.
/// e.g. "unsloth/gemma-4-26B-A4B-it-GGUF" → "gemma-4-26b-a4b-it" (lowercased)
fn model_stem_from_repo(repo_id: &str) -> String {
let repo_name = repo_id.rsplit('/').next().unwrap_or(repo_id);
let stem = repo_name
.strip_suffix("-GGUF")
.or_else(|| repo_name.strip_suffix("-gguf"))
.unwrap_or(repo_name);
stem.to_lowercase()
const AUXILIARY_TOKENS: &[&str] = &["encoder", "draft", "drafter", "adapter", "lora"];
fn contains_auxiliary_token(value: &str) -> bool {
value
.split(['-', '_', '.'])
.any(|token| AUXILIARY_TOKENS.contains(&token))
}
/// Check whether a GGUF file belongs to the main model (vs auxiliary files like mmproj).
/// Matches files whose basename starts with the model stem derived from the repo name.
fn is_model_file(filename: &str, model_stem_lower: &str) -> bool {
let basename = filename.rsplit('/').next().unwrap_or(filename);
basename.to_lowercase().starts_with(model_stem_lower)
/// Check whether a GGUF file ships alongside the weights rather than being a
/// downloadable variant itself (projectors, vision encoders, speculative drafters).
///
/// Filenames are matched by exclusion rather than by a repo-name prefix, because
/// publishers routinely rename files relative to the repo — Google drops the `qat`
/// segment, unsloth drops `MTP`, Qwen writes `Qwen3VL` for `Qwen3-VL`.
///
/// MTP drafters are published either under an `MTP/` directory or with a leading
/// `mtp-` on the basename; models whose *name* contains MTP carry it mid-name and
/// are real weights, so only the leading/directory forms count as auxiliary.
pub fn is_auxiliary_gguf_file(filename: &str) -> bool {
let lowercase = filename.to_lowercase();
if lowercase.contains("mmproj") {
return true;
}
if parent_components(&lowercase)
.into_iter()
.any(|component| component == "mtp" || contains_auxiliary_token(component))
{
return true;
}
let basename = lowercase.rsplit('/').next().unwrap_or(&lowercase);
let stem = basename.trim_end_matches(".gguf");
stem.split(['-', '_', '.']).next() == Some("mtp") || contains_auxiliary_token(stem)
}
/// Collect GGUF files into quantization variants.
/// Single-file quants use the file directly.
/// Sharded quants (multiple files for one quantization) aggregate sizes and use the
/// first shard filename as the representative — the download path must handle all shards.
fn group_into_variants(repo_id: &str, files: Vec<HfApiSibling>) -> Vec<HfQuantVariant> {
fn group_into_variants(repo_id: &str, files: Vec<HfApiSibling>) -> Result<Vec<HfQuantVariant>> {
use std::collections::HashMap;
let stem = model_stem_from_repo(repo_id);
let gguf_files: Vec<_> = files
.into_iter()
.filter(|s| {
s.rfilename.ends_with(".gguf")
&& is_model_file(&s.rfilename, &stem)
&& !is_auxiliary_gguf_file(&s.rfilename)
&& parse_quantization(&s.rfilename) != "unknown"
})
.collect();
let mut files_by_quant: HashMap<String, Vec<&HfApiSibling>> = HashMap::new();
for file in &gguf_files {
files_by_quant
.entry(parse_quantization(&file.rfilename))
.or_default()
.push(file);
}
for (quantization, files) in files_by_quant {
ensure_unambiguous_model_family(repo_id, &quantization, &files)?;
}
// Separate single files from shards
let mut single_files: Vec<&HfApiSibling> = Vec::new();
let mut shard_groups: HashMap<String, Vec<&HfApiSibling>> = HashMap::new();
@@ -500,6 +706,17 @@ fn group_into_variants(repo_id: &str, files: Vec<HfApiSibling>) -> Vec<HfQuantVa
}
}
// A repo can expose the same family and quantization through multiple paths;
// keep the plainest path so each quantization appears once.
single_files.sort_by_key(|s| {
(
parse_quantization(&s.rfilename),
s.rfilename.len(),
s.rfilename.clone(),
)
});
single_files.dedup_by_key(|s| parse_quantization(&s.rfilename));
let mut variants: Vec<HfQuantVariant> = Vec::new();
let mut seen_quants: std::collections::HashSet<String> = std::collections::HashSet::new();
@@ -521,10 +738,11 @@ fn group_into_variants(repo_id: &str, files: Vec<HfApiSibling>) -> Vec<HfQuantVa
}
// Add shard-only variants (quants that only exist as sharded files)
for (quant, mut shards) in shard_groups {
for (quant, shards) in shard_groups {
if seen_quants.contains(&quant) {
continue;
}
let mut shards = select_preferred_shard_set(shards);
shards.sort_by(|a, b| a.rfilename.cmp(&b.rfilename));
let total_size: u64 = shards.iter().map(|s| s.size.unwrap_or(0)).sum();
let info = quant_info(&quant);
@@ -547,7 +765,7 @@ fn group_into_variants(repo_id: &str, files: Vec<HfApiSibling>) -> Vec<HfQuantVa
.cmp(&a.quality_rank)
.then_with(|| b.size_bytes.cmp(&a.size_bytes))
});
variants
Ok(variants)
}
pub async fn search_local_models(query: &str, limit: usize) -> Result<Vec<HfModelInfo>> {
@@ -686,7 +904,7 @@ pub async fn get_repo_gguf_variants(repo_id: &str) -> Result<Vec<HfQuantVariant>
let model: HfApiModel = response.json().await?;
let siblings = model.siblings.unwrap_or_default();
Ok(group_into_variants(repo_id, siblings))
group_into_variants(repo_id, siblings)
}
/// Fetch raw GGUF files (kept for resolve_model_spec).
@@ -711,13 +929,11 @@ pub async fn get_repo_gguf_files(repo_id: &str) -> Result<Vec<HfGgufFile>> {
let model: HfApiModel = response.json().await?;
let siblings = model.siblings.unwrap_or_default();
let stem = model_stem_from_repo(repo_id);
let files = siblings
.into_iter()
.filter(|s| s.rfilename.ends_with(".gguf"))
.filter(|s| !is_shard_file(&s.rfilename))
.filter(|s| is_model_file(&s.rfilename, &stem))
.filter(|s| !is_auxiliary_gguf_file(&s.rfilename))
.map(|s| {
let quantization = parse_quantization(&s.rfilename);
let download_url = build_download_url(repo_id, &s.rfilename);
@@ -778,14 +994,13 @@ pub async fn resolve_model_spec_full(spec: &str) -> Result<(String, ResolvedMode
let model: HfApiModel = response.json().await?;
let siblings = model.siblings.unwrap_or_default();
let stem = model_stem_from_repo(&repo_id);
// Collect all GGUF files matching the quantization
let matching: Vec<_> = siblings
.iter()
.filter(|s| {
s.rfilename.ends_with(".gguf")
&& is_model_file(&s.rfilename, &stem)
&& !is_auxiliary_gguf_file(&s.rfilename)
&& parse_quantization(&s.rfilename).eq_ignore_ascii_case(&quant)
})
.collect();
@@ -797,6 +1012,7 @@ pub async fn resolve_model_spec_full(spec: &str) -> Result<(String, ResolvedMode
repo_id
);
}
ensure_unambiguous_model_family(&repo_id, &quant, &matching)?;
// Separate single files from shards
let mut single_files: Vec<&HfApiSibling> = Vec::new();
@@ -809,7 +1025,9 @@ pub async fn resolve_model_spec_full(spec: &str) -> Result<(String, ResolvedMode
}
}
// Prefer single file if available
// Prefer single file if available, picking the same plainest-name candidate
// that `group_into_variants` advertised for this quantization.
single_files.sort_by_key(|s| (s.rfilename.len(), s.rfilename.clone()));
if let Some(single) = single_files.first() {
let mmproj = select_best_mmproj(&repo_id, &siblings, &single.rfilename, &quant);
let file = HfGgufFile {
@@ -829,6 +1047,8 @@ pub async fn resolve_model_spec_full(spec: &str) -> Result<(String, ResolvedMode
));
}
let mut shard_files = select_preferred_shard_set(shard_files);
// Use shards, sorted by filename so shard 1 is first
shard_files.sort_by(|a, b| a.rfilename.cmp(&b.rfilename));
@@ -1264,19 +1484,6 @@ mod tests {
assert_eq!(recommend_variant(&variants, 1_000_000_000), None);
}
#[test]
fn test_model_stem_from_repo() {
assert_eq!(
model_stem_from_repo("unsloth/gemma-4-26B-A4B-it-GGUF"),
"gemma-4-26b-a4b-it"
);
assert_eq!(
model_stem_from_repo("bartowski/Llama-3.2-3B-Instruct-GGUF"),
"llama-3.2-3b-instruct"
);
assert_eq!(model_stem_from_repo("someone/SomeModel"), "somemodel");
}
#[test]
fn hf_download_progress_init_preserves_cancelled_reservation() {
let model_id = "test-cancelled-hf-progress-init";
@@ -1310,15 +1517,62 @@ mod tests {
}
#[test]
fn test_is_model_file() {
let stem = "gemma-3-27b-it";
assert!(is_model_file("gemma-3-27b-it-Q4_K_M.gguf", stem));
assert!(is_model_file(
"BF16/gemma-3-27b-it-BF16-00001-of-00002.gguf",
stem
fn test_is_auxiliary_gguf_file() {
assert!(!is_auxiliary_gguf_file("gemma-3-27b-it-Q4_K_M.gguf"));
assert!(!is_auxiliary_gguf_file(
"BF16/gemma-3-27b-it-BF16-00001-of-00002.gguf"
));
assert!(!is_model_file("mmproj-BF16.gguf", stem));
assert!(!is_model_file("vision-encoder-Q4_K_M.gguf", stem));
assert!(is_auxiliary_gguf_file("mmproj-BF16.gguf"));
assert!(is_auxiliary_gguf_file("gemma-4-26B-it-mmproj.gguf"));
assert!(is_auxiliary_gguf_file("vision-encoder-Q4_K_M.gguf"));
}
#[test]
fn test_is_auxiliary_gguf_file_distinguishes_mtp_drafters_from_mtp_models() {
assert!(is_auxiliary_gguf_file(
"MTP/mtp-gemma-4-26B-A4B-it-BF16.gguf"
));
assert!(is_auxiliary_gguf_file("mtp-Qwen3.6-35B-A3B-BF16.gguf"));
// "MTP" mid-name is part of the model name, not a drafter marker.
assert!(!is_auxiliary_gguf_file(
"Qwopus3.6-27B-Coder-MTP-Q3_K_M.gguf"
));
assert!(!is_auxiliary_gguf_file("Ornith-1.0-9B-MTP-BF16.gguf"));
}
#[test]
fn test_parse_quantization_tag_not_in_final_position() {
// google/gemma-4-26B-A4B-it-qat-q4_0-gguf
assert_eq!(parse_quantization("gemma-4-26B_q4_0-it.gguf"), "Q4_0");
// google/gemma-3-27b-it-qat-q4_0-gguf
assert_eq!(parse_quantization("gemma-3-27b-it-q4_0.gguf"), "Q4_0");
// Qwen/Qwen3-VL-30B-A3B-Instruct-GGUF
assert_eq!(
parse_quantization("Qwen3VL-30B-A3B-Instruct-F16-split-00001-of-00002.gguf"),
"F16"
);
assert_eq!(
parse_quantization("Wan2_1-InfiniteTalk_Multi_Q4_K_M.gguf"),
"Q4_K_M"
);
}
#[test]
fn test_parse_quantization_prefers_longest_match() {
// "Q2_K" is a delimiter-bounded substring of "Q2_K_L"; the longer tag wins.
assert_eq!(parse_quantization("Model-Q2_K_L.gguf"), "Q2_K_L");
assert_eq!(parse_quantization("Model-Q4_K_M.gguf"), "Q4_K_M");
// "F16" must not match inside "BF16".
assert_eq!(parse_quantization("mmproj-BF16.gguf"), "BF16");
}
#[test]
fn test_parse_quantization_canonicalizes_case() {
assert_eq!(parse_quantization("Model-q4_k_m.gguf"), "Q4_K_M");
assert_eq!(
quant_info(&parse_quantization("gemma-4-26B_q4_0-it.gguf")).quality_rank,
42
);
}
#[test]
@@ -1333,7 +1587,7 @@ mod tests {
size: Some(800_000_000),
},
];
let variants = group_into_variants("unsloth/gemma-3-27b-it-GGUF", files);
let variants = group_into_variants("unsloth/gemma-3-27b-it-GGUF", files).unwrap();
assert_eq!(variants.len(), 1);
assert_eq!(variants[0].quantization, "Q4_K_M");
}
@@ -1354,7 +1608,7 @@ mod tests {
size: Some(4_000_000_000),
},
];
let variants = group_into_variants("unsloth/gemma-3-27b-it-GGUF", files);
let variants = group_into_variants("unsloth/gemma-3-27b-it-GGUF", files).unwrap();
assert_eq!(variants.len(), 2);
// Sorted descending by quality_rank: BF16 (91) > Q4_K_M (45)
assert_eq!(variants[0].quantization, "BF16");
@@ -1364,6 +1618,111 @@ mod tests {
assert!(!variants[1].sharded);
}
#[test]
fn test_group_into_variants_selects_plainest_duplicate_shard_set() {
let files = vec![
HfApiSibling {
rfilename: "Model-Q4_K_M-00001-of-00002.gguf".into(),
size: Some(2_000_000_000),
},
HfApiSibling {
rfilename: "Model-Q4_K_M-00002-of-00002.gguf".into(),
size: Some(1_000_000_000),
},
HfApiSibling {
rfilename: "Q4_K_M/Model-Q4_K_M-00001-of-00002.gguf".into(),
size: Some(2_500_000_000),
},
HfApiSibling {
rfilename: "Q4_K_M/Model-Q4_K_M-00002-of-00002.gguf".into(),
size: Some(1_500_000_000),
},
];
let variants = group_into_variants("someone/Model-GGUF", files).unwrap();
assert_eq!(variants.len(), 1);
assert_eq!(variants[0].quantization, "Q4_K_M");
assert_eq!(variants[0].size_bytes, 3_000_000_000);
assert_eq!(variants[0].filename, "Model-Q4_K_M-00001-of-00002.gguf");
}
#[test]
fn test_group_into_variants_prefers_complete_duplicate_shard_set() {
let files = vec![
HfApiSibling {
rfilename: "Q4_K_M/Model-Q4_K_M-00001-of-00002.gguf".into(),
size: Some(2_000_000_000),
},
HfApiSibling {
rfilename: "Model-Q4_K_M-00001-of-00002.gguf".into(),
size: Some(2_500_000_000),
},
HfApiSibling {
rfilename: "Model-Q4_K_M-00002-of-00002.gguf".into(),
size: Some(1_500_000_000),
},
];
let variants = group_into_variants("someone/Model-GGUF", files).unwrap();
assert_eq!(variants.len(), 1);
assert_eq!(variants[0].size_bytes, 4_000_000_000);
assert_eq!(variants[0].filename, "Model-Q4_K_M-00001-of-00002.gguf");
}
#[test]
fn test_group_into_variants_rejects_same_quant_different_families() {
let files = vec![
HfApiSibling {
rfilename: "ModelA-Q4_K_M.gguf".into(),
size: Some(4_000_000_000),
},
HfApiSibling {
rfilename: "ModelB-Q4_K_M.gguf".into(),
size: Some(5_000_000_000),
},
];
let error = group_into_variants("someone/models-GGUF", files).unwrap_err();
assert!(error
.to_string()
.contains("Quantization 'Q4_K_M' is ambiguous"));
}
#[test]
fn test_group_into_variants_keeps_different_shard_totals_separate() {
let files = vec![
HfApiSibling {
rfilename: "Model-Q4_K_M-00001-of-00002.gguf".into(),
size: Some(2_000_000_000),
},
HfApiSibling {
rfilename: "Model-Q4_K_M-00002-of-00002.gguf".into(),
size: Some(1_000_000_000),
},
HfApiSibling {
rfilename: "Model-Q4_K_M-00001-of-00003.gguf".into(),
size: Some(2_500_000_000),
},
HfApiSibling {
rfilename: "Model-Q4_K_M-00002-of-00003.gguf".into(),
size: Some(1_500_000_000),
},
HfApiSibling {
rfilename: "Model-Q4_K_M-00003-of-00003.gguf".into(),
size: Some(500_000_000),
},
];
let variants = group_into_variants("someone/Model-GGUF", files).unwrap();
assert_eq!(variants.len(), 1);
assert_eq!(variants[0].size_bytes, 3_000_000_000);
assert_eq!(variants[0].filename, "Model-Q4_K_M-00001-of-00002.gguf");
}
#[test]
fn test_group_into_variants_sorted_descending() {
let files = vec![
@@ -1380,7 +1739,7 @@ mod tests {
size: Some(8_000_000_000),
},
];
let variants = group_into_variants("someone/Model-GGUF", files);
let variants = group_into_variants("someone/Model-GGUF", files).unwrap();
assert_eq!(variants.len(), 3);
assert_eq!(variants[0].quantization, "Q8_0");
assert_eq!(variants[1].quantization, "Q4_K_M");
@@ -1489,6 +1848,59 @@ mod tests {
assert_eq!(mmproj.filename, "mmproj-F32.gguf");
}
#[test]
fn test_select_best_mmproj_ignores_ambiguous_projector() {
let files = vec![
HfApiSibling {
rfilename: "ModelA-Q4_K_M.gguf".into(),
size: Some(4_000),
},
HfApiSibling {
rfilename: "ModelB-Q8_0.gguf".into(),
size: Some(8_000),
},
HfApiSibling {
rfilename: "mmproj-BF16.gguf".into(),
size: Some(2_000),
},
];
assert!(
select_best_mmproj("someone/models-GGUF", &files, "ModelB-Q8_0.gguf", "Q8_0").is_none()
);
}
#[test]
fn test_select_best_mmproj_matches_named_model_family() {
let files = vec![
HfApiSibling {
rfilename: "ModelA-Q4_K_M.gguf".into(),
size: Some(4_000),
},
HfApiSibling {
rfilename: "ModelB-Q8_0.gguf".into(),
size: Some(8_000),
},
HfApiSibling {
rfilename: "ModelA-mmproj-BF16.gguf".into(),
size: Some(2_000),
},
];
let mmproj = select_best_mmproj(
"someone/models-GGUF",
&files,
"ModelA-Q4_K_M.gguf",
"Q4_K_M",
)
.unwrap();
assert_eq!(mmproj.filename, "ModelA-mmproj-BF16.gguf");
assert!(
select_best_mmproj("someone/models-GGUF", &files, "ModelB-Q8_0.gguf", "Q8_0").is_none()
);
}
}
async fn hf_client() -> Result<HFClient> {
@@ -1598,8 +2010,9 @@ async fn model_info_to_local_model_info(
downloads_hint: Option<u64>,
) -> Result<Option<HfModelInfo>> {
let repo_id = info.id.clone();
let mut variants: Vec<HfModelVariant> = get_repo_gguf_variants(&repo_id)
.await
let gguf_variants = get_repo_gguf_variants(&repo_id).await;
let mut variants: Vec<HfModelVariant> = gguf_variants
.as_deref()
.unwrap_or_default()
.iter()
.map(|variant| variant.to_model_variant(&repo_id))
@@ -1613,6 +2026,23 @@ async fn model_info_to_local_model_info(
}
if variants.is_empty() {
drop(gguf_variants?);
let unusable: Vec<&str> = info
.siblings
.as_deref()
.unwrap_or(&[])
.iter()
.map(|s| s.rfilename.as_str())
.filter(|name| name.ends_with(".gguf"))
.collect();
if !unusable.is_empty() {
tracing::warn!(
repo_id,
files = ?unusable,
"Dropping repo from results: no GGUF file yielded a recognizable quantization"
);
}
return Ok(None);
}
crates/goose-local-inference/tests/gguf_filename_parsing.rs
+132
View File
@@ -0,0 +1,132 @@
//! Filename shapes captured from real HuggingFace repos, to keep GGUF discovery
//! working across publishers that disagree about where the quantization tag goes.
use goose_local_inference::hf_models::{is_auxiliary_gguf_file, parse_quantization_from_filename};
fn quant(filename: &str) -> String {
parse_quantization_from_filename(filename)
}
#[test]
fn parses_conventional_trailing_quant_tags() {
assert_eq!(quant("Meta-Llama-3.1-8B-Instruct-Q4_K_M.gguf"), "Q4_K_M");
assert_eq!(quant("Qwen3.6-27B-IQ4_NL.gguf"), "IQ4_NL");
assert_eq!(quant("Qwen3.6-35B-A3B-MXFP4_MOE.gguf"), "MXFP4_MOE");
assert_eq!(quant("Model-UD-IQ1_M.gguf"), "IQ1_M");
assert_eq!(quant("Model-UD-Q4_K_XL.gguf"), "Q4_K_XL");
}
#[test]
fn parses_quant_tags_buried_mid_filename() {
// google/gemma-4-*-it-qat-q4_0-gguf publish the tag before the "-it" suffix.
assert_eq!(quant("gemma-4-26B_q4_0-it.gguf"), "Q4_0");
assert_eq!(quant("gemma-4-31B_q4_0-it.gguf"), "Q4_0");
assert_eq!(quant("gemma-4-E4B_q4_0-it.gguf"), "Q4_0");
// google/gemma-3-*-it-qat-q4_0-gguf drop the "qat" segment the repo name carries.
assert_eq!(quant("gemma-3-27b-it-q4_0.gguf"), "Q4_0");
// Qwen/Qwen3-VL-30B-A3B-Instruct-GGUF interleaves a "split" marker.
assert_eq!(
quant("Qwen3VL-30B-A3B-Instruct-F16-split-00001-of-00002.gguf"),
"F16"
);
}
#[test]
fn canonicalizes_quant_case_so_lookups_match_the_quant_table() {
assert_eq!(quant("Model-q4_k_m.gguf"), "Q4_K_M");
assert_eq!(quant("Gemma-3-1B-Heretic_Q3_k_m.gguf"), "Q3_K_M");
}
#[test]
fn strips_shard_and_directory_prefixes() {
assert_eq!(quant("Q5_K_M/Model-Q5_K_M-00001-of-00002.gguf"), "Q5_K_M");
assert_eq!(quant("BF16/Qwen3.6-27B-BF16-00001-of-00002.gguf"), "BF16");
}
#[test]
fn does_not_match_a_quant_tag_inside_a_longer_token() {
// "F16" sits inside "BF16"; "Q2_K" sits inside "Q2_K_L".
assert_eq!(quant("mmproj-BF16.gguf"), "BF16");
assert_eq!(quant("Model-Q2_K_L.gguf"), "Q2_K_L");
assert_eq!(quant("Model-Q3_K_XL.gguf"), "Q3_K_XL");
}
#[test]
fn preserves_quant_tags_that_are_absent_from_the_quant_table() {
// Matching must not degrade these to the shorter table entry inside them.
assert_eq!(quant("Model-Q6_K_L.gguf"), "Q6_K_L");
assert_eq!(quant("Model-Q5_K_L.gguf"), "Q5_K_L");
assert_eq!(quant("Model-Q4_0_4_4.gguf"), "Q4_0_4_4");
assert_eq!(quant("Model-Q2_K_P.gguf"), "Q2_K_P");
assert_eq!(quant("ggml-model-Q3_K.gguf"), "Q3_K");
}
#[test]
fn reports_unknown_when_no_quant_tag_is_present() {
assert_eq!(quant("random-name.gguf"), "unknown");
assert_eq!(quant("imatrix.gguf"), "unknown");
assert_eq!(quant("surya-2.gguf"), "unknown");
// A quant tag's family prefix must be followed by a digit, so a model name
// is never mistaken for a quantization by the component scan.
assert_eq!(quant("Qwen3.6-27B-Instruct.gguf"), "unknown");
}
#[test]
fn keeps_named_presets_in_the_tag_position_as_variant_labels() {
// "APEX"/"OPAL" builds ship a preset name where the quant tag normally sits.
// These are the only variant these repos expose, so they stay listed.
assert_eq!(quant("Qwen3.6-35B-A3B-APEX-I-Quality.gguf"), "Quality");
assert_eq!(quant("SIQ-1-35B-OPAL-quality.gguf"), "quality");
assert_eq!(quant("Model.Quality.gguf"), "Quality");
// A real quant tag elsewhere in the name still wins over the preset word.
assert_eq!(quant("Model-Q4_0-quality.gguf"), "Q4_0");
}
#[test]
fn parses_dot_separated_quant_tags() {
assert_eq!(
quant("DeepSeek-V3-0324.IQ1_M.gguf-00001-of-00009.gguf"),
"IQ1_M"
);
assert_eq!(quant("model.Q4_K_M.gguf"), "Q4_K_M");
}
#[test]
fn treats_projectors_and_encoders_as_auxiliary() {
assert!(is_auxiliary_gguf_file("mmproj-BF16.gguf"));
assert!(is_auxiliary_gguf_file("gemma-4-26B-it-mmproj.gguf"));
assert!(is_auxiliary_gguf_file("mmproj-model-f16-27B.gguf"));
assert!(is_auxiliary_gguf_file("vision-encoder-Q4_K_M.gguf"));
assert!(is_auxiliary_gguf_file("Q4_K_M/mmproj-F32.gguf"));
assert!(is_auxiliary_gguf_file("draft/Model-Q4_K_M.gguf"));
assert!(is_auxiliary_gguf_file("adapter/Model-Q4_K_M.gguf"));
assert!(is_auxiliary_gguf_file("lora/Model-Q4_K_M.gguf"));
}
#[test]
fn treats_mtp_drafters_as_auxiliary_but_keeps_mtp_named_models() {
// Drafters: leading "mtp-" basename, or an "MTP/" directory.
assert!(is_auxiliary_gguf_file(
"MTP/mtp-gemma-4-26B-A4B-it-BF16.gguf"
));
assert!(is_auxiliary_gguf_file("mtp-Qwen3.6-35B-A3B-BF16.gguf"));
assert!(is_auxiliary_gguf_file("mtp-gemma-4-26B-A4B-it.gguf"));
// Real weights whose model name happens to contain "MTP".
assert!(!is_auxiliary_gguf_file(
"Qwopus3.6-27B-Coder-MTP-Q3_K_M.gguf"
));
assert!(!is_auxiliary_gguf_file("Ornith-1.0-9B-MTP-BF16.gguf"));
assert!(!is_auxiliary_gguf_file(
"Qwen3.6-27B-Fable-Fus-711-NEO-MAX-MTP-IQ4_XS.gguf"
));
}
#[test]
fn keeps_ordinary_model_files() {
assert!(!is_auxiliary_gguf_file("gemma-4-26B_q4_0-it.gguf"));
assert!(!is_auxiliary_gguf_file("gemma-3-27b-it-Q4_K_M.gguf"));
assert!(!is_auxiliary_gguf_file(
"BF16/gemma-3-27b-it-BF16-00001-of-00002.gguf"
));
}