compress canonical_models.json (#12385)

This commit is contained in:
Jack Amadeo
2026-09-22 13:37:09 +00:00
committed by GitHub
parent bfbbf44631
commit 48ce701f84
4 changed files with 25 additions and 3 deletions
Generated
+1
View File
@@ -5506,6 +5506,7 @@ dependencies = [
"tracing",
"unicode-normalization",
"uuid",
"zstd",
]
[[package]]
+4
View File
@@ -31,6 +31,7 @@ tokio = { workspace = true, features = ["sync", "time"] }
tracing = { workspace = true }
unicode-normalization = { version = "0.1.22", default-features = false, features = ["std"] }
uuid = { workspace = true, features = ["v4", "std"] }
zstd = "0.14.0"
[dev-dependencies]
env-lock = { workspace = true }
@@ -38,3 +39,6 @@ tempfile = { workspace = true }
test-case = { workspace = true }
tokio = { workspace = true, features = ["rt-multi-thread"] }
tokio-stream = { workspace = true }
[build-dependencies]
zstd = "0.14.0"
+14
View File
@@ -0,0 +1,14 @@
use std::fs::File;
use std::io::{BufReader, BufWriter};
use std::path::PathBuf;
fn main() {
const CATALOG: &str = "src/canonical/data/canonical_models.json";
println!("cargo:rerun-if-changed={CATALOG}");
let output = PathBuf::from(std::env::var_os("OUT_DIR").expect("OUT_DIR must be set"))
.join("canonical_models.json.zst");
let input = BufReader::new(File::open(CATALOG).expect("canonical model catalog must open"));
let output = BufWriter::new(File::create(output).expect("compressed catalog must be created"));
zstd::stream::copy_encode(input, output, 19).expect("canonical model catalog must compress");
}
@@ -6,10 +6,13 @@ use std::path::Path;
/// Cached bundled canonical model registry
static BUNDLED_REGISTRY: Lazy<Result<CanonicalModelRegistry>> = Lazy::new(|| {
const CANONICAL_MODELS_JSON: &str = include_str!("data/canonical_models.json");
const CANONICAL_MODELS_ZSTD: &[u8] =
include_bytes!(concat!(env!("OUT_DIR"), "/canonical_models.json.zst"));
let models: Vec<CanonicalModel> = serde_json::from_str(CANONICAL_MODELS_JSON)
.context("Failed to parse bundled canonical models JSON")?;
let json = zstd::decode_all(CANONICAL_MODELS_ZSTD)
.context("Failed to decompress bundled canonical models JSON")?;
let models: Vec<CanonicalModel> =
serde_json::from_slice(&json).context("Failed to parse bundled canonical models JSON")?;
let mut registry = CanonicalModelRegistry::new();
for model in models {