diff --git a/Cargo.lock b/Cargo.lock index b6b1f3e23..bd6328d05 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5506,6 +5506,7 @@ dependencies = [ "tracing", "unicode-normalization", "uuid", + "zstd", ] [[package]] diff --git a/crates/goose-provider-types/Cargo.toml b/crates/goose-provider-types/Cargo.toml index e4f550d0e..8a2601bf5 100644 --- a/crates/goose-provider-types/Cargo.toml +++ b/crates/goose-provider-types/Cargo.toml @@ -31,6 +31,7 @@ tokio = { workspace = true, features = ["sync", "time"] } tracing = { workspace = true } unicode-normalization = { version = "0.1.22", default-features = false, features = ["std"] } uuid = { workspace = true, features = ["v4", "std"] } +zstd = "0.14.0" [dev-dependencies] env-lock = { workspace = true } @@ -38,3 +39,6 @@ tempfile = { workspace = true } test-case = { workspace = true } tokio = { workspace = true, features = ["rt-multi-thread"] } tokio-stream = { workspace = true } + +[build-dependencies] +zstd = "0.14.0" diff --git a/crates/goose-provider-types/build.rs b/crates/goose-provider-types/build.rs new file mode 100644 index 000000000..61c0f8e55 --- /dev/null +++ b/crates/goose-provider-types/build.rs @@ -0,0 +1,14 @@ +use std::fs::File; +use std::io::{BufReader, BufWriter}; +use std::path::PathBuf; + +fn main() { + const CATALOG: &str = "src/canonical/data/canonical_models.json"; + println!("cargo:rerun-if-changed={CATALOG}"); + + let output = PathBuf::from(std::env::var_os("OUT_DIR").expect("OUT_DIR must be set")) + .join("canonical_models.json.zst"); + let input = BufReader::new(File::open(CATALOG).expect("canonical model catalog must open")); + let output = BufWriter::new(File::create(output).expect("compressed catalog must be created")); + zstd::stream::copy_encode(input, output, 19).expect("canonical model catalog must compress"); +} diff --git a/crates/goose-provider-types/src/canonical/registry.rs b/crates/goose-provider-types/src/canonical/registry.rs index da3ed24e3..4e52fd651 100644 --- a/crates/goose-provider-types/src/canonical/registry.rs +++ b/crates/goose-provider-types/src/canonical/registry.rs @@ -6,10 +6,13 @@ use std::path::Path; /// Cached bundled canonical model registry static BUNDLED_REGISTRY: Lazy> = Lazy::new(|| { - const CANONICAL_MODELS_JSON: &str = include_str!("data/canonical_models.json"); + const CANONICAL_MODELS_ZSTD: &[u8] = + include_bytes!(concat!(env!("OUT_DIR"), "/canonical_models.json.zst")); - let models: Vec = serde_json::from_str(CANONICAL_MODELS_JSON) - .context("Failed to parse bundled canonical models JSON")?; + let json = zstd::decode_all(CANONICAL_MODELS_ZSTD) + .context("Failed to decompress bundled canonical models JSON")?; + let models: Vec = + serde_json::from_slice(&json).context("Failed to parse bundled canonical models JSON")?; let mut registry = CanonicalModelRegistry::new(); for model in models {