compress canonical_models.json (#12385)
This commit is contained in:
Generated
+1
@@ -5506,6 +5506,7 @@ dependencies = [
|
||||
"tracing",
|
||||
"unicode-normalization",
|
||||
"uuid",
|
||||
"zstd",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
@@ -31,6 +31,7 @@ tokio = { workspace = true, features = ["sync", "time"] }
|
||||
tracing = { workspace = true }
|
||||
unicode-normalization = { version = "0.1.22", default-features = false, features = ["std"] }
|
||||
uuid = { workspace = true, features = ["v4", "std"] }
|
||||
zstd = "0.14.0"
|
||||
|
||||
[dev-dependencies]
|
||||
env-lock = { workspace = true }
|
||||
@@ -38,3 +39,6 @@ tempfile = { workspace = true }
|
||||
test-case = { workspace = true }
|
||||
tokio = { workspace = true, features = ["rt-multi-thread"] }
|
||||
tokio-stream = { workspace = true }
|
||||
|
||||
[build-dependencies]
|
||||
zstd = "0.14.0"
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
use std::fs::File;
|
||||
use std::io::{BufReader, BufWriter};
|
||||
use std::path::PathBuf;
|
||||
|
||||
fn main() {
|
||||
const CATALOG: &str = "src/canonical/data/canonical_models.json";
|
||||
println!("cargo:rerun-if-changed={CATALOG}");
|
||||
|
||||
let output = PathBuf::from(std::env::var_os("OUT_DIR").expect("OUT_DIR must be set"))
|
||||
.join("canonical_models.json.zst");
|
||||
let input = BufReader::new(File::open(CATALOG).expect("canonical model catalog must open"));
|
||||
let output = BufWriter::new(File::create(output).expect("compressed catalog must be created"));
|
||||
zstd::stream::copy_encode(input, output, 19).expect("canonical model catalog must compress");
|
||||
}
|
||||
@@ -6,10 +6,13 @@ use std::path::Path;
|
||||
|
||||
/// Cached bundled canonical model registry
|
||||
static BUNDLED_REGISTRY: Lazy<Result<CanonicalModelRegistry>> = Lazy::new(|| {
|
||||
const CANONICAL_MODELS_JSON: &str = include_str!("data/canonical_models.json");
|
||||
const CANONICAL_MODELS_ZSTD: &[u8] =
|
||||
include_bytes!(concat!(env!("OUT_DIR"), "/canonical_models.json.zst"));
|
||||
|
||||
let models: Vec<CanonicalModel> = serde_json::from_str(CANONICAL_MODELS_JSON)
|
||||
.context("Failed to parse bundled canonical models JSON")?;
|
||||
let json = zstd::decode_all(CANONICAL_MODELS_ZSTD)
|
||||
.context("Failed to decompress bundled canonical models JSON")?;
|
||||
let models: Vec<CanonicalModel> =
|
||||
serde_json::from_slice(&json).context("Failed to parse bundled canonical models JSON")?;
|
||||
|
||||
let mut registry = CanonicalModelRegistry::new();
|
||||
for model in models {
|
||||
|
||||
Reference in New Issue
Block a user