diff --git a/Cargo.lock b/Cargo.lock index 4c5a788a3..838f7dc42 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11216,9 +11216,9 @@ checksum = "a7973cce6668464ea31f176d85b13c7ab3bba2cb3b77a2ed26abd7801688010a" [[package]] name = "symphonia" -version = "0.5.5" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5773a4c030a19d9bfaa090f49746ff35c75dfddfa700df7a5939d5e076a57039" +checksum = "a7edef6a96b696d4e0cab5ee9ebb7ca155ed95f30a6b45bbb8b97d2727f02424" dependencies = [ "lazy_static", "symphonia-bundle-mp3", @@ -11236,32 +11236,32 @@ dependencies = [ [[package]] name = "symphonia-bundle-mp3" -version = "0.5.5" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4872dd6bb56bf5eac799e3e957aa1981086c3e613b27e0ac23b176054f7c57ed" +checksum = "98ea5ffc8716bff677dfb3b01b420c7b758de901a72b8c330bf2040ab74b4add" dependencies = [ "lazy_static", "log", "symphonia-core", - "symphonia-metadata", ] [[package]] name = "symphonia-codec-aac" -version = "0.5.5" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c263845aa86881416849c1729a54c7f55164f8b96111dba59de46849e73a790" +checksum = "f5bf8e39552d34a3c4c98333370e62f48c92456d2e814f273b5c3ad7c4a5f45c" dependencies = [ "lazy_static", "log", + "symphonia-common", "symphonia-core", ] [[package]] name = "symphonia-codec-adpcm" -version = "0.5.5" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dddc50e2bbea4cfe027441eece77c46b9f319748605ab8f3443350129ddd07f" +checksum = "445932ecb0c59362fde9c082dd63a75380650c5077fdb8b80b8cac850442a74c" dependencies = [ "log", "symphonia-core", @@ -11269,19 +11269,20 @@ dependencies = [ [[package]] name = "symphonia-codec-alac" -version = "0.5.5" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8413fa754942ac16a73634c9dfd1500ed5c61430956b33728567f667fdd393ab" +checksum = "920a78f96f3cf62932d0c497959c0cc2100ef36b03c7ca1417b260f432a482d8" dependencies = [ "log", + "symphonia-common", "symphonia-core", ] [[package]] name = "symphonia-codec-pcm" -version = "0.5.5" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e89d716c01541ad3ebe7c91ce4c8d38a7cf266a3f7b2f090b108fb0cb031d95" +checksum = "e04ba75686acbe43542fdd374571195f0530c0b7785ca25cc6840e9c6c4b6eea" dependencies = [ "log", "symphonia-core", @@ -11289,59 +11290,69 @@ dependencies = [ [[package]] name = "symphonia-codec-vorbis" -version = "0.5.5" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f025837c309cd69ffef572750b4a2257b59552c5399a5e49707cc5b1b85d1c73" +checksum = "73d90b4fcf796137cc683c538282804ff9629f8ad9dbfd881fcbba331ac4e986" +dependencies = [ + "log", + "symphonia-common", + "symphonia-core", +] + +[[package]] +name = "symphonia-common" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acc3fcc18ec9b8cdd48614e259c4cf0d27b71d41e5d9b120b42c5adab12d7c4" dependencies = [ "log", "symphonia-core", - "symphonia-utils-xiph", + "symphonia-metadata", ] [[package]] name = "symphonia-core" -version = "0.5.5" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea00cc4f79b7f6bb7ff87eddc065a1066f3a43fe1875979056672c9ef948c2af" +checksum = "01c412864d599d4750d0c3d684d7e093ec05e5309681ef5252cc1096a437f6e0" dependencies = [ - "arrayvec", - "bitflags 1.3.2", + "bitflags 2.13.0", "bytemuck", "lazy_static", "log", + "num-complex", + "smallvec", ] [[package]] name = "symphonia-format-isomp4" -version = "0.5.5" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "243739585d11f81daf8dac8d9f3d18cc7898f6c09a259675fc364b382c30e0a5" +checksum = "0e681a70e1870d34e02abf1dbc51e4267c3f1827801474e8870be8c689fc4dc3" dependencies = [ - "encoding_rs", "log", + "symphonia-common", "symphonia-core", "symphonia-metadata", - "symphonia-utils-xiph", ] [[package]] name = "symphonia-format-mkv" -version = "0.5.5" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "122d786d2c43a49beb6f397551b4a050d8229eaa54c7ddf9ee4b98899b8742d0" +checksum = "d015c5c0558864665894b3f4cbd95e10abb01b9c868e751c72670f326a56360e" dependencies = [ "lazy_static", "log", + "symphonia-common", "symphonia-core", - "symphonia-metadata", - "symphonia-utils-xiph", ] [[package]] name = "symphonia-format-riff" -version = "0.5.5" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2d7c3df0e7d94efb68401d81906eae73c02b40d5ec1a141962c592d0f11a96f" +checksum = "1ff70929083a8c1a5f6cd7c904b6071c7914ad04739b510c2f7239dfc9b7dabe" dependencies = [ "extended", "log", @@ -11351,26 +11362,17 @@ dependencies = [ [[package]] name = "symphonia-metadata" -version = "0.5.5" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36306ff42b9ffe6e5afc99d49e121e0bd62fe79b9db7b9681d48e29fa19e6b16" +checksum = "83713a97705d77bdef7cdbc0768fd6e5a54e4cd7e48d60a806ae85639e2c87c6" dependencies = [ - "encoding_rs", "lazy_static", "log", + "regex-lite", + "smallvec", "symphonia-core", ] -[[package]] -name = "symphonia-utils-xiph" -version = "0.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee27c85ab799a338446b68eec77abf42e1a6f1bb490656e121c6e27bfbab9f16" -dependencies = [ - "symphonia-core", - "symphonia-metadata", -] - [[package]] name = "syn" version = "1.0.109" diff --git a/crates/goose/Cargo.toml b/crates/goose/Cargo.toml index f60a1013d..816c65367 100644 --- a/crates/goose/Cargo.toml +++ b/crates/goose/Cargo.toml @@ -185,7 +185,7 @@ candle-nn = { workspace = true, optional = true } candle-transformers = { version = "0.11", default-features = false, optional = true } byteorder = { version = "1.5", default-features = false, features = ["std"], optional = true } tokenizers = { version = "0.23", default-features = false, features = ["onig"], optional = true } -symphonia = { version = "0.5", default-features = false, features = ["aac", "adpcm", "alac", "isomp4", "mkv", "mp3", "pcm", "vorbis", "wav"], optional = true } +symphonia = { version = "0.6", default-features = false, features = ["aac", "adpcm", "alac", "isomp4", "mkv", "mp3", "pcm", "vorbis", "wav"], optional = true } rubato = { version = "5.0", default-features = false, optional = true } sys-info = { version = "0.9", default-features = false } diff --git a/crates/goose/src/dictation/whisper.rs b/crates/goose/src/dictation/whisper.rs index d17e9fcfe..a4110db0d 100644 --- a/crates/goose/src/dictation/whisper.rs +++ b/crates/goose/src/dictation/whisper.rs @@ -19,12 +19,12 @@ use candle_transformers::models::whisper::{self as m, audio, Config, N_FRAMES}; use serde::{Deserialize, Serialize}; use std::io::Cursor; use std::path::{Path, PathBuf}; -use symphonia::core::audio::{AudioBufferRef, Layout, Signal}; -use symphonia::core::codecs::DecoderOptions; -use symphonia::core::formats::FormatOptions; +use symphonia::core::audio::GenericAudioBufferRef; +use symphonia::core::codecs::audio::AudioDecoderOptions; +use symphonia::core::formats::probe::Hint; +use symphonia::core::formats::{FormatOptions, TrackType}; use symphonia::core::io::MediaSourceStream; use symphonia::core::meta::MetadataOptions; -use symphonia::core::probe::Hint; use tokenizers::Tokenizer; // Common suppress tokens for all Whisper models @@ -935,42 +935,40 @@ fn decode_audio_simple(audio_data: &[u8]) -> Result> { let hint = Hint::new(); - let probed = symphonia::default::get_probe() - .format( + let mut format = symphonia::default::get_probe() + .probe( &hint, mss, - &FormatOptions::default(), - &MetadataOptions::default(), + FormatOptions::default(), + MetadataOptions::default(), ) .context("Failed to probe audio format - unsupported format")?; - let mut format = probed.format; - let track = format - .default_track() + .default_track(TrackType::Audio) .context("No default audio track found")?; - let sample_rate = track + let codec_params = track .codec_params + .as_ref() + .and_then(|params| params.audio()) + .context("No audio codec parameters found")? + .clone(); + + let sample_rate = codec_params .sample_rate .context("No sample rate in audio track")?; - let channels = if let Some(ch) = track.codec_params.channels { - ch.count() - } else if let Some(layout) = track.codec_params.channel_layout { - match layout { - Layout::Mono => 1, - Layout::Stereo => 2, - _ => 1, - } - } else { - anyhow::bail!("No channel information in audio track (neither channels nor channel_layout)") - }; + let channels = codec_params + .channels + .as_ref() + .map(|channels| channels.count()) + .context("No channel information in audio track")?; tracing::debug!(sample_rate, channels, "audio format detected"); let mut decoder = symphonia::default::get_codecs() - .make(&track.codec_params, &DecoderOptions::default()) + .make_audio_decoder(&codec_params, &AudioDecoderOptions::default()) .context("Failed to create audio decoder - please ensure browser sends WAV format audio")?; let mut pcm_data = Vec::new(); @@ -978,7 +976,8 @@ fn decode_audio_simple(audio_data: &[u8]) -> Result> { loop { let packet = match format.next_packet() { - Ok(packet) => packet, + Ok(Some(packet)) => packet, + Ok(None) => break, Err(symphonia::core::errors::Error::IoError(e)) if e.kind() == std::io::ErrorKind::UnexpectedEof => { @@ -1040,45 +1039,9 @@ fn decode_audio_simple(audio_data: &[u8]) -> Result> { Ok(resampled) } -fn audio_buffer_to_f32(buffer: &AudioBufferRef) -> Vec { - let num_channels = buffer.spec().channels.count(); - let num_frames = buffer.frames(); - let mut samples = Vec::with_capacity(num_frames * num_channels); - - match buffer { - AudioBufferRef::F32(buf) => { - for frame_idx in 0..num_frames { - for ch_idx in 0..num_channels { - samples.push(buf.chan(ch_idx)[frame_idx]); - } - } - } - AudioBufferRef::S16(buf) => { - for frame_idx in 0..num_frames { - for ch_idx in 0..num_channels { - samples.push(buf.chan(ch_idx)[frame_idx] as f32 / 32768.0); - } - } - } - AudioBufferRef::S32(buf) => { - for frame_idx in 0..num_frames { - for ch_idx in 0..num_channels { - samples.push(buf.chan(ch_idx)[frame_idx] as f32 / 2147483648.0); - } - } - } - AudioBufferRef::F64(buf) => { - for frame_idx in 0..num_frames { - for ch_idx in 0..num_channels { - samples.push(buf.chan(ch_idx)[frame_idx] as f32); - } - } - } - _ => { - tracing::warn!("Unsupported audio buffer format, returning silence"); - } - } - +fn audio_buffer_to_f32(buffer: &GenericAudioBufferRef<'_>) -> Vec { + let mut samples = Vec::with_capacity(buffer.samples_interleaved()); + buffer.copy_to_vec_interleaved(&mut samples); samples } @@ -1152,6 +1115,35 @@ mod tests { const TS: u32 = 50364; // A timestamp token for tests + #[test] + fn decodes_pcm_wav() { + let input = [-32768i16, 0, 32767]; + let data_size = (input.len() * std::mem::size_of::()) as u32; + let mut wav = Vec::new(); + wav.extend_from_slice(b"RIFF"); + wav.extend_from_slice(&(36 + data_size).to_le_bytes()); + wav.extend_from_slice(b"WAVEfmt "); + wav.extend_from_slice(&16u32.to_le_bytes()); + wav.extend_from_slice(&1u16.to_le_bytes()); + wav.extend_from_slice(&1u16.to_le_bytes()); + wav.extend_from_slice(&16000u32.to_le_bytes()); + wav.extend_from_slice(&32000u32.to_le_bytes()); + wav.extend_from_slice(&2u16.to_le_bytes()); + wav.extend_from_slice(&16u16.to_le_bytes()); + wav.extend_from_slice(b"data"); + wav.extend_from_slice(&data_size.to_le_bytes()); + for sample in input { + wav.extend_from_slice(&sample.to_le_bytes()); + } + + let decoded = decode_audio_simple(&wav).unwrap(); + + assert_eq!(decoded.len(), input.len()); + assert_eq!(decoded[0], -1.0); + assert_eq!(decoded[1], 0.0); + assert!((decoded[2] - 32767.0 / 32768.0).abs() < f32::EPSILON); + } + #[test_case(None, ENGLISH_LANGUAGE_TOKEN ; "unset falls back to english")] #[test_case(Some("en"), ENGLISH_LANGUAGE_TOKEN ; "english")] #[test_case(Some("de"), 50261 ; "german")]