fix(dictation): bound local audio resampling (#11379)
This commit is contained in:
@@ -19,7 +19,9 @@ use candle_transformers::models::whisper::{self as m, audio, Config, N_FRAMES};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::io::Cursor;
|
||||
use std::path::{Path, PathBuf};
|
||||
use symphonia::core::audio::GenericAudioBufferRef;
|
||||
use symphonia::core::audio::conv::FromSample;
|
||||
use symphonia::core::audio::sample::Sample;
|
||||
use symphonia::core::audio::{Audio, AudioBuffer, GenericAudioBufferRef};
|
||||
use symphonia::core::codecs::audio::AudioDecoderOptions;
|
||||
use symphonia::core::formats::probe::Hint;
|
||||
use symphonia::core::formats::{FormatOptions, TrackType};
|
||||
@@ -43,6 +45,9 @@ const TRANSCRIBE_TOKEN: u32 = 50359;
|
||||
const EOT_TOKEN: u32 = 50257;
|
||||
const TIMESTAMP_BEGIN: u32 = 50364;
|
||||
const SAMPLE_BEGIN: usize = 3;
|
||||
const MIN_SUPPORTED_AUDIO_SAMPLE_RATE: u32 = 8_000;
|
||||
const MAX_SUPPORTED_AUDIO_SAMPLE_RATE: u32 = 96_000;
|
||||
const MAX_WHISPER_AUDIO_SAMPLES: usize = 50 * 1024 * 1024;
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct WhisperModel {
|
||||
@@ -958,6 +963,7 @@ fn decode_audio_simple(audio_data: &[u8]) -> Result<Vec<f32>> {
|
||||
let sample_rate = codec_params
|
||||
.sample_rate
|
||||
.context("No sample rate in audio track")?;
|
||||
checked_resampled_sample_count(0, sample_rate, 16_000)?;
|
||||
|
||||
let channels = codec_params
|
||||
.channels
|
||||
@@ -971,7 +977,7 @@ fn decode_audio_simple(audio_data: &[u8]) -> Result<Vec<f32>> {
|
||||
.make_audio_decoder(&codec_params, &AudioDecoderOptions::default())
|
||||
.context("Failed to create audio decoder - please ensure browser sends WAV format audio")?;
|
||||
|
||||
let mut pcm_data = Vec::new();
|
||||
let mut mono_data = Vec::new();
|
||||
let mut packet_count = 0;
|
||||
|
||||
loop {
|
||||
@@ -988,7 +994,9 @@ fn decode_audio_simple(audio_data: &[u8]) -> Result<Vec<f32>> {
|
||||
|
||||
match decoder.decode(&packet) {
|
||||
Ok(decoded) => {
|
||||
pcm_data.extend(audio_buffer_to_f32(&decoded));
|
||||
checked_decoded_sample_count(mono_data.len(), decoded.frames(), sample_rate)?;
|
||||
mono_data.try_reserve(decoded.frames())?;
|
||||
append_audio_buffer_as_mono(&decoded, &mut mono_data);
|
||||
packet_count += 1;
|
||||
}
|
||||
Err(symphonia::core::errors::Error::DecodeError(_)) => {
|
||||
@@ -1000,17 +1008,10 @@ fn decode_audio_simple(audio_data: &[u8]) -> Result<Vec<f32>> {
|
||||
|
||||
tracing::debug!(
|
||||
packet_count,
|
||||
pcm_samples = pcm_data.len(),
|
||||
mono_samples = mono_data.len(),
|
||||
"decoded audio packets"
|
||||
);
|
||||
|
||||
let mono_data = if channels > 1 {
|
||||
tracing::debug!(channels, "converting to mono");
|
||||
convert_to_mono(&pcm_data, channels)
|
||||
} else {
|
||||
pcm_data
|
||||
};
|
||||
|
||||
let resampled = if sample_rate != 16000 {
|
||||
tracing::debug!(from_rate = sample_rate, to_rate = 16000, "resampling audio");
|
||||
resample_audio(&mono_data, sample_rate, 16000)?
|
||||
@@ -1039,29 +1040,38 @@ fn decode_audio_simple(audio_data: &[u8]) -> Result<Vec<f32>> {
|
||||
Ok(resampled)
|
||||
}
|
||||
|
||||
fn audio_buffer_to_f32(buffer: &GenericAudioBufferRef<'_>) -> Vec<f32> {
|
||||
let mut samples = Vec::with_capacity(buffer.samples_interleaved());
|
||||
buffer.copy_to_vec_interleaved(&mut samples);
|
||||
samples
|
||||
fn append_audio_buffer_as_mono(buffer: &GenericAudioBufferRef<'_>, samples: &mut Vec<f32>) {
|
||||
match buffer {
|
||||
GenericAudioBufferRef::U8(buffer) => append_typed_audio_buffer_as_mono(buffer, samples),
|
||||
GenericAudioBufferRef::U16(buffer) => append_typed_audio_buffer_as_mono(buffer, samples),
|
||||
GenericAudioBufferRef::U24(buffer) => append_typed_audio_buffer_as_mono(buffer, samples),
|
||||
GenericAudioBufferRef::U32(buffer) => append_typed_audio_buffer_as_mono(buffer, samples),
|
||||
GenericAudioBufferRef::S8(buffer) => append_typed_audio_buffer_as_mono(buffer, samples),
|
||||
GenericAudioBufferRef::S16(buffer) => append_typed_audio_buffer_as_mono(buffer, samples),
|
||||
GenericAudioBufferRef::S24(buffer) => append_typed_audio_buffer_as_mono(buffer, samples),
|
||||
GenericAudioBufferRef::S32(buffer) => append_typed_audio_buffer_as_mono(buffer, samples),
|
||||
GenericAudioBufferRef::F32(buffer) => append_typed_audio_buffer_as_mono(buffer, samples),
|
||||
GenericAudioBufferRef::F64(buffer) => append_typed_audio_buffer_as_mono(buffer, samples),
|
||||
}
|
||||
}
|
||||
|
||||
fn convert_to_mono(data: &[f32], channels: usize) -> Vec<f32> {
|
||||
if channels == 1 {
|
||||
return data.to_vec();
|
||||
}
|
||||
fn append_typed_audio_buffer_as_mono<S>(buffer: &AudioBuffer<S>, samples: &mut Vec<f32>)
|
||||
where
|
||||
S: Sample,
|
||||
f32: FromSample<S>,
|
||||
{
|
||||
let num_channels = buffer.num_planes();
|
||||
let channel_scale = 1.0 / num_channels as f32;
|
||||
|
||||
let frames = data.len() / channels;
|
||||
let mut mono = Vec::with_capacity(frames);
|
||||
|
||||
for frame_idx in 0..frames {
|
||||
for frame_idx in 0..buffer.frames() {
|
||||
let mut sum = 0.0;
|
||||
for ch in 0..channels {
|
||||
sum += data[frame_idx * channels + ch];
|
||||
for ch_idx in 0..num_channels {
|
||||
if let Some(plane) = buffer.plane(ch_idx) {
|
||||
sum += f32::from_sample(plane[frame_idx]);
|
||||
}
|
||||
}
|
||||
mono.push(sum / channels as f32);
|
||||
samples.push(sum * channel_scale);
|
||||
}
|
||||
|
||||
mono
|
||||
}
|
||||
|
||||
fn resample_audio(data: &[f32], from_rate: u32, to_rate: u32) -> Result<Vec<f32>> {
|
||||
@@ -1070,6 +1080,8 @@ fn resample_audio(data: &[f32], from_rate: u32, to_rate: u32) -> Result<Vec<f32>
|
||||
SincInterpolationParameters, SincInterpolationType, WindowFunction,
|
||||
};
|
||||
|
||||
let output_samples = checked_resampled_sample_count(data.len(), from_rate, to_rate)?;
|
||||
|
||||
if from_rate == to_rate {
|
||||
return Ok(data.to_vec());
|
||||
}
|
||||
@@ -1078,6 +1090,7 @@ fn resample_audio(data: &[f32], from_rate: u32, to_rate: u32) -> Result<Vec<f32>
|
||||
from_rate,
|
||||
to_rate,
|
||||
input_samples = data.len(),
|
||||
output_samples,
|
||||
"resampling audio"
|
||||
);
|
||||
|
||||
@@ -1107,10 +1120,60 @@ fn resample_audio(data: &[f32], from_rate: u32, to_rate: u32) -> Result<Vec<f32>
|
||||
Ok(output)
|
||||
}
|
||||
|
||||
fn checked_resampled_sample_count(
|
||||
input_samples: usize,
|
||||
from_rate: u32,
|
||||
to_rate: u32,
|
||||
) -> Result<usize> {
|
||||
anyhow::ensure!(
|
||||
(MIN_SUPPORTED_AUDIO_SAMPLE_RATE..=MAX_SUPPORTED_AUDIO_SAMPLE_RATE).contains(&from_rate),
|
||||
"Unsupported audio sample rate {from_rate} Hz; supported range is {MIN_SUPPORTED_AUDIO_SAMPLE_RATE}-{MAX_SUPPORTED_AUDIO_SAMPLE_RATE} Hz"
|
||||
);
|
||||
|
||||
let from_rate = from_rate as usize;
|
||||
let to_rate = to_rate as usize;
|
||||
let whole_seconds = input_samples / from_rate;
|
||||
let partial_second_samples = input_samples % from_rate;
|
||||
let output_samples = whole_seconds
|
||||
.checked_mul(to_rate)
|
||||
.and_then(|whole| {
|
||||
partial_second_samples
|
||||
.checked_mul(to_rate)
|
||||
.and_then(|partial| whole.checked_add(partial.div_ceil(from_rate)))
|
||||
})
|
||||
.ok_or_else(|| anyhow::anyhow!("Resampled audio size overflow"))?;
|
||||
|
||||
anyhow::ensure!(
|
||||
output_samples <= MAX_WHISPER_AUDIO_SAMPLES,
|
||||
"Whisper audio would contain {output_samples} samples; maximum is {MAX_WHISPER_AUDIO_SAMPLES}"
|
||||
);
|
||||
|
||||
Ok(output_samples)
|
||||
}
|
||||
|
||||
fn checked_decoded_sample_count(
|
||||
current_samples: usize,
|
||||
decoded_frames: usize,
|
||||
sample_rate: u32,
|
||||
) -> Result<usize> {
|
||||
let decoded_samples = current_samples
|
||||
.checked_add(decoded_frames)
|
||||
.ok_or_else(|| anyhow::anyhow!("Decoded audio size overflow"))?;
|
||||
|
||||
anyhow::ensure!(
|
||||
decoded_samples <= MAX_WHISPER_AUDIO_SAMPLES,
|
||||
"Decoded audio would contain {decoded_samples} samples; maximum is {MAX_WHISPER_AUDIO_SAMPLES}"
|
||||
);
|
||||
checked_resampled_sample_count(decoded_samples, sample_rate, 16_000)?;
|
||||
|
||||
Ok(decoded_samples)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
use symphonia::core::audio::{AudioSpec, Position};
|
||||
use test_case::test_case;
|
||||
|
||||
const TS: u32 = 50364; // A timestamp token for tests
|
||||
@@ -1255,4 +1318,91 @@ mod tests {
|
||||
fn test_split_into_sentences(input: &str, expected: Vec<&str>) {
|
||||
assert_eq!(split_into_sentences(input), expected);
|
||||
}
|
||||
|
||||
#[test_case(8_000, 48_000, 96_000 ; "minimum supported rate")]
|
||||
#[test_case(16_000, 48_000, 48_000 ; "whisper rate no-op")]
|
||||
#[test_case(96_000, 96_000, 16_000 ; "maximum supported rate")]
|
||||
fn supported_resample_rates_preserve_expected_size(
|
||||
from_rate: u32,
|
||||
input_samples: usize,
|
||||
expected_output_samples: usize,
|
||||
) {
|
||||
assert_eq!(
|
||||
checked_resampled_sample_count(input_samples, from_rate, 16_000).unwrap(),
|
||||
expected_output_samples
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resampling_rejects_attacker_controlled_extreme_rate() {
|
||||
let error = resample_audio(&[0.0; 16], 1, 16_000).unwrap_err();
|
||||
|
||||
assert!(error.to_string().contains("Unsupported audio sample rate"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resampled_sample_budget_accepts_boundary_and_rejects_excess() {
|
||||
let boundary_input = MAX_WHISPER_AUDIO_SAMPLES / 2;
|
||||
assert_eq!(
|
||||
checked_resampled_sample_count(boundary_input, 8_000, 16_000).unwrap(),
|
||||
MAX_WHISPER_AUDIO_SAMPLES
|
||||
);
|
||||
|
||||
let error = checked_resampled_sample_count(boundary_input + 1, 8_000, 16_000).unwrap_err();
|
||||
assert!(error.to_string().contains("maximum"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decoded_audio_budget_enforces_native_whisper_rate() {
|
||||
assert_eq!(
|
||||
checked_decoded_sample_count(MAX_WHISPER_AUDIO_SAMPLES - 1, 1, 16_000).unwrap(),
|
||||
MAX_WHISPER_AUDIO_SAMPLES
|
||||
);
|
||||
|
||||
let error =
|
||||
checked_decoded_sample_count(MAX_WHISPER_AUDIO_SAMPLES - 1, 2, 16_000).unwrap_err();
|
||||
assert!(error.to_string().contains("maximum"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decoded_audio_budget_caps_high_rate_pcm() {
|
||||
let error = checked_decoded_sample_count(MAX_WHISPER_AUDIO_SAMPLES, 1, 96_000).unwrap_err();
|
||||
|
||||
assert!(error.to_string().contains("maximum"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decoded_stereo_frames_are_appended_as_mono() {
|
||||
let spec = AudioSpec::new(
|
||||
16_000,
|
||||
(Position::FRONT_LEFT | Position::FRONT_RIGHT).into(),
|
||||
);
|
||||
let mut buffer = AudioBuffer::<f32>::new(spec, 2);
|
||||
buffer
|
||||
.render_with(Some(2), |frame, planes| {
|
||||
planes[0][frame] = if frame == 0 { 1.0 } else { -1.0 };
|
||||
planes[1][frame] = if frame == 0 { 3.0 } else { 1.0 };
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
let mut samples = vec![4.0];
|
||||
|
||||
append_audio_buffer_as_mono(&GenericAudioBufferRef::F32(&buffer), &mut samples);
|
||||
|
||||
assert_eq!(samples, vec![4.0, 2.0, 0.0]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resampled_sample_count_rejects_overflow() {
|
||||
let error = checked_resampled_sample_count(usize::MAX, 8_000, 16_000).unwrap_err();
|
||||
|
||||
assert!(error.to_string().contains("overflow"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn same_rate_resampling_preserves_samples() {
|
||||
let samples = vec![0.25, -0.5, 0.75];
|
||||
|
||||
assert_eq!(resample_audio(&samples, 16_000, 16_000).unwrap(), samples);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user