gpt-live API support (#12011)

Co-authored-by: Lifei Zhou <lifei@squareup.com>
This commit is contained in:
Jack Amadeo
2026-09-15 15:55:23 +00:00
committed by GitHub
parent a23a8cd5b1
commit 53672c3f14
13 changed files with 3336 additions and 477 deletions
Generated
+537 -471
View File
File diff suppressed because it is too large Load Diff
+13 -1
View File
@@ -13,6 +13,7 @@ workspace = true
[features]
default = []
test-utils = []
local-inference = ["dep:goose-local-inference"]
local-inference-hf-hub = ["local-inference", "goose-local-inference/hf-hub"]
cuda = ["local-inference", "goose-local-inference/cuda"]
@@ -21,13 +22,16 @@ mlx = ["local-inference", "goose-local-inference/mlx"]
rustls-tls = [
"reqwest/rustls",
"rmcp/reqwest",
"tokio-tungstenite?/rustls-tls-webpki-roots",
]
live-websocket = ["dep:tokio-tungstenite", "dep:http"]
native-tls = [
"dep:pem",
"dep:pkcs8",
"dep:sec1",
"reqwest/native-tls",
"rmcp/reqwest-native-tls",
"tokio-tungstenite?/native-tls",
]
[dependencies]
@@ -48,6 +52,10 @@ tokio-stream = { workspace = true, features = ["io-util"] }
tokio-util = { workspace = true, features = ["compat"] }
url = { workspace = true }
urlencoding = { workspace = true }
uuid = { workspace = true, features = ["v4"] }
base64 = { workspace = true }
tokio-tungstenite = { version = "0.28", default-features = false, features = ["connect", "handshake"], optional = true }
http = { workspace = true, optional = true }
pem = { version = "4.0.0", default-features = false, features = ["std"], optional = true }
pkcs8 = { version = "0.11", default-features = false, features = ["alloc", "std"], optional = true }
sec1 = { version = "0.8", default-features = false, features = ["der", "std"], optional = true }
@@ -56,7 +64,7 @@ include_dir = { workspace = true }
[dev-dependencies]
test-case = { workspace = true }
tempfile = { workspace = true }
tokio = { workspace = true, features = ["io-util", "macros", "net", "rt-multi-thread", "time"] }
tokio = { workspace = true, features = ["io-util", "macros", "net", "rt-multi-thread", "test-util", "time"] }
tokio-stream = { workspace = true }
env-lock = { workspace = true }
wiremock.workspace = true
@@ -66,3 +74,7 @@ reqwest = { workspace = true, features = ["gzip"] }
[[example]]
name = "streaming"
required-features = ["rustls-tls"]
[[example]]
name = "openai_live"
required-features = ["live-websocket", "rustls-tls"]
@@ -0,0 +1,28 @@
//! OpenAI Live low-level API examples.
use anyhow::Result;
use goose_providers::openai_live::{OpenAiLiveClient, OpenAiLiveSessionConfig};
fn config() -> OpenAiLiveSessionConfig {
OpenAiLiveSessionConfig {
model: "gpt-live-1".into(),
instructions: "Be concise.".into(),
voice: Some("marin".into()),
input_messages: vec![],
extra_session_fields: Default::default(),
}
}
#[cfg(feature = "live-websocket")]
async fn primary_websocket(client: &OpenAiLiveClient) -> Result<()> {
let mut connected = client.websocket(config()).connect().await?;
let event = connected.recv().await?;
println!("{event:?}");
connected.close().await
}
fn main() {
let _ = OpenAiLiveClient::from_env;
#[cfg(feature = "live-websocket")]
let _ = primary_websocket;
}
@@ -0,0 +1,103 @@
//! Message bridge for browser-owned WebRTC data channels.
//!
//! Audio remains on WebRTC media tracks; this transport carries only protocol
//! control messages and events between Rust and the browser data channel.
use crate::live::LiveTransport;
use anyhow::Result;
use async_trait::async_trait;
use serde_json::Value;
use tokio::sync::{mpsc, watch, Mutex};
pub struct BrowserLiveOutbound {
receiver: mpsc::Receiver<Value>,
closed: watch::Receiver<bool>,
}
impl BrowserLiveOutbound {
pub async fn recv(&mut self) -> Option<Value> {
tokio::select! {
biased;
_ = self.closed.wait_for(|closed| *closed) => None,
message = self.receiver.recv() => message,
}
}
}
pub struct BrowserLiveTransport {
outbound_tx: mpsc::Sender<Value>,
outbound_rx: Mutex<Option<mpsc::Receiver<Value>>>,
incoming_tx: mpsc::Sender<Value>,
incoming_rx: Mutex<mpsc::Receiver<Value>>,
closed_tx: watch::Sender<bool>,
}
impl BrowserLiveTransport {
pub fn new() -> Self {
let (outbound_tx, outbound_rx) = mpsc::channel(256);
let (incoming_tx, incoming_rx) = mpsc::channel(256);
let (closed_tx, _) = watch::channel(false);
Self {
outbound_tx,
outbound_rx: Mutex::new(Some(outbound_rx)),
incoming_tx,
incoming_rx: Mutex::new(incoming_rx),
closed_tx,
}
}
pub async fn take_outbound(&self) -> Result<BrowserLiveOutbound> {
let receiver = self
.outbound_rx
.lock()
.await
.take()
.ok_or_else(|| anyhow::anyhow!("browser live receiver was already taken"))?;
Ok(BrowserLiveOutbound {
receiver,
closed: self.closed_tx.subscribe(),
})
}
pub async fn push_incoming(&self, message: Value) -> Result<()> {
let mut closed = self.closed_tx.subscribe();
tokio::select! {
biased;
_ = closed.wait_for(|closed| *closed) => anyhow::bail!("browser live transport is closed"),
result = self.incoming_tx.send(message) => result.map_err(Into::into),
}
}
}
impl Default for BrowserLiveTransport {
fn default() -> Self {
Self::new()
}
}
#[async_trait]
impl LiveTransport for BrowserLiveTransport {
async fn send(&self, message: Value) -> Result<()> {
let mut closed = self.closed_tx.subscribe();
tokio::select! {
biased;
_ = closed.wait_for(|closed| *closed) => anyhow::bail!("browser live transport is closed"),
result = self.outbound_tx.send(message) => result.map_err(Into::into),
}
}
async fn receive(&self) -> Result<Option<Value>> {
let mut closed = self.closed_tx.subscribe();
let mut incoming = self.incoming_rx.lock().await;
tokio::select! {
biased;
_ = closed.wait_for(|closed| *closed) => Ok(None),
message = incoming.recv() => Ok(message),
}
}
async fn close(&self) -> Result<()> {
self.closed_tx.send_replace(true);
Ok(())
}
}
+14
View File
@@ -1,3 +1,9 @@
#[cfg(all(
feature = "live-websocket",
not(any(feature = "rustls-tls", feature = "native-tls"))
))]
compile_error!("feature `live-websocket` requires either `rustls-tls` or `native-tls`");
pub mod anthropic;
pub mod api_client;
pub mod azure_foundry;
@@ -9,13 +15,21 @@ pub use goose_provider_types::{
base, cache_semantics, canonical, context_limit, conversation, documents, errors, formats,
goose_mode, images, json, model, permission, request_log, retry, thinking, utils,
};
pub mod browser_live_transport;
pub mod declarative;
pub mod http_status;
pub mod live;
#[cfg(feature = "live-websocket")]
pub mod live_transport_websocket;
pub mod live_voice_provider;
#[cfg(feature = "local-inference")]
pub mod local_inference;
pub mod ollama;
pub mod openai;
pub mod openai_compatible;
pub mod openai_live;
#[cfg(feature = "live-websocket")]
pub mod openai_live_voice_provider;
pub mod openrouter;
pub mod openrouter_format;
+298
View File
@@ -0,0 +1,298 @@
//! Runtime primitives for provider-specific live sessions.
//!
//! A single actor owns transport I/O and lifecycle transitions so sends,
//! shutdown, and incoming events have deterministic ordering.
use anyhow::{anyhow, bail, Result};
use async_trait::async_trait;
use serde_json::Value;
use std::sync::Arc;
use tokio::{
sync::{broadcast, mpsc, oneshot, watch},
time::{timeout, Duration},
};
pub(crate) const LIVE_EVENT_CHANNEL_CAPACITY: usize = 256;
const TRANSPORT_SEND_TIMEOUT: Duration = Duration::from_secs(5);
const TRANSPORT_CLOSE_TIMEOUT: Duration = Duration::from_secs(5);
const CLOSE_ACKNOWLEDGEMENT_TIMEOUT: Duration = Duration::from_secs(12);
#[async_trait]
pub trait LiveTransport: Send + Sync {
async fn send(&self, message: Value) -> Result<()>;
async fn receive(&self) -> Result<Option<Value>>;
async fn close(&self) -> Result<()>;
}
pub trait LiveProtocol: Send + Sync + 'static {
type Command: Send + 'static;
type Event: Clone + Send + 'static;
fn encode(&self, command: Self::Command) -> Result<Value>;
fn decode(&self, message: Value) -> Result<Self::Event>;
fn close_command(&self) -> Option<Self::Command>;
fn is_close_acknowledgement(&self, event: &Self::Event) -> bool;
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum LiveSessionEndReason {
Closed,
CloseTimedOut,
TransportClosed,
TransportFailed,
ProtocolFailed,
DecodeFailed,
}
#[derive(Debug, Clone)]
struct LiveSessionOutcome {
reason: LiveSessionEndReason,
error: Option<Arc<anyhow::Error>>,
}
impl LiveSessionOutcome {
fn result(&self) -> Result<()> {
match &self.error {
Some(error) => Err(anyhow!(error.to_string())),
None if self.reason == LiveSessionEndReason::Closed => Ok(()),
None => Err(anyhow!("live session ended ({:?})", self.reason)),
}
}
}
#[derive(Debug, Clone)]
pub enum LiveSessionEvent<E> {
Message(E),
Ended {
reason: LiveSessionEndReason,
error: Option<Arc<anyhow::Error>>,
},
}
impl<E> LiveSessionEvent<E> {
fn ended(reason: LiveSessionEndReason, error: Option<anyhow::Error>) -> Self {
Self::Ended {
reason,
error: error.map(Arc::new),
}
}
}
enum ActorCommand<C> {
Send {
command: C,
result: oneshot::Sender<Result<()>>,
},
Close {
result: oneshot::Sender<Result<()>>,
},
}
pub struct LiveSession<P: LiveProtocol> {
commands: mpsc::Sender<ActorCommand<P::Command>>,
events: broadcast::Sender<LiveSessionEvent<P::Event>>,
ended: watch::Receiver<Option<LiveSessionOutcome>>,
}
impl<P: LiveProtocol> Clone for LiveSession<P> {
fn clone(&self) -> Self {
Self {
commands: self.commands.clone(),
events: self.events.clone(),
ended: self.ended.clone(),
}
}
}
impl<P: LiveProtocol> LiveSession<P> {
pub fn connect(
protocol: Arc<P>,
transport: Arc<dyn LiveTransport>,
) -> (Self, broadcast::Receiver<LiveSessionEvent<P::Event>>) {
let (commands, command_rx) = mpsc::channel(256);
let (events, initial_events) = broadcast::channel(LIVE_EVENT_CHANNEL_CAPACITY);
let (ended_tx, ended) = watch::channel(None);
tokio::spawn(run_actor(
protocol,
transport,
command_rx,
events.clone(),
ended_tx,
));
(
Self {
commands,
events,
ended,
},
initial_events,
)
}
pub fn subscribe(&self) -> broadcast::Receiver<LiveSessionEvent<P::Event>> {
self.events.subscribe()
}
pub fn end_reason(&self) -> Option<LiveSessionEndReason> {
self.ended.borrow().as_ref().map(|outcome| outcome.reason)
}
pub async fn send(&self, command: P::Command) -> Result<()> {
if let Some(reason) = self.end_reason() {
bail!("live session ended ({reason:?})");
}
let (result_tx, result_rx) = oneshot::channel();
self.commands
.send(ActorCommand::Send {
command,
result: result_tx,
})
.await
.map_err(|_| anyhow!("live session ended"))?;
result_rx.await.map_err(|_| anyhow!("live session ended"))?
}
pub async fn close(&self) -> Result<()> {
let outcome = self.ended.borrow().clone();
if let Some(outcome) = outcome {
return outcome.result();
}
let (result_tx, result_rx) = oneshot::channel();
if self
.commands
.send(ActorCommand::Close { result: result_tx })
.await
.is_err()
{
return self.wait_for_end().await;
}
match result_rx.await {
Ok(result) => result,
Err(_) => self.wait_for_end().await,
}
}
async fn wait_for_end(&self) -> Result<()> {
let mut ended = self.ended.clone();
loop {
let outcome = ended.borrow().clone();
if let Some(outcome) = outcome {
return outcome.result();
}
ended
.changed()
.await
.map_err(|_| anyhow!("live session ended without an outcome"))?;
}
}
}
async fn run_actor<P: LiveProtocol>(
protocol: Arc<P>,
transport: Arc<dyn LiveTransport>,
mut commands: mpsc::Receiver<ActorCommand<P::Command>>,
events: broadcast::Sender<LiveSessionEvent<P::Event>>,
ended: watch::Sender<Option<LiveSessionOutcome>>,
) {
let mut close_waiters = Vec::new();
let mut closing = false;
let close_timeout = tokio::time::sleep(CLOSE_ACKNOWLEDGEMENT_TIMEOUT);
tokio::pin!(close_timeout);
let (reason, error) = loop {
tokio::select! {
command = commands.recv() => match command {
Some(ActorCommand::Send { command, result }) if !closing => {
let message = match protocol.encode(command) {
Ok(message) => message,
Err(error) => {
let _ = result.send(Err(error));
continue;
}
};
let send_result = send_transport(&transport, message).await;
let failed = send_result.is_err();
let error = send_result.as_ref().err().map(|error| anyhow!(error.to_string()));
let _ = result.send(send_result);
if failed {
break (LiveSessionEndReason::TransportFailed, error);
}
}
Some(ActorCommand::Send { result, .. }) => {
let _ = result.send(Err(anyhow!("live session is closing")));
}
Some(ActorCommand::Close { result }) => {
close_waiters.push(result);
if !closing {
closing = true;
if let Some(command) = protocol.close_command() {
let close_result = match protocol.encode(command) {
Ok(message) => send_transport(&transport, message).await,
Err(error) => {
break (LiveSessionEndReason::ProtocolFailed, Some(error));
}
};
if let Err(error) = close_result {
break (LiveSessionEndReason::TransportFailed, Some(error));
}
close_timeout.as_mut().reset(
tokio::time::Instant::now() + CLOSE_ACKNOWLEDGEMENT_TIMEOUT,
);
} else {
break (LiveSessionEndReason::Closed, None);
}
}
}
None => break (LiveSessionEndReason::Closed, None),
},
incoming = transport.receive() => match incoming {
Ok(Some(message)) => match protocol.decode(message) {
Ok(event) => {
let acknowledged = protocol.is_close_acknowledgement(&event);
let _ = events.send(LiveSessionEvent::Message(event));
if acknowledged {
break (LiveSessionEndReason::Closed, None);
}
}
Err(error) => break (LiveSessionEndReason::DecodeFailed, Some(error)),
},
Ok(None) => break (LiveSessionEndReason::TransportClosed, None),
Err(error) => break (LiveSessionEndReason::TransportFailed, Some(error)),
},
_ = &mut close_timeout, if closing => {
break (
LiveSessionEndReason::CloseTimedOut,
Some(anyhow!("live session close acknowledgement timed out")),
);
},
}
};
let close_result = timeout(TRANSPORT_CLOSE_TIMEOUT, transport.close()).await;
let close_error = match close_result {
Ok(Ok(())) => None,
Ok(Err(error)) => Some(error),
Err(_) => Some(anyhow!("live transport close timed out")),
};
let final_error = error.or(close_error);
let outcome = LiveSessionOutcome {
reason,
error: final_error
.as_ref()
.map(|error| Arc::new(anyhow!(error.to_string()))),
};
ended.send_replace(Some(outcome.clone()));
let _ = events.send(LiveSessionEvent::ended(
reason,
final_error.as_ref().map(|error| anyhow!(error.to_string())),
));
for waiter in close_waiters {
let _ = waiter.send(outcome.result());
}
}
async fn send_transport(transport: &Arc<dyn LiveTransport>, message: Value) -> Result<()> {
timeout(TRANSPORT_SEND_TIMEOUT, transport.send(message))
.await
.map_err(|_| anyhow!("live transport send timed out"))?
}
@@ -0,0 +1,86 @@
//! Native WebSocket transport for OpenAI Live sessions.
use crate::{live::LiveTransport, openai_live::OpenAiLiveWebSocketRequest};
use anyhow::{Context, Result};
use async_trait::async_trait;
use futures::{stream::SplitSink, stream::SplitStream, SinkExt, StreamExt};
use serde_json::Value;
use tokio::{
sync::Mutex,
time::{timeout, Duration},
};
use tokio_tungstenite::{
connect_async,
tungstenite::{client::IntoClientRequest, Message},
MaybeTlsStream, WebSocketStream,
};
const CONNECT_TIMEOUT: Duration = Duration::from_secs(10);
type Socket = WebSocketStream<MaybeTlsStream<tokio::net::TcpStream>>;
type SocketSink = SplitSink<Socket, Message>;
type SocketStream = SplitStream<Socket>;
pub struct WebSocketLiveTransport {
sink: Mutex<SocketSink>,
stream: Mutex<SocketStream>,
}
impl WebSocketLiveTransport {
pub async fn connect(request: OpenAiLiveWebSocketRequest) -> Result<Self> {
let mut websocket_request = request.endpoint.into_client_request()?;
for (name, value) in request.headers {
websocket_request.headers_mut().insert(
http::header::HeaderName::try_from(name)?,
http::header::HeaderValue::try_from(value)?,
);
}
let (socket, _) = timeout(CONNECT_TIMEOUT, connect_async(websocket_request))
.await
.context("live WebSocket connection timed out")?
.context("live WebSocket connection failed")?;
let (mut sink, stream) = socket.split();
timeout(CONNECT_TIMEOUT, async {
for message in request.initial_messages {
sink.send(Message::Text(message.to_string().into())).await?;
}
Result::<()>::Ok(())
})
.await
.context("live WebSocket initialization timed out")??;
Ok(Self {
sink: Mutex::new(sink),
stream: Mutex::new(stream),
})
}
}
#[async_trait]
impl LiveTransport for WebSocketLiveTransport {
async fn send(&self, message: Value) -> Result<()> {
self.sink
.lock()
.await
.send(Message::Text(message.to_string().into()))
.await?;
Ok(())
}
async fn receive(&self) -> Result<Option<Value>> {
let stream = &mut *self.stream.lock().await;
while let Some(message) = stream.next().await {
match message? {
Message::Text(text) => return Ok(Some(serde_json::from_str(&text)?)),
Message::Binary(bytes) => return Ok(Some(serde_json::from_slice(&bytes)?)),
Message::Close(_) => return Ok(None),
_ => {}
}
}
Ok(None)
}
async fn close(&self) -> Result<()> {
self.sink.lock().await.close().await?;
Ok(())
}
}
@@ -0,0 +1,225 @@
//! The application-facing boundary for live voice providers.
use anyhow::Result;
use async_trait::async_trait;
const MAX_SDP_BYTES: usize = 256 * 1024;
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct WebRtcOffer(String);
impl WebRtcOffer {
pub fn new(sdp: String) -> Option<Self> {
valid_sdp(&sdp).then_some(Self(sdp))
}
pub fn into_sdp(self) -> String {
self.0
}
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct WebRtcAnswer(String);
impl WebRtcAnswer {
pub fn new(sdp: String) -> Option<Self> {
valid_sdp(&sdp).then_some(Self(sdp))
}
pub fn into_sdp(self) -> String {
self.0
}
}
fn valid_sdp(sdp: &str) -> bool {
!sdp.is_empty() && sdp.len() <= MAX_SDP_BYTES
}
#[async_trait]
pub trait ProviderConnection: Send {
async fn next_event(&mut self) -> ProviderConnectionEvent;
async fn stop(&mut self) -> Result<()>;
}
#[derive(Debug, PartialEq)]
pub enum ProviderConnectionEvent {
TranscriptDelta {
event_id: String,
role: rmcp::model::Role,
text: String,
},
ReceiverLagged,
Closed,
Failed,
}
#[derive(Clone, Debug, PartialEq)]
pub struct LiveVoiceInputMessage {
pub role: rmcp::model::Role,
pub text: String,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum LiveVoiceProviderAvailability {
Ready,
Disabled,
Unavailable,
}
#[async_trait]
pub trait LiveVoiceProvider: Send + Sync {
fn availability(&self) -> LiveVoiceProviderAvailability;
async fn start(
&self,
offer: WebRtcOffer,
input_messages: Vec<LiveVoiceInputMessage>,
) -> Result<(WebRtcAnswer, Box<dyn ProviderConnection>)>;
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn sdp_values_must_be_present_and_bounded() {
assert!(WebRtcOffer::new(String::new()).is_none());
assert!(WebRtcOffer::new("x".repeat(MAX_SDP_BYTES + 1)).is_none());
assert!(WebRtcOffer::new("offer".into()).is_some());
assert!(WebRtcAnswer::new("answer".into()).is_some());
}
}
#[cfg(any(test, feature = "test-utils"))]
pub mod fake {
use super::*;
use tokio::sync::{mpsc, oneshot};
pub fn provider_channel() -> (
std::sync::Arc<FakeLiveVoiceProvider>,
mpsc::UnboundedReceiver<FakeStartRequest>,
) {
provider_channel_with_availability(LiveVoiceProviderAvailability::Ready)
}
pub fn provider_channel_with_availability(
availability: LiveVoiceProviderAvailability,
) -> (
std::sync::Arc<FakeLiveVoiceProvider>,
mpsc::UnboundedReceiver<FakeStartRequest>,
) {
let (start_tx, start_rx) = mpsc::unbounded_channel();
(
std::sync::Arc::new(FakeLiveVoiceProvider {
availability,
start_tx,
}),
start_rx,
)
}
pub struct FakeLiveVoiceProvider {
availability: LiveVoiceProviderAvailability,
start_tx: mpsc::UnboundedSender<FakeStartRequest>,
}
#[async_trait]
impl LiveVoiceProvider for FakeLiveVoiceProvider {
fn availability(&self) -> LiveVoiceProviderAvailability {
self.availability
}
async fn start(
&self,
offer: WebRtcOffer,
input_messages: Vec<LiveVoiceInputMessage>,
) -> Result<(WebRtcAnswer, Box<dyn ProviderConnection>)> {
let (response_tx, response_rx) = oneshot::channel();
self.start_tx
.send(FakeStartRequest {
offer,
input_messages,
response_tx,
})
.map_err(|_| anyhow::anyhow!("fake provider driver dropped"))?;
response_rx
.await
.map_err(|_| anyhow::anyhow!("fake provider start response dropped"))?
}
}
pub struct FakeStartRequest {
pub offer: WebRtcOffer,
pub input_messages: Vec<LiveVoiceInputMessage>,
response_tx: oneshot::Sender<Result<(WebRtcAnswer, Box<dyn ProviderConnection>)>>,
}
impl FakeStartRequest {
pub fn accept(self, answer: WebRtcAnswer) -> Result<FakeConnectionDriver> {
let (stop_request_tx, stop_request_rx) = mpsc::unbounded_channel();
let (event_tx, event_rx) = mpsc::unbounded_channel();
self.response_tx
.send(Ok((
answer,
Box::new(FakeProviderConnection {
stop_request_tx,
event_rx,
}),
)))
.map_err(|_| anyhow::anyhow!("fake provider start caller dropped"))?;
Ok(FakeConnectionDriver {
stop_request_rx,
event_tx,
})
}
pub fn reject(self, message: impl Into<String>) -> Result<()> {
self.response_tx
.send(Err(anyhow::anyhow!(message.into())))
.map_err(|_| anyhow::anyhow!("fake provider start caller dropped"))
}
}
pub struct FakeConnectionDriver {
stop_request_rx: mpsc::UnboundedReceiver<oneshot::Sender<Result<(), String>>>,
event_tx: mpsc::UnboundedSender<ProviderConnectionEvent>,
}
impl FakeConnectionDriver {
pub async fn next_stop_request(&mut self) -> Option<oneshot::Sender<Result<(), String>>> {
self.stop_request_rx.recv().await
}
pub fn send_event(&self, event: ProviderConnectionEvent) -> Result<()> {
self.event_tx
.send(event)
.map_err(|_| anyhow::anyhow!("fake provider event receiver dropped"))
}
}
struct FakeProviderConnection {
stop_request_tx: mpsc::UnboundedSender<oneshot::Sender<Result<(), String>>>,
event_rx: mpsc::UnboundedReceiver<ProviderConnectionEvent>,
}
#[async_trait]
impl ProviderConnection for FakeProviderConnection {
async fn next_event(&mut self) -> ProviderConnectionEvent {
self.event_rx
.recv()
.await
.unwrap_or(ProviderConnectionEvent::Failed)
}
async fn stop(&mut self) -> Result<()> {
let (response_tx, response_rx) = oneshot::channel();
self.stop_request_tx
.send(response_tx)
.map_err(|_| anyhow::anyhow!("fake provider driver dropped"))?;
response_rx
.await
.map_err(|_| anyhow::anyhow!("fake provider stop response dropped"))?
.map_err(anyhow::Error::msg)
}
}
}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,308 @@
//! OpenAI implementation of the live voice provider boundary.
use crate::{
live::{LiveSessionEndReason, LiveSessionEvent},
live_voice_provider::{
LiveVoiceInputMessage, LiveVoiceProvider, LiveVoiceProviderAvailability,
ProviderConnection, ProviderConnectionEvent, WebRtcAnswer, WebRtcOffer,
},
openai_live::{
ConnectedOpenAiLiveSession, OpenAiLiveClient, OpenAiLiveEvent, OpenAiLiveEventKind,
OpenAiLiveMessage, OpenAiLiveMessageRole, OpenAiLiveSessionConfig, OpenAiLiveSessionId,
},
};
use anyhow::{bail, Result};
use async_trait::async_trait;
use std::time::Duration;
use tokio::{
sync::broadcast::error::RecvError,
time::{sleep_until, timeout, timeout_at, Instant},
};
pub const OPENAI_LIVE_VOICE_GATE_ENV: &str = "GOOSE_LIVE_VOICE_ENABLED";
pub const OPENAI_LIVE_MODEL_ENV: &str = "GOOSE_LIVE_VOICE_MODEL";
pub const OPENAI_LIVE_VOICE_ENV: &str = "GOOSE_LIVE_VOICE";
pub const OPENAI_LIVE_API_KEY_ENV: &str = "OPENAI_API_KEY";
pub const DEFAULT_OPENAI_LIVE_MODEL: &str = "gpt-live-1";
pub const DEFAULT_OPENAI_LIVE_VOICE: &str = "marin";
const HTTP_SETUP_TIMEOUT: Duration = Duration::from_secs(15);
const SIDEBAND_ATTACH_TIMEOUT: Duration = Duration::from_secs(10);
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct OpenAiLiveVoiceConfig {
pub enabled: bool,
pub model: String,
pub voice: String,
}
impl OpenAiLiveVoiceConfig {
pub fn from_env() -> Self {
Self {
enabled: std::env::var(OPENAI_LIVE_VOICE_GATE_ENV)
.is_ok_and(|value| value.eq_ignore_ascii_case("true") || value == "1"),
model: std::env::var(OPENAI_LIVE_MODEL_ENV)
.unwrap_or_else(|_| DEFAULT_OPENAI_LIVE_MODEL.into()),
voice: std::env::var(OPENAI_LIVE_VOICE_ENV)
.unwrap_or_else(|_| DEFAULT_OPENAI_LIVE_VOICE.into()),
}
}
}
pub struct OpenAiLiveVoiceProvider {
client: Option<OpenAiLiveClient>,
config: OpenAiLiveVoiceConfig,
}
impl OpenAiLiveVoiceProvider {
pub fn from_env() -> Self {
let config = OpenAiLiveVoiceConfig::from_env();
let client = std::env::var(OPENAI_LIVE_API_KEY_ENV)
.ok()
.filter(|key| !key.trim().is_empty())
.map(OpenAiLiveClient::new);
Self { client, config }
}
pub fn new(api_key: impl Into<String>, config: OpenAiLiveVoiceConfig) -> Result<Self> {
let api_key = api_key.into();
if api_key.trim().is_empty() {
bail!("OpenAI API key is empty");
}
Ok(Self {
client: Some(OpenAiLiveClient::new(api_key)),
config,
})
}
}
#[async_trait]
impl LiveVoiceProvider for OpenAiLiveVoiceProvider {
fn availability(&self) -> LiveVoiceProviderAvailability {
if !self.config.enabled {
LiveVoiceProviderAvailability::Disabled
} else if self.client.is_none() {
LiveVoiceProviderAvailability::Unavailable
} else {
LiveVoiceProviderAvailability::Ready
}
}
async fn start(
&self,
offer: WebRtcOffer,
input_messages: Vec<LiveVoiceInputMessage>,
) -> Result<(WebRtcAnswer, Box<dyn ProviderConnection>)> {
if !self.config.enabled {
bail!("OpenAI Live voice is disabled");
}
let client = self
.client
.as_ref()
.ok_or_else(|| anyhow::anyhow!("OpenAI Live credentials are unavailable"))?
.clone();
let config = OpenAiLiveSessionConfig {
model: self.config.model.clone(),
instructions: String::new(),
voice: Some(self.config.voice.clone()),
input_messages: input_messages
.into_iter()
.map(|message| OpenAiLiveMessage {
role: match message.role {
rmcp::model::Role::User => OpenAiLiveMessageRole::User,
rmcp::model::Role::Assistant => OpenAiLiveMessageRole::Assistant,
},
text: message.text,
})
.collect(),
extra_session_fields: Default::default(),
};
let negotiation = timeout(
HTTP_SETUP_TIMEOUT,
client.webrtc(config).negotiate(offer.into_sdp()),
)
.await
.map_err(|_| anyhow::anyhow!("OpenAI Live HTTP setup timed out"))??;
let session_id = negotiation.session_id;
let answer = WebRtcAnswer::new(negotiation.answer_sdp)
.ok_or_else(|| anyhow::anyhow!("OpenAI Live returned an invalid WebRTC answer"))?;
let sideband = connect_sideband(&client, session_id).await?;
Ok((answer, Box::new(OpenAiProviderConnection { sideband })))
}
}
struct OpenAiProviderConnection {
sideband: ConnectedOpenAiLiveSession,
}
#[async_trait]
impl ProviderConnection for OpenAiProviderConnection {
async fn next_event(&mut self) -> ProviderConnectionEvent {
loop {
if let Some(event) = provider_connection_event(self.sideband.recv().await) {
return event;
}
}
}
async fn stop(&mut self) -> Result<()> {
self.sideband.close().await
}
}
fn provider_connection_event(
event: std::result::Result<LiveSessionEvent<OpenAiLiveEvent>, RecvError>,
) -> Option<ProviderConnectionEvent> {
match event {
Ok(LiveSessionEvent::Message(event)) => match event.kind {
OpenAiLiveEventKind::TranscriptDelta {
event_id,
role,
delta,
..
} => Some(ProviderConnectionEvent::TranscriptDelta {
event_id,
role,
text: delta,
}),
OpenAiLiveEventKind::SessionClosed { .. } => Some(ProviderConnectionEvent::Closed),
_ => None,
},
Ok(LiveSessionEvent::Ended {
reason: LiveSessionEndReason::Closed,
error: None,
}) => Some(ProviderConnectionEvent::Closed),
Err(RecvError::Lagged(_)) => Some(ProviderConnectionEvent::ReceiverLagged),
Ok(LiveSessionEvent::Ended { .. }) | Err(RecvError::Closed) => {
Some(ProviderConnectionEvent::Failed)
}
}
}
async fn connect_sideband(
client: &OpenAiLiveClient,
session_id: OpenAiLiveSessionId,
) -> Result<ConnectedOpenAiLiveSession> {
let deadline = Instant::now() + SIDEBAND_ATTACH_TIMEOUT;
loop {
match timeout_at(
deadline,
client.existing_session(session_id.clone()).connect(),
)
.await
{
Ok(Ok(sideband)) => return Ok(sideband),
Ok(Err(error)) => {
let retry_at = Instant::now() + Duration::from_millis(200);
if retry_at >= deadline {
return Err(error);
}
sleep_until(retry_at).await;
}
Err(_) => bail!("OpenAI Live sideband attachment timed out"),
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn configuration_is_gated() {
let provider = OpenAiLiveVoiceProvider::new(
"key",
OpenAiLiveVoiceConfig {
enabled: false,
model: DEFAULT_OPENAI_LIVE_MODEL.into(),
voice: DEFAULT_OPENAI_LIVE_VOICE.into(),
},
)
.unwrap();
assert_eq!(
provider.availability(),
LiveVoiceProviderAvailability::Disabled
);
}
#[test]
fn maps_live_voice_observations() {
let message = |kind| {
Ok(LiveSessionEvent::Message(OpenAiLiveEvent {
kind,
raw: None,
}))
};
let cases = [
(
message(OpenAiLiveEventKind::TranscriptDelta {
event_id: "event_transcript_1".into(),
client_event_id: None,
role: rmcp::model::Role::Assistant,
delta: "hello".into(),
start_ms: 10,
end_ms: 20,
}),
Some(ProviderConnectionEvent::TranscriptDelta {
event_id: "event_transcript_1".into(),
role: rmcp::model::Role::Assistant,
text: "hello".into(),
}),
),
(
message(OpenAiLiveEventKind::SessionClosed {
event_id: "event_closed_1".into(),
client_event_id: Some("client_close_1".into()),
reason: "close_requested".into(),
session: serde_json::json!({ "id": "session_1" }),
usage: serde_json::json!({ "seconds": 1 }),
}),
Some(ProviderConnectionEvent::Closed),
),
(
message(OpenAiLiveEventKind::Error {
event_id: "event_error_1".into(),
error_type: "server_error".into(),
code: "provider_failed".into(),
message: "provider failed".into(),
parameter: None,
client_event_id: None,
}),
None,
),
(
Ok(LiveSessionEvent::Ended {
reason: LiveSessionEndReason::Closed,
error: None,
}),
Some(ProviderConnectionEvent::Closed),
),
(
Ok(LiveSessionEvent::Ended {
reason: LiveSessionEndReason::TransportFailed,
error: None,
}),
Some(ProviderConnectionEvent::Failed),
),
(
Err(RecvError::Lagged(1)),
Some(ProviderConnectionEvent::ReceiverLagged),
),
(
Err(RecvError::Closed),
Some(ProviderConnectionEvent::Failed),
),
(
message(OpenAiLiveEventKind::InputMuted {
event_id: "event_muted_1".into(),
client_event_id: None,
}),
None,
),
];
for (event, expected) in cases {
assert_eq!(provider_connection_event(event), expected);
}
}
}
@@ -0,0 +1,396 @@
use anyhow::{anyhow, Result};
use async_trait::async_trait;
use goose_providers::live::{
LiveProtocol, LiveSession, LiveSessionEndReason, LiveSessionEvent, LiveTransport,
};
use serde_json::{json, Value};
use std::sync::{
atomic::{AtomicUsize, Ordering},
Arc,
};
use tokio::sync::{mpsc, Mutex};
#[derive(Clone, Debug, PartialEq, Eq)]
enum TestEvent {
Message(String),
Closed,
}
struct TestProtocol {
graceful: bool,
}
impl LiveProtocol for TestProtocol {
type Command = String;
type Event = TestEvent;
fn encode(&self, command: Self::Command) -> Result<Value> {
if command == "invalid" {
return Err(anyhow!("invalid command"));
}
Ok(json!({ "text": command }))
}
fn decode(&self, message: Value) -> Result<Self::Event> {
let text = message
.get("text")
.and_then(Value::as_str)
.ok_or_else(|| anyhow!("missing text"))?;
Ok(if text == "closed" {
TestEvent::Closed
} else {
TestEvent::Message(text.to_owned())
})
}
fn close_command(&self) -> Option<Self::Command> {
self.graceful.then(|| "close".to_string())
}
fn is_close_acknowledgement(&self, event: &Self::Event) -> bool {
matches!(event, TestEvent::Closed)
}
}
struct TestTransport {
incoming: Mutex<mpsc::Receiver<Result<Option<Value>>>>,
sent: mpsc::UnboundedSender<Value>,
closes: Arc<AtomicUsize>,
stall_sends: bool,
}
/// Test-side handles for driving and observing a `TestTransport`.
struct Controls {
incoming: mpsc::Sender<Result<Option<Value>>>,
sent: mpsc::UnboundedReceiver<Value>,
closes: Arc<AtomicUsize>,
}
impl Controls {
fn close_count(&self) -> usize {
self.closes.load(Ordering::SeqCst)
}
}
impl TestTransport {
fn new() -> (Arc<Self>, Controls) {
Self::with_stalled_sends(false)
}
fn with_stalled_sends(stall_sends: bool) -> (Arc<Self>, Controls) {
let (incoming_tx, incoming_rx) = mpsc::channel(16);
let (sent_tx, sent_rx) = mpsc::unbounded_channel();
let closes = Arc::new(AtomicUsize::new(0));
let transport = Arc::new(Self {
incoming: Mutex::new(incoming_rx),
sent: sent_tx,
closes: closes.clone(),
stall_sends,
});
let controls = Controls {
incoming: incoming_tx,
sent: sent_rx,
closes,
};
(transport, controls)
}
}
#[async_trait]
impl LiveTransport for TestTransport {
async fn send(&self, message: Value) -> Result<()> {
if self.stall_sends {
return std::future::pending().await;
}
self.sent.send(message)?;
Ok(())
}
async fn receive(&self) -> Result<Option<Value>> {
match self.incoming.lock().await.recv().await {
Some(result) => result,
None => Ok(None),
}
}
async fn close(&self) -> Result<()> {
self.closes.fetch_add(1, Ordering::SeqCst);
Ok(())
}
}
fn session(graceful: bool) -> (LiveSession<TestProtocol>, Controls) {
let (transport, controls) = TestTransport::new();
let (session, _events) = LiveSession::connect(Arc::new(TestProtocol { graceful }), transport);
(session, controls)
}
#[tokio::test]
async fn transport_failure_is_reported_not_swallowed() {
let (session, controls) = session(true);
let mut events = session.subscribe();
controls
.incoming
.send(Err(anyhow!("socket exploded")))
.await
.unwrap();
match events.recv().await.unwrap() {
LiveSessionEvent::Ended { reason, error } => {
assert_eq!(reason, LiveSessionEndReason::TransportFailed);
assert_eq!(error.unwrap().to_string(), "socket exploded");
}
other => panic!("expected Ended, got {other:?}"),
}
assert_eq!(
session.end_reason(),
Some(LiveSessionEndReason::TransportFailed)
);
}
#[tokio::test]
async fn decode_failure_is_reported_not_swallowed() {
let (session, controls) = session(true);
let mut events = session.subscribe();
controls
.incoming
.send(Ok(Some(json!({ "nope": 1 }))))
.await
.unwrap();
match events.recv().await.unwrap() {
LiveSessionEvent::Ended { reason, error } => {
assert_eq!(reason, LiveSessionEndReason::DecodeFailed);
assert!(error.is_some());
}
other => panic!("expected Ended, got {other:?}"),
}
}
#[tokio::test]
async fn clean_transport_shutdown_is_not_a_graceful_acknowledgement() {
let (session, mut controls) = session(true);
let mut events = session.subscribe();
let close = session.close();
let disconnect = async {
controls.sent.recv().await.unwrap();
controls.incoming.send(Ok(None)).await.unwrap();
};
let (result, ()) = tokio::join!(close, disconnect);
assert!(result.is_err());
match events.recv().await.unwrap() {
LiveSessionEvent::Ended { reason, error } => {
assert_eq!(reason, LiveSessionEndReason::TransportClosed);
assert!(error.is_none());
}
other => panic!("expected Ended, got {other:?}"),
}
}
#[tokio::test]
async fn close_sends_protocol_close_command_then_closes_transport() {
let (session, mut controls) = session(true);
let mut events = session.subscribe();
let close = tokio::spawn({
let session = session.clone();
async move { session.close().await }
});
assert_eq!(
controls.sent.recv().await.unwrap(),
json!({ "text": "close" })
);
controls
.incoming
.send(Ok(Some(json!({ "text": "final message" }))))
.await
.unwrap();
controls
.incoming
.send(Ok(Some(json!({ "text": "closed" }))))
.await
.unwrap();
close.await.unwrap().unwrap();
assert!(matches!(
events.recv().await.unwrap(),
LiveSessionEvent::Message(TestEvent::Message(message)) if message == "final message"
));
assert!(matches!(
events.recv().await.unwrap(),
LiveSessionEvent::Message(TestEvent::Closed)
));
assert_eq!(controls.close_count(), 1);
assert_eq!(session.end_reason(), Some(LiveSessionEndReason::Closed));
}
#[tokio::test]
async fn close_without_protocol_command_only_closes_transport() {
let (session, mut controls) = session(false);
session.close().await.unwrap();
assert!(controls.sent.try_recv().is_err());
assert_eq!(controls.close_count(), 1);
}
#[tokio::test]
async fn close_is_idempotent() {
let (session, controls) = session(false);
session.close().await.unwrap();
session.close().await.unwrap();
assert_eq!(controls.close_count(), 1);
assert_eq!(session.end_reason(), Some(LiveSessionEndReason::Closed));
}
#[tokio::test]
async fn concurrent_close_has_one_owner() {
let (session, mut controls) = session(true);
let first = tokio::spawn({
let session = session.clone();
async move { session.close().await }
});
assert_eq!(
controls.sent.recv().await.unwrap(),
json!({ "text": "close" })
);
let second = tokio::spawn({
let session = session.clone();
async move { session.close().await }
});
tokio::task::yield_now().await;
assert!(controls.sent.try_recv().is_err());
controls
.incoming
.send(Ok(Some(json!({ "text": "closed" }))))
.await
.unwrap();
tokio::time::timeout(std::time::Duration::from_secs(2), first)
.await
.expect("owner timed out")
.unwrap()
.unwrap();
tokio::time::timeout(std::time::Duration::from_secs(2), second)
.await
.expect("waiter timed out")
.unwrap()
.unwrap();
assert_eq!(controls.close_count(), 1);
}
#[tokio::test]
async fn send_after_end_fails() {
let (session, controls) = session(true);
let mut events = session.subscribe();
controls.incoming.send(Ok(None)).await.unwrap();
events.recv().await.unwrap();
assert!(session.send("hello".into()).await.is_err());
}
#[tokio::test]
async fn invalid_command_does_not_end_a_healthy_session() {
let (session, mut controls) = session(false);
let error = session.send("invalid".into()).await.unwrap_err();
assert_eq!(error.to_string(), "invalid command");
assert_eq!(session.end_reason(), None);
session.send("valid".into()).await.unwrap();
assert_eq!(
controls.sent.recv().await.unwrap(),
json!({ "text": "valid" })
);
session.close().await.unwrap();
}
#[tokio::test(start_paused = true)]
async fn stalled_send_times_out_and_releases_transport() {
let (transport, controls) = TestTransport::with_stalled_sends(true);
let (session, mut events) =
LiveSession::connect(Arc::new(TestProtocol { graceful: true }), transport);
let error = session.send("blocked".into()).await.unwrap_err();
assert_eq!(error.to_string(), "live transport send timed out");
match events.recv().await.unwrap() {
LiveSessionEvent::Ended { reason, error } => {
assert_eq!(reason, LiveSessionEndReason::TransportFailed);
assert_eq!(error.unwrap().to_string(), "live transport send timed out");
}
other => panic!("expected Ended, got {other:?}"),
}
assert_eq!(controls.close_count(), 1);
}
#[tokio::test(start_paused = true)]
async fn missing_close_acknowledgement_is_reported_as_timeout() {
let (session, mut controls) = session(true);
let close = tokio::spawn({
let session = session.clone();
async move { session.close().await }
});
assert_eq!(
controls.sent.recv().await.unwrap(),
json!({ "text": "close" })
);
let error = close.await.unwrap().unwrap_err();
assert_eq!(
error.to_string(),
"live session close acknowledgement timed out"
);
assert_eq!(
session.end_reason(),
Some(LiveSessionEndReason::CloseTimedOut)
);
assert_eq!(controls.close_count(), 1);
}
#[tokio::test]
async fn messages_are_delivered_before_end() {
let (session, controls) = session(true);
let mut events = session.subscribe();
controls
.incoming
.send(Ok(Some(json!({ "text": "hi" }))))
.await
.unwrap();
controls.incoming.send(Ok(None)).await.unwrap();
assert!(matches!(
events.recv().await.unwrap(),
LiveSessionEvent::Message(TestEvent::Message(text)) if text == "hi"
));
assert!(matches!(
events.recv().await.unwrap(),
LiveSessionEvent::Ended { .. }
));
}
#[tokio::test]
async fn dropping_session_stops_actor() {
let (transport, controls) = TestTransport::new();
let (session, events) =
LiveSession::connect(Arc::new(TestProtocol { graceful: true }), transport);
drop(session);
drop(events);
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
assert!(controls.close_count() >= 1);
}
@@ -0,0 +1,58 @@
#![cfg(feature = "live-websocket")]
use anyhow::Result;
use futures::{SinkExt, StreamExt};
use goose_providers::openai_live::{OpenAiLiveClient, OpenAiLiveSessionConfig};
use serde_json::{json, Value};
use tokio::net::TcpListener;
use tokio_tungstenite::{accept_hdr_async, tungstenite::Message};
#[tokio::test]
#[allow(clippy::result_large_err)]
async fn websocket_connect_performs_handshake_and_waits_until_ready() -> Result<()> {
let listener = TcpListener::bind("127.0.0.1:0").await?;
let address = listener.local_addr()?;
let server = tokio::spawn(async move {
let (stream, _) = listener.accept().await.unwrap();
let mut socket = accept_hdr_async(stream, |request: &http::Request<()>, response| {
assert_eq!(request.headers()["authorization"], "Bearer test-key");
Ok(response)
})
.await
.unwrap();
let message = socket.next().await.unwrap().unwrap();
let Message::Text(text) = message else {
panic!("expected initial text message");
};
let session_start: Value = serde_json::from_str(&text).unwrap();
assert_eq!(session_start["type"], "session.start");
socket
.send(Message::Text(
json!({
"type": "session.started",
"event_id": "event_started_1",
"session": { "id": "session_1" }
})
.to_string()
.into(),
))
.await
.unwrap();
});
let connected = OpenAiLiveClient::new("test-key")
.with_websocket_endpoint(format!("ws://{address}/v1/live/sessions"))
.websocket(OpenAiLiveSessionConfig {
model: "gpt-live-test".into(),
instructions: "test".into(),
voice: None,
input_messages: vec![],
extra_session_fields: Default::default(),
})
.connect()
.await?;
let _connected = connected;
server.await?;
Ok(())
}
+5 -5
View File
@@ -155,14 +155,14 @@ sqlx = { version = "0.9.0", default-features = false, features = [
] }
# For Bedrock provider (optional, behind "aws-providers" feature)
aws-config = { version = "1.8", default-features = false, features = ["credentials-process", "rt-tokio", "sso", "behavior-version-latest"], optional = true }
aws-smithy-types = { version = "1.3.4", default-features = false, features = ["rt-tokio"], optional = true }
aws-sdk-bedrockruntime = { version = "1.132", default-features = false, features = ["rt-tokio"], optional = true }
aws-smithy-runtime-api = { version = "1.13", default-features = false, features = ["client"], optional = true }
aws-config = { version = "=1.12.0", default-features = false, features = ["credentials-process", "rt-tokio", "sso", "behavior-version-latest"], optional = true }
aws-smithy-types = { version = "=1.6.3", default-features = false, features = ["rt-tokio"], optional = true }
aws-sdk-bedrockruntime = { version = "=1.143.0", default-features = false, features = ["rt-tokio"], optional = true }
aws-smithy-runtime-api = { version = "=1.16.0", default-features = false, features = ["client"], optional = true }
smithy-transport-reqwest = { version = "0.1", default-features = false, features = ["http2", "system-proxy"], optional = true }
# For SageMaker TGI provider (optional, behind "aws-providers" feature)
aws-sdk-sagemakerruntime = { version = "1.104", default-features = false, features = ["rt-tokio"], optional = true }
aws-sdk-sagemakerruntime = { version = "=1.113.0", default-features = false, features = ["rt-tokio"], optional = true }
# For GCP Vertex AI provider auth
jsonwebtoken = { version = "11.0", default-features = false, features = ["use_pem"] }