diff --git a/crates/goose/src/providers/litellm.rs b/crates/goose/src/providers/litellm.rs index 285f220e9..a9263079b 100644 --- a/crates/goose/src/providers/litellm.rs +++ b/crates/goose/src/providers/litellm.rs @@ -14,7 +14,9 @@ use super::base::{ ConfigKey, MessageStream, ModelInfo, Provider, ProviderDef, ProviderMetadata, DEFAULT_PROVIDER_TIMEOUT_SECS, }; -use super::openai_compatible::handle_response_openai_compat; +use super::openai_compatible::{ + handle_response_openai_compat, handle_status, stream_openai_compat, +}; use super::retry::ProviderRetry; use super::utils::get_model; use crate::conversation::message::Message; @@ -41,6 +43,7 @@ pub struct LiteLLMProvider { #[serde(skip)] api_client: ApiClient, base_path: String, + supports_streaming: bool, #[serde(skip)] name: String, #[serde(skip)] @@ -69,6 +72,9 @@ impl LiteLLMProvider { let timeout_secs: u64 = config .get_param("LITELLM_TIMEOUT") .unwrap_or(DEFAULT_PROVIDER_TIMEOUT_SECS); + let supports_streaming: bool = config + .get_param("LITELLM_SUPPORTS_STREAMING") + .unwrap_or(true); let auth = if api_key.is_empty() { AuthMethod::NoAuth @@ -97,6 +103,7 @@ impl LiteLLMProvider { Ok(Self { api_client, base_path, + supports_streaming, name: LITELLM_PROVIDER_NAME.to_string(), cached_model_info: tokio::sync::Mutex::new(None), }) @@ -180,14 +187,14 @@ impl LiteLLMProvider { &self, model_config: &ModelConfig, payload: &Value, - ) -> Result { - let response = self + ) -> Result { + Ok(self .api_client .request(&self.base_path) .model_headers(model_config)? + .streaming(self.supports_streaming) .response_post(payload) - .await?; - handle_response_openai_compat(response).await + .await?) } async fn supports_cache_control(&self, model: &ModelConfig) -> bool { @@ -228,6 +235,13 @@ impl goose_providers::base::ProviderDescriptor for LiteLLMProvider { ), ConfigKey::new("LITELLM_CUSTOM_HEADERS", false, true, None, false), ConfigKey::new("LITELLM_TIMEOUT", false, false, Some("600"), false), + ConfigKey::new( + "LITELLM_SUPPORTS_STREAMING", + false, + false, + Some("true"), + false, + ), ], ) .with_setup( @@ -295,7 +309,7 @@ impl Provider for LiteLLMProvider { messages, tools, &ImageFormat::OpenAi, - false, + self.supports_streaming, )?; if !model_config.prompt_cache_disabled() && self.supports_cache_control(model_config).await @@ -303,17 +317,37 @@ impl Provider for LiteLLMProvider { apply_chat_payload_breakpoints(&mut payload); } + let mut log = start_log(model_config, &payload)?; + + if self.supports_streaming { + let response = self + .with_retry(|| async { + let payload_clone = payload.clone(); + let response = self.post(model_config, &payload_clone).await?; + handle_status(response).await + }) + .await + .inspect_err(|error| { + let _ = log.error(error); + })?; + + return stream_openai_compat(response, log); + } + let response = self .with_retry(|| async { let payload_clone = payload.clone(); - self.post(model_config, &payload_clone).await + let response = self.post(model_config, &payload_clone).await?; + handle_response_openai_compat(response).await }) - .await?; + .await + .inspect_err(|error| { + let _ = log.error(error); + })?; let message = goose_providers::formats::openai::response_to_message(&response)?; let usage = goose_providers::formats::openai::get_usage(&response); let response_model = get_model(&response); - let mut log = start_log(model_config, &payload)?; log.write(&response, Some(&usage))?; let provider_usage = ProviderUsage::new(response_model, usage); Ok(super::base::stream_from_single_message( @@ -345,6 +379,213 @@ fn parse_custom_headers(headers_str: String) -> HashMap { #[cfg(test)] mod tests { use super::*; + use futures::StreamExt; + use goose_providers::request_log::{install_logger, RequestLogHandle, RequestLogger}; + use serde_json::json; + use std::sync::{Mutex, Once}; + use wiremock::matchers::{method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + static CAPTURED_REQUEST_LOGS: Mutex>> = Mutex::new(Vec::new()); + static INSTALL_REQUEST_LOGGER: Once = Once::new(); + + struct CapturingRequestLogger; + struct CapturingRequestLogHandle(usize); + + impl RequestLogger for CapturingRequestLogger { + fn start( + &self, + ) -> std::result::Result, Box> + { + let mut requests = CAPTURED_REQUEST_LOGS.lock().unwrap(); + requests.push(Vec::new()); + Ok(Box::new(CapturingRequestLogHandle(requests.len() - 1))) + } + } + + impl RequestLogHandle for CapturingRequestLogHandle { + fn write( + &mut self, + line: &str, + ) -> std::result::Result<(), Box> { + CAPTURED_REQUEST_LOGS.lock().unwrap()[self.0].push(line.to_string()); + Ok(()) + } + } + + fn install_capturing_request_logger() { + INSTALL_REQUEST_LOGGER.call_once(|| { + install_logger(CapturingRequestLogger) + .expect("test request logger should install once"); + }); + } + + fn captured_request_log(model: &str) -> Vec { + CAPTURED_REQUEST_LOGS + .lock() + .unwrap() + .iter() + .find(|request| { + request + .first() + .is_some_and(|line| line.contains(&format!(r#""model_name":"{model}""#))) + }) + .unwrap_or_else(|| panic!("request log for {model} should exist")) + .iter() + .map(|line| serde_json::from_str(line).unwrap()) + .collect() + } + + #[tokio::test] + async fn streaming_request_consumes_sse_and_reports_usage() { + install_capturing_request_logger(); + let server = MockServer::start().await; + let sse_body = concat!( + r#"data: {"id":"chatcmpl-litellm","object":"chat.completion.chunk","created":0,"model":"test-model","choices":[{"index":0,"delta":{"role":"assistant","content":"hello"},"finish_reason":null}]}"#, + "\n\n", + r#"data: {"id":"chatcmpl-litellm","object":"chat.completion.chunk","created":0,"model":"test-model","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}"#, + "\n\n", + r#"data: {"id":"chatcmpl-litellm","object":"chat.completion.chunk","created":0,"model":"test-model","choices":[],"usage":{"prompt_tokens":3,"completion_tokens":1,"total_tokens":4}}"#, + "\n\n", + "data: [DONE]", + ); + Mock::given(method("POST")) + .and(path("/v1/chat/completions")) + .respond_with(ResponseTemplate::new(200).set_body_raw(sse_body, "text/event-stream")) + .mount(&server) + .await; + + let provider = LiteLLMProvider { + api_client: ApiClient::new_with_tls(server.uri(), AuthMethod::NoAuth, None).unwrap(), + base_path: "v1/chat/completions".to_string(), + supports_streaming: true, + name: LITELLM_PROVIDER_NAME.to_string(), + cached_model_info: tokio::sync::Mutex::new(None), + }; + let model = ModelConfig::new("litellm-streaming-log-test").with_prompt_cache_disabled(); + + let mut stream = provider + .stream(&model, "", &[], &[]) + .await + .expect("LiteLLM streaming request should consume SSE"); + let mut text = String::new(); + let mut usage = None; + while let Some(item) = stream.next().await { + let (message, item_usage) = item.expect("SSE item should parse"); + if let Some(message) = message { + for content in message.content { + text.push_str(&content.to_string()); + } + } + if item_usage.is_some() { + usage = item_usage; + } + } + + assert_eq!(text, "hello"); + let usage = usage.expect("SSE usage chunk should be reported"); + assert_eq!(usage.model, "test-model"); + assert_eq!(usage.usage.input_tokens, Some(3)); + assert_eq!(usage.usage.output_tokens, Some(1)); + assert_eq!(usage.usage.total_tokens, Some(4)); + + let requests = server.received_requests().await.unwrap(); + assert_eq!(requests.len(), 1); + let body: Value = serde_json::from_slice(&requests[0].body).unwrap(); + assert_eq!(body["stream"], json!(true)); + assert_eq!(body["stream_options"], json!({"include_usage": true})); + + let log = captured_request_log("litellm-streaming-log-test"); + assert_eq!(log[0]["input"]["stream"], json!(true)); + assert!(log.iter().skip(1).any(|entry| !entry["data"].is_null())); + assert!(log + .iter() + .skip(1) + .any(|entry| entry["usage"]["total_tokens"] == json!(4))); + } + + #[tokio::test] + async fn non_streaming_request_preserves_json_response_path() { + install_capturing_request_logger(); + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/v1/chat/completions")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "id": "chatcmpl-litellm-json", + "object": "chat.completion", + "created": 0, + "model": "test-model", + "choices": [{ + "index": 0, + "message": {"role": "assistant", "content": "hello from JSON"}, + "finish_reason": "stop" + }], + "usage": {"prompt_tokens": 5, "completion_tokens": 3, "total_tokens": 8} + }))) + .mount(&server) + .await; + + let provider = LiteLLMProvider { + api_client: ApiClient::new_with_tls(server.uri(), AuthMethod::NoAuth, None).unwrap(), + base_path: "v1/chat/completions".to_string(), + supports_streaming: false, + name: LITELLM_PROVIDER_NAME.to_string(), + cached_model_info: tokio::sync::Mutex::new(None), + }; + let model = ModelConfig::new("litellm-json-log-test").with_prompt_cache_disabled(); + + let mut stream = provider + .stream(&model, "", &[], &[]) + .await + .expect("LiteLLM JSON response should remain supported"); + let (message, usage) = stream + .next() + .await + .expect("JSON response should yield one item") + .expect("JSON response item should parse"); + + let text = message + .expect("JSON response should contain a message") + .content + .into_iter() + .map(|content| content.to_string()) + .collect::(); + assert_eq!(text, "hello from JSON"); + let usage = usage.expect("JSON response usage should be reported"); + assert_eq!(usage.model, "test-model"); + assert_eq!(usage.usage.input_tokens, Some(5)); + assert_eq!(usage.usage.output_tokens, Some(3)); + assert_eq!(usage.usage.total_tokens, Some(8)); + assert!(stream.next().await.is_none()); + + let requests = server.received_requests().await.unwrap(); + assert_eq!(requests.len(), 1); + let body: Value = serde_json::from_slice(&requests[0].body).unwrap(); + assert!(body.get("stream").is_none()); + assert!(body.get("stream_options").is_none()); + + let log = captured_request_log("litellm-json-log-test"); + assert!(log[0]["input"].get("stream").is_none()); + assert!(log.iter().skip(1).any(|entry| !entry["data"].is_null())); + assert!(log + .iter() + .skip(1) + .any(|entry| entry["usage"]["total_tokens"] == json!(8))); + } + + #[test] + fn streaming_config_defaults_to_true() { + let metadata = ::metadata(); + let key = metadata + .config_keys + .iter() + .find(|key| key.name == "LITELLM_SUPPORTS_STREAMING") + .expect("streaming config key should be exposed"); + + assert_eq!(key.default.as_deref(), Some("true")); + assert!(!key.required); + assert!(!key.secret); + } #[tokio::test] async fn context_limit_negative_caches_failed_model_info() { @@ -356,6 +597,7 @@ mod tests { ) .unwrap(), base_path: "v1/chat/completions".to_string(), + supports_streaming: true, name: LITELLM_PROVIDER_NAME.to_string(), cached_model_info: tokio::sync::Mutex::new(None), }; @@ -384,6 +626,7 @@ mod tests { ) .unwrap(), base_path: "v1/chat/completions".to_string(), + supports_streaming: true, name: LITELLM_PROVIDER_NAME.to_string(), cached_model_info: tokio::sync::Mutex::new(Some(CachedModelInfo::Failure( Instant::now() - MODEL_INFO_FAILURE_TTL, @@ -413,6 +656,7 @@ mod tests { ) .unwrap(), base_path: "v1/chat/completions".to_string(), + supports_streaming: true, name: LITELLM_PROVIDER_NAME.to_string(), cached_model_info, };