fix(litellm): stream chat completions by default (#11870)

This commit is contained in:
Fnine59
2026-09-08 19:22:20 +00:00
committed by GitHub
parent 6d663d884a
commit 1c3c53066f
+253 -9
View File
@@ -14,7 +14,9 @@ use super::base::{
ConfigKey, MessageStream, ModelInfo, Provider, ProviderDef, ProviderMetadata,
DEFAULT_PROVIDER_TIMEOUT_SECS,
};
use super::openai_compatible::handle_response_openai_compat;
use super::openai_compatible::{
handle_response_openai_compat, handle_status, stream_openai_compat,
};
use super::retry::ProviderRetry;
use super::utils::get_model;
use crate::conversation::message::Message;
@@ -41,6 +43,7 @@ pub struct LiteLLMProvider {
#[serde(skip)]
api_client: ApiClient,
base_path: String,
supports_streaming: bool,
#[serde(skip)]
name: String,
#[serde(skip)]
@@ -69,6 +72,9 @@ impl LiteLLMProvider {
let timeout_secs: u64 = config
.get_param("LITELLM_TIMEOUT")
.unwrap_or(DEFAULT_PROVIDER_TIMEOUT_SECS);
let supports_streaming: bool = config
.get_param("LITELLM_SUPPORTS_STREAMING")
.unwrap_or(true);
let auth = if api_key.is_empty() {
AuthMethod::NoAuth
@@ -97,6 +103,7 @@ impl LiteLLMProvider {
Ok(Self {
api_client,
base_path,
supports_streaming,
name: LITELLM_PROVIDER_NAME.to_string(),
cached_model_info: tokio::sync::Mutex::new(None),
})
@@ -180,14 +187,14 @@ impl LiteLLMProvider {
&self,
model_config: &ModelConfig,
payload: &Value,
) -> Result<Value, ProviderError> {
let response = self
) -> Result<reqwest::Response, ProviderError> {
Ok(self
.api_client
.request(&self.base_path)
.model_headers(model_config)?
.streaming(self.supports_streaming)
.response_post(payload)
.await?;
handle_response_openai_compat(response).await
.await?)
}
async fn supports_cache_control(&self, model: &ModelConfig) -> bool {
@@ -228,6 +235,13 @@ impl goose_providers::base::ProviderDescriptor for LiteLLMProvider {
),
ConfigKey::new("LITELLM_CUSTOM_HEADERS", false, true, None, false),
ConfigKey::new("LITELLM_TIMEOUT", false, false, Some("600"), false),
ConfigKey::new(
"LITELLM_SUPPORTS_STREAMING",
false,
false,
Some("true"),
false,
),
],
)
.with_setup(
@@ -295,7 +309,7 @@ impl Provider for LiteLLMProvider {
messages,
tools,
&ImageFormat::OpenAi,
false,
self.supports_streaming,
)?;
if !model_config.prompt_cache_disabled() && self.supports_cache_control(model_config).await
@@ -303,17 +317,37 @@ impl Provider for LiteLLMProvider {
apply_chat_payload_breakpoints(&mut payload);
}
let mut log = start_log(model_config, &payload)?;
if self.supports_streaming {
let response = self
.with_retry(|| async {
let payload_clone = payload.clone();
let response = self.post(model_config, &payload_clone).await?;
handle_status(response).await
})
.await
.inspect_err(|error| {
let _ = log.error(error);
})?;
return stream_openai_compat(response, log);
}
let response = self
.with_retry(|| async {
let payload_clone = payload.clone();
self.post(model_config, &payload_clone).await
let response = self.post(model_config, &payload_clone).await?;
handle_response_openai_compat(response).await
})
.await?;
.await
.inspect_err(|error| {
let _ = log.error(error);
})?;
let message = goose_providers::formats::openai::response_to_message(&response)?;
let usage = goose_providers::formats::openai::get_usage(&response);
let response_model = get_model(&response);
let mut log = start_log(model_config, &payload)?;
log.write(&response, Some(&usage))?;
let provider_usage = ProviderUsage::new(response_model, usage);
Ok(super::base::stream_from_single_message(
@@ -345,6 +379,213 @@ fn parse_custom_headers(headers_str: String) -> HashMap<String, String> {
#[cfg(test)]
mod tests {
use super::*;
use futures::StreamExt;
use goose_providers::request_log::{install_logger, RequestLogHandle, RequestLogger};
use serde_json::json;
use std::sync::{Mutex, Once};
use wiremock::matchers::{method, path};
use wiremock::{Mock, MockServer, ResponseTemplate};
static CAPTURED_REQUEST_LOGS: Mutex<Vec<Vec<String>>> = Mutex::new(Vec::new());
static INSTALL_REQUEST_LOGGER: Once = Once::new();
struct CapturingRequestLogger;
struct CapturingRequestLogHandle(usize);
impl RequestLogger for CapturingRequestLogger {
fn start(
&self,
) -> std::result::Result<Box<dyn RequestLogHandle>, Box<dyn std::error::Error + Send + Sync>>
{
let mut requests = CAPTURED_REQUEST_LOGS.lock().unwrap();
requests.push(Vec::new());
Ok(Box::new(CapturingRequestLogHandle(requests.len() - 1)))
}
}
impl RequestLogHandle for CapturingRequestLogHandle {
fn write(
&mut self,
line: &str,
) -> std::result::Result<(), Box<dyn std::error::Error + Send + Sync>> {
CAPTURED_REQUEST_LOGS.lock().unwrap()[self.0].push(line.to_string());
Ok(())
}
}
fn install_capturing_request_logger() {
INSTALL_REQUEST_LOGGER.call_once(|| {
install_logger(CapturingRequestLogger)
.expect("test request logger should install once");
});
}
fn captured_request_log(model: &str) -> Vec<Value> {
CAPTURED_REQUEST_LOGS
.lock()
.unwrap()
.iter()
.find(|request| {
request
.first()
.is_some_and(|line| line.contains(&format!(r#""model_name":"{model}""#)))
})
.unwrap_or_else(|| panic!("request log for {model} should exist"))
.iter()
.map(|line| serde_json::from_str(line).unwrap())
.collect()
}
#[tokio::test]
async fn streaming_request_consumes_sse_and_reports_usage() {
install_capturing_request_logger();
let server = MockServer::start().await;
let sse_body = concat!(
r#"data: {"id":"chatcmpl-litellm","object":"chat.completion.chunk","created":0,"model":"test-model","choices":[{"index":0,"delta":{"role":"assistant","content":"hello"},"finish_reason":null}]}"#,
"\n\n",
r#"data: {"id":"chatcmpl-litellm","object":"chat.completion.chunk","created":0,"model":"test-model","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}"#,
"\n\n",
r#"data: {"id":"chatcmpl-litellm","object":"chat.completion.chunk","created":0,"model":"test-model","choices":[],"usage":{"prompt_tokens":3,"completion_tokens":1,"total_tokens":4}}"#,
"\n\n",
"data: [DONE]",
);
Mock::given(method("POST"))
.and(path("/v1/chat/completions"))
.respond_with(ResponseTemplate::new(200).set_body_raw(sse_body, "text/event-stream"))
.mount(&server)
.await;
let provider = LiteLLMProvider {
api_client: ApiClient::new_with_tls(server.uri(), AuthMethod::NoAuth, None).unwrap(),
base_path: "v1/chat/completions".to_string(),
supports_streaming: true,
name: LITELLM_PROVIDER_NAME.to_string(),
cached_model_info: tokio::sync::Mutex::new(None),
};
let model = ModelConfig::new("litellm-streaming-log-test").with_prompt_cache_disabled();
let mut stream = provider
.stream(&model, "", &[], &[])
.await
.expect("LiteLLM streaming request should consume SSE");
let mut text = String::new();
let mut usage = None;
while let Some(item) = stream.next().await {
let (message, item_usage) = item.expect("SSE item should parse");
if let Some(message) = message {
for content in message.content {
text.push_str(&content.to_string());
}
}
if item_usage.is_some() {
usage = item_usage;
}
}
assert_eq!(text, "hello");
let usage = usage.expect("SSE usage chunk should be reported");
assert_eq!(usage.model, "test-model");
assert_eq!(usage.usage.input_tokens, Some(3));
assert_eq!(usage.usage.output_tokens, Some(1));
assert_eq!(usage.usage.total_tokens, Some(4));
let requests = server.received_requests().await.unwrap();
assert_eq!(requests.len(), 1);
let body: Value = serde_json::from_slice(&requests[0].body).unwrap();
assert_eq!(body["stream"], json!(true));
assert_eq!(body["stream_options"], json!({"include_usage": true}));
let log = captured_request_log("litellm-streaming-log-test");
assert_eq!(log[0]["input"]["stream"], json!(true));
assert!(log.iter().skip(1).any(|entry| !entry["data"].is_null()));
assert!(log
.iter()
.skip(1)
.any(|entry| entry["usage"]["total_tokens"] == json!(4)));
}
#[tokio::test]
async fn non_streaming_request_preserves_json_response_path() {
install_capturing_request_logger();
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/v1/chat/completions"))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
"id": "chatcmpl-litellm-json",
"object": "chat.completion",
"created": 0,
"model": "test-model",
"choices": [{
"index": 0,
"message": {"role": "assistant", "content": "hello from JSON"},
"finish_reason": "stop"
}],
"usage": {"prompt_tokens": 5, "completion_tokens": 3, "total_tokens": 8}
})))
.mount(&server)
.await;
let provider = LiteLLMProvider {
api_client: ApiClient::new_with_tls(server.uri(), AuthMethod::NoAuth, None).unwrap(),
base_path: "v1/chat/completions".to_string(),
supports_streaming: false,
name: LITELLM_PROVIDER_NAME.to_string(),
cached_model_info: tokio::sync::Mutex::new(None),
};
let model = ModelConfig::new("litellm-json-log-test").with_prompt_cache_disabled();
let mut stream = provider
.stream(&model, "", &[], &[])
.await
.expect("LiteLLM JSON response should remain supported");
let (message, usage) = stream
.next()
.await
.expect("JSON response should yield one item")
.expect("JSON response item should parse");
let text = message
.expect("JSON response should contain a message")
.content
.into_iter()
.map(|content| content.to_string())
.collect::<String>();
assert_eq!(text, "hello from JSON");
let usage = usage.expect("JSON response usage should be reported");
assert_eq!(usage.model, "test-model");
assert_eq!(usage.usage.input_tokens, Some(5));
assert_eq!(usage.usage.output_tokens, Some(3));
assert_eq!(usage.usage.total_tokens, Some(8));
assert!(stream.next().await.is_none());
let requests = server.received_requests().await.unwrap();
assert_eq!(requests.len(), 1);
let body: Value = serde_json::from_slice(&requests[0].body).unwrap();
assert!(body.get("stream").is_none());
assert!(body.get("stream_options").is_none());
let log = captured_request_log("litellm-json-log-test");
assert!(log[0]["input"].get("stream").is_none());
assert!(log.iter().skip(1).any(|entry| !entry["data"].is_null()));
assert!(log
.iter()
.skip(1)
.any(|entry| entry["usage"]["total_tokens"] == json!(8)));
}
#[test]
fn streaming_config_defaults_to_true() {
let metadata = <LiteLLMProvider as goose_providers::base::ProviderDescriptor>::metadata();
let key = metadata
.config_keys
.iter()
.find(|key| key.name == "LITELLM_SUPPORTS_STREAMING")
.expect("streaming config key should be exposed");
assert_eq!(key.default.as_deref(), Some("true"));
assert!(!key.required);
assert!(!key.secret);
}
#[tokio::test]
async fn context_limit_negative_caches_failed_model_info() {
@@ -356,6 +597,7 @@ mod tests {
)
.unwrap(),
base_path: "v1/chat/completions".to_string(),
supports_streaming: true,
name: LITELLM_PROVIDER_NAME.to_string(),
cached_model_info: tokio::sync::Mutex::new(None),
};
@@ -384,6 +626,7 @@ mod tests {
)
.unwrap(),
base_path: "v1/chat/completions".to_string(),
supports_streaming: true,
name: LITELLM_PROVIDER_NAME.to_string(),
cached_model_info: tokio::sync::Mutex::new(Some(CachedModelInfo::Failure(
Instant::now() - MODEL_INFO_FAILURE_TTL,
@@ -413,6 +656,7 @@ mod tests {
)
.unwrap(),
base_path: "v1/chat/completions".to_string(),
supports_streaming: true,
name: LITELLM_PROVIDER_NAME.to_string(),
cached_model_info,
};