fix(litellm): stream chat completions by default (#11870)
This commit is contained in:
@@ -14,7 +14,9 @@ use super::base::{
|
||||
ConfigKey, MessageStream, ModelInfo, Provider, ProviderDef, ProviderMetadata,
|
||||
DEFAULT_PROVIDER_TIMEOUT_SECS,
|
||||
};
|
||||
use super::openai_compatible::handle_response_openai_compat;
|
||||
use super::openai_compatible::{
|
||||
handle_response_openai_compat, handle_status, stream_openai_compat,
|
||||
};
|
||||
use super::retry::ProviderRetry;
|
||||
use super::utils::get_model;
|
||||
use crate::conversation::message::Message;
|
||||
@@ -41,6 +43,7 @@ pub struct LiteLLMProvider {
|
||||
#[serde(skip)]
|
||||
api_client: ApiClient,
|
||||
base_path: String,
|
||||
supports_streaming: bool,
|
||||
#[serde(skip)]
|
||||
name: String,
|
||||
#[serde(skip)]
|
||||
@@ -69,6 +72,9 @@ impl LiteLLMProvider {
|
||||
let timeout_secs: u64 = config
|
||||
.get_param("LITELLM_TIMEOUT")
|
||||
.unwrap_or(DEFAULT_PROVIDER_TIMEOUT_SECS);
|
||||
let supports_streaming: bool = config
|
||||
.get_param("LITELLM_SUPPORTS_STREAMING")
|
||||
.unwrap_or(true);
|
||||
|
||||
let auth = if api_key.is_empty() {
|
||||
AuthMethod::NoAuth
|
||||
@@ -97,6 +103,7 @@ impl LiteLLMProvider {
|
||||
Ok(Self {
|
||||
api_client,
|
||||
base_path,
|
||||
supports_streaming,
|
||||
name: LITELLM_PROVIDER_NAME.to_string(),
|
||||
cached_model_info: tokio::sync::Mutex::new(None),
|
||||
})
|
||||
@@ -180,14 +187,14 @@ impl LiteLLMProvider {
|
||||
&self,
|
||||
model_config: &ModelConfig,
|
||||
payload: &Value,
|
||||
) -> Result<Value, ProviderError> {
|
||||
let response = self
|
||||
) -> Result<reqwest::Response, ProviderError> {
|
||||
Ok(self
|
||||
.api_client
|
||||
.request(&self.base_path)
|
||||
.model_headers(model_config)?
|
||||
.streaming(self.supports_streaming)
|
||||
.response_post(payload)
|
||||
.await?;
|
||||
handle_response_openai_compat(response).await
|
||||
.await?)
|
||||
}
|
||||
|
||||
async fn supports_cache_control(&self, model: &ModelConfig) -> bool {
|
||||
@@ -228,6 +235,13 @@ impl goose_providers::base::ProviderDescriptor for LiteLLMProvider {
|
||||
),
|
||||
ConfigKey::new("LITELLM_CUSTOM_HEADERS", false, true, None, false),
|
||||
ConfigKey::new("LITELLM_TIMEOUT", false, false, Some("600"), false),
|
||||
ConfigKey::new(
|
||||
"LITELLM_SUPPORTS_STREAMING",
|
||||
false,
|
||||
false,
|
||||
Some("true"),
|
||||
false,
|
||||
),
|
||||
],
|
||||
)
|
||||
.with_setup(
|
||||
@@ -295,7 +309,7 @@ impl Provider for LiteLLMProvider {
|
||||
messages,
|
||||
tools,
|
||||
&ImageFormat::OpenAi,
|
||||
false,
|
||||
self.supports_streaming,
|
||||
)?;
|
||||
|
||||
if !model_config.prompt_cache_disabled() && self.supports_cache_control(model_config).await
|
||||
@@ -303,17 +317,37 @@ impl Provider for LiteLLMProvider {
|
||||
apply_chat_payload_breakpoints(&mut payload);
|
||||
}
|
||||
|
||||
let mut log = start_log(model_config, &payload)?;
|
||||
|
||||
if self.supports_streaming {
|
||||
let response = self
|
||||
.with_retry(|| async {
|
||||
let payload_clone = payload.clone();
|
||||
let response = self.post(model_config, &payload_clone).await?;
|
||||
handle_status(response).await
|
||||
})
|
||||
.await
|
||||
.inspect_err(|error| {
|
||||
let _ = log.error(error);
|
||||
})?;
|
||||
|
||||
return stream_openai_compat(response, log);
|
||||
}
|
||||
|
||||
let response = self
|
||||
.with_retry(|| async {
|
||||
let payload_clone = payload.clone();
|
||||
self.post(model_config, &payload_clone).await
|
||||
let response = self.post(model_config, &payload_clone).await?;
|
||||
handle_response_openai_compat(response).await
|
||||
})
|
||||
.await?;
|
||||
.await
|
||||
.inspect_err(|error| {
|
||||
let _ = log.error(error);
|
||||
})?;
|
||||
|
||||
let message = goose_providers::formats::openai::response_to_message(&response)?;
|
||||
let usage = goose_providers::formats::openai::get_usage(&response);
|
||||
let response_model = get_model(&response);
|
||||
let mut log = start_log(model_config, &payload)?;
|
||||
log.write(&response, Some(&usage))?;
|
||||
let provider_usage = ProviderUsage::new(response_model, usage);
|
||||
Ok(super::base::stream_from_single_message(
|
||||
@@ -345,6 +379,213 @@ fn parse_custom_headers(headers_str: String) -> HashMap<String, String> {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use futures::StreamExt;
|
||||
use goose_providers::request_log::{install_logger, RequestLogHandle, RequestLogger};
|
||||
use serde_json::json;
|
||||
use std::sync::{Mutex, Once};
|
||||
use wiremock::matchers::{method, path};
|
||||
use wiremock::{Mock, MockServer, ResponseTemplate};
|
||||
|
||||
static CAPTURED_REQUEST_LOGS: Mutex<Vec<Vec<String>>> = Mutex::new(Vec::new());
|
||||
static INSTALL_REQUEST_LOGGER: Once = Once::new();
|
||||
|
||||
struct CapturingRequestLogger;
|
||||
struct CapturingRequestLogHandle(usize);
|
||||
|
||||
impl RequestLogger for CapturingRequestLogger {
|
||||
fn start(
|
||||
&self,
|
||||
) -> std::result::Result<Box<dyn RequestLogHandle>, Box<dyn std::error::Error + Send + Sync>>
|
||||
{
|
||||
let mut requests = CAPTURED_REQUEST_LOGS.lock().unwrap();
|
||||
requests.push(Vec::new());
|
||||
Ok(Box::new(CapturingRequestLogHandle(requests.len() - 1)))
|
||||
}
|
||||
}
|
||||
|
||||
impl RequestLogHandle for CapturingRequestLogHandle {
|
||||
fn write(
|
||||
&mut self,
|
||||
line: &str,
|
||||
) -> std::result::Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
CAPTURED_REQUEST_LOGS.lock().unwrap()[self.0].push(line.to_string());
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn install_capturing_request_logger() {
|
||||
INSTALL_REQUEST_LOGGER.call_once(|| {
|
||||
install_logger(CapturingRequestLogger)
|
||||
.expect("test request logger should install once");
|
||||
});
|
||||
}
|
||||
|
||||
fn captured_request_log(model: &str) -> Vec<Value> {
|
||||
CAPTURED_REQUEST_LOGS
|
||||
.lock()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.find(|request| {
|
||||
request
|
||||
.first()
|
||||
.is_some_and(|line| line.contains(&format!(r#""model_name":"{model}""#)))
|
||||
})
|
||||
.unwrap_or_else(|| panic!("request log for {model} should exist"))
|
||||
.iter()
|
||||
.map(|line| serde_json::from_str(line).unwrap())
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn streaming_request_consumes_sse_and_reports_usage() {
|
||||
install_capturing_request_logger();
|
||||
let server = MockServer::start().await;
|
||||
let sse_body = concat!(
|
||||
r#"data: {"id":"chatcmpl-litellm","object":"chat.completion.chunk","created":0,"model":"test-model","choices":[{"index":0,"delta":{"role":"assistant","content":"hello"},"finish_reason":null}]}"#,
|
||||
"\n\n",
|
||||
r#"data: {"id":"chatcmpl-litellm","object":"chat.completion.chunk","created":0,"model":"test-model","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}"#,
|
||||
"\n\n",
|
||||
r#"data: {"id":"chatcmpl-litellm","object":"chat.completion.chunk","created":0,"model":"test-model","choices":[],"usage":{"prompt_tokens":3,"completion_tokens":1,"total_tokens":4}}"#,
|
||||
"\n\n",
|
||||
"data: [DONE]",
|
||||
);
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/v1/chat/completions"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_raw(sse_body, "text/event-stream"))
|
||||
.mount(&server)
|
||||
.await;
|
||||
|
||||
let provider = LiteLLMProvider {
|
||||
api_client: ApiClient::new_with_tls(server.uri(), AuthMethod::NoAuth, None).unwrap(),
|
||||
base_path: "v1/chat/completions".to_string(),
|
||||
supports_streaming: true,
|
||||
name: LITELLM_PROVIDER_NAME.to_string(),
|
||||
cached_model_info: tokio::sync::Mutex::new(None),
|
||||
};
|
||||
let model = ModelConfig::new("litellm-streaming-log-test").with_prompt_cache_disabled();
|
||||
|
||||
let mut stream = provider
|
||||
.stream(&model, "", &[], &[])
|
||||
.await
|
||||
.expect("LiteLLM streaming request should consume SSE");
|
||||
let mut text = String::new();
|
||||
let mut usage = None;
|
||||
while let Some(item) = stream.next().await {
|
||||
let (message, item_usage) = item.expect("SSE item should parse");
|
||||
if let Some(message) = message {
|
||||
for content in message.content {
|
||||
text.push_str(&content.to_string());
|
||||
}
|
||||
}
|
||||
if item_usage.is_some() {
|
||||
usage = item_usage;
|
||||
}
|
||||
}
|
||||
|
||||
assert_eq!(text, "hello");
|
||||
let usage = usage.expect("SSE usage chunk should be reported");
|
||||
assert_eq!(usage.model, "test-model");
|
||||
assert_eq!(usage.usage.input_tokens, Some(3));
|
||||
assert_eq!(usage.usage.output_tokens, Some(1));
|
||||
assert_eq!(usage.usage.total_tokens, Some(4));
|
||||
|
||||
let requests = server.received_requests().await.unwrap();
|
||||
assert_eq!(requests.len(), 1);
|
||||
let body: Value = serde_json::from_slice(&requests[0].body).unwrap();
|
||||
assert_eq!(body["stream"], json!(true));
|
||||
assert_eq!(body["stream_options"], json!({"include_usage": true}));
|
||||
|
||||
let log = captured_request_log("litellm-streaming-log-test");
|
||||
assert_eq!(log[0]["input"]["stream"], json!(true));
|
||||
assert!(log.iter().skip(1).any(|entry| !entry["data"].is_null()));
|
||||
assert!(log
|
||||
.iter()
|
||||
.skip(1)
|
||||
.any(|entry| entry["usage"]["total_tokens"] == json!(4)));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn non_streaming_request_preserves_json_response_path() {
|
||||
install_capturing_request_logger();
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/v1/chat/completions"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
|
||||
"id": "chatcmpl-litellm-json",
|
||||
"object": "chat.completion",
|
||||
"created": 0,
|
||||
"model": "test-model",
|
||||
"choices": [{
|
||||
"index": 0,
|
||||
"message": {"role": "assistant", "content": "hello from JSON"},
|
||||
"finish_reason": "stop"
|
||||
}],
|
||||
"usage": {"prompt_tokens": 5, "completion_tokens": 3, "total_tokens": 8}
|
||||
})))
|
||||
.mount(&server)
|
||||
.await;
|
||||
|
||||
let provider = LiteLLMProvider {
|
||||
api_client: ApiClient::new_with_tls(server.uri(), AuthMethod::NoAuth, None).unwrap(),
|
||||
base_path: "v1/chat/completions".to_string(),
|
||||
supports_streaming: false,
|
||||
name: LITELLM_PROVIDER_NAME.to_string(),
|
||||
cached_model_info: tokio::sync::Mutex::new(None),
|
||||
};
|
||||
let model = ModelConfig::new("litellm-json-log-test").with_prompt_cache_disabled();
|
||||
|
||||
let mut stream = provider
|
||||
.stream(&model, "", &[], &[])
|
||||
.await
|
||||
.expect("LiteLLM JSON response should remain supported");
|
||||
let (message, usage) = stream
|
||||
.next()
|
||||
.await
|
||||
.expect("JSON response should yield one item")
|
||||
.expect("JSON response item should parse");
|
||||
|
||||
let text = message
|
||||
.expect("JSON response should contain a message")
|
||||
.content
|
||||
.into_iter()
|
||||
.map(|content| content.to_string())
|
||||
.collect::<String>();
|
||||
assert_eq!(text, "hello from JSON");
|
||||
let usage = usage.expect("JSON response usage should be reported");
|
||||
assert_eq!(usage.model, "test-model");
|
||||
assert_eq!(usage.usage.input_tokens, Some(5));
|
||||
assert_eq!(usage.usage.output_tokens, Some(3));
|
||||
assert_eq!(usage.usage.total_tokens, Some(8));
|
||||
assert!(stream.next().await.is_none());
|
||||
|
||||
let requests = server.received_requests().await.unwrap();
|
||||
assert_eq!(requests.len(), 1);
|
||||
let body: Value = serde_json::from_slice(&requests[0].body).unwrap();
|
||||
assert!(body.get("stream").is_none());
|
||||
assert!(body.get("stream_options").is_none());
|
||||
|
||||
let log = captured_request_log("litellm-json-log-test");
|
||||
assert!(log[0]["input"].get("stream").is_none());
|
||||
assert!(log.iter().skip(1).any(|entry| !entry["data"].is_null()));
|
||||
assert!(log
|
||||
.iter()
|
||||
.skip(1)
|
||||
.any(|entry| entry["usage"]["total_tokens"] == json!(8)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn streaming_config_defaults_to_true() {
|
||||
let metadata = <LiteLLMProvider as goose_providers::base::ProviderDescriptor>::metadata();
|
||||
let key = metadata
|
||||
.config_keys
|
||||
.iter()
|
||||
.find(|key| key.name == "LITELLM_SUPPORTS_STREAMING")
|
||||
.expect("streaming config key should be exposed");
|
||||
|
||||
assert_eq!(key.default.as_deref(), Some("true"));
|
||||
assert!(!key.required);
|
||||
assert!(!key.secret);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn context_limit_negative_caches_failed_model_info() {
|
||||
@@ -356,6 +597,7 @@ mod tests {
|
||||
)
|
||||
.unwrap(),
|
||||
base_path: "v1/chat/completions".to_string(),
|
||||
supports_streaming: true,
|
||||
name: LITELLM_PROVIDER_NAME.to_string(),
|
||||
cached_model_info: tokio::sync::Mutex::new(None),
|
||||
};
|
||||
@@ -384,6 +626,7 @@ mod tests {
|
||||
)
|
||||
.unwrap(),
|
||||
base_path: "v1/chat/completions".to_string(),
|
||||
supports_streaming: true,
|
||||
name: LITELLM_PROVIDER_NAME.to_string(),
|
||||
cached_model_info: tokio::sync::Mutex::new(Some(CachedModelInfo::Failure(
|
||||
Instant::now() - MODEL_INFO_FAILURE_TTL,
|
||||
@@ -413,6 +656,7 @@ mod tests {
|
||||
)
|
||||
.unwrap(),
|
||||
base_path: "v1/chat/completions".to_string(),
|
||||
supports_streaming: true,
|
||||
name: LITELLM_PROVIDER_NAME.to_string(),
|
||||
cached_model_info,
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user