chore: introduce DEFAULT_PROVIDER_TIMEOUT_SECS constant for providers (#8816)
Signed-off-by: Rodolfo Olivieri <rolivier@redhat.com> Signed-off-by: Douwe Osinga <douwe@squareup.com> Co-authored-by: Douwe Osinga <douwe@squareup.com>
This commit is contained in:
@@ -1,3 +1,4 @@
|
|||||||
|
use crate::providers::base::DEFAULT_PROVIDER_TIMEOUT_SECS;
|
||||||
use crate::session_context::SESSION_ID_HEADER;
|
use crate::session_context::SESSION_ID_HEADER;
|
||||||
use anyhow::Result;
|
use anyhow::Result;
|
||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
@@ -278,7 +279,11 @@ pub struct ApiRequestBuilder<'a> {
|
|||||||
|
|
||||||
impl ApiClient {
|
impl ApiClient {
|
||||||
pub fn new(host: String, auth: AuthMethod) -> Result<Self> {
|
pub fn new(host: String, auth: AuthMethod) -> Result<Self> {
|
||||||
Self::with_timeout(host, auth, Duration::from_secs(600))
|
Self::with_timeout(
|
||||||
|
host,
|
||||||
|
auth,
|
||||||
|
Duration::from_secs(DEFAULT_PROVIDER_TIMEOUT_SECS),
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn with_timeout(host: String, auth: AuthMethod, timeout: Duration) -> Result<Self> {
|
pub fn with_timeout(host: String, auth: AuthMethod, timeout: Duration) -> Result<Self> {
|
||||||
|
|||||||
@@ -4,6 +4,11 @@ use futures::future::BoxFuture;
|
|||||||
use futures::Stream;
|
use futures::Stream;
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
|
/// Default HTTP timeout for all provider API calls.
|
||||||
|
/// Long-running model inference can take several minutes, so we allow up to 10 minutes
|
||||||
|
/// before giving up. Individual providers may override this via their own config key.
|
||||||
|
pub const DEFAULT_PROVIDER_TIMEOUT_SECS: u64 = 600;
|
||||||
|
|
||||||
use super::canonical::{map_to_canonical_model, CanonicalModelRegistry};
|
use super::canonical::{map_to_canonical_model, CanonicalModelRegistry};
|
||||||
use super::errors::ProviderError;
|
use super::errors::ProviderError;
|
||||||
use super::inventory::{default_inventory_identity, InventoryIdentityInput};
|
use super::inventory::{default_inventory_identity, InventoryIdentityInput};
|
||||||
|
|||||||
@@ -13,7 +13,10 @@ use tokio_util::codec::{FramedRead, LinesCodec};
|
|||||||
use tokio_util::io::StreamReader;
|
use tokio_util::io::StreamReader;
|
||||||
|
|
||||||
use super::api_client::{ApiClient, AuthMethod, AuthProvider};
|
use super::api_client::{ApiClient, AuthMethod, AuthProvider};
|
||||||
use super::base::{ConfigKey, MessageStream, Provider, ProviderDef, ProviderMetadata};
|
use super::base::{
|
||||||
|
ConfigKey, MessageStream, Provider, ProviderDef, ProviderMetadata,
|
||||||
|
DEFAULT_PROVIDER_TIMEOUT_SECS,
|
||||||
|
};
|
||||||
use super::embedding::EmbeddingCapable;
|
use super::embedding::EmbeddingCapable;
|
||||||
use super::errors::ProviderError;
|
use super::errors::ProviderError;
|
||||||
use super::formats::databricks::create_request;
|
use super::formats::databricks::create_request;
|
||||||
@@ -41,7 +44,6 @@ use serde_json::json;
|
|||||||
const DEFAULT_CLIENT_ID: &str = "databricks-cli";
|
const DEFAULT_CLIENT_ID: &str = "databricks-cli";
|
||||||
const DEFAULT_REDIRECT_URL: &str = "http://localhost";
|
const DEFAULT_REDIRECT_URL: &str = "http://localhost";
|
||||||
const DEFAULT_SCOPES: &[&str] = &["all-apis", "offline_access"];
|
const DEFAULT_SCOPES: &[&str] = &["all-apis", "offline_access"];
|
||||||
const DEFAULT_TIMEOUT_SECS: u64 = 600;
|
|
||||||
|
|
||||||
const DATABRICKS_PROVIDER_NAME: &str = "databricks";
|
const DATABRICKS_PROVIDER_NAME: &str = "databricks";
|
||||||
pub const DATABRICKS_DEFAULT_MODEL: &str = "databricks-claude-sonnet-4";
|
pub const DATABRICKS_DEFAULT_MODEL: &str = "databricks-claude-sonnet-4";
|
||||||
@@ -177,8 +179,11 @@ impl DatabricksProvider {
|
|||||||
token_cache: token_cache.clone(),
|
token_cache: token_cache.clone(),
|
||||||
}));
|
}));
|
||||||
|
|
||||||
let api_client =
|
let api_client = ApiClient::with_timeout(
|
||||||
ApiClient::with_timeout(host, auth_method, Duration::from_secs(DEFAULT_TIMEOUT_SECS))?;
|
host,
|
||||||
|
auth_method,
|
||||||
|
Duration::from_secs(DEFAULT_PROVIDER_TIMEOUT_SECS),
|
||||||
|
)?;
|
||||||
|
|
||||||
let mut provider = Self {
|
let mut provider = Self {
|
||||||
api_client,
|
api_client,
|
||||||
@@ -242,7 +247,11 @@ impl DatabricksProvider {
|
|||||||
token_cache: token_cache.clone(),
|
token_cache: token_cache.clone(),
|
||||||
}));
|
}));
|
||||||
|
|
||||||
let api_client = ApiClient::with_timeout(host, auth_method, Duration::from_secs(600))?;
|
let api_client = ApiClient::with_timeout(
|
||||||
|
host,
|
||||||
|
auth_method,
|
||||||
|
Duration::from_secs(DEFAULT_PROVIDER_TIMEOUT_SECS),
|
||||||
|
)?;
|
||||||
|
|
||||||
Ok(Self {
|
Ok(Self {
|
||||||
api_client,
|
api_client,
|
||||||
|
|||||||
@@ -16,7 +16,10 @@ use url::Url;
|
|||||||
|
|
||||||
use crate::conversation::message::Message;
|
use crate::conversation::message::Message;
|
||||||
use crate::model::ModelConfig;
|
use crate::model::ModelConfig;
|
||||||
use crate::providers::base::{ConfigKey, MessageStream, Provider, ProviderDef, ProviderMetadata};
|
use crate::providers::base::{
|
||||||
|
ConfigKey, MessageStream, Provider, ProviderDef, ProviderMetadata,
|
||||||
|
DEFAULT_PROVIDER_TIMEOUT_SECS,
|
||||||
|
};
|
||||||
|
|
||||||
use crate::providers::errors::ProviderError;
|
use crate::providers::errors::ProviderError;
|
||||||
use crate::providers::formats::gcpvertexai::{
|
use crate::providers::formats::gcpvertexai::{
|
||||||
@@ -33,8 +36,6 @@ use rmcp::model::Tool;
|
|||||||
const GCP_VERTEX_AI_PROVIDER_NAME: &str = "gcp_vertex_ai";
|
const GCP_VERTEX_AI_PROVIDER_NAME: &str = "gcp_vertex_ai";
|
||||||
/// Base URL for GCP Vertex AI documentation
|
/// Base URL for GCP Vertex AI documentation
|
||||||
const GCP_VERTEX_AI_DOC_URL: &str = "https://cloud.google.com/vertex-ai";
|
const GCP_VERTEX_AI_DOC_URL: &str = "https://cloud.google.com/vertex-ai";
|
||||||
/// Default timeout for API requests in seconds
|
|
||||||
const DEFAULT_TIMEOUT_SECS: u64 = 600;
|
|
||||||
/// Default initial interval for retry (in milliseconds)
|
/// Default initial interval for retry (in milliseconds)
|
||||||
const DEFAULT_INITIAL_RETRY_INTERVAL_MS: u64 = 5000;
|
const DEFAULT_INITIAL_RETRY_INTERVAL_MS: u64 = 5000;
|
||||||
/// Default maximum number of retries
|
/// Default maximum number of retries
|
||||||
@@ -171,7 +172,7 @@ impl GcpVertexAIProvider {
|
|||||||
let host = Self::build_host_url(&location);
|
let host = Self::build_host_url(&location);
|
||||||
|
|
||||||
let client = Client::builder()
|
let client = Client::builder()
|
||||||
.timeout(Duration::from_secs(DEFAULT_TIMEOUT_SECS))
|
.timeout(Duration::from_secs(DEFAULT_PROVIDER_TIMEOUT_SECS))
|
||||||
.build()?;
|
.build()?;
|
||||||
|
|
||||||
let auth = GcpAuth::new().await?;
|
let auth = GcpAuth::new().await?;
|
||||||
|
|||||||
@@ -1,7 +1,10 @@
|
|||||||
use crate::config::paths::Paths;
|
use crate::config::paths::Paths;
|
||||||
use crate::conversation::message::Message;
|
use crate::conversation::message::Message;
|
||||||
use crate::model::ModelConfig;
|
use crate::model::ModelConfig;
|
||||||
use crate::providers::base::{ConfigKey, MessageStream, Provider, ProviderDef, ProviderMetadata};
|
use crate::providers::base::{
|
||||||
|
ConfigKey, MessageStream, Provider, ProviderDef, ProviderMetadata,
|
||||||
|
DEFAULT_PROVIDER_TIMEOUT_SECS,
|
||||||
|
};
|
||||||
use crate::providers::errors::ProviderError;
|
use crate::providers::errors::ProviderError;
|
||||||
use crate::providers::formats::google::{create_request, response_to_streaming_message};
|
use crate::providers::formats::google::{create_request, response_to_streaming_message};
|
||||||
use crate::providers::google::GOOGLE_DOC_URL;
|
use crate::providers::google::GOOGLE_DOC_URL;
|
||||||
@@ -35,11 +38,9 @@ use tokio_stream::StreamExt;
|
|||||||
use tokio_util::codec::{FramedRead, LinesCodec};
|
use tokio_util::codec::{FramedRead, LinesCodec};
|
||||||
use tokio_util::io::StreamReader;
|
use tokio_util::io::StreamReader;
|
||||||
|
|
||||||
const HTTP_TIMEOUT_SECS: u64 = 600;
|
|
||||||
|
|
||||||
static HTTP_CLIENT: LazyLock<reqwest::Client> = LazyLock::new(|| {
|
static HTTP_CLIENT: LazyLock<reqwest::Client> = LazyLock::new(|| {
|
||||||
reqwest::Client::builder()
|
reqwest::Client::builder()
|
||||||
.timeout(Duration::from_secs(HTTP_TIMEOUT_SECS))
|
.timeout(Duration::from_secs(DEFAULT_PROVIDER_TIMEOUT_SECS))
|
||||||
.build()
|
.build()
|
||||||
.expect("failed to build HTTP client")
|
.expect("failed to build HTTP client")
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -14,7 +14,9 @@ use std::collections::HashMap;
|
|||||||
use std::path::PathBuf;
|
use std::path::PathBuf;
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
|
|
||||||
use super::base::{Provider, ProviderDef, ProviderMetadata, ProviderUsage, Usage};
|
use super::base::{
|
||||||
|
Provider, ProviderDef, ProviderMetadata, ProviderUsage, Usage, DEFAULT_PROVIDER_TIMEOUT_SECS,
|
||||||
|
};
|
||||||
use super::errors::ProviderError;
|
use super::errors::ProviderError;
|
||||||
use super::formats::openai::{create_request, get_usage, response_to_message};
|
use super::formats::openai::{create_request, get_usage, response_to_message};
|
||||||
use super::openai_compatible::handle_response_openai_compat;
|
use super::openai_compatible::handle_response_openai_compat;
|
||||||
@@ -224,7 +226,7 @@ impl GithubCopilotProvider {
|
|||||||
let copilot_token_url: Option<String> = config.get_param("GITHUB_COPILOT_TOKEN_URL").ok();
|
let copilot_token_url: Option<String> = config.get_param("GITHUB_COPILOT_TOKEN_URL").ok();
|
||||||
let urls = GithubCopilotUrls::new(&host, copilot_token_url.as_deref());
|
let urls = GithubCopilotUrls::new(&host, copilot_token_url.as_deref());
|
||||||
let client = Client::builder()
|
let client = Client::builder()
|
||||||
.timeout(Duration::from_secs(600))
|
.timeout(Duration::from_secs(DEFAULT_PROVIDER_TIMEOUT_SECS))
|
||||||
.build()?;
|
.build()?;
|
||||||
let cache = DiskCache::new(&host);
|
let cache = DiskCache::new(&host);
|
||||||
let mu = tokio::sync::Mutex::new(RefCell::new(None));
|
let mu = tokio::sync::Mutex::new(RefCell::new(None));
|
||||||
|
|||||||
@@ -16,7 +16,10 @@ use tokio::pin;
|
|||||||
use tokio_util::io::StreamReader;
|
use tokio_util::io::StreamReader;
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
|
||||||
use super::base::{ConfigKey, MessageStream, Provider, ProviderDef, ProviderMetadata};
|
use super::base::{
|
||||||
|
ConfigKey, MessageStream, Provider, ProviderDef, ProviderMetadata,
|
||||||
|
DEFAULT_PROVIDER_TIMEOUT_SECS,
|
||||||
|
};
|
||||||
use super::errors::ProviderError;
|
use super::errors::ProviderError;
|
||||||
use super::formats::anthropic::{create_request, response_to_streaming_message};
|
use super::formats::anthropic::{create_request, response_to_streaming_message};
|
||||||
use super::oauth_device_flow::{
|
use super::oauth_device_flow::{
|
||||||
@@ -162,7 +165,7 @@ impl KimiCodeProvider {
|
|||||||
pub async fn from_env(model: ModelConfig) -> Result<Self> {
|
pub async fn from_env(model: ModelConfig) -> Result<Self> {
|
||||||
let model = model.with_fast(KIMI_CODE_DEFAULT_FAST_MODEL, KIMI_CODE_PROVIDER_NAME)?;
|
let model = model.with_fast(KIMI_CODE_DEFAULT_FAST_MODEL, KIMI_CODE_PROVIDER_NAME)?;
|
||||||
let client = Client::builder()
|
let client = Client::builder()
|
||||||
.timeout(StdDuration::from_secs(600))
|
.timeout(StdDuration::from_secs(DEFAULT_PROVIDER_TIMEOUT_SECS))
|
||||||
.build()?;
|
.build()?;
|
||||||
let device_id = Self::get_or_create_device_id().await?;
|
let device_id = Self::get_or_create_device_id().await?;
|
||||||
Ok(Self {
|
Ok(Self {
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ use std::collections::HashMap;
|
|||||||
use super::api_client::{ApiClient, AuthMethod};
|
use super::api_client::{ApiClient, AuthMethod};
|
||||||
use super::base::{
|
use super::base::{
|
||||||
ConfigKey, MessageStream, ModelInfo, Provider, ProviderDef, ProviderMetadata, ProviderUsage,
|
ConfigKey, MessageStream, ModelInfo, Provider, ProviderDef, ProviderMetadata, ProviderUsage,
|
||||||
|
DEFAULT_PROVIDER_TIMEOUT_SECS,
|
||||||
};
|
};
|
||||||
use super::embedding::EmbeddingCapable;
|
use super::embedding::EmbeddingCapable;
|
||||||
use super::errors::ProviderError;
|
use super::errors::ProviderError;
|
||||||
@@ -48,7 +49,9 @@ impl LiteLLMProvider {
|
|||||||
.get("LITELLM_CUSTOM_HEADERS")
|
.get("LITELLM_CUSTOM_HEADERS")
|
||||||
.cloned()
|
.cloned()
|
||||||
.map(parse_custom_headers);
|
.map(parse_custom_headers);
|
||||||
let timeout_secs: u64 = config.get_param("LITELLM_TIMEOUT").unwrap_or(600);
|
let timeout_secs: u64 = config
|
||||||
|
.get_param("LITELLM_TIMEOUT")
|
||||||
|
.unwrap_or(DEFAULT_PROVIDER_TIMEOUT_SECS);
|
||||||
|
|
||||||
let auth = if api_key.is_empty() {
|
let auth = if api_key.is_empty() {
|
||||||
AuthMethod::NoAuth
|
AuthMethod::NoAuth
|
||||||
|
|||||||
@@ -1,5 +1,8 @@
|
|||||||
use super::api_client::{ApiClient, AuthMethod};
|
use super::api_client::{ApiClient, AuthMethod};
|
||||||
use super::base::{ConfigKey, MessageStream, Provider, ProviderDef, ProviderMetadata};
|
use super::base::{
|
||||||
|
ConfigKey, MessageStream, Provider, ProviderDef, ProviderMetadata,
|
||||||
|
DEFAULT_PROVIDER_TIMEOUT_SECS,
|
||||||
|
};
|
||||||
use super::errors::ProviderError;
|
use super::errors::ProviderError;
|
||||||
use super::inventory::InventoryIdentityInput;
|
use super::inventory::InventoryIdentityInput;
|
||||||
use super::openai_compatible::handle_status;
|
use super::openai_compatible::handle_status;
|
||||||
@@ -26,7 +29,7 @@ use url::Url;
|
|||||||
|
|
||||||
const OLLAMA_PROVIDER_NAME: &str = "ollama";
|
const OLLAMA_PROVIDER_NAME: &str = "ollama";
|
||||||
pub const OLLAMA_HOST: &str = "localhost";
|
pub const OLLAMA_HOST: &str = "localhost";
|
||||||
pub const OLLAMA_TIMEOUT: u64 = 600;
|
pub const OLLAMA_TIMEOUT: u64 = DEFAULT_PROVIDER_TIMEOUT_SECS;
|
||||||
pub const OLLAMA_DEFAULT_PORT: u16 = 11434;
|
pub const OLLAMA_DEFAULT_PORT: u16 = 11434;
|
||||||
pub const OLLAMA_DEFAULT_MODEL: &str = "qwen3";
|
pub const OLLAMA_DEFAULT_MODEL: &str = "qwen3";
|
||||||
pub const OLLAMA_KNOWN_MODELS: &[&str] = &[
|
pub const OLLAMA_KNOWN_MODELS: &[&str] = &[
|
||||||
|
|||||||
@@ -1,5 +1,7 @@
|
|||||||
use super::api_client::{ApiClient, AuthMethod};
|
use super::api_client::{ApiClient, AuthMethod};
|
||||||
use super::base::{ConfigKey, ModelInfo, Provider, ProviderDef, ProviderMetadata};
|
use super::base::{
|
||||||
|
ConfigKey, ModelInfo, Provider, ProviderDef, ProviderMetadata, DEFAULT_PROVIDER_TIMEOUT_SECS,
|
||||||
|
};
|
||||||
use super::embedding::{EmbeddingCapable, EmbeddingRequest, EmbeddingResponse};
|
use super::embedding::{EmbeddingCapable, EmbeddingRequest, EmbeddingResponse};
|
||||||
use super::errors::ProviderError;
|
use super::errors::ProviderError;
|
||||||
use super::formats::openai::{create_request, get_usage, response_to_message};
|
use super::formats::openai::{create_request, get_usage, response_to_message};
|
||||||
@@ -229,7 +231,9 @@ impl OpenAiProvider {
|
|||||||
|
|
||||||
let organization: Option<String> = config.get_param("OPENAI_ORGANIZATION").ok();
|
let organization: Option<String> = config.get_param("OPENAI_ORGANIZATION").ok();
|
||||||
let project: Option<String> = config.get_param("OPENAI_PROJECT").ok();
|
let project: Option<String> = config.get_param("OPENAI_PROJECT").ok();
|
||||||
let timeout_secs: u64 = config.get_param("OPENAI_TIMEOUT").unwrap_or(600);
|
let timeout_secs: u64 = config
|
||||||
|
.get_param("OPENAI_TIMEOUT")
|
||||||
|
.unwrap_or(DEFAULT_PROVIDER_TIMEOUT_SECS);
|
||||||
|
|
||||||
let auth = match api_key {
|
let auth = match api_key {
|
||||||
Some(key) if !key.is_empty() => AuthMethod::BearerToken(key),
|
Some(key) if !key.is_empty() => AuthMethod::BearerToken(key),
|
||||||
@@ -334,7 +338,9 @@ impl OpenAiProvider {
|
|||||||
Self::derive_base_path(url.path())
|
Self::derive_base_path(url.path())
|
||||||
};
|
};
|
||||||
|
|
||||||
let timeout_secs = config.timeout_seconds.unwrap_or(600);
|
let timeout_secs = config
|
||||||
|
.timeout_seconds
|
||||||
|
.unwrap_or(DEFAULT_PROVIDER_TIMEOUT_SECS);
|
||||||
|
|
||||||
let auth = match api_key {
|
let auth = match api_key {
|
||||||
Some(key) if !key.is_empty() => AuthMethod::BearerToken(key),
|
Some(key) if !key.is_empty() => AuthMethod::BearerToken(key),
|
||||||
|
|||||||
@@ -36,6 +36,7 @@ use super::ollama::OLLAMA_HOST;
|
|||||||
use crate::conversation::message::{Message, MessageContent};
|
use crate::conversation::message::{Message, MessageContent};
|
||||||
use crate::conversation::Conversation;
|
use crate::conversation::Conversation;
|
||||||
use crate::model::ModelConfig;
|
use crate::model::ModelConfig;
|
||||||
|
use crate::providers::base::DEFAULT_PROVIDER_TIMEOUT_SECS;
|
||||||
use crate::providers::formats::openai::create_request;
|
use crate::providers::formats::openai::create_request;
|
||||||
use anyhow::Result;
|
use anyhow::Result;
|
||||||
use reqwest::Client;
|
use reqwest::Client;
|
||||||
@@ -71,7 +72,7 @@ pub struct OllamaInterpreter {
|
|||||||
impl OllamaInterpreter {
|
impl OllamaInterpreter {
|
||||||
pub fn new() -> Result<Self, ProviderError> {
|
pub fn new() -> Result<Self, ProviderError> {
|
||||||
let client = Client::builder()
|
let client = Client::builder()
|
||||||
.timeout(Duration::from_secs(600))
|
.timeout(Duration::from_secs(DEFAULT_PROVIDER_TIMEOUT_SECS))
|
||||||
.build()
|
.build()
|
||||||
.expect("Failed to create HTTP client");
|
.expect("Failed to create HTTP client");
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user