feat: Add configurable Bedrock retry parameters (#4316)
Signed-off-by: Dan Wuensch <dan.wuensch@ncino.com> Co-authored-by: angiejones <jones.angie@gmail.com>
This commit is contained in:
@@ -2,7 +2,7 @@ use std::collections::HashMap;
|
||||
|
||||
use super::base::{ConfigKey, Provider, ProviderMetadata, ProviderUsage};
|
||||
use super::errors::ProviderError;
|
||||
use super::retry::ProviderRetry;
|
||||
use super::retry::{ProviderRetry, RetryConfig};
|
||||
use crate::conversation::message::Message;
|
||||
use crate::impl_provider_default;
|
||||
use crate::model::ModelConfig;
|
||||
@@ -23,17 +23,28 @@ use super::formats::bedrock::{
|
||||
pub const BEDROCK_DOC_LINK: &str =
|
||||
"https://docs.aws.amazon.com/bedrock/latest/userguide/models-supported.html";
|
||||
|
||||
pub const BEDROCK_DEFAULT_MODEL: &str = "anthropic.claude-3-5-sonnet-20240620-v1:0";
|
||||
pub const BEDROCK_DEFAULT_MODEL: &str = "anthropic.claude-sonnet-4-20250514-v1:0";
|
||||
pub const BEDROCK_KNOWN_MODELS: &[&str] = &[
|
||||
"anthropic.claude-3-5-sonnet-20240620-v1:0",
|
||||
"anthropic.claude-3-5-sonnet-20241022-v2:0",
|
||||
"anthropic.claude-3-7-sonnet-20250219-v1:0",
|
||||
"anthropic.claude-sonnet-4-20250514-v1:0",
|
||||
"anthropic.claude-opus-4-20250514-v1:0",
|
||||
"anthropic.claude-opus-4-1-20250805-v1:0",
|
||||
];
|
||||
|
||||
pub const BEDROCK_DEFAULT_MAX_RETRIES: usize = 6;
|
||||
pub const BEDROCK_DEFAULT_INITIAL_RETRY_INTERVAL_MS: u64 = 2000;
|
||||
pub const BEDROCK_DEFAULT_BACKOFF_MULTIPLIER: f64 = 2.0;
|
||||
pub const BEDROCK_DEFAULT_MAX_RETRY_INTERVAL_MS: u64 = 120_000;
|
||||
|
||||
#[derive(Debug, serde::Serialize)]
|
||||
pub struct BedrockProvider {
|
||||
#[serde(skip)]
|
||||
client: Client,
|
||||
model: ModelConfig,
|
||||
#[serde(skip)]
|
||||
retry_config: RetryConfig,
|
||||
}
|
||||
|
||||
impl BedrockProvider {
|
||||
@@ -65,7 +76,38 @@ impl BedrockProvider {
|
||||
)?;
|
||||
let client = Client::new(&sdk_config);
|
||||
|
||||
Ok(Self { client, model })
|
||||
let retry_config = Self::load_retry_config(config);
|
||||
|
||||
Ok(Self {
|
||||
client,
|
||||
model,
|
||||
retry_config,
|
||||
})
|
||||
}
|
||||
|
||||
fn load_retry_config(config: &crate::config::Config) -> RetryConfig {
|
||||
let max_retries = config
|
||||
.get_param::<usize>("BEDROCK_MAX_RETRIES")
|
||||
.unwrap_or(BEDROCK_DEFAULT_MAX_RETRIES);
|
||||
|
||||
let initial_interval_ms = config
|
||||
.get_param::<u64>("BEDROCK_INITIAL_RETRY_INTERVAL_MS")
|
||||
.unwrap_or(BEDROCK_DEFAULT_INITIAL_RETRY_INTERVAL_MS);
|
||||
|
||||
let backoff_multiplier = config
|
||||
.get_param::<f64>("BEDROCK_BACKOFF_MULTIPLIER")
|
||||
.unwrap_or(BEDROCK_DEFAULT_BACKOFF_MULTIPLIER);
|
||||
|
||||
let max_interval_ms = config
|
||||
.get_param::<u64>("BEDROCK_MAX_RETRY_INTERVAL_MS")
|
||||
.unwrap_or(BEDROCK_DEFAULT_MAX_RETRY_INTERVAL_MS);
|
||||
|
||||
RetryConfig {
|
||||
max_retries,
|
||||
initial_interval_ms,
|
||||
backoff_multiplier,
|
||||
max_interval_ms,
|
||||
}
|
||||
}
|
||||
|
||||
async fn converse(
|
||||
@@ -147,6 +189,10 @@ impl Provider for BedrockProvider {
|
||||
)
|
||||
}
|
||||
|
||||
fn retry_config(&self) -> RetryConfig {
|
||||
self.retry_config.clone()
|
||||
}
|
||||
|
||||
fn get_model_config(&self) -> ModelConfig {
|
||||
self.model.clone()
|
||||
}
|
||||
|
||||
@@ -114,4 +114,9 @@ pub trait ProviderRetry {
|
||||
}
|
||||
}
|
||||
|
||||
impl<P: Provider> ProviderRetry for P {}
|
||||
// Let specific providers define their retry config if desired
|
||||
impl<P: Provider> ProviderRetry for P {
|
||||
fn retry_config(&self) -> RetryConfig {
|
||||
Provider::retry_config(self)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user