feat: Add configurable Bedrock retry parameters (#4316)

Signed-off-by: Dan Wuensch <dan.wuensch@ncino.com>
Co-authored-by: angiejones <jones.angie@gmail.com>
This commit is contained in:
Dan Wuensch
2025-09-04 12:26:29 -04:00
committed by GitHub
parent 429aeffc77
commit 627696a68a
4 changed files with 98 additions and 6 deletions
+49 -3
View File
@@ -2,7 +2,7 @@ use std::collections::HashMap;
use super::base::{ConfigKey, Provider, ProviderMetadata, ProviderUsage};
use super::errors::ProviderError;
use super::retry::ProviderRetry;
use super::retry::{ProviderRetry, RetryConfig};
use crate::conversation::message::Message;
use crate::impl_provider_default;
use crate::model::ModelConfig;
@@ -23,17 +23,28 @@ use super::formats::bedrock::{
pub const BEDROCK_DOC_LINK: &str =
"https://docs.aws.amazon.com/bedrock/latest/userguide/models-supported.html";
pub const BEDROCK_DEFAULT_MODEL: &str = "anthropic.claude-3-5-sonnet-20240620-v1:0";
pub const BEDROCK_DEFAULT_MODEL: &str = "anthropic.claude-sonnet-4-20250514-v1:0";
pub const BEDROCK_KNOWN_MODELS: &[&str] = &[
"anthropic.claude-3-5-sonnet-20240620-v1:0",
"anthropic.claude-3-5-sonnet-20241022-v2:0",
"anthropic.claude-3-7-sonnet-20250219-v1:0",
"anthropic.claude-sonnet-4-20250514-v1:0",
"anthropic.claude-opus-4-20250514-v1:0",
"anthropic.claude-opus-4-1-20250805-v1:0",
];
pub const BEDROCK_DEFAULT_MAX_RETRIES: usize = 6;
pub const BEDROCK_DEFAULT_INITIAL_RETRY_INTERVAL_MS: u64 = 2000;
pub const BEDROCK_DEFAULT_BACKOFF_MULTIPLIER: f64 = 2.0;
pub const BEDROCK_DEFAULT_MAX_RETRY_INTERVAL_MS: u64 = 120_000;
#[derive(Debug, serde::Serialize)]
pub struct BedrockProvider {
#[serde(skip)]
client: Client,
model: ModelConfig,
#[serde(skip)]
retry_config: RetryConfig,
}
impl BedrockProvider {
@@ -65,7 +76,38 @@ impl BedrockProvider {
)?;
let client = Client::new(&sdk_config);
Ok(Self { client, model })
let retry_config = Self::load_retry_config(config);
Ok(Self {
client,
model,
retry_config,
})
}
fn load_retry_config(config: &crate::config::Config) -> RetryConfig {
let max_retries = config
.get_param::<usize>("BEDROCK_MAX_RETRIES")
.unwrap_or(BEDROCK_DEFAULT_MAX_RETRIES);
let initial_interval_ms = config
.get_param::<u64>("BEDROCK_INITIAL_RETRY_INTERVAL_MS")
.unwrap_or(BEDROCK_DEFAULT_INITIAL_RETRY_INTERVAL_MS);
let backoff_multiplier = config
.get_param::<f64>("BEDROCK_BACKOFF_MULTIPLIER")
.unwrap_or(BEDROCK_DEFAULT_BACKOFF_MULTIPLIER);
let max_interval_ms = config
.get_param::<u64>("BEDROCK_MAX_RETRY_INTERVAL_MS")
.unwrap_or(BEDROCK_DEFAULT_MAX_RETRY_INTERVAL_MS);
RetryConfig {
max_retries,
initial_interval_ms,
backoff_multiplier,
max_interval_ms,
}
}
async fn converse(
@@ -147,6 +189,10 @@ impl Provider for BedrockProvider {
)
}
fn retry_config(&self) -> RetryConfig {
self.retry_config.clone()
}
fn get_model_config(&self) -> ModelConfig {
self.model.clone()
}
+6 -1
View File
@@ -114,4 +114,9 @@ pub trait ProviderRetry {
}
}
impl<P: Provider> ProviderRetry for P {}
// Let specific providers define their retry config if desired
impl<P: Provider> ProviderRetry for P {
fn retry_config(&self) -> RetryConfig {
Provider::retry_config(self)
}
}