fix: we load extensions when agent starts so don't do it up front (#6350)

crates/goose/src/agents/extension_manager.rs

@@ -477,6 +477,11 @@ impl ExtensionManager {
     pub async fn add_extension(&self, config: ExtensionConfig) -> ExtensionResult<()> {
         let config_name = config.key().to_string();
         let sanitized_name = normalize(config_name.clone());
+
+        if self.extensions.lock().await.contains_key(&sanitized_name) {
+            return Ok(());
+        }
+
         let mut temp_dir = None;
 
         let client: Box<dyn McpClientTrait> = match &config {