feat: add local inference provider with llama.cpp backend and HuggingFace model management (#6933)

Co-authored-by: Douwe Osinga <douwe@squareup.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: jh-block <jhugo@block.xyz>
Co-authored-by: Spence <spencermartin@squareup.com>
Co-authored-by: Michael Neale <michael.neale@gmail.com>

crates/goose/src/providers/base.rs

@@ -17,10 +17,22 @@ use rmcp::model::Tool;
 use utoipa::ToSchema;
 
 use once_cell::sync::Lazy;
+use regex::Regex;
 use std::ops::{Add, AddAssign};
 use std::pin::Pin;
+use std::sync::LazyLock;
 use std::sync::Mutex;
 
+fn strip_xml_tags(text: &str) -> String {
+    static BLOCK_RE: LazyLock<Regex> = LazyLock::new(|| {
+        Regex::new(r"(?s)<([a-zA-Z][a-zA-Z0-9_]*)[^>]*>.*?</[a-zA-Z][a-zA-Z0-9_]*>").unwrap()
+    });
+    static TAG_RE: LazyLock<Regex> =
+        LazyLock::new(|| Regex::new(r"</?[a-zA-Z][a-zA-Z0-9_]*[^>]*>").unwrap());
+    let pass1 = BLOCK_RE.replace_all(text, "");
+    TAG_RE.replace_all(&pass1, "").into_owned()
+}
+
 /// A global store for the current model being used, we use this as when a provider returns, it tells us the real model, not an alias
 pub static CURRENT_MODEL: Lazy<Mutex<Option<String>>> = Lazy::new(|| Mutex::new(None));
 
@@ -594,20 +606,28 @@ pub trait Provider: Send + Sync {
         messages: &Conversation,
     ) -> Result<String, ProviderError> {
         let context = self.get_initial_user_messages(messages);
-        let prompt = self.create_session_name_prompt(&context);
-        let message = Message::user().with_text(&prompt);
+        let system = crate::prompt_template::render_template(
+            "session_name.md",
+            &std::collections::HashMap::<String, String>::new(),
+        )
+        .map_err(|e| ProviderError::ContextLengthExceeded(e.to_string()))?;
+
+        let user_text = format!(
+            "---BEGIN USER MESSAGES---\n{}\n---END USER MESSAGES---\n\nGenerate a short title for the above messages.",
+            context.join("\n")
+        );
+        let message = Message::user().with_text(&user_text);
         let result = self
-            .complete_fast(
-                session_id,
-                "Reply with only a description in four words or less",
-                &[message],
-                &[],
-            )
+            .complete_fast(session_id, &system, &[message], &[])
             .await?;
 
-        let description = result
+        let raw: String = result
             .0
-            .as_concat_text()
+            .content
+            .iter()
+            .filter_map(|c| c.as_text())
+            .collect();
+        let description = strip_xml_tags(&raw)
             .split_whitespace()
             .collect::<Vec<_>>()
             .join(" ");
@@ -615,21 +635,6 @@ pub trait Provider: Send + Sync {
         Ok(safe_truncate(&description, 100))
     }
 
-    // Generate a prompt for a session name based on the conversation history
-    fn create_session_name_prompt(&self, context: &[String]) -> String {
-        // Create a prompt for a concise description
-        let mut prompt = "Based on the conversation so far, provide a concise description of this session in 4 words or less. This will be used for finding the session later in a UI with limited space - reply *ONLY* with the description".to_string();
-
-        if !context.is_empty() {
-            prompt = format!(
-                "Here are the first few user messages:\n{}\n\n{}",
-                context.join("\n"),
-                prompt
-            );
-        }
-        prompt
-    }
-
     /// Configure OAuth authentication for this provider
     ///
     /// This method is called when a provider has configuration keys marked with oauth_flow = true.
@@ -719,6 +724,42 @@ mod tests {
     use test_case::test_case;
 
     use serde_json::json;
+
+    #[test]
+    fn test_strip_xml_tags() {
+        assert_eq!(strip_xml_tags("<think>reasoning</think>answer"), "answer");
+        assert_eq!(strip_xml_tags("before<t>mid</t>after"), "beforeafter");
+        assert_eq!(strip_xml_tags("<a>x</a><b>y</b>z"), "z");
+        assert_eq!(strip_xml_tags("no tags here"), "no tags here");
+        assert_eq!(strip_xml_tags("a < b > c"), "a < b > c");
+        assert_eq!(strip_xml_tags("<think>über</think>ok"), "ok");
+        assert_eq!(strip_xml_tags("<think>日本語</think>hello"), "hello");
+        assert_eq!(strip_xml_tags(""), "");
+        assert_eq!(strip_xml_tags("<>stuff</>"), "<>stuff</>");
+        // attributes
+        assert_eq!(
+            strip_xml_tags(r#"<think class="deep">reasoning</think>answer"#),
+            "answer"
+        );
+        // self-closing tags
+        assert_eq!(strip_xml_tags("<br/>self closing"), "self closing");
+        // orphan closing tags
+        assert_eq!(strip_xml_tags("orphan </think> tag"), "orphan  tag");
+        // multiline content
+        assert_eq!(
+            strip_xml_tags("<think>\nline1\nline2\n</think>result"),
+            "result"
+        );
+    }
+
+    #[test]
+    fn test_usage_creation() {
+        let usage = Usage::new(Some(10), Some(20), Some(30));
+        assert_eq!(usage.input_tokens, Some(10));
+        assert_eq!(usage.output_tokens, Some(20));
+        assert_eq!(usage.total_tokens, Some(30));
+    }
+
     fn content_from_str(s: String) -> MessageContent {
         if let Some(img_data) = s.strip_prefix("*img:") {
             MessageContent::image(format!("http://example.com/{}", img_data), "image/png")
@@ -798,11 +839,10 @@ mod tests {
 
     #[tokio::test]
     async fn test_collect_stream_defaults_usage() {
-        // Should not error when usage is missing
         let stream = create_test_stream(vec!["Hello".to_string()]);
         let (msg, usage) = collect_stream(Box::pin(stream)).await.unwrap();
         assert_eq!(content_to_strings(&msg), vec!["Hello"]);
-        assert_eq!(usage.model, "unknown"); // Default usage
+        assert_eq!(usage.model, "unknown");
     }
 
     #[test]