fix: lazy init local inference runtime in router instead of app start (#8656)

crates/goose-server/src/commands/agent.rs

@@ -9,6 +9,10 @@ use goose_server::tls::setup_tls;
 use tower_http::cors::{Any, CorsLayer};
 use tracing::info;
 
+fn boot_marker(message: &str) {
+    eprintln!("GOOSED_BOOT: {message}");
+}
+
 #[cfg(unix)]
 async fn shutdown_signal() {
     use tokio::signal::unix::{signal, SignalKind};
@@ -35,6 +39,7 @@ pub async fn run() -> Result<()> {
     #[cfg(feature = "rustls-tls")]
     let _ = rustls::crypto::ring::default_provider().install_default();
 
+    boot_marker("main entered");
     crate::logging::setup_logging(Some("goosed"))?;
 
     let settings = configuration::Settings::new()?;
@@ -42,6 +47,7 @@ pub async fn run() -> Result<()> {
     let secret_key = std::env::var("GOOSE_SERVER__SECRET_KEY")
         .unwrap_or_else(|_| hex::encode(rand::random::<[u8; 32]>()));
 
+    boot_marker("appstate init start");
     let app_state = state::AppState::new(settings.tls).await?;
 
     // Share the server secret with the tunnel manager so it uses the same
@@ -78,6 +84,7 @@ pub async fn run() -> Result<()> {
     if settings.tls {
         #[cfg(any(feature = "rustls-tls", feature = "native-tls"))]
         {
+            boot_marker("tls setup start");
             let tls_setup = setup_tls(
                 settings.tls_cert_path.as_deref(),
                 settings.tls_key_path.as_deref(),
@@ -92,6 +99,7 @@ pub async fn run() -> Result<()> {
             });
 
             info!("listening on https://{}", addr);
+            boot_marker("listening");
 
             #[cfg(feature = "rustls-tls")]
             axum_server::bind_rustls(addr, tls_setup.config)
@@ -114,9 +122,11 @@ pub async fn run() -> Result<()> {
             );
         }
     } else {
+        boot_marker("tcp bind start");
         let listener = tokio::net::TcpListener::bind(addr).await?;
 
         info!("listening on http://{}", addr);
+        boot_marker("listening");
 
         axum::serve(listener, app)
             .with_graceful_shutdown(async { shutdown_signal().await })

crates/goose-server/src/main.rs

@@ -9,6 +9,7 @@ mod state;
 mod tunnel;
 
 use std::path::PathBuf;
+use std::{backtrace::Backtrace, panic::PanicHookInfo};
 
 use clap::{Parser, Subcommand};
 use goose::agents::validate_extensions;
@@ -42,9 +43,42 @@ enum Commands {
     },
 }
 
+fn boot_marker(message: &str) {
+    eprintln!("GOOSED_BOOT: {message}");
+}
+
+fn install_panic_hook() {
+    let default_hook = std::panic::take_hook();
+    std::panic::set_hook(Box::new(move |panic_info: &PanicHookInfo<'_>| {
+        let location = panic_info
+            .location()
+            .map(|location| format!("{}:{}", location.file(), location.line()))
+            .unwrap_or_else(|| "unknown".to_string());
+
+        let payload = panic_info
+            .payload()
+            .downcast_ref::<&str>()
+            .map(|msg| (*msg).to_string())
+            .or_else(|| panic_info.payload().downcast_ref::<String>().cloned())
+            .unwrap_or_else(|| "unknown panic payload".to_string());
+
+        eprintln!("GOOSED_BOOT: panic at {location}: {payload}");
+        eprintln!("GOOSED_BOOT: backtrace:\n{}", Backtrace::force_capture());
+
+        default_hook(panic_info);
+    }));
+}
+
 #[tokio::main]
 async fn main() -> anyhow::Result<()> {
+    install_panic_hook();
+    boot_marker("main entered");
+
     let cli = Cli::parse();
+    boot_marker(&format!(
+        "command parsed: {:?}",
+        std::mem::discriminant(&cli.command)
+    ));
 
     match cli.command {
         Commands::Agent => {

crates/goose-server/src/routes/local_inference.rs

@@ -230,7 +230,8 @@ pub async fn sync_featured_models() -> Result<StatusCode, ErrorResponse> {
 pub async fn list_local_models(
     axum::extract::State(state): axum::extract::State<Arc<AppState>>,
 ) -> Result<Json<Vec<LocalModelResponse>>, ErrorResponse> {
-    let recommended_id = recommend_local_model(&state.inference_runtime);
+    let runtime = state.get_inference_runtime()?;
+    let recommended_id = recommend_local_model(&runtime);
 
     let registry = get_registry()
         .lock()
@@ -360,7 +361,8 @@ pub async fn get_repo_files(
         .await
         .map_err(|e| ErrorResponse::internal(format!("Failed to fetch repo files: {}", e)))?;
 
-    let available_memory = available_inference_memory_bytes(&state.inference_runtime);
+    let runtime = state.get_inference_runtime()?;
+    let available_memory = available_inference_memory_bytes(&runtime);
     let recommended_index = hf_models::recommend_variant(&variants, available_memory);
 
     let downloaded_quants = {

crates/goose-server/src/state.rs

@@ -5,7 +5,7 @@ use goose::scheduler_trait::SchedulerTrait;
 use goose::session::SessionManager;
 use std::collections::{HashMap, HashSet};
 use std::path::PathBuf;
-use std::sync::Arc;
+use std::sync::{Arc, OnceLock};
 use tokio::sync::Mutex;
 use tokio::task::JoinHandle;
 
@@ -28,7 +28,7 @@ pub struct AppState {
     pub gateway_manager: Arc<GatewayManager>,
     pub extension_loading_tasks: ExtensionLoadingTasks,
     #[cfg(feature = "local-inference")]
-    pub inference_runtime: Arc<InferenceRuntime>,
+    inference_runtime: Arc<OnceLock<Arc<InferenceRuntime>>>,
     session_buses: Arc<Mutex<HashMap<String, Arc<SessionEventBus>>>>,
 }
 
@@ -48,11 +48,31 @@ impl AppState {
             gateway_manager,
             extension_loading_tasks: Arc::new(Mutex::new(HashMap::new())),
             #[cfg(feature = "local-inference")]
-            inference_runtime: InferenceRuntime::get_or_init(),
+            inference_runtime: Arc::new(OnceLock::new()),
             session_buses: Arc::new(Mutex::new(HashMap::new())),
         }))
     }
 
+    #[cfg(feature = "local-inference")]
+    pub fn get_inference_runtime(&self) -> anyhow::Result<Arc<InferenceRuntime>> {
+        if let Some(runtime) = self.inference_runtime.get() {
+            return Ok(runtime.clone());
+        }
+
+        let runtime = InferenceRuntime::get_or_init()?;
+
+        // Another thread may win the race to cache the runtime in AppState.
+        // In that case, return the already-initialized cached runtime.
+        match self.inference_runtime.set(runtime.clone()) {
+            Ok(()) => Ok(runtime),
+            Err(_) => Ok(self
+                .inference_runtime
+                .get()
+                .expect("inference runtime initialized by another thread")
+                .clone()),
+        }
+    }
+
     pub async fn set_extension_loading_task(
         &self,
         session_id: String,

crates/goose/src/providers/local_inference.rs

@@ -62,10 +62,10 @@ pub struct InferenceRuntime {
 static RUNTIME: StdMutex<Weak<InferenceRuntime>> = StdMutex::new(Weak::new());
 
 impl InferenceRuntime {
-    pub fn get_or_init() -> Arc<Self> {
+    pub fn get_or_init() -> Result<Arc<Self>> {
         let mut guard = RUNTIME.lock().expect("runtime lock poisoned");
         if let Some(runtime) = guard.upgrade() {
-            return runtime;
+            return Ok(runtime);
         }
         // Safety invariant: the Weak::upgrade() check and LlamaBackend::init()
         // both execute inside this same mutex guard, so there is no window where
@@ -80,7 +80,10 @@ impl InferenceRuntime {
                      the mutex guard prevents concurrent re-init"
                 )
             }
-            Err(e) => panic!("Failed to init llama backend: {}", e),
+            Err(e) => {
+                tracing::error!(error = %e, "failed to initialize local inference runtime");
+                return Err(anyhow::anyhow!("Failed to init llama backend: {}", e));
+            }
         };
         llama_cpp_2::send_logs_to_tracing(LogOptions::default());
         let runtime = Arc::new(Self {
@@ -88,7 +91,7 @@ impl InferenceRuntime {
             backend,
         });
         *guard = Arc::downgrade(&runtime);
-        runtime
+        Ok(runtime)
     }
 
     pub fn backend(&self) -> &LlamaBackend {
@@ -357,7 +360,7 @@ pub struct LocalInferenceProvider {
 
 impl LocalInferenceProvider {
     pub async fn from_env(model: ModelConfig, _extensions: Vec<ExtensionConfig>) -> Result<Self> {
-        let runtime = InferenceRuntime::get_or_init();
+        let runtime = InferenceRuntime::get_or_init()?;
         let model_slot = runtime.get_or_create_model_slot(&model.model_name);
         Ok(Self {
             runtime,