Add Linux Vulkan support for local inference (#9038)

Signed-off-by: jh-block <jhugo@block.xyz>

.github/workflows/build-cli.yml

@@ -117,7 +117,11 @@ jobs:
           cross --version
 
           export CC="${{ matrix.cc || ''}}"
-          cross build --release --target ${TARGET} -p goose-cli
+          FEATURE_ARGS=()
+          if [[ "${TARGET}" == *-unknown-linux-gnu ]]; then
+            FEATURE_ARGS=(--features vulkan)
+          fi
+          cross build --release --target ${TARGET} -p goose-cli "${FEATURE_ARGS[@]}"
 
       - name: Setup Rust (Windows)
         if: matrix.os == 'windows'

.github/workflows/bundle-desktop-linux.yml

@@ -79,7 +79,10 @@ jobs:
             protobuf-compiler \
             flatpak \
             flatpak-builder \
-            elfutils
+            elfutils \
+            libvulkan-dev \
+            libvulkan1 \
+            glslc
 
       - name: Setup Flatpak Runtimes
         run: |
@@ -110,7 +113,7 @@ jobs:
           source ./bin/activate-hermit
           export TARGET="x86_64-unknown-linux-gnu"
           rustup target add "${TARGET}"
-          cross build --release --target ${TARGET} -p goose-server
+          cross build --release --target ${TARGET} -p goose-server --features vulkan
 
       - name: Copy binaries into Electron folder
         run: |

.github/workflows/bundle-goose2.yml

@@ -444,7 +444,10 @@ jobs:
             libappindicator3-dev \
             librsvg2-dev \
             patchelf \
-            protobuf-compiler
+            protobuf-compiler \
+            libvulkan-dev \
+            libvulkan1 \
+            glslc
 
       - name: Update versions
         if: inputs.version != ''
@@ -489,7 +492,7 @@ jobs:
         if: inputs.cli-run-id == ''
         run: |
           source ./bin/activate-hermit
-          cargo build --release -p goose-cli --bin goose
+          cargo build --release -p goose-cli --bin goose --features vulkan
 
       - name: Prepare goose binary with target triple
         if: inputs.cli-run-id == ''

BUILDING_LINUX.md

@@ -9,22 +9,22 @@ This guide covers building the goose Desktop application from source on various
 **Debian/Ubuntu:**
 ```bash
 sudo apt update
-sudo apt install -y dpkg fakeroot build-essential clang libxcb1-dev libxcb-util-dev protobuf-compiler
+sudo apt install -y dpkg fakeroot build-essential clang libxcb1-dev libxcb-util-dev protobuf-compiler libvulkan-dev libvulkan1 glslc
 ```
 
 **Arch/Manjaro:**
 ```bash
-sudo pacman -S --needed dpkg fakeroot base-devel
+sudo pacman -S --needed dpkg fakeroot base-devel vulkan-headers vulkan-icd-loader shaderc
 ```
 
 **Fedora/RHEL/CentOS:**
 ```bash
-sudo dnf install dpkg-dev fakeroot gcc gcc-c++ make libxcb-devel
+sudo dnf install dpkg-dev fakeroot gcc gcc-c++ make libxcb-devel vulkan-headers vulkan-loader glslc
 ```
 
 **openSUSE:**
 ```bash
-sudo zypper install dpkg fakeroot gcc gcc-c++ make
+sudo zypper install dpkg fakeroot gcc gcc-c++ make vulkan-headers vulkan-loader glslc
 ```
 
 **android / termux:**
@@ -125,7 +125,7 @@ sudo dpkg -i out/make/deb/x64/goose_*.deb
 ### Common Issues
 
 #### Missing System Dependencies
-If you see errors about missing `dpkg` or `fakeroot`:
+If you see errors about missing `dpkg`, `fakeroot`, Vulkan headers, or `glslc`:
 ```bash
 # Install the missing packages for your distribution (see Prerequisites above)
 ```

Cross.toml

@@ -15,6 +15,8 @@ pre-build = [
      libssl-dev:arm64 \
      libdbus-1-dev:arm64 \
      libxcb1-dev:arm64 \
+     libvulkan-dev:arm64 \
+     glslc \
      gcc-10
     """
 ]
@@ -30,6 +32,8 @@ pre-build = [
      libssl-dev \
      libdbus-1-dev \
      libxcb1-dev \
+     libvulkan-dev \
+     glslc \
      gcc-10
     """
 ]

Justfile

@@ -394,6 +394,7 @@ release-notes old:
 
 ### s = file separator based on OS
 s := if os() == "windows" { "\\" } else { "/" }
+linux_vulkan_features := if os() == "linux" { "--features vulkan" } else { "" }
 
 ### testing/debugging
 os:
@@ -486,6 +487,6 @@ record-mcp-tests: build-test-tools
   git add crates/goose/tests/mcp_replays/
 
 bundle-goose2:
-  cargo build --release --package goose-cli --bin goose
+  cargo build --release --package goose-cli --bin goose {{linux_vulkan_features}}
   cp target/release/goose target/release/goose-$(rustc --print host-tuple)
   @just goose2::bundle

crates/goose-cli/Cargo.toml

@@ -74,6 +74,7 @@ code-mode = ["goose/code-mode"]
 local-inference = ["goose/local-inference"]
 aws-providers = ["goose/aws-providers"]
 cuda = ["goose/cuda", "local-inference"]
+vulkan = ["goose/vulkan", "local-inference"]
 telemetry = ["goose/telemetry"]
 otel = ["goose/otel"]
 # disables the update command

crates/goose-server/Cargo.toml

@@ -17,6 +17,7 @@ code-mode = ["goose/code-mode"]
 local-inference = ["goose/local-inference"]
 aws-providers = ["goose/aws-providers"]
 cuda = ["goose/cuda", "local-inference"]
+vulkan = ["goose/vulkan", "local-inference"]
 telemetry = ["goose/telemetry"]
 otel = ["goose/otel"]
 rustls-tls = [

crates/goose/Cargo.toml

@@ -37,6 +37,7 @@ aws-providers = [
     "dep:aws-sdk-sagemakerruntime",
 ]
 cuda = ["local-inference", "candle-core/cuda", "candle-nn/cuda", "llama-cpp-2/cuda"]
+vulkan = ["local-inference", "llama-cpp-2/vulkan"]
 rustls-tls = [
     "reqwest/rustls",
     "rmcp/reqwest",

crates/goose/src/providers/local_inference.rs

@@ -86,6 +86,7 @@ impl InferenceRuntime {
             }
         };
         llama_cpp_2::send_logs_to_tracing(LogOptions::default());
+        log_inference_backend_devices();
         let runtime = Arc::new(Self {
             models: StdMutex::new(HashMap::new()),
             backend,
@@ -156,20 +157,55 @@ pub fn resolve_model_path(model_id: &str) -> Option<ResolvedModelPaths> {
     None
 }
 
+fn is_accelerator_device(device_type: LlamaBackendDeviceType) -> bool {
+    matches!(
+        device_type,
+        LlamaBackendDeviceType::Gpu
+            | LlamaBackendDeviceType::IntegratedGpu
+            | LlamaBackendDeviceType::Accelerator
+    )
+}
+
+fn is_non_cpu_device(device_type: LlamaBackendDeviceType) -> bool {
+    !matches!(device_type, LlamaBackendDeviceType::Cpu)
+}
+
+fn log_inference_backend_devices() {
+    let devices = list_llama_ggml_backend_devices();
+    let non_cpu_devices: Vec<_> = devices
+        .iter()
+        .filter(|device| is_non_cpu_device(device.device_type))
+        .collect();
+
+    if non_cpu_devices.is_empty() {
+        tracing::info!(
+            device_count = devices.len(),
+            "No non-CPU llama.cpp backend devices detected for local inference"
+        );
+        return;
+    }
+
+    for device in non_cpu_devices {
+        tracing::info!(
+            index = device.index,
+            backend = %device.backend,
+            name = %device.name,
+            description = %device.description,
+            device_type = ?device.device_type,
+            memory_total_bytes = device.memory_total as u64,
+            memory_free_bytes = device.memory_free as u64,
+            "Non-CPU llama.cpp backend device detected for local inference"
+        );
+    }
+}
+
 pub fn available_inference_memory_bytes(runtime: &InferenceRuntime) -> u64 {
     let _ = &runtime.backend;
     let devices = list_llama_ggml_backend_devices();
 
     let accel_memory = devices
         .iter()
-        .filter(|d| {
-            matches!(
-                d.device_type,
-                LlamaBackendDeviceType::Gpu
-                    | LlamaBackendDeviceType::IntegratedGpu
-                    | LlamaBackendDeviceType::Accelerator
-            )
-        })
+        .filter(|d| is_accelerator_device(d.device_type))
         .map(|d| d.memory_free as u64)
         .max()
         .unwrap_or(0);

ui/desktop/forge.config.ts

@@ -92,6 +92,7 @@ module.exports = {
         categories: ['Development'],
         desktopTemplate: './forge.deb.desktop',
         options: {
+          depends: ['libvulkan1'],
           icon: 'src/images/icon.png',
           prefix: '/opt',
         },
@@ -107,6 +108,7 @@ module.exports = {
         categories: ['Development'],
         desktopTemplate: './forge.rpm.desktop',
         options: {
+          requires: ['vulkan-loader'],
           icon: 'src/images/icon.png',
           prefix: '/opt',
           fpm: ['--rpm-rpmbuild-define', '_build_id_links none'],

ui/goose-binary/README.md

@@ -32,6 +32,11 @@ npx tsx scripts/build-native.ts darwin-arm64 linux-x64
 The built binaries are placed into `ui/goose-binary/goose-binary-{platform}/bin/`.
 These directories are git-ignored.
 
+Linux native binaries are built with local inference Vulkan support. Linux build
+hosts need Vulkan headers and `glslc`; Linux runtime hosts need the Vulkan loader
+package, such as `libvulkan1` on Debian/Ubuntu or `vulkan-loader` on RPM-based
+distributions.
+
 ## Publishing
 
 Publishing is handled by GitHub Actions. See `.github/workflows/publish-npm.yml`.

ui/goose2/src-tauri/tauri.conf.json

@@ -50,6 +50,14 @@
       "../distro": "distro"
     },
     "externalBin": ["../../../target/release/goose"],
+    "linux": {
+      "deb": {
+        "depends": ["libvulkan1"]
+      },
+      "rpm": {
+        "depends": ["vulkan-loader"]
+      }
+    },
     "macOS": {
       "entitlements": "entitlements.plist",
       "hardenedRuntime": true

ui/scripts/publish.sh

@@ -75,9 +75,10 @@ set -euo pipefail
 apt-get update -qq
 apt-get install -y -qq --no-install-recommends \
   build-essential cmake pkg-config libssl-dev libdbus-1-dev \
-  libclang-dev protobuf-compiler libprotobuf-dev ca-certificates >/dev/null 2>&1
+  libclang-dev protobuf-compiler libprotobuf-dev ca-certificates \
+  libvulkan-dev libvulkan1 glslc >/dev/null 2>&1
 echo "==> Compiling goose (this takes a while)..."
-cargo build --release --bin goose
+cargo build --release --bin goose --features vulkan
 cp /build/target/release/goose /output/goose
 echo "==> Done"
 '
@@ -116,12 +117,13 @@ FROM rust:1.92-bookworm
 RUN apt-get update -qq && \
     apt-get install -y -qq --no-install-recommends \
       build-essential cmake pkg-config libssl-dev libdbus-1-dev \
-      libclang-dev protobuf-compiler libprotobuf-dev ca-certificates >/dev/null 2>&1 && \
+      libclang-dev protobuf-compiler libprotobuf-dev ca-certificates \
+      libvulkan-dev libvulkan1 glslc >/dev/null 2>&1 && \
     rm -rf /var/lib/apt/lists/*
 WORKDIR /build
 COPY . .
 RUN mkdir -p /output && \
-    cargo build --release --bin goose && \
+    cargo build --release --bin goose --features vulkan && \
     cp target/release/goose /output/goose
 DEOF
 

ui/sdk/scripts/build-native.ts

@@ -57,10 +57,16 @@ function buildTarget(platform: string): void {
   console.log(`==> Building goose for ${platform} (${rustTarget})`);
 
   try {
-    execSync(`cargo build --release --target ${rustTarget} --bin goose`, {
-      cwd: ROOT,
-      stdio: "inherit",
-    });
+    const featureArgs = platform.startsWith("linux-")
+      ? " --features vulkan"
+      : "";
+    execSync(
+      `cargo build --release --target ${rustTarget} --bin goose${featureArgs}`,
+      {
+        cwd: ROOT,
+        stdio: "inherit",
+      },
+    );
   } catch (err) {
     console.error(`Failed to build for ${platform}`);
     throw err;