Add vision/image support for local inference models (#8442)

Signed-off-by: jh-block <jhugo@block.xyz>

ui/desktop/src/api/types.gen.ts

@@ -612,12 +612,14 @@ export type LoadedProvider = {
 export type LocalModelResponse = {
     filename: string;
     id: string;
+    mmproj_status?: ModelDownloadStatus | null;
     quantization: string;
     recommended: boolean;
     repo_id: string;
     settings: ModelSettings;
     size_bytes: number;
     status: ModelDownloadStatus;
+    vision_capable: boolean;
 };
 
 /**
@@ -821,7 +823,16 @@ export type ModelSettings = {
     enable_thinking?: boolean;
     flash_attention?: boolean | null;
     frequency_penalty?: number;
+    /**
+     * Estimated tokens per image for budget planning before mtmd tokenization.
+     * The actual count is determined after tokenization via `chunks.total_tokens()`.
+     */
+    image_token_estimate?: number;
     max_output_tokens?: number | null;
+    /**
+     * Size of the mmproj file in bytes, used for memory accounting.
+     */
+    mmproj_size_bytes?: number;
     n_batch?: number | null;
     n_gpu_layers?: number | null;
     n_threads?: number | null;
@@ -832,6 +843,11 @@ export type ModelSettings = {
     sampling?: SamplingConfig;
     use_jinja?: boolean;
     use_mlock?: boolean;
+    /**
+     * Whether this model architecture supports vision input.
+     * Derived from the featured model table, not user-configurable.
+     */
+    vision_capable?: boolean;
 };
 
 export type ModelTemplate = {

ui/desktop/src/components/settings/localInference/LocalInferenceSettings.tsx

@@ -1,5 +1,5 @@
 import { useState, useEffect, useCallback, useRef } from 'react';
-import { Download, Trash2, X, ChevronDown, ChevronUp, Settings2 } from 'lucide-react';
+import { Download, Trash2, X, ChevronDown, ChevronUp, Settings2, Eye } from 'lucide-react';
 import { Button } from '../../ui/button';
 import { useModelAndProvider } from '../../ModelAndProviderContext';
 import { defineMessages, useIntl } from '../../../i18n';
@@ -83,8 +83,57 @@ const i18n = defineMessages({
     id: 'localInferenceSettings.modelSettingsTitle',
     defaultMessage: 'Model settings',
   },
+  vision: {
+    id: 'localInferenceSettings.vision',
+    defaultMessage: 'Vision',
+  },
+  visionEncoderDownloading: {
+    id: 'localInferenceSettings.visionEncoderDownloading',
+    defaultMessage: 'Vision encoder downloading…',
+  },
+  visionEncoderNotDownloaded: {
+    id: 'localInferenceSettings.visionEncoderNotDownloaded',
+    defaultMessage: 'Vision encoder not downloaded',
+  },
 });
 
+const VisionBadge = ({ model, intl }: { model: LocalModelResponse; intl: ReturnType<typeof useIntl> }) => {
+  if (!model.vision_capable) return null;
+
+  const mmproj = model.mmproj_status;
+  const isDownloaded = mmproj?.state === 'Downloaded';
+  const isDownloading = mmproj?.state === 'Downloading';
+
+  if (isDownloaded) {
+    return (
+      <span className="inline-flex items-center gap-1 text-xs text-green-400 bg-green-500/10 px-2 py-0.5 rounded">
+        <Eye className="w-3 h-3" />
+        {intl.formatMessage(i18n.vision)}
+      </span>
+    );
+  }
+
+  if (isDownloading) {
+    const percent = mmproj && 'progress_percent' in mmproj
+      ? Math.round(mmproj.progress_percent)
+      : null;
+    return (
+      <span className="inline-flex items-center gap-1 text-xs text-yellow-400 bg-yellow-500/10 px-2 py-0.5 rounded">
+        <Eye className="w-3 h-3" />
+        {intl.formatMessage(i18n.visionEncoderDownloading)}
+        {percent != null && ` ${percent}%`}
+      </span>
+    );
+  }
+
+  return (
+    <span className="inline-flex items-center gap-1 text-xs text-text-muted bg-background-subtle px-2 py-0.5 rounded">
+      <Eye className="w-3 h-3" />
+      {intl.formatMessage(i18n.vision)}
+    </span>
+  );
+};
+
 const formatBytes = (bytes: number): string => {
   if (bytes < 1024) return `${bytes}B`;
   if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(0)}KB`;
@@ -128,6 +177,19 @@ export const LocalInferenceSettings = () => {
     // eslint-disable-next-line react-hooks/exhaustive-deps
   }, []);
 
+  // Poll model list while any vision encoder is downloading
+  useEffect(() => {
+    const hasDownloadingMmproj = models.some(
+      (m) => m.vision_capable && m.mmproj_status?.state === 'Downloading'
+    );
+    if (!hasDownloadingMmproj) return;
+
+    const interval = setInterval(() => {
+      loadModels();
+    }, 2000);
+    return () => clearInterval(interval);
+  }, [models, loadModels]);
+
   const selectModel = async (modelId: string) => {
     try {
       await setConfigProvider({
@@ -365,6 +427,7 @@ export const LocalInferenceSettings = () => {
                           {intl.formatMessage(i18n.recommended)}
                         </span>
                       )}
+                      <VisionBadge model={model} intl={intl} />
                     </div>
                     <div className="flex items-center gap-1">
                       <Button
@@ -414,6 +477,7 @@ export const LocalInferenceSettings = () => {
                           {intl.formatMessage(i18n.recommended)}
                         </span>
                       )}
+                      <VisionBadge model={model} intl={intl} />
                     </div>
                   </div>
                   <Button

ui/desktop/src/i18n/messages/en.json

@@ -1838,6 +1838,15 @@
   "localInferenceSettings.title": {
     "defaultMessage": "Local Inference Models"
   },
+  "localInferenceSettings.vision": {
+    "defaultMessage": "Vision"
+  },
+  "localInferenceSettings.visionEncoderDownloading": {
+    "defaultMessage": "Vision encoder downloading\u2026"
+  },
+  "localInferenceSettings.visionEncoderNotDownloaded": {
+    "defaultMessage": "Vision encoder not downloaded"
+  },
   "localModelManager.active": {
     "defaultMessage": "Active"
   },