feat: add local inference provider with llama.cpp backend and HuggingFace model management (#6933)

Co-authored-by: Douwe Osinga <douwe@squareup.com> Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Co-authored-by: jh-block <jhugo@block.xyz> Co-authored-by: Spence <spencermartin@squareup.com> Co-authored-by: Michael Neale <michael.neale@gmail.com>
2026-02-19 18:30:05 +00:00
parent 6928c8cee1
commit ddd35f6d47
44 changed files with 7171 additions and 181 deletions
@@ -1827,6 +1827,308 @@
        }
      }
    },
+    "/local-inference/download": {
+      "post": {
+        "tags": [
+          "super::routes::local_inference"
+        ],
+        "operationId": "download_hf_model",
+        "requestBody": {
+          "content": {
+            "application/json": {
+              "schema": {
+                "$ref": "#/components/schemas/DownloadModelRequest"
+              }
+            }
+          },
+          "required": true
+        },
+        "responses": {
+          "202": {
+            "description": "Download started",
+            "content": {
+              "text/plain": {
+                "schema": {
+                  "type": "string"
+                }
+              }
+            }
+          },
+          "400": {
+            "description": "Invalid request"
+          }
+        }
+      }
+    },
+    "/local-inference/models": {
+      "get": {
+        "tags": [
+          "super::routes::local_inference"
+        ],
+        "operationId": "list_local_models",
+        "responses": {
+          "200": {
+            "description": "List of available local LLM models",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "type": "array",
+                  "items": {
+                    "$ref": "#/components/schemas/LocalModelResponse"
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    },
+    "/local-inference/models/{model_id}": {
+      "delete": {
+        "tags": [
+          "super::routes::local_inference"
+        ],
+        "operationId": "delete_local_model",
+        "parameters": [
+          {
+            "name": "model_id",
+            "in": "path",
+            "required": true,
+            "schema": {
+              "type": "string"
+            }
+          }
+        ],
+        "responses": {
+          "200": {
+            "description": "Model deleted"
+          },
+          "404": {
+            "description": "Model not found"
+          }
+        }
+      }
+    },
+    "/local-inference/models/{model_id}/download": {
+      "get": {
+        "tags": [
+          "super::routes::local_inference"
+        ],
+        "operationId": "get_local_model_download_progress",
+        "parameters": [
+          {
+            "name": "model_id",
+            "in": "path",
+            "required": true,
+            "schema": {
+              "type": "string"
+            }
+          }
+        ],
+        "responses": {
+          "200": {
+            "description": "Download progress",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/DownloadProgress"
+                }
+              }
+            }
+          },
+          "404": {
+            "description": "No active download"
+          }
+        }
+      },
+      "delete": {
+        "tags": [
+          "super::routes::local_inference"
+        ],
+        "operationId": "cancel_local_model_download",
+        "parameters": [
+          {
+            "name": "model_id",
+            "in": "path",
+            "required": true,
+            "schema": {
+              "type": "string"
+            }
+          }
+        ],
+        "responses": {
+          "200": {
+            "description": "Download cancelled"
+          },
+          "404": {
+            "description": "No active download"
+          }
+        }
+      }
+    },
+    "/local-inference/models/{model_id}/settings": {
+      "get": {
+        "tags": [
+          "super::routes::local_inference"
+        ],
+        "operationId": "get_model_settings",
+        "parameters": [
+          {
+            "name": "model_id",
+            "in": "path",
+            "required": true,
+            "schema": {
+              "type": "string"
+            }
+          }
+        ],
+        "responses": {
+          "200": {
+            "description": "Model settings",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/ModelSettings"
+                }
+              }
+            }
+          },
+          "404": {
+            "description": "Model not found"
+          }
+        }
+      },
+      "put": {
+        "tags": [
+          "super::routes::local_inference"
+        ],
+        "operationId": "update_model_settings",
+        "parameters": [
+          {
+            "name": "model_id",
+            "in": "path",
+            "required": true,
+            "schema": {
+              "type": "string"
+            }
+          }
+        ],
+        "requestBody": {
+          "content": {
+            "application/json": {
+              "schema": {
+                "$ref": "#/components/schemas/ModelSettings"
+              }
+            }
+          },
+          "required": true
+        },
+        "responses": {
+          "200": {
+            "description": "Settings updated",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/ModelSettings"
+                }
+              }
+            }
+          },
+          "404": {
+            "description": "Model not found"
+          },
+          "500": {
+            "description": "Failed to save settings"
+          }
+        }
+      }
+    },
+    "/local-inference/repo/{author}/{repo}/files": {
+      "get": {
+        "tags": [
+          "super::routes::local_inference"
+        ],
+        "operationId": "get_repo_files",
+        "parameters": [
+          {
+            "name": "author",
+            "in": "path",
+            "required": true,
+            "schema": {
+              "type": "string"
+            }
+          },
+          {
+            "name": "repo",
+            "in": "path",
+            "required": true,
+            "schema": {
+              "type": "string"
+            }
+          }
+        ],
+        "responses": {
+          "200": {
+            "description": "GGUF files in the repo",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/RepoVariantsResponse"
+                }
+              }
+            }
+          }
+        }
+      }
+    },
+    "/local-inference/search": {
+      "get": {
+        "tags": [
+          "super::routes::local_inference"
+        ],
+        "operationId": "search_hf_models",
+        "parameters": [
+          {
+            "name": "q",
+            "in": "query",
+            "description": "Search query",
+            "required": true,
+            "schema": {
+              "type": "string"
+            }
+          },
+          {
+            "name": "limit",
+            "in": "query",
+            "description": "Max results",
+            "required": false,
+            "schema": {
+              "type": "integer",
+              "nullable": true,
+              "minimum": 0
+            }
+          }
+        ],
+        "responses": {
+          "200": {
+            "description": "Search results",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "type": "array",
+                  "items": {
+                    "$ref": "#/components/schemas/HfModelInfo"
+                  }
+                }
+              }
+            }
+          },
+          "500": {
+            "description": "Search failed"
+          }
+        }
+      }
+    },
    "/mcp-ui-proxy": {
      "get": {
        "tags": [
@@ -3954,6 +4256,18 @@
          }
        }
      },
+      "DownloadModelRequest": {
+        "type": "object",
+        "required": [
+          "spec"
+        ],
+        "properties": {
+          "spec": {
+            "type": "string",
+            "description": "Model spec like \"bartowski/Llama-3.2-3B-Instruct-GGUF:Q4_K_M\""
+          }
+        }
+      },
      "DownloadProgress": {
        "type": "object",
        "required": [
@@ -4588,6 +4902,100 @@
          }
        ]
      },
+      "HfGgufFile": {
+        "type": "object",
+        "description": "A single downloadable GGUF file (used internally and for downloads).",
+        "required": [
+          "filename",
+          "size_bytes",
+          "quantization",
+          "download_url"
+        ],
+        "properties": {
+          "download_url": {
+            "type": "string"
+          },
+          "filename": {
+            "type": "string"
+          },
+          "quantization": {
+            "type": "string"
+          },
+          "size_bytes": {
+            "type": "integer",
+            "format": "int64",
+            "minimum": 0
+          }
+        }
+      },
+      "HfModelInfo": {
+        "type": "object",
+        "required": [
+          "repo_id",
+          "author",
+          "model_name",
+          "downloads",
+          "gguf_files"
+        ],
+        "properties": {
+          "author": {
+            "type": "string"
+          },
+          "downloads": {
+            "type": "integer",
+            "format": "int64",
+            "minimum": 0
+          },
+          "gguf_files": {
+            "type": "array",
+            "items": {
+              "$ref": "#/components/schemas/HfGgufFile"
+            }
+          },
+          "model_name": {
+            "type": "string"
+          },
+          "repo_id": {
+            "type": "string"
+          }
+        }
+      },
+      "HfQuantVariant": {
+        "type": "object",
+        "description": "A quantization variant — groups sharded files into one logical entry.",
+        "required": [
+          "quantization",
+          "size_bytes",
+          "filename",
+          "download_url",
+          "description",
+          "quality_rank"
+        ],
+        "properties": {
+          "description": {
+            "type": "string"
+          },
+          "download_url": {
+            "type": "string"
+          },
+          "filename": {
+            "type": "string"
+          },
+          "quality_rank": {
+            "type": "integer",
+            "format": "int32",
+            "minimum": 0
+          },
+          "quantization": {
+            "type": "string"
+          },
+          "size_bytes": {
+            "type": "integer",
+            "format": "int64",
+            "minimum": 0
+          }
+        }
+      },
      "Icon": {
        "type": "object",
        "required": [
@@ -4773,6 +5181,51 @@
          }
        }
      },
+      "LocalModelResponse": {
+        "type": "object",
+        "required": [
+          "id",
+          "display_name",
+          "repo_id",
+          "filename",
+          "quantization",
+          "size_bytes",
+          "status",
+          "recommended",
+          "settings"
+        ],
+        "properties": {
+          "display_name": {
+            "type": "string"
+          },
+          "filename": {
+            "type": "string"
+          },
+          "id": {
+            "type": "string"
+          },
+          "quantization": {
+            "type": "string"
+          },
+          "recommended": {
+            "type": "boolean"
+          },
+          "repo_id": {
+            "type": "string"
+          },
+          "settings": {
+            "$ref": "#/components/schemas/ModelSettings"
+          },
+          "size_bytes": {
+            "type": "integer",
+            "format": "int64",
+            "minimum": 0
+          },
+          "status": {
+            "$ref": "#/components/schemas/ModelDownloadStatus"
+          }
+        }
+      },
      "McpAppResource": {
        "type": "object",
        "description": "MCP App Resource\nRepresents a UI resource that can be rendered in an MCP App",
@@ -5293,6 +5746,78 @@
          }
        }
      },
+      "ModelDownloadStatus": {
+        "oneOf": [
+          {
+            "type": "object",
+            "required": [
+              "state"
+            ],
+            "properties": {
+              "state": {
+                "type": "string",
+                "enum": [
+                  "NotDownloaded"
+                ]
+              }
+            }
+          },
+          {
+            "type": "object",
+            "required": [
+              "progress_percent",
+              "bytes_downloaded",
+              "total_bytes",
+              "state"
+            ],
+            "properties": {
+              "bytes_downloaded": {
+                "type": "integer",
+                "format": "int64",
+                "minimum": 0
+              },
+              "progress_percent": {
+                "type": "number",
+                "format": "float"
+              },
+              "speed_bps": {
+                "type": "integer",
+                "format": "int64",
+                "nullable": true,
+                "minimum": 0
+              },
+              "state": {
+                "type": "string",
+                "enum": [
+                  "Downloading"
+                ]
+              },
+              "total_bytes": {
+                "type": "integer",
+                "format": "int64",
+                "minimum": 0
+              }
+            }
+          },
+          {
+            "type": "object",
+            "required": [
+              "state"
+            ],
+            "properties": {
+              "state": {
+                "type": "string",
+                "enum": [
+                  "Downloaded"
+                ]
+              }
+            }
+          }
+        ],
+        "discriminator": {
+          "propertyName": "state"
+        }
+      },
      "ModelInfo": {
        "type": "object",
        "description": "Information about a model's capabilities",
@@ -5417,6 +5942,71 @@
          }
        }
      },
+      "ModelSettings": {
+        "type": "object",
+        "properties": {
+          "context_size": {
+            "type": "integer",
+            "format": "int32",
+            "nullable": true,
+            "minimum": 0
+          },
+          "flash_attention": {
+            "type": "boolean",
+            "nullable": true
+          },
+          "frequency_penalty": {
+            "type": "number",
+            "format": "float"
+          },
+          "max_output_tokens": {
+            "type": "integer",
+            "nullable": true,
+            "minimum": 0
+          },
+          "n_batch": {
+            "type": "integer",
+            "format": "int32",
+            "nullable": true,
+            "minimum": 0
+          },
+          "n_gpu_layers": {
+            "type": "integer",
+            "format": "int32",
+            "nullable": true,
+            "minimum": 0
+          },
+          "n_threads": {
+            "type": "integer",
+            "format": "int32",
+            "nullable": true
+          },
+          "native_tool_calling": {
+            "type": "boolean"
+          },
+          "presence_penalty": {
+            "type": "number",
+            "format": "float"
+          },
+          "repeat_last_n": {
+            "type": "integer",
+            "format": "int32"
+          },
+          "repeat_penalty": {
+            "type": "number",
+            "format": "float"
+          },
+          "sampling": {
+            "$ref": "#/components/schemas/SamplingConfig"
+          },
+          "use_jinja": {
+            "type": "boolean"
+          },
+          "use_mlock": {
+            "type": "boolean"
+          }
+        }
+      },
      "ParseRecipeRequest": {
        "type": "object",
        "required": [
@@ -6005,6 +6595,25 @@
          }
        }
      },
+      "RepoVariantsResponse": {
+        "type": "object",
+        "required": [
+          "variants"
+        ],
+        "properties": {
+          "recommended_index": {
+            "type": "integer",
+            "nullable": true,
+            "minimum": 0
+          },
+          "variants": {
+            "type": "array",
+            "items": {
+              "$ref": "#/components/schemas/HfQuantVariant"
+            }
+          }
+        }
+      },
      "ResourceContents": {
        "anyOf": [
          {
@@ -6196,6 +6805,97 @@
          }
        }
      },
+      "SamplingConfig": {
+        "oneOf": [
+          {
+            "type": "object",
+            "required": [
+              "type"
+            ],
+            "properties": {
+              "type": {
+                "type": "string",
+                "enum": [
+                  "Greedy"
+                ]
+              }
+            }
+          },
+          {
+            "type": "object",
+            "required": [
+              "temperature",
+              "top_k",
+              "top_p",
+              "min_p",
+              "type"
+            ],
+            "properties": {
+              "min_p": {
+                "type": "number",
+                "format": "float"
+              },
+              "seed": {
+                "type": "integer",
+                "format": "int32",
+                "nullable": true,
+                "minimum": 0
+              },
+              "temperature": {
+                "type": "number",
+                "format": "float"
+              },
+              "top_k": {
+                "type": "integer",
+                "format": "int32"
+              },
+              "top_p": {
+                "type": "number",
+                "format": "float"
+              },
+              "type": {
+                "type": "string",
+                "enum": [
+                  "Temperature"
+                ]
+              }
+            }
+          },
+          {
+            "type": "object",
+            "required": [
+              "tau",
+              "eta",
+              "type"
+            ],
+            "properties": {
+              "eta": {
+                "type": "number",
+                "format": "float"
+              },
+              "seed": {
+                "type": "integer",
+                "format": "int32",
+                "nullable": true,
+                "minimum": 0
+              },
+              "tau": {
+                "type": "number",
+                "format": "float"
+              },
+              "type": {
+                "type": "string",
+                "enum": [
+                  "MirostatV2"
+                ]
+              }
+            }
+          }
+        ],
+        "discriminator": {
+          "propertyName": "type"
+        }
+      },
      "SavePromptRequest": {
        "type": "object",
        "required": [
@@ -232,6 +232,13 @@ export type DictationProviderStatus = {
    uses_provider_config: boolean;
 };

+export type DownloadModelRequest = {
+    /**
+     * Model spec like "bartowski/Llama-3.2-3B-Instruct-GGUF:Q4_K_M"
+     */
+    spec: string;
+};
+
 export type DownloadProgress = {
    /**
     * Bytes downloaded so far
@@ -446,6 +453,36 @@ export type GooseApp = McpAppResource & (WindowProps | null) & {
    prd?: string | null;
 };

+/**
+ * A single downloadable GGUF file (used internally and for downloads).
+ */
+export type HfGgufFile = {
+    download_url: string;
+    filename: string;
+    quantization: string;
+    size_bytes: number;
+};
+
+export type HfModelInfo = {
+    author: string;
+    downloads: number;
+    gguf_files: Array<HfGgufFile>;
+    model_name: string;
+    repo_id: string;
+};
+
+/**
+ * A quantization variant — groups sharded files into one logical entry.
+ */
+export type HfQuantVariant = {
+    description: string;
+    download_url: string;
+    filename: string;
+    quality_rank: number;
+    quantization: string;
+    size_bytes: number;
+};
+
 export type Icon = {
    mimeType?: string;
    sizes?: Array<string>;
@@ -511,6 +548,18 @@ export type LoadedProvider = {
    is_editable: boolean;
 };

+export type LocalModelResponse = {
+    display_name: string;
+    filename: string;
+    id: string;
+    quantization: string;
+    recommended: boolean;
+    repo_id: string;
+    settings: ModelSettings;
+    size_bytes: number;
+    status: ModelDownloadStatus;
+};
+
 /**
 * MCP App Resource
 * Represents a UI resource that can be rendered in an MCP App
@@ -638,6 +687,18 @@ export type ModelConfig = {
    toolshim_model?: string | null;
 };

+export type ModelDownloadStatus = {
+    state: 'NotDownloaded';
+} | {
+    bytes_downloaded: number;
+    progress_percent: number;
+    speed_bps?: number | null;
+    state: 'Downloading';
+    total_bytes: number;
+} | {
+    state: 'Downloaded';
+};
+
 /**
 * Information about a model's capabilities
 */
@@ -690,6 +751,23 @@ export type ModelInfoResponse = {
    source: string;
 };

+export type ModelSettings = {
+    context_size?: number | null;
+    flash_attention?: boolean | null;
+    frequency_penalty?: number;
+    max_output_tokens?: number | null;
+    n_batch?: number | null;
+    n_gpu_layers?: number | null;
+    n_threads?: number | null;
+    native_tool_calling?: boolean;
+    presence_penalty?: number;
+    repeat_last_n?: number;
+    repeat_penalty?: number;
+    sampling?: SamplingConfig;
+    use_jinja?: boolean;
+    use_mlock?: boolean;
+};
+
 export type ParseRecipeRequest = {
    content: string;
 };
@@ -909,6 +987,11 @@ export type RemoveExtensionRequest = {
    session_id: string;
 };

+export type RepoVariantsResponse = {
+    recommended_index?: number | null;
+    variants: Array<HfQuantVariant>;
+};
+
 export type ResourceContents = {
    _meta?: {
        [key: string]: unknown;
@@ -986,6 +1069,22 @@ export type RunNowResponse = {
    session_id: string;
 };

+export type SamplingConfig = {
+    type: 'Greedy';
+} | {
+    min_p: number;
+    seed?: number | null;
+    temperature: number;
+    top_k: number;
+    top_p: number;
+    type: 'Temperature';
+} | {
+    eta: number;
+    seed?: number | null;
+    tau: number;
+    type: 'MirostatV2';
+};
+
 export type SavePromptRequest = {
    content: string;
 };
@@ -2836,6 +2935,221 @@ export type StartTetrateSetupResponses = {

 export type StartTetrateSetupResponse = StartTetrateSetupResponses[keyof StartTetrateSetupResponses];

+export type DownloadHfModelData = {
+    body: DownloadModelRequest;
+    path?: never;
+    query?: never;
+    url: '/local-inference/download';
+};
+
+export type DownloadHfModelErrors = {
+    /**
+     * Invalid request
+     */
+    400: unknown;
+};
+
+export type DownloadHfModelResponses = {
+    /**
+     * Download started
+     */
+    202: string;
+};
+
+export type DownloadHfModelResponse = DownloadHfModelResponses[keyof DownloadHfModelResponses];
+
+export type ListLocalModelsData = {
+    body?: never;
+    path?: never;
+    query?: never;
+    url: '/local-inference/models';
+};
+
+export type ListLocalModelsResponses = {
+    /**
+     * List of available local LLM models
+     */
+    200: Array<LocalModelResponse>;
+};
+
+export type ListLocalModelsResponse = ListLocalModelsResponses[keyof ListLocalModelsResponses];
+
+export type DeleteLocalModelData = {
+    body?: never;
+    path: {
+        model_id: string;
+    };
+    query?: never;
+    url: '/local-inference/models/{model_id}';
+};
+
+export type DeleteLocalModelErrors = {
+    /**
+     * Model not found
+     */
+    404: unknown;
+};
+
+export type DeleteLocalModelResponses = {
+    /**
+     * Model deleted
+     */
+    200: unknown;
+};
+
+export type CancelLocalModelDownloadData = {
+    body?: never;
+    path: {
+        model_id: string;
+    };
+    query?: never;
+    url: '/local-inference/models/{model_id}/download';
+};
+
+export type CancelLocalModelDownloadErrors = {
+    /**
+     * No active download
+     */
+    404: unknown;
+};
+
+export type CancelLocalModelDownloadResponses = {
+    /**
+     * Download cancelled
+     */
+    200: unknown;
+};
+
+export type GetLocalModelDownloadProgressData = {
+    body?: never;
+    path: {
+        model_id: string;
+    };
+    query?: never;
+    url: '/local-inference/models/{model_id}/download';
+};
+
+export type GetLocalModelDownloadProgressErrors = {
+    /**
+     * No active download
+     */
+    404: unknown;
+};
+
+export type GetLocalModelDownloadProgressResponses = {
+    /**
+     * Download progress
+     */
+    200: DownloadProgress;
+};
+
+export type GetLocalModelDownloadProgressResponse = GetLocalModelDownloadProgressResponses[keyof GetLocalModelDownloadProgressResponses];
+
+export type GetModelSettingsData = {
+    body?: never;
+    path: {
+        model_id: string;
+    };
+    query?: never;
+    url: '/local-inference/models/{model_id}/settings';
+};
+
+export type GetModelSettingsErrors = {
+    /**
+     * Model not found
+     */
+    404: unknown;
+};
+
+export type GetModelSettingsResponses = {
+    /**
+     * Model settings
+     */
+    200: ModelSettings;
+};
+
+export type GetModelSettingsResponse = GetModelSettingsResponses[keyof GetModelSettingsResponses];
+
+export type UpdateModelSettingsData = {
+    body: ModelSettings;
+    path: {
+        model_id: string;
+    };
+    query?: never;
+    url: '/local-inference/models/{model_id}/settings';
+};
+
+export type UpdateModelSettingsErrors = {
+    /**
+     * Model not found
+     */
+    404: unknown;
+    /**
+     * Failed to save settings
+     */
+    500: unknown;
+};
+
+export type UpdateModelSettingsResponses = {
+    /**
+     * Settings updated
+     */
+    200: ModelSettings;
+};
+
+export type UpdateModelSettingsResponse = UpdateModelSettingsResponses[keyof UpdateModelSettingsResponses];
+
+export type GetRepoFilesData = {
+    body?: never;
+    path: {
+        author: string;
+        repo: string;
+    };
+    query?: never;
+    url: '/local-inference/repo/{author}/{repo}/files';
+};
+
+export type GetRepoFilesResponses = {
+    /**
+     * GGUF files in the repo
+     */
+    200: RepoVariantsResponse;
+};
+
+export type GetRepoFilesResponse = GetRepoFilesResponses[keyof GetRepoFilesResponses];
+
+export type SearchHfModelsData = {
+    body?: never;
+    path?: never;
+    query: {
+        /**
+         * Search query
+         */
+        q: string;
+        /**
+         * Max results
+         */
+        limit?: number | null;
+    };
+    url: '/local-inference/search';
+};
+
+export type SearchHfModelsErrors = {
+    /**
+     * Search failed
+     */
+    500: unknown;
+};
+
+export type SearchHfModelsResponses = {
+    /**
+     * Search results
+     */
+    200: Array<HfModelInfo>;
+};
+
+export type SearchHfModelsResponse = SearchHfModelsResponses[keyof SearchHfModelsResponses];
+
 export type McpUiProxyData = {
    body?: never;
    path?: never;
@@ -0,0 +1,397 @@
+import { useState, useEffect, useCallback, useRef } from 'react';
+import { useConfig } from './ConfigContext';
+import {
+  listLocalModels,
+  downloadHfModel,
+  getLocalModelDownloadProgress,
+  cancelLocalModelDownload,
+  type DownloadProgress,
+  type LocalModelResponse,
+} from '../api';
+import { toastService } from '../toasts';
+import { trackOnboardingSetupFailed } from '../utils/analytics';
+import { Goose } from './icons';
+
+interface LocalModelSetupProps {
+  onSuccess: () => void;
+  onCancel: () => void;
+}
+
+const formatBytes = (bytes: number): string => {
+  if (bytes < 1024) return `${bytes}B`;
+  if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(0)}KB`;
+  if (bytes < 1024 * 1024 * 1024) return `${(bytes / (1024 * 1024)).toFixed(0)}MB`;
+  return `${(bytes / (1024 * 1024 * 1024)).toFixed(1)}GB`;
+};
+
+const formatSize = (bytes: number): string => {
+  const mb = bytes / (1024 * 1024);
+  return mb >= 1024 ? `${(mb / 1024).toFixed(1)}GB` : `${mb.toFixed(0)}MB`;
+};
+
+type SetupPhase = 'loading' | 'select' | 'downloading' | 'error';
+
+export function LocalModelSetup({ onSuccess, onCancel }: LocalModelSetupProps) {
+  const { upsert } = useConfig();
+  const [phase, setPhase] = useState<SetupPhase>('loading');
+  const [models, setModels] = useState<LocalModelResponse[]>([]);
+  const [selectedModelId, setSelectedModelId] = useState<string | null>(null);
+  const [downloadProgress, setDownloadProgress] = useState<DownloadProgress | null>(null);
+  const [errorMessage, setErrorMessage] = useState<string | null>(null);
+  const [showAllModels, setShowAllModels] = useState(false);
+  const pollRef = useRef<ReturnType<typeof setInterval> | null>(null);
+
+  const cleanup = useCallback(() => {
+    if (pollRef.current) {
+      clearInterval(pollRef.current);
+      pollRef.current = null;
+    }
+  }, []);
+
+  useEffect(() => cleanup, [cleanup]);
+
+  useEffect(() => {
+    const load = async () => {
+      try {
+        const response = await listLocalModels();
+        if (response.data) {
+          setModels(response.data);
+
+          const alreadyDownloaded = response.data.find((m) => m.status.state === 'Downloaded');
+          if (alreadyDownloaded) {
+            setSelectedModelId(alreadyDownloaded.id);
+          } else {
+            const recommended = response.data.find((m: LocalModelResponse) => m.recommended);
+            if (recommended) setSelectedModelId(recommended.id);
+          }
+        }
+      } catch (error) {
+        console.error('Failed to load local models:', error);
+        setErrorMessage('Failed to load available models. Please try again.');
+        setPhase('error');
+        return;
+      }
+      setPhase('select');
+    };
+    load();
+  }, []);
+
+  const finishSetup = async (modelId: string) => {
+    await upsert('GOOSE_PROVIDER', 'local', false);
+    await upsert('GOOSE_MODEL', modelId, false);
+    toastService.success({
+      title: 'Local Model Ready',
+      msg: `Running entirely on your machine with ${modelId}.`,
+    });
+    onSuccess();
+  };
+
+  const startDownload = async (modelId: string) => {
+    setPhase('downloading');
+    setDownloadProgress(null);
+    setErrorMessage(null);
+
+    const model = models.find((m) => m.id === modelId);
+    if (!model) {
+      setErrorMessage('Model not found');
+      setPhase('error');
+      return;
+    }
+
+    try {
+      await downloadHfModel({ body: { spec: model.id } });
+    } catch (error) {
+      console.error('Failed to start download:', error);
+      setErrorMessage('Failed to start download. Please try again.');
+      trackOnboardingSetupFailed('local', 'download_start_failed');
+      setPhase('error');
+      return;
+    }
+
+    pollRef.current = setInterval(async () => {
+      try {
+        const response = await getLocalModelDownloadProgress({ path: { model_id: modelId } });
+        if (response.data) {
+          setDownloadProgress(response.data);
+          if (response.data.status === 'completed') {
+            cleanup();
+            await finishSetup(modelId);
+          } else if (response.data.status === 'failed') {
+            cleanup();
+            setErrorMessage(response.data.error || 'Download failed.');
+            trackOnboardingSetupFailed('local', response.data.error || 'download_failed');
+            setPhase('error');
+          } else if (response.data.status === 'cancelled') {
+            cleanup();
+            setPhase('select');
+          }
+        }
+      } catch {
+        cleanup();
+        setErrorMessage('Lost connection to download. Please try again.');
+        trackOnboardingSetupFailed('local', 'progress_poll_failed');
+        setPhase('error');
+      }
+    }, 500);
+  };
+
+  const handleCancel = async () => {
+    if (phase === 'downloading' && selectedModelId) {
+      cleanup();
+      try {
+        await cancelLocalModelDownload({ path: { model_id: selectedModelId } });
+      } catch {
+        // best-effort
+      }
+      setDownloadProgress(null);
+      setPhase('select');
+    } else {
+      onCancel();
+    }
+  };
+
+  const handlePrimaryAction = async () => {
+    if (!selectedModelId) return;
+    const model = models.find((m) => m.id === selectedModelId);
+    if (!model) return;
+    if (model.status.state === 'Downloaded') {
+      await finishSetup(model.id);
+    } else {
+      await startDownload(model.id);
+    }
+  };
+
+  const recommended = models.find((m) => m.recommended);
+  const otherModels = models.filter((m) => m.id !== recommended?.id);
+  const selectedModel = models.find((m) => m.id === selectedModelId);
+
+  if (phase === 'loading') {
+    return (
+      <div className="flex flex-col items-center justify-center py-16">
+        <div className="animate-spin rounded-full h-8 w-8 border-t-2 border-b-2 border-text-muted mb-4"></div>
+        <p className="text-text-muted text-sm">Checking available models...</p>
+      </div>
+    );
+  }
+
+  return (
+    <div className="space-y-6">
+      {/* Header */}
+      <div className="text-left space-y-3">
+        <div className="origin-bottom-left goose-icon-animation">
+          <Goose className="size-6 sm:size-8" />
+        </div>
+        <h1 className="text-2xl sm:text-4xl font-light">Run Locally</h1>
+        <p className="text-text-muted text-base sm:text-lg">
+          Download a model to run Goose entirely on your machine — no API keys, no accounts, completely free and private.
+        </p>
+      </div>
+
+      {/* Error state */}
+      {phase === 'error' && (
+        <div className="space-y-4">
+          <div className="border border-red-500/30 rounded-xl p-4 bg-red-500/5">
+            <p className="text-sm text-red-400">{errorMessage}</p>
+          </div>
+          <button
+            onClick={() => {
+              setErrorMessage(null);
+              setPhase('select');
+            }}
+            className="w-full px-6 py-3 bg-background-muted text-text-default rounded-lg transition-colors font-medium hover:bg-background-muted/80"
+          >
+            Try Again
+          </button>
+          <button
+            onClick={onCancel}
+            className="w-full px-6 py-3 bg-transparent text-text-muted rounded-lg hover:bg-background-muted transition-colors"
+          >
+            Back
+          </button>
+        </div>
+      )}
+
+      {/* Model selection */}
+      {phase === 'select' && (
+        <div className="space-y-5">
+          {/* Recommended model card */}
+          {recommended && (
+            <div
+              onClick={() => setSelectedModelId(recommended.id)}
+              className={`relative w-full p-4 sm:p-6 border rounded-xl cursor-pointer transition-all duration-200 group ${
+                selectedModelId === recommended.id
+                  ? 'border-blue-500 bg-blue-500/5'
+                  : 'border-border-subtle hover:border-border-default'
+              }`}
+            >
+              <div className="absolute -top-2 -right-2 sm:-top-3 sm:-right-3 z-10">
+                <span className="inline-block px-2 py-1 text-xs font-medium bg-blue-600 text-white rounded-full">
+                  Best for your machine
+                </span>
+              </div>
+              <div className="flex items-start gap-3">
+                <input
+                  type="radio"
+                  checked={selectedModelId === recommended.id}
+                  onChange={() => setSelectedModelId(recommended.id)}
+                  className="cursor-pointer flex-shrink-0 mt-1"
+                />
+                <div className="flex-1 min-w-0">
+                  <div className="flex items-center gap-2 flex-wrap">
+                    <span className="font-medium text-text-default text-sm sm:text-base">
+                      {recommended.display_name}
+                    </span>
+                    {recommended.status.state === 'Downloaded' && (
+                      <span className="text-xs bg-green-600 text-white px-2 py-0.5 rounded-full">
+                        Ready
+                      </span>
+                    )}
+                  </div>
+                  <p className="text-text-muted text-xs mt-1">
+                    {formatSize(recommended.size_bytes)}
+                  </p>
+                </div>
+              </div>
+            </div>
+          )}
+
+          {/* Expandable other models */}
+          {otherModels.length > 0 && (
+            <div>
+              <button
+                onClick={() => setShowAllModels(!showAllModels)}
+                className="text-sm text-blue-500 hover:text-blue-400 transition-colors flex items-center gap-1"
+              >
+                {showAllModels ? 'Hide other sizes' : `Show ${otherModels.length} other sizes`}
+                <svg
+                  className={`w-3.5 h-3.5 transition-transform ${showAllModels ? 'rotate-180' : ''}`}
+                  fill="none"
+                  stroke="currentColor"
+                  viewBox="0 0 24 24"
+                >
+                  <path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M19 9l-7 7-7-7" />
+                </svg>
+              </button>
+
+              {showAllModels && (
+                <div className="mt-3 space-y-2">
+                  {otherModels.map((model) => (
+                    <div
+                      key={model.id}
+                      onClick={() => setSelectedModelId(model.id)}
+                      className={`w-full p-4 border rounded-xl cursor-pointer transition-all duration-200 ${
+                        selectedModelId === model.id
+                          ? 'border-blue-500 bg-blue-500/5'
+                          : 'border-border-subtle hover:border-border-default'
+                      }`}
+                    >
+                      <div className="flex items-start gap-3">
+                        <input
+                          type="radio"
+                          checked={selectedModelId === model.id}
+                          onChange={() => setSelectedModelId(model.id)}
+                          className="cursor-pointer flex-shrink-0 mt-0.5"
+                        />
+                        <div className="flex-1 min-w-0">
+                          <div className="flex items-center gap-2 flex-wrap">
+                            <span className="font-medium text-text-default text-sm">{model.display_name}</span>
+                            <span className="text-xs text-text-muted">{formatSize(model.size_bytes)}</span>
+                            {model.status.state === 'Downloaded' && (
+                              <span className="text-xs bg-green-600 text-white px-2 py-0.5 rounded-full">
+                                Ready
+                              </span>
+                            )}
+                          </div>
+                        </div>
+                      </div>
+                    </div>
+                  ))}
+                </div>
+              )}
+            </div>
+          )}
+
+          {/* Primary action */}
+          <button
+            onClick={handlePrimaryAction}
+            disabled={!selectedModelId}
+            className="w-full px-6 py-3 bg-background-muted text-text-default rounded-lg transition-colors font-medium disabled:opacity-40 disabled:cursor-not-allowed hover:bg-background-muted/80"
+          >
+            {selectedModel?.status.state === 'Downloaded'
+              ? `Use ${selectedModel.display_name}`
+              : selectedModel
+                ? `Download ${selectedModel.display_name} (${formatSize(selectedModel.size_bytes)})`
+                : 'Select a model'}
+          </button>
+
+          <button
+            onClick={onCancel}
+            className="w-full px-6 py-3 bg-transparent text-text-muted rounded-lg hover:bg-background-muted transition-colors"
+          >
+            Back
+          </button>
+        </div>
+      )}
+
+      {/* Downloading state */}
+      {phase === 'downloading' && selectedModel && (
+        <div className="space-y-6">
+          <div className="border border-border-subtle rounded-xl p-5 sm:p-6 bg-background-default">
+            <p className="font-medium text-text-default text-sm sm:text-base mb-4">
+              Downloading {selectedModel.display_name}
+            </p>
+
+            {downloadProgress ? (
+              <div className="space-y-3">
+                {/* Progress bar */}
+                <div className="w-full bg-background-subtle rounded-full h-2 overflow-hidden">
+                  <div
+                    className="bg-blue-500 h-2 rounded-full transition-all duration-500 ease-out"
+                    style={{ width: `${downloadProgress.progress_percent}%` }}
+                  />
+                </div>
+
+                {/* Stats row */}
+                <div className="flex justify-between text-xs text-text-muted">
+                  <span>
+                    {formatBytes(downloadProgress.bytes_downloaded)} of{' '}
+                    {formatBytes(downloadProgress.total_bytes)}
+                  </span>
+                  <span>{downloadProgress.progress_percent.toFixed(0)}%</span>
+                </div>
+
+                <div className="flex justify-between text-xs text-text-muted">
+                  {downloadProgress.speed_bps ? (
+                    <span>{formatBytes(downloadProgress.speed_bps)}/s</span>
+                  ) : (
+                    <span />
+                  )}
+                  {downloadProgress.eta_seconds != null && downloadProgress.eta_seconds > 0 && (
+                    <span>
+                      ~{downloadProgress.eta_seconds < 60
+                        ? `${Math.round(downloadProgress.eta_seconds)}s`
+                        : `${Math.round(downloadProgress.eta_seconds / 60)}m`}{' '}
+                      remaining
+                    </span>
+                  )}
+                </div>
+              </div>
+            ) : (
+              <div className="flex items-center gap-3">
+                <div className="animate-spin rounded-full h-4 w-4 border-t-2 border-b-2 border-text-muted"></div>
+                <span className="text-sm text-text-muted">Starting download...</span>
+              </div>
+            )}
+          </div>
+
+          <button
+            onClick={handleCancel}
+            className="w-full px-6 py-3 bg-transparent text-text-muted rounded-lg hover:bg-background-muted transition-colors border border-border-subtle"
+          >
+            Cancel Download
+          </button>
+        </div>
+      )}
+    </div>
+  );
+}
@@ -15,7 +15,7 @@
 * - "standalone" — Goose-specific mode for dedicated Electron windows
 */

-import { AppRenderer, type RequestHandlerExtra } from '@mcp-ui/client';
+import { AppRenderer } from '@mcp-ui/client';
 import type {
  McpUiDisplayMode,
  McpUiHostContext,
@@ -23,7 +23,7 @@ import type {
  McpUiResourcePermissions,
  McpUiSizeChangedNotification,
 } from '@modelcontextprotocol/ext-apps/app-bridge';
-import type { CallToolResult, JSONRPCRequest } from '@modelcontextprotocol/sdk/types.js';
+import type { CallToolResult } from '@modelcontextprotocol/sdk/types.js';
 import { useCallback, useEffect, useMemo, useReducer, useRef, useState } from 'react';
 import { callTool, readResource } from '../../api';
 import { AppEvents } from '../../constants/events';
@@ -40,8 +40,6 @@ import {
  McpAppToolInputPartial,
  McpAppToolResult,
  DimensionLayout,
-  SamplingCreateMessageParams,
-  SamplingCreateMessageResponse,
 } from './types';

 const DEFAULT_IFRAME_HEIGHT = 200;
@@ -267,13 +265,7 @@ export default function McpAppRenderer({
  const containerRef = useRef<HTMLDivElement>(null);
  const [containerWidth, setContainerWidth] = useState<number>(0);
  const [containerHeight, setContainerHeight] = useState<number>(0);
-  const [apiHost, setApiHost] = useState<string | null>(null);
-  const [secretKey, setSecretKey] = useState<string | null>(null);

-  useEffect(() => {
-    window.electron.getGoosedHostPort().then(setApiHost);
-    window.electron.getSecretKey().then(setSecretKey);
-  }, []);

  // Fetch the resource from the extension to get HTML and metadata (CSP, permissions, etc.).
  // If cachedHtml is provided we show it immediately; the fetch updates metadata and
@@ -535,42 +527,6 @@ export default function McpAppRenderer({
    return () => observer.disconnect();
  }, []);

-  const handleFallbackRequest = useCallback(
-    async (request: JSONRPCRequest, _extra: RequestHandlerExtra) => {
-      if (request.method === 'sampling/createMessage') {
-        if (!sessionId || !apiHost || !secretKey) {
-          throw new Error('Session not initialized for sampling request');
-        }
-        const { messages, systemPrompt, maxTokens } =
-          request.params as unknown as SamplingCreateMessageParams;
-        const response = await fetch(`${apiHost}/sessions/${sessionId}/sampling/message`, {
-          method: 'POST',
-          headers: {
-            'Content-Type': 'application/json',
-            'X-Secret-Key': secretKey,
-          },
-          body: JSON.stringify({
-            messages: messages.map((m) => ({
-              role: m.role,
-              content: m.content,
-            })),
-            systemPrompt,
-            maxTokens,
-          }),
-        });
-        if (!response.ok) {
-          throw new Error(`Sampling request failed: ${response.statusText}`);
-        }
-        return (await response.json()) as SamplingCreateMessageResponse;
-      }
-      return {
-        status: 'error' as const,
-        message: `Unhandled JSON-RPC method: ${request.method ?? '<unknown>'}`,
-      };
-    },
-    [sessionId, apiHost, secretKey]
-  );
-
  const handleError = useCallback((err: Error) => {
    console.error('[MCP App Error]:', err);
    dispatch({ type: 'ERROR', message: errorMessage(err) });
@@ -687,7 +643,6 @@ export default function McpAppRenderer({
        onReadResource={handleReadResource}
        onLoggingMessage={handleLoggingMessage}
        onSizeChanged={handleSizeChanged}
-        onFallbackRequest={handleFallbackRequest}
        onError={handleError}
      />
    );
@@ -8,6 +8,7 @@ import { startChatGptCodexSetup } from '../utils/chatgptCodexSetup';
 import WelcomeGooseLogo from './WelcomeGooseLogo';
 import { toastService } from '../toasts';
 import { OllamaSetup } from './OllamaSetup';
+import { LocalModelSetup } from './LocalModelSetup';
 import ApiKeyTester from './ApiKeyTester';
 import { SwitchModelModal } from './settings/models/subcomponents/SwitchModelModal';
 import { createNavigationHandler } from '../utils/navigationUtils';
@@ -34,6 +35,7 @@ export default function ProviderGuard({ didSelectProvider, children }: ProviderG
  const [hasProvider, setHasProvider] = useState(false);
  const [showFirstTimeSetup, setShowFirstTimeSetup] = useState(false);
  const [showOllamaSetup, setShowOllamaSetup] = useState(false);
+  const [showLocalModelSetup, setShowLocalModelSetup] = useState(false);
  const [userInActiveSetup, setUserInActiveSetup] = useState(false);
  const [showSwitchModelModal, setShowSwitchModelModal] = useState(false);
  const [switchModelProvider, setSwitchModelProvider] = useState<string | null>(null);
@@ -200,6 +202,19 @@ export default function ProviderGuard({ didSelectProvider, children }: ProviderG
    setShowOllamaSetup(false);
  };

+  const handleLocalModelComplete = () => {
+    trackOnboardingCompleted('local');
+    setShowLocalModelSetup(false);
+    setShowFirstTimeSetup(false);
+    setHasProvider(true);
+    navigate('/', { replace: true });
+  };
+
+  const handleLocalModelCancel = () => {
+    trackOnboardingAbandoned('local_model_setup');
+    setShowLocalModelSetup(false);
+  };
+
  const handleRetrySetup = (setupType: 'openrouter' | 'tetrate' | 'chatgpt_codex') => {
    if (setupType === 'openrouter') {
      setOpenRouterSetupState(null);
@@ -285,6 +300,23 @@ export default function ProviderGuard({ didSelectProvider, children }: ProviderG
    return <OllamaSetup onSuccess={handleOllamaComplete} onCancel={handleOllamaCancel} />;
  }

+  if (showLocalModelSetup) {
+    return (
+      <div className="h-screen w-full bg-background-default overflow-hidden">
+        <div className="h-full overflow-y-auto">
+          <div className="min-h-full flex flex-col items-center justify-center p-4 py-8">
+            <div className="max-w-2xl w-full mx-auto p-8">
+              <LocalModelSetup
+                onSuccess={handleLocalModelComplete}
+                onCancel={handleLocalModelCancel}
+              />
+            </div>
+          </div>
+        </div>
+      </div>
+    );
+  }
+
  if (!hasProvider && showFirstTimeSetup) {
    return (
      <div className="h-screen w-full bg-background-default overflow-hidden relative">
@@ -316,6 +348,48 @@ export default function ProviderGuard({ didSelectProvider, children }: ProviderG
                }}
              />

+              {/* Run Locally Card */}
+              <div className="relative w-full mb-4">
+                <div className="absolute -top-2 -right-2 sm:-top-3 sm:-right-3 z-20">
+                  <span className="inline-block px-2 py-1 text-xs font-medium bg-green-600 text-white rounded-full">
+                    Free &amp; Private
+                  </span>
+                </div>
+                <div
+                  onClick={() => {
+                    trackOnboardingProviderSelected('local');
+                    setShowLocalModelSetup(true);
+                  }}
+                  className="w-full p-4 sm:p-6 bg-transparent border rounded-xl transition-all duration-200 cursor-pointer group"
+                >
+                  <div className="flex items-start justify-between mb-3">
+                    <div className="flex items-center gap-2">
+                      <span className="font-medium text-text-default text-sm sm:text-base">
+                        Run Locally
+                      </span>
+                    </div>
+                    <div className="text-text-muted group-hover:text-text-default transition-colors">
+                      <svg
+                        className="w-4 h-4 sm:w-5 sm:h-5"
+                        fill="none"
+                        stroke="currentColor"
+                        viewBox="0 0 24 24"
+                      >
+                        <path
+                          strokeLinecap="round"
+                          strokeLinejoin="round"
+                          strokeWidth={2}
+                          d="M9 5l7 7-7 7"
+                        />
+                      </svg>
+                    </div>
+                  </div>
+                  <p className="text-text-muted text-sm sm:text-base">
+                    Download a model and run entirely on your machine. No API keys, no accounts.
+                  </p>
+                </div>
+              </div>
+
              {/* ChatGPT Subscription Card - Full Width */}
              <div className="relative w-full mb-4">
                <div className="absolute -top-2 -right-2 sm:-top-3 sm:-right-3 z-20">
@@ -9,10 +9,11 @@ import ConfigSettings from './config/ConfigSettings';
 import PromptsSettingsSection from './PromptsSettingsSection';
 import { ExtensionConfig } from '../../api';
 import { MainPanelLayout } from '../Layout/MainPanelLayout';
-import { Bot, Share2, Monitor, MessageSquare, FileText, Keyboard } from 'lucide-react';
+import { Bot, Share2, Monitor, MessageSquare, FileText, Keyboard, HardDrive } from 'lucide-react';
 import { useState, useEffect, useRef } from 'react';
 import ChatSettingsSection from './chat/ChatSettingsSection';
 import KeyboardShortcutsSection from './keyboard/KeyboardShortcutsSection';
+import LocalInferenceSection from './localInference/LocalInferenceSection';
 import { CONFIGURATION_ENABLED } from '../../updates';
 import { trackSettingsTabViewed } from '../../utils/analytics';

@@ -54,6 +55,7 @@ export default function SettingsView({
        chat: 'chat',
        prompts: 'prompts',
        keyboard: 'keyboard',
+        'local-inference': 'local-inference',
      };

      const targetTab = sectionToTab[viewOptions.section];
@@ -112,6 +114,14 @@ export default function SettingsView({
                    <Bot className="h-4 w-4" />
                    Models
                  </TabsTrigger>
+                  <TabsTrigger
+                    value="local-inference"
+                    className="flex gap-2"
+                    data-testid="settings-local-inference-tab"
+                  >
+                    <HardDrive className="h-4 w-4" />
+                    Local Inference
+                  </TabsTrigger>
                  <TabsTrigger value="chat" className="flex gap-2" data-testid="settings-chat-tab">
                    <MessageSquare className="h-4 w-4" />
                    Chat
@@ -155,6 +165,13 @@ export default function SettingsView({
                  <ModelsSection setView={setView} />
                </TabsContent>

+                <TabsContent
+                  value="local-inference"
+                  className="mt-0 focus-visible:outline-none focus-visible:ring-0"
+                >
+                  <LocalInferenceSection />
+                </TabsContent>
+
                <TabsContent
                  value="chat"
                  className="mt-0 focus-visible:outline-none focus-visible:ring-0"
@@ -38,19 +38,6 @@ export const LocalModelManager = () => {
    // eslint-disable-next-line react-hooks/exhaustive-deps
  }, []);

-  // Determine if we should show all models by default (if non-recommended models are downloaded)
-  useEffect(() => {
-    if (models.length === 0) return;
-
-    const hasDownloadedNonRecommended = models.some(
-      (model) => model.downloaded && !model.recommended
-    );
-
-    if (hasDownloadedNonRecommended && !showAllModels) {
-      setShowAllModels(true);
-    }
-  }, [models, showAllModels]);
-
  const loadSelectedModel = async () => {
    try {
      const value = await read(LOCAL_WHISPER_MODEL_CONFIG_KEY, false);
@@ -145,8 +132,14 @@ export const LocalModelManager = () => {
    }
  };

-  const displayedModels = showAllModels ? models : models.filter((m) => m.recommended);
+  const hasDownloadedNonRecommended = models.some(
+    (model) => model.downloaded && !model.recommended
+  );
+  const displayedModels = showAllModels || hasDownloadedNonRecommended
+    ? models
+    : models.filter((m) => m.recommended);
  const hasNonRecommendedModels = models.some((m) => !m.recommended);
+  const showToggleButton = hasNonRecommendedModels && !hasDownloadedNonRecommended;

  return (
    <div className="space-y-3">
@@ -267,7 +260,7 @@ export const LocalModelManager = () => {
        })}
      </div>

-      {hasNonRecommendedModels && (
+      {showToggleButton && (
        <Button
          variant="ghost"
          size="sm"
@@ -282,7 +275,7 @@ export const LocalModelManager = () => {
          ) : (
            <>
              <ChevronDown className="w-4 h-4 mr-1" />
-              Show all models
+              Show all models ({models.length - displayedModels.length} more)
            </>
          )}
        </Button>
@@ -0,0 +1,358 @@
+import { useState, useCallback, useRef } from 'react';
+import { Search, Download, ChevronDown, ChevronUp, Loader2, Star } from 'lucide-react';
+import { Button } from '../../ui/button';
+import {
+  searchHfModels,
+  getRepoFiles,
+  downloadHfModel,
+  type HfModelInfo,
+  type HfQuantVariant,
+} from '../../../api';
+
+const formatBytes = (bytes: number): string => {
+  if (bytes === 0) return 'unknown';
+  if (bytes < 1024) return `${bytes}B`;
+  if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(0)}KB`;
+  if (bytes < 1024 * 1024 * 1024) return `${(bytes / (1024 * 1024)).toFixed(0)}MB`;
+  return `${(bytes / (1024 * 1024 * 1024)).toFixed(1)}GB`;
+};
+
+const formatDownloads = (n: number): string => {
+  if (n >= 1_000_000) return `${(n / 1_000_000).toFixed(1)}M`;
+  if (n >= 1_000) return `${(n / 1_000).toFixed(1)}K`;
+  return `${n}`;
+};
+
+interface RepoData {
+  variants: HfQuantVariant[];
+  recommendedIndex: number | null;
+}
+
+interface Props {
+  onDownloadStarted: (modelId: string) => void;
+}
+
+export const HuggingFaceModelSearch = ({ onDownloadStarted }: Props) => {
+  const [query, setQuery] = useState('');
+  const [results, setResults] = useState<HfModelInfo[]>([]);
+  const [expandedRepo, setExpandedRepo] = useState<string | null>(null);
+  const [repoData, setRepoData] = useState<Record<string, RepoData>>({});
+  const [searching, setSearching] = useState(false);
+  const [downloading, setDownloading] = useState<Set<string>>(new Set());
+  const [loadingFiles, setLoadingFiles] = useState<Set<string>>(new Set());
+  const [directSpec, setDirectSpec] = useState('');
+  const [error, setError] = useState<string | null>(null);
+  const debounceRef = useRef<ReturnType<typeof setTimeout> | null>(null);
+
+  const doSearch = useCallback(async (q: string) => {
+    if (!q.trim()) {
+      setResults([]);
+      setError(null);
+      return;
+    }
+    setSearching(true);
+    setError(null);
+    try {
+      const response = await searchHfModels({
+        query: { q, limit: 20 },
+      });
+      if (response.data) {
+        // Pre-fetch variants for all results and filter out repos with no suitable quantizations
+        const modelsWithVariants = await Promise.all(
+          response.data.map(async (model) => {
+            try {
+              const [author, repo] = model.repo_id.split('/');
+              const filesResponse = await getRepoFiles({ path: { author, repo } });
+              if (filesResponse.data && filesResponse.data.variants.length > 0) {
+                return { model, data: filesResponse.data };
+              }
+            } catch {
+              // Skip repos we can't fetch
+            }
+            return null;
+          })
+        );
+
+        const validResults = modelsWithVariants.filter(Boolean) as {
+          model: HfModelInfo;
+          data: { variants: HfQuantVariant[]; recommended_index?: number | null };
+        }[];
+
+        setResults(validResults.map((r) => r.model));
+        setRepoData((prev) => {
+          const next = { ...prev };
+          for (const r of validResults) {
+            next[r.model.repo_id] = {
+              variants: r.data.variants,
+              recommendedIndex: r.data.recommended_index ?? null,
+            };
+          }
+          return next;
+        });
+
+        if (validResults.length === 0) {
+          setError('No GGUF models found for this query.');
+        }
+      } else {
+        console.error('Search response:', response);
+        const errMsg = response.error
+          ? `Search error: ${JSON.stringify(response.error)}`
+          : 'Search returned no data.';
+        setError(errMsg);
+      }
+    } catch (e) {
+      console.error('Search failed:', e);
+      setError('Search failed. Please try again.');
+    } finally {
+      setSearching(false);
+    }
+  }, []);
+
+  const handleQueryChange = (value: string) => {
+    setQuery(value);
+    if (debounceRef.current) clearTimeout(debounceRef.current);
+    debounceRef.current = setTimeout(() => doSearch(value), 300);
+  };
+
+  const toggleRepo = async (repoId: string) => {
+    if (expandedRepo === repoId) {
+      setExpandedRepo(null);
+      return;
+    }
+    setExpandedRepo(repoId);
+
+    if (!repoData[repoId]?.variants.length) {
+      setLoadingFiles((prev) => new Set(prev).add(repoId));
+      try {
+        const [author, repo] = repoId.split('/');
+        const response = await getRepoFiles({
+          path: { author, repo },
+        });
+        if (response.data) {
+          const variants = response.data.variants;
+          setRepoData((prev) => ({
+            ...prev,
+            [repoId]: {
+              variants,
+              recommendedIndex: response.data!.recommended_index ?? null,
+            },
+          }));
+        }
+      } catch (e) {
+        console.error('Failed to fetch repo files:', e);
+      } finally {
+        setLoadingFiles((prev) => {
+          const next = new Set(prev);
+          next.delete(repoId);
+          return next;
+        });
+      }
+    }
+  };
+
+  const startDownload = async (repoId: string, quantization: string) => {
+    const spec = `${repoId}:${quantization}`;
+    setDownloading((prev) => new Set(prev).add(spec));
+    try {
+      const response = await downloadHfModel({
+        body: { spec },
+      });
+      if (response.data) {
+        onDownloadStarted(response.data);
+      }
+    } catch (e) {
+      console.error('Download failed:', e);
+    } finally {
+      setDownloading((prev) => {
+        const next = new Set(prev);
+        next.delete(spec);
+        return next;
+      });
+    }
+  };
+
+  const startDirectDownload = async () => {
+    const spec = directSpec.trim();
+    if (!spec) return;
+    const key = `direct:${spec}`;
+    setDownloading((prev) => new Set(prev).add(key));
+    try {
+      const response = await downloadHfModel({
+        body: { spec },
+      });
+      if (response.data) {
+        onDownloadStarted(response.data);
+        setDirectSpec('');
+      }
+    } catch (e) {
+      console.error('Direct download failed:', e);
+    } finally {
+      setDownloading((prev) => {
+        const next = new Set(prev);
+        next.delete(key);
+        return next;
+      });
+    }
+  };
+
+  return (
+    <div className="space-y-4">
+      <div>
+        <h4 className="text-sm font-medium text-text-default mb-2">Search HuggingFace</h4>
+        <div className="relative">
+          <Search className="absolute left-3 top-1/2 -translate-y-1/2 w-4 h-4 text-text-muted" />
+          <input
+            type="text"
+            value={query}
+            onChange={(e) => handleQueryChange(e.target.value)}
+            placeholder="Search for GGUF models..."
+            className="w-full pl-9 pr-4 py-2 text-sm border border-border-subtle rounded-lg bg-background-default text-text-default placeholder:text-text-muted focus:outline-none focus:border-accent-primary"
+          />
+          {searching && (
+            <Loader2 className="absolute right-3 top-1/2 -translate-y-1/2 w-4 h-4 text-text-muted animate-spin" />
+          )}
+        </div>
+      </div>
+
+      {error && !searching && (
+        <p className="text-xs text-text-muted">{error}</p>
+      )}
+
+      {results.length > 0 && (
+        <div className="space-y-1 max-h-96 overflow-y-auto">
+          {results.map((model) => {
+            const isExpanded = expandedRepo === model.repo_id;
+            const data = repoData[model.repo_id];
+            const variants = data?.variants || [];
+            const recommendedIndex = data?.recommendedIndex ?? null;
+
+            return (
+              <div key={model.repo_id} className="border border-border-subtle rounded-lg">
+                <button
+                  onClick={() => toggleRepo(model.repo_id)}
+                  className="w-full flex items-center justify-between p-3 text-left hover:bg-background-subtle rounded-lg"
+                >
+                  <div className="flex-1 min-w-0">
+                    <div className="flex items-center gap-2">
+                      <span className="text-sm font-medium text-text-default truncate">
+                        {model.repo_id}
+                      </span>
+                    </div>
+                    <div className="flex items-center gap-3 mt-0.5">
+                      <span className="text-xs text-text-muted">
+                        ↓ {formatDownloads(model.downloads)}
+                      </span>
+                    </div>
+                  </div>
+                  {isExpanded ? (
+                    <ChevronUp className="w-4 h-4 text-text-muted flex-shrink-0" />
+                  ) : (
+                    <ChevronDown className="w-4 h-4 text-text-muted flex-shrink-0" />
+                  )}
+                </button>
+
+                {isExpanded && (
+                  <div className="border-t border-border-subtle px-3 pb-3 space-y-1">
+                    {loadingFiles.has(model.repo_id) && (
+                      <div className="flex items-center gap-2 py-2 text-xs text-text-muted">
+                        <Loader2 className="w-3 h-3 animate-spin" />
+                        Loading variants...
+                      </div>
+                    )}
+                    {variants.map((variant, idx) => {
+                      const dlKey = `${model.repo_id}:${variant.quantization}`;
+                      const isStarting = downloading.has(dlKey);
+                      const isRecommended = idx === recommendedIndex;
+
+                      return (
+                        <div
+                          key={variant.quantization}
+                          className={`flex items-center justify-between py-2 px-2 rounded ${
+                            isRecommended
+                              ? 'bg-blue-500/5 border border-blue-500/20'
+                              : 'hover:bg-background-subtle'
+                          }`}
+                        >
+                          <div className="flex flex-col gap-0.5 min-w-0 flex-1 mr-3">
+                            <div className="flex items-center gap-2">
+                              <span className="text-xs font-mono font-medium text-text-default">
+                                {variant.quantization}
+                              </span>
+                              <span className="text-xs text-text-muted">
+                                {formatBytes(variant.size_bytes)}
+                              </span>
+                              {isRecommended && (
+                                <span className="inline-flex items-center gap-1 text-xs bg-blue-500 text-white px-1.5 py-0.5 rounded">
+                                  <Star className="w-3 h-3" />
+                                  Recommended
+                                </span>
+                              )}
+                            </div>
+                            {variant.description && (
+                              <span className="text-xs text-text-muted">
+                                {variant.description}
+                              </span>
+                            )}
+                          </div>
+                          <Button
+                            variant="outline"
+                            size="sm"
+                            disabled={isStarting}
+                            onClick={() => startDownload(model.repo_id, variant.quantization)}
+                          >
+                            {isStarting ? (
+                              <Loader2 className="w-3 h-3 animate-spin" />
+                            ) : (
+                              <>
+                                <Download className="w-3 h-3 mr-1" />
+                                Download
+                              </>
+                            )}
+                          </Button>
+                        </div>
+                      );
+                    })}
+                  </div>
+                )}
+              </div>
+            );
+          })}
+        </div>
+      )}
+
+      <div>
+        <h4 className="text-sm font-medium text-text-default mb-2">Direct Download</h4>
+        <p className="text-xs text-text-muted mb-2">
+          Specify a model directly: <code className="bg-background-subtle px-1 rounded">user/repo:quantization</code>
+        </p>
+        <div className="flex gap-2">
+          <input
+            type="text"
+            value={directSpec}
+            onChange={(e) => setDirectSpec(e.target.value)}
+            placeholder="bartowski/Llama-3.2-1B-Instruct-GGUF:Q4_K_M"
+            className="flex-1 px-3 py-2 text-sm border border-border-subtle rounded-lg bg-background-default text-text-default placeholder:text-text-muted focus:outline-none focus:border-accent-primary"
+            onKeyDown={(e) => {
+              if (e.key === 'Enter') startDirectDownload();
+            }}
+          />
+          <Button
+            variant="outline"
+            size="sm"
+            disabled={!directSpec.trim() || downloading.has(`direct:${directSpec}`)}
+            onClick={startDirectDownload}
+          >
+            {downloading.has(`direct:${directSpec}`) ? (
+              <Loader2 className="w-4 h-4 animate-spin" />
+            ) : (
+              <>
+                <Download className="w-4 h-4 mr-1" />
+                Download
+              </>
+            )}
+          </Button>
+        </div>
+      </div>
+    </div>
+  );
+};
@@ -0,0 +1,9 @@
+import { LocalInferenceSettings } from './LocalInferenceSettings';
+
+export default function LocalInferenceSection() {
+  return (
+    <section id="local-inference" className="space-y-4 pr-4 pb-8">
+      <LocalInferenceSettings />
+    </section>
+  );
+}
@@ -0,0 +1,410 @@
+import { useState, useEffect, useCallback, useRef } from 'react';
+import { Download, Trash2, X, ChevronDown, ChevronUp, Settings2 } from 'lucide-react';
+import { Button } from '../../ui/button';
+import { useModelAndProvider } from '../../ModelAndProviderContext';
+import {
+  listLocalModels,
+  downloadHfModel,
+  getLocalModelDownloadProgress,
+  cancelLocalModelDownload,
+  deleteLocalModel,
+  setConfigProvider,
+  type DownloadProgress,
+  type LocalModelResponse,
+} from '../../../api';
+import { HuggingFaceModelSearch } from './HuggingFaceModelSearch';
+import { ModelSettingsPanel } from './ModelSettingsPanel';
+import {
+  Dialog,
+  DialogContent,
+  DialogHeader,
+  DialogTitle,
+} from '../../ui/dialog';
+
+const formatBytes = (bytes: number): string => {
+  if (bytes < 1024) return `${bytes}B`;
+  if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(0)}KB`;
+  if (bytes < 1024 * 1024 * 1024) return `${(bytes / (1024 * 1024)).toFixed(0)}MB`;
+  return `${(bytes / (1024 * 1024 * 1024)).toFixed(1)}GB`;
+};
+
+export const LocalInferenceSettings = () => {
+  const [models, setModels] = useState<LocalModelResponse[]>([]);
+  const [downloads, setDownloads] = useState<Map<string, DownloadProgress>>(new Map());
+  const [showAllFeatured, setShowAllFeatured] = useState(false);
+  const [settingsOpenFor, setSettingsOpenFor] = useState<string | null>(null);
+  const { currentModel, currentProvider, setProviderAndModel } = useModelAndProvider();
+  const downloadSectionRef = useRef<HTMLDivElement>(null);
+  const selectedModelId = currentProvider === 'local' ? currentModel : null;
+
+  const getDisplayName = useCallback(
+    (modelId: string): string => {
+      const model = models.find((m) => m.id === modelId);
+      return model?.display_name || modelId;
+    },
+    [models]
+  );
+
+  const loadModels = useCallback(async () => {
+    try {
+      const response = await listLocalModels();
+      if (response.data) {
+        setModels(response.data);
+      }
+    } catch (error) {
+      console.error('Failed to load models:', error);
+    }
+  }, []);
+
+  // Check for any in-progress downloads when models list changes
+  const detectActiveDownloads = useCallback(async () => {
+    for (const model of models) {
+      if (downloads.has(model.id)) continue;
+      // Check models that the API reports as downloading
+      if (model.status.state === 'Downloading') {
+        pollDownloadProgress(model.id);
+      }
+    }
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [models, downloads]);
+
+  useEffect(() => {
+    loadModels();
+  }, [loadModels]);
+
+  useEffect(() => {
+    if (models.length > 0) {
+      detectActiveDownloads();
+    }
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [models]);
+
+  const selectModel = async (modelId: string) => {
+    setProviderAndModel('local', modelId);
+    try {
+      await setConfigProvider({
+        body: { provider: 'local', model: modelId },
+        throwOnError: true,
+      });
+    } catch (error) {
+      console.error('Failed to select model:', error);
+    }
+  };
+
+  const startFeaturedDownload = async (modelId: string) => {
+    const model = models.find((m) => m.id === modelId);
+    if (!model) return;
+    try {
+      await downloadHfModel({ body: { spec: model.id } });
+      pollDownloadProgress(modelId);
+      scrollToDownloads();
+    } catch (error) {
+      console.error('Failed to start download:', error);
+    }
+  };
+
+  const scrollToDownloads = useCallback(() => {
+    requestAnimationFrame(() => {
+      downloadSectionRef.current?.scrollIntoView({ behavior: 'smooth', block: 'nearest' });
+    });
+  }, []);
+
+  const pollDownloadProgress = (modelId: string) => {
+    const interval = setInterval(async () => {
+      try {
+        const response = await getLocalModelDownloadProgress({ path: { model_id: modelId } });
+        if (response.data) {
+          const progress = response.data;
+          setDownloads((prev) => new Map(prev).set(modelId, progress));
+
+          if (progress.status === 'completed') {
+            clearInterval(interval);
+            setDownloads((prev) => {
+              const next = new Map(prev);
+              next.delete(modelId);
+              return next;
+            });
+            await loadModels();
+            await selectModel(modelId);
+          } else if (progress.status === 'failed') {
+            clearInterval(interval);
+            await loadModels();
+          }
+        } else {
+          clearInterval(interval);
+        }
+      } catch {
+        clearInterval(interval);
+      }
+    }, 1000);
+  };
+
+  const cancelDownload = async (modelId: string) => {
+    try {
+      await cancelLocalModelDownload({ path: { model_id: modelId } });
+      setDownloads((prev) => {
+        const next = new Map(prev);
+        next.delete(modelId);
+        return next;
+      });
+    } catch (error) {
+      console.error('Failed to cancel download:', error);
+    }
+  };
+
+  const handleDeleteModel = async (modelId: string) => {
+    if (!window.confirm('Delete this model? You can re-download it later.')) return;
+    try {
+      await deleteLocalModel({ path: { model_id: modelId } });
+      await loadModels();
+    } catch (error) {
+      console.error('Failed to delete model:', error);
+    }
+  };
+
+  const handleHfDownloadStarted = (modelId: string) => {
+    pollDownloadProgress(modelId);
+    loadModels();
+    scrollToDownloads();
+  };
+
+  const isDownloaded = (model: LocalModelResponse) => model.status.state === 'Downloaded';
+  const isNotDownloaded = (model: LocalModelResponse) =>
+    model.status.state === 'NotDownloaded' && !downloads.has(model.id);
+
+  const downloadedModels = models.filter(isDownloaded);
+  const notDownloadedModels = models.filter(isNotDownloaded);
+  const recommendedModels = notDownloadedModels.filter((m) => m.recommended);
+  const displayedFeatured = showAllFeatured ? notDownloadedModels : recommendedModels;
+  const showFeaturedToggle = notDownloadedModels.length > recommendedModels.length;
+
+  return (
+    <div className="space-y-6">
+      <div>
+        <h3 className="text-text-default font-medium">Local Inference Models</h3>
+        <p className="text-xs text-text-muted max-w-2xl mt-1">
+          Download and manage local LLM models for inference without API keys. Search HuggingFace
+          for any GGUF model or use the featured picks below.
+        </p>
+      </div>
+
+      {/* Active Downloads */}
+      {downloads.size > 0 && (
+        <div ref={downloadSectionRef}>
+          <h4 className="text-sm font-medium text-text-default mb-2">Downloading</h4>
+          <div className="space-y-2">
+            {Array.from(downloads.entries()).map(([modelId, progress]) => {
+              if (progress.status === 'completed') return null;
+              const displayName = getDisplayName(modelId);
+              return (
+                <div
+                  key={modelId}
+                  className="border rounded-lg p-3 border-border-subtle bg-background-default"
+                >
+                  <div className="flex items-center justify-between mb-2">
+                    <span className="text-sm font-medium text-text-default truncate">
+                      {displayName}
+                    </span>
+                    {progress.status === 'downloading' && (
+                      <Button
+                        variant="ghost"
+                        size="sm"
+                        onClick={() => cancelDownload(modelId)}
+                        className="text-destructive hover:text-destructive"
+                      >
+                        <X className="w-4 h-4" />
+                      </Button>
+                    )}
+                  </div>
+                  {progress.status === 'downloading' && (
+                    <div className="space-y-1">
+                      <div className="w-full bg-background-subtle rounded-full h-2">
+                        <div
+                          className="bg-accent-primary h-2 rounded-full transition-all duration-300"
+                          style={{ width: `${progress.progress_percent}%` }}
+                        />
+                      </div>
+                      <div className="flex justify-between text-xs text-text-muted">
+                        <span>
+                          {formatBytes(progress.bytes_downloaded)} /{' '}
+                          {formatBytes(progress.total_bytes)} (
+                          {progress.progress_percent.toFixed(0)}%)
+                        </span>
+                        <span className="flex gap-2">
+                          {progress.eta_seconds != null && progress.eta_seconds > 0 && (
+                            <span>
+                              {progress.eta_seconds < 60
+                                ? `${Math.round(progress.eta_seconds)}s`
+                                : `${Math.round(progress.eta_seconds / 60)}m`}{' '}
+                              remaining
+                            </span>
+                          )}
+                          {progress.speed_bps != null && progress.speed_bps > 0 && (
+                            <span>{formatBytes(progress.speed_bps)}/s</span>
+                          )}
+                        </span>
+                      </div>
+                    </div>
+                  )}
+                  {progress.status === 'failed' && (
+                    <p className="text-xs text-destructive">
+                      {progress.error || 'Download failed'}
+                    </p>
+                  )}
+                </div>
+              );
+            })}
+          </div>
+        </div>
+      )}
+
+      {/* Downloaded Models */}
+      {downloadedModels.length > 0 && (
+        <div>
+          <h4 className="text-sm font-medium text-text-default mb-2">Downloaded Models</h4>
+          <div className="space-y-2">
+            {downloadedModels.map((model) => {
+              const isSelected = selectedModelId === model.id;
+              return (
+                <div
+                  key={model.id}
+                  className={`border rounded-lg p-3 transition-colors ${
+                    isSelected
+                      ? 'border-accent-primary bg-accent-primary/5'
+                      : 'border-border-subtle bg-background-default hover:border-border-default'
+                  }`}
+                >
+                  <div className="flex items-center justify-between">
+                    <div className="flex items-center gap-2">
+                      <input
+                        type="radio"
+                        checked={isSelected}
+                        onChange={() => selectModel(model.id)}
+                        className="cursor-pointer"
+                      />
+                      <span className="text-sm font-medium text-text-default">
+                        {model.display_name}
+                      </span>
+                      <span className="text-xs text-text-muted">
+                        {formatBytes(model.size_bytes)}
+                      </span>
+                      {model.recommended && (
+                        <span className="text-xs bg-blue-500 text-white px-2 py-0.5 rounded">
+                          Recommended
+                        </span>
+                      )}
+                    </div>
+                    <div className="flex items-center gap-1">
+                      <Button
+                        variant="ghost"
+                        size="sm"
+                        onClick={() => setSettingsOpenFor(model.id)}
+                        title="Model settings"
+                      >
+                        <Settings2 className="w-4 h-4" />
+                      </Button>
+                      <Button
+                        variant="ghost"
+                        size="sm"
+                        onClick={() => handleDeleteModel(model.id)}
+                        className="text-destructive hover:text-destructive"
+                      >
+                        <Trash2 className="w-4 h-4" />
+                      </Button>
+                    </div>
+                  </div>
+                </div>
+              );
+            })}
+          </div>
+        </div>
+      )}
+
+      {/* Featured Models (not yet downloaded) */}
+      {displayedFeatured.length > 0 && (
+        <div>
+          <h4 className="text-sm font-medium text-text-default mb-2">Featured Models</h4>
+          <div className="space-y-2">
+            {displayedFeatured.map((model) => (
+              <div
+                key={model.id}
+                className="border rounded-lg p-3 border-border-subtle bg-background-default hover:border-border-default"
+              >
+                <div className="flex items-center justify-between gap-3">
+                  <div className="flex-1 min-w-0">
+                    <div className="flex items-center gap-2 flex-wrap">
+                      <h4 className="text-sm font-medium text-text-default">
+                        {model.display_name}
+                      </h4>
+                      <span className="text-xs text-text-muted">
+                        {formatBytes(model.size_bytes)}
+                      </span>
+                      {model.recommended && (
+                        <span className="text-xs bg-blue-500 text-white px-2 py-0.5 rounded">
+                          Recommended
+                        </span>
+                      )}
+                    </div>
+                  </div>
+                  <Button
+                    variant="outline"
+                    size="sm"
+                    onClick={() => startFeaturedDownload(model.id)}
+                  >
+                    <Download className="w-4 h-4 mr-1" />
+                    Download
+                  </Button>
+                </div>
+              </div>
+            ))}
+          </div>
+
+          {showFeaturedToggle && (
+            <Button
+              variant="ghost"
+              size="sm"
+              onClick={() => setShowAllFeatured(!showAllFeatured)}
+              className="w-full text-text-muted hover:text-text-default mt-2"
+            >
+              {showAllFeatured ? (
+                <>
+                  <ChevronUp className="w-4 h-4 mr-1" />
+                  Show recommended only
+                </>
+              ) : (
+                <>
+                  <ChevronDown className="w-4 h-4 mr-1" />
+                  Show all featured ({notDownloadedModels.length - displayedFeatured.length} more)
+                </>
+              )}
+            </Button>
+          )}
+        </div>
+      )}
+
+      {/* HuggingFace Search */}
+      <div className="border-t border-border-subtle pt-4">
+        <HuggingFaceModelSearch onDownloadStarted={handleHfDownloadStarted} />
+      </div>
+
+      {models.length === 0 && (
+        <div className="text-center py-6 text-text-muted text-sm">No models available</div>
+      )}
+
+      <Dialog
+        open={!!settingsOpenFor}
+        onOpenChange={(open) => {
+          if (!open) setSettingsOpenFor(null);
+        }}
+      >
+        <DialogContent className="max-h-[80vh] overflow-y-auto sm:max-w-xl">
+          <DialogHeader>
+            <DialogTitle>Model Settings</DialogTitle>
+            <p className="text-sm text-text-muted">{getDisplayName(settingsOpenFor || '')}</p>
+          </DialogHeader>
+          {settingsOpenFor && <ModelSettingsPanel modelId={settingsOpenFor} />}
+        </DialogContent>
+      </Dialog>
+    </div>
+  );
+};
@@ -0,0 +1,415 @@
+import { useState, useEffect, useCallback } from 'react';
+import { RotateCcw } from 'lucide-react';
+import { Button } from '../../ui/button';
+import { Switch } from '../../ui/switch';
+import {
+  getModelSettings,
+  updateModelSettings,
+  type ModelSettings,
+  type SamplingConfig,
+} from '../../../api';
+
+const DEFAULT_SETTINGS: ModelSettings = {
+  context_size: null,
+  max_output_tokens: null,
+  sampling: { type: 'Temperature', temperature: 0.8, top_k: 40, top_p: 0.95, min_p: 0.05, seed: null },
+  repeat_penalty: 1.0,
+  repeat_last_n: 64,
+  frequency_penalty: 0.0,
+  presence_penalty: 0.0,
+  n_batch: null,
+  n_gpu_layers: null,
+  use_mlock: false,
+  flash_attention: null,
+  n_threads: null,
+  native_tool_calling: false,
+};
+
+type SamplingType = SamplingConfig['type'];
+
+function NumberField({
+  label,
+  description,
+  value,
+  onChange,
+  placeholder,
+  min,
+  max,
+  step,
+  allowNull,
+}: {
+  label: string;
+  description?: string;
+  value: number | null | undefined;
+  onChange: (v: number | null) => void;
+  placeholder?: string;
+  min?: number;
+  max?: number;
+  step?: number;
+  allowNull?: boolean;
+}) {
+  return (
+    <div className="flex flex-col gap-1">
+      <label className="text-xs font-medium text-text-default">{label}</label>
+      {description && <span className="text-xs text-text-muted">{description}</span>}
+      <input
+        type="number"
+        className="w-full rounded border border-border-subtle bg-background-default px-2 py-1 text-sm text-text-default"
+        value={value ?? ''}
+        onChange={(e) => {
+          const raw = e.target.value;
+          if (raw === '' && allowNull) {
+            onChange(null);
+          } else {
+            const n = step && step < 1 ? parseFloat(raw) : parseInt(raw, 10);
+            if (!isNaN(n)) onChange(n);
+          }
+        }}
+        placeholder={placeholder ?? 'Auto'}
+        min={min}
+        max={max}
+        step={step}
+      />
+    </div>
+  );
+}
+
+function ToggleField({
+  label,
+  description,
+  value,
+  onChange,
+}: {
+  label: string;
+  description?: string;
+  value: boolean;
+  onChange: (v: boolean) => void;
+}) {
+  return (
+    <div className="flex items-center justify-between gap-2">
+      <div>
+        <div className="text-xs font-medium text-text-default">{label}</div>
+        {description && <span className="text-xs text-text-muted">{description}</span>}
+      </div>
+      <Switch checked={value} onCheckedChange={onChange} variant="mono" />
+    </div>
+  );
+}
+
+function SelectField<T extends string>({
+  label,
+  description,
+  value,
+  options,
+  onChange,
+}: {
+  label: string;
+  description?: string;
+  value: T;
+  options: { value: T; label: string }[];
+  onChange: (v: T) => void;
+}) {
+  return (
+    <div className="flex items-center justify-between gap-2">
+      <div>
+        <div className="text-xs font-medium text-text-default">{label}</div>
+        {description && <span className="text-xs text-text-muted">{description}</span>}
+      </div>
+      <select
+        value={value}
+        onChange={(e) => onChange(e.target.value as T)}
+        className="rounded border border-border-subtle bg-background-default px-2 py-1 text-xs text-text-default"
+      >
+        {options.map((opt) => (
+          <option key={opt.value} value={opt.value}>
+            {opt.label}
+          </option>
+        ))}
+      </select>
+    </div>
+  );
+}
+
+export const ModelSettingsPanel = ({ modelId }: { modelId: string }) => {
+  const [settings, setSettings] = useState<ModelSettings>(DEFAULT_SETTINGS);
+  const [loading, setLoading] = useState(true);
+  const [saving, setSaving] = useState(false);
+
+  const load = useCallback(async () => {
+    try {
+      const res = await getModelSettings({ path: { model_id: modelId } });
+      if (res.data) setSettings(res.data);
+    } catch {
+      // use defaults
+    } finally {
+      setLoading(false);
+    }
+  }, [modelId]);
+
+  useEffect(() => {
+    load();
+  }, [load]);
+
+  const save = async (updated: ModelSettings) => {
+    setSettings(updated);
+    setSaving(true);
+    try {
+      await updateModelSettings({ path: { model_id: modelId }, body: updated });
+    } catch (e) {
+      console.error('Failed to save settings:', e);
+    } finally {
+      setSaving(false);
+    }
+  };
+
+  const resetDefaults = () => save(DEFAULT_SETTINGS);
+
+  const updateField = <K extends keyof ModelSettings>(key: K, value: ModelSettings[K]) => {
+    save({ ...settings, [key]: value });
+  };
+
+  const samplingType: SamplingType = settings.sampling?.type ?? 'Temperature';
+
+  const setSamplingType = (type: SamplingType) => {
+    let sampling: SamplingConfig;
+    if (type === 'Greedy') {
+      sampling = { type: 'Greedy' };
+    } else if (type === 'MirostatV2') {
+      sampling = { type: 'MirostatV2', tau: 5.0, eta: 0.1, seed: null };
+    } else {
+      sampling = { type: 'Temperature', temperature: 0.8, top_k: 40, top_p: 0.95, min_p: 0.05, seed: null };
+    }
+    save({ ...settings, sampling });
+  };
+
+  const updateSampling = (partial: Partial<SamplingConfig>) => {
+    save({ ...settings, sampling: { ...settings.sampling!, ...partial } as SamplingConfig });
+  };
+
+  if (loading) {
+    return <div className="py-2 text-xs text-text-muted">Loading settings...</div>;
+  }
+
+  return (
+    <div className="space-y-4">
+      <div className="flex items-center justify-end">
+        {saving && <span className="text-xs text-text-muted mr-auto">Saving...</span>}
+        <Button variant="ghost" size="sm" onClick={resetDefaults} title="Reset to defaults">
+          <RotateCcw className="w-3.5 h-3.5 mr-1" />
+          <span className="text-xs">Reset</span>
+        </Button>
+      </div>
+
+      {/* Context & Generation */}
+      <div className="space-y-2">
+        <h5 className="text-xs font-medium text-text-default">Context & Generation</h5>
+        <div className="grid grid-cols-2 gap-3">
+          <NumberField
+            label="Context size"
+            description="Max context window (0 = model default)"
+            value={settings.context_size}
+            onChange={(v) => updateField('context_size', v)}
+            placeholder="Auto"
+            min={0}
+            allowNull
+          />
+          <NumberField
+            label="Max output tokens"
+            description="Cap on generated tokens"
+            value={settings.max_output_tokens}
+            onChange={(v) => updateField('max_output_tokens', v)}
+            placeholder="No limit"
+            min={1}
+            allowNull
+          />
+        </div>
+      </div>
+
+      {/* Sampling */}
+      <div className="space-y-2">
+        <SelectField
+          label="Sampling Strategy"
+          value={samplingType}
+          options={[
+            { value: 'Greedy' as SamplingType, label: 'Greedy' },
+            { value: 'Temperature' as SamplingType, label: 'Temperature' },
+            { value: 'MirostatV2' as SamplingType, label: 'Mirostat v2' },
+          ]}
+          onChange={(v) => setSamplingType(v)}
+        />
+
+        {samplingType === 'Temperature' && settings.sampling?.type === 'Temperature' && (
+          <div className="grid grid-cols-2 gap-3">
+            <NumberField
+              label="Temperature"
+              value={settings.sampling.temperature}
+              onChange={(v) => updateSampling({ temperature: v ?? 0.8 })}
+              min={0}
+              max={2}
+              step={0.05}
+            />
+            <NumberField
+              label="Top K"
+              value={settings.sampling.top_k}
+              onChange={(v) => updateSampling({ top_k: v ?? 40 })}
+              min={0}
+            />
+            <NumberField
+              label="Top P"
+              value={settings.sampling.top_p}
+              onChange={(v) => updateSampling({ top_p: v ?? 0.95 })}
+              min={0}
+              max={1}
+              step={0.01}
+            />
+            <NumberField
+              label="Min P"
+              value={settings.sampling.min_p}
+              onChange={(v) => updateSampling({ min_p: v ?? 0.05 })}
+              min={0}
+              max={1}
+              step={0.01}
+            />
+            <NumberField
+              label="Seed"
+              value={settings.sampling.seed}
+              onChange={(v) => updateSampling({ seed: v })}
+              placeholder="Random"
+              min={0}
+              allowNull
+            />
+          </div>
+        )}
+
+        {samplingType === 'MirostatV2' && settings.sampling?.type === 'MirostatV2' && (
+          <div className="grid grid-cols-2 gap-3">
+            <NumberField
+              label="Tau (target entropy)"
+              value={settings.sampling.tau}
+              onChange={(v) => updateSampling({ tau: v ?? 5.0 })}
+              min={0}
+              step={0.1}
+            />
+            <NumberField
+              label="Eta (learning rate)"
+              value={settings.sampling.eta}
+              onChange={(v) => updateSampling({ eta: v ?? 0.1 })}
+              min={0}
+              max={1}
+              step={0.01}
+            />
+            <NumberField
+              label="Seed"
+              value={settings.sampling.seed}
+              onChange={(v) => updateSampling({ seed: v })}
+              placeholder="Random"
+              min={0}
+              allowNull
+            />
+          </div>
+        )}
+      </div>
+
+      {/* Repetition Penalty */}
+      <div className="space-y-2">
+        <h5 className="text-xs font-medium text-text-default">Repetition Penalty</h5>
+        <div className="grid grid-cols-2 gap-3">
+          <NumberField
+            label="Repeat penalty"
+            description="1.0 = off"
+            value={settings.repeat_penalty}
+            onChange={(v) => updateField('repeat_penalty', v ?? 1.0)}
+            min={0}
+            step={0.05}
+          />
+          <NumberField
+            label="Repeat window"
+            description="Tokens to look back"
+            value={settings.repeat_last_n}
+            onChange={(v) => updateField('repeat_last_n', v ?? 64)}
+            min={0}
+          />
+          <NumberField
+            label="Frequency penalty"
+            description="0.0 = off"
+            value={settings.frequency_penalty}
+            onChange={(v) => updateField('frequency_penalty', v ?? 0.0)}
+            min={0}
+            max={2}
+            step={0.05}
+          />
+          <NumberField
+            label="Presence penalty"
+            description="0.0 = off"
+            value={settings.presence_penalty}
+            onChange={(v) => updateField('presence_penalty', v ?? 0.0)}
+            min={0}
+            max={2}
+            step={0.05}
+          />
+        </div>
+      </div>
+
+      {/* Performance */}
+      <div className="space-y-2">
+        <h5 className="text-xs font-medium text-text-default">Performance</h5>
+        <div className="grid grid-cols-2 gap-3">
+          <NumberField
+            label="Batch size"
+            description="Prompt processing batch"
+            value={settings.n_batch}
+            onChange={(v) => updateField('n_batch', v)}
+            placeholder="Auto"
+            min={1}
+            allowNull
+          />
+          <NumberField
+            label="GPU layers"
+            description="Layers to offload to GPU"
+            value={settings.n_gpu_layers}
+            onChange={(v) => updateField('n_gpu_layers', v)}
+            placeholder="All"
+            min={0}
+            allowNull
+          />
+          <NumberField
+            label="Threads"
+            description="CPU threads for generation"
+            value={settings.n_threads}
+            onChange={(v) => updateField('n_threads', v)}
+            placeholder="Auto"
+            min={1}
+            allowNull
+          />
+        </div>
+        <ToggleField
+          label="Lock model in RAM (mlock)"
+          description="Prevent model from being swapped to disk"
+          value={settings.use_mlock ?? false}
+          onChange={(v) => updateField('use_mlock', v)}
+        />
+        <SelectField
+          label="Flash attention"
+          description="Enable flash attention optimization"
+          value={settings.flash_attention === null || settings.flash_attention === undefined ? 'auto' : settings.flash_attention ? 'on' : 'off'}
+          options={[
+            { value: 'auto', label: 'Auto' },
+            { value: 'on', label: 'On' },
+            { value: 'off', label: 'Off' },
+          ]}
+          onChange={(v) => updateField('flash_attention', v === 'auto' ? null : v === 'on')}
+        />
+      </div>
+      {/* Tool Calling */}
+      <div className="space-y-2">
+        <h5 className="text-xs font-medium text-text-default">Tool Calling</h5>
+        <ToggleField
+          label="Native tool calling"
+          description="Use the model's built-in tool-call format instead of the shell-command emulator. Enable for large models that reliably support tool calling."
+          value={settings.native_tool_calling ?? false}
+          onChange={(v) => updateField('native_tool_calling', v)}
+        />
+      </div>
+    </div>
+  );
+};
@@ -1,4 +1,4 @@
-import { ProviderDetails, getProviderModels } from '../../../api';
+import { ProviderDetails, getProviderModels, listLocalModels } from '../../../api';
 import { errorMessage as getErrorMessage } from '../../../utils/conversionUtils';

 export default interface Model {
@@ -54,6 +54,16 @@ export async function fetchModelsForProviders(
 ): Promise<ProviderModelsResult[]> {
  const modelPromises = activeProviders.map(async (p) => {
    try {
+      // For local provider, use listLocalModels and filter to only downloaded models
+      if (p.name === 'local') {
+        const response = await listLocalModels();
+        const allModels = response.data || [];
+        const downloadedModels = allModels
+          .filter((m) => m.status.state === 'Downloaded')
+          .map((m) => m.id);
+        return { provider: p, models: downloadedModels, error: null };
+      }
+
      const response = await getProviderModels({
        path: { name: p.name },
        throwOnError: true,
@@ -486,7 +486,36 @@ export const SwitchModelModal = ({

              {provider && (
                <>
-                  {providerErrors[provider] ? (
+                  {provider === 'local' &&
+                  !loadingModels &&
+                  filteredModelOptions.flatMap((g) => g.options).filter((o) => o.value !== 'custom')
+                    .length === 0 ? (
+                    /* Show special UI for local provider when no models are downloaded */
+                    <div className="rounded-md bg-blue-50 dark:bg-blue-900/20 border border-blue-200 dark:border-blue-800 p-4">
+                      <div className="flex flex-col gap-3">
+                        <div>
+                          <h3 className="text-sm font-medium text-blue-800 dark:text-blue-200">
+                            Local models need to be downloaded first
+                          </h3>
+                          <div className="mt-1 text-sm text-blue-700 dark:text-blue-300">
+                            To use local inference, you need to download a model to your computer
+                            first. Go to Settings → Models to manage local models.
+                          </div>
+                        </div>
+                        <Button
+                          variant="outline"
+                          size="sm"
+                          onClick={() => {
+                            setView('settings');
+                            onClose();
+                          }}
+                          className="self-start border-blue-300 dark:border-blue-700 text-blue-700 dark:text-blue-300 hover:bg-blue-100 dark:hover:bg-blue-900/40"
+                        >
+                          Go to Settings
+                        </Button>
+                      </div>
+                    </div>
+                  ) : providerErrors[provider] ? (
                    /* Show error message when provider failed to connect */
                    <div className="rounded-md bg-red-50 dark:bg-red-900/20 border border-red-200 dark:border-red-800 p-3">
                      <div className="flex items-start">
@@ -70,7 +70,7 @@ export type AnalyticsEvent =
  | {
      name: 'onboarding_provider_selected';
      properties: {
-        method: 'api_key' | 'openrouter' | 'tetrate' | 'chatgpt_codex' | 'ollama' | 'other';
+        method: 'api_key' | 'openrouter' | 'tetrate' | 'chatgpt_codex' | 'ollama' | 'local' | 'other';
      };
    }
  | {
@@ -80,7 +80,7 @@ export type AnalyticsEvent =
  | { name: 'onboarding_abandoned'; properties: { step: string; duration_seconds?: number } }
  | {
      name: 'onboarding_setup_failed';
-      properties: { provider: 'openrouter' | 'tetrate' | 'chatgpt_codex'; error_message?: string };
+      properties: { provider: 'openrouter' | 'tetrate' | 'chatgpt_codex' | 'local'; error_message?: string };
    }
  | {
      name: 'error_occurred';
@@ -284,7 +284,7 @@ export function trackOnboardingStarted(): void {
 }

 export function trackOnboardingProviderSelected(
-  method: 'api_key' | 'openrouter' | 'tetrate' | 'chatgpt_codex' | 'ollama' | 'other'
+  method: 'api_key' | 'openrouter' | 'tetrate' | 'chatgpt_codex' | 'ollama' | 'local' | 'other'
 ): void {
  trackEvent({
    name: 'onboarding_provider_selected',
@@ -317,7 +317,7 @@ export function trackOnboardingAbandoned(step: string): void {
 }

 export function trackOnboardingSetupFailed(
-  provider: 'openrouter' | 'tetrate' | 'chatgpt_codex',
+  provider: 'openrouter' | 'tetrate' | 'chatgpt_codex' | 'local',
  errorMessage?: string
 ): void {
  trackEvent({