feat: add local inference provider with llama.cpp backend and HuggingFace model management (#6933)

Co-authored-by: Douwe Osinga <douwe@squareup.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: jh-block <jhugo@block.xyz>
Co-authored-by: Spence <spencermartin@squareup.com>
Co-authored-by: Michael Neale <michael.neale@gmail.com>

ui/desktop/src/api/types.gen.ts

@@ -232,6 +232,13 @@ export type DictationProviderStatus = {
     uses_provider_config: boolean;
 };
 
+export type DownloadModelRequest = {
+    /**
+     * Model spec like "bartowski/Llama-3.2-3B-Instruct-GGUF:Q4_K_M"
+     */
+    spec: string;
+};
+
 export type DownloadProgress = {
     /**
      * Bytes downloaded so far
@@ -446,6 +453,36 @@ export type GooseApp = McpAppResource & (WindowProps | null) & {
     prd?: string | null;
 };
 
+/**
+ * A single downloadable GGUF file (used internally and for downloads).
+ */
+export type HfGgufFile = {
+    download_url: string;
+    filename: string;
+    quantization: string;
+    size_bytes: number;
+};
+
+export type HfModelInfo = {
+    author: string;
+    downloads: number;
+    gguf_files: Array<HfGgufFile>;
+    model_name: string;
+    repo_id: string;
+};
+
+/**
+ * A quantization variant — groups sharded files into one logical entry.
+ */
+export type HfQuantVariant = {
+    description: string;
+    download_url: string;
+    filename: string;
+    quality_rank: number;
+    quantization: string;
+    size_bytes: number;
+};
+
 export type Icon = {
     mimeType?: string;
     sizes?: Array<string>;
@@ -511,6 +548,18 @@ export type LoadedProvider = {
     is_editable: boolean;
 };
 
+export type LocalModelResponse = {
+    display_name: string;
+    filename: string;
+    id: string;
+    quantization: string;
+    recommended: boolean;
+    repo_id: string;
+    settings: ModelSettings;
+    size_bytes: number;
+    status: ModelDownloadStatus;
+};
+
 /**
  * MCP App Resource
  * Represents a UI resource that can be rendered in an MCP App
@@ -638,6 +687,18 @@ export type ModelConfig = {
     toolshim_model?: string | null;
 };
 
+export type ModelDownloadStatus = {
+    state: 'NotDownloaded';
+} | {
+    bytes_downloaded: number;
+    progress_percent: number;
+    speed_bps?: number | null;
+    state: 'Downloading';
+    total_bytes: number;
+} | {
+    state: 'Downloaded';
+};
+
 /**
  * Information about a model's capabilities
  */
@@ -690,6 +751,23 @@ export type ModelInfoResponse = {
     source: string;
 };
 
+export type ModelSettings = {
+    context_size?: number | null;
+    flash_attention?: boolean | null;
+    frequency_penalty?: number;
+    max_output_tokens?: number | null;
+    n_batch?: number | null;
+    n_gpu_layers?: number | null;
+    n_threads?: number | null;
+    native_tool_calling?: boolean;
+    presence_penalty?: number;
+    repeat_last_n?: number;
+    repeat_penalty?: number;
+    sampling?: SamplingConfig;
+    use_jinja?: boolean;
+    use_mlock?: boolean;
+};
+
 export type ParseRecipeRequest = {
     content: string;
 };
@@ -909,6 +987,11 @@ export type RemoveExtensionRequest = {
     session_id: string;
 };
 
+export type RepoVariantsResponse = {
+    recommended_index?: number | null;
+    variants: Array<HfQuantVariant>;
+};
+
 export type ResourceContents = {
     _meta?: {
         [key: string]: unknown;
@@ -986,6 +1069,22 @@ export type RunNowResponse = {
     session_id: string;
 };
 
+export type SamplingConfig = {
+    type: 'Greedy';
+} | {
+    min_p: number;
+    seed?: number | null;
+    temperature: number;
+    top_k: number;
+    top_p: number;
+    type: 'Temperature';
+} | {
+    eta: number;
+    seed?: number | null;
+    tau: number;
+    type: 'MirostatV2';
+};
+
 export type SavePromptRequest = {
     content: string;
 };
@@ -2836,6 +2935,221 @@ export type StartTetrateSetupResponses = {
 
 export type StartTetrateSetupResponse = StartTetrateSetupResponses[keyof StartTetrateSetupResponses];
 
+export type DownloadHfModelData = {
+    body: DownloadModelRequest;
+    path?: never;
+    query?: never;
+    url: '/local-inference/download';
+};
+
+export type DownloadHfModelErrors = {
+    /**
+     * Invalid request
+     */
+    400: unknown;
+};
+
+export type DownloadHfModelResponses = {
+    /**
+     * Download started
+     */
+    202: string;
+};
+
+export type DownloadHfModelResponse = DownloadHfModelResponses[keyof DownloadHfModelResponses];
+
+export type ListLocalModelsData = {
+    body?: never;
+    path?: never;
+    query?: never;
+    url: '/local-inference/models';
+};
+
+export type ListLocalModelsResponses = {
+    /**
+     * List of available local LLM models
+     */
+    200: Array<LocalModelResponse>;
+};
+
+export type ListLocalModelsResponse = ListLocalModelsResponses[keyof ListLocalModelsResponses];
+
+export type DeleteLocalModelData = {
+    body?: never;
+    path: {
+        model_id: string;
+    };
+    query?: never;
+    url: '/local-inference/models/{model_id}';
+};
+
+export type DeleteLocalModelErrors = {
+    /**
+     * Model not found
+     */
+    404: unknown;
+};
+
+export type DeleteLocalModelResponses = {
+    /**
+     * Model deleted
+     */
+    200: unknown;
+};
+
+export type CancelLocalModelDownloadData = {
+    body?: never;
+    path: {
+        model_id: string;
+    };
+    query?: never;
+    url: '/local-inference/models/{model_id}/download';
+};
+
+export type CancelLocalModelDownloadErrors = {
+    /**
+     * No active download
+     */
+    404: unknown;
+};
+
+export type CancelLocalModelDownloadResponses = {
+    /**
+     * Download cancelled
+     */
+    200: unknown;
+};
+
+export type GetLocalModelDownloadProgressData = {
+    body?: never;
+    path: {
+        model_id: string;
+    };
+    query?: never;
+    url: '/local-inference/models/{model_id}/download';
+};
+
+export type GetLocalModelDownloadProgressErrors = {
+    /**
+     * No active download
+     */
+    404: unknown;
+};
+
+export type GetLocalModelDownloadProgressResponses = {
+    /**
+     * Download progress
+     */
+    200: DownloadProgress;
+};
+
+export type GetLocalModelDownloadProgressResponse = GetLocalModelDownloadProgressResponses[keyof GetLocalModelDownloadProgressResponses];
+
+export type GetModelSettingsData = {
+    body?: never;
+    path: {
+        model_id: string;
+    };
+    query?: never;
+    url: '/local-inference/models/{model_id}/settings';
+};
+
+export type GetModelSettingsErrors = {
+    /**
+     * Model not found
+     */
+    404: unknown;
+};
+
+export type GetModelSettingsResponses = {
+    /**
+     * Model settings
+     */
+    200: ModelSettings;
+};
+
+export type GetModelSettingsResponse = GetModelSettingsResponses[keyof GetModelSettingsResponses];
+
+export type UpdateModelSettingsData = {
+    body: ModelSettings;
+    path: {
+        model_id: string;
+    };
+    query?: never;
+    url: '/local-inference/models/{model_id}/settings';
+};
+
+export type UpdateModelSettingsErrors = {
+    /**
+     * Model not found
+     */
+    404: unknown;
+    /**
+     * Failed to save settings
+     */
+    500: unknown;
+};
+
+export type UpdateModelSettingsResponses = {
+    /**
+     * Settings updated
+     */
+    200: ModelSettings;
+};
+
+export type UpdateModelSettingsResponse = UpdateModelSettingsResponses[keyof UpdateModelSettingsResponses];
+
+export type GetRepoFilesData = {
+    body?: never;
+    path: {
+        author: string;
+        repo: string;
+    };
+    query?: never;
+    url: '/local-inference/repo/{author}/{repo}/files';
+};
+
+export type GetRepoFilesResponses = {
+    /**
+     * GGUF files in the repo
+     */
+    200: RepoVariantsResponse;
+};
+
+export type GetRepoFilesResponse = GetRepoFilesResponses[keyof GetRepoFilesResponses];
+
+export type SearchHfModelsData = {
+    body?: never;
+    path?: never;
+    query: {
+        /**
+         * Search query
+         */
+        q: string;
+        /**
+         * Max results
+         */
+        limit?: number | null;
+    };
+    url: '/local-inference/search';
+};
+
+export type SearchHfModelsErrors = {
+    /**
+     * Search failed
+     */
+    500: unknown;
+};
+
+export type SearchHfModelsResponses = {
+    /**
+     * Search results
+     */
+    200: Array<HfModelInfo>;
+};
+
+export type SearchHfModelsResponse = SearchHfModelsResponses[keyof SearchHfModelsResponses];
+
 export type McpUiProxyData = {
     body?: never;
     path?: never;