feat: add local inference provider with llama.cpp backend and HuggingFace model management (#6933)
Co-authored-by: Douwe Osinga <douwe@squareup.com> Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Co-authored-by: jh-block <jhugo@block.xyz> Co-authored-by: Spence <spencermartin@squareup.com> Co-authored-by: Michael Neale <michael.neale@gmail.com>
This commit is contained in:
Douwe Osinga
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -232,6 +232,13 @@ export type DictationProviderStatus = {
|
||||
uses_provider_config: boolean;
|
||||
};
|
||||
|
||||
export type DownloadModelRequest = {
|
||||
/**
|
||||
* Model spec like "bartowski/Llama-3.2-3B-Instruct-GGUF:Q4_K_M"
|
||||
*/
|
||||
spec: string;
|
||||
};
|
||||
|
||||
export type DownloadProgress = {
|
||||
/**
|
||||
* Bytes downloaded so far
|
||||
@@ -446,6 +453,36 @@ export type GooseApp = McpAppResource & (WindowProps | null) & {
|
||||
prd?: string | null;
|
||||
};
|
||||
|
||||
/**
|
||||
* A single downloadable GGUF file (used internally and for downloads).
|
||||
*/
|
||||
export type HfGgufFile = {
|
||||
download_url: string;
|
||||
filename: string;
|
||||
quantization: string;
|
||||
size_bytes: number;
|
||||
};
|
||||
|
||||
export type HfModelInfo = {
|
||||
author: string;
|
||||
downloads: number;
|
||||
gguf_files: Array<HfGgufFile>;
|
||||
model_name: string;
|
||||
repo_id: string;
|
||||
};
|
||||
|
||||
/**
|
||||
* A quantization variant — groups sharded files into one logical entry.
|
||||
*/
|
||||
export type HfQuantVariant = {
|
||||
description: string;
|
||||
download_url: string;
|
||||
filename: string;
|
||||
quality_rank: number;
|
||||
quantization: string;
|
||||
size_bytes: number;
|
||||
};
|
||||
|
||||
export type Icon = {
|
||||
mimeType?: string;
|
||||
sizes?: Array<string>;
|
||||
@@ -511,6 +548,18 @@ export type LoadedProvider = {
|
||||
is_editable: boolean;
|
||||
};
|
||||
|
||||
export type LocalModelResponse = {
|
||||
display_name: string;
|
||||
filename: string;
|
||||
id: string;
|
||||
quantization: string;
|
||||
recommended: boolean;
|
||||
repo_id: string;
|
||||
settings: ModelSettings;
|
||||
size_bytes: number;
|
||||
status: ModelDownloadStatus;
|
||||
};
|
||||
|
||||
/**
|
||||
* MCP App Resource
|
||||
* Represents a UI resource that can be rendered in an MCP App
|
||||
@@ -638,6 +687,18 @@ export type ModelConfig = {
|
||||
toolshim_model?: string | null;
|
||||
};
|
||||
|
||||
export type ModelDownloadStatus = {
|
||||
state: 'NotDownloaded';
|
||||
} | {
|
||||
bytes_downloaded: number;
|
||||
progress_percent: number;
|
||||
speed_bps?: number | null;
|
||||
state: 'Downloading';
|
||||
total_bytes: number;
|
||||
} | {
|
||||
state: 'Downloaded';
|
||||
};
|
||||
|
||||
/**
|
||||
* Information about a model's capabilities
|
||||
*/
|
||||
@@ -690,6 +751,23 @@ export type ModelInfoResponse = {
|
||||
source: string;
|
||||
};
|
||||
|
||||
export type ModelSettings = {
|
||||
context_size?: number | null;
|
||||
flash_attention?: boolean | null;
|
||||
frequency_penalty?: number;
|
||||
max_output_tokens?: number | null;
|
||||
n_batch?: number | null;
|
||||
n_gpu_layers?: number | null;
|
||||
n_threads?: number | null;
|
||||
native_tool_calling?: boolean;
|
||||
presence_penalty?: number;
|
||||
repeat_last_n?: number;
|
||||
repeat_penalty?: number;
|
||||
sampling?: SamplingConfig;
|
||||
use_jinja?: boolean;
|
||||
use_mlock?: boolean;
|
||||
};
|
||||
|
||||
export type ParseRecipeRequest = {
|
||||
content: string;
|
||||
};
|
||||
@@ -909,6 +987,11 @@ export type RemoveExtensionRequest = {
|
||||
session_id: string;
|
||||
};
|
||||
|
||||
export type RepoVariantsResponse = {
|
||||
recommended_index?: number | null;
|
||||
variants: Array<HfQuantVariant>;
|
||||
};
|
||||
|
||||
export type ResourceContents = {
|
||||
_meta?: {
|
||||
[key: string]: unknown;
|
||||
@@ -986,6 +1069,22 @@ export type RunNowResponse = {
|
||||
session_id: string;
|
||||
};
|
||||
|
||||
export type SamplingConfig = {
|
||||
type: 'Greedy';
|
||||
} | {
|
||||
min_p: number;
|
||||
seed?: number | null;
|
||||
temperature: number;
|
||||
top_k: number;
|
||||
top_p: number;
|
||||
type: 'Temperature';
|
||||
} | {
|
||||
eta: number;
|
||||
seed?: number | null;
|
||||
tau: number;
|
||||
type: 'MirostatV2';
|
||||
};
|
||||
|
||||
export type SavePromptRequest = {
|
||||
content: string;
|
||||
};
|
||||
@@ -2836,6 +2935,221 @@ export type StartTetrateSetupResponses = {
|
||||
|
||||
export type StartTetrateSetupResponse = StartTetrateSetupResponses[keyof StartTetrateSetupResponses];
|
||||
|
||||
export type DownloadHfModelData = {
|
||||
body: DownloadModelRequest;
|
||||
path?: never;
|
||||
query?: never;
|
||||
url: '/local-inference/download';
|
||||
};
|
||||
|
||||
export type DownloadHfModelErrors = {
|
||||
/**
|
||||
* Invalid request
|
||||
*/
|
||||
400: unknown;
|
||||
};
|
||||
|
||||
export type DownloadHfModelResponses = {
|
||||
/**
|
||||
* Download started
|
||||
*/
|
||||
202: string;
|
||||
};
|
||||
|
||||
export type DownloadHfModelResponse = DownloadHfModelResponses[keyof DownloadHfModelResponses];
|
||||
|
||||
export type ListLocalModelsData = {
|
||||
body?: never;
|
||||
path?: never;
|
||||
query?: never;
|
||||
url: '/local-inference/models';
|
||||
};
|
||||
|
||||
export type ListLocalModelsResponses = {
|
||||
/**
|
||||
* List of available local LLM models
|
||||
*/
|
||||
200: Array<LocalModelResponse>;
|
||||
};
|
||||
|
||||
export type ListLocalModelsResponse = ListLocalModelsResponses[keyof ListLocalModelsResponses];
|
||||
|
||||
export type DeleteLocalModelData = {
|
||||
body?: never;
|
||||
path: {
|
||||
model_id: string;
|
||||
};
|
||||
query?: never;
|
||||
url: '/local-inference/models/{model_id}';
|
||||
};
|
||||
|
||||
export type DeleteLocalModelErrors = {
|
||||
/**
|
||||
* Model not found
|
||||
*/
|
||||
404: unknown;
|
||||
};
|
||||
|
||||
export type DeleteLocalModelResponses = {
|
||||
/**
|
||||
* Model deleted
|
||||
*/
|
||||
200: unknown;
|
||||
};
|
||||
|
||||
export type CancelLocalModelDownloadData = {
|
||||
body?: never;
|
||||
path: {
|
||||
model_id: string;
|
||||
};
|
||||
query?: never;
|
||||
url: '/local-inference/models/{model_id}/download';
|
||||
};
|
||||
|
||||
export type CancelLocalModelDownloadErrors = {
|
||||
/**
|
||||
* No active download
|
||||
*/
|
||||
404: unknown;
|
||||
};
|
||||
|
||||
export type CancelLocalModelDownloadResponses = {
|
||||
/**
|
||||
* Download cancelled
|
||||
*/
|
||||
200: unknown;
|
||||
};
|
||||
|
||||
export type GetLocalModelDownloadProgressData = {
|
||||
body?: never;
|
||||
path: {
|
||||
model_id: string;
|
||||
};
|
||||
query?: never;
|
||||
url: '/local-inference/models/{model_id}/download';
|
||||
};
|
||||
|
||||
export type GetLocalModelDownloadProgressErrors = {
|
||||
/**
|
||||
* No active download
|
||||
*/
|
||||
404: unknown;
|
||||
};
|
||||
|
||||
export type GetLocalModelDownloadProgressResponses = {
|
||||
/**
|
||||
* Download progress
|
||||
*/
|
||||
200: DownloadProgress;
|
||||
};
|
||||
|
||||
export type GetLocalModelDownloadProgressResponse = GetLocalModelDownloadProgressResponses[keyof GetLocalModelDownloadProgressResponses];
|
||||
|
||||
export type GetModelSettingsData = {
|
||||
body?: never;
|
||||
path: {
|
||||
model_id: string;
|
||||
};
|
||||
query?: never;
|
||||
url: '/local-inference/models/{model_id}/settings';
|
||||
};
|
||||
|
||||
export type GetModelSettingsErrors = {
|
||||
/**
|
||||
* Model not found
|
||||
*/
|
||||
404: unknown;
|
||||
};
|
||||
|
||||
export type GetModelSettingsResponses = {
|
||||
/**
|
||||
* Model settings
|
||||
*/
|
||||
200: ModelSettings;
|
||||
};
|
||||
|
||||
export type GetModelSettingsResponse = GetModelSettingsResponses[keyof GetModelSettingsResponses];
|
||||
|
||||
export type UpdateModelSettingsData = {
|
||||
body: ModelSettings;
|
||||
path: {
|
||||
model_id: string;
|
||||
};
|
||||
query?: never;
|
||||
url: '/local-inference/models/{model_id}/settings';
|
||||
};
|
||||
|
||||
export type UpdateModelSettingsErrors = {
|
||||
/**
|
||||
* Model not found
|
||||
*/
|
||||
404: unknown;
|
||||
/**
|
||||
* Failed to save settings
|
||||
*/
|
||||
500: unknown;
|
||||
};
|
||||
|
||||
export type UpdateModelSettingsResponses = {
|
||||
/**
|
||||
* Settings updated
|
||||
*/
|
||||
200: ModelSettings;
|
||||
};
|
||||
|
||||
export type UpdateModelSettingsResponse = UpdateModelSettingsResponses[keyof UpdateModelSettingsResponses];
|
||||
|
||||
export type GetRepoFilesData = {
|
||||
body?: never;
|
||||
path: {
|
||||
author: string;
|
||||
repo: string;
|
||||
};
|
||||
query?: never;
|
||||
url: '/local-inference/repo/{author}/{repo}/files';
|
||||
};
|
||||
|
||||
export type GetRepoFilesResponses = {
|
||||
/**
|
||||
* GGUF files in the repo
|
||||
*/
|
||||
200: RepoVariantsResponse;
|
||||
};
|
||||
|
||||
export type GetRepoFilesResponse = GetRepoFilesResponses[keyof GetRepoFilesResponses];
|
||||
|
||||
export type SearchHfModelsData = {
|
||||
body?: never;
|
||||
path?: never;
|
||||
query: {
|
||||
/**
|
||||
* Search query
|
||||
*/
|
||||
q: string;
|
||||
/**
|
||||
* Max results
|
||||
*/
|
||||
limit?: number | null;
|
||||
};
|
||||
url: '/local-inference/search';
|
||||
};
|
||||
|
||||
export type SearchHfModelsErrors = {
|
||||
/**
|
||||
* Search failed
|
||||
*/
|
||||
500: unknown;
|
||||
};
|
||||
|
||||
export type SearchHfModelsResponses = {
|
||||
/**
|
||||
* Search results
|
||||
*/
|
||||
200: Array<HfModelInfo>;
|
||||
};
|
||||
|
||||
export type SearchHfModelsResponse = SearchHfModelsResponses[keyof SearchHfModelsResponses];
|
||||
|
||||
export type McpUiProxyData = {
|
||||
body?: never;
|
||||
path?: never;
|
||||
|
||||
@@ -0,0 +1,397 @@
|
||||
import { useState, useEffect, useCallback, useRef } from 'react';
|
||||
import { useConfig } from './ConfigContext';
|
||||
import {
|
||||
listLocalModels,
|
||||
downloadHfModel,
|
||||
getLocalModelDownloadProgress,
|
||||
cancelLocalModelDownload,
|
||||
type DownloadProgress,
|
||||
type LocalModelResponse,
|
||||
} from '../api';
|
||||
import { toastService } from '../toasts';
|
||||
import { trackOnboardingSetupFailed } from '../utils/analytics';
|
||||
import { Goose } from './icons';
|
||||
|
||||
interface LocalModelSetupProps {
|
||||
onSuccess: () => void;
|
||||
onCancel: () => void;
|
||||
}
|
||||
|
||||
const formatBytes = (bytes: number): string => {
|
||||
if (bytes < 1024) return `${bytes}B`;
|
||||
if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(0)}KB`;
|
||||
if (bytes < 1024 * 1024 * 1024) return `${(bytes / (1024 * 1024)).toFixed(0)}MB`;
|
||||
return `${(bytes / (1024 * 1024 * 1024)).toFixed(1)}GB`;
|
||||
};
|
||||
|
||||
const formatSize = (bytes: number): string => {
|
||||
const mb = bytes / (1024 * 1024);
|
||||
return mb >= 1024 ? `${(mb / 1024).toFixed(1)}GB` : `${mb.toFixed(0)}MB`;
|
||||
};
|
||||
|
||||
type SetupPhase = 'loading' | 'select' | 'downloading' | 'error';
|
||||
|
||||
export function LocalModelSetup({ onSuccess, onCancel }: LocalModelSetupProps) {
|
||||
const { upsert } = useConfig();
|
||||
const [phase, setPhase] = useState<SetupPhase>('loading');
|
||||
const [models, setModels] = useState<LocalModelResponse[]>([]);
|
||||
const [selectedModelId, setSelectedModelId] = useState<string | null>(null);
|
||||
const [downloadProgress, setDownloadProgress] = useState<DownloadProgress | null>(null);
|
||||
const [errorMessage, setErrorMessage] = useState<string | null>(null);
|
||||
const [showAllModels, setShowAllModels] = useState(false);
|
||||
const pollRef = useRef<ReturnType<typeof setInterval> | null>(null);
|
||||
|
||||
const cleanup = useCallback(() => {
|
||||
if (pollRef.current) {
|
||||
clearInterval(pollRef.current);
|
||||
pollRef.current = null;
|
||||
}
|
||||
}, []);
|
||||
|
||||
useEffect(() => cleanup, [cleanup]);
|
||||
|
||||
useEffect(() => {
|
||||
const load = async () => {
|
||||
try {
|
||||
const response = await listLocalModels();
|
||||
if (response.data) {
|
||||
setModels(response.data);
|
||||
|
||||
const alreadyDownloaded = response.data.find((m) => m.status.state === 'Downloaded');
|
||||
if (alreadyDownloaded) {
|
||||
setSelectedModelId(alreadyDownloaded.id);
|
||||
} else {
|
||||
const recommended = response.data.find((m: LocalModelResponse) => m.recommended);
|
||||
if (recommended) setSelectedModelId(recommended.id);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Failed to load local models:', error);
|
||||
setErrorMessage('Failed to load available models. Please try again.');
|
||||
setPhase('error');
|
||||
return;
|
||||
}
|
||||
setPhase('select');
|
||||
};
|
||||
load();
|
||||
}, []);
|
||||
|
||||
const finishSetup = async (modelId: string) => {
|
||||
await upsert('GOOSE_PROVIDER', 'local', false);
|
||||
await upsert('GOOSE_MODEL', modelId, false);
|
||||
toastService.success({
|
||||
title: 'Local Model Ready',
|
||||
msg: `Running entirely on your machine with ${modelId}.`,
|
||||
});
|
||||
onSuccess();
|
||||
};
|
||||
|
||||
const startDownload = async (modelId: string) => {
|
||||
setPhase('downloading');
|
||||
setDownloadProgress(null);
|
||||
setErrorMessage(null);
|
||||
|
||||
const model = models.find((m) => m.id === modelId);
|
||||
if (!model) {
|
||||
setErrorMessage('Model not found');
|
||||
setPhase('error');
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
await downloadHfModel({ body: { spec: model.id } });
|
||||
} catch (error) {
|
||||
console.error('Failed to start download:', error);
|
||||
setErrorMessage('Failed to start download. Please try again.');
|
||||
trackOnboardingSetupFailed('local', 'download_start_failed');
|
||||
setPhase('error');
|
||||
return;
|
||||
}
|
||||
|
||||
pollRef.current = setInterval(async () => {
|
||||
try {
|
||||
const response = await getLocalModelDownloadProgress({ path: { model_id: modelId } });
|
||||
if (response.data) {
|
||||
setDownloadProgress(response.data);
|
||||
if (response.data.status === 'completed') {
|
||||
cleanup();
|
||||
await finishSetup(modelId);
|
||||
} else if (response.data.status === 'failed') {
|
||||
cleanup();
|
||||
setErrorMessage(response.data.error || 'Download failed.');
|
||||
trackOnboardingSetupFailed('local', response.data.error || 'download_failed');
|
||||
setPhase('error');
|
||||
} else if (response.data.status === 'cancelled') {
|
||||
cleanup();
|
||||
setPhase('select');
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
cleanup();
|
||||
setErrorMessage('Lost connection to download. Please try again.');
|
||||
trackOnboardingSetupFailed('local', 'progress_poll_failed');
|
||||
setPhase('error');
|
||||
}
|
||||
}, 500);
|
||||
};
|
||||
|
||||
const handleCancel = async () => {
|
||||
if (phase === 'downloading' && selectedModelId) {
|
||||
cleanup();
|
||||
try {
|
||||
await cancelLocalModelDownload({ path: { model_id: selectedModelId } });
|
||||
} catch {
|
||||
// best-effort
|
||||
}
|
||||
setDownloadProgress(null);
|
||||
setPhase('select');
|
||||
} else {
|
||||
onCancel();
|
||||
}
|
||||
};
|
||||
|
||||
const handlePrimaryAction = async () => {
|
||||
if (!selectedModelId) return;
|
||||
const model = models.find((m) => m.id === selectedModelId);
|
||||
if (!model) return;
|
||||
if (model.status.state === 'Downloaded') {
|
||||
await finishSetup(model.id);
|
||||
} else {
|
||||
await startDownload(model.id);
|
||||
}
|
||||
};
|
||||
|
||||
const recommended = models.find((m) => m.recommended);
|
||||
const otherModels = models.filter((m) => m.id !== recommended?.id);
|
||||
const selectedModel = models.find((m) => m.id === selectedModelId);
|
||||
|
||||
if (phase === 'loading') {
|
||||
return (
|
||||
<div className="flex flex-col items-center justify-center py-16">
|
||||
<div className="animate-spin rounded-full h-8 w-8 border-t-2 border-b-2 border-text-muted mb-4"></div>
|
||||
<p className="text-text-muted text-sm">Checking available models...</p>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="space-y-6">
|
||||
{/* Header */}
|
||||
<div className="text-left space-y-3">
|
||||
<div className="origin-bottom-left goose-icon-animation">
|
||||
<Goose className="size-6 sm:size-8" />
|
||||
</div>
|
||||
<h1 className="text-2xl sm:text-4xl font-light">Run Locally</h1>
|
||||
<p className="text-text-muted text-base sm:text-lg">
|
||||
Download a model to run Goose entirely on your machine — no API keys, no accounts, completely free and private.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
{/* Error state */}
|
||||
{phase === 'error' && (
|
||||
<div className="space-y-4">
|
||||
<div className="border border-red-500/30 rounded-xl p-4 bg-red-500/5">
|
||||
<p className="text-sm text-red-400">{errorMessage}</p>
|
||||
</div>
|
||||
<button
|
||||
onClick={() => {
|
||||
setErrorMessage(null);
|
||||
setPhase('select');
|
||||
}}
|
||||
className="w-full px-6 py-3 bg-background-muted text-text-default rounded-lg transition-colors font-medium hover:bg-background-muted/80"
|
||||
>
|
||||
Try Again
|
||||
</button>
|
||||
<button
|
||||
onClick={onCancel}
|
||||
className="w-full px-6 py-3 bg-transparent text-text-muted rounded-lg hover:bg-background-muted transition-colors"
|
||||
>
|
||||
Back
|
||||
</button>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Model selection */}
|
||||
{phase === 'select' && (
|
||||
<div className="space-y-5">
|
||||
{/* Recommended model card */}
|
||||
{recommended && (
|
||||
<div
|
||||
onClick={() => setSelectedModelId(recommended.id)}
|
||||
className={`relative w-full p-4 sm:p-6 border rounded-xl cursor-pointer transition-all duration-200 group ${
|
||||
selectedModelId === recommended.id
|
||||
? 'border-blue-500 bg-blue-500/5'
|
||||
: 'border-border-subtle hover:border-border-default'
|
||||
}`}
|
||||
>
|
||||
<div className="absolute -top-2 -right-2 sm:-top-3 sm:-right-3 z-10">
|
||||
<span className="inline-block px-2 py-1 text-xs font-medium bg-blue-600 text-white rounded-full">
|
||||
Best for your machine
|
||||
</span>
|
||||
</div>
|
||||
<div className="flex items-start gap-3">
|
||||
<input
|
||||
type="radio"
|
||||
checked={selectedModelId === recommended.id}
|
||||
onChange={() => setSelectedModelId(recommended.id)}
|
||||
className="cursor-pointer flex-shrink-0 mt-1"
|
||||
/>
|
||||
<div className="flex-1 min-w-0">
|
||||
<div className="flex items-center gap-2 flex-wrap">
|
||||
<span className="font-medium text-text-default text-sm sm:text-base">
|
||||
{recommended.display_name}
|
||||
</span>
|
||||
{recommended.status.state === 'Downloaded' && (
|
||||
<span className="text-xs bg-green-600 text-white px-2 py-0.5 rounded-full">
|
||||
Ready
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
<p className="text-text-muted text-xs mt-1">
|
||||
{formatSize(recommended.size_bytes)}
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Expandable other models */}
|
||||
{otherModels.length > 0 && (
|
||||
<div>
|
||||
<button
|
||||
onClick={() => setShowAllModels(!showAllModels)}
|
||||
className="text-sm text-blue-500 hover:text-blue-400 transition-colors flex items-center gap-1"
|
||||
>
|
||||
{showAllModels ? 'Hide other sizes' : `Show ${otherModels.length} other sizes`}
|
||||
<svg
|
||||
className={`w-3.5 h-3.5 transition-transform ${showAllModels ? 'rotate-180' : ''}`}
|
||||
fill="none"
|
||||
stroke="currentColor"
|
||||
viewBox="0 0 24 24"
|
||||
>
|
||||
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M19 9l-7 7-7-7" />
|
||||
</svg>
|
||||
</button>
|
||||
|
||||
{showAllModels && (
|
||||
<div className="mt-3 space-y-2">
|
||||
{otherModels.map((model) => (
|
||||
<div
|
||||
key={model.id}
|
||||
onClick={() => setSelectedModelId(model.id)}
|
||||
className={`w-full p-4 border rounded-xl cursor-pointer transition-all duration-200 ${
|
||||
selectedModelId === model.id
|
||||
? 'border-blue-500 bg-blue-500/5'
|
||||
: 'border-border-subtle hover:border-border-default'
|
||||
}`}
|
||||
>
|
||||
<div className="flex items-start gap-3">
|
||||
<input
|
||||
type="radio"
|
||||
checked={selectedModelId === model.id}
|
||||
onChange={() => setSelectedModelId(model.id)}
|
||||
className="cursor-pointer flex-shrink-0 mt-0.5"
|
||||
/>
|
||||
<div className="flex-1 min-w-0">
|
||||
<div className="flex items-center gap-2 flex-wrap">
|
||||
<span className="font-medium text-text-default text-sm">{model.display_name}</span>
|
||||
<span className="text-xs text-text-muted">{formatSize(model.size_bytes)}</span>
|
||||
{model.status.state === 'Downloaded' && (
|
||||
<span className="text-xs bg-green-600 text-white px-2 py-0.5 rounded-full">
|
||||
Ready
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Primary action */}
|
||||
<button
|
||||
onClick={handlePrimaryAction}
|
||||
disabled={!selectedModelId}
|
||||
className="w-full px-6 py-3 bg-background-muted text-text-default rounded-lg transition-colors font-medium disabled:opacity-40 disabled:cursor-not-allowed hover:bg-background-muted/80"
|
||||
>
|
||||
{selectedModel?.status.state === 'Downloaded'
|
||||
? `Use ${selectedModel.display_name}`
|
||||
: selectedModel
|
||||
? `Download ${selectedModel.display_name} (${formatSize(selectedModel.size_bytes)})`
|
||||
: 'Select a model'}
|
||||
</button>
|
||||
|
||||
<button
|
||||
onClick={onCancel}
|
||||
className="w-full px-6 py-3 bg-transparent text-text-muted rounded-lg hover:bg-background-muted transition-colors"
|
||||
>
|
||||
Back
|
||||
</button>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Downloading state */}
|
||||
{phase === 'downloading' && selectedModel && (
|
||||
<div className="space-y-6">
|
||||
<div className="border border-border-subtle rounded-xl p-5 sm:p-6 bg-background-default">
|
||||
<p className="font-medium text-text-default text-sm sm:text-base mb-4">
|
||||
Downloading {selectedModel.display_name}
|
||||
</p>
|
||||
|
||||
{downloadProgress ? (
|
||||
<div className="space-y-3">
|
||||
{/* Progress bar */}
|
||||
<div className="w-full bg-background-subtle rounded-full h-2 overflow-hidden">
|
||||
<div
|
||||
className="bg-blue-500 h-2 rounded-full transition-all duration-500 ease-out"
|
||||
style={{ width: `${downloadProgress.progress_percent}%` }}
|
||||
/>
|
||||
</div>
|
||||
|
||||
{/* Stats row */}
|
||||
<div className="flex justify-between text-xs text-text-muted">
|
||||
<span>
|
||||
{formatBytes(downloadProgress.bytes_downloaded)} of{' '}
|
||||
{formatBytes(downloadProgress.total_bytes)}
|
||||
</span>
|
||||
<span>{downloadProgress.progress_percent.toFixed(0)}%</span>
|
||||
</div>
|
||||
|
||||
<div className="flex justify-between text-xs text-text-muted">
|
||||
{downloadProgress.speed_bps ? (
|
||||
<span>{formatBytes(downloadProgress.speed_bps)}/s</span>
|
||||
) : (
|
||||
<span />
|
||||
)}
|
||||
{downloadProgress.eta_seconds != null && downloadProgress.eta_seconds > 0 && (
|
||||
<span>
|
||||
~{downloadProgress.eta_seconds < 60
|
||||
? `${Math.round(downloadProgress.eta_seconds)}s`
|
||||
: `${Math.round(downloadProgress.eta_seconds / 60)}m`}{' '}
|
||||
remaining
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
) : (
|
||||
<div className="flex items-center gap-3">
|
||||
<div className="animate-spin rounded-full h-4 w-4 border-t-2 border-b-2 border-text-muted"></div>
|
||||
<span className="text-sm text-text-muted">Starting download...</span>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<button
|
||||
onClick={handleCancel}
|
||||
className="w-full px-6 py-3 bg-transparent text-text-muted rounded-lg hover:bg-background-muted transition-colors border border-border-subtle"
|
||||
>
|
||||
Cancel Download
|
||||
</button>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -15,7 +15,7 @@
|
||||
* - "standalone" — Goose-specific mode for dedicated Electron windows
|
||||
*/
|
||||
|
||||
import { AppRenderer, type RequestHandlerExtra } from '@mcp-ui/client';
|
||||
import { AppRenderer } from '@mcp-ui/client';
|
||||
import type {
|
||||
McpUiDisplayMode,
|
||||
McpUiHostContext,
|
||||
@@ -23,7 +23,7 @@ import type {
|
||||
McpUiResourcePermissions,
|
||||
McpUiSizeChangedNotification,
|
||||
} from '@modelcontextprotocol/ext-apps/app-bridge';
|
||||
import type { CallToolResult, JSONRPCRequest } from '@modelcontextprotocol/sdk/types.js';
|
||||
import type { CallToolResult } from '@modelcontextprotocol/sdk/types.js';
|
||||
import { useCallback, useEffect, useMemo, useReducer, useRef, useState } from 'react';
|
||||
import { callTool, readResource } from '../../api';
|
||||
import { AppEvents } from '../../constants/events';
|
||||
@@ -40,8 +40,6 @@ import {
|
||||
McpAppToolInputPartial,
|
||||
McpAppToolResult,
|
||||
DimensionLayout,
|
||||
SamplingCreateMessageParams,
|
||||
SamplingCreateMessageResponse,
|
||||
} from './types';
|
||||
|
||||
const DEFAULT_IFRAME_HEIGHT = 200;
|
||||
@@ -267,13 +265,7 @@ export default function McpAppRenderer({
|
||||
const containerRef = useRef<HTMLDivElement>(null);
|
||||
const [containerWidth, setContainerWidth] = useState<number>(0);
|
||||
const [containerHeight, setContainerHeight] = useState<number>(0);
|
||||
const [apiHost, setApiHost] = useState<string | null>(null);
|
||||
const [secretKey, setSecretKey] = useState<string | null>(null);
|
||||
|
||||
useEffect(() => {
|
||||
window.electron.getGoosedHostPort().then(setApiHost);
|
||||
window.electron.getSecretKey().then(setSecretKey);
|
||||
}, []);
|
||||
|
||||
// Fetch the resource from the extension to get HTML and metadata (CSP, permissions, etc.).
|
||||
// If cachedHtml is provided we show it immediately; the fetch updates metadata and
|
||||
@@ -535,42 +527,6 @@ export default function McpAppRenderer({
|
||||
return () => observer.disconnect();
|
||||
}, []);
|
||||
|
||||
const handleFallbackRequest = useCallback(
|
||||
async (request: JSONRPCRequest, _extra: RequestHandlerExtra) => {
|
||||
if (request.method === 'sampling/createMessage') {
|
||||
if (!sessionId || !apiHost || !secretKey) {
|
||||
throw new Error('Session not initialized for sampling request');
|
||||
}
|
||||
const { messages, systemPrompt, maxTokens } =
|
||||
request.params as unknown as SamplingCreateMessageParams;
|
||||
const response = await fetch(`${apiHost}/sessions/${sessionId}/sampling/message`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'X-Secret-Key': secretKey,
|
||||
},
|
||||
body: JSON.stringify({
|
||||
messages: messages.map((m) => ({
|
||||
role: m.role,
|
||||
content: m.content,
|
||||
})),
|
||||
systemPrompt,
|
||||
maxTokens,
|
||||
}),
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw new Error(`Sampling request failed: ${response.statusText}`);
|
||||
}
|
||||
return (await response.json()) as SamplingCreateMessageResponse;
|
||||
}
|
||||
return {
|
||||
status: 'error' as const,
|
||||
message: `Unhandled JSON-RPC method: ${request.method ?? '<unknown>'}`,
|
||||
};
|
||||
},
|
||||
[sessionId, apiHost, secretKey]
|
||||
);
|
||||
|
||||
const handleError = useCallback((err: Error) => {
|
||||
console.error('[MCP App Error]:', err);
|
||||
dispatch({ type: 'ERROR', message: errorMessage(err) });
|
||||
@@ -687,7 +643,6 @@ export default function McpAppRenderer({
|
||||
onReadResource={handleReadResource}
|
||||
onLoggingMessage={handleLoggingMessage}
|
||||
onSizeChanged={handleSizeChanged}
|
||||
onFallbackRequest={handleFallbackRequest}
|
||||
onError={handleError}
|
||||
/>
|
||||
);
|
||||
|
||||
@@ -8,6 +8,7 @@ import { startChatGptCodexSetup } from '../utils/chatgptCodexSetup';
|
||||
import WelcomeGooseLogo from './WelcomeGooseLogo';
|
||||
import { toastService } from '../toasts';
|
||||
import { OllamaSetup } from './OllamaSetup';
|
||||
import { LocalModelSetup } from './LocalModelSetup';
|
||||
import ApiKeyTester from './ApiKeyTester';
|
||||
import { SwitchModelModal } from './settings/models/subcomponents/SwitchModelModal';
|
||||
import { createNavigationHandler } from '../utils/navigationUtils';
|
||||
@@ -34,6 +35,7 @@ export default function ProviderGuard({ didSelectProvider, children }: ProviderG
|
||||
const [hasProvider, setHasProvider] = useState(false);
|
||||
const [showFirstTimeSetup, setShowFirstTimeSetup] = useState(false);
|
||||
const [showOllamaSetup, setShowOllamaSetup] = useState(false);
|
||||
const [showLocalModelSetup, setShowLocalModelSetup] = useState(false);
|
||||
const [userInActiveSetup, setUserInActiveSetup] = useState(false);
|
||||
const [showSwitchModelModal, setShowSwitchModelModal] = useState(false);
|
||||
const [switchModelProvider, setSwitchModelProvider] = useState<string | null>(null);
|
||||
@@ -200,6 +202,19 @@ export default function ProviderGuard({ didSelectProvider, children }: ProviderG
|
||||
setShowOllamaSetup(false);
|
||||
};
|
||||
|
||||
const handleLocalModelComplete = () => {
|
||||
trackOnboardingCompleted('local');
|
||||
setShowLocalModelSetup(false);
|
||||
setShowFirstTimeSetup(false);
|
||||
setHasProvider(true);
|
||||
navigate('/', { replace: true });
|
||||
};
|
||||
|
||||
const handleLocalModelCancel = () => {
|
||||
trackOnboardingAbandoned('local_model_setup');
|
||||
setShowLocalModelSetup(false);
|
||||
};
|
||||
|
||||
const handleRetrySetup = (setupType: 'openrouter' | 'tetrate' | 'chatgpt_codex') => {
|
||||
if (setupType === 'openrouter') {
|
||||
setOpenRouterSetupState(null);
|
||||
@@ -285,6 +300,23 @@ export default function ProviderGuard({ didSelectProvider, children }: ProviderG
|
||||
return <OllamaSetup onSuccess={handleOllamaComplete} onCancel={handleOllamaCancel} />;
|
||||
}
|
||||
|
||||
if (showLocalModelSetup) {
|
||||
return (
|
||||
<div className="h-screen w-full bg-background-default overflow-hidden">
|
||||
<div className="h-full overflow-y-auto">
|
||||
<div className="min-h-full flex flex-col items-center justify-center p-4 py-8">
|
||||
<div className="max-w-2xl w-full mx-auto p-8">
|
||||
<LocalModelSetup
|
||||
onSuccess={handleLocalModelComplete}
|
||||
onCancel={handleLocalModelCancel}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
if (!hasProvider && showFirstTimeSetup) {
|
||||
return (
|
||||
<div className="h-screen w-full bg-background-default overflow-hidden relative">
|
||||
@@ -316,6 +348,48 @@ export default function ProviderGuard({ didSelectProvider, children }: ProviderG
|
||||
}}
|
||||
/>
|
||||
|
||||
{/* Run Locally Card */}
|
||||
<div className="relative w-full mb-4">
|
||||
<div className="absolute -top-2 -right-2 sm:-top-3 sm:-right-3 z-20">
|
||||
<span className="inline-block px-2 py-1 text-xs font-medium bg-green-600 text-white rounded-full">
|
||||
Free & Private
|
||||
</span>
|
||||
</div>
|
||||
<div
|
||||
onClick={() => {
|
||||
trackOnboardingProviderSelected('local');
|
||||
setShowLocalModelSetup(true);
|
||||
}}
|
||||
className="w-full p-4 sm:p-6 bg-transparent border rounded-xl transition-all duration-200 cursor-pointer group"
|
||||
>
|
||||
<div className="flex items-start justify-between mb-3">
|
||||
<div className="flex items-center gap-2">
|
||||
<span className="font-medium text-text-default text-sm sm:text-base">
|
||||
Run Locally
|
||||
</span>
|
||||
</div>
|
||||
<div className="text-text-muted group-hover:text-text-default transition-colors">
|
||||
<svg
|
||||
className="w-4 h-4 sm:w-5 sm:h-5"
|
||||
fill="none"
|
||||
stroke="currentColor"
|
||||
viewBox="0 0 24 24"
|
||||
>
|
||||
<path
|
||||
strokeLinecap="round"
|
||||
strokeLinejoin="round"
|
||||
strokeWidth={2}
|
||||
d="M9 5l7 7-7 7"
|
||||
/>
|
||||
</svg>
|
||||
</div>
|
||||
</div>
|
||||
<p className="text-text-muted text-sm sm:text-base">
|
||||
Download a model and run entirely on your machine. No API keys, no accounts.
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* ChatGPT Subscription Card - Full Width */}
|
||||
<div className="relative w-full mb-4">
|
||||
<div className="absolute -top-2 -right-2 sm:-top-3 sm:-right-3 z-20">
|
||||
|
||||
@@ -9,10 +9,11 @@ import ConfigSettings from './config/ConfigSettings';
|
||||
import PromptsSettingsSection from './PromptsSettingsSection';
|
||||
import { ExtensionConfig } from '../../api';
|
||||
import { MainPanelLayout } from '../Layout/MainPanelLayout';
|
||||
import { Bot, Share2, Monitor, MessageSquare, FileText, Keyboard } from 'lucide-react';
|
||||
import { Bot, Share2, Monitor, MessageSquare, FileText, Keyboard, HardDrive } from 'lucide-react';
|
||||
import { useState, useEffect, useRef } from 'react';
|
||||
import ChatSettingsSection from './chat/ChatSettingsSection';
|
||||
import KeyboardShortcutsSection from './keyboard/KeyboardShortcutsSection';
|
||||
import LocalInferenceSection from './localInference/LocalInferenceSection';
|
||||
import { CONFIGURATION_ENABLED } from '../../updates';
|
||||
import { trackSettingsTabViewed } from '../../utils/analytics';
|
||||
|
||||
@@ -54,6 +55,7 @@ export default function SettingsView({
|
||||
chat: 'chat',
|
||||
prompts: 'prompts',
|
||||
keyboard: 'keyboard',
|
||||
'local-inference': 'local-inference',
|
||||
};
|
||||
|
||||
const targetTab = sectionToTab[viewOptions.section];
|
||||
@@ -112,6 +114,14 @@ export default function SettingsView({
|
||||
<Bot className="h-4 w-4" />
|
||||
Models
|
||||
</TabsTrigger>
|
||||
<TabsTrigger
|
||||
value="local-inference"
|
||||
className="flex gap-2"
|
||||
data-testid="settings-local-inference-tab"
|
||||
>
|
||||
<HardDrive className="h-4 w-4" />
|
||||
Local Inference
|
||||
</TabsTrigger>
|
||||
<TabsTrigger value="chat" className="flex gap-2" data-testid="settings-chat-tab">
|
||||
<MessageSquare className="h-4 w-4" />
|
||||
Chat
|
||||
@@ -155,6 +165,13 @@ export default function SettingsView({
|
||||
<ModelsSection setView={setView} />
|
||||
</TabsContent>
|
||||
|
||||
<TabsContent
|
||||
value="local-inference"
|
||||
className="mt-0 focus-visible:outline-none focus-visible:ring-0"
|
||||
>
|
||||
<LocalInferenceSection />
|
||||
</TabsContent>
|
||||
|
||||
<TabsContent
|
||||
value="chat"
|
||||
className="mt-0 focus-visible:outline-none focus-visible:ring-0"
|
||||
|
||||
@@ -38,19 +38,6 @@ export const LocalModelManager = () => {
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, []);
|
||||
|
||||
// Determine if we should show all models by default (if non-recommended models are downloaded)
|
||||
useEffect(() => {
|
||||
if (models.length === 0) return;
|
||||
|
||||
const hasDownloadedNonRecommended = models.some(
|
||||
(model) => model.downloaded && !model.recommended
|
||||
);
|
||||
|
||||
if (hasDownloadedNonRecommended && !showAllModels) {
|
||||
setShowAllModels(true);
|
||||
}
|
||||
}, [models, showAllModels]);
|
||||
|
||||
const loadSelectedModel = async () => {
|
||||
try {
|
||||
const value = await read(LOCAL_WHISPER_MODEL_CONFIG_KEY, false);
|
||||
@@ -145,8 +132,14 @@ export const LocalModelManager = () => {
|
||||
}
|
||||
};
|
||||
|
||||
const displayedModels = showAllModels ? models : models.filter((m) => m.recommended);
|
||||
const hasDownloadedNonRecommended = models.some(
|
||||
(model) => model.downloaded && !model.recommended
|
||||
);
|
||||
const displayedModels = showAllModels || hasDownloadedNonRecommended
|
||||
? models
|
||||
: models.filter((m) => m.recommended);
|
||||
const hasNonRecommendedModels = models.some((m) => !m.recommended);
|
||||
const showToggleButton = hasNonRecommendedModels && !hasDownloadedNonRecommended;
|
||||
|
||||
return (
|
||||
<div className="space-y-3">
|
||||
@@ -267,7 +260,7 @@ export const LocalModelManager = () => {
|
||||
})}
|
||||
</div>
|
||||
|
||||
{hasNonRecommendedModels && (
|
||||
{showToggleButton && (
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="sm"
|
||||
@@ -282,7 +275,7 @@ export const LocalModelManager = () => {
|
||||
) : (
|
||||
<>
|
||||
<ChevronDown className="w-4 h-4 mr-1" />
|
||||
Show all models
|
||||
Show all models ({models.length - displayedModels.length} more)
|
||||
</>
|
||||
)}
|
||||
</Button>
|
||||
|
||||
@@ -0,0 +1,358 @@
|
||||
import { useState, useCallback, useRef } from 'react';
|
||||
import { Search, Download, ChevronDown, ChevronUp, Loader2, Star } from 'lucide-react';
|
||||
import { Button } from '../../ui/button';
|
||||
import {
|
||||
searchHfModels,
|
||||
getRepoFiles,
|
||||
downloadHfModel,
|
||||
type HfModelInfo,
|
||||
type HfQuantVariant,
|
||||
} from '../../../api';
|
||||
|
||||
const formatBytes = (bytes: number): string => {
|
||||
if (bytes === 0) return 'unknown';
|
||||
if (bytes < 1024) return `${bytes}B`;
|
||||
if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(0)}KB`;
|
||||
if (bytes < 1024 * 1024 * 1024) return `${(bytes / (1024 * 1024)).toFixed(0)}MB`;
|
||||
return `${(bytes / (1024 * 1024 * 1024)).toFixed(1)}GB`;
|
||||
};
|
||||
|
||||
const formatDownloads = (n: number): string => {
|
||||
if (n >= 1_000_000) return `${(n / 1_000_000).toFixed(1)}M`;
|
||||
if (n >= 1_000) return `${(n / 1_000).toFixed(1)}K`;
|
||||
return `${n}`;
|
||||
};
|
||||
|
||||
interface RepoData {
|
||||
variants: HfQuantVariant[];
|
||||
recommendedIndex: number | null;
|
||||
}
|
||||
|
||||
interface Props {
|
||||
onDownloadStarted: (modelId: string) => void;
|
||||
}
|
||||
|
||||
export const HuggingFaceModelSearch = ({ onDownloadStarted }: Props) => {
|
||||
const [query, setQuery] = useState('');
|
||||
const [results, setResults] = useState<HfModelInfo[]>([]);
|
||||
const [expandedRepo, setExpandedRepo] = useState<string | null>(null);
|
||||
const [repoData, setRepoData] = useState<Record<string, RepoData>>({});
|
||||
const [searching, setSearching] = useState(false);
|
||||
const [downloading, setDownloading] = useState<Set<string>>(new Set());
|
||||
const [loadingFiles, setLoadingFiles] = useState<Set<string>>(new Set());
|
||||
const [directSpec, setDirectSpec] = useState('');
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const debounceRef = useRef<ReturnType<typeof setTimeout> | null>(null);
|
||||
|
||||
const doSearch = useCallback(async (q: string) => {
|
||||
if (!q.trim()) {
|
||||
setResults([]);
|
||||
setError(null);
|
||||
return;
|
||||
}
|
||||
setSearching(true);
|
||||
setError(null);
|
||||
try {
|
||||
const response = await searchHfModels({
|
||||
query: { q, limit: 20 },
|
||||
});
|
||||
if (response.data) {
|
||||
// Pre-fetch variants for all results and filter out repos with no suitable quantizations
|
||||
const modelsWithVariants = await Promise.all(
|
||||
response.data.map(async (model) => {
|
||||
try {
|
||||
const [author, repo] = model.repo_id.split('/');
|
||||
const filesResponse = await getRepoFiles({ path: { author, repo } });
|
||||
if (filesResponse.data && filesResponse.data.variants.length > 0) {
|
||||
return { model, data: filesResponse.data };
|
||||
}
|
||||
} catch {
|
||||
// Skip repos we can't fetch
|
||||
}
|
||||
return null;
|
||||
})
|
||||
);
|
||||
|
||||
const validResults = modelsWithVariants.filter(Boolean) as {
|
||||
model: HfModelInfo;
|
||||
data: { variants: HfQuantVariant[]; recommended_index?: number | null };
|
||||
}[];
|
||||
|
||||
setResults(validResults.map((r) => r.model));
|
||||
setRepoData((prev) => {
|
||||
const next = { ...prev };
|
||||
for (const r of validResults) {
|
||||
next[r.model.repo_id] = {
|
||||
variants: r.data.variants,
|
||||
recommendedIndex: r.data.recommended_index ?? null,
|
||||
};
|
||||
}
|
||||
return next;
|
||||
});
|
||||
|
||||
if (validResults.length === 0) {
|
||||
setError('No GGUF models found for this query.');
|
||||
}
|
||||
} else {
|
||||
console.error('Search response:', response);
|
||||
const errMsg = response.error
|
||||
? `Search error: ${JSON.stringify(response.error)}`
|
||||
: 'Search returned no data.';
|
||||
setError(errMsg);
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('Search failed:', e);
|
||||
setError('Search failed. Please try again.');
|
||||
} finally {
|
||||
setSearching(false);
|
||||
}
|
||||
}, []);
|
||||
|
||||
const handleQueryChange = (value: string) => {
|
||||
setQuery(value);
|
||||
if (debounceRef.current) clearTimeout(debounceRef.current);
|
||||
debounceRef.current = setTimeout(() => doSearch(value), 300);
|
||||
};
|
||||
|
||||
const toggleRepo = async (repoId: string) => {
|
||||
if (expandedRepo === repoId) {
|
||||
setExpandedRepo(null);
|
||||
return;
|
||||
}
|
||||
setExpandedRepo(repoId);
|
||||
|
||||
if (!repoData[repoId]?.variants.length) {
|
||||
setLoadingFiles((prev) => new Set(prev).add(repoId));
|
||||
try {
|
||||
const [author, repo] = repoId.split('/');
|
||||
const response = await getRepoFiles({
|
||||
path: { author, repo },
|
||||
});
|
||||
if (response.data) {
|
||||
const variants = response.data.variants;
|
||||
setRepoData((prev) => ({
|
||||
...prev,
|
||||
[repoId]: {
|
||||
variants,
|
||||
recommendedIndex: response.data!.recommended_index ?? null,
|
||||
},
|
||||
}));
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('Failed to fetch repo files:', e);
|
||||
} finally {
|
||||
setLoadingFiles((prev) => {
|
||||
const next = new Set(prev);
|
||||
next.delete(repoId);
|
||||
return next;
|
||||
});
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const startDownload = async (repoId: string, quantization: string) => {
|
||||
const spec = `${repoId}:${quantization}`;
|
||||
setDownloading((prev) => new Set(prev).add(spec));
|
||||
try {
|
||||
const response = await downloadHfModel({
|
||||
body: { spec },
|
||||
});
|
||||
if (response.data) {
|
||||
onDownloadStarted(response.data);
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('Download failed:', e);
|
||||
} finally {
|
||||
setDownloading((prev) => {
|
||||
const next = new Set(prev);
|
||||
next.delete(spec);
|
||||
return next;
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
const startDirectDownload = async () => {
|
||||
const spec = directSpec.trim();
|
||||
if (!spec) return;
|
||||
const key = `direct:${spec}`;
|
||||
setDownloading((prev) => new Set(prev).add(key));
|
||||
try {
|
||||
const response = await downloadHfModel({
|
||||
body: { spec },
|
||||
});
|
||||
if (response.data) {
|
||||
onDownloadStarted(response.data);
|
||||
setDirectSpec('');
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('Direct download failed:', e);
|
||||
} finally {
|
||||
setDownloading((prev) => {
|
||||
const next = new Set(prev);
|
||||
next.delete(key);
|
||||
return next;
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="space-y-4">
|
||||
<div>
|
||||
<h4 className="text-sm font-medium text-text-default mb-2">Search HuggingFace</h4>
|
||||
<div className="relative">
|
||||
<Search className="absolute left-3 top-1/2 -translate-y-1/2 w-4 h-4 text-text-muted" />
|
||||
<input
|
||||
type="text"
|
||||
value={query}
|
||||
onChange={(e) => handleQueryChange(e.target.value)}
|
||||
placeholder="Search for GGUF models..."
|
||||
className="w-full pl-9 pr-4 py-2 text-sm border border-border-subtle rounded-lg bg-background-default text-text-default placeholder:text-text-muted focus:outline-none focus:border-accent-primary"
|
||||
/>
|
||||
{searching && (
|
||||
<Loader2 className="absolute right-3 top-1/2 -translate-y-1/2 w-4 h-4 text-text-muted animate-spin" />
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{error && !searching && (
|
||||
<p className="text-xs text-text-muted">{error}</p>
|
||||
)}
|
||||
|
||||
{results.length > 0 && (
|
||||
<div className="space-y-1 max-h-96 overflow-y-auto">
|
||||
{results.map((model) => {
|
||||
const isExpanded = expandedRepo === model.repo_id;
|
||||
const data = repoData[model.repo_id];
|
||||
const variants = data?.variants || [];
|
||||
const recommendedIndex = data?.recommendedIndex ?? null;
|
||||
|
||||
return (
|
||||
<div key={model.repo_id} className="border border-border-subtle rounded-lg">
|
||||
<button
|
||||
onClick={() => toggleRepo(model.repo_id)}
|
||||
className="w-full flex items-center justify-between p-3 text-left hover:bg-background-subtle rounded-lg"
|
||||
>
|
||||
<div className="flex-1 min-w-0">
|
||||
<div className="flex items-center gap-2">
|
||||
<span className="text-sm font-medium text-text-default truncate">
|
||||
{model.repo_id}
|
||||
</span>
|
||||
</div>
|
||||
<div className="flex items-center gap-3 mt-0.5">
|
||||
<span className="text-xs text-text-muted">
|
||||
↓ {formatDownloads(model.downloads)}
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
{isExpanded ? (
|
||||
<ChevronUp className="w-4 h-4 text-text-muted flex-shrink-0" />
|
||||
) : (
|
||||
<ChevronDown className="w-4 h-4 text-text-muted flex-shrink-0" />
|
||||
)}
|
||||
</button>
|
||||
|
||||
{isExpanded && (
|
||||
<div className="border-t border-border-subtle px-3 pb-3 space-y-1">
|
||||
{loadingFiles.has(model.repo_id) && (
|
||||
<div className="flex items-center gap-2 py-2 text-xs text-text-muted">
|
||||
<Loader2 className="w-3 h-3 animate-spin" />
|
||||
Loading variants...
|
||||
</div>
|
||||
)}
|
||||
{variants.map((variant, idx) => {
|
||||
const dlKey = `${model.repo_id}:${variant.quantization}`;
|
||||
const isStarting = downloading.has(dlKey);
|
||||
const isRecommended = idx === recommendedIndex;
|
||||
|
||||
return (
|
||||
<div
|
||||
key={variant.quantization}
|
||||
className={`flex items-center justify-between py-2 px-2 rounded ${
|
||||
isRecommended
|
||||
? 'bg-blue-500/5 border border-blue-500/20'
|
||||
: 'hover:bg-background-subtle'
|
||||
}`}
|
||||
>
|
||||
<div className="flex flex-col gap-0.5 min-w-0 flex-1 mr-3">
|
||||
<div className="flex items-center gap-2">
|
||||
<span className="text-xs font-mono font-medium text-text-default">
|
||||
{variant.quantization}
|
||||
</span>
|
||||
<span className="text-xs text-text-muted">
|
||||
{formatBytes(variant.size_bytes)}
|
||||
</span>
|
||||
{isRecommended && (
|
||||
<span className="inline-flex items-center gap-1 text-xs bg-blue-500 text-white px-1.5 py-0.5 rounded">
|
||||
<Star className="w-3 h-3" />
|
||||
Recommended
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
{variant.description && (
|
||||
<span className="text-xs text-text-muted">
|
||||
{variant.description}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
<Button
|
||||
variant="outline"
|
||||
size="sm"
|
||||
disabled={isStarting}
|
||||
onClick={() => startDownload(model.repo_id, variant.quantization)}
|
||||
>
|
||||
{isStarting ? (
|
||||
<Loader2 className="w-3 h-3 animate-spin" />
|
||||
) : (
|
||||
<>
|
||||
<Download className="w-3 h-3 mr-1" />
|
||||
Download
|
||||
</>
|
||||
)}
|
||||
</Button>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div>
|
||||
<h4 className="text-sm font-medium text-text-default mb-2">Direct Download</h4>
|
||||
<p className="text-xs text-text-muted mb-2">
|
||||
Specify a model directly: <code className="bg-background-subtle px-1 rounded">user/repo:quantization</code>
|
||||
</p>
|
||||
<div className="flex gap-2">
|
||||
<input
|
||||
type="text"
|
||||
value={directSpec}
|
||||
onChange={(e) => setDirectSpec(e.target.value)}
|
||||
placeholder="bartowski/Llama-3.2-1B-Instruct-GGUF:Q4_K_M"
|
||||
className="flex-1 px-3 py-2 text-sm border border-border-subtle rounded-lg bg-background-default text-text-default placeholder:text-text-muted focus:outline-none focus:border-accent-primary"
|
||||
onKeyDown={(e) => {
|
||||
if (e.key === 'Enter') startDirectDownload();
|
||||
}}
|
||||
/>
|
||||
<Button
|
||||
variant="outline"
|
||||
size="sm"
|
||||
disabled={!directSpec.trim() || downloading.has(`direct:${directSpec}`)}
|
||||
onClick={startDirectDownload}
|
||||
>
|
||||
{downloading.has(`direct:${directSpec}`) ? (
|
||||
<Loader2 className="w-4 h-4 animate-spin" />
|
||||
) : (
|
||||
<>
|
||||
<Download className="w-4 h-4 mr-1" />
|
||||
Download
|
||||
</>
|
||||
)}
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,9 @@
|
||||
import { LocalInferenceSettings } from './LocalInferenceSettings';
|
||||
|
||||
export default function LocalInferenceSection() {
|
||||
return (
|
||||
<section id="local-inference" className="space-y-4 pr-4 pb-8">
|
||||
<LocalInferenceSettings />
|
||||
</section>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,410 @@
|
||||
import { useState, useEffect, useCallback, useRef } from 'react';
|
||||
import { Download, Trash2, X, ChevronDown, ChevronUp, Settings2 } from 'lucide-react';
|
||||
import { Button } from '../../ui/button';
|
||||
import { useModelAndProvider } from '../../ModelAndProviderContext';
|
||||
import {
|
||||
listLocalModels,
|
||||
downloadHfModel,
|
||||
getLocalModelDownloadProgress,
|
||||
cancelLocalModelDownload,
|
||||
deleteLocalModel,
|
||||
setConfigProvider,
|
||||
type DownloadProgress,
|
||||
type LocalModelResponse,
|
||||
} from '../../../api';
|
||||
import { HuggingFaceModelSearch } from './HuggingFaceModelSearch';
|
||||
import { ModelSettingsPanel } from './ModelSettingsPanel';
|
||||
import {
|
||||
Dialog,
|
||||
DialogContent,
|
||||
DialogHeader,
|
||||
DialogTitle,
|
||||
} from '../../ui/dialog';
|
||||
|
||||
const formatBytes = (bytes: number): string => {
|
||||
if (bytes < 1024) return `${bytes}B`;
|
||||
if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(0)}KB`;
|
||||
if (bytes < 1024 * 1024 * 1024) return `${(bytes / (1024 * 1024)).toFixed(0)}MB`;
|
||||
return `${(bytes / (1024 * 1024 * 1024)).toFixed(1)}GB`;
|
||||
};
|
||||
|
||||
export const LocalInferenceSettings = () => {
|
||||
const [models, setModels] = useState<LocalModelResponse[]>([]);
|
||||
const [downloads, setDownloads] = useState<Map<string, DownloadProgress>>(new Map());
|
||||
const [showAllFeatured, setShowAllFeatured] = useState(false);
|
||||
const [settingsOpenFor, setSettingsOpenFor] = useState<string | null>(null);
|
||||
const { currentModel, currentProvider, setProviderAndModel } = useModelAndProvider();
|
||||
const downloadSectionRef = useRef<HTMLDivElement>(null);
|
||||
const selectedModelId = currentProvider === 'local' ? currentModel : null;
|
||||
|
||||
const getDisplayName = useCallback(
|
||||
(modelId: string): string => {
|
||||
const model = models.find((m) => m.id === modelId);
|
||||
return model?.display_name || modelId;
|
||||
},
|
||||
[models]
|
||||
);
|
||||
|
||||
const loadModels = useCallback(async () => {
|
||||
try {
|
||||
const response = await listLocalModels();
|
||||
if (response.data) {
|
||||
setModels(response.data);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Failed to load models:', error);
|
||||
}
|
||||
}, []);
|
||||
|
||||
// Check for any in-progress downloads when models list changes
|
||||
const detectActiveDownloads = useCallback(async () => {
|
||||
for (const model of models) {
|
||||
if (downloads.has(model.id)) continue;
|
||||
// Check models that the API reports as downloading
|
||||
if (model.status.state === 'Downloading') {
|
||||
pollDownloadProgress(model.id);
|
||||
}
|
||||
}
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [models, downloads]);
|
||||
|
||||
useEffect(() => {
|
||||
loadModels();
|
||||
}, [loadModels]);
|
||||
|
||||
useEffect(() => {
|
||||
if (models.length > 0) {
|
||||
detectActiveDownloads();
|
||||
}
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [models]);
|
||||
|
||||
const selectModel = async (modelId: string) => {
|
||||
setProviderAndModel('local', modelId);
|
||||
try {
|
||||
await setConfigProvider({
|
||||
body: { provider: 'local', model: modelId },
|
||||
throwOnError: true,
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('Failed to select model:', error);
|
||||
}
|
||||
};
|
||||
|
||||
const startFeaturedDownload = async (modelId: string) => {
|
||||
const model = models.find((m) => m.id === modelId);
|
||||
if (!model) return;
|
||||
try {
|
||||
await downloadHfModel({ body: { spec: model.id } });
|
||||
pollDownloadProgress(modelId);
|
||||
scrollToDownloads();
|
||||
} catch (error) {
|
||||
console.error('Failed to start download:', error);
|
||||
}
|
||||
};
|
||||
|
||||
const scrollToDownloads = useCallback(() => {
|
||||
requestAnimationFrame(() => {
|
||||
downloadSectionRef.current?.scrollIntoView({ behavior: 'smooth', block: 'nearest' });
|
||||
});
|
||||
}, []);
|
||||
|
||||
const pollDownloadProgress = (modelId: string) => {
|
||||
const interval = setInterval(async () => {
|
||||
try {
|
||||
const response = await getLocalModelDownloadProgress({ path: { model_id: modelId } });
|
||||
if (response.data) {
|
||||
const progress = response.data;
|
||||
setDownloads((prev) => new Map(prev).set(modelId, progress));
|
||||
|
||||
if (progress.status === 'completed') {
|
||||
clearInterval(interval);
|
||||
setDownloads((prev) => {
|
||||
const next = new Map(prev);
|
||||
next.delete(modelId);
|
||||
return next;
|
||||
});
|
||||
await loadModels();
|
||||
await selectModel(modelId);
|
||||
} else if (progress.status === 'failed') {
|
||||
clearInterval(interval);
|
||||
await loadModels();
|
||||
}
|
||||
} else {
|
||||
clearInterval(interval);
|
||||
}
|
||||
} catch {
|
||||
clearInterval(interval);
|
||||
}
|
||||
}, 1000);
|
||||
};
|
||||
|
||||
const cancelDownload = async (modelId: string) => {
|
||||
try {
|
||||
await cancelLocalModelDownload({ path: { model_id: modelId } });
|
||||
setDownloads((prev) => {
|
||||
const next = new Map(prev);
|
||||
next.delete(modelId);
|
||||
return next;
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('Failed to cancel download:', error);
|
||||
}
|
||||
};
|
||||
|
||||
const handleDeleteModel = async (modelId: string) => {
|
||||
if (!window.confirm('Delete this model? You can re-download it later.')) return;
|
||||
try {
|
||||
await deleteLocalModel({ path: { model_id: modelId } });
|
||||
await loadModels();
|
||||
} catch (error) {
|
||||
console.error('Failed to delete model:', error);
|
||||
}
|
||||
};
|
||||
|
||||
const handleHfDownloadStarted = (modelId: string) => {
|
||||
pollDownloadProgress(modelId);
|
||||
loadModels();
|
||||
scrollToDownloads();
|
||||
};
|
||||
|
||||
const isDownloaded = (model: LocalModelResponse) => model.status.state === 'Downloaded';
|
||||
const isNotDownloaded = (model: LocalModelResponse) =>
|
||||
model.status.state === 'NotDownloaded' && !downloads.has(model.id);
|
||||
|
||||
const downloadedModels = models.filter(isDownloaded);
|
||||
const notDownloadedModels = models.filter(isNotDownloaded);
|
||||
const recommendedModels = notDownloadedModels.filter((m) => m.recommended);
|
||||
const displayedFeatured = showAllFeatured ? notDownloadedModels : recommendedModels;
|
||||
const showFeaturedToggle = notDownloadedModels.length > recommendedModels.length;
|
||||
|
||||
return (
|
||||
<div className="space-y-6">
|
||||
<div>
|
||||
<h3 className="text-text-default font-medium">Local Inference Models</h3>
|
||||
<p className="text-xs text-text-muted max-w-2xl mt-1">
|
||||
Download and manage local LLM models for inference without API keys. Search HuggingFace
|
||||
for any GGUF model or use the featured picks below.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
{/* Active Downloads */}
|
||||
{downloads.size > 0 && (
|
||||
<div ref={downloadSectionRef}>
|
||||
<h4 className="text-sm font-medium text-text-default mb-2">Downloading</h4>
|
||||
<div className="space-y-2">
|
||||
{Array.from(downloads.entries()).map(([modelId, progress]) => {
|
||||
if (progress.status === 'completed') return null;
|
||||
const displayName = getDisplayName(modelId);
|
||||
return (
|
||||
<div
|
||||
key={modelId}
|
||||
className="border rounded-lg p-3 border-border-subtle bg-background-default"
|
||||
>
|
||||
<div className="flex items-center justify-between mb-2">
|
||||
<span className="text-sm font-medium text-text-default truncate">
|
||||
{displayName}
|
||||
</span>
|
||||
{progress.status === 'downloading' && (
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="sm"
|
||||
onClick={() => cancelDownload(modelId)}
|
||||
className="text-destructive hover:text-destructive"
|
||||
>
|
||||
<X className="w-4 h-4" />
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
{progress.status === 'downloading' && (
|
||||
<div className="space-y-1">
|
||||
<div className="w-full bg-background-subtle rounded-full h-2">
|
||||
<div
|
||||
className="bg-accent-primary h-2 rounded-full transition-all duration-300"
|
||||
style={{ width: `${progress.progress_percent}%` }}
|
||||
/>
|
||||
</div>
|
||||
<div className="flex justify-between text-xs text-text-muted">
|
||||
<span>
|
||||
{formatBytes(progress.bytes_downloaded)} /{' '}
|
||||
{formatBytes(progress.total_bytes)} (
|
||||
{progress.progress_percent.toFixed(0)}%)
|
||||
</span>
|
||||
<span className="flex gap-2">
|
||||
{progress.eta_seconds != null && progress.eta_seconds > 0 && (
|
||||
<span>
|
||||
{progress.eta_seconds < 60
|
||||
? `${Math.round(progress.eta_seconds)}s`
|
||||
: `${Math.round(progress.eta_seconds / 60)}m`}{' '}
|
||||
remaining
|
||||
</span>
|
||||
)}
|
||||
{progress.speed_bps != null && progress.speed_bps > 0 && (
|
||||
<span>{formatBytes(progress.speed_bps)}/s</span>
|
||||
)}
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
{progress.status === 'failed' && (
|
||||
<p className="text-xs text-destructive">
|
||||
{progress.error || 'Download failed'}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Downloaded Models */}
|
||||
{downloadedModels.length > 0 && (
|
||||
<div>
|
||||
<h4 className="text-sm font-medium text-text-default mb-2">Downloaded Models</h4>
|
||||
<div className="space-y-2">
|
||||
{downloadedModels.map((model) => {
|
||||
const isSelected = selectedModelId === model.id;
|
||||
return (
|
||||
<div
|
||||
key={model.id}
|
||||
className={`border rounded-lg p-3 transition-colors ${
|
||||
isSelected
|
||||
? 'border-accent-primary bg-accent-primary/5'
|
||||
: 'border-border-subtle bg-background-default hover:border-border-default'
|
||||
}`}
|
||||
>
|
||||
<div className="flex items-center justify-between">
|
||||
<div className="flex items-center gap-2">
|
||||
<input
|
||||
type="radio"
|
||||
checked={isSelected}
|
||||
onChange={() => selectModel(model.id)}
|
||||
className="cursor-pointer"
|
||||
/>
|
||||
<span className="text-sm font-medium text-text-default">
|
||||
{model.display_name}
|
||||
</span>
|
||||
<span className="text-xs text-text-muted">
|
||||
{formatBytes(model.size_bytes)}
|
||||
</span>
|
||||
{model.recommended && (
|
||||
<span className="text-xs bg-blue-500 text-white px-2 py-0.5 rounded">
|
||||
Recommended
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
<div className="flex items-center gap-1">
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="sm"
|
||||
onClick={() => setSettingsOpenFor(model.id)}
|
||||
title="Model settings"
|
||||
>
|
||||
<Settings2 className="w-4 h-4" />
|
||||
</Button>
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="sm"
|
||||
onClick={() => handleDeleteModel(model.id)}
|
||||
className="text-destructive hover:text-destructive"
|
||||
>
|
||||
<Trash2 className="w-4 h-4" />
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Featured Models (not yet downloaded) */}
|
||||
{displayedFeatured.length > 0 && (
|
||||
<div>
|
||||
<h4 className="text-sm font-medium text-text-default mb-2">Featured Models</h4>
|
||||
<div className="space-y-2">
|
||||
{displayedFeatured.map((model) => (
|
||||
<div
|
||||
key={model.id}
|
||||
className="border rounded-lg p-3 border-border-subtle bg-background-default hover:border-border-default"
|
||||
>
|
||||
<div className="flex items-center justify-between gap-3">
|
||||
<div className="flex-1 min-w-0">
|
||||
<div className="flex items-center gap-2 flex-wrap">
|
||||
<h4 className="text-sm font-medium text-text-default">
|
||||
{model.display_name}
|
||||
</h4>
|
||||
<span className="text-xs text-text-muted">
|
||||
{formatBytes(model.size_bytes)}
|
||||
</span>
|
||||
{model.recommended && (
|
||||
<span className="text-xs bg-blue-500 text-white px-2 py-0.5 rounded">
|
||||
Recommended
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
<Button
|
||||
variant="outline"
|
||||
size="sm"
|
||||
onClick={() => startFeaturedDownload(model.id)}
|
||||
>
|
||||
<Download className="w-4 h-4 mr-1" />
|
||||
Download
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
|
||||
{showFeaturedToggle && (
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="sm"
|
||||
onClick={() => setShowAllFeatured(!showAllFeatured)}
|
||||
className="w-full text-text-muted hover:text-text-default mt-2"
|
||||
>
|
||||
{showAllFeatured ? (
|
||||
<>
|
||||
<ChevronUp className="w-4 h-4 mr-1" />
|
||||
Show recommended only
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
<ChevronDown className="w-4 h-4 mr-1" />
|
||||
Show all featured ({notDownloadedModels.length - displayedFeatured.length} more)
|
||||
</>
|
||||
)}
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* HuggingFace Search */}
|
||||
<div className="border-t border-border-subtle pt-4">
|
||||
<HuggingFaceModelSearch onDownloadStarted={handleHfDownloadStarted} />
|
||||
</div>
|
||||
|
||||
{models.length === 0 && (
|
||||
<div className="text-center py-6 text-text-muted text-sm">No models available</div>
|
||||
)}
|
||||
|
||||
<Dialog
|
||||
open={!!settingsOpenFor}
|
||||
onOpenChange={(open) => {
|
||||
if (!open) setSettingsOpenFor(null);
|
||||
}}
|
||||
>
|
||||
<DialogContent className="max-h-[80vh] overflow-y-auto sm:max-w-xl">
|
||||
<DialogHeader>
|
||||
<DialogTitle>Model Settings</DialogTitle>
|
||||
<p className="text-sm text-text-muted">{getDisplayName(settingsOpenFor || '')}</p>
|
||||
</DialogHeader>
|
||||
{settingsOpenFor && <ModelSettingsPanel modelId={settingsOpenFor} />}
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,415 @@
|
||||
import { useState, useEffect, useCallback } from 'react';
|
||||
import { RotateCcw } from 'lucide-react';
|
||||
import { Button } from '../../ui/button';
|
||||
import { Switch } from '../../ui/switch';
|
||||
import {
|
||||
getModelSettings,
|
||||
updateModelSettings,
|
||||
type ModelSettings,
|
||||
type SamplingConfig,
|
||||
} from '../../../api';
|
||||
|
||||
const DEFAULT_SETTINGS: ModelSettings = {
|
||||
context_size: null,
|
||||
max_output_tokens: null,
|
||||
sampling: { type: 'Temperature', temperature: 0.8, top_k: 40, top_p: 0.95, min_p: 0.05, seed: null },
|
||||
repeat_penalty: 1.0,
|
||||
repeat_last_n: 64,
|
||||
frequency_penalty: 0.0,
|
||||
presence_penalty: 0.0,
|
||||
n_batch: null,
|
||||
n_gpu_layers: null,
|
||||
use_mlock: false,
|
||||
flash_attention: null,
|
||||
n_threads: null,
|
||||
native_tool_calling: false,
|
||||
};
|
||||
|
||||
type SamplingType = SamplingConfig['type'];
|
||||
|
||||
function NumberField({
|
||||
label,
|
||||
description,
|
||||
value,
|
||||
onChange,
|
||||
placeholder,
|
||||
min,
|
||||
max,
|
||||
step,
|
||||
allowNull,
|
||||
}: {
|
||||
label: string;
|
||||
description?: string;
|
||||
value: number | null | undefined;
|
||||
onChange: (v: number | null) => void;
|
||||
placeholder?: string;
|
||||
min?: number;
|
||||
max?: number;
|
||||
step?: number;
|
||||
allowNull?: boolean;
|
||||
}) {
|
||||
return (
|
||||
<div className="flex flex-col gap-1">
|
||||
<label className="text-xs font-medium text-text-default">{label}</label>
|
||||
{description && <span className="text-xs text-text-muted">{description}</span>}
|
||||
<input
|
||||
type="number"
|
||||
className="w-full rounded border border-border-subtle bg-background-default px-2 py-1 text-sm text-text-default"
|
||||
value={value ?? ''}
|
||||
onChange={(e) => {
|
||||
const raw = e.target.value;
|
||||
if (raw === '' && allowNull) {
|
||||
onChange(null);
|
||||
} else {
|
||||
const n = step && step < 1 ? parseFloat(raw) : parseInt(raw, 10);
|
||||
if (!isNaN(n)) onChange(n);
|
||||
}
|
||||
}}
|
||||
placeholder={placeholder ?? 'Auto'}
|
||||
min={min}
|
||||
max={max}
|
||||
step={step}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function ToggleField({
|
||||
label,
|
||||
description,
|
||||
value,
|
||||
onChange,
|
||||
}: {
|
||||
label: string;
|
||||
description?: string;
|
||||
value: boolean;
|
||||
onChange: (v: boolean) => void;
|
||||
}) {
|
||||
return (
|
||||
<div className="flex items-center justify-between gap-2">
|
||||
<div>
|
||||
<div className="text-xs font-medium text-text-default">{label}</div>
|
||||
{description && <span className="text-xs text-text-muted">{description}</span>}
|
||||
</div>
|
||||
<Switch checked={value} onCheckedChange={onChange} variant="mono" />
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function SelectField<T extends string>({
|
||||
label,
|
||||
description,
|
||||
value,
|
||||
options,
|
||||
onChange,
|
||||
}: {
|
||||
label: string;
|
||||
description?: string;
|
||||
value: T;
|
||||
options: { value: T; label: string }[];
|
||||
onChange: (v: T) => void;
|
||||
}) {
|
||||
return (
|
||||
<div className="flex items-center justify-between gap-2">
|
||||
<div>
|
||||
<div className="text-xs font-medium text-text-default">{label}</div>
|
||||
{description && <span className="text-xs text-text-muted">{description}</span>}
|
||||
</div>
|
||||
<select
|
||||
value={value}
|
||||
onChange={(e) => onChange(e.target.value as T)}
|
||||
className="rounded border border-border-subtle bg-background-default px-2 py-1 text-xs text-text-default"
|
||||
>
|
||||
{options.map((opt) => (
|
||||
<option key={opt.value} value={opt.value}>
|
||||
{opt.label}
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export const ModelSettingsPanel = ({ modelId }: { modelId: string }) => {
|
||||
const [settings, setSettings] = useState<ModelSettings>(DEFAULT_SETTINGS);
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [saving, setSaving] = useState(false);
|
||||
|
||||
const load = useCallback(async () => {
|
||||
try {
|
||||
const res = await getModelSettings({ path: { model_id: modelId } });
|
||||
if (res.data) setSettings(res.data);
|
||||
} catch {
|
||||
// use defaults
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
}, [modelId]);
|
||||
|
||||
useEffect(() => {
|
||||
load();
|
||||
}, [load]);
|
||||
|
||||
const save = async (updated: ModelSettings) => {
|
||||
setSettings(updated);
|
||||
setSaving(true);
|
||||
try {
|
||||
await updateModelSettings({ path: { model_id: modelId }, body: updated });
|
||||
} catch (e) {
|
||||
console.error('Failed to save settings:', e);
|
||||
} finally {
|
||||
setSaving(false);
|
||||
}
|
||||
};
|
||||
|
||||
const resetDefaults = () => save(DEFAULT_SETTINGS);
|
||||
|
||||
const updateField = <K extends keyof ModelSettings>(key: K, value: ModelSettings[K]) => {
|
||||
save({ ...settings, [key]: value });
|
||||
};
|
||||
|
||||
const samplingType: SamplingType = settings.sampling?.type ?? 'Temperature';
|
||||
|
||||
const setSamplingType = (type: SamplingType) => {
|
||||
let sampling: SamplingConfig;
|
||||
if (type === 'Greedy') {
|
||||
sampling = { type: 'Greedy' };
|
||||
} else if (type === 'MirostatV2') {
|
||||
sampling = { type: 'MirostatV2', tau: 5.0, eta: 0.1, seed: null };
|
||||
} else {
|
||||
sampling = { type: 'Temperature', temperature: 0.8, top_k: 40, top_p: 0.95, min_p: 0.05, seed: null };
|
||||
}
|
||||
save({ ...settings, sampling });
|
||||
};
|
||||
|
||||
const updateSampling = (partial: Partial<SamplingConfig>) => {
|
||||
save({ ...settings, sampling: { ...settings.sampling!, ...partial } as SamplingConfig });
|
||||
};
|
||||
|
||||
if (loading) {
|
||||
return <div className="py-2 text-xs text-text-muted">Loading settings...</div>;
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="space-y-4">
|
||||
<div className="flex items-center justify-end">
|
||||
{saving && <span className="text-xs text-text-muted mr-auto">Saving...</span>}
|
||||
<Button variant="ghost" size="sm" onClick={resetDefaults} title="Reset to defaults">
|
||||
<RotateCcw className="w-3.5 h-3.5 mr-1" />
|
||||
<span className="text-xs">Reset</span>
|
||||
</Button>
|
||||
</div>
|
||||
|
||||
{/* Context & Generation */}
|
||||
<div className="space-y-2">
|
||||
<h5 className="text-xs font-medium text-text-default">Context & Generation</h5>
|
||||
<div className="grid grid-cols-2 gap-3">
|
||||
<NumberField
|
||||
label="Context size"
|
||||
description="Max context window (0 = model default)"
|
||||
value={settings.context_size}
|
||||
onChange={(v) => updateField('context_size', v)}
|
||||
placeholder="Auto"
|
||||
min={0}
|
||||
allowNull
|
||||
/>
|
||||
<NumberField
|
||||
label="Max output tokens"
|
||||
description="Cap on generated tokens"
|
||||
value={settings.max_output_tokens}
|
||||
onChange={(v) => updateField('max_output_tokens', v)}
|
||||
placeholder="No limit"
|
||||
min={1}
|
||||
allowNull
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Sampling */}
|
||||
<div className="space-y-2">
|
||||
<SelectField
|
||||
label="Sampling Strategy"
|
||||
value={samplingType}
|
||||
options={[
|
||||
{ value: 'Greedy' as SamplingType, label: 'Greedy' },
|
||||
{ value: 'Temperature' as SamplingType, label: 'Temperature' },
|
||||
{ value: 'MirostatV2' as SamplingType, label: 'Mirostat v2' },
|
||||
]}
|
||||
onChange={(v) => setSamplingType(v)}
|
||||
/>
|
||||
|
||||
{samplingType === 'Temperature' && settings.sampling?.type === 'Temperature' && (
|
||||
<div className="grid grid-cols-2 gap-3">
|
||||
<NumberField
|
||||
label="Temperature"
|
||||
value={settings.sampling.temperature}
|
||||
onChange={(v) => updateSampling({ temperature: v ?? 0.8 })}
|
||||
min={0}
|
||||
max={2}
|
||||
step={0.05}
|
||||
/>
|
||||
<NumberField
|
||||
label="Top K"
|
||||
value={settings.sampling.top_k}
|
||||
onChange={(v) => updateSampling({ top_k: v ?? 40 })}
|
||||
min={0}
|
||||
/>
|
||||
<NumberField
|
||||
label="Top P"
|
||||
value={settings.sampling.top_p}
|
||||
onChange={(v) => updateSampling({ top_p: v ?? 0.95 })}
|
||||
min={0}
|
||||
max={1}
|
||||
step={0.01}
|
||||
/>
|
||||
<NumberField
|
||||
label="Min P"
|
||||
value={settings.sampling.min_p}
|
||||
onChange={(v) => updateSampling({ min_p: v ?? 0.05 })}
|
||||
min={0}
|
||||
max={1}
|
||||
step={0.01}
|
||||
/>
|
||||
<NumberField
|
||||
label="Seed"
|
||||
value={settings.sampling.seed}
|
||||
onChange={(v) => updateSampling({ seed: v })}
|
||||
placeholder="Random"
|
||||
min={0}
|
||||
allowNull
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{samplingType === 'MirostatV2' && settings.sampling?.type === 'MirostatV2' && (
|
||||
<div className="grid grid-cols-2 gap-3">
|
||||
<NumberField
|
||||
label="Tau (target entropy)"
|
||||
value={settings.sampling.tau}
|
||||
onChange={(v) => updateSampling({ tau: v ?? 5.0 })}
|
||||
min={0}
|
||||
step={0.1}
|
||||
/>
|
||||
<NumberField
|
||||
label="Eta (learning rate)"
|
||||
value={settings.sampling.eta}
|
||||
onChange={(v) => updateSampling({ eta: v ?? 0.1 })}
|
||||
min={0}
|
||||
max={1}
|
||||
step={0.01}
|
||||
/>
|
||||
<NumberField
|
||||
label="Seed"
|
||||
value={settings.sampling.seed}
|
||||
onChange={(v) => updateSampling({ seed: v })}
|
||||
placeholder="Random"
|
||||
min={0}
|
||||
allowNull
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Repetition Penalty */}
|
||||
<div className="space-y-2">
|
||||
<h5 className="text-xs font-medium text-text-default">Repetition Penalty</h5>
|
||||
<div className="grid grid-cols-2 gap-3">
|
||||
<NumberField
|
||||
label="Repeat penalty"
|
||||
description="1.0 = off"
|
||||
value={settings.repeat_penalty}
|
||||
onChange={(v) => updateField('repeat_penalty', v ?? 1.0)}
|
||||
min={0}
|
||||
step={0.05}
|
||||
/>
|
||||
<NumberField
|
||||
label="Repeat window"
|
||||
description="Tokens to look back"
|
||||
value={settings.repeat_last_n}
|
||||
onChange={(v) => updateField('repeat_last_n', v ?? 64)}
|
||||
min={0}
|
||||
/>
|
||||
<NumberField
|
||||
label="Frequency penalty"
|
||||
description="0.0 = off"
|
||||
value={settings.frequency_penalty}
|
||||
onChange={(v) => updateField('frequency_penalty', v ?? 0.0)}
|
||||
min={0}
|
||||
max={2}
|
||||
step={0.05}
|
||||
/>
|
||||
<NumberField
|
||||
label="Presence penalty"
|
||||
description="0.0 = off"
|
||||
value={settings.presence_penalty}
|
||||
onChange={(v) => updateField('presence_penalty', v ?? 0.0)}
|
||||
min={0}
|
||||
max={2}
|
||||
step={0.05}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Performance */}
|
||||
<div className="space-y-2">
|
||||
<h5 className="text-xs font-medium text-text-default">Performance</h5>
|
||||
<div className="grid grid-cols-2 gap-3">
|
||||
<NumberField
|
||||
label="Batch size"
|
||||
description="Prompt processing batch"
|
||||
value={settings.n_batch}
|
||||
onChange={(v) => updateField('n_batch', v)}
|
||||
placeholder="Auto"
|
||||
min={1}
|
||||
allowNull
|
||||
/>
|
||||
<NumberField
|
||||
label="GPU layers"
|
||||
description="Layers to offload to GPU"
|
||||
value={settings.n_gpu_layers}
|
||||
onChange={(v) => updateField('n_gpu_layers', v)}
|
||||
placeholder="All"
|
||||
min={0}
|
||||
allowNull
|
||||
/>
|
||||
<NumberField
|
||||
label="Threads"
|
||||
description="CPU threads for generation"
|
||||
value={settings.n_threads}
|
||||
onChange={(v) => updateField('n_threads', v)}
|
||||
placeholder="Auto"
|
||||
min={1}
|
||||
allowNull
|
||||
/>
|
||||
</div>
|
||||
<ToggleField
|
||||
label="Lock model in RAM (mlock)"
|
||||
description="Prevent model from being swapped to disk"
|
||||
value={settings.use_mlock ?? false}
|
||||
onChange={(v) => updateField('use_mlock', v)}
|
||||
/>
|
||||
<SelectField
|
||||
label="Flash attention"
|
||||
description="Enable flash attention optimization"
|
||||
value={settings.flash_attention === null || settings.flash_attention === undefined ? 'auto' : settings.flash_attention ? 'on' : 'off'}
|
||||
options={[
|
||||
{ value: 'auto', label: 'Auto' },
|
||||
{ value: 'on', label: 'On' },
|
||||
{ value: 'off', label: 'Off' },
|
||||
]}
|
||||
onChange={(v) => updateField('flash_attention', v === 'auto' ? null : v === 'on')}
|
||||
/>
|
||||
</div>
|
||||
{/* Tool Calling */}
|
||||
<div className="space-y-2">
|
||||
<h5 className="text-xs font-medium text-text-default">Tool Calling</h5>
|
||||
<ToggleField
|
||||
label="Native tool calling"
|
||||
description="Use the model's built-in tool-call format instead of the shell-command emulator. Enable for large models that reliably support tool calling."
|
||||
value={settings.native_tool_calling ?? false}
|
||||
onChange={(v) => updateField('native_tool_calling', v)}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -1,4 +1,4 @@
|
||||
import { ProviderDetails, getProviderModels } from '../../../api';
|
||||
import { ProviderDetails, getProviderModels, listLocalModels } from '../../../api';
|
||||
import { errorMessage as getErrorMessage } from '../../../utils/conversionUtils';
|
||||
|
||||
export default interface Model {
|
||||
@@ -54,6 +54,16 @@ export async function fetchModelsForProviders(
|
||||
): Promise<ProviderModelsResult[]> {
|
||||
const modelPromises = activeProviders.map(async (p) => {
|
||||
try {
|
||||
// For local provider, use listLocalModels and filter to only downloaded models
|
||||
if (p.name === 'local') {
|
||||
const response = await listLocalModels();
|
||||
const allModels = response.data || [];
|
||||
const downloadedModels = allModels
|
||||
.filter((m) => m.status.state === 'Downloaded')
|
||||
.map((m) => m.id);
|
||||
return { provider: p, models: downloadedModels, error: null };
|
||||
}
|
||||
|
||||
const response = await getProviderModels({
|
||||
path: { name: p.name },
|
||||
throwOnError: true,
|
||||
|
||||
@@ -486,7 +486,36 @@ export const SwitchModelModal = ({
|
||||
|
||||
{provider && (
|
||||
<>
|
||||
{providerErrors[provider] ? (
|
||||
{provider === 'local' &&
|
||||
!loadingModels &&
|
||||
filteredModelOptions.flatMap((g) => g.options).filter((o) => o.value !== 'custom')
|
||||
.length === 0 ? (
|
||||
/* Show special UI for local provider when no models are downloaded */
|
||||
<div className="rounded-md bg-blue-50 dark:bg-blue-900/20 border border-blue-200 dark:border-blue-800 p-4">
|
||||
<div className="flex flex-col gap-3">
|
||||
<div>
|
||||
<h3 className="text-sm font-medium text-blue-800 dark:text-blue-200">
|
||||
Local models need to be downloaded first
|
||||
</h3>
|
||||
<div className="mt-1 text-sm text-blue-700 dark:text-blue-300">
|
||||
To use local inference, you need to download a model to your computer
|
||||
first. Go to Settings → Models to manage local models.
|
||||
</div>
|
||||
</div>
|
||||
<Button
|
||||
variant="outline"
|
||||
size="sm"
|
||||
onClick={() => {
|
||||
setView('settings');
|
||||
onClose();
|
||||
}}
|
||||
className="self-start border-blue-300 dark:border-blue-700 text-blue-700 dark:text-blue-300 hover:bg-blue-100 dark:hover:bg-blue-900/40"
|
||||
>
|
||||
Go to Settings
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
) : providerErrors[provider] ? (
|
||||
/* Show error message when provider failed to connect */
|
||||
<div className="rounded-md bg-red-50 dark:bg-red-900/20 border border-red-200 dark:border-red-800 p-3">
|
||||
<div className="flex items-start">
|
||||
|
||||
@@ -70,7 +70,7 @@ export type AnalyticsEvent =
|
||||
| {
|
||||
name: 'onboarding_provider_selected';
|
||||
properties: {
|
||||
method: 'api_key' | 'openrouter' | 'tetrate' | 'chatgpt_codex' | 'ollama' | 'other';
|
||||
method: 'api_key' | 'openrouter' | 'tetrate' | 'chatgpt_codex' | 'ollama' | 'local' | 'other';
|
||||
};
|
||||
}
|
||||
| {
|
||||
@@ -80,7 +80,7 @@ export type AnalyticsEvent =
|
||||
| { name: 'onboarding_abandoned'; properties: { step: string; duration_seconds?: number } }
|
||||
| {
|
||||
name: 'onboarding_setup_failed';
|
||||
properties: { provider: 'openrouter' | 'tetrate' | 'chatgpt_codex'; error_message?: string };
|
||||
properties: { provider: 'openrouter' | 'tetrate' | 'chatgpt_codex' | 'local'; error_message?: string };
|
||||
}
|
||||
| {
|
||||
name: 'error_occurred';
|
||||
@@ -284,7 +284,7 @@ export function trackOnboardingStarted(): void {
|
||||
}
|
||||
|
||||
export function trackOnboardingProviderSelected(
|
||||
method: 'api_key' | 'openrouter' | 'tetrate' | 'chatgpt_codex' | 'ollama' | 'other'
|
||||
method: 'api_key' | 'openrouter' | 'tetrate' | 'chatgpt_codex' | 'ollama' | 'local' | 'other'
|
||||
): void {
|
||||
trackEvent({
|
||||
name: 'onboarding_provider_selected',
|
||||
@@ -317,7 +317,7 @@ export function trackOnboardingAbandoned(step: string): void {
|
||||
}
|
||||
|
||||
export function trackOnboardingSetupFailed(
|
||||
provider: 'openrouter' | 'tetrate' | 'chatgpt_codex',
|
||||
provider: 'openrouter' | 'tetrate' | 'chatgpt_codex' | 'local',
|
||||
errorMessage?: string
|
||||
): void {
|
||||
trackEvent({
|
||||
|
||||
Reference in New Issue
Block a user