tkmind/tkmind_go
1
0
Fork 0
Code Issues Pull Requests Actions 30 Packages Projects Releases Wiki Activity

Migrate local inference model management to ACP (#10124)

Browse Source
This commit is contained in:
Jasper
2026-07-01 00:40:04 +02:00
committed by GitHub
parent 159fb4d53e
commit d1cc1439ec
26 changed files with 3517 additions and 2579 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ui/desktop/src/components/onboarding/LocalModelPicker.tsx
+42 -41
View File
@@ -1,13 +1,12 @@
import { useState, useEffect, useCallback, useRef } from 'react';
import {
listLocalModels,
syncFeaturedModels,
downloadHfModel,
getLocalModelDownloadProgress,
cancelLocalModelDownload,
type DownloadProgress,
type LocalModelResponse,
} from '../../api';
} from '../../acp/local-inference';
import { trackOnboardingSetupFailed } from '../../utils/analytics';
import { defineMessages, useIntl } from '../../i18n';
@@ -124,16 +123,15 @@ export default function LocalModelPicker({ onConfigured }: LocalModelPickerProps
useEffect(() => {
const load = async () => {
try {
await syncFeaturedModels();
const response = await listLocalModels({ throwOnError: true });
if (response.data) {
setModels(response.data);
const models = await listLocalModels();
if (models) {
setModels(models);
const alreadyDownloaded = response.data.find((m) => m.status.state === 'Downloaded');
const alreadyDownloaded = models.find((m) => m.status.state === 'Downloaded');
if (alreadyDownloaded) {
setSelectedModelId(alreadyDownloaded.id);
} else {
const recommended = response.data.find((m: LocalModelResponse) => m.recommended);
const recommended = models.find((m: LocalModelResponse) => m.recommended);
if (recommended) setSelectedModelId(recommended.id);
}
}
@@ -165,7 +163,7 @@ export default function LocalModelPicker({ onConfigured }: LocalModelPickerProps
}
try {
await downloadHfModel({ body: { spec: model.id }, throwOnError: true });
await downloadHfModel({ spec: model.id });
} catch (error) {
console.error('Failed to start download:', error);
setErrorMessage(intl.formatMessage(i18n.failedToStartDownload));
@@ -176,24 +174,27 @@ export default function LocalModelPicker({ onConfigured }: LocalModelPickerProps
pollRef.current = setInterval(async () => {
try {
const response = await getLocalModelDownloadProgress({
path: { model_id: modelId },
throwOnError: true,
});
if (response.data) {
setDownloadProgress(response.data);
if (response.data.status === 'completed') {
cleanup();
finishSetup(modelId);
} else if (response.data.status === 'failed') {
cleanup();
setErrorMessage(response.data.error || 'Download failed.');
trackOnboardingSetupFailed(LOCAL_PROVIDER, response.data.error || 'download_failed');
setPhase('error');
} else if (response.data.status === 'cancelled') {
cleanup();
setPhase('select');
}
const progress = await getLocalModelDownloadProgress(modelId);
if (!progress) {
cleanup();
setErrorMessage(intl.formatMessage(i18n.lostConnection));
trackOnboardingSetupFailed(LOCAL_PROVIDER, 'progress_missing');
setPhase('error');
return;
}
setDownloadProgress(progress);
if (progress.status === 'completed') {
cleanup();
finishSetup(modelId);
} else if (progress.status === 'failed') {
cleanup();
setErrorMessage(progress.error || 'Download failed.');
trackOnboardingSetupFailed(LOCAL_PROVIDER, progress.error || 'download_failed');
setPhase('error');
} else if (progress.status === 'cancelled') {
cleanup();
setPhase('select');
}
} catch {
cleanup();
@@ -208,7 +209,7 @@ export default function LocalModelPicker({ onConfigured }: LocalModelPickerProps
if (phase === 'downloading' && selectedModelId) {
cleanup();
try {
await cancelLocalModelDownload({ path: { model_id: selectedModelId } });
await cancelLocalModelDownload(selectedModelId);
} catch {
// best-effort
}
@@ -296,7 +297,7 @@ export default function LocalModelPicker({ onConfigured }: LocalModelPickerProps
)}
</div>
<p className="text-text-muted text-xs mt-1">
{formatSize(recommended.size_bytes)}
{formatSize(recommended.sizeBytes)}
</p>
</div>
</div>
@@ -350,7 +351,7 @@ export default function LocalModelPicker({ onConfigured }: LocalModelPickerProps
{model.id}
</span>
<span className="text-xs text-text-muted">
{formatSize(model.size_bytes)}
{formatSize(model.sizeBytes)}
</span>
{model.status.state === 'Downloaded' && (
<span className="text-xs bg-green-600 text-white px-2 py-0.5 rounded-full">
@@ -375,7 +376,7 @@ export default function LocalModelPicker({ onConfigured }: LocalModelPickerProps
{selectedModel?.status.state === 'Downloaded'
? intl.formatMessage(i18n.useModel, { modelId: selectedModel.id })
: selectedModel
? intl.formatMessage(i18n.downloadModel, { modelId: selectedModel.id, size: formatSize(selectedModel.size_bytes) })
? intl.formatMessage(i18n.downloadModel, { modelId: selectedModel.id, size: formatSize(selectedModel.sizeBytes) })
: intl.formatMessage(i18n.selectModel)}
</button>
@@ -394,30 +395,30 @@ export default function LocalModelPicker({ onConfigured }: LocalModelPickerProps
<div className="w-full bg-background-subtle rounded-full h-2 overflow-hidden">
<div
className="bg-blue-500 h-2 rounded-full transition-all duration-500 ease-out"
style={{ width: `${downloadProgress.progress_percent}%` }}
style={{ width: `${downloadProgress.progressPercent}%` }}
/>
</div>
<div className="flex justify-between text-xs text-text-muted">
<span>
{formatBytes(downloadProgress.bytes_downloaded)} of{' '}
{formatBytes(downloadProgress.total_bytes)}
{formatBytes(downloadProgress.bytesDownloaded)} of{' '}
{formatBytes(downloadProgress.totalBytes)}
</span>
<span>{downloadProgress.progress_percent.toFixed(0)}%</span>
<span>{downloadProgress.progressPercent.toFixed(0)}%</span>
</div>
<div className="flex justify-between text-xs text-text-muted">
{downloadProgress.speed_bps ? (
<span>{formatBytes(downloadProgress.speed_bps)}/s</span>
{downloadProgress.speedBps ? (
<span>{formatBytes(downloadProgress.speedBps)}/s</span>
) : (
<span />
)}
{downloadProgress.eta_seconds != null && downloadProgress.eta_seconds > 0 && (
{downloadProgress.etaSeconds != null && downloadProgress.etaSeconds > 0 && (
<span>
~
{downloadProgress.eta_seconds < 60
? `${Math.round(downloadProgress.eta_seconds)}s`
: `${Math.round(downloadProgress.eta_seconds / 60)}m`}{' '}
{downloadProgress.etaSeconds < 60
? `${Math.round(downloadProgress.etaSeconds)}s`
: `${Math.round(downloadProgress.etaSeconds / 60)}m`}{' '}
remaining
</span>
)}
ui/desktop/src/components/settings/localInference/HuggingFaceModelSearch.tsx
+67 -93
View File
@@ -17,7 +17,8 @@ import {
type DownloadModelRequest,
type HfModelInfo,
type HfModelVariant,
} from '../../../api';
type RepoVariantsResponse,
} from '../../../acp/local-inference';
import { defineMessages, useIntl } from '../../../i18n';
const i18n = defineMessages({
@@ -127,61 +128,43 @@ export const HuggingFaceModelSearch = ({
setSearching(true);
setError(null);
try {
const response = await searchHfModels({
query: { q, limit: 20 },
});
if (response.data) {
// Pre-fetch variants for all results and filter out repos with no compatible local variants
const modelsWithVariants = await Promise.all(
response.data.map(async (model) => {
try {
const [author, repo] = model.repo_id.split('/');
const filesResponse = await getRepoFiles({ path: { author, repo } });
if (filesResponse.data && filesResponse.data.variants.length > 0) {
return { model, data: filesResponse.data };
}
} catch {
// Skip repos we can't fetch
const models = await searchHfModels(q, 20);
const modelsWithVariants = await Promise.all(
models.map(async (model) => {
try {
const repoData = await getRepoFiles(model.repoId);
if (repoData.variants.length > 0) {
return { model, data: repoData };
}
return null;
})
);
const validResults = modelsWithVariants.filter(Boolean) as {
model: HfModelInfo;
data: {
variants: HfModelVariant[];
recommended_index?: number | null;
available_memory_bytes: number;
downloaded_quants: string[];
downloaded_variants: string[];
};
}[];
setResults(validResults.map((r) => r.model));
setRepoData((prev) => {
const next = { ...prev };
for (const r of validResults) {
next[r.model.repo_id] = {
variants: r.data.variants,
recommendedIndex: r.data.recommended_index ?? null,
availableMemoryBytes: r.data.available_memory_bytes,
downloadedQuants: new Set(r.data.downloaded_quants),
downloadedVariants: new Set(r.data.downloaded_variants),
};
} catch {
// Skip repos we can't fetch
}
return next;
});
return null;
})
);
if (validResults.length === 0) {
setError(intl.formatMessage(i18n.noGgufModels));
const validResults = modelsWithVariants.filter(Boolean) as {
model: HfModelInfo;
data: RepoVariantsResponse;
}[];
setResults(validResults.map((r) => r.model));
setRepoData((prev) => {
const next = { ...prev };
for (const r of validResults) {
next[r.model.repoId] = {
variants: r.data.variants,
recommendedIndex: r.data.recommendedIndex ?? null,
availableMemoryBytes: r.data.availableMemoryBytes,
downloadedQuants: new Set(r.data.downloadedQuants),
downloadedVariants: new Set(r.data.downloadedVariants),
};
}
} else {
console.error('Search response:', response);
const errMsg = response.error
? intl.formatMessage(i18n.searchError, { details: JSON.stringify(response.error) })
: intl.formatMessage(i18n.searchNoData);
setError(errMsg);
return next;
});
if (validResults.length === 0) {
setError(intl.formatMessage(i18n.noGgufModels));
}
} catch (e) {
console.error('Search failed:', e);
@@ -209,22 +192,17 @@ export const HuggingFaceModelSearch = ({
if (!repoData[repoId]?.variants.length) {
setLoadingFiles((prev) => new Set(prev).add(repoId));
try {
const [author, repo] = repoId.split('/');
const response = await getRepoFiles({
path: { author, repo },
});
if (response.data) {
setRepoData((prev) => ({
...prev,
[repoId]: {
variants: response.data!.variants,
recommendedIndex: response.data!.recommended_index ?? null,
availableMemoryBytes: response.data!.available_memory_bytes,
downloadedQuants: new Set(response.data!.downloaded_quants),
downloadedVariants: new Set(response.data!.downloaded_variants),
},
}));
}
const response = await getRepoFiles(repoId);
setRepoData((prev) => ({
...prev,
[repoId]: {
variants: response.variants,
recommendedIndex: response.recommendedIndex ?? null,
availableMemoryBytes: response.availableMemoryBytes,
downloadedQuants: new Set(response.downloadedQuants),
downloadedVariants: new Set(response.downloadedVariants),
},
}));
} catch (e) {
console.error('Failed to fetch repo files:', e);
} finally {
@@ -238,20 +216,16 @@ export const HuggingFaceModelSearch = ({
};
const startDownload = async (repoId: string, variant: HfModelVariant) => {
const downloadKey = variant.download_id;
const downloadKey = variant.downloadId;
const request: DownloadModelRequest = {
spec: repoId,
backend_id: variant.backend_id,
variant_id: variant.variant_id,
backendId: variant.backendId,
variantId: variant.variantId,
};
setDownloading((prev) => new Set(prev).add(downloadKey));
try {
const response = await downloadHfModel({
body: request,
});
if (response.data) {
onDownloadStarted(response.data, request);
}
const modelId = await downloadHfModel(request);
onDownloadStarted(modelId, request);
} catch (e) {
console.error('Download failed:', e);
} finally {
@@ -289,8 +263,8 @@ export const HuggingFaceModelSearch = ({
{results.length > 0 && (
<div className="space-y-1">
{results.map((model) => {
const isExpanded = expandedRepo === model.repo_id;
const data = repoData[model.repo_id];
const isExpanded = expandedRepo === model.repoId;
const data = repoData[model.repoId];
const variants = data?.variants || [];
const recommendedIndex = data?.recommendedIndex ?? null;
const availableMemory = data?.availableMemoryBytes ?? 0;
@@ -298,15 +272,15 @@ export const HuggingFaceModelSearch = ({
const downloadedVariants = data?.downloadedVariants ?? new Set<string>();
return (
<div key={model.repo_id} className="border border-border-subtle rounded-lg">
<div key={model.repoId} className="border border-border-subtle rounded-lg">
<button
onClick={() => toggleRepo(model.repo_id)}
onClick={() => toggleRepo(model.repoId)}
className="w-full flex items-center justify-between p-3 text-left hover:bg-background-subtle rounded-lg"
>
<div className="flex-1 min-w-0">
<div className="flex items-center gap-2">
<span className="text-sm font-medium text-text-default truncate">
{model.repo_id}
{model.repoId}
</span>
</div>
<div className="flex items-center gap-3 mt-0.5">
@@ -324,28 +298,28 @@ export const HuggingFaceModelSearch = ({
{isExpanded && (
<div className="border-t border-border-subtle px-3 pb-3 space-y-1">
{loadingFiles.has(model.repo_id) && (
{loadingFiles.has(model.repoId) && (
<div className="flex items-center gap-2 py-2 text-xs text-text-muted">
<Loader2 className="w-3 h-3 animate-spin" />
{intl.formatMessage(i18n.loadingVariants)}
</div>
)}
{variants.map((variant, idx) => {
const isStarting = downloading.has(variant.download_id);
const isStarting = downloading.has(variant.downloadId);
const isRecommended = idx === recommendedIndex;
const modelId = variant.model_id;
const modelId = variant.modelId;
const isActiveDownload = activeDownloadIds?.has(modelId) ?? false;
const isDownloaded = downloadedModelIds
? downloadedModelIds.has(modelId)
: downloadedVariants.has(modelId) ||
downloadedQuants.has(variant.variant_id);
downloadedQuants.has(variant.variantId);
const tooLarge =
availableMemory > 0 && variant.size_bytes > availableMemory * 0.85;
availableMemory > 0 && variant.sizeBytes > availableMemory * 0.85;
const isSupported = variant.supported ?? true;
return (
<div
key={variant.download_id}
key={variant.downloadId}
className={`flex items-center justify-between py-2 px-2 rounded ${
isDownloaded
? 'bg-green-500/5 border border-green-500/20'
@@ -363,7 +337,7 @@ export const HuggingFaceModelSearch = ({
{variant.label}
</span>
<span className="text-xs text-text-muted">
{formatBytes(variant.size_bytes)}
{formatBytes(variant.sizeBytes)}
</span>
{isRecommended && !isDownloaded && (
<span className="inline-flex items-center gap-1 text-xs bg-blue-500 text-white px-1.5 py-0.5 rounded">
@@ -375,17 +349,17 @@ export const HuggingFaceModelSearch = ({
{variant.description && (
<span className="text-xs text-text-muted">{variant.description}</span>
)}
{!isSupported && variant.unsupported_reason && (
{!isSupported && variant.unsupportedReason && (
<span className="inline-flex items-center gap-1 text-xs text-amber-500">
<AlertTriangle className="w-3 h-3" />
{variant.unsupported_reason}
{variant.unsupportedReason}
</span>
)}
{tooLarge && !isDownloaded && (
<span className="inline-flex items-center gap-1 text-xs text-amber-500">
<AlertTriangle className="w-3 h-3" />
{intl.formatMessage(i18n.tooLarge, {
size: formatBytes(variant.size_bytes),
size: formatBytes(variant.sizeBytes),
available: formatBytes(availableMemory),
})}
</span>
@@ -406,7 +380,7 @@ export const HuggingFaceModelSearch = ({
variant="outline"
size="sm"
disabled={isStarting || !isSupported}
onClick={() => startDownload(model.repo_id, variant)}
onClick={() => startDownload(model.repoId, variant)}
>
{isStarting ? (
<Loader2 className="w-3 h-3 animate-spin" />
ui/desktop/src/components/settings/localInference/LocalInferenceSettings.tsx
+29 -33
View File
@@ -14,7 +14,6 @@ import { useModelAndProvider } from '../../ModelAndProviderContext';
import { defineMessages, useIntl } from '../../../i18n';
import {
listLocalModels,
syncFeaturedModels,
downloadHfModel,
getLocalModelDownloadProgress,
cancelLocalModelDownload,
@@ -22,7 +21,7 @@ import {
type DownloadProgress,
type DownloadModelRequest,
type LocalModelResponse,
} from '../../../api';
} from '../../../acp/local-inference';
import { HuggingFaceModelSearch } from './HuggingFaceModelSearch';
import { ModelSettingsPanel } from './ModelSettingsPanel';
import { Dialog, DialogContent, DialogHeader, DialogTitle } from '../../ui/dialog';
@@ -133,9 +132,9 @@ const VisionBadge = ({
model: LocalModelResponse;
intl: ReturnType<typeof useIntl>;
}) => {
if (!model.vision_capable) return null;
if (!model.visionCapable) return null;
const mmproj = model.mmproj_status;
const mmproj = model.mmprojStatus;
const isDownloaded = mmproj?.state === 'Downloaded';
const isDownloading = mmproj?.state === 'Downloading';
@@ -150,7 +149,7 @@ const VisionBadge = ({
if (isDownloading) {
const percent =
mmproj && 'progress_percent' in mmproj ? Math.round(mmproj.progress_percent) : null;
mmproj && mmproj.progressPercent != null ? Math.round(mmproj.progressPercent) : null;
return (
<span className="inline-flex items-center gap-1 text-xs text-yellow-400 bg-yellow-500/10 px-2 py-0.5 rounded">
<Eye className="w-3 h-3" />
@@ -191,12 +190,11 @@ export const LocalInferenceSettings = () => {
const loadModels = useCallback(async (): Promise<LocalModelResponse[] | undefined> => {
try {
await syncFeaturedModels();
const response = await listLocalModels();
if (response.data) {
setModels(response.data);
const models = await listLocalModels();
if (models) {
setModels(models);
const downloadedIds = new Set(
response.data
models
.filter((model) => model.status.state === 'Downloaded')
.map((model) => model.id)
);
@@ -212,13 +210,13 @@ export const LocalInferenceSettings = () => {
return next;
});
}
response.data.forEach((model) => {
models.forEach((model) => {
if (model.status.state === 'Downloading') {
pollDownloadProgress(model.id);
}
});
return response.data;
return models;
}
} catch (error) {
console.error('Failed to load models:', error);
@@ -235,7 +233,7 @@ export const LocalInferenceSettings = () => {
// Poll model list while any vision encoder is downloading
useEffect(() => {
const hasDownloadingMmproj = models.some(
(m) => m.vision_capable && m.mmproj_status?.state === 'Downloading'
(m) => m.visionCapable && m.mmprojStatus?.state === 'Downloading'
);
if (!hasDownloadingMmproj) return;
@@ -259,7 +257,7 @@ export const LocalInferenceSettings = () => {
if (!model) return;
const request = { spec: model.id };
try {
await downloadHfModel({ body: request });
await downloadHfModel(request);
setDownloadRequests((prev) => new Map(prev).set(modelId, request));
pollDownloadProgress(modelId);
scrollToDownloads();
@@ -285,9 +283,8 @@ export const LocalInferenceSettings = () => {
const interval = setInterval(async () => {
try {
const response = await getLocalModelDownloadProgress({ path: { model_id: modelId } });
if (response.data) {
const progress = response.data;
const progress = await getLocalModelDownloadProgress(modelId);
if (progress) {
setDownloads((prev) => new Map(prev).set(modelId, progress));
if (progress.status === 'completed') {
@@ -314,7 +311,7 @@ export const LocalInferenceSettings = () => {
const cancelDownload = async (modelId: string) => {
try {
await cancelLocalModelDownload({ path: { model_id: modelId } });
await cancelLocalModelDownload(modelId);
setDownloads((prev) => {
const next = new Map(prev);
const progress = next.get(modelId);
@@ -346,8 +343,7 @@ export const LocalInferenceSettings = () => {
const request = downloadRequests.get(modelId) ?? { spec: modelId };
dismissDownload(modelId);
try {
const response = await downloadHfModel({ body: request });
const nextModelId = response.data ?? modelId;
const nextModelId = await downloadHfModel(request);
setDownloadRequests((prev) => new Map(prev).set(nextModelId, request));
pollDownloadProgress(nextModelId);
scrollToDownloads();
@@ -359,7 +355,7 @@ export const LocalInferenceSettings = () => {
const handleDeleteModel = async (modelId: string) => {
if (!window.confirm(intl.formatMessage(i18n.deleteConfirm))) return;
try {
await deleteLocalModel({ path: { model_id: modelId } });
await deleteLocalModel(modelId);
const updatedModels = await loadModels();
if (selectedModelId === modelId && updatedModels) {
@@ -442,30 +438,30 @@ export const LocalInferenceSettings = () => {
<div className="w-full bg-gray-700 rounded-full h-2">
<div
className="bg-blue-500 h-2 rounded-full transition-all duration-300"
style={{ width: `${progress.progress_percent}%` }}
style={{ width: `${progress.progressPercent}%` }}
/>
</div>
<div className="flex justify-between text-xs text-text-muted">
<span>
{intl.formatMessage(i18n.downloadProgress, {
downloaded: formatBytes(progress.bytes_downloaded),
total: formatBytes(progress.total_bytes),
percent: progress.progress_percent.toFixed(0),
downloaded: formatBytes(progress.bytesDownloaded),
total: formatBytes(progress.totalBytes),
percent: progress.progressPercent.toFixed(0),
})}
</span>
<span className="flex gap-2">
{progress.eta_seconds != null && progress.eta_seconds > 0 && (
{progress.etaSeconds != null && progress.etaSeconds > 0 && (
<span>
{intl.formatMessage(i18n.remaining, {
time:
progress.eta_seconds < 60
? `${Math.round(progress.eta_seconds)}s`
: `${Math.round(progress.eta_seconds / 60)}m`,
progress.etaSeconds < 60
? `${Math.round(progress.etaSeconds)}s`
: `${Math.round(progress.etaSeconds / 60)}m`,
})}
</span>
)}
{progress.speed_bps != null && progress.speed_bps > 0 && (
<span>{formatBytes(progress.speed_bps)}/s</span>
{progress.speedBps != null && progress.speedBps > 0 && (
<span>{formatBytes(progress.speedBps)}/s</span>
)}
</span>
</div>
@@ -540,7 +536,7 @@ export const LocalInferenceSettings = () => {
/>
<span className="text-sm font-medium text-text-default">{model.id}</span>
<span className="text-xs text-text-muted">
{formatBytes(model.size_bytes)}
{formatBytes(model.sizeBytes)}
</span>
{model.recommended && (
<span className="text-xs bg-blue-500 text-white px-2 py-0.5 rounded">
@@ -592,7 +588,7 @@ export const LocalInferenceSettings = () => {
<div className="flex items-center gap-2 flex-wrap">
<h4 className="text-sm font-medium text-text-default">{model.id}</h4>
<span className="text-xs text-text-muted">
{formatBytes(model.size_bytes)}
{formatBytes(model.sizeBytes)}
</span>
{model.recommended && (
<span className="text-xs bg-blue-500 text-white px-2 py-0.5 rounded">
ui/desktop/src/components/settings/localInference/ModelSettingsPanel.tsx
+75 -66
View File
@@ -10,7 +10,7 @@ import {
type ModelSettings,
type SamplingConfig,
type ToolCallingMode,
} from '../../../api';
} from '../../../acp/local-inference';
import { defineMessages, useIntl } from '../../../i18n';
const i18n = defineMessages({
@@ -221,27 +221,33 @@ const i18n = defineMessages({
});
const DEFAULT_SETTINGS: ModelSettings = {
context_size: null,
max_output_tokens: null,
backendId: null,
contextSize: null,
maxOutputTokens: null,
draftModel: null,
sampling: {
type: 'Temperature',
temperature: 0.8,
top_k: 40,
top_p: 0.95,
min_p: 0.05,
topK: 40,
topP: 0.95,
minP: 0.05,
seed: null,
},
repeat_penalty: 1.0,
repeat_last_n: 64,
frequency_penalty: 0.0,
presence_penalty: 0.0,
n_batch: null,
n_gpu_layers: null,
use_mlock: false,
flash_attention: null,
n_threads: null,
tool_calling: 'auto',
chat_template: { type: 'embedded' },
repeatPenalty: 1.0,
repeatLastN: 64,
frequencyPenalty: 0.0,
presencePenalty: 0.0,
nBatch: null,
nGpuLayers: null,
useMlock: false,
flashAttention: null,
nThreads: null,
toolCalling: 'auto',
chatTemplate: { type: 'embedded' },
enableThinking: true,
visionCapable: false,
imageTokenEstimate: 256,
mmprojSizeBytes: 0,
};
type SamplingType = SamplingConfig['type'];
@@ -390,17 +396,17 @@ export const ModelSettingsPanel = ({ modelId }: { modelId: string }) => {
const load = useCallback(async () => {
try {
const [settingsResult, builtinsResult] = await Promise.allSettled([
getModelSettings({ path: { model_id: modelId } }),
getModelSettings(modelId),
listBuiltinChatTemplates(),
]);
if (builtinsResult.status === 'fulfilled' && builtinsResult.value.data?.length) {
setBuiltinTemplateOptions(builtinsResult.value.data);
if (builtinsResult.status === 'fulfilled' && builtinsResult.value.length) {
setBuiltinTemplateOptions(builtinsResult.value);
}
if (settingsResult.status === 'fulfilled' && settingsResult.value.data) {
if (settingsResult.status === 'fulfilled') {
setSettings({
...settingsResult.value.data,
tool_calling: settingsResult.value.data.tool_calling ?? 'auto',
chat_template: settingsResult.value.data.chat_template ?? { type: 'embedded' },
...settingsResult.value,
toolCalling: settingsResult.value.toolCalling ?? 'auto',
chatTemplate: settingsResult.value.chatTemplate ?? { type: 'embedded' },
});
}
} catch {
@@ -415,7 +421,7 @@ export const ModelSettingsPanel = ({ modelId }: { modelId: string }) => {
}, [load]);
useEffect(() => {
const chatTemplate = settings.chat_template;
const chatTemplate = settings.chatTemplate;
if (chatTemplate?.type === 'custom_inline') {
setChatTemplateDraft(chatTemplate.template ?? '');
} else {
@@ -424,13 +430,13 @@ export const ModelSettingsPanel = ({ modelId }: { modelId: string }) => {
if (chatTemplate?.type === 'builtin') {
setBuiltinTemplateDraft(chatTemplate.name ?? 'chatml');
}
}, [settings.chat_template]);
}, [settings.chatTemplate]);
const save = async (updated: ModelSettings) => {
setSettings(updated);
setSaving(true);
try {
await updateModelSettings({ path: { model_id: modelId }, body: updated });
await updateModelSettings(modelId, updated);
} catch (e) {
console.error('Failed to save settings:', e);
} finally {
@@ -445,7 +451,7 @@ export const ModelSettingsPanel = ({ modelId }: { modelId: string }) => {
};
const samplingType: SamplingType = settings.sampling?.type ?? 'Temperature';
const chatTemplate = settings.chat_template ?? { type: 'embedded' };
const chatTemplate = settings.chatTemplate ?? { type: 'embedded' };
const chatTemplateMode: ChatTemplateMode =
chatTemplate.type === 'custom_inline'
? 'custom_inline'
@@ -458,23 +464,26 @@ export const ModelSettingsPanel = ({ modelId }: { modelId: string }) => {
if (mode === 'custom_inline') {
next = { type: 'custom_inline', template: chatTemplateDraft };
} else if (mode === 'builtin') {
next = { type: 'builtin', name: builtinTemplateDraft.trim() || builtinTemplateOptions[0] || 'chatml' };
next = {
type: 'builtin',
name: builtinTemplateDraft.trim() || builtinTemplateOptions[0] || 'chatml',
};
} else {
next = { type: 'embedded' };
}
updateField('chat_template', next);
updateField('chatTemplate', next);
};
const setBuiltinTemplateName = (name: string) => {
setBuiltinTemplateDraft(name);
if (chatTemplateMode === 'builtin') {
updateField('chat_template', { type: 'builtin', name });
updateField('chatTemplate', { type: 'builtin', name });
}
};
const saveChatTemplateDraft = () => {
if (chatTemplateMode === 'custom_inline') {
updateField('chat_template', { type: 'custom_inline', template: chatTemplateDraft });
updateField('chatTemplate', { type: 'custom_inline', template: chatTemplateDraft });
}
};
@@ -488,9 +497,9 @@ export const ModelSettingsPanel = ({ modelId }: { modelId: string }) => {
sampling = {
type: 'Temperature',
temperature: 0.8,
top_k: 40,
top_p: 0.95,
min_p: 0.05,
topK: 40,
topP: 0.95,
minP: 0.05,
seed: null,
};
}
@@ -526,8 +535,8 @@ export const ModelSettingsPanel = ({ modelId }: { modelId: string }) => {
<NumberField
label={intl.formatMessage(i18n.contextSize)}
description={intl.formatMessage(i18n.contextSizeDescription)}
value={settings.context_size}
onChange={(v) => updateField('context_size', v)}
value={settings.contextSize}
onChange={(v) => updateField('contextSize', v)}
placeholder="Auto"
min={0}
allowNull
@@ -535,8 +544,8 @@ export const ModelSettingsPanel = ({ modelId }: { modelId: string }) => {
<NumberField
label={intl.formatMessage(i18n.maxOutputTokens)}
description={intl.formatMessage(i18n.maxOutputTokensDescription)}
value={settings.max_output_tokens}
onChange={(v) => updateField('max_output_tokens', v)}
value={settings.maxOutputTokens}
onChange={(v) => updateField('maxOutputTokens', v)}
placeholder="No limit"
min={1}
allowNull
@@ -569,22 +578,22 @@ export const ModelSettingsPanel = ({ modelId }: { modelId: string }) => {
/>
<NumberField
label={intl.formatMessage(i18n.topK)}
value={settings.sampling.top_k}
onChange={(v) => updateSampling({ top_k: v ?? 40 })}
value={settings.sampling.topK}
onChange={(v) => updateSampling({ topK: v ?? 40 })}
min={0}
/>
<NumberField
label={intl.formatMessage(i18n.topP)}
value={settings.sampling.top_p}
onChange={(v) => updateSampling({ top_p: v ?? 0.95 })}
value={settings.sampling.topP}
onChange={(v) => updateSampling({ topP: v ?? 0.95 })}
min={0}
max={1}
step={0.01}
/>
<NumberField
label={intl.formatMessage(i18n.minP)}
value={settings.sampling.min_p}
onChange={(v) => updateSampling({ min_p: v ?? 0.05 })}
value={settings.sampling.minP}
onChange={(v) => updateSampling({ minP: v ?? 0.05 })}
min={0}
max={1}
step={0.01}
@@ -636,23 +645,23 @@ export const ModelSettingsPanel = ({ modelId }: { modelId: string }) => {
<NumberField
label={intl.formatMessage(i18n.repeatPenalty)}
description={intl.formatMessage(i18n.repeatPenaltyDescription)}
value={settings.repeat_penalty}
onChange={(v) => updateField('repeat_penalty', v ?? 1.0)}
value={settings.repeatPenalty}
onChange={(v) => updateField('repeatPenalty', v ?? 1.0)}
min={0}
step={0.05}
/>
<NumberField
label={intl.formatMessage(i18n.repeatWindow)}
description={intl.formatMessage(i18n.repeatWindowDescription)}
value={settings.repeat_last_n}
onChange={(v) => updateField('repeat_last_n', v ?? 64)}
value={settings.repeatLastN}
onChange={(v) => updateField('repeatLastN', v ?? 64)}
min={0}
/>
<NumberField
label={intl.formatMessage(i18n.frequencyPenalty)}
description={intl.formatMessage(i18n.frequencyPenaltyDescription)}
value={settings.frequency_penalty}
onChange={(v) => updateField('frequency_penalty', v ?? 0.0)}
value={settings.frequencyPenalty}
onChange={(v) => updateField('frequencyPenalty', v ?? 0.0)}
min={0}
max={2}
step={0.05}
@@ -660,8 +669,8 @@ export const ModelSettingsPanel = ({ modelId }: { modelId: string }) => {
<NumberField
label={intl.formatMessage(i18n.presencePenalty)}
description={intl.formatMessage(i18n.presencePenaltyDescription)}
value={settings.presence_penalty}
onChange={(v) => updateField('presence_penalty', v ?? 0.0)}
value={settings.presencePenalty}
onChange={(v) => updateField('presencePenalty', v ?? 0.0)}
min={0}
max={2}
step={0.05}
@@ -676,8 +685,8 @@ export const ModelSettingsPanel = ({ modelId }: { modelId: string }) => {
<NumberField
label={intl.formatMessage(i18n.batchSize)}
description={intl.formatMessage(i18n.batchSizeDescription)}
value={settings.n_batch}
onChange={(v) => updateField('n_batch', v)}
value={settings.nBatch}
onChange={(v) => updateField('nBatch', v)}
placeholder="Auto"
min={1}
allowNull
@@ -685,8 +694,8 @@ export const ModelSettingsPanel = ({ modelId }: { modelId: string }) => {
<NumberField
label={intl.formatMessage(i18n.gpuLayers)}
description={intl.formatMessage(i18n.gpuLayersDescription)}
value={settings.n_gpu_layers}
onChange={(v) => updateField('n_gpu_layers', v)}
value={settings.nGpuLayers}
onChange={(v) => updateField('nGpuLayers', v)}
placeholder="All"
min={0}
allowNull
@@ -694,8 +703,8 @@ export const ModelSettingsPanel = ({ modelId }: { modelId: string }) => {
<NumberField
label={intl.formatMessage(i18n.threads)}
description={intl.formatMessage(i18n.threadsDescription)}
value={settings.n_threads}
onChange={(v) => updateField('n_threads', v)}
value={settings.nThreads}
onChange={(v) => updateField('nThreads', v)}
placeholder="Auto"
min={1}
allowNull
@@ -704,16 +713,16 @@ export const ModelSettingsPanel = ({ modelId }: { modelId: string }) => {
<ToggleField
label={intl.formatMessage(i18n.lockModelInRam)}
description={intl.formatMessage(i18n.lockModelInRamDescription)}
value={settings.use_mlock ?? false}
onChange={(v) => updateField('use_mlock', v)}
value={settings.useMlock ?? false}
onChange={(v) => updateField('useMlock', v)}
/>
<SelectField
label={intl.formatMessage(i18n.flashAttention)}
description={intl.formatMessage(i18n.flashAttentionDescription)}
value={
settings.flash_attention === null || settings.flash_attention === undefined
settings.flashAttention === null || settings.flashAttention === undefined
? 'auto'
: settings.flash_attention
: settings.flashAttention
? 'on'
: 'off'
}
@@ -722,18 +731,18 @@ export const ModelSettingsPanel = ({ modelId }: { modelId: string }) => {
{ value: 'on', label: 'On' },
{ value: 'off', label: 'Off' },
]}
onChange={(v) => updateField('flash_attention', v === 'auto' ? null : v === 'on')}
onChange={(v) => updateField('flashAttention', v === 'auto' ? null : v === 'on')}
/>
<SelectField<ToolCallingMode>
label={intl.formatMessage(i18n.toolCalling)}
description={intl.formatMessage(i18n.toolCallingDescription)}
value={settings.tool_calling ?? 'auto'}
value={settings.toolCalling ?? 'auto'}
options={[
{ value: 'auto', label: intl.formatMessage(i18n.toolCallingAuto) },
{ value: 'force_native', label: intl.formatMessage(i18n.toolCallingForceNative) },
{ value: 'force_emulated', label: intl.formatMessage(i18n.toolCallingForceEmulated) },
]}
onChange={(v) => updateField('tool_calling', v)}
onChange={(v) => updateField('toolCalling', v)}
/>
<SelectField<ChatTemplateMode>
label={intl.formatMessage(i18n.chatTemplate)}
ui/desktop/src/components/settings/models/modelInterface.ts
+3 -3
View File
@@ -1,4 +1,5 @@
import { ProviderDetails, ThinkingEffort, listLocalModels } from '../../../api';
import { ProviderDetails, ThinkingEffort } from '../../../api';
import { listLocalModels } from '../../../acp/local-inference';
import { acpListProviderDetails, acpListProviderModels } from '../../../acp/providers';
import { errorMessage as getErrorMessage } from '../../../utils/conversionUtils';
@@ -56,8 +57,7 @@ export async function fetchModelsForProviders(
try {
// For local provider, use listLocalModels and filter to only downloaded models
if (p.name === 'local') {
const response = await listLocalModels();
const allModels = response.data || [];
const allModels = await listLocalModels();
const downloadedModels = allModels
.filter((m) => m.status.state === 'Downloaded')
.map((m) => ({ name: m.id, provider: p.name }) as Model);