tkmind/tkmind_go
1
0
Fork 0
Code Issues Pull Requests Actions 4 Packages Projects Releases Wiki Activity

Improve local inference settings and model downloader (#8467)

Browse Source 
Signed-off-by: jh-block <jhugo@block.xyz>
This commit is contained in:
jh-block
2026-04-14 10:50:50 +02:00
committed by GitHub
parent 2e529b7497
commit 65e5d97cac
10 changed files with 1009 additions and 297 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ui/desktop/openapi.json
+17 -1
View File
@@ -5522,6 +5522,9 @@
"quantization": {
"type": "string"
},
"sharded": {
"type": "boolean"
},
"size_bytes": {
"type": "integer",
"format": "int64",
@@ -7261,9 +7264,22 @@
"RepoVariantsResponse": {
"type": "object",
"required": [
"variants"
"variants",
"available_memory_bytes",
"downloaded_quants"
],
"properties": {
"available_memory_bytes": {
"type": "integer",
"format": "int64",
"minimum": 0
},
"downloaded_quants": {
"type": "array",
"items": {
"type": "string"
}
},
"recommended_index": {
"type": "integer",
"nullable": true,
ui/desktop/src/api/types.gen.ts
+3
View File
@@ -541,6 +541,7 @@ export type HfQuantVariant = {
filename: string;
quality_rank: number;
quantization: string;
sharded?: boolean;
size_bytes: number;
};
@@ -1095,6 +1096,8 @@ export type RemoveExtensionRequest = {
};
export type RepoVariantsResponse = {
available_memory_bytes: number;
downloaded_quants: Array<string>;
recommended_index?: number | null;
variants: Array<HfQuantVariant>;
};
ui/desktop/src/components/settings/localInference/HuggingFaceModelSearch.tsx
+84 -104
View File
@@ -1,5 +1,5 @@
import { useState, useCallback, useRef } from 'react';
import { Search, Download, ChevronDown, ChevronUp, Loader2, Star } from 'lucide-react';
import { Search, Download, ChevronDown, ChevronUp, Loader2, Star, Check, AlertTriangle } from 'lucide-react';
import { Button } from '../../ui/button';
import {
searchHfModels,
@@ -8,8 +8,6 @@ import {
type HfModelInfo,
type HfQuantVariant,
} from '../../../api';
import { toastError } from '../../../toasts';
import { errorMessage } from '../../../utils/conversionUtils';
import { defineMessages, useIntl } from '../../../i18n';
const i18n = defineMessages({
@@ -33,21 +31,17 @@ const i18n = defineMessages({
id: 'huggingFaceModelSearch.download',
defaultMessage: 'Download',
},
directDownload: {
id: 'huggingFaceModelSearch.directDownload',
defaultMessage: 'Direct Download',
downloaded: {
id: 'huggingFaceModelSearch.downloaded',
defaultMessage: 'Downloaded',
},
directDownloadDescription: {
id: 'huggingFaceModelSearch.directDownloadDescription',
defaultMessage: 'Specify a model directly: {format}',
downloading: {
id: 'huggingFaceModelSearch.downloading',
defaultMessage: 'Downloading…',
},
directDownloadFailed: {
id: 'huggingFaceModelSearch.directDownloadFailed',
defaultMessage: 'Direct download failed',
},
directDownloadErrorMsg: {
id: 'huggingFaceModelSearch.directDownloadErrorMsg',
defaultMessage: 'Failed to start the download. Check the spec: {error}',
tooLarge: {
id: 'huggingFaceModelSearch.tooLarge',
defaultMessage: 'May not fit in memory ({size} model, {available} available)',
},
noGgufModels: {
id: 'huggingFaceModelSearch.noGgufModels',
@@ -84,13 +78,19 @@ const formatDownloads = (n: number): string => {
interface RepoData {
variants: HfQuantVariant[];
recommendedIndex: number | null;
availableMemoryBytes: number;
downloadedQuants: Set<string>;
}
interface Props {
onDownloadStarted: (modelId: string) => void;
/** Model IDs (repo:quant) with an active download in progress */
activeDownloadIds?: Set<string>;
/** Model IDs (repo:quant) confirmed downloaded on disk */
downloadedModelIds?: Set<string>;
}
export const HuggingFaceModelSearch = ({ onDownloadStarted }: Props) => {
export const HuggingFaceModelSearch = ({ onDownloadStarted, activeDownloadIds, downloadedModelIds }: Props) => {
const intl = useIntl();
const [query, setQuery] = useState('');
const [results, setResults] = useState<HfModelInfo[]>([]);
@@ -99,7 +99,6 @@ export const HuggingFaceModelSearch = ({ onDownloadStarted }: Props) => {
const [searching, setSearching] = useState(false);
const [downloading, setDownloading] = useState<Set<string>>(new Set());
const [loadingFiles, setLoadingFiles] = useState<Set<string>>(new Set());
const [directSpec, setDirectSpec] = useState('');
const [error, setError] = useState<string | null>(null);
const debounceRef = useRef<ReturnType<typeof setTimeout> | null>(null);
@@ -134,7 +133,7 @@ export const HuggingFaceModelSearch = ({ onDownloadStarted }: Props) => {
const validResults = modelsWithVariants.filter(Boolean) as {
model: HfModelInfo;
data: { variants: HfQuantVariant[]; recommended_index?: number | null };
data: { variants: HfQuantVariant[]; recommended_index?: number | null; available_memory_bytes: number; downloaded_quants: string[] };
}[];
setResults(validResults.map((r) => r.model));
@@ -144,6 +143,8 @@ export const HuggingFaceModelSearch = ({ onDownloadStarted }: Props) => {
next[r.model.repo_id] = {
variants: r.data.variants,
recommendedIndex: r.data.recommended_index ?? null,
availableMemoryBytes: r.data.available_memory_bytes,
downloadedQuants: new Set(r.data.downloaded_quants),
};
}
return next;
@@ -188,12 +189,13 @@ export const HuggingFaceModelSearch = ({ onDownloadStarted }: Props) => {
path: { author, repo },
});
if (response.data) {
const variants = response.data.variants;
setRepoData((prev) => ({
...prev,
[repoId]: {
variants,
variants: response.data!.variants,
recommendedIndex: response.data!.recommended_index ?? null,
availableMemoryBytes: response.data!.available_memory_bytes,
downloadedQuants: new Set(response.data!.downloaded_quants),
},
}));
}
@@ -230,34 +232,6 @@ export const HuggingFaceModelSearch = ({ onDownloadStarted }: Props) => {
}
};
const startDirectDownload = async () => {
const spec = directSpec.trim();
if (!spec) return;
const key = `direct:${spec}`;
setDownloading((prev) => new Set(prev).add(key));
try {
const response = await downloadHfModel({
body: { spec },
throwOnError: true,
});
if (response.data) {
onDownloadStarted(response.data);
setDirectSpec('');
}
} catch (e) {
toastError({
title: intl.formatMessage(i18n.directDownloadFailed),
msg: intl.formatMessage(i18n.directDownloadErrorMsg, { error: errorMessage(e) }),
});
} finally {
setDownloading((prev) => {
const next = new Set(prev);
next.delete(key);
return next;
});
}
};
return (
<div className="space-y-4">
<div>
@@ -280,12 +254,14 @@ export const HuggingFaceModelSearch = ({ onDownloadStarted }: Props) => {
{error && !searching && <p className="text-xs text-text-muted">{error}</p>}
{results.length > 0 && (
<div className="space-y-1 max-h-96 overflow-y-auto">
<div className="space-y-1">
{results.map((model) => {
const isExpanded = expandedRepo === model.repo_id;
const data = repoData[model.repo_id];
const variants = data?.variants || [];
const recommendedIndex = data?.recommendedIndex ?? null;
const availableMemory = data?.availableMemoryBytes ?? 0;
const downloadedQuants = data?.downloadedQuants ?? new Set<string>();
return (
<div key={model.repo_id} className="border border-border-subtle rounded-lg">
@@ -324,14 +300,22 @@ export const HuggingFaceModelSearch = ({ onDownloadStarted }: Props) => {
const dlKey = `${model.repo_id}:${variant.quantization}`;
const isStarting = downloading.has(dlKey);
const isRecommended = idx === recommendedIndex;
const modelId = `${model.repo_id}:${variant.quantization}`;
const isActiveDownload = activeDownloadIds?.has(modelId) ?? false;
const isDownloaded = downloadedModelIds
? downloadedModelIds.has(modelId)
: downloadedQuants.has(variant.quantization);
const tooLarge = availableMemory > 0 && variant.size_bytes > availableMemory * 0.85;
return (
<div
key={variant.quantization}
className={`flex items-center justify-between py-2 px-2 rounded ${
isRecommended
? 'bg-blue-500/5 border border-blue-500/20'
: 'hover:bg-background-subtle'
isDownloaded
? 'bg-green-500/5 border border-green-500/20'
: isRecommended
? 'bg-blue-500/5 border border-blue-500/20'
: 'hover:bg-background-subtle'
}`}
>
<div className="flex flex-col gap-0.5 min-w-0 flex-1 mr-3">
@@ -342,7 +326,7 @@ export const HuggingFaceModelSearch = ({ onDownloadStarted }: Props) => {
<span className="text-xs text-text-muted">
{formatBytes(variant.size_bytes)}
</span>
{isRecommended && (
{isRecommended && !isDownloaded && (
<span className="inline-flex items-center gap-1 text-xs bg-blue-500 text-white px-1.5 py-0.5 rounded">
<Star className="w-3 h-3" />
{intl.formatMessage(i18n.recommended)}
@@ -352,22 +336,53 @@ export const HuggingFaceModelSearch = ({ onDownloadStarted }: Props) => {
{variant.description && (
<span className="text-xs text-text-muted">{variant.description}</span>
)}
</div>
<Button
variant="outline"
size="sm"
disabled={isStarting}
onClick={() => startDownload(model.repo_id, variant.quantization)}
>
{isStarting ? (
<Loader2 className="w-3 h-3 animate-spin" />
) : (
<>
<Download className="w-3 h-3 mr-1" />
{intl.formatMessage(i18n.download)}
</>
{tooLarge && !isDownloaded && (
<span className="inline-flex items-center gap-1 text-xs text-amber-500">
<AlertTriangle className="w-3 h-3" />
{intl.formatMessage(i18n.tooLarge, {
size: formatBytes(variant.size_bytes),
available: formatBytes(availableMemory),
})}
</span>
)}
</Button>
</div>
{isDownloaded ? (
<Button
variant="outline"
size="sm"
disabled
className="opacity-60"
>
<Check className="w-3 h-3 mr-1" />
{intl.formatMessage(i18n.downloaded)}
</Button>
) : isActiveDownload ? (
<Button
variant="outline"
size="sm"
disabled
className="opacity-60"
>
<Loader2 className="w-3 h-3 mr-1 animate-spin" />
{intl.formatMessage(i18n.downloading)}
</Button>
) : (
<Button
variant="outline"
size="sm"
disabled={isStarting}
onClick={() => startDownload(model.repo_id, variant.quantization)}
>
{isStarting ? (
<Loader2 className="w-3 h-3 animate-spin" />
) : (
<>
<Download className="w-3 h-3 mr-1" />
{intl.formatMessage(i18n.download)}
</>
)}
</Button>
)}
</div>
);
})}
@@ -379,41 +394,6 @@ export const HuggingFaceModelSearch = ({ onDownloadStarted }: Props) => {
</div>
)}
<div>
<h4 className="text-sm font-medium text-text-default mb-2">{intl.formatMessage(i18n.directDownload)}</h4>
<p className="text-xs text-text-muted mb-2">
{intl.formatMessage(i18n.directDownloadDescription, {
format: 'user/repo:quantization',
})}
</p>
<div className="flex gap-2">
<input
type="text"
value={directSpec}
onChange={(e) => setDirectSpec(e.target.value)}
placeholder="bartowski/Llama-3.2-1B-Instruct-GGUF:Q4_K_M"
className="flex-1 px-3 py-2 text-sm border border-border-subtle rounded-lg bg-background-default text-text-default placeholder:text-text-muted focus:outline-none focus:border-accent-primary"
onKeyDown={(e) => {
if (e.key === 'Enter') startDirectDownload();
}}
/>
<Button
variant="outline"
size="sm"
disabled={!directSpec.trim() || downloading.has(`direct:${directSpec}`)}
onClick={startDirectDownload}
>
{downloading.has(`direct:${directSpec}`) ? (
<Loader2 className="w-4 h-4 animate-spin" />
) : (
<>
<Download className="w-4 h-4 mr-1" />
{intl.formatMessage(i18n.download)}
</>
)}
</Button>
</div>
</div>
</div>
);
};
ui/desktop/src/components/settings/localInference/LocalInferenceSettings.tsx
+5 -1
View File
@@ -520,7 +520,11 @@ export const LocalInferenceSettings = () => {
{/* HuggingFace Search */}
<div className="border-t border-border-subtle pt-4">
<HuggingFaceModelSearch onDownloadStarted={handleHfDownloadStarted} />
<HuggingFaceModelSearch
onDownloadStarted={handleHfDownloadStarted}
activeDownloadIds={new Set(downloads.keys())}
downloadedModelIds={new Set(models.filter(m => m.status.state === 'Downloaded').map(m => m.id))}
/>
</div>
{models.length === 0 && (
ui/desktop/src/i18n/messages/en.json
+9 -12
View File
@@ -1436,21 +1436,15 @@
"headersSection.value": {
"defaultMessage": "Value"
},
"huggingFaceModelSearch.directDownload": {
"defaultMessage": "Direct Download"
},
"huggingFaceModelSearch.directDownloadDescription": {
"defaultMessage": "Specify a model directly: {format}"
},
"huggingFaceModelSearch.directDownloadErrorMsg": {
"defaultMessage": "Failed to start the download. Check the spec: {error}"
},
"huggingFaceModelSearch.directDownloadFailed": {
"defaultMessage": "Direct download failed"
},
"huggingFaceModelSearch.download": {
"defaultMessage": "Download"
},
"huggingFaceModelSearch.downloaded": {
"defaultMessage": "Downloaded"
},
"huggingFaceModelSearch.downloading": {
"defaultMessage": "Downloading\u2026"
},
"huggingFaceModelSearch.loadingVariants": {
"defaultMessage": "Loading variants..."
},
@@ -1475,6 +1469,9 @@
"huggingFaceModelSearch.searchPlaceholder": {
"defaultMessage": "Search for GGUF models..."
},
"huggingFaceModelSearch.tooLarge": {
"defaultMessage": "May not fit in memory ({size} model, {available} available)"
},
"imagePreview.altText": {
"defaultMessage": "goose image"
},