feat: add local inference provider with llama.cpp backend and HuggingFace model management (#6933)
Co-authored-by: Douwe Osinga <douwe@squareup.com> Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Co-authored-by: jh-block <jhugo@block.xyz> Co-authored-by: Spence <spencermartin@squareup.com> Co-authored-by: Michael Neale <michael.neale@gmail.com>
This commit is contained in:
Douwe Osinga
@@ -0,0 +1,358 @@
|
||||
import { useState, useCallback, useRef } from 'react';
|
||||
import { Search, Download, ChevronDown, ChevronUp, Loader2, Star } from 'lucide-react';
|
||||
import { Button } from '../../ui/button';
|
||||
import {
|
||||
searchHfModels,
|
||||
getRepoFiles,
|
||||
downloadHfModel,
|
||||
type HfModelInfo,
|
||||
type HfQuantVariant,
|
||||
} from '../../../api';
|
||||
|
||||
const formatBytes = (bytes: number): string => {
|
||||
if (bytes === 0) return 'unknown';
|
||||
if (bytes < 1024) return `${bytes}B`;
|
||||
if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(0)}KB`;
|
||||
if (bytes < 1024 * 1024 * 1024) return `${(bytes / (1024 * 1024)).toFixed(0)}MB`;
|
||||
return `${(bytes / (1024 * 1024 * 1024)).toFixed(1)}GB`;
|
||||
};
|
||||
|
||||
const formatDownloads = (n: number): string => {
|
||||
if (n >= 1_000_000) return `${(n / 1_000_000).toFixed(1)}M`;
|
||||
if (n >= 1_000) return `${(n / 1_000).toFixed(1)}K`;
|
||||
return `${n}`;
|
||||
};
|
||||
|
||||
interface RepoData {
|
||||
variants: HfQuantVariant[];
|
||||
recommendedIndex: number | null;
|
||||
}
|
||||
|
||||
interface Props {
|
||||
onDownloadStarted: (modelId: string) => void;
|
||||
}
|
||||
|
||||
export const HuggingFaceModelSearch = ({ onDownloadStarted }: Props) => {
|
||||
const [query, setQuery] = useState('');
|
||||
const [results, setResults] = useState<HfModelInfo[]>([]);
|
||||
const [expandedRepo, setExpandedRepo] = useState<string | null>(null);
|
||||
const [repoData, setRepoData] = useState<Record<string, RepoData>>({});
|
||||
const [searching, setSearching] = useState(false);
|
||||
const [downloading, setDownloading] = useState<Set<string>>(new Set());
|
||||
const [loadingFiles, setLoadingFiles] = useState<Set<string>>(new Set());
|
||||
const [directSpec, setDirectSpec] = useState('');
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const debounceRef = useRef<ReturnType<typeof setTimeout> | null>(null);
|
||||
|
||||
const doSearch = useCallback(async (q: string) => {
|
||||
if (!q.trim()) {
|
||||
setResults([]);
|
||||
setError(null);
|
||||
return;
|
||||
}
|
||||
setSearching(true);
|
||||
setError(null);
|
||||
try {
|
||||
const response = await searchHfModels({
|
||||
query: { q, limit: 20 },
|
||||
});
|
||||
if (response.data) {
|
||||
// Pre-fetch variants for all results and filter out repos with no suitable quantizations
|
||||
const modelsWithVariants = await Promise.all(
|
||||
response.data.map(async (model) => {
|
||||
try {
|
||||
const [author, repo] = model.repo_id.split('/');
|
||||
const filesResponse = await getRepoFiles({ path: { author, repo } });
|
||||
if (filesResponse.data && filesResponse.data.variants.length > 0) {
|
||||
return { model, data: filesResponse.data };
|
||||
}
|
||||
} catch {
|
||||
// Skip repos we can't fetch
|
||||
}
|
||||
return null;
|
||||
})
|
||||
);
|
||||
|
||||
const validResults = modelsWithVariants.filter(Boolean) as {
|
||||
model: HfModelInfo;
|
||||
data: { variants: HfQuantVariant[]; recommended_index?: number | null };
|
||||
}[];
|
||||
|
||||
setResults(validResults.map((r) => r.model));
|
||||
setRepoData((prev) => {
|
||||
const next = { ...prev };
|
||||
for (const r of validResults) {
|
||||
next[r.model.repo_id] = {
|
||||
variants: r.data.variants,
|
||||
recommendedIndex: r.data.recommended_index ?? null,
|
||||
};
|
||||
}
|
||||
return next;
|
||||
});
|
||||
|
||||
if (validResults.length === 0) {
|
||||
setError('No GGUF models found for this query.');
|
||||
}
|
||||
} else {
|
||||
console.error('Search response:', response);
|
||||
const errMsg = response.error
|
||||
? `Search error: ${JSON.stringify(response.error)}`
|
||||
: 'Search returned no data.';
|
||||
setError(errMsg);
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('Search failed:', e);
|
||||
setError('Search failed. Please try again.');
|
||||
} finally {
|
||||
setSearching(false);
|
||||
}
|
||||
}, []);
|
||||
|
||||
const handleQueryChange = (value: string) => {
|
||||
setQuery(value);
|
||||
if (debounceRef.current) clearTimeout(debounceRef.current);
|
||||
debounceRef.current = setTimeout(() => doSearch(value), 300);
|
||||
};
|
||||
|
||||
const toggleRepo = async (repoId: string) => {
|
||||
if (expandedRepo === repoId) {
|
||||
setExpandedRepo(null);
|
||||
return;
|
||||
}
|
||||
setExpandedRepo(repoId);
|
||||
|
||||
if (!repoData[repoId]?.variants.length) {
|
||||
setLoadingFiles((prev) => new Set(prev).add(repoId));
|
||||
try {
|
||||
const [author, repo] = repoId.split('/');
|
||||
const response = await getRepoFiles({
|
||||
path: { author, repo },
|
||||
});
|
||||
if (response.data) {
|
||||
const variants = response.data.variants;
|
||||
setRepoData((prev) => ({
|
||||
...prev,
|
||||
[repoId]: {
|
||||
variants,
|
||||
recommendedIndex: response.data!.recommended_index ?? null,
|
||||
},
|
||||
}));
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('Failed to fetch repo files:', e);
|
||||
} finally {
|
||||
setLoadingFiles((prev) => {
|
||||
const next = new Set(prev);
|
||||
next.delete(repoId);
|
||||
return next;
|
||||
});
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const startDownload = async (repoId: string, quantization: string) => {
|
||||
const spec = `${repoId}:${quantization}`;
|
||||
setDownloading((prev) => new Set(prev).add(spec));
|
||||
try {
|
||||
const response = await downloadHfModel({
|
||||
body: { spec },
|
||||
});
|
||||
if (response.data) {
|
||||
onDownloadStarted(response.data);
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('Download failed:', e);
|
||||
} finally {
|
||||
setDownloading((prev) => {
|
||||
const next = new Set(prev);
|
||||
next.delete(spec);
|
||||
return next;
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
const startDirectDownload = async () => {
|
||||
const spec = directSpec.trim();
|
||||
if (!spec) return;
|
||||
const key = `direct:${spec}`;
|
||||
setDownloading((prev) => new Set(prev).add(key));
|
||||
try {
|
||||
const response = await downloadHfModel({
|
||||
body: { spec },
|
||||
});
|
||||
if (response.data) {
|
||||
onDownloadStarted(response.data);
|
||||
setDirectSpec('');
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('Direct download failed:', e);
|
||||
} finally {
|
||||
setDownloading((prev) => {
|
||||
const next = new Set(prev);
|
||||
next.delete(key);
|
||||
return next;
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="space-y-4">
|
||||
<div>
|
||||
<h4 className="text-sm font-medium text-text-default mb-2">Search HuggingFace</h4>
|
||||
<div className="relative">
|
||||
<Search className="absolute left-3 top-1/2 -translate-y-1/2 w-4 h-4 text-text-muted" />
|
||||
<input
|
||||
type="text"
|
||||
value={query}
|
||||
onChange={(e) => handleQueryChange(e.target.value)}
|
||||
placeholder="Search for GGUF models..."
|
||||
className="w-full pl-9 pr-4 py-2 text-sm border border-border-subtle rounded-lg bg-background-default text-text-default placeholder:text-text-muted focus:outline-none focus:border-accent-primary"
|
||||
/>
|
||||
{searching && (
|
||||
<Loader2 className="absolute right-3 top-1/2 -translate-y-1/2 w-4 h-4 text-text-muted animate-spin" />
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{error && !searching && (
|
||||
<p className="text-xs text-text-muted">{error}</p>
|
||||
)}
|
||||
|
||||
{results.length > 0 && (
|
||||
<div className="space-y-1 max-h-96 overflow-y-auto">
|
||||
{results.map((model) => {
|
||||
const isExpanded = expandedRepo === model.repo_id;
|
||||
const data = repoData[model.repo_id];
|
||||
const variants = data?.variants || [];
|
||||
const recommendedIndex = data?.recommendedIndex ?? null;
|
||||
|
||||
return (
|
||||
<div key={model.repo_id} className="border border-border-subtle rounded-lg">
|
||||
<button
|
||||
onClick={() => toggleRepo(model.repo_id)}
|
||||
className="w-full flex items-center justify-between p-3 text-left hover:bg-background-subtle rounded-lg"
|
||||
>
|
||||
<div className="flex-1 min-w-0">
|
||||
<div className="flex items-center gap-2">
|
||||
<span className="text-sm font-medium text-text-default truncate">
|
||||
{model.repo_id}
|
||||
</span>
|
||||
</div>
|
||||
<div className="flex items-center gap-3 mt-0.5">
|
||||
<span className="text-xs text-text-muted">
|
||||
↓ {formatDownloads(model.downloads)}
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
{isExpanded ? (
|
||||
<ChevronUp className="w-4 h-4 text-text-muted flex-shrink-0" />
|
||||
) : (
|
||||
<ChevronDown className="w-4 h-4 text-text-muted flex-shrink-0" />
|
||||
)}
|
||||
</button>
|
||||
|
||||
{isExpanded && (
|
||||
<div className="border-t border-border-subtle px-3 pb-3 space-y-1">
|
||||
{loadingFiles.has(model.repo_id) && (
|
||||
<div className="flex items-center gap-2 py-2 text-xs text-text-muted">
|
||||
<Loader2 className="w-3 h-3 animate-spin" />
|
||||
Loading variants...
|
||||
</div>
|
||||
)}
|
||||
{variants.map((variant, idx) => {
|
||||
const dlKey = `${model.repo_id}:${variant.quantization}`;
|
||||
const isStarting = downloading.has(dlKey);
|
||||
const isRecommended = idx === recommendedIndex;
|
||||
|
||||
return (
|
||||
<div
|
||||
key={variant.quantization}
|
||||
className={`flex items-center justify-between py-2 px-2 rounded ${
|
||||
isRecommended
|
||||
? 'bg-blue-500/5 border border-blue-500/20'
|
||||
: 'hover:bg-background-subtle'
|
||||
}`}
|
||||
>
|
||||
<div className="flex flex-col gap-0.5 min-w-0 flex-1 mr-3">
|
||||
<div className="flex items-center gap-2">
|
||||
<span className="text-xs font-mono font-medium text-text-default">
|
||||
{variant.quantization}
|
||||
</span>
|
||||
<span className="text-xs text-text-muted">
|
||||
{formatBytes(variant.size_bytes)}
|
||||
</span>
|
||||
{isRecommended && (
|
||||
<span className="inline-flex items-center gap-1 text-xs bg-blue-500 text-white px-1.5 py-0.5 rounded">
|
||||
<Star className="w-3 h-3" />
|
||||
Recommended
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
{variant.description && (
|
||||
<span className="text-xs text-text-muted">
|
||||
{variant.description}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
<Button
|
||||
variant="outline"
|
||||
size="sm"
|
||||
disabled={isStarting}
|
||||
onClick={() => startDownload(model.repo_id, variant.quantization)}
|
||||
>
|
||||
{isStarting ? (
|
||||
<Loader2 className="w-3 h-3 animate-spin" />
|
||||
) : (
|
||||
<>
|
||||
<Download className="w-3 h-3 mr-1" />
|
||||
Download
|
||||
</>
|
||||
)}
|
||||
</Button>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div>
|
||||
<h4 className="text-sm font-medium text-text-default mb-2">Direct Download</h4>
|
||||
<p className="text-xs text-text-muted mb-2">
|
||||
Specify a model directly: <code className="bg-background-subtle px-1 rounded">user/repo:quantization</code>
|
||||
</p>
|
||||
<div className="flex gap-2">
|
||||
<input
|
||||
type="text"
|
||||
value={directSpec}
|
||||
onChange={(e) => setDirectSpec(e.target.value)}
|
||||
placeholder="bartowski/Llama-3.2-1B-Instruct-GGUF:Q4_K_M"
|
||||
className="flex-1 px-3 py-2 text-sm border border-border-subtle rounded-lg bg-background-default text-text-default placeholder:text-text-muted focus:outline-none focus:border-accent-primary"
|
||||
onKeyDown={(e) => {
|
||||
if (e.key === 'Enter') startDirectDownload();
|
||||
}}
|
||||
/>
|
||||
<Button
|
||||
variant="outline"
|
||||
size="sm"
|
||||
disabled={!directSpec.trim() || downloading.has(`direct:${directSpec}`)}
|
||||
onClick={startDirectDownload}
|
||||
>
|
||||
{downloading.has(`direct:${directSpec}`) ? (
|
||||
<Loader2 className="w-4 h-4 animate-spin" />
|
||||
) : (
|
||||
<>
|
||||
<Download className="w-4 h-4 mr-1" />
|
||||
Download
|
||||
</>
|
||||
)}
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,9 @@
|
||||
import { LocalInferenceSettings } from './LocalInferenceSettings';
|
||||
|
||||
export default function LocalInferenceSection() {
|
||||
return (
|
||||
<section id="local-inference" className="space-y-4 pr-4 pb-8">
|
||||
<LocalInferenceSettings />
|
||||
</section>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,410 @@
|
||||
import { useState, useEffect, useCallback, useRef } from 'react';
|
||||
import { Download, Trash2, X, ChevronDown, ChevronUp, Settings2 } from 'lucide-react';
|
||||
import { Button } from '../../ui/button';
|
||||
import { useModelAndProvider } from '../../ModelAndProviderContext';
|
||||
import {
|
||||
listLocalModels,
|
||||
downloadHfModel,
|
||||
getLocalModelDownloadProgress,
|
||||
cancelLocalModelDownload,
|
||||
deleteLocalModel,
|
||||
setConfigProvider,
|
||||
type DownloadProgress,
|
||||
type LocalModelResponse,
|
||||
} from '../../../api';
|
||||
import { HuggingFaceModelSearch } from './HuggingFaceModelSearch';
|
||||
import { ModelSettingsPanel } from './ModelSettingsPanel';
|
||||
import {
|
||||
Dialog,
|
||||
DialogContent,
|
||||
DialogHeader,
|
||||
DialogTitle,
|
||||
} from '../../ui/dialog';
|
||||
|
||||
const formatBytes = (bytes: number): string => {
|
||||
if (bytes < 1024) return `${bytes}B`;
|
||||
if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(0)}KB`;
|
||||
if (bytes < 1024 * 1024 * 1024) return `${(bytes / (1024 * 1024)).toFixed(0)}MB`;
|
||||
return `${(bytes / (1024 * 1024 * 1024)).toFixed(1)}GB`;
|
||||
};
|
||||
|
||||
export const LocalInferenceSettings = () => {
|
||||
const [models, setModels] = useState<LocalModelResponse[]>([]);
|
||||
const [downloads, setDownloads] = useState<Map<string, DownloadProgress>>(new Map());
|
||||
const [showAllFeatured, setShowAllFeatured] = useState(false);
|
||||
const [settingsOpenFor, setSettingsOpenFor] = useState<string | null>(null);
|
||||
const { currentModel, currentProvider, setProviderAndModel } = useModelAndProvider();
|
||||
const downloadSectionRef = useRef<HTMLDivElement>(null);
|
||||
const selectedModelId = currentProvider === 'local' ? currentModel : null;
|
||||
|
||||
const getDisplayName = useCallback(
|
||||
(modelId: string): string => {
|
||||
const model = models.find((m) => m.id === modelId);
|
||||
return model?.display_name || modelId;
|
||||
},
|
||||
[models]
|
||||
);
|
||||
|
||||
const loadModels = useCallback(async () => {
|
||||
try {
|
||||
const response = await listLocalModels();
|
||||
if (response.data) {
|
||||
setModels(response.data);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Failed to load models:', error);
|
||||
}
|
||||
}, []);
|
||||
|
||||
// Check for any in-progress downloads when models list changes
|
||||
const detectActiveDownloads = useCallback(async () => {
|
||||
for (const model of models) {
|
||||
if (downloads.has(model.id)) continue;
|
||||
// Check models that the API reports as downloading
|
||||
if (model.status.state === 'Downloading') {
|
||||
pollDownloadProgress(model.id);
|
||||
}
|
||||
}
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [models, downloads]);
|
||||
|
||||
useEffect(() => {
|
||||
loadModels();
|
||||
}, [loadModels]);
|
||||
|
||||
useEffect(() => {
|
||||
if (models.length > 0) {
|
||||
detectActiveDownloads();
|
||||
}
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [models]);
|
||||
|
||||
const selectModel = async (modelId: string) => {
|
||||
setProviderAndModel('local', modelId);
|
||||
try {
|
||||
await setConfigProvider({
|
||||
body: { provider: 'local', model: modelId },
|
||||
throwOnError: true,
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('Failed to select model:', error);
|
||||
}
|
||||
};
|
||||
|
||||
const startFeaturedDownload = async (modelId: string) => {
|
||||
const model = models.find((m) => m.id === modelId);
|
||||
if (!model) return;
|
||||
try {
|
||||
await downloadHfModel({ body: { spec: model.id } });
|
||||
pollDownloadProgress(modelId);
|
||||
scrollToDownloads();
|
||||
} catch (error) {
|
||||
console.error('Failed to start download:', error);
|
||||
}
|
||||
};
|
||||
|
||||
const scrollToDownloads = useCallback(() => {
|
||||
requestAnimationFrame(() => {
|
||||
downloadSectionRef.current?.scrollIntoView({ behavior: 'smooth', block: 'nearest' });
|
||||
});
|
||||
}, []);
|
||||
|
||||
const pollDownloadProgress = (modelId: string) => {
|
||||
const interval = setInterval(async () => {
|
||||
try {
|
||||
const response = await getLocalModelDownloadProgress({ path: { model_id: modelId } });
|
||||
if (response.data) {
|
||||
const progress = response.data;
|
||||
setDownloads((prev) => new Map(prev).set(modelId, progress));
|
||||
|
||||
if (progress.status === 'completed') {
|
||||
clearInterval(interval);
|
||||
setDownloads((prev) => {
|
||||
const next = new Map(prev);
|
||||
next.delete(modelId);
|
||||
return next;
|
||||
});
|
||||
await loadModels();
|
||||
await selectModel(modelId);
|
||||
} else if (progress.status === 'failed') {
|
||||
clearInterval(interval);
|
||||
await loadModels();
|
||||
}
|
||||
} else {
|
||||
clearInterval(interval);
|
||||
}
|
||||
} catch {
|
||||
clearInterval(interval);
|
||||
}
|
||||
}, 1000);
|
||||
};
|
||||
|
||||
const cancelDownload = async (modelId: string) => {
|
||||
try {
|
||||
await cancelLocalModelDownload({ path: { model_id: modelId } });
|
||||
setDownloads((prev) => {
|
||||
const next = new Map(prev);
|
||||
next.delete(modelId);
|
||||
return next;
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('Failed to cancel download:', error);
|
||||
}
|
||||
};
|
||||
|
||||
const handleDeleteModel = async (modelId: string) => {
|
||||
if (!window.confirm('Delete this model? You can re-download it later.')) return;
|
||||
try {
|
||||
await deleteLocalModel({ path: { model_id: modelId } });
|
||||
await loadModels();
|
||||
} catch (error) {
|
||||
console.error('Failed to delete model:', error);
|
||||
}
|
||||
};
|
||||
|
||||
const handleHfDownloadStarted = (modelId: string) => {
|
||||
pollDownloadProgress(modelId);
|
||||
loadModels();
|
||||
scrollToDownloads();
|
||||
};
|
||||
|
||||
const isDownloaded = (model: LocalModelResponse) => model.status.state === 'Downloaded';
|
||||
const isNotDownloaded = (model: LocalModelResponse) =>
|
||||
model.status.state === 'NotDownloaded' && !downloads.has(model.id);
|
||||
|
||||
const downloadedModels = models.filter(isDownloaded);
|
||||
const notDownloadedModels = models.filter(isNotDownloaded);
|
||||
const recommendedModels = notDownloadedModels.filter((m) => m.recommended);
|
||||
const displayedFeatured = showAllFeatured ? notDownloadedModels : recommendedModels;
|
||||
const showFeaturedToggle = notDownloadedModels.length > recommendedModels.length;
|
||||
|
||||
return (
|
||||
<div className="space-y-6">
|
||||
<div>
|
||||
<h3 className="text-text-default font-medium">Local Inference Models</h3>
|
||||
<p className="text-xs text-text-muted max-w-2xl mt-1">
|
||||
Download and manage local LLM models for inference without API keys. Search HuggingFace
|
||||
for any GGUF model or use the featured picks below.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
{/* Active Downloads */}
|
||||
{downloads.size > 0 && (
|
||||
<div ref={downloadSectionRef}>
|
||||
<h4 className="text-sm font-medium text-text-default mb-2">Downloading</h4>
|
||||
<div className="space-y-2">
|
||||
{Array.from(downloads.entries()).map(([modelId, progress]) => {
|
||||
if (progress.status === 'completed') return null;
|
||||
const displayName = getDisplayName(modelId);
|
||||
return (
|
||||
<div
|
||||
key={modelId}
|
||||
className="border rounded-lg p-3 border-border-subtle bg-background-default"
|
||||
>
|
||||
<div className="flex items-center justify-between mb-2">
|
||||
<span className="text-sm font-medium text-text-default truncate">
|
||||
{displayName}
|
||||
</span>
|
||||
{progress.status === 'downloading' && (
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="sm"
|
||||
onClick={() => cancelDownload(modelId)}
|
||||
className="text-destructive hover:text-destructive"
|
||||
>
|
||||
<X className="w-4 h-4" />
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
{progress.status === 'downloading' && (
|
||||
<div className="space-y-1">
|
||||
<div className="w-full bg-background-subtle rounded-full h-2">
|
||||
<div
|
||||
className="bg-accent-primary h-2 rounded-full transition-all duration-300"
|
||||
style={{ width: `${progress.progress_percent}%` }}
|
||||
/>
|
||||
</div>
|
||||
<div className="flex justify-between text-xs text-text-muted">
|
||||
<span>
|
||||
{formatBytes(progress.bytes_downloaded)} /{' '}
|
||||
{formatBytes(progress.total_bytes)} (
|
||||
{progress.progress_percent.toFixed(0)}%)
|
||||
</span>
|
||||
<span className="flex gap-2">
|
||||
{progress.eta_seconds != null && progress.eta_seconds > 0 && (
|
||||
<span>
|
||||
{progress.eta_seconds < 60
|
||||
? `${Math.round(progress.eta_seconds)}s`
|
||||
: `${Math.round(progress.eta_seconds / 60)}m`}{' '}
|
||||
remaining
|
||||
</span>
|
||||
)}
|
||||
{progress.speed_bps != null && progress.speed_bps > 0 && (
|
||||
<span>{formatBytes(progress.speed_bps)}/s</span>
|
||||
)}
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
{progress.status === 'failed' && (
|
||||
<p className="text-xs text-destructive">
|
||||
{progress.error || 'Download failed'}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Downloaded Models */}
|
||||
{downloadedModels.length > 0 && (
|
||||
<div>
|
||||
<h4 className="text-sm font-medium text-text-default mb-2">Downloaded Models</h4>
|
||||
<div className="space-y-2">
|
||||
{downloadedModels.map((model) => {
|
||||
const isSelected = selectedModelId === model.id;
|
||||
return (
|
||||
<div
|
||||
key={model.id}
|
||||
className={`border rounded-lg p-3 transition-colors ${
|
||||
isSelected
|
||||
? 'border-accent-primary bg-accent-primary/5'
|
||||
: 'border-border-subtle bg-background-default hover:border-border-default'
|
||||
}`}
|
||||
>
|
||||
<div className="flex items-center justify-between">
|
||||
<div className="flex items-center gap-2">
|
||||
<input
|
||||
type="radio"
|
||||
checked={isSelected}
|
||||
onChange={() => selectModel(model.id)}
|
||||
className="cursor-pointer"
|
||||
/>
|
||||
<span className="text-sm font-medium text-text-default">
|
||||
{model.display_name}
|
||||
</span>
|
||||
<span className="text-xs text-text-muted">
|
||||
{formatBytes(model.size_bytes)}
|
||||
</span>
|
||||
{model.recommended && (
|
||||
<span className="text-xs bg-blue-500 text-white px-2 py-0.5 rounded">
|
||||
Recommended
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
<div className="flex items-center gap-1">
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="sm"
|
||||
onClick={() => setSettingsOpenFor(model.id)}
|
||||
title="Model settings"
|
||||
>
|
||||
<Settings2 className="w-4 h-4" />
|
||||
</Button>
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="sm"
|
||||
onClick={() => handleDeleteModel(model.id)}
|
||||
className="text-destructive hover:text-destructive"
|
||||
>
|
||||
<Trash2 className="w-4 h-4" />
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Featured Models (not yet downloaded) */}
|
||||
{displayedFeatured.length > 0 && (
|
||||
<div>
|
||||
<h4 className="text-sm font-medium text-text-default mb-2">Featured Models</h4>
|
||||
<div className="space-y-2">
|
||||
{displayedFeatured.map((model) => (
|
||||
<div
|
||||
key={model.id}
|
||||
className="border rounded-lg p-3 border-border-subtle bg-background-default hover:border-border-default"
|
||||
>
|
||||
<div className="flex items-center justify-between gap-3">
|
||||
<div className="flex-1 min-w-0">
|
||||
<div className="flex items-center gap-2 flex-wrap">
|
||||
<h4 className="text-sm font-medium text-text-default">
|
||||
{model.display_name}
|
||||
</h4>
|
||||
<span className="text-xs text-text-muted">
|
||||
{formatBytes(model.size_bytes)}
|
||||
</span>
|
||||
{model.recommended && (
|
||||
<span className="text-xs bg-blue-500 text-white px-2 py-0.5 rounded">
|
||||
Recommended
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
<Button
|
||||
variant="outline"
|
||||
size="sm"
|
||||
onClick={() => startFeaturedDownload(model.id)}
|
||||
>
|
||||
<Download className="w-4 h-4 mr-1" />
|
||||
Download
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
|
||||
{showFeaturedToggle && (
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="sm"
|
||||
onClick={() => setShowAllFeatured(!showAllFeatured)}
|
||||
className="w-full text-text-muted hover:text-text-default mt-2"
|
||||
>
|
||||
{showAllFeatured ? (
|
||||
<>
|
||||
<ChevronUp className="w-4 h-4 mr-1" />
|
||||
Show recommended only
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
<ChevronDown className="w-4 h-4 mr-1" />
|
||||
Show all featured ({notDownloadedModels.length - displayedFeatured.length} more)
|
||||
</>
|
||||
)}
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* HuggingFace Search */}
|
||||
<div className="border-t border-border-subtle pt-4">
|
||||
<HuggingFaceModelSearch onDownloadStarted={handleHfDownloadStarted} />
|
||||
</div>
|
||||
|
||||
{models.length === 0 && (
|
||||
<div className="text-center py-6 text-text-muted text-sm">No models available</div>
|
||||
)}
|
||||
|
||||
<Dialog
|
||||
open={!!settingsOpenFor}
|
||||
onOpenChange={(open) => {
|
||||
if (!open) setSettingsOpenFor(null);
|
||||
}}
|
||||
>
|
||||
<DialogContent className="max-h-[80vh] overflow-y-auto sm:max-w-xl">
|
||||
<DialogHeader>
|
||||
<DialogTitle>Model Settings</DialogTitle>
|
||||
<p className="text-sm text-text-muted">{getDisplayName(settingsOpenFor || '')}</p>
|
||||
</DialogHeader>
|
||||
{settingsOpenFor && <ModelSettingsPanel modelId={settingsOpenFor} />}
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,415 @@
|
||||
import { useState, useEffect, useCallback } from 'react';
|
||||
import { RotateCcw } from 'lucide-react';
|
||||
import { Button } from '../../ui/button';
|
||||
import { Switch } from '../../ui/switch';
|
||||
import {
|
||||
getModelSettings,
|
||||
updateModelSettings,
|
||||
type ModelSettings,
|
||||
type SamplingConfig,
|
||||
} from '../../../api';
|
||||
|
||||
const DEFAULT_SETTINGS: ModelSettings = {
|
||||
context_size: null,
|
||||
max_output_tokens: null,
|
||||
sampling: { type: 'Temperature', temperature: 0.8, top_k: 40, top_p: 0.95, min_p: 0.05, seed: null },
|
||||
repeat_penalty: 1.0,
|
||||
repeat_last_n: 64,
|
||||
frequency_penalty: 0.0,
|
||||
presence_penalty: 0.0,
|
||||
n_batch: null,
|
||||
n_gpu_layers: null,
|
||||
use_mlock: false,
|
||||
flash_attention: null,
|
||||
n_threads: null,
|
||||
native_tool_calling: false,
|
||||
};
|
||||
|
||||
type SamplingType = SamplingConfig['type'];
|
||||
|
||||
function NumberField({
|
||||
label,
|
||||
description,
|
||||
value,
|
||||
onChange,
|
||||
placeholder,
|
||||
min,
|
||||
max,
|
||||
step,
|
||||
allowNull,
|
||||
}: {
|
||||
label: string;
|
||||
description?: string;
|
||||
value: number | null | undefined;
|
||||
onChange: (v: number | null) => void;
|
||||
placeholder?: string;
|
||||
min?: number;
|
||||
max?: number;
|
||||
step?: number;
|
||||
allowNull?: boolean;
|
||||
}) {
|
||||
return (
|
||||
<div className="flex flex-col gap-1">
|
||||
<label className="text-xs font-medium text-text-default">{label}</label>
|
||||
{description && <span className="text-xs text-text-muted">{description}</span>}
|
||||
<input
|
||||
type="number"
|
||||
className="w-full rounded border border-border-subtle bg-background-default px-2 py-1 text-sm text-text-default"
|
||||
value={value ?? ''}
|
||||
onChange={(e) => {
|
||||
const raw = e.target.value;
|
||||
if (raw === '' && allowNull) {
|
||||
onChange(null);
|
||||
} else {
|
||||
const n = step && step < 1 ? parseFloat(raw) : parseInt(raw, 10);
|
||||
if (!isNaN(n)) onChange(n);
|
||||
}
|
||||
}}
|
||||
placeholder={placeholder ?? 'Auto'}
|
||||
min={min}
|
||||
max={max}
|
||||
step={step}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function ToggleField({
|
||||
label,
|
||||
description,
|
||||
value,
|
||||
onChange,
|
||||
}: {
|
||||
label: string;
|
||||
description?: string;
|
||||
value: boolean;
|
||||
onChange: (v: boolean) => void;
|
||||
}) {
|
||||
return (
|
||||
<div className="flex items-center justify-between gap-2">
|
||||
<div>
|
||||
<div className="text-xs font-medium text-text-default">{label}</div>
|
||||
{description && <span className="text-xs text-text-muted">{description}</span>}
|
||||
</div>
|
||||
<Switch checked={value} onCheckedChange={onChange} variant="mono" />
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function SelectField<T extends string>({
|
||||
label,
|
||||
description,
|
||||
value,
|
||||
options,
|
||||
onChange,
|
||||
}: {
|
||||
label: string;
|
||||
description?: string;
|
||||
value: T;
|
||||
options: { value: T; label: string }[];
|
||||
onChange: (v: T) => void;
|
||||
}) {
|
||||
return (
|
||||
<div className="flex items-center justify-between gap-2">
|
||||
<div>
|
||||
<div className="text-xs font-medium text-text-default">{label}</div>
|
||||
{description && <span className="text-xs text-text-muted">{description}</span>}
|
||||
</div>
|
||||
<select
|
||||
value={value}
|
||||
onChange={(e) => onChange(e.target.value as T)}
|
||||
className="rounded border border-border-subtle bg-background-default px-2 py-1 text-xs text-text-default"
|
||||
>
|
||||
{options.map((opt) => (
|
||||
<option key={opt.value} value={opt.value}>
|
||||
{opt.label}
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export const ModelSettingsPanel = ({ modelId }: { modelId: string }) => {
|
||||
const [settings, setSettings] = useState<ModelSettings>(DEFAULT_SETTINGS);
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [saving, setSaving] = useState(false);
|
||||
|
||||
const load = useCallback(async () => {
|
||||
try {
|
||||
const res = await getModelSettings({ path: { model_id: modelId } });
|
||||
if (res.data) setSettings(res.data);
|
||||
} catch {
|
||||
// use defaults
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
}, [modelId]);
|
||||
|
||||
useEffect(() => {
|
||||
load();
|
||||
}, [load]);
|
||||
|
||||
const save = async (updated: ModelSettings) => {
|
||||
setSettings(updated);
|
||||
setSaving(true);
|
||||
try {
|
||||
await updateModelSettings({ path: { model_id: modelId }, body: updated });
|
||||
} catch (e) {
|
||||
console.error('Failed to save settings:', e);
|
||||
} finally {
|
||||
setSaving(false);
|
||||
}
|
||||
};
|
||||
|
||||
const resetDefaults = () => save(DEFAULT_SETTINGS);
|
||||
|
||||
const updateField = <K extends keyof ModelSettings>(key: K, value: ModelSettings[K]) => {
|
||||
save({ ...settings, [key]: value });
|
||||
};
|
||||
|
||||
const samplingType: SamplingType = settings.sampling?.type ?? 'Temperature';
|
||||
|
||||
const setSamplingType = (type: SamplingType) => {
|
||||
let sampling: SamplingConfig;
|
||||
if (type === 'Greedy') {
|
||||
sampling = { type: 'Greedy' };
|
||||
} else if (type === 'MirostatV2') {
|
||||
sampling = { type: 'MirostatV2', tau: 5.0, eta: 0.1, seed: null };
|
||||
} else {
|
||||
sampling = { type: 'Temperature', temperature: 0.8, top_k: 40, top_p: 0.95, min_p: 0.05, seed: null };
|
||||
}
|
||||
save({ ...settings, sampling });
|
||||
};
|
||||
|
||||
const updateSampling = (partial: Partial<SamplingConfig>) => {
|
||||
save({ ...settings, sampling: { ...settings.sampling!, ...partial } as SamplingConfig });
|
||||
};
|
||||
|
||||
if (loading) {
|
||||
return <div className="py-2 text-xs text-text-muted">Loading settings...</div>;
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="space-y-4">
|
||||
<div className="flex items-center justify-end">
|
||||
{saving && <span className="text-xs text-text-muted mr-auto">Saving...</span>}
|
||||
<Button variant="ghost" size="sm" onClick={resetDefaults} title="Reset to defaults">
|
||||
<RotateCcw className="w-3.5 h-3.5 mr-1" />
|
||||
<span className="text-xs">Reset</span>
|
||||
</Button>
|
||||
</div>
|
||||
|
||||
{/* Context & Generation */}
|
||||
<div className="space-y-2">
|
||||
<h5 className="text-xs font-medium text-text-default">Context & Generation</h5>
|
||||
<div className="grid grid-cols-2 gap-3">
|
||||
<NumberField
|
||||
label="Context size"
|
||||
description="Max context window (0 = model default)"
|
||||
value={settings.context_size}
|
||||
onChange={(v) => updateField('context_size', v)}
|
||||
placeholder="Auto"
|
||||
min={0}
|
||||
allowNull
|
||||
/>
|
||||
<NumberField
|
||||
label="Max output tokens"
|
||||
description="Cap on generated tokens"
|
||||
value={settings.max_output_tokens}
|
||||
onChange={(v) => updateField('max_output_tokens', v)}
|
||||
placeholder="No limit"
|
||||
min={1}
|
||||
allowNull
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Sampling */}
|
||||
<div className="space-y-2">
|
||||
<SelectField
|
||||
label="Sampling Strategy"
|
||||
value={samplingType}
|
||||
options={[
|
||||
{ value: 'Greedy' as SamplingType, label: 'Greedy' },
|
||||
{ value: 'Temperature' as SamplingType, label: 'Temperature' },
|
||||
{ value: 'MirostatV2' as SamplingType, label: 'Mirostat v2' },
|
||||
]}
|
||||
onChange={(v) => setSamplingType(v)}
|
||||
/>
|
||||
|
||||
{samplingType === 'Temperature' && settings.sampling?.type === 'Temperature' && (
|
||||
<div className="grid grid-cols-2 gap-3">
|
||||
<NumberField
|
||||
label="Temperature"
|
||||
value={settings.sampling.temperature}
|
||||
onChange={(v) => updateSampling({ temperature: v ?? 0.8 })}
|
||||
min={0}
|
||||
max={2}
|
||||
step={0.05}
|
||||
/>
|
||||
<NumberField
|
||||
label="Top K"
|
||||
value={settings.sampling.top_k}
|
||||
onChange={(v) => updateSampling({ top_k: v ?? 40 })}
|
||||
min={0}
|
||||
/>
|
||||
<NumberField
|
||||
label="Top P"
|
||||
value={settings.sampling.top_p}
|
||||
onChange={(v) => updateSampling({ top_p: v ?? 0.95 })}
|
||||
min={0}
|
||||
max={1}
|
||||
step={0.01}
|
||||
/>
|
||||
<NumberField
|
||||
label="Min P"
|
||||
value={settings.sampling.min_p}
|
||||
onChange={(v) => updateSampling({ min_p: v ?? 0.05 })}
|
||||
min={0}
|
||||
max={1}
|
||||
step={0.01}
|
||||
/>
|
||||
<NumberField
|
||||
label="Seed"
|
||||
value={settings.sampling.seed}
|
||||
onChange={(v) => updateSampling({ seed: v })}
|
||||
placeholder="Random"
|
||||
min={0}
|
||||
allowNull
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{samplingType === 'MirostatV2' && settings.sampling?.type === 'MirostatV2' && (
|
||||
<div className="grid grid-cols-2 gap-3">
|
||||
<NumberField
|
||||
label="Tau (target entropy)"
|
||||
value={settings.sampling.tau}
|
||||
onChange={(v) => updateSampling({ tau: v ?? 5.0 })}
|
||||
min={0}
|
||||
step={0.1}
|
||||
/>
|
||||
<NumberField
|
||||
label="Eta (learning rate)"
|
||||
value={settings.sampling.eta}
|
||||
onChange={(v) => updateSampling({ eta: v ?? 0.1 })}
|
||||
min={0}
|
||||
max={1}
|
||||
step={0.01}
|
||||
/>
|
||||
<NumberField
|
||||
label="Seed"
|
||||
value={settings.sampling.seed}
|
||||
onChange={(v) => updateSampling({ seed: v })}
|
||||
placeholder="Random"
|
||||
min={0}
|
||||
allowNull
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Repetition Penalty */}
|
||||
<div className="space-y-2">
|
||||
<h5 className="text-xs font-medium text-text-default">Repetition Penalty</h5>
|
||||
<div className="grid grid-cols-2 gap-3">
|
||||
<NumberField
|
||||
label="Repeat penalty"
|
||||
description="1.0 = off"
|
||||
value={settings.repeat_penalty}
|
||||
onChange={(v) => updateField('repeat_penalty', v ?? 1.0)}
|
||||
min={0}
|
||||
step={0.05}
|
||||
/>
|
||||
<NumberField
|
||||
label="Repeat window"
|
||||
description="Tokens to look back"
|
||||
value={settings.repeat_last_n}
|
||||
onChange={(v) => updateField('repeat_last_n', v ?? 64)}
|
||||
min={0}
|
||||
/>
|
||||
<NumberField
|
||||
label="Frequency penalty"
|
||||
description="0.0 = off"
|
||||
value={settings.frequency_penalty}
|
||||
onChange={(v) => updateField('frequency_penalty', v ?? 0.0)}
|
||||
min={0}
|
||||
max={2}
|
||||
step={0.05}
|
||||
/>
|
||||
<NumberField
|
||||
label="Presence penalty"
|
||||
description="0.0 = off"
|
||||
value={settings.presence_penalty}
|
||||
onChange={(v) => updateField('presence_penalty', v ?? 0.0)}
|
||||
min={0}
|
||||
max={2}
|
||||
step={0.05}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Performance */}
|
||||
<div className="space-y-2">
|
||||
<h5 className="text-xs font-medium text-text-default">Performance</h5>
|
||||
<div className="grid grid-cols-2 gap-3">
|
||||
<NumberField
|
||||
label="Batch size"
|
||||
description="Prompt processing batch"
|
||||
value={settings.n_batch}
|
||||
onChange={(v) => updateField('n_batch', v)}
|
||||
placeholder="Auto"
|
||||
min={1}
|
||||
allowNull
|
||||
/>
|
||||
<NumberField
|
||||
label="GPU layers"
|
||||
description="Layers to offload to GPU"
|
||||
value={settings.n_gpu_layers}
|
||||
onChange={(v) => updateField('n_gpu_layers', v)}
|
||||
placeholder="All"
|
||||
min={0}
|
||||
allowNull
|
||||
/>
|
||||
<NumberField
|
||||
label="Threads"
|
||||
description="CPU threads for generation"
|
||||
value={settings.n_threads}
|
||||
onChange={(v) => updateField('n_threads', v)}
|
||||
placeholder="Auto"
|
||||
min={1}
|
||||
allowNull
|
||||
/>
|
||||
</div>
|
||||
<ToggleField
|
||||
label="Lock model in RAM (mlock)"
|
||||
description="Prevent model from being swapped to disk"
|
||||
value={settings.use_mlock ?? false}
|
||||
onChange={(v) => updateField('use_mlock', v)}
|
||||
/>
|
||||
<SelectField
|
||||
label="Flash attention"
|
||||
description="Enable flash attention optimization"
|
||||
value={settings.flash_attention === null || settings.flash_attention === undefined ? 'auto' : settings.flash_attention ? 'on' : 'off'}
|
||||
options={[
|
||||
{ value: 'auto', label: 'Auto' },
|
||||
{ value: 'on', label: 'On' },
|
||||
{ value: 'off', label: 'Off' },
|
||||
]}
|
||||
onChange={(v) => updateField('flash_attention', v === 'auto' ? null : v === 'on')}
|
||||
/>
|
||||
</div>
|
||||
{/* Tool Calling */}
|
||||
<div className="space-y-2">
|
||||
<h5 className="text-xs font-medium text-text-default">Tool Calling</h5>
|
||||
<ToggleField
|
||||
label="Native tool calling"
|
||||
description="Use the model's built-in tool-call format instead of the shell-command emulator. Enable for large models that reliably support tool calling."
|
||||
value={settings.native_tool_calling ?? false}
|
||||
onChange={(v) => updateField('native_tool_calling', v)}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
Reference in New Issue
Block a user