tkmind/tkmind_go
1
0
Fork 0
Code Issues Pull Requests Actions 5 Packages Projects Releases Wiki Activity

Native images (#6619)

Browse Source 
Co-authored-by: Douwe Osinga <douwe@squareup.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
This commit is contained in:
Douwe Osinga
2026-01-22 18:53:57 -05:00
committed by GitHub
parent 4578c77576
commit f8faf9620a
23 changed files with 398 additions and 602 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ui/desktop/src/App.tsx
+14 -5
View File
@@ -20,10 +20,11 @@ import { createSession } from './sessions';
import { ChatType } from './types/chat';
import Hub from './components/Hub';
import { UserInput } from './types/message';
interface PairRouteState {
resumeSessionId?: string;
initialMessage?: string;
initialMessage?: UserInput;
}
import SettingsView, { SettingsViewOptions } from './components/settings/SettingsView';
import SessionsView from './components/sessions/SessionsView';
@@ -68,8 +69,13 @@ const HubRouteWrapper = () => {
const PairRouteWrapper = ({
activeSessions,
}: {
activeSessions: Array<{ sessionId: string; initialMessage?: string }>;
setActiveSessions: (sessions: Array<{ sessionId: string; initialMessage?: string }>) => void;
activeSessions: Array<{
sessionId: string;
initialMessage?: UserInput;
}>;
setActiveSessions: (
sessions: Array<{ sessionId: string; initialMessage?: UserInput }>
) => void;
}) => {
const { extensionsList } = useConfig();
const location = useLocation();
@@ -354,13 +360,16 @@ export function AppInner() {
const MAX_ACTIVE_SESSIONS = 10;
const [activeSessions, setActiveSessions] = useState<
Array<{ sessionId: string; initialMessage?: string }>
Array<{ sessionId: string; initialMessage?: UserInput }>
>([]);
useEffect(() => {
const handleAddActiveSession = (event: Event) => {
const { sessionId, initialMessage } = (
event as CustomEvent<{ sessionId: string; initialMessage?: string }>
event as CustomEvent<{
sessionId: string;
initialMessage?: UserInput;
}>
).detail;
setActiveSessions((prev) => {
ui/desktop/src/components/BaseChat.tsx
+13 -20
View File
@@ -28,17 +28,18 @@ import { useNavigation } from '../hooks/useNavigation';
import { RecipeHeader } from './RecipeHeader';
import { RecipeWarningModal } from './ui/RecipeWarningModal';
import { scanRecipe } from '../recipe';
import { UserInput } from '../types/message';
import { useCostTracking } from '../hooks/useCostTracking';
import RecipeActivities from './recipes/RecipeActivities';
import { useToolCount } from './alerts/useToolCount';
import { getThinkingMessage, getTextContent } from '../types/message';
import { getThinkingMessage, getTextAndImageContent } from '../types/message';
import ParameterInputModal from './ParameterInputModal';
import { substituteParameters } from '../utils/providerUtils';
import CreateRecipeFromSessionModal from './recipes/CreateRecipeFromSessionModal';
import { toastSuccess } from '../toasts';
import { Recipe } from '../recipe';
import { useAutoSubmit } from '../hooks/useAutoSubmit';
import { Goose } from './icons/Goose';
import { Goose } from './icons';
import EnvironmentBadge from './GooseSidebar/EnvironmentBadge';
const CurrentModelContext = createContext<{ model: string; mode: string } | null>(null);
@@ -56,10 +57,10 @@ interface BaseChatProps {
suppressEmptyState: boolean;
sessionId: string;
isActiveSession: boolean;
initialMessage?: string;
initialMessage?: UserInput;
}
function BaseChatContent({
export default function BaseChat({
setChat,
renderHeader,
customChatInputProps = {},
@@ -149,7 +150,7 @@ function BaseChatContent({
return messages
.reduce<string[]>((history, message) => {
if (message.role === 'user') {
const text = getTextContent(message).trim();
const text = getTextAndImageContent(message).textContent.trim();
if (text) {
history.push(text);
}
@@ -159,14 +160,11 @@ function BaseChatContent({
.reverse();
}, [messages]);
const handleFormSubmit = (e: React.FormEvent) => {
const customEvent = e as unknown as CustomEvent;
const textValue = customEvent.detail?.value || '';
if (recipe && textValue.trim()) {
const chatInputSubmit = (input: UserInput) => {
if (recipe && input.msg.trim()) {
setHasStartedUsingRecipe(true);
}
handleSubmit(textValue);
handleSubmit(input);
};
const { sessionCosts } = useCostTracking({
@@ -326,7 +324,6 @@ function BaseChatContent({
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [session?.name, setChat]);
// Only use initialMessage for the prompt if it hasn't been submitted yet
// If we have a recipe prompt and user recipe values, substitute parameters
let recipePrompt = '';
if (messages.length === 0 && recipe?.prompt) {
@@ -413,7 +410,7 @@ function BaseChatContent({
{recipe && (
<div className={hasStartedUsingRecipe ? 'mb-6' : ''}>
<RecipeActivities
append={(text: string) => handleSubmit(text)}
append={(text: string) => handleSubmit({ msg: text, images: [] })}
activities={Array.isArray(recipe.activities) ? recipe.activities : null}
title={recipe.title}
parameterValues={session?.user_recipe_values || {}}
@@ -428,7 +425,7 @@ function BaseChatContent({
messages={messages}
chat={{ sessionId }}
toolCallNotifications={toolCallNotifications}
append={(text: string) => handleSubmit(text)}
append={(text: string) => handleSubmit({ msg: text, images: [] })}
isUserMessage={(m: Message) => m.role === 'user'}
isStreamingMessage={chatState !== ChatState.Idle}
onRenderingComplete={handleRenderingComplete}
@@ -440,7 +437,7 @@ function BaseChatContent({
<div className="block h-8" />
</>
) : !recipe && showPopularTopics ? (
<PopularChatTopics append={(text: string) => handleSubmit(text)} />
<PopularChatTopics append={(text: string) => handleSubmit({ msg: text, images: [] })} />
) : null}
</ScrollArea>
@@ -464,7 +461,7 @@ function BaseChatContent({
<ChatInput
inputRef={chatInputRef}
sessionId={sessionId}
handleSubmit={handleFormSubmit}
handleSubmit={chatInputSubmit}
chatState={chatState}
setChatState={setChatState}
onStop={stopStreaming}
@@ -527,7 +524,3 @@ function BaseChatContent({
</div>
);
}
export default function BaseChat(props: BaseChatProps) {
return <BaseChatContent {...props} />;
}
ui/desktop/src/components/ChatInput.tsx
+172 -237
View File
@@ -25,7 +25,7 @@ import { COST_TRACKING_ENABLED, VOICE_DICTATION_ELEVENLABS_ENABLED } from '../up
import { CostTracker } from './bottom_menu/CostTracker';
import { DroppedFile, useFileDrop } from '../hooks/useFileDrop';
import { Recipe } from '../recipe';
import MessageQueue from './MessageQueue';
import { MessageQueue, QueuedMessage } from './MessageQueue';
import { detectInterruption } from '../utils/interruptionDetector';
import { DiagnosticsModal } from './ui/Diagnostics';
import { getSession, Message } from '../api';
@@ -41,24 +41,17 @@ import {
trackEditRecipeOpened,
} from '../utils/analytics';
import { getNavigationShortcutText } from '../utils/keyboardShortcuts';
interface QueuedMessage {
id: string;
content: string;
timestamp: number;
}
import { UserInput, ImageData } from '../types/message';
import { compressImageDataUrl } from '../utils/conversionUtils';
interface PastedImage {
id: string;
dataUrl: string; // For immediate preview
filePath?: string; // Path on filesystem after saving
dataUrl: string;
isLoading: boolean;
error?: string;
}
// Constants for image handling
const MAX_IMAGES_PER_MESSAGE = 5;
const MAX_IMAGE_SIZE_MB = 5;
const MAX_IMAGES_PER_MESSAGE = 10;
// Constants for token and tool alerts
const TOKEN_LIMIT_DEFAULT = 128000; // fallback for custom models that the backend doesn't know about
@@ -74,7 +67,7 @@ interface ModelLimit {
interface ChatInputProps {
sessionId: string | null;
handleSubmit: (e: React.FormEvent) => void;
handleSubmit: (input: UserInput) => void;
chatState: ChatState;
setChatState?: (state: ChatState) => void;
onStop?: () => void;
@@ -219,11 +212,7 @@ export default function ChatInput({
if (shouldProcessQueue) {
const nextMessage = queuedMessages[0];
LocalMessageStorage.addMessage(nextMessage.content);
handleSubmit(
new CustomEvent('submit', {
detail: { value: nextMessage.content },
}) as unknown as React.FormEvent
);
handleSubmit({ msg: nextMessage.content, images: nextMessage.images });
setQueuedMessages((prev) => {
const newQueue = prev.slice(1);
// If queue becomes empty after processing, clear the paused state
@@ -309,15 +298,7 @@ export default function ChatInput({
useEffect(() => {
setValue(initialValue);
setDisplayValue(initialValue);
setPastedImages((currentPastedImages) => {
currentPastedImages.forEach((img) => {
if (img.filePath) {
window.electron.deleteTempFile(img.filePath);
}
});
return [];
});
setPastedImages([]);
setHistoryIndex(-1);
setIsInGlobalHistory(false);
setHasUserTyped(false);
@@ -366,43 +347,9 @@ export default function ChatInput({
};
const handleRemovePastedImage = (idToRemove: string) => {
const imageToRemove = pastedImages.find((img) => img.id === idToRemove);
if (imageToRemove?.filePath) {
window.electron.deleteTempFile(imageToRemove.filePath);
}
setPastedImages((currentImages) => currentImages.filter((img) => img.id !== idToRemove));
};
const handleRetryImageSave = async (imageId: string) => {
const imageToRetry = pastedImages.find((img) => img.id === imageId);
if (!imageToRetry || !imageToRetry.dataUrl) return;
// Set the image to loading state
setPastedImages((prev) =>
prev.map((img) => (img.id === imageId ? { ...img, isLoading: true, error: undefined } : img))
);
try {
const result = await window.electron.saveDataUrlToTemp(imageToRetry.dataUrl, imageId);
setPastedImages((prev) =>
prev.map((img) =>
img.id === result.id
? { ...img, filePath: result.filePath, error: result.error, isLoading: false }
: img
)
);
} catch (err) {
console.error('Error retrying image save:', err);
setPastedImages((prev) =>
prev.map((img) =>
img.id === imageId
? { ...img, error: 'Failed to save image via Electron.', isLoading: false }
: img
)
);
}
};
useEffect(() => {
if (textAreaRef.current) {
textAreaRef.current.focus();
@@ -423,7 +370,6 @@ export default function ChatInput({
return [];
};
// Helper function to find model limit using pattern matching
const findModelLimit = (modelName: string, modelLimits: ModelLimit[]): number | null => {
if (!modelName) return null;
const matchingLimit = modelLimits.find((limit) =>
@@ -512,12 +458,7 @@ export default function ChatInput({
compactButtonDisabled: !totalTokens,
onCompact: () => {
window.dispatchEvent(new CustomEvent(AppEvents.HIDE_ALERT_POPOVER));
const customEvent = new CustomEvent('submit', {
detail: { value: MANUAL_COMPACT_TRIGGER },
}) as unknown as React.FormEvent;
handleSubmit(customEvent);
handleSubmit({ msg: MANUAL_COMPACT_TRIGGER, images: [] });
},
compactIcon: <ScrollText size={12} />,
});
@@ -542,20 +483,6 @@ export default function ChatInput({
// Cleanup effect for component unmount - prevent memory leaks
useEffect(() => {
return () => {
// Clear any pending timeouts from image processing
setPastedImages((currentImages) => {
currentImages.forEach((img) => {
if (img.filePath) {
try {
window.electron.deleteTempFile(img.filePath);
} catch (error) {
console.error('Error deleting temp file:', error);
}
}
});
return [];
});
// Clear all tracked timeouts
// eslint-disable-next-line react-hooks/exhaustive-deps
const timeouts = timeoutRefsRef.current;
@@ -659,6 +586,62 @@ export default function ChatInput({
}));
};
const convertImagesToImageData = useCallback((): ImageData[] => {
const pastedImageData: ImageData[] = pastedImages
.filter((img) => img.dataUrl && !img.error && !img.isLoading)
.map((img) => {
const matches = img.dataUrl.match(/^data:([^;]+);base64,(.+)$/);
if (matches) {
return {
data: matches[2],
mimeType: matches[1],
};
}
return null;
})
.filter((img): img is ImageData => img !== null);
const droppedImageData: ImageData[] = allDroppedFiles
.filter((file) => file.isImage && file.dataUrl && !file.error && !file.isLoading)
.map((file) => {
const matches = file.dataUrl!.match(/^data:([^;]+);base64,(.+)$/);
if (matches) {
return {
data: matches[2],
mimeType: matches[1],
};
}
return null;
})
.filter((img): img is ImageData => img !== null);
return [...pastedImageData, ...droppedImageData];
}, [pastedImages, allDroppedFiles]);
const appendDroppedFilePaths = useCallback((text: string): string => {
const droppedFilePaths = allDroppedFiles
.filter((file) => !file.isImage && !file.error && !file.isLoading)
.map((file) => file.path);
if (droppedFilePaths.length > 0) {
const pathsString = droppedFilePaths.join(' ');
return text ? `${text} ${pathsString}` : pathsString;
}
return text;
}, [allDroppedFiles]);
const clearInputState = useCallback(() => {
setDisplayValue('');
setValue('');
setPastedImages([]);
if (onFilesProcessed && droppedFiles.length > 0) {
onFilesProcessed();
}
if (localDroppedFiles.length > 0) {
setLocalDroppedFiles([]);
}
}, [droppedFiles.length, localDroppedFiles.length, onFilesProcessed, setLocalDroppedFiles]);
const handlePaste = async (evt: React.ClipboardEvent<HTMLTextAreaElement>) => {
const files = Array.from(evt.clipboardData.files || []);
const imageFiles = files.filter((file) => file.type.startsWith('image/'));
@@ -694,26 +677,6 @@ export default function ChatInput({
const newImages: PastedImage[] = [];
for (const file of imageFiles) {
// Check individual file size before processing
if (file.size > MAX_IMAGE_SIZE_MB * 1024 * 1024) {
const errorId = `error-${Date.now()}-${Math.random().toString(36).substring(2, 9)}`;
newImages.push({
id: errorId,
dataUrl: '',
isLoading: false,
error: `Image too large (${Math.round(file.size / (1024 * 1024))}MB). Maximum ${MAX_IMAGE_SIZE_MB}MB allowed.`,
});
// Remove the error message after 5 seconds with cleanup tracking
const timeoutId = setTimeout(() => {
setPastedImages((prev) => prev.filter((img) => img.id !== errorId));
timeoutRefsRef.current.delete(timeoutId);
}, 5000);
timeoutRefsRef.current.add(timeoutId);
continue;
}
const imageId = `img-${Date.now()}-${Math.random().toString(36).substring(2, 9)}`;
// Add the image with loading state
@@ -728,30 +691,12 @@ export default function ChatInput({
reader.onload = async (e) => {
const dataUrl = e.target?.result as string;
if (dataUrl) {
// Update the image with the data URL
const compressedDataUrl = await compressImageDataUrl(dataUrl);
setPastedImages((prev) =>
prev.map((img) => (img.id === imageId ? { ...img, dataUrl, isLoading: true } : img))
prev.map((img) =>
img.id === imageId ? { ...img, dataUrl: compressedDataUrl, isLoading: false } : img
)
);
try {
const result = await window.electron.saveDataUrlToTemp(dataUrl, imageId);
setPastedImages((prev) =>
prev.map((img) =>
img.id === result.id
? { ...img, filePath: result.filePath, error: result.error, isLoading: false }
: img
)
);
} catch (err) {
console.error('Error saving pasted image:', err);
setPastedImages((prev) =>
prev.map((img) =>
img.id === imageId
? { ...img, error: 'Failed to save image via Electron.', isLoading: false }
: img
)
);
}
}
};
reader.onerror = () => {
@@ -865,25 +810,13 @@ export default function ChatInput({
}
};
// Helper function to handle interruption and queue logic when loading
const handleInterruptionAndQueue = () => {
if (!isLoading || !hasSubmittableContent) {
return false;
}
const validPastedImageFilesPaths = pastedImages
.filter((img) => img.filePath && !img.error && !img.isLoading)
.map((img) => img.filePath as string);
const droppedFilePaths = allDroppedFiles
.filter((file) => !file.error && !file.isLoading)
.map((file) => file.path);
let contentToQueue = displayValue.trim();
const allFilePaths = [...validPastedImageFilesPaths, ...droppedFilePaths];
if (allFilePaths.length > 0) {
const pathsString = allFilePaths.join(' ');
contentToQueue = contentToQueue ? `${contentToQueue} ${pathsString}` : pathsString;
}
const imageData = convertImagesToImageData();
const contentToQueue = appendDroppedFilePaths(displayValue.trim());
const interruptionMatch = detectInterruption(displayValue.trim());
@@ -894,31 +827,25 @@ export default function ChatInput({
// For interruptions, we need to queue the message to be sent after the stop completes
// rather than trying to send it immediately while the system is still loading
const interruptionMessage = {
const interruptionMessage: QueuedMessage = {
id: Date.now().toString() + Math.random().toString(36).substr(2, 9),
content: contentToQueue,
timestamp: Date.now(),
images: imageData,
};
// Add the interruption message to the front of the queue so it gets sent first
setQueuedMessages((prev) => [interruptionMessage, ...prev]);
setDisplayValue('');
setValue('');
setPastedImages([]);
if (onFilesProcessed && droppedFiles.length > 0) {
onFilesProcessed();
}
if (localDroppedFiles.length > 0) {
setLocalDroppedFiles([]);
}
clearInputState();
return true;
}
const newMessage = {
const newMessage: QueuedMessage = {
id: Date.now().toString() + Math.random().toString(36).substr(2, 9),
content: contentToQueue,
timestamp: Date.now(),
images: imageData,
};
setQueuedMessages((prev) => {
const newQueue = [...prev, newMessage];
@@ -929,53 +856,35 @@ export default function ChatInput({
}
return newQueue;
});
setDisplayValue('');
setValue('');
setPastedImages([]);
if (onFilesProcessed && droppedFiles.length > 0) {
onFilesProcessed();
}
if (localDroppedFiles.length > 0) {
setLocalDroppedFiles([]);
}
clearInputState();
return true;
};
const canSubmit =
!isLoading &&
(displayValue.trim() ||
pastedImages.some((img) => img.filePath && !img.error && !img.isLoading) ||
pastedImages.some((img) => img.dataUrl && !img.error && !img.isLoading) ||
allDroppedFiles.some((file) => !file.error && !file.isLoading));
const performSubmit = useCallback(
(text?: string) => {
const validPastedImageFilesPaths = pastedImages
.filter((img) => img.filePath && !img.error && !img.isLoading)
.map((img) => img.filePath as string);
// Get paths from all dropped files (both parent and local)
const droppedFilePaths = allDroppedFiles
.filter((file) => !file.error && !file.isLoading)
.map((file) => file.path);
const imageData = convertImagesToImageData();
const textToSend = appendDroppedFilePaths(text ?? displayValue.trim());
let textToSend = text ?? displayValue.trim();
// Combine pasted images and dropped files
const allFilePaths = [...validPastedImageFilesPaths, ...droppedFilePaths];
if (allFilePaths.length > 0) {
const pathsString = allFilePaths.join(' ');
textToSend = textToSend ? `${textToSend} ${pathsString}` : pathsString;
}
if (textToSend) {
if (textToSend || imageData.length > 0) {
// Store original message in history
if (displayValue.trim()) {
LocalMessageStorage.addMessage(displayValue);
} else if (allFilePaths.length > 0) {
LocalMessageStorage.addMessage(allFilePaths.join(' '));
} else {
const droppedFilePaths = allDroppedFiles
.filter((file) => !file.isImage && !file.error && !file.isLoading)
.map((file) => file.path);
if (droppedFilePaths.length > 0) {
LocalMessageStorage.addMessage(droppedFilePaths.join(' '));
}
}
handleSubmit(
new CustomEvent('submit', { detail: { value: textToSend } }) as unknown as React.FormEvent
);
handleSubmit({ msg: textToSend, images: imageData });
// Auto-resume queue after sending a NON-interruption message (if it was paused due to interruption)
if (
@@ -988,33 +897,21 @@ export default function ChatInput({
setLastInterruption(null);
}
setDisplayValue('');
setValue('');
setPastedImages([]);
clearInputState();
setHistoryIndex(-1);
setSavedInput('');
setIsInGlobalHistory(false);
setHasUserTyped(false);
// Clear both parent and local dropped files after processing
if (onFilesProcessed && droppedFiles.length > 0) {
onFilesProcessed();
}
if (localDroppedFiles.length > 0) {
setLocalDroppedFiles([]);
}
}
},
[
allDroppedFiles,
convertImagesToImageData,
appendDroppedFilePaths,
displayValue,
droppedFiles.length,
allDroppedFiles,
handleSubmit,
lastInterruption,
localDroppedFiles.length,
onFilesProcessed,
pastedImages,
setLocalDroppedFiles,
clearInputState,
]
);
@@ -1090,29 +987,85 @@ export default function ChatInput({
const canSubmit =
!isLoading &&
(displayValue.trim() ||
pastedImages.some((img) => img.filePath && !img.error && !img.isLoading) ||
pastedImages.some((img) => img.dataUrl && !img.error && !img.isLoading) ||
allDroppedFiles.some((file) => !file.error && !file.isLoading));
if (canSubmit) {
performSubmit();
}
};
const handleFileSelect = async () => {
if (isFilePickerOpen) return;
setIsFilePickerOpen(true);
try {
const path = await window.electron.selectFileOrDirectory();
if (path) {
const isDirectory = !path.includes('.') || path.endsWith('/');
trackFileAttached(isDirectory ? 'directory' : 'file');
const fileInputRef = React.useRef<HTMLInputElement>(null);
const newValue = displayValue.trim() ? `${displayValue.trim()} ${path}` : path;
setDisplayValue(newValue);
setValue(newValue);
textAreaRef.current?.focus();
const handleFileSelect = () => {
if (isFilePickerOpen) return;
fileInputRef.current?.click();
};
const handleFileInputChange = async (e: React.ChangeEvent<HTMLInputElement>) => {
const files = e.target.files;
if (!files || files.length === 0) return;
setIsFilePickerOpen(true);
const file = files[0];
const isImage = file.type.startsWith('image/');
if (isImage) {
trackFileAttached('file');
if (pastedImages.length >= MAX_IMAGES_PER_MESSAGE) {
console.warn(`Maximum ${MAX_IMAGES_PER_MESSAGE} images per message`);
setIsFilePickerOpen(false);
return;
}
} finally {
setIsFilePickerOpen(false);
const uniqueId = `upload-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
setPastedImages((prev) => [
...prev,
{
id: uniqueId,
dataUrl: '',
isLoading: true,
error: undefined,
},
]);
const reader = new FileReader();
reader.onload = async (evt) => {
const dataUrl = evt.target?.result as string;
if (dataUrl) {
const compressedDataUrl = await compressImageDataUrl(dataUrl);
setPastedImages((prev) =>
prev.map((img) =>
img.id === uniqueId
? { ...img, dataUrl: compressedDataUrl, isLoading: false, error: undefined }
: img
)
);
}
};
reader.onerror = () => {
setPastedImages((prev) =>
prev.map((img) =>
img.id === uniqueId
? { ...img, isLoading: false, error: 'Failed to read image file' }
: img
)
);
};
reader.readAsDataURL(file);
} else {
trackFileAttached('file');
const path = window.electron.getPathForFile(file);
const newValue = displayValue.trim() ? `${displayValue.trim()} ${path}` : path;
setDisplayValue(newValue);
setValue(newValue);
}
textAreaRef.current?.focus();
setIsFilePickerOpen(false);
if (e.target) {
e.target.value = '';
}
};
@@ -1140,7 +1093,7 @@ export default function ChatInput({
const hasSubmittableContent =
displayValue.trim() ||
pastedImages.some((img) => img.filePath && !img.error && !img.isLoading) ||
pastedImages.some((img) => img.dataUrl && !img.error && !img.isLoading) ||
allDroppedFiles.some((file) => !file.error && !file.isLoading);
const isAnyImageLoading = pastedImages.some((img) => img.isLoading);
const isAnyDroppedFileLoading = allDroppedFiles.some((file) => file.isLoading);
@@ -1196,11 +1149,7 @@ export default function ChatInput({
// Remove the message from queue and send it immediately
setQueuedMessages((prev) => prev.filter((msg) => msg.id !== messageId));
LocalMessageStorage.addMessage(messageToSend.content);
handleSubmit(
new CustomEvent('submit', {
detail: { value: messageToSend.content },
}) as unknown as React.FormEvent
);
handleSubmit({ msg: messageToSend.content, images: messageToSend.images });
// Restore previous pause state after a brief delay to prevent race condition
setTimeout(() => {
@@ -1214,11 +1163,7 @@ export default function ChatInput({
if (!isLoading && queuedMessages.length > 0) {
const nextMessage = queuedMessages[0];
LocalMessageStorage.addMessage(nextMessage.content);
handleSubmit(
new CustomEvent('submit', {
detail: { value: nextMessage.content },
}) as unknown as React.FormEvent
);
handleSubmit({ msg: nextMessage.content, images: nextMessage.images });
setQueuedMessages((prev) => {
const newQueue = prev.slice(1);
// If queue becomes empty after processing, clear the paused state
@@ -1244,6 +1189,7 @@ export default function ChatInput({
onDrop={handleLocalDrop}
onDragOver={handleLocalDragOver}
>
<input ref={fileInputRef} type="file" onChange={handleFileInputChange} style={{ display: 'none' }} accept="*/*" />
{/* Message Queue Display */}
{queuedMessages.length > 0 && (
<MessageQueue
@@ -1451,20 +1397,9 @@ export default function ChatInput({
)}
{img.error && !img.isLoading && (
<div className="absolute inset-0 flex flex-col items-center justify-center bg-black bg-opacity-75 rounded p-1 text-center">
<p className="text-red-400 text-[10px] leading-tight break-all mb-1">
<p className="text-red-400 text-[10px] leading-tight break-all">
{img.error.substring(0, 50)}
</p>
{img.dataUrl && (
<Button
type="button"
onClick={() => handleRetryImageSave(img.id)}
title="Retry saving image"
variant="outline"
size="xs"
>
Retry
</Button>
)}
</div>
)}
{!img.isLoading && (
@@ -1570,7 +1505,7 @@ export default function ChatInput({
<Attach className="w-4 h-4" />
</Button>
</TooltipTrigger>
<TooltipContent>Attach file or directory</TooltipContent>
<TooltipContent>Attach file</TooltipContent>
</Tooltip>
<div className="w-px h-4 bg-border-default mx-2" />
{/* Model selector, mode selector, alerts, summarize button */}
ui/desktop/src/components/ChatSessionsContainer.tsx
+5 -1
View File
@@ -1,10 +1,14 @@
import { useSearchParams } from 'react-router-dom';
import BaseChat from './BaseChat';
import { ChatType } from '../types/chat';
import { UserInput } from '../types/message';
interface ChatSessionsContainerProps {
setChat: (chat: ChatType) => void;
activeSessions: Array<{ sessionId: string; initialMessage?: string }>;
activeSessions: Array<{
sessionId: string;
initialMessage?: UserInput;
}>;
}
/**
ui/desktop/src/components/GooseMessage.tsx
+13 -15
View File
@@ -1,11 +1,10 @@
import { useMemo, useRef } from 'react';
import ImagePreview from './ImagePreview';
import { extractImagePaths, removeImagePathsFromText } from '../utils/imageUtils';
import { formatMessageTimestamp } from '../utils/timeUtils';
import MarkdownContent from './MarkdownContent';
import ToolCallWithResponse from './ToolCallWithResponse';
import {
getTextContent,
getTextAndImageContent,
getToolRequests,
getToolResponses,
getToolConfirmationContent,
@@ -45,28 +44,25 @@ export default function GooseMessage({
}: GooseMessageProps) {
const contentRef = useRef<HTMLDivElement | null>(null);
let textContent = getTextContent(message);
let { textContent, imagePaths } = getTextAndImageContent(message);
const splitChainOfThought = (text: string): { visibleText: string; cotText: string | null } => {
const splitChainOfThought = (text: string): { displayText: string; cotText: string | null } => {
const regex = /<think>([\s\S]*?)<\/think>/i;
const match = text.match(regex);
if (!match) {
return { visibleText: text, cotText: null };
return { displayText: text, cotText: null };
}
const cotRaw = match[1].trim();
const visibleText = text.replace(regex, '').trim();
const displayText = text.replace(regex, '').trim();
return {
visibleText,
displayText,
cotText: cotRaw || null,
};
};
const { visibleText, cotText } = splitChainOfThought(textContent);
const imagePaths = extractImagePaths(visibleText);
const displayText =
imagePaths.length > 0 ? removeImagePathsFromText(visibleText, imagePaths) : visibleText;
const { displayText, cotText } = splitChainOfThought(textContent);
const timestamp = useMemo(() => formatMessageTimestamp(message.created), [message.created]);
const toolRequests = getToolRequests(message);
@@ -116,11 +112,13 @@ export default function GooseMessage({
</details>
)}
{displayText && (
{(displayText.trim() || imagePaths.length > 0) && (
<div className="flex flex-col group">
<div ref={contentRef} className="w-full">
<MarkdownContent content={displayText} />
</div>
{displayText.trim() && (
<div ref={contentRef} className="w-full">
<MarkdownContent content={displayText} />
</div>
)}
{imagePaths.length > 0 && (
<div className="mt-4">
ui/desktop/src/components/GooseSidebar/AppSidebar.tsx
+1 -1
View File
@@ -263,7 +263,7 @@ const AppSidebar: React.FC<SidebarProps> = ({ currentPath }) => {
let isPolling = false;
const handleSessionCreated = (event: Event) => {
const { session } = (event as CustomEvent<{ session?: Session }>).detail;
const { session } = (event as CustomEvent<{ session?: Session }>).detail || {};
// If session data is provided, add it immediately to the sidebar
// This is for displaying sessions that won't be returned by the API due to not having messages yet
if (session) {
ui/desktop/src/components/Hub.tsx
+6 -9
View File
@@ -29,6 +29,7 @@ import {
import { getInitialWorkingDir } from '../utils/workingDir';
import { createSession } from '../sessions';
import LoadingGoose from './LoadingGoose';
import { UserInput } from '../types/message';
export default function Hub({
setView,
@@ -39,11 +40,9 @@ export default function Hub({
const [workingDir, setWorkingDir] = useState(getInitialWorkingDir());
const [isCreatingSession, setIsCreatingSession] = useState(false);
const handleSubmit = async (e: React.FormEvent) => {
const customEvent = e as unknown as CustomEvent;
const combinedTextFromInput = customEvent.detail?.value || '';
if (combinedTextFromInput.trim() && !isCreatingSession) {
const handleSubmit = async (input: UserInput) => {
const { msg: userMessage, images } = input;
if ((images.length > 0 || userMessage.trim()) && !isCreatingSession) {
const extensionConfigs = getExtensionConfigsWithOverrides(extensionsList);
clearExtensionOverrides();
setIsCreatingSession(true);
@@ -57,21 +56,19 @@ export default function Hub({
window.dispatchEvent(new CustomEvent(AppEvents.SESSION_CREATED));
window.dispatchEvent(
new CustomEvent(AppEvents.ADD_ACTIVE_SESSION, {
detail: { sessionId: session.id, initialMessage: combinedTextFromInput },
detail: { sessionId: session.id, initialMessage: { msg: userMessage, images } },
})
);
setView('pair', {
disableAnimation: true,
resumeSessionId: session.id,
initialMessage: combinedTextFromInput,
initialMessage: { msg: userMessage, images },
});
} catch (error) {
console.error('Failed to create session:', error);
setIsCreatingSession(false);
}
e.preventDefault();
}
};
ui/desktop/src/components/ImagePreview.tsx
+17 -73
View File
@@ -1,88 +1,32 @@
import { useState, useEffect } from 'react';
import { useState } from 'react';
interface ImagePreviewProps {
src: string;
alt?: string;
className?: string;
}
export default function ImagePreview({
src,
alt = 'Pasted image',
className = '',
}: ImagePreviewProps) {
export default function ImagePreview({ src }: ImagePreviewProps) {
const [isExpanded, setIsExpanded] = useState(false);
const [error, setError] = useState(false);
const [isLoading, setIsLoading] = useState(true);
const [imageData, setImageData] = useState<string | null>(null);
useEffect(() => {
const loadImage = async () => {
try {
// Use the IPC handler to get the image data
const data = await window.electron.getTempImage(src);
if (data) {
setImageData(data);
setIsLoading(false);
} else {
setError(true);
setIsLoading(false);
}
} catch (err) {
console.error('Error loading image:', err);
setError(true);
setIsLoading(false);
}
};
loadImage();
}, [src]);
const handleError = () => {
setError(true);
setIsLoading(false);
};
const toggleExpand = () => {
if (!error) {
setIsExpanded(!isExpanded);
}
};
// Validate that this is a safe file path (should contain goose-pasted-images)
if (!src.includes('goose-pasted-images')) {
return <div className="text-red-500 text-xs italic mt-1 mb-1">Invalid image path: {src}</div>;
}
if (error) {
return <div className="text-red-500 text-xs italic mt-1 mb-1">Unable to load image: {src}</div>;
return <div className="text-red-500 text-xs italic mt-1 mb-1">Unable to load image</div>;
}
return (
<div className={`image-preview mt-2 mb-2 ${className}`}>
{isLoading && (
<div className="animate-pulse bg-gray-200 rounded w-40 h-40 flex items-center justify-center">
<span className="text-gray-500 text-xs">Loading...</span>
</div>
)}
{imageData && (
<img
src={imageData}
alt={alt}
onError={handleError}
onClick={toggleExpand}
className={`rounded border border-borderSubtle cursor-pointer hover:border-borderStandard transition-all ${
isExpanded ? 'max-w-full max-h-96' : 'max-h-40 max-w-40'
} ${isLoading ? 'hidden' : ''}`}
style={{ objectFit: 'contain' }}
/>
)}
{isExpanded && !error && !isLoading && imageData && (
<div className="text-xs text-textSubtle mt-1">Click to collapse</div>
)}
{!isExpanded && !error && !isLoading && imageData && (
<div className="text-xs text-textSubtle mt-1">Click to expand</div>
)}
<div className={`image-preview mt-2 mb-2`}>
<img
src={src}
alt="goose image"
onError={() => setError(true)}
onClick={() => setIsExpanded(!isExpanded)}
className={`rounded border border-borderSubtle cursor-pointer hover:border-borderStandard transition-all ${
isExpanded ? 'max-w-full max-h-96' : 'max-h-40 max-w-40'
}`}
style={{ objectFit: 'contain' }}
/>
<div className="text-xs text-textSubtle mt-1">
Click to {isExpanded ? 'collapse' : 'expand'}
</div>
</div>
);
}
ui/desktop/src/components/Layout/AppLayout.tsx
+9 -2
View File
@@ -7,9 +7,13 @@ import { Button } from '../ui/button';
import { Sidebar, SidebarInset, SidebarProvider, SidebarTrigger, useSidebar } from '../ui/sidebar';
import ChatSessionsContainer from '../ChatSessionsContainer';
import { useChatContext } from '../../contexts/ChatContext';
import { UserInput } from '../../types/message';
interface AppLayoutContentProps {
activeSessions: Array<{ sessionId: string; initialMessage?: string }>;
activeSessions: Array<{
sessionId: string;
initialMessage?: UserInput;
}>;
}
const AppLayoutContent: React.FC<AppLayoutContentProps> = ({ activeSessions }) => {
@@ -126,7 +130,10 @@ const AppLayoutContent: React.FC<AppLayoutContentProps> = ({ activeSessions }) =
};
interface AppLayoutProps {
activeSessions: Array<{ sessionId: string; initialMessage?: string }>;
activeSessions: Array<{
sessionId: string;
initialMessage?: UserInput;
}>;
}
export const AppLayout: React.FC<AppLayoutProps> = ({ activeSessions }) => {
ui/desktop/src/components/MessageQueue.tsx
+3 -1
View File
@@ -1,11 +1,13 @@
import React, { useState } from 'react';
import { X, Clock, Send, GripVertical, Zap, Sparkles, ChevronDown, ChevronUp } from 'lucide-react';
import { Button } from './ui/button';
import { ImageData } from '../types/message';
interface QueuedMessage {
export interface QueuedMessage {
id: string;
content: string;
timestamp: number;
images: ImageData[];
}
interface MessageQueueProps {
ui/desktop/src/components/UserMessage.tsx
+25 -37
View File
@@ -1,8 +1,7 @@
import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
import { useCallback, useEffect, useRef, useState } from 'react';
import ImagePreview from './ImagePreview';
import { extractImagePaths, removeImagePathsFromText } from '../utils/imageUtils';
import MarkdownContent from './MarkdownContent';
import { getTextContent } from '../types/message';
import { getTextAndImageContent } from '../types/message';
import { Message } from '../api';
import MessageCopyLink from './MessageCopyLink';
import { formatMessageTimestamp } from '../utils/timeUtils';
@@ -21,35 +20,23 @@ export default function UserMessage({ message, onMessageUpdate }: UserMessagePro
const [editContent, setEditContent] = useState('');
const [error, setError] = useState<string | null>(null);
// Extract text content from the message
const textContent = getTextContent(message);
// Extract image paths from the message
const imagePaths = extractImagePaths(textContent);
// Remove image paths from text for display - memoized for performance
const displayText = useMemo(
() => removeImagePathsFromText(textContent, imagePaths),
[textContent, imagePaths]
);
// Memoize the timestamp
const timestamp = useMemo(() => formatMessageTimestamp(message.created), [message.created]);
const { textContent, imagePaths } = getTextAndImageContent(message);
const timestamp = formatMessageTimestamp(message.created);
// Effect to handle message content changes and ensure persistence
useEffect(() => {
// If we're not editing, update the edit content to match the current message
if (!isEditing) {
setEditContent(displayText);
setEditContent(textContent);
}
}, [message.content, displayText, message.id, isEditing]);
}, [message.content, textContent, message.id, isEditing]);
// Initialize edit mode with current message content
const initializeEditMode = useCallback(() => {
setEditContent(displayText);
setEditContent(textContent);
setError(null);
window.electron.logInfo(`Entering edit mode with content: ${displayText}`);
}, [displayText]);
window.electron.logInfo(`Entering edit mode with content: ${textContent}`);
}, [textContent]);
// Handle edit button click
const handleEditClick = useCallback(() => {
@@ -93,7 +80,7 @@ export default function UserMessage({ message, onMessageUpdate }: UserMessagePro
setIsEditing(false);
if (editContent.trim() === displayText.trim()) {
if (editContent.trim() === textContent.trim()) {
return;
}
@@ -101,16 +88,16 @@ export default function UserMessage({ message, onMessageUpdate }: UserMessagePro
onMessageUpdate(message.id, editContent, editType);
}
},
[editContent, displayText, onMessageUpdate, message.id]
[editContent, textContent, onMessageUpdate, message.id]
);
// Handle cancel action
const handleCancel = useCallback(() => {
window.electron.logInfo('Cancel clicked - reverting to original content');
setIsEditing(false);
setEditContent(displayText); // Reset to original content
setEditContent(textContent); // Reset to original content
setError(null);
}, [displayText]);
}, [textContent]);
// Handle keyboard events for accessibility
const handleKeyDown = useCallback(
@@ -207,20 +194,21 @@ export default function UserMessage({ message, onMessageUpdate }: UserMessagePro
<div className="message flex justify-end w-full">
<div className="flex-col max-w-[85%] w-fit">
<div className="flex flex-col group">
<div className="flex bg-background-accent text-text-on-accent rounded-xl py-2.5 px-4">
<div ref={contentRef}>
<MarkdownContent
content={displayText}
className="text-text-on-accent prose-a:text-text-on-accent prose-headings:text-text-on-accent prose-strong:text-text-on-accent prose-em:text-text-on-accent user-message"
/>
{textContent.trim() && (
<div className="flex bg-background-accent text-text-on-accent rounded-xl py-2.5 px-4">
<div ref={contentRef}>
<MarkdownContent
content={textContent}
className="text-text-on-accent prose-a:text-text-on-accent prose-headings:text-text-on-accent prose-strong:text-text-on-accent prose-em:text-text-on-accent user-message"
/>
</div>
</div>
</div>
)}
{/* Render images if any */}
{imagePaths.length > 0 && (
<div className="flex flex-wrap gap-2 mt-2">
{imagePaths.map((imagePath, index) => (
<ImagePreview key={index} src={imagePath} alt={`Pasted image ${index + 1}`} />
<ImagePreview key={index} src={imagePath} />
))}
</div>
)}
@@ -239,14 +227,14 @@ export default function UserMessage({ message, onMessageUpdate }: UserMessagePro
}
}}
className="flex items-center gap-1 text-xs text-text-subtle hover:cursor-pointer hover:text-text-prominent transition-all duration-200 opacity-0 group-hover:opacity-100 -translate-y-4 group-hover:translate-y-0 focus:outline-none focus:ring-2 focus:ring-blue-400 focus:ring-opacity-50 rounded"
aria-label={`Edit message: ${displayText.substring(0, 50)}${displayText.length > 50 ? '...' : ''}`}
aria-label={`Edit message: ${textContent.substring(0, 50)}${textContent.length > 50 ? '...' : ''}`}
aria-expanded={isEditing}
title="Edit message"
>
<Edit className="h-3 w-3" />
<span>Edit</span>
</button>
<MessageCopyLink text={displayText} contentRef={contentRef} />
<MessageCopyLink text={textContent} contentRef={contentRef} />
</div>
</div>
</div>
ui/desktop/src/components/sessions/SessionViewComponents.tsx
+5 -20
View File
@@ -7,12 +7,11 @@ import MarkdownContent from '../MarkdownContent';
import ToolCallWithResponse from '../ToolCallWithResponse';
import ImagePreview from '../ImagePreview';
import {
getTextContent,
getTextAndImageContent,
ToolRequestMessageContent,
ToolResponseMessageContent,
} from '../../types/message';
import { formatMessageTimestamp } from '../../utils/timeUtils';
import { extractImagePaths, removeImagePathsFromText } from '../../utils/imageUtils';
import { Message } from '../../api';
/**
@@ -82,15 +81,7 @@ export const SessionMessages: React.FC<SessionMessagesProps> = ({
) : messages?.length > 0 ? (
messages
.map((message, index) => {
const textContent = getTextContent(message);
// Extract image paths from the message
const imagePaths = extractImagePaths(textContent);
// Remove image paths from text for display
const displayText =
imagePaths.length > 0
? removeImagePathsFromText(textContent, imagePaths)
: textContent;
const { textContent, imagePaths } = getTextAndImageContent(message);
// Get tool requests from the message
const toolRequests = message.content
@@ -128,24 +119,18 @@ export const SessionMessages: React.FC<SessionMessagesProps> = ({
</div>
<div className="flex flex-col w-full">
{/* Text content */}
{displayText && (
{textContent && (
<div
className={`${toolRequests.length > 0 || imagePaths.length > 0 ? 'mb-4' : ''}`}
>
<MarkdownContent content={displayText} />
<MarkdownContent content={textContent} />
</div>
)}
{/* Render images if any */}
{imagePaths.length > 0 && (
<div className="flex flex-wrap gap-2 mt-2 mb-2">
{imagePaths.map((imagePath, imageIndex) => (
<ImagePreview
key={imageIndex}
src={imagePath}
alt={`Image ${imageIndex + 1}`}
/>
<ImagePreview key={imageIndex} src={imagePath} />
))}
</div>
)}
ui/desktop/src/hooks/useAutoSubmit.ts
+4 -3
View File
@@ -4,6 +4,7 @@ import { useSearchParams } from 'react-router-dom';
import { Session } from '../api';
import { Message } from '../api';
import { ChatState } from '../types/chatState';
import { UserInput } from '../types/message';
/**
* Auto-submit scenarios:
@@ -17,8 +18,8 @@ interface UseAutoSubmitProps {
session: Session | undefined;
messages: Message[];
chatState: ChatState;
initialMessage: string | undefined;
handleSubmit: (message: string) => void;
initialMessage: UserInput | undefined;
handleSubmit: (input: UserInput) => void;
}
interface UseAutoSubmitReturn {
@@ -84,7 +85,7 @@ export function useAutoSubmit({
// Scenario 3: Resume with shouldStartAgent (continue existing conversation)
if (shouldStartAgent) {
hasAutoSubmittedRef.current = true;
handleSubmit('');
handleSubmit({ msg: '', images: [] });
}
}, [
session,
ui/desktop/src/hooks/useChatStream.ts
+7 -5
View File
@@ -21,6 +21,7 @@ import {
getCompactingMessage,
getThinkingMessage,
NotificationEvent,
UserInput,
} from '../types/message';
import { errorMessage } from '../utils/conversionUtils';
import { showExtensionLoadResults } from '../utils/extensionErrorUtils';
@@ -39,7 +40,7 @@ interface UseChatStreamReturn {
messages: Message[];
chatState: ChatState;
setChatState: (state: ChatState) => void;
handleSubmit: (userMessage: string) => Promise<void>;
handleSubmit: (input: UserInput) => Promise<void>;
submitElicitationResponse: (
elicitationId: string,
userData: Record<string, unknown>
@@ -449,7 +450,8 @@ export function useChatStream({
}, [sessionId, onSessionLoaded]);
const handleSubmit = useCallback(
async (userMessage: string) => {
async (input: UserInput) => {
const { msg: userMessage, images } = input;
const currentState = stateRef.current;
// Guard: Don't submit if session hasn't been loaded yet
@@ -458,7 +460,7 @@ export function useChatStream({
}
const hasExistingMessages = currentState.messages.length > 0;
const hasNewMessage = userMessage.trim().length > 0;
const hasNewMessage = userMessage.trim().length > 0 || images.length > 0;
// Don't submit if there's no message and no conversation to continue
if (!hasNewMessage && !hasExistingMessages) {
@@ -520,7 +522,7 @@ export function useChatStream({
}
const newMessage = hasNewMessage
? createUserMessage(userMessage)
? createUserMessage(userMessage, images)
: currentState.messages[currentState.messages.length - 1];
const currentMessages = hasNewMessage
? [...currentState.messages, newMessage]
@@ -697,7 +699,7 @@ export function useChatStream({
if (sessionResponse.data?.conversation) {
dispatch({ type: 'SET_MESSAGES', payload: sessionResponse.data.conversation });
}
await handleSubmit(newContent);
await handleSubmit({ msg: newContent, images: [] });
}
} catch (error) {
const errorMsg = errorMessage(error);
ui/desktop/src/hooks/useFileDrop.ts
+25 -8
View File
@@ -1,4 +1,5 @@
import { useCallback, useState, useRef, useEffect } from 'react';
import { compressImageDataUrl } from '../utils/conversionUtils';
export interface DroppedFile {
id: string;
@@ -6,7 +7,7 @@ export interface DroppedFile {
name: string;
type: string;
isImage: boolean;
dataUrl?: string; // For image previews
dataUrl?: string;
isLoading?: boolean;
error?: string;
}
@@ -71,25 +72,41 @@ export const useFileDrop = () => {
droppedFileObjects.push(droppedFile);
// For images, generate a preview (only if successfully processed)
if (droppedFile.isImage && !droppedFile.error) {
const reader = new FileReader();
activeReadersRef.current.add(reader);
reader.onload = (event) => {
reader.onload = async (event) => {
const dataUrl = event.target?.result as string;
setDroppedFiles((prev) =>
prev.map((f) => (f.id === droppedFile.id ? { ...f, dataUrl, isLoading: false } : f))
);
try {
// Compress the image
const compressedDataUrl = await compressImageDataUrl(dataUrl);
setDroppedFiles((prev) =>
prev.map((f) =>
f.id === droppedFile.id
? { ...f, dataUrl: compressedDataUrl, isLoading: false }
: f
)
);
} catch (compressionError) {
console.error('Failed to compress image:', file.name, compressionError);
setDroppedFiles((prev) =>
prev.map((f) =>
f.id === droppedFile.id
? { ...f, error: 'Failed to compress image', isLoading: false }
: f
)
);
}
activeReadersRef.current.delete(reader);
};
reader.onerror = () => {
console.error('Failed to generate preview for:', file.name);
console.error('Failed to read image:', file.name);
setDroppedFiles((prev) =>
prev.map((f) =>
f.id === droppedFile.id
? { ...f, error: 'Failed to load image preview', isLoading: false }
? { ...f, error: 'Failed to load image', isLoading: false }
: f
)
);
ui/desktop/src/main.ts
-73
View File
@@ -1576,79 +1576,6 @@ ipcMain.handle('save-data-url-to-temp', async (_event, dataUrl: string, uniqueId
}
});
// IPC handler to serve temporary image files
ipcMain.handle('get-temp-image', async (_event, filePath: string) => {
console.log(`[Main] Received get-temp-image for path: ${filePath}`);
// Input validation
if (!filePath || typeof filePath !== 'string') {
console.warn('[Main] Invalid file path provided for image serving');
return null;
}
// Ensure the path is within the designated temp directory
const resolvedPath = path.resolve(filePath);
const resolvedTempDir = path.resolve(gooseTempDir);
if (!resolvedPath.startsWith(resolvedTempDir + path.sep)) {
console.warn(`[Main] Attempted to access file outside designated temp directory: ${filePath}`);
return null;
}
try {
// Check if it's a regular file first, before trying realpath
const stats = await fs.lstat(filePath);
if (!stats.isFile()) {
console.warn(`[Main] Not a regular file, refusing to serve: ${filePath}`);
return null;
}
// Get the real paths for both the temp directory and the file to handle symlinks properly
let realTempDir: string;
let actualPath = filePath;
try {
realTempDir = await fs.realpath(gooseTempDir);
const realPath = await fs.realpath(filePath);
// Double-check that the real path is still within our real temp directory
if (!realPath.startsWith(realTempDir + path.sep)) {
console.warn(
`[Main] Real path is outside designated temp directory: ${realPath} not in ${realTempDir}`
);
return null;
}
actualPath = realPath;
} catch (realpathError) {
// If realpath fails, use the original path validation
console.log(
`[Main] realpath failed for ${filePath}, using original path validation:`,
realpathError instanceof Error ? realpathError.message : String(realpathError)
);
}
// Read the file and return as base64 data URL
const fileBuffer = await fs.readFile(actualPath);
const fileExtension = path.extname(actualPath).toLowerCase().substring(1);
// Validate file extension
const allowedExtensions = ['png', 'jpg', 'jpeg', 'gif', 'webp'];
if (!allowedExtensions.includes(fileExtension)) {
console.warn(`[Main] Unsupported file extension: ${fileExtension}`);
return null;
}
const mimeType = fileExtension === 'jpg' ? 'image/jpeg' : `image/${fileExtension}`;
const base64Data = fileBuffer.toString('base64');
const dataUrl = `data:${mimeType};base64,${base64Data}`;
console.log(`[Main] Served temp image: ${filePath}`);
return dataUrl;
} catch (error) {
console.error(`[Main] Failed to serve temp image: ${filePath}`, error);
return null;
}
});
ipcMain.on('delete-temp-file', async (_event, filePath: string) => {
console.log(`[Main] Received delete-temp-file for path: ${filePath}`);
ui/desktop/src/preload.ts
-5
View File
@@ -121,8 +121,6 @@ type ElectronAPI = {
deleteTempFile: (filePath: string) => void;
// Function for opening external URLs securely
openExternal: (url: string) => Promise<void>;
// Function to serve temp images
getTempImage: (filePath: string) => Promise<string | null>;
// Update-related functions
getVersion: () => string;
checkForUpdates: () => Promise<{ updateInfo: unknown; error: string | null }>;
@@ -246,9 +244,6 @@ const electronAPI: ElectronAPI = {
openExternal: (url: string): Promise<void> => {
return ipcRenderer.invoke('open-external', url);
},
getTempImage: (filePath: string): Promise<string | null> => {
return ipcRenderer.invoke('get-temp-image', filePath);
},
getVersion: (): string => {
return config.GOOSE_VERSION || ipcRenderer.sendSync('get-app-version') || '';
},
ui/desktop/src/sessions.ts
+4 -2
View File
@@ -88,9 +88,11 @@ export async function startNewSession(
// Include session data so sidebar can add it immediately (before it has messages)
window.dispatchEvent(new CustomEvent(AppEvents.SESSION_CREATED, { detail: { session } }));
const initialMessage = initialText ? { msg: initialText, images: [] } : undefined;
const eventDetail = {
sessionId: session.id,
initialMessage: initialText,
initialMessage,
};
window.dispatchEvent(
@@ -101,7 +103,7 @@ export async function startNewSession(
setView('pair', {
disableAnimation: true,
initialMessage: initialText,
initialMessage,
resumeSessionId: session.id,
});
return session;
ui/desktop/src/types/message.ts
+44 -9
View File
@@ -7,12 +7,38 @@ export type NotificationEvent = Extract<MessageEvent, { type: 'Notification' }>;
// Compaction response message - must match backend constant
const COMPACTION_THINKING_TEXT = 'goose is compacting the conversation...';
export function createUserMessage(text: string): Message {
export interface ImageData {
data: string; // base64 encoded image data
mimeType: string;
}
export interface UserInput {
msg: string;
images: ImageData[];
}
export function createUserMessage(text: string, images?: ImageData[]): Message {
const content: Message['content'] = [];
if (text.trim()) {
content.push({ type: 'text', text });
}
if (images && images.length > 0) {
images.forEach((img) => {
content.push({
type: 'image',
data: img.data,
mimeType: img.mimeType,
});
});
}
return {
id: generateMessageId(),
role: 'user',
created: Math.floor(Date.now() / 1000),
content: [{ type: 'text', text }],
content,
metadata: { userVisible: true, agentVisible: true },
};
}
@@ -43,13 +69,22 @@ export function generateMessageId(): string {
return Math.random().toString(36).substring(2, 10);
}
export function getTextContent(message: Message): string {
return message.content
.map((content) => {
if (content.type === 'text') return content.text;
return '';
})
.join('');
export function getTextAndImageContent(message: Message): {
textContent: string;
imagePaths: string[];
} {
let textContent = '';
const imagePaths: string[] = [];
for (const content of message.content) {
if (content.type === 'text') {
textContent += content.text;
} else if (content.type === 'image') {
imagePaths.push(`data:${content.mimeType};base64,${content.data}`);
}
}
return { textContent, imagePaths };
}
export function getToolRequests(message: Message): (ToolRequest & { type: 'toolRequest' })[] {
ui/desktop/src/utils/conversionUtils.ts
+26
View File
@@ -22,6 +22,32 @@ export function errorMessage(err: Error | unknown, default_value?: string) {
}
}
export async function compressImageDataUrl(dataUrl: string): Promise<string> {
return new Promise((resolve, reject) => {
const img = new globalThis.Image();
img.onload = () => {
const maxDim = 1024;
const scale = Math.min(1, maxDim / Math.max(img.width, img.height));
const width = Math.floor(img.width * scale);
const height = Math.floor(img.height * scale);
const canvas = document.createElement('canvas');
canvas.width = width;
canvas.height = height;
const ctx = canvas.getContext('2d');
if (!ctx) {
reject(new Error('Failed to get canvas context'));
return;
}
ctx.drawImage(img, 0, 0, width, height);
resolve(canvas.toDataURL('image/jpeg', 0.85));
};
img.onerror = () => reject(new Error('Failed to load image'));
img.src = dataUrl;
});
}
export function formatAppName(name: string): string {
return name
.split(/[-_\s]+/)
ui/desktop/src/utils/imageUtils.ts
-59
View File
@@ -1,59 +0,0 @@
/**
* Utility functions for detecting and handling image paths in messages
*/
/**
* Extracts image file paths from a message text
* Looks for paths that match the pattern of pasted images from the temp directory
*
* @param text The message text to extract image paths from
* @returns An array of image file paths found in the message
*/
export function extractImagePaths(text: string): string[] {
if (!text) return [];
// Match paths that look like pasted image paths from the temp directory
// Pattern: /path/to/goose-pasted-images/pasted-img-TIMESTAMP-RANDOM.ext
// This regex looks for:
// - Word boundary or start of string
// - A path containing "goose-pasted-images"
// - Followed by a filename starting with "pasted-"
// - Ending with common image extensions
// - Word boundary or end of string
const regex =
/(?:^|\s)((?:[^\s]*\/)?goose-pasted-images\/pasted-[^\s]+\.(png|jpg|jpeg|gif|webp))(?=\s|$)/gi;
const matches = [];
let match;
while ((match = regex.exec(text)) !== null) {
matches.push(match[1]);
}
return matches;
}
/**
* Removes image paths from the text
*
* @param text The original text
* @param imagePaths Array of image paths to remove
* @returns Text with image paths removed
*/
export function removeImagePathsFromText(text: string, imagePaths: string[]): string {
if (!text || imagePaths.length === 0) return text;
let result = text;
// Remove each image path from the text
imagePaths.forEach((path) => {
// Escape special regex characters in the path
const escapedPath = path.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
// Create a regex that matches the path with optional surrounding whitespace
const pathRegex = new RegExp(`(^|\\s)${escapedPath}(?=\\s|$)`, 'g');
result = result.replace(pathRegex, '$1');
});
// Clean up any extra whitespace
return result.replace(/\s+/g, ' ').trim();
}
ui/desktop/src/utils/navigationUtils.ts
+3 -2
View File
@@ -1,5 +1,6 @@
import { NavigateFunction } from 'react-router-dom';
import { Recipe } from '../api/types.gen';
import { Recipe } from '../api';
import { UserInput } from '../types/message';
export type View =
| 'welcome'
@@ -29,7 +30,7 @@ export type ViewOptions = {
parentView?: View;
parentViewOptions?: ViewOptions;
disableAnimation?: boolean;
initialMessage?: string;
initialMessage?: UserInput;
shareToken?: string;
resumeSessionId?: string;
pendingScheduleDeepLink?: string;
ui/desktop/src/utils/toolCallChaining.ts
+2 -15
View File
@@ -1,4 +1,4 @@
import { getToolRequests, getTextContent, getToolResponses } from '../types/message';
import { getToolRequests, getTextAndImageContent, getToolResponses } from '../types/message';
import { Message } from '../api';
export function identifyConsecutiveToolCalls(messages: Message[]): number[][] {
@@ -9,7 +9,7 @@ export function identifyConsecutiveToolCalls(messages: Message[]): number[][] {
const message = messages[i];
const toolRequests = getToolRequests(message);
const toolResponses = getToolResponses(message);
const textContent = getTextContent(message);
const { textContent } = getTextAndImageContent(message);
const hasText = textContent.trim().length > 0;
if (toolResponses.length > 0 && toolRequests.length === 0) {
@@ -47,15 +47,6 @@ export function identifyConsecutiveToolCalls(messages: Message[]): number[][] {
return chains;
}
export function shouldHideMessage(messageIndex: number, chains: number[][]): boolean {
for (const chain of chains) {
if (chain.includes(messageIndex)) {
return chain[0] !== messageIndex;
}
}
return false;
}
export function shouldHideTimestamp(messageIndex: number, chains: number[][]): boolean {
for (const chain of chains) {
if (chain.includes(messageIndex)) {
@@ -69,7 +60,3 @@ export function shouldHideTimestamp(messageIndex: number, chains: number[][]): b
export function isInChain(messageIndex: number, chains: number[][]): boolean {
return chains.some((chain) => chain.includes(messageIndex));
}
export function getChainForMessage(messageIndex: number, chains: number[][]): number[] | null {
return chains.find((chain) => chain.includes(messageIndex)) || null;
}