Native images (#6619)

Co-authored-by: Douwe Osinga <douwe@squareup.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

ui/desktop/src/App.tsx

@@ -20,10 +20,11 @@ import { createSession } from './sessions';
 
 import { ChatType } from './types/chat';
 import Hub from './components/Hub';
+import { UserInput } from './types/message';
 
 interface PairRouteState {
   resumeSessionId?: string;
-  initialMessage?: string;
+  initialMessage?: UserInput;
 }
 import SettingsView, { SettingsViewOptions } from './components/settings/SettingsView';
 import SessionsView from './components/sessions/SessionsView';
@@ -68,8 +69,13 @@ const HubRouteWrapper = () => {
 const PairRouteWrapper = ({
   activeSessions,
 }: {
-  activeSessions: Array<{ sessionId: string; initialMessage?: string }>;
+  activeSessions: Array<{
-  setActiveSessions: (sessions: Array<{ sessionId: string; initialMessage?: string }>) => void;
+    sessionId: string;
+    initialMessage?: UserInput;
+  }>;
+  setActiveSessions: (
+    sessions: Array<{ sessionId: string; initialMessage?: UserInput }>
+  ) => void;
 }) => {
   const { extensionsList } = useConfig();
   const location = useLocation();
@@ -354,13 +360,16 @@ export function AppInner() {
   const MAX_ACTIVE_SESSIONS = 10;
 
   const [activeSessions, setActiveSessions] = useState<
-    Array<{ sessionId: string; initialMessage?: string }>
+    Array<{ sessionId: string; initialMessage?: UserInput }>
   >([]);
 
   useEffect(() => {
     const handleAddActiveSession = (event: Event) => {
       const { sessionId, initialMessage } = (
-        event as CustomEvent<{ sessionId: string; initialMessage?: string }>
+        event as CustomEvent<{
+          sessionId: string;
+          initialMessage?: UserInput;
+        }>
       ).detail;
 
       setActiveSessions((prev) => {

ui/desktop/src/components/BaseChat.tsx

@@ -28,17 +28,18 @@ import { useNavigation } from '../hooks/useNavigation';
 import { RecipeHeader } from './RecipeHeader';
 import { RecipeWarningModal } from './ui/RecipeWarningModal';
 import { scanRecipe } from '../recipe';
+import { UserInput } from '../types/message';
 import { useCostTracking } from '../hooks/useCostTracking';
 import RecipeActivities from './recipes/RecipeActivities';
 import { useToolCount } from './alerts/useToolCount';
-import { getThinkingMessage, getTextContent } from '../types/message';
+import { getThinkingMessage, getTextAndImageContent } from '../types/message';
 import ParameterInputModal from './ParameterInputModal';
 import { substituteParameters } from '../utils/providerUtils';
 import CreateRecipeFromSessionModal from './recipes/CreateRecipeFromSessionModal';
 import { toastSuccess } from '../toasts';
 import { Recipe } from '../recipe';
 import { useAutoSubmit } from '../hooks/useAutoSubmit';
-import { Goose } from './icons/Goose';
+import { Goose } from './icons';
 import EnvironmentBadge from './GooseSidebar/EnvironmentBadge';
 
 const CurrentModelContext = createContext<{ model: string; mode: string } | null>(null);
@@ -56,10 +57,10 @@ interface BaseChatProps {
   suppressEmptyState: boolean;
   sessionId: string;
   isActiveSession: boolean;
-  initialMessage?: string;
+  initialMessage?: UserInput;
 }
 
-function BaseChatContent({
+export default function BaseChat({
   setChat,
   renderHeader,
   customChatInputProps = {},
@@ -149,7 +150,7 @@ function BaseChatContent({
     return messages
       .reduce<string[]>((history, message) => {
         if (message.role === 'user') {
-          const text = getTextContent(message).trim();
+          const text = getTextAndImageContent(message).textContent.trim();
           if (text) {
             history.push(text);
           }
@@ -159,14 +160,11 @@ function BaseChatContent({
       .reverse();
   }, [messages]);
 
-  const handleFormSubmit = (e: React.FormEvent) => {
+  const chatInputSubmit = (input: UserInput) => {
-    const customEvent = e as unknown as CustomEvent;
+    if (recipe && input.msg.trim()) {
-    const textValue = customEvent.detail?.value || '';
-
-    if (recipe && textValue.trim()) {
       setHasStartedUsingRecipe(true);
     }
-    handleSubmit(textValue);
+    handleSubmit(input);
   };
 
   const { sessionCosts } = useCostTracking({
@@ -326,7 +324,6 @@ function BaseChatContent({
     // eslint-disable-next-line react-hooks/exhaustive-deps
   }, [session?.name, setChat]);
 
-  // Only use initialMessage for the prompt if it hasn't been submitted yet
   // If we have a recipe prompt and user recipe values, substitute parameters
   let recipePrompt = '';
   if (messages.length === 0 && recipe?.prompt) {
@@ -413,7 +410,7 @@ function BaseChatContent({
             {recipe && (
               <div className={hasStartedUsingRecipe ? 'mb-6' : ''}>
                 <RecipeActivities
-                  append={(text: string) => handleSubmit(text)}
+                  append={(text: string) => handleSubmit({ msg: text, images: [] })}
                   activities={Array.isArray(recipe.activities) ? recipe.activities : null}
                   title={recipe.title}
                   parameterValues={session?.user_recipe_values || {}}
@@ -428,7 +425,7 @@ function BaseChatContent({
                     messages={messages}
                     chat={{ sessionId }}
                     toolCallNotifications={toolCallNotifications}
-                    append={(text: string) => handleSubmit(text)}
+                    append={(text: string) => handleSubmit({ msg: text, images: [] })}
                     isUserMessage={(m: Message) => m.role === 'user'}
                     isStreamingMessage={chatState !== ChatState.Idle}
                     onRenderingComplete={handleRenderingComplete}
@@ -440,7 +437,7 @@ function BaseChatContent({
                 <div className="block h-8" />
               </>
             ) : !recipe && showPopularTopics ? (
-              <PopularChatTopics append={(text: string) => handleSubmit(text)} />
+              <PopularChatTopics append={(text: string) => handleSubmit({ msg: text, images: [] })} />
             ) : null}
           </ScrollArea>
 
@@ -464,7 +461,7 @@ function BaseChatContent({
           <ChatInput
             inputRef={chatInputRef}
             sessionId={sessionId}
-            handleSubmit={handleFormSubmit}
+            handleSubmit={chatInputSubmit}
             chatState={chatState}
             setChatState={setChatState}
             onStop={stopStreaming}
@@ -527,7 +524,3 @@ function BaseChatContent({
     </div>
   );
 }
-
-export default function BaseChat(props: BaseChatProps) {
-  return <BaseChatContent {...props} />;
-}

ui/desktop/src/components/ChatInput.tsx

@@ -25,7 +25,7 @@ import { COST_TRACKING_ENABLED, VOICE_DICTATION_ELEVENLABS_ENABLED } from '../up
 import { CostTracker } from './bottom_menu/CostTracker';
 import { DroppedFile, useFileDrop } from '../hooks/useFileDrop';
 import { Recipe } from '../recipe';
-import MessageQueue from './MessageQueue';
+import { MessageQueue, QueuedMessage } from './MessageQueue';
 import { detectInterruption } from '../utils/interruptionDetector';
 import { DiagnosticsModal } from './ui/Diagnostics';
 import { getSession, Message } from '../api';
@@ -41,24 +41,17 @@ import {
   trackEditRecipeOpened,
 } from '../utils/analytics';
 import { getNavigationShortcutText } from '../utils/keyboardShortcuts';
-
+import { UserInput, ImageData } from '../types/message';
-interface QueuedMessage {
+import { compressImageDataUrl } from '../utils/conversionUtils';
-  id: string;
-  content: string;
-  timestamp: number;
-}
 
 interface PastedImage {
   id: string;
-  dataUrl: string; // For immediate preview
+  dataUrl: string;
-  filePath?: string; // Path on filesystem after saving
   isLoading: boolean;
   error?: string;
 }
 
-// Constants for image handling
+const MAX_IMAGES_PER_MESSAGE = 10;
-const MAX_IMAGES_PER_MESSAGE = 5;
-const MAX_IMAGE_SIZE_MB = 5;
 
 // Constants for token and tool alerts
 const TOKEN_LIMIT_DEFAULT = 128000; // fallback for custom models that the backend doesn't know about
@@ -74,7 +67,7 @@ interface ModelLimit {
 
 interface ChatInputProps {
   sessionId: string | null;
-  handleSubmit: (e: React.FormEvent) => void;
+  handleSubmit: (input: UserInput) => void;
   chatState: ChatState;
   setChatState?: (state: ChatState) => void;
   onStop?: () => void;
@@ -219,11 +212,7 @@ export default function ChatInput({
       if (shouldProcessQueue) {
         const nextMessage = queuedMessages[0];
         LocalMessageStorage.addMessage(nextMessage.content);
-        handleSubmit(
+        handleSubmit({ msg: nextMessage.content, images: nextMessage.images });
-          new CustomEvent('submit', {
-            detail: { value: nextMessage.content },
-          }) as unknown as React.FormEvent
-        );
         setQueuedMessages((prev) => {
           const newQueue = prev.slice(1);
           // If queue becomes empty after processing, clear the paused state
@@ -309,15 +298,7 @@ export default function ChatInput({
   useEffect(() => {
     setValue(initialValue);
     setDisplayValue(initialValue);
-    setPastedImages((currentPastedImages) => {
+    setPastedImages([]);
-      currentPastedImages.forEach((img) => {
-        if (img.filePath) {
-          window.electron.deleteTempFile(img.filePath);
-        }
-      });
-      return [];
-    });
-
     setHistoryIndex(-1);
     setIsInGlobalHistory(false);
     setHasUserTyped(false);
@@ -366,43 +347,9 @@ export default function ChatInput({
   };
 
   const handleRemovePastedImage = (idToRemove: string) => {
-    const imageToRemove = pastedImages.find((img) => img.id === idToRemove);
-    if (imageToRemove?.filePath) {
-      window.electron.deleteTempFile(imageToRemove.filePath);
-    }
     setPastedImages((currentImages) => currentImages.filter((img) => img.id !== idToRemove));
   };
 
-  const handleRetryImageSave = async (imageId: string) => {
-    const imageToRetry = pastedImages.find((img) => img.id === imageId);
-    if (!imageToRetry || !imageToRetry.dataUrl) return;
-
-    // Set the image to loading state
-    setPastedImages((prev) =>
-      prev.map((img) => (img.id === imageId ? { ...img, isLoading: true, error: undefined } : img))
-    );
-
-    try {
-      const result = await window.electron.saveDataUrlToTemp(imageToRetry.dataUrl, imageId);
-      setPastedImages((prev) =>
-        prev.map((img) =>
-          img.id === result.id
-            ? { ...img, filePath: result.filePath, error: result.error, isLoading: false }
-            : img
-        )
-      );
-    } catch (err) {
-      console.error('Error retrying image save:', err);
-      setPastedImages((prev) =>
-        prev.map((img) =>
-          img.id === imageId
-            ? { ...img, error: 'Failed to save image via Electron.', isLoading: false }
-            : img
-        )
-      );
-    }
-  };
-
   useEffect(() => {
     if (textAreaRef.current) {
       textAreaRef.current.focus();
@@ -423,7 +370,6 @@ export default function ChatInput({
     return [];
   };
 
-  // Helper function to find model limit using pattern matching
   const findModelLimit = (modelName: string, modelLimits: ModelLimit[]): number | null => {
     if (!modelName) return null;
     const matchingLimit = modelLimits.find((limit) =>
@@ -512,12 +458,7 @@ export default function ChatInput({
         compactButtonDisabled: !totalTokens,
         onCompact: () => {
           window.dispatchEvent(new CustomEvent(AppEvents.HIDE_ALERT_POPOVER));
-
+          handleSubmit({ msg: MANUAL_COMPACT_TRIGGER, images: [] });
-          const customEvent = new CustomEvent('submit', {
-            detail: { value: MANUAL_COMPACT_TRIGGER },
-          }) as unknown as React.FormEvent;
-
-          handleSubmit(customEvent);
         },
         compactIcon: <ScrollText size={12} />,
       });
@@ -542,20 +483,6 @@ export default function ChatInput({
   // Cleanup effect for component unmount - prevent memory leaks
   useEffect(() => {
     return () => {
-      // Clear any pending timeouts from image processing
-      setPastedImages((currentImages) => {
-        currentImages.forEach((img) => {
-          if (img.filePath) {
-            try {
-              window.electron.deleteTempFile(img.filePath);
-            } catch (error) {
-              console.error('Error deleting temp file:', error);
-            }
-          }
-        });
-        return [];
-      });
-
       // Clear all tracked timeouts
       // eslint-disable-next-line react-hooks/exhaustive-deps
       const timeouts = timeoutRefsRef.current;
@@ -659,6 +586,62 @@ export default function ChatInput({
     }));
   };
 
+  const convertImagesToImageData = useCallback((): ImageData[] => {
+    const pastedImageData: ImageData[] = pastedImages
+      .filter((img) => img.dataUrl && !img.error && !img.isLoading)
+      .map((img) => {
+        const matches = img.dataUrl.match(/^data:([^;]+);base64,(.+)$/);
+        if (matches) {
+          return {
+            data: matches[2],
+            mimeType: matches[1],
+          };
+        }
+        return null;
+      })
+      .filter((img): img is ImageData => img !== null);
+
+    const droppedImageData: ImageData[] = allDroppedFiles
+      .filter((file) => file.isImage && file.dataUrl && !file.error && !file.isLoading)
+      .map((file) => {
+        const matches = file.dataUrl!.match(/^data:([^;]+);base64,(.+)$/);
+        if (matches) {
+          return {
+            data: matches[2],
+            mimeType: matches[1],
+          };
+        }
+        return null;
+      })
+      .filter((img): img is ImageData => img !== null);
+
+    return [...pastedImageData, ...droppedImageData];
+  }, [pastedImages, allDroppedFiles]);
+
+  const appendDroppedFilePaths = useCallback((text: string): string => {
+    const droppedFilePaths = allDroppedFiles
+      .filter((file) => !file.isImage && !file.error && !file.isLoading)
+      .map((file) => file.path);
+
+    if (droppedFilePaths.length > 0) {
+      const pathsString = droppedFilePaths.join(' ');
+      return text ? `${text} ${pathsString}` : pathsString;
+    }
+    return text;
+  }, [allDroppedFiles]);
+
+  const clearInputState = useCallback(() => {
+    setDisplayValue('');
+    setValue('');
+    setPastedImages([]);
+    if (onFilesProcessed && droppedFiles.length > 0) {
+      onFilesProcessed();
+    }
+    if (localDroppedFiles.length > 0) {
+      setLocalDroppedFiles([]);
+    }
+  }, [droppedFiles.length, localDroppedFiles.length, onFilesProcessed, setLocalDroppedFiles]);
+
   const handlePaste = async (evt: React.ClipboardEvent<HTMLTextAreaElement>) => {
     const files = Array.from(evt.clipboardData.files || []);
     const imageFiles = files.filter((file) => file.type.startsWith('image/'));
@@ -694,26 +677,6 @@ export default function ChatInput({
     const newImages: PastedImage[] = [];
 
     for (const file of imageFiles) {
-      // Check individual file size before processing
-      if (file.size > MAX_IMAGE_SIZE_MB * 1024 * 1024) {
-        const errorId = `error-${Date.now()}-${Math.random().toString(36).substring(2, 9)}`;
-        newImages.push({
-          id: errorId,
-          dataUrl: '',
-          isLoading: false,
-          error: `Image too large (${Math.round(file.size / (1024 * 1024))}MB). Maximum ${MAX_IMAGE_SIZE_MB}MB allowed.`,
-        });
-
-        // Remove the error message after 5 seconds with cleanup tracking
-        const timeoutId = setTimeout(() => {
-          setPastedImages((prev) => prev.filter((img) => img.id !== errorId));
-          timeoutRefsRef.current.delete(timeoutId);
-        }, 5000);
-        timeoutRefsRef.current.add(timeoutId);
-
-        continue;
-      }
-
       const imageId = `img-${Date.now()}-${Math.random().toString(36).substring(2, 9)}`;
 
       // Add the image with loading state
@@ -728,30 +691,12 @@ export default function ChatInput({
       reader.onload = async (e) => {
         const dataUrl = e.target?.result as string;
         if (dataUrl) {
-          // Update the image with the data URL
+          const compressedDataUrl = await compressImageDataUrl(dataUrl);
           setPastedImages((prev) =>
-            prev.map((img) => (img.id === imageId ? { ...img, dataUrl, isLoading: true } : img))
+            prev.map((img) =>
+              img.id === imageId ? { ...img, dataUrl: compressedDataUrl, isLoading: false } : img
+            )
           );
-
-          try {
-            const result = await window.electron.saveDataUrlToTemp(dataUrl, imageId);
-            setPastedImages((prev) =>
-              prev.map((img) =>
-                img.id === result.id
-                  ? { ...img, filePath: result.filePath, error: result.error, isLoading: false }
-                  : img
-              )
-            );
-          } catch (err) {
-            console.error('Error saving pasted image:', err);
-            setPastedImages((prev) =>
-              prev.map((img) =>
-                img.id === imageId
-                  ? { ...img, error: 'Failed to save image via Electron.', isLoading: false }
-                  : img
-              )
-            );
-          }
         }
       };
       reader.onerror = () => {
@@ -865,25 +810,13 @@ export default function ChatInput({
     }
   };
 
-  // Helper function to handle interruption and queue logic when loading
   const handleInterruptionAndQueue = () => {
     if (!isLoading || !hasSubmittableContent) {
       return false;
     }
 
-    const validPastedImageFilesPaths = pastedImages
+    const imageData = convertImagesToImageData();
-      .filter((img) => img.filePath && !img.error && !img.isLoading)
+    const contentToQueue = appendDroppedFilePaths(displayValue.trim());
-      .map((img) => img.filePath as string);
-    const droppedFilePaths = allDroppedFiles
-      .filter((file) => !file.error && !file.isLoading)
-      .map((file) => file.path);
-
-    let contentToQueue = displayValue.trim();
-    const allFilePaths = [...validPastedImageFilesPaths, ...droppedFilePaths];
-    if (allFilePaths.length > 0) {
-      const pathsString = allFilePaths.join(' ');
-      contentToQueue = contentToQueue ? `${contentToQueue} ${pathsString}` : pathsString;
-    }
 
     const interruptionMatch = detectInterruption(displayValue.trim());
 
@@ -894,31 +827,25 @@ export default function ChatInput({
 
       // For interruptions, we need to queue the message to be sent after the stop completes
       // rather than trying to send it immediately while the system is still loading
-      const interruptionMessage = {
+      const interruptionMessage: QueuedMessage = {
         id: Date.now().toString() + Math.random().toString(36).substr(2, 9),
         content: contentToQueue,
         timestamp: Date.now(),
+        images: imageData,
       };
 
       // Add the interruption message to the front of the queue so it gets sent first
       setQueuedMessages((prev) => [interruptionMessage, ...prev]);
 
-      setDisplayValue('');
+      clearInputState();
-      setValue('');
-      setPastedImages([]);
-      if (onFilesProcessed && droppedFiles.length > 0) {
-        onFilesProcessed();
-      }
-      if (localDroppedFiles.length > 0) {
-        setLocalDroppedFiles([]);
-      }
       return true;
     }
 
-    const newMessage = {
+    const newMessage: QueuedMessage = {
       id: Date.now().toString() + Math.random().toString(36).substr(2, 9),
       content: contentToQueue,
       timestamp: Date.now(),
+      images: imageData,
     };
     setQueuedMessages((prev) => {
       const newQueue = [...prev, newMessage];
@@ -929,53 +856,35 @@ export default function ChatInput({
       }
       return newQueue;
     });
-    setDisplayValue('');
+    clearInputState();
-    setValue('');
-    setPastedImages([]);
-    if (onFilesProcessed && droppedFiles.length > 0) {
-      onFilesProcessed();
-    }
-    if (localDroppedFiles.length > 0) {
-      setLocalDroppedFiles([]);
-    }
     return true;
   };
 
   const canSubmit =
     !isLoading &&
     (displayValue.trim() ||
-      pastedImages.some((img) => img.filePath && !img.error && !img.isLoading) ||
+      pastedImages.some((img) => img.dataUrl && !img.error && !img.isLoading) ||
       allDroppedFiles.some((file) => !file.error && !file.isLoading));
 
   const performSubmit = useCallback(
     (text?: string) => {
-      const validPastedImageFilesPaths = pastedImages
+      const imageData = convertImagesToImageData();
-        .filter((img) => img.filePath && !img.error && !img.isLoading)
+      const textToSend = appendDroppedFilePaths(text ?? displayValue.trim());
-        .map((img) => img.filePath as string);
-      // Get paths from all dropped files (both parent and local)
-      const droppedFilePaths = allDroppedFiles
-        .filter((file) => !file.error && !file.isLoading)
-        .map((file) => file.path);
 
-      let textToSend = text ?? displayValue.trim();
+      if (textToSend || imageData.length > 0) {
-
+        // Store original message in history
-      // Combine pasted images and dropped files
-      const allFilePaths = [...validPastedImageFilesPaths, ...droppedFilePaths];
-      if (allFilePaths.length > 0) {
-        const pathsString = allFilePaths.join(' ');
-        textToSend = textToSend ? `${textToSend} ${pathsString}` : pathsString;
-      }
-
-      if (textToSend) {
         if (displayValue.trim()) {
           LocalMessageStorage.addMessage(displayValue);
-        } else if (allFilePaths.length > 0) {
+        } else {
-          LocalMessageStorage.addMessage(allFilePaths.join(' '));
+          const droppedFilePaths = allDroppedFiles
+            .filter((file) => !file.isImage && !file.error && !file.isLoading)
+            .map((file) => file.path);
+          if (droppedFilePaths.length > 0) {
+            LocalMessageStorage.addMessage(droppedFilePaths.join(' '));
+          }
         }
 
-        handleSubmit(
+        handleSubmit({ msg: textToSend, images: imageData });
-          new CustomEvent('submit', { detail: { value: textToSend } }) as unknown as React.FormEvent
-        );
 
         // Auto-resume queue after sending a NON-interruption message (if it was paused due to interruption)
         if (
@@ -988,33 +897,21 @@ export default function ChatInput({
           setLastInterruption(null);
         }
 
-        setDisplayValue('');
+        clearInputState();
-        setValue('');
-        setPastedImages([]);
         setHistoryIndex(-1);
         setSavedInput('');
         setIsInGlobalHistory(false);
         setHasUserTyped(false);
-
-        // Clear both parent and local dropped files after processing
-        if (onFilesProcessed && droppedFiles.length > 0) {
-          onFilesProcessed();
-        }
-        if (localDroppedFiles.length > 0) {
-          setLocalDroppedFiles([]);
-        }
       }
     },
     [
-      allDroppedFiles,
+      convertImagesToImageData,
+      appendDroppedFilePaths,
       displayValue,
-      droppedFiles.length,
+      allDroppedFiles,
       handleSubmit,
       lastInterruption,
-      localDroppedFiles.length,
+      clearInputState,
-      onFilesProcessed,
-      pastedImages,
-      setLocalDroppedFiles,
     ]
   );
 
@@ -1090,29 +987,85 @@ export default function ChatInput({
     const canSubmit =
       !isLoading &&
       (displayValue.trim() ||
-        pastedImages.some((img) => img.filePath && !img.error && !img.isLoading) ||
+        pastedImages.some((img) => img.dataUrl && !img.error && !img.isLoading) ||
         allDroppedFiles.some((file) => !file.error && !file.isLoading));
     if (canSubmit) {
       performSubmit();
     }
   };
 
-  const handleFileSelect = async () => {
+  const fileInputRef = React.useRef<HTMLInputElement>(null);
-    if (isFilePickerOpen) return;
-    setIsFilePickerOpen(true);
-    try {
-      const path = await window.electron.selectFileOrDirectory();
-      if (path) {
-        const isDirectory = !path.includes('.') || path.endsWith('/');
-        trackFileAttached(isDirectory ? 'directory' : 'file');
 
-        const newValue = displayValue.trim() ? `${displayValue.trim()} ${path}` : path;
+  const handleFileSelect = () => {
-        setDisplayValue(newValue);
+    if (isFilePickerOpen) return;
-        setValue(newValue);
+    fileInputRef.current?.click();
-        textAreaRef.current?.focus();
+  };
+
+  const handleFileInputChange = async (e: React.ChangeEvent<HTMLInputElement>) => {
+    const files = e.target.files;
+    if (!files || files.length === 0) return;
+
+    setIsFilePickerOpen(true);
+    const file = files[0];
+    const isImage = file.type.startsWith('image/');
+
+    if (isImage) {
+      trackFileAttached('file');
+
+      if (pastedImages.length >= MAX_IMAGES_PER_MESSAGE) {
+        console.warn(`Maximum ${MAX_IMAGES_PER_MESSAGE} images per message`);
+        setIsFilePickerOpen(false);
+        return;
       }
-    } finally {
+
-      setIsFilePickerOpen(false);
+      const uniqueId = `upload-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
+
+      setPastedImages((prev) => [
+        ...prev,
+        {
+          id: uniqueId,
+          dataUrl: '',
+          isLoading: true,
+          error: undefined,
+        },
+      ]);
+
+      const reader = new FileReader();
+      reader.onload = async (evt) => {
+        const dataUrl = evt.target?.result as string;
+        if (dataUrl) {
+          const compressedDataUrl = await compressImageDataUrl(dataUrl);
+          setPastedImages((prev) =>
+            prev.map((img) =>
+              img.id === uniqueId
+                ? { ...img, dataUrl: compressedDataUrl, isLoading: false, error: undefined }
+                : img
+            )
+          );
+        }
+      };
+      reader.onerror = () => {
+        setPastedImages((prev) =>
+          prev.map((img) =>
+            img.id === uniqueId
+              ? { ...img, isLoading: false, error: 'Failed to read image file' }
+              : img
+          )
+        );
+      };
+      reader.readAsDataURL(file);
+    } else {
+      trackFileAttached('file');
+      const path = window.electron.getPathForFile(file);
+      const newValue = displayValue.trim() ? `${displayValue.trim()} ${path}` : path;
+      setDisplayValue(newValue);
+      setValue(newValue);
+    }
+
+    textAreaRef.current?.focus();
+    setIsFilePickerOpen(false);
+    if (e.target) {
+      e.target.value = '';
     }
   };
 
@@ -1140,7 +1093,7 @@ export default function ChatInput({
 
   const hasSubmittableContent =
     displayValue.trim() ||
-    pastedImages.some((img) => img.filePath && !img.error && !img.isLoading) ||
+    pastedImages.some((img) => img.dataUrl && !img.error && !img.isLoading) ||
     allDroppedFiles.some((file) => !file.error && !file.isLoading);
   const isAnyImageLoading = pastedImages.some((img) => img.isLoading);
   const isAnyDroppedFileLoading = allDroppedFiles.some((file) => file.isLoading);
@@ -1196,11 +1149,7 @@ export default function ChatInput({
     // Remove the message from queue and send it immediately
     setQueuedMessages((prev) => prev.filter((msg) => msg.id !== messageId));
     LocalMessageStorage.addMessage(messageToSend.content);
-    handleSubmit(
+    handleSubmit({ msg: messageToSend.content, images: messageToSend.images });
-      new CustomEvent('submit', {
-        detail: { value: messageToSend.content },
-      }) as unknown as React.FormEvent
-    );
 
     // Restore previous pause state after a brief delay to prevent race condition
     setTimeout(() => {
@@ -1214,11 +1163,7 @@ export default function ChatInput({
     if (!isLoading && queuedMessages.length > 0) {
       const nextMessage = queuedMessages[0];
       LocalMessageStorage.addMessage(nextMessage.content);
-      handleSubmit(
+      handleSubmit({ msg: nextMessage.content, images: nextMessage.images });
-        new CustomEvent('submit', {
-          detail: { value: nextMessage.content },
-        }) as unknown as React.FormEvent
-      );
       setQueuedMessages((prev) => {
         const newQueue = prev.slice(1);
         // If queue becomes empty after processing, clear the paused state
@@ -1244,6 +1189,7 @@ export default function ChatInput({
       onDrop={handleLocalDrop}
       onDragOver={handleLocalDragOver}
     >
+      <input ref={fileInputRef} type="file" onChange={handleFileInputChange} style={{ display: 'none' }} accept="*/*" />
       {/* Message Queue Display */}
       {queuedMessages.length > 0 && (
         <MessageQueue
@@ -1451,20 +1397,9 @@ export default function ChatInput({
               )}
               {img.error && !img.isLoading && (
                 <div className="absolute inset-0 flex flex-col items-center justify-center bg-black bg-opacity-75 rounded p-1 text-center">
-                  <p className="text-red-400 text-[10px] leading-tight break-all mb-1">
+                  <p className="text-red-400 text-[10px] leading-tight break-all">
                     {img.error.substring(0, 50)}
                   </p>
-                  {img.dataUrl && (
-                    <Button
-                      type="button"
-                      onClick={() => handleRetryImageSave(img.id)}
-                      title="Retry saving image"
-                      variant="outline"
-                      size="xs"
-                    >
-                      Retry
-                    </Button>
-                  )}
                 </div>
               )}
               {!img.isLoading && (
@@ -1570,7 +1505,7 @@ export default function ChatInput({
               <Attach className="w-4 h-4" />
             </Button>
           </TooltipTrigger>
-          <TooltipContent>Attach file or directory</TooltipContent>
+          <TooltipContent>Attach file</TooltipContent>
         </Tooltip>
         <div className="w-px h-4 bg-border-default mx-2" />
         {/* Model selector, mode selector, alerts, summarize button */}

ui/desktop/src/components/ChatSessionsContainer.tsx

@@ -1,10 +1,14 @@
 import { useSearchParams } from 'react-router-dom';
 import BaseChat from './BaseChat';
 import { ChatType } from '../types/chat';
+import { UserInput } from '../types/message';
 
 interface ChatSessionsContainerProps {
   setChat: (chat: ChatType) => void;
-  activeSessions: Array<{ sessionId: string; initialMessage?: string }>;
+  activeSessions: Array<{
+    sessionId: string;
+    initialMessage?: UserInput;
+  }>;
 }
 
 /**

ui/desktop/src/components/GooseMessage.tsx

@@ -1,11 +1,10 @@
 import { useMemo, useRef } from 'react';
 import ImagePreview from './ImagePreview';
-import { extractImagePaths, removeImagePathsFromText } from '../utils/imageUtils';
 import { formatMessageTimestamp } from '../utils/timeUtils';
 import MarkdownContent from './MarkdownContent';
 import ToolCallWithResponse from './ToolCallWithResponse';
 import {
-  getTextContent,
+  getTextAndImageContent,
   getToolRequests,
   getToolResponses,
   getToolConfirmationContent,
@@ -45,28 +44,25 @@ export default function GooseMessage({
 }: GooseMessageProps) {
   const contentRef = useRef<HTMLDivElement | null>(null);
 
-  let textContent = getTextContent(message);
+  let { textContent, imagePaths } = getTextAndImageContent(message);
 
-  const splitChainOfThought = (text: string): { visibleText: string; cotText: string | null } => {
+  const splitChainOfThought = (text: string): { displayText: string; cotText: string | null } => {
     const regex = /<think>([\s\S]*?)<\/think>/i;
     const match = text.match(regex);
     if (!match) {
-      return { visibleText: text, cotText: null };
+      return { displayText: text, cotText: null };
     }
 
     const cotRaw = match[1].trim();
-    const visibleText = text.replace(regex, '').trim();
+    const displayText = text.replace(regex, '').trim();
 
     return {
-      visibleText,
+      displayText,
       cotText: cotRaw || null,
     };
   };
 
-  const { visibleText, cotText } = splitChainOfThought(textContent);
+  const { displayText, cotText } = splitChainOfThought(textContent);
-  const imagePaths = extractImagePaths(visibleText);
-  const displayText =
-    imagePaths.length > 0 ? removeImagePathsFromText(visibleText, imagePaths) : visibleText;
 
   const timestamp = useMemo(() => formatMessageTimestamp(message.created), [message.created]);
   const toolRequests = getToolRequests(message);
@@ -116,11 +112,13 @@ export default function GooseMessage({
           </details>
         )}
 
-        {displayText && (
+        {(displayText.trim() || imagePaths.length > 0) && (
           <div className="flex flex-col group">
-            <div ref={contentRef} className="w-full">
+            {displayText.trim() && (
-              <MarkdownContent content={displayText} />
+              <div ref={contentRef} className="w-full">
-            </div>
+                <MarkdownContent content={displayText} />
+              </div>
+            )}
 
             {imagePaths.length > 0 && (
               <div className="mt-4">

ui/desktop/src/components/GooseSidebar/AppSidebar.tsx

@@ -263,7 +263,7 @@ const AppSidebar: React.FC<SidebarProps> = ({ currentPath }) => {
     let isPolling = false;
 
     const handleSessionCreated = (event: Event) => {
-      const { session } = (event as CustomEvent<{ session?: Session }>).detail;
+      const { session } = (event as CustomEvent<{ session?: Session }>).detail || {};
       // If session data is provided, add it immediately to the sidebar
       // This is for displaying sessions that won't be returned by the API due to not having messages yet
       if (session) {

ui/desktop/src/components/Hub.tsx

@@ -29,6 +29,7 @@ import {
 import { getInitialWorkingDir } from '../utils/workingDir';
 import { createSession } from '../sessions';
 import LoadingGoose from './LoadingGoose';
+import { UserInput } from '../types/message';
 
 export default function Hub({
   setView,
@@ -39,11 +40,9 @@ export default function Hub({
   const [workingDir, setWorkingDir] = useState(getInitialWorkingDir());
   const [isCreatingSession, setIsCreatingSession] = useState(false);
 
-  const handleSubmit = async (e: React.FormEvent) => {
+  const handleSubmit = async (input: UserInput) => {
-    const customEvent = e as unknown as CustomEvent;
+    const { msg: userMessage, images } = input;
-    const combinedTextFromInput = customEvent.detail?.value || '';
+    if ((images.length > 0 || userMessage.trim()) && !isCreatingSession) {
-
-    if (combinedTextFromInput.trim() && !isCreatingSession) {
       const extensionConfigs = getExtensionConfigsWithOverrides(extensionsList);
       clearExtensionOverrides();
       setIsCreatingSession(true);
@@ -57,21 +56,19 @@ export default function Hub({
         window.dispatchEvent(new CustomEvent(AppEvents.SESSION_CREATED));
         window.dispatchEvent(
           new CustomEvent(AppEvents.ADD_ACTIVE_SESSION, {
-            detail: { sessionId: session.id, initialMessage: combinedTextFromInput },
+            detail: { sessionId: session.id, initialMessage: { msg: userMessage, images } },
           })
         );
 
         setView('pair', {
           disableAnimation: true,
           resumeSessionId: session.id,
-          initialMessage: combinedTextFromInput,
+          initialMessage: { msg: userMessage, images },
         });
       } catch (error) {
         console.error('Failed to create session:', error);
         setIsCreatingSession(false);
       }
-
-      e.preventDefault();
     }
   };
 

ui/desktop/src/components/ImagePreview.tsx

@@ -1,88 +1,32 @@
-import { useState, useEffect } from 'react';
+import { useState } from 'react';
 
 interface ImagePreviewProps {
   src: string;
-  alt?: string;
-  className?: string;
 }
 
-export default function ImagePreview({
+export default function ImagePreview({ src }: ImagePreviewProps) {
-  src,
-  alt = 'Pasted image',
-  className = '',
-}: ImagePreviewProps) {
   const [isExpanded, setIsExpanded] = useState(false);
   const [error, setError] = useState(false);
-  const [isLoading, setIsLoading] = useState(true);
-  const [imageData, setImageData] = useState<string | null>(null);
-
-  useEffect(() => {
-    const loadImage = async () => {
-      try {
-        // Use the IPC handler to get the image data
-        const data = await window.electron.getTempImage(src);
-        if (data) {
-          setImageData(data);
-          setIsLoading(false);
-        } else {
-          setError(true);
-          setIsLoading(false);
-        }
-      } catch (err) {
-        console.error('Error loading image:', err);
-        setError(true);
-        setIsLoading(false);
-      }
-    };
-
-    loadImage();
-  }, [src]);
-
-  const handleError = () => {
-    setError(true);
-    setIsLoading(false);
-  };
-
-  const toggleExpand = () => {
-    if (!error) {
-      setIsExpanded(!isExpanded);
-    }
-  };
-
-  // Validate that this is a safe file path (should contain goose-pasted-images)
-  if (!src.includes('goose-pasted-images')) {
-    return <div className="text-red-500 text-xs italic mt-1 mb-1">Invalid image path: {src}</div>;
-  }
 
   if (error) {
-    return <div className="text-red-500 text-xs italic mt-1 mb-1">Unable to load image: {src}</div>;
+    return <div className="text-red-500 text-xs italic mt-1 mb-1">Unable to load image</div>;
   }
 
   return (
-    <div className={`image-preview mt-2 mb-2 ${className}`}>
+    <div className={`image-preview mt-2 mb-2`}>
-      {isLoading && (
+      <img
-        <div className="animate-pulse bg-gray-200 rounded w-40 h-40 flex items-center justify-center">
+        src={src}
-          <span className="text-gray-500 text-xs">Loading...</span>
+        alt="goose image"
-        </div>
+        onError={() => setError(true)}
-      )}
+        onClick={() => setIsExpanded(!isExpanded)}
-      {imageData && (
+        className={`rounded border border-borderSubtle cursor-pointer hover:border-borderStandard transition-all ${
-        <img
+          isExpanded ? 'max-w-full max-h-96' : 'max-h-40 max-w-40'
-          src={imageData}
+        }`}
-          alt={alt}
+        style={{ objectFit: 'contain' }}
-          onError={handleError}
+      />
-          onClick={toggleExpand}
+      <div className="text-xs text-textSubtle mt-1">
-          className={`rounded border border-borderSubtle cursor-pointer hover:border-borderStandard transition-all ${
+        Click to {isExpanded ? 'collapse' : 'expand'}
-            isExpanded ? 'max-w-full max-h-96' : 'max-h-40 max-w-40'
+      </div>
-          } ${isLoading ? 'hidden' : ''}`}
-          style={{ objectFit: 'contain' }}
-        />
-      )}
-      {isExpanded && !error && !isLoading && imageData && (
-        <div className="text-xs text-textSubtle mt-1">Click to collapse</div>
-      )}
-      {!isExpanded && !error && !isLoading && imageData && (
-        <div className="text-xs text-textSubtle mt-1">Click to expand</div>
-      )}
     </div>
   );
 }

ui/desktop/src/components/Layout/AppLayout.tsx

@@ -7,9 +7,13 @@ import { Button } from '../ui/button';
 import { Sidebar, SidebarInset, SidebarProvider, SidebarTrigger, useSidebar } from '../ui/sidebar';
 import ChatSessionsContainer from '../ChatSessionsContainer';
 import { useChatContext } from '../../contexts/ChatContext';
+import { UserInput } from '../../types/message';
 
 interface AppLayoutContentProps {
-  activeSessions: Array<{ sessionId: string; initialMessage?: string }>;
+  activeSessions: Array<{
+    sessionId: string;
+    initialMessage?: UserInput;
+  }>;
 }
 
 const AppLayoutContent: React.FC<AppLayoutContentProps> = ({ activeSessions }) => {
@@ -126,7 +130,10 @@ const AppLayoutContent: React.FC<AppLayoutContentProps> = ({ activeSessions }) =
 };
 
 interface AppLayoutProps {
-  activeSessions: Array<{ sessionId: string; initialMessage?: string }>;
+  activeSessions: Array<{
+    sessionId: string;
+    initialMessage?: UserInput;
+  }>;
 }
 
 export const AppLayout: React.FC<AppLayoutProps> = ({ activeSessions }) => {

ui/desktop/src/components/MessageQueue.tsx

@@ -1,11 +1,13 @@
 import React, { useState } from 'react';
 import { X, Clock, Send, GripVertical, Zap, Sparkles, ChevronDown, ChevronUp } from 'lucide-react';
 import { Button } from './ui/button';
+import { ImageData } from '../types/message';
 
-interface QueuedMessage {
+export interface QueuedMessage {
   id: string;
   content: string;
   timestamp: number;
+  images: ImageData[];
 }
 
 interface MessageQueueProps {

ui/desktop/src/components/UserMessage.tsx

@@ -1,8 +1,7 @@
-import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
+import { useCallback, useEffect, useRef, useState } from 'react';
 import ImagePreview from './ImagePreview';
-import { extractImagePaths, removeImagePathsFromText } from '../utils/imageUtils';
 import MarkdownContent from './MarkdownContent';
-import { getTextContent } from '../types/message';
+import { getTextAndImageContent } from '../types/message';
 import { Message } from '../api';
 import MessageCopyLink from './MessageCopyLink';
 import { formatMessageTimestamp } from '../utils/timeUtils';
@@ -21,35 +20,23 @@ export default function UserMessage({ message, onMessageUpdate }: UserMessagePro
   const [editContent, setEditContent] = useState('');
   const [error, setError] = useState<string | null>(null);
 
-  // Extract text content from the message
+  const { textContent, imagePaths } = getTextAndImageContent(message);
-  const textContent = getTextContent(message);
+  const timestamp = formatMessageTimestamp(message.created);
-
-  // Extract image paths from the message
-  const imagePaths = extractImagePaths(textContent);
-
-  // Remove image paths from text for display - memoized for performance
-  const displayText = useMemo(
-    () => removeImagePathsFromText(textContent, imagePaths),
-    [textContent, imagePaths]
-  );
-
-  // Memoize the timestamp
-  const timestamp = useMemo(() => formatMessageTimestamp(message.created), [message.created]);
 
   // Effect to handle message content changes and ensure persistence
   useEffect(() => {
     // If we're not editing, update the edit content to match the current message
     if (!isEditing) {
-      setEditContent(displayText);
+      setEditContent(textContent);
     }
-  }, [message.content, displayText, message.id, isEditing]);
+  }, [message.content, textContent, message.id, isEditing]);
 
   // Initialize edit mode with current message content
   const initializeEditMode = useCallback(() => {
-    setEditContent(displayText);
+    setEditContent(textContent);
     setError(null);
-    window.electron.logInfo(`Entering edit mode with content: ${displayText}`);
+    window.electron.logInfo(`Entering edit mode with content: ${textContent}`);
-  }, [displayText]);
+  }, [textContent]);
 
   // Handle edit button click
   const handleEditClick = useCallback(() => {
@@ -93,7 +80,7 @@ export default function UserMessage({ message, onMessageUpdate }: UserMessagePro
 
       setIsEditing(false);
 
-      if (editContent.trim() === displayText.trim()) {
+      if (editContent.trim() === textContent.trim()) {
         return;
       }
 
@@ -101,16 +88,16 @@ export default function UserMessage({ message, onMessageUpdate }: UserMessagePro
         onMessageUpdate(message.id, editContent, editType);
       }
     },
-    [editContent, displayText, onMessageUpdate, message.id]
+    [editContent, textContent, onMessageUpdate, message.id]
   );
 
   // Handle cancel action
   const handleCancel = useCallback(() => {
     window.electron.logInfo('Cancel clicked - reverting to original content');
     setIsEditing(false);
-    setEditContent(displayText); // Reset to original content
+    setEditContent(textContent); // Reset to original content
     setError(null);
-  }, [displayText]);
+  }, [textContent]);
 
   // Handle keyboard events for accessibility
   const handleKeyDown = useCallback(
@@ -207,20 +194,21 @@ export default function UserMessage({ message, onMessageUpdate }: UserMessagePro
           <div className="message flex justify-end w-full">
             <div className="flex-col max-w-[85%] w-fit">
               <div className="flex flex-col group">
-                <div className="flex bg-background-accent text-text-on-accent rounded-xl py-2.5 px-4">
+                {textContent.trim() && (
-                  <div ref={contentRef}>
+                  <div className="flex bg-background-accent text-text-on-accent rounded-xl py-2.5 px-4">
-                    <MarkdownContent
+                    <div ref={contentRef}>
-                      content={displayText}
+                      <MarkdownContent
-                      className="text-text-on-accent prose-a:text-text-on-accent prose-headings:text-text-on-accent prose-strong:text-text-on-accent prose-em:text-text-on-accent user-message"
+                        content={textContent}
-                    />
+                        className="text-text-on-accent prose-a:text-text-on-accent prose-headings:text-text-on-accent prose-strong:text-text-on-accent prose-em:text-text-on-accent user-message"
+                      />
+                    </div>
                   </div>
-                </div>
+                )}
 
-                {/* Render images if any */}
                 {imagePaths.length > 0 && (
                   <div className="flex flex-wrap gap-2 mt-2">
                     {imagePaths.map((imagePath, index) => (
-                      <ImagePreview key={index} src={imagePath} alt={`Pasted image ${index + 1}`} />
+                      <ImagePreview key={index} src={imagePath} />
                     ))}
                   </div>
                 )}
@@ -239,14 +227,14 @@ export default function UserMessage({ message, onMessageUpdate }: UserMessagePro
                         }
                       }}
                       className="flex items-center gap-1 text-xs text-text-subtle hover:cursor-pointer hover:text-text-prominent transition-all duration-200 opacity-0 group-hover:opacity-100 -translate-y-4 group-hover:translate-y-0 focus:outline-none focus:ring-2 focus:ring-blue-400 focus:ring-opacity-50 rounded"
-                      aria-label={`Edit message: ${displayText.substring(0, 50)}${displayText.length > 50 ? '...' : ''}`}
+                      aria-label={`Edit message: ${textContent.substring(0, 50)}${textContent.length > 50 ? '...' : ''}`}
                       aria-expanded={isEditing}
                       title="Edit message"
                     >
                       <Edit className="h-3 w-3" />
                       <span>Edit</span>
                     </button>
-                    <MessageCopyLink text={displayText} contentRef={contentRef} />
+                    <MessageCopyLink text={textContent} contentRef={contentRef} />
                   </div>
                 </div>
               </div>

ui/desktop/src/components/sessions/SessionViewComponents.tsx

@@ -7,12 +7,11 @@ import MarkdownContent from '../MarkdownContent';
 import ToolCallWithResponse from '../ToolCallWithResponse';
 import ImagePreview from '../ImagePreview';
 import {
-  getTextContent,
+  getTextAndImageContent,
   ToolRequestMessageContent,
   ToolResponseMessageContent,
 } from '../../types/message';
 import { formatMessageTimestamp } from '../../utils/timeUtils';
-import { extractImagePaths, removeImagePathsFromText } from '../../utils/imageUtils';
 import { Message } from '../../api';
 
 /**
@@ -82,15 +81,7 @@ export const SessionMessages: React.FC<SessionMessagesProps> = ({
             ) : messages?.length > 0 ? (
               messages
                 .map((message, index) => {
-                  const textContent = getTextContent(message);
+                  const { textContent, imagePaths } = getTextAndImageContent(message);
-                  // Extract image paths from the message
-                  const imagePaths = extractImagePaths(textContent);
-
-                  // Remove image paths from text for display
-                  const displayText =
-                    imagePaths.length > 0
-                      ? removeImagePathsFromText(textContent, imagePaths)
-                      : textContent;
 
                   // Get tool requests from the message
                   const toolRequests = message.content
@@ -128,24 +119,18 @@ export const SessionMessages: React.FC<SessionMessagesProps> = ({
                       </div>
 
                       <div className="flex flex-col w-full">
-                        {/* Text content */}
+                        {textContent && (
-                        {displayText && (
                           <div
                             className={`${toolRequests.length > 0 || imagePaths.length > 0 ? 'mb-4' : ''}`}
                           >
-                            <MarkdownContent content={displayText} />
+                            <MarkdownContent content={textContent} />
                           </div>
                         )}
 
-                        {/* Render images if any */}
                         {imagePaths.length > 0 && (
                           <div className="flex flex-wrap gap-2 mt-2 mb-2">
                             {imagePaths.map((imagePath, imageIndex) => (
-                              <ImagePreview
+                              <ImagePreview key={imageIndex} src={imagePath} />
-                                key={imageIndex}
-                                src={imagePath}
-                                alt={`Image ${imageIndex + 1}`}
-                              />
                             ))}
                           </div>
                         )}

ui/desktop/src/hooks/useAutoSubmit.ts

@@ -4,6 +4,7 @@ import { useSearchParams } from 'react-router-dom';
 import { Session } from '../api';
 import { Message } from '../api';
 import { ChatState } from '../types/chatState';
+import { UserInput } from '../types/message';
 
 /**
  * Auto-submit scenarios:
@@ -17,8 +18,8 @@ interface UseAutoSubmitProps {
   session: Session | undefined;
   messages: Message[];
   chatState: ChatState;
-  initialMessage: string | undefined;
+  initialMessage: UserInput | undefined;
-  handleSubmit: (message: string) => void;
+  handleSubmit: (input: UserInput) => void;
 }
 
 interface UseAutoSubmitReturn {
@@ -84,7 +85,7 @@ export function useAutoSubmit({
     // Scenario 3: Resume with shouldStartAgent (continue existing conversation)
     if (shouldStartAgent) {
       hasAutoSubmittedRef.current = true;
-      handleSubmit('');
+      handleSubmit({ msg: '', images: [] });
     }
   }, [
     session,

ui/desktop/src/hooks/useChatStream.ts

@@ -21,6 +21,7 @@ import {
   getCompactingMessage,
   getThinkingMessage,
   NotificationEvent,
+  UserInput,
 } from '../types/message';
 import { errorMessage } from '../utils/conversionUtils';
 import { showExtensionLoadResults } from '../utils/extensionErrorUtils';
@@ -39,7 +40,7 @@ interface UseChatStreamReturn {
   messages: Message[];
   chatState: ChatState;
   setChatState: (state: ChatState) => void;
-  handleSubmit: (userMessage: string) => Promise<void>;
+  handleSubmit: (input: UserInput) => Promise<void>;
   submitElicitationResponse: (
     elicitationId: string,
     userData: Record<string, unknown>
@@ -449,7 +450,8 @@ export function useChatStream({
   }, [sessionId, onSessionLoaded]);
 
   const handleSubmit = useCallback(
-    async (userMessage: string) => {
+    async (input: UserInput) => {
+      const { msg: userMessage, images } = input;
       const currentState = stateRef.current;
 
       // Guard: Don't submit if session hasn't been loaded yet
@@ -458,7 +460,7 @@ export function useChatStream({
       }
 
       const hasExistingMessages = currentState.messages.length > 0;
-      const hasNewMessage = userMessage.trim().length > 0;
+      const hasNewMessage = userMessage.trim().length > 0 || images.length > 0;
 
       // Don't submit if there's no message and no conversation to continue
       if (!hasNewMessage && !hasExistingMessages) {
@@ -520,7 +522,7 @@ export function useChatStream({
       }
 
       const newMessage = hasNewMessage
-        ? createUserMessage(userMessage)
+        ? createUserMessage(userMessage, images)
         : currentState.messages[currentState.messages.length - 1];
       const currentMessages = hasNewMessage
         ? [...currentState.messages, newMessage]
@@ -697,7 +699,7 @@ export function useChatStream({
           if (sessionResponse.data?.conversation) {
             dispatch({ type: 'SET_MESSAGES', payload: sessionResponse.data.conversation });
           }
-          await handleSubmit(newContent);
+          await handleSubmit({ msg: newContent, images: [] });
         }
       } catch (error) {
         const errorMsg = errorMessage(error);

ui/desktop/src/hooks/useFileDrop.ts

@@ -1,4 +1,5 @@
 import { useCallback, useState, useRef, useEffect } from 'react';
+import { compressImageDataUrl } from '../utils/conversionUtils';
 
 export interface DroppedFile {
   id: string;
@@ -6,7 +7,7 @@ export interface DroppedFile {
   name: string;
   type: string;
   isImage: boolean;
-  dataUrl?: string; // For image previews
+  dataUrl?: string;
   isLoading?: boolean;
   error?: string;
 }
@@ -71,25 +72,41 @@ export const useFileDrop = () => {
 
         droppedFileObjects.push(droppedFile);
 
-        // For images, generate a preview (only if successfully processed)
         if (droppedFile.isImage && !droppedFile.error) {
           const reader = new FileReader();
           activeReadersRef.current.add(reader);
 
-          reader.onload = (event) => {
+          reader.onload = async (event) => {
             const dataUrl = event.target?.result as string;
-            setDroppedFiles((prev) =>
+            try {
-              prev.map((f) => (f.id === droppedFile.id ? { ...f, dataUrl, isLoading: false } : f))
+              // Compress the image
-            );
+              const compressedDataUrl = await compressImageDataUrl(dataUrl);
+              setDroppedFiles((prev) =>
+                prev.map((f) =>
+                  f.id === droppedFile.id
+                    ? { ...f, dataUrl: compressedDataUrl, isLoading: false }
+                    : f
+                )
+              );
+            } catch (compressionError) {
+              console.error('Failed to compress image:', file.name, compressionError);
+              setDroppedFiles((prev) =>
+                prev.map((f) =>
+                  f.id === droppedFile.id
+                    ? { ...f, error: 'Failed to compress image', isLoading: false }
+                    : f
+                )
+              );
+            }
             activeReadersRef.current.delete(reader);
           };
 
           reader.onerror = () => {
-            console.error('Failed to generate preview for:', file.name);
+            console.error('Failed to read image:', file.name);
             setDroppedFiles((prev) =>
               prev.map((f) =>
                 f.id === droppedFile.id
-                  ? { ...f, error: 'Failed to load image preview', isLoading: false }
+                  ? { ...f, error: 'Failed to load image', isLoading: false }
                   : f
               )
             );

ui/desktop/src/main.ts

@@ -1576,79 +1576,6 @@ ipcMain.handle('save-data-url-to-temp', async (_event, dataUrl: string, uniqueId
   }
 });
 
-// IPC handler to serve temporary image files
-ipcMain.handle('get-temp-image', async (_event, filePath: string) => {
-  console.log(`[Main] Received get-temp-image for path: ${filePath}`);
-
-  // Input validation
-  if (!filePath || typeof filePath !== 'string') {
-    console.warn('[Main] Invalid file path provided for image serving');
-    return null;
-  }
-
-  // Ensure the path is within the designated temp directory
-  const resolvedPath = path.resolve(filePath);
-  const resolvedTempDir = path.resolve(gooseTempDir);
-
-  if (!resolvedPath.startsWith(resolvedTempDir + path.sep)) {
-    console.warn(`[Main] Attempted to access file outside designated temp directory: ${filePath}`);
-    return null;
-  }
-
-  try {
-    // Check if it's a regular file first, before trying realpath
-    const stats = await fs.lstat(filePath);
-    if (!stats.isFile()) {
-      console.warn(`[Main] Not a regular file, refusing to serve: ${filePath}`);
-      return null;
-    }
-
-    // Get the real paths for both the temp directory and the file to handle symlinks properly
-    let realTempDir: string;
-    let actualPath = filePath;
-
-    try {
-      realTempDir = await fs.realpath(gooseTempDir);
-      const realPath = await fs.realpath(filePath);
-
-      // Double-check that the real path is still within our real temp directory
-      if (!realPath.startsWith(realTempDir + path.sep)) {
-        console.warn(
-          `[Main] Real path is outside designated temp directory: ${realPath} not in ${realTempDir}`
-        );
-        return null;
-      }
-      actualPath = realPath;
-    } catch (realpathError) {
-      // If realpath fails, use the original path validation
-      console.log(
-        `[Main] realpath failed for ${filePath}, using original path validation:`,
-        realpathError instanceof Error ? realpathError.message : String(realpathError)
-      );
-    }
-
-    // Read the file and return as base64 data URL
-    const fileBuffer = await fs.readFile(actualPath);
-    const fileExtension = path.extname(actualPath).toLowerCase().substring(1);
-
-    // Validate file extension
-    const allowedExtensions = ['png', 'jpg', 'jpeg', 'gif', 'webp'];
-    if (!allowedExtensions.includes(fileExtension)) {
-      console.warn(`[Main] Unsupported file extension: ${fileExtension}`);
-      return null;
-    }
-
-    const mimeType = fileExtension === 'jpg' ? 'image/jpeg' : `image/${fileExtension}`;
-    const base64Data = fileBuffer.toString('base64');
-    const dataUrl = `data:${mimeType};base64,${base64Data}`;
-
-    console.log(`[Main] Served temp image: ${filePath}`);
-    return dataUrl;
-  } catch (error) {
-    console.error(`[Main] Failed to serve temp image: ${filePath}`, error);
-    return null;
-  }
-});
 ipcMain.on('delete-temp-file', async (_event, filePath: string) => {
   console.log(`[Main] Received delete-temp-file for path: ${filePath}`);
 

ui/desktop/src/preload.ts

@@ -121,8 +121,6 @@ type ElectronAPI = {
   deleteTempFile: (filePath: string) => void;
   // Function for opening external URLs securely
   openExternal: (url: string) => Promise<void>;
-  // Function to serve temp images
-  getTempImage: (filePath: string) => Promise<string | null>;
   // Update-related functions
   getVersion: () => string;
   checkForUpdates: () => Promise<{ updateInfo: unknown; error: string | null }>;
@@ -246,9 +244,6 @@ const electronAPI: ElectronAPI = {
   openExternal: (url: string): Promise<void> => {
     return ipcRenderer.invoke('open-external', url);
   },
-  getTempImage: (filePath: string): Promise<string | null> => {
-    return ipcRenderer.invoke('get-temp-image', filePath);
-  },
   getVersion: (): string => {
     return config.GOOSE_VERSION || ipcRenderer.sendSync('get-app-version') || '';
   },

ui/desktop/src/sessions.ts

@@ -88,9 +88,11 @@ export async function startNewSession(
   // Include session data so sidebar can add it immediately (before it has messages)
   window.dispatchEvent(new CustomEvent(AppEvents.SESSION_CREATED, { detail: { session } }));
 
+  const initialMessage = initialText ? { msg: initialText, images: [] } : undefined;
+
   const eventDetail = {
     sessionId: session.id,
-    initialMessage: initialText,
+    initialMessage,
   };
 
   window.dispatchEvent(
@@ -101,7 +103,7 @@ export async function startNewSession(
 
   setView('pair', {
     disableAnimation: true,
-    initialMessage: initialText,
+    initialMessage,
     resumeSessionId: session.id,
   });
   return session;

ui/desktop/src/types/message.ts

@@ -7,12 +7,38 @@ export type NotificationEvent = Extract<MessageEvent, { type: 'Notification' }>;
 // Compaction response message - must match backend constant
 const COMPACTION_THINKING_TEXT = 'goose is compacting the conversation...';
 
-export function createUserMessage(text: string): Message {
+export interface ImageData {
+  data: string; // base64 encoded image data
+  mimeType: string;
+}
+
+export interface UserInput {
+  msg: string;
+  images: ImageData[];
+}
+
+export function createUserMessage(text: string, images?: ImageData[]): Message {
+  const content: Message['content'] = [];
+
+  if (text.trim()) {
+    content.push({ type: 'text', text });
+  }
+
+  if (images && images.length > 0) {
+    images.forEach((img) => {
+      content.push({
+        type: 'image',
+        data: img.data,
+        mimeType: img.mimeType,
+      });
+    });
+  }
+
   return {
     id: generateMessageId(),
     role: 'user',
     created: Math.floor(Date.now() / 1000),
-    content: [{ type: 'text', text }],
+    content,
     metadata: { userVisible: true, agentVisible: true },
   };
 }
@@ -43,13 +69,22 @@ export function generateMessageId(): string {
   return Math.random().toString(36).substring(2, 10);
 }
 
-export function getTextContent(message: Message): string {
+export function getTextAndImageContent(message: Message): {
-  return message.content
+  textContent: string;
-    .map((content) => {
+  imagePaths: string[];
-      if (content.type === 'text') return content.text;
+} {
-      return '';
+  let textContent = '';
-    })
+  const imagePaths: string[] = [];
-    .join('');
+
+  for (const content of message.content) {
+    if (content.type === 'text') {
+      textContent += content.text;
+    } else if (content.type === 'image') {
+      imagePaths.push(`data:${content.mimeType};base64,${content.data}`);
+    }
+  }
+
+  return { textContent, imagePaths };
 }
 
 export function getToolRequests(message: Message): (ToolRequest & { type: 'toolRequest' })[] {

ui/desktop/src/utils/conversionUtils.ts

@@ -22,6 +22,32 @@ export function errorMessage(err: Error | unknown, default_value?: string) {
   }
 }
 
+export async function compressImageDataUrl(dataUrl: string): Promise<string> {
+  return new Promise((resolve, reject) => {
+    const img = new globalThis.Image();
+    img.onload = () => {
+      const maxDim = 1024;
+      const scale = Math.min(1, maxDim / Math.max(img.width, img.height));
+      const width = Math.floor(img.width * scale);
+      const height = Math.floor(img.height * scale);
+
+      const canvas = document.createElement('canvas');
+      canvas.width = width;
+      canvas.height = height;
+      const ctx = canvas.getContext('2d');
+      if (!ctx) {
+        reject(new Error('Failed to get canvas context'));
+        return;
+      }
+      ctx.drawImage(img, 0, 0, width, height);
+
+      resolve(canvas.toDataURL('image/jpeg', 0.85));
+    };
+    img.onerror = () => reject(new Error('Failed to load image'));
+    img.src = dataUrl;
+  });
+}
+
 export function formatAppName(name: string): string {
   return name
     .split(/[-_\s]+/)

ui/desktop/src/utils/imageUtils.ts

@@ -1,59 +0,0 @@
-/**
- * Utility functions for detecting and handling image paths in messages
- */
-
-/**
- * Extracts image file paths from a message text
- * Looks for paths that match the pattern of pasted images from the temp directory
- *
- * @param text The message text to extract image paths from
- * @returns An array of image file paths found in the message
- */
-export function extractImagePaths(text: string): string[] {
-  if (!text) return [];
-
-  // Match paths that look like pasted image paths from the temp directory
-  // Pattern: /path/to/goose-pasted-images/pasted-img-TIMESTAMP-RANDOM.ext
-  // This regex looks for:
-  // - Word boundary or start of string
-  // - A path containing "goose-pasted-images"
-  // - Followed by a filename starting with "pasted-"
-  // - Ending with common image extensions
-  // - Word boundary or end of string
-  const regex =
-    /(?:^|\s)((?:[^\s]*\/)?goose-pasted-images\/pasted-[^\s]+\.(png|jpg|jpeg|gif|webp))(?=\s|$)/gi;
-
-  const matches = [];
-  let match;
-
-  while ((match = regex.exec(text)) !== null) {
-    matches.push(match[1]);
-  }
-
-  return matches;
-}
-
-/**
- * Removes image paths from the text
- *
- * @param text The original text
- * @param imagePaths Array of image paths to remove
- * @returns Text with image paths removed
- */
-export function removeImagePathsFromText(text: string, imagePaths: string[]): string {
-  if (!text || imagePaths.length === 0) return text;
-
-  let result = text;
-
-  // Remove each image path from the text
-  imagePaths.forEach((path) => {
-    // Escape special regex characters in the path
-    const escapedPath = path.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
-    // Create a regex that matches the path with optional surrounding whitespace
-    const pathRegex = new RegExp(`(^|\\s)${escapedPath}(?=\\s|$)`, 'g');
-    result = result.replace(pathRegex, '$1');
-  });
-
-  // Clean up any extra whitespace
-  return result.replace(/\s+/g, ' ').trim();
-}

ui/desktop/src/utils/navigationUtils.ts

@@ -1,5 +1,6 @@
 import { NavigateFunction } from 'react-router-dom';
-import { Recipe } from '../api/types.gen';
+import { Recipe } from '../api';
+import { UserInput } from '../types/message';
 
 export type View =
   | 'welcome'
@@ -29,7 +30,7 @@ export type ViewOptions = {
   parentView?: View;
   parentViewOptions?: ViewOptions;
   disableAnimation?: boolean;
-  initialMessage?: string;
+  initialMessage?: UserInput;
   shareToken?: string;
   resumeSessionId?: string;
   pendingScheduleDeepLink?: string;

ui/desktop/src/utils/toolCallChaining.ts

@@ -1,4 +1,4 @@
-import { getToolRequests, getTextContent, getToolResponses } from '../types/message';
+import { getToolRequests, getTextAndImageContent, getToolResponses } from '../types/message';
 import { Message } from '../api';
 
 export function identifyConsecutiveToolCalls(messages: Message[]): number[][] {
@@ -9,7 +9,7 @@ export function identifyConsecutiveToolCalls(messages: Message[]): number[][] {
     const message = messages[i];
     const toolRequests = getToolRequests(message);
     const toolResponses = getToolResponses(message);
-    const textContent = getTextContent(message);
+    const { textContent } = getTextAndImageContent(message);
     const hasText = textContent.trim().length > 0;
 
     if (toolResponses.length > 0 && toolRequests.length === 0) {
@@ -47,15 +47,6 @@ export function identifyConsecutiveToolCalls(messages: Message[]): number[][] {
   return chains;
 }
 
-export function shouldHideMessage(messageIndex: number, chains: number[][]): boolean {
-  for (const chain of chains) {
-    if (chain.includes(messageIndex)) {
-      return chain[0] !== messageIndex;
-    }
-  }
-  return false;
-}
-
 export function shouldHideTimestamp(messageIndex: number, chains: number[][]): boolean {
   for (const chain of chains) {
     if (chain.includes(messageIndex)) {
@@ -69,7 +60,3 @@ export function shouldHideTimestamp(messageIndex: number, chains: number[][]): b
 export function isInChain(messageIndex: number, chains: number[][]): boolean {
   return chains.some((chain) => chain.includes(messageIndex));
 }
-
-export function getChainForMessage(messageIndex: number, chains: number[][]): number[] | null {
-  return chains.find((chain) => chain.includes(messageIndex)) || null;
-}