Native images (#6619)

Co-authored-by: Douwe Osinga <douwe@squareup.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

ui/desktop/src/hooks/useAutoSubmit.ts

@@ -4,6 +4,7 @@ import { useSearchParams } from 'react-router-dom';
 import { Session } from '../api';
 import { Message } from '../api';
 import { ChatState } from '../types/chatState';
+import { UserInput } from '../types/message';
 
 /**
  * Auto-submit scenarios:
@@ -17,8 +18,8 @@ interface UseAutoSubmitProps {
   session: Session | undefined;
   messages: Message[];
   chatState: ChatState;
-  initialMessage: string | undefined;
-  handleSubmit: (message: string) => void;
+  initialMessage: UserInput | undefined;
+  handleSubmit: (input: UserInput) => void;
 }
 
 interface UseAutoSubmitReturn {
@@ -84,7 +85,7 @@ export function useAutoSubmit({
     // Scenario 3: Resume with shouldStartAgent (continue existing conversation)
     if (shouldStartAgent) {
       hasAutoSubmittedRef.current = true;
-      handleSubmit('');
+      handleSubmit({ msg: '', images: [] });
     }
   }, [
     session,

ui/desktop/src/hooks/useChatStream.ts

@@ -21,6 +21,7 @@ import {
   getCompactingMessage,
   getThinkingMessage,
   NotificationEvent,
+  UserInput,
 } from '../types/message';
 import { errorMessage } from '../utils/conversionUtils';
 import { showExtensionLoadResults } from '../utils/extensionErrorUtils';
@@ -39,7 +40,7 @@ interface UseChatStreamReturn {
   messages: Message[];
   chatState: ChatState;
   setChatState: (state: ChatState) => void;
-  handleSubmit: (userMessage: string) => Promise<void>;
+  handleSubmit: (input: UserInput) => Promise<void>;
   submitElicitationResponse: (
     elicitationId: string,
     userData: Record<string, unknown>
@@ -449,7 +450,8 @@ export function useChatStream({
   }, [sessionId, onSessionLoaded]);
 
   const handleSubmit = useCallback(
-    async (userMessage: string) => {
+    async (input: UserInput) => {
+      const { msg: userMessage, images } = input;
       const currentState = stateRef.current;
 
       // Guard: Don't submit if session hasn't been loaded yet
@@ -458,7 +460,7 @@ export function useChatStream({
       }
 
       const hasExistingMessages = currentState.messages.length > 0;
-      const hasNewMessage = userMessage.trim().length > 0;
+      const hasNewMessage = userMessage.trim().length > 0 || images.length > 0;
 
       // Don't submit if there's no message and no conversation to continue
       if (!hasNewMessage && !hasExistingMessages) {
@@ -520,7 +522,7 @@ export function useChatStream({
       }
 
       const newMessage = hasNewMessage
-        ? createUserMessage(userMessage)
+        ? createUserMessage(userMessage, images)
         : currentState.messages[currentState.messages.length - 1];
       const currentMessages = hasNewMessage
         ? [...currentState.messages, newMessage]
@@ -697,7 +699,7 @@ export function useChatStream({
           if (sessionResponse.data?.conversation) {
             dispatch({ type: 'SET_MESSAGES', payload: sessionResponse.data.conversation });
           }
-          await handleSubmit(newContent);
+          await handleSubmit({ msg: newContent, images: [] });
         }
       } catch (error) {
         const errorMsg = errorMessage(error);

ui/desktop/src/hooks/useFileDrop.ts

@@ -1,4 +1,5 @@
 import { useCallback, useState, useRef, useEffect } from 'react';
+import { compressImageDataUrl } from '../utils/conversionUtils';
 
 export interface DroppedFile {
   id: string;
@@ -6,7 +7,7 @@ export interface DroppedFile {
   name: string;
   type: string;
   isImage: boolean;
-  dataUrl?: string; // For image previews
+  dataUrl?: string;
   isLoading?: boolean;
   error?: string;
 }
@@ -71,25 +72,41 @@ export const useFileDrop = () => {
 
         droppedFileObjects.push(droppedFile);
 
-        // For images, generate a preview (only if successfully processed)
         if (droppedFile.isImage && !droppedFile.error) {
           const reader = new FileReader();
           activeReadersRef.current.add(reader);
 
-          reader.onload = (event) => {
+          reader.onload = async (event) => {
             const dataUrl = event.target?.result as string;
-            setDroppedFiles((prev) =>
-              prev.map((f) => (f.id === droppedFile.id ? { ...f, dataUrl, isLoading: false } : f))
-            );
+            try {
+              // Compress the image
+              const compressedDataUrl = await compressImageDataUrl(dataUrl);
+              setDroppedFiles((prev) =>
+                prev.map((f) =>
+                  f.id === droppedFile.id
+                    ? { ...f, dataUrl: compressedDataUrl, isLoading: false }
+                    : f
+                )
+              );
+            } catch (compressionError) {
+              console.error('Failed to compress image:', file.name, compressionError);
+              setDroppedFiles((prev) =>
+                prev.map((f) =>
+                  f.id === droppedFile.id
+                    ? { ...f, error: 'Failed to compress image', isLoading: false }
+                    : f
+                )
+              );
+            }
             activeReadersRef.current.delete(reader);
           };
 
           reader.onerror = () => {
-            console.error('Failed to generate preview for:', file.name);
+            console.error('Failed to read image:', file.name);
             setDroppedFiles((prev) =>
               prev.map((f) =>
                 f.id === droppedFile.id
-                  ? { ...f, error: 'Failed to load image preview', isLoading: false }
+                  ? { ...f, error: 'Failed to load image', isLoading: false }
                   : f
               )
             );