feat(goose2): voice dictation via direct-ACP pattern (#8609)

Signed-off-by: tulsi <tulsi@block.xyz> Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-04-20 12:26:50 -07:00
parent 2efa625da9
commit 5642e0b81e
40 changed files with 4588 additions and 25 deletions
@@ -0,0 +1,140 @@
+import { describe, it, expect, vi, beforeEach } from "vitest";
+import {
+  cancelDictationLocalModelDownload,
+  deleteDictationLocalModel,
+  downloadDictationLocalModel,
+  getDictationConfig,
+  getDictationLocalModelDownloadProgress,
+  listDictationLocalModels,
+  saveDictationModelSelection,
+  transcribeDictation,
+} from "../dictation";
+import { getClient } from "../acpConnection";
+
+vi.mock("../acpConnection", () => ({
+  getClient: vi.fn(),
+}));
+
+describe("dictation SDK wiring", () => {
+  let client: any;
+  beforeEach(() => {
+    client = {
+      goose: {
+        GooseDictationConfig: vi.fn().mockResolvedValue({
+          providers: {
+            openai: {
+              configured: true,
+              description: "OpenAI transcription",
+              usesProviderConfig: true,
+              availableModels: [],
+            },
+          },
+        }),
+        GooseDictationTranscribe: vi.fn().mockResolvedValue({ text: "hello" }),
+      },
+    };
+    vi.mocked(getClient).mockResolvedValue(client);
+  });
+
+  it("getDictationConfig calls GooseDictationConfig and returns providers map", async () => {
+    const result = await getDictationConfig();
+    expect(client.goose.GooseDictationConfig).toHaveBeenCalledWith({});
+    expect(result.openai.configured).toBe(true);
+  });
+
+  it("transcribeDictation forwards audio + mimeType + provider", async () => {
+    const result = await transcribeDictation({
+      audio: "base64==",
+      mimeType: "audio/webm",
+      provider: "openai" as any,
+    });
+    expect(client.goose.GooseDictationTranscribe).toHaveBeenCalledWith({
+      audio: "base64==",
+      mimeType: "audio/webm",
+      provider: "openai",
+    });
+    expect(result.text).toBe("hello");
+  });
+
+  it("saveDictationModelSelection calls GooseDictationModelSelect", async () => {
+    client.goose.GooseDictationModelSelect = vi.fn().mockResolvedValue({});
+    await saveDictationModelSelection("local" as any, "tiny");
+    expect(client.goose.GooseDictationModelSelect).toHaveBeenCalledWith({
+      provider: "local",
+      modelId: "tiny",
+    });
+  });
+
+  it("listDictationLocalModels returns the models array", async () => {
+    client.goose.GooseDictationModelsList = vi.fn().mockResolvedValue({
+      models: [
+        {
+          id: "tiny",
+          description: "Tiny",
+          sizeMb: 75,
+          downloaded: true,
+          downloadInProgress: false,
+        },
+      ],
+    });
+    const result = await listDictationLocalModels();
+    expect(client.goose.GooseDictationModelsList).toHaveBeenCalledWith({});
+    expect(result).toHaveLength(1);
+    expect(result[0].id).toBe("tiny");
+  });
+
+  it("downloadDictationLocalModel forwards modelId", async () => {
+    client.goose.GooseDictationModelsDownload = vi.fn().mockResolvedValue({});
+    await downloadDictationLocalModel("tiny");
+    expect(client.goose.GooseDictationModelsDownload).toHaveBeenCalledWith({
+      modelId: "tiny",
+    });
+  });
+
+  it("getDictationLocalModelDownloadProgress returns progress or null", async () => {
+    client.goose.GooseDictationModelsDownloadProgress = vi
+      .fn()
+      .mockResolvedValue({
+        progress: {
+          bytesDownloaded: 100,
+          totalBytes: 1000,
+          progressPercent: 10,
+          status: "downloading",
+          error: null,
+        },
+      });
+    const result = await getDictationLocalModelDownloadProgress("tiny");
+    expect(result?.bytesDownloaded).toBe(100);
+    expect(
+      client.goose.GooseDictationModelsDownloadProgress,
+    ).toHaveBeenCalledWith({
+      modelId: "tiny",
+    });
+  });
+
+  it("getDictationLocalModelDownloadProgress returns null when no download", async () => {
+    client.goose.GooseDictationModelsDownloadProgress = vi
+      .fn()
+      .mockResolvedValue({
+        progress: undefined,
+      });
+    const result = await getDictationLocalModelDownloadProgress("tiny");
+    expect(result).toBeNull();
+  });
+
+  it("cancelDictationLocalModelDownload forwards modelId", async () => {
+    client.goose.GooseDictationModelsCancel = vi.fn().mockResolvedValue({});
+    await cancelDictationLocalModelDownload("tiny");
+    expect(client.goose.GooseDictationModelsCancel).toHaveBeenCalledWith({
+      modelId: "tiny",
+    });
+  });
+
+  it("deleteDictationLocalModel forwards modelId", async () => {
+    client.goose.GooseDictationModelsDelete = vi.fn().mockResolvedValue({});
+    await deleteDictationLocalModel("tiny");
+    expect(client.goose.GooseDictationModelsDelete).toHaveBeenCalledWith({
+      modelId: "tiny",
+    });
+  });
+});
@@ -0,0 +1,106 @@
+import { invoke } from "@tauri-apps/api/core";
+import type {
+  DictationDownloadProgress,
+  DictationProvider,
+  DictationProviderStatus,
+  DictationTranscribeResponse,
+  WhisperModelStatus,
+} from "@/shared/types/dictation";
+import { getClient } from "./acpConnection";
+
+export async function getDictationConfig(): Promise<
+  Record<DictationProvider, DictationProviderStatus>
+> {
+  const client = await getClient();
+  const response = await client.goose.GooseDictationConfig({});
+  return response.providers as Record<
+    DictationProvider,
+    DictationProviderStatus
+  >;
+}
+
+export async function transcribeDictation(request: {
+  audio: string;
+  mimeType: string;
+  provider: DictationProvider;
+}): Promise<DictationTranscribeResponse> {
+  const client = await getClient();
+  return client.goose.GooseDictationTranscribe({
+    audio: request.audio,
+    mimeType: request.mimeType,
+    provider: request.provider,
+  });
+}
+
+export async function saveDictationModelSelection(
+  provider: DictationProvider,
+  modelId: string,
+): Promise<void> {
+  const client = await getClient();
+  await client.goose.GooseDictationModelSelect({ provider, modelId });
+}
+
+export async function saveDictationProviderSecret(
+  _provider: DictationProvider,
+  value: string,
+  configKey?: string,
+): Promise<void> {
+  if (!configKey) {
+    throw new Error("No config key for this provider");
+  }
+  return invoke("save_provider_field", { key: configKey, value });
+}
+
+export async function deleteDictationProviderSecret(
+  provider: DictationProvider,
+  _configKey?: string,
+): Promise<void> {
+  const providerIdMap: Record<string, string> = {
+    groq: "dictation_groq",
+    elevenlabs: "dictation_elevenlabs",
+  };
+  const providerId = providerIdMap[provider];
+  if (!providerId) {
+    throw new Error("Cannot delete secrets for this provider");
+  }
+  return invoke("delete_provider_config", { providerId });
+}
+
+export async function listDictationLocalModels(): Promise<
+  WhisperModelStatus[]
+> {
+  const client = await getClient();
+  const response = await client.goose.GooseDictationModelsList({});
+  return response.models as unknown as WhisperModelStatus[];
+}
+
+export async function downloadDictationLocalModel(
+  modelId: string,
+): Promise<void> {
+  const client = await getClient();
+  await client.goose.GooseDictationModelsDownload({ modelId });
+}
+
+export async function getDictationLocalModelDownloadProgress(
+  modelId: string,
+): Promise<DictationDownloadProgress | null> {
+  const client = await getClient();
+  const response = await client.goose.GooseDictationModelsDownloadProgress({
+    modelId,
+  });
+  return (response.progress ?? null) as DictationDownloadProgress | null;
+}
+
+export async function cancelDictationLocalModelDownload(
+  modelId: string,
+): Promise<void> {
+  const client = await getClient();
+  await client.goose.GooseDictationModelsCancel({ modelId });
+}
+
+export async function deleteDictationLocalModel(
+  modelId: string,
+): Promise<void> {
+  const client = await getClient();
+  await client.goose.GooseDictationModelsDelete({ modelId });
+}
@@ -169,7 +169,11 @@
    "selectProject": "Select project",
    "sendMessage": "Send message",
    "stopGeneration": "Stop generation",
-    "voiceInputSoon": "Voice input (coming soon)"
+    "voiceInput": "Voice dictation",
+    "voiceInputDisabled": "Configure a voice provider in Settings to enable dictation",
+    "voiceInputRecording": "Listening...",
+    "voiceInputTranscribing": "Transcribing...",
+    "voiceInputAutoSubmitHint": "Say \"submit\" to send"
  },
  "tools": {
    "fileNotFound": "File not found: {{path}}",
@@ -124,7 +124,49 @@
      "spanish": "Spanish",
      "system": "System default ({{language}})"
    },
-    "title": "General"
+    "title": "General",
+    "voiceInput": {
+      "label": "Voice Input",
+      "description": "Configure voice dictation for hands-free input.",
+      "providerLabel": "Transcription Provider",
+      "disabled": "Disabled",
+      "notConfiguredSuffix": "(not configured)",
+      "placeholder": "Select a provider",
+      "modelLabel": "Model",
+      "apiKeyLabel": "API Key",
+      "apiKeyDescription": "Enter your API key for this provider.",
+      "apiKeyPlaceholder": "sk-...",
+      "apiKeyConfigured": "API key configured",
+      "addApiKey": "Add API key",
+      "updateApiKey": "Update API key",
+      "removeApiKey": "Remove API key",
+      "localModelLabel": "Local Whisper Model",
+      "localModelDescription": "Download a Whisper model to run transcription locally. Selecting a model sets it as your active local transcription model.",
+      "noLocalModels": "No local Whisper models available.",
+      "download": "Download",
+      "selectModel": "Select",
+      "selectedModel": "Selected",
+      "deleteModel": "Delete",
+      "microphoneLabel": "Microphone",
+      "microphoneDescription": "Choose which microphone to use for voice input.",
+      "microphoneUnavailable": "Microphone access is not available in this environment.",
+      "microphoneAccessPrompt": "Click \"Grant access\" to allow microphone use.",
+      "grantMicrophone": "Grant access",
+      "systemMicrophone": "System default",
+      "unknownMicrophone": "Unknown microphone",
+      "autoSubmitLabel": "Auto-submit Phrases",
+      "autoSubmitDescription": "Comma-separated words that trigger automatic send (e.g. \"submit\").",
+      "providers": {
+        "openai": "OpenAI Whisper",
+        "groq": "Groq",
+        "elevenlabs": "ElevenLabs",
+        "local": "Local Whisper"
+      },
+      "downloadProgress": "Downloading... {{percent}}%",
+      "loadError": "Failed to load voice settings.",
+      "saveError": "Failed to save.",
+      "deleteError": "Failed to delete."
+    }
  },
  "nav": {
    "about": "About",
@@ -134,7 +176,8 @@
    "general": "General",
    "projects": "Projects",
    "extensions": "Extensions",
-    "providers": "Providers"
+    "providers": "Providers",
+    "voice": "Voice"
  },
  "projects": {
    "description": "Manage your projects.",
@@ -169,7 +169,11 @@
    "selectProject": "Seleccionar proyecto",
    "sendMessage": "Enviar mensaje",
    "stopGeneration": "Detener generación",
-    "voiceInputSoon": "Entrada de voz (pronto)"
+    "voiceInput": "Dictado por voz",
+    "voiceInputDisabled": "Configura un proveedor de voz en Ajustes para activar el dictado",
+    "voiceInputRecording": "Escuchando...",
+    "voiceInputTranscribing": "Transcribiendo...",
+    "voiceInputAutoSubmitHint": "Di \"enviar\" para enviar"
  },
  "tools": {
    "fileNotFound": "Archivo no encontrado: {{path}}",
@@ -124,7 +124,49 @@
      "spanish": "Español",
      "system": "Predeterminado del sistema ({{language}})"
    },
-    "title": "General"
+    "title": "General",
+    "voiceInput": {
+      "label": "Entrada de voz",
+      "description": "Configura el dictado por voz para entrada manos libres.",
+      "providerLabel": "Proveedor de transcripción",
+      "disabled": "Desactivado",
+      "notConfiguredSuffix": "(no configurado)",
+      "placeholder": "Selecciona un proveedor",
+      "modelLabel": "Modelo",
+      "apiKeyLabel": "Clave API",
+      "apiKeyDescription": "Ingresa tu clave API para este proveedor.",
+      "apiKeyPlaceholder": "sk-...",
+      "apiKeyConfigured": "Clave API configurada",
+      "addApiKey": "Agregar clave API",
+      "updateApiKey": "Actualizar clave API",
+      "removeApiKey": "Eliminar clave API",
+      "localModelLabel": "Modelo Whisper local",
+      "localModelDescription": "Descarga un modelo Whisper para transcribir localmente. Seleccionar un modelo lo establece como tu modelo de transcripción local activo.",
+      "noLocalModels": "No hay modelos Whisper locales disponibles.",
+      "download": "Descargar",
+      "selectModel": "Seleccionar",
+      "selectedModel": "Seleccionado",
+      "deleteModel": "Eliminar",
+      "microphoneLabel": "Micrófono",
+      "microphoneDescription": "Elige qué micrófono usar para la entrada de voz.",
+      "microphoneUnavailable": "El acceso al micrófono no está disponible en este entorno.",
+      "microphoneAccessPrompt": "Haz clic en \"Permitir acceso\" para usar el micrófono.",
+      "grantMicrophone": "Permitir acceso",
+      "systemMicrophone": "Predeterminado del sistema",
+      "unknownMicrophone": "Micrófono desconocido",
+      "autoSubmitLabel": "Frases de envío automático",
+      "autoSubmitDescription": "Palabras separadas por coma que activan el envío automático (ej. \"enviar\").",
+      "providers": {
+        "openai": "OpenAI Whisper",
+        "groq": "Groq",
+        "elevenlabs": "ElevenLabs",
+        "local": "Whisper local"
+      },
+      "downloadProgress": "Descargando... {{percent}}%",
+      "loadError": "Error al cargar ajustes de voz.",
+      "saveError": "Error al guardar.",
+      "deleteError": "Error al eliminar."
+    }
  },
  "nav": {
    "about": "Acerca de",
@@ -134,7 +176,8 @@
    "general": "General",
    "projects": "Proyectos",
    "extensions": "Extensiones",
-    "providers": "Proveedores"
+    "providers": "Proveedores",
+    "voice": "Voz"
  },
  "projects": {
    "description": "Administra tus proyectos.",
@@ -0,0 +1,47 @@
+export type DictationProvider = "openai" | "groq" | "elevenlabs" | "local";
+
+export interface DictationModelOption {
+  id: string;
+  label: string;
+  description: string;
+}
+
+export interface DictationProviderStatus {
+  configured: boolean;
+  host?: string | null;
+  description: string;
+  usesProviderConfig: boolean;
+  settingsPath?: string | null;
+  configKey?: string | null;
+  modelConfigKey?: string | null;
+  defaultModel?: string | null;
+  selectedModel?: string | null;
+  availableModels: DictationModelOption[];
+}
+
+export interface DictationTranscribeResponse {
+  text: string;
+}
+
+export type MicrophonePermissionStatus =
+  | "not_determined"
+  | "authorized"
+  | "denied"
+  | "restricted"
+  | "unsupported";
+
+export interface WhisperModelStatus {
+  id: string;
+  sizeMb: number;
+  description: string;
+  downloaded: boolean;
+  downloadInProgress: boolean;
+}
+
+export interface DictationDownloadProgress {
+  bytesDownloaded: number;
+  totalBytes: number;
+  progressPercent: number;
+  status: string;
+  error?: string | null;
+}
@@ -74,10 +74,6 @@ export const useAudioDevices = () => {
  }, []);

  const loadDevicesWithPermission = useCallback(async () => {
-    if (loading) {
-      return;
-    }
-
    try {
      setLoading(true);
      setError(null);
@@ -108,11 +104,57 @@ export const useAudioDevices = () => {
    } finally {
      setLoading(false);
    }
-  }, [loading]);
+  }, []);

  useEffect(() => {
-    loadDevicesWithoutPermission();
-  }, [loadDevicesWithoutPermission]);
+    let cancelled = false;
+    let status: PermissionStatus | null = null;
+    const onChange = () => {
+      if (cancelled || !status) return;
+      const granted = status.state === "granted";
+      setHasPermission(granted);
+      // When permission flips to granted mid-session (e.g. the user enabled
+      // mic access via OS settings), re-enumerate devices so we pick up the
+      // real deviceIds/labels — the prior enumeration may have returned
+      // empty-string entries that VoiceInputSettings filters out.
+      if (granted) {
+        void loadDevicesWithPermission();
+      }
+    };
+
+    const init = async () => {
+      let alreadyGranted = false;
+      try {
+        status = await navigator.permissions.query({
+          name: "microphone" as PermissionName,
+        });
+        if (cancelled) return;
+        alreadyGranted = status.state === "granted";
+        setHasPermission(alreadyGranted);
+        status.addEventListener("change", onChange);
+      } catch {
+        // Permissions API not available for microphone; fall back silently.
+      }
+      if (cancelled) return;
+      // If OS-level permission is already granted, enumerate through the
+      // permission-ful path — otherwise enumerateDevices() may return
+      // entries with empty deviceId/label, which Radix Select rejects.
+      if (alreadyGranted) {
+        await loadDevicesWithPermission();
+      } else {
+        await loadDevicesWithoutPermission();
+      }
+    };
+
+    void init();
+
+    return () => {
+      cancelled = true;
+      if (status) {
+        status.removeEventListener("change", onChange);
+      }
+    };
+  }, [loadDevicesWithPermission, loadDevicesWithoutPermission]);

  useEffect(() => {
    const handleDeviceChange = () => {