feat(goose2): voice dictation via direct-ACP pattern (#8609)
Signed-off-by: tulsi <tulsi@block.xyz> Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
tulsi
@@ -13,6 +13,18 @@ import type {
|
||||
CheckSecretRequest,
|
||||
CheckSecretResponse,
|
||||
DeleteSessionRequest,
|
||||
DictationConfigRequest,
|
||||
DictationConfigResponse,
|
||||
DictationModelCancelRequest,
|
||||
DictationModelDeleteRequest,
|
||||
DictationModelDownloadProgressRequest,
|
||||
DictationModelDownloadProgressResponse,
|
||||
DictationModelDownloadRequest,
|
||||
DictationModelSelectRequest,
|
||||
DictationModelsListRequest,
|
||||
DictationModelsListResponse,
|
||||
DictationTranscribeRequest,
|
||||
DictationTranscribeResponse,
|
||||
ExportSessionRequest,
|
||||
ExportSessionResponse,
|
||||
GetExtensionsRequest,
|
||||
@@ -43,6 +55,10 @@ import type {
|
||||
} from './types.gen.js';
|
||||
import {
|
||||
zCheckSecretResponse,
|
||||
zDictationConfigResponse,
|
||||
zDictationModelDownloadProgressResponse,
|
||||
zDictationModelsListResponse,
|
||||
zDictationTranscribeResponse,
|
||||
zExportSessionResponse,
|
||||
zGetExtensionsResponse,
|
||||
zGetProviderDetailsResponse,
|
||||
@@ -174,4 +190,71 @@ export class GooseExtClient {
|
||||
async GooseSessionUnarchive(params: UnarchiveSessionRequest): Promise<void> {
|
||||
await this.conn.extMethod("_goose/session/unarchive", params);
|
||||
}
|
||||
|
||||
async GooseDictationTranscribe(
|
||||
params: DictationTranscribeRequest,
|
||||
): Promise<DictationTranscribeResponse> {
|
||||
const raw = await this.conn.extMethod(
|
||||
"_goose/dictation/transcribe",
|
||||
params,
|
||||
);
|
||||
return zDictationTranscribeResponse.parse(
|
||||
raw,
|
||||
) as DictationTranscribeResponse;
|
||||
}
|
||||
|
||||
async GooseDictationConfig(
|
||||
params: DictationConfigRequest,
|
||||
): Promise<DictationConfigResponse> {
|
||||
const raw = await this.conn.extMethod("_goose/dictation/config", params);
|
||||
return zDictationConfigResponse.parse(raw) as DictationConfigResponse;
|
||||
}
|
||||
|
||||
async GooseDictationModelsList(
|
||||
params: DictationModelsListRequest,
|
||||
): Promise<DictationModelsListResponse> {
|
||||
const raw = await this.conn.extMethod(
|
||||
"_goose/dictation/models/list",
|
||||
params,
|
||||
);
|
||||
return zDictationModelsListResponse.parse(
|
||||
raw,
|
||||
) as DictationModelsListResponse;
|
||||
}
|
||||
|
||||
async GooseDictationModelsDownload(
|
||||
params: DictationModelDownloadRequest,
|
||||
): Promise<void> {
|
||||
await this.conn.extMethod("_goose/dictation/models/download", params);
|
||||
}
|
||||
|
||||
async GooseDictationModelsDownloadProgress(
|
||||
params: DictationModelDownloadProgressRequest,
|
||||
): Promise<DictationModelDownloadProgressResponse> {
|
||||
const raw = await this.conn.extMethod(
|
||||
"_goose/dictation/models/download/progress",
|
||||
params,
|
||||
);
|
||||
return zDictationModelDownloadProgressResponse.parse(
|
||||
raw,
|
||||
) as DictationModelDownloadProgressResponse;
|
||||
}
|
||||
|
||||
async GooseDictationModelsCancel(
|
||||
params: DictationModelCancelRequest,
|
||||
): Promise<void> {
|
||||
await this.conn.extMethod("_goose/dictation/models/cancel", params);
|
||||
}
|
||||
|
||||
async GooseDictationModelsDelete(
|
||||
params: DictationModelDeleteRequest,
|
||||
): Promise<void> {
|
||||
await this.conn.extMethod("_goose/dictation/models/delete", params);
|
||||
}
|
||||
|
||||
async GooseDictationModelSelect(
|
||||
params: DictationModelSelectRequest,
|
||||
): Promise<void> {
|
||||
await this.conn.extMethod("_goose/dictation/model/select", params);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
// This file is auto-generated by @hey-api/openapi-ts
|
||||
|
||||
export type { AddExtensionRequest, ArchiveSessionRequest, CheckSecretRequest, CheckSecretResponse, DeleteSessionRequest, EmptyResponse, ExportSessionRequest, ExportSessionResponse, ExtRequest, ExtResponse, GetExtensionsRequest, GetExtensionsResponse, GetProviderDetailsRequest, GetProviderDetailsResponse, GetProviderModelsRequest, GetProviderModelsResponse, GetSessionExtensionsRequest, GetSessionExtensionsResponse, GetToolsRequest, GetToolsResponse, ImportSessionRequest, ImportSessionResponse, ListProvidersRequest, ListProvidersResponse, ModelEntry, ProviderConfigKey, ProviderDetailEntry, ProviderListEntry, ReadConfigRequest, ReadConfigResponse, ReadResourceRequest, ReadResourceResponse, RemoveConfigRequest, RemoveExtensionRequest, RemoveSecretRequest, UnarchiveSessionRequest, UpdateWorkingDirRequest, UpsertConfigRequest, UpsertSecretRequest } from './types.gen.js';
|
||||
export type { AddExtensionRequest, ArchiveSessionRequest, CheckSecretRequest, CheckSecretResponse, DeleteSessionRequest, DictationConfigRequest, DictationConfigResponse, DictationDownloadProgress, DictationLocalModelStatus, DictationModelCancelRequest, DictationModelDeleteRequest, DictationModelDownloadProgressRequest, DictationModelDownloadProgressResponse, DictationModelDownloadRequest, DictationModelOption, DictationModelSelectRequest, DictationModelsListRequest, DictationModelsListResponse, DictationProviderStatusEntry, DictationTranscribeRequest, DictationTranscribeResponse, EmptyResponse, ExportSessionRequest, ExportSessionResponse, ExtRequest, ExtResponse, GetExtensionsRequest, GetExtensionsResponse, GetProviderDetailsRequest, GetProviderDetailsResponse, GetProviderModelsRequest, GetProviderModelsResponse, GetSessionExtensionsRequest, GetSessionExtensionsResponse, GetToolsRequest, GetToolsResponse, ImportSessionRequest, ImportSessionResponse, ListProvidersRequest, ListProvidersResponse, ModelEntry, ProviderConfigKey, ProviderDetailEntry, ProviderListEntry, ReadConfigRequest, ReadConfigResponse, ReadResourceRequest, ReadResourceResponse, RemoveConfigRequest, RemoveExtensionRequest, RemoveSecretRequest, UnarchiveSessionRequest, UpdateWorkingDirRequest, UpsertConfigRequest, UpsertSecretRequest } from './types.gen.js';
|
||||
|
||||
export const GOOSE_EXT_METHODS = [
|
||||
{
|
||||
@@ -108,6 +108,46 @@ export const GOOSE_EXT_METHODS = [
|
||||
requestType: "UnarchiveSessionRequest",
|
||||
responseType: "EmptyResponse",
|
||||
},
|
||||
{
|
||||
method: "_goose/dictation/transcribe",
|
||||
requestType: "DictationTranscribeRequest",
|
||||
responseType: "DictationTranscribeResponse",
|
||||
},
|
||||
{
|
||||
method: "_goose/dictation/config",
|
||||
requestType: "DictationConfigRequest",
|
||||
responseType: "DictationConfigResponse",
|
||||
},
|
||||
{
|
||||
method: "_goose/dictation/models/list",
|
||||
requestType: "DictationModelsListRequest",
|
||||
responseType: "DictationModelsListResponse",
|
||||
},
|
||||
{
|
||||
method: "_goose/dictation/models/download",
|
||||
requestType: "DictationModelDownloadRequest",
|
||||
responseType: "EmptyResponse",
|
||||
},
|
||||
{
|
||||
method: "_goose/dictation/models/download/progress",
|
||||
requestType: "DictationModelDownloadProgressRequest",
|
||||
responseType: "DictationModelDownloadProgressResponse",
|
||||
},
|
||||
{
|
||||
method: "_goose/dictation/models/cancel",
|
||||
requestType: "DictationModelCancelRequest",
|
||||
responseType: "EmptyResponse",
|
||||
},
|
||||
{
|
||||
method: "_goose/dictation/models/delete",
|
||||
requestType: "DictationModelDeleteRequest",
|
||||
responseType: "EmptyResponse",
|
||||
},
|
||||
{
|
||||
method: "_goose/dictation/model/select",
|
||||
requestType: "DictationModelSelectRequest",
|
||||
responseType: "EmptyResponse",
|
||||
},
|
||||
] as const;
|
||||
|
||||
export type GooseExtMethod = (typeof GOOSE_EXT_METHODS)[number];
|
||||
|
||||
@@ -281,17 +281,154 @@ export type UnarchiveSessionRequest = {
|
||||
sessionId: string;
|
||||
};
|
||||
|
||||
/**
|
||||
* Transcribe audio via a dictation provider.
|
||||
*/
|
||||
export type DictationTranscribeRequest = {
|
||||
/**
|
||||
* Base64-encoded audio data
|
||||
*/
|
||||
audio: string;
|
||||
/**
|
||||
* MIME type (e.g. "audio/wav", "audio/webm")
|
||||
*/
|
||||
mimeType: string;
|
||||
/**
|
||||
* Provider to use: "openai", "groq", "elevenlabs", or "local"
|
||||
*/
|
||||
provider: string;
|
||||
};
|
||||
|
||||
/**
|
||||
* Transcription result.
|
||||
*/
|
||||
export type DictationTranscribeResponse = {
|
||||
text: string;
|
||||
};
|
||||
|
||||
/**
|
||||
* Get the configuration status of all dictation providers.
|
||||
*/
|
||||
export type DictationConfigRequest = {
|
||||
[key: string]: unknown;
|
||||
};
|
||||
|
||||
/**
|
||||
* Dictation config response — map of provider name to status.
|
||||
*/
|
||||
export type DictationConfigResponse = {
|
||||
providers: {
|
||||
[key: string]: DictationProviderStatusEntry;
|
||||
};
|
||||
};
|
||||
|
||||
/**
|
||||
* Per-provider configuration status.
|
||||
*/
|
||||
export type DictationProviderStatusEntry = {
|
||||
configured: boolean;
|
||||
host?: string | null;
|
||||
description: string;
|
||||
usesProviderConfig: boolean;
|
||||
settingsPath?: string | null;
|
||||
configKey?: string | null;
|
||||
modelConfigKey?: string | null;
|
||||
defaultModel?: string | null;
|
||||
selectedModel?: string | null;
|
||||
availableModels?: Array<DictationModelOption>;
|
||||
};
|
||||
|
||||
export type DictationModelOption = {
|
||||
id: string;
|
||||
label: string;
|
||||
description: string;
|
||||
};
|
||||
|
||||
/**
|
||||
* List available local Whisper models with their download status.
|
||||
*/
|
||||
export type DictationModelsListRequest = {
|
||||
[key: string]: unknown;
|
||||
};
|
||||
|
||||
export type DictationModelsListResponse = {
|
||||
models: Array<DictationLocalModelStatus>;
|
||||
};
|
||||
|
||||
export type DictationLocalModelStatus = {
|
||||
id: string;
|
||||
label: string;
|
||||
description: string;
|
||||
sizeMb: number;
|
||||
downloaded: boolean;
|
||||
downloadInProgress: boolean;
|
||||
};
|
||||
|
||||
/**
|
||||
* Kick off a background download of a local Whisper model.
|
||||
*/
|
||||
export type DictationModelDownloadRequest = {
|
||||
modelId: string;
|
||||
};
|
||||
|
||||
/**
|
||||
* Poll the progress of an in-flight download.
|
||||
*/
|
||||
export type DictationModelDownloadProgressRequest = {
|
||||
modelId: string;
|
||||
};
|
||||
|
||||
export type DictationModelDownloadProgressResponse = {
|
||||
/**
|
||||
* None when no download is active for this model id.
|
||||
*/
|
||||
progress?: DictationDownloadProgress | null;
|
||||
};
|
||||
|
||||
export type DictationDownloadProgress = {
|
||||
bytesDownloaded: number;
|
||||
totalBytes: number;
|
||||
progressPercent: number;
|
||||
/**
|
||||
* serde lowercase of DownloadStatus: "downloading" | "completed" | "failed" | "cancelled"
|
||||
*/
|
||||
status: string;
|
||||
error?: string | null;
|
||||
};
|
||||
|
||||
/**
|
||||
* Cancel an in-flight download.
|
||||
*/
|
||||
export type DictationModelCancelRequest = {
|
||||
modelId: string;
|
||||
};
|
||||
|
||||
/**
|
||||
* Delete a downloaded local Whisper model from disk.
|
||||
*/
|
||||
export type DictationModelDeleteRequest = {
|
||||
modelId: string;
|
||||
};
|
||||
|
||||
/**
|
||||
* Persist the user's model selection for a given provider.
|
||||
*/
|
||||
export type DictationModelSelectRequest = {
|
||||
provider: string;
|
||||
modelId: string;
|
||||
};
|
||||
|
||||
export type ExtRequest = {
|
||||
id: string;
|
||||
method: string;
|
||||
params?: AddExtensionRequest | RemoveExtensionRequest | GetToolsRequest | ReadResourceRequest | UpdateWorkingDirRequest | DeleteSessionRequest | GetExtensionsRequest | GetSessionExtensionsRequest | ListProvidersRequest | GetProviderDetailsRequest | GetProviderModelsRequest | ReadConfigRequest | UpsertConfigRequest | RemoveConfigRequest | CheckSecretRequest | UpsertSecretRequest | RemoveSecretRequest | ExportSessionRequest | ImportSessionRequest | ArchiveSessionRequest | UnarchiveSessionRequest | {
|
||||
params?: AddExtensionRequest | RemoveExtensionRequest | GetToolsRequest | ReadResourceRequest | UpdateWorkingDirRequest | DeleteSessionRequest | GetExtensionsRequest | GetSessionExtensionsRequest | ListProvidersRequest | GetProviderDetailsRequest | GetProviderModelsRequest | ReadConfigRequest | UpsertConfigRequest | RemoveConfigRequest | CheckSecretRequest | UpsertSecretRequest | RemoveSecretRequest | ExportSessionRequest | ImportSessionRequest | ArchiveSessionRequest | UnarchiveSessionRequest | DictationTranscribeRequest | DictationConfigRequest | DictationModelsListRequest | DictationModelDownloadRequest | DictationModelDownloadProgressRequest | DictationModelCancelRequest | DictationModelDeleteRequest | DictationModelSelectRequest | {
|
||||
[key: string]: unknown;
|
||||
} | null;
|
||||
};
|
||||
|
||||
export type ExtResponse = {
|
||||
id: string;
|
||||
result?: EmptyResponse | GetToolsResponse | ReadResourceResponse | GetExtensionsResponse | GetSessionExtensionsResponse | ListProvidersResponse | GetProviderDetailsResponse | GetProviderModelsResponse | ReadConfigResponse | CheckSecretResponse | ExportSessionResponse | ImportSessionResponse | unknown;
|
||||
result?: EmptyResponse | GetToolsResponse | ReadResourceResponse | GetExtensionsResponse | GetSessionExtensionsResponse | ListProvidersResponse | GetProviderDetailsResponse | GetProviderModelsResponse | ReadConfigResponse | CheckSecretResponse | ExportSessionResponse | ImportSessionResponse | DictationTranscribeResponse | DictationConfigResponse | DictationModelsListResponse | DictationModelDownloadProgressResponse | unknown;
|
||||
} | {
|
||||
error: {
|
||||
code: number;
|
||||
|
||||
@@ -271,6 +271,146 @@ export const zUnarchiveSessionRequest = z.object({
|
||||
sessionId: z.string()
|
||||
});
|
||||
|
||||
/**
|
||||
* Transcribe audio via a dictation provider.
|
||||
*/
|
||||
export const zDictationTranscribeRequest = z.object({
|
||||
audio: z.string(),
|
||||
mimeType: z.string(),
|
||||
provider: z.string()
|
||||
});
|
||||
|
||||
/**
|
||||
* Transcription result.
|
||||
*/
|
||||
export const zDictationTranscribeResponse = z.object({
|
||||
text: z.string()
|
||||
});
|
||||
|
||||
/**
|
||||
* Get the configuration status of all dictation providers.
|
||||
*/
|
||||
export const zDictationConfigRequest = z.record(z.unknown());
|
||||
|
||||
export const zDictationModelOption = z.object({
|
||||
id: z.string(),
|
||||
label: z.string(),
|
||||
description: z.string()
|
||||
});
|
||||
|
||||
/**
|
||||
* Per-provider configuration status.
|
||||
*/
|
||||
export const zDictationProviderStatusEntry = z.object({
|
||||
configured: z.boolean(),
|
||||
host: z.union([
|
||||
z.string(),
|
||||
z.null()
|
||||
]).optional(),
|
||||
description: z.string(),
|
||||
usesProviderConfig: z.boolean(),
|
||||
settingsPath: z.union([
|
||||
z.string(),
|
||||
z.null()
|
||||
]).optional(),
|
||||
configKey: z.union([
|
||||
z.string(),
|
||||
z.null()
|
||||
]).optional(),
|
||||
modelConfigKey: z.union([
|
||||
z.string(),
|
||||
z.null()
|
||||
]).optional(),
|
||||
defaultModel: z.union([
|
||||
z.string(),
|
||||
z.null()
|
||||
]).optional(),
|
||||
selectedModel: z.union([
|
||||
z.string(),
|
||||
z.null()
|
||||
]).optional(),
|
||||
availableModels: z.array(zDictationModelOption).optional().default([])
|
||||
});
|
||||
|
||||
/**
|
||||
* Dictation config response — map of provider name to status.
|
||||
*/
|
||||
export const zDictationConfigResponse = z.object({
|
||||
providers: z.record(zDictationProviderStatusEntry)
|
||||
});
|
||||
|
||||
/**
|
||||
* List available local Whisper models with their download status.
|
||||
*/
|
||||
export const zDictationModelsListRequest = z.record(z.unknown());
|
||||
|
||||
export const zDictationLocalModelStatus = z.object({
|
||||
id: z.string(),
|
||||
label: z.string(),
|
||||
description: z.string(),
|
||||
sizeMb: z.number().int().gte(0),
|
||||
downloaded: z.boolean(),
|
||||
downloadInProgress: z.boolean()
|
||||
});
|
||||
|
||||
export const zDictationModelsListResponse = z.object({
|
||||
models: z.array(zDictationLocalModelStatus)
|
||||
});
|
||||
|
||||
/**
|
||||
* Kick off a background download of a local Whisper model.
|
||||
*/
|
||||
export const zDictationModelDownloadRequest = z.object({
|
||||
modelId: z.string()
|
||||
});
|
||||
|
||||
/**
|
||||
* Poll the progress of an in-flight download.
|
||||
*/
|
||||
export const zDictationModelDownloadProgressRequest = z.object({
|
||||
modelId: z.string()
|
||||
});
|
||||
|
||||
export const zDictationDownloadProgress = z.object({
|
||||
bytesDownloaded: z.number().int().gte(0),
|
||||
totalBytes: z.number().int().gte(0),
|
||||
progressPercent: z.number(),
|
||||
status: z.string(),
|
||||
error: z.union([
|
||||
z.string(),
|
||||
z.null()
|
||||
]).optional()
|
||||
});
|
||||
|
||||
export const zDictationModelDownloadProgressResponse = z.object({
|
||||
progress: z.union([
|
||||
zDictationDownloadProgress,
|
||||
z.null()
|
||||
]).optional()
|
||||
});
|
||||
|
||||
/**
|
||||
* Cancel an in-flight download.
|
||||
*/
|
||||
export const zDictationModelCancelRequest = z.object({
|
||||
modelId: z.string()
|
||||
});
|
||||
|
||||
/**
|
||||
* Delete a downloaded local Whisper model from disk.
|
||||
*/
|
||||
export const zDictationModelDeleteRequest = z.object({
|
||||
modelId: z.string()
|
||||
});
|
||||
|
||||
/**
|
||||
* Persist the user's model selection for a given provider.
|
||||
*/
|
||||
export const zDictationModelSelectRequest = z.object({
|
||||
provider: z.string(),
|
||||
modelId: z.string()
|
||||
});
|
||||
|
||||
export const zExtRequest = z.object({
|
||||
id: z.string(),
|
||||
method: z.string(),
|
||||
@@ -296,7 +436,15 @@ export const zExtRequest = z.object({
|
||||
zExportSessionRequest,
|
||||
zImportSessionRequest,
|
||||
zArchiveSessionRequest,
|
||||
zUnarchiveSessionRequest
|
||||
zUnarchiveSessionRequest,
|
||||
zDictationTranscribeRequest,
|
||||
zDictationConfigRequest,
|
||||
zDictationModelsListRequest,
|
||||
zDictationModelDownloadRequest,
|
||||
zDictationModelDownloadProgressRequest,
|
||||
zDictationModelCancelRequest,
|
||||
zDictationModelDeleteRequest,
|
||||
zDictationModelSelectRequest
|
||||
]),
|
||||
z.union([
|
||||
z.record(z.unknown()),
|
||||
@@ -321,7 +469,11 @@ export const zExtResponse = z.union([
|
||||
zReadConfigResponse,
|
||||
zCheckSecretResponse,
|
||||
zExportSessionResponse,
|
||||
zImportSessionResponse
|
||||
zImportSessionResponse,
|
||||
zDictationTranscribeResponse,
|
||||
zDictationConfigResponse,
|
||||
zDictationModelsListResponse,
|
||||
zDictationModelDownloadProgressResponse
|
||||
]),
|
||||
z.unknown()
|
||||
]).optional()
|
||||
|
||||
Reference in New Issue
Block a user