Initial commit: Memind H5 portal with MindSpace, Plaza, and agent jobs.

Track application source and tests; exclude local env, user workspaces, and runtime data via .gitignore.

Co-authored-by: Cursor <cursoragent@cursor.com>

src/voice/asrTransport.ts

@@ -0,0 +1,38 @@
+import type { VoiceSessionMode } from './types';
+
+function blobExtension(mimeType: string) {
+  if (mimeType.includes('webm')) return 'webm';
+  if (mimeType.includes('mp4')) return 'm4a';
+  if (mimeType.includes('ogg')) return 'ogg';
+  if (mimeType.includes('wav')) return 'wav';
+  return 'webm';
+}
+
+async function parseAsrError(res: Response) {
+  const text = await res.text().catch(() => '');
+  try {
+    const body = JSON.parse(text) as Record<string, unknown>;
+    const nested =
+      body.error && typeof body.error === 'object'
+        ? (body.error as Record<string, unknown>)
+        : body;
+    return (
+      (typeof nested.message === 'string' && nested.message) ||
+      (typeof body.message === 'string' && body.message) ||
+      text ||
+      `${res.status} ${res.statusText}`
+    );
+  } catch {
+    return text || `${res.status} ${res.statusText}`;
+  }
+}
+
+/** 一次性识别（push-to-talk）；call 模式后续可扩展 openStream() */
+export async function transcribeOneShot(blob: Blob, _mode: VoiceSessionMode = 'push-to-talk') {
+  const form = new FormData();
+  form.append('file', blob, `recording.${blobExtension(blob.type)}`);
+  const res = await fetch('/api/asr/oneshot', { method: 'POST', body: form });
+  if (!res.ok) throw new Error(await parseAsrError(res));
+  const body = (await res.json()) as { data?: { text?: string } };
+  return (body.data?.text ?? '').trim();
+}

src/voice/audioAnalyser.ts

@@ -0,0 +1,62 @@
+import { extendMicRelease, waitForMicRelease } from './micSession';
+import { mapMicError } from './micCapture';
+
+export class AudioAnalyser {
+  private stream: MediaStream | null = null;
+  private context: AudioContext | null = null;
+  private analyser: AnalyserNode | null = null;
+  private source: MediaStreamAudioSourceNode | null = null;
+
+  get node() {
+    return this.analyser;
+  }
+
+  async start() {
+    if (this.analyser) return this.analyser;
+    await waitForMicRelease();
+
+    this.stream = await navigator.mediaDevices.getUserMedia({
+      audio: {
+        echoCancellation: true,
+        noiseSuppression: true,
+        autoGainControl: true,
+      },
+    });
+
+    this.context = new AudioContext();
+    this.analyser = this.context.createAnalyser();
+    this.analyser.fftSize = 256;
+    this.analyser.smoothingTimeConstant = 0.75;
+    this.source = this.context.createMediaStreamSource(this.stream);
+    this.source.connect(this.analyser);
+    if (this.context.state === 'suspended') {
+      await this.context.resume();
+    }
+    return this.analyser;
+  }
+
+  stop() {
+    const disposePromise = this.dispose();
+    extendMicRelease(disposePromise);
+    void disposePromise;
+  }
+
+  private async dispose() {
+    this.source?.disconnect();
+    this.source = null;
+    this.analyser = null;
+    const context = this.context;
+    this.context = null;
+    if (context) {
+      try {
+        await context.close();
+      } catch {
+        // ignore
+      }
+    }
+    this.stream?.getTracks().forEach((track) => track.stop());
+    this.stream = null;
+  }
+}
+
+export { mapMicError };

src/voice/capabilities.ts

@@ -0,0 +1,15 @@
+import { MicCapture } from './micCapture';
+import { isSpeechRecognitionSupported } from './speechRecognition';
+
+/** Whether voice input may work in this browser (mic and/or live speech). */
+export function isVoiceInputAvailable(): boolean {
+  if (typeof navigator === 'undefined') return false;
+  if (MicCapture.isSupported()) return true;
+  if (isSpeechRecognitionSupported()) return true;
+  return Boolean(navigator.mediaDevices?.getUserMedia);
+}
+
+/** Show the mic control in chat; availability is checked on click. */
+export function shouldShowVoiceInputControl(): boolean {
+  return typeof window !== 'undefined';
+}

src/voice/micCapture.ts

@@ -0,0 +1,147 @@
+import { extendMicRelease, waitForMicRelease } from './micSession';
+
+const PREFERRED_MIME_TYPES = [
+  'audio/webm;codecs=opus',
+  'audio/webm',
+  'audio/mp4',
+  'audio/ogg;codecs=opus',
+  'audio/ogg',
+];
+
+export class MicCapture {
+  private stream: MediaStream | null = null;
+  private recorder: MediaRecorder | null = null;
+  private chunks: Blob[] = [];
+  private context: AudioContext | null = null;
+  private analyser: AnalyserNode | null = null;
+  private source: MediaStreamAudioSourceNode | null = null;
+
+  static isSupported() {
+    return (
+      typeof navigator !== 'undefined' &&
+      !!navigator.mediaDevices?.getUserMedia &&
+      typeof MediaRecorder !== 'undefined'
+    );
+  }
+
+  static pickMimeType() {
+    for (const type of PREFERRED_MIME_TYPES) {
+      if (MediaRecorder.isTypeSupported(type)) return type;
+    }
+    return undefined;
+  }
+
+  get recording() {
+    return this.recorder?.state === 'recording';
+  }
+
+  get levelAnalyser() {
+    return this.analyser;
+  }
+
+  /** 开始采集；后续 call 模式可改为 onChunk 流式回调 */
+  async start(): Promise<void> {
+    if (this.recording) return;
+    await waitForMicRelease();
+    this.chunks = [];
+    this.stream = await navigator.mediaDevices.getUserMedia({
+      audio: {
+        echoCancellation: true,
+        noiseSuppression: true,
+        autoGainControl: true,
+      },
+    });
+    const mimeType = MicCapture.pickMimeType();
+    this.recorder = mimeType
+      ? new MediaRecorder(this.stream, { mimeType })
+      : new MediaRecorder(this.stream);
+    this.recorder.ondataavailable = (event) => {
+      if (event.data.size > 0) this.chunks.push(event.data);
+    };
+    this.context = new AudioContext();
+    this.analyser = this.context.createAnalyser();
+    this.analyser.fftSize = 256;
+    this.analyser.smoothingTimeConstant = 0.75;
+    this.source = this.context.createMediaStreamSource(this.stream);
+    this.source.connect(this.analyser);
+    if (this.context.state === 'suspended') {
+      await this.context.resume();
+    }
+    this.recorder.start(250);
+  }
+
+  async stop(): Promise<Blob> {
+    const recorder = this.recorder;
+    if (!recorder || recorder.state === 'inactive') {
+      await this.dispose();
+      return new Blob();
+    }
+    const disposePromise = new Promise<Blob>((resolve) => {
+      recorder.onstop = () => {
+        const blob = new Blob(this.chunks, { type: recorder.mimeType || 'audio/webm' });
+        void this.dispose().then(() => resolve(blob));
+      };
+      try {
+        if (recorder.state === 'recording') {
+          recorder.requestData();
+        }
+      } catch {
+        // Some WebViews omit requestData(); stop() still flushes on stop.
+      }
+      recorder.stop();
+    });
+    extendMicRelease(disposePromise);
+    return disposePromise;
+  }
+
+  async cancel(): Promise<void> {
+    const recorder = this.recorder;
+    if (recorder && recorder.state !== 'inactive') {
+      recorder.onstop = null;
+      try {
+        if (recorder.state === 'recording') {
+          recorder.requestData();
+        }
+      } catch {
+        // ignore
+      }
+      recorder.stop();
+    }
+    const disposePromise = this.dispose();
+    extendMicRelease(disposePromise);
+    await disposePromise;
+  }
+
+  private async dispose() {
+    this.source?.disconnect();
+    this.source = null;
+    this.analyser = null;
+    const context = this.context;
+    this.context = null;
+    if (context) {
+      try {
+        await context.close();
+      } catch {
+        // ignore close races
+      }
+    }
+    this.stream?.getTracks().forEach((track) => track.stop());
+    this.stream = null;
+    this.recorder = null;
+    this.chunks = [];
+  }
+}
+
+export function mapMicError(err: unknown) {
+  const name = err instanceof DOMException ? err.name : '';
+  if (name === 'NotAllowedError' || name === 'PermissionDeniedError') {
+    return '请在浏览器设置中允许麦克风';
+  }
+  if (name === 'NotFoundError' || name === 'DevicesNotFoundError') {
+    return '未检测到麦克风设备';
+  }
+  if (name === 'NotReadableError') {
+    return '麦克风被其他应用占用';
+  }
+  return err instanceof Error ? err.message : '无法访问麦克风';
+}

src/voice/micSession.ts

@@ -0,0 +1,15 @@
+/** Serialize mic open/close so the second session waits for tracks to release. */
+let releaseChain = Promise.resolve();
+
+export function waitForMicRelease() {
+  return releaseChain;
+}
+
+export function extendMicRelease(until: Promise<unknown>) {
+  let release!: () => void;
+  const gate = new Promise<void>((resolve) => {
+    release = resolve;
+  });
+  releaseChain = releaseChain.then(() => gate);
+  void until.finally(() => release());
+}

src/voice/speechRecognition.ts

@@ -0,0 +1,129 @@
+type BrowserSpeechRecognition = SpeechRecognition & {
+  onresult: ((this: SpeechRecognition, ev: SpeechRecognitionEvent) => void) | null;
+  onerror: ((this: SpeechRecognition, ev: SpeechRecognitionErrorEvent) => void) | null;
+  onend: ((this: SpeechRecognition, ev: Event) => void) | null;
+};
+
+type SpeechRecognitionCtor = new () => BrowserSpeechRecognition;
+
+declare global {
+  interface Window {
+    webkitSpeechRecognition?: SpeechRecognitionCtor;
+  }
+}
+
+export function isSpeechRecognitionSupported() {
+  return typeof window !== 'undefined' && !!(window.SpeechRecognition || window.webkitSpeechRecognition);
+}
+
+function getSpeechRecognitionCtor(): SpeechRecognitionCtor | null {
+  if (typeof window === 'undefined') return null;
+  return window.SpeechRecognition ?? window.webkitSpeechRecognition ?? null;
+}
+
+export function createLiveSpeechRecognition({
+  onInterim,
+  onFinal,
+  onError,
+}: {
+  onInterim: (text: string) => void;
+  onFinal: (text: string) => void;
+  onError: (message: string) => void;
+}) {
+  const Ctor = getSpeechRecognitionCtor();
+  if (!Ctor) return null;
+
+  const recognition = new Ctor();
+  recognition.lang = 'zh-CN';
+  recognition.interimResults = true;
+  recognition.continuous = true;
+  recognition.maxAlternatives = 1;
+
+  let active = false;
+  let restarting = false;
+  let restartTimer: number | null = null;
+
+  const clearRestartTimer = () => {
+    if (restartTimer != null) {
+      window.clearTimeout(restartTimer);
+      restartTimer = null;
+    }
+    restarting = false;
+  };
+
+  recognition.onresult = (event) => {
+    let interim = '';
+    let finals = '';
+    for (let i = event.resultIndex; i < event.results.length; i += 1) {
+      const result = event.results[i];
+      const chunk = result[0]?.transcript ?? '';
+      if (result.isFinal) finals += chunk;
+      else interim += chunk;
+    }
+    if (finals) onFinal(finals);
+    onInterim(interim);
+  };
+
+  recognition.onerror = (event) => {
+    if (event.error === 'aborted' || event.error === 'no-speech') return;
+    if (event.error === 'not-allowed') {
+      onError('请在浏览器设置中允许麦克风');
+      return;
+    }
+    if (event.error === 'network') {
+      onError('语音识别网络异常，请检查连接');
+      return;
+    }
+    onError(`语音识别失败：${event.error}`);
+  };
+
+  recognition.onend = () => {
+    if (!active || restarting) return;
+    restarting = true;
+    restartTimer = window.setTimeout(() => {
+      restartTimer = null;
+      restarting = false;
+      if (!active) return;
+      try {
+        recognition.start();
+      } catch {
+        // ignore restart races
+      }
+    }, 120);
+  };
+
+  return {
+    start() {
+      clearRestartTimer();
+      active = true;
+      try {
+        recognition.start();
+      } catch (err) {
+        active = false;
+        const message = err instanceof Error ? err.message : String(err);
+        if (/already started/i.test(message)) {
+          window.setTimeout(() => {
+            if (!active) return;
+            try {
+              recognition.start();
+            } catch {
+              onError('无法启动语音识别，请关闭后重试');
+            }
+          }, 150);
+          return;
+        }
+        onError('无法启动语音识别，请重试');
+      }
+    },
+    stop() {
+      active = false;
+      clearRestartTimer();
+      recognition.stop();
+    },
+    abort() {
+      active = false;
+      clearRestartTimer();
+      recognition.abort();
+    },
+  };
+}

src/voice/types.ts

@@ -0,0 +1,11 @@
+/** push-to-talk: 按住说话；call: 后续连续通话（流式 ASR + Agent） */
+export type VoiceSessionMode = 'push-to-talk' | 'call';
+
+export type VoiceCaptureState = 'idle' | 'requesting' | 'recording' | 'transcribing' | 'error';
+
+export type VoiceSessionPhase = 'idle' | 'requesting' | 'listening' | 'transcribing' | 'ready' | 'error';
+
+export interface VoiceTranscript {
+  text: string;
+  final: boolean;
+}