diff --git a/api-core-retry.test.mjs b/api-core-retry.test.mjs new file mode 100644 index 0000000..e4f807b --- /dev/null +++ b/api-core-retry.test.mjs @@ -0,0 +1,86 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { fetchWithTransientRetry } from './src/api/core.ts'; + +async function withMockFetch(mock, run) { + const originalFetch = globalThis.fetch; + globalThis.fetch = mock; + try { + await run(); + } finally { + globalThis.fetch = originalFetch; + } +} + +test('idempotent requests retry transient network failures', async () => { + let calls = 0; + await withMockFetch( + async () => { + calls += 1; + if (calls < 3) throw new TypeError('fetch failed'); + return new Response('{}', { status: 200 }); + }, + async () => { + const response = await fetchWithTransientRetry('/api/sessions', undefined, 1_000, [0, 0]); + assert.equal(response.status, 200); + }, + ); + assert.equal(calls, 3); +}); + +test('idempotent requests retry transient gateway responses', async () => { + let calls = 0; + await withMockFetch( + async () => { + calls += 1; + return calls === 1 + ? new Response('temporary', { status: 503 }) + : new Response('{}', { status: 200 }); + }, + async () => { + const response = await fetchWithTransientRetry('/api/sessions', undefined, 1_000, [0]); + assert.equal(response.status, 200); + }, + ); + assert.equal(calls, 2); +}); + +test('non-idempotent requests are never retried', async () => { + let calls = 0; + await withMockFetch( + async () => { + calls += 1; + throw new TypeError('fetch failed'); + }, + async () => { + await assert.rejects( + fetchWithTransientRetry( + '/api/agent/start', + { method: 'POST', body: '{}' }, + 1_000, + [0, 0], + ), + /fetch failed/, + ); + }, + ); + assert.equal(calls, 1); +}); + +test('aborted requests are never retried', async () => { + let calls = 0; + await withMockFetch( + async () => { + calls += 1; + throw new DOMException('Aborted', 'AbortError'); + }, + async () => { + await assert.rejects( + fetchWithTransientRetry('/api/sessions', undefined, 1_000, [0, 0]), + (error) => error instanceof DOMException && error.name === 'AbortError', + ); + }, + ); + assert.equal(calls, 1); +}); diff --git a/package.json b/package.json index d034cca..38e9329 100644 --- a/package.json +++ b/package.json @@ -74,7 +74,7 @@ "ci:page-data-dev-loop-smoke": "node scripts/ci-page-data-dev-loop-smoke.mjs", "migrate:agent-code-run-config": "node scripts/migrate-agent-code-run-config-from-env.mjs", "test:scenario:john4-diet": "node scripts/run-scenario-test.mjs --scenario john4-children-hobby-diet-update", - "test": "node --test auth.test.mjs asr-proxy.test.mjs billing.test.mjs billing-token-state.test.mjs billing-recharge.test.mjs wechat-pay.test.mjs wechat-oauth.test.mjs wechat-mp.test.mjs wechat-media.test.mjs wechat/image-generation-policy.test.mjs wechat/verify/generated-thumbnail.test.mjs schedule-intent.test.mjs schedule-reminder-worker.test.mjs capabilities.test.mjs policies.test.mjs server/portal-api-auth-middleware.test.mjs server/portal-config-routes.test.mjs server/portal-plaza-discovery-routes.test.mjs server/portal-runtime-routes.test.mjs server/portal-gateway-services-bootstrap.test.mjs chat-skills.test.mjs chat-intent-router.test.mjs chat-finish-sync.test.mjs chat-agent-run-gate.test.mjs conversation-display.test.mjs user-publish.test.mjs user-memory-profile.test.mjs skills-registry.test.mjs skill-runtime-policy.test.mjs excel-analyst.test.mjs agent-run-gateway.test.mjs agent-run-routes.test.mjs session-broker.test.mjs sse-event-taxonomy.test.mjs goosed-proxy-boundary.test.mjs agent-run-stream.test.mjs mindspace-h5-html-finish-guard.test.mjs admin-routes.test.mjs services/orchestrator/admin-config.test.mjs services/orchestrator/contracts.test.mjs services/orchestrator/checkpoint.test.mjs services/orchestrator/runtime.test.mjs services/orchestrator/app.test.mjs services/orchestrator/server.test.mjs services/orchestrator/shadow-dispatcher.test.mjs services/orchestrator/shadow-observer.test.mjs services/orchestrator/observability.test.mjs services/orchestrator/executor-gateway.test.mjs services/orchestrator/executor-job-store.test.mjs image-make-admin-config.test.mjs asset-gateway.test.mjs image-make-client.test.mjs mindspace-image-generation.test.mjs mindspace-image-generation-routes.test.mjs mindspace-image-review.test.mjs mindspace-run-public-html-scope.test.mjs direct-chat-service.test.mjs tool-gateway.test.mjs mindspace.test.mjs mindspace-scan.test.mjs mindspace-assets.test.mjs mindspace-local-runtime-services.test.mjs mindspace-local-server-adapter.test.mjs mindspace-public-asset-token.test.mjs mindspace-remote-server-adapter.test.mjs mindspace-server-adapter.test.mjs mindspace-pages.test.mjs mindspace-page-sync-service.test.mjs public-site-bases.test.mjs mindspace-html-download-links.test.mjs mindspace-page-purge.test.mjs mindspace-public-delivery.test.mjs mindspace-public-page-context.test.mjs mindspace-published-page-csp.test.mjs mindspace-published-script-localize.test.mjs agent-run-deliverable-check.test.mjs mindspace-public-route.test.mjs mindspace-publications.test.mjs mindspace-public-links.test.mjs mindspace-chat-save.test.mjs mindspace-chat-docx-package.test.mjs mindspace-public-finish-sync.test.mjs mindspace-chat-context.test.mjs mindspace-canonical-url.test.mjs mindspace-conversation-package.test.mjs mindspace-conversation-package-backfill.test.mjs mindspace-conversation-package-public-html.test.mjs mindspace-conversation-package-verify.test.mjs mindspace-conversation-package-registry.test.mjs mindspace-conversation-package-routes.test.mjs mindspace-conversation-package-store.test.mjs mindspace-conversation-schema.test.mjs mindspace-runtime-config.test.mjs mindspace-config.test.mjs mindspace-analytics.test.mjs mindspace-rybbit.test.mjs mindspace-service.test.mjs mindspace-storage-adapter.test.mjs mindspace-content-scan.test.mjs mindspace-html-localize.test.mjs mindspace-visual-editor.test.mjs mindspace-cleanup.test.mjs mindspace-thumbnails.test.mjs mindspace-workspace-thumbnails.test.mjs mindspace-workspace-sync.test.mjs mindspace-asset-preview.test.mjs mindspace-agent-jobs.test.mjs mindspace-agent-runner.test.mjs mindspace-sandbox-mcp.test.mjs mindspace-userdata-postgres.test.mjs postgres-user-data-space-service.test.mjs user-data-space-service.test.mjs page-data-routes.test.mjs page-access-policy.test.mjs page-access-visitor.test.mjs page-data-public-service.test.mjs page-data-integration.test.mjs page-data-log-store.test.mjs page-data-ops.test.mjs page-data-session-store.test.mjs page-data-browser-client.test.mjs page-data-policy-index.test.mjs message-stream.test.mjs mindspace-service/mindspace-rpc-server.test.mjs plaza-posts.test.mjs plaza-interactions.test.mjs plaza-algorithm.test.mjs plaza-seo.test.mjs plaza-ops.test.mjs user-auth.test.mjs llm-providers.test.mjs admin-guard.test.mjs user-feedback.test.mjs memory-v2.test.mjs memory-v2-admin-config.test.mjs memory-v2-lifecycle.test.mjs memory-v2-adapter-scaffold.test.mjs memory-v2-backend-contract.test.mjs memory-v2-health.test.mjs memory-v2-runtime.test.mjs memory-v2-plugin-backends.test.mjs memory-v2-pgvector.test.mjs memory-v2-pgvector-schema.test.mjs memory-v2-pgvector-backfill.test.mjs memory-v2-pgvector-smoke.test.mjs memory-v2-qdrant.test.mjs memory-v2-weaviate.test.mjs memory-v2-mem0.test.mjs memory-v2-letta.test.mjs memory-v2-external-adapters.test.mjs scripts/embed-memory-v2-local-hash.test.mjs scripts/check-memory-v2-app-canary.test.mjs scripts/check-memory-v2-config-gaps.test.mjs scripts/check-memory-v2-contracts.test.mjs scripts/check-memory-v2-health.test.mjs scripts/check-memory-v2-session-flow.test.mjs scripts/check-memory-v2-stack.test.mjs scripts/setup-memory-v2-pgvector-schema.test.mjs scripts/backfill-memory-v2-pgvector.test.mjs scripts/scaffold-memory-v2-backend.test.mjs scripts/smoke-memory-v2-pgvector.test.mjs scripts/smoke-memory-v2-qdrant.test.mjs scripts/smoke-memory-v2-external.test.mjs scripts/mock-memory-v2-services.test.mjs", + "test": "node --test api-core-retry.test.mjs auth.test.mjs asr-proxy.test.mjs billing.test.mjs billing-token-state.test.mjs billing-recharge.test.mjs wechat-pay.test.mjs wechat-oauth.test.mjs wechat-mp.test.mjs wechat-media.test.mjs wechat/image-generation-policy.test.mjs wechat/verify/generated-thumbnail.test.mjs schedule-intent.test.mjs schedule-reminder-worker.test.mjs capabilities.test.mjs policies.test.mjs server/portal-api-auth-middleware.test.mjs server/portal-config-routes.test.mjs server/portal-plaza-discovery-routes.test.mjs server/portal-runtime-routes.test.mjs server/portal-gateway-services-bootstrap.test.mjs chat-skills.test.mjs chat-intent-router.test.mjs chat-finish-sync.test.mjs chat-agent-run-gate.test.mjs conversation-display.test.mjs user-publish.test.mjs user-memory-profile.test.mjs skills-registry.test.mjs skill-runtime-policy.test.mjs excel-analyst.test.mjs agent-run-gateway.test.mjs agent-run-routes.test.mjs session-broker.test.mjs sse-event-taxonomy.test.mjs goosed-proxy-boundary.test.mjs agent-run-stream.test.mjs mindspace-h5-html-finish-guard.test.mjs admin-routes.test.mjs services/orchestrator/admin-config.test.mjs services/orchestrator/contracts.test.mjs services/orchestrator/checkpoint.test.mjs services/orchestrator/runtime.test.mjs services/orchestrator/app.test.mjs services/orchestrator/server.test.mjs services/orchestrator/shadow-dispatcher.test.mjs services/orchestrator/shadow-observer.test.mjs services/orchestrator/observability.test.mjs services/orchestrator/executor-gateway.test.mjs services/orchestrator/executor-job-store.test.mjs image-make-admin-config.test.mjs asset-gateway.test.mjs image-make-client.test.mjs mindspace-image-generation.test.mjs mindspace-image-generation-routes.test.mjs mindspace-image-review.test.mjs mindspace-run-public-html-scope.test.mjs direct-chat-service.test.mjs tool-gateway.test.mjs mindspace.test.mjs mindspace-scan.test.mjs mindspace-assets.test.mjs mindspace-local-runtime-services.test.mjs mindspace-local-server-adapter.test.mjs mindspace-public-asset-token.test.mjs mindspace-remote-server-adapter.test.mjs mindspace-server-adapter.test.mjs mindspace-pages.test.mjs mindspace-page-sync-service.test.mjs public-site-bases.test.mjs mindspace-html-download-links.test.mjs mindspace-page-purge.test.mjs mindspace-public-delivery.test.mjs mindspace-public-page-context.test.mjs mindspace-published-page-csp.test.mjs mindspace-published-script-localize.test.mjs agent-run-deliverable-check.test.mjs mindspace-public-route.test.mjs mindspace-publications.test.mjs mindspace-public-links.test.mjs mindspace-chat-save.test.mjs mindspace-chat-docx-package.test.mjs mindspace-public-finish-sync.test.mjs mindspace-chat-context.test.mjs mindspace-canonical-url.test.mjs mindspace-conversation-package.test.mjs mindspace-conversation-package-backfill.test.mjs mindspace-conversation-package-public-html.test.mjs mindspace-conversation-package-verify.test.mjs mindspace-conversation-package-registry.test.mjs mindspace-conversation-package-routes.test.mjs mindspace-conversation-package-store.test.mjs mindspace-conversation-schema.test.mjs mindspace-runtime-config.test.mjs mindspace-config.test.mjs mindspace-analytics.test.mjs mindspace-rybbit.test.mjs mindspace-service.test.mjs mindspace-storage-adapter.test.mjs mindspace-content-scan.test.mjs mindspace-html-localize.test.mjs mindspace-visual-editor.test.mjs mindspace-cleanup.test.mjs mindspace-thumbnails.test.mjs mindspace-workspace-thumbnails.test.mjs mindspace-workspace-sync.test.mjs mindspace-asset-preview.test.mjs mindspace-agent-jobs.test.mjs mindspace-agent-runner.test.mjs mindspace-sandbox-mcp.test.mjs mindspace-userdata-postgres.test.mjs postgres-user-data-space-service.test.mjs user-data-space-service.test.mjs page-data-routes.test.mjs page-access-policy.test.mjs page-access-visitor.test.mjs page-data-public-service.test.mjs page-data-integration.test.mjs page-data-log-store.test.mjs page-data-ops.test.mjs page-data-session-store.test.mjs page-data-browser-client.test.mjs page-data-policy-index.test.mjs message-stream.test.mjs mindspace-service/mindspace-rpc-server.test.mjs plaza-posts.test.mjs plaza-interactions.test.mjs plaza-algorithm.test.mjs plaza-seo.test.mjs plaza-ops.test.mjs user-auth.test.mjs llm-providers.test.mjs admin-guard.test.mjs user-feedback.test.mjs memory-v2.test.mjs memory-v2-admin-config.test.mjs memory-v2-lifecycle.test.mjs memory-v2-adapter-scaffold.test.mjs memory-v2-backend-contract.test.mjs memory-v2-health.test.mjs memory-v2-runtime.test.mjs memory-v2-plugin-backends.test.mjs memory-v2-pgvector.test.mjs memory-v2-pgvector-schema.test.mjs memory-v2-pgvector-backfill.test.mjs memory-v2-pgvector-smoke.test.mjs memory-v2-qdrant.test.mjs memory-v2-weaviate.test.mjs memory-v2-mem0.test.mjs memory-v2-letta.test.mjs memory-v2-external-adapters.test.mjs scripts/embed-memory-v2-local-hash.test.mjs scripts/check-memory-v2-app-canary.test.mjs scripts/check-memory-v2-config-gaps.test.mjs scripts/check-memory-v2-contracts.test.mjs scripts/check-memory-v2-health.test.mjs scripts/check-memory-v2-session-flow.test.mjs scripts/check-memory-v2-stack.test.mjs scripts/setup-memory-v2-pgvector-schema.test.mjs scripts/backfill-memory-v2-pgvector.test.mjs scripts/scaffold-memory-v2-backend.test.mjs scripts/smoke-memory-v2-pgvector.test.mjs scripts/smoke-memory-v2-qdrant.test.mjs scripts/smoke-memory-v2-external.test.mjs scripts/mock-memory-v2-services.test.mjs", "test:episodic-memory": "node --test episodic-memory.test.mjs direct-chat-service.test.mjs chat-intent-router.test.mjs", "test:deep-search": "node --test deep-search.test.mjs mindsearch.test.mjs", "test:image-review": "node --test mindspace-image-review.test.mjs mindspace-image-generation.test.mjs", diff --git a/src/api/client.ts b/src/api/client.ts index 91d15df..43957f4 100644 --- a/src/api/client.ts +++ b/src/api/client.ts @@ -1443,7 +1443,9 @@ export async function listSessions(options?: { if (options?.offset != null) params.set('offset', String(options.offset)); if (options?.query?.trim()) params.set('query', options.query.trim()); const qs = params.size ? `?${params.toString()}` : ''; - const result = await apiFetch(`/sessions${qs}`); + const result = await apiFetch(`/sessions${qs}`, undefined, { + retryOnTransientFailure: true, + }); return { items: result.sessions ?? [], page: result.page ?? {} }; } @@ -1452,7 +1454,9 @@ function sessionPath(sessionId: string, suffix = '') { } export async function getSession(sessionId: string): Promise { - return apiFetch(sessionPath(sessionId)); + return apiFetch(sessionPath(sessionId), undefined, { + retryOnTransientFailure: true, + }); } export async function deleteChatSession(sessionId: string): Promise { @@ -1474,7 +1478,9 @@ export async function loadSessionDetail( if (history?.before != null) params.set('history_before', String(history.before)); if (history?.limit != null) params.set('history_limit', String(history.limit)); const qs = params.size ? `?${params.toString()}` : ''; - const detail = await apiFetch(`${sessionPath(sessionId)}${qs}`); + const detail = await apiFetch(`${sessionPath(sessionId)}${qs}`, undefined, { + retryOnTransientFailure: true, + }); const messages = normalizeConversationMessages( (detail.conversation ?? []).filter((m) => m.metadata?.userVisible), ); diff --git a/src/api/core.ts b/src/api/core.ts index de3fdf2..022c5ea 100644 --- a/src/api/core.ts +++ b/src/api/core.ts @@ -2,6 +2,8 @@ import type { InsufficientBalanceDetails, SessionEvent } from '../types'; export const API = '/api'; const DEFAULT_API_TIMEOUT_MS = 20_000; +const DEFAULT_TRANSIENT_RETRY_DELAYS_MS = [500, 1_000] as const; +const RETRYABLE_RESPONSE_STATUSES = new Set([502, 503, 504]); export class ApiError extends Error { readonly status: number; @@ -123,7 +125,7 @@ export async function fetchWithTimeout( ): Promise { const controller = new AbortController(); const upstreamSignal = init?.signal; - const timeout = window.setTimeout(() => controller.abort(), timeoutMs); + const timeout = globalThis.setTimeout(() => controller.abort(), timeoutMs); const abortFromUpstream = () => controller.abort(); if (upstreamSignal) { @@ -137,11 +139,68 @@ export async function fetchWithTimeout( signal: controller.signal, }); } finally { - window.clearTimeout(timeout); + globalThis.clearTimeout(timeout); upstreamSignal?.removeEventListener('abort', abortFromUpstream); } } +function isIdempotentRequest(init?: RequestInit) { + const method = String(init?.method ?? 'GET').toUpperCase(); + return method === 'GET' || method === 'HEAD' || method === 'OPTIONS'; +} + +function isRetryableNetworkError(err: unknown) { + if (err instanceof DOMException && err.name === 'AbortError') return false; + const message = err instanceof Error ? err.message : String(err ?? ''); + return /failed to fetch|networkerror|fetch failed|econn|enotfound|network/i.test(message); +} + +async function waitForTransientRetry(delayMs: number, signal?: AbortSignal) { + if (signal?.aborted) throw new DOMException('Aborted', 'AbortError'); + await new Promise((resolve, reject) => { + const finish = () => { + signal?.removeEventListener('abort', abort); + resolve(); + }; + const abort = () => { + globalThis.clearTimeout(timeout); + reject(new DOMException('Aborted', 'AbortError')); + }; + const timeout = globalThis.setTimeout(finish, delayMs); + signal?.addEventListener('abort', abort, { once: true }); + }); +} + +export async function fetchWithTransientRetry( + input: RequestInfo | URL, + init: RequestInit | undefined, + timeoutMs: number, + retryDelaysMs: readonly number[] = DEFAULT_TRANSIENT_RETRY_DELAYS_MS, +): Promise { + if (!isIdempotentRequest(init) || retryDelaysMs.length === 0) { + return fetchWithTimeout(input, init, timeoutMs); + } + + let attempt = 0; + while (true) { + try { + const response = await fetchWithTimeout(input, init, timeoutMs); + if ( + !RETRYABLE_RESPONSE_STATUSES.has(response.status) + || attempt >= retryDelaysMs.length + ) { + return response; + } + await response.body?.cancel().catch(() => undefined); + } catch (err) { + if (!isRetryableNetworkError(err) || attempt >= retryDelaysMs.length) throw err; + } + + await waitForTransientRetry(retryDelaysMs[attempt], init?.signal ?? undefined); + attempt += 1; + } +} + export async function portalFetch(path: string, init?: RequestInit): Promise { let res: Response; try { @@ -206,21 +265,21 @@ export function sanitizeSessionEvent(event: SessionEvent): SessionEvent { export async function apiFetch( path: string, init?: RequestInit, - options?: { timeoutMs?: number }, + options?: { timeoutMs?: number; retryOnTransientFailure?: boolean }, ): Promise { let res: Response; try { - res = await fetchWithTimeout( - `${API}${path}`, - { - ...init, - headers: { - 'Content-Type': 'application/json', - ...init?.headers, - }, + const requestInit = { + ...init, + headers: { + 'Content-Type': 'application/json', + ...init?.headers, }, - options?.timeoutMs, - ); + }; + const timeoutMs = options?.timeoutMs ?? DEFAULT_API_TIMEOUT_MS; + res = options?.retryOnTransientFailure + ? await fetchWithTransientRetry(`${API}${path}`, requestInit, timeoutMs) + : await fetchWithTimeout(`${API}${path}`, requestInit, timeoutMs); } catch (err) { throw new ApiError(0, formatNetworkError(err)); }