feat(desktop): report the real cause of external backend connection failures (#11828)
This commit is contained in:
@@ -2,7 +2,6 @@ import { describe, expect, it, vi } from 'vitest';
|
||||
import { checkBackendStatus } from './backendStatus';
|
||||
|
||||
type FetchInput = Parameters<typeof globalThis.fetch>[0];
|
||||
type FetchInit = NonNullable<Parameters<typeof globalThis.fetch>[1]>;
|
||||
|
||||
const fetchInputUrl = (input: FetchInput): string => {
|
||||
if (typeof input === 'string') {
|
||||
@@ -14,24 +13,14 @@ const fetchInputUrl = (input: FetchInput): string => {
|
||||
return input.url;
|
||||
};
|
||||
|
||||
type FetchSignal = NonNullable<FetchInit['signal']>;
|
||||
|
||||
const expectAbortSignal = (init?: FetchInit): FetchSignal => {
|
||||
expect(init?.signal).toBeInstanceOf(globalThis.AbortSignal);
|
||||
return init!.signal!;
|
||||
};
|
||||
|
||||
describe('checkBackendStatus', () => {
|
||||
it('checks /status and validates the secret against /acp', async () => {
|
||||
const fetch = vi.fn(async (input: FetchInput, init?: FetchInit) => {
|
||||
const fetch = vi.fn(async (input: FetchInput) => {
|
||||
const url = fetchInputUrl(input);
|
||||
if (url === 'https://example.com/goose/status') {
|
||||
expect(init?.headers).toEqual({ 'X-Secret-Key': 'test-secret' });
|
||||
expectAbortSignal(init);
|
||||
return new Response(null, { status: 200 });
|
||||
}
|
||||
if (url === 'https://example.com/goose/acp?token=test-secret') {
|
||||
expectAbortSignal(init);
|
||||
return new Response(null, { status: 406 });
|
||||
}
|
||||
|
||||
@@ -44,17 +33,15 @@ describe('checkBackendStatus', () => {
|
||||
serverSecret: 'test-secret',
|
||||
fetch,
|
||||
})
|
||||
).resolves.toBe(true);
|
||||
).resolves.toMatchObject({ ok: true, failure: null });
|
||||
|
||||
expect(fetch).toHaveBeenCalledTimes(2);
|
||||
expect(fetch.mock.calls.map(([input]) => fetchInputUrl(input))).toEqual([
|
||||
'https://example.com/goose/status',
|
||||
'https://example.com/goose/acp?token=test-secret',
|
||||
]);
|
||||
});
|
||||
|
||||
it('fails immediately when the ACP auth probe rejects the secret', async () => {
|
||||
const onEvent = vi.fn();
|
||||
it('reports the rejected secret without retrying', async () => {
|
||||
const fetch = vi.fn(async (input: FetchInput) => {
|
||||
const url = fetchInputUrl(input);
|
||||
if (url === 'https://example.com/status') {
|
||||
@@ -67,57 +54,44 @@ describe('checkBackendStatus', () => {
|
||||
throw new Error(`Unexpected URL: ${url}`);
|
||||
});
|
||||
|
||||
await expect(
|
||||
checkBackendStatus({
|
||||
baseUrl: 'https://example.com',
|
||||
serverSecret: 'wrong-secret',
|
||||
fetch,
|
||||
options: { onEvent },
|
||||
})
|
||||
).resolves.toBe(false);
|
||||
const result = await checkBackendStatus({
|
||||
baseUrl: 'https://example.com',
|
||||
serverSecret: 'wrong-secret',
|
||||
fetch,
|
||||
});
|
||||
|
||||
expect(result.ok).toBe(false);
|
||||
expect(result.failure).toContain('Secret key: The backend rejected the secret key (HTTP 401)');
|
||||
expect(fetch).toHaveBeenCalledTimes(2);
|
||||
expect(onEvent).toHaveBeenCalledWith('healthcheck_auth_failed', { attempt: 1 });
|
||||
});
|
||||
|
||||
it('aborts hanging ACP auth probes and reports the healthcheck timeout', async () => {
|
||||
vi.useFakeTimers();
|
||||
it('reports an unusable URL without any request', async () => {
|
||||
const fetch = vi.fn();
|
||||
|
||||
try {
|
||||
const onEvent = vi.fn();
|
||||
const acpSignals: FetchSignal[] = [];
|
||||
const fetch = vi.fn((input: FetchInput, init?: FetchInit): Promise<Response> => {
|
||||
const url = fetchInputUrl(input);
|
||||
if (url === 'https://example.com/status') {
|
||||
expectAbortSignal(init);
|
||||
return Promise.resolve(new Response(null, { status: 200 }));
|
||||
}
|
||||
if (url === 'https://example.com/acp?token=test-secret') {
|
||||
const signal = expectAbortSignal(init);
|
||||
acpSignals.push(signal);
|
||||
return new Promise<Response>((_, reject) => {
|
||||
signal.addEventListener('abort', () => reject(new Error('aborted')), { once: true });
|
||||
});
|
||||
}
|
||||
const result = await checkBackendStatus({
|
||||
baseUrl: 'https://example.com/acp',
|
||||
serverSecret: 'test-secret',
|
||||
fetch,
|
||||
});
|
||||
|
||||
throw new Error(`Unexpected URL: ${url}`);
|
||||
});
|
||||
expect(result.ok).toBe(false);
|
||||
expect(result.failure).toContain('URL:');
|
||||
expect(fetch).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
const result = checkBackendStatus({
|
||||
baseUrl: 'https://example.com',
|
||||
serverSecret: 'test-secret',
|
||||
fetch,
|
||||
options: { onEvent },
|
||||
});
|
||||
it('stops retrying a fatal network failure', async () => {
|
||||
const fetch = vi.fn(async () => {
|
||||
throw new Error('net::ERR_NAME_NOT_RESOLVED');
|
||||
});
|
||||
|
||||
await vi.advanceTimersByTimeAsync(31000);
|
||||
const result = await checkBackendStatus({
|
||||
baseUrl: 'https://nope.example.com',
|
||||
serverSecret: 'test-secret',
|
||||
fetch,
|
||||
});
|
||||
|
||||
await expect(result).resolves.toBe(false);
|
||||
expect(acpSignals.length).toBeGreaterThan(0);
|
||||
expect(acpSignals.every((signal) => signal.aborted)).toBe(true);
|
||||
expect(onEvent).toHaveBeenCalledWith('healthcheck_timeout', { timeoutMs: 30000 });
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
expect(result.ok).toBe(false);
|
||||
expect(result.failure).toContain('ERR_NAME_NOT_RESOLVED');
|
||||
expect(fetch).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
|
||||
+126
-60
@@ -1,95 +1,161 @@
|
||||
import { acpHttpUrlFromHttpBase, statusHttpUrlFromHttpBase } from './acp/url';
|
||||
import {
|
||||
acpHttpUrlFromHttpBase,
|
||||
normalizeAcpHttpBaseUrl,
|
||||
statusHttpUrlFromHttpBase,
|
||||
} from './acp/url';
|
||||
|
||||
const HEALTHCHECK_TIMEOUT_MS = 30000;
|
||||
const HEALTHCHECK_INTERVAL_MS = 100;
|
||||
const PROBE_TIMEOUT_MS = 1000;
|
||||
const RETRY_BUDGET_MS = 15000;
|
||||
const RETRY_INTERVAL_MS = 250;
|
||||
const PROBE_TIMEOUT_MS = 5000;
|
||||
|
||||
type FetchInput = Parameters<typeof globalThis.fetch>[0];
|
||||
type FetchInit = Parameters<typeof globalThis.fetch>[1];
|
||||
const FATAL_ERROR_PATTERN = /panicked at|RUST_BACKTRACE|fatal error/i;
|
||||
const FATAL_NETWORK_PATTERN = /NAME_NOT_RESOLVED|CERT|SSL|CLIENT_AUTH|INVALID_URL|UNSAFE_PORT/;
|
||||
|
||||
export interface CheckServerStatusOptions {
|
||||
onEvent?: (name: string, details?: Record<string, unknown>) => void;
|
||||
export interface BackendCheckStep {
|
||||
name: string;
|
||||
ok: boolean;
|
||||
detail: string;
|
||||
}
|
||||
|
||||
export interface CheckBackendStatusParams {
|
||||
export interface BackendCheckResult {
|
||||
ok: boolean;
|
||||
steps: BackendCheckStep[];
|
||||
failure: string | null;
|
||||
}
|
||||
|
||||
export interface BackendCheckParams {
|
||||
baseUrl: string;
|
||||
serverSecret: string;
|
||||
fetch: typeof globalThis.fetch;
|
||||
errorLog?: string[];
|
||||
options?: CheckServerStatusOptions;
|
||||
}
|
||||
|
||||
export const isFatalError = (line: string): boolean => {
|
||||
const fatalPatterns = [/panicked at/, /RUST_BACKTRACE/, /fatal error/i];
|
||||
return fatalPatterns.some((pattern) => pattern.test(line));
|
||||
};
|
||||
interface Probe {
|
||||
ok: boolean;
|
||||
detail: string;
|
||||
retryable: boolean;
|
||||
}
|
||||
|
||||
const delay = (timeoutMs: number): Promise<void> =>
|
||||
new Promise((resolve) => setTimeout(resolve, timeoutMs));
|
||||
|
||||
const fetchWithTimeout = async (
|
||||
const errorText = (error: unknown): string =>
|
||||
error instanceof Error ? error.message : String(error);
|
||||
|
||||
const proxyNote = (response: Response): string => {
|
||||
const doormanError = response.headers.get('x-sq-cf-doorman-error');
|
||||
return doormanError && doormanError !== 'none'
|
||||
? ` A proxy in front of the backend reported "${doormanError}".`
|
||||
: '';
|
||||
};
|
||||
|
||||
const request = async (
|
||||
fetch: typeof globalThis.fetch,
|
||||
input: FetchInput,
|
||||
init?: FetchInit
|
||||
): Promise<Response> => {
|
||||
url: string,
|
||||
init: Parameters<typeof globalThis.fetch>[1],
|
||||
expect: (response: Response) => Probe
|
||||
): Promise<Probe> => {
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(() => controller.abort(), PROBE_TIMEOUT_MS);
|
||||
|
||||
try {
|
||||
return await fetch(input, { ...init, signal: controller.signal });
|
||||
return expect(await fetch(url, { ...init, signal: controller.signal }));
|
||||
} catch (error) {
|
||||
const detail = errorText(error);
|
||||
return { ok: false, detail, retryable: !FATAL_NETWORK_PATTERN.test(detail) };
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
};
|
||||
|
||||
const probeStatus = (
|
||||
fetch: typeof globalThis.fetch,
|
||||
baseUrl: string,
|
||||
secret: string
|
||||
): Promise<Probe> =>
|
||||
request(
|
||||
fetch,
|
||||
statusHttpUrlFromHttpBase(baseUrl),
|
||||
{ headers: { 'X-Secret-Key': secret } },
|
||||
(response) =>
|
||||
response.ok
|
||||
? { ok: true, detail: `GET /status returned ${response.status}.`, retryable: false }
|
||||
: {
|
||||
ok: false,
|
||||
detail: `GET /status returned ${response.status} ${response.statusText}.${proxyNote(response)}`,
|
||||
retryable: response.status >= 500,
|
||||
}
|
||||
);
|
||||
|
||||
const probeSecret = (
|
||||
fetch: typeof globalThis.fetch,
|
||||
baseUrl: string,
|
||||
secret: string
|
||||
): Promise<Probe> =>
|
||||
request(fetch, acpHttpUrlFromHttpBase(baseUrl, secret), {}, (response) => {
|
||||
if (response.status === 406) {
|
||||
return { ok: true, detail: 'The backend accepted the secret key.', retryable: false };
|
||||
}
|
||||
if (response.status === 401 || response.status === 403) {
|
||||
return {
|
||||
ok: false,
|
||||
detail: `The backend rejected the secret key (HTTP ${response.status}). It must match GOOSE_SERVER__SECRET_KEY on the backend.${proxyNote(response)}`,
|
||||
retryable: false,
|
||||
};
|
||||
}
|
||||
return {
|
||||
ok: false,
|
||||
detail: `GET /acp returned ${response.status} ${response.statusText}, expected 406.${proxyNote(response)}`,
|
||||
retryable: response.status >= 500,
|
||||
};
|
||||
});
|
||||
|
||||
export const isFatalError = (line: string): boolean => FATAL_ERROR_PATTERN.test(line);
|
||||
|
||||
export const checkBackendStatus = async ({
|
||||
baseUrl,
|
||||
serverSecret,
|
||||
fetch,
|
||||
errorLog = [],
|
||||
options = {},
|
||||
}: CheckBackendStatusParams): Promise<boolean> => {
|
||||
const deadline = Date.now() + HEALTHCHECK_TIMEOUT_MS;
|
||||
const statusUrl = statusHttpUrlFromHttpBase(baseUrl);
|
||||
const acpUrl = acpHttpUrlFromHttpBase(baseUrl, serverSecret);
|
||||
options.onEvent?.('healthcheck_start', {
|
||||
timeoutMs: HEALTHCHECK_TIMEOUT_MS,
|
||||
intervalMs: HEALTHCHECK_INTERVAL_MS,
|
||||
});
|
||||
}: BackendCheckParams): Promise<BackendCheckResult> => {
|
||||
const steps: BackendCheckStep[] = [];
|
||||
|
||||
let attempt = 1;
|
||||
while (Date.now() < deadline) {
|
||||
if (errorLog.some(isFatalError)) {
|
||||
options.onEvent?.('healthcheck_fatal_error', { attempt });
|
||||
return false;
|
||||
const run = async (name: string, probe: () => Promise<Probe>): Promise<boolean> => {
|
||||
const deadline = Date.now() + RETRY_BUDGET_MS;
|
||||
let result = await probe();
|
||||
while (
|
||||
!result.ok &&
|
||||
result.retryable &&
|
||||
Date.now() < deadline &&
|
||||
!errorLog.some(isFatalError)
|
||||
) {
|
||||
await delay(RETRY_INTERVAL_MS);
|
||||
result = await probe();
|
||||
}
|
||||
steps.push({ name, ok: result.ok, detail: result.detail });
|
||||
return result.ok;
|
||||
};
|
||||
|
||||
try {
|
||||
const response = await fetchWithTimeout(fetch, statusUrl, {
|
||||
headers: {
|
||||
'X-Secret-Key': serverSecret,
|
||||
},
|
||||
});
|
||||
if (response.ok) {
|
||||
const authResponse = await fetchWithTimeout(fetch, acpUrl);
|
||||
// GET /acp without an SSE Accept header returns 406 after auth succeeds.
|
||||
if (authResponse.status === 406) {
|
||||
options.onEvent?.('healthcheck_success', { attempt });
|
||||
return true;
|
||||
}
|
||||
if (authResponse.status === 401 || authResponse.status === 403) {
|
||||
options.onEvent?.('healthcheck_auth_failed', { attempt });
|
||||
return false;
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Retry until the backend is ready or the timeout expires.
|
||||
}
|
||||
|
||||
await delay(HEALTHCHECK_INTERVAL_MS);
|
||||
attempt += 1;
|
||||
let normalizedBaseUrl = '';
|
||||
try {
|
||||
normalizedBaseUrl = normalizeAcpHttpBaseUrl(baseUrl);
|
||||
steps.push({ name: 'URL', ok: true, detail: normalizedBaseUrl });
|
||||
} catch (error) {
|
||||
steps.push({ name: 'URL', ok: false, detail: errorText(error) });
|
||||
}
|
||||
|
||||
options.onEvent?.('healthcheck_timeout', { timeoutMs: HEALTHCHECK_TIMEOUT_MS });
|
||||
return false;
|
||||
if (normalizedBaseUrl) {
|
||||
const reachable = await run('Reachable', () =>
|
||||
probeStatus(fetch, normalizedBaseUrl, serverSecret)
|
||||
);
|
||||
if (reachable) {
|
||||
await run('Secret key', () => probeSecret(fetch, normalizedBaseUrl, serverSecret));
|
||||
}
|
||||
}
|
||||
|
||||
const failed = steps.find((step) => !step.ok);
|
||||
return {
|
||||
ok: !failed,
|
||||
steps,
|
||||
failure: failed ? `${failed.name}: ${failed.detail}`.trim() : null,
|
||||
};
|
||||
};
|
||||
|
||||
@@ -1116,20 +1116,20 @@ const createChat = async (
|
||||
);
|
||||
}
|
||||
|
||||
const externalBackendReady = await checkBackendStatus({
|
||||
const externalBackendCheck = await checkBackendStatus({
|
||||
baseUrl: externalBaseUrl,
|
||||
serverSecret,
|
||||
fetch: net.fetch as unknown as typeof globalThis.fetch,
|
||||
});
|
||||
if (!externalBackendReady) {
|
||||
if (!externalBackendCheck.ok) {
|
||||
externalCertificateTrust?.release();
|
||||
log.error(`External backend check failed: ${externalBackendCheck.failure}`);
|
||||
const canDisableExternalBackend = externalBackend.source === 'settings';
|
||||
const response = dialog.showMessageBoxSync({
|
||||
type: 'error',
|
||||
title: 'External Backend Unreachable',
|
||||
message: `Could not connect to external backend at ${externalBaseUrl}`,
|
||||
detail:
|
||||
'The external backend must be running and the configured secret must match GOOSE_SERVER__SECRET_KEY on the server.',
|
||||
detail: externalBackendCheck.failure ?? undefined,
|
||||
buttons: canDisableExternalBackend
|
||||
? ['Disable External Backend & Retry', 'Quit']
|
||||
: ['Quit'],
|
||||
|
||||
Reference in New Issue
Block a user