feat(desktop): report the real cause of external backend connection failures (#11828)

This commit is contained in:
Alex Hancock
2026-09-09 14:09:28 -04:00
committed by GitHub
parent 13c7081076
commit a8a4dbce82
3 changed files with 163 additions and 123 deletions
+33 -59
View File
@@ -2,7 +2,6 @@ import { describe, expect, it, vi } from 'vitest';
import { checkBackendStatus } from './backendStatus';
type FetchInput = Parameters<typeof globalThis.fetch>[0];
type FetchInit = NonNullable<Parameters<typeof globalThis.fetch>[1]>;
const fetchInputUrl = (input: FetchInput): string => {
if (typeof input === 'string') {
@@ -14,24 +13,14 @@ const fetchInputUrl = (input: FetchInput): string => {
return input.url;
};
type FetchSignal = NonNullable<FetchInit['signal']>;
const expectAbortSignal = (init?: FetchInit): FetchSignal => {
expect(init?.signal).toBeInstanceOf(globalThis.AbortSignal);
return init!.signal!;
};
describe('checkBackendStatus', () => {
it('checks /status and validates the secret against /acp', async () => {
const fetch = vi.fn(async (input: FetchInput, init?: FetchInit) => {
const fetch = vi.fn(async (input: FetchInput) => {
const url = fetchInputUrl(input);
if (url === 'https://example.com/goose/status') {
expect(init?.headers).toEqual({ 'X-Secret-Key': 'test-secret' });
expectAbortSignal(init);
return new Response(null, { status: 200 });
}
if (url === 'https://example.com/goose/acp?token=test-secret') {
expectAbortSignal(init);
return new Response(null, { status: 406 });
}
@@ -44,17 +33,15 @@ describe('checkBackendStatus', () => {
serverSecret: 'test-secret',
fetch,
})
).resolves.toBe(true);
).resolves.toMatchObject({ ok: true, failure: null });
expect(fetch).toHaveBeenCalledTimes(2);
expect(fetch.mock.calls.map(([input]) => fetchInputUrl(input))).toEqual([
'https://example.com/goose/status',
'https://example.com/goose/acp?token=test-secret',
]);
});
it('fails immediately when the ACP auth probe rejects the secret', async () => {
const onEvent = vi.fn();
it('reports the rejected secret without retrying', async () => {
const fetch = vi.fn(async (input: FetchInput) => {
const url = fetchInputUrl(input);
if (url === 'https://example.com/status') {
@@ -67,57 +54,44 @@ describe('checkBackendStatus', () => {
throw new Error(`Unexpected URL: ${url}`);
});
await expect(
checkBackendStatus({
baseUrl: 'https://example.com',
serverSecret: 'wrong-secret',
fetch,
options: { onEvent },
})
).resolves.toBe(false);
const result = await checkBackendStatus({
baseUrl: 'https://example.com',
serverSecret: 'wrong-secret',
fetch,
});
expect(result.ok).toBe(false);
expect(result.failure).toContain('Secret key: The backend rejected the secret key (HTTP 401)');
expect(fetch).toHaveBeenCalledTimes(2);
expect(onEvent).toHaveBeenCalledWith('healthcheck_auth_failed', { attempt: 1 });
});
it('aborts hanging ACP auth probes and reports the healthcheck timeout', async () => {
vi.useFakeTimers();
it('reports an unusable URL without any request', async () => {
const fetch = vi.fn();
try {
const onEvent = vi.fn();
const acpSignals: FetchSignal[] = [];
const fetch = vi.fn((input: FetchInput, init?: FetchInit): Promise<Response> => {
const url = fetchInputUrl(input);
if (url === 'https://example.com/status') {
expectAbortSignal(init);
return Promise.resolve(new Response(null, { status: 200 }));
}
if (url === 'https://example.com/acp?token=test-secret') {
const signal = expectAbortSignal(init);
acpSignals.push(signal);
return new Promise<Response>((_, reject) => {
signal.addEventListener('abort', () => reject(new Error('aborted')), { once: true });
});
}
const result = await checkBackendStatus({
baseUrl: 'https://example.com/acp',
serverSecret: 'test-secret',
fetch,
});
throw new Error(`Unexpected URL: ${url}`);
});
expect(result.ok).toBe(false);
expect(result.failure).toContain('URL:');
expect(fetch).not.toHaveBeenCalled();
});
const result = checkBackendStatus({
baseUrl: 'https://example.com',
serverSecret: 'test-secret',
fetch,
options: { onEvent },
});
it('stops retrying a fatal network failure', async () => {
const fetch = vi.fn(async () => {
throw new Error('net::ERR_NAME_NOT_RESOLVED');
});
await vi.advanceTimersByTimeAsync(31000);
const result = await checkBackendStatus({
baseUrl: 'https://nope.example.com',
serverSecret: 'test-secret',
fetch,
});
await expect(result).resolves.toBe(false);
expect(acpSignals.length).toBeGreaterThan(0);
expect(acpSignals.every((signal) => signal.aborted)).toBe(true);
expect(onEvent).toHaveBeenCalledWith('healthcheck_timeout', { timeoutMs: 30000 });
} finally {
vi.useRealTimers();
}
expect(result.ok).toBe(false);
expect(result.failure).toContain('ERR_NAME_NOT_RESOLVED');
expect(fetch).toHaveBeenCalledTimes(1);
});
});
+126 -60
View File
@@ -1,95 +1,161 @@
import { acpHttpUrlFromHttpBase, statusHttpUrlFromHttpBase } from './acp/url';
import {
acpHttpUrlFromHttpBase,
normalizeAcpHttpBaseUrl,
statusHttpUrlFromHttpBase,
} from './acp/url';
const HEALTHCHECK_TIMEOUT_MS = 30000;
const HEALTHCHECK_INTERVAL_MS = 100;
const PROBE_TIMEOUT_MS = 1000;
const RETRY_BUDGET_MS = 15000;
const RETRY_INTERVAL_MS = 250;
const PROBE_TIMEOUT_MS = 5000;
type FetchInput = Parameters<typeof globalThis.fetch>[0];
type FetchInit = Parameters<typeof globalThis.fetch>[1];
const FATAL_ERROR_PATTERN = /panicked at|RUST_BACKTRACE|fatal error/i;
const FATAL_NETWORK_PATTERN = /NAME_NOT_RESOLVED|CERT|SSL|CLIENT_AUTH|INVALID_URL|UNSAFE_PORT/;
export interface CheckServerStatusOptions {
onEvent?: (name: string, details?: Record<string, unknown>) => void;
export interface BackendCheckStep {
name: string;
ok: boolean;
detail: string;
}
export interface CheckBackendStatusParams {
export interface BackendCheckResult {
ok: boolean;
steps: BackendCheckStep[];
failure: string | null;
}
export interface BackendCheckParams {
baseUrl: string;
serverSecret: string;
fetch: typeof globalThis.fetch;
errorLog?: string[];
options?: CheckServerStatusOptions;
}
export const isFatalError = (line: string): boolean => {
const fatalPatterns = [/panicked at/, /RUST_BACKTRACE/, /fatal error/i];
return fatalPatterns.some((pattern) => pattern.test(line));
};
interface Probe {
ok: boolean;
detail: string;
retryable: boolean;
}
const delay = (timeoutMs: number): Promise<void> =>
new Promise((resolve) => setTimeout(resolve, timeoutMs));
const fetchWithTimeout = async (
const errorText = (error: unknown): string =>
error instanceof Error ? error.message : String(error);
const proxyNote = (response: Response): string => {
const doormanError = response.headers.get('x-sq-cf-doorman-error');
return doormanError && doormanError !== 'none'
? ` A proxy in front of the backend reported "${doormanError}".`
: '';
};
const request = async (
fetch: typeof globalThis.fetch,
input: FetchInput,
init?: FetchInit
): Promise<Response> => {
url: string,
init: Parameters<typeof globalThis.fetch>[1],
expect: (response: Response) => Probe
): Promise<Probe> => {
const controller = new AbortController();
const timeout = setTimeout(() => controller.abort(), PROBE_TIMEOUT_MS);
try {
return await fetch(input, { ...init, signal: controller.signal });
return expect(await fetch(url, { ...init, signal: controller.signal }));
} catch (error) {
const detail = errorText(error);
return { ok: false, detail, retryable: !FATAL_NETWORK_PATTERN.test(detail) };
} finally {
clearTimeout(timeout);
}
};
const probeStatus = (
fetch: typeof globalThis.fetch,
baseUrl: string,
secret: string
): Promise<Probe> =>
request(
fetch,
statusHttpUrlFromHttpBase(baseUrl),
{ headers: { 'X-Secret-Key': secret } },
(response) =>
response.ok
? { ok: true, detail: `GET /status returned ${response.status}.`, retryable: false }
: {
ok: false,
detail: `GET /status returned ${response.status} ${response.statusText}.${proxyNote(response)}`,
retryable: response.status >= 500,
}
);
const probeSecret = (
fetch: typeof globalThis.fetch,
baseUrl: string,
secret: string
): Promise<Probe> =>
request(fetch, acpHttpUrlFromHttpBase(baseUrl, secret), {}, (response) => {
if (response.status === 406) {
return { ok: true, detail: 'The backend accepted the secret key.', retryable: false };
}
if (response.status === 401 || response.status === 403) {
return {
ok: false,
detail: `The backend rejected the secret key (HTTP ${response.status}). It must match GOOSE_SERVER__SECRET_KEY on the backend.${proxyNote(response)}`,
retryable: false,
};
}
return {
ok: false,
detail: `GET /acp returned ${response.status} ${response.statusText}, expected 406.${proxyNote(response)}`,
retryable: response.status >= 500,
};
});
export const isFatalError = (line: string): boolean => FATAL_ERROR_PATTERN.test(line);
export const checkBackendStatus = async ({
baseUrl,
serverSecret,
fetch,
errorLog = [],
options = {},
}: CheckBackendStatusParams): Promise<boolean> => {
const deadline = Date.now() + HEALTHCHECK_TIMEOUT_MS;
const statusUrl = statusHttpUrlFromHttpBase(baseUrl);
const acpUrl = acpHttpUrlFromHttpBase(baseUrl, serverSecret);
options.onEvent?.('healthcheck_start', {
timeoutMs: HEALTHCHECK_TIMEOUT_MS,
intervalMs: HEALTHCHECK_INTERVAL_MS,
});
}: BackendCheckParams): Promise<BackendCheckResult> => {
const steps: BackendCheckStep[] = [];
let attempt = 1;
while (Date.now() < deadline) {
if (errorLog.some(isFatalError)) {
options.onEvent?.('healthcheck_fatal_error', { attempt });
return false;
const run = async (name: string, probe: () => Promise<Probe>): Promise<boolean> => {
const deadline = Date.now() + RETRY_BUDGET_MS;
let result = await probe();
while (
!result.ok &&
result.retryable &&
Date.now() < deadline &&
!errorLog.some(isFatalError)
) {
await delay(RETRY_INTERVAL_MS);
result = await probe();
}
steps.push({ name, ok: result.ok, detail: result.detail });
return result.ok;
};
try {
const response = await fetchWithTimeout(fetch, statusUrl, {
headers: {
'X-Secret-Key': serverSecret,
},
});
if (response.ok) {
const authResponse = await fetchWithTimeout(fetch, acpUrl);
// GET /acp without an SSE Accept header returns 406 after auth succeeds.
if (authResponse.status === 406) {
options.onEvent?.('healthcheck_success', { attempt });
return true;
}
if (authResponse.status === 401 || authResponse.status === 403) {
options.onEvent?.('healthcheck_auth_failed', { attempt });
return false;
}
}
} catch {
// Retry until the backend is ready or the timeout expires.
}
await delay(HEALTHCHECK_INTERVAL_MS);
attempt += 1;
let normalizedBaseUrl = '';
try {
normalizedBaseUrl = normalizeAcpHttpBaseUrl(baseUrl);
steps.push({ name: 'URL', ok: true, detail: normalizedBaseUrl });
} catch (error) {
steps.push({ name: 'URL', ok: false, detail: errorText(error) });
}
options.onEvent?.('healthcheck_timeout', { timeoutMs: HEALTHCHECK_TIMEOUT_MS });
return false;
if (normalizedBaseUrl) {
const reachable = await run('Reachable', () =>
probeStatus(fetch, normalizedBaseUrl, serverSecret)
);
if (reachable) {
await run('Secret key', () => probeSecret(fetch, normalizedBaseUrl, serverSecret));
}
}
const failed = steps.find((step) => !step.ok);
return {
ok: !failed,
steps,
failure: failed ? `${failed.name}: ${failed.detail}`.trim() : null,
};
};
+4 -4
View File
@@ -1116,20 +1116,20 @@ const createChat = async (
);
}
const externalBackendReady = await checkBackendStatus({
const externalBackendCheck = await checkBackendStatus({
baseUrl: externalBaseUrl,
serverSecret,
fetch: net.fetch as unknown as typeof globalThis.fetch,
});
if (!externalBackendReady) {
if (!externalBackendCheck.ok) {
externalCertificateTrust?.release();
log.error(`External backend check failed: ${externalBackendCheck.failure}`);
const canDisableExternalBackend = externalBackend.source === 'settings';
const response = dialog.showMessageBoxSync({
type: 'error',
title: 'External Backend Unreachable',
message: `Could not connect to external backend at ${externalBaseUrl}`,
detail:
'The external backend must be running and the configured secret must match GOOSE_SERVER__SECRET_KEY on the server.',
detail: externalBackendCheck.failure ?? undefined,
buttons: canDisableExternalBackend
? ['Disable External Backend & Retry', 'Quit']
: ['Quit'],