fix(ci): prevent flaky smoke test timeouts from failing the build (#8837)

Signed-off-by: Michael Neale <michael.neale@gmail.com>
This commit is contained in:
Michael Neale
2026-04-24 16:30:53 -07:00
committed by GitHub
parent e790d37959
commit 097e1bc99c
@@ -313,13 +313,19 @@ function registerTests(label: string, cases: TestCase[], fn: ProviderTestFn): vo
} }
if (flaky.length > 0) { if (flaky.length > 0) {
test.each(flaky)(`${label}$provider / $model (flaky)`, async (tc) => { // Use a longer vitest timeout (90s) so the internal runGoose timeout (55s)
try { // fires first — that rejection is catchable and the test passes as "allowed".
await fn(tc); test.each(flaky)(
} catch (err) { `${label}$provider / $model (flaky)`,
console.warn(`Flaky test ${tc.provider}/${tc.model} failed (allowed): ${err}`); async (tc) => {
} try {
}); await fn(tc);
} catch (err) {
console.warn(`Flaky test ${tc.provider}/${tc.model} failed (allowed): ${err}`);
}
},
90_000
);
} }
if (skipped.length > 0) { if (skipped.length > 0) {
@@ -357,9 +363,10 @@ export function runGoose(
cwd: string, cwd: string,
prompt: string, prompt: string,
builtins: string, builtins: string,
env: Record<string, string> env: Record<string, string>,
timeoutMs: number = 55_000
): Promise<string> { ): Promise<string> {
return new Promise((resolve) => { return new Promise((resolve, reject) => {
const child: ChildProcess = spawn( const child: ChildProcess = spawn(
gooseBin, gooseBin,
['run', '--text', prompt, '--with-builtin', builtins], ['run', '--text', prompt, '--with-builtin', builtins],
@@ -371,6 +378,16 @@ export function runGoose(
); );
let output = ''; let output = '';
let settled = false;
const timer = setTimeout(() => {
if (!settled) {
settled = true;
child.kill('SIGKILL');
reject(new Error(`goose timed out after ${timeoutMs}ms\n\nPartial output:\n${output}`));
}
}, timeoutMs);
child.stdout?.on('data', (d) => { child.stdout?.on('data', (d) => {
output += String(d); output += String(d);
}); });
@@ -379,11 +396,19 @@ export function runGoose(
}); });
child.on('close', () => { child.on('close', () => {
resolve(output); if (!settled) {
settled = true;
clearTimeout(timer);
resolve(output);
}
}); });
child.on('error', (err) => { child.on('error', (err) => {
resolve(`spawn error: ${err.message}`); if (!settled) {
settled = true;
clearTimeout(timer);
resolve(`spawn error: ${err.message}`);
}
}); });
}); });
} }