fix(ci): prevent flaky smoke test timeouts from failing the build (#8837)
Signed-off-by: Michael Neale <michael.neale@gmail.com>
This commit is contained in:
@@ -313,13 +313,19 @@ function registerTests(label: string, cases: TestCase[], fn: ProviderTestFn): vo
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (flaky.length > 0) {
|
if (flaky.length > 0) {
|
||||||
test.each(flaky)(`${label} — $provider / $model (flaky)`, async (tc) => {
|
// Use a longer vitest timeout (90s) so the internal runGoose timeout (55s)
|
||||||
try {
|
// fires first — that rejection is catchable and the test passes as "allowed".
|
||||||
await fn(tc);
|
test.each(flaky)(
|
||||||
} catch (err) {
|
`${label} — $provider / $model (flaky)`,
|
||||||
console.warn(`Flaky test ${tc.provider}/${tc.model} failed (allowed): ${err}`);
|
async (tc) => {
|
||||||
}
|
try {
|
||||||
});
|
await fn(tc);
|
||||||
|
} catch (err) {
|
||||||
|
console.warn(`Flaky test ${tc.provider}/${tc.model} failed (allowed): ${err}`);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
90_000
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (skipped.length > 0) {
|
if (skipped.length > 0) {
|
||||||
@@ -357,9 +363,10 @@ export function runGoose(
|
|||||||
cwd: string,
|
cwd: string,
|
||||||
prompt: string,
|
prompt: string,
|
||||||
builtins: string,
|
builtins: string,
|
||||||
env: Record<string, string>
|
env: Record<string, string>,
|
||||||
|
timeoutMs: number = 55_000
|
||||||
): Promise<string> {
|
): Promise<string> {
|
||||||
return new Promise((resolve) => {
|
return new Promise((resolve, reject) => {
|
||||||
const child: ChildProcess = spawn(
|
const child: ChildProcess = spawn(
|
||||||
gooseBin,
|
gooseBin,
|
||||||
['run', '--text', prompt, '--with-builtin', builtins],
|
['run', '--text', prompt, '--with-builtin', builtins],
|
||||||
@@ -371,6 +378,16 @@ export function runGoose(
|
|||||||
);
|
);
|
||||||
|
|
||||||
let output = '';
|
let output = '';
|
||||||
|
let settled = false;
|
||||||
|
|
||||||
|
const timer = setTimeout(() => {
|
||||||
|
if (!settled) {
|
||||||
|
settled = true;
|
||||||
|
child.kill('SIGKILL');
|
||||||
|
reject(new Error(`goose timed out after ${timeoutMs}ms\n\nPartial output:\n${output}`));
|
||||||
|
}
|
||||||
|
}, timeoutMs);
|
||||||
|
|
||||||
child.stdout?.on('data', (d) => {
|
child.stdout?.on('data', (d) => {
|
||||||
output += String(d);
|
output += String(d);
|
||||||
});
|
});
|
||||||
@@ -379,11 +396,19 @@ export function runGoose(
|
|||||||
});
|
});
|
||||||
|
|
||||||
child.on('close', () => {
|
child.on('close', () => {
|
||||||
resolve(output);
|
if (!settled) {
|
||||||
|
settled = true;
|
||||||
|
clearTimeout(timer);
|
||||||
|
resolve(output);
|
||||||
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
child.on('error', (err) => {
|
child.on('error', (err) => {
|
||||||
resolve(`spawn error: ${err.message}`);
|
if (!settled) {
|
||||||
|
settled = true;
|
||||||
|
clearTimeout(timer);
|
||||||
|
resolve(`spawn error: ${err.message}`);
|
||||||
|
}
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user