Make provider smoke tests faster and more reliable (#10605)

Co-authored-by: Douwe M Osinga <douwe@sidewalklabs.com>

crates/goose/src/providers/claude_code.rs

@@ -854,6 +854,42 @@ impl Provider for ClaudeCodeProvider {
                                 }
                                 Some("result") => {
                                     process.needs_drain = false;
+                                    if parsed
+                                        .get("is_error")
+                                        .and_then(Value::as_bool)
+                                        .unwrap_or(false)
+                                    {
+                                        let subtype = parsed
+                                            .get("subtype")
+                                            .and_then(Value::as_str)
+                                            .unwrap_or("error");
+                                        let mut details = Vec::new();
+                                        if let Some(error) =
+                                            parsed.get("error").and_then(Value::as_str)
+                                        {
+                                            details.push(error);
+                                        }
+                                        if let Some(errors) =
+                                            parsed.get("errors").and_then(Value::as_array)
+                                        {
+                                            details.extend(errors.iter().filter_map(Value::as_str));
+                                        }
+                                        if let Some(result) =
+                                            parsed.get("result").and_then(Value::as_str)
+                                        {
+                                            details.push(result);
+                                        }
+                                        let details = details.join("; ");
+                                        let message = match (subtype, details.is_empty()) {
+                                            ("success", false) => details,
+                                            (_, false) => format!("{subtype}: {details}"),
+                                            _ => subtype.to_string(),
+                                        };
+                                        stream_error = Some(ProviderError::RequestFailed(format!(
+                                            "Claude CLI error: {message}"
+                                        )));
+                                        break;
+                                    }
                                     if let Some(usage_info) = parsed.get("usage") {
                                         let new = extract_usage_tokens(usage_info);
                                         let reports_own_cache = new.cache_read_input_tokens.is_some()

ui/desktop/tests/integration/test_providers.test.ts

@@ -6,7 +6,7 @@
  * and validates the output.
  */
 
-import { expect, beforeAll } from 'vitest';
+import { beforeAll } from 'vitest';
 import fs from 'node:fs';
 import os from 'node:os';
 import path from 'node:path';
@@ -29,7 +29,7 @@ beforeAll(() => {
 
 const { testAgentic, testNonAgentic } = providerTest(discoverTestCases());
 
-testNonAgentic('reads files via shell tool', async (tc) => {
+testNonAgentic('reads files via shell tool', async (tc, { expect }) => {
   const testdir = fs.mkdtempSync(path.join(os.tmpdir(), 'goose-test-'));
   try {
     const tokenA = `smoke-alpha-${Math.floor(Math.random() * 32768)}`;
@@ -68,7 +68,7 @@ testNonAgentic('reads files via shell tool', async (tc) => {
   }
 });
 
-testAgentic('reads file contents', async (tc) => {
+testAgentic('reads file contents', async (tc, { expect }) => {
   const testdir = fs.mkdtempSync(path.join(os.tmpdir(), 'goose-test-'));
   try {
     fs.copyFileSync(testFile, path.join(testdir, 'test-content.txt'));

ui/desktop/tests/integration/test_providers_code_exec.test.ts

@@ -6,7 +6,7 @@
  * that the code_execution tool was invoked.
  */
 
-import { expect, beforeAll } from 'vitest';
+import { beforeAll } from 'vitest';
 import fs from 'node:fs';
 import os from 'node:os';
 import path from 'node:path';
@@ -22,7 +22,7 @@ beforeAll(() => {
 
 const { testAll } = providerTest(discoverTestCases({ skipAgentic: true }));
 
-testAll('invokes code_execution tool', async (tc) => {
+testAll('invokes code_execution tool', async (tc, { expect }) => {
   const testdir = fs.mkdtempSync(path.join(os.tmpdir(), 'goose-codeexec-'));
   try {
     const output = await runGoose(

ui/desktop/tests/integration/test_providers_lib.ts

@@ -5,7 +5,7 @@
  * allowed-failure list, agentic-provider list, and environment detection.
  */
 
-import { test } from 'vitest';
+import { test, type TestContext } from 'vitest';
 import { execSync, spawn, type ChildProcess } from 'node:child_process';
 import fs from 'node:fs';
 import os from 'node:os';
@@ -299,7 +299,7 @@ export function discoverTestCases(options?: { skipAgentic?: boolean }): TestCase
 // Test registration helpers
 // ---------------------------------------------------------------------------
 
-type ProviderTestFn = (tc: TestCase) => Promise<void>;
+type ProviderTestFn = (tc: TestCase, context: TestContext) => Promise<void>;
 
 function registerTests(label: string, cases: TestCase[], fn: ProviderTestFn): void {
   const available = cases.filter((tc) => tc.available && !tc.flaky);
@@ -307,24 +307,24 @@ function registerTests(label: string, cases: TestCase[], fn: ProviderTestFn): vo
   const skipped = cases.filter((tc) => !tc.available);
 
   if (available.length > 0) {
-    test.each(available)(`${label} — $provider / $model`, async (tc) => {
-      await fn(tc);
+    test.concurrent.for(available)(`${label} — $provider / $model`, async (tc, context) => {
+      await fn(tc, context);
     });
   }
 
   if (flaky.length > 0) {
     // Use a longer vitest timeout (90s) so the internal runGoose timeout (55s)
     // fires first — that rejection is catchable and the test passes as "allowed".
-    test.each(flaky)(
+    test.concurrent.for(flaky)(
       `${label} — $provider / $model (flaky)`,
-      async (tc) => {
+      { timeout: 90_000 },
+      async (tc, context) => {
         try {
-          await fn(tc);
+          await fn(tc, context);
         } catch (err) {
           console.warn(`Flaky test ${tc.provider}/${tc.model} failed (allowed): ${err}`);
         }
-      },
-      90_000
+      }
     );
   }
 
@@ -373,7 +373,7 @@ export function runGoose(
       ['run', '--text', prompt, '--with-builtin', builtins],
       {
         cwd,
-        env: { ...process.env, ...env },
+        env: { ...process.env, ...env, GOOSE_MODE: 'auto' },
         stdio: ['ignore', 'pipe', 'pipe'],
       }
     );

ui/desktop/vitest.integration.config.ts

@@ -15,6 +15,7 @@ export default defineConfig({
     hookTimeout: 60000,
     pool: 'forks',
     singleFork: true,
+    maxConcurrency: 4,
     silent: 'passed-only',
   },
 });