Port provider tests to typescript (#8237)

Signed-off-by: Douwe Osinga <douwe@squareup.com>
Co-authored-by: Douwe Osinga <douwe@squareup.com>

ui/desktop/package.json

@@ -35,6 +35,9 @@
     "test:ui": "vitest --ui",
     "test:coverage": "vitest run --coverage",
     "test:integration": "vitest run --config vitest.integration.config.ts",
+    "test:integration:goosed": "vitest run --config vitest.integration.config.ts tests/integration/goosed.test.ts",
+    "test:integration:providers": "vitest run --config vitest.integration.config.ts tests/integration/test_providers.test.ts",
+    "test:integration:providers-code-exec": "vitest run --config vitest.integration.config.ts tests/integration/test_providers_code_exec.test.ts",
     "test:integration:watch": "vitest --config vitest.integration.config.ts",
     "test:integration:debug": "DEBUG=1 vitest run --config vitest.integration.config.ts",
     "i18n:extract": "formatjs extract 'src/**/*.{ts,tsx}' --out-file src/i18n/messages/en.json --flatten && pnpm run i18n:compile",

ui/desktop/tests/integration/test_providers.test.ts

@@ -0,0 +1,86 @@
+/**
+ * Provider smoke tests — normal mode (direct tool calls).
+ *
+ * Each available provider/model pair gets its own test that spawns `goose run`
+ * with the developer builtin, asks the model to read files via the shell tool,
+ * and validates the output.
+ */
+
+import { expect, beforeAll } from 'vitest';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { buildGoose, discoverTestCases, runGoose, providerTest } from './test_providers_lib';
+
+const BUILTINS = 'developer';
+const TEST_CONTENT = 'test-content-abc123';
+
+let gooseBin: string;
+let testFile: string;
+
+beforeAll(() => {
+  gooseBin = buildGoose();
+
+  const targetDir = path.resolve(process.cwd(), '..', '..', 'target');
+  fs.mkdirSync(targetDir, { recursive: true });
+  testFile = path.join(targetDir, 'test-content.txt');
+  fs.writeFileSync(testFile, TEST_CONTENT + '\n');
+});
+
+const { testAgentic, testNonAgentic } = providerTest(discoverTestCases());
+
+testNonAgentic('reads files via shell tool', async (tc) => {
+  const testdir = fs.mkdtempSync(path.join(os.tmpdir(), 'goose-test-'));
+  try {
+    const tokenA = `smoke-alpha-${Math.floor(Math.random() * 32768)}`;
+    const tokenB = `smoke-bravo-${Math.floor(Math.random() * 32768)}`;
+    fs.writeFileSync(path.join(testdir, 'part-a.txt'), tokenA + '\n');
+    fs.writeFileSync(path.join(testdir, 'part-b.txt'), tokenB + '\n');
+
+    const output = await runGoose(
+      gooseBin,
+      testdir,
+      'Use the shell tool to cat ./part-a.txt and ./part-b.txt, then reply with ONLY the contents of both files, one per line, nothing else.',
+      BUILTINS,
+      { GOOSE_PROVIDER: tc.provider, GOOSE_MODEL: tc.model }
+    );
+
+    const shellToolPattern = /(shell \| developer)|(▸.*shell)/;
+    expect(
+      shellToolPattern.test(output),
+      `Expected model to use shell tool\n\nFull output:\n${output}`
+    ).toBe(true);
+    expect(
+      output,
+      `Expected output to contain token from part-a.txt (${tokenA})\n\nFull output:\n${output}`
+    ).toContain(tokenA);
+    expect(
+      output,
+      `Expected output to contain token from part-b.txt (${tokenB})\n\nFull output:\n${output}`
+    ).toContain(tokenB);
+  } finally {
+    fs.rmSync(testdir, { recursive: true, force: true });
+  }
+});
+
+testAgentic('reads file contents', async (tc) => {
+  const testdir = fs.mkdtempSync(path.join(os.tmpdir(), 'goose-test-'));
+  try {
+    fs.copyFileSync(testFile, path.join(testdir, 'test-content.txt'));
+
+    const output = await runGoose(
+      gooseBin,
+      testdir,
+      'read ./test-content.txt and output its contents exactly',
+      BUILTINS,
+      { GOOSE_PROVIDER: tc.provider, GOOSE_MODEL: tc.model }
+    );
+
+    expect(
+      output.toLowerCase(),
+      `Expected model output to contain "${TEST_CONTENT}"\n\nFull output:\n${output}`
+    ).toContain(TEST_CONTENT.toLowerCase());
+  } finally {
+    fs.rmSync(testdir, { recursive: true, force: true });
+  }
+});

ui/desktop/tests/integration/test_providers_code_exec.test.ts

@@ -0,0 +1,50 @@
+/**
+ * Provider smoke tests — code execution mode (JS batching).
+ *
+ * Each available (non-agentic) provider/model pair gets its own test that
+ * spawns `goose run` with the memory + code_execution builtins and validates
+ * that the code_execution tool was invoked.
+ */
+
+import { expect, beforeAll } from 'vitest';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { buildGoose, discoverTestCases, runGoose, providerTest } from './test_providers_lib';
+
+const BUILTINS = 'memory,code_execution';
+
+let gooseBin: string;
+
+beforeAll(() => {
+  gooseBin = buildGoose();
+});
+
+const { testAll } = providerTest(discoverTestCases({ skipAgentic: true }));
+
+testAll('invokes code_execution tool', async (tc) => {
+  const testdir = fs.mkdtempSync(path.join(os.tmpdir(), 'goose-codeexec-'));
+  try {
+    const output = await runGoose(
+      gooseBin,
+      testdir,
+      "Store a memory with category 'test' and data 'hello world', then retrieve all memories from category 'test'.",
+      BUILTINS,
+      { GOOSE_PROVIDER: tc.provider, GOOSE_MODEL: tc.model }
+    );
+
+    // Matches: "execute_typescript | code_execution", "get_function_details | code_execution",
+    //           "tool call | execute", "tool calls | execute" (old format)
+    //           "▸ execute N tool call" (new format with tool_graph)
+    //           "▸ execute_typescript" (plain tool name in output)
+    const codeExecPattern =
+      /(execute_typescript \| code_execution)|(get_function_details \| code_execution)|(tool calls? \| execute)|(▸.*execute.*tool call)|(▸ execute_typescript)/;
+
+    expect(
+      codeExecPattern.test(output),
+      `Expected code_execution tool to be called\n\nFull output:\n${output}`
+    ).toBe(true);
+  } finally {
+    fs.rmSync(testdir, { recursive: true, force: true });
+  }
+});

ui/desktop/tests/integration/test_providers_lib.ts

@@ -0,0 +1,389 @@
+/**
+ * Shared library for provider smoke tests.
+ *
+ * Ported from scripts/test_providers_lib.sh — keeps the same provider config,
+ * allowed-failure list, agentic-provider list, and environment detection.
+ */
+
+import { test } from 'vitest';
+import { execSync, spawn, type ChildProcess } from 'node:child_process';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+
+// ---------------------------------------------------------------------------
+// Provider configuration
+// ---------------------------------------------------------------------------
+
+type ModelEntry = string | { name: string; flaky: true };
+
+interface ProviderConfig {
+  provider: string;
+  models: ModelEntry[];
+  agentic?: boolean;
+  available: () => boolean;
+}
+
+function modelName(entry: ModelEntry): string {
+  return typeof entry === 'string' ? entry : entry.name;
+}
+
+function modelFlaky(entry: ModelEntry): boolean {
+  return typeof entry !== 'string' && entry.flaky;
+}
+
+function hasEnv(name: string): boolean {
+  return !!process.env[name];
+}
+
+function hasCmd(name: string): boolean {
+  try {
+    execSync(`command -v ${name}`, { stdio: 'ignore' });
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+function hasFile(p: string): boolean {
+  return fs.existsSync(p);
+}
+
+function getProviders(): ProviderConfig[] {
+  return [
+    {
+      provider: 'openrouter',
+      models: [
+        'google/gemini-2.5-pro',
+        'anthropic/claude-sonnet-4.5',
+        { name: 'qwen/qwen3-coder:exacto', flaky: true },
+        'z-ai/glm-4.6:exacto',
+        { name: 'nvidia/nemotron-3-nano-30b-a3b', flaky: true },
+      ],
+      available: () => hasEnv('OPENROUTER_API_KEY'),
+    },
+    {
+      provider: 'xai',
+      models: ['grok-3'],
+      available: () => hasEnv('XAI_API_KEY'),
+    },
+    {
+      provider: 'openai',
+      models: ['gpt-4o', 'gpt-4o-mini', { name: 'gpt-3.5-turbo', flaky: true }, 'gpt-5'],
+      available: () => hasEnv('OPENAI_API_KEY'),
+    },
+    {
+      provider: 'anthropic',
+      models: ['claude-sonnet-4-5-20250929', 'claude-opus-4-5-20251101'],
+      available: () => hasEnv('ANTHROPIC_API_KEY'),
+    },
+    {
+      provider: 'google',
+      models: [
+        'gemini-2.5-pro',
+        { name: 'gemini-2.5-flash', flaky: true },
+        { name: 'gemini-3-pro-preview', flaky: true },
+        'gemini-3-flash-preview',
+      ],
+      available: () => hasEnv('GOOGLE_API_KEY'),
+    },
+    {
+      provider: 'tetrate',
+      models: ['claude-sonnet-4-20250514'],
+      available: () => hasEnv('TETRATE_API_KEY'),
+    },
+    {
+      provider: 'databricks',
+      models: ['databricks-claude-sonnet-4', 'gemini-2-5-flash', 'gpt-4o'],
+      available: () => hasEnv('DATABRICKS_HOST') && hasEnv('DATABRICKS_TOKEN'),
+    },
+    {
+      provider: 'azure_openai',
+      models: [process.env.AZURE_OPENAI_DEPLOYMENT_NAME ?? ''],
+      available: () => hasEnv('AZURE_OPENAI_ENDPOINT') && hasEnv('AZURE_OPENAI_DEPLOYMENT_NAME'),
+    },
+    {
+      provider: 'aws_bedrock',
+      models: ['us.anthropic.claude-sonnet-4-5-20250929-v1:0'],
+      available: () =>
+        hasEnv('AWS_REGION') && (hasEnv('AWS_PROFILE') || hasEnv('AWS_ACCESS_KEY_ID')),
+    },
+    {
+      provider: 'gcp_vertex_ai',
+      models: ['gemini-2.5-pro'],
+      available: () => hasEnv('GCP_PROJECT_ID'),
+    },
+    {
+      provider: 'snowflake',
+      models: ['claude-sonnet-4-5'],
+      available: () => hasEnv('SNOWFLAKE_HOST') && hasEnv('SNOWFLAKE_TOKEN'),
+    },
+    {
+      provider: 'venice',
+      models: ['llama-3.3-70b'],
+      available: () => hasEnv('VENICE_API_KEY'),
+    },
+    {
+      provider: 'litellm',
+      models: ['gpt-4o-mini'],
+      available: () => hasEnv('LITELLM_API_KEY'),
+    },
+    {
+      provider: 'sagemaker_tgi',
+      models: ['sagemaker-tgi-endpoint'],
+      available: () => hasEnv('SAGEMAKER_ENDPOINT_NAME') && hasEnv('AWS_REGION'),
+    },
+    {
+      provider: 'github_copilot',
+      models: ['gpt-4.1'],
+      available: () =>
+        hasEnv('GITHUB_COPILOT_TOKEN') ||
+        hasFile(path.join(os.homedir(), '.config/goose/github_copilot_token.json')),
+    },
+    {
+      provider: 'chatgpt_codex',
+      models: ['gpt-5.4'],
+      available: () =>
+        hasEnv('CHATGPT_CODEX_TOKEN') ||
+        hasFile(path.join(os.homedir(), '.config/goose/chatgpt_codex/tokens.json')),
+    },
+    {
+      provider: 'claude-code',
+      models: ['default'],
+      agentic: true,
+      available: () => hasCmd('claude'),
+    },
+    {
+      provider: 'cursor-agent',
+      models: ['auto'],
+      agentic: true,
+      available: () => hasCmd('cursor-agent'),
+    },
+    {
+      provider: 'ollama',
+      models: ['qwen3'],
+      available: () => hasEnv('OLLAMA_HOST') || hasCmd('ollama'),
+    },
+  ];
+}
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+function stripQuotes(s: string): string {
+  if (
+    s.length >= 2 &&
+    ((s.startsWith('"') && s.endsWith('"')) || (s.startsWith("'") && s.endsWith("'")))
+  ) {
+    return s.slice(1, -1);
+  }
+  return s;
+}
+
+function loadDotenv(): void {
+  // Resolve .env from the repository root (two levels up from ui/desktop).
+  const repoRoot = path.resolve(__dirname, '..', '..', '..', '..');
+  const envPath = path.join(repoRoot, '.env');
+  if (!fs.existsSync(envPath)) return;
+  const lines = fs.readFileSync(envPath, 'utf-8').split('\n');
+  for (const line of lines) {
+    const trimmed = line.trim();
+    if (!trimmed || trimmed.startsWith('#')) continue;
+    const eqIdx = trimmed.indexOf('=');
+    if (eqIdx === -1) continue;
+    const key = trimmed.slice(0, eqIdx);
+    const value = stripQuotes(trimmed.slice(eqIdx + 1));
+    if (!(key in process.env)) {
+      process.env[key] = value;
+    }
+  }
+}
+
+function shouldSkipProvider(provider: string): boolean {
+  const skip = process.env.SKIP_PROVIDERS;
+  if (!skip) return false;
+  return skip
+    .split(',')
+    .map((s) => s.trim())
+    .includes(provider);
+}
+
+// ---------------------------------------------------------------------------
+// Build goose binary
+// ---------------------------------------------------------------------------
+
+export function buildGoose(): string {
+  if (!process.env.SKIP_BUILD) {
+    console.error('Building goose...');
+    execSync('cargo build --bin goose', { stdio: 'inherit' });
+    console.error('');
+  } else {
+    console.error('Skipping build (SKIP_BUILD is set)...');
+    console.error('');
+  }
+  return path.resolve(process.cwd(), '..', '..', 'target/debug/goose');
+}
+
+// ---------------------------------------------------------------------------
+// Test case discovery
+// ---------------------------------------------------------------------------
+
+export interface TestCase {
+  provider: string;
+  model: string;
+  available: boolean;
+  flaky: boolean;
+  agentic: boolean;
+  skippedReason?: string;
+}
+
+export function discoverTestCases(options?: { skipAgentic?: boolean }): TestCase[] {
+  loadDotenv();
+  const skipAgentic = options?.skipAgentic ?? false;
+  const providers = getProviders();
+
+  const testCases: TestCase[] = [];
+
+  for (const pc of providers) {
+    const providerAvailable = pc.available();
+    const agentic = pc.agentic ?? false;
+
+    for (const entry of pc.models) {
+      const model = modelName(entry);
+      const flaky = modelFlaky(entry);
+
+      if (!providerAvailable) {
+        testCases.push({
+          provider: pc.provider,
+          model,
+          available: false,
+          flaky,
+          agentic,
+          skippedReason: 'prerequisites not met',
+        });
+      } else if (shouldSkipProvider(pc.provider)) {
+        testCases.push({
+          provider: pc.provider,
+          model,
+          available: false,
+          flaky,
+          agentic,
+          skippedReason: 'SKIP_PROVIDERS',
+        });
+      } else if (skipAgentic && agentic) {
+        testCases.push({
+          provider: pc.provider,
+          model,
+          available: false,
+          flaky,
+          agentic,
+          skippedReason: 'agentic provider skipped in this mode',
+        });
+      } else {
+        testCases.push({
+          provider: pc.provider,
+          model,
+          available: true,
+          flaky,
+          agentic,
+        });
+      }
+    }
+  }
+
+  return testCases;
+}
+
+// ---------------------------------------------------------------------------
+// Test registration helpers
+// ---------------------------------------------------------------------------
+
+type ProviderTestFn = (tc: TestCase) => Promise<void>;
+
+function registerTests(label: string, cases: TestCase[], fn: ProviderTestFn): void {
+  const available = cases.filter((tc) => tc.available && !tc.flaky);
+  const flaky = cases.filter((tc) => tc.available && tc.flaky);
+  const skipped = cases.filter((tc) => !tc.available);
+
+  if (available.length > 0) {
+    test.each(available)(`${label} — $provider / $model`, async (tc) => {
+      await fn(tc);
+    });
+  }
+
+  if (flaky.length > 0) {
+    test.each(flaky)(`${label} — $provider / $model (flaky)`, async (tc) => {
+      try {
+        await fn(tc);
+      } catch (err) {
+        console.warn(`Flaky test ${tc.provider}/${tc.model} failed (allowed): ${err}`);
+      }
+    });
+  }
+
+  if (skipped.length > 0) {
+    test.skip.each(skipped)(`${label} — $provider / $model — $skippedReason`, () => {});
+  }
+}
+
+/**
+ * Build decorator-style test registrars from a set of discovered test cases.
+ *
+ * Usage:
+ *   const { testAll, testAgentic, testNonAgentic } = providerTest(cases);
+ *
+ *   testAll('reads a file', async (tc) => { ... });
+ *   testAgentic('delegates work', async (tc) => { ... });
+ *   testNonAgentic('uses shell tool', async (tc) => { ... });
+ */
+export function providerTest(cases: TestCase[]) {
+  const agentic = cases.filter((tc) => tc.agentic);
+  const nonAgentic = cases.filter((tc) => !tc.agentic);
+
+  return {
+    testAll: (label: string, fn: ProviderTestFn) => registerTests(label, cases, fn),
+    testAgentic: (label: string, fn: ProviderTestFn) => registerTests(label, agentic, fn),
+    testNonAgentic: (label: string, fn: ProviderTestFn) => registerTests(label, nonAgentic, fn),
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Utility: run goose binary and capture output
+// ---------------------------------------------------------------------------
+
+export function runGoose(
+  gooseBin: string,
+  cwd: string,
+  prompt: string,
+  builtins: string,
+  env: Record<string, string>
+): Promise<string> {
+  return new Promise((resolve) => {
+    const child: ChildProcess = spawn(
+      gooseBin,
+      ['run', '--text', prompt, '--with-builtin', builtins],
+      {
+        cwd,
+        env: { ...process.env, ...env },
+        stdio: ['ignore', 'pipe', 'pipe'],
+      }
+    );
+
+    let output = '';
+    child.stdout?.on('data', (d) => {
+      output += String(d);
+    });
+    child.stderr?.on('data', (d) => {
+      output += String(d);
+    });
+
+    child.on('close', () => {
+      resolve(output);
+    });
+
+    child.on('error', (err) => {
+      resolve(`spawn error: ${err.message}`);
+    });
+  });
+}

ui/desktop/vitest.config.ts

@@ -1,7 +1,7 @@
 /// <reference types="vitest" />
-import { defineConfig } from 'vitest/config'
-import react from '@vitejs/plugin-react'
-import { resolve } from 'node:path'
+import { defineConfig } from 'vitest/config';
+import react from '@vitejs/plugin-react';
+import { resolve } from 'node:path';
 
 const cfg = {
   plugins: [react()],
@@ -17,6 +17,6 @@ const cfg = {
     css: true,
     include: ['src/**/*.{test,spec}.{js,jsx,ts,tsx}'],
   },
-} satisfies Record<string, any>
+} satisfies Record<string, any>;
 
-export default defineConfig(cfg as any)
+export default defineConfig(cfg as any);