diff --git a/crates/goose/src/providers/databricks.rs b/crates/goose/src/providers/databricks.rs index 30cd7c0a..ea9d6fbb 100644 --- a/crates/goose/src/providers/databricks.rs +++ b/crates/goose/src/providers/databricks.rs @@ -134,7 +134,6 @@ impl DatabricksProvider { let api_client = ApiClient::with_timeout(host, auth_method, Duration::from_secs(DEFAULT_TIMEOUT_SECS))?; - // Create the provider without the fast model first let mut provider = Self { api_client, auth, @@ -143,31 +142,7 @@ impl DatabricksProvider { retry_config, name: Self::metadata().name, }; - - // Check if the default fast model exists in the workspace - // Generate UUID for this initialization request since no user session exists yet - let session_id = uuid::Uuid::new_v4().to_string(); - let model_with_fast = - if let Ok(Some(models)) = provider.fetch_supported_models(&session_id).await { - if models.contains(&DATABRICKS_DEFAULT_FAST_MODEL.to_string()) { - tracing::debug!( - "Found {} in Databricks workspace, setting as fast model", - DATABRICKS_DEFAULT_FAST_MODEL - ); - model.with_fast(DATABRICKS_DEFAULT_FAST_MODEL.to_string()) - } else { - tracing::debug!( - "{} not found in Databricks workspace, not setting fast model", - DATABRICKS_DEFAULT_FAST_MODEL - ); - model - } - } else { - tracing::debug!("Could not fetch Databricks models, not setting fast model"); - model - }; - - provider.model = model_with_fast; + provider.model = model.with_fast(DATABRICKS_DEFAULT_FAST_MODEL.to_string()); Ok(provider) } diff --git a/ui/desktop/playwright.config.ts b/ui/desktop/playwright.config.ts index 2923eddb..2e1fe4a2 100644 --- a/ui/desktop/playwright.config.ts +++ b/ui/desktop/playwright.config.ts @@ -20,7 +20,7 @@ const config: PlaywrightTestConfig = { screenshot: 'only-on-failure' }, outputDir: 'test-results', - preserveOutput: 'failures-only' + preserveOutput: 'always' }; export default config; \ No newline at end of file diff --git a/ui/desktop/src/main.ts b/ui/desktop/src/main.ts index 10293068..1a0c0ec6 100644 --- a/ui/desktop/src/main.ts +++ b/ui/desktop/src/main.ts @@ -154,8 +154,9 @@ async function configureProxy() { if (started) app.quit(); if (process.env.ENABLE_PLAYWRIGHT) { - console.log('[Main] Enabling Playwright remote debugging on port 9222'); - app.commandLine.appendSwitch('remote-debugging-port', '9222'); + const debugPort = process.env.PLAYWRIGHT_DEBUG_PORT || '9222'; + console.log(`[Main] Enabling Playwright remote debugging on port ${debugPort}`); + app.commandLine.appendSwitch('remote-debugging-port', debugPort); } // In development mode, force registration as the default protocol client diff --git a/ui/desktop/tests/e2e/app.spec.ts b/ui/desktop/tests/e2e/app.spec.ts index d545c0f8..d9f5071a 100644 --- a/ui/desktop/tests/e2e/app.spec.ts +++ b/ui/desktop/tests/e2e/app.spec.ts @@ -1,12 +1,9 @@ -import { test as base, expect } from '@playwright/test'; -import { _electron as electron } from '@playwright/test'; -import { join } from 'path'; -import { spawn, exec } from 'child_process'; -import { promisify } from 'util'; +import { test as base, expect } from './fixtures'; +import { Page } from '@playwright/test'; import { showTestName, clearTestName } from './test-overlay'; +import { join } from 'path'; const { runningQuotes } = require('./basic-mcp'); -const execAsync = promisify(exec); // Define provider interface type Provider = { @@ -28,23 +25,18 @@ const test = base.extend({ provider: [providers[0], { option: true }], // Default to first provider (Databricks) }); -// Store mainWindow reference -let mainWindow; +let mainWindow: Page; -// Add hooks for test name overlay -// eslint-disable-next-line no-empty-pattern -test.beforeEach(async ({ }, testInfo) => { - if (mainWindow) { - // Get a clean test name without the full hierarchy - const testName = testInfo.titlePath[testInfo.titlePath.length - 1]; +test.beforeEach(async ({ goosePage }, testInfo) => { + mainWindow = goosePage; - // Get provider name if we're in a provider suite - const providerSuite = testInfo.titlePath.find(t => t.startsWith('Provider:')); - const providerName = providerSuite ? providerSuite.split(': ')[1] : undefined; + const testName = testInfo.titlePath[testInfo.titlePath.length - 1]; - console.log(`Setting overlay for test: "${testName}"${providerName ? ` (Provider: ${providerName})` : ''}`); - await showTestName(mainWindow, testName, providerName); - } + const providerSuite = testInfo.titlePath.find(t => t.startsWith('Provider:')); + const providerName = providerSuite ? providerSuite.split(': ')[1] : undefined; + + console.log(`Setting overlay for test: "${testName}"${providerName ? ` (Provider: ${providerName})` : ''}`); + await showTestName(mainWindow, testName, providerName); }); test.afterEach(async () => { @@ -116,184 +108,105 @@ async function selectProvider(mainWindow: any, provider: Provider) { return; // Provider is already selected, no need to do anything } - // Check if we need to click "configure other providers (advanced)" button - const configureAdvancedButton = await mainWindow.waitForSelector('h3:has-text("Other providers")', { + // Check if we're on the welcome screen with "Other Providers" section + const otherProvidersSection = await mainWindow.waitForSelector('text="Other Providers"', { timeout: 3000, state: 'visible' }).catch(() => null); - if (configureAdvancedButton) { - console.log('Found "configure other providers" button, clicking it...'); - await configureAdvancedButton.click(); - await mainWindow.waitForTimeout(1500); + if (otherProvidersSection) { + console.log('Found "Other Providers" section, clicking "Go to Provider Settings" link...'); + // Click the "Go to Provider Settings" link (includes arrow →) + const providerSettingsLink = await mainWindow.waitForSelector('button:has-text("Go to Provider Settings")', { + timeout: 3000, + state: 'visible' + }); + await providerSettingsLink.click(); + await mainWindow.waitForTimeout(1000); + + // We should now be in Settings -> Models tab + console.log('Navigated to Provider Settings'); } - // We should now be at provider selection - await mainWindow.waitForSelector('[data-testid="provider-selection-heading"]'); + // Now we should be on the "Other providers" page with provider cards + console.log(`Looking for ${provider.name} provider card...`); + + // Wait for the provider cards to load + await mainWindow.waitForTimeout(1000); + + // Find the Launch button within the specific provider card using its data-testid + console.log(`Looking for ${provider.name} card with Launch button...`); - // Find and verify the provider card container - console.log(`Looking for ${provider.name} card...`); - let providerContainer; try { - providerContainer = await mainWindow.waitForSelector(`[data-testid="provider-card-${provider.name.toLowerCase()}"]`); - expect(await providerContainer.isVisible()).toBe(true); + // Each provider card has data-testid="provider-card-{provider-name-lowercase}" + const providerCardTestId = `provider-card-${provider.name.toLowerCase()}`; + const launchButton = mainWindow.locator(`[data-testid="${providerCardTestId}"] button:has-text("Launch")`); + + await launchButton.waitFor({ state: 'visible', timeout: 5000 }); + console.log(`Found Launch button in ${provider.name} card, clicking it...`); + await launchButton.click(); + await mainWindow.waitForTimeout(1000); + + // Wait for "Choose Model" dialog to appear and select a model + console.log('Waiting for model selection dialog...'); + const chooseModelDialog = await mainWindow.waitForSelector('text="Choose Model"', { + timeout: 5000, + state: 'visible' + }).catch(() => null); + + if (chooseModelDialog) { + console.log('Model selection dialog appeared, waiting for models to load...'); + + // The "Select model" button starts enabled and only disables during loading (UI bug) + // So we wait for a fixed timeout to ensure models are loaded + await mainWindow.waitForTimeout(5000); + console.log('Waited for models to load'); + + const confirmButton = await mainWindow.waitForSelector('button:has-text("Select model")', { + timeout: 5000, + state: 'visible' + }); + + console.log('Clicking "Select model" button'); + await confirmButton.click(); + await mainWindow.waitForTimeout(2000); + } } catch (error) { - console.error(`Provider card not found for ${provider.name}. This could indicate a missing or incorrectly configured provider.`); + console.error(`Failed to find or click Launch button in ${provider.name} card:`, error); throw error; } - // Find the Launch button within the provider container - console.log(`Looking for Launch button in ${provider.name} card...`); - const launchButton = await providerContainer.waitForSelector('[data-testid="provider-launch-button"]'); - expect(await launchButton.isVisible()).toBe(true); + // Navigate to home/chat after provider configuration + console.log('Navigating to home/chat...'); + const homeButton = await mainWindow.waitForSelector('[data-testid="sidebar-home-button"]', { + timeout: 5000 + }).catch(() => null); - // Take screenshot before clicking - await mainWindow.screenshot({ path: `test-results/before-${provider.name.toLowerCase()}-click.png` }); - - // Click the Launch button - await launchButton.click(); + if (homeButton) { + await homeButton.click(); + await mainWindow.waitForTimeout(1000); + } // Wait for chat interface to appear - const chatTextareaAfterClick = await mainWindow.waitForSelector('[data-testid="chat-input"]', + const chatTextareaAfterConfig = await mainWindow.waitForSelector('[data-testid="chat-input"]', { timeout: 10000 }); - expect(await chatTextareaAfterClick.isVisible()).toBe(true); + expect(await chatTextareaAfterConfig.isVisible()).toBe(true); // Take screenshot of chat interface await mainWindow.screenshot({ path: `test-results/chat-interface-${provider.name.toLowerCase()}.png` }); } test.describe('Goose App', () => { - let electronApp; - let appProcess; - - test.beforeAll(async () => { - console.log('Starting Electron app...'); - - // Start the electron-forge process - appProcess = spawn('npm', ['run', 'start-gui'], { - cwd: join(__dirname, '../..'), - stdio: 'pipe', - shell: true, - env: { - ...process.env, - ELECTRON_IS_DEV: '1', - NODE_ENV: 'development', - GOOSE_ALLOWLIST_BYPASS: 'true', - } - }); - - // Log process output - appProcess.stdout.on('data', (data) => { - console.log('App stdout:', data.toString()); - }); - - appProcess.stderr.on('data', (data) => { - console.log('App stderr:', data.toString()); - }); - - // Wait a bit for the app to start - console.log('Waiting for app to start...'); - await new Promise(resolve => setTimeout(resolve, 5000)); - - // Launch Electron for testing - electronApp = await electron.launch({ - args: ['.vite/build/main.js'], - cwd: join(__dirname, '../..'), - env: { - ...process.env, - ELECTRON_IS_DEV: '1', - NODE_ENV: 'development', - }, - recordVideo: { - dir: 'test-results/videos/', - size: { width: 620, height: 680 } - } - }); - - // Get the main window once for all tests - mainWindow = await electronApp.firstWindow(); - await mainWindow.waitForLoadState('domcontentloaded'); - - // Try to wait for networkidle, but don't fail if it times out due to MCP activity - try { - await mainWindow.waitForLoadState('networkidle', { timeout: 10000 }); - } catch (error) { - console.log('NetworkIdle timeout (likely due to MCP activity), continuing with test...'); - } - - // Wait for React app to be ready by checking for the root element to have content - await mainWindow.waitForFunction(() => { - const root = document.getElementById('root'); - return root && root.children.length > 0; - }); - - // Wait for any animations to complete - await mainWindow.waitForTimeout(2000); - - // Take a screenshot to debug what's on the screen - await mainWindow.screenshot({ path: 'test-results/initial-load.png' }); - - // Debug: print out the page content - const content = await mainWindow.content(); - console.log('Page content:', content); - }); - - test.afterAll(async () => { - console.log('Final cleanup...'); - - // Close the test instance - if (electronApp) { - await electronApp.close().catch(console.error); - } - - // Kill any remaining electron processes - try { - if (process.platform === 'win32') { - await execAsync('taskkill /F /IM electron.exe'); - } else { - await execAsync('pkill -f electron || true'); - } - } catch (error) { - if (!error.message?.includes('no process found')) { - console.error('Error killing electron processes:', error); - } - } - - // Kill any remaining npm processes from start-gui - try { - if (process.platform === 'win32') { - await execAsync('taskkill /F /IM node.exe'); - } else { - await execAsync('pkill -f "start-gui" || true'); - } - } catch (error) { - if (!error.message?.includes('no process found')) { - console.error('Error killing npm processes:', error); - } - } - - // Kill the specific npm process if it's still running - try { - if (appProcess && appProcess.pid) { - process.kill(appProcess.pid); - } - } catch (error) { - if (error.code !== 'ESRCH') { - console.error('Error killing npm process:', error); - } - } - }); + // No need for beforeAll/afterAll - the fixture handles app launch and cleanup! test.describe('General UI', () => { test('dark mode toggle', async () => { console.log('Testing dark mode toggle...'); - const chatTextarea = await mainWindow.waitForSelector('[data-testid="chat-input"]', { - timeout: 2000 - }).catch(() => null); - if (!chatTextarea) { - await selectProvider(mainWindow, providers[0]); - } + // Assume the app is already configured and wait for chat input + await mainWindow.waitForSelector('[data-testid="chat-input"]', { + timeout: 10000 + }); // Navigate to Settings via sidebar const settingsButton = await mainWindow.waitForSelector('[data-testid="sidebar-settings-button"]', { @@ -356,8 +269,8 @@ test.describe('Goose App', () => { for (const provider of providers) { test.describe(`Provider: ${provider.name}`, () => { - test.beforeAll(async () => { - // Select the provider once before all tests for this provider + test.beforeEach(async () => { + // Select the provider before each test for this provider await selectProvider(mainWindow, provider); }); @@ -378,19 +291,19 @@ test.describe('Goose App', () => { // Send message await chatInput.press('Enter'); - // Wait for loading indicator to appear - console.log('Waiting for loading indicator...'); - const loadingGoose = await mainWindow.waitForSelector('[data-testid="loading-indicator"]', - { timeout: 2000 }); - expect(await loadingGoose.isVisible()).toBe(true); - - // Take screenshot of loading state - await mainWindow.screenshot({ path: `test-results/${provider.name.toLowerCase()}-loading-state.png` }); - - // Wait for loading indicator to disappear + // Wait for loading indicator to appear and then disappear console.log('Waiting for response...'); - await mainWindow.waitForSelector('[data-testid="loading-indicator"]', - { state: 'hidden', timeout: 30000 }); + await mainWindow.waitForSelector('[data-testid="loading-indicator"]', { + state: 'visible', + timeout: 5000 + }); + console.log('Loading indicator appeared'); + + await mainWindow.waitForSelector('[data-testid="loading-indicator"]', { + state: 'hidden', + timeout: 30000 + }); + console.log('Loading indicator disappeared'); // Get the latest response const response = await mainWindow.locator('[data-testid="message-container"]').last(); @@ -433,9 +346,10 @@ test.describe('Goose App', () => { // Take screenshot of chat history await mainWindow.screenshot({ path: `test-results/${provider.name.toLowerCase()}-chat-history.png` }); - // Test command history (up arrow) - await chatInput.press('Control+ArrowUp'); - const inputValue = await chatInput.inputValue(); + // Test command history (up arrow) - re-query for the input since the element may have been re-rendered + const chatInputForHistory = await mainWindow.waitForSelector('[data-testid="chat-input"]'); + await chatInputForHistory.press('Control+ArrowUp'); + const inputValue = await chatInputForHistory.inputValue(); expect(inputValue).toBe('What is 2+2?'); }); }); diff --git a/ui/desktop/tests/e2e/context-management.spec.ts b/ui/desktop/tests/e2e/context-management.spec.ts index 7b003c51..28121950 100644 --- a/ui/desktop/tests/e2e/context-management.spec.ts +++ b/ui/desktop/tests/e2e/context-management.spec.ts @@ -1,74 +1,42 @@ -import { test, expect, Page } from '@playwright/test'; -import { ElectronApplication, _electron as electron } from 'playwright'; -import path from 'path'; - -let electronApp: ElectronApplication; -let page: Page; +import { test, expect } from './fixtures'; test.describe('Context Management E2E Tests', () => { - test.beforeAll(async () => { - // Launch Electron app - electronApp = await electron.launch({ - args: [path.join(__dirname, '../../.vite/build/main.js')], - env: { - ...process.env, - NODE_ENV: 'test', - GOOSE_TEST_MODE: 'true', - }, - }); - - // Get the main window - page = await electronApp.firstWindow(); - await page.waitForLoadState('domcontentloaded'); - - // Wait for the app to be ready - await page.waitForSelector('[data-testid="chat-input"]', { timeout: 10000 }); + test.beforeEach(async ({ goosePage }) => { + // Ensure the app is ready before each test + await goosePage.waitForSelector('[data-testid="chat-input"]', { timeout: 10000 }); }); - test.afterAll(async () => { - if (electronApp) { - await electronApp.close(); - } - }); - - test.beforeEach(async () => { - // Reset to a clean state before each test - await page.reload(); - await page.waitForLoadState('domcontentloaded'); - await page.waitForSelector('[data-testid="chat-input"]', { timeout: 10000 }); - }); - - test('should show context window alert when tokens are being used', async () => { + test('should show context window alert when tokens are being used', async ({ goosePage }) => { // Type a message to generate some token usage - const chatInput = page.locator('[data-testid="chat-input"]'); + const chatInput = goosePage.locator('[data-testid="chat-input"]'); await chatInput.fill('Hello, this is a test message to generate some token usage.'); // Submit the message - await page.keyboard.press('Enter'); + await goosePage.keyboard.press('Enter'); // Wait for response and check for context window alert - await page.waitForSelector('[data-testid="alert-indicator"]', { timeout: 15000 }); + await goosePage.waitForSelector('[data-testid="alert-indicator"]', { timeout: 15000 }); // Click on the alert indicator to open the popover - await page.click('[data-testid="alert-indicator"]'); + await goosePage.click('[data-testid="alert-indicator"]'); // Verify the context window alert is shown - const alertBox = page.locator('[role="alert"]'); + const alertBox = goosePage.locator('[role="alert"]'); await expect(alertBox).toBeVisible(); await expect(alertBox).toContainText('Context window'); // Verify progress bar is shown - const progressBar = page.locator('[role="progressbar"]'); + const progressBar = goosePage.locator('[role="progressbar"]'); await expect(progressBar).toBeVisible(); // Verify compact button is present - const compactButton = page.locator('text=Compact now'); + const compactButton = goosePage.locator('text=Compact now'); await expect(compactButton).toBeVisible(); }); - test('should perform manual compaction when compact button is clicked', async () => { + test('should perform manual compaction when compact button is clicked', async ({ goosePage }) => { // First, generate enough conversation to have tokens - const chatInput = page.locator('[data-testid="chat-input"]'); + const chatInput = goosePage.locator('[data-testid="chat-input"]'); // Send multiple messages to build up context const messages = [ @@ -80,42 +48,42 @@ test.describe('Context Management E2E Tests', () => { for (const message of messages) { await chatInput.fill(message); - await page.keyboard.press('Enter'); + await goosePage.keyboard.press('Enter'); // Wait for response before sending next message - await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); - await page.waitForTimeout(1000); // Brief pause between messages + await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); + await goosePage.waitForTimeout(1000); // Brief pause between messages } // Open the alert popover - await page.waitForSelector('[data-testid="alert-indicator"]', { timeout: 15000 }); - await page.click('[data-testid="alert-indicator"]'); + await goosePage.waitForSelector('[data-testid="alert-indicator"]', { timeout: 15000 }); + await goosePage.click('[data-testid="alert-indicator"]'); // Click the compact button - const compactButton = page.locator('text=Compact now'); + const compactButton = goosePage.locator('text=Compact now'); await expect(compactButton).toBeVisible(); await compactButton.click(); // Verify compaction loading state - const loadingGoose = page.locator('[data-testid="loading-goose"]'); + const loadingGoose = goosePage.locator('[data-testid="loading-goose"]'); await expect(loadingGoose).toBeVisible(); await expect(loadingGoose).toContainText('goose is compacting the conversation...'); // Wait for compaction to complete - await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); + await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); // Verify compaction marker appears - const compactionMarker = page.locator('text=Conversation compacted and summarized'); + const compactionMarker = goosePage.locator('text=Conversation compacted and summarized'); await expect(compactionMarker).toBeVisible(); // Verify alert popover is closed after compaction - const alertBox = page.locator('[role="alert"]'); + const alertBox = goosePage.locator('[role="alert"]'); await expect(alertBox).not.toBeVisible(); }); - test('should allow scrolling to see past messages after compaction', async () => { + test('should allow scrolling to see past messages after compaction', async ({ goosePage }) => { // Generate conversation content - const chatInput = page.locator('[data-testid="chat-input"]'); + const chatInput = goosePage.locator('[data-testid="chat-input"]'); const testMessages = [ 'First message in the conversation', @@ -126,42 +94,42 @@ test.describe('Context Management E2E Tests', () => { // Send messages and store their content for verification for (const message of testMessages) { await chatInput.fill(message); - await page.keyboard.press('Enter'); - await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); - await page.waitForTimeout(1000); + await goosePage.keyboard.press('Enter'); + await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); + await goosePage.waitForTimeout(1000); } // Perform manual compaction - await page.waitForSelector('[data-testid="alert-indicator"]', { timeout: 15000 }); - await page.click('[data-testid="alert-indicator"]'); - await page.click('text=Compact now'); + await goosePage.waitForSelector('[data-testid="alert-indicator"]', { timeout: 15000 }); + await goosePage.click('[data-testid="alert-indicator"]'); + await goosePage.click('text=Compact now'); // Wait for compaction to complete - await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); - await expect(page.locator('text=Conversation compacted and summarized')).toBeVisible(); + await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); + await expect(goosePage.locator('text=Conversation compacted and summarized')).toBeVisible(); // Scroll up to verify past messages are still visible - const chatContainer = page.locator('[data-testid="chat-container"]'); + const chatContainer = goosePage.locator('[data-testid="chat-container"]'); await chatContainer.hover(); // Scroll up multiple times to reach earlier messages for (let i = 0; i < 5; i++) { - await page.mouse.wheel(0, -500); - await page.waitForTimeout(200); + await goosePage.mouse.wheel(0, -500); + await goosePage.waitForTimeout(200); } // Verify that we can still see the original messages // Note: The exact messages might be in ancestor messages, so we check for partial content - const messageElements = page.locator('[data-testid="message"]'); + const messageElements = goosePage.locator('[data-testid="message"]'); const messageCount = await messageElements.count(); // Should have more than just the compaction marker and summary expect(messageCount).toBeGreaterThan(2); }); - test('should handle compaction errors gracefully', async () => { + test('should handle compaction errors gracefully', async ({ goosePage }) => { // Mock a backend error by intercepting the compaction request - await page.route('**/api/sessions/*/manage-context', async (route) => { + await goosePage.route('**/api/sessions/*/manage-context', async (route) => { await route.fulfill({ status: 500, contentType: 'application/json', @@ -170,126 +138,126 @@ test.describe('Context Management E2E Tests', () => { }); // Generate some conversation - const chatInput = page.locator('[data-testid="chat-input"]'); + const chatInput = goosePage.locator('[data-testid="chat-input"]'); await chatInput.fill('Test message for error handling'); - await page.keyboard.press('Enter'); + await goosePage.keyboard.press('Enter'); - await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); + await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); // Attempt compaction - await page.waitForSelector('[data-testid="alert-indicator"]', { timeout: 15000 }); - await page.click('[data-testid="alert-indicator"]'); - await page.click('text=Compact now'); + await goosePage.waitForSelector('[data-testid="alert-indicator"]', { timeout: 15000 }); + await goosePage.click('[data-testid="alert-indicator"]'); + await goosePage.click('text=Compact now'); // Wait for compaction to fail - await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); + await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); // Verify error message appears - const errorMarker = page.locator('text=Compaction failed. Please try again or start a new session.'); + const errorMarker = goosePage.locator('text=Compaction failed. Please try again or start a new session.'); await expect(errorMarker).toBeVisible(); }); - test('should not show compaction UI when no tokens are used', async () => { - // On a fresh page with no messages, there should be no alert indicator - const alertIndicator = page.locator('[data-testid="alert-indicator"]'); + test('should not show compaction UI when no tokens are used', async ({ goosePage }) => { + // On a fresh goosePage with no messages, there should be no alert indicator + const alertIndicator = goosePage.locator('[data-testid="alert-indicator"]'); await expect(alertIndicator).not.toBeVisible(); // The chat input should be available but no context alerts - const chatInput = page.locator('[data-testid="chat-input"]'); + const chatInput = goosePage.locator('[data-testid="chat-input"]'); await expect(chatInput).toBeVisible(); }); - test('should maintain conversation flow after compaction', async () => { + test('should maintain conversation flow after compaction', async ({ goosePage }) => { // Generate initial conversation - const chatInput = page.locator('[data-testid="chat-input"]'); + const chatInput = goosePage.locator('[data-testid="chat-input"]'); await chatInput.fill('What is React?'); - await page.keyboard.press('Enter'); - await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); + await goosePage.keyboard.press('Enter'); + await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); await chatInput.fill('Can you give me an example?'); - await page.keyboard.press('Enter'); - await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); + await goosePage.keyboard.press('Enter'); + await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); // Perform compaction - await page.waitForSelector('[data-testid="alert-indicator"]', { timeout: 15000 }); - await page.click('[data-testid="alert-indicator"]'); - await page.click('text=Compact now'); - await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); + await goosePage.waitForSelector('[data-testid="alert-indicator"]', { timeout: 15000 }); + await goosePage.click('[data-testid="alert-indicator"]'); + await goosePage.click('text=Compact now'); + await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); // Verify compaction marker - await expect(page.locator('text=Conversation compacted and summarized')).toBeVisible(); + await expect(goosePage.locator('text=Conversation compacted and summarized')).toBeVisible(); // Continue conversation after compaction await chatInput.fill('Thank you, that was helpful. What about Vue.js?'); - await page.keyboard.press('Enter'); + await goosePage.keyboard.press('Enter'); // Verify that the conversation continues normally - await page.waitForSelector('[data-testid="loading-goose"]', { timeout: 30000 }); - await expect(page.locator('[data-testid="loading-goose"]')).toBeVisible(); + await goosePage.waitForSelector('[data-testid="loading-goose"]', { timeout: 30000 }); + await expect(goosePage.locator('[data-testid="loading-goose"]')).toBeVisible(); // Wait for response - await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); + await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); // Verify new message appears after compaction - const messages = page.locator('[data-testid="message"]'); + const messages = goosePage.locator('[data-testid="message"]'); const messageCount = await messages.count(); expect(messageCount).toBeGreaterThan(1); // Should have compaction marker + new messages }); - test('should show appropriate loading states during compaction', async () => { + test('should show appropriate loading states during compaction', async ({ goosePage }) => { // Generate conversation - const chatInput = page.locator('[data-testid="chat-input"]'); + const chatInput = goosePage.locator('[data-testid="chat-input"]'); await chatInput.fill('Test message for loading state verification'); - await page.keyboard.press('Enter'); - await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); + await goosePage.keyboard.press('Enter'); + await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); // Start compaction - await page.waitForSelector('[data-testid="alert-indicator"]', { timeout: 15000 }); - await page.click('[data-testid="alert-indicator"]'); - await page.click('text=Compact now'); + await goosePage.waitForSelector('[data-testid="alert-indicator"]', { timeout: 15000 }); + await goosePage.click('[data-testid="alert-indicator"]'); + await goosePage.click('text=Compact now'); // Verify loading state immediately after clicking compact - const loadingGoose = page.locator('[data-testid="loading-goose"]'); + const loadingGoose = goosePage.locator('[data-testid="loading-goose"]'); await expect(loadingGoose).toBeVisible(); await expect(loadingGoose).toContainText('goose is compacting the conversation...'); // Verify chat input is disabled during compaction - const submitButton = page.locator('[data-testid="submit-button"]'); + const submitButton = goosePage.locator('[data-testid="submit-button"]'); await expect(submitButton).toBeDisabled(); // Wait for compaction to complete - await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); + await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); // Verify chat input is re-enabled after compaction await expect(submitButton).toBeEnabled(); }); - test('should handle multiple rapid compaction attempts', async () => { + test('should handle multiple rapid compaction attempts', async ({ goosePage }) => { // Generate conversation - const chatInput = page.locator('[data-testid="chat-input"]'); + const chatInput = goosePage.locator('[data-testid="chat-input"]'); await chatInput.fill('Test message for rapid compaction test'); - await page.keyboard.press('Enter'); - await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); + await goosePage.keyboard.press('Enter'); + await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); // Open alert and try to click compact multiple times rapidly - await page.waitForSelector('[data-testid="alert-indicator"]', { timeout: 15000 }); - await page.click('[data-testid="alert-indicator"]'); + await goosePage.waitForSelector('[data-testid="alert-indicator"]', { timeout: 15000 }); + await goosePage.click('[data-testid="alert-indicator"]'); - const compactButton = page.locator('text=Compact now'); + const compactButton = goosePage.locator('text=Compact now'); await expect(compactButton).toBeVisible(); // Click multiple times rapidly await compactButton.click(); // The alert should be hidden immediately after first click - const alertBox = page.locator('[role="alert"]'); + const alertBox = goosePage.locator('[role="alert"]'); await expect(alertBox).not.toBeVisible(); // Verify only one compaction occurs - await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); + await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); - const compactionMarkers = page.locator('text=Conversation compacted and summarized'); + const compactionMarkers = goosePage.locator('text=Conversation compacted and summarized'); await expect(compactionMarkers).toHaveCount(1); }); }); diff --git a/ui/desktop/tests/e2e/enhanced-context-management.spec.ts b/ui/desktop/tests/e2e/enhanced-context-management.spec.ts index 094b6b75..f59a3b61 100644 --- a/ui/desktop/tests/e2e/enhanced-context-management.spec.ts +++ b/ui/desktop/tests/e2e/enhanced-context-management.spec.ts @@ -1,103 +1,71 @@ -import { test, expect, Page } from '@playwright/test'; -import { ElectronApplication, _electron as electron } from 'playwright'; -import path from 'path'; - -let electronApp: ElectronApplication; -let page: Page; +import { test, expect } from './fixtures'; test.describe('Enhanced Context Management E2E Tests', () => { - test.beforeAll(async () => { - // Launch Electron app - electronApp = await electron.launch({ - args: [path.join(__dirname, '../../.vite/build/main.js')], - env: { - ...process.env, - NODE_ENV: 'test', - GOOSE_TEST_MODE: 'true', - }, - }); - - // Get the main window - page = await electronApp.firstWindow(); - await page.waitForLoadState('domcontentloaded'); - - // Wait for the app to be ready - await page.waitForSelector('[data-testid="chat-input"]', { timeout: 10000 }); - }); - - test.afterAll(async () => { - if (electronApp) { - await electronApp.close(); - } - }); - - test.beforeEach(async () => { - // Reset to a clean state before each test - await page.reload(); - await page.waitForLoadState('domcontentloaded'); - await page.waitForSelector('[data-testid="chat-input"]', { timeout: 10000 }); + test.beforeEach(async ({ goosePage }) => { + // Ensure the app is ready before each test + await goosePage.waitForSelector('[data-testid="chat-input"]', { timeout: 10000 }); }); test.describe('Context Window Alert System', () => { - test('should show context window alert only when tokens are being used', async () => { + test('should show context window alert only when tokens are being used', async ({ goosePage }) => { // Initially, no alert should be visible - const alertIndicator = page.locator('[data-testid="alert-indicator"]'); + const alertIndicator = goosePage.locator('[data-testid="alert-indicator"]'); await expect(alertIndicator).not.toBeVisible(); // Type and send a message to generate token usage - const chatInput = page.locator('[data-testid="chat-input"]'); + const chatInput = goosePage.locator('[data-testid="chat-input"]'); await chatInput.fill('Hello, this is a test message to generate some token usage.'); - await page.keyboard.press('Enter'); + await goosePage.keyboard.press('Enter'); // Wait for response and check for context window alert - await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); - await page.waitForSelector('[data-testid="alert-indicator"]', { timeout: 15000 }); + await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); + await goosePage.waitForSelector('[data-testid="alert-indicator"]', { timeout: 15000 }); // Click on the alert indicator to open the popover - await page.click('[data-testid="alert-indicator"]'); + await goosePage.click('[data-testid="alert-indicator"]'); // Verify the context window alert is shown - const alertBox = page.locator('[role="alert"]'); + const alertBox = goosePage.locator('[role="alert"]'); await expect(alertBox).toBeVisible(); await expect(alertBox).toContainText('Context window'); // Verify progress bar is shown - const progressBar = page.locator('[role="progressbar"]'); + const progressBar = goosePage.locator('[role="progressbar"]'); await expect(progressBar).toBeVisible(); // Verify compact button is present - const compactButton = page.locator('text=Compact now'); + const compactButton = goosePage.locator('text=Compact now'); await expect(compactButton).toBeVisible(); }); - test('should update progress bar as conversation grows', async () => { - const chatInput = page.locator('[data-testid="chat-input"]'); + test('should update progress bar as conversation grows', async ({ goosePage }) => { + const chatInput = goosePage.locator('[data-testid="chat-input"]'); // Send first message await chatInput.fill('First message'); - await page.keyboard.press('Enter'); - await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); + await goosePage.keyboard.press('Enter'); + await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); // Get initial progress - await page.waitForSelector('[data-testid="alert-indicator"]', { timeout: 15000 }); - await page.click('[data-testid="alert-indicator"]'); + await goosePage.waitForSelector('[data-testid="alert-indicator"]', { timeout: 15000 }); + await goosePage.click('[data-testid="alert-indicator"]'); - const progressText1 = await page.locator('[role="alert"]').textContent(); + const progressText1 = await goosePage.locator('[role="alert"]').textContent(); const match1 = progressText1?.match(/(\d+(?:,\d+)*)\s*\/\s*(\d+(?:,\d+)*)/); const initialTokens = match1 ? parseInt(match1[1].replace(/,/g, '')) : 0; // Close the alert popover - await page.keyboard.press('Escape'); + await goosePage.keyboard.press('Escape'); // Send second message await chatInput.fill('Second message with more content to increase token usage significantly'); - await page.keyboard.press('Enter'); - await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); + await goosePage.keyboard.press('Enter'); + await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); // Get updated progress - await page.click('[data-testid="alert-indicator"]'); + await goosePage.click('[data-testid="alert-indicator"]'); - const progressText2 = await page.locator('[role="alert"]').textContent(); + const progressText2 = await goosePage.locator('[role="alert"]').textContent(); const match2 = progressText2?.match(/(\d+(?:,\d+)*)\s*\/\s*(\d+(?:,\d+)*)/); const updatedTokens = match2 ? parseInt(match2[1].replace(/,/g, '')) : 0; @@ -107,8 +75,8 @@ test.describe('Enhanced Context Management E2E Tests', () => { }); test.describe('Manual Compaction Workflow', () => { - test('should perform complete manual compaction workflow', async () => { - const chatInput = page.locator('[data-testid="chat-input"]'); + test('should perform complete manual compaction workflow', async ({ goosePage }) => { + const chatInput = goosePage.locator('[data-testid="chat-input"]'); // Build up conversation with multiple exchanges const messages = [ @@ -120,64 +88,64 @@ test.describe('Enhanced Context Management E2E Tests', () => { for (const message of messages) { await chatInput.fill(message); - await page.keyboard.press('Enter'); - await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); - await page.waitForTimeout(1000); // Brief pause between messages + await goosePage.keyboard.press('Enter'); + await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); + await goosePage.waitForTimeout(1000); // Brief pause between messages } // Open the alert popover and initiate compaction - await page.waitForSelector('[data-testid="alert-indicator"]', { timeout: 15000 }); - await page.click('[data-testid="alert-indicator"]'); + await goosePage.waitForSelector('[data-testid="alert-indicator"]', { timeout: 15000 }); + await goosePage.click('[data-testid="alert-indicator"]'); - const compactButton = page.locator('text=Compact now'); + const compactButton = goosePage.locator('text=Compact now'); await expect(compactButton).toBeVisible(); await compactButton.click(); // Verify alert popover closes immediately - const alertBox = page.locator('[role="alert"]'); + const alertBox = goosePage.locator('[role="alert"]'); await expect(alertBox).not.toBeVisible(); // Verify compaction loading state - const loadingGoose = page.locator('[data-testid="loading-goose"]'); + const loadingGoose = goosePage.locator('[data-testid="loading-goose"]'); await expect(loadingGoose).toBeVisible(); await expect(loadingGoose).toContainText('goose is compacting the conversation...'); // Wait for compaction to complete - await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); + await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); // Verify compaction marker appears - const compactionMarker = page.locator('text=Conversation compacted and summarized'); + const compactionMarker = goosePage.locator('text=Conversation compacted and summarized'); await expect(compactionMarker).toBeVisible(); // Verify chat input is re-enabled - const submitButton = page.locator('[data-testid="submit-button"]'); + const submitButton = goosePage.locator('[data-testid="submit-button"]'); await expect(submitButton).toBeEnabled(); }); - test('should hide alert indicator after successful compaction', async () => { - const chatInput = page.locator('[data-testid="chat-input"]'); + test('should hide alert indicator after successful compaction', async ({ goosePage }) => { + const chatInput = goosePage.locator('[data-testid="chat-input"]'); // Generate conversation await chatInput.fill('Test message for compaction'); - await page.keyboard.press('Enter'); - await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); + await goosePage.keyboard.press('Enter'); + await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); // Perform compaction - await page.waitForSelector('[data-testid="alert-indicator"]', { timeout: 15000 }); - await page.click('[data-testid="alert-indicator"]'); - await page.click('text=Compact now'); + await goosePage.waitForSelector('[data-testid="alert-indicator"]', { timeout: 15000 }); + await goosePage.click('[data-testid="alert-indicator"]'); + await goosePage.click('text=Compact now'); // Wait for compaction to complete - await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); + await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); // Verify alert indicator is no longer visible (or shows reduced token count) - const alertIndicator = page.locator('[data-testid="alert-indicator"]'); + const alertIndicator = goosePage.locator('[data-testid="alert-indicator"]'); // Either the indicator is hidden, or if visible, the token count should be much lower const isVisible = await alertIndicator.isVisible(); if (isVisible) { await alertIndicator.click(); - const alertContent = await page.locator('[role="alert"]').textContent(); + const alertContent = await goosePage.locator('[role="alert"]').textContent(); const match = alertContent?.match(/(\d+(?:,\d+)*)\s*\/\s*(\d+(?:,\d+)*)/); const currentTokens = match ? parseInt(match[1].replace(/,/g, '')) : 0; @@ -186,42 +154,42 @@ test.describe('Enhanced Context Management E2E Tests', () => { } }); - test('should prevent multiple simultaneous compaction attempts', async () => { - const chatInput = page.locator('[data-testid="chat-input"]'); + test('should prevent multiple simultaneous compaction attempts', async ({ goosePage }) => { + const chatInput = goosePage.locator('[data-testid="chat-input"]'); // Generate conversation await chatInput.fill('Test message for multiple compaction prevention'); - await page.keyboard.press('Enter'); - await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); + await goosePage.keyboard.press('Enter'); + await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); // Open alert and click compact button - await page.waitForSelector('[data-testid="alert-indicator"]', { timeout: 15000 }); - await page.click('[data-testid="alert-indicator"]'); + await goosePage.waitForSelector('[data-testid="alert-indicator"]', { timeout: 15000 }); + await goosePage.click('[data-testid="alert-indicator"]'); - const compactButton = page.locator('text=Compact now'); + const compactButton = goosePage.locator('text=Compact now'); await expect(compactButton).toBeVisible(); await compactButton.click(); // Alert should close immediately, preventing further clicks - const alertBox = page.locator('[role="alert"]'); + const alertBox = goosePage.locator('[role="alert"]'); await expect(alertBox).not.toBeVisible(); // Verify loading state appears - const loadingGoose = page.locator('[data-testid="loading-goose"]'); + const loadingGoose = goosePage.locator('[data-testid="loading-goose"]'); await expect(loadingGoose).toBeVisible(); // Wait for compaction to complete - await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); + await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); // Verify only one compaction marker exists - const compactionMarkers = page.locator('text=Conversation compacted and summarized'); + const compactionMarkers = goosePage.locator('text=Conversation compacted and summarized'); await expect(compactionMarkers).toHaveCount(1); }); }); test.describe('Post-Compaction Behavior', () => { - test('should allow scrolling to view ancestor messages after compaction', async () => { - const chatInput = page.locator('[data-testid="chat-input"]'); + test('should allow scrolling to view ancestor messages after compaction', async ({ goosePage }) => { + const chatInput = goosePage.locator('[data-testid="chat-input"]'); // Create identifiable messages const testMessages = [ @@ -233,98 +201,98 @@ test.describe('Enhanced Context Management E2E Tests', () => { // Send messages for (const message of testMessages) { await chatInput.fill(message); - await page.keyboard.press('Enter'); - await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); - await page.waitForTimeout(1000); + await goosePage.keyboard.press('Enter'); + await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); + await goosePage.waitForTimeout(1000); } // Perform compaction - await page.waitForSelector('[data-testid="alert-indicator"]', { timeout: 15000 }); - await page.click('[data-testid="alert-indicator"]'); - await page.click('text=Compact now'); - await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); + await goosePage.waitForSelector('[data-testid="alert-indicator"]', { timeout: 15000 }); + await goosePage.click('[data-testid="alert-indicator"]'); + await goosePage.click('text=Compact now'); + await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); // Verify compaction marker is visible - await expect(page.locator('text=Conversation compacted and summarized')).toBeVisible(); + await expect(goosePage.locator('text=Conversation compacted and summarized')).toBeVisible(); // Scroll up to find ancestor messages - const chatContainer = page.locator('[data-testid="chat-container"]'); + const chatContainer = goosePage.locator('[data-testid="chat-container"]'); await chatContainer.hover(); // Scroll up multiple times for (let i = 0; i < 10; i++) { - await page.mouse.wheel(0, -500); - await page.waitForTimeout(100); + await goosePage.mouse.wheel(0, -500); + await goosePage.waitForTimeout(100); } // Check if we can find at least one of our original messages - const hasFirstMessage = await page.locator('text=FIRST_UNIQUE_MESSAGE').isVisible(); - const hasSecondMessage = await page.locator('text=SECOND_UNIQUE_MESSAGE').isVisible(); - const hasThirdMessage = await page.locator('text=THIRD_UNIQUE_MESSAGE').isVisible(); + const hasFirstMessage = await goosePage.locator('text=FIRST_UNIQUE_MESSAGE').isVisible(); + const hasSecondMessage = await goosePage.locator('text=SECOND_UNIQUE_MESSAGE').isVisible(); + const hasThirdMessage = await goosePage.locator('text=THIRD_UNIQUE_MESSAGE').isVisible(); // At least one original message should be visible in the ancestor messages expect(hasFirstMessage || hasSecondMessage || hasThirdMessage).toBe(true); }); - test('should continue conversation normally after compaction', async () => { - const chatInput = page.locator('[data-testid="chat-input"]'); + test('should continue conversation normally after compaction', async ({ goosePage }) => { + const chatInput = goosePage.locator('[data-testid="chat-input"]'); // Generate initial conversation await chatInput.fill('What is TypeScript?'); - await page.keyboard.press('Enter'); - await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); + await goosePage.keyboard.press('Enter'); + await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); await chatInput.fill('Can you give me an example?'); - await page.keyboard.press('Enter'); - await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); + await goosePage.keyboard.press('Enter'); + await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); // Perform compaction - await page.waitForSelector('[data-testid="alert-indicator"]', { timeout: 15000 }); - await page.click('[data-testid="alert-indicator"]'); - await page.click('text=Compact now'); - await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); + await goosePage.waitForSelector('[data-testid="alert-indicator"]', { timeout: 15000 }); + await goosePage.click('[data-testid="alert-indicator"]'); + await goosePage.click('text=Compact now'); + await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); // Verify compaction completed - await expect(page.locator('text=Conversation compacted and summarized')).toBeVisible(); + await expect(goosePage.locator('text=Conversation compacted and summarized')).toBeVisible(); // Continue conversation after compaction await chatInput.fill('POST_COMPACTION_MESSAGE: Thank you, what about React?'); - await page.keyboard.press('Enter'); + await goosePage.keyboard.press('Enter'); // Verify conversation continues normally - await expect(page.locator('[data-testid="loading-goose"]')).toBeVisible(); - await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); + await expect(goosePage.locator('[data-testid="loading-goose"]')).toBeVisible(); + await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); // Verify the new message appears - await expect(page.locator('text=POST_COMPACTION_MESSAGE')).toBeVisible(); + await expect(goosePage.locator('text=POST_COMPACTION_MESSAGE')).toBeVisible(); // Verify we get a response - const messages = page.locator('[data-testid="message"]'); + const messages = goosePage.locator('[data-testid="message"]'); const messageCount = await messages.count(); expect(messageCount).toBeGreaterThan(2); // Should have compaction marker + new messages }); - test('should maintain proper message ordering after compaction', async () => { - const chatInput = page.locator('[data-testid="chat-input"]'); + test('should maintain proper message ordering after compaction', async ({ goosePage }) => { + const chatInput = goosePage.locator('[data-testid="chat-input"]'); // Generate conversation await chatInput.fill('First question about programming'); - await page.keyboard.press('Enter'); - await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); + await goosePage.keyboard.press('Enter'); + await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); // Perform compaction - await page.waitForSelector('[data-testid="alert-indicator"]', { timeout: 15000 }); - await page.click('[data-testid="alert-indicator"]'); - await page.click('text=Compact now'); - await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); + await goosePage.waitForSelector('[data-testid="alert-indicator"]', { timeout: 15000 }); + await goosePage.click('[data-testid="alert-indicator"]'); + await goosePage.click('text=Compact now'); + await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); // Send new message after compaction await chatInput.fill('NEW_MESSAGE_AFTER_COMPACTION'); - await page.keyboard.press('Enter'); - await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); + await goosePage.keyboard.press('Enter'); + await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); // Verify message order: compaction marker should come before new messages - const allMessages = page.locator('[data-testid="message"]'); + const allMessages = goosePage.locator('[data-testid="message"]'); const messageTexts = await allMessages.allTextContents(); const compactionIndex = messageTexts.findIndex(text => @@ -340,9 +308,9 @@ test.describe('Enhanced Context Management E2E Tests', () => { }); test.describe('Error Handling', () => { - test('should handle compaction errors gracefully', async () => { + test('should handle compaction errors gracefully', async ({ goosePage }) => { // Mock a backend error - await page.route('**/api/sessions/*/manage-context', async (route) => { + await goosePage.route('**/api/sessions/*/manage-context', async (route) => { await route.fulfill({ status: 500, contentType: 'application/json', @@ -350,33 +318,33 @@ test.describe('Enhanced Context Management E2E Tests', () => { }); }); - const chatInput = page.locator('[data-testid="chat-input"]'); + const chatInput = goosePage.locator('[data-testid="chat-input"]'); // Generate conversation await chatInput.fill('Test message for error handling'); - await page.keyboard.press('Enter'); - await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); + await goosePage.keyboard.press('Enter'); + await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); // Attempt compaction - await page.waitForSelector('[data-testid="alert-indicator"]', { timeout: 15000 }); - await page.click('[data-testid="alert-indicator"]'); - await page.click('text=Compact now'); + await goosePage.waitForSelector('[data-testid="alert-indicator"]', { timeout: 15000 }); + await goosePage.click('[data-testid="alert-indicator"]'); + await goosePage.click('text=Compact now'); // Wait for compaction to fail - await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); + await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); // Verify error message appears - const errorMarker = page.locator('text=Compaction failed. Please try again or start a new session.'); + const errorMarker = goosePage.locator('text=Compaction failed. Please try again or start a new session.'); await expect(errorMarker).toBeVisible(); // Verify chat input is still functional after error - const submitButton = page.locator('[data-testid="submit-button"]'); + const submitButton = goosePage.locator('[data-testid="submit-button"]'); await expect(submitButton).toBeEnabled(); }); - test('should handle network timeouts during compaction', async () => { + test('should handle network timeouts during compaction', async ({ goosePage }) => { // Mock a timeout - await page.route('**/api/sessions/*/manage-context', async (route) => { + await goosePage.route('**/api/sessions/*/manage-context', async (route) => { // Delay response to simulate timeout await new Promise(resolve => setTimeout(resolve, 5000)); await route.fulfill({ @@ -386,86 +354,86 @@ test.describe('Enhanced Context Management E2E Tests', () => { }); }); - const chatInput = page.locator('[data-testid="chat-input"]'); + const chatInput = goosePage.locator('[data-testid="chat-input"]'); // Generate conversation await chatInput.fill('Test message for timeout handling'); - await page.keyboard.press('Enter'); - await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); + await goosePage.keyboard.press('Enter'); + await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); // Attempt compaction - await page.waitForSelector('[data-testid="alert-indicator"]', { timeout: 15000 }); - await page.click('[data-testid="alert-indicator"]'); - await page.click('text=Compact now'); + await goosePage.waitForSelector('[data-testid="alert-indicator"]', { timeout: 15000 }); + await goosePage.click('[data-testid="alert-indicator"]'); + await goosePage.click('text=Compact now'); // Verify loading state persists during timeout - const loadingGoose = page.locator('[data-testid="loading-goose"]'); + const loadingGoose = goosePage.locator('[data-testid="loading-goose"]'); await expect(loadingGoose).toBeVisible(); await expect(loadingGoose).toContainText('goose is compacting the conversation...'); // Wait for timeout to complete - await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 35000 }); + await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 35000 }); // Should show error message - const errorMarker = page.locator('text=Compaction failed. Please try again or start a new session.'); + const errorMarker = goosePage.locator('text=Compaction failed. Please try again or start a new session.'); await expect(errorMarker).toBeVisible(); }); }); test.describe('UI State Management', () => { - test('should disable chat input during compaction', async () => { - const chatInput = page.locator('[data-testid="chat-input"]'); + test('should disable chat input during compaction', async ({ goosePage }) => { + const chatInput = goosePage.locator('[data-testid="chat-input"]'); // Generate conversation await chatInput.fill('Test message for UI state verification'); - await page.keyboard.press('Enter'); - await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); + await goosePage.keyboard.press('Enter'); + await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); // Start compaction - await page.waitForSelector('[data-testid="alert-indicator"]', { timeout: 15000 }); - await page.click('[data-testid="alert-indicator"]'); - await page.click('text=Compact now'); + await goosePage.waitForSelector('[data-testid="alert-indicator"]', { timeout: 15000 }); + await goosePage.click('[data-testid="alert-indicator"]'); + await goosePage.click('text=Compact now'); // Verify chat input is disabled during compaction - const submitButton = page.locator('[data-testid="submit-button"]'); + const submitButton = goosePage.locator('[data-testid="submit-button"]'); await expect(submitButton).toBeDisabled(); // Verify loading message - const loadingGoose = page.locator('[data-testid="loading-goose"]'); + const loadingGoose = goosePage.locator('[data-testid="loading-goose"]'); await expect(loadingGoose).toBeVisible(); await expect(loadingGoose).toContainText('goose is compacting the conversation...'); // Wait for compaction to complete - await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); + await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); // Verify chat input is re-enabled await expect(submitButton).toBeEnabled(); }); - test('should show appropriate loading states', async () => { - const chatInput = page.locator('[data-testid="chat-input"]'); + test('should show appropriate loading states', async ({ goosePage }) => { + const chatInput = goosePage.locator('[data-testid="chat-input"]'); // Generate conversation await chatInput.fill('Test loading state message'); - await page.keyboard.press('Enter'); - await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); + await goosePage.keyboard.press('Enter'); + await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); // Start compaction and immediately check loading state - await page.waitForSelector('[data-testid="alert-indicator"]', { timeout: 15000 }); - await page.click('[data-testid="alert-indicator"]'); - await page.click('text=Compact now'); + await goosePage.waitForSelector('[data-testid="alert-indicator"]', { timeout: 15000 }); + await goosePage.click('[data-testid="alert-indicator"]'); + await goosePage.click('text=Compact now'); // Verify loading goose appears with correct message - const loadingGoose = page.locator('[data-testid="loading-goose"]'); + const loadingGoose = goosePage.locator('[data-testid="loading-goose"]'); await expect(loadingGoose).toBeVisible(); await expect(loadingGoose).toContainText('goose is compacting the conversation...'); // Verify no other loading indicators are shown - const regularLoadingMessages = page.locator('[data-testid="loading-goose"]:not(:has-text("compacting"))'); + const regularLoadingMessages = goosePage.locator('[data-testid="loading-goose"]:not(:has-text("compacting"))'); await expect(regularLoadingMessages).not.toBeVisible(); // Wait for completion - await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); + await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); // Verify loading state is cleared await expect(loadingGoose).not.toBeVisible(); @@ -473,8 +441,8 @@ test.describe('Enhanced Context Management E2E Tests', () => { }); test.describe('Performance and Reliability', () => { - test('should handle large conversations efficiently', async () => { - const chatInput = page.locator('[data-testid="chat-input"]'); + test('should handle large conversations efficiently', async ({ goosePage }) => { + const chatInput = goosePage.locator('[data-testid="chat-input"]'); // Generate a larger conversation const messages = Array.from({ length: 8 }, (_, i) => @@ -483,54 +451,54 @@ test.describe('Enhanced Context Management E2E Tests', () => { for (const message of messages) { await chatInput.fill(message); - await page.keyboard.press('Enter'); - await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); - await page.waitForTimeout(500); + await goosePage.keyboard.press('Enter'); + await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); + await goosePage.waitForTimeout(500); } // Perform compaction - await page.waitForSelector('[data-testid="alert-indicator"]', { timeout: 15000 }); - await page.click('[data-testid="alert-indicator"]'); - await page.click('text=Compact now'); + await goosePage.waitForSelector('[data-testid="alert-indicator"]', { timeout: 15000 }); + await goosePage.click('[data-testid="alert-indicator"]'); + await goosePage.click('text=Compact now'); // Verify compaction completes within reasonable time - await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 45000 }); + await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 45000 }); // Verify compaction marker appears - await expect(page.locator('text=Conversation compacted and summarized')).toBeVisible(); + await expect(goosePage.locator('text=Conversation compacted and summarized')).toBeVisible(); // Verify system remains responsive await chatInput.fill('Post-compaction test message'); - await page.keyboard.press('Enter'); - await expect(page.locator('[data-testid="loading-goose"]')).toBeVisible(); + await goosePage.keyboard.press('Enter'); + await expect(goosePage.locator('[data-testid="loading-goose"]')).toBeVisible(); }); - test('should maintain conversation context after compaction', async () => { - const chatInput = page.locator('[data-testid="chat-input"]'); + test('should maintain conversation context after compaction', async ({ goosePage }) => { + const chatInput = goosePage.locator('[data-testid="chat-input"]'); // Create conversation with specific context await chatInput.fill('My name is Alice and I am a software developer working on React applications.'); - await page.keyboard.press('Enter'); - await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); + await goosePage.keyboard.press('Enter'); + await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); await chatInput.fill('I am having trouble with useState hooks. Can you help?'); - await page.keyboard.press('Enter'); - await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); + await goosePage.keyboard.press('Enter'); + await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); // Perform compaction - await page.waitForSelector('[data-testid="alert-indicator"]', { timeout: 15000 }); - await page.click('[data-testid="alert-indicator"]'); - await page.click('text=Compact now'); - await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); + await goosePage.waitForSelector('[data-testid="alert-indicator"]', { timeout: 15000 }); + await goosePage.click('[data-testid="alert-indicator"]'); + await goosePage.click('text=Compact now'); + await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); // Test if context is maintained by asking a follow-up question await chatInput.fill('What did I tell you my name was?'); - await page.keyboard.press('Enter'); - await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); + await goosePage.keyboard.press('Enter'); + await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 }); // The response should ideally reference the name Alice or indicate context retention // Note: This is a behavioral test that depends on the AI's ability to use the summary - const messages = page.locator('[data-testid="message"]'); + const messages = goosePage.locator('[data-testid="message"]'); const lastMessageText = await messages.last().textContent(); // The system should have some response (not just an error) diff --git a/ui/desktop/tests/e2e/fixtures.ts b/ui/desktop/tests/e2e/fixtures.ts new file mode 100644 index 00000000..1db875f4 --- /dev/null +++ b/ui/desktop/tests/e2e/fixtures.ts @@ -0,0 +1,170 @@ +import { test as base, Page, Browser, chromium } from '@playwright/test'; +import { spawn, ChildProcess } from 'child_process'; +import { join } from 'path'; +import { promisify } from 'util'; + +const execAsync = promisify(require('child_process').exec); + +type GooseTestFixtures = { + goosePage: Page; +}; + +/** + * Test-scoped fixture that launches a fresh Electron app for EACH test. + * + * Isolation: ⚠️ Partial - each test gets a fresh app instance, but uses ambient user config + * Speed: ⚠️ Slow - ~3s startup overhead per test + * + * This ensures each test starts with a fresh app instance, but the app uses the + * user's existing Goose configuration (providers, models, etc.). + * + * Usage: + * import { test, expect } from './fixtures'; + * + * test('my test', async ({ goosePage }) => { + * await goosePage.waitForSelector('[data-testid="chat-input"]'); + * // ... test code + * }); + */ +export const test = base.extend({ + // Test-scoped fixture: launches a fresh Electron app for each test + goosePage: async ({}, use, testInfo) => { + console.log(`Launching fresh Electron app for test: ${testInfo.title}`); + + let appProcess: ChildProcess | null = null; + let browser: Browser | null = null; + + try { + // Assign a unique debug port for this test to enable parallel execution + // Base port 9222, offset by worker index * 100 + parallel slot + const debugPort = 9222 + (testInfo.parallelIndex * 10); + console.log(`Using debug port ${debugPort} for parallel test execution`); + + // Start the electron-forge process with Playwright remote debugging enabled + // Use detached mode on Unix to create a process group we can kill together + appProcess = spawn('npm', ['run', 'start-gui'], { + cwd: join(__dirname, '../..'), + stdio: 'pipe', + detached: process.platform !== 'win32', + env: { + ...process.env, + ELECTRON_IS_DEV: '1', + NODE_ENV: 'development', + GOOSE_ALLOWLIST_BYPASS: 'true', + ENABLE_PLAYWRIGHT: 'true', + PLAYWRIGHT_DEBUG_PORT: debugPort.toString(), // Unique port per test for parallel execution + RUST_LOG: 'info', // Enable info-level logging for goosed backend + } + }); + + // Log process output for debugging + if (process.env.DEBUG_TESTS) { + appProcess.stdout?.on('data', (data) => { + console.log('App stdout:', data.toString()); + }); + + appProcess.stderr?.on('data', (data) => { + console.log('App stderr:', data.toString()); + }); + } + + // Wait for the app to start and remote debugging to be available + // Retry connection until it succeeds (app is ready) or timeout + console.log(`Waiting for Electron app to start on port ${debugPort}...`); + const maxRetries = 100; // 100 retries * 100ms = 10 seconds max + const retryDelay = 100; // 100ms between retries + + for (let attempt = 1; attempt <= maxRetries; attempt++) { + try { + browser = await chromium.connectOverCDP(`http://127.0.0.1:${debugPort}`); + console.log(`Connected to Electron app on attempt ${attempt} (~${(attempt * retryDelay) / 1000}s)`); + break; + } catch (error) { + if (attempt === maxRetries) { + throw new Error(`Failed to connect to Electron app after ${maxRetries} attempts (${(maxRetries * retryDelay) / 1000}s). Last error: ${error.message}`); + } + // Wait before next retry + await new Promise(resolve => setTimeout(resolve, retryDelay)); + } + } + + if (!browser) { + throw new Error('Browser connection failed unexpectedly'); + } + + // Get the electron app context and first page + const contexts = browser.contexts(); + if (contexts.length === 0) { + throw new Error('No browser contexts found'); + } + + const pages = contexts[0].pages(); + if (pages.length === 0) { + throw new Error('No windows/pages found'); + } + + const page = pages[0]; + + // Wait for page to be ready + await page.waitForLoadState('domcontentloaded'); + + // Try to wait for networkidle + try { + await page.waitForLoadState('networkidle', { timeout: 10000 }); + } catch (error) { + console.log('NetworkIdle timeout (likely due to MCP activity), continuing...'); + } + + // Wait for React app to be ready + await page.waitForFunction(() => { + const root = document.getElementById('root'); + return root && root.children.length > 0; + }, { timeout: 30000 }); + + console.log('App ready, starting test...'); + + // Provide the page to the test + await use(page); + + } finally { + console.log('Cleaning up Electron app for this test...'); + + // Close the CDP connection + if (browser) { + await browser.close().catch(console.error); + } + + // Kill the npm process tree + if (appProcess && appProcess.pid) { + try { + if (process.platform === 'win32') { + // On Windows, kill the entire process tree + await execAsync(`taskkill /F /T /PID ${appProcess.pid}`); + } else { + // On Unix, kill the entire process group + try { + // First try SIGTERM for graceful shutdown + process.kill(-appProcess.pid, 'SIGTERM'); + await new Promise(resolve => setTimeout(resolve, 2000)); + } catch (e) { + // Process might already be dead + } + // Then SIGKILL if still running + try { + process.kill(-appProcess.pid, 'SIGKILL'); + } catch (e) { + // Process already exited + } + } + console.log('Cleaned up app process'); + } catch (error) { + if (error.code !== 'ESRCH' && !error.message?.includes('No such process')) { + console.error('Error killing app process:', error); + } + } + } + } + }, +}); + +export { expect } from '@playwright/test'; diff --git a/ui/desktop/tests/e2e/performance.spec.ts b/ui/desktop/tests/e2e/performance.spec.ts new file mode 100644 index 00000000..3bec5433 --- /dev/null +++ b/ui/desktop/tests/e2e/performance.spec.ts @@ -0,0 +1,330 @@ +import { test, expect } from './fixtures'; + +test.describe('Performance Tests', () => { + test('measure end-to-end performance for prompt submission', async ({ goosePage }) => { + // Start Playwright tracing to capture all performance data + await goosePage.context().tracing.start({ + screenshots: true, + snapshots: true, + sources: true + }); + + console.log('\n=== Performance Test Started ===\n'); + + // Mark: App ready + await goosePage.waitForSelector('[data-testid="chat-input"]', { timeout: 30000 }); + await goosePage.evaluate(() => performance.mark('app-ready')); + console.log('✓ App ready'); + + // Prepare prompt + const chatInput = await goosePage.waitForSelector('[data-testid="chat-input"]'); + const testPrompt = 'Write a haiku about testing software'; + await chatInput.fill(testPrompt); + + // Mark: Prompt submit + await goosePage.evaluate(() => performance.mark('prompt-submit-start')); + await chatInput.press('Enter'); + await goosePage.evaluate(() => performance.mark('prompt-submitted')); + + // Wait for loading indicator to appear and check if it's "loading conversation..." + await goosePage.waitForSelector('[data-testid="loading-indicator"]', { + state: 'visible', + timeout: 5000 + }); + + const loadingText = await goosePage.locator('[data-testid="loading-indicator"]').textContent(); + if (loadingText?.includes('loading conversation')) { + await goosePage.evaluate(() => performance.mark('loading-conversation-start')); + console.log('✓ Loading conversation detected'); + + // Wait for it to change or disappear + await goosePage.waitForFunction(() => { + const indicator = document.querySelector('[data-testid="loading-indicator"]'); + if (!indicator) return true; // Disappeared + const text = indicator.textContent || ''; + return !text.includes('loading conversation'); // Changed to different state + }, { timeout: 30000 }); + + await goosePage.evaluate(() => performance.mark('loading-conversation-end')); + console.log('✓ Loading conversation complete'); + } + + await goosePage.evaluate(() => performance.mark('loading-started')); + + // Monitor for first token (first visible response content) + let firstTokenDetected = false; + const checkForFirstToken = async () => { + while (!firstTokenDetected) { + try { + const messageContainers = await goosePage.locator('[data-testid="message-container"]').all(); + if (messageContainers.length > 0) { + const lastMessage = messageContainers[messageContainers.length - 1]; + const content = await lastMessage.textContent(); + if (content && content.trim().length > 0) { + await goosePage.evaluate(() => performance.mark('first-token-received')); + firstTokenDetected = true; + console.log('✓ First token detected'); + break; + } + } + } catch (e) { + // Continue checking + } + await goosePage.waitForTimeout(50); + } + }; + + // Start checking for first token + const firstTokenPromise = checkForFirstToken(); + await firstTokenPromise; + + // Wait for response to complete + await goosePage.waitForSelector('[data-testid="loading-indicator"]', { + state: 'hidden', + timeout: 60000 + }); + await goosePage.evaluate(() => performance.mark('response-complete')); + console.log('✓ Response complete'); + + // Create performance measures + await goosePage.evaluate(() => { + // Measure loading conversation if it was detected + const marks = performance.getEntriesByType('mark').map(m => m.name); + if (marks.includes('loading-conversation-start') && marks.includes('loading-conversation-end')) { + performance.measure('loading-conversation-duration', 'loading-conversation-start', 'loading-conversation-end'); + } + + performance.measure('time-to-prompt-submit', 'prompt-submit-start', 'prompt-submitted'); + performance.measure('time-to-first-token', 'prompt-submitted', 'first-token-received'); + performance.measure('time-to-complete-response', 'prompt-submitted', 'response-complete'); + performance.measure('streaming-duration', 'first-token-received', 'response-complete'); + performance.measure('total-interaction', 'prompt-submit-start', 'response-complete'); + }); + + // Extract and display performance metrics + const metrics = await goosePage.evaluate(() => { + const measures = performance.getEntriesByType('measure'); + const result: Record = {}; + measures.forEach(measure => { + result[measure.name] = Math.round(measure.duration); + }); + return result; + }); + + console.log('\n=== Performance Metrics ==='); + if (metrics['loading-conversation-duration']) { + console.log(`Loading Conversation: ${metrics['loading-conversation-duration']}ms`); + } + console.log(`Time to Submit Prompt: ${metrics['time-to-prompt-submit']}ms`); + console.log(`Time to First Token (TTFT): ${metrics['time-to-first-token']}ms`); + console.log(`Time to Complete Response: ${metrics['time-to-complete-response']}ms`); + console.log(`Streaming Duration: ${metrics['streaming-duration']}ms`); + console.log(`Total Interaction Time: ${metrics['total-interaction']}ms`); + console.log('===========================\n'); + + // Verify we got a response + const response = await goosePage.locator('[data-testid="message-container"]').last(); + const responseText = await response.textContent(); + expect(responseText).toBeTruthy(); + expect(responseText!.length).toBeGreaterThan(0); + + // Assert performance thresholds + expect(metrics['time-to-first-token']).toBeLessThan(10000); // First token in < 10s + expect(metrics['time-to-complete-response']).toBeLessThan(60000); // Complete in < 60s + + // Stop tracing and save + const tracePath = test.info().outputPath('trace.zip'); + await goosePage.context().tracing.stop({ path: tracePath }); + console.log(`✓ Performance trace saved to: ${tracePath}`); + console.log(` View with: npx playwright show-trace ${tracePath}\n`); + + // Attach metrics as JSON + await test.info().attach('performance-metrics.json', { + body: JSON.stringify(metrics, null, 2), + contentType: 'application/json', + }); + }); + + test('measure cold start vs warm cache performance', async ({ goosePage }) => { + await goosePage.context().tracing.start({ screenshots: true, snapshots: true }); + + console.log('\n=== Cold vs Warm Performance ===\n'); + + // Cold start measurement + await goosePage.waitForSelector('[data-testid="chat-input"]', { timeout: 30000 }); + await goosePage.evaluate(() => performance.mark('app-ready')); + + // First prompt (cold) + const chatInput = await goosePage.waitForSelector('[data-testid="chat-input"]'); + await chatInput.fill('Say hello'); + + await goosePage.evaluate(() => performance.mark('cold-prompt-start')); + await chatInput.press('Enter'); + + await goosePage.waitForSelector('[data-testid="loading-indicator"]', { + state: 'visible', + timeout: 5000 + }); + + await goosePage.waitForSelector('[data-testid="loading-indicator"]', { + state: 'hidden', + timeout: 60000 + }); + + await goosePage.evaluate(() => { + performance.mark('cold-prompt-complete'); + performance.measure('cold-prompt-duration', 'cold-prompt-start', 'cold-prompt-complete'); + }); + + console.log('✓ Cold prompt complete'); + + // Second prompt (warm) + const chatInput2 = await goosePage.waitForSelector('[data-testid="chat-input"]'); + await chatInput2.fill('Say goodbye'); + + await goosePage.evaluate(() => performance.mark('warm-prompt-start')); + await chatInput2.press('Enter'); + + await goosePage.waitForSelector('[data-testid="loading-indicator"]', { + state: 'visible', + timeout: 5000 + }); + + await goosePage.waitForSelector('[data-testid="loading-indicator"]', { + state: 'hidden', + timeout: 60000 + }); + + await goosePage.evaluate(() => { + performance.mark('warm-prompt-complete'); + performance.measure('warm-prompt-duration', 'warm-prompt-start', 'warm-prompt-complete'); + }); + + console.log('✓ Warm prompt complete'); + + // Extract metrics + const metrics = await goosePage.evaluate(() => { + const measures = performance.getEntriesByType('measure'); + const result: Record = {}; + measures.forEach(measure => { + result[measure.name] = Math.round(measure.duration); + }); + return result; + }); + + const coldDuration = metrics['cold-prompt-duration']; + const warmDuration = metrics['warm-prompt-duration']; + const improvement = ((coldDuration - warmDuration) / coldDuration * 100).toFixed(1); + + console.log('\n=== Results ==='); + console.log(`Cold Prompt Duration: ${coldDuration}ms`); + console.log(`Warm Prompt Duration: ${warmDuration}ms`); + console.log(`Improvement: ${improvement}%`); + console.log('================\n'); + + // Save trace + const tracePath = test.info().outputPath('cold-vs-warm-trace.zip'); + await goosePage.context().tracing.stop({ path: tracePath }); + console.log(`✓ Trace saved to: ${tracePath}\n`); + + // Attach results + await test.info().attach('cold-vs-warm.json', { + body: JSON.stringify({ + coldDuration, + warmDuration, + improvement: `${improvement}%` + }, null, 2), + contentType: 'application/json', + }); + }); + + test('capture full performance profile with navigation timing', async ({ goosePage }) => { + await goosePage.context().tracing.start({ screenshots: true, snapshots: true }); + + console.log('\n=== Full Performance Profile ===\n'); + + // Get navigation timing + const navigationTiming = await goosePage.evaluate(() => { + const perf = performance.getEntriesByType('navigation')[0] as PerformanceNavigationTiming; + return { + domContentLoaded: Math.round(perf.domContentLoadedEventEnd - perf.fetchStart), + loadComplete: Math.round(perf.loadEventEnd - perf.fetchStart), + domInteractive: Math.round(perf.domInteractive - perf.fetchStart), + }; + }); + + console.log('Navigation Timing:'); + console.log(` DOM Content Loaded: ${navigationTiming.domContentLoaded}ms`); + console.log(` DOM Interactive: ${navigationTiming.domInteractive}ms`); + console.log(` Load Complete: ${navigationTiming.loadComplete}ms`); + + // Wait for app ready + await goosePage.waitForSelector('[data-testid="chat-input"]', { timeout: 30000 }); + await goosePage.evaluate(() => performance.mark('app-interactive')); + + // Measure time from navigation to interactive + const appReadyTime = await goosePage.evaluate(() => { + const appInteractive = performance.getEntriesByName('app-interactive')[0]; + return Math.round(appInteractive.startTime); + }); + + console.log(` App Interactive: ${appReadyTime}ms\n`); + + // Submit a prompt and measure + const chatInput = await goosePage.waitForSelector('[data-testid="chat-input"]'); + await chatInput.fill('Hello'); + + await goosePage.evaluate(() => performance.mark('user-interaction-start')); + await chatInput.press('Enter'); + + await goosePage.waitForSelector('[data-testid="loading-indicator"]', { state: 'visible', timeout: 5000 }); + await goosePage.waitForSelector('[data-testid="loading-indicator"]', { state: 'hidden', timeout: 60000 }); + + await goosePage.evaluate(() => { + performance.mark('user-interaction-complete'); + performance.measure('user-interaction-duration', 'user-interaction-start', 'user-interaction-complete'); + }); + + const interactionTime = await goosePage.evaluate(() => { + const measure = performance.getEntriesByName('user-interaction-duration')[0]; + return Math.round(measure.duration); + }); + + console.log(`User Interaction Duration: ${interactionTime}ms\n`); + + // Get resource timing summary + const resourceStats = await goosePage.evaluate(() => { + const resources = performance.getEntriesByType('resource'); + const types: Record = {}; + resources.forEach(resource => { + const type = (resource as PerformanceResourceTiming).initiatorType; + types[type] = (types[type] || 0) + 1; + }); + return { + total: resources.length, + byType: types + }; + }); + + console.log('Resource Loading:'); + console.log(` Total Resources: ${resourceStats.total}`); + console.log(` By Type:`, resourceStats.byType); + console.log('\n==============================\n'); + + // Save trace + const tracePath = test.info().outputPath('full-profile-trace.zip'); + await goosePage.context().tracing.stop({ path: tracePath }); + console.log(`✓ Full trace saved to: ${tracePath}\n`); + + // Attach all metrics + await test.info().attach('full-performance-profile.json', { + body: JSON.stringify({ + navigationTiming, + appReadyTime, + interactionTime, + resourceStats + }, null, 2), + contentType: 'application/json', + }); + }); +});