tkmind/tkmind_go
1
0
Fork 0
Code Issues Pull Requests Actions 4 Packages Projects Releases Wiki Activity

Speed up Databricks provider init by removing fetch of supported models (#6616)

Browse Source
This commit is contained in:
jh-block
2026-01-26 16:51:49 +01:00
committed by GitHub
parent 5971e2f187
commit 82ba07ff1d
8 changed files with 865 additions and 539 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/goose/src/providers/databricks.rs
+1 -26
View File
@@ -134,7 +134,6 @@ impl DatabricksProvider {
let api_client =
ApiClient::with_timeout(host, auth_method, Duration::from_secs(DEFAULT_TIMEOUT_SECS))?;
// Create the provider without the fast model first
let mut provider = Self {
api_client,
auth,
@@ -143,31 +142,7 @@ impl DatabricksProvider {
retry_config,
name: Self::metadata().name,
};
// Check if the default fast model exists in the workspace
// Generate UUID for this initialization request since no user session exists yet
let session_id = uuid::Uuid::new_v4().to_string();
let model_with_fast =
if let Ok(Some(models)) = provider.fetch_supported_models(&session_id).await {
if models.contains(&DATABRICKS_DEFAULT_FAST_MODEL.to_string()) {
tracing::debug!(
"Found {} in Databricks workspace, setting as fast model",
DATABRICKS_DEFAULT_FAST_MODEL
);
model.with_fast(DATABRICKS_DEFAULT_FAST_MODEL.to_string())
} else {
tracing::debug!(
"{} not found in Databricks workspace, not setting fast model",
DATABRICKS_DEFAULT_FAST_MODEL
);
model
}
} else {
tracing::debug!("Could not fetch Databricks models, not setting fast model");
model
};
provider.model = model_with_fast;
provider.model = model.with_fast(DATABRICKS_DEFAULT_FAST_MODEL.to_string());
Ok(provider)
}
ui/desktop/playwright.config.ts
+1 -1
View File
@@ -20,7 +20,7 @@ const config: PlaywrightTestConfig = {
screenshot: 'only-on-failure'
},
outputDir: 'test-results',
preserveOutput: 'failures-only'
preserveOutput: 'always'
};
export default config;
ui/desktop/src/main.ts
+3 -2
View File
@@ -154,8 +154,9 @@ async function configureProxy() {
if (started) app.quit();
if (process.env.ENABLE_PLAYWRIGHT) {
console.log('[Main] Enabling Playwright remote debugging on port 9222');
app.commandLine.appendSwitch('remote-debugging-port', '9222');
const debugPort = process.env.PLAYWRIGHT_DEBUG_PORT || '9222';
console.log(`[Main] Enabling Playwright remote debugging on port ${debugPort}`);
app.commandLine.appendSwitch('remote-debugging-port', debugPort);
}
// In development mode, force registration as the default protocol client
ui/desktop/tests/e2e/app.spec.ts
+102 -188
View File
@@ -1,12 +1,9 @@
import { test as base, expect } from '@playwright/test';
import { _electron as electron } from '@playwright/test';
import { join } from 'path';
import { spawn, exec } from 'child_process';
import { promisify } from 'util';
import { test as base, expect } from './fixtures';
import { Page } from '@playwright/test';
import { showTestName, clearTestName } from './test-overlay';
import { join } from 'path';
const { runningQuotes } = require('./basic-mcp');
const execAsync = promisify(exec);
// Define provider interface
type Provider = {
@@ -28,23 +25,18 @@ const test = base.extend<TestFixtures>({
provider: [providers[0], { option: true }], // Default to first provider (Databricks)
});
// Store mainWindow reference
let mainWindow;
let mainWindow: Page;
// Add hooks for test name overlay
// eslint-disable-next-line no-empty-pattern
test.beforeEach(async ({ }, testInfo) => {
if (mainWindow) {
// Get a clean test name without the full hierarchy
const testName = testInfo.titlePath[testInfo.titlePath.length - 1];
test.beforeEach(async ({ goosePage }, testInfo) => {
mainWindow = goosePage;
// Get provider name if we're in a provider suite
const providerSuite = testInfo.titlePath.find(t => t.startsWith('Provider:'));
const providerName = providerSuite ? providerSuite.split(': ')[1] : undefined;
const testName = testInfo.titlePath[testInfo.titlePath.length - 1];
console.log(`Setting overlay for test: "${testName}"${providerName ? ` (Provider: ${providerName})` : ''}`);
await showTestName(mainWindow, testName, providerName);
}
const providerSuite = testInfo.titlePath.find(t => t.startsWith('Provider:'));
const providerName = providerSuite ? providerSuite.split(': ')[1] : undefined;
console.log(`Setting overlay for test: "${testName}"${providerName ? ` (Provider: ${providerName})` : ''}`);
await showTestName(mainWindow, testName, providerName);
});
test.afterEach(async () => {
@@ -116,184 +108,105 @@ async function selectProvider(mainWindow: any, provider: Provider) {
return; // Provider is already selected, no need to do anything
}
// Check if we need to click "configure other providers (advanced)" button
const configureAdvancedButton = await mainWindow.waitForSelector('h3:has-text("Other providers")', {
// Check if we're on the welcome screen with "Other Providers" section
const otherProvidersSection = await mainWindow.waitForSelector('text="Other Providers"', {
timeout: 3000,
state: 'visible'
}).catch(() => null);
if (configureAdvancedButton) {
console.log('Found "configure other providers" button, clicking it...');
await configureAdvancedButton.click();
await mainWindow.waitForTimeout(1500);
if (otherProvidersSection) {
console.log('Found "Other Providers" section, clicking "Go to Provider Settings" link...');
// Click the "Go to Provider Settings" link (includes arrow →)
const providerSettingsLink = await mainWindow.waitForSelector('button:has-text("Go to Provider Settings")', {
timeout: 3000,
state: 'visible'
});
await providerSettingsLink.click();
await mainWindow.waitForTimeout(1000);
// We should now be in Settings -> Models tab
console.log('Navigated to Provider Settings');
}
// We should now be at provider selection
await mainWindow.waitForSelector('[data-testid="provider-selection-heading"]');
// Now we should be on the "Other providers" page with provider cards
console.log(`Looking for ${provider.name} provider card...`);
// Wait for the provider cards to load
await mainWindow.waitForTimeout(1000);
// Find the Launch button within the specific provider card using its data-testid
console.log(`Looking for ${provider.name} card with Launch button...`);
// Find and verify the provider card container
console.log(`Looking for ${provider.name} card...`);
let providerContainer;
try {
providerContainer = await mainWindow.waitForSelector(`[data-testid="provider-card-${provider.name.toLowerCase()}"]`);
expect(await providerContainer.isVisible()).toBe(true);
// Each provider card has data-testid="provider-card-{provider-name-lowercase}"
const providerCardTestId = `provider-card-${provider.name.toLowerCase()}`;
const launchButton = mainWindow.locator(`[data-testid="${providerCardTestId}"] button:has-text("Launch")`);
await launchButton.waitFor({ state: 'visible', timeout: 5000 });
console.log(`Found Launch button in ${provider.name} card, clicking it...`);
await launchButton.click();
await mainWindow.waitForTimeout(1000);
// Wait for "Choose Model" dialog to appear and select a model
console.log('Waiting for model selection dialog...');
const chooseModelDialog = await mainWindow.waitForSelector('text="Choose Model"', {
timeout: 5000,
state: 'visible'
}).catch(() => null);
if (chooseModelDialog) {
console.log('Model selection dialog appeared, waiting for models to load...');
// The "Select model" button starts enabled and only disables during loading (UI bug)
// So we wait for a fixed timeout to ensure models are loaded
await mainWindow.waitForTimeout(5000);
console.log('Waited for models to load');
const confirmButton = await mainWindow.waitForSelector('button:has-text("Select model")', {
timeout: 5000,
state: 'visible'
});
console.log('Clicking "Select model" button');
await confirmButton.click();
await mainWindow.waitForTimeout(2000);
}
} catch (error) {
console.error(`Provider card not found for ${provider.name}. This could indicate a missing or incorrectly configured provider.`);
console.error(`Failed to find or click Launch button in ${provider.name} card:`, error);
throw error;
}
// Find the Launch button within the provider container
console.log(`Looking for Launch button in ${provider.name} card...`);
const launchButton = await providerContainer.waitForSelector('[data-testid="provider-launch-button"]');
expect(await launchButton.isVisible()).toBe(true);
// Navigate to home/chat after provider configuration
console.log('Navigating to home/chat...');
const homeButton = await mainWindow.waitForSelector('[data-testid="sidebar-home-button"]', {
timeout: 5000
}).catch(() => null);
// Take screenshot before clicking
await mainWindow.screenshot({ path: `test-results/before-${provider.name.toLowerCase()}-click.png` });
// Click the Launch button
await launchButton.click();
if (homeButton) {
await homeButton.click();
await mainWindow.waitForTimeout(1000);
}
// Wait for chat interface to appear
const chatTextareaAfterClick = await mainWindow.waitForSelector('[data-testid="chat-input"]',
const chatTextareaAfterConfig = await mainWindow.waitForSelector('[data-testid="chat-input"]',
{ timeout: 10000 });
expect(await chatTextareaAfterClick.isVisible()).toBe(true);
expect(await chatTextareaAfterConfig.isVisible()).toBe(true);
// Take screenshot of chat interface
await mainWindow.screenshot({ path: `test-results/chat-interface-${provider.name.toLowerCase()}.png` });
}
test.describe('Goose App', () => {
let electronApp;
let appProcess;
test.beforeAll(async () => {
console.log('Starting Electron app...');
// Start the electron-forge process
appProcess = spawn('npm', ['run', 'start-gui'], {
cwd: join(__dirname, '../..'),
stdio: 'pipe',
shell: true,
env: {
...process.env,
ELECTRON_IS_DEV: '1',
NODE_ENV: 'development',
GOOSE_ALLOWLIST_BYPASS: 'true',
}
});
// Log process output
appProcess.stdout.on('data', (data) => {
console.log('App stdout:', data.toString());
});
appProcess.stderr.on('data', (data) => {
console.log('App stderr:', data.toString());
});
// Wait a bit for the app to start
console.log('Waiting for app to start...');
await new Promise(resolve => setTimeout(resolve, 5000));
// Launch Electron for testing
electronApp = await electron.launch({
args: ['.vite/build/main.js'],
cwd: join(__dirname, '../..'),
env: {
...process.env,
ELECTRON_IS_DEV: '1',
NODE_ENV: 'development',
},
recordVideo: {
dir: 'test-results/videos/',
size: { width: 620, height: 680 }
}
});
// Get the main window once for all tests
mainWindow = await electronApp.firstWindow();
await mainWindow.waitForLoadState('domcontentloaded');
// Try to wait for networkidle, but don't fail if it times out due to MCP activity
try {
await mainWindow.waitForLoadState('networkidle', { timeout: 10000 });
} catch (error) {
console.log('NetworkIdle timeout (likely due to MCP activity), continuing with test...');
}
// Wait for React app to be ready by checking for the root element to have content
await mainWindow.waitForFunction(() => {
const root = document.getElementById('root');
return root && root.children.length > 0;
});
// Wait for any animations to complete
await mainWindow.waitForTimeout(2000);
// Take a screenshot to debug what's on the screen
await mainWindow.screenshot({ path: 'test-results/initial-load.png' });
// Debug: print out the page content
const content = await mainWindow.content();
console.log('Page content:', content);
});
test.afterAll(async () => {
console.log('Final cleanup...');
// Close the test instance
if (electronApp) {
await electronApp.close().catch(console.error);
}
// Kill any remaining electron processes
try {
if (process.platform === 'win32') {
await execAsync('taskkill /F /IM electron.exe');
} else {
await execAsync('pkill -f electron || true');
}
} catch (error) {
if (!error.message?.includes('no process found')) {
console.error('Error killing electron processes:', error);
}
}
// Kill any remaining npm processes from start-gui
try {
if (process.platform === 'win32') {
await execAsync('taskkill /F /IM node.exe');
} else {
await execAsync('pkill -f "start-gui" || true');
}
} catch (error) {
if (!error.message?.includes('no process found')) {
console.error('Error killing npm processes:', error);
}
}
// Kill the specific npm process if it's still running
try {
if (appProcess && appProcess.pid) {
process.kill(appProcess.pid);
}
} catch (error) {
if (error.code !== 'ESRCH') {
console.error('Error killing npm process:', error);
}
}
});
// No need for beforeAll/afterAll - the fixture handles app launch and cleanup!
test.describe('General UI', () => {
test('dark mode toggle', async () => {
console.log('Testing dark mode toggle...');
const chatTextarea = await mainWindow.waitForSelector('[data-testid="chat-input"]', {
timeout: 2000
}).catch(() => null);
if (!chatTextarea) {
await selectProvider(mainWindow, providers[0]);
}
// Assume the app is already configured and wait for chat input
await mainWindow.waitForSelector('[data-testid="chat-input"]', {
timeout: 10000
});
// Navigate to Settings via sidebar
const settingsButton = await mainWindow.waitForSelector('[data-testid="sidebar-settings-button"]', {
@@ -356,8 +269,8 @@ test.describe('Goose App', () => {
for (const provider of providers) {
test.describe(`Provider: ${provider.name}`, () => {
test.beforeAll(async () => {
// Select the provider once before all tests for this provider
test.beforeEach(async () => {
// Select the provider before each test for this provider
await selectProvider(mainWindow, provider);
});
@@ -378,19 +291,19 @@ test.describe('Goose App', () => {
// Send message
await chatInput.press('Enter');
// Wait for loading indicator to appear
console.log('Waiting for loading indicator...');
const loadingGoose = await mainWindow.waitForSelector('[data-testid="loading-indicator"]',
{ timeout: 2000 });
expect(await loadingGoose.isVisible()).toBe(true);
// Take screenshot of loading state
await mainWindow.screenshot({ path: `test-results/${provider.name.toLowerCase()}-loading-state.png` });
// Wait for loading indicator to disappear
// Wait for loading indicator to appear and then disappear
console.log('Waiting for response...');
await mainWindow.waitForSelector('[data-testid="loading-indicator"]',
{ state: 'hidden', timeout: 30000 });
await mainWindow.waitForSelector('[data-testid="loading-indicator"]', {
state: 'visible',
timeout: 5000
});
console.log('Loading indicator appeared');
await mainWindow.waitForSelector('[data-testid="loading-indicator"]', {
state: 'hidden',
timeout: 30000
});
console.log('Loading indicator disappeared');
// Get the latest response
const response = await mainWindow.locator('[data-testid="message-container"]').last();
@@ -433,9 +346,10 @@ test.describe('Goose App', () => {
// Take screenshot of chat history
await mainWindow.screenshot({ path: `test-results/${provider.name.toLowerCase()}-chat-history.png` });
// Test command history (up arrow)
await chatInput.press('Control+ArrowUp');
const inputValue = await chatInput.inputValue();
// Test command history (up arrow) - re-query for the input since the element may have been re-rendered
const chatInputForHistory = await mainWindow.waitForSelector('[data-testid="chat-input"]');
await chatInputForHistory.press('Control+ArrowUp');
const inputValue = await chatInputForHistory.inputValue();
expect(inputValue).toBe('What is 2+2?');
});
});
ui/desktop/tests/e2e/context-management.spec.ts
+88 -120
View File
@@ -1,74 +1,42 @@
import { test, expect, Page } from '@playwright/test';
import { ElectronApplication, _electron as electron } from 'playwright';
import path from 'path';
let electronApp: ElectronApplication;
let page: Page;
import { test, expect } from './fixtures';
test.describe('Context Management E2E Tests', () => {
test.beforeAll(async () => {
// Launch Electron app
electronApp = await electron.launch({
args: [path.join(__dirname, '../../.vite/build/main.js')],
env: {
...process.env,
NODE_ENV: 'test',
GOOSE_TEST_MODE: 'true',
},
});
// Get the main window
page = await electronApp.firstWindow();
await page.waitForLoadState('domcontentloaded');
// Wait for the app to be ready
await page.waitForSelector('[data-testid="chat-input"]', { timeout: 10000 });
test.beforeEach(async ({ goosePage }) => {
// Ensure the app is ready before each test
await goosePage.waitForSelector('[data-testid="chat-input"]', { timeout: 10000 });
});
test.afterAll(async () => {
if (electronApp) {
await electronApp.close();
}
});
test.beforeEach(async () => {
// Reset to a clean state before each test
await page.reload();
await page.waitForLoadState('domcontentloaded');
await page.waitForSelector('[data-testid="chat-input"]', { timeout: 10000 });
});
test('should show context window alert when tokens are being used', async () => {
test('should show context window alert when tokens are being used', async ({ goosePage }) => {
// Type a message to generate some token usage
const chatInput = page.locator('[data-testid="chat-input"]');
const chatInput = goosePage.locator('[data-testid="chat-input"]');
await chatInput.fill('Hello, this is a test message to generate some token usage.');
// Submit the message
await page.keyboard.press('Enter');
await goosePage.keyboard.press('Enter');
// Wait for response and check for context window alert
await page.waitForSelector('[data-testid="alert-indicator"]', { timeout: 15000 });
await goosePage.waitForSelector('[data-testid="alert-indicator"]', { timeout: 15000 });
// Click on the alert indicator to open the popover
await page.click('[data-testid="alert-indicator"]');
await goosePage.click('[data-testid="alert-indicator"]');
// Verify the context window alert is shown
const alertBox = page.locator('[role="alert"]');
const alertBox = goosePage.locator('[role="alert"]');
await expect(alertBox).toBeVisible();
await expect(alertBox).toContainText('Context window');
// Verify progress bar is shown
const progressBar = page.locator('[role="progressbar"]');
const progressBar = goosePage.locator('[role="progressbar"]');
await expect(progressBar).toBeVisible();
// Verify compact button is present
const compactButton = page.locator('text=Compact now');
const compactButton = goosePage.locator('text=Compact now');
await expect(compactButton).toBeVisible();
});
test('should perform manual compaction when compact button is clicked', async () => {
test('should perform manual compaction when compact button is clicked', async ({ goosePage }) => {
// First, generate enough conversation to have tokens
const chatInput = page.locator('[data-testid="chat-input"]');
const chatInput = goosePage.locator('[data-testid="chat-input"]');
// Send multiple messages to build up context
const messages = [
@@ -80,42 +48,42 @@ test.describe('Context Management E2E Tests', () => {
for (const message of messages) {
await chatInput.fill(message);
await page.keyboard.press('Enter');
await goosePage.keyboard.press('Enter');
// Wait for response before sending next message
await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
await page.waitForTimeout(1000); // Brief pause between messages
await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
await goosePage.waitForTimeout(1000); // Brief pause between messages
}
// Open the alert popover
await page.waitForSelector('[data-testid="alert-indicator"]', { timeout: 15000 });
await page.click('[data-testid="alert-indicator"]');
await goosePage.waitForSelector('[data-testid="alert-indicator"]', { timeout: 15000 });
await goosePage.click('[data-testid="alert-indicator"]');
// Click the compact button
const compactButton = page.locator('text=Compact now');
const compactButton = goosePage.locator('text=Compact now');
await expect(compactButton).toBeVisible();
await compactButton.click();
// Verify compaction loading state
const loadingGoose = page.locator('[data-testid="loading-goose"]');
const loadingGoose = goosePage.locator('[data-testid="loading-goose"]');
await expect(loadingGoose).toBeVisible();
await expect(loadingGoose).toContainText('goose is compacting the conversation...');
// Wait for compaction to complete
await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
// Verify compaction marker appears
const compactionMarker = page.locator('text=Conversation compacted and summarized');
const compactionMarker = goosePage.locator('text=Conversation compacted and summarized');
await expect(compactionMarker).toBeVisible();
// Verify alert popover is closed after compaction
const alertBox = page.locator('[role="alert"]');
const alertBox = goosePage.locator('[role="alert"]');
await expect(alertBox).not.toBeVisible();
});
test('should allow scrolling to see past messages after compaction', async () => {
test('should allow scrolling to see past messages after compaction', async ({ goosePage }) => {
// Generate conversation content
const chatInput = page.locator('[data-testid="chat-input"]');
const chatInput = goosePage.locator('[data-testid="chat-input"]');
const testMessages = [
'First message in the conversation',
@@ -126,42 +94,42 @@ test.describe('Context Management E2E Tests', () => {
// Send messages and store their content for verification
for (const message of testMessages) {
await chatInput.fill(message);
await page.keyboard.press('Enter');
await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
await page.waitForTimeout(1000);
await goosePage.keyboard.press('Enter');
await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
await goosePage.waitForTimeout(1000);
}
// Perform manual compaction
await page.waitForSelector('[data-testid="alert-indicator"]', { timeout: 15000 });
await page.click('[data-testid="alert-indicator"]');
await page.click('text=Compact now');
await goosePage.waitForSelector('[data-testid="alert-indicator"]', { timeout: 15000 });
await goosePage.click('[data-testid="alert-indicator"]');
await goosePage.click('text=Compact now');
// Wait for compaction to complete
await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
await expect(page.locator('text=Conversation compacted and summarized')).toBeVisible();
await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
await expect(goosePage.locator('text=Conversation compacted and summarized')).toBeVisible();
// Scroll up to verify past messages are still visible
const chatContainer = page.locator('[data-testid="chat-container"]');
const chatContainer = goosePage.locator('[data-testid="chat-container"]');
await chatContainer.hover();
// Scroll up multiple times to reach earlier messages
for (let i = 0; i < 5; i++) {
await page.mouse.wheel(0, -500);
await page.waitForTimeout(200);
await goosePage.mouse.wheel(0, -500);
await goosePage.waitForTimeout(200);
}
// Verify that we can still see the original messages
// Note: The exact messages might be in ancestor messages, so we check for partial content
const messageElements = page.locator('[data-testid="message"]');
const messageElements = goosePage.locator('[data-testid="message"]');
const messageCount = await messageElements.count();
// Should have more than just the compaction marker and summary
expect(messageCount).toBeGreaterThan(2);
});
test('should handle compaction errors gracefully', async () => {
test('should handle compaction errors gracefully', async ({ goosePage }) => {
// Mock a backend error by intercepting the compaction request
await page.route('**/api/sessions/*/manage-context', async (route) => {
await goosePage.route('**/api/sessions/*/manage-context', async (route) => {
await route.fulfill({
status: 500,
contentType: 'application/json',
@@ -170,126 +138,126 @@ test.describe('Context Management E2E Tests', () => {
});
// Generate some conversation
const chatInput = page.locator('[data-testid="chat-input"]');
const chatInput = goosePage.locator('[data-testid="chat-input"]');
await chatInput.fill('Test message for error handling');
await page.keyboard.press('Enter');
await goosePage.keyboard.press('Enter');
await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
// Attempt compaction
await page.waitForSelector('[data-testid="alert-indicator"]', { timeout: 15000 });
await page.click('[data-testid="alert-indicator"]');
await page.click('text=Compact now');
await goosePage.waitForSelector('[data-testid="alert-indicator"]', { timeout: 15000 });
await goosePage.click('[data-testid="alert-indicator"]');
await goosePage.click('text=Compact now');
// Wait for compaction to fail
await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
// Verify error message appears
const errorMarker = page.locator('text=Compaction failed. Please try again or start a new session.');
const errorMarker = goosePage.locator('text=Compaction failed. Please try again or start a new session.');
await expect(errorMarker).toBeVisible();
});
test('should not show compaction UI when no tokens are used', async () => {
// On a fresh page with no messages, there should be no alert indicator
const alertIndicator = page.locator('[data-testid="alert-indicator"]');
test('should not show compaction UI when no tokens are used', async ({ goosePage }) => {
// On a fresh goosePage with no messages, there should be no alert indicator
const alertIndicator = goosePage.locator('[data-testid="alert-indicator"]');
await expect(alertIndicator).not.toBeVisible();
// The chat input should be available but no context alerts
const chatInput = page.locator('[data-testid="chat-input"]');
const chatInput = goosePage.locator('[data-testid="chat-input"]');
await expect(chatInput).toBeVisible();
});
test('should maintain conversation flow after compaction', async () => {
test('should maintain conversation flow after compaction', async ({ goosePage }) => {
// Generate initial conversation
const chatInput = page.locator('[data-testid="chat-input"]');
const chatInput = goosePage.locator('[data-testid="chat-input"]');
await chatInput.fill('What is React?');
await page.keyboard.press('Enter');
await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
await goosePage.keyboard.press('Enter');
await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
await chatInput.fill('Can you give me an example?');
await page.keyboard.press('Enter');
await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
await goosePage.keyboard.press('Enter');
await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
// Perform compaction
await page.waitForSelector('[data-testid="alert-indicator"]', { timeout: 15000 });
await page.click('[data-testid="alert-indicator"]');
await page.click('text=Compact now');
await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
await goosePage.waitForSelector('[data-testid="alert-indicator"]', { timeout: 15000 });
await goosePage.click('[data-testid="alert-indicator"]');
await goosePage.click('text=Compact now');
await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
// Verify compaction marker
await expect(page.locator('text=Conversation compacted and summarized')).toBeVisible();
await expect(goosePage.locator('text=Conversation compacted and summarized')).toBeVisible();
// Continue conversation after compaction
await chatInput.fill('Thank you, that was helpful. What about Vue.js?');
await page.keyboard.press('Enter');
await goosePage.keyboard.press('Enter');
// Verify that the conversation continues normally
await page.waitForSelector('[data-testid="loading-goose"]', { timeout: 30000 });
await expect(page.locator('[data-testid="loading-goose"]')).toBeVisible();
await goosePage.waitForSelector('[data-testid="loading-goose"]', { timeout: 30000 });
await expect(goosePage.locator('[data-testid="loading-goose"]')).toBeVisible();
// Wait for response
await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
// Verify new message appears after compaction
const messages = page.locator('[data-testid="message"]');
const messages = goosePage.locator('[data-testid="message"]');
const messageCount = await messages.count();
expect(messageCount).toBeGreaterThan(1); // Should have compaction marker + new messages
});
test('should show appropriate loading states during compaction', async () => {
test('should show appropriate loading states during compaction', async ({ goosePage }) => {
// Generate conversation
const chatInput = page.locator('[data-testid="chat-input"]');
const chatInput = goosePage.locator('[data-testid="chat-input"]');
await chatInput.fill('Test message for loading state verification');
await page.keyboard.press('Enter');
await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
await goosePage.keyboard.press('Enter');
await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
// Start compaction
await page.waitForSelector('[data-testid="alert-indicator"]', { timeout: 15000 });
await page.click('[data-testid="alert-indicator"]');
await page.click('text=Compact now');
await goosePage.waitForSelector('[data-testid="alert-indicator"]', { timeout: 15000 });
await goosePage.click('[data-testid="alert-indicator"]');
await goosePage.click('text=Compact now');
// Verify loading state immediately after clicking compact
const loadingGoose = page.locator('[data-testid="loading-goose"]');
const loadingGoose = goosePage.locator('[data-testid="loading-goose"]');
await expect(loadingGoose).toBeVisible();
await expect(loadingGoose).toContainText('goose is compacting the conversation...');
// Verify chat input is disabled during compaction
const submitButton = page.locator('[data-testid="submit-button"]');
const submitButton = goosePage.locator('[data-testid="submit-button"]');
await expect(submitButton).toBeDisabled();
// Wait for compaction to complete
await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
// Verify chat input is re-enabled after compaction
await expect(submitButton).toBeEnabled();
});
test('should handle multiple rapid compaction attempts', async () => {
test('should handle multiple rapid compaction attempts', async ({ goosePage }) => {
// Generate conversation
const chatInput = page.locator('[data-testid="chat-input"]');
const chatInput = goosePage.locator('[data-testid="chat-input"]');
await chatInput.fill('Test message for rapid compaction test');
await page.keyboard.press('Enter');
await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
await goosePage.keyboard.press('Enter');
await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
// Open alert and try to click compact multiple times rapidly
await page.waitForSelector('[data-testid="alert-indicator"]', { timeout: 15000 });
await page.click('[data-testid="alert-indicator"]');
await goosePage.waitForSelector('[data-testid="alert-indicator"]', { timeout: 15000 });
await goosePage.click('[data-testid="alert-indicator"]');
const compactButton = page.locator('text=Compact now');
const compactButton = goosePage.locator('text=Compact now');
await expect(compactButton).toBeVisible();
// Click multiple times rapidly
await compactButton.click();
// The alert should be hidden immediately after first click
const alertBox = page.locator('[role="alert"]');
const alertBox = goosePage.locator('[role="alert"]');
await expect(alertBox).not.toBeVisible();
// Verify only one compaction occurs
await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
const compactionMarkers = page.locator('text=Conversation compacted and summarized');
const compactionMarkers = goosePage.locator('text=Conversation compacted and summarized');
await expect(compactionMarkers).toHaveCount(1);
});
});
ui/desktop/tests/e2e/enhanced-context-management.spec.ts
+170 -202
View File
@@ -1,103 +1,71 @@
import { test, expect, Page } from '@playwright/test';
import { ElectronApplication, _electron as electron } from 'playwright';
import path from 'path';
let electronApp: ElectronApplication;
let page: Page;
import { test, expect } from './fixtures';
test.describe('Enhanced Context Management E2E Tests', () => {
test.beforeAll(async () => {
// Launch Electron app
electronApp = await electron.launch({
args: [path.join(__dirname, '../../.vite/build/main.js')],
env: {
...process.env,
NODE_ENV: 'test',
GOOSE_TEST_MODE: 'true',
},
});
// Get the main window
page = await electronApp.firstWindow();
await page.waitForLoadState('domcontentloaded');
// Wait for the app to be ready
await page.waitForSelector('[data-testid="chat-input"]', { timeout: 10000 });
});
test.afterAll(async () => {
if (electronApp) {
await electronApp.close();
}
});
test.beforeEach(async () => {
// Reset to a clean state before each test
await page.reload();
await page.waitForLoadState('domcontentloaded');
await page.waitForSelector('[data-testid="chat-input"]', { timeout: 10000 });
test.beforeEach(async ({ goosePage }) => {
// Ensure the app is ready before each test
await goosePage.waitForSelector('[data-testid="chat-input"]', { timeout: 10000 });
});
test.describe('Context Window Alert System', () => {
test('should show context window alert only when tokens are being used', async () => {
test('should show context window alert only when tokens are being used', async ({ goosePage }) => {
// Initially, no alert should be visible
const alertIndicator = page.locator('[data-testid="alert-indicator"]');
const alertIndicator = goosePage.locator('[data-testid="alert-indicator"]');
await expect(alertIndicator).not.toBeVisible();
// Type and send a message to generate token usage
const chatInput = page.locator('[data-testid="chat-input"]');
const chatInput = goosePage.locator('[data-testid="chat-input"]');
await chatInput.fill('Hello, this is a test message to generate some token usage.');
await page.keyboard.press('Enter');
await goosePage.keyboard.press('Enter');
// Wait for response and check for context window alert
await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
await page.waitForSelector('[data-testid="alert-indicator"]', { timeout: 15000 });
await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
await goosePage.waitForSelector('[data-testid="alert-indicator"]', { timeout: 15000 });
// Click on the alert indicator to open the popover
await page.click('[data-testid="alert-indicator"]');
await goosePage.click('[data-testid="alert-indicator"]');
// Verify the context window alert is shown
const alertBox = page.locator('[role="alert"]');
const alertBox = goosePage.locator('[role="alert"]');
await expect(alertBox).toBeVisible();
await expect(alertBox).toContainText('Context window');
// Verify progress bar is shown
const progressBar = page.locator('[role="progressbar"]');
const progressBar = goosePage.locator('[role="progressbar"]');
await expect(progressBar).toBeVisible();
// Verify compact button is present
const compactButton = page.locator('text=Compact now');
const compactButton = goosePage.locator('text=Compact now');
await expect(compactButton).toBeVisible();
});
test('should update progress bar as conversation grows', async () => {
const chatInput = page.locator('[data-testid="chat-input"]');
test('should update progress bar as conversation grows', async ({ goosePage }) => {
const chatInput = goosePage.locator('[data-testid="chat-input"]');
// Send first message
await chatInput.fill('First message');
await page.keyboard.press('Enter');
await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
await goosePage.keyboard.press('Enter');
await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
// Get initial progress
await page.waitForSelector('[data-testid="alert-indicator"]', { timeout: 15000 });
await page.click('[data-testid="alert-indicator"]');
await goosePage.waitForSelector('[data-testid="alert-indicator"]', { timeout: 15000 });
await goosePage.click('[data-testid="alert-indicator"]');
const progressText1 = await page.locator('[role="alert"]').textContent();
const progressText1 = await goosePage.locator('[role="alert"]').textContent();
const match1 = progressText1?.match(/(\d+(?:,\d+)*)\s*\/\s*(\d+(?:,\d+)*)/);
const initialTokens = match1 ? parseInt(match1[1].replace(/,/g, '')) : 0;
// Close the alert popover
await page.keyboard.press('Escape');
await goosePage.keyboard.press('Escape');
// Send second message
await chatInput.fill('Second message with more content to increase token usage significantly');
await page.keyboard.press('Enter');
await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
await goosePage.keyboard.press('Enter');
await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
// Get updated progress
await page.click('[data-testid="alert-indicator"]');
await goosePage.click('[data-testid="alert-indicator"]');
const progressText2 = await page.locator('[role="alert"]').textContent();
const progressText2 = await goosePage.locator('[role="alert"]').textContent();
const match2 = progressText2?.match(/(\d+(?:,\d+)*)\s*\/\s*(\d+(?:,\d+)*)/);
const updatedTokens = match2 ? parseInt(match2[1].replace(/,/g, '')) : 0;
@@ -107,8 +75,8 @@ test.describe('Enhanced Context Management E2E Tests', () => {
});
test.describe('Manual Compaction Workflow', () => {
test('should perform complete manual compaction workflow', async () => {
const chatInput = page.locator('[data-testid="chat-input"]');
test('should perform complete manual compaction workflow', async ({ goosePage }) => {
const chatInput = goosePage.locator('[data-testid="chat-input"]');
// Build up conversation with multiple exchanges
const messages = [
@@ -120,64 +88,64 @@ test.describe('Enhanced Context Management E2E Tests', () => {
for (const message of messages) {
await chatInput.fill(message);
await page.keyboard.press('Enter');
await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
await page.waitForTimeout(1000); // Brief pause between messages
await goosePage.keyboard.press('Enter');
await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
await goosePage.waitForTimeout(1000); // Brief pause between messages
}
// Open the alert popover and initiate compaction
await page.waitForSelector('[data-testid="alert-indicator"]', { timeout: 15000 });
await page.click('[data-testid="alert-indicator"]');
await goosePage.waitForSelector('[data-testid="alert-indicator"]', { timeout: 15000 });
await goosePage.click('[data-testid="alert-indicator"]');
const compactButton = page.locator('text=Compact now');
const compactButton = goosePage.locator('text=Compact now');
await expect(compactButton).toBeVisible();
await compactButton.click();
// Verify alert popover closes immediately
const alertBox = page.locator('[role="alert"]');
const alertBox = goosePage.locator('[role="alert"]');
await expect(alertBox).not.toBeVisible();
// Verify compaction loading state
const loadingGoose = page.locator('[data-testid="loading-goose"]');
const loadingGoose = goosePage.locator('[data-testid="loading-goose"]');
await expect(loadingGoose).toBeVisible();
await expect(loadingGoose).toContainText('goose is compacting the conversation...');
// Wait for compaction to complete
await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
// Verify compaction marker appears
const compactionMarker = page.locator('text=Conversation compacted and summarized');
const compactionMarker = goosePage.locator('text=Conversation compacted and summarized');
await expect(compactionMarker).toBeVisible();
// Verify chat input is re-enabled
const submitButton = page.locator('[data-testid="submit-button"]');
const submitButton = goosePage.locator('[data-testid="submit-button"]');
await expect(submitButton).toBeEnabled();
});
test('should hide alert indicator after successful compaction', async () => {
const chatInput = page.locator('[data-testid="chat-input"]');
test('should hide alert indicator after successful compaction', async ({ goosePage }) => {
const chatInput = goosePage.locator('[data-testid="chat-input"]');
// Generate conversation
await chatInput.fill('Test message for compaction');
await page.keyboard.press('Enter');
await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
await goosePage.keyboard.press('Enter');
await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
// Perform compaction
await page.waitForSelector('[data-testid="alert-indicator"]', { timeout: 15000 });
await page.click('[data-testid="alert-indicator"]');
await page.click('text=Compact now');
await goosePage.waitForSelector('[data-testid="alert-indicator"]', { timeout: 15000 });
await goosePage.click('[data-testid="alert-indicator"]');
await goosePage.click('text=Compact now');
// Wait for compaction to complete
await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
// Verify alert indicator is no longer visible (or shows reduced token count)
const alertIndicator = page.locator('[data-testid="alert-indicator"]');
const alertIndicator = goosePage.locator('[data-testid="alert-indicator"]');
// Either the indicator is hidden, or if visible, the token count should be much lower
const isVisible = await alertIndicator.isVisible();
if (isVisible) {
await alertIndicator.click();
const alertContent = await page.locator('[role="alert"]').textContent();
const alertContent = await goosePage.locator('[role="alert"]').textContent();
const match = alertContent?.match(/(\d+(?:,\d+)*)\s*\/\s*(\d+(?:,\d+)*)/);
const currentTokens = match ? parseInt(match[1].replace(/,/g, '')) : 0;
@@ -186,42 +154,42 @@ test.describe('Enhanced Context Management E2E Tests', () => {
}
});
test('should prevent multiple simultaneous compaction attempts', async () => {
const chatInput = page.locator('[data-testid="chat-input"]');
test('should prevent multiple simultaneous compaction attempts', async ({ goosePage }) => {
const chatInput = goosePage.locator('[data-testid="chat-input"]');
// Generate conversation
await chatInput.fill('Test message for multiple compaction prevention');
await page.keyboard.press('Enter');
await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
await goosePage.keyboard.press('Enter');
await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
// Open alert and click compact button
await page.waitForSelector('[data-testid="alert-indicator"]', { timeout: 15000 });
await page.click('[data-testid="alert-indicator"]');
await goosePage.waitForSelector('[data-testid="alert-indicator"]', { timeout: 15000 });
await goosePage.click('[data-testid="alert-indicator"]');
const compactButton = page.locator('text=Compact now');
const compactButton = goosePage.locator('text=Compact now');
await expect(compactButton).toBeVisible();
await compactButton.click();
// Alert should close immediately, preventing further clicks
const alertBox = page.locator('[role="alert"]');
const alertBox = goosePage.locator('[role="alert"]');
await expect(alertBox).not.toBeVisible();
// Verify loading state appears
const loadingGoose = page.locator('[data-testid="loading-goose"]');
const loadingGoose = goosePage.locator('[data-testid="loading-goose"]');
await expect(loadingGoose).toBeVisible();
// Wait for compaction to complete
await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
// Verify only one compaction marker exists
const compactionMarkers = page.locator('text=Conversation compacted and summarized');
const compactionMarkers = goosePage.locator('text=Conversation compacted and summarized');
await expect(compactionMarkers).toHaveCount(1);
});
});
test.describe('Post-Compaction Behavior', () => {
test('should allow scrolling to view ancestor messages after compaction', async () => {
const chatInput = page.locator('[data-testid="chat-input"]');
test('should allow scrolling to view ancestor messages after compaction', async ({ goosePage }) => {
const chatInput = goosePage.locator('[data-testid="chat-input"]');
// Create identifiable messages
const testMessages = [
@@ -233,98 +201,98 @@ test.describe('Enhanced Context Management E2E Tests', () => {
// Send messages
for (const message of testMessages) {
await chatInput.fill(message);
await page.keyboard.press('Enter');
await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
await page.waitForTimeout(1000);
await goosePage.keyboard.press('Enter');
await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
await goosePage.waitForTimeout(1000);
}
// Perform compaction
await page.waitForSelector('[data-testid="alert-indicator"]', { timeout: 15000 });
await page.click('[data-testid="alert-indicator"]');
await page.click('text=Compact now');
await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
await goosePage.waitForSelector('[data-testid="alert-indicator"]', { timeout: 15000 });
await goosePage.click('[data-testid="alert-indicator"]');
await goosePage.click('text=Compact now');
await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
// Verify compaction marker is visible
await expect(page.locator('text=Conversation compacted and summarized')).toBeVisible();
await expect(goosePage.locator('text=Conversation compacted and summarized')).toBeVisible();
// Scroll up to find ancestor messages
const chatContainer = page.locator('[data-testid="chat-container"]');
const chatContainer = goosePage.locator('[data-testid="chat-container"]');
await chatContainer.hover();
// Scroll up multiple times
for (let i = 0; i < 10; i++) {
await page.mouse.wheel(0, -500);
await page.waitForTimeout(100);
await goosePage.mouse.wheel(0, -500);
await goosePage.waitForTimeout(100);
}
// Check if we can find at least one of our original messages
const hasFirstMessage = await page.locator('text=FIRST_UNIQUE_MESSAGE').isVisible();
const hasSecondMessage = await page.locator('text=SECOND_UNIQUE_MESSAGE').isVisible();
const hasThirdMessage = await page.locator('text=THIRD_UNIQUE_MESSAGE').isVisible();
const hasFirstMessage = await goosePage.locator('text=FIRST_UNIQUE_MESSAGE').isVisible();
const hasSecondMessage = await goosePage.locator('text=SECOND_UNIQUE_MESSAGE').isVisible();
const hasThirdMessage = await goosePage.locator('text=THIRD_UNIQUE_MESSAGE').isVisible();
// At least one original message should be visible in the ancestor messages
expect(hasFirstMessage || hasSecondMessage || hasThirdMessage).toBe(true);
});
test('should continue conversation normally after compaction', async () => {
const chatInput = page.locator('[data-testid="chat-input"]');
test('should continue conversation normally after compaction', async ({ goosePage }) => {
const chatInput = goosePage.locator('[data-testid="chat-input"]');
// Generate initial conversation
await chatInput.fill('What is TypeScript?');
await page.keyboard.press('Enter');
await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
await goosePage.keyboard.press('Enter');
await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
await chatInput.fill('Can you give me an example?');
await page.keyboard.press('Enter');
await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
await goosePage.keyboard.press('Enter');
await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
// Perform compaction
await page.waitForSelector('[data-testid="alert-indicator"]', { timeout: 15000 });
await page.click('[data-testid="alert-indicator"]');
await page.click('text=Compact now');
await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
await goosePage.waitForSelector('[data-testid="alert-indicator"]', { timeout: 15000 });
await goosePage.click('[data-testid="alert-indicator"]');
await goosePage.click('text=Compact now');
await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
// Verify compaction completed
await expect(page.locator('text=Conversation compacted and summarized')).toBeVisible();
await expect(goosePage.locator('text=Conversation compacted and summarized')).toBeVisible();
// Continue conversation after compaction
await chatInput.fill('POST_COMPACTION_MESSAGE: Thank you, what about React?');
await page.keyboard.press('Enter');
await goosePage.keyboard.press('Enter');
// Verify conversation continues normally
await expect(page.locator('[data-testid="loading-goose"]')).toBeVisible();
await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
await expect(goosePage.locator('[data-testid="loading-goose"]')).toBeVisible();
await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
// Verify the new message appears
await expect(page.locator('text=POST_COMPACTION_MESSAGE')).toBeVisible();
await expect(goosePage.locator('text=POST_COMPACTION_MESSAGE')).toBeVisible();
// Verify we get a response
const messages = page.locator('[data-testid="message"]');
const messages = goosePage.locator('[data-testid="message"]');
const messageCount = await messages.count();
expect(messageCount).toBeGreaterThan(2); // Should have compaction marker + new messages
});
test('should maintain proper message ordering after compaction', async () => {
const chatInput = page.locator('[data-testid="chat-input"]');
test('should maintain proper message ordering after compaction', async ({ goosePage }) => {
const chatInput = goosePage.locator('[data-testid="chat-input"]');
// Generate conversation
await chatInput.fill('First question about programming');
await page.keyboard.press('Enter');
await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
await goosePage.keyboard.press('Enter');
await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
// Perform compaction
await page.waitForSelector('[data-testid="alert-indicator"]', { timeout: 15000 });
await page.click('[data-testid="alert-indicator"]');
await page.click('text=Compact now');
await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
await goosePage.waitForSelector('[data-testid="alert-indicator"]', { timeout: 15000 });
await goosePage.click('[data-testid="alert-indicator"]');
await goosePage.click('text=Compact now');
await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
// Send new message after compaction
await chatInput.fill('NEW_MESSAGE_AFTER_COMPACTION');
await page.keyboard.press('Enter');
await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
await goosePage.keyboard.press('Enter');
await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
// Verify message order: compaction marker should come before new messages
const allMessages = page.locator('[data-testid="message"]');
const allMessages = goosePage.locator('[data-testid="message"]');
const messageTexts = await allMessages.allTextContents();
const compactionIndex = messageTexts.findIndex(text =>
@@ -340,9 +308,9 @@ test.describe('Enhanced Context Management E2E Tests', () => {
});
test.describe('Error Handling', () => {
test('should handle compaction errors gracefully', async () => {
test('should handle compaction errors gracefully', async ({ goosePage }) => {
// Mock a backend error
await page.route('**/api/sessions/*/manage-context', async (route) => {
await goosePage.route('**/api/sessions/*/manage-context', async (route) => {
await route.fulfill({
status: 500,
contentType: 'application/json',
@@ -350,33 +318,33 @@ test.describe('Enhanced Context Management E2E Tests', () => {
});
});
const chatInput = page.locator('[data-testid="chat-input"]');
const chatInput = goosePage.locator('[data-testid="chat-input"]');
// Generate conversation
await chatInput.fill('Test message for error handling');
await page.keyboard.press('Enter');
await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
await goosePage.keyboard.press('Enter');
await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
// Attempt compaction
await page.waitForSelector('[data-testid="alert-indicator"]', { timeout: 15000 });
await page.click('[data-testid="alert-indicator"]');
await page.click('text=Compact now');
await goosePage.waitForSelector('[data-testid="alert-indicator"]', { timeout: 15000 });
await goosePage.click('[data-testid="alert-indicator"]');
await goosePage.click('text=Compact now');
// Wait for compaction to fail
await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
// Verify error message appears
const errorMarker = page.locator('text=Compaction failed. Please try again or start a new session.');
const errorMarker = goosePage.locator('text=Compaction failed. Please try again or start a new session.');
await expect(errorMarker).toBeVisible();
// Verify chat input is still functional after error
const submitButton = page.locator('[data-testid="submit-button"]');
const submitButton = goosePage.locator('[data-testid="submit-button"]');
await expect(submitButton).toBeEnabled();
});
test('should handle network timeouts during compaction', async () => {
test('should handle network timeouts during compaction', async ({ goosePage }) => {
// Mock a timeout
await page.route('**/api/sessions/*/manage-context', async (route) => {
await goosePage.route('**/api/sessions/*/manage-context', async (route) => {
// Delay response to simulate timeout
await new Promise(resolve => setTimeout(resolve, 5000));
await route.fulfill({
@@ -386,86 +354,86 @@ test.describe('Enhanced Context Management E2E Tests', () => {
});
});
const chatInput = page.locator('[data-testid="chat-input"]');
const chatInput = goosePage.locator('[data-testid="chat-input"]');
// Generate conversation
await chatInput.fill('Test message for timeout handling');
await page.keyboard.press('Enter');
await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
await goosePage.keyboard.press('Enter');
await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
// Attempt compaction
await page.waitForSelector('[data-testid="alert-indicator"]', { timeout: 15000 });
await page.click('[data-testid="alert-indicator"]');
await page.click('text=Compact now');
await goosePage.waitForSelector('[data-testid="alert-indicator"]', { timeout: 15000 });
await goosePage.click('[data-testid="alert-indicator"]');
await goosePage.click('text=Compact now');
// Verify loading state persists during timeout
const loadingGoose = page.locator('[data-testid="loading-goose"]');
const loadingGoose = goosePage.locator('[data-testid="loading-goose"]');
await expect(loadingGoose).toBeVisible();
await expect(loadingGoose).toContainText('goose is compacting the conversation...');
// Wait for timeout to complete
await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 35000 });
await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 35000 });
// Should show error message
const errorMarker = page.locator('text=Compaction failed. Please try again or start a new session.');
const errorMarker = goosePage.locator('text=Compaction failed. Please try again or start a new session.');
await expect(errorMarker).toBeVisible();
});
});
test.describe('UI State Management', () => {
test('should disable chat input during compaction', async () => {
const chatInput = page.locator('[data-testid="chat-input"]');
test('should disable chat input during compaction', async ({ goosePage }) => {
const chatInput = goosePage.locator('[data-testid="chat-input"]');
// Generate conversation
await chatInput.fill('Test message for UI state verification');
await page.keyboard.press('Enter');
await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
await goosePage.keyboard.press('Enter');
await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
// Start compaction
await page.waitForSelector('[data-testid="alert-indicator"]', { timeout: 15000 });
await page.click('[data-testid="alert-indicator"]');
await page.click('text=Compact now');
await goosePage.waitForSelector('[data-testid="alert-indicator"]', { timeout: 15000 });
await goosePage.click('[data-testid="alert-indicator"]');
await goosePage.click('text=Compact now');
// Verify chat input is disabled during compaction
const submitButton = page.locator('[data-testid="submit-button"]');
const submitButton = goosePage.locator('[data-testid="submit-button"]');
await expect(submitButton).toBeDisabled();
// Verify loading message
const loadingGoose = page.locator('[data-testid="loading-goose"]');
const loadingGoose = goosePage.locator('[data-testid="loading-goose"]');
await expect(loadingGoose).toBeVisible();
await expect(loadingGoose).toContainText('goose is compacting the conversation...');
// Wait for compaction to complete
await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
// Verify chat input is re-enabled
await expect(submitButton).toBeEnabled();
});
test('should show appropriate loading states', async () => {
const chatInput = page.locator('[data-testid="chat-input"]');
test('should show appropriate loading states', async ({ goosePage }) => {
const chatInput = goosePage.locator('[data-testid="chat-input"]');
// Generate conversation
await chatInput.fill('Test loading state message');
await page.keyboard.press('Enter');
await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
await goosePage.keyboard.press('Enter');
await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
// Start compaction and immediately check loading state
await page.waitForSelector('[data-testid="alert-indicator"]', { timeout: 15000 });
await page.click('[data-testid="alert-indicator"]');
await page.click('text=Compact now');
await goosePage.waitForSelector('[data-testid="alert-indicator"]', { timeout: 15000 });
await goosePage.click('[data-testid="alert-indicator"]');
await goosePage.click('text=Compact now');
// Verify loading goose appears with correct message
const loadingGoose = page.locator('[data-testid="loading-goose"]');
const loadingGoose = goosePage.locator('[data-testid="loading-goose"]');
await expect(loadingGoose).toBeVisible();
await expect(loadingGoose).toContainText('goose is compacting the conversation...');
// Verify no other loading indicators are shown
const regularLoadingMessages = page.locator('[data-testid="loading-goose"]:not(:has-text("compacting"))');
const regularLoadingMessages = goosePage.locator('[data-testid="loading-goose"]:not(:has-text("compacting"))');
await expect(regularLoadingMessages).not.toBeVisible();
// Wait for completion
await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
// Verify loading state is cleared
await expect(loadingGoose).not.toBeVisible();
@@ -473,8 +441,8 @@ test.describe('Enhanced Context Management E2E Tests', () => {
});
test.describe('Performance and Reliability', () => {
test('should handle large conversations efficiently', async () => {
const chatInput = page.locator('[data-testid="chat-input"]');
test('should handle large conversations efficiently', async ({ goosePage }) => {
const chatInput = goosePage.locator('[data-testid="chat-input"]');
// Generate a larger conversation
const messages = Array.from({ length: 8 }, (_, i) =>
@@ -483,54 +451,54 @@ test.describe('Enhanced Context Management E2E Tests', () => {
for (const message of messages) {
await chatInput.fill(message);
await page.keyboard.press('Enter');
await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
await page.waitForTimeout(500);
await goosePage.keyboard.press('Enter');
await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
await goosePage.waitForTimeout(500);
}
// Perform compaction
await page.waitForSelector('[data-testid="alert-indicator"]', { timeout: 15000 });
await page.click('[data-testid="alert-indicator"]');
await page.click('text=Compact now');
await goosePage.waitForSelector('[data-testid="alert-indicator"]', { timeout: 15000 });
await goosePage.click('[data-testid="alert-indicator"]');
await goosePage.click('text=Compact now');
// Verify compaction completes within reasonable time
await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 45000 });
await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 45000 });
// Verify compaction marker appears
await expect(page.locator('text=Conversation compacted and summarized')).toBeVisible();
await expect(goosePage.locator('text=Conversation compacted and summarized')).toBeVisible();
// Verify system remains responsive
await chatInput.fill('Post-compaction test message');
await page.keyboard.press('Enter');
await expect(page.locator('[data-testid="loading-goose"]')).toBeVisible();
await goosePage.keyboard.press('Enter');
await expect(goosePage.locator('[data-testid="loading-goose"]')).toBeVisible();
});
test('should maintain conversation context after compaction', async () => {
const chatInput = page.locator('[data-testid="chat-input"]');
test('should maintain conversation context after compaction', async ({ goosePage }) => {
const chatInput = goosePage.locator('[data-testid="chat-input"]');
// Create conversation with specific context
await chatInput.fill('My name is Alice and I am a software developer working on React applications.');
await page.keyboard.press('Enter');
await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
await goosePage.keyboard.press('Enter');
await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
await chatInput.fill('I am having trouble with useState hooks. Can you help?');
await page.keyboard.press('Enter');
await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
await goosePage.keyboard.press('Enter');
await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
// Perform compaction
await page.waitForSelector('[data-testid="alert-indicator"]', { timeout: 15000 });
await page.click('[data-testid="alert-indicator"]');
await page.click('text=Compact now');
await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
await goosePage.waitForSelector('[data-testid="alert-indicator"]', { timeout: 15000 });
await goosePage.click('[data-testid="alert-indicator"]');
await goosePage.click('text=Compact now');
await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
// Test if context is maintained by asking a follow-up question
await chatInput.fill('What did I tell you my name was?');
await page.keyboard.press('Enter');
await page.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
await goosePage.keyboard.press('Enter');
await goosePage.waitForSelector('[data-testid="loading-goose"]', { state: 'hidden', timeout: 30000 });
// The response should ideally reference the name Alice or indicate context retention
// Note: This is a behavioral test that depends on the AI's ability to use the summary
const messages = page.locator('[data-testid="message"]');
const messages = goosePage.locator('[data-testid="message"]');
const lastMessageText = await messages.last().textContent();
// The system should have some response (not just an error)
ui/desktop/tests/e2e/fixtures.ts
+170
View File
@@ -0,0 +1,170 @@
import { test as base, Page, Browser, chromium } from '@playwright/test';
import { spawn, ChildProcess } from 'child_process';
import { join } from 'path';
import { promisify } from 'util';
const execAsync = promisify(require('child_process').exec);
type GooseTestFixtures = {
goosePage: Page;
};
/**
* Test-scoped fixture that launches a fresh Electron app for EACH test.
*
* Isolation: ⚠️ Partial - each test gets a fresh app instance, but uses ambient user config
* Speed: ⚠️ Slow - ~3s startup overhead per test
*
* This ensures each test starts with a fresh app instance, but the app uses the
* user's existing Goose configuration (providers, models, etc.).
*
* Usage:
* import { test, expect } from './fixtures';
*
* test('my test', async ({ goosePage }) => {
* await goosePage.waitForSelector('[data-testid="chat-input"]');
* // ... test code
* });
*/
export const test = base.extend<GooseTestFixtures>({
// Test-scoped fixture: launches a fresh Electron app for each test
goosePage: async ({}, use, testInfo) => {
console.log(`Launching fresh Electron app for test: ${testInfo.title}`);
let appProcess: ChildProcess | null = null;
let browser: Browser | null = null;
try {
// Assign a unique debug port for this test to enable parallel execution
// Base port 9222, offset by worker index * 100 + parallel slot
const debugPort = 9222 + (testInfo.parallelIndex * 10);
console.log(`Using debug port ${debugPort} for parallel test execution`);
// Start the electron-forge process with Playwright remote debugging enabled
// Use detached mode on Unix to create a process group we can kill together
appProcess = spawn('npm', ['run', 'start-gui'], {
cwd: join(__dirname, '../..'),
stdio: 'pipe',
detached: process.platform !== 'win32',
env: {
...process.env,
ELECTRON_IS_DEV: '1',
NODE_ENV: 'development',
GOOSE_ALLOWLIST_BYPASS: 'true',
ENABLE_PLAYWRIGHT: 'true',
PLAYWRIGHT_DEBUG_PORT: debugPort.toString(), // Unique port per test for parallel execution
RUST_LOG: 'info', // Enable info-level logging for goosed backend
}
});
// Log process output for debugging
if (process.env.DEBUG_TESTS) {
appProcess.stdout?.on('data', (data) => {
console.log('App stdout:', data.toString());
});
appProcess.stderr?.on('data', (data) => {
console.log('App stderr:', data.toString());
});
}
// Wait for the app to start and remote debugging to be available
// Retry connection until it succeeds (app is ready) or timeout
console.log(`Waiting for Electron app to start on port ${debugPort}...`);
const maxRetries = 100; // 100 retries * 100ms = 10 seconds max
const retryDelay = 100; // 100ms between retries
for (let attempt = 1; attempt <= maxRetries; attempt++) {
try {
browser = await chromium.connectOverCDP(`http://127.0.0.1:${debugPort}`);
console.log(`Connected to Electron app on attempt ${attempt} (~${(attempt * retryDelay) / 1000}s)`);
break;
} catch (error) {
if (attempt === maxRetries) {
throw new Error(`Failed to connect to Electron app after ${maxRetries} attempts (${(maxRetries * retryDelay) / 1000}s). Last error: ${error.message}`);
}
// Wait before next retry
await new Promise(resolve => setTimeout(resolve, retryDelay));
}
}
if (!browser) {
throw new Error('Browser connection failed unexpectedly');
}
// Get the electron app context and first page
const contexts = browser.contexts();
if (contexts.length === 0) {
throw new Error('No browser contexts found');
}
const pages = contexts[0].pages();
if (pages.length === 0) {
throw new Error('No windows/pages found');
}
const page = pages[0];
// Wait for page to be ready
await page.waitForLoadState('domcontentloaded');
// Try to wait for networkidle
try {
await page.waitForLoadState('networkidle', { timeout: 10000 });
} catch (error) {
console.log('NetworkIdle timeout (likely due to MCP activity), continuing...');
}
// Wait for React app to be ready
await page.waitForFunction(() => {
const root = document.getElementById('root');
return root && root.children.length > 0;
}, { timeout: 30000 });
console.log('App ready, starting test...');
// Provide the page to the test
await use(page);
} finally {
console.log('Cleaning up Electron app for this test...');
// Close the CDP connection
if (browser) {
await browser.close().catch(console.error);
}
// Kill the npm process tree
if (appProcess && appProcess.pid) {
try {
if (process.platform === 'win32') {
// On Windows, kill the entire process tree
await execAsync(`taskkill /F /T /PID ${appProcess.pid}`);
} else {
// On Unix, kill the entire process group
try {
// First try SIGTERM for graceful shutdown
process.kill(-appProcess.pid, 'SIGTERM');
await new Promise(resolve => setTimeout(resolve, 2000));
} catch (e) {
// Process might already be dead
}
// Then SIGKILL if still running
try {
process.kill(-appProcess.pid, 'SIGKILL');
} catch (e) {
// Process already exited
}
}
console.log('Cleaned up app process');
} catch (error) {
if (error.code !== 'ESRCH' && !error.message?.includes('No such process')) {
console.error('Error killing app process:', error);
}
}
}
}
},
});
export { expect } from '@playwright/test';
ui/desktop/tests/e2e/performance.spec.ts
+330
View File
@@ -0,0 +1,330 @@
import { test, expect } from './fixtures';
test.describe('Performance Tests', () => {
test('measure end-to-end performance for prompt submission', async ({ goosePage }) => {
// Start Playwright tracing to capture all performance data
await goosePage.context().tracing.start({
screenshots: true,
snapshots: true,
sources: true
});
console.log('\n=== Performance Test Started ===\n');
// Mark: App ready
await goosePage.waitForSelector('[data-testid="chat-input"]', { timeout: 30000 });
await goosePage.evaluate(() => performance.mark('app-ready'));
console.log('✓ App ready');
// Prepare prompt
const chatInput = await goosePage.waitForSelector('[data-testid="chat-input"]');
const testPrompt = 'Write a haiku about testing software';
await chatInput.fill(testPrompt);
// Mark: Prompt submit
await goosePage.evaluate(() => performance.mark('prompt-submit-start'));
await chatInput.press('Enter');
await goosePage.evaluate(() => performance.mark('prompt-submitted'));
// Wait for loading indicator to appear and check if it's "loading conversation..."
await goosePage.waitForSelector('[data-testid="loading-indicator"]', {
state: 'visible',
timeout: 5000
});
const loadingText = await goosePage.locator('[data-testid="loading-indicator"]').textContent();
if (loadingText?.includes('loading conversation')) {
await goosePage.evaluate(() => performance.mark('loading-conversation-start'));
console.log('✓ Loading conversation detected');
// Wait for it to change or disappear
await goosePage.waitForFunction(() => {
const indicator = document.querySelector('[data-testid="loading-indicator"]');
if (!indicator) return true; // Disappeared
const text = indicator.textContent || '';
return !text.includes('loading conversation'); // Changed to different state
}, { timeout: 30000 });
await goosePage.evaluate(() => performance.mark('loading-conversation-end'));
console.log('✓ Loading conversation complete');
}
await goosePage.evaluate(() => performance.mark('loading-started'));
// Monitor for first token (first visible response content)
let firstTokenDetected = false;
const checkForFirstToken = async () => {
while (!firstTokenDetected) {
try {
const messageContainers = await goosePage.locator('[data-testid="message-container"]').all();
if (messageContainers.length > 0) {
const lastMessage = messageContainers[messageContainers.length - 1];
const content = await lastMessage.textContent();
if (content && content.trim().length > 0) {
await goosePage.evaluate(() => performance.mark('first-token-received'));
firstTokenDetected = true;
console.log('✓ First token detected');
break;
}
}
} catch (e) {
// Continue checking
}
await goosePage.waitForTimeout(50);
}
};
// Start checking for first token
const firstTokenPromise = checkForFirstToken();
await firstTokenPromise;
// Wait for response to complete
await goosePage.waitForSelector('[data-testid="loading-indicator"]', {
state: 'hidden',
timeout: 60000
});
await goosePage.evaluate(() => performance.mark('response-complete'));
console.log('✓ Response complete');
// Create performance measures
await goosePage.evaluate(() => {
// Measure loading conversation if it was detected
const marks = performance.getEntriesByType('mark').map(m => m.name);
if (marks.includes('loading-conversation-start') && marks.includes('loading-conversation-end')) {
performance.measure('loading-conversation-duration', 'loading-conversation-start', 'loading-conversation-end');
}
performance.measure('time-to-prompt-submit', 'prompt-submit-start', 'prompt-submitted');
performance.measure('time-to-first-token', 'prompt-submitted', 'first-token-received');
performance.measure('time-to-complete-response', 'prompt-submitted', 'response-complete');
performance.measure('streaming-duration', 'first-token-received', 'response-complete');
performance.measure('total-interaction', 'prompt-submit-start', 'response-complete');
});
// Extract and display performance metrics
const metrics = await goosePage.evaluate(() => {
const measures = performance.getEntriesByType('measure');
const result: Record<string, number> = {};
measures.forEach(measure => {
result[measure.name] = Math.round(measure.duration);
});
return result;
});
console.log('\n=== Performance Metrics ===');
if (metrics['loading-conversation-duration']) {
console.log(`Loading Conversation: ${metrics['loading-conversation-duration']}ms`);
}
console.log(`Time to Submit Prompt: ${metrics['time-to-prompt-submit']}ms`);
console.log(`Time to First Token (TTFT): ${metrics['time-to-first-token']}ms`);
console.log(`Time to Complete Response: ${metrics['time-to-complete-response']}ms`);
console.log(`Streaming Duration: ${metrics['streaming-duration']}ms`);
console.log(`Total Interaction Time: ${metrics['total-interaction']}ms`);
console.log('===========================\n');
// Verify we got a response
const response = await goosePage.locator('[data-testid="message-container"]').last();
const responseText = await response.textContent();
expect(responseText).toBeTruthy();
expect(responseText!.length).toBeGreaterThan(0);
// Assert performance thresholds
expect(metrics['time-to-first-token']).toBeLessThan(10000); // First token in < 10s
expect(metrics['time-to-complete-response']).toBeLessThan(60000); // Complete in < 60s
// Stop tracing and save
const tracePath = test.info().outputPath('trace.zip');
await goosePage.context().tracing.stop({ path: tracePath });
console.log(`✓ Performance trace saved to: ${tracePath}`);
console.log(` View with: npx playwright show-trace ${tracePath}\n`);
// Attach metrics as JSON
await test.info().attach('performance-metrics.json', {
body: JSON.stringify(metrics, null, 2),
contentType: 'application/json',
});
});
test('measure cold start vs warm cache performance', async ({ goosePage }) => {
await goosePage.context().tracing.start({ screenshots: true, snapshots: true });
console.log('\n=== Cold vs Warm Performance ===\n');
// Cold start measurement
await goosePage.waitForSelector('[data-testid="chat-input"]', { timeout: 30000 });
await goosePage.evaluate(() => performance.mark('app-ready'));
// First prompt (cold)
const chatInput = await goosePage.waitForSelector('[data-testid="chat-input"]');
await chatInput.fill('Say hello');
await goosePage.evaluate(() => performance.mark('cold-prompt-start'));
await chatInput.press('Enter');
await goosePage.waitForSelector('[data-testid="loading-indicator"]', {
state: 'visible',
timeout: 5000
});
await goosePage.waitForSelector('[data-testid="loading-indicator"]', {
state: 'hidden',
timeout: 60000
});
await goosePage.evaluate(() => {
performance.mark('cold-prompt-complete');
performance.measure('cold-prompt-duration', 'cold-prompt-start', 'cold-prompt-complete');
});
console.log('✓ Cold prompt complete');
// Second prompt (warm)
const chatInput2 = await goosePage.waitForSelector('[data-testid="chat-input"]');
await chatInput2.fill('Say goodbye');
await goosePage.evaluate(() => performance.mark('warm-prompt-start'));
await chatInput2.press('Enter');
await goosePage.waitForSelector('[data-testid="loading-indicator"]', {
state: 'visible',
timeout: 5000
});
await goosePage.waitForSelector('[data-testid="loading-indicator"]', {
state: 'hidden',
timeout: 60000
});
await goosePage.evaluate(() => {
performance.mark('warm-prompt-complete');
performance.measure('warm-prompt-duration', 'warm-prompt-start', 'warm-prompt-complete');
});
console.log('✓ Warm prompt complete');
// Extract metrics
const metrics = await goosePage.evaluate(() => {
const measures = performance.getEntriesByType('measure');
const result: Record<string, number> = {};
measures.forEach(measure => {
result[measure.name] = Math.round(measure.duration);
});
return result;
});
const coldDuration = metrics['cold-prompt-duration'];
const warmDuration = metrics['warm-prompt-duration'];
const improvement = ((coldDuration - warmDuration) / coldDuration * 100).toFixed(1);
console.log('\n=== Results ===');
console.log(`Cold Prompt Duration: ${coldDuration}ms`);
console.log(`Warm Prompt Duration: ${warmDuration}ms`);
console.log(`Improvement: ${improvement}%`);
console.log('================\n');
// Save trace
const tracePath = test.info().outputPath('cold-vs-warm-trace.zip');
await goosePage.context().tracing.stop({ path: tracePath });
console.log(`✓ Trace saved to: ${tracePath}\n`);
// Attach results
await test.info().attach('cold-vs-warm.json', {
body: JSON.stringify({
coldDuration,
warmDuration,
improvement: `${improvement}%`
}, null, 2),
contentType: 'application/json',
});
});
test('capture full performance profile with navigation timing', async ({ goosePage }) => {
await goosePage.context().tracing.start({ screenshots: true, snapshots: true });
console.log('\n=== Full Performance Profile ===\n');
// Get navigation timing
const navigationTiming = await goosePage.evaluate(() => {
const perf = performance.getEntriesByType('navigation')[0] as PerformanceNavigationTiming;
return {
domContentLoaded: Math.round(perf.domContentLoadedEventEnd - perf.fetchStart),
loadComplete: Math.round(perf.loadEventEnd - perf.fetchStart),
domInteractive: Math.round(perf.domInteractive - perf.fetchStart),
};
});
console.log('Navigation Timing:');
console.log(` DOM Content Loaded: ${navigationTiming.domContentLoaded}ms`);
console.log(` DOM Interactive: ${navigationTiming.domInteractive}ms`);
console.log(` Load Complete: ${navigationTiming.loadComplete}ms`);
// Wait for app ready
await goosePage.waitForSelector('[data-testid="chat-input"]', { timeout: 30000 });
await goosePage.evaluate(() => performance.mark('app-interactive'));
// Measure time from navigation to interactive
const appReadyTime = await goosePage.evaluate(() => {
const appInteractive = performance.getEntriesByName('app-interactive')[0];
return Math.round(appInteractive.startTime);
});
console.log(` App Interactive: ${appReadyTime}ms\n`);
// Submit a prompt and measure
const chatInput = await goosePage.waitForSelector('[data-testid="chat-input"]');
await chatInput.fill('Hello');
await goosePage.evaluate(() => performance.mark('user-interaction-start'));
await chatInput.press('Enter');
await goosePage.waitForSelector('[data-testid="loading-indicator"]', { state: 'visible', timeout: 5000 });
await goosePage.waitForSelector('[data-testid="loading-indicator"]', { state: 'hidden', timeout: 60000 });
await goosePage.evaluate(() => {
performance.mark('user-interaction-complete');
performance.measure('user-interaction-duration', 'user-interaction-start', 'user-interaction-complete');
});
const interactionTime = await goosePage.evaluate(() => {
const measure = performance.getEntriesByName('user-interaction-duration')[0];
return Math.round(measure.duration);
});
console.log(`User Interaction Duration: ${interactionTime}ms\n`);
// Get resource timing summary
const resourceStats = await goosePage.evaluate(() => {
const resources = performance.getEntriesByType('resource');
const types: Record<string, number> = {};
resources.forEach(resource => {
const type = (resource as PerformanceResourceTiming).initiatorType;
types[type] = (types[type] || 0) + 1;
});
return {
total: resources.length,
byType: types
};
});
console.log('Resource Loading:');
console.log(` Total Resources: ${resourceStats.total}`);
console.log(` By Type:`, resourceStats.byType);
console.log('\n==============================\n');
// Save trace
const tracePath = test.info().outputPath('full-profile-trace.zip');
await goosePage.context().tracing.stop({ path: tracePath });
console.log(`✓ Full trace saved to: ${tracePath}\n`);
// Attach all metrics
await test.info().attach('full-performance-profile.json', {
body: JSON.stringify({
navigationTiming,
appReadyTime,
interactionTime,
resourceStats
}, null, 2),
contentType: 'application/json',
});
});
});