Fix e2e tests (#2256)

ui/desktop/package.json

@@ -20,6 +20,7 @@
     "bundle:intel": "npm run make -- --arch=x64 && cd out/Goose-darwin-x64 && ditto -c -k --sequesterRsrc --keepParent Goose.app Goose_intel_mac.zip",
     "debug": "echo 'run --remote-debugging-port=8315' && lldb out/Goose-darwin-arm64/Goose.app",
     "test-e2e": "npm run generate-api && playwright test",
+    "test-e2e:dev": "npm run generate-api && playwright test --reporter=list --retries=0 --max-failures=1",
     "test-e2e:ui": "npm run generate-api && playwright test --ui",
     "test-e2e:debug": "npm run generate-api && playwright test --debug",
     "test-e2e:report": "playwright show-report",

ui/desktop/src/components/ChatView.tsx

@@ -391,7 +391,11 @@ export default function ChatView({
           <ScrollArea ref={scrollRef} className="flex-1" autoScroll>
             <SearchView>
               {filteredMessages.map((message, index) => (
-                <div key={message.id || index} className="mt-4 px-4">
+                <div
+                  key={message.id || index}
+                  className="mt-4 px-4"
+                  data-testid="message-container"
+                >
                   {isUserMessage(message) ? (
                     <UserMessage message={message} />
                   ) : (

ui/desktop/src/components/Input.tsx

@@ -194,6 +194,7 @@ export default function Input({
       className="flex relative h-auto px-[16px] pr-[68px] py-[1rem] border-t border-borderSubtle"
     >
       <textarea
+        data-testid="chat-input"
         autoFocus
         id="dynamic-textarea"
         placeholder="What can goose help with?   ⌘↑/⌘↓"

ui/desktop/src/components/LoadingGoose.tsx

@@ -4,7 +4,10 @@ import GooseLogo from './GooseLogo';
 const LoadingGoose = () => {
   return (
     <div className="w-full pb-[2px]">
-      <div className="flex items-center text-xs text-textStandard mb-2 mt-2 pl-4 animate-[appear_250ms_ease-in_forwards]">
+      <div
+        data-testid="loading-indicator"
+        className="flex items-center text-xs text-textStandard mb-2 mt-2 pl-4 animate-[appear_250ms_ease-in_forwards]"
+      >
         <GooseLogo className="mr-2" size="small" hover={false} />
         goose is working on it…
       </div>

ui/desktop/src/components/WelcomeView.tsx

@@ -50,7 +50,10 @@ export default function WelcomeScreen({ onSubmit }: WelcomeScreenProps) {
 
             {/* ProviderGrid */}
             <div className="w-full">
-              <h2 className="text-3xl font-medium text-textStandard tracking-tight mb-2">
+              <h2
+                className="text-3xl font-medium text-textStandard tracking-tight mb-2"
+                data-testid="provider-selection-heading"
+              >
                 Choose a Provider
               </h2>
               <p className="text-xl text-textStandard mb-4">

ui/desktop/src/components/more_menu/MoreMenu.tsx

@@ -52,6 +52,7 @@ const ThemeSelect: React.FC<ThemeSelectProps> = ({ themeMode, onThemeChange }) =
       <div className="text-sm mb-2">Theme</div>
       <div className="grid grid-cols-3 gap-2">
         <button
+          data-testid="light-mode-button"
           onClick={() => onThemeChange('light')}
           className={`flex items-center justify-center gap-2 p-2 rounded-md border transition-colors ${
             themeMode === 'light'
@@ -64,6 +65,7 @@ const ThemeSelect: React.FC<ThemeSelectProps> = ({ themeMode, onThemeChange }) =
         </button>
 
         <button
+          data-testid="dark-mode-button"
           onClick={() => onThemeChange('dark')}
           className={`flex items-center justify-center gap-2 p-2 rounded-md border transition-colors ${
             themeMode === 'dark'
@@ -76,6 +78,7 @@ const ThemeSelect: React.FC<ThemeSelectProps> = ({ themeMode, onThemeChange }) =
         </button>
 
         <button
+          data-testid="system-mode-button"
           onClick={() => onThemeChange('system')}
           className={`flex items-center justify-center gap-2 p-2 rounded-md border transition-colors ${
             themeMode === 'system'
@@ -162,9 +165,9 @@ export default function MoreMenu({
     <Popover open={open} onOpenChange={setOpen}>
       <PopoverTrigger asChild>
         <button
+          data-testid="more-options-button"
           className={`z-[100] absolute top-2 right-4 w-[20px] h-[20px] transition-colors cursor-pointer no-drag hover:text-textProminent ${open ? 'text-textProminent' : 'text-textSubtle'}`}
           role="button"
-          aria-label="More options"
         >
           <More />
         </button>
@@ -265,6 +268,7 @@ export default function MoreMenu({
 
               {settingsV2Enabled && (
                 <MenuButton
+                  data-testid="reset-provider-button"
                   onClick={async () => {
                     await remove('GOOSE_PROVIDER', false);
                     await remove('GOOSE_MODEL', false);
@@ -282,6 +286,7 @@ export default function MoreMenu({
 
               {!settingsV2Enabled && (
                 <MenuButton
+                  data-testid="reset-provider-button"
                   onClick={() => {
                     localStorage.removeItem('GOOSE_PROVIDER');
                     setOpen(false);

ui/desktop/src/components/settings/providers/BaseProviderGrid.tsx

@@ -67,7 +67,10 @@ function BaseProviderCard({
   const tooltipText = numRequiredKeys === 1 ? `Add ${name} API Key` : `Add ${name} API Keys`;
 
   return (
-    <div className="relative h-full p-[2px] overflow-hidden rounded-[9px] group/card bg-borderSubtle hover:bg-transparent hover:duration-300">
+    <div
+      className="relative h-full p-[2px] overflow-hidden rounded-[9px] group/card bg-borderSubtle hover:bg-transparent hover:duration-300"
+      data-testid={`provider-card-${name.toLowerCase()}`}
+    >
       {/* Glowing ring */}
       <div
         className={`absolute pointer-events-none w-[260px] h-[260px] top-[-50px] left-[-30px] origin-center bg-[linear-gradient(45deg,#13BBAF,#FF4F00)] animate-[rotate_6s_linear_infinite] z-[-1] ${
@@ -195,6 +198,7 @@ function BaseProviderCard({
               <Tooltip>
                 <TooltipTrigger asChild>
                   <Button
+                    data-testid="provider-launch-button"
                     variant="default"
                     size="sm"
                     onClick={(e) => {

ui/desktop/src/components/settings_v2/providers/ProviderSettingsPage.tsx

@@ -98,7 +98,10 @@ export default function ProviderSettings({ onClose, isOnboarding }: ProviderSett
         <div className="px-8 pt-6 pb-4">
           {/* Only show back button if not in onboarding mode */}
           {!isOnboarding && <BackButton onClick={onClose} />}
-          <h1 className="text-3xl font-medium text-textStandard mt-1">
+          <h1
+            className="text-3xl font-medium text-textStandard mt-1"
+            data-testid="provider-selection-heading"
+          >
             {isOnboarding ? 'Configure your providers' : 'Provider Configuration Settings'}
           </h1>
           {isOnboarding && (

ui/desktop/src/components/settings_v2/providers/subcomponents/CardContainer.tsx

@@ -5,6 +5,7 @@ interface CardContainerProps {
   body: React.ReactNode;
   onClick: () => void;
   grayedOut: boolean;
+  testId?: string;
 }
 
 function GlowingRing() {
@@ -31,9 +32,11 @@ export default function CardContainer({
   body,
   onClick,
   grayedOut = false,
+  testId,
 }: CardContainerProps) {
   return (
     <div
+      data-testid={testId}
       className={`relative h-full p-[1px] overflow-hidden rounded-[9px] group/card 
                  ${
                    grayedOut

ui/desktop/src/components/settings_v2/providers/subcomponents/ProviderCard.tsx

@@ -37,6 +37,7 @@ export const ProviderCard = memo(function ProviderCard({
 
   return (
     <CardContainer
+      testId={`provider-card-${provider.name.toLowerCase()}`}
       grayedOut={!provider.is_configured && isOnboarding} // onboarding page will have grayed out cards if not configured
       onClick={handleCardClick}
       header={

ui/desktop/src/components/settings_v2/providers/subcomponents/buttons/CardButtons.tsx

@@ -110,6 +110,7 @@ export function ConfigureSettingsButton({ tooltip, className, ...props }: Action
 export function RocketButton({ tooltip, className, ...props }: ActionButtonProps) {
   return (
     <ActionButton
+      data-testid="provider-launch-button"
       icon={Rocket}
       tooltip={tooltip}
       className={className}

ui/desktop/tests/e2e/app.spec.ts

@@ -11,7 +11,6 @@ const execAsync = promisify(exec);
 // Define provider interface
 type Provider = {
   name: string;
-  testPath: string;
 };
 
 // Create test fixture type
@@ -19,10 +18,10 @@ type TestFixtures = {
   provider: Provider;
 };
 
-// Define available providers
+// Define available providers, keeping as a list of objects for easy expansion
 const providers: Provider[] = [
-  { name: 'Databricks', testPath: 'div:has(h3:text("Databricks"))[class*="relative bg-bgApp rounded-lg"]' },
-  { name: 'Google', testPath: 'div:has(h3:text("Google"))[class*="relative bg-bgApp rounded-lg"]' }
+  { name: 'Databricks' },
+  { name: 'Google' }
 ];
 
 // Create test with fixtures
@@ -59,49 +58,53 @@ test.afterEach(async () => {
 async function selectProvider(mainWindow: any, provider: Provider) {
   console.log(`Selecting provider: ${provider.name}`);
   
-  // Click the menu button (3 dots) if we're in chat
-  try {
-    // Wait for header and menu button to be visible
-    await mainWindow.waitForSelector('div[class*="bg-bgSubtle border-b border-borderSubtle"]', { timeout: 5000 });
-    await mainWindow.waitForTimeout(1000); // Give UI time to stabilize
-    
-    const menuButton = await mainWindow.waitForSelector('button[aria-label="More options"]', { 
-      timeout: 2000,
+  // If we're already in the chat interface, we need to reset providers
+  const chatTextarea = await mainWindow.waitForSelector('[data-testid="chat-input"]', { 
+    timeout: 2000
+  }).catch(() => null);
+
+  if (chatTextarea) {
+    // Click menu button to reset providers
+    console.log('Opening menu to reset providers...');
+    const menuButton = await mainWindow.waitForSelector('[data-testid="more-options-button"]', {
+      timeout: 5000,
       state: 'visible'
     });
     await menuButton.click();
-    
-    // Wait for menu to be visible
-    await mainWindow.waitForTimeout(1000);
-    
-    // Click "Reset provider and model"
-    const resetProviderButton = await mainWindow.waitForSelector('button:has-text("Reset provider and model")', { timeout: 2000 });
-    await resetProviderButton.click();
 
-    // Wait for page to start refreshing
+    // Wait for menu to appear and be interactive
     await mainWindow.waitForTimeout(1000);
 
-    // Wait for page to finish reloading
-    await mainWindow.reload();
-    await mainWindow.waitForLoadState('networkidle');
-    
-  } catch (e) {
-    console.log('Already on provider selection screen or error:', e);
+    // Click Reset Provider and Model
+    console.log('Clicking Reset provider and model...');
+    const resetButton = await mainWindow.waitForSelector('button:has-text("Reset provider and model")', {
+      timeout: 5000,
+      state: 'visible'
+    });
+    await resetButton.click();
   }
 
-  // Wait for provider selection screen
-  const heading = await mainWindow.waitForSelector('h2:has-text("Choose a Provider")', { timeout: 2000 });
-  const headingText = await heading.textContent();
-  expect(headingText).toBe('Choose a Provider');
+  // Wait for React app to be ready and animations to complete
+  await mainWindow.waitForFunction(() => {
+    const root = document.getElementById('root');
+    return root && root.children.length > 0;
+  });
+  await mainWindow.waitForTimeout(2000);
+
+  // Take a screenshot before proceeding
+  await mainWindow.screenshot({ path: `test-results/before-provider-${provider.name.toLowerCase()}-check.png` });
+
+  // We should now be at provider selection
+  await mainWindow.waitForSelector('[data-testid="provider-selection-heading"]');
 
   // Find and verify the provider card container
   console.log(`Looking for ${provider.name} card...`);
-  const providerContainer = await mainWindow.waitForSelector(provider.testPath);
+  const providerContainer = await mainWindow.waitForSelector(`[data-testid="provider-card-${provider.name.toLowerCase()}"]`);
   expect(await providerContainer.isVisible()).toBe(true);
 
   // Find the Launch button within the provider container
   console.log(`Looking for Launch button in ${provider.name} card...`);
-  const launchButton = await providerContainer.waitForSelector('button:has-text("Launch")');
+  const launchButton = await providerContainer.waitForSelector('[data-testid="provider-launch-button"]');
   expect(await launchButton.isVisible()).toBe(true);
 
   // Take screenshot before clicking
@@ -111,9 +114,9 @@ async function selectProvider(mainWindow: any, provider: Provider) {
   await launchButton.click();
 
   // Wait for chat interface to appear
-  const chatTextarea = await mainWindow.waitForSelector('textarea[placeholder*="What can goose help with?"]',
-    { timeout: 5000 });
-  expect(await chatTextarea.isVisible()).toBe(true);
+  const chatTextareaAfterClick = await mainWindow.waitForSelector('[data-testid="chat-input"]',
+    { timeout: 2000 });
+  expect(await chatTextareaAfterClick.isVisible()).toBe(true);
 
   // Take screenshot of chat interface
   await mainWindow.screenshot({ path: `test-results/chat-interface-${provider.name.toLowerCase()}.png` });
@@ -134,7 +137,8 @@ test.describe('Goose App', () => {
       env: {
         ...process.env,
         ELECTRON_IS_DEV: '1',
-        NODE_ENV: 'development'
+        NODE_ENV: 'development',
+        GOOSE_ALLOWLIST_BYPASS: 'true',
       }
     });
 
@@ -169,6 +173,23 @@ test.describe('Goose App', () => {
     // Get the main window once for all tests
     mainWindow = await electronApp.firstWindow();
     await mainWindow.waitForLoadState('domcontentloaded');
+    await mainWindow.waitForLoadState('networkidle');
+
+    // Wait for React app to be ready by checking for the root element to have content
+    await mainWindow.waitForFunction(() => {
+      const root = document.getElementById('root');
+      return root && root.children.length > 0;
+    });
+
+    // Wait for any animations to complete
+    await mainWindow.waitForTimeout(2000);
+
+    // Take a screenshot to debug what's on the screen
+    await mainWindow.screenshot({ path: 'test-results/initial-load.png' });
+
+    // Debug: print out the page content
+    const content = await mainWindow.content();
+    console.log('Page content:', content);
   });
 
   test.afterAll(async () => {
@@ -225,16 +246,16 @@ test.describe('Goose App', () => {
   
       // Click the three dots menu button in the top right
       await mainWindow.waitForSelector('div[class*="bg-bgSubtle border-b border-borderSubtle"]');
-      const menuButton = await mainWindow.waitForSelector('button[aria-label="More options"]', {
+      const menuButton = await mainWindow.waitForSelector('[data-testid="more-options-button"]', {
         timeout: 5000,
         state: 'visible'
       });
       await menuButton.click();
   
       // Find and click the dark mode toggle button
-      const darkModeButton = await mainWindow.waitForSelector('button:has-text("Dark")');
-      const lightModeButton = await mainWindow.waitForSelector('button:has-text("Light")');
-      const systemModeButton = await mainWindow.waitForSelector('button:has-text("System")');
+      const darkModeButton = await mainWindow.waitForSelector('[data-testid="dark-mode-button"]');
+      const lightModeButton = await mainWindow.waitForSelector('[data-testid="light-mode-button"]');
+      const systemModeButton = await mainWindow.waitForSelector('[data-testid="system-mode-button"]');
 
       // Get initial state
       const isDarkMode = await mainWindow.evaluate(() => document.documentElement.classList.contains('dark'));
@@ -282,7 +303,7 @@ test.describe('Goose App', () => {
           console.log(`Testing chat interaction with ${provider.name}...`);
     
           // Find the chat input
-          const chatInput = await mainWindow.waitForSelector('textarea[placeholder*="What can goose help with?"]');
+          const chatInput = await mainWindow.waitForSelector('[data-testid="chat-input"]');
           expect(await chatInput.isVisible()).toBe(true);
     
           // Type a message
@@ -292,15 +313,15 @@ test.describe('Goose App', () => {
           await mainWindow.screenshot({ path: `test-results/${provider.name.toLowerCase()}-before-send.png` });
     
           // Get initial message count
-          const initialMessages = await mainWindow.locator('.prose').count();
+          const initialMessages = await mainWindow.locator('[data-testid="message-container"]').count();
     
           // Send message
           await chatInput.press('Enter');
     
           // Wait for loading indicator to appear
           console.log('Waiting for loading indicator...');
-          const loadingGoose = await mainWindow.waitForSelector('.text-textStandard >> text="goose is working on it…"',
-            { timeout: 10000 });
+          const loadingGoose = await mainWindow.waitForSelector('[data-testid="loading-indicator"]',
+            { timeout: 2000 });
           expect(await loadingGoose.isVisible()).toBe(true);
     
           // Take screenshot of loading state
@@ -308,17 +329,17 @@ test.describe('Goose App', () => {
     
           // Wait for loading indicator to disappear
           console.log('Waiting for response...');
-          await mainWindow.waitForSelector('.text-textStandard >> text="goose is working on it…"',
+          await mainWindow.waitForSelector('[data-testid="loading-indicator"]',
             { state: 'hidden', timeout: 30000 });
     
           // Wait for new message to appear
           await mainWindow.waitForFunction((count) => {
-            const messages = document.querySelectorAll('.prose');
+            const messages = document.querySelectorAll('[data-testid="message-container"]');
             return messages.length > count;
           }, initialMessages, { timeout: 30000 });
     
           // Get the latest response
-          const response = await mainWindow.locator('.prose').last();
+          const response = await mainWindow.locator('[data-testid="message-container"]').last();
           expect(await response.isVisible()).toBe(true);
     
           // Verify response has content
@@ -334,34 +355,34 @@ test.describe('Goose App', () => {
           console.log(`Testing chat history with ${provider.name}...`);
     
           // Find the chat input again
-          const chatInput = await mainWindow.waitForSelector('textarea[placeholder*="What can goose help with?"]');
+          const chatInput = await mainWindow.waitForSelector('[data-testid="chat-input"]');
     
           // Test message sending with a specific question
           await chatInput.fill('What is 2+2?');
     
           // Get initial message count
-          const initialMessages = await mainWindow.locator('.prose').count();
+          const initialMessages = await mainWindow.locator('[data-testid="message-container"]').count();
     
           // Send message
           await chatInput.press('Enter');
     
           // Wait for loading indicator and response
-          await mainWindow.waitForSelector('.text-textStandard >> text="goose is working on it…"',
+          await mainWindow.waitForSelector('[data-testid="loading-indicator"]',
             { state: 'hidden', timeout: 30000 });
     
           // Wait for new message
           await mainWindow.waitForFunction((count) => {
-            const messages = document.querySelectorAll('.prose');
+            const messages = document.querySelectorAll('[data-testid="message-container"]');
             return messages.length > count;
           }, initialMessages, { timeout: 30000 });
     
           // Get the latest response
-          const response = await mainWindow.locator('.prose').last();
+          const response = await mainWindow.locator('[data-testid="message-container"]').last();
           const responseText = await response.textContent();
           expect(responseText).toBeTruthy();
     
           // Check for message history
-          const messages = await mainWindow.locator('.prose').all();
+          const messages = await mainWindow.locator('[data-testid="message-container"]').all();
           expect(messages.length).toBeGreaterThanOrEqual(2);
     
           // Take screenshot of chat history
@@ -378,7 +399,14 @@ test.describe('Goose App', () => {
         test('running quotes MCP server integration', async () => {
           console.log(`Testing Running Quotes MCP server integration with ${provider.name}...`);
       
+          // Create test-results directory if it doesn't exist
+          const fs = require('fs');
+          if (!fs.existsSync('test-results')) {
+            fs.mkdirSync('test-results', { recursive: true });
+          }
+
           // Clean up any existing running-quotes extensions from localStorage
+          // todo: extensions are no longer in localstorage so we need a way to delete extensions from the ui or config to actually clean these up
           await mainWindow.evaluate(() => {
             const USER_SETTINGS_KEY = 'user_settings';
             const settings = JSON.parse(localStorage.getItem(USER_SETTINGS_KEY) || '{"extensions":[]}');
@@ -389,91 +417,184 @@ test.describe('Goose App', () => {
             // Save back to localStorage
             localStorage.setItem(USER_SETTINGS_KEY, JSON.stringify(settings));
             
-            // Log the cleanup
             console.log('Cleaned up existing running-quotes extensions');
           });
       
-          // Reload the page to ensure settings are fresh
-          await mainWindow.reload();
-          await mainWindow.waitForLoadState('networkidle');
-      
-          // Debug: Print HTML structure before trying to find menu button
-          // console.log('Debug: Current HTML structure before menu button selection:');
-          // const htmlStructure = await mainWindow.evaluate(() => document.documentElement.outerHTML);
-          // console.log(htmlStructure);
-
-          // Take screenshot before attempting to find menu button
-          await mainWindow.screenshot({ path: `test-results/${provider.name.toLowerCase()}-before-menu-button.png` });
-
-          // Click the menu button (3 dots)
-          console.log('Attempting to find menu button...');
-          const menuButton = await mainWindow.waitForSelector('button[aria-label="More options"]', {
-            timeout: 5000,
-            state: 'visible'
-          });
-          console.log('Menu button found, clicking...');
-          await menuButton.click();
-      
-          // Click Advanced settings
-          const advancedSettingsButton = await mainWindow.waitForSelector('button:has-text("Advanced settings")');
-          await advancedSettingsButton.click();
-      
-          // Wait for settings page and take screenshot
-          await mainWindow.screenshot({ path: `test-results/${provider.name.toLowerCase()}-mcp-settings-page.png` });
-      
-          // Click Add Custom Extension button and wait for modal
-          const addExtensionButton = await mainWindow.waitForSelector('button:has-text("Add Custom Extension")');
-          await addExtensionButton.click();
-      
-          // Wait for modal and form to be fully rendered
-          await mainWindow.waitForSelector('form', { state: 'visible', timeout: 10000 });
-          console.log('Form found, waiting for modal animation...');
-          await mainWindow.waitForTimeout(1000); // Wait for modal animation
-      
           try {
-            // Fill ID (find by label text)
-            console.log('Filling ID field...');
-            await mainWindow.locator('label:has-text("ID *") + input[type="text"]').fill('running-quotes');
-      
-            // Fill Name (find by label text)
-            console.log('Filling Name field...');
-            await mainWindow.locator('label:has-text("Name *") + input[type="text"]').fill('Running Quotes');
-      
-            // Fill Description (find by label text)
-            console.log('Filling Description field...');
-            await mainWindow.locator('label:has-text("Description *") + input[type="text"]').fill('Inspirational running quotes MCP server');
-      
-            // Fill Command (find by label text and placeholder)
-            console.log('Filling Command field...');
+            // Reload the page to ensure settings are fresh
+            await mainWindow.reload();
+            await mainWindow.waitForLoadState('networkidle');
+            await mainWindow.waitForLoadState('domcontentloaded');
+            
+            // Wait for React app to be ready
+            await mainWindow.waitForFunction(() => {
+              const root = document.getElementById('root');
+              return root && root.children.length > 0;
+            });
+
+            // Take screenshot of initial state
+            await mainWindow.screenshot({ path: `test-results/${provider.name.toLowerCase()}-initial-state.png` });
+
+            // Click the menu button (3 dots)
+            console.log('Attempting to find menu button...');
+            const menuButton = await mainWindow.waitForSelector('[data-testid="more-options-button"]', {
+              timeout: 2000,
+              state: 'visible'
+            });
+            
+            // Verify menu button is visible and clickable
+            const isMenuButtonVisible = await menuButton.isVisible();
+            console.log('Menu button visible:', isMenuButtonVisible);
+            
+            // Take screenshot before clicking menu
+            await mainWindow.screenshot({ path: `test-results/${provider.name.toLowerCase()}-before-menu.png` });
+            
+            await menuButton.click();
+            console.log('Clicked menu button');
+            
+            // Wait for menu to appear and take screenshot
+            await mainWindow.waitForTimeout(1000);
+            await mainWindow.screenshot({ path: `test-results/${provider.name.toLowerCase()}-after-menu.png` });
+
+            // Click Advanced settings
+            console.log('Looking for Advanced settings button...');
+            const advancedSettingsButton = await mainWindow.waitForSelector('button:has-text("Advanced settings")', {
+              timeout: 2000,
+              state: 'visible'
+            });
+            
+            // Verify advanced settings button is visible
+            const isAdvancedSettingsVisible = await advancedSettingsButton.isVisible();
+            console.log('Advanced settings button visible:', isAdvancedSettingsVisible);
+            
+            await advancedSettingsButton.click();
+            console.log('Clicked Advanced settings');
+            
+            // Wait for navigation and take screenshot
+            await mainWindow.waitForTimeout(1000);
+            await mainWindow.screenshot({ path: `test-results/${provider.name.toLowerCase()}-settings-page.png` });
+
+            // Click Add Custom Extension button
+            console.log('Looking for Add Custom Extension button...');
+            const addExtensionButton = await mainWindow.waitForSelector('button:has-text("Add Custom Extension")', {
+              timeout: 2000,
+              state: 'visible'
+            });
+            
+            // Verify add extension button is visible
+            const isAddExtensionVisible = await addExtensionButton.isVisible();
+            console.log('Add Extension button visible:', isAddExtensionVisible);
+            
+            await addExtensionButton.click();
+            console.log('Clicked Add Custom Extension');
+
+            // Wait for modal and take screenshot
+            await mainWindow.waitForTimeout(1000);
+            await mainWindow.screenshot({ path: `test-results/${provider.name.toLowerCase()}-modal.png` });
+
+            // Get page content for debugging
+            const modalContent = await mainWindow.evaluate(() => document.body.innerHTML);
+            console.log('Modal content:', modalContent);
+
+            // Fill the form
+            console.log('Filling form fields...');
+            
+            // Fill Extension Name
+            const nameInput = await mainWindow.waitForSelector('input[placeholder="Enter extension name..."]', {
+              timeout: 2000,
+              state: 'visible'
+            });
+            await nameInput.fill('Running Quotes');
+            
+            // Fill Description
+            const descriptionInput = await mainWindow.waitForSelector('input[placeholder="Optional description..."]', {
+              timeout: 2000,
+              state: 'visible'
+            });
+            await descriptionInput.fill('Inspirational running quotes MCP server');
+            
+            // Fill Command
             const mcpScriptPath = join(__dirname, 'basic-mcp.ts');
-            await mainWindow.locator('label:has-text("Command *") + input[placeholder="e.g. goosed mcp example"]')
-              .fill(`node ${mcpScriptPath}`);
-      
+            const commandInput = await mainWindow.waitForSelector('input[placeholder*="npx -y @modelcontextprotocol"]', {
+              timeout: 2000,
+              state: 'visible'
+            });
+            await commandInput.fill(`node ${mcpScriptPath}`);
+
             // Take screenshot of filled form
-            await mainWindow.screenshot({ path: `test-results/${provider.name.toLowerCase()}-mcp-form-filled.png` });
-      
-            // Add a delay to inspect the form
-            console.log('Waiting 5 seconds to inspect form...');
-            await mainWindow.waitForTimeout(5000);
-      
-            // Click Add button (it's a submit button)
-            console.log('Clicking Add button...');
-            await mainWindow.locator('button[type="submit"]').click();
-      
-            // Wait for success toast and take screenshot
-            await mainWindow.waitForSelector('.Toastify__toast-body div div:has-text("Successfully enabled extension")',
-              { state: 'visible', timeout: 10000 });
-            await mainWindow.screenshot({ path: `test-results/${provider.name.toLowerCase()}-mcp-extension-added.png` });
-            console.log('Extension added successfully');
-      
-            // Click Exit button to return to chat
-            const exitButton = await mainWindow.waitForSelector('button:has-text("Back")', { timeout: 5000 });
-            await exitButton.click();
-      
+            await mainWindow.screenshot({ path: `test-results/${provider.name.toLowerCase()}-filled-form.png` });
+
+            // Click Add Extension button
+            console.log('Looking for Add Extension button...');
+            const addButton = await mainWindow.waitForSelector('button:has-text("Add Extension")', {
+              timeout: 2000,
+              state: 'visible'
+            });
+            
+            // Verify add button is visible
+            const isAddButtonVisible = await addButton.isVisible();
+            console.log('Add button visible:', isAddButtonVisible);
+            
+            await addButton.click();
+            console.log('Clicked Add Extension button');
+
+            // Wait for the Running Quotes extension to appear in the list
+            console.log('Waiting for Running Quotes extension to appear...');
+            try {
+              const extensionCard = await mainWindow.waitForSelector(
+                'div.flex:has-text("Running Quotes")', 
+                {
+                  timeout: 30000,
+                  state: 'visible'
+                }
+              );
+              
+              // Verify the extension is enabled
+              const toggleButton = await extensionCard.$('button[role="switch"][data-state="checked"]');
+              const isEnabled = !!toggleButton;
+              console.log('Extension enabled:', isEnabled);
+              
+              if (!isEnabled) {
+                throw new Error('Running Quotes extension was added but not enabled');
+              }
+              
+              await mainWindow.screenshot({ path: `test-results/${provider.name.toLowerCase()}-extension-added.png` });
+              console.log('Extension added successfully');
+            } catch (error) {
+              console.error('Error verifying extension:', error);
+              
+              // Get current page content
+              const pageContent = await mainWindow.evaluate(() => document.body.innerHTML);
+              console.log('Page content after clicking Add Extension:', pageContent);
+              
+              // Get any error messages that might be visible
+              const errorElements = await mainWindow.$$eval('.text-red-500, .text-error', 
+                elements => elements.map(el => el.textContent)
+              );
+              if (errorElements.length > 0) {
+                console.log('Found error messages:', errorElements);
+              }
+              
+              throw error;
+            }
+
+            // Click Back button
+            const backButton = await mainWindow.waitForSelector('button:has-text("Back")', { 
+              timeout: 2000,
+              state: 'visible'
+            });
+            await backButton.click();
+            console.log('Clicked Back button');
+
           } catch (error) {
-            // Take error screenshot
-            await mainWindow.screenshot({ path: `test-results/${provider.name.toLowerCase()}-mcp-form-error.png` });
-            console.error('Error during form filling:', error);
+            // Take error screenshot and log details
+            await mainWindow.screenshot({ path: `test-results/${provider.name.toLowerCase()}-error.png` });
+            
+            // Get page content
+            const pageContent = await mainWindow.evaluate(() => document.body.innerHTML);
+            console.log('Page content at error:', pageContent);
+            
+            console.error('Test failed:', error);
             throw error;
           }
         });
@@ -482,7 +603,7 @@ test.describe('Goose App', () => {
           console.log(`Testing running quotes functionality with ${provider.name}...`);
       
           // Find the chat input
-          const chatInput = await mainWindow.waitForSelector('textarea[placeholder*="What can goose help with?"]');
+          const chatInput = await mainWindow.waitForSelector('[data-testid="chat-input"]');
           expect(await chatInput.isVisible()).toBe(true);
       
           // Type a message requesting a running quote
@@ -492,13 +613,13 @@ test.describe('Goose App', () => {
           await mainWindow.screenshot({ path: `test-results/${provider.name.toLowerCase()}-before-quote-request.png` });
       
           // Get initial message count
-          const initialMessages = await mainWindow.locator('.prose').count();
+          const initialMessages = await mainWindow.locator('[data-testid="message-container"]').count();
       
           // Send message
           await chatInput.press('Enter');
       
           // Wait for loading indicator
-          const loadingIndicator = await mainWindow.waitForSelector('.text-textStandard >> text="goose is working on it…"',
+          const loadingIndicator = await mainWindow.waitForSelector('[data-testid="loading-indicator"]',
             { timeout: 30000 });
           expect(await loadingIndicator.isVisible()).toBe(true);
       
@@ -506,17 +627,17 @@ test.describe('Goose App', () => {
           await mainWindow.screenshot({ path: `test-results/${provider.name.toLowerCase()}-quote-loading.png` });
       
           // Wait for loading indicator to disappear
-          await mainWindow.waitForSelector('.text-textStandard >> text="goose is working on it…"',
+          await mainWindow.waitForSelector('[data-testid="loading-indicator"]',
             { state: 'hidden', timeout: 30000 });
       
           // Wait for new message to appear
           await mainWindow.waitForFunction((count) => {
-            const messages = document.querySelectorAll('.prose');
+            const messages = document.querySelectorAll('[data-testid="message-container"]');
             return messages.length > count;
           }, initialMessages, { timeout: 30000 });
       
           // Get the latest response
-          const response = await mainWindow.locator('.prose').last();
+          const response = await mainWindow.locator('[data-testid="message-container"]').last();
           expect(await response.isVisible()).toBe(true);
       
           // Click the Output dropdown to reveal the actual quote