make video better

ui/desktop/playwright.config.ts

@@ -15,8 +15,7 @@ const config: PlaywrightTestConfig = {
   use: {
     actionTimeout: 30000,
     navigationTimeout: 30000,
-    trace: 'on-first-retry',
-    video: 'retain-on-failure',
+    video: 'on',
     screenshot: 'only-on-failure'
   },
   outputDir: 'test-results',

ui/desktop/tests/e2e/app.spec.ts

@@ -1,4 +1,4 @@
-import { test as base, expect } from './fixtures.electron.packaged';
+import { test as base, expect, waitForLoadingDone } from './fixtures.electron.packaged';
 import { Page } from '@playwright/test';
 import { showTestName, clearTestName } from './test-overlay';
 import { join } from 'path';
@@ -169,7 +169,7 @@ test.describe('Goose App', () => {
 
           // Wait for response to complete
           console.log('Waiting for response...');
-          await expect(mainWindow.getByTestId('loading-indicator')).toHaveCount(0, { timeout: LLM_TIMEOUT });
+          await waitForLoadingDone(mainWindow, LLM_TIMEOUT);
           console.log('Response complete');
 
           // Verify response
@@ -188,7 +188,7 @@ test.describe('Goose App', () => {
           await chatInput.press('Enter');
 
           // Wait for response to complete
-          await expect(mainWindow.getByTestId('loading-indicator')).toHaveCount(0, { timeout: LLM_TIMEOUT });
+          await waitForLoadingDone(mainWindow, LLM_TIMEOUT);
 
           // Verify response
           const response = mainWindow.getByTestId('message-container').last();
@@ -236,7 +236,7 @@ test.describe('Goose App', () => {
           await chatInput.press('Enter');
 
           // Wait for goose to finish responding
-          await expect(mainWindow.getByTestId('loading-indicator')).toHaveCount(0, { timeout: LLM_TIMEOUT });
+          await waitForLoadingDone(mainWindow, LLM_TIMEOUT);
 
           // Verify the response contains a known quote
           const lastMessage = mainWindow.locator('.goose-message').last();

ui/desktop/tests/e2e/fixtures.electron.packaged.ts

@@ -1,11 +1,12 @@
-import { test as base, expect, Page, _electron as electron } from '@playwright/test';
+import { test as base, expect, Locator, Page, _electron as electron } from '@playwright/test';
 import { promisify } from 'util';
-import { exec } from 'child_process';
+import { exec, execFile } from 'child_process';
 import { join } from 'path';
 import * as fs from 'fs';
 import * as os from 'os';
 
 const execAsync = promisify(exec);
+const execFileAsync = promisify(execFile);
 
 type GooseTestFixtures = {
   goosePage: Page;
@@ -18,11 +19,148 @@ const debugLog = (message: string) => {
   }
 };
 
+export const isVideoRecording = () => process.env.PW_ELECTRON_VIDEO === '1';
+const VISUAL_DELAY_MS = Number(process.env.E2E_VISUAL_DELAY_MS ?? (isVideoRecording() ? '500' : '0'));
+const POST_LOAD_HOLD_MS = Number(process.env.E2E_POST_LOAD_HOLD_MS ?? (isVideoRecording() ? '1000' : '0'));
+const VIDEO_TRIM_READY_BUFFER_MS = Number(process.env.E2E_VIDEO_TRIM_READY_BUFFER_MS ?? (isVideoRecording() ? '300' : '0'));
+
+const PAGE_ACTION_METHODS = new Set([
+  'click',
+  'dblclick',
+  'tap',
+  'fill',
+  'press',
+  'check',
+  'uncheck',
+  'setChecked',
+  'selectOption',
+  'dragAndDrop',
+  'goto',
+  'reload'
+]);
+
+const PAGE_LOCATOR_METHODS = new Set([
+  'locator',
+  'getByAltText',
+  'getByLabel',
+  'getByPlaceholder',
+  'getByRole',
+  'getByTestId',
+  'getByText',
+  'getByTitle'
+]);
+
+const LOCATOR_ACTION_METHODS = new Set([
+  'click',
+  'dblclick',
+  'tap',
+  'fill',
+  'press',
+  'check',
+  'uncheck',
+  'setChecked',
+  'selectOption',
+  'hover',
+  'focus',
+  'blur'
+]);
+
+const LOCATOR_CHAIN_METHODS = new Set([
+  'locator',
+  'getByAltText',
+  'getByLabel',
+  'getByPlaceholder',
+  'getByRole',
+  'getByTestId',
+  'getByText',
+  'getByTitle'
+]);
+const VISUAL_DELAY_DECORATED = Symbol('visual-delay-decorated');
+
+async function applyVisualDelay(page: Page): Promise<void> {
+  if (VISUAL_DELAY_MS > 0) {
+    await page.waitForTimeout(VISUAL_DELAY_MS);
+  }
+}
+
+function withVisualDelayLocator(locator: Locator, page: Page): Locator {
+  const existingMark = (locator as unknown as Record<PropertyKey, unknown>)[VISUAL_DELAY_DECORATED];
+  if (existingMark) {
+    return locator;
+  }
+
+  for (const methodName of LOCATOR_ACTION_METHODS) {
+    const original = (locator as unknown as Record<string, unknown>)[methodName];
+    if (typeof original === 'function') {
+      (locator as unknown as Record<string, unknown>)[methodName] = async (...args: unknown[]) => {
+        const resolved = await (original as (...params: unknown[]) => unknown).apply(locator, args);
+        await applyVisualDelay(page);
+        return resolved;
+      };
+    }
+  }
+
+  for (const methodName of LOCATOR_CHAIN_METHODS) {
+    const original = (locator as unknown as Record<string, unknown>)[methodName];
+    if (typeof original === 'function') {
+      (locator as unknown as Record<string, unknown>)[methodName] = (...args: unknown[]) => {
+        const next = (original as (...params: unknown[]) => unknown).apply(locator, args);
+        if (next && typeof next === 'object') {
+          return withVisualDelayLocator(next as Locator, page);
+        }
+        return next;
+      };
+    }
+  }
+
+  Object.defineProperty(locator, VISUAL_DELAY_DECORATED, {
+    value: true,
+    enumerable: false,
+    configurable: false,
+    writable: false
+  });
+
+  return locator;
+}
+
+function withVisualDelayPage(page: Page): Page {
+  if (VISUAL_DELAY_MS <= 0) {
+    return page;
+  }
+
+  return new Proxy(page, {
+    get(target, prop, receiver) {
+      const propName = String(prop);
+      const value = Reflect.get(target, prop, receiver);
+
+      if (typeof value !== 'function') {
+        return value;
+      }
+
+      return (...args: unknown[]) => {
+        const result = value.apply(target, args);
+        if (PAGE_LOCATOR_METHODS.has(propName) && result && typeof result === 'object') {
+          return withVisualDelayLocator(result as Locator, target);
+        }
+        if (PAGE_ACTION_METHODS.has(propName)) {
+          return Promise.resolve(result).then(async (resolved) => {
+            await applyVisualDelay(target);
+            return resolved;
+          });
+        }
+        return result;
+      };
+    }
+  }) as Page;
+}
+
+
 export const test = base.extend<GooseTestFixtures>({
   goosePage: async ({}, use, testInfo) => {
     let electronApp: Awaited<ReturnType<typeof electron.launch>> | null = null;
     let page: Page | null = null;
-    let tracingStarted = false;
+    let videoDir: string | undefined;
+    let videoTrimStartMs = 0;
     const appRoot = join(__dirname, '../..');
     const executablePath = resolvePackagedExecutable(appRoot);
 
@@ -35,7 +173,7 @@ export const test = base.extend<GooseTestFixtures>({
     const tempDir = createIsolatedGoosePathRoot();
 
     try {
-      const videoDir = process.env.PW_ELECTRON_VIDEO === '1' ? testInfo.outputPath('videos') : undefined;
+      videoDir = isVideoRecording() ? testInfo.outputPath('videos') : undefined;
       const launchOptions = buildLaunchOptions(executablePath, tempDir, videoDir);
       debugLog(`Launching packaged Electron for test: ${testInfo.title}`);
       debugLog(`Using packaged executable: ${executablePath}`);
@@ -44,31 +182,37 @@ export const test = base.extend<GooseTestFixtures>({
       attachAppDebugLogs(electronApp);
 
       await electronApp.firstWindow({ timeout: 30000 });
+      const recordingStartMs = Date.now();
+      await waitForRootWindow(electronApp, 60000);
+      const rootReadyElapsedMs = Date.now() - recordingStartMs;
       page = await waitForReadyAppWindow(electronApp, 60000);
       await page.waitForLoadState('domcontentloaded', { timeout: 10000 }).catch(() => {});
       debugLog(`Selected app window URL: ${page.url()}`);
       attachPageDebugLogs(page);
-
-      await page.context().tracing.start({
-        screenshots: true,
-        snapshots: true,
-        sources: true,
-      });
-      tracingStarted = true;
-
-      await use(page);
-    } finally {
-      if (page && tracingStarted) {
-        try {
-          await page.context().tracing.stop({ path: testInfo.outputPath('trace.zip') });
-        } catch {
-          // Tracing stop can fail if context has already been torn down.
-        }
+      if (isVideoRecording()) {
+        await enableCursorHighlight(page);
+        page.on('domcontentloaded', () => {
+          void enableCursorHighlight(page);
+        });
+      }
+      const overrideTrimStartMs = process.env.E2E_VIDEO_TRIM_START_MS;
+      if (overrideTrimStartMs !== undefined) {
+        const parsedOverride = Number(overrideTrimStartMs);
+        videoTrimStartMs = Number.isFinite(parsedOverride) && parsedOverride > 0 ? parsedOverride : 0;
+      } else {
+        videoTrimStartMs = Math.max(0, rootReadyElapsedMs - VIDEO_TRIM_READY_BUFFER_MS);
       }
 
+      await use(withVisualDelayPage(page));
+    } finally {
       if (electronApp) {
         await closeWithFallback(electronApp);
       }
+
+      if (videoDir && videoTrimStartMs > 0) {
+        await trimVideosInDirectory(videoDir, videoTrimStartMs);
+      }
+
       fs.rmSync(tempDir, { recursive: true, force: true });
     }
   },
@@ -105,6 +249,30 @@ async function waitForReadyAppWindow(
   throw new Error(`No app-ready window found within ${timeoutMs}ms. Window URLs: ${urls.join(', ')}`);
 }
 
+async function waitForRootWindow(
+  electronApp: Awaited<ReturnType<typeof electron.launch>>,
+  timeoutMs: number
+): Promise<Page> {
+  const start = Date.now();
+  while (Date.now() - start < timeoutMs) {
+    const windows = electronApp.windows();
+    for (const page of windows) {
+      try {
+        const hasRoot = await page.evaluate(() => !!document.getElementById('root'));
+        if (hasRoot) {
+          return page;
+        }
+      } catch {
+        // Window may not be ready for evaluation yet.
+      }
+    }
+    await new Promise((resolve) => setTimeout(resolve, 100));
+  }
+
+  const urls = electronApp.windows().map((w) => w.url() || '<empty>');
+  throw new Error(`No root-ready window found within ${timeoutMs}ms. Window URLs: ${urls.join(', ')}`);
+}
+
 function resolvePackagedExecutable(appRoot: string): string {
   if (process.env.GOOSE_PACKAGED_EXECUTABLE) {
     return process.env.GOOSE_PACKAGED_EXECUTABLE;
@@ -131,6 +299,18 @@ function createIsolatedGoosePathRoot(): string {
     join(configDir, 'config.yaml'),
     'GOOSE_PROVIDER: databricks\nGOOSE_MODEL: databricks-claude-haiku-4-5\nGOOSE_TELEMETRY_ENABLED: false\nDATABRICKS_HOST: https://block-lakehouse-production.cloud.databricks.com/\n'
   );
+
+  // Copy OAuth token cache so the app can silently refresh without opening a browser.
+  // If the refresh token is expired, the app falls back to browser OAuth as usual.
+  const realOAuthDir = join(os.homedir(), '.config', 'goose', 'databricks', 'oauth');
+  if (fs.existsSync(realOAuthDir)) {
+    const testOAuthDir = join(configDir, 'databricks', 'oauth');
+    fs.mkdirSync(testOAuthDir, { recursive: true });
+    for (const file of fs.readdirSync(realOAuthDir)) {
+      fs.copyFileSync(join(realOAuthDir, file), join(testOAuthDir, file));
+    }
+  }
+
   return tempDir;
 }
 
@@ -146,6 +326,7 @@ function buildLaunchOptions(
     env: {
       ...process.env,
       GOOSE_ALLOWLIST_BYPASS: 'true',
+      GOOSE_DISABLE_KEYRING: '1',
       GOOSE_PATH_ROOT: tempDir,
       RUST_LOG: 'info',
     },
@@ -219,4 +400,191 @@ async function closeWithFallback(electronApp: Awaited<ReturnType<typeof electron
   }
 }
 
+async function enableCursorHighlight(page: Page): Promise<void> {
+  try {
+    await page.evaluate(() => {
+      const markerAttr = 'data-test-cursor-highlight';
+      if (document.querySelector(`[${markerAttr}]`)) {
+        return;
+      }
+
+      const style = document.createElement('style');
+      style.setAttribute(markerAttr, 'style');
+      style.textContent = `
+        .test-cursor-overlay {
+          position: fixed;
+          inset: 0;
+          pointer-events: none;
+          z-index: 2147483647;
+          overflow: hidden;
+        }
+        .test-cursor-highlight {
+          position: absolute;
+          width: 22px;
+          height: 22px;
+          border-radius: 9999px;
+          border: 2px solid rgba(255, 255, 255, 0.95);
+          background: rgba(239, 68, 68, 0.45);
+          box-shadow: 0 0 0 1px rgba(0, 0, 0, 0.2), 0 4px 14px rgba(0, 0, 0, 0.25);
+          transform: none;
+          transition: width 100ms ease, height 100ms ease, background 100ms ease;
+          mix-blend-mode: normal;
+          opacity: 0;
+        }
+        .test-cursor-highlight.clicking {
+          width: 28px;
+          height: 28px;
+          background: rgba(239, 68, 68, 0.65);
+        }
+        .test-cursor-click-ring {
+          position: absolute;
+          width: 12px;
+          height: 12px;
+          border-radius: 9999px;
+          border: 2px solid rgba(239, 68, 68, 0.8);
+          transform: none;
+          animation: test-cursor-ring 420ms ease-out forwards;
+        }
+        @keyframes test-cursor-ring {
+          0% { opacity: 0.95; width: 12px; height: 12px; }
+          100% { opacity: 0; width: 54px; height: 54px; }
+        }
+      `;
+      document.head.appendChild(style);
+
+      const overlay = document.createElement('div');
+      overlay.className = 'test-cursor-overlay';
+      overlay.setAttribute(markerAttr, 'overlay');
+      document.body.appendChild(overlay);
+
+      const cursor = document.createElement('div');
+      cursor.className = 'test-cursor-highlight';
+      cursor.setAttribute(markerAttr, 'cursor');
+      overlay.appendChild(cursor);
+
+      const setElementCenter = (element: HTMLElement, x: number, y: number) => {
+        const width = element.offsetWidth || Number.parseInt(getComputedStyle(element).width, 10) || 0;
+        const height = element.offsetHeight || Number.parseInt(getComputedStyle(element).height, 10) || 0;
+        element.style.left = `${x - width / 2}px`;
+        element.style.top = `${y - height / 2}px`;
+      };
+
+      const moveCursor = (x: number, y: number) => {
+        setElementCenter(cursor, x, y);
+        cursor.style.opacity = '1';
+      };
+
+      const spawnClickRing = (x: number, y: number) => {
+        const ring = document.createElement('div');
+        ring.className = 'test-cursor-click-ring';
+        setElementCenter(ring, x, y);
+        ring.setAttribute(markerAttr, 'ring');
+        overlay.appendChild(ring);
+        window.setTimeout(() => ring.remove(), 500);
+      };
+
+      document.addEventListener(
+        'mousemove',
+        (event) => {
+          moveCursor(event.clientX, event.clientY);
+        },
+        { passive: true }
+      );
+
+      document.addEventListener(
+        'mousedown',
+        (event) => {
+          moveCursor(event.clientX, event.clientY);
+          cursor.classList.add('clicking');
+          spawnClickRing(event.clientX, event.clientY);
+        },
+        { passive: true }
+      );
+
+      document.addEventListener(
+        'mouseup',
+        () => {
+          cursor.classList.remove('clicking');
+        },
+        { passive: true }
+      );
+    });
+  } catch (error) {
+    debugLog(`Failed to enable cursor highlight: ${String(error)}`);
+  }
+}
+
+async function trimVideosInDirectory(videoDir: string, trimStartMs: number): Promise<void> {
+  const requestedTrimSeconds = Math.max(0, trimStartMs) / 1000;
+  if (requestedTrimSeconds <= 0 || !fs.existsSync(videoDir)) {
+    return;
+  }
+
+  const files = fs.readdirSync(videoDir).filter((name) => name.endsWith('.webm'));
+  if (files.length === 0) {
+    return;
+  }
+
+  for (const fileName of files) {
+    const sourcePath = join(videoDir, fileName);
+    const trimmedPath = join(videoDir, `${fileName}.trimmed.webm`);
+    try {
+      const durationSeconds = await getVideoDurationSeconds(sourcePath);
+      const maxTrimSeconds = Math.max(0, durationSeconds - 0.25);
+      const trimSeconds = Math.min(requestedTrimSeconds, maxTrimSeconds);
+      if (trimSeconds <= 0) {
+        continue;
+      }
+
+      await execFileAsync('ffmpeg', [
+        '-y',
+        '-ss',
+        trimSeconds.toFixed(3),
+        '-i',
+        sourcePath,
+        '-c:v',
+        'libvpx-vp9',
+        '-b:v',
+        '0',
+        '-crf',
+        '32',
+        '-an',
+        trimmedPath
+      ]);
+      fs.renameSync(trimmedPath, sourcePath);
+    } catch (error) {
+      debugLog(`Failed to trim video ${sourcePath}: ${String(error)}`);
+      if (fs.existsSync(trimmedPath)) {
+        fs.rmSync(trimmedPath, { force: true });
+      }
+    }
+  }
+}
+
+async function getVideoDurationSeconds(videoPath: string): Promise<number> {
+  try {
+    const { stdout } = await execFileAsync('ffprobe', [
+      '-v',
+      'error',
+      '-show_entries',
+      'format=duration',
+      '-of',
+      'default=noprint_wrappers=1:nokey=1',
+      videoPath
+    ]);
+    const duration = Number(stdout.trim());
+    return Number.isFinite(duration) && duration > 0 ? duration : 0;
+  } catch (error) {
+    debugLog(`Failed to probe video duration ${videoPath}: ${String(error)}`);
+    return 0;
+  }
+}
+
+export async function waitForLoadingDone(page: Page, timeout: number): Promise<void> {
+  await expect(page.getByTestId('loading-indicator')).toHaveCount(0, { timeout });
+  if (POST_LOAD_HOLD_MS > 0) {
+    await page.waitForTimeout(POST_LOAD_HOLD_MS);
+  }
+}
+
 export { expect } from '@playwright/test';

ui/desktop/tests/e2e/test-overlay.ts

@@ -1,3 +1,5 @@
+const isVideoRecording = () => process.env.PW_ELECTRON_VIDEO === '1';
+
 // Helper function to show test name overlay
 async function showTestName(mainWindow: any, testName: string, providerName?: string) {
   await mainWindow.evaluate(({ name, provider }: { name: string, provider?: string }) => {
@@ -28,7 +30,7 @@ async function showTestName(mainWindow: any, testName: string, providerName?: st
       user-select: none;
       -webkit-user-select: none;
     `;
-    
+
     const testText = `Running: ${name}`;
     const providerText = provider ? `\nProvider: ${provider}` : '';
     overlay.textContent = testText + providerText;
@@ -39,10 +41,19 @@ async function showTestName(mainWindow: any, testName: string, providerName?: st
     // Force a repaint to ensure the overlay is visible
     overlay.getBoundingClientRect();
   }, { name: testName, provider: providerName });
+
+  // Hold overlay visible so it's readable in the video
+  if (isVideoRecording()) {
+    await mainWindow.waitForTimeout(1500);
+  }
 }
 
 // Helper function to clear test name overlay
 async function clearTestName(mainWindow: any) {
+  // Pause before clearing so the final state is captured in the video
+  if (isVideoRecording()) {
+    await mainWindow.waitForTimeout(1000);
+  }
   await mainWindow.evaluate(() => {
     const overlay = document.getElementById('test-overlay');
     if (overlay) overlay.remove();