From 8c2b6f978545ffa2dd46e5f3c026cd36e940c0de Mon Sep 17 00:00:00 2001 From: Andrew Harvard Date: Thu, 26 Feb 2026 13:30:16 -0500 Subject: [PATCH] feat(ui): implement fullscreen and pip display modes for MCP Apps (#7312) Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .../src/components/McpApps/McpAppRenderer.tsx | 261 ++++++++++++-- ui/desktop/src/components/McpApps/types.ts | 6 + .../src/components/McpApps/useDisplayMode.ts | 338 ++++++++++++++++++ ui/desktop/src/styles/main.css | 49 +++ 4 files changed, 625 insertions(+), 29 deletions(-) create mode 100644 ui/desktop/src/components/McpApps/useDisplayMode.ts diff --git a/ui/desktop/src/components/McpApps/McpAppRenderer.tsx b/ui/desktop/src/components/McpApps/McpAppRenderer.tsx index b53c380e1c..c2142df87f 100644 --- a/ui/desktop/src/components/McpApps/McpAppRenderer.tsx +++ b/ui/desktop/src/components/McpApps/McpAppRenderer.tsx @@ -24,6 +24,7 @@ import type { McpUiSizeChangedNotification, } from '@modelcontextprotocol/ext-apps/app-bridge'; import type { CallToolResult, JSONRPCRequest } from '@modelcontextprotocol/sdk/types.js'; +import { GripHorizontal, Maximize2, PictureInPicture2, X } from 'lucide-react'; import { useCallback, useEffect, useMemo, useReducer, useRef, useState } from 'react'; import { callTool, readResource } from '../../api'; import { AppEvents } from '../../constants/events'; @@ -40,14 +41,21 @@ import { McpAppToolInputPartial, McpAppToolResult, DimensionLayout, + OnDisplayModeChange, SamplingCreateMessageParams, SamplingCreateMessageResponse, } from './types'; +import { + useDisplayMode, + AVAILABLE_DISPLAY_MODES, + PIP_WIDTH, + PIP_HEIGHT, + PIP_MARGIN_RIGHT, + PIP_MARGIN_BOTTOM, +} from './useDisplayMode'; const DEFAULT_IFRAME_HEIGHT = 200; -const AVAILABLE_DISPLAY_MODES: McpUiDisplayMode[] = ['inline']; - const DISPLAY_MODE_LAYOUTS: Record = { inline: { width: 'fixed', height: 'unbounded' }, fullscreen: { width: 'fixed', height: 'fixed' }, @@ -139,6 +147,7 @@ interface McpAppRendererProps { append?: (text: string) => void; displayMode?: GooseDisplayMode; cachedHtml?: string; + onDisplayModeChange?: OnDisplayModeChange; } interface ResourceMeta { @@ -235,8 +244,27 @@ export default function McpAppRenderer({ append, displayMode = 'inline', cachedHtml, + onDisplayModeChange, }: McpAppRendererProps) { - const isExpandedView = displayMode === 'fullscreen' || displayMode === 'standalone'; + const containerRef = useRef(null); + + const dm = useDisplayMode({ displayMode, onDisplayModeChange, containerRef }); + const { + activeDisplayMode, + effectiveDisplayModes, + isStandalone, + isFullscreen, + isPip, + isFillsViewport, + isInline, + appSupportsFullscreen, + appSupportsPip, + changeDisplayMode, + inlineHeight, + pipPosition, + pipHandlers, + fullscreenCloseRef, + } = dm; const { resolvedTheme, mcpHostStyles } = useTheme(); @@ -264,7 +292,18 @@ export default function McpAppRenderer({ }); const [iframeHeight, setIframeHeight] = useState(DEFAULT_IFRAME_HEIGHT); - const containerRef = useRef(null); + // Restore iframeHeight from the saved snapshot when returning to inline. + // While in fullscreen/pip, handleSizeChanged ignores size notifications, so + // iframeHeight may be stale. This ensures the container starts at the correct + // height the moment the mode flips back to inline. + useEffect(() => { + if (isInline) { + setIframeHeight(inlineHeight); + } + }, [isInline, inlineHeight]); + + const effectiveInlineHeight = iframeHeight || DEFAULT_IFRAME_HEIGHT; + const [containerWidth, setContainerWidth] = useState(0); const [containerHeight, setContainerHeight] = useState(0); const [apiHost, setApiHost] = useState(null); @@ -512,11 +551,14 @@ export default function McpAppRenderer({ [] ); - const handleSizeChanged = useCallback(({ height }: McpUiSizeChangedNotification['params']) => { - if (height !== undefined && height > 0) { - setIframeHeight(height); - } - }, []); + const handleSizeChanged = useCallback( + ({ height }: McpUiSizeChangedNotification['params']) => { + if (height !== undefined && height > 0 && isInline) { + setIframeHeight(height); + } + }, + [isInline] + ); // Track the container's pixel dimensions so we can report them to apps via containerDimensions. useEffect(() => { @@ -605,12 +647,17 @@ export default function McpAppRenderer({ // todo: toolInfo: {} theme: resolvedTheme, styles: mcpHostStyles, - // 'standalone' is a Goose-specific display mode (dedicated Electron window) - // that maps to the spec's inline | fullscreen | pip modes. - displayMode: displayMode as McpUiDisplayMode, - availableDisplayModes: - displayMode === 'standalone' ? [displayMode as McpUiDisplayMode] : AVAILABLE_DISPLAY_MODES, - containerDimensions: getContainerDimensions(displayMode, containerWidth, containerHeight), + displayMode: activeDisplayMode as McpUiDisplayMode, + availableDisplayModes: isStandalone + ? [activeDisplayMode as McpUiDisplayMode] + : effectiveDisplayModes.length > 0 + ? effectiveDisplayModes + : AVAILABLE_DISPLAY_MODES, + containerDimensions: getContainerDimensions( + activeDisplayMode, + containerWidth, + containerHeight + ), locale: navigator.language, timeZone: Intl.DateTimeFormat().resolvedOptions().timeZone, userAgent: navigator.userAgent, @@ -628,7 +675,15 @@ export default function McpAppRenderer({ }; return context; - }, [resolvedTheme, mcpHostStyles, displayMode, containerWidth, containerHeight]); + }, [ + resolvedTheme, + mcpHostStyles, + activeDisplayMode, + isStandalone, + containerWidth, + containerHeight, + effectiveDisplayModes, + ]); const appToolResult = useMemo((): CallToolResult | undefined => { if (!toolResult) return undefined; @@ -694,23 +749,171 @@ export default function McpAppRenderer({ ); }; + const showControls = !isStandalone && !isError && (appSupportsFullscreen || appSupportsPip); + + const renderDisplayModeControls = () => { + if (!showControls) return null; + + if (activeDisplayMode === 'fullscreen') { + return ( +
+ {appSupportsPip && ( + + )} + +
+ ); + } + + if (activeDisplayMode === 'pip') { + return ( + <> + {appSupportsFullscreen && ( + + )} + + + ); + } + + // Inline mode — show controls on hover or keyboard focus + return ( +
+ {appSupportsFullscreen && ( + + )} + {appSupportsPip && ( + + )} +
+ ); + }; + + // Single stable container — CSS switches between inline/fullscreen/pip positioning. + // The AppRenderer and its iframe are never unmounted, preserving app state across mode changes. const containerClasses = cn( - 'bg-background-primary overflow-hidden [&_iframe]:!w-full', - isError && 'border border-red-500 rounded-lg bg-red-50 dark:bg-red-900/20', - !isError && !isExpandedView && 'mt-6 mb-2', - !isError && !isExpandedView && meta.prefersBorder && 'border border-border-primary rounded-lg' + 'mcp-app-container bg-background-primary [&_iframe]:!w-full', + isFillsViewport && 'fixed inset-0 z-[1000] overflow-hidden [&_iframe]:!h-full', + isPip && + 'fixed z-[900] overflow-y-auto overflow-x-hidden rounded-xl border border-border-primary shadow-2xl', + isInline && 'group/mcp-app relative overflow-hidden', + isInline && !isError && 'mt-6 mb-2', + isInline && !isError && meta.prefersBorder && 'border border-border-primary rounded-lg', + isError && 'border border-red-500 rounded-lg bg-red-50 dark:bg-red-900/20' ); - const containerStyle = isExpandedView - ? { width: '100%', height: '100%' } - : { - width: '100%', - height: `${iframeHeight || DEFAULT_IFRAME_HEIGHT}px`, - }; + const containerStyle: React.CSSProperties = { + ...(isFillsViewport + ? {} + : isPip + ? { + width: `${PIP_WIDTH}px`, + height: `${PIP_HEIGHT}px`, + right: `${PIP_MARGIN_RIGHT - pipPosition.x}px`, + bottom: `${PIP_MARGIN_BOTTOM - pipPosition.y}px`, + } + : { + width: '100%', + height: `${effectiveInlineHeight}px`, + }), + }; return ( -
- {renderContent()} -
+ <> + {/* Placeholder in chat flow when app is detached (fullscreen or pip) */} + {isFullscreen && ( +
+ )} + {isPip && ( +
+ +
+ )} + + {/* Stable app container — never unmounted, only repositioned via CSS */} +
+ {isPip && ( +
+
+ +
+
{renderDisplayModeControls()}
+
+ )} +
+ {!isPip && renderDisplayModeControls()} + {renderContent()} +
+
+ ); } diff --git a/ui/desktop/src/components/McpApps/types.ts b/ui/desktop/src/components/McpApps/types.ts index c2efed80cc..144f257608 100644 --- a/ui/desktop/src/components/McpApps/types.ts +++ b/ui/desktop/src/components/McpApps/types.ts @@ -43,6 +43,12 @@ export type McpAppToolResult = { _meta?: { [key: string]: unknown }; }; +/** + * Callback fired when the display mode changes, either via user-initiated + * host-side controls or app-initiated `ui/request-display-mode` changes. + */ +export type OnDisplayModeChange = (mode: GooseDisplayMode) => void; + export type SamplingMessage = { role: 'user' | 'assistant'; content: { type: 'text'; text: string } | { type: 'image'; data: string; mimeType: string }; diff --git a/ui/desktop/src/components/McpApps/useDisplayMode.ts b/ui/desktop/src/components/McpApps/useDisplayMode.ts new file mode 100644 index 0000000000..9f644ffdd9 --- /dev/null +++ b/ui/desktop/src/components/McpApps/useDisplayMode.ts @@ -0,0 +1,338 @@ +/** + * useDisplayMode — Manages display mode state for MCP App containers. + * + * Encapsulates the display mode state machine, capability negotiation, + * PiP drag handling, entrance animations, and postMessage interception + * for ui/initialize and ui/request-display-mode. + */ + +import type { McpUiDisplayMode } from '@modelcontextprotocol/ext-apps/app-bridge'; +import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react'; +import type { GooseDisplayMode, OnDisplayModeChange } from './types'; + +const DEFAULT_IFRAME_HEIGHT = 200; + +const AVAILABLE_DISPLAY_MODES: McpUiDisplayMode[] = ['inline', 'fullscreen', 'pip']; + +const PIP_WIDTH = 400; +const PIP_HEIGHT = 300; +const PIP_MARGIN_RIGHT = 16; +// Keeps the PiP window above the chat input area (~120px) plus padding. +const PIP_MARGIN_BOTTOM = 140; + +interface UseDisplayModeOptions { + displayMode: GooseDisplayMode; + onDisplayModeChange?: OnDisplayModeChange; + containerRef: React.RefObject; +} + +export interface DisplayModeState { + activeDisplayMode: GooseDisplayMode; + effectiveDisplayModes: McpUiDisplayMode[]; + isStandalone: boolean; + isFullscreen: boolean; + isPip: boolean; + isFillsViewport: boolean; + isInline: boolean; + appSupportsFullscreen: boolean; + appSupportsPip: boolean; + + changeDisplayMode: (mode: GooseDisplayMode) => void; + + /** Remembered inline height for placeholders when detached. */ + inlineHeight: number; + + /** PiP position offset from the default bottom-right corner. */ + pipPosition: { x: number; y: number }; + + /** PiP drag handle event handlers. */ + pipHandlers: { + onPointerDown: (e: React.PointerEvent) => void; + onPointerMove: (e: React.PointerEvent) => void; + onPointerUp: (e: React.PointerEvent) => void; + onLostPointerCapture: () => void; + onKeyDown: (e: React.KeyboardEvent) => void; + }; + + /** Ref for the fullscreen close button (auto-focused on enter). */ + fullscreenCloseRef: React.RefObject; +} + +export { AVAILABLE_DISPLAY_MODES, PIP_WIDTH, PIP_HEIGHT, PIP_MARGIN_RIGHT, PIP_MARGIN_BOTTOM }; + +export function useDisplayMode({ + displayMode, + onDisplayModeChange, + containerRef, +}: UseDisplayModeOptions): DisplayModeState { + const [activeDisplayMode, setActiveDisplayMode] = useState(displayMode); + + useEffect(() => { + setActiveDisplayMode(displayMode); + }, [displayMode]); + + const isStandalone = displayMode === 'standalone'; + + // Display modes the app declared support for during ui/initialize. + // null = not yet known (controls stay hidden until initialize), empty = app didn't declare any. + const [appDeclaredModes, setAppDeclaredModes] = useState(null); + + const effectiveDisplayModes = useMemo((): McpUiDisplayMode[] => { + if (!appDeclaredModes) return []; + return AVAILABLE_DISPLAY_MODES.filter((m) => appDeclaredModes.includes(m)); + }, [appDeclaredModes]); + + // Snapshot of the container height captured when leaving inline mode. + // Stored as state (not a ref) so consumers re-render with the correct value + // for placeholders and for restoring the inline container on return. + const [savedInlineHeight, setSavedInlineHeight] = useState(DEFAULT_IFRAME_HEIGHT); + + // Cache iframe contentWindows for O(1) message source matching. + // eslint-disable-next-line no-undef + const iframeWindowsRef = useRef>(new Set()); + + const enterAnimRef = useRef(null); + const fullscreenCloseRef = useRef(null); + + // ── Mode transitions ────────────────────────────────────────────────── + + const changeDisplayMode = useCallback( + (mode: GooseDisplayMode) => { + const el = containerRef.current; + const prefersReducedMotion = window.matchMedia('(prefers-reduced-motion: reduce)').matches; + + if (activeDisplayMode === 'inline' && el) { + setSavedInlineHeight(el.getBoundingClientRect().height || DEFAULT_IFRAME_HEIGHT); + } + + if (enterAnimRef.current && el) { + el.classList.remove(enterAnimRef.current); + enterAnimRef.current = null; + } + + setActiveDisplayMode(mode); + onDisplayModeChange?.(mode); + + if (el && !prefersReducedMotion && mode !== activeDisplayMode) { + const animClass = + mode === 'pip' + ? 'mcp-enter-pip' + : mode === 'fullscreen' + ? 'mcp-enter-fullscreen' + : 'mcp-enter-inline'; + + requestAnimationFrame(() => { + el.classList.add(animClass); + enterAnimRef.current = animClass; + + el.addEventListener( + 'animationend', + () => { + el.classList.remove(animClass); + if (enterAnimRef.current === animClass) { + enterAnimRef.current = null; + } + }, + { once: true } + ); + }); + } + }, + [onDisplayModeChange, activeDisplayMode, containerRef] + ); + + // ── PiP drag ────────────────────────────────────────────────────────── + + const [pipPosition, setPipPosition] = useState({ x: 0, y: 0 }); + const pipPositionRef = useRef(pipPosition); + const pipDragRef = useRef<{ + startX: number; + startY: number; + originX: number; + originY: number; + } | null>(null); + + useEffect(() => { + pipPositionRef.current = pipPosition; + }, [pipPosition]); + + const clampPipPosition = useCallback((pos: { x: number; y: number }) => { + const minX = PIP_WIDTH + PIP_MARGIN_RIGHT - window.innerWidth; + const maxX = PIP_MARGIN_RIGHT; + const minY = PIP_HEIGHT + PIP_MARGIN_BOTTOM - window.innerHeight; + const maxY = PIP_MARGIN_BOTTOM; + return { + x: minX > maxX ? 0 : Math.max(minX, Math.min(maxX, pos.x)), + y: minY > maxY ? 0 : Math.max(minY, Math.min(maxY, pos.y)), + }; + }, []); + + const handlePipPointerDown = useCallback((e: React.PointerEvent) => { + e.preventDefault(); + (e.currentTarget as HTMLElement).setPointerCapture(e.pointerId); + const { x, y } = pipPositionRef.current; + pipDragRef.current = { startX: e.clientX, startY: e.clientY, originX: x, originY: y }; + }, []); + + const handlePipPointerMove = useCallback( + (e: React.PointerEvent) => { + if (!pipDragRef.current) return; + const dx = e.clientX - pipDragRef.current.startX; + const dy = e.clientY - pipDragRef.current.startY; + setPipPosition( + clampPipPosition({ + x: pipDragRef.current.originX + dx, + y: pipDragRef.current.originY + dy, + }) + ); + }, + [clampPipPosition] + ); + + const handlePipPointerUp = useCallback((e: React.PointerEvent) => { + (e.currentTarget as HTMLElement).releasePointerCapture(e.pointerId); + pipDragRef.current = null; + }, []); + + const handlePipLostPointerCapture = useCallback(() => { + pipDragRef.current = null; + }, []); + + const handlePipKeyDown = useCallback( + (e: React.KeyboardEvent) => { + const step = e.shiftKey ? 32 : 8; + let dx = 0; + let dy = 0; + switch (e.key) { + case 'ArrowUp': + dy = -step; + break; + case 'ArrowDown': + dy = step; + break; + case 'ArrowLeft': + dx = -step; + break; + case 'ArrowRight': + dx = step; + break; + default: + return; + } + e.preventDefault(); + setPipPosition((prev) => clampPipPosition({ x: prev.x + dx, y: prev.y + dy })); + }, + [clampPipPosition] + ); + + // ── Effects ─────────────────────────────────────────────────────────── + + // Cache iframe contentWindows for O(1) source matching via MutationObserver. + useEffect(() => { + const container = containerRef.current; + if (!container) return; + + const refreshCache = () => { + const windows = iframeWindowsRef.current; + windows.clear(); + container.querySelectorAll('iframe').forEach((iframe) => { + if (iframe.contentWindow) windows.add(iframe.contentWindow); + }); + }; + + refreshCache(); + const observer = new MutationObserver(refreshCache); + observer.observe(container, { childList: true, subtree: true }); + return () => observer.disconnect(); + }, [containerRef]); + + // Intercept app postMessages for: + // 1. ui/initialize — extract appCapabilities.availableDisplayModes + // 2. ui/request-display-mode — change display mode on behalf of the app + useEffect(() => { + if (isStandalone) return; + + const handleMessage = (e: MessageEvent) => { + const data = e.data; + if (!data || typeof data !== 'object') return; + // eslint-disable-next-line no-undef + if (!e.source || !iframeWindowsRef.current.has(e.source as Window)) return; + + if (data.method === 'ui/initialize' && data.params) { + const caps = data.params.appCapabilities || data.params.capabilities; + if (caps?.availableDisplayModes && Array.isArray(caps.availableDisplayModes)) { + setAppDeclaredModes(caps.availableDisplayModes); + } + } + + // After initialize, only allow modes both host and app agree on. + // Before initialize (effectiveDisplayModes empty), fall back to the full host list. + if (data.method === 'ui/request-display-mode' && data.params?.mode) { + const requested = data.params.mode as McpUiDisplayMode; + const allowed = + effectiveDisplayModes.length > 0 ? effectiveDisplayModes : AVAILABLE_DISPLAY_MODES; + if (allowed.includes(requested)) { + changeDisplayMode(requested); + } + } + }; + + window.addEventListener('message', handleMessage); + return () => window.removeEventListener('message', handleMessage); + }, [isStandalone, changeDisplayMode, effectiveDisplayModes]); + + // Escape key exits fullscreen. + useEffect(() => { + if (activeDisplayMode !== 'fullscreen') return; + fullscreenCloseRef.current?.focus(); + const handleKeyDown = (e: KeyboardEvent) => { + if (e.key === 'Escape') changeDisplayMode('inline'); + }; + window.addEventListener('keydown', handleKeyDown); + return () => window.removeEventListener('keydown', handleKeyDown); + }, [activeDisplayMode, changeDisplayMode]); + + // Reset PiP position when entering PiP mode. + useEffect(() => { + if (activeDisplayMode === 'pip') { + setPipPosition({ x: 0, y: 0 }); + } + }, [activeDisplayMode]); + + // ── Derived state ───────────────────────────────────────────────────── + + const isFullscreen = activeDisplayMode === 'fullscreen'; + const isPip = activeDisplayMode === 'pip'; + const isFillsViewport = isFullscreen || isStandalone; + const isInline = !isFillsViewport && !isPip; + + const appSupportsFullscreen = effectiveDisplayModes.includes('fullscreen'); + const appSupportsPip = effectiveDisplayModes.includes('pip'); + + return { + activeDisplayMode, + effectiveDisplayModes, + isStandalone, + isFullscreen, + isPip, + isFillsViewport, + isInline, + appSupportsFullscreen, + appSupportsPip, + + changeDisplayMode, + + inlineHeight: savedInlineHeight, + pipPosition, + + pipHandlers: { + onPointerDown: handlePipPointerDown, + onPointerMove: handlePipPointerMove, + onPointerUp: handlePipPointerUp, + onLostPointerCapture: handlePipLostPointerCapture, + onKeyDown: handlePipKeyDown, + }, + + fullscreenCloseRef, + }; +} diff --git a/ui/desktop/src/styles/main.css b/ui/desktop/src/styles/main.css index 1d9e181cc9..f4f96837f9 100644 --- a/ui/desktop/src/styles/main.css +++ b/ui/desktop/src/styles/main.css @@ -878,3 +878,52 @@ p > code.bg-inline-code { max-height: var(--search-bar-height); } } + +/* ========================================================================== + MCP App Display Mode Entrance Animations + ========================================================================== */ + +@keyframes mcp-enter-pip { + from { + opacity: 0; + transform: scale(0.85) translate(12px, -12px); + } + to { + opacity: 1; + transform: none; + } +} + +@keyframes mcp-enter-fullscreen { + from { + opacity: 0; + transform: scale(0.95); + } + to { + opacity: 1; + transform: none; + } +} + +@keyframes mcp-enter-inline { + from { + opacity: 0; + transform: scale(1.03); + } + to { + opacity: 1; + transform: none; + } +} + +.mcp-app-container.mcp-enter-pip { + animation: mcp-enter-pip 200ms cubic-bezier(0.2, 0, 0, 1) both; +} + +.mcp-app-container.mcp-enter-fullscreen { + animation: mcp-enter-fullscreen 200ms cubic-bezier(0.2, 0, 0, 1) both; +} + +.mcp-app-container.mcp-enter-inline { + animation: mcp-enter-inline 180ms cubic-bezier(0.2, 0, 0, 1) both; +}