From 568604259f77563d1f72f3cb5cc3eeebec1b6b1e Mon Sep 17 00:00:00 2001 From: spencrmartin Date: Wed, 12 Nov 2025 13:14:40 -0500 Subject: [PATCH] Complete webviewer context injection system for AI prompts - Created WebViewerContext to centralize active webviewer information - WebViewer components now register/update/unregister with context - Added WebViewerProvider to App component for application-wide access - Integrated context injection into BaseChat handleSubmit function - AI prompts now automatically include webviewer context when active - Context provides structured information about open tools/apps with: - URL, title, domain, and security information - Context-specific suggestions based on detected services - Clear categorization between sidecar and main webviewers This helps the AI understand what tools/apps the user currently has open and provide more contextually relevant assistance. --- ui/desktop/src/App.tsx | 19 ++- ui/desktop/src/components/BaseChat.tsx | 15 +- ui/desktop/src/components/WebViewer.tsx | 49 +++++- ui/desktop/src/contexts/WebViewerContext.tsx | 154 +++++++++++++++++++ 4 files changed, 227 insertions(+), 10 deletions(-) create mode 100644 ui/desktop/src/contexts/WebViewerContext.tsx diff --git a/ui/desktop/src/App.tsx b/ui/desktop/src/App.tsx index cee8830da1..7fd4327815 100644 --- a/ui/desktop/src/App.tsx +++ b/ui/desktop/src/App.tsx @@ -28,6 +28,7 @@ import ProviderSettings from './components/settings/providers/ProviderSettingsPa import { AppLayout } from './components/Layout/AppLayout'; import { ChatProvider } from './contexts/ChatContext'; import { DraftProvider } from './contexts/DraftContext'; +import { WebViewerProvider } from './contexts/WebViewerContext'; import 'react-toastify/dist/ReactToastify.css'; import { useConfig, ConfigProvider } from './components/ConfigContext'; @@ -546,14 +547,16 @@ export function AppInner() { path="/" element={ - - - + + + + + } > diff --git a/ui/desktop/src/components/BaseChat.tsx b/ui/desktop/src/components/BaseChat.tsx index 975bc1632f..8df879a455 100644 --- a/ui/desktop/src/components/BaseChat.tsx +++ b/ui/desktop/src/components/BaseChat.tsx @@ -67,6 +67,7 @@ import { ChatType } from '../types/chat'; import { useToolCount } from './alerts/useToolCount'; import { SidecarInvoker } from './Layout/SidecarInvoker'; import { useSidecar } from './SidecarLayout'; +import { useWebViewerContextOptional } from '../contexts/WebViewerContext'; // Context for sharing current model info const CurrentModelContext = createContext<{ model: string; mode: string } | null>(null); @@ -125,6 +126,9 @@ function BaseChatContent({ // Sidecar functionality const sidecar = useSidecar(); + // WebViewer context for prompt injection + const webViewerContext = useWebViewerContextOptional(); + const handleShowLocalhost = () => { console.log('Localhost viewer requested'); if (sidecar) { @@ -325,7 +329,16 @@ function BaseChatContent({ onMessageSubmit(combinedTextFromInput); } - engineHandleSubmit(combinedTextFromInput); + // Inject webviewer context into the prompt if webviewers are active + let messageWithContext = combinedTextFromInput; + if (webViewerContext?.getWebViewerContext && combinedTextFromInput.trim()) { + const contextInfo = webViewerContext.getWebViewerContext(); + if (contextInfo.trim()) { + messageWithContext = `${contextInfo}\n\n---\n\n${combinedTextFromInput}`; + } + } + + engineHandleSubmit(messageWithContext); }; const toolCount = useToolCount(chat.sessionId); diff --git a/ui/desktop/src/components/WebViewer.tsx b/ui/desktop/src/components/WebViewer.tsx index d92a44dd1a..ed3240a642 100644 --- a/ui/desktop/src/components/WebViewer.tsx +++ b/ui/desktop/src/components/WebViewer.tsx @@ -2,6 +2,7 @@ import { useRef, useState, useEffect, useCallback } from 'react'; import { RefreshCw, ExternalLink, ChevronLeft, ChevronRight, Home, Globe, Shield, ShieldOff } from 'lucide-react'; import { Button } from './ui/button'; import { Tooltip, TooltipTrigger, TooltipContent } from './ui/Tooltip'; +import { useWebViewerContextOptional } from '../contexts/WebViewerContext'; interface WebViewerProps { initialUrl?: string; @@ -86,6 +87,9 @@ export function WebViewer({ const containerRef = useRef(null); const updateBoundsTimeoutRef = useRef(null); + // WebViewer context for AI prompt injection + const webViewerContext = useWebViewerContextOptional(); + // Initialize from localStorage or use initialUrl const storageKey = allowAllSites ? 'goose-webviewer-url' : 'goose-sidecar-url'; const [url, setUrl] = useState(() => { @@ -184,10 +188,53 @@ export function WebViewer({ }; }, [url, isVisible]); + // Register/unregister with WebViewer context for AI prompt injection + useEffect(() => { + if (!webViewerContext || !childWindowCreated || !actualUrl) return; + + const webViewerInfo = { + id: childWindowId.current, + url: actualUrl, + title: pageTitle || 'Loading...', + domain: getDomainFromUrl(actualUrl), + isSecure: isSecure, + isLocalhost: isLocalhostUrl(actualUrl), + lastUpdated: new Date(), + type: (allowAllSites ? 'main' : 'sidecar') as 'sidecar' | 'main', + }; + + webViewerContext.registerWebViewer(webViewerInfo); + console.log('[WebViewer] Registered with context:', webViewerInfo); + + return () => { + webViewerContext.unregisterWebViewer(childWindowId.current); + console.log('[WebViewer] Unregistered from context:', childWindowId.current); + }; + }, [webViewerContext, childWindowCreated, actualUrl, pageTitle, isSecure, allowAllSites]); + + // Update WebViewer context when URL or title changes + useEffect(() => { + if (!webViewerContext || !childWindowCreated || !actualUrl) return; + + webViewerContext.updateWebViewer(childWindowId.current, { + url: actualUrl, + title: pageTitle || 'Loading...', + domain: getDomainFromUrl(actualUrl), + isSecure: isSecure, + isLocalhost: isLocalhostUrl(actualUrl), + }); + }, [webViewerContext, childWindowCreated, actualUrl, pageTitle, isSecure]); + // Cleanup child window on component unmount with robust error handling useEffect(() => { const cleanup = async () => { console.log('WebViewer component unmounting, cleaning up child window:', childWindowId.current); + + // Unregister from context first + if (webViewerContext) { + webViewerContext.unregisterWebViewer(childWindowId.current); + } + if (childWindowId.current && childWindowCreated) { try { // First hide the child window @@ -210,7 +257,7 @@ export function WebViewer({ return () => { cleanup(); }; - }, [childWindowCreated]); + }, [childWindowCreated, webViewerContext]); // Update child window bounds when container resizes (throttled) useEffect(() => { diff --git a/ui/desktop/src/contexts/WebViewerContext.tsx b/ui/desktop/src/contexts/WebViewerContext.tsx new file mode 100644 index 0000000000..5bcb92ffc1 --- /dev/null +++ b/ui/desktop/src/contexts/WebViewerContext.tsx @@ -0,0 +1,154 @@ +import React, { createContext, useContext, useState, useCallback } from 'react'; + +export interface WebViewerInfo { + id: string; + url: string; + title: string; + domain: string; + isSecure: boolean; + isLocalhost: boolean; + lastUpdated: Date; + type: 'sidecar' | 'main'; // Type of webviewer (sidecar for sidebar, main for full screen) +} + +interface WebViewerContextType { + activeWebViewers: WebViewerInfo[]; + registerWebViewer: (info: WebViewerInfo) => void; + updateWebViewer: (id: string, updates: Partial) => void; + unregisterWebViewer: (id: string) => void; + getWebViewerContext: () => string; +} + +const WebViewerContext = createContext(null); + +export const useWebViewerContext = () => { + const context = useContext(WebViewerContext); + if (!context) { + throw new Error('useWebViewerContext must be used within a WebViewerProvider'); + } + return context; +}; + +export const useWebViewerContextOptional = () => { + return useContext(WebViewerContext); +}; + +export const WebViewerProvider: React.FC<{ children: React.ReactNode }> = ({ children }) => { + const [activeWebViewers, setActiveWebViewers] = useState([]); + + const registerWebViewer = useCallback((info: WebViewerInfo) => { + setActiveWebViewers(prev => { + // Remove any existing webviewer with the same ID + const filtered = prev.filter(viewer => viewer.id !== info.id); + return [...filtered, info]; + }); + }, []); + + const updateWebViewer = useCallback((id: string, updates: Partial) => { + setActiveWebViewers(prev => + prev.map(viewer => + viewer.id === id + ? { ...viewer, ...updates, lastUpdated: new Date() } + : viewer + ) + ); + }, []); + + const unregisterWebViewer = useCallback((id: string) => { + setActiveWebViewers(prev => prev.filter(viewer => viewer.id !== id)); + }, []); + + const getWebViewerContext = useCallback((): string => { + if (activeWebViewers.length === 0) { + return ''; + } + + const contextParts: string[] = []; + + // Group webviewers by type + const sidecarViewers = activeWebViewers.filter(v => v.type === 'sidecar'); + const mainViewers = activeWebViewers.filter(v => v.type === 'main'); + + if (sidecarViewers.length > 0) { + contextParts.push('\n## Currently Open Sidecar Web Tools'); + contextParts.push('The user has the following web tools open in the sidecar (sidebar) that you can reference and help with:'); + + sidecarViewers.forEach((viewer, index) => { + const securityInfo = viewer.isSecure ? 'HTTPS (secure)' : 'HTTP (insecure)'; + const locationInfo = viewer.isLocalhost ? 'Local development server' : 'External website'; + + contextParts.push(`\n${index + 1}. **${viewer.title || 'Untitled Page'}**`); + contextParts.push(` - URL: ${viewer.url}`); + contextParts.push(` - Domain: ${viewer.domain}`); + contextParts.push(` - Type: ${locationInfo} (${securityInfo})`); + contextParts.push(` - Last updated: ${viewer.lastUpdated.toLocaleTimeString()}`); + + // Add context-specific suggestions based on the URL/domain + if (viewer.isLocalhost) { + if (viewer.url.includes(':3000') || viewer.url.includes('localhost:3000')) { + contextParts.push(` - Context: This appears to be a React development server. You can help with React development, debugging, component issues, etc.`); + } else if (viewer.url.includes(':8000') || viewer.url.includes(':8080')) { + contextParts.push(` - Context: This appears to be a local web server. You can help with web development, API testing, server configuration, etc.`); + } else if (viewer.url.includes(':5000')) { + contextParts.push(` - Context: This might be a Flask/Python development server. You can help with Python web development, API debugging, etc.`); + } else { + contextParts.push(` - Context: This is a local development server. You can help with development tasks, debugging, testing, etc.`); + } + } else { + // External websites - provide context based on domain + if (viewer.domain.includes('github.com')) { + contextParts.push(` - Context: GitHub repository or page. You can help with Git operations, code review, repository management, etc.`); + } else if (viewer.domain.includes('stackoverflow.com')) { + contextParts.push(` - Context: Stack Overflow page. You can help explain solutions, debug similar issues, or provide alternative approaches.`); + } else if (viewer.domain.includes('docs.') || viewer.domain.includes('documentation')) { + contextParts.push(` - Context: Documentation page. You can help explain concepts, provide examples, or clarify usage.`); + } else { + contextParts.push(` - Context: External website. You can help with web-related tasks or provide information about this site.`); + } + } + }); + + contextParts.push('\n**How to help with sidecar tools:**'); + contextParts.push('- You can reference these tools when providing assistance'); + contextParts.push('- For localhost servers, you can help with development, debugging, and testing'); + contextParts.push('- For external sites, you can provide context-aware help based on what the user is viewing'); + contextParts.push('- Always mention when you\'re referencing information from their open sidecar tools'); + } + + if (mainViewers.length > 0) { + contextParts.push('\n## Currently Open Main Web Tools'); + contextParts.push('The user has the following web tools open in the main view:'); + + mainViewers.forEach((viewer, index) => { + const securityInfo = viewer.isSecure ? 'HTTPS (secure)' : 'HTTP (insecure)'; + const locationInfo = viewer.isLocalhost ? 'Local development server' : 'External website'; + + contextParts.push(`\n${index + 1}. **${viewer.title || 'Untitled Page'}**`); + contextParts.push(` - URL: ${viewer.url}`); + contextParts.push(` - Domain: ${viewer.domain}`); + contextParts.push(` - Type: ${locationInfo} (${securityInfo})`); + contextParts.push(` - Last updated: ${viewer.lastUpdated.toLocaleTimeString()}`); + }); + } + + if (contextParts.length > 0) { + contextParts.push('\n---\n'); + } + + return contextParts.join('\n'); + }, [activeWebViewers]); + + const value: WebViewerContextType = { + activeWebViewers, + registerWebViewer, + updateWebViewer, + unregisterWebViewer, + getWebViewerContext, + }; + + return ( + + {children} + + ); +};