ai-space/goose
3
0
Fork 0
mirror of https://github.com/aaif-goose/goose.git synced 2026-07-03 14:10:03 +02:00
Code Issues Packages Projects Releases Wiki Activity

Fix eleven labs audio transcription and added more logging (#4358)

Browse Source
This commit is contained in:
Zane
2025-08-27 12:53:05 -07:00
committed by GitHub
parent 862dd97ca4
commit 1014d9e519
7 changed files with 316 additions and 189 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/goose-server/src/routes/audio.rs
+182 -137
View File
@@ -42,6 +42,155 @@ struct WhisperResponse {
text: String,
}
/// Validate audio input and return decoded bytes and file extension
fn validate_audio_input(
audio: &str,
mime_type: &str,
) -> Result<(Vec<u8>, &'static str), StatusCode> {
// Decode the base64 audio data
let audio_bytes = BASE64.decode(audio).map_err(|_| StatusCode::BAD_REQUEST)?;
// Check file size
if audio_bytes.len() > MAX_AUDIO_SIZE_BYTES {
tracing::warn!(
"Audio file too large: {} bytes (max: {} bytes)",
audio_bytes.len(),
MAX_AUDIO_SIZE_BYTES
);
return Err(StatusCode::PAYLOAD_TOO_LARGE);
}
// Determine file extension based on MIME type
let file_extension = match mime_type {
"audio/webm" => "webm",
"audio/webm;codecs=opus" => "webm",
"audio/mp4" => "mp4",
"audio/mpeg" => "mp3",
"audio/mpga" => "mpga",
"audio/m4a" => "m4a",
"audio/wav" => "wav",
"audio/x-wav" => "wav",
_ => return Err(StatusCode::UNSUPPORTED_MEDIA_TYPE),
};
Ok((audio_bytes, file_extension))
}
/// Get OpenAI configuration (API key and host)
fn get_openai_config() -> Result<(String, String), StatusCode> {
let config = goose::config::Config::global();
let api_key: String = config.get_secret("OPENAI_API_KEY").map_err(|e| {
tracing::error!("Failed to get OpenAI API key: {:?}", e);
StatusCode::PRECONDITION_FAILED
})?;
let openai_host = match config.get("OPENAI_HOST", false) {
Ok(value) => value
.as_str()
.map(|s| s.to_string())
.unwrap_or_else(|| "https://api.openai.com".to_string()),
Err(_) => "https://api.openai.com".to_string(),
};
Ok((api_key, openai_host))
}
/// Send transcription request to OpenAI Whisper API
async fn send_openai_request(
audio_bytes: Vec<u8>,
file_extension: &str,
mime_type: &str,
api_key: &str,
openai_host: &str,
) -> Result<WhisperResponse, StatusCode> {
tracing::info!("Using OpenAI host: {}", openai_host);
tracing::info!(
"Audio file size: {} bytes, extension: {}, mime_type: {}",
audio_bytes.len(),
file_extension,
mime_type
);
// Create a multipart form with the audio file
let part = reqwest::multipart::Part::bytes(audio_bytes)
.file_name(format!("audio.{}", file_extension))
.mime_str(mime_type)
.map_err(|e| {
tracing::error!("Failed to create multipart part: {:?}", e);
StatusCode::INTERNAL_SERVER_ERROR
})?;
let form = reqwest::multipart::Form::new()
.part("file", part)
.text("model", "whisper-1")
.text("response_format", "json");
tracing::info!("Created multipart form for OpenAI Whisper API");
// Make request to OpenAI Whisper API
let client = Client::builder()
.timeout(Duration::from_secs(OPENAI_TIMEOUT_SECONDS))
.build()
.map_err(|e| {
tracing::error!("Failed to create HTTP client: {}", e);
StatusCode::INTERNAL_SERVER_ERROR
})?;
tracing::info!(
"Sending request to OpenAI: {}/v1/audio/transcriptions",
openai_host
);
let response = client
.post(format!("{}/v1/audio/transcriptions", openai_host))
.header("Authorization", format!("Bearer {}", api_key))
.multipart(form)
.send()
.await
.map_err(|e| {
if e.is_timeout() {
tracing::error!(
"OpenAI API request timed out after {}s",
OPENAI_TIMEOUT_SECONDS
);
StatusCode::GATEWAY_TIMEOUT
} else {
tracing::error!("Failed to send request to OpenAI: {}", e);
StatusCode::SERVICE_UNAVAILABLE
}
})?;
tracing::info!(
"Received response from OpenAI with status: {}",
response.status()
);
if !response.status().is_success() {
let status = response.status();
let error_text = response.text().await.unwrap_or_default();
tracing::error!("OpenAI API error (status: {}): {}", status, error_text);
// Check for specific error codes
if status == 401 {
tracing::error!("OpenAI API key appears to be invalid or unauthorized");
return Err(StatusCode::UNAUTHORIZED);
} else if status == 429 {
tracing::error!("OpenAI API quota or rate limit exceeded");
return Err(StatusCode::TOO_MANY_REQUESTS);
}
return Err(StatusCode::BAD_GATEWAY);
}
let whisper_response: WhisperResponse = response.json().await.map_err(|e| {
tracing::error!("Failed to parse OpenAI response: {}", e);
StatusCode::INTERNAL_SERVER_ERROR
})?;
Ok(whisper_response)
}
/// Transcribe audio using OpenAI's Whisper API
///
/// # Request
@@ -66,100 +215,17 @@ async fn transcribe_handler(
) -> Result<Json<TranscribeResponse>, StatusCode> {
verify_secret_key(&headers, &state)?;
// Validate input first before checking API key configuration
// Decode the base64 audio data
let audio_bytes = BASE64
.decode(&request.audio)
.map_err(|_| StatusCode::BAD_REQUEST)?;
let (audio_bytes, file_extension) = validate_audio_input(&request.audio, &request.mime_type)?;
let (api_key, openai_host) = get_openai_config()?;
// Check file size
if audio_bytes.len() > MAX_AUDIO_SIZE_BYTES {
tracing::warn!(
"Audio file too large: {} bytes (max: {} bytes)",
audio_bytes.len(),
MAX_AUDIO_SIZE_BYTES
);
return Err(StatusCode::PAYLOAD_TOO_LARGE);
}
// Determine file extension based on MIME type
let file_extension = match request.mime_type.as_str() {
"audio/webm" => "webm",
"audio/mp4" => "mp4",
"audio/mpeg" => "mp3",
"audio/mpga" => "mpga",
"audio/m4a" => "m4a",
"audio/wav" => "wav",
"audio/x-wav" => "wav",
_ => return Err(StatusCode::UNSUPPORTED_MEDIA_TYPE),
};
// Get the OpenAI API key from config (after input validation)
let config = goose::config::Config::global();
let api_key: String = config
.get_secret("OPENAI_API_KEY")
.map_err(|_| StatusCode::PRECONDITION_FAILED)?;
// Get the OpenAI host from config (with default)
let openai_host = match config.get("OPENAI_HOST", false) {
Ok(value) => value
.as_str()
.map(|s| s.to_string())
.unwrap_or_else(|| "https://api.openai.com".to_string()),
Err(_) => "https://api.openai.com".to_string(),
};
tracing::debug!("Using OpenAI host: {}", openai_host);
// Create a multipart form with the audio file
let part = reqwest::multipart::Part::bytes(audio_bytes)
.file_name(format!("audio.{}", file_extension))
.mime_str(&request.mime_type)
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
let form = reqwest::multipart::Form::new()
.part("file", part)
.text("model", "whisper-1")
.text("response_format", "json");
// Make request to OpenAI Whisper API
let client = Client::builder()
.timeout(Duration::from_secs(OPENAI_TIMEOUT_SECONDS))
.build()
.map_err(|e| {
tracing::error!("Failed to create HTTP client: {}", e);
StatusCode::INTERNAL_SERVER_ERROR
})?;
let response = client
.post(format!("{}/v1/audio/transcriptions", openai_host))
.header("Authorization", format!("Bearer {}", api_key))
.multipart(form)
.send()
.await
.map_err(|e| {
if e.is_timeout() {
tracing::error!(
"OpenAI API request timed out after {}s",
OPENAI_TIMEOUT_SECONDS
);
StatusCode::GATEWAY_TIMEOUT
} else {
tracing::error!("Failed to send request to OpenAI: {}", e);
StatusCode::SERVICE_UNAVAILABLE
}
})?;
if !response.status().is_success() {
let error_text = response.text().await.unwrap_or_default();
tracing::error!("OpenAI API error: {}", error_text);
return Err(StatusCode::BAD_GATEWAY);
}
let whisper_response: WhisperResponse = response.json().await.map_err(|e| {
tracing::error!("Failed to parse OpenAI response: {}", e);
StatusCode::INTERNAL_SERVER_ERROR
})?;
let whisper_response = send_openai_request(
audio_bytes,
file_extension,
&request.mime_type,
&api_key,
&openai_host,
)
.await?;
Ok(Json(TranscribeResponse {
text: whisper_response.text,
@@ -177,39 +243,13 @@ async fn transcribe_elevenlabs_handler(
) -> Result<Json<TranscribeResponse>, StatusCode> {
verify_secret_key(&headers, &state)?;
// Validate input first before checking API key configuration
// Decode the base64 audio data
let audio_bytes = BASE64
.decode(&request.audio)
.map_err(|_| StatusCode::BAD_REQUEST)?;
// Check file size
if audio_bytes.len() > MAX_AUDIO_SIZE_BYTES {
tracing::warn!(
"Audio file too large: {} bytes (max: {} bytes)",
audio_bytes.len(),
MAX_AUDIO_SIZE_BYTES
);
return Err(StatusCode::PAYLOAD_TOO_LARGE);
}
// Determine file extension and content type based on MIME type
let (file_extension, content_type) = match request.mime_type.as_str() {
"audio/webm" => ("webm", "audio/webm"),
"audio/mp4" => ("mp4", "audio/mp4"),
"audio/mpeg" => ("mp3", "audio/mpeg"),
"audio/mpga" => ("mp3", "audio/mpeg"),
"audio/m4a" => ("m4a", "audio/m4a"),
"audio/wav" => ("wav", "audio/wav"),
"audio/x-wav" => ("wav", "audio/wav"),
_ => return Err(StatusCode::UNSUPPORTED_MEDIA_TYPE),
};
let (audio_bytes, file_extension) = validate_audio_input(&request.audio, &request.mime_type)?;
// Get the ElevenLabs API key from config (after input validation)
let config = goose::config::Config::global();
// First try to get it as a secret
let api_key: String = match config.get_secret("ELEVENLABS_API_KEY") {
let api_key: String = match config.get_secret::<String>("ELEVENLABS_API_KEY") {
Ok(key) => key,
Err(_) => {
// Try to get it as non-secret (for backward compatibility)
@@ -217,7 +257,6 @@ async fn transcribe_elevenlabs_handler(
Ok(value) => {
match value.as_str() {
Some(key_str) => {
tracing::info!("Migrating ElevenLabs API key to secret storage");
let key = key_str.to_string();
// Migrate to secret storage
if let Err(e) = config.set(
@@ -228,17 +267,25 @@ async fn transcribe_elevenlabs_handler(
tracing::error!("Failed to migrate ElevenLabs API key: {:?}", e);
}
// Delete the non-secret version
let _ = config.delete("ELEVENLABS_API_KEY");
if let Err(e) = config.delete("ELEVENLABS_API_KEY") {
tracing::warn!(
"Failed to delete non-secret ElevenLabs API key: {:?}",
e
);
}
key
}
None => {
tracing::error!("ElevenLabs API key is not a string");
tracing::error!(
"ElevenLabs API key is not a string, found: {:?}",
value
);
return Err(StatusCode::PRECONDITION_FAILED);
}
}
}
Err(e) => {
tracing::error!("Failed to get ElevenLabs API key from config: {:?}", e);
Err(_) => {
tracing::error!("No ElevenLabs API key found in configuration");
return Err(StatusCode::PRECONDITION_FAILED);
}
}
@@ -248,7 +295,7 @@ async fn transcribe_elevenlabs_handler(
// Create multipart form for ElevenLabs API
let part = reqwest::multipart::Part::bytes(audio_bytes)
.file_name(format!("audio.{}", file_extension))
.mime_str(content_type)
.mime_str(&request.mime_type)
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
let form = reqwest::multipart::Form::new()
@@ -286,8 +333,9 @@ async fn transcribe_elevenlabs_handler(
})?;
if !response.status().is_success() {
let status = response.status();
let error_text = response.text().await.unwrap_or_default();
tracing::error!("ElevenLabs API error: {}", error_text);
tracing::error!("ElevenLabs API error (status: {}): {}", status, error_text);
// Check for specific error codes
if error_text.contains("Unauthorized") || error_text.contains("Invalid API key") {
@@ -330,16 +378,13 @@ async fn check_dictation_config(
let config = goose::config::Config::global();
// Check if ElevenLabs API key is configured
let has_elevenlabs = config
.get_secret::<String>("ELEVENLABS_API_KEY")
.map(|_| true)
.unwrap_or_else(|_| {
let has_elevenlabs = match config.get_secret::<String>("ELEVENLABS_API_KEY") {
Ok(_) => true,
Err(_) => {
// Check non-secret for backward compatibility
config
.get("ELEVENLABS_API_KEY", false)
.map(|_| true)
.unwrap_or(false)
});
config.get("ELEVENLABS_API_KEY", false).is_ok()
}
};
Ok(Json(serde_json::json!({
"elevenlabs": has_elevenlabs
crates/goose-server/src/routes/config_management.rs
+4 -1
View File
@@ -167,6 +167,7 @@ pub async fn read_config(
}
let config = Config::global();
let response_value = match config.get(&query.key, query.is_secret) {
Ok(value) => {
if query.is_secret {
@@ -182,7 +183,9 @@ pub async fn read_config(
Value::Null
}
}
Err(_) => return Err(StatusCode::INTERNAL_SERVER_ERROR),
Err(_) => {
return Err(StatusCode::INTERNAL_SERVER_ERROR);
}
};
Ok(Json(response_value))
}
ui/desktop/src/components/ChatInput.tsx
+2 -2
View File
@@ -1259,8 +1259,8 @@ export default function ChatInput({
{/* Inline action buttons on the right */}
<div className="flex items-center gap-1 px-2 relative">
{/* Microphone button - show if dictation is enabled, disable if not configured */}
{(dictationSettings?.enabled || dictationSettings?.provider === null) && (
{/* Microphone button - show only if dictation is enabled */}
{dictationSettings?.enabled && (
<>
{!canUseDictation ? (
<Tooltip>
ui/desktop/src/components/settings/dictation/DictationSection.tsx
+23 -13
View File
@@ -4,9 +4,11 @@ import { ChevronDown } from 'lucide-react';
import { Input } from '../../ui/input';
import { useConfig } from '../../ConfigContext';
import { DictationProvider, DictationSettings } from '../../../hooks/useDictationSettings';
const DICTATION_SETTINGS_KEY = 'dictation_settings';
const ELEVENLABS_API_KEY = 'ELEVENLABS_API_KEY';
import {
DICTATION_SETTINGS_KEY,
ELEVENLABS_API_KEY,
getDefaultDictationSettings,
} from '../../../hooks/dictationConstants';
export default function DictationSection() {
const [settings, setSettings] = useState<DictationSettings>({
@@ -27,20 +29,19 @@ export default function DictationSection() {
useEffect(() => {
const loadSettings = async () => {
const savedSettings = localStorage.getItem(DICTATION_SETTINGS_KEY);
let loadedSettings: DictationSettings;
if (savedSettings) {
const parsed = JSON.parse(savedSettings);
setSettings(parsed);
setShowElevenLabsKey(parsed.provider === 'elevenlabs');
loadedSettings = parsed;
} else {
// Default settings
const defaultSettings: DictationSettings = {
enabled: true,
provider: 'openai',
};
setSettings(defaultSettings);
localStorage.setItem(DICTATION_SETTINGS_KEY, JSON.stringify(defaultSettings));
loadedSettings = await getDefaultDictationSettings(getProviders);
}
setSettings(loadedSettings);
setShowElevenLabsKey(loadedSettings.provider === 'elevenlabs');
// Load ElevenLabs API key from storage
setIsLoadingKey(true);
try {
@@ -58,7 +59,7 @@ export default function DictationSection() {
};
loadSettings();
}, [read]);
}, [read, getProviders]);
// Save ElevenLabs key on unmount if it has changed
useEffect(() => {
@@ -109,6 +110,7 @@ export default function DictationSection() {
};
const saveSettings = (newSettings: DictationSettings) => {
console.log('Saving dictation settings to localStorage:', newSettings);
setSettings(newSettings);
localStorage.setItem(DICTATION_SETTINGS_KEY, JSON.stringify(newSettings));
};
@@ -130,18 +132,26 @@ export default function DictationSection() {
const handleElevenLabsKeyChange = (key: string) => {
setElevenLabsApiKey(key);
elevenLabsApiKeyRef.current = key;
// If user starts typing, they're updating the key
if (key.length > 0) {
setHasElevenLabsKey(false); // Hide "configured" while typing
}
};
const saveElevenLabsKey = async () => {
// Save to secure storage
try {
if (elevenLabsApiKey.trim()) {
console.log('Saving ElevenLabs API key to secure storage...');
await upsert(ELEVENLABS_API_KEY, elevenLabsApiKey, true);
setHasElevenLabsKey(true);
console.log('ElevenLabs API key saved successfully');
} else {
// If key is empty, remove it from storage
console.log('Removing ElevenLabs API key from secure storage...');
await upsert(ELEVENLABS_API_KEY, null, true);
setHasElevenLabsKey(false);
console.log('ElevenLabs API key removed successfully');
}
} catch (error) {
console.error('Error saving ElevenLabs API key:', error);
ui/desktop/src/hooks/dictationConstants.ts
+25
View File
@@ -0,0 +1,25 @@
import { DictationSettings, DictationProvider } from './useDictationSettings';
export const DICTATION_SETTINGS_KEY = 'dictation_settings';
export const ELEVENLABS_API_KEY = 'ELEVENLABS_API_KEY';
export const getDefaultDictationSettings = async (
getProviders: (refresh: boolean) => Promise<Array<{ name: string; is_configured: boolean }>>
): Promise<DictationSettings> => {
const providers = await getProviders(false);
// Check if we have an OpenAI API key as primary default
const openAIProvider = providers.find((p) => p.name === 'openai');
if (openAIProvider && openAIProvider.is_configured) {
return {
enabled: true,
provider: 'openai' as DictationProvider,
};
} else {
return {
enabled: false,
provider: null as DictationProvider,
};
}
};
ui/desktop/src/hooks/useDictationSettings.ts
+10 -18
View File
@@ -1,5 +1,10 @@
import { useState, useEffect } from 'react';
import { useConfig } from '../components/ConfigContext';
import {
DICTATION_SETTINGS_KEY,
ELEVENLABS_API_KEY,
getDefaultDictationSettings,
} from './dictationConstants';
export type DictationProvider = 'openai' | 'elevenlabs' | null;
@@ -8,9 +13,6 @@ export interface DictationSettings {
provider: DictationProvider;
}
const DICTATION_SETTINGS_KEY = 'dictation_settings';
const ELEVENLABS_API_KEY = 'ELEVENLABS_API_KEY';
export const useDictationSettings = () => {
const [settings, setSettings] = useState<DictationSettings | null>(null);
const [hasElevenLabsKey, setHasElevenLabsKey] = useState<boolean>(false);
@@ -20,23 +22,13 @@ export const useDictationSettings = () => {
const loadSettings = async () => {
// Load settings from localStorage
const saved = localStorage.getItem(DICTATION_SETTINGS_KEY);
if (saved) {
setSettings(JSON.parse(saved));
const parsedSettings = JSON.parse(saved);
setSettings(parsedSettings);
} else {
const providers = await getProviders(false);
// Check if we have an OpenAI API key as primary default
const openAIProvider = providers.find((p) => p.name === 'openai');
if (openAIProvider && openAIProvider.is_configured) {
setSettings({
enabled: true,
provider: 'openai',
});
} else {
setSettings({
enabled: false,
provider: null,
});
}
const defaultSettings = await getDefaultDictationSettings(getProviders);
setSettings(defaultSettings);
}
// Load ElevenLabs API key from storage (non-secret for frontend access)
ui/desktop/src/hooks/useWhisper.ts
+70 -18
View File
@@ -87,7 +87,7 @@ export const useWhisper = ({ onTranscription, onError, onSizeWarning }: UseWhisp
// Define stopRecording before startRecording to avoid circular dependency
const stopRecording = useCallback(() => {
setIsRecording(false); // Always update the visual state
setIsRecording(false);
if (mediaRecorderRef.current && mediaRecorderRef.current.state !== 'inactive') {
mediaRecorderRef.current.stop();
@@ -159,14 +159,20 @@ export const useWhisper = ({ onTranscription, onError, onSizeWarning }: UseWhisp
reader.readAsDataURL(audioBlob);
});
const mimeType = audioBlob.type;
if (!mimeType) {
throw new Error('Unable to determine audio format. Please try again.');
}
let endpoint = '';
let headers: Record<string, string> = {
'Content-Type': 'application/json',
'X-Secret-Key': await window.electron.getSecretKey(),
};
let body: Record<string, string> = {
audio: base64Audio,
mime_type: 'audio/webm',
mime_type: mimeType,
};
// Choose endpoint based on provider
@@ -234,23 +240,32 @@ export const useWhisper = ({ onTranscription, onError, onSizeWarning }: UseWhisp
try {
// Request microphone permission
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
const stream = await navigator.mediaDevices.getUserMedia({
audio: {
echoCancellation: true,
noiseSuppression: true,
autoGainControl: true,
sampleRate: 44100,
},
});
streamRef.current = stream;
// Create audio context and analyser for visualization
const context = new AudioContext();
const source = context.createMediaStreamSource(stream);
const analyserNode = context.createAnalyser();
analyserNode.fftSize = 2048;
source.connect(analyserNode);
// Verify we have valid audio tracks
const audioTracks = stream.getAudioTracks();
if (audioTracks.length === 0) {
throw new Error('No audio tracks available in the microphone stream');
}
setAudioContext(context);
setAnalyser(analyserNode);
// AudioContext creation is disabled to prevent MediaRecorder conflicts
setAudioContext(null);
setAnalyser(null);
// Create MediaRecorder
const mediaRecorder = new MediaRecorder(stream, {
mimeType: 'audio/webm',
});
// Determine best supported MIME type
const supportedTypes = ['audio/webm;codecs=opus', 'audio/webm', 'audio/mp4', 'audio/wav'];
const mimeType = supportedTypes.find((type) => MediaRecorder.isTypeSupported(type)) || '';
const mediaRecorder = new MediaRecorder(stream, mimeType ? { mimeType } : {});
mediaRecorderRef.current = mediaRecorder;
audioChunksRef.current = [];
@@ -297,12 +312,49 @@ export const useWhisper = ({ onTranscription, onError, onSizeWarning }: UseWhisp
};
mediaRecorder.onstop = async () => {
const audioBlob = new Blob(audioChunksRef.current, { type: 'audio/webm' });
const audioBlob = new Blob(audioChunksRef.current, { type: mimeType || 'audio/webm' });
// Check if the blob is empty
if (audioBlob.size === 0) {
onError?.(
new Error(
'No audio data was recorded. Please check your microphone permissions and try again.'
)
);
return;
}
await transcribeAudio(audioBlob);
};
mediaRecorder.start(1000); // Collect data every second for size monitoring
setIsRecording(true);
// Add error handler for MediaRecorder
mediaRecorder.onerror = (event) => {
console.error('MediaRecorder error:', event);
onError?.(new Error('Recording failed: Unknown error'));
};
if (!stream.active) {
throw new Error('Audio stream became inactive before recording could start');
}
// Check audio tracks again before starting recording
if (audioTracks.length === 0) {
throw new Error('No audio tracks available in the stream');
}
const activeAudioTracks = audioTracks.filter((track) => track.readyState === 'live');
if (activeAudioTracks.length === 0) {
throw new Error('No live audio tracks available');
}
try {
mediaRecorder.start(100);
setIsRecording(true);
} catch (startError) {
console.error('Error calling mediaRecorder.start():', startError);
const errorMessage = startError instanceof Error ? startError.message : String(startError);
throw new Error(`Failed to start recording: ${errorMessage}`);
}
} catch (error) {
console.error('Error starting recording:', error);
stopRecording();