From 4e8f39b2821113aeb0df18f38bd453bc7a8c4eca Mon Sep 17 00:00:00 2001 From: angiejones Date: Wed, 5 Nov 2025 01:38:56 -0600 Subject: [PATCH] rest of the fix --- crates/goose-server/src/routes/mod.rs | 1 + crates/goose-server/src/routes/recipe.rs | 49 +++++- .../src/routes/subrecipe_fetcher.rs | 144 ++++++++++++++++++ crates/goose/src/recipe/mod.rs | 21 ++- 4 files changed, 210 insertions(+), 5 deletions(-) create mode 100644 crates/goose-server/src/routes/subrecipe_fetcher.rs diff --git a/crates/goose-server/src/routes/mod.rs b/crates/goose-server/src/routes/mod.rs index 9ca3799e5d..2bf8911bca 100644 --- a/crates/goose-server/src/routes/mod.rs +++ b/crates/goose-server/src/routes/mod.rs @@ -9,6 +9,7 @@ pub mod schedule; pub mod session; pub mod setup; pub mod status; +pub mod subrecipe_fetcher; pub mod utils; use std::sync::Arc; diff --git a/crates/goose-server/src/routes/recipe.rs b/crates/goose-server/src/routes/recipe.rs index 45e74c5db5..e445d1aecd 100644 --- a/crates/goose-server/src/routes/recipe.rs +++ b/crates/goose-server/src/routes/recipe.rs @@ -41,6 +41,7 @@ use crate::routes::recipe_utils::{ get_all_recipes_manifests, get_recipe_file_path_by_id, short_id_from_path, validate_recipe, RecipeValidationError, }; +use crate::routes::subrecipe_fetcher::fetch_and_store_subrecipes; use crate::state::AppState; #[derive(Debug, Deserialize, ToSchema)] @@ -238,10 +239,50 @@ async fn decode_recipe( Json(request): Json, ) -> Result, StatusCode> { match recipe_deeplink::decode(&request.deeplink) { - Ok(recipe) => match validate_recipe(&recipe) { - Ok(_) => Ok(Json(DecodeRecipeResponse { recipe })), - Err(RecipeValidationError { status, .. }) => Err(status), - }, + Ok(mut recipe) => { + if let Some(sub_recipes) = recipe.sub_recipes.clone() { + if !sub_recipes.is_empty() { + tracing::info!( + "Recipe '{}' has {} subrecipe(s), fetching from GitHub...", + recipe.title, + sub_recipes.len() + ); + + let recipe_id = recipe.title.to_lowercase().replace(' ', "-"); + + match fetch_and_store_subrecipes(sub_recipes, &recipe_id).await { + Ok(updated_sub_recipes) => { + if !updated_sub_recipes.is_empty() { + tracing::info!( + "Successfully fetched {} subrecipe(s) for '{}'", + updated_sub_recipes.len(), + recipe.title + ); + recipe.sub_recipes = Some(updated_sub_recipes); + } else { + tracing::warn!( + "No subrecipes were successfully fetched for '{}'", + recipe.title + ); + } + } + Err(e) => { + tracing::error!( + "Failed to fetch subrecipes for '{}': {}", + recipe.title, + e + ); + // Continue anyway - the recipe might still work without subrecipes + } + } + } + } + + match validate_recipe(&recipe) { + Ok(_) => Ok(Json(DecodeRecipeResponse { recipe })), + Err(RecipeValidationError { status, .. }) => Err(status), + } + } Err(err) => { tracing::error!("Failed to decode deeplink: {}", err); Err(StatusCode::BAD_REQUEST) diff --git a/crates/goose-server/src/routes/subrecipe_fetcher.rs b/crates/goose-server/src/routes/subrecipe_fetcher.rs new file mode 100644 index 0000000000..2db470496f --- /dev/null +++ b/crates/goose-server/src/routes/subrecipe_fetcher.rs @@ -0,0 +1,144 @@ +use anyhow::{anyhow, Result}; +use goose::recipe::SubRecipe; +use std::fs; +use std::path::Path; +use std::time::Duration; + +const GOOSE_RECIPES_REPO: &str = "block/goose"; +const GOOSE_RECIPES_BRANCH: &str = "main"; +const RECIPES_BASE_PATH: &str = "documentation/src/pages/recipes/data/recipes"; +const REQUEST_TIMEOUT_SECS: u64 = 10; + +async fn fetch_subrecipe_from_github(sub_recipe: &SubRecipe) -> Result { + let path = Path::new(&sub_recipe.path); + + let filename = if sub_recipe.path.starts_with("./") || sub_recipe.path.starts_with("../") { + path.file_name() + .and_then(|f| f.to_str()) + .ok_or_else(|| anyhow!("Invalid subrecipe path: {}", sub_recipe.path))? + } else { + sub_recipe.path.as_str() + }; + + let url = format!( + "https://raw.githubusercontent.com/{}/{}/{}/subrecipes/{}", + GOOSE_RECIPES_REPO, GOOSE_RECIPES_BRANCH, RECIPES_BASE_PATH, filename + ); + + tracing::info!("Fetching subrecipe from: {}", url); + + let client = reqwest::Client::builder() + .timeout(Duration::from_secs(REQUEST_TIMEOUT_SECS)) + .build()?; + let response = client.get(&url).send().await.map_err(|e| { + if e.is_timeout() { + anyhow!( + "Request timed out after {}s fetching subrecipe from GitHub: {}", + REQUEST_TIMEOUT_SECS, + e + ) + } else { + anyhow!("Failed to fetch subrecipe from GitHub: {}", e) + } + })?; + + if !response.status().is_success() { + return Err(anyhow!( + "Failed to fetch subrecipe '{}': HTTP {}", + filename, + response.status() + )); + } + + let content = response + .text() + .await + .map_err(|e| anyhow!("Failed to read subrecipe content: {}", e))?; + + Ok(content) +} + +pub async fn fetch_and_store_subrecipes( + sub_recipes: Vec, + recipe_id: &str, +) -> Result> { + // Sanitize recipe_id to prevent directory traversal + let recipe_id = recipe_id.replace(['/', '\\', '.'], "_"); + let mut updated_sub_recipes = Vec::new(); + + for sub_recipe in sub_recipes { + match fetch_subrecipe_from_github(&sub_recipe).await { + Ok(content) => { + let temp_dir = std::env::temp_dir() + .join("goose_subrecipes") + .join(&recipe_id); + fs::create_dir_all(&temp_dir)?; + + let filename = Path::new(&sub_recipe.path) + .file_name() + .and_then(|f| f.to_str()) + .ok_or_else(|| anyhow!("Invalid subrecipe path: {}", sub_recipe.path))?; + + let local_path = temp_dir.join(filename); + fs::write(&local_path, content)?; + + tracing::info!( + "Stored subrecipe '{}' at: {}", + sub_recipe.name, + local_path.display() + ); + + let updated_sub_recipe = SubRecipe { + name: sub_recipe.name.clone(), + path: local_path.to_string_lossy().to_string(), + values: sub_recipe.values.clone(), + sequential_when_repeated: sub_recipe.sequential_when_repeated, + description: sub_recipe.description.clone(), + }; + + updated_sub_recipes.push(updated_sub_recipe); + } + Err(e) => { + tracing::warn!( + "Failed to fetch subrecipe '{}': {}. Skipping.", + sub_recipe.name, + e + ); + // Continue with other subrecipes even if one fails + } + } + } + + Ok(updated_sub_recipes) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn test_fetch_subrecipe_from_github() { + let sub_recipe = SubRecipe { + name: "experiment_tracker".to_string(), + path: "./subrecipes/experiment-tracker.yaml".to_string(), + values: None, + sequential_when_repeated: false, + description: None, + }; + + let result = fetch_subrecipe_from_github(&sub_recipe).await; + + assert!( + result.is_ok(), + "Failed to fetch subrecipe: {:?}", + result.err() + ); + + let content = result.unwrap(); + assert!(!content.is_empty(), "Subrecipe content should not be empty"); + assert!( + content.contains("title:") || content.contains("description:"), + "Subrecipe content should contain expected YAML fields" + ); + } +} diff --git a/crates/goose/src/recipe/mod.rs b/crates/goose/src/recipe/mod.rs index 9f6e27ce11..31c1371fc9 100644 --- a/crates/goose/src/recipe/mod.rs +++ b/crates/goose/src/recipe/mod.rs @@ -140,6 +140,21 @@ where } } +fn deserialize_string_or_number<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + let value = Option::::deserialize(deserializer)?; + + match value { + None => Ok(None), + Some(serde_json::Value::String(s)) => Ok(Some(s)), + Some(serde_json::Value::Number(n)) => Ok(Some(n.to_string())), + Some(serde_json::Value::Null) => Ok(None), + Some(_) => Err(serde::de::Error::custom("expected string or number")), + } +} + #[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] #[serde(rename_all = "snake_case")] pub enum RecipeParameterRequirement { @@ -187,7 +202,11 @@ pub struct RecipeParameter { pub input_type: RecipeParameterInputType, pub requirement: RecipeParameterRequirement, pub description: String, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde( + default, + skip_serializing_if = "Option::is_none", + deserialize_with = "deserialize_string_or_number" + )] pub default: Option, #[serde(skip_serializing_if = "Option::is_none")] pub options: Option>,