[feat] goosebenchv2 additions for eval post-processing (#2619)

Co-authored-by: Alice Hau <ahau@squareup.com>

crates/goose-bench/Cargo.toml

@@ -25,6 +25,7 @@ include_dir = "0.7.4"
 once_cell = "1.19"
 regex = "1.11.1"
 toml = "0.8.20"
+dotenvy = "0.15.7"
 
 [target.'cfg(target_os = "windows")'.dependencies]
 winapi = { version = "0.3", features = ["wincred"] }

crates/goose-bench/README.md

@@ -0,0 +1,273 @@
+# Goose Benchmarking Framework
+
+The `goose-bench` crate provides a framework for benchmarking and evaluating LLM models with the Goose framework. This tool helps quantify model performance across various tasks and generate structured reports.
+
+## Features
+
+- Run benchmark suites across multiple LLM models
+- Execute evaluations in parallel when supported
+- Generate structured JSON and CSV reports
+- Process evaluation results with custom scripts
+- Calculate aggregate metrics across evaluations
+- Support for tool-shim evaluation
+- Generate leaderboards and comparative metrics
+
+## Prerequisites
+
+- **Python Environment**: The `generate-leaderboard` command executes Python scripts and requires a valid Python environment with necessary dependencies (pandas, etc.)
+- **OpenAI API Key**: For evaluations using LLM-as-judge (like `blog_summary` and `restaurant_research`), you must have an `OPENAI_API_KEY` environment variable set, as the judge uses the OpenAI GPT-4o model
+
+## Benchmark Workflow
+
+Running benchmarks is a two-step process:
+
+### Step 1: Run Benchmarks
+
+First, run the benchmark evaluations with your configuration:
+
+```bash
+goose bench run --config /path/to/your-config.json
+```
+
+This will execute all evaluations for all models specified in your configuration and create a benchmark directory with results.
+
+### Step 2: Generate Leaderboard
+
+After the benchmarks complete, generate the leaderboard and aggregated metrics:
+
+```bash
+goose bench generate-leaderboard --benchmark-dir /path/to/benchmark-output-directory
+```
+
+The benchmark directory path will be shown in the output of the previous command, typically in the format `benchmark-YYYY-MM-DD-HH:MM:SS`.
+
+**Note**: This command requires a valid Python environment as it executes Python scripts for data aggregation and leaderboard generation.
+
+## Configuration
+
+Benchmark configuration is provided through a JSON file. Here's a sample configuration file (leaderboard-config.json) that you can use as a template:
+
+```json
+{
+  "models": [
+    {
+      "provider": "databricks",
+      "name": "gpt-4-1-mini",
+      "parallel_safe": true,
+      "tool_shim": {
+        "use_tool_shim": false,
+        "tool_shim_model": null
+      }
+    },
+    {
+      "provider": "databricks",
+      "name": "claude-3-5-sonnet",
+      "parallel_safe": true,
+      "tool_shim": null
+    },
+    {
+      "provider": "databricks",
+      "name": "gpt-4o",
+      "parallel_safe": true,
+      "tool_shim": null
+    }
+  ],
+  "evals": [
+    {
+      "selector": "core:developer",
+      "post_process_cmd": null,
+      "parallel_safe": true
+    },
+    {
+      "selector": "core:developer_search_replace",
+      "post_process_cmd": null,
+      "parallel_safe": true
+    },
+    {
+      "selector": "vibes:blog_summary",
+      "post_process_cmd": "/Users/ahau/Development/goose-1.0/goose/scripts/bench-postprocess-scripts/llm-judges/run_vibes_judge.sh",
+      "parallel_safe": true
+    },
+    {
+      "selector": "vibes:restaurant_research",
+      "post_process_cmd": "/Users/ahau/Development/goose-1.0/goose/scripts/bench-postprocess-scripts/llm-judges/run_vibes_judge.sh",
+      "parallel_safe": true
+    }
+  ],
+  "include_dirs": [],
+  "repeat": 3,
+  "run_id": null,
+  "output_dir": "/path/to/output/directory",
+  "eval_result_filename": "eval-results.json",
+  "run_summary_filename": "run-results-summary.json",
+  "env_file": "/path/to/.goosebench.env"
+}
+```
+
+## Configuration Options
+
+### Models
+
+- `provider`: The LLM provider (e.g., "databricks", "openai")
+- `name`: The model name
+- `parallel_safe`: Whether the model can be run in parallel
+- `tool_shim`: Configuration for tool-shim support
+  - `use_tool_shim`: Whether to use tool-shim
+  - `tool_shim_model`: Optional custom model for tool-shim
+
+### Evaluations
+
+- `selector`: The evaluation selector in format `suite:evaluation`
+- `post_process_cmd`: Optional path to a post-processing script
+- `parallel_safe`: Whether the evaluation can be run in parallel
+
+### Global Configuration
+
+- `include_dirs`: Additional directories to include in the benchmark environment
+- `repeat`: Number of times to repeat evaluations (for statistical significance)
+- `run_id`: Optional identifier for the run (defaults to timestamp)
+- `output_dir`: Directory to store benchmark results (must be absolute path)
+- `eval_result_filename`: Filename for individual evaluation results
+- `run_summary_filename`: Filename for run summary
+- `env_file`: Optional path to environment variables file
+
+## Environment Variables
+
+You can provide environment variables through the `env_file` configuration option. This is useful for provider API keys and other sensitive information. Example `.goosebench.env` file:
+
+```bash
+OPENAI_API_KEY=your_openai_api_key_here
+DATABRICKS_TOKEN=your_databricks_token_here
+# Add other environment variables as needed
+```
+
+**Important**: For evaluations that use LLM-as-judge (like `blog_summary` and `restaurant_research`), you must set `OPENAI_API_KEY` as the judging system uses OpenAI's GPT-4o model.
+
+## Post-Processing
+
+You can specify post-processing commands for evaluations, which will be executed after each evaluation completes. The command receives the path to the evaluation results file as its first argument.
+
+For example, the `run_vibes_judge.sh` script processes outputs from the `blog_summary` and `restaurant_research` evaluations, using LLM-based judging to assign scores.
+
+## Output Structure
+
+Results are organized in a directory structure that follows this pattern:
+
+```
+{benchmark_dir}/
+├── config.cfg                           # Configuration used for the benchmark
+├── {provider}-{model}/
+│   ├── eval-results/
+│   │   └── aggregate_metrics.csv        # Aggregated metrics for this model
+│   └── run-{run_id}/
+│       ├── {suite}/
+│       │   └── {evaluation}/
+│       │       ├── eval-results.json    # Individual evaluation results
+│       │       ├── {eval_name}.jsonl    # Session logs
+│       │       └── work_dir.json        # Info about evaluation working dir
+│       └── run-results-summary.json     # Summary of all evaluations in this run
+├── leaderboard.csv                      # Final leaderboard comparing all models
+└── all_metrics.csv                      # Union of all metrics across all models
+```
+
+### Output Files Explained
+
+#### Per-Model Files
+
+- **`eval-results/aggregate_metrics.csv`**: Contains aggregated metrics for each evaluation, averaged across all runs. Includes metrics like `score_mean`, `total_tokens_mean`, `prompt_execution_time_seconds_mean`, etc.
+
+#### Global Output Files
+
+- **`leaderboard.csv`**: Final leaderboard ranking all models by their average performance across evaluations. Contains columns like:
+  - `provider`, `model_name`: Model identification
+  - `avg_score_mean`: Average score across all evaluations
+  - `avg_prompt_execution_time_seconds_mean`: Average execution time
+  - `avg_total_tool_calls_mean`: Average number of tool calls
+  - `avg_total_tokens_mean`: Average token usage
+
+- **`all_metrics.csv`**: Comprehensive dataset containing detailed metrics for every model-evaluation combination. This is a union of all individual model metrics, useful for detailed analysis and custom reporting.
+
+Each model gets its own directory, containing run results and aggregated CSV files for analysis. The `generate-leaderboard` command processes all individual evaluation results and creates the comparative metrics files.
+
+## Error Handling and Troubleshooting
+
+**Important**: The current version of goose-bench does not have robust error handling for common issues that can occur during evaluation runs, such as:
+
+- Rate limiting from inference providers
+- Network timeouts or connection errors
+- Provider API errors that cause early session termination
+- Resource exhaustion or memory issues
+
+### Checking for Failed Evaluations
+
+After running benchmarks, you should inspect the generated metrics files to identify any evaluations that may have failed or terminated early:
+
+1. **Check the `aggregate_metrics.csv` files** in each model's `eval-results/` directory for:
+   - Missing evaluations (fewer rows than expected)
+   - Unusually low scores or metrics
+   - Zero or near-zero execution times
+   - Missing or NaN values
+
+2. **Look for `server_error_mean` column** in the aggregate metrics - values greater than 0 indicate server errors occurred during evaluation
+
+3. **Review session logs** (`.jsonl` files) in individual evaluation directories for error messages like:
+   - "Server error"
+   - "Rate limit exceeded" 
+   - "TEMPORARILY_UNAVAILABLE"
+   - Unexpected session terminations
+
+### Re-running Failed Evaluations
+
+If you identify failed evaluations, you may need to:
+
+1. **Adjust rate limiting**: Add delays between requests or reduce parallel execution
+2. **Update environment variables**: Ensure API keys and tokens are valid
+3. **Re-run specific model/evaluation combinations**: Create a new config with only the failed combinations
+4. **Check provider status**: Verify the inference provider is operational
+
+Example of creating a config to re-run failed evaluations:
+
+```json
+{
+  "models": [
+    {
+      "provider": "databricks",
+      "name": "claude-3-5-sonnet",
+      "parallel_safe": false
+    }
+  ],
+  "evals": [
+    {
+      "selector": "vibes:blog_summary",
+      "post_process_cmd": "/path/to/scripts/bench-postprocess-scripts/llm-judges/run_vibes_judge.sh",
+      "parallel_safe": false
+    }
+  ],
+  "repeat": 1,
+  "output_dir": "/path/to/retry-benchmark"
+}
+```
+
+We recommend monitoring evaluation progress and checking for errors regularly, especially when running large benchmark suites across multiple models.
+
+## Available Commands
+
+### List Evaluations
+```bash
+goose bench selectors --config /path/to/config.json
+```
+
+### Generate Initial Config
+```bash
+goose bench init-config --name my-benchmark-config.json
+```
+
+### Run Benchmarks
+```bash
+goose bench run --config /path/to/config.json
+```
+
+### Generate Leaderboard
+```bash
+goose bench generate-leaderboard --benchmark-dir /path/to/benchmark-output
+```

crates/goose-bench/src/eval_suites/evaluation.rs

@@ -3,17 +3,29 @@ use crate::bench_work_dir::BenchmarkWorkDir;
 use anyhow::Result;
 use async_trait::async_trait;
 use serde::{Deserialize, Serialize};
+use std::fmt;
 
 pub type Model = (String, String);
 pub type Extension = String;
 
-#[derive(Debug, Deserialize, Serialize)]
+#[derive(Debug, Deserialize, Serialize, Clone)]
 pub enum EvalMetricValue {
     Integer(i64),
     Float(f64),
     String(String),
     Boolean(bool),
 }
+
+impl fmt::Display for EvalMetricValue {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            EvalMetricValue::Integer(i) => write!(f, "{}", i),
+            EvalMetricValue::Float(fl) => write!(f, "{:.2}", fl),
+            EvalMetricValue::String(s) => write!(f, "{}", s),
+            EvalMetricValue::Boolean(b) => write!(f, "{}", b),
+        }
+    }
+}
 #[derive(Debug, Serialize)]
 pub struct EvalMetric {
     pub name: String,

crates/goose-bench/src/reporting.rs

@@ -98,17 +98,6 @@ impl BenchmarkResults {
     }
 }
 
-impl fmt::Display for EvalMetricValue {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        match self {
-            EvalMetricValue::Integer(i) => write!(f, "{}", i),
-            EvalMetricValue::Float(fl) => write!(f, "{:.2}", fl),
-            EvalMetricValue::String(s) => write!(f, "{}", s),
-            EvalMetricValue::Boolean(b) => write!(f, "{}", b),
-        }
-    }
-}
-
 impl fmt::Display for BenchmarkResults {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         writeln!(f, "Benchmark Results")?;

crates/goose-bench/src/runners/eval_runner.rs

@@ -4,12 +4,14 @@ use crate::bench_work_dir::BenchmarkWorkDir;
 use crate::eval_suites::{EvaluationSuite, ExtensionRequirements};
 use crate::reporting::EvaluationResult;
 use crate::utilities::await_process_exits;
+use anyhow::{bail, Context, Result};
 use std::env;
 use std::fs;
 use std::future::Future;
 use std::path::PathBuf;
 use std::process::Command;
 use std::time::{SystemTime, UNIX_EPOCH};
+use tracing;
 
 #[derive(Clone)]
 pub struct EvalRunner {
@@ -17,13 +19,17 @@ pub struct EvalRunner {
 }
 
 impl EvalRunner {
-    pub fn from(config: String) -> anyhow::Result<EvalRunner> {
-        let config = BenchRunConfig::from_string(config)?;
+    pub fn from(config: String) -> Result<EvalRunner> {
+        let config = BenchRunConfig::from_string(config)
+            .context("Failed to parse evaluation configuration")?;
         Ok(EvalRunner { config })
     }
 
-    fn create_work_dir(&self, config: &BenchRunConfig) -> anyhow::Result<BenchmarkWorkDir> {
-        let goose_model = config.models.first().unwrap();
+    fn create_work_dir(&self, config: &BenchRunConfig) -> Result<BenchmarkWorkDir> {
+        let goose_model = config
+            .models
+            .first()
+            .context("No model specified in configuration")?;
         let model_name = goose_model.name.clone();
         let provider_name = goose_model.provider.clone();
 
@@ -48,13 +54,21 @@ impl EvalRunner {
         let work_dir = BenchmarkWorkDir::new(work_dir_name, include_dir);
         Ok(work_dir)
     }
-    pub async fn run<F, Fut>(&mut self, agent_generator: F) -> anyhow::Result<()>
+
+    pub async fn run<F, Fut>(&mut self, agent_generator: F) -> Result<()>
     where
         F: Fn(ExtensionRequirements, String) -> Fut,
         Fut: Future<Output = BenchAgent> + Send,
     {
-        let mut work_dir = self.create_work_dir(&self.config)?;
-        let bench_eval = self.config.evals.first().unwrap();
+        let mut work_dir = self
+            .create_work_dir(&self.config)
+            .context("Failed to create evaluation work directory")?;
+
+        let bench_eval = self
+            .config
+            .evals
+            .first()
+            .context("No evaluations specified in configuration")?;
 
         let run_id = &self
             .config
@@ -65,41 +79,89 @@ impl EvalRunner {
 
         // create entire dir subtree for eval and cd into dir for running eval
         work_dir.set_eval(&bench_eval.selector, run_id);
+        tracing::info!("Set evaluation directory for {}", bench_eval.selector);
 
         if let Some(eval) = EvaluationSuite::from(&bench_eval.selector) {
-            let now_stamp = SystemTime::now().duration_since(UNIX_EPOCH)?.as_nanos();
+            let now_stamp = SystemTime::now()
+                .duration_since(UNIX_EPOCH)
+                .context("Failed to get current timestamp")?
+                .as_nanos();
+
             let session_id = format!("{}-{}", bench_eval.selector.clone(), now_stamp);
             let mut agent = agent_generator(eval.required_extensions(), session_id).await;
+            tracing::info!("Agent created for {}", eval.name());
 
             let mut result = EvaluationResult::new(eval.name().to_string());
 
-            if let Ok(metrics) = eval.run(&mut agent, &mut work_dir).await {
-                for (name, metric) in metrics {
-                    result.add_metric(name, metric);
+            match eval.run(&mut agent, &mut work_dir).await {
+                Ok(metrics) => {
+                    tracing::info!("Evaluation run successful with {} metrics", metrics.len());
+                    for (name, metric) in metrics {
+                        result.add_metric(name, metric);
+                    }
                 }
-
-                // Add any errors that occurred
-                for error in agent.get_errors().await {
-                    result.add_error(error);
+                Err(e) => {
+                    tracing::error!("Evaluation run failed: {}", e);
                 }
             }
 
-            let eval_results = serde_json::to_string_pretty(&result)?;
+            // Add any errors that occurred
+            let errors = agent.get_errors().await;
+            tracing::info!("Agent reported {} errors", errors.len());
+            for error in errors {
+                result.add_error(error);
+            }
+
+            // Write results to file
+            let eval_results = serde_json::to_string_pretty(&result)
+                .context("Failed to serialize evaluation results to JSON")?;
+
+            let eval_results_file = env::current_dir()
+                .context("Failed to get current directory")?
+                .join(&self.config.eval_result_filename);
+
+            fs::write(&eval_results_file, &eval_results).with_context(|| {
+                format!(
+                    "Failed to write evaluation results to {}",
+                    eval_results_file.display()
+                )
+            })?;
+
+            tracing::info!(
+                "Wrote evaluation results to {}",
+                eval_results_file.display()
+            );
 
-            let eval_results_file = env::current_dir()?.join(&self.config.eval_result_filename);
-            fs::write(&eval_results_file, &eval_results)?;
             self.config.save("config.cfg".to_string());
             work_dir.save();
 
             // handle running post-process cmd if configured
             if let Some(cmd) = &bench_eval.post_process_cmd {
-                let handle = Command::new(cmd).arg(&eval_results_file).spawn()?;
+                tracing::info!("Running post-process command: {:?}", cmd);
+
+                let handle = Command::new(cmd)
+                    .arg(&eval_results_file)
+                    .spawn()
+                    .with_context(|| {
+                        format!("Failed to execute post-process command: {:?}", cmd)
+                    })?;
+
                 await_process_exits(&mut [handle], Vec::new());
             }
 
             // copy session file into eval-dir
-            let here = env::current_dir()?.canonicalize()?;
-            BenchmarkWorkDir::deep_copy(agent.session_file().as_path(), here.as_path(), false)?;
+            let here = env::current_dir()
+                .context("Failed to get current directory")?
+                .canonicalize()
+                .context("Failed to canonicalize current directory path")?;
+
+            BenchmarkWorkDir::deep_copy(agent.session_file().as_path(), here.as_path(), false)
+                .context("Failed to copy session file to evaluation directory")?;
+
+            tracing::info!("Evaluation completed successfully");
+        } else {
+            tracing::error!("No evaluation found for selector: {}", bench_eval.selector);
+            bail!("No evaluation found for selector: {}", bench_eval.selector);
         }
 
         Ok(())

crates/goose-bench/src/runners/metric_aggregator.rs

@@ -0,0 +1,81 @@
+use anyhow::{bail, ensure, Context, Result};
+use std::path::PathBuf;
+use tracing;
+
+pub struct MetricAggregator;
+
+impl MetricAggregator {
+    /// Generate leaderboard and aggregated metrics CSV files from benchmark directory
+    pub fn generate_csv_from_benchmark_dir(benchmark_dir: &PathBuf) -> Result<()> {
+        use std::process::Command;
+
+        // Step 1: Run prepare_aggregate_metrics.py to create aggregate_metrics.csv files
+        let prepare_script_path = std::env::current_dir()
+            .context("Failed to get current working directory")?
+            .join("scripts")
+            .join("bench-postprocess-scripts")
+            .join("prepare_aggregate_metrics.py");
+
+        ensure!(
+            prepare_script_path.exists(),
+            "Prepare script not found: {}",
+            prepare_script_path.display()
+        );
+
+        tracing::info!(
+            "Preparing aggregate metrics from benchmark directory: {}",
+            benchmark_dir.display()
+        );
+
+        let output = Command::new(&prepare_script_path)
+            .arg("--benchmark-dir")
+            .arg(benchmark_dir)
+            .output()
+            .context("Failed to execute prepare_aggregate_metrics.py script")?;
+
+        if !output.status.success() {
+            let error_message = String::from_utf8_lossy(&output.stderr);
+            bail!("Failed to prepare aggregate metrics: {}", error_message);
+        }
+
+        let success_message = String::from_utf8_lossy(&output.stdout);
+        tracing::info!("{}", success_message);
+
+        // Step 2: Run generate_leaderboard.py to create the final leaderboard
+        let leaderboard_script_path = std::env::current_dir()
+            .context("Failed to get current working directory")?
+            .join("scripts")
+            .join("bench-postprocess-scripts")
+            .join("generate_leaderboard.py");
+
+        ensure!(
+            leaderboard_script_path.exists(),
+            "Leaderboard script not found: {}",
+            leaderboard_script_path.display()
+        );
+
+        tracing::info!(
+            "Generating leaderboard from benchmark directory: {}",
+            benchmark_dir.display()
+        );
+
+        let output = Command::new(&leaderboard_script_path)
+            .arg("--benchmark-dir")
+            .arg(benchmark_dir)
+            .arg("--leaderboard-output")
+            .arg("leaderboard.csv")
+            .arg("--union-output")
+            .arg("all_metrics.csv")
+            .output()
+            .context("Failed to execute generate_leaderboard.py script")?;
+
+        if !output.status.success() {
+            let error_message = String::from_utf8_lossy(&output.stderr);
+            bail!("Failed to generate leaderboard: {}", error_message);
+        }
+
+        let success_message = String::from_utf8_lossy(&output.stdout);
+        tracing::info!("{}", success_message);
+        Ok(())
+    }
+}

crates/goose-bench/src/runners/mod.rs

@@ -1,3 +1,4 @@
 pub mod bench_runner;
 pub mod eval_runner;
+pub mod metric_aggregator;
 pub mod model_runner;

crates/goose-bench/src/runners/model_runner.rs

@@ -3,12 +3,14 @@ use crate::eval_suites::EvaluationSuite;
 use crate::reporting::{BenchmarkResults, SuiteResult};
 use crate::runners::eval_runner::EvalRunner;
 use crate::utilities::{await_process_exits, parallel_bench_cmd};
+use anyhow::{Context, Result};
+use dotenvy::from_path_iter;
 use std::collections::HashMap;
 use std::fs::read_to_string;
-use std::io::{self, BufRead};
 use std::path::PathBuf;
 use std::process::Child;
 use std::thread;
+use tracing;
 
 #[derive(Clone)]
 pub struct ModelRunner {
@@ -16,23 +18,27 @@ pub struct ModelRunner {
 }
 
 impl ModelRunner {
-    pub fn from(config: String) -> anyhow::Result<ModelRunner> {
-        let config = BenchRunConfig::from_string(config)?;
+    pub fn from(config: String) -> Result<ModelRunner> {
+        let config =
+            BenchRunConfig::from_string(config).context("Failed to parse configuration")?;
         Ok(ModelRunner { config })
     }
 
-    pub fn run(&self) -> anyhow::Result<()> {
-        let model = self.config.models.first().unwrap();
+    pub fn run(&self) -> Result<()> {
+        let model = self
+            .config
+            .models
+            .first()
+            .context("No model specified in config")?;
         let suites = self.collect_evals_for_run();
 
         let mut handles = vec![];
 
         for i in 0..self.config.repeat.unwrap_or(1) {
-            let mut self_copy = self.clone();
+            let self_copy = self.clone();
             let model_clone = model.clone();
             let suites_clone = suites.clone();
-            // create thread to handle launching parallel processes to run model's evals in parallel
-            let handle = thread::spawn(move || {
+            let handle = thread::spawn(move || -> Result<()> {
                 self_copy.run_benchmark(&model_clone, suites_clone, i.to_string())
             });
             handles.push(handle);
@@ -41,55 +47,32 @@ impl ModelRunner {
 
         let mut all_runs_results: Vec<BenchmarkResults> = Vec::new();
         for i in 0..self.config.repeat.unwrap_or(1) {
-            let run_results =
-                self.collect_run_results(model.clone(), suites.clone(), i.to_string())?;
-            all_runs_results.push(run_results);
+            match self.collect_run_results(model.clone(), suites.clone(), i.to_string()) {
+                Ok(run_results) => all_runs_results.push(run_results),
+                Err(e) => {
+                    tracing::error!("Failed to collect results for run {}: {}", i, e)
+                }
+            }
         }
-        // write summary file
 
         Ok(())
     }
 
-    fn load_env_file(&self, path: &PathBuf) -> anyhow::Result<Vec<(String, String)>> {
-        let file = std::fs::File::open(path)?;
-        let reader = io::BufReader::new(file);
-        let mut env_vars = Vec::new();
-
-        for line in reader.lines() {
-            let line = line?;
-            // Skip empty lines and comments
-            if line.trim().is_empty() || line.trim_start().starts_with('#') {
-                continue;
-            }
-
-            // Split on first '=' only
-            if let Some((key, value)) = line.split_once('=') {
-                let key = key.trim().to_string();
-                // Remove quotes if present
-                let value = value
-                    .trim()
-                    .trim_matches('"')
-                    .trim_matches('\'')
-                    .to_string();
-                env_vars.push((key, value));
-            }
-        }
-
-        Ok(env_vars)
-    }
-
     fn run_benchmark(
-        &mut self,
+        &self,
         model: &BenchModel,
         suites: HashMap<String, Vec<BenchEval>>,
         run_id: String,
-    ) -> anyhow::Result<()> {
+    ) -> Result<()> {
         let mut results_handles = HashMap::<String, Vec<Child>>::new();
 
         // Load environment variables from file if specified
         let mut envs = self.toolshim_envs();
         if let Some(env_file) = &self.config.env_file {
-            let env_vars = self.load_env_file(env_file)?;
+            let env_vars = ModelRunner::load_env_file(env_file).context(format!(
+                "Failed to load environment file: {}",
+                env_file.display()
+            ))?;
             envs.extend(env_vars);
         }
         envs.push(("GOOSE_MODEL".to_string(), model.clone().name));
@@ -116,9 +99,13 @@ impl ModelRunner {
             // Run parallel-safe evaluations in parallel
             if !parallel_evals.is_empty() {
                 for eval_selector in &parallel_evals {
-                    self.config.run_id = Some(run_id.clone());
-                    self.config.evals = vec![(*eval_selector).clone()];
-                    let cfg = self.config.to_string()?;
+                    let mut config_copy = self.config.clone();
+                    config_copy.run_id = Some(run_id.clone());
+                    config_copy.evals = vec![(*eval_selector).clone()];
+                    let cfg = config_copy
+                        .to_string()
+                        .context("Failed to serialize configuration")?;
+
                     let handle = parallel_bench_cmd("exec-eval".to_string(), cfg, envs.clone());
                     results_handles.get_mut(suite).unwrap().push(handle);
                 }
@@ -126,9 +113,13 @@ impl ModelRunner {
 
             // Run non-parallel-safe evaluations sequentially
             for eval_selector in &sequential_evals {
-                self.config.run_id = Some(run_id.clone());
-                self.config.evals = vec![(*eval_selector).clone()];
-                let cfg = self.config.to_string()?;
+                let mut config_copy = self.config.clone();
+                config_copy.run_id = Some(run_id.clone());
+                config_copy.evals = vec![(*eval_selector).clone()];
+                let cfg = config_copy
+                    .to_string()
+                    .context("Failed to serialize configuration")?;
+
                 let handle = parallel_bench_cmd("exec-eval".to_string(), cfg, envs.clone());
 
                 // Wait for this process to complete before starting the next one
@@ -150,7 +141,7 @@ impl ModelRunner {
         model: BenchModel,
         suites: HashMap<String, Vec<BenchEval>>,
         run_id: String,
-    ) -> anyhow::Result<BenchmarkResults> {
+    ) -> Result<BenchmarkResults> {
         let mut results = BenchmarkResults::new(model.provider.clone());
 
         let mut summary_path: Option<PathBuf> = None;
@@ -161,7 +152,17 @@ impl ModelRunner {
                 let mut eval_path =
                     EvalRunner::path_for_eval(&model, eval_selector, run_id.clone());
                 eval_path.push(self.config.eval_result_filename.clone());
-                let eval_result = serde_json::from_str(&read_to_string(&eval_path)?)?;
+
+                let content = read_to_string(&eval_path).with_context(|| {
+                    format!(
+                        "Failed to read evaluation results from {}",
+                        eval_path.display()
+                    )
+                })?;
+
+                let eval_result = serde_json::from_str(&content)
+                    .context("Failed to parse evaluation results JSON")?;
+
                 suite_result.add_evaluation(eval_result);
 
                 // use current eval to determine where the summary should be written
@@ -180,12 +181,21 @@ impl ModelRunner {
             results.add_suite(suite_result);
         }
 
-        let mut run_summary = PathBuf::new();
-        run_summary.push(summary_path.clone().unwrap());
-        run_summary.push(&self.config.run_summary_filename);
+        if let Some(path) = summary_path {
+            let mut run_summary = PathBuf::new();
+            run_summary.push(path);
+            run_summary.push(&self.config.run_summary_filename);
 
-        let output_str = serde_json::to_string_pretty(&results)?;
-        std::fs::write(run_summary, &output_str)?;
+            let output_str = serde_json::to_string_pretty(&results)
+                .context("Failed to serialize benchmark results to JSON")?;
+
+            std::fs::write(&run_summary, &output_str).with_context(|| {
+                format!(
+                    "Failed to write results summary to {}",
+                    run_summary.display()
+                )
+            })?;
+        }
 
         Ok(results)
     }
@@ -210,20 +220,29 @@ impl ModelRunner {
 
     fn toolshim_envs(&self) -> Vec<(String, String)> {
         // read tool-shim preference from config, set respective env vars accordingly
-        let model = self.config.models.first().unwrap();
-
         let mut shim_envs: Vec<(String, String)> = Vec::new();
-        if let Some(shim_opt) = &model.tool_shim {
-            if shim_opt.use_tool_shim {
-                shim_envs.push(("GOOSE_TOOLSHIM".to_string(), "true".to_string()));
-                if let Some(shim_model) = &shim_opt.tool_shim_model {
-                    shim_envs.push((
-                        "GOOSE_TOOLSHIM_OLLAMA_MODEL".to_string(),
-                        shim_model.clone(),
-                    ));
+        if let Some(model) = self.config.models.first() {
+            if let Some(shim_opt) = &model.tool_shim {
+                if shim_opt.use_tool_shim {
+                    shim_envs.push(("GOOSE_TOOLSHIM".to_string(), "true".to_string()));
+                    if let Some(shim_model) = &shim_opt.tool_shim_model {
+                        shim_envs.push((
+                            "GOOSE_TOOLSHIM_OLLAMA_MODEL".to_string(),
+                            shim_model.clone(),
+                        ));
+                    }
                 }
             }
         }
         shim_envs
     }
+
+    fn load_env_file(path: &PathBuf) -> Result<Vec<(String, String)>> {
+        let iter =
+            from_path_iter(path).context("Failed to read environment variables from file")?;
+        let env_vars = iter
+            .map(|item| item.context("Failed to parse environment variable"))
+            .collect::<Result<_, _>>()?;
+        Ok(env_vars)
+    }
 }

crates/goose-bench/src/utilities.rs

@@ -1,15 +1,14 @@
+use anyhow::Result;
 use std::env;
 use std::process::{Child, Command};
 use std::thread::JoinHandle;
+use tracing;
 
-pub fn await_process_exits(
-    child_processes: &mut [Child],
-    handles: Vec<JoinHandle<anyhow::Result<()>>>,
-) {
+pub fn await_process_exits(child_processes: &mut [Child], handles: Vec<JoinHandle<Result<()>>>) {
     for child in child_processes.iter_mut() {
         match child.wait() {
-            Ok(status) => println!("Child exited with status: {}", status),
-            Err(e) => println!("Error waiting for child: {}", e),
+            Ok(status) => tracing::info!("Child exited with status: {}", status),
+            Err(e) => tracing::error!("Error waiting for child: {}", e),
         }
     }
 
@@ -18,7 +17,7 @@ pub fn await_process_exits(
             Ok(_res) => (),
             Err(e) => {
                 // Handle thread panic
-                println!("Thread panicked: {:?}", e);
+                tracing::error!("Thread panicked: {:?}", e);
             }
         }
     }

crates/goose-cli/src/cli.rs

@@ -17,6 +17,7 @@ use crate::session::{build_session, SessionBuilderConfig};
 use goose_bench::bench_config::BenchRunConfig;
 use goose_bench::runners::bench_runner::BenchRunner;
 use goose_bench::runners::eval_runner::EvalRunner;
+use goose_bench::runners::metric_aggregator::MetricAggregator;
 use goose_bench::runners::model_runner::ModelRunner;
 use std::io::Read;
 use std::path::PathBuf;
@@ -142,6 +143,19 @@ pub enum BenchCommand {
         #[arg(short, long, help = "A serialized config file for the eval only.")]
         config: String,
     },
+
+    #[command(
+        name = "generate-leaderboard",
+        about = "Generate a leaderboard CSV from benchmark results"
+    )]
+    GenerateLeaderboard {
+        #[arg(
+            short,
+            long,
+            help = "Path to the benchmark directory containing model evaluation results"
+        )]
+        benchmark_dir: PathBuf,
+    },
 }
 
 #[derive(Subcommand)]
@@ -651,6 +665,9 @@ pub async fn cli() -> Result<()> {
                 BenchCommand::ExecEval { config } => {
                     EvalRunner::from(config)?.run(agent_generator).await?
                 }
+                BenchCommand::GenerateLeaderboard { benchmark_dir } => {
+                    MetricAggregator::generate_csv_from_benchmark_dir(&benchmark_dir)?
+                }
             }
             return Ok(());
         }