mirror of
https://github.com/aaif-goose/goose.git
synced 2026-07-03 14:10:03 +02:00
feat: goose bench framework for functional and regression testing
Co-authored-by: Zaki Ali <zaki@squareup.com>
This commit is contained in:
marcelle
@@ -0,0 +1,83 @@
|
||||
# Goose Benchmark Scripts
|
||||
|
||||
This directory contains scripts for running and analyzing Goose benchmarks.
|
||||
|
||||
## run-benchmarks.sh
|
||||
|
||||
This script runs Goose benchmarks across multiple provider:model pairs and analyzes the results.
|
||||
|
||||
### Prerequisites
|
||||
|
||||
- Goose CLI must be built or installed
|
||||
- `jq` command-line tool for JSON processing (optional, but recommended for result analysis)
|
||||
|
||||
### Usage
|
||||
|
||||
```bash
|
||||
./scripts/run-benchmarks.sh [options]
|
||||
```
|
||||
|
||||
#### Options
|
||||
|
||||
- `-p, --provider-models`: Comma-separated list of provider:model pairs (e.g., 'openai:gpt-4o,anthropic:claude-3-5-sonnet')
|
||||
- `-s, --suites`: Comma-separated list of benchmark suites to run (e.g., 'core,small_models')
|
||||
- `-o, --output-dir`: Directory to store benchmark results (default: './benchmark-results')
|
||||
- `-d, --debug`: Use debug build instead of release build
|
||||
- `-h, --help`: Show help message
|
||||
|
||||
#### Examples
|
||||
|
||||
```bash
|
||||
# Run with release build (default)
|
||||
./scripts/run-benchmarks.sh --provider-models 'openai:gpt-4o,anthropic:claude-3-5-sonnet' --suites 'core,small_models'
|
||||
|
||||
# Run with debug build
|
||||
./scripts/run-benchmarks.sh --provider-models 'openai:gpt-4o' --suites 'core' --debug
|
||||
```
|
||||
|
||||
### How It Works
|
||||
|
||||
The script:
|
||||
1. Parses the provider:model pairs and benchmark suites
|
||||
2. Determines whether to use the debug or release binary
|
||||
3. For each provider:model pair:
|
||||
- Sets the `GOOSE_PROVIDER` and `GOOSE_MODEL` environment variables
|
||||
- Runs the benchmark with the specified suites
|
||||
- Analyzes the results for failures
|
||||
4. Generates a summary of all benchmark runs
|
||||
|
||||
### Output
|
||||
|
||||
The script creates the following files in the output directory:
|
||||
|
||||
- `summary.md`: A summary of all benchmark results
|
||||
- `{provider}-{model}.json`: Raw JSON output from each benchmark run
|
||||
- `{provider}-{model}-analysis.txt`: Analysis of each benchmark run
|
||||
|
||||
### Exit Codes
|
||||
|
||||
- `0`: All benchmarks completed successfully
|
||||
- `1`: One or more benchmarks failed
|
||||
|
||||
## parse-benchmark-results.sh
|
||||
|
||||
This script analyzes a single benchmark JSON result file and identifies any failures.
|
||||
|
||||
### Usage
|
||||
|
||||
```bash
|
||||
./scripts/parse-benchmark-results.sh path/to/benchmark-results.json
|
||||
```
|
||||
|
||||
### Output
|
||||
|
||||
The script outputs an analysis of the benchmark results to stdout, including:
|
||||
|
||||
- Basic information about the benchmark run
|
||||
- Results for each evaluation in each suite
|
||||
- Summary of passed and failed metrics
|
||||
|
||||
### Exit Codes
|
||||
|
||||
- `0`: All metrics passed successfully
|
||||
- `1`: One or more metrics failed
|
||||
Executable
+93
@@ -0,0 +1,93 @@
|
||||
#!/usr/bin/env bash
|
||||
# Script to parse goose-bench results and check for failures
|
||||
|
||||
set -e
|
||||
|
||||
if [ "$#" -ne 1 ]; then
|
||||
echo "Usage: $0 <benchmark-result-json-file>"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
RESULT_FILE="$1"
|
||||
|
||||
if [ ! -f "$RESULT_FILE" ]; then
|
||||
echo "Error: Result file not found: $RESULT_FILE"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Extract basic information
|
||||
PROVIDER=$(jq -r '.provider' "$RESULT_FILE")
|
||||
START_TIME=$(jq -r '.start_time' "$RESULT_FILE")
|
||||
SUITE_COUNT=$(jq '.suites | length' "$RESULT_FILE")
|
||||
|
||||
echo "Benchmark Results Analysis"
|
||||
echo "-------------------------"
|
||||
echo "Provider: $PROVIDER"
|
||||
echo "Start Time: $START_TIME"
|
||||
echo "Number of Suites: $SUITE_COUNT"
|
||||
echo ""
|
||||
|
||||
# Initialize counters
|
||||
TOTAL_EVALS=0
|
||||
TOTAL_METRICS=0
|
||||
FAILED_METRICS=0
|
||||
PASSED_METRICS=0
|
||||
|
||||
# Process each suite
|
||||
for i in $(seq 0 $((SUITE_COUNT-1))); do
|
||||
SUITE_NAME=$(jq -r ".suites[$i].name" "$RESULT_FILE")
|
||||
EVAL_COUNT=$(jq ".suites[$i].evaluations | length" "$RESULT_FILE")
|
||||
TOTAL_EVALS=$((TOTAL_EVALS + EVAL_COUNT))
|
||||
|
||||
echo "Suite: $SUITE_NAME ($EVAL_COUNT evaluations)"
|
||||
|
||||
# Process each evaluation in this suite
|
||||
for j in $(seq 0 $((EVAL_COUNT-1))); do
|
||||
EVAL_NAME=$(jq -r ".suites[$i].evaluations[$j].name" "$RESULT_FILE")
|
||||
METRIC_COUNT=$(jq ".suites[$i].evaluations[$j].metrics | length" "$RESULT_FILE")
|
||||
TOTAL_METRICS=$((TOTAL_METRICS + METRIC_COUNT))
|
||||
|
||||
# Check for failures in this evaluation
|
||||
# This assumes metrics with names containing "success", "pass", or "correct"
|
||||
# and boolean values of false indicate failures
|
||||
FAILURES=$(jq -r ".suites[$i].evaluations[$j].metrics[] |
|
||||
select(
|
||||
(.[0] | test(\"success|pass|correct\"; \"i\")) and
|
||||
(.[1] == false or .[1] == \"false\" or .[1] == 0 or .[1] == \"0\")
|
||||
) | .[0]" "$RESULT_FILE" | wc -l | tr -d ' ')
|
||||
|
||||
if [ "$FAILURES" -gt 0 ]; then
|
||||
FAILED_METRICS=$((FAILED_METRICS + FAILURES))
|
||||
echo " ❌ $EVAL_NAME: $FAILURES failures detected"
|
||||
|
||||
# Print the specific failing metrics
|
||||
FAILING_METRICS=$(jq -r ".suites[$i].evaluations[$j].metrics[] |
|
||||
select(
|
||||
(.[0] | test(\"success|pass|correct\"; \"i\")) and
|
||||
(.[1] == false or .[1] == \"false\" or .[1] == 0 or .[1] == \"0\")
|
||||
) | \" - \" + .[0]" "$RESULT_FILE")
|
||||
echo "$FAILING_METRICS"
|
||||
else
|
||||
PASSED_METRICS=$((PASSED_METRICS + METRIC_COUNT))
|
||||
echo " ✅ $EVAL_NAME: All metrics passed"
|
||||
fi
|
||||
done
|
||||
echo ""
|
||||
done
|
||||
|
||||
# Print summary
|
||||
echo "Summary:"
|
||||
echo "-------"
|
||||
echo "Total Evaluations: $TOTAL_EVALS"
|
||||
echo "Total Metrics: $TOTAL_METRICS"
|
||||
echo "Passed Metrics: $PASSED_METRICS"
|
||||
echo "Failed Metrics: $FAILED_METRICS"
|
||||
|
||||
# Set exit code based on failures
|
||||
if [ "$FAILED_METRICS" -gt 0 ]; then
|
||||
echo "❌ Benchmark has $FAILED_METRICS failures"
|
||||
exit 1
|
||||
else
|
||||
echo "✅ All metrics passed successfully"
|
||||
exit 0
|
||||
fi
|
||||
Executable
+286
@@ -0,0 +1,286 @@
|
||||
#!/usr/bin/env bash
|
||||
# run-benchmarks.sh - Script to run goose benchmarks across multiple provider:model pairs
|
||||
|
||||
set -e
|
||||
|
||||
# Display usage information
|
||||
function show_usage() {
|
||||
echo "Usage: $0 [options]"
|
||||
echo ""
|
||||
echo "Options:"
|
||||
echo " -p, --provider-models Comma-separated list of provider:model pairs (e.g., 'openai:gpt-4o,anthropic:claude-3-5-sonnet')"
|
||||
echo " -s, --suites Comma-separated list of benchmark suites to run (e.g., 'core,small_models')"
|
||||
echo " -o, --output-dir Directory to store benchmark results (default: './benchmark-results')"
|
||||
echo " -d, --debug Use debug build instead of release build"
|
||||
echo " -h, --help Show this help message"
|
||||
echo ""
|
||||
echo "Example:"
|
||||
echo " $0 --provider-models 'openai:gpt-4o,anthropic:claude-3-5-sonnet' --suites 'core,small_models'"
|
||||
}
|
||||
|
||||
# Parse command line arguments
|
||||
PROVIDER_MODELS=""
|
||||
SUITES=""
|
||||
OUTPUT_DIR="./benchmark-results"
|
||||
DEBUG_MODE=false
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
-p|--provider-models)
|
||||
PROVIDER_MODELS="$2"
|
||||
shift 2
|
||||
;;
|
||||
-s|--suites)
|
||||
SUITES="$2"
|
||||
shift 2
|
||||
;;
|
||||
-o|--output-dir)
|
||||
OUTPUT_DIR="$2"
|
||||
shift 2
|
||||
;;
|
||||
-d|--debug)
|
||||
DEBUG_MODE=true
|
||||
shift
|
||||
;;
|
||||
-h|--help)
|
||||
show_usage
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo "Error: Unknown option: $1"
|
||||
show_usage
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Validate required parameters
|
||||
if [[ -z "$PROVIDER_MODELS" ]]; then
|
||||
echo "Error: Provider-model pairs must be specified"
|
||||
show_usage
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ -z "$SUITES" ]]; then
|
||||
echo "Error: Benchmark suites must be specified"
|
||||
show_usage
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Create output directory
|
||||
mkdir -p "$OUTPUT_DIR"
|
||||
|
||||
# Create a results summary file
|
||||
SUMMARY_FILE="$OUTPUT_DIR/summary.md"
|
||||
echo "# Benchmark Results Summary" > "$SUMMARY_FILE"
|
||||
echo "Run date: $(date)" >> "$SUMMARY_FILE"
|
||||
echo "Suites: $SUITES" >> "$SUMMARY_FILE"
|
||||
if [ "$DEBUG_MODE" = true ]; then
|
||||
echo "Mode: Debug" >> "$SUMMARY_FILE"
|
||||
else
|
||||
echo "Mode: Release" >> "$SUMMARY_FILE"
|
||||
fi
|
||||
echo "" >> "$SUMMARY_FILE"
|
||||
|
||||
# Determine which binary to use
|
||||
GOOSE_CMD="goose"
|
||||
if [ "$DEBUG_MODE" = true ]; then
|
||||
if [ -f "./target/debug/goose" ]; then
|
||||
GOOSE_CMD="./target/debug/goose"
|
||||
echo "Using debug binary: $GOOSE_CMD"
|
||||
else
|
||||
echo "Warning: Debug binary not found at ./target/debug/goose. Falling back to system-installed goose."
|
||||
fi
|
||||
else
|
||||
if [ -f "./target/release/goose" ]; then
|
||||
GOOSE_CMD="./target/release/goose"
|
||||
echo "Using release binary: $GOOSE_CMD"
|
||||
else
|
||||
echo "Warning: Release binary not found at ./target/release/goose. Falling back to system-installed goose."
|
||||
fi
|
||||
fi
|
||||
|
||||
# Parse provider:model pairs
|
||||
PROVIDERS=()
|
||||
MODELS=()
|
||||
|
||||
# Read provider:model pairs
|
||||
IFS=',' read -ra PAIRS <<< "$PROVIDER_MODELS"
|
||||
for pair in "${PAIRS[@]}"; do
|
||||
# Split by colon
|
||||
IFS=':' read -r provider model <<< "$pair"
|
||||
if [[ -n "$provider" && -n "$model" ]]; then
|
||||
PROVIDERS+=("$provider")
|
||||
MODELS+=("$model")
|
||||
else
|
||||
echo "Warning: Invalid provider:model pair: $pair. Skipping."
|
||||
fi
|
||||
done
|
||||
|
||||
# Track overall success
|
||||
OVERALL_SUCCESS=true
|
||||
COUNT=${#PROVIDERS[@]}
|
||||
|
||||
echo "Running benchmarks for $COUNT provider:model pairs..."
|
||||
echo "Benchmark suites: $SUITES"
|
||||
echo ""
|
||||
|
||||
# Loop through each provider-model pair
|
||||
for ((i=0; i<$COUNT; i++)); do
|
||||
provider="${PROVIDERS[i]}"
|
||||
model="${MODELS[i]}"
|
||||
|
||||
echo "=========================================================="
|
||||
echo "Provider: $provider, Model: $model"
|
||||
echo "=========================================================="
|
||||
|
||||
echo "## Provider: $provider, Model: $model" >> "$SUMMARY_FILE"
|
||||
|
||||
# Set environment variables for this provider/model instead of using configure
|
||||
export GOOSE_PROVIDER="$provider"
|
||||
export GOOSE_MODEL="$model"
|
||||
|
||||
# Run the benchmark and save results to JSON
|
||||
echo "Running benchmark for $provider/$model with suites: $SUITES"
|
||||
OUTPUT_FILE="$OUTPUT_DIR/${provider}-${model}.json"
|
||||
ANALYSIS_FILE="$OUTPUT_DIR/${provider}-${model}-analysis.txt"
|
||||
|
||||
if $GOOSE_CMD bench --suites "$SUITES" --output "$OUTPUT_FILE" --format json; then
|
||||
echo "✅ Benchmark completed successfully" | tee -a "$SUMMARY_FILE"
|
||||
|
||||
# Parse the JSON to check for failures
|
||||
if [ -f "$OUTPUT_FILE" ]; then
|
||||
# Check if jq is installed
|
||||
if ! command -v jq &> /dev/null; then
|
||||
echo "Warning: jq not found. Cannot parse JSON results."
|
||||
echo "⚠️ Could not parse results (jq not installed)" >> "$SUMMARY_FILE"
|
||||
else
|
||||
# Basic validation of the JSON file
|
||||
if jq empty "$OUTPUT_FILE" 2>/dev/null; then
|
||||
# Extract basic information
|
||||
PROVIDER_NAME=$(jq -r '.provider' "$OUTPUT_FILE")
|
||||
START_TIME=$(jq -r '.start_time' "$OUTPUT_FILE")
|
||||
SUITE_COUNT=$(jq '.suites | length' "$OUTPUT_FILE")
|
||||
|
||||
echo "Benchmark Results Analysis" > "$ANALYSIS_FILE"
|
||||
echo "-------------------------" >> "$ANALYSIS_FILE"
|
||||
echo "Provider: $PROVIDER_NAME" >> "$ANALYSIS_FILE"
|
||||
echo "Start Time: $START_TIME" >> "$ANALYSIS_FILE"
|
||||
echo "Number of Suites: $SUITE_COUNT" >> "$ANALYSIS_FILE"
|
||||
echo "" >> "$ANALYSIS_FILE"
|
||||
|
||||
# Initialize counters
|
||||
TOTAL_EVALS=0
|
||||
TOTAL_METRICS=0
|
||||
FAILED_METRICS=0
|
||||
PASSED_METRICS=0
|
||||
TOTAL_ERRORS=0
|
||||
|
||||
# Process each suite
|
||||
for j in $(seq 0 $((SUITE_COUNT-1))); do
|
||||
SUITE_NAME=$(jq -r ".suites[$j].name" "$OUTPUT_FILE")
|
||||
EVAL_COUNT=$(jq ".suites[$j].evaluations | length" "$OUTPUT_FILE")
|
||||
TOTAL_EVALS=$((TOTAL_EVALS + EVAL_COUNT))
|
||||
|
||||
echo "Suite: $SUITE_NAME ($EVAL_COUNT evaluations)" >> "$ANALYSIS_FILE"
|
||||
|
||||
# Process each evaluation in this suite
|
||||
for k in $(seq 0 $((EVAL_COUNT-1))); do
|
||||
EVAL_NAME=$(jq -r ".suites[$j].evaluations[$k].name" "$OUTPUT_FILE")
|
||||
METRIC_COUNT=$(jq ".suites[$j].evaluations[$k].metrics | length" "$OUTPUT_FILE")
|
||||
TOTAL_METRICS=$((TOTAL_METRICS + METRIC_COUNT))
|
||||
|
||||
# Check for errors in this evaluation
|
||||
ERROR_COUNT=$(jq ".suites[$j].evaluations[$k].errors | length" "$OUTPUT_FILE")
|
||||
TOTAL_ERRORS=$((TOTAL_ERRORS + ERROR_COUNT))
|
||||
|
||||
# Check for failures in metrics
|
||||
FAILURES=$(jq -r ".suites[$j].evaluations[$k].metrics[] |
|
||||
select(
|
||||
.[1].Boolean == false or .[1].Boolean == \"false\" or .[1].Boolean == 0 or .[1].Boolean == \"0\"
|
||||
) | .[0]" "$OUTPUT_FILE" | wc -l | tr -d ' ')
|
||||
|
||||
if [ "$FAILURES" -gt 0 ] || [ "$ERROR_COUNT" -gt 0 ]; then
|
||||
FAILED_METRICS=$((FAILED_METRICS + FAILURES))
|
||||
echo " ❌ $EVAL_NAME:" >> "$ANALYSIS_FILE"
|
||||
|
||||
if [ "$FAILURES" -gt 0 ]; then
|
||||
echo " - $FAILURES metric failures detected" >> "$ANALYSIS_FILE"
|
||||
# Print the specific failing metrics
|
||||
FAILING_METRICS=$(jq -r ".suites[$j].evaluations[$k].metrics[] |
|
||||
select(
|
||||
.[1].Boolean == false or .[1].Boolean == \"false\" or .[1].Boolean == 0 or .[1].Boolean == \"0\"
|
||||
) | .[0]" "$OUTPUT_FILE")
|
||||
echo " Failed metrics:" >> "$ANALYSIS_FILE"
|
||||
echo "$FAILING_METRICS" | sed 's/^/ - /' >> "$ANALYSIS_FILE"
|
||||
fi
|
||||
|
||||
if [ "$ERROR_COUNT" -gt 0 ]; then
|
||||
echo " - $ERROR_COUNT errors detected" >> "$ANALYSIS_FILE"
|
||||
# Print the errors
|
||||
jq -r ".suites[$j].evaluations[$k].errors[] | \" [\(.level)] \(.message)\"" "$OUTPUT_FILE" >> "$ANALYSIS_FILE"
|
||||
fi
|
||||
else
|
||||
PASSED_METRICS=$((PASSED_METRICS + METRIC_COUNT))
|
||||
echo " ✅ $EVAL_NAME: All metrics passed, no errors" >> "$ANALYSIS_FILE"
|
||||
fi
|
||||
done
|
||||
echo "" >> "$ANALYSIS_FILE"
|
||||
done
|
||||
|
||||
# Print summary
|
||||
echo "Summary:" >> "$ANALYSIS_FILE"
|
||||
echo "-------" >> "$ANALYSIS_FILE"
|
||||
echo "Total Evaluations: $TOTAL_EVALS" >> "$ANALYSIS_FILE"
|
||||
echo "Total Metrics: $TOTAL_METRICS" >> "$ANALYSIS_FILE"
|
||||
echo "Passed Metrics: $PASSED_METRICS" >> "$ANALYSIS_FILE"
|
||||
echo "Failed Metrics: $FAILED_METRICS" >> "$ANALYSIS_FILE"
|
||||
echo "Total Errors: $TOTAL_ERRORS" >> "$ANALYSIS_FILE"
|
||||
|
||||
# Determine success/failure
|
||||
if [ "$FAILED_METRICS" -gt 0 ] || [ "$TOTAL_ERRORS" -gt 0 ]; then
|
||||
if [ "$FAILED_METRICS" -gt 0 ]; then
|
||||
echo "❌ Benchmark has $FAILED_METRICS failed metrics" >> "$ANALYSIS_FILE"
|
||||
fi
|
||||
if [ "$TOTAL_ERRORS" -gt 0 ]; then
|
||||
echo "❌ Benchmark has $TOTAL_ERRORS errors" >> "$ANALYSIS_FILE"
|
||||
fi
|
||||
echo "❌ Tests failed for $provider/$model" | tee -a "$SUMMARY_FILE"
|
||||
cat "$ANALYSIS_FILE" >> "$SUMMARY_FILE"
|
||||
OVERALL_SUCCESS=false
|
||||
else
|
||||
echo "✅ All metrics passed successfully, no errors" >> "$ANALYSIS_FILE"
|
||||
echo "✅ All tests passed for $provider/$model" | tee -a "$SUMMARY_FILE"
|
||||
cat "$ANALYSIS_FILE" >> "$SUMMARY_FILE"
|
||||
fi
|
||||
else
|
||||
echo "❌ Invalid JSON in benchmark output" | tee -a "$SUMMARY_FILE"
|
||||
OVERALL_SUCCESS=false
|
||||
fi
|
||||
fi
|
||||
else
|
||||
echo "❌ Benchmark output file not found" | tee -a "$SUMMARY_FILE"
|
||||
OVERALL_SUCCESS=false
|
||||
fi
|
||||
else
|
||||
echo "❌ Benchmark failed to run" | tee -a "$SUMMARY_FILE"
|
||||
OVERALL_SUCCESS=false
|
||||
fi
|
||||
|
||||
echo "" >> "$SUMMARY_FILE"
|
||||
echo ""
|
||||
done
|
||||
|
||||
echo "=========================================================="
|
||||
echo "Benchmark run completed"
|
||||
echo "Results saved to: $OUTPUT_DIR"
|
||||
echo "Summary file: $SUMMARY_FILE"
|
||||
|
||||
# Output final status
|
||||
if [ "$OVERALL_SUCCESS" = false ]; then
|
||||
echo "❌ Some benchmarks failed. Check the summary for details."
|
||||
exit 1
|
||||
else
|
||||
echo "✅ All benchmarks completed successfully."
|
||||
exit 0
|
||||
fi
|
||||
Reference in New Issue
Block a user