learning from test finder

.github/workflows/ci.yml

@@ -148,6 +148,7 @@ jobs:
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
           PROVIDERS: ${{ matrix.provider }}
           VERBOSE: 1
+          GOOSE_DISABLE_KEYRING: 1
         run: ./tests/smoke/run_smoke_tests.sh
 
   desktop-lint:

tests/smoke/config.anthropic.yaml

@@ -1,12 +0,0 @@
-GOOSE_PROVIDER: anthropic
-GOOSE_MODEL: claude-sonnet-4-5-20250929
-extensions:
-  developer:
-    enabled: true
-    type: builtin
-    name: developer
-    display_name: Developer
-    description: null
-    timeout: 300
-    bundled: true
-    available_tools: []

tests/smoke/config.openai.yaml

@@ -1,12 +0,0 @@
-GOOSE_PROVIDER: openai
-GOOSE_MODEL: gpt-4o
-extensions:
-  developer:
-    enabled: true
-    type: builtin
-    name: developer
-    display_name: Developer
-    description: null
-    timeout: 300
-    bundled: true
-    available_tools: []

tests/smoke/run_smoke_tests.sh

@@ -4,14 +4,14 @@
 # Usage: ./run_smoke_tests.sh
 # Requires: ANTHROPIC_API_KEY and OPENAI_API_KEY environment variables
 
-set -e  # Exit on error
-set -u  # Exit on undefined variable
+set -e
+set -u
 
 # Colors for output
 RED='\033[0;31m'
 GREEN='\033[0;32m'
 YELLOW='\033[1;33m'
-NC='\033[0m' # No Color
+NC='\033[0m'
 
 # Configuration
 TIMEOUT=${TIMEOUT:-60}
@@ -21,7 +21,7 @@ PROVIDERS=${PROVIDERS:-"anthropic openai"}
 # Get script directory and repo root
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
-# Look for binary in multiple locations (for CI sparse checkout)
+
 if [ -f "${REPO_ROOT}/target/release/goose" ]; then
     GOOSE_BINARY="${REPO_ROOT}/target/release/goose"
 elif [ -f "target/release/goose" ]; then
@@ -52,10 +52,6 @@ log_info() {
     echo -e "${GREEN}[INFO]${NC} $*"
 }
 
-log_warn() {
-    echo -e "${YELLOW}[WARN]${NC} $*"
-}
-
 log_error() {
     echo -e "${RED}[ERROR]${NC} $*"
 }
@@ -68,15 +64,14 @@ log_verbose() {
 
 # Setup test environment
 setup_test_env() {
-    # Create temporary directory for this test run
     TEST_TEMP_DIR=$(mktemp -d -t goose-smoke-test.XXXXXX)
     log_verbose "Created temp directory: ${TEST_TEMP_DIR}"
     
-    # Set environment variables to use temp directory
     export HOME="${TEST_TEMP_DIR}"
     export GOOSE_DISABLE_KEYRING=1
     
-    # Create config directory (config will be swapped per test)
+    # Create goose directory structure
+    mkdir -p "${HOME}/.local/share/goose/sessions"
     mkdir -p "${HOME}/.config/goose"
     
     log_verbose "Test environment configured"
@@ -86,7 +81,6 @@ setup_test_env() {
 check_prerequisites() {
     log_info "Checking prerequisites..."
     
-    # Check if binary exists
     if [ ! -f "${GOOSE_BINARY}" ]; then
         log_error "Goose binary not found at: ${GOOSE_BINARY}"
         log_error "Please build it first: cargo build --release"
@@ -94,7 +88,6 @@ check_prerequisites() {
     fi
     log_verbose "✓ Binary found: ${GOOSE_BINARY}"
     
-    # Check for required environment variables based on selected providers
     if echo "${PROVIDERS}" | grep -q "anthropic"; then
         if [ -z "${ANTHROPIC_API_KEY:-}" ]; then
             log_error "ANTHROPIC_API_KEY environment variable not set"
@@ -118,31 +111,27 @@ check_prerequisites() {
 run_test() {
     local test_name="$1"
     local provider="$2"
-    local prompt="$3"
-    local expected_pattern="$4"
-    local use_developer="${5:-no}"
+    local model="$3"
+    local prompt="$4"
+    local expected_pattern="$5"
     
     TESTS_RUN=$((TESTS_RUN + 1))
     
     log_info "Running test: ${test_name}"
     log_verbose "  Provider: ${provider}"
+    log_verbose "  Model: ${model}"
     log_verbose "  Prompt: ${prompt}"
     
-    # Build command (model comes from config file)
-    local cmd="${GOOSE_BINARY} run --text \"${prompt}\" --no-session --provider ${provider}"
+    # Set provider env vars
+    export GOOSE_PROVIDER="${provider}"
+    export GOOSE_MODEL="${model}"
     
-    # Add developer extension if requested
-    if [ "${use_developer}" = "yes" ]; then
-        cmd="${cmd} --with-builtin developer"
-        log_verbose "  Using developer extension"
-    fi
-    
-    # Create output file for this test
+    local cmd="${GOOSE_BINARY} run --text \"${prompt}\" --no-session --with-builtin developer"
     local output_file="${TEST_TEMP_DIR}/test_${TESTS_RUN}.out"
     
     log_verbose "  Running command: ${cmd}"
     
-    # Run the command with timeout (use gtimeout on macOS if available, otherwise skip timeout)
+    # Run with timeout if available
     local timeout_cmd=""
     if command -v timeout >/dev/null 2>&1; then
         timeout_cmd="timeout ${TIMEOUT}"
@@ -154,7 +143,6 @@ run_test() {
         eval "${timeout_cmd} bash -c \"${cmd}\"" > "${output_file}" 2>&1 || local exit_code=$?
         [ -z "${exit_code:-}" ] && local exit_code=0
     else
-        # No timeout available, run without it
         bash -c "${cmd}" > "${output_file}" 2>&1 || local exit_code=$?
         [ -z "${exit_code:-}" ] && local exit_code=0
     fi
@@ -162,7 +150,6 @@ run_test() {
     if [ ${exit_code} -eq 0 ]; then
         log_verbose "  Command completed successfully"
         
-        # Check if output matches expected pattern
         if grep -q "${expected_pattern}" "${output_file}"; then
             log_info "✓ Test passed: ${test_name}"
             TESTS_PASSED=$((TESTS_PASSED + 1))
@@ -181,7 +168,6 @@ run_test() {
             return 1
         fi
     else
-        local exit_code=$?
         log_error "✗ Test failed: ${test_name}"
         log_error "  Command failed with exit code: ${exit_code}"
         log_error "  Output:"
@@ -194,29 +180,19 @@ run_test() {
 # Smoke test: Developer extension with file listing
 test_developer_extension() {
     local provider="$1"
-    
-    # Swap in the provider-specific config
-    local config_file="${SCRIPT_DIR}/config.${provider}.yaml"
-    if [ -f "${config_file}" ]; then
-        cp "${config_file}" "${HOME}/.config/goose/config.yaml"
-        log_verbose "Swapped in config for ${provider}"
-    else
-        log_error "Config file not found: ${config_file}"
-        return 1
-    fi
+    local model="$2"
     
     # Create hello.txt in temp directory
     echo "hello" > "${TEST_TEMP_DIR}/hello.txt"
     
-    # Change to temp directory and run test
     cd "${TEST_TEMP_DIR}"
     
     run_test \
         "Developer extension smoke test (${provider})" \
         "${provider}" \
+        "${model}" \
         "list files" \
-        "hello.txt" \
-        "yes"
+        "hello.txt"
     
     cd "${REPO_ROOT}"
 }
@@ -224,12 +200,13 @@ test_developer_extension() {
 # Run all tests for a provider
 run_provider_tests() {
     local provider="$1"
+    local model="$2"
     
     log_info "========================================="
-    log_info "Testing provider: ${provider}"
+    log_info "Testing provider: ${provider} (${model})"
     log_info "========================================="
     
-    test_developer_extension "${provider}"
+    test_developer_extension "${provider}" "${model}"
     
     echo ""
 }
@@ -244,11 +221,11 @@ main() {
     
     # Run tests for each provider
     if echo "${PROVIDERS}" | grep -q "anthropic"; then
-        run_provider_tests "anthropic"
+        run_provider_tests "anthropic" "claude-sonnet-4-5-20250929"
     fi
     
     if echo "${PROVIDERS}" | grep -q "openai"; then
-        run_provider_tests "openai"
+        run_provider_tests "openai" "gpt-5"
     fi
     
     # Print summary