live testing script (#5263)

Co-authored-by: Jack Amadeo <jackamadeo@squareup.com>

.github/workflows/pr-smoke-test.yml

@@ -0,0 +1,104 @@
+on:
+  pull_request:
+    branches:
+      - main
+
+name: Live Provider Tests
+
+jobs:
+  changes:
+    runs-on: ubuntu-latest
+    outputs:
+      code: ${{ steps.filter.outputs.code }}
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # pin@v4
+
+      - name: Check for code changes
+        uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # pin@v3
+        id: filter
+        with:
+          filters: |
+            code:
+              - '!documentation/**'
+
+  build-binary:
+    name: Build Release Binary
+    runs-on: goose
+    needs: changes
+    if: needs.changes.outputs.code == 'true'
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # pin@v4
+
+      - name: Activate hermit and set CARGO_HOME
+        run: |
+          source bin/activate-hermit
+          echo "CARGO_HOME=$CARGO_HOME" >> $GITHUB_ENV
+          echo "RUSTUP_HOME=$RUSTUP_HOME" >> $GITHUB_ENV
+
+      - name: Install Dependencies
+        run: |
+          sudo apt update -y
+          sudo apt install -y libdbus-1-dev gnome-keyring libxcb1-dev
+
+      - name: Cache Cargo artifacts
+        uses: actions/cache@2f8e54208210a422b2efd51efaa6bd6d7ca8920f # pin@v3
+        with:
+          path: |
+            ${{ env.CARGO_HOME }}/bin/
+            ${{ env.CARGO_HOME }}/registry/index/
+            ${{ env.CARGO_HOME }}/registry/cache/
+            ${{ env.CARGO_HOME }}/git/db/
+            target/
+          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-cargo-
+
+      - name: Build Release Binary for Smoke Tests
+        run: |
+          source ./bin/activate-hermit
+          cargo build --release
+
+      - name: Upload Binary for Smoke Tests
+        uses: actions/upload-artifact@v4
+        with:
+          name: goose-binary
+          path: target/release/goose
+          retention-days: 1
+
+  smoke-tests:
+    name: Smoke Tests
+    runs-on: ubuntu-latest
+    needs: build-binary
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # pin@v4
+
+      - name: Download Binary
+        uses: actions/download-artifact@v4
+        with:
+          name: goose-binary
+          path: target/release
+
+      - name: Make Binary Executable
+        run: chmod +x target/release/goose
+
+      - name: Run Smoke Tests with Provider Script
+        env:
+          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
+          DATABRICKS_HOST: ${{ secrets.DATABRICKS_HOST }}
+          DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }}
+          OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
+          HOME: /tmp/goose-home
+          GOOSE_DISABLE_KEYRING: 1
+          SKIP_BUILD: 1
+        run: |
+          # Ensure the HOME directory structure exists
+          mkdir -p $HOME/.local/share/goose/sessions
+          mkdir -p $HOME/.config/goose
+
+          # Run the provider test script (binary already built and downloaded)
+          bash scripts/test_providers.sh

scripts/test_providers.sh

@@ -3,20 +3,38 @@ if [ -f .env ]; then
   export $(grep -v '^#' .env | xargs)
 fi
 
-echo "Building goose..."
-cargo build --release --bin goose
-echo ""
+if [ -z "$SKIP_BUILD" ]; then
+  echo "Building goose..."
+  cargo build --release --bin goose
+  echo ""
+else
+  echo "Skipping build (SKIP_BUILD is set)..."
+  echo ""
+fi
 
 SCRIPT_DIR=$(pwd)
 
 PROVIDERS=(
-  "openrouter:anthropic/claude-sonnet-4.5:google/gemini-flash-2.5:qwen/qwen3-coder"
+  "openrouter:anthropic/claude-sonnet-4.5:qwen/qwen3-coder"
   "openai:gpt-4o:gpt-4o-mini:gpt-3.5-turbo"
-  "anthropic:claude-sonnet-4-0:claude-3-7-sonnet-latest"
+  "anthropic:claude-sonnet-4-5-20250929:claude-opus-4-1-20250805"
   "google:gemini-2.5-pro:gemini-2.5-pro:gemini-2.5-flash"
-  "databricks:databricks-claude-sonnet-4:gemini-2-5-flash:gpt-4o"
 )
 
+# In CI, only run Databricks tests if DATABRICKS_HOST and DATABRICKS_TOKEN are set
+# Locally, always run Databricks tests
+if [ -n "$CI" ]; then
+  if [ -n "$DATABRICKS_HOST" ] && [ -n "$DATABRICKS_TOKEN" ]; then
+    echo "✓ Including Databricks tests"
+    PROVIDERS+=("databricks:databricks-claude-sonnet-4:gemini-2-5-flash:gpt-4o")
+  else
+    echo "⚠️  Skipping Databricks tests (DATABRICKS_HOST and DATABRICKS_TOKEN required in CI)"
+  fi
+else
+  echo "✓ Including Databricks tests"
+  PROVIDERS+=("databricks:databricks-claude-sonnet-4:gemini-2-5-flash:gpt-4o")
+fi
+
 RESULTS=()
 
 for provider_config in "${PROVIDERS[@]}"; do