From ef0252bb3f2bc95683a1aa850231585fe22b2c7a Mon Sep 17 00:00:00 2001 From: Bradley Axen Date: Wed, 24 Dec 2025 20:39:09 -0800 Subject: [PATCH] fix: improve smoke test prompt for reliable tool calling The previous prompt 'please list files in the current directory' was ambiguous and didn't explicitly require tool usage. Models like qwen/qwen3-coder and z-ai/glm-4.6 would sometimes respond with text describing what they would do instead of actually calling the tool. The new prompt explicitly instructs the model to immediately call the shell tool without asking for confirmation, which should improve reliability for models with weaker tool-calling capabilities. --- scripts/test_providers.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/test_providers.sh b/scripts/test_providers.sh index e8fdc28fb0..088a588c18 100755 --- a/scripts/test_providers.sh +++ b/scripts/test_providers.sh @@ -85,7 +85,7 @@ for provider_config in "${PROVIDERS[@]}"; do echo "Model: ${MODEL}" echo "" TMPFILE=$(mktemp) - (cd "$TESTDIR" && "$SCRIPT_DIR/target/release/goose" run --text "please list files in the current directory" --with-builtin "$BUILTINS" 2>&1) | tee "$TMPFILE" + (cd "$TESTDIR" && "$SCRIPT_DIR/target/release/goose" run --text "Immediately call the shell tool to run 'ls -la'. Do not ask for confirmation." --with-builtin "$BUILTINS" 2>&1) | tee "$TMPFILE" echo "" if grep -qE "$SUCCESS_PATTERN" "$TMPFILE"; then echo "✓ SUCCESS: Test passed - $SUCCESS_MSG"