mirror of
https://github.com/block/goose.git
synced 2026-07-17 12:56:20 +02:00
feat: simplify developer extension (#7466)
Co-authored-by: Alex Hancock <alexhancock@block.xyz>
This commit is contained in:
@@ -23,9 +23,9 @@ run_test() {
|
||||
cp "$TEST_FILE" "$testdir/test-content.txt"
|
||||
prompt="read ./test-content.txt and output its contents exactly"
|
||||
else
|
||||
# Write two files with unique random tokens. Validation checks that text_editor
|
||||
# was used and that both tokens appear in the output, proving the model actually
|
||||
# read the files (random tokens can't be guessed or hallucinated).
|
||||
# Write two files with unique random tokens. Validation checks that the shell
|
||||
# tool was used and that both tokens appear in the output, proving the model
|
||||
# actually read the files (random tokens can't be guessed or hallucinated).
|
||||
local token_a="smoke-alpha-$RANDOM"
|
||||
local token_b="smoke-bravo-$RANDOM"
|
||||
echo "$token_a" > "$testdir/part-a.txt"
|
||||
@@ -33,7 +33,7 @@ run_test() {
|
||||
# Store tokens so validation can check them
|
||||
echo "$token_a" > "$testdir/.token_a"
|
||||
echo "$token_b" > "$testdir/.token_b"
|
||||
prompt="Use the text_editor view command to read ./part-a.txt and ./part-b.txt, then reply with ONLY the contents of both files, one per line, nothing else. Do NOT use any other tool in Developer."
|
||||
prompt="Use the shell tool to cat ./part-a.txt and ./part-b.txt, then reply with ONLY the contents of both files, one per line, nothing else."
|
||||
fi
|
||||
|
||||
(
|
||||
@@ -52,8 +52,8 @@ run_test() {
|
||||
local token_a token_b
|
||||
token_a=$(cat "$testdir/.token_a")
|
||||
token_b=$(cat "$testdir/.token_b")
|
||||
if ! grep -qE "(text_editor \| developer)|(▸.*text_editor.*developer)" "$output_file"; then
|
||||
echo "failure|model did not use text_editor tool" > "$result_file"
|
||||
if ! grep -qE "(shell \| developer)|(▸.*shell)" "$output_file"; then
|
||||
echo "failure|model did not use shell tool" > "$result_file"
|
||||
elif ! grep -q "$token_a" "$output_file"; then
|
||||
echo "failure|model did not return contents of part-a.txt ($token_a)" > "$result_file"
|
||||
elif ! grep -q "$token_b" "$output_file"; then
|
||||
|
||||
@@ -10,7 +10,7 @@ echo ""
|
||||
# --- Setup ---
|
||||
|
||||
GOOSE_BIN=$(build_goose)
|
||||
BUILTINS="developer,code_execution"
|
||||
BUILTINS="memory,code_execution"
|
||||
|
||||
# --- Test case ---
|
||||
|
||||
@@ -18,8 +18,7 @@ run_test() {
|
||||
local provider="$1" model="$2" result_file="$3" output_file="$4"
|
||||
local testdir=$(mktemp -d)
|
||||
|
||||
echo "hello" > "$testdir/hello.txt"
|
||||
local prompt="Run 'ls' to list files in the current directory."
|
||||
local prompt="Store a memory with category 'test' and data 'hello world', then retrieve all memories from category 'test'."
|
||||
|
||||
# Run goose
|
||||
(
|
||||
@@ -28,7 +27,6 @@ run_test() {
|
||||
cd "$testdir" && "$GOOSE_BIN" run --text "$prompt" --with-builtin "$BUILTINS" 2>&1
|
||||
) > "$output_file" 2>&1
|
||||
|
||||
# Verify: code_execution tool must be called
|
||||
# Matches: "execute | code_execution", "get_function_details | code_execution",
|
||||
# "tool call | execute", "tool calls | execute" (old format)
|
||||
# "▸ execute N tool call" (new format with tool_graph)
|
||||
|
||||
@@ -29,6 +29,7 @@ ALLOWED_FAILURES=(
|
||||
"google:gemini-2.5-flash"
|
||||
"google:gemini-3-pro-preview"
|
||||
"openrouter:nvidia/nemotron-3-nano-30b-a3b"
|
||||
"openrouter:qwen/qwen3-coder:exacto"
|
||||
"openai:gpt-3.5-turbo"
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user