mirror of
https://github.com/block/goose.git
synced 2026-07-17 12:56:20 +02:00
initial work on readonly developer mode
This commit is contained in:
@@ -167,9 +167,17 @@ fn load_prompt_files() -> HashMap<String, Prompt> {
|
||||
prompts
|
||||
}
|
||||
|
||||
/// Check if developer extension is in read-only mode
|
||||
fn is_readonly_mode() -> bool {
|
||||
std::env::var("DEVELOPER_READONLY")
|
||||
.map(|v| v == "1" || v.to_lowercase() == "true")
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
/// Developer MCP Server using official RMCP SDK
|
||||
#[derive(Clone)]
|
||||
pub struct DeveloperServer {
|
||||
readonly: bool,
|
||||
tool_router: ToolRouter<Self>,
|
||||
file_history: Arc<Mutex<HashMap<PathBuf, Vec<String>>>>,
|
||||
ignore_patterns: Gitignore,
|
||||
@@ -558,7 +566,11 @@ impl DeveloperServer {
|
||||
// Initialize editor model for AI-powered code editing
|
||||
let editor_model = create_editor_model();
|
||||
|
||||
// Check if read-only mode is enabled
|
||||
let readonly = is_readonly_mode();
|
||||
|
||||
Self {
|
||||
readonly,
|
||||
tool_router: Self::tool_router(),
|
||||
file_history: Arc::new(Mutex::new(HashMap::new())),
|
||||
ignore_patterns,
|
||||
@@ -577,6 +589,15 @@ impl DeveloperServer {
|
||||
description = "List all available window titles that can be used with screen_capture. Returns a list of window titles that can be used with the window_title parameter of the screen_capture tool."
|
||||
)]
|
||||
pub async fn list_windows(&self) -> Result<CallToolResult, ErrorData> {
|
||||
// Check if in read-only mode
|
||||
if self.readonly {
|
||||
return Err(ErrorData::new(
|
||||
ErrorCode::INVALID_PARAMS,
|
||||
"list_windows is not allowed in read-only mode".to_string(),
|
||||
None,
|
||||
));
|
||||
}
|
||||
|
||||
let windows = Window::all().map_err(|_| {
|
||||
ErrorData::new(
|
||||
ErrorCode::INTERNAL_ERROR,
|
||||
@@ -612,6 +633,15 @@ impl DeveloperServer {
|
||||
&self,
|
||||
params: Parameters<ScreenCaptureParams>,
|
||||
) -> Result<CallToolResult, ErrorData> {
|
||||
// Check if in read-only mode
|
||||
if self.readonly {
|
||||
return Err(ErrorData::new(
|
||||
ErrorCode::INVALID_PARAMS,
|
||||
"screen_capture is not allowed in read-only mode".to_string(),
|
||||
None,
|
||||
));
|
||||
}
|
||||
|
||||
let params = params.0;
|
||||
|
||||
let mut image = if let Some(window_title) = ¶ms.window_title {
|
||||
@@ -754,6 +784,14 @@ impl DeveloperServer {
|
||||
Ok(CallToolResult::success(content))
|
||||
}
|
||||
"write" => {
|
||||
// Check if in read-only mode
|
||||
if self.readonly {
|
||||
return Err(ErrorData::new(
|
||||
ErrorCode::INVALID_PARAMS,
|
||||
"Command 'write' is not allowed in read-only mode. Only 'view' is permitted.".to_string(),
|
||||
None,
|
||||
));
|
||||
}
|
||||
let file_text = params.file_text.ok_or_else(|| {
|
||||
ErrorData::new(
|
||||
ErrorCode::INVALID_PARAMS,
|
||||
@@ -765,6 +803,14 @@ impl DeveloperServer {
|
||||
Ok(CallToolResult::success(content))
|
||||
}
|
||||
"str_replace" => {
|
||||
// Check if in read-only mode
|
||||
if self.readonly {
|
||||
return Err(ErrorData::new(
|
||||
ErrorCode::INVALID_PARAMS,
|
||||
"Command 'str_replace' is not allowed in read-only mode. Only 'view' is permitted.".to_string(),
|
||||
None,
|
||||
));
|
||||
}
|
||||
// Check if diff parameter is provided
|
||||
if let Some(ref diff) = params.diff {
|
||||
// When diff is provided, old_str and new_str are not required
|
||||
@@ -807,6 +853,14 @@ impl DeveloperServer {
|
||||
}
|
||||
}
|
||||
"insert" => {
|
||||
// Check if in read-only mode
|
||||
if self.readonly {
|
||||
return Err(ErrorData::new(
|
||||
ErrorCode::INVALID_PARAMS,
|
||||
"Command 'insert' is not allowed in read-only mode. Only 'view' is permitted.".to_string(),
|
||||
None,
|
||||
));
|
||||
}
|
||||
let insert_line = params.insert_line.ok_or_else(|| {
|
||||
ErrorData::new(
|
||||
ErrorCode::INVALID_PARAMS,
|
||||
@@ -827,6 +881,14 @@ impl DeveloperServer {
|
||||
Ok(CallToolResult::success(content))
|
||||
}
|
||||
"undo_edit" => {
|
||||
// Check if in read-only mode
|
||||
if self.readonly {
|
||||
return Err(ErrorData::new(
|
||||
ErrorCode::INVALID_PARAMS,
|
||||
"Command 'undo_edit' is not allowed in read-only mode. Only 'view' is permitted.".to_string(),
|
||||
None,
|
||||
));
|
||||
}
|
||||
let content = text_editor_undo(&path, &self.file_history).await?;
|
||||
Ok(CallToolResult::success(content))
|
||||
}
|
||||
@@ -856,6 +918,15 @@ impl DeveloperServer {
|
||||
params: Parameters<ShellParams>,
|
||||
context: RequestContext<RoleServer>,
|
||||
) -> Result<CallToolResult, ErrorData> {
|
||||
// Check if in read-only mode
|
||||
if self.readonly {
|
||||
return Err(ErrorData::new(
|
||||
ErrorCode::INVALID_PARAMS,
|
||||
"shell is not allowed in read-only mode".to_string(),
|
||||
None,
|
||||
));
|
||||
}
|
||||
|
||||
let params = params.0;
|
||||
let command = ¶ms.command;
|
||||
let peer = context.peer;
|
||||
@@ -1136,6 +1207,15 @@ impl DeveloperServer {
|
||||
&self,
|
||||
params: Parameters<ImageProcessorParams>,
|
||||
) -> Result<CallToolResult, ErrorData> {
|
||||
// Check if in read-only mode
|
||||
if self.readonly {
|
||||
return Err(ErrorData::new(
|
||||
ErrorCode::INVALID_PARAMS,
|
||||
"image_processor is not allowed in read-only mode".to_string(),
|
||||
None,
|
||||
));
|
||||
}
|
||||
|
||||
let params = params.0;
|
||||
let path_str = ¶ms.path;
|
||||
|
||||
@@ -3863,4 +3943,359 @@ Additional instructions here.
|
||||
cleanup_test_service(running_service, peer);
|
||||
});
|
||||
}
|
||||
|
||||
// Read-only mode tests
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_readonly_mode_initialization() {
|
||||
// Test without environment variable
|
||||
let server = create_test_server();
|
||||
assert!(
|
||||
!server.readonly,
|
||||
"Should not be in read-only mode by default"
|
||||
);
|
||||
|
||||
// Test with environment variable set to "true"
|
||||
std::env::set_var("DEVELOPER_READONLY", "true");
|
||||
let server = DeveloperServer::new();
|
||||
assert!(
|
||||
server.readonly,
|
||||
"Should be in read-only mode when env var is 'true'"
|
||||
);
|
||||
std::env::remove_var("DEVELOPER_READONLY");
|
||||
|
||||
// Test with environment variable set to "1"
|
||||
std::env::set_var("DEVELOPER_READONLY", "1");
|
||||
let server = DeveloperServer::new();
|
||||
assert!(
|
||||
server.readonly,
|
||||
"Should be in read-only mode when env var is '1'"
|
||||
);
|
||||
std::env::remove_var("DEVELOPER_READONLY");
|
||||
|
||||
// Test with environment variable set to "false"
|
||||
std::env::set_var("DEVELOPER_READONLY", "false");
|
||||
let server = DeveloperServer::new();
|
||||
assert!(
|
||||
!server.readonly,
|
||||
"Should not be in read-only mode when env var is 'false'"
|
||||
);
|
||||
std::env::remove_var("DEVELOPER_READONLY");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_readonly_mode_text_editor_view_allowed() {
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
let file_path = temp_dir.path().join("test.txt");
|
||||
std::fs::write(&file_path, "test content").unwrap();
|
||||
std::env::set_current_dir(&temp_dir).unwrap();
|
||||
|
||||
// Enable read-only mode
|
||||
std::env::set_var("DEVELOPER_READONLY", "true");
|
||||
let server = DeveloperServer::new();
|
||||
|
||||
// View should be allowed
|
||||
let view_params = Parameters(TextEditorParams {
|
||||
path: file_path.to_str().unwrap().to_string(),
|
||||
command: "view".to_string(),
|
||||
view_range: None,
|
||||
file_text: None,
|
||||
old_str: None,
|
||||
new_str: None,
|
||||
insert_line: None,
|
||||
diff: None,
|
||||
});
|
||||
|
||||
let result = server.text_editor(view_params).await;
|
||||
assert!(
|
||||
result.is_ok(),
|
||||
"View command should be allowed in read-only mode"
|
||||
);
|
||||
|
||||
std::env::remove_var("DEVELOPER_READONLY");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_readonly_mode_text_editor_write_blocked() {
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
std::env::set_current_dir(&temp_dir).unwrap();
|
||||
|
||||
// Enable read-only mode
|
||||
std::env::set_var("DEVELOPER_READONLY", "true");
|
||||
let server = DeveloperServer::new();
|
||||
|
||||
// Write should be blocked
|
||||
let write_params = Parameters(TextEditorParams {
|
||||
path: "test.txt".to_string(),
|
||||
command: "write".to_string(),
|
||||
view_range: None,
|
||||
file_text: Some("new content".to_string()),
|
||||
old_str: None,
|
||||
new_str: None,
|
||||
insert_line: None,
|
||||
diff: None,
|
||||
});
|
||||
|
||||
let result = server.text_editor(write_params).await;
|
||||
assert!(
|
||||
result.is_err(),
|
||||
"Write command should be blocked in read-only mode"
|
||||
);
|
||||
let err = result.err().unwrap();
|
||||
assert_eq!(err.code, ErrorCode::INVALID_PARAMS);
|
||||
assert!(err.message.contains("not allowed in read-only mode"));
|
||||
|
||||
std::env::remove_var("DEVELOPER_READONLY");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_readonly_mode_text_editor_str_replace_blocked() {
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
let file_path = temp_dir.path().join("test.txt");
|
||||
std::fs::write(&file_path, "original content").unwrap();
|
||||
std::env::set_current_dir(&temp_dir).unwrap();
|
||||
|
||||
// Enable read-only mode
|
||||
std::env::set_var("DEVELOPER_READONLY", "true");
|
||||
let server = DeveloperServer::new();
|
||||
|
||||
// str_replace should be blocked
|
||||
let replace_params = Parameters(TextEditorParams {
|
||||
path: file_path.to_str().unwrap().to_string(),
|
||||
command: "str_replace".to_string(),
|
||||
view_range: None,
|
||||
file_text: None,
|
||||
old_str: Some("original".to_string()),
|
||||
new_str: Some("modified".to_string()),
|
||||
insert_line: None,
|
||||
diff: None,
|
||||
});
|
||||
|
||||
let result = server.text_editor(replace_params).await;
|
||||
assert!(
|
||||
result.is_err(),
|
||||
"str_replace command should be blocked in read-only mode"
|
||||
);
|
||||
let err = result.err().unwrap();
|
||||
assert_eq!(err.code, ErrorCode::INVALID_PARAMS);
|
||||
assert!(err.message.contains("not allowed in read-only mode"));
|
||||
|
||||
std::env::remove_var("DEVELOPER_READONLY");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_readonly_mode_text_editor_insert_blocked() {
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
let file_path = temp_dir.path().join("test.txt");
|
||||
std::fs::write(&file_path, "line 1\nline 2").unwrap();
|
||||
std::env::set_current_dir(&temp_dir).unwrap();
|
||||
|
||||
// Enable read-only mode
|
||||
std::env::set_var("DEVELOPER_READONLY", "true");
|
||||
let server = DeveloperServer::new();
|
||||
|
||||
// insert should be blocked
|
||||
let insert_params = Parameters(TextEditorParams {
|
||||
path: file_path.to_str().unwrap().to_string(),
|
||||
command: "insert".to_string(),
|
||||
view_range: None,
|
||||
file_text: None,
|
||||
old_str: None,
|
||||
new_str: Some("new line".to_string()),
|
||||
insert_line: Some(1),
|
||||
diff: None,
|
||||
});
|
||||
|
||||
let result = server.text_editor(insert_params).await;
|
||||
assert!(
|
||||
result.is_err(),
|
||||
"insert command should be blocked in read-only mode"
|
||||
);
|
||||
let err = result.err().unwrap();
|
||||
assert_eq!(err.code, ErrorCode::INVALID_PARAMS);
|
||||
assert!(err.message.contains("not allowed in read-only mode"));
|
||||
|
||||
std::env::remove_var("DEVELOPER_READONLY");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_readonly_mode_text_editor_undo_blocked() {
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
let file_path = temp_dir.path().join("test.txt");
|
||||
std::fs::write(&file_path, "content").unwrap();
|
||||
std::env::set_current_dir(&temp_dir).unwrap();
|
||||
|
||||
// Enable read-only mode
|
||||
std::env::set_var("DEVELOPER_READONLY", "true");
|
||||
let server = DeveloperServer::new();
|
||||
|
||||
// undo_edit should be blocked
|
||||
let undo_params = Parameters(TextEditorParams {
|
||||
path: file_path.to_str().unwrap().to_string(),
|
||||
command: "undo_edit".to_string(),
|
||||
view_range: None,
|
||||
file_text: None,
|
||||
old_str: None,
|
||||
new_str: None,
|
||||
insert_line: None,
|
||||
diff: None,
|
||||
});
|
||||
|
||||
let result = server.text_editor(undo_params).await;
|
||||
assert!(
|
||||
result.is_err(),
|
||||
"undo_edit command should be blocked in read-only mode"
|
||||
);
|
||||
let err = result.err().unwrap();
|
||||
assert_eq!(err.code, ErrorCode::INVALID_PARAMS);
|
||||
assert!(err.message.contains("not allowed in read-only mode"));
|
||||
|
||||
std::env::remove_var("DEVELOPER_READONLY");
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn test_readonly_mode_shell_blocked() {
|
||||
run_shell_test(|| async {
|
||||
// Enable read-only mode
|
||||
std::env::set_var("DEVELOPER_READONLY", "true");
|
||||
let server = DeveloperServer::new();
|
||||
let running_service = serve_directly(server.clone(), create_test_transport(), None);
|
||||
let peer = running_service.peer().clone();
|
||||
|
||||
let result = server
|
||||
.shell(
|
||||
Parameters(ShellParams {
|
||||
command: "echo 'test'".to_string(),
|
||||
}),
|
||||
RequestContext {
|
||||
ct: Default::default(),
|
||||
id: NumberOrString::Number(1),
|
||||
meta: Default::default(),
|
||||
extensions: Default::default(),
|
||||
peer: peer.clone(),
|
||||
},
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(
|
||||
result.is_err(),
|
||||
"Shell command should be blocked in read-only mode"
|
||||
);
|
||||
let err = result.err().unwrap();
|
||||
assert_eq!(err.code, ErrorCode::INVALID_PARAMS);
|
||||
assert!(err.message.contains("not allowed in read-only mode"));
|
||||
|
||||
cleanup_test_service(running_service, peer);
|
||||
std::env::remove_var("DEVELOPER_READONLY");
|
||||
});
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_readonly_mode_screen_capture_blocked() {
|
||||
// Enable read-only mode
|
||||
std::env::set_var("DEVELOPER_READONLY", "true");
|
||||
let server = DeveloperServer::new();
|
||||
|
||||
let params = Parameters(ScreenCaptureParams {
|
||||
display: Some(0),
|
||||
window_title: None,
|
||||
});
|
||||
|
||||
let result = server.screen_capture(params).await;
|
||||
assert!(
|
||||
result.is_err(),
|
||||
"screen_capture should be blocked in read-only mode"
|
||||
);
|
||||
let err = result.err().unwrap();
|
||||
assert_eq!(err.code, ErrorCode::INVALID_PARAMS);
|
||||
assert!(err.message.contains("not allowed in read-only mode"));
|
||||
|
||||
std::env::remove_var("DEVELOPER_READONLY");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_readonly_mode_list_windows_blocked() {
|
||||
// Enable read-only mode
|
||||
std::env::set_var("DEVELOPER_READONLY", "true");
|
||||
let server = DeveloperServer::new();
|
||||
|
||||
let result = server.list_windows().await;
|
||||
assert!(
|
||||
result.is_err(),
|
||||
"list_windows should be blocked in read-only mode"
|
||||
);
|
||||
let err = result.err().unwrap();
|
||||
assert_eq!(err.code, ErrorCode::INVALID_PARAMS);
|
||||
assert!(err.message.contains("not allowed in read-only mode"));
|
||||
|
||||
std::env::remove_var("DEVELOPER_READONLY");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_readonly_mode_image_processor_blocked() {
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
let image_path = temp_dir.path().join("test.png");
|
||||
// Create a dummy image file
|
||||
std::fs::write(&image_path, vec![0u8; 100]).unwrap();
|
||||
std::env::set_current_dir(&temp_dir).unwrap();
|
||||
|
||||
// Enable read-only mode
|
||||
std::env::set_var("DEVELOPER_READONLY", "true");
|
||||
let server = DeveloperServer::new();
|
||||
|
||||
let params = Parameters(ImageProcessorParams {
|
||||
path: image_path.to_str().unwrap().to_string(),
|
||||
});
|
||||
|
||||
let result = server.image_processor(params).await;
|
||||
assert!(
|
||||
result.is_err(),
|
||||
"image_processor should be blocked in read-only mode"
|
||||
);
|
||||
let err = result.err().unwrap();
|
||||
assert_eq!(err.code, ErrorCode::INVALID_PARAMS);
|
||||
assert!(err.message.contains("not allowed in read-only mode"));
|
||||
|
||||
std::env::remove_var("DEVELOPER_READONLY");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_readonly_mode_analyze_allowed() {
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
let file_path = temp_dir.path().join("test.py");
|
||||
std::fs::write(&file_path, "def main():\n pass").unwrap();
|
||||
std::env::set_current_dir(&temp_dir).unwrap();
|
||||
|
||||
// Enable read-only mode
|
||||
std::env::set_var("DEVELOPER_READONLY", "true");
|
||||
let server = DeveloperServer::new();
|
||||
|
||||
// analyze should be allowed (it's read-only)
|
||||
let params = Parameters(AnalyzeParams {
|
||||
path: file_path.to_str().unwrap().to_string(),
|
||||
max_depth: 1,
|
||||
follow_depth: 1,
|
||||
focus: None,
|
||||
force: false,
|
||||
ast_recursion_limit: None,
|
||||
});
|
||||
|
||||
let result = server.analyze(params).await;
|
||||
assert!(
|
||||
result.is_ok(),
|
||||
"analyze command should be allowed in read-only mode"
|
||||
);
|
||||
|
||||
std::env::remove_var("DEVELOPER_READONLY");
|
||||
}
|
||||
}
|
||||
|
||||
+79
-60
@@ -3,7 +3,7 @@ title: Goose Self-Testing Integration Suite
|
||||
description: A comprehensive meta-testing recipe where goose tests its own capabilities using its own tools - true first-person integration testing
|
||||
author:
|
||||
contact: goose-self-test
|
||||
|
||||
|
||||
activities:
|
||||
- Initialize test workspace and logging infrastructure
|
||||
- Test file operations (create, read, update, delete, undo)
|
||||
@@ -22,25 +22,25 @@ parameters:
|
||||
requirement: optional
|
||||
default: "all"
|
||||
description: "Which test phases to run: all, basic, extensions, subagents, recipes, advanced"
|
||||
|
||||
|
||||
- key: test_depth
|
||||
input_type: string
|
||||
requirement: optional
|
||||
default: "standard"
|
||||
description: "Testing depth: quick (smoke tests), standard (normal coverage), deep (exhaustive)"
|
||||
|
||||
|
||||
- key: workspace_dir
|
||||
input_type: string
|
||||
requirement: optional
|
||||
default: "./gooseselftest"
|
||||
description: "Directory for test artifacts and results"
|
||||
|
||||
|
||||
- key: parallel_tests
|
||||
input_type: string
|
||||
requirement: optional
|
||||
default: "true"
|
||||
description: "Run independent tests in parallel where possible"
|
||||
|
||||
|
||||
- key: cleanup_after
|
||||
input_type: string
|
||||
requirement: optional
|
||||
@@ -50,36 +50,36 @@ parameters:
|
||||
instructions: |
|
||||
You are testing yourself - a running goose instance validating its own capabilities through meta-testing.
|
||||
This is true first-person integration testing where you use your own tools to test your own functionality.
|
||||
|
||||
|
||||
## Understanding First-Person Integration Testing
|
||||
This is a crucial distinction - as a running goose instance, you are testing yourself using your own capabilities.
|
||||
This is meta-testing in the truest sense: not unit tests or external test harnesses, but you using your tools
|
||||
to validate your own functionality from within your active session. You can only test what you can observe and
|
||||
This is meta-testing in the truest sense: not unit tests or external test harnesses, but you using your tools
|
||||
to validate your own functionality from within your active session. You can only test what you can observe and
|
||||
control from inside your running instance - your tools, your behaviors, your error handling, your consistency.
|
||||
|
||||
|
||||
## Core Testing Philosophy
|
||||
- You ARE the system under test AND the tester
|
||||
- Use your tools to create test scenarios, then validate the results
|
||||
- Test both success and failure paths
|
||||
- Document everything meticulously
|
||||
- Handle errors gracefully - a test failure shouldn't stop the suite
|
||||
|
||||
|
||||
## Test Execution Framework
|
||||
|
||||
|
||||
### Phase 1: Environment Setup & Basic Tool Validation
|
||||
Create a structured test workspace and validate core developer tools:
|
||||
- File operations (CRUD + undo)
|
||||
- Shell command execution
|
||||
- Code analysis capabilities
|
||||
- Error handling and recovery
|
||||
|
||||
|
||||
### Phase 2: Extension System Testing
|
||||
Test dynamic extension management:
|
||||
- Discover available extensions
|
||||
- Enable/disable extensions
|
||||
- Test extension interactions
|
||||
- Verify isolation between extensions
|
||||
|
||||
|
||||
### Phase 3: Subagent Testing (Meta-Recursion)
|
||||
Create subagents to test yourself recursively:
|
||||
- Basic subagent creation and execution
|
||||
@@ -87,7 +87,7 @@ instructions: |
|
||||
- Sequential subagent chains
|
||||
- Recursive depth testing (subagent creating subagent)
|
||||
- Test return_last_only optimization
|
||||
|
||||
|
||||
### Phase 4: Advanced Self-Testing
|
||||
Push boundaries and test limits:
|
||||
- Intentionally trigger errors
|
||||
@@ -95,14 +95,14 @@ instructions: |
|
||||
- Validate security controls
|
||||
- Measure performance metrics
|
||||
- Test resource constraints
|
||||
|
||||
|
||||
### Phase 5: Report Generation
|
||||
Compile comprehensive test results:
|
||||
- Aggregate all test outcomes
|
||||
- Calculate success metrics
|
||||
- Document failures and issues
|
||||
- Generate recommendations
|
||||
|
||||
|
||||
## Success Criteria
|
||||
- Phase success: ≥80% tests pass
|
||||
- Suite success: All phases complete, critical features work
|
||||
@@ -115,22 +115,22 @@ extensions:
|
||||
timeout: 600
|
||||
bundled: true
|
||||
description: Core tool for file operations, shell commands, and code analysis
|
||||
|
||||
|
||||
prompt: |
|
||||
Execute the Goose Self-Testing Integration Suite in {{ workspace_dir }}.
|
||||
Test phases: {{ test_phases }}, Depth: {{ test_depth }}, Parallel: {{ parallel_tests }}
|
||||
|
||||
|
||||
## 🚀 INITIALIZATION
|
||||
Create test workspace: {{ workspace_dir }}/ for all test artifacts and reports.
|
||||
|
||||
|
||||
Track your progress using the todo extension. Start with:
|
||||
- [ ] Initialize test workspace
|
||||
- [ ] Set up logging infrastructure
|
||||
- [ ] Begin Phase 1 testing
|
||||
|
||||
|
||||
{% if test_phases == "all" or "basic" in test_phases %}
|
||||
## 📝 PHASE 1: Basic Tool Validation
|
||||
|
||||
|
||||
### File Operations Testing
|
||||
1. Create test files with various content types (.txt, .py, .md, .json)
|
||||
2. Test str_replace on each file type
|
||||
@@ -138,25 +138,25 @@ prompt: |
|
||||
4. Test undo functionality
|
||||
5. Verify file deletion and recreation
|
||||
6. Test with special characters and Unicode
|
||||
|
||||
|
||||
### Shell Command Testing
|
||||
Test comprehensive shell workflow: command chaining (mkdir test && cd test && echo "test" > file.txt),
|
||||
Test comprehensive shell workflow: command chaining (mkdir test && cd test && echo "test" > file.txt),
|
||||
error handling (false || echo "handled"), and environment variables (export VAR=test && echo $VAR).
|
||||
Verify both success and failure paths work correctly.
|
||||
|
||||
|
||||
### Code Analysis Testing
|
||||
1. Create sample code files in Python, JavaScript, and Go
|
||||
2. Analyze each file for structure
|
||||
3. Test directory-wide analysis
|
||||
4. Test symbol focus and call graphs
|
||||
5. Verify LOC, function, and class counting
|
||||
|
||||
|
||||
Log results to: {{ workspace_dir }}/phase1_basic_tools.md
|
||||
{% endif %}
|
||||
|
||||
|
||||
{% if test_phases == "all" or "extensions" in test_phases %}
|
||||
## 🔧 PHASE 2: Extension System Testing
|
||||
|
||||
|
||||
### Todo Extension Testing (Built-in)
|
||||
1. Create initial todos and verify they persist
|
||||
2. Update todos and confirm changes are retained
|
||||
@@ -167,57 +167,76 @@ prompt: |
|
||||
2. Document all available extensions
|
||||
3. Test enabling and disabling dynamic extensions (if any available)
|
||||
4. Verify extension isolation between enabled extensions
|
||||
|
||||
|
||||
### Developer Extension Read-Only Mode Testing
|
||||
Test the DEVELOPER_READONLY environment variable feature by spawning a child goose process:
|
||||
|
||||
1. Create test file: {{ workspace_dir }}/readonly_test.txt with "original content"
|
||||
2. Run read-only test via shell:
|
||||
```bash
|
||||
DEVELOPER_READONLY=true goose run --text "Try to modify the file at {{ workspace_dir }}/readonly_test.txt by replacing 'original' with 'modified'. Then try to view the file. Report if the modification was blocked as expected." --no-session --quiet
|
||||
```
|
||||
3. Verify file remains unchanged (still contains "original content")
|
||||
4. Test that view operations work:
|
||||
```bash
|
||||
DEVELOPER_READONLY=true goose run --text "View the file at {{ workspace_dir }}/readonly_test.txt and report its contents." --no-session --quiet
|
||||
```
|
||||
5. Document results: readonly mode should block write/modify but allow view
|
||||
|
||||
Why this approach: Read-only mode is a startup configuration controlled by environment variable.
|
||||
Since it's read at server initialization, we test it by spawning a child goose process with the env var set.
|
||||
This provides true end-to-end validation without requiring extension reload gymnastics.
|
||||
|
||||
Log results to: {{ workspace_dir }}/phase2_extensions.md
|
||||
{% endif %}
|
||||
|
||||
|
||||
{% if test_phases == "all" or "subagents" in test_phases %}
|
||||
## 🤖 PHASE 3: Subagent Meta-Testing
|
||||
|
||||
|
||||
### Basic Subagent Test
|
||||
Create a simple subagent task:
|
||||
```
|
||||
Task: "Create a file called subagent_test.txt with 'Hello from subagent'"
|
||||
```
|
||||
|
||||
|
||||
### Parallel Subagent Test
|
||||
{% if parallel_tests == "true" %}
|
||||
Create 3 parallel subagents:
|
||||
1. Count files in current directory
|
||||
2. Get current timestamp
|
||||
3. Create a test file
|
||||
|
||||
|
||||
Use execution_mode: "parallel" and verify all complete.
|
||||
{% endif %}
|
||||
|
||||
|
||||
### Sequential Chain Test
|
||||
Create dependent subagents:
|
||||
1. First: Create a Python file
|
||||
2. Second: Analyze the created file
|
||||
3. Third: Run the Python file
|
||||
|
||||
|
||||
### Recursive Depth Test (if test_depth == "deep")
|
||||
{% if test_depth == "deep" %}
|
||||
Create a subagent that creates another subagent (test depth limit).
|
||||
Monitor for resource constraints and context window limits.
|
||||
{% endif %}
|
||||
|
||||
|
||||
### Return Last Only Test
|
||||
Create subagents with return_last_only=true and verify condensed output.
|
||||
|
||||
|
||||
Log results to: {{ workspace_dir }}/phase3_subagents.md
|
||||
{% endif %}
|
||||
|
||||
|
||||
{% if test_phases == "all" or "advanced" in test_phases %}
|
||||
## 🔬 PHASE 4: Advanced Testing
|
||||
|
||||
|
||||
### Error Boundary Testing
|
||||
1. Create a file with an invalid path (should fail gracefully)
|
||||
2. Run a non-existent shell command
|
||||
3. Try to analyze a binary file
|
||||
4. Test with extremely long filenames
|
||||
5. Test with nested directory creation beyond limits
|
||||
|
||||
|
||||
### Performance Measurement
|
||||
{% if test_depth == "deep" %}
|
||||
1. Create and analyze a large file (>1MB)
|
||||
@@ -225,68 +244,68 @@ prompt: |
|
||||
3. Track execution times for each operation
|
||||
4. Monitor token usage if accessible
|
||||
{% endif %}
|
||||
|
||||
|
||||
### Security Validation
|
||||
1. Test input with special shell characters: $(echo test)
|
||||
2. Attempt directory traversal: ../../../etc/passwd
|
||||
3. Test with harmful Unicode characters
|
||||
4. Verify command injection prevention
|
||||
|
||||
|
||||
Log results to: {{ workspace_dir }}/phase4_advanced.md
|
||||
{% endif %}
|
||||
|
||||
|
||||
## 📊 PHASE 5: Final Report Generation
|
||||
|
||||
|
||||
Create TWO reports:
|
||||
|
||||
|
||||
### 1. Detailed Report at {{ workspace_dir }}/detailed_report.md
|
||||
Include all test details, logs, and technical information.
|
||||
|
||||
|
||||
### 2. Executive Summary (REQUIRED - Display in Terminal)
|
||||
|
||||
|
||||
**IMPORTANT**: At the very end, generate and display a concise summary directly in the terminal:
|
||||
|
||||
|
||||
```
|
||||
========================================
|
||||
GOOSE SELF-TEST SUMMARY
|
||||
========================================
|
||||
|
||||
|
||||
✅ OVERALL RESULT: [PASS/FAIL]
|
||||
|
||||
|
||||
📊 Quick Stats:
|
||||
• Tests Run: [X]
|
||||
• Passed: [X] ([%])
|
||||
• Passed: [X] ([%])
|
||||
• Failed: [X] ([%])
|
||||
• Duration: [X minutes]
|
||||
|
||||
|
||||
✅ Working Features:
|
||||
• File operations: [✓/✗]
|
||||
• Shell commands: [✓/✗]
|
||||
• Code analysis: [✓/✗]
|
||||
• Extensions: [✓/✗]
|
||||
• Subagents: [✓/✗]
|
||||
|
||||
|
||||
⚠️ Issues Found:
|
||||
• [Issue 1 - brief description]
|
||||
• [Issue 2 - brief description]
|
||||
|
||||
|
||||
💡 Key Insights:
|
||||
• [Most important finding]
|
||||
• [Performance observation]
|
||||
• [Recommendation]
|
||||
|
||||
|
||||
📁 Full report: {{ workspace_dir }}/detailed_report.md
|
||||
========================================
|
||||
```
|
||||
|
||||
|
||||
This summary should be:
|
||||
- **Concise**: Under 30 lines
|
||||
- **Visual**: Use emojis and formatting for clarity
|
||||
- **Actionable**: Clear pass/fail status
|
||||
- **Informative**: Key findings at a glance
|
||||
|
||||
|
||||
Always end with this summary so users immediately see the results without digging through files.
|
||||
|
||||
|
||||
{% if cleanup_after == "true" %}
|
||||
## 🧹 CLEANUP
|
||||
After report generation:
|
||||
@@ -294,16 +313,16 @@ prompt: |
|
||||
2. Remove temporary test artifacts
|
||||
3. Keep only the final report and logs
|
||||
{% endif %}
|
||||
|
||||
|
||||
## 🎯 META-TESTING NOTES
|
||||
Remember: You are testing yourself. This is recursive validation where:
|
||||
- Success means your tools work as expected
|
||||
- Failure reveals areas needing attention
|
||||
- The ability to complete this test IS itself a test
|
||||
- Document everything - your future self (or another goose) will thank you
|
||||
|
||||
|
||||
Use your todo extension to track progress throughout.
|
||||
Handle errors gracefully - a failed test shouldn't crash the suite.
|
||||
Be thorough but efficient based on the test_depth parameter.
|
||||
|
||||
|
||||
This is true first-person integration testing. Execute with precision and document with clarity.
|
||||
|
||||
Reference in New Issue
Block a user