initial work on readonly developer mode

This commit is contained in:
Tyler Longwell
2025-10-15 15:37:56 -04:00
parent b23e1e6329
commit 124ef5a3ce
2 changed files with 514 additions and 60 deletions
@@ -167,9 +167,17 @@ fn load_prompt_files() -> HashMap<String, Prompt> {
prompts
}
/// Check if developer extension is in read-only mode
fn is_readonly_mode() -> bool {
std::env::var("DEVELOPER_READONLY")
.map(|v| v == "1" || v.to_lowercase() == "true")
.unwrap_or(false)
}
/// Developer MCP Server using official RMCP SDK
#[derive(Clone)]
pub struct DeveloperServer {
readonly: bool,
tool_router: ToolRouter<Self>,
file_history: Arc<Mutex<HashMap<PathBuf, Vec<String>>>>,
ignore_patterns: Gitignore,
@@ -558,7 +566,11 @@ impl DeveloperServer {
// Initialize editor model for AI-powered code editing
let editor_model = create_editor_model();
// Check if read-only mode is enabled
let readonly = is_readonly_mode();
Self {
readonly,
tool_router: Self::tool_router(),
file_history: Arc::new(Mutex::new(HashMap::new())),
ignore_patterns,
@@ -577,6 +589,15 @@ impl DeveloperServer {
description = "List all available window titles that can be used with screen_capture. Returns a list of window titles that can be used with the window_title parameter of the screen_capture tool."
)]
pub async fn list_windows(&self) -> Result<CallToolResult, ErrorData> {
// Check if in read-only mode
if self.readonly {
return Err(ErrorData::new(
ErrorCode::INVALID_PARAMS,
"list_windows is not allowed in read-only mode".to_string(),
None,
));
}
let windows = Window::all().map_err(|_| {
ErrorData::new(
ErrorCode::INTERNAL_ERROR,
@@ -612,6 +633,15 @@ impl DeveloperServer {
&self,
params: Parameters<ScreenCaptureParams>,
) -> Result<CallToolResult, ErrorData> {
// Check if in read-only mode
if self.readonly {
return Err(ErrorData::new(
ErrorCode::INVALID_PARAMS,
"screen_capture is not allowed in read-only mode".to_string(),
None,
));
}
let params = params.0;
let mut image = if let Some(window_title) = &params.window_title {
@@ -754,6 +784,14 @@ impl DeveloperServer {
Ok(CallToolResult::success(content))
}
"write" => {
// Check if in read-only mode
if self.readonly {
return Err(ErrorData::new(
ErrorCode::INVALID_PARAMS,
"Command 'write' is not allowed in read-only mode. Only 'view' is permitted.".to_string(),
None,
));
}
let file_text = params.file_text.ok_or_else(|| {
ErrorData::new(
ErrorCode::INVALID_PARAMS,
@@ -765,6 +803,14 @@ impl DeveloperServer {
Ok(CallToolResult::success(content))
}
"str_replace" => {
// Check if in read-only mode
if self.readonly {
return Err(ErrorData::new(
ErrorCode::INVALID_PARAMS,
"Command 'str_replace' is not allowed in read-only mode. Only 'view' is permitted.".to_string(),
None,
));
}
// Check if diff parameter is provided
if let Some(ref diff) = params.diff {
// When diff is provided, old_str and new_str are not required
@@ -807,6 +853,14 @@ impl DeveloperServer {
}
}
"insert" => {
// Check if in read-only mode
if self.readonly {
return Err(ErrorData::new(
ErrorCode::INVALID_PARAMS,
"Command 'insert' is not allowed in read-only mode. Only 'view' is permitted.".to_string(),
None,
));
}
let insert_line = params.insert_line.ok_or_else(|| {
ErrorData::new(
ErrorCode::INVALID_PARAMS,
@@ -827,6 +881,14 @@ impl DeveloperServer {
Ok(CallToolResult::success(content))
}
"undo_edit" => {
// Check if in read-only mode
if self.readonly {
return Err(ErrorData::new(
ErrorCode::INVALID_PARAMS,
"Command 'undo_edit' is not allowed in read-only mode. Only 'view' is permitted.".to_string(),
None,
));
}
let content = text_editor_undo(&path, &self.file_history).await?;
Ok(CallToolResult::success(content))
}
@@ -856,6 +918,15 @@ impl DeveloperServer {
params: Parameters<ShellParams>,
context: RequestContext<RoleServer>,
) -> Result<CallToolResult, ErrorData> {
// Check if in read-only mode
if self.readonly {
return Err(ErrorData::new(
ErrorCode::INVALID_PARAMS,
"shell is not allowed in read-only mode".to_string(),
None,
));
}
let params = params.0;
let command = &params.command;
let peer = context.peer;
@@ -1136,6 +1207,15 @@ impl DeveloperServer {
&self,
params: Parameters<ImageProcessorParams>,
) -> Result<CallToolResult, ErrorData> {
// Check if in read-only mode
if self.readonly {
return Err(ErrorData::new(
ErrorCode::INVALID_PARAMS,
"image_processor is not allowed in read-only mode".to_string(),
None,
));
}
let params = params.0;
let path_str = &params.path;
@@ -3863,4 +3943,359 @@ Additional instructions here.
cleanup_test_service(running_service, peer);
});
}
// Read-only mode tests
#[tokio::test]
#[serial]
async fn test_readonly_mode_initialization() {
// Test without environment variable
let server = create_test_server();
assert!(
!server.readonly,
"Should not be in read-only mode by default"
);
// Test with environment variable set to "true"
std::env::set_var("DEVELOPER_READONLY", "true");
let server = DeveloperServer::new();
assert!(
server.readonly,
"Should be in read-only mode when env var is 'true'"
);
std::env::remove_var("DEVELOPER_READONLY");
// Test with environment variable set to "1"
std::env::set_var("DEVELOPER_READONLY", "1");
let server = DeveloperServer::new();
assert!(
server.readonly,
"Should be in read-only mode when env var is '1'"
);
std::env::remove_var("DEVELOPER_READONLY");
// Test with environment variable set to "false"
std::env::set_var("DEVELOPER_READONLY", "false");
let server = DeveloperServer::new();
assert!(
!server.readonly,
"Should not be in read-only mode when env var is 'false'"
);
std::env::remove_var("DEVELOPER_READONLY");
}
#[tokio::test]
#[serial]
async fn test_readonly_mode_text_editor_view_allowed() {
let temp_dir = tempfile::tempdir().unwrap();
let file_path = temp_dir.path().join("test.txt");
std::fs::write(&file_path, "test content").unwrap();
std::env::set_current_dir(&temp_dir).unwrap();
// Enable read-only mode
std::env::set_var("DEVELOPER_READONLY", "true");
let server = DeveloperServer::new();
// View should be allowed
let view_params = Parameters(TextEditorParams {
path: file_path.to_str().unwrap().to_string(),
command: "view".to_string(),
view_range: None,
file_text: None,
old_str: None,
new_str: None,
insert_line: None,
diff: None,
});
let result = server.text_editor(view_params).await;
assert!(
result.is_ok(),
"View command should be allowed in read-only mode"
);
std::env::remove_var("DEVELOPER_READONLY");
}
#[tokio::test]
#[serial]
async fn test_readonly_mode_text_editor_write_blocked() {
let temp_dir = tempfile::tempdir().unwrap();
std::env::set_current_dir(&temp_dir).unwrap();
// Enable read-only mode
std::env::set_var("DEVELOPER_READONLY", "true");
let server = DeveloperServer::new();
// Write should be blocked
let write_params = Parameters(TextEditorParams {
path: "test.txt".to_string(),
command: "write".to_string(),
view_range: None,
file_text: Some("new content".to_string()),
old_str: None,
new_str: None,
insert_line: None,
diff: None,
});
let result = server.text_editor(write_params).await;
assert!(
result.is_err(),
"Write command should be blocked in read-only mode"
);
let err = result.err().unwrap();
assert_eq!(err.code, ErrorCode::INVALID_PARAMS);
assert!(err.message.contains("not allowed in read-only mode"));
std::env::remove_var("DEVELOPER_READONLY");
}
#[tokio::test]
#[serial]
async fn test_readonly_mode_text_editor_str_replace_blocked() {
let temp_dir = tempfile::tempdir().unwrap();
let file_path = temp_dir.path().join("test.txt");
std::fs::write(&file_path, "original content").unwrap();
std::env::set_current_dir(&temp_dir).unwrap();
// Enable read-only mode
std::env::set_var("DEVELOPER_READONLY", "true");
let server = DeveloperServer::new();
// str_replace should be blocked
let replace_params = Parameters(TextEditorParams {
path: file_path.to_str().unwrap().to_string(),
command: "str_replace".to_string(),
view_range: None,
file_text: None,
old_str: Some("original".to_string()),
new_str: Some("modified".to_string()),
insert_line: None,
diff: None,
});
let result = server.text_editor(replace_params).await;
assert!(
result.is_err(),
"str_replace command should be blocked in read-only mode"
);
let err = result.err().unwrap();
assert_eq!(err.code, ErrorCode::INVALID_PARAMS);
assert!(err.message.contains("not allowed in read-only mode"));
std::env::remove_var("DEVELOPER_READONLY");
}
#[tokio::test]
#[serial]
async fn test_readonly_mode_text_editor_insert_blocked() {
let temp_dir = tempfile::tempdir().unwrap();
let file_path = temp_dir.path().join("test.txt");
std::fs::write(&file_path, "line 1\nline 2").unwrap();
std::env::set_current_dir(&temp_dir).unwrap();
// Enable read-only mode
std::env::set_var("DEVELOPER_READONLY", "true");
let server = DeveloperServer::new();
// insert should be blocked
let insert_params = Parameters(TextEditorParams {
path: file_path.to_str().unwrap().to_string(),
command: "insert".to_string(),
view_range: None,
file_text: None,
old_str: None,
new_str: Some("new line".to_string()),
insert_line: Some(1),
diff: None,
});
let result = server.text_editor(insert_params).await;
assert!(
result.is_err(),
"insert command should be blocked in read-only mode"
);
let err = result.err().unwrap();
assert_eq!(err.code, ErrorCode::INVALID_PARAMS);
assert!(err.message.contains("not allowed in read-only mode"));
std::env::remove_var("DEVELOPER_READONLY");
}
#[tokio::test]
#[serial]
async fn test_readonly_mode_text_editor_undo_blocked() {
let temp_dir = tempfile::tempdir().unwrap();
let file_path = temp_dir.path().join("test.txt");
std::fs::write(&file_path, "content").unwrap();
std::env::set_current_dir(&temp_dir).unwrap();
// Enable read-only mode
std::env::set_var("DEVELOPER_READONLY", "true");
let server = DeveloperServer::new();
// undo_edit should be blocked
let undo_params = Parameters(TextEditorParams {
path: file_path.to_str().unwrap().to_string(),
command: "undo_edit".to_string(),
view_range: None,
file_text: None,
old_str: None,
new_str: None,
insert_line: None,
diff: None,
});
let result = server.text_editor(undo_params).await;
assert!(
result.is_err(),
"undo_edit command should be blocked in read-only mode"
);
let err = result.err().unwrap();
assert_eq!(err.code, ErrorCode::INVALID_PARAMS);
assert!(err.message.contains("not allowed in read-only mode"));
std::env::remove_var("DEVELOPER_READONLY");
}
#[test]
#[serial]
fn test_readonly_mode_shell_blocked() {
run_shell_test(|| async {
// Enable read-only mode
std::env::set_var("DEVELOPER_READONLY", "true");
let server = DeveloperServer::new();
let running_service = serve_directly(server.clone(), create_test_transport(), None);
let peer = running_service.peer().clone();
let result = server
.shell(
Parameters(ShellParams {
command: "echo 'test'".to_string(),
}),
RequestContext {
ct: Default::default(),
id: NumberOrString::Number(1),
meta: Default::default(),
extensions: Default::default(),
peer: peer.clone(),
},
)
.await;
assert!(
result.is_err(),
"Shell command should be blocked in read-only mode"
);
let err = result.err().unwrap();
assert_eq!(err.code, ErrorCode::INVALID_PARAMS);
assert!(err.message.contains("not allowed in read-only mode"));
cleanup_test_service(running_service, peer);
std::env::remove_var("DEVELOPER_READONLY");
});
}
#[tokio::test]
#[serial]
async fn test_readonly_mode_screen_capture_blocked() {
// Enable read-only mode
std::env::set_var("DEVELOPER_READONLY", "true");
let server = DeveloperServer::new();
let params = Parameters(ScreenCaptureParams {
display: Some(0),
window_title: None,
});
let result = server.screen_capture(params).await;
assert!(
result.is_err(),
"screen_capture should be blocked in read-only mode"
);
let err = result.err().unwrap();
assert_eq!(err.code, ErrorCode::INVALID_PARAMS);
assert!(err.message.contains("not allowed in read-only mode"));
std::env::remove_var("DEVELOPER_READONLY");
}
#[tokio::test]
#[serial]
async fn test_readonly_mode_list_windows_blocked() {
// Enable read-only mode
std::env::set_var("DEVELOPER_READONLY", "true");
let server = DeveloperServer::new();
let result = server.list_windows().await;
assert!(
result.is_err(),
"list_windows should be blocked in read-only mode"
);
let err = result.err().unwrap();
assert_eq!(err.code, ErrorCode::INVALID_PARAMS);
assert!(err.message.contains("not allowed in read-only mode"));
std::env::remove_var("DEVELOPER_READONLY");
}
#[tokio::test]
#[serial]
async fn test_readonly_mode_image_processor_blocked() {
let temp_dir = tempfile::tempdir().unwrap();
let image_path = temp_dir.path().join("test.png");
// Create a dummy image file
std::fs::write(&image_path, vec![0u8; 100]).unwrap();
std::env::set_current_dir(&temp_dir).unwrap();
// Enable read-only mode
std::env::set_var("DEVELOPER_READONLY", "true");
let server = DeveloperServer::new();
let params = Parameters(ImageProcessorParams {
path: image_path.to_str().unwrap().to_string(),
});
let result = server.image_processor(params).await;
assert!(
result.is_err(),
"image_processor should be blocked in read-only mode"
);
let err = result.err().unwrap();
assert_eq!(err.code, ErrorCode::INVALID_PARAMS);
assert!(err.message.contains("not allowed in read-only mode"));
std::env::remove_var("DEVELOPER_READONLY");
}
#[tokio::test]
#[serial]
async fn test_readonly_mode_analyze_allowed() {
let temp_dir = tempfile::tempdir().unwrap();
let file_path = temp_dir.path().join("test.py");
std::fs::write(&file_path, "def main():\n pass").unwrap();
std::env::set_current_dir(&temp_dir).unwrap();
// Enable read-only mode
std::env::set_var("DEVELOPER_READONLY", "true");
let server = DeveloperServer::new();
// analyze should be allowed (it's read-only)
let params = Parameters(AnalyzeParams {
path: file_path.to_str().unwrap().to_string(),
max_depth: 1,
follow_depth: 1,
focus: None,
force: false,
ast_recursion_limit: None,
});
let result = server.analyze(params).await;
assert!(
result.is_ok(),
"analyze command should be allowed in read-only mode"
);
std::env::remove_var("DEVELOPER_READONLY");
}
}
+79 -60
View File
@@ -3,7 +3,7 @@ title: Goose Self-Testing Integration Suite
description: A comprehensive meta-testing recipe where goose tests its own capabilities using its own tools - true first-person integration testing
author:
contact: goose-self-test
activities:
- Initialize test workspace and logging infrastructure
- Test file operations (create, read, update, delete, undo)
@@ -22,25 +22,25 @@ parameters:
requirement: optional
default: "all"
description: "Which test phases to run: all, basic, extensions, subagents, recipes, advanced"
- key: test_depth
input_type: string
requirement: optional
default: "standard"
description: "Testing depth: quick (smoke tests), standard (normal coverage), deep (exhaustive)"
- key: workspace_dir
input_type: string
requirement: optional
default: "./gooseselftest"
description: "Directory for test artifacts and results"
- key: parallel_tests
input_type: string
requirement: optional
default: "true"
description: "Run independent tests in parallel where possible"
- key: cleanup_after
input_type: string
requirement: optional
@@ -50,36 +50,36 @@ parameters:
instructions: |
You are testing yourself - a running goose instance validating its own capabilities through meta-testing.
This is true first-person integration testing where you use your own tools to test your own functionality.
## Understanding First-Person Integration Testing
This is a crucial distinction - as a running goose instance, you are testing yourself using your own capabilities.
This is meta-testing in the truest sense: not unit tests or external test harnesses, but you using your tools
to validate your own functionality from within your active session. You can only test what you can observe and
This is meta-testing in the truest sense: not unit tests or external test harnesses, but you using your tools
to validate your own functionality from within your active session. You can only test what you can observe and
control from inside your running instance - your tools, your behaviors, your error handling, your consistency.
## Core Testing Philosophy
- You ARE the system under test AND the tester
- Use your tools to create test scenarios, then validate the results
- Test both success and failure paths
- Document everything meticulously
- Handle errors gracefully - a test failure shouldn't stop the suite
## Test Execution Framework
### Phase 1: Environment Setup & Basic Tool Validation
Create a structured test workspace and validate core developer tools:
- File operations (CRUD + undo)
- Shell command execution
- Code analysis capabilities
- Error handling and recovery
### Phase 2: Extension System Testing
Test dynamic extension management:
- Discover available extensions
- Enable/disable extensions
- Test extension interactions
- Verify isolation between extensions
### Phase 3: Subagent Testing (Meta-Recursion)
Create subagents to test yourself recursively:
- Basic subagent creation and execution
@@ -87,7 +87,7 @@ instructions: |
- Sequential subagent chains
- Recursive depth testing (subagent creating subagent)
- Test return_last_only optimization
### Phase 4: Advanced Self-Testing
Push boundaries and test limits:
- Intentionally trigger errors
@@ -95,14 +95,14 @@ instructions: |
- Validate security controls
- Measure performance metrics
- Test resource constraints
### Phase 5: Report Generation
Compile comprehensive test results:
- Aggregate all test outcomes
- Calculate success metrics
- Document failures and issues
- Generate recommendations
## Success Criteria
- Phase success: ≥80% tests pass
- Suite success: All phases complete, critical features work
@@ -115,22 +115,22 @@ extensions:
timeout: 600
bundled: true
description: Core tool for file operations, shell commands, and code analysis
prompt: |
Execute the Goose Self-Testing Integration Suite in {{ workspace_dir }}.
Test phases: {{ test_phases }}, Depth: {{ test_depth }}, Parallel: {{ parallel_tests }}
## 🚀 INITIALIZATION
Create test workspace: {{ workspace_dir }}/ for all test artifacts and reports.
Track your progress using the todo extension. Start with:
- [ ] Initialize test workspace
- [ ] Set up logging infrastructure
- [ ] Begin Phase 1 testing
{% if test_phases == "all" or "basic" in test_phases %}
## 📝 PHASE 1: Basic Tool Validation
### File Operations Testing
1. Create test files with various content types (.txt, .py, .md, .json)
2. Test str_replace on each file type
@@ -138,25 +138,25 @@ prompt: |
4. Test undo functionality
5. Verify file deletion and recreation
6. Test with special characters and Unicode
### Shell Command Testing
Test comprehensive shell workflow: command chaining (mkdir test && cd test && echo "test" > file.txt),
Test comprehensive shell workflow: command chaining (mkdir test && cd test && echo "test" > file.txt),
error handling (false || echo "handled"), and environment variables (export VAR=test && echo $VAR).
Verify both success and failure paths work correctly.
### Code Analysis Testing
1. Create sample code files in Python, JavaScript, and Go
2. Analyze each file for structure
3. Test directory-wide analysis
4. Test symbol focus and call graphs
5. Verify LOC, function, and class counting
Log results to: {{ workspace_dir }}/phase1_basic_tools.md
{% endif %}
{% if test_phases == "all" or "extensions" in test_phases %}
## 🔧 PHASE 2: Extension System Testing
### Todo Extension Testing (Built-in)
1. Create initial todos and verify they persist
2. Update todos and confirm changes are retained
@@ -167,57 +167,76 @@ prompt: |
2. Document all available extensions
3. Test enabling and disabling dynamic extensions (if any available)
4. Verify extension isolation between enabled extensions
### Developer Extension Read-Only Mode Testing
Test the DEVELOPER_READONLY environment variable feature by spawning a child goose process:
1. Create test file: {{ workspace_dir }}/readonly_test.txt with "original content"
2. Run read-only test via shell:
```bash
DEVELOPER_READONLY=true goose run --text "Try to modify the file at {{ workspace_dir }}/readonly_test.txt by replacing 'original' with 'modified'. Then try to view the file. Report if the modification was blocked as expected." --no-session --quiet
```
3. Verify file remains unchanged (still contains "original content")
4. Test that view operations work:
```bash
DEVELOPER_READONLY=true goose run --text "View the file at {{ workspace_dir }}/readonly_test.txt and report its contents." --no-session --quiet
```
5. Document results: readonly mode should block write/modify but allow view
Why this approach: Read-only mode is a startup configuration controlled by environment variable.
Since it's read at server initialization, we test it by spawning a child goose process with the env var set.
This provides true end-to-end validation without requiring extension reload gymnastics.
Log results to: {{ workspace_dir }}/phase2_extensions.md
{% endif %}
{% if test_phases == "all" or "subagents" in test_phases %}
## 🤖 PHASE 3: Subagent Meta-Testing
### Basic Subagent Test
Create a simple subagent task:
```
Task: "Create a file called subagent_test.txt with 'Hello from subagent'"
```
### Parallel Subagent Test
{% if parallel_tests == "true" %}
Create 3 parallel subagents:
1. Count files in current directory
2. Get current timestamp
3. Create a test file
Use execution_mode: "parallel" and verify all complete.
{% endif %}
### Sequential Chain Test
Create dependent subagents:
1. First: Create a Python file
2. Second: Analyze the created file
3. Third: Run the Python file
### Recursive Depth Test (if test_depth == "deep")
{% if test_depth == "deep" %}
Create a subagent that creates another subagent (test depth limit).
Monitor for resource constraints and context window limits.
{% endif %}
### Return Last Only Test
Create subagents with return_last_only=true and verify condensed output.
Log results to: {{ workspace_dir }}/phase3_subagents.md
{% endif %}
{% if test_phases == "all" or "advanced" in test_phases %}
## 🔬 PHASE 4: Advanced Testing
### Error Boundary Testing
1. Create a file with an invalid path (should fail gracefully)
2. Run a non-existent shell command
3. Try to analyze a binary file
4. Test with extremely long filenames
5. Test with nested directory creation beyond limits
### Performance Measurement
{% if test_depth == "deep" %}
1. Create and analyze a large file (>1MB)
@@ -225,68 +244,68 @@ prompt: |
3. Track execution times for each operation
4. Monitor token usage if accessible
{% endif %}
### Security Validation
1. Test input with special shell characters: $(echo test)
2. Attempt directory traversal: ../../../etc/passwd
3. Test with harmful Unicode characters
4. Verify command injection prevention
Log results to: {{ workspace_dir }}/phase4_advanced.md
{% endif %}
## 📊 PHASE 5: Final Report Generation
Create TWO reports:
### 1. Detailed Report at {{ workspace_dir }}/detailed_report.md
Include all test details, logs, and technical information.
### 2. Executive Summary (REQUIRED - Display in Terminal)
**IMPORTANT**: At the very end, generate and display a concise summary directly in the terminal:
```
========================================
GOOSE SELF-TEST SUMMARY
========================================
✅ OVERALL RESULT: [PASS/FAIL]
📊 Quick Stats:
• Tests Run: [X]
• Passed: [X] ([%])
• Passed: [X] ([%])
• Failed: [X] ([%])
• Duration: [X minutes]
✅ Working Features:
• File operations: [✓/✗]
• Shell commands: [✓/✗]
• Code analysis: [✓/✗]
• Extensions: [✓/✗]
• Subagents: [✓/✗]
⚠️ Issues Found:
• [Issue 1 - brief description]
• [Issue 2 - brief description]
💡 Key Insights:
• [Most important finding]
• [Performance observation]
• [Recommendation]
📁 Full report: {{ workspace_dir }}/detailed_report.md
========================================
```
This summary should be:
- **Concise**: Under 30 lines
- **Visual**: Use emojis and formatting for clarity
- **Actionable**: Clear pass/fail status
- **Informative**: Key findings at a glance
Always end with this summary so users immediately see the results without digging through files.
{% if cleanup_after == "true" %}
## 🧹 CLEANUP
After report generation:
@@ -294,16 +313,16 @@ prompt: |
2. Remove temporary test artifacts
3. Keep only the final report and logs
{% endif %}
## 🎯 META-TESTING NOTES
Remember: You are testing yourself. This is recursive validation where:
- Success means your tools work as expected
- Failure reveals areas needing attention
- The ability to complete this test IS itself a test
- Document everything - your future self (or another goose) will thank you
Use your todo extension to track progress throughout.
Handle errors gracefully - a failed test shouldn't crash the suite.
Be thorough but efficient based on the test_depth parameter.
This is true first-person integration testing. Execute with precision and document with clarity.