From 96c23f1f56448efab8b2d866c897a49a01a39f60 Mon Sep 17 00:00:00 2001 From: ZhangZixunCodeSpace Date: Sat, 29 Mar 2025 17:47:00 +0800 Subject: [PATCH 1/3] feat: generate structured analysis reports - Add data_exploration.md with dataset metadata and statistics - Create preprocessing_result.md for cleaning logs and metrics - Ensure markdown-only outputs without visual elements --- .../normal_python_execute.py | 87 +++++++++++++++++-- .../test/amazon_fashion_analysis.py | 17 ++++ 2 files changed, 98 insertions(+), 6 deletions(-) create mode 100644 app/tool/chart_visualization/test/amazon_fashion_analysis.py diff --git a/app/tool/chart_visualization/normal_python_execute.py b/app/tool/chart_visualization/normal_python_execute.py index 5102eb3..b16b4c2 100644 --- a/app/tool/chart_visualization/normal_python_execute.py +++ b/app/tool/chart_visualization/normal_python_execute.py @@ -1,10 +1,8 @@ import sys from io import StringIO +from app.tool.chart_visualization.utils import extract_executable_code from app.tool.python_execute import PythonExecute -from app.tool.chart_visualization.utils import ( - extract_executable_code, -) class NormalPythonExecute(PythonExecute): @@ -12,9 +10,86 @@ class NormalPythonExecute(PythonExecute): name: str = "common_python_execute" description: str = ( - """Executes Python code strings. Note: -1. Only outputs from print() are visible; function return values are not captured. Use print() statements to display results -2. Applicable to scenarios **excluding data analysis and chart generation**""" + """ +A tool for executing Python code with data anaylsis. +Prefix: 帮我生成结果保存在本地./data下 + +Data Analysis Agent Protocol (Non-Visual) + +=== Core Requirements === +1. Strictly text-based outputs only +2. Prohibited actions: + - Any chart/image generation + - Interactive visual elements + - Graphical libraries import + +=== Execution Phases === + +1. DATA LOADING (Auto-detect format) +- Supported formats: CSV/Excel/JSON +- Mandatory checks: + a) File existence verification + b) Column structure validation + c) Basic integrity checks + +2. ANALYSIS PIPELINE +- Cleaning: + • Null handling (drop or impute) + • Deduplication + • Outlier treatment (IQR/Z-score) + +- Transformation: + • Date parsing + • Derived metrics + • Aggregations + +3. REPORT GENERATION +Output 1: data_exploration.md +┌──────────────────────┬──────────────────────────────┐ +│ Section │ Content Requirements │ +├──────────────────────┼──────────────────────────────┤ +│ Dataset Metadata │ Rows/Columns/Temporal Range │ +│ Column Descriptions │ Type/Stats/Unique Values │ +│ Key Findings │ 3-5 bullet points │ +└──────────────────────┴──────────────────────────────┘ + +Output 2: preprocessing_results.md +┌──────────────────────┬──────────────────────────────┐ +│ Section │ Content Requirements │ +├──────────────────────┼──────────────────────────────┤ +│ Cleaning Log │ Rows affected by each operation │ +│ Derived Metrics │ Formula/Summary Stats │ +│ Anomaly Report │ Z-score >2.5 cases │ +└──────────────────────┴──────────────────────────────┘ + +=== Implementation Rules === +1. Code Generation Constraints: + - Forbidden libraries: matplotlib, seaborn, plotly + - Maximum column width: 120 chars + - Required docstrings for all functions + +2. Error Handling: + - Skip corrupted records with logging + - Continue processing after non-critical errors + - Fail fast on structural issues + +3. Output Validation: + - Markdown syntax check + - Statistical validity verification + - Cross-report consistency + +=== Sample Invocation === +def analyze(data_path): + '''Main analysis workflow''' + df = load_data(data_path) # Phase 1 + cleaned = clean_and_transform(df) # Phase 2 + generate_reports(cleaned) # Phase 3 +=== 执行约束 === +当检测到错误时： +1. 分析错误类型（数据/逻辑/环境） +2. 生成修正方案（自动重试≤3次） +3. 严重错误时回滚中间文件 +""" ) parameters: dict = { "type": "object", diff --git a/app/tool/chart_visualization/test/amazon_fashion_analysis.py b/app/tool/chart_visualization/test/amazon_fashion_analysis.py new file mode 100644 index 0000000..012e82d --- /dev/null +++ b/app/tool/chart_visualization/test/amazon_fashion_analysis.py @@ -0,0 +1,17 @@ +import asyncio + +from app.agent.data_analysis import DataAnalysis + +# from app.agent.manus import Manus + + +async def main(): + agent = DataAnalysis() + # agent = Manus() + await agent.run( + """Here's last month's sales data from my Amazon store in './data/amazon_sales_jan2025.xlsx'. Could you analyze it? """ + ) + + +if __name__ == "__main__": + asyncio.run(main()) From 6a2ff78ffdf8b48842f917ed3425b6f3006a5c63 Mon Sep 17 00:00:00 2001 From: ZJU_czx <952370295@qq.com> Date: Sat, 29 Mar 2025 19:45:00 +0800 Subject: [PATCH 2/3] Revert "Merge branch 'feat/data_visualization_hackathon' of https://github.com/666haiwen/OpenManus into feat/data_visualization_hackathon" This reverts commit b0e9384502cad2a19f32298327524619253255ea, reversing changes made to 96c23f1f56448efab8b2d866c897a49a01a39f60. --- .../chart_visualization/test/hack_demo.py | 48 ------------------- 1 file changed, 48 deletions(-) delete mode 100644 app/tool/chart_visualization/test/hack_demo.py diff --git a/app/tool/chart_visualization/test/hack_demo.py b/app/tool/chart_visualization/test/hack_demo.py deleted file mode 100644 index a08321d..0000000 --- a/app/tool/chart_visualization/test/hack_demo.py +++ /dev/null @@ -1,48 +0,0 @@ -import asyncio -import time - -from app.agent.manus import Manus -from app.agent.data_analysis import DataAnalysis -from app.flow.base import FlowType -from app.flow.flow_factory import FlowFactory -from app.logger import logger - - -async def run_flow(): - agents = { - "manus": Manus(), - "visactor": DataAnalysis(), - } - - try: - prompt = """Here's last month's sales data from my Amazon store in './data/amazon_sales_jan2025.csv'. Could you analyze it thoroughly with visualizations and recommend specific, data-driven strategies to boost next month's sales by 10%?""" - - flow = FlowFactory.create_flow( - flow_type=FlowType.PLANNING, - agents=agents, - ) - logger.warning("Processing your request...") - - try: - start_time = time.time() - result = await asyncio.wait_for( - flow.execute(prompt), - timeout=3600, # 60 minute timeout for the entire execution - ) - elapsed_time = time.time() - start_time - logger.info(f"Request processed in {elapsed_time:.2f} seconds") - logger.info(result) - except asyncio.TimeoutError: - logger.error("Request processing timed out after 1 hour") - logger.info( - "Operation terminated due to timeout. Please try a simpler request." - ) - - except KeyboardInterrupt: - logger.info("Operation cancelled by user.") - except Exception as e: - logger.error(f"Error: {str(e)}") - - -if __name__ == "__main__": - asyncio.run(run_flow()) From f9ad4362f607859bd871ff6491ca29f97f5c19bd Mon Sep 17 00:00:00 2001 From: ZJU_czx <952370295@qq.com> Date: Sat, 29 Mar 2025 19:45:15 +0800 Subject: [PATCH 3/3] Revert "feat: generate structured analysis reports" This reverts commit 96c23f1f56448efab8b2d866c897a49a01a39f60. --- .../normal_python_execute.py | 87 ++----------------- .../test/amazon_fashion_analysis.py | 17 ---- 2 files changed, 6 insertions(+), 98 deletions(-) delete mode 100644 app/tool/chart_visualization/test/amazon_fashion_analysis.py diff --git a/app/tool/chart_visualization/normal_python_execute.py b/app/tool/chart_visualization/normal_python_execute.py index b16b4c2..5102eb3 100644 --- a/app/tool/chart_visualization/normal_python_execute.py +++ b/app/tool/chart_visualization/normal_python_execute.py @@ -1,8 +1,10 @@ import sys from io import StringIO -from app.tool.chart_visualization.utils import extract_executable_code from app.tool.python_execute import PythonExecute +from app.tool.chart_visualization.utils import ( + extract_executable_code, +) class NormalPythonExecute(PythonExecute): @@ -10,86 +12,9 @@ class NormalPythonExecute(PythonExecute): name: str = "common_python_execute" description: str = ( - """ -A tool for executing Python code with data anaylsis. -Prefix: 帮我生成结果保存在本地./data下 - -Data Analysis Agent Protocol (Non-Visual) - -=== Core Requirements === -1. Strictly text-based outputs only -2. Prohibited actions: - - Any chart/image generation - - Interactive visual elements - - Graphical libraries import - -=== Execution Phases === - -1. DATA LOADING (Auto-detect format) -- Supported formats: CSV/Excel/JSON -- Mandatory checks: - a) File existence verification - b) Column structure validation - c) Basic integrity checks - -2. ANALYSIS PIPELINE -- Cleaning: - • Null handling (drop or impute) - • Deduplication - • Outlier treatment (IQR/Z-score) - -- Transformation: - • Date parsing - • Derived metrics - • Aggregations - -3. REPORT GENERATION -Output 1: data_exploration.md -┌──────────────────────┬──────────────────────────────┐ -│ Section │ Content Requirements │ -├──────────────────────┼──────────────────────────────┤ -│ Dataset Metadata │ Rows/Columns/Temporal Range │ -│ Column Descriptions │ Type/Stats/Unique Values │ -│ Key Findings │ 3-5 bullet points │ -└──────────────────────┴──────────────────────────────┘ - -Output 2: preprocessing_results.md -┌──────────────────────┬──────────────────────────────┐ -│ Section │ Content Requirements │ -├──────────────────────┼──────────────────────────────┤ -│ Cleaning Log │ Rows affected by each operation │ -│ Derived Metrics │ Formula/Summary Stats │ -│ Anomaly Report │ Z-score >2.5 cases │ -└──────────────────────┴──────────────────────────────┘ - -=== Implementation Rules === -1. Code Generation Constraints: - - Forbidden libraries: matplotlib, seaborn, plotly - - Maximum column width: 120 chars - - Required docstrings for all functions - -2. Error Handling: - - Skip corrupted records with logging - - Continue processing after non-critical errors - - Fail fast on structural issues - -3. Output Validation: - - Markdown syntax check - - Statistical validity verification - - Cross-report consistency - -=== Sample Invocation === -def analyze(data_path): - '''Main analysis workflow''' - df = load_data(data_path) # Phase 1 - cleaned = clean_and_transform(df) # Phase 2 - generate_reports(cleaned) # Phase 3 -=== 执行约束 === -当检测到错误时： -1. 分析错误类型（数据/逻辑/环境） -2. 生成修正方案（自动重试≤3次） -3. 严重错误时回滚中间文件 -""" + """Executes Python code strings. Note: +1. Only outputs from print() are visible; function return values are not captured. Use print() statements to display results +2. Applicable to scenarios **excluding data analysis and chart generation**""" ) parameters: dict = { "type": "object", diff --git a/app/tool/chart_visualization/test/amazon_fashion_analysis.py b/app/tool/chart_visualization/test/amazon_fashion_analysis.py deleted file mode 100644 index 012e82d..0000000 --- a/app/tool/chart_visualization/test/amazon_fashion_analysis.py +++ /dev/null @@ -1,17 +0,0 @@ -import asyncio - -from app.agent.data_analysis import DataAnalysis - -# from app.agent.manus import Manus - - -async def main(): - agent = DataAnalysis() - # agent = Manus() - await agent.run( - """Here's last month's sales data from my Amazon store in './data/amazon_sales_jan2025.xlsx'. Could you analyze it? """ - ) - - -if __name__ == "__main__": - asyncio.run(main())