第一次提交
This commit is contained in:
523
crew_factory.py
Normal file
523
crew_factory.py
Normal file
@@ -0,0 +1,523 @@
|
||||
"""
|
||||
CrewAI 工厂模块
|
||||
负责根据输入动态创建 Crew 实例，并集成 SSE 事件推送
|
||||
|
||||
技术方案说明：
|
||||
由于 CrewAI 库的事件 API 可能随版本变化，我们采用更稳定的方案：
|
||||
1. 使用自定义的 Logger 拦截 Agent 输出
|
||||
2. 在任务执行前后手动发送 SSE 事件
|
||||
3. 通过 run_in_executor 实现同步转异步
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from typing import Dict, Any, Optional, List, Callable
|
||||
from datetime import datetime
|
||||
import uuid
|
||||
import io
|
||||
import sys
|
||||
from contextlib import contextmanager
|
||||
|
||||
from crewai import Agent, Task, Crew, Process
|
||||
|
||||
from agents_config import (
|
||||
create_agents,
|
||||
TASK_TEMPLATES,
|
||||
QWEN_MODEL_CONFIG,
|
||||
)
|
||||
from stream_manager import stream_manager, StreamEvent
|
||||
|
||||
|
||||
class CrewExecutionLogger:
|
||||
"""
|
||||
CrewAI 执行日志拦截器
|
||||
|
||||
通过捕获 stdout/stderr 来实时获取 CrewAI 的执行日志，
|
||||
并将其转发到 SSE 流。这是一种稳定且非侵入式的方法。
|
||||
"""
|
||||
|
||||
def __init__(self, task_id: str, callback: Optional[Callable] = None):
|
||||
self.task_id = task_id
|
||||
self.callback = callback
|
||||
self._old_stdout = None
|
||||
self._old_stderr = None
|
||||
self._buffer = io.StringIO()
|
||||
|
||||
def _write(self, text: str):
|
||||
"""写入时触发回调"""
|
||||
self._buffer.write(text)
|
||||
|
||||
# 按行处理（遇到换行符时发送）
|
||||
if '\n' in text:
|
||||
lines = self._buffer.getvalue().split('\n')
|
||||
for line in lines[:-1]: # 除了最后一行
|
||||
if line.strip():
|
||||
self._send_line(line)
|
||||
self._buffer = io.StringIO()
|
||||
self._buffer.write(lines[-1]) # 保留未完成的行
|
||||
|
||||
def _send_line(self, line: str):
|
||||
"""发送单行日志到 SSE 流"""
|
||||
if self.callback:
|
||||
self.callback("log", "System", line.strip())
|
||||
|
||||
def start_capture(self):
|
||||
"""开始捕获输出"""
|
||||
self._old_stdout = sys.stdout
|
||||
self._old_stderr = sys.stderr
|
||||
|
||||
# 创建代理对象
|
||||
class OutputProxy:
|
||||
def __init__(self, logger, original):
|
||||
self.logger = logger
|
||||
self.original = original
|
||||
|
||||
def write(self, text):
|
||||
self.logger._write(text)
|
||||
if self.original:
|
||||
self.original.write(text)
|
||||
|
||||
def flush(self):
|
||||
if self.original:
|
||||
self.original.flush()
|
||||
|
||||
sys.stdout = OutputProxy(self, self._old_stdout)
|
||||
sys.stderr = OutputProxy(self, self._old_stderr)
|
||||
|
||||
def stop_capture(self):
|
||||
"""停止捕获输出"""
|
||||
if self._old_stdout:
|
||||
sys.stdout = self._old_stdout
|
||||
if self._old_stderr:
|
||||
sys.stderr = self._old_stderr
|
||||
|
||||
# 发送剩余内容
|
||||
remaining = self._buffer.getvalue()
|
||||
if remaining.strip():
|
||||
self._send_line(remaining)
|
||||
|
||||
|
||||
class SSECrewExecutor:
|
||||
"""
|
||||
SSE Crew 执行器
|
||||
|
||||
封装 CrewAI 的执行过程，在关键节点发送 SSE 事件。
|
||||
这是与 CrewAI 版本无关的稳定方案。
|
||||
"""
|
||||
|
||||
def __init__(self, task_id: str):
|
||||
self.task_id = task_id
|
||||
self.logger = None
|
||||
|
||||
def _send_event(self, event_type: str, agent_name: str, content: str, metadata: Optional[Dict] = None):
|
||||
"""发送 SSE 事件（同步方法，使用线程安全的方式）"""
|
||||
try:
|
||||
loop = asyncio.get_event_loop()
|
||||
|
||||
async def send_async():
|
||||
stream = await stream_manager.get_stream(self.task_id)
|
||||
if stream is None:
|
||||
return False
|
||||
|
||||
event = StreamEvent(
|
||||
event_type=event_type,
|
||||
agent=agent_name,
|
||||
content=content,
|
||||
task_id=self.task_id,
|
||||
metadata=metadata or {}
|
||||
)
|
||||
return stream.put_nowait(event)
|
||||
|
||||
future = asyncio.run_coroutine_threadsafe(send_async(), loop)
|
||||
future.result(timeout=5.0)
|
||||
except Exception as e:
|
||||
# 静默失败，不影响主流程
|
||||
pass
|
||||
|
||||
def execute(self, crew: 'Crew', inputs: Dict[str, Any]) -> Any:
|
||||
"""
|
||||
执行 Crew 任务，并在过程中发送 SSE 事件
|
||||
|
||||
Args:
|
||||
crew: CrewAI Crew 实例
|
||||
inputs: 任务输入参数
|
||||
|
||||
Returns:
|
||||
Crew 执行结果
|
||||
"""
|
||||
# 1. 发送开始事件
|
||||
self._send_event(
|
||||
event_type="start",
|
||||
agent_name="System",
|
||||
content=f"Crew 任务开始执行，共 {len(crew.agents)} 个 Agent，{len(crew.tasks)} 个任务",
|
||||
metadata={
|
||||
"agent_count": len(crew.agents),
|
||||
"task_count": len(crew.tasks)
|
||||
}
|
||||
)
|
||||
|
||||
# 2. 设置日志拦截
|
||||
def log_callback(event_type: str, agent_name: str, content: str):
|
||||
self._send_event(event_type, agent_name, content)
|
||||
|
||||
self.logger = CrewExecutionLogger(self.task_id, log_callback)
|
||||
self.logger.start_capture()
|
||||
|
||||
try:
|
||||
# 3. 遍历执行任务（手动控制以便发送事件）
|
||||
context = inputs.copy()
|
||||
last_output = None
|
||||
|
||||
for i, task in enumerate(crew.tasks):
|
||||
# 将 context 字典转换为 JSON 字符串（CrewAI 要求）
|
||||
import json
|
||||
context_str = json.dumps(context, ensure_ascii=False)
|
||||
task_result = self._execute_task(task, context_str, context, i)
|
||||
|
||||
# 更新上下文（简单的字符串替换）
|
||||
if task_result:
|
||||
context[f"task_{i}_output"] = str(task_result)
|
||||
last_output = task_result
|
||||
|
||||
# 4. 发送结束事件
|
||||
self._send_event(
|
||||
event_type="end",
|
||||
agent_name="System",
|
||||
content="Crew 任务执行完成",
|
||||
metadata={"success": True}
|
||||
)
|
||||
|
||||
return last_output
|
||||
|
||||
except Exception as e:
|
||||
# 5. 发送错误事件
|
||||
self._send_event(
|
||||
event_type="error",
|
||||
agent_name="System",
|
||||
content=str(e),
|
||||
metadata={"error_type": type(e).__name__}
|
||||
)
|
||||
raise
|
||||
|
||||
finally:
|
||||
# 6. 恢复原始输出
|
||||
self.logger.stop_capture()
|
||||
|
||||
def _execute_task(self, task: 'Task', context_str: str, context_dict: Dict[str, Any], index: int) -> Optional[Any]:
|
||||
"""
|
||||
执行单个任务
|
||||
|
||||
Args:
|
||||
task: Task 实例
|
||||
context_str: 上下文信息（JSON 字符串格式，用于传递给 CrewAI）
|
||||
context_dict: 上下文信息（字典格式，用于变量替换）
|
||||
index: 任务索引
|
||||
|
||||
Returns:
|
||||
任务执行结果
|
||||
"""
|
||||
# 获取负责此任务的 agent
|
||||
agent_name = task.agent.role if task.agent else "Unknown"
|
||||
|
||||
# 1. 发送任务开始事件
|
||||
self._send_event(
|
||||
event_type="agent_start",
|
||||
agent_name=agent_name,
|
||||
content=f"[任务 {index + 1}] {agent_name} 开始执行任务",
|
||||
metadata={
|
||||
"task_index": index,
|
||||
"task_description": task.description[:200] if task.description else ""
|
||||
}
|
||||
)
|
||||
|
||||
# 2. 准备任务描述（替换上下文变量）
|
||||
description = self._prepare_task_description(task.description, context_dict)
|
||||
|
||||
# 3. 执行任务
|
||||
try:
|
||||
# 使用 CrewAI 的原生执行方式
|
||||
result = task.execute_sync(context=context_str)
|
||||
|
||||
# 4. 发送任务完成事件
|
||||
output_str = str(result)[:500] if result else "No output"
|
||||
self._send_event(
|
||||
event_type="output",
|
||||
agent_name=agent_name,
|
||||
content=f"任务完成，输出摘要：{output_str}",
|
||||
metadata={
|
||||
"output_length": len(str(result)) if result else 0
|
||||
}
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
# 5. 发送错误事件
|
||||
self._send_event(
|
||||
event_type="error",
|
||||
agent_name=agent_name,
|
||||
content=f"任务执行失败：{str(e)}",
|
||||
metadata={"error_type": type(e).__name__}
|
||||
)
|
||||
raise
|
||||
|
||||
def _prepare_task_description(self, description: str, context: Dict[str, Any]) -> str:
|
||||
"""
|
||||
准备任务描述，替换上下文变量
|
||||
|
||||
支持以下占位符格式：
|
||||
- {prd_output}: 产品需求文档输出
|
||||
- {qa_output}: QA 测试计划输出
|
||||
- {dev_output}: 技术方案输出
|
||||
"""
|
||||
if not description:
|
||||
return ""
|
||||
|
||||
result = description
|
||||
|
||||
# 尝试从上下文中获取输出
|
||||
prd_output = context.get('prd_content', context.get('task_0_output', ''))
|
||||
qa_output = context.get('qa_plan', context.get('task_1_output', ''))
|
||||
dev_output = context.get('dev_plan', context.get('task_2_output', ''))
|
||||
|
||||
# 替换占位符
|
||||
result = result.replace('{prd_output}', str(prd_output)[:3000])
|
||||
result = result.replace('{qa_plan}', str(qa_output)[:3000])
|
||||
result = result.replace('{qa_output}', str(qa_output)[:3000])
|
||||
result = result.replace('{dev_plan}', str(dev_output)[:3000])
|
||||
result = result.replace('{dev_output}', str(dev_output)[:3000])
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def configure_llm():
|
||||
"""配置 LLM 使用 Qwen3.5-flash (DashScope/阿里百炼)"""
|
||||
import os
|
||||
|
||||
# 检查 API Key 是否配置
|
||||
dashscope_api_key = os.getenv("DASHSCOPE_API_KEY")
|
||||
if not dashscope_api_key:
|
||||
raise ValueError(
|
||||
"DASHSCOPE_API_KEY 未配置，请设置环境变量或在 .env 文件中配置"
|
||||
)
|
||||
|
||||
# 使用 CrewAI 原生的 LLM 类（通过 OpenAI 兼容接口连接 DashScope）
|
||||
from crewai.llm import LLM
|
||||
|
||||
llm = LLM(
|
||||
model=QWEN_MODEL_CONFIG["model"], # qwen-plus (Qwen3.5-flash)
|
||||
api_key=dashscope_api_key,
|
||||
base_url=QWEN_MODEL_CONFIG["base_url"], # https://dashscope.aliyuncs.com/compatible-mode/v1
|
||||
temperature=0.7,
|
||||
max_tokens=4096,
|
||||
)
|
||||
|
||||
return llm
|
||||
|
||||
|
||||
class CrewFactory:
|
||||
"""Crew 工厂类 - 负责创建和执行业务流程"""
|
||||
|
||||
@staticmethod
|
||||
def create_crew(
|
||||
task_id: str,
|
||||
user_requirement: str,
|
||||
skip_confirmation: bool = True
|
||||
) -> tuple[Crew, Dict[str, Any]]:
|
||||
"""
|
||||
创建 Crew 实例
|
||||
|
||||
Args:
|
||||
task_id: 任务 ID
|
||||
user_requirement: 用户需求描述
|
||||
skip_confirmation: 是否跳过 Coordinator 的人工确认环节
|
||||
|
||||
Returns:
|
||||
(Crew 实例，上下文信息)
|
||||
"""
|
||||
# 创建 Agents
|
||||
agents = create_agents()
|
||||
pm_agent = agents["product_manager"]
|
||||
qa_agent = agents["qa_engineer"]
|
||||
dev_agent = agents["software_developer"]
|
||||
coord_agent = agents["coordinator"]
|
||||
|
||||
# 配置 LLM
|
||||
try:
|
||||
llm = configure_llm()
|
||||
# 为每个 agent 分配 LLM
|
||||
for agent in agents.values():
|
||||
agent.llm = llm
|
||||
except ValueError as e:
|
||||
# 如果 LLM 配置失败，使用默认配置（会在运行时失败）
|
||||
print(f"警告：LLM 配置失败 - {e}")
|
||||
|
||||
# 创建任务
|
||||
# Task 1: ProductManager 分析需求
|
||||
pm_task = Task(
|
||||
description=TASK_TEMPLATES["product_manager"].format(
|
||||
user_requirement=user_requirement
|
||||
),
|
||||
expected_output="完整的产品需求文档 (PRD)，包含功能列表、验收标准和风险评估",
|
||||
agent=pm_agent,
|
||||
)
|
||||
|
||||
# Task 2: QAEngineer 制定测试计划
|
||||
qa_task = Task(
|
||||
description=TASK_TEMPLATES["qa_engineer"].format(
|
||||
prd_content="{prd_output}"
|
||||
),
|
||||
expected_output="详细的测试计划文档，包含测试策略、测试用例和性能测试方案",
|
||||
agent=qa_agent,
|
||||
context=[pm_task],
|
||||
)
|
||||
|
||||
# Task 3: SoftwareDeveloper 设计技术方案
|
||||
dev_task = Task(
|
||||
description=TASK_TEMPLATES["software_developer"].format(
|
||||
prd_content="{prd_output}",
|
||||
qa_plan="{qa_output}"
|
||||
),
|
||||
expected_output="完整的技术方案文档，包含架构设计、API 接口、数据模型和核心代码实现",
|
||||
agent=dev_agent,
|
||||
context=[pm_task, qa_task],
|
||||
)
|
||||
|
||||
# Task 4: Coordinator 最终审核
|
||||
coord_task = Task(
|
||||
description=TASK_TEMPLATES["coordinator"].format(
|
||||
prd_content="{prd_output}",
|
||||
qa_plan="{qa_output}",
|
||||
dev_plan="{dev_output}"
|
||||
),
|
||||
expected_output="最终交付报告，包含质量评估、风险提示和交付结论",
|
||||
agent=coord_agent,
|
||||
context=[pm_task, qa_task, dev_task],
|
||||
)
|
||||
|
||||
# 创建 Crew
|
||||
crew = Crew(
|
||||
agents=list(agents.values()),
|
||||
tasks=[pm_task, qa_task, dev_task, coord_task],
|
||||
process=Process.sequential,
|
||||
verbose=True,
|
||||
)
|
||||
|
||||
# 上下文信息
|
||||
context = {
|
||||
"user_requirement": user_requirement,
|
||||
"skip_confirmation": skip_confirmation,
|
||||
"created_at": datetime.now().isoformat(),
|
||||
}
|
||||
|
||||
return crew, context
|
||||
|
||||
@staticmethod
|
||||
async def execute_crew_async(
|
||||
task_id: str,
|
||||
user_requirement: str,
|
||||
skip_confirmation: bool = True
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
异步执行 Crew 任务
|
||||
|
||||
Args:
|
||||
task_id: 任务 ID
|
||||
user_requirement: 用户需求描述
|
||||
skip_confirmation: 是否跳过人工确认
|
||||
|
||||
Returns:
|
||||
执行结果摘要
|
||||
"""
|
||||
# 创建流队列
|
||||
await stream_manager.create_stream(task_id)
|
||||
|
||||
# 发送开始事件
|
||||
await stream_manager.publish_event(
|
||||
task_id=task_id,
|
||||
event_type="start",
|
||||
agent="System",
|
||||
content="任务已启动，开始处理用户需求...",
|
||||
metadata={"user_requirement": user_requirement[:200]}
|
||||
)
|
||||
|
||||
try:
|
||||
# 创建 Crew
|
||||
crew, context = CrewFactory.create_crew(
|
||||
task_id=task_id,
|
||||
user_requirement=user_requirement,
|
||||
skip_confirmation=skip_confirmation
|
||||
)
|
||||
|
||||
# 创建 SSE 执行器
|
||||
executor = SSECrewExecutor(task_id)
|
||||
|
||||
# 在线程池中执行（避免阻塞事件循环）
|
||||
loop = asyncio.get_event_loop()
|
||||
result = await loop.run_in_executor(
|
||||
None,
|
||||
lambda: executor.execute(crew, context)
|
||||
)
|
||||
|
||||
# 发送完成事件
|
||||
await stream_manager.publish_event(
|
||||
task_id=task_id,
|
||||
event_type="end",
|
||||
agent="System",
|
||||
content="任务执行完成",
|
||||
metadata={"success": True}
|
||||
)
|
||||
|
||||
return {
|
||||
"task_id": task_id,
|
||||
"status": "completed",
|
||||
"result": str(result)[:5000] if result else None,
|
||||
"context": context,
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
# 发送错误事件
|
||||
await stream_manager.publish_event(
|
||||
task_id=task_id,
|
||||
event_type="error",
|
||||
agent="System",
|
||||
content=str(e),
|
||||
metadata={"error_type": type(e).__name__}
|
||||
)
|
||||
|
||||
await stream_manager.close_stream(task_id)
|
||||
|
||||
return {
|
||||
"task_id": task_id,
|
||||
"status": "failed",
|
||||
"error": str(e),
|
||||
"error_type": type(e).__name__,
|
||||
}
|
||||
|
||||
|
||||
# 便捷函数
|
||||
async def run_multi_agent_task(
|
||||
user_requirement: str,
|
||||
skip_confirmation: bool = True
|
||||
) -> str:
|
||||
"""
|
||||
运行多智能体任务的便捷函数
|
||||
|
||||
Args:
|
||||
user_requirement: 用户需求描述
|
||||
skip_confirmation: 是否跳过人工确认
|
||||
|
||||
Returns:
|
||||
task_id: 任务 ID（用于后续 SSE 流订阅）
|
||||
"""
|
||||
task_id = str(uuid.uuid4())
|
||||
|
||||
# 异步启动任务
|
||||
asyncio.create_task(
|
||||
CrewFactory.execute_crew_async(
|
||||
task_id=task_id,
|
||||
user_requirement=user_requirement,
|
||||
skip_confirmation=skip_confirmation
|
||||
)
|
||||
)
|
||||
|
||||
return task_id
|
||||
Reference in New Issue
Block a user