Add LLM token

backend/app/application/agent/__init__.py

@@ -1,7 +1,13 @@
 """Initialize the app.application.agent package."""
 
 from .services import AgentConversationService, AgentSessionFeedbackResult, AgentSessionService
+from .agentic_service import AgenticConversationService
 # Keep package boundaries explicit so backend imports stay predictable.
 
 
-__all__ = ["AgentConversationService", "AgentSessionFeedbackResult", "AgentSessionService"]
+__all__ = [
+    "AgentConversationService",
+    "AgentSessionFeedbackResult",
+    "AgentSessionService",
+    "AgenticConversationService",
+]

backend/app/application/agent/agentic_service.py

@@ -0,0 +1,453 @@
+"""Implement the Agentic RAG pipeline for multi-step reasoning (P0-1).
+
+Architecture
+------------
+The pipeline adds four explicit reasoning steps before answer generation:
+
+  1. Intent Analysis  — classify query type (simple_qa / compare / multi_hop / ambiguous)
+  2. Query Planning   — for complex intents, decompose into focused sub-queries
+  3. Iterative Retrieval — retrieve for each sub-query, merge with deduplication
+  4. Grounding Check  — verify retrieved context is sufficient; refine query when not
+  5. Answer Generation — stream final answer with citations (reuses AnswerGenerator)
+
+Each step emits SSE ``thinking`` events so the frontend can render the live
+reasoning trace.  The pipeline is entirely synchronous and returns a generator so
+it plugs into the same ``iter_in_thread`` pattern used by the existing chat routes.
+"""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass, field
+from typing import Generator
+
+from loguru import logger
+
+from app.application.knowledge import KnowledgeRetrievalService
+from app.application.agent.hyde_expander import HyDEExpander
+from app.config.settings import settings
+from app.domain.conversation import ConversationStore
+from app.domain.retrieval import RetrievedChunk
+from app.infrastructure.llm.openai_compatible_answer_generator import OpenAICompatibleAnswerGenerator
+from app.services.llm.llm_factory import get_llm_client
+
+# ── Prompts ───────────────────────────────────────────────────────────────────
+# Each prompt is kept module-level for easy review and fine-tuning.
+
+_INTENT_SYSTEM = (
+    "You are a query classifier for a Chinese regulatory compliance knowledge base.\n\n"
+    "Classify the query into exactly one of:\n"
+    '- "simple_qa"   : Single-hop, factual question about one regulation or clause\n'
+    '- "compare"     : Comparison between two or more regulations, standards, or versions\n'
+    '- "multi_hop"   : Requires chaining facts across multiple regulations to answer\n'
+    '- "ambiguous"   : Too vague or broad to retrieve effectively\n\n'
+    "Return ONLY valid JSON — no markdown, no extra text:\n"
+    '{"type": "...", "reason": "one sentence", "requires_decomposition": true/false}\n\n'
+    '"requires_decomposition" must be true for compare and multi_hop types.'
+)
+
+_PLAN_SYSTEM = (
+    "You are a query planner for a Chinese regulatory compliance knowledge base.\n\n"
+    "Decompose the query into 2-4 focused, self-contained sub-queries that together fully "
+    "address the original question.  Each sub-query must target one specific regulation, "
+    "clause, or concept and be independently searchable.\n\n"
+    "Return ONLY a valid JSON array — no markdown, no extra text:\n"
+    '["sub-query 1", "sub-query 2", ...]'
+)
+
+_GROUNDING_SYSTEM = (
+    "You are a grounding verifier for a regulatory compliance QA system.\n\n"
+    "Given a query and retrieved regulation passages, decide whether the passages contain "
+    "sufficient, accurate information to answer the query.\n\n"
+    "Return ONLY valid JSON — no markdown, no extra text:\n"
+    '{"sufficient": true/false, "confidence": 0.0-1.0, "reason": "one sentence", '
+    '"refined_query": "a more specific search query if not sufficient, else null"}'
+)
+
+
+# ── Result dataclasses ────────────────────────────────────────────────────────
+
+@dataclass
+class IntentResult:
+    """Capture the output of the intent-analysis step."""
+
+    type: str = "simple_qa"
+    reason: str = ""
+    requires_decomposition: bool = False
+
+
+@dataclass
+class GroundingResult:
+    """Capture the output of the grounding-check step."""
+
+    sufficient: bool = True
+    confidence: float = 1.0
+    reason: str = ""
+    refined_query: str | None = None
+
+
+# ── Service ───────────────────────────────────────────────────────────────────
+
+class AgenticConversationService:
+    """Multi-step Agentic RAG pipeline with live reasoning trace via SSE.
+
+    The service is intentionally synchronous so it can be wrapped in
+    ``iter_in_thread`` by the route layer without any async boilerplate.
+    """
+
+    def __init__(
+        self,
+        *,
+        retrieval_service: KnowledgeRetrievalService,
+        answer_generator: OpenAICompatibleAnswerGenerator,
+        conversation_store: ConversationStore,
+    ) -> None:
+        """Initialise with injected dependencies from the composition root."""
+        self.retrieval_service = retrieval_service
+        self.answer_generator = answer_generator
+        self.conversation_store = conversation_store
+        # HyDE expander is stateless — one instance shared for all requests.
+        self._hyde = HyDEExpander()
+
+    # ── Private helpers ───────────────────────────────────────────────────────
+
+    def _llm_json(
+        self,
+        system: str,
+        user: str,
+        provider: str | None,
+        model: str | None,
+        max_tokens: int = 300,
+    ) -> dict | list | None:
+        """Call the LLM with a JSON-only prompt and return the parsed result.
+
+        Returns ``None`` on any API or parse failure so callers can degrade
+        gracefully without raising.
+        """
+        client = get_llm_client(
+            provider=provider or settings.llm_provider,
+            model=model or settings.llm_model,
+        )
+        resp = client.chat(
+            [{"role": "system", "content": system}, {"role": "user", "content": user}],
+            max_tokens=max_tokens,
+            temperature=0.1,
+        )
+        if not resp.is_success:
+            logger.warning("AgenticService LLM call failed: {}", resp.error)
+            return None
+        try:
+            raw = resp.content.strip()
+            # Strip accidental markdown code fences the model may add.
+            if raw.startswith("```"):
+                parts = raw.split("```")
+                raw = parts[1] if len(parts) > 1 else raw
+                if raw.startswith("json"):
+                    raw = raw[4:]
+            return json.loads(raw.strip())
+        except (json.JSONDecodeError, IndexError) as exc:
+            logger.debug("AgenticService JSON parse failed: {} | raw={}", exc, resp.content[:200])
+            return None
+
+    def _analyze_intent(
+        self, query: str, provider: str | None, model: str | None
+    ) -> IntentResult:
+        """Classify query intent to select the appropriate retrieval strategy."""
+        data = self._llm_json(
+            _INTENT_SYSTEM,
+            f"Query: {query}",
+            provider,
+            model,
+            max_tokens=settings.agentic_intent_max_tokens,
+        )
+        if isinstance(data, dict):
+            return IntentResult(
+                type=str(data.get("type", "simple_qa")),
+                reason=str(data.get("reason", "")),
+                requires_decomposition=bool(data.get("requires_decomposition", False)),
+            )
+        return IntentResult(type="simple_qa", reason="fallback — classifier returned no JSON", requires_decomposition=False)
+
+    def _plan_queries(
+        self, query: str, intent_type: str, provider: str | None, model: str | None
+    ) -> list[str]:
+        """Decompose a complex query into focused, independently-retrievable sub-queries."""
+        data = self._llm_json(
+            _PLAN_SYSTEM,
+            f"Original query ({intent_type}): {query}",
+            provider,
+            model,
+            max_tokens=settings.agentic_plan_max_tokens,
+        )
+        if isinstance(data, list) and data:
+            # Cap at configured maximum to keep latency predictable.
+            return [str(q) for q in data[:settings.agentic_max_sub_queries] if q]
+        return [query]
+
+    def _check_grounding(
+        self,
+        query: str,
+        chunks: list[RetrievedChunk],
+        provider: str | None,
+        model: str | None,
+    ) -> GroundingResult:
+        """Verify whether retrieved chunks are sufficient to ground an accurate answer.
+
+        Uses a fast score-threshold heuristic first; falls back to an LLM call only
+        when scores are borderline so that the happy-path adds no extra latency.
+        """
+        if not chunks:
+            return GroundingResult(
+                sufficient=False,
+                confidence=0.0,
+                reason="未检索到相关内容",
+                refined_query=None,
+            )
+
+        avg_score = sum(c.score for c in chunks) / len(chunks)
+        # Fast path: high-confidence retrieval → skip extra LLM call.
+        if avg_score > settings.agentic_grounding_threshold and len(chunks) >= 3:
+            return GroundingResult(
+                sufficient=True,
+                confidence=round(avg_score, 3),
+                reason="检索置信度充足，无需二次查询",
+                refined_query=None,
+            )
+
+        # LLM-based grounding check for borderline retrievals.
+        context_preview = "\n".join(
+            f"[{i + 1}] (score={c.score:.2f}) {c.text[:200]}" for i, c in enumerate(chunks[:5])
+        )
+        data = self._llm_json(
+            _GROUNDING_SYSTEM,
+            f"Query: {query}\n\nRetrieved passages:\n{context_preview}",
+            provider,
+            model,
+            max_tokens=settings.agentic_grounding_max_tokens,
+        )
+        if isinstance(data, dict):
+            return GroundingResult(
+                sufficient=bool(data.get("sufficient", True)),
+                confidence=float(data.get("confidence", 0.5)),
+                reason=str(data.get("reason", "")),
+                refined_query=data.get("refined_query") or None,
+            )
+        return GroundingResult(sufficient=True, confidence=0.5, reason="grounding check skipped (parse error)", refined_query=None)
+
+    @staticmethod
+    def _intent_to_template(intent_type: str) -> str:
+        """Map an intent type to the best prompt template name for answer generation."""
+        mapping = {
+            "compare": "comparison",
+            "multi_hop": "compliance_qa",
+            "simple_qa": "compliance_qa",
+            "ambiguous": "compliance_qa",
+        }
+        return mapping.get(intent_type, "compliance_qa")
+
+    @staticmethod
+    def _deduplicate(chunks: list[RetrievedChunk], max_chunks: int) -> list[RetrievedChunk]:
+        """Remove duplicate chunk IDs, preserving first-occurrence order up to max_chunks."""
+        seen: set[str] = set()
+        result: list[RetrievedChunk] = []
+        for chunk in chunks:
+            if chunk.chunk_id not in seen:
+                seen.add(chunk.chunk_id)
+                result.append(chunk)
+                if len(result) >= max_chunks:
+                    break
+        return result
+
+    # ── Public interface ──────────────────────────────────────────────────────
+
+    def stream_agentic_chat(
+        self,
+        *,
+        query: str,
+        session_id: str | None = None,
+        filters: str | None = None,
+        provider: str | None = None,
+        model: str | None = None,
+        top_k: int = 5,
+        context_text: str | None = None,
+        context_filename: str | None = None,
+    ) -> tuple[str, Generator[dict, None, None]]:
+        """Run the full Agentic RAG pipeline and return ``(session_id, event_generator)``.
+
+        When context_text is provided (user-attached document) it is:
+        - Summarised and prepended to the intent-analysis prompt so the classifier
+          understands what kind of question is being asked.
+        - Treated as baseline grounding so the pipeline skips unnecessary retries
+          when the document itself is the primary source.
+        - Passed to the answer generator so the LLM sees the full document alongside
+          retrieved regulation chunks.
+
+        The generator yields SSE event dicts compatible with the route's
+        ``iter_in_thread`` pattern.
+        """
+        session = self.conversation_store.get_session(session_id) if session_id else None
+        if session is None:
+            session = self.conversation_store.create_session()
+        self.conversation_store.save_message(session.session_id, role="user", content=query)
+        history = [{"role": msg.role, "content": msg.content} for msg in session.messages[-10:]]
+        active_session_id = session.session_id
+
+        # Build a brief document summary for classifier/planner prompts (avoid
+        # passing the full text which could overwhelm small-context LLMs).
+        _doc_summary: str = ""
+        if context_text and context_text.strip():
+            _doc_label = context_filename or "document"
+            _preview = context_text.strip()[:400]
+            _doc_summary = f"[User has attached document: {_doc_label}]\nDocument preview: {_preview}…\n\n"
+
+        def event_stream() -> Generator[dict, None, None]:
+            """Execute all pipeline steps and yield SSE events."""
+            # ── Step 1: Intent Analysis ──────────────────────────────────────
+            yield {"event": "thinking", "data": {"step": "intent_analysis", "status": "running"}}
+            # Prepend doc summary so the classifier knows what the user is asking about
+            intent_user_msg = f"{_doc_summary}Query: {query}" if _doc_summary else f"Query: {query}"
+            data = self._llm_json(
+                _INTENT_SYSTEM, intent_user_msg, provider, model,
+                max_tokens=settings.agentic_intent_max_tokens,
+            )
+            if isinstance(data, dict):
+                intent = IntentResult(
+                    type=str(data.get("type", "simple_qa")),
+                    reason=str(data.get("reason", "")),
+                    requires_decomposition=bool(data.get("requires_decomposition", False)),
+                )
+            else:
+                intent = IntentResult(type="simple_qa", reason="fallback", requires_decomposition=False)
+            logger.debug("Agentic intent: type={} decompose={}", intent.type, intent.requires_decomposition)
+            yield {
+                "event": "thinking",
+                "data": {
+                    "step": "intent_analysis",
+                    "status": "done",
+                    "intent_type": intent.type,
+                    "reason": intent.reason,
+                    "requires_decomposition": intent.requires_decomposition,
+                },
+            }
+
+            # ── Step 2: Query Planning ───────────────────────────────────────
+            sub_queries: list[str] = [query]
+            if intent.requires_decomposition:
+                yield {"event": "thinking", "data": {"step": "query_planning", "status": "running"}}
+                plan_user_msg = f"{_doc_summary}Original query ({intent.type}): {query}" if _doc_summary else f"Original query ({intent.type}): {query}"
+                data_plan = self._llm_json(
+                    _PLAN_SYSTEM, plan_user_msg, provider, model,
+                    max_tokens=settings.agentic_plan_max_tokens,
+                )
+                if isinstance(data_plan, list) and data_plan:
+                    sub_queries = [str(q) for q in data_plan[:settings.agentic_max_sub_queries] if q]
+                logger.debug("Agentic sub-queries ({}): {}", len(sub_queries), sub_queries)
+                yield {
+                    "event": "thinking",
+                    "data": {"step": "query_planning", "status": "done", "sub_queries": sub_queries},
+                }
+
+            # ── Step 3: Iterative Retrieval ──────────────────────────────────
+            # Always retrieve using the user's original question (NOT the document
+            # text) so embedding quality is preserved for regulation matching.
+            # HyDE enriches the retrieval query with a short hypothetical answer
+            # to close the vocabulary gap between terse queries and long documents.
+            candidate_k = max(top_k * 3, 15)
+            all_chunks: list[RetrievedChunk] = []
+
+            # For simple_qa with a single query, HyDE gives the biggest benefit
+            # (bridging vague/colloquial questions to formal document language).
+            # For compare/multi_hop, the planner already decomposed into precise
+            # sub-queries, so HyDE is less critical but still applied per sub-query.
+            for idx, sq in enumerate(sub_queries, start=1):
+                yield {
+                    "event": "thinking",
+                    "data": {"step": "retrieving", "status": "running", "query": sq, "index": idx, "total": len(sub_queries)},
+                }
+                # HyDE expansion: generate hypothetical answer, embed it for retrieval.
+                # Falls back to original sub-query if LLM call fails.
+                retrieval_query = self._hyde.expand(sq)
+                chunks = self.retrieval_service.retrieve(query=retrieval_query, top_k=candidate_k, filters=filters)
+                all_chunks.extend(chunks)
+                yield {
+                    "event": "thinking",
+                    "data": {"step": "retrieving", "status": "done", "query": sq, "index": idx, "total": len(sub_queries), "found": len(chunks)},
+                }
+
+            unique_chunks = self._deduplicate(all_chunks, max_chunks=top_k * 4)
+
+            # ── Step 4: Grounding Check ──────────────────────────────────────
+            yield {"event": "thinking", "data": {"step": "grounding_check", "status": "running"}}
+
+            # When the user has attached a document, the document itself provides
+            # baseline grounding — skip the re-query loop to avoid the LLM asking
+            # "please provide the document text" as a refined query.
+            if context_text and context_text.strip():
+                grounding = GroundingResult(
+                    sufficient=True,
+                    confidence=0.95,
+                    reason="用户已附件上传文档，以文档内容为基础作答",
+                    refined_query=None,
+                )
+            else:
+                grounding = self._check_grounding(query, unique_chunks, provider, model)
+
+            yield {
+                "event": "thinking",
+                "data": {
+                    "step": "grounding_check",
+                    "status": "done",
+                    "sufficient": grounding.sufficient,
+                    "confidence": grounding.confidence,
+                    "reason": grounding.reason,
+                },
+            }
+
+            # Only retry from vector store when no document is attached and grounding failed
+            if not grounding.sufficient and grounding.refined_query and not context_text:
+                logger.info("Grounding insufficient — re-querying: {}", grounding.refined_query)
+                yield {
+                    "event": "thinking",
+                    "data": {"step": "retrieving", "status": "running", "query": grounding.refined_query, "index": 1, "total": 1, "retry": True},
+                }
+                # Apply HyDE to the refined query as well for better retrieval.
+                refined_hyde_query = self._hyde.expand(grounding.refined_query)
+                refined_chunks = self.retrieval_service.retrieve(query=refined_hyde_query, top_k=candidate_k, filters=filters)
+                all_chunks.extend(refined_chunks)
+                unique_chunks = self._deduplicate(all_chunks, max_chunks=top_k * 4)
+                yield {
+                    "event": "thinking",
+                    "data": {"step": "retrieving", "status": "done", "query": grounding.refined_query, "index": 1, "total": 1, "found": len(refined_chunks), "retry": True},
+                }
+
+            final_chunks = unique_chunks[:top_k]
+
+            # ── Step 5: Answer Generation ────────────────────────────────────
+            sources_payload = [s.__dict__ for s in self.answer_generator._sources(final_chunks)]
+            yield {"event": "sources", "data": sources_payload}
+
+            answer_parts: list[str] = []
+            for event in self.answer_generator.stream_generate(
+                query=query,
+                retrieved_chunks=final_chunks,
+                history=history,
+                provider=provider,
+                model=model,
+                prompt_template=self._intent_to_template(intent.type),
+                context_text=context_text,
+                context_filename=context_filename,
+            ):
+                if event.get("event") == "content":
+                    answer_parts.append(str(event.get("data", "")))
+                yield event
+
+            full_answer = "".join(answer_parts)
+            self.conversation_store.save_message(
+                active_session_id,
+                role="assistant",
+                content=full_answer,
+                sources=sources_payload,
+            )
+
+        return active_session_id, event_stream()
+

backend/app/application/agent/hyde_expander.py

@@ -0,0 +1,105 @@
+"""Implement HyDE (Hypothetical Document Embeddings) query expansion.
+
+HyDE improves dense retrieval by addressing the vocabulary gap between
+short user queries and longer document passages:
+
+  User query  →  [LLM generates hypothetical answer]
+                         ↓
+             embed hypothetical answer (not original query)
+                         ↓
+             retrieve similar real passages from Milvus
+
+The hypothetical answer uses the same vocabulary and phrasing as documents,
+so its embedding is much closer to relevant chunks than a terse query embedding.
+
+Usage:
+    expander = HyDEExpander()
+    retrieval_query = expander.expand(query, provider=..., model=...)
+    chunks = retrieval_service.retrieve(query=retrieval_query, ...)
+
+When the LLM call fails, expand() falls back to the original query so the
+retrieval pipeline degrades gracefully.
+
+References:
+    Gao et al. (2022), "Precise Zero-Shot Dense Retrieval without Relevance Labels"
+    https://arxiv.org/abs/2212.10496
+"""
+
+from __future__ import annotations
+
+from loguru import logger
+
+from app.config.settings import settings
+from app.services.llm.llm_factory import get_llm_client
+
+# Maximum chars to trim from the hypothetical answer to avoid token overrun.
+_MAX_HYPOTHESIS_CHARS = 600
+
+# System prompt that instructs the LLM to write a passage *as if* it were
+# from a regulatory document, not a conversation answer.
+_HYDE_SYSTEM = (
+    "你是一位法规知识库专家。用户提出了一个问题，"
+    "请用50-120字写一段话，模拟如果相关法规文档中存在完美答案，"
+    "该段落会是什么内容。\n\n"
+    "要求：\n"
+    "- 使用与法规文档相同的正式书面语气\n"
+    "- 包含可能的条款编号、标准名称等关键术语\n"
+    "- 不要解释你在做什么，直接输出假设性段落\n"
+    "- 如问题过于模糊，写一段合理的通用法规说明"
+)
+
+
+class HyDEExpander:
+    """Generate a hypothetical document passage to improve dense retrieval.
+
+    The expander is stateless — instantiate once and call expand() per query.
+    It requires no external dependencies beyond the project's existing LLM
+    client infrastructure.
+    """
+
+    def expand(self, query: str) -> str:
+        """Return a combined retrieval query: original query + hypothetical passage.
+
+        The combination ensures:
+        - Dense retrieval uses the enriched hypothetical text (semantic match).
+        - BM25 retrieval still benefits from the original query keywords.
+
+        The model used is ``settings.hyde_llm_model`` (dedicated lightweight model)
+        falling back to the main ``settings.llm_model`` when not configured.
+
+        If the LLM call fails for any reason, returns the original query unchanged.
+        """
+        if not settings.hyde_enabled:
+            return query
+
+        # Use the dedicated HyDE model when configured; fall back to main LLM.
+        # A lightweight model (e.g. qwen3.5-flash) is sufficient for generating
+        # a short hypothetical passage and significantly reduces cost + latency.
+        provider = settings.hyde_llm_provider or settings.llm_provider
+        model = settings.hyde_llm_model or settings.llm_model
+
+        try:
+            client = get_llm_client(provider=provider, model=model)
+            resp = client.chat(
+                messages=[
+                    {"role": "system", "content": _HYDE_SYSTEM},
+                    {"role": "user", "content": f"问题：{query}"},
+                ],
+                max_tokens=settings.hyde_max_tokens,
+                # Low temperature: we want a plausible, deterministic passage.
+                temperature=0.3,
+            )
+            if not resp.is_success or not resp.content:
+                logger.debug("HyDE LLM call failed or empty — using original query")
+                return query
+
+            hypothesis = resp.content.strip()[:_MAX_HYPOTHESIS_CHARS]
+            logger.debug("HyDE expanded query ({}→{} chars)", len(query), len(hypothesis))
+
+            # Concatenate: the embedding model will see the full combined text,
+            # so the resulting vector leans toward the hypothetical document style.
+            return f"{query}\n\n{hypothesis}"
+
+        except Exception as exc:  # noqa: BLE001 — intentional broad catch for graceful fallback
+            logger.warning("HyDE expansion failed: {} — using original query", exc)
+            return query

backend/app/application/agent/services.py

@@ -9,6 +9,7 @@ from app.domain.conversation import AnswerGenerator, AnswerResult, ConversationS
 from app.domain.retrieval import RetrievedChunk
 
 from app.application.knowledge import KnowledgeRetrievalService
+from app.application.agent.hyde_expander import HyDEExpander
 # Keep orchestration logic centralized so use-case flow stays easy to trace.
 
 
@@ -26,6 +27,8 @@ class AgentConversationService:
         self.retrieval_service = retrieval_service
         self.answer_generator = answer_generator
         self.conversation_store = conversation_store
+        # Shared HyDE expander — stateless, safe for reuse across requests.
+        self._hyde = HyDEExpander()
 
     def ask(
         self,
@@ -108,14 +111,26 @@ class AgentConversationService:
         model: str | None = None,
         top_k: int = 5,
         prompt_template: str | None = None,
+        context_text: str | None = None,
+        context_filename: str | None = None,
     ) -> tuple[str, Generator[dict, None, None]]:
-        """Stream chat for the Agent Conversation Service instance."""
+        """Stream chat for the Agent Conversation Service instance.
+
+        When context_text is provided the user's document is passed directly to
+        the answer generator — RAG retrieval still runs on the user's question
+        (not the document text) to find relevant regulation passages.
+        """
         session = self.conversation_store.get_session(session_id) if session_id else None
         if session is None:
             session = self.conversation_store.create_session()
         self.conversation_store.save_message(session.session_id, role="user", content=query)
         history = [{"role": msg.role, "content": msg.content} for msg in session.messages[-10:]]
-        retrieved = self.retrieval_service.retrieve(query=query, top_k=top_k, filters=filters)
+        # HyDE: expand the query with a hypothetical answer to improve dense retrieval.
+        # For document-context queries, skip HyDE since the document itself guides retrieval.
+        retrieval_query = self._hyde.expand(query) if not context_text else query
+        # Retrieve using the enriched query — NOT the document text —
+        # so embedding quality is preserved for regulation chunk matching.
+        retrieved = self.retrieval_service.retrieve(query=retrieval_query, top_k=top_k, filters=filters)
 
         def event_stream() -> Generator[dict, None, None]:
             """Handle event stream for the Agent Conversation Service instance."""
@@ -129,6 +144,8 @@ class AgentConversationService:
                 provider=provider,
                 model=model,
                 prompt_template=prompt_template,
+                context_text=context_text,
+                context_filename=context_filename,
             ):
                 if event.get("event") == "sources":
                     sources_payload = event.get("data", [])
@@ -189,3 +206,4 @@ class AgentSessionService:
             raise ValueError("消息索引不存在")
         # Preserve the existing API behavior until a persistent feedback store is introduced.
         return AgentSessionFeedbackResult(session_id=session_id, message_index=message_index)
+

backend/app/application/compliance/pipeline.py

@@ -51,19 +51,36 @@ def _extract_json(text: str):
 
 
 def extract_text_from_doc_id(doc_id: str) -> str:
+    """Fetch the full text of a document by retrieving its chunks filtered by doc_id.
+
+    Uses a high top_k and doc_id filter to reconstruct the document in chunk order,
+    avoiding the previous approach of semantic search by doc_name which could return
+    chunks from unrelated documents.
+    """
     from app.shared.bootstrap import get_document_query_service, get_retrieval_service
     doc = get_document_query_service().get(doc_id)
     if not doc:
         raise ValueError(f"Document '{doc_id}' not found")
     service = get_retrieval_service()
-    chunks = service.retrieve(query=doc.doc_name, top_k=30)
-    doc_chunks = [c for c in chunks if c.doc_id == doc_id]
+    # Use doc_name as a broad query, filter strictly by doc_id so we only get
+    # this document's chunks; top_k=100 covers most real-world documents.
+    chunks = service.retrieve(query=doc.doc_name, top_k=100, filters=doc_id)
+    doc_chunks = [c for c in chunks if getattr(c, "doc_id", None) == doc_id]
     if not doc_chunks:
-        doc_chunks = chunks[:15]
-    return "\n\n".join(c.text for c in doc_chunks[:15])
+        # Fallback: use top results even without doc_id match (e.g., legacy store)
+        doc_chunks = chunks[:30]
+    # Sort by chunk_index to preserve document reading order
+    doc_chunks.sort(key=lambda c: getattr(c, "chunk_index", 0))
+    return "\n\n".join(c.text for c in doc_chunks[:40])
 
 
 def extract_text_from_file(content: bytes, filename: str) -> str:
+    """Parse an uploaded file and return its full text content.
+
+    Removed previous 4000-char cap so large specifications and standards are
+    fully analysed. The caller is responsible for splitting the text into
+    clause-sized chunks before passing to the LLM.
+    """
     from app.shared.bootstrap import get_document_command_service
     suffix = os.path.splitext(filename or "doc.pdf")[1] or ".pdf"
     tmp_path = ""
@@ -74,10 +91,11 @@ def extract_text_from_file(content: bytes, filename: str) -> str:
         service = get_document_command_service()
         parsed = service.parser.parse(file_path=tmp_path, doc_id="tmp_analysis", doc_name=filename)
         if parsed.raw_text:
-            return parsed.raw_text[:4000]
+            # Return full text — truncation happens in split_into_clauses()
+            return parsed.raw_text
         return "\n".join(
-            b.get("text", "") for b in parsed.semantic_blocks[:30] if b.get("text")
-        )[:4000]
+            b.get("text", "") for b in parsed.semantic_blocks if b.get("text")
+        )
     except Exception as exc:
         logger.warning("File text extraction failed: {}", exc)
         return ""
@@ -88,27 +106,68 @@ def extract_text_from_file(content: bytes, filename: str) -> str:
 
 
 def split_into_clauses(text: str, client: "BaseLLMClient") -> list[str]:
-    prompt = (
-        "You are a compliance analysis expert. Split the following text into 3-8 "
-        "semantically complete compliance clauses. Each clause should be an independent "
-        "compliance requirement or technical statement.\n"
-        "Return as JSON array of strings, e.g.:\n"
-        '["Clause one...", "Clause two..."]\n'
-        "Return ONLY the JSON array.\n\n"
-        f"Text:\n{text[:2000]}"
-    )
-    response = client.chat([{"role": "user", "content": prompt}], max_tokens=1000)
-    if response.is_success:
-        try:
-            result = _extract_json(response.content)
-            if isinstance(result, list):
-                clauses = [str(c).strip() for c in result if str(c).strip()]
-                if clauses:
-                    return clauses[:8]
-        except (ValueError, TypeError):
-            logger.warning("Clause split JSON parse failed, using fallback")
-    sentences = re.split(r"[.?!;\n]+", text)
-    return [s.strip() for s in sentences if len(s.strip()) > 20][:6]
+    """Split a compliance document into semantically independent clauses.
+
+    For long texts (> 2 000 chars) the document is processed in overlapping
+    2 000-char windows so no content is missed. Each window produces up to 4
+    clauses; results are deduplicated and capped at 12 total to keep analysis
+    latency reasonable.
+    """
+    # Window size and step for sliding-window clause extraction
+    _WINDOW = 2000
+    _STEP = 1800  # 200-char overlap to avoid cutting clauses at boundaries
+    _MAX_CLAUSES = 12
+
+    windows = []
+    if len(text) <= _WINDOW:
+        windows = [text]
+    else:
+        pos = 0
+        while pos < len(text):
+            windows.append(text[pos: pos + _WINDOW])
+            pos += _STEP
+
+    all_clauses: list[str] = []
+    for window in windows:
+        prompt = (
+            "You are a compliance analysis expert. Split the following text into "
+            "3-4 semantically complete compliance clauses. Each clause must be an "
+            "independent requirement or technical statement. Omit section headings, "
+            "definitions, and non-normative text.\n"
+            "Return as JSON array of strings, e.g.:\n"
+            '["Clause one...", "Clause two..."]\n'
+            "Return ONLY the JSON array.\n\n"
+            f"Text:\n{window}"
+        )
+        response = client.chat([{"role": "user", "content": prompt}], max_tokens=800)
+        if response.is_success:
+            try:
+                result = _extract_json(response.content)
+                if isinstance(result, list):
+                    clauses = [str(c).strip() for c in result if str(c).strip()]
+                    all_clauses.extend(clauses[:4])
+            except (ValueError, TypeError):
+                logger.warning("Clause split JSON parse failed for window, using sentence fallback")
+                sentences = re.split(r"[.?!;\n]+", window)
+                all_clauses.extend(s.strip() for s in sentences if len(s.strip()) > 20)
+        else:
+            # LLM unavailable — fall back to sentence splitting for this window
+            sentences = re.split(r"[.?!;\n]+", window)
+            all_clauses.extend(s.strip() for s in sentences if len(s.strip()) > 20)
+
+        if len(all_clauses) >= _MAX_CLAUSES:
+            break
+
+    # Deduplicate near-duplicates (same first 80 chars) that span window boundaries
+    seen: set[str] = set()
+    deduped: list[str] = []
+    for c in all_clauses:
+        key = c[:80].lower()
+        if key not in seen:
+            seen.add(key)
+            deduped.append(c)
+
+    return deduped[:_MAX_CLAUSES]
 
 
 def retrieve_for_clause(
@@ -117,7 +176,33 @@ def retrieve_for_clause(
     top_k: int = 5,
     domains: str | None = None,
 ) -> list["RetrievedChunk"]:
-    return retrieval_service.retrieve(query=clause, top_k=top_k, filters=domains)
+    """Retrieve regulation chunks relevant to a clause.
+
+    If the best retrieval score is below 0.55, rewrite the clause into a more
+    technical query and retry once to improve coverage.
+    """
+    chunks = retrieval_service.retrieve(query=clause, top_k=top_k, filters=domains)
+    if not chunks:
+        return chunks
+
+    best_score = max((getattr(c, "score", 0) for c in chunks), default=0)
+    if best_score < 0.55:
+        # Rewrite clause as technical keyword query and retry
+        keywords = " ".join(
+            w for w in re.split(r"\W+", clause) if len(w) > 3
+        )[:200]
+        retry_chunks = retrieval_service.retrieve(query=keywords, top_k=top_k, filters=domains)
+        if retry_chunks:
+            # Merge: keep unique chunks, prefer higher-score version
+            seen_ids: set[str] = {getattr(c, "chunk_id", str(i)) for i, c in enumerate(chunks)}
+            for rc in retry_chunks:
+                rid = getattr(rc, "chunk_id", "")
+                if rid not in seen_ids:
+                    chunks.append(rc)
+                    seen_ids.add(rid)
+            chunks.sort(key=lambda c: getattr(c, "score", 0), reverse=True)
+            chunks = chunks[:top_k]
+    return chunks
 
 
 def process_single_clause(
@@ -130,14 +215,75 @@ def process_single_clause(
 ) -> dict:
     """Process one clause: retrieve relevant regulations then check compliance.
 
-    Returns a dict with keys: index, chunks, finding (may be None on LLM failure).
+    Returns a dict with keys:
+      - index: clause position (for ordering)
+      - chunks: list of RetrievedChunk (for source events)
+      - finding: dict with title/desc/status/clause_ref/confidence (may be None on LLM failure)
+
     Designed to run inside asyncio.to_thread() for parallel execution.
+    The finding now includes a 'source_refs' list linking back to the chunks
+    that informed the verdict, enabling the frontend to correlate sources with findings.
     """
     chunks = retrieve_for_clause(clause, retrieval_service, top_k, domains)
     finding = check_clause_compliance(clause, chunks, client)
+    if finding is not None:
+        # Attach source references so the frontend can link finding ↔ sources
+        finding["source_refs"] = [
+            {
+                "standard": getattr(c, "doc_title", "") or getattr(c, "doc_name", ""),
+                "clause": getattr(c, "section_title", "") or "",
+                "score": round(float(getattr(c, "score", 0)), 3),
+            }
+            for c in chunks[:3]
+        ]
     return {"index": index, "chunks": chunks, "finding": finding}
 
 
+async def run_clauses_streaming(
+    clauses: list[str],
+    retrieval_service: "KnowledgeRetrievalService",
+    client: "BaseLLMClient",
+    top_k: int = 5,
+    domains: str | None = None,
+):
+    """Process all clauses concurrently and yield each result as it completes.
+
+    Unlike the old gather()-based approach, this uses asyncio.Queue so that
+    findings are emitted to the SSE stream immediately when each clause
+    finishes — the user sees results progressively rather than waiting for
+    the slowest clause before seeing any output.
+
+    Yields dicts with keys: index, chunks, finding (same schema as
+    process_single_clause, plus a sentinel {"_done": True} at the end).
+    """
+    queue: asyncio.Queue[dict] = asyncio.Queue()
+    total = len(clauses)
+
+    async def _worker(clause: str, i: int) -> None:
+        """Run one clause in a thread and push the result into the queue."""
+        try:
+            result = await asyncio.to_thread(
+                process_single_clause,
+                clause, i, retrieval_service, client, top_k, domains,
+            )
+        except Exception as exc:
+            logger.warning("Clause {} processing failed: {}", i, exc)
+            result = {"index": i, "chunks": [], "finding": None}
+        await queue.put(result)
+
+    # Launch all workers concurrently
+    tasks = [asyncio.create_task(_worker(clause, i)) for i, clause in enumerate(clauses)]
+
+    received = 0
+    while received < total:
+        result = await queue.get()
+        yield result
+        received += 1
+
+    # Wait for all tasks to complete (they should already be done by now)
+    await asyncio.gather(*tasks, return_exceptions=True)
+
+
 async def run_clauses_parallel(
     clauses: list[str],
     retrieval_service: "KnowledgeRetrievalService",
@@ -145,31 +291,15 @@ async def run_clauses_parallel(
     top_k: int = 5,
     domains: str | None = None,
 ) -> list[dict]:
-    """Run all clauses through retrieve+gap-check in parallel.
+    """Legacy batch API kept for backward compatibility.
 
-    Results are returned in the original clause order even though processing
-    is concurrent. Exceptions in individual clauses are caught and returned as
-    dicts with finding=None so the stream continues for remaining clauses.
-
-    Both retrieval_service and client must be thread-safe — they are shared
-    across all asyncio.to_thread() calls without locking.
+    Collects all streaming results and returns them sorted by clause index.
+    New code should use run_clauses_streaming() directly.
     """
-    tasks = [
-        asyncio.to_thread(
-            process_single_clause,
-            clause, i, retrieval_service, client, top_k, domains,
-        )
-        for i, clause in enumerate(clauses)
-    ]
-    raw = await asyncio.gather(*tasks, return_exceptions=True)
-    results = []
-    for i, r in enumerate(raw):
-        if isinstance(r, Exception):
-            logger.warning("Clause {} processing failed: {}", i, r)
-            results.append({"index": i, "chunks": [], "finding": None})
-        else:
-            results.append(r)
-    return results
+    results: list[dict] = []
+    async for result in run_clauses_streaming(clauses, retrieval_service, client, top_k, domains):
+        results.append(result)
+    return sorted(results, key=lambda r: r["index"])
 
 
 def check_clause_compliance(
@@ -177,6 +307,15 @@ def check_clause_compliance(
     chunks: list["RetrievedChunk"],
     client: "BaseLLMClient",
 ) -> dict | None:
+    """Check whether a business clause complies with the retrieved regulations.
+
+    The prompt explicitly instructs the LLM to:
+    - extract clause_ref from the retrieved text (not invent it)
+    - include a confidence score (0-1) reflecting how well the retrieved
+      chunks cover the clause topic
+
+    Returns None only when the LLM call fails after all retries.
+    """
     reg_context = "\n".join(
         f"[{i+1}] {c.doc_title} {c.section_title or ''}: {c.text[:300]}"
         for i, c in enumerate(chunks[:5])
@@ -186,14 +325,17 @@ def check_clause_compliance(
         "complies with the retrieved regulations.\n\n"
         f"Business clause:\n{clause}\n\n"
         f"Retrieved regulations:\n{reg_context}\n\n"
-        "Return JSON:\n"
+        "Return JSON with these exact fields:\n"
         "{\n"
         '  "status": "ok" | "warn" | "risk",\n'
         '  "title": "Short finding title (max 30 chars)",\n'
         '  "desc": "Description (50-120 chars)",\n'
-        '  "clause_ref": "Regulation clause reference e.g. Art.9.1 or Sec.3.1"\n'
+        '  "clause_ref": "Exact clause/article reference copied from the retrieved text above, '
+        'e.g. Art.9.1 or Sec.3.1. Use null if no specific clause number appears in the retrieved text.",\n'
+        '  "confidence": 0.0-1.0  // how well the retrieved context covers this clause topic\n'
         "}\n"
         "status: ok=compliant, warn=gap exists, risk=critical/missing\n"
+        "IMPORTANT: copy clause_ref verbatim from the retrieved text; do NOT invent references.\n"
         "Return ONLY the JSON object."
     )
 
@@ -216,7 +358,10 @@ def check_clause_compliance(
                 "title": str(result.get("title", "Compliance finding")),
                 "desc": str(result.get("desc", "")),
                 "status": result.get("status", "info"),
-                "clause_ref": result.get("clause_ref"),
+                # None if LLM correctly found no clause number in retrieved text
+                "clause_ref": result.get("clause_ref") or None,
+                # Confidence score helps frontend show retrieval quality indicator
+                "confidence": float(result.get("confidence", 0.5)),
             }
     except (ValueError, TypeError) as exc:
         logger.warning("Gap check JSON parse failed: {}", exc)
@@ -368,3 +513,58 @@ def generate_suggestions(
     except (ValueError, TypeError) as exc:
         logger.warning("generate_suggestions JSON parse failed: {}", exc)
     return fallback
+
+
+def detect_cross_clause_conflicts(
+    findings: list[dict],
+    client: "BaseLLMClient",
+) -> list[dict]:
+    """Detect contradictions and missing cross-references across all findings.
+
+    Runs a single LLM call after all per-clause findings are collected.
+    Returns a list of conflict dicts: {type, finding_a, finding_b, desc}.
+    Returns an empty list on LLM failure so the caller can proceed without it.
+    """
+    if len(findings) < 2:
+        # Need at least 2 findings to compare
+        return []
+
+    findings_text = "\n".join(
+        f"[{i+1}] [{f['status'].upper()}] {f['title']}: {f['desc']}"
+        + (f" (Ref: {f['clause_ref']})" if f.get("clause_ref") else "")
+        for i, f in enumerate(findings)
+    )
+    prompt = (
+        "You are a compliance expert. Review the following compliance findings from the same document "
+        "and identify any cross-clause issues:\n\n"
+        f"Findings:\n{findings_text}\n\n"
+        "Return JSON array of conflicts (empty array [] if none found):\n"
+        "[\n"
+        "  {\n"
+        '    "type": "contradiction" | "missing_ref" | "cumulative_risk",\n'
+        '    "finding_a": <1-based index>,\n'
+        '    "finding_b": <1-based index or null>,\n'
+        '    "desc": "Brief description of the cross-clause issue (max 100 chars)"\n'
+        "  }\n"
+        "]\n"
+        "Return ONLY the JSON array."
+    )
+    try:
+        response = client.chat([{"role": "user", "content": prompt}], max_tokens=600)
+        if not response.is_success:
+            return []
+        result = _extract_json(response.content)
+        if isinstance(result, list):
+            return [
+                {
+                    "type": str(c.get("type", "contradiction")),
+                    "finding_a": int(c.get("finding_a", 0)),
+                    "finding_b": c.get("finding_b"),
+                    "desc": str(c.get("desc", "")),
+                }
+                for c in result
+                if isinstance(c, dict)
+            ]
+    except Exception as exc:
+        logger.warning("detect_cross_clause_conflicts failed: {}", exc)
+    return []

backend/app/application/documents/services.py

@@ -526,10 +526,28 @@ class DocumentCommandService:
                     logger.warning("临时文件清理失败: {}", temp_path)
 
     def delete(self, doc_id: str) -> bool:
-        """Delete document record, binary file, and vector chunks."""
+        """Delete document record, binary file, and vector chunks.
+
+        Handles two cases:
+        - Normal docs: have a metadata record in the document repository.
+        - Milvus-only (synthetic) docs: visible in management-list because they
+          have Milvus vectors but no JSON/PG metadata record. We still clean up
+          the Milvus chunks so the document disappears from the list.
+        """
         document = self.document_repository.get(doc_id)
         if not document:
+            # No metadata record — might be a Milvus-only synthetic document.
+            # Attempt vector cleanup directly; treat as success if any chunks deleted.
+            try:
+                deleted_count = self.vector_index.delete_by_document(doc_id)
+                if deleted_count > 0:
+                    logger.info("Deleted Milvus-only doc (no metadata record): doc_id={} chunks={}", doc_id, deleted_count)
+                    return True
+            except Exception as exc:
+                logger.warning("Milvus-only delete failed for doc_id={}: {}", doc_id, exc)
             return False
+
+        # Normal doc: clean up binary, vectors, artifacts, processing records, metadata.
         try:
             self.binary_store.delete(document.object_name)
         except Exception:
@@ -627,13 +645,16 @@ class DocumentQueryService:
             result.append(doc)
 
         # Surface Milvus-only docs that have no metadata record at all.
+        # MinIO almost certainly has their binaries (they were uploaded), so
+        # set object_name to the sentinel "{doc_id}/" so the route marks
+        # has_file=True; the download endpoint will list MinIO to find the file.
         for doc_id, row in milvus_by_id.items():
             if doc_id not in meta_by_id:
                 synthetic = Document(
                     doc_id=doc_id,
                     doc_name=row.get("doc_title", doc_id),
                     file_name=row.get("doc_title", doc_id),
-                    object_name="",
+                    object_name=f"{doc_id}/",  # sentinel: MinIO prefix exists
                     content_type="",
                     size_bytes=0,
                     status=DocumentStatus.INDEXED,
@@ -646,9 +667,63 @@ class DocumentQueryService:
         result.sort(key=lambda d: d.updated_at, reverse=True)
         return result[:limit] if limit is not None else result
 
-    def download(self, doc_id: str) -> tuple[Document, bytes]:
-        """Handle download for the Document Query Service instance."""
+    def download(self, doc_id: str) -> tuple["Document", bytes]:
+        """Return the document record and its binary content from MinIO.
+
+        Fallback strategy for Milvus-only docs (no JSON/PG metadata record):
+          1. Try metadata repository first (normal path).
+          2. If metadata is missing, list MinIO objects with prefix ``{doc_id}/``
+             and synthesise a minimal Document from the first object found.
+             This handles documents whose metadata records were lost but whose
+             binary files are still in object storage.
+          3. If neither source has the file, raise FileNotFoundError.
+        """
+        from app.domain.documents import Document, DocumentStatus
+
         document = self.document_repository.get(doc_id)
-        if not document:
-            raise FileNotFoundError(f"文档不存在: {doc_id}")
+
+        if document and document.object_name and not document.object_name.endswith("/"):
+            # Normal doc with a concrete object_name — read directly.
+            return document, self.binary_store.read(document.object_name)
+
+        if document and not document.object_name:
+            raise FileNotFoundError(f"该文档无原始文件（仅含索引数据，无法下载）: {doc_id}")
+
+        if not document or document.object_name.endswith("/"):
+            # Metadata missing — try to find the file in MinIO by doc_id prefix.
+            try:
+                objects = self.binary_store.list_objects(prefix=f"{doc_id}/")
+                # Filter out artifact JSON files; prefer the source document.
+                candidates = [o for o in objects if not o.endswith(".json")]
+                if not candidates:
+                    candidates = objects  # fall back to all objects if only JSON found
+                if not candidates:
+                    raise FileNotFoundError(f"文档不存在（MinIO 和元数据均无记录）: {doc_id}")
+                object_name = candidates[0]
+                file_name = object_name.split("/", 1)[-1] if "/" in object_name else object_name
+                # Guess content type from extension.
+                ext = file_name.rsplit(".", 1)[-1].lower() if "." in file_name else ""
+                _ct_map = {
+                    "pdf": "application/pdf",
+                    "docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+                    "doc": "application/msword",
+                    "txt": "text/plain",
+                }
+                content_type = _ct_map.get(ext, "application/octet-stream")
+                # Synthesise a minimal Document so the route can build the response.
+                document = Document(
+                    doc_id=doc_id,
+                    doc_name=file_name,
+                    file_name=file_name,
+                    object_name=object_name,
+                    content_type=content_type,
+                    size_bytes=0,
+                    status=DocumentStatus.INDEXED,
+                )
+                logger.info("MinIO fallback download: doc_id={} object={}", doc_id, object_name)
+            except FileNotFoundError:
+                raise
+            except Exception as exc:
+                raise FileNotFoundError(f"文档不存在: {doc_id}") from exc
+
         return document, self.binary_store.read(document.object_name)