init

vw-agentic-rag/tests/integration/test_full_workflow.py

@@ -0,0 +1,402 @@
+"""
+Full Workflow Integration Tests
+
+These tests validate complete end-to-end workflows by connecting to a running service.
+They test realistic user scenarios and complex interactions.
+"""
+import pytest
+import asyncio
+import httpx
+import time
+import os
+from typing import List, Dict, Any
+
+
+# Configuration for remote service connection
+DEFAULT_SERVICE_URL = "http://127.0.0.1:8000"
+SERVICE_URL = os.getenv("AGENTIC_RAG_SERVICE_URL", DEFAULT_SERVICE_URL)
+
+
+@pytest.fixture(scope="session")
+def service_url() -> str:
+    """Get the service URL for testing"""
+    return SERVICE_URL
+
+
+class TestCompleteWorkflows:
+    """Test complete user workflows"""
+
+    @pytest.mark.asyncio
+    async def test_standards_research_workflow(self, service_url: str):
+        """Test a complete standards research workflow"""
+        session_id = f"standards_workflow_{int(time.time())}"
+        
+        # Simulate a user researching ISO 26262
+        conversation_flow = [
+            "What is ISO 26262 and what does it cover?",
+            "What are the ASIL levels in ISO 26262?", 
+            "Can you explain ASIL D requirements in detail?",
+            "How does ISO 26262 relate to vehicle cybersecurity?"
+        ]
+        
+        async with httpx.AsyncClient(timeout=60.0) as client:
+            for i, question in enumerate(conversation_flow):
+                request_data = {
+                    "session_id": session_id,
+                    "messages": [{"role": "user", "content": question}]
+                }
+                
+                response = await client.post(
+                    f"{service_url}/api/ai-sdk/chat",
+                    json=request_data,
+                    headers={"Content-Type": "application/json"}
+                )
+                
+                assert response.status_code == 200
+                
+                # Read the streaming response
+                content = ""
+                async for chunk in response.aiter_text():
+                    content += chunk
+                    if len(content) > 200:  # Get substantial response
+                        break
+                
+                # Verify we get meaningful content
+                assert len(content) > 50
+                print(f"Question {i+1} response length: {len(content)} chars")
+                
+                # Small delay between questions
+                await asyncio.sleep(0.5)
+
+    @pytest.mark.asyncio
+    async def test_manufacturing_safety_workflow(self, service_url: str):
+        """Test manufacturing safety standards workflow"""
+        session_id = f"manufacturing_workflow_{int(time.time())}"
+        
+        conversation_flow = [
+            "What are the key safety standards for manufacturing equipment?",
+            "How do ISO 13849 and IEC 62061 compare?",
+            "What is the process for safety risk assessment in manufacturing?"
+        ]
+        
+        async with httpx.AsyncClient(timeout=60.0) as client:
+            responses = []
+            
+            for question in conversation_flow:
+                request_data = {
+                    "session_id": session_id,
+                    "messages": [{"role": "user", "content": question}]
+                }
+                
+                response = await client.post(
+                    f"{service_url}/api/chat",
+                    json=request_data,
+                    headers={"Content-Type": "application/json"}
+                )
+                
+                assert response.status_code == 200
+                
+                # Collect response content
+                content = ""
+                async for chunk in response.aiter_text():
+                    content += chunk
+                    if len(content) > 300:
+                        break
+                
+                responses.append(content)
+                await asyncio.sleep(0.5)
+            
+            # Verify we got responses for all questions
+            assert len(responses) == len(conversation_flow)
+            for response_content in responses:
+                assert len(response_content) > 30
+
+    @pytest.mark.asyncio
+    async def test_session_context_continuity(self, service_url: str):
+        """Test that session context is maintained across requests"""
+        session_id = f"context_test_{int(time.time())}"
+        
+        async with httpx.AsyncClient(timeout=60.0) as client:
+            # First message - establish context
+            request1 = {
+                "session_id": session_id,
+                "messages": [{"role": "user", "content": "I'm working on a safety system for automotive braking. What standard should I follow?"}]
+            }
+            
+            response1 = await client.post(
+                f"{service_url}/api/chat",
+                json=request1,
+                headers={"Content-Type": "application/json"}
+            )
+            assert response1.status_code == 200
+            
+            # Wait for processing
+            await asyncio.sleep(2)
+            
+            # Follow-up question that depends on context
+            request2 = {
+                "session_id": session_id,
+                "messages": [{"role": "user", "content": "What are the specific testing requirements for this standard?"}]
+            }
+            
+            response2 = await client.post(
+                f"{service_url}/api/chat",
+                json=request2,
+                headers={"Content-Type": "application/json"}
+            )
+            assert response2.status_code == 200
+            
+            # Verify both responses are meaningful
+            content1 = ""
+            async for chunk in response1.aiter_text():
+                content1 += chunk
+                if len(content1) > 100:
+                    break
+            
+            content2 = ""
+            async for chunk in response2.aiter_text():
+                content2 += chunk
+                if len(content2) > 100:
+                    break
+            
+            assert len(content1) > 50
+            assert len(content2) > 50
+
+
+class TestErrorRecoveryWorkflows:
+    """Test error recovery and edge case workflows"""
+
+    @pytest.mark.asyncio
+    async def test_session_recovery_after_error(self, service_url: str):
+        """Test that sessions can recover after encountering errors"""
+        session_id = f"error_recovery_{int(time.time())}"
+        
+        async with httpx.AsyncClient(timeout=60.0) as client:
+            # Valid request
+            valid_request = {
+                "session_id": session_id,
+                "messages": [{"role": "user", "content": "What is ISO 9001?"}]
+            }
+            
+            response = await client.post(
+                f"{service_url}/api/chat",
+                json=valid_request,
+                headers={"Content-Type": "application/json"}
+            )
+            assert response.status_code == 200
+            
+            # Try an invalid request that might cause issues
+            invalid_request = {
+                "session_id": session_id,
+                "messages": [{"role": "user", "content": ""}]  # Empty content
+            }
+            
+            try:
+                await client.post(
+                    f"{service_url}/api/chat", 
+                    json=invalid_request,
+                    headers={"Content-Type": "application/json"}
+                )
+            except Exception:
+                pass  # Expected to potentially fail
+            
+            await asyncio.sleep(1)
+            
+            # Another valid request to test recovery
+            recovery_request = {
+                "session_id": session_id,
+                "messages": [{"role": "user", "content": "Can you summarize what we discussed?"}]
+            }
+            
+            recovery_response = await client.post(
+                f"{service_url}/api/chat",
+                json=recovery_request,
+                headers={"Content-Type": "application/json"}
+            )
+            
+            # Session should still work
+            assert recovery_response.status_code == 200
+
+    @pytest.mark.asyncio
+    async def test_concurrent_sessions(self, service_url: str):
+        """Test multiple concurrent sessions"""
+        base_time = int(time.time())
+        sessions = [f"concurrent_{base_time}_{i}" for i in range(3)]
+        
+        async def test_session(session_id: str, question: str):
+            """Test a single session"""
+            async with httpx.AsyncClient(timeout=60.0) as client:
+                request = {
+                    "session_id": session_id,
+                    "messages": [{"role": "user", "content": question}]
+                }
+                
+                response = await client.post(
+                    f"{service_url}/api/chat",
+                    json=request,
+                    headers={"Content-Type": "application/json"}
+                )
+                
+                assert response.status_code == 200
+                return session_id
+        
+        # Run concurrent sessions
+        questions = [
+            "What is ISO 27001?",
+            "What is NIST Cybersecurity Framework?", 
+            "What is GDPR compliance?"
+        ]
+        
+        tasks = [
+            test_session(session_id, question)
+            for session_id, question in zip(sessions, questions)
+        ]
+        
+        results = await asyncio.gather(*tasks, return_exceptions=True)
+        
+        # All sessions should complete successfully
+        assert len(results) == 3
+        for result in results:
+            assert not isinstance(result, Exception)
+
+
+class TestPerformanceWorkflows:
+    """Test performance-related workflows"""
+
+    @pytest.mark.asyncio
+    async def test_rapid_fire_requests(self, service_url: str):
+        """Test rapid consecutive requests in same session"""
+        session_id = f"rapid_fire_{int(time.time())}"
+        
+        questions = [
+            "Hello",
+            "What is ISO 14001?",
+            "Thank you",
+            "Goodbye"
+        ]
+        
+        async with httpx.AsyncClient(timeout=60.0) as client:
+            for i, question in enumerate(questions):
+                request = {
+                    "session_id": session_id,
+                    "messages": [{"role": "user", "content": question}]
+                }
+                
+                response = await client.post(
+                    f"{service_url}/api/chat",
+                    json=request,
+                    headers={"Content-Type": "application/json"}
+                )
+                
+                assert response.status_code == 200
+                print(f"Rapid request {i+1} completed")
+                
+                # Very short delay
+                await asyncio.sleep(0.1)
+
+    @pytest.mark.asyncio
+    async def test_large_context_workflow(self, service_url: str):
+        """Test workflow with gradually increasing context"""
+        session_id = f"large_context_{int(time.time())}"
+        
+        async with httpx.AsyncClient(timeout=60.0) as client:
+            # Build up context over multiple turns
+            conversation = [
+                "I need to understand automotive safety standards",
+                "Specifically, tell me about ISO 26262 functional safety",
+                "What are the different ASIL levels and their requirements?",
+                "How do I implement ASIL D for a braking system?",
+                "What testing and validation is required for ASIL D?",
+                "Can you provide a summary of everything we've discussed?"
+            ]
+            
+            for i, message in enumerate(conversation):
+                request = {
+                    "session_id": session_id,
+                    "messages": [{"role": "user", "content": message}]
+                }
+                
+                response = await client.post(
+                    f"{service_url}/api/chat",
+                    json=request,
+                    headers={"Content-Type": "application/json"}
+                )
+                
+                assert response.status_code == 200
+                print(f"Context turn {i+1} completed")
+                
+                # Allow time for processing
+                await asyncio.sleep(1)
+
+
+class TestRealWorldScenarios:
+    """Test realistic user scenarios"""
+
+    @pytest.mark.asyncio
+    async def test_compliance_officer_scenario(self, service_url: str):
+        """Simulate a compliance officer's typical workflow"""
+        session_id = f"compliance_officer_{int(time.time())}"
+        
+        # Typical compliance questions
+        scenario_questions = [
+            "I need to ensure our new product meets regulatory requirements. What standards apply to automotive safety systems?",
+            "Our system is classified as ASIL C. What does this mean for our development process?",
+            "What documentation do we need to prepare for safety assessment?",
+            "How often do we need to review and update our safety processes?"
+        ]
+        
+        async with httpx.AsyncClient(timeout=90.0) as client:
+            for i, question in enumerate(scenario_questions):
+                request = {
+                    "session_id": session_id,
+                    "messages": [{"role": "user", "content": question}]
+                }
+                
+                response = await client.post(
+                    f"{service_url}/api/ai-sdk/chat",
+                    json=request,
+                    headers={"Content-Type": "application/json"}
+                )
+                
+                assert response.status_code == 200
+                
+                # Allow realistic time between questions
+                await asyncio.sleep(2)
+                print(f"Compliance scenario step {i+1} completed")
+
+    @pytest.mark.asyncio
+    async def test_engineer_research_scenario(self, service_url: str):
+        """Simulate an engineer researching technical details"""
+        session_id = f"engineer_research_{int(time.time())}"
+        
+        research_flow = [
+            "I'm designing a safety-critical system. What's the difference between ISO 26262 and IEC 61508?",
+            "For automotive applications, which standard takes precedence?",
+            "What are the specific requirements for software development under ISO 26262?",
+            "Can you explain the V-model development process required by the standard?"
+        ]
+        
+        async with httpx.AsyncClient(timeout=90.0) as client:
+            for question in research_flow:
+                request = {
+                    "session_id": session_id,
+                    "messages": [{"role": "user", "content": question}]
+                }
+                
+                response = await client.post(
+                    f"{service_url}/api/chat",
+                    json=request,
+                    headers={"Content-Type": "application/json"}
+                )
+                
+                assert response.status_code == 200
+                
+                # Read some response to verify it's working
+                content = ""
+                async for chunk in response.aiter_text():
+                    content += chunk
+                    if len(content) > 150:
+                        break
+                
+                assert len(content) > 50
+                await asyncio.sleep(1.5)