403 lines
15 KiB
Python
403 lines
15 KiB
Python
|
"""
|
||
|
|
Full Workflow Integration Tests
|
||
|
|
|
||
|
|
These tests validate complete end-to-end workflows by connecting to a running service.
|
||
|
|
They test realistic user scenarios and complex interactions.
|
||
|
|
"""
|
||
|
|
import pytest
|
||
|
|
import asyncio
|
||
|
|
import httpx
|
||
|
|
import time
|
||
|
|
import os
|
||
|
|
from typing import List, Dict, Any
|
||
|
|
|
||
|
|
|
||
|
|
# Configuration for remote service connection
|
||
|
|
DEFAULT_SERVICE_URL = "http://127.0.0.1:8000"
|
||
|
|
SERVICE_URL = os.getenv("AGENTIC_RAG_SERVICE_URL", DEFAULT_SERVICE_URL)
|
||
|
|
|
||
|
|
|
||
|
|
@pytest.fixture(scope="session")
|
||
|
|
def service_url() -> str:
|
||
|
|
"""Get the service URL for testing"""
|
||
|
|
return SERVICE_URL
|
||
|
|
|
||
|
|
|
||
|
|
class TestCompleteWorkflows:
|
||
|
|
"""Test complete user workflows"""
|
||
|
|
|
||
|
|
@pytest.mark.asyncio
|
||
|
|
async def test_standards_research_workflow(self, service_url: str):
|
||
|
|
"""Test a complete standards research workflow"""
|
||
|
|
session_id = f"standards_workflow_{int(time.time())}"
|
||
|
|
|
||
|
|
# Simulate a user researching ISO 26262
|
||
|
|
conversation_flow = [
|
||
|
|
"What is ISO 26262 and what does it cover?",
|
||
|
|
"What are the ASIL levels in ISO 26262?",
|
||
|
|
"Can you explain ASIL D requirements in detail?",
|
||
|
|
"How does ISO 26262 relate to vehicle cybersecurity?"
|
||
|
|
]
|
||
|
|
|
||
|
|
async with httpx.AsyncClient(timeout=60.0) as client:
|
||
|
|
for i, question in enumerate(conversation_flow):
|
||
|
|
request_data = {
|
||
|
|
"session_id": session_id,
|
||
|
|
"messages": [{"role": "user", "content": question}]
|
||
|
|
}
|
||
|
|
|
||
|
|
response = await client.post(
|
||
|
|
f"{service_url}/api/ai-sdk/chat",
|
||
|
|
json=request_data,
|
||
|
|
headers={"Content-Type": "application/json"}
|
||
|
|
)
|
||
|
|
|
||
|
|
assert response.status_code == 200
|
||
|
|
|
||
|
|
# Read the streaming response
|
||
|
|
content = ""
|
||
|
|
async for chunk in response.aiter_text():
|
||
|
|
content += chunk
|
||
|
|
if len(content) > 200: # Get substantial response
|
||
|
|
break
|
||
|
|
|
||
|
|
# Verify we get meaningful content
|
||
|
|
assert len(content) > 50
|
||
|
|
print(f"Question {i+1} response length: {len(content)} chars")
|
||
|
|
|
||
|
|
# Small delay between questions
|
||
|
|
await asyncio.sleep(0.5)
|
||
|
|
|
||
|
|
@pytest.mark.asyncio
|
||
|
|
async def test_manufacturing_safety_workflow(self, service_url: str):
|
||
|
|
"""Test manufacturing safety standards workflow"""
|
||
|
|
session_id = f"manufacturing_workflow_{int(time.time())}"
|
||
|
|
|
||
|
|
conversation_flow = [
|
||
|
|
"What are the key safety standards for manufacturing equipment?",
|
||
|
|
"How do ISO 13849 and IEC 62061 compare?",
|
||
|
|
"What is the process for safety risk assessment in manufacturing?"
|
||
|
|
]
|
||
|
|
|
||
|
|
async with httpx.AsyncClient(timeout=60.0) as client:
|
||
|
|
responses = []
|
||
|
|
|
||
|
|
for question in conversation_flow:
|
||
|
|
request_data = {
|
||
|
|
"session_id": session_id,
|
||
|
|
"messages": [{"role": "user", "content": question}]
|
||
|
|
}
|
||
|
|
|
||
|
|
response = await client.post(
|
||
|
|
f"{service_url}/api/chat",
|
||
|
|
json=request_data,
|
||
|
|
headers={"Content-Type": "application/json"}
|
||
|
|
)
|
||
|
|
|
||
|
|
assert response.status_code == 200
|
||
|
|
|
||
|
|
# Collect response content
|
||
|
|
content = ""
|
||
|
|
async for chunk in response.aiter_text():
|
||
|
|
content += chunk
|
||
|
|
if len(content) > 300:
|
||
|
|
break
|
||
|
|
|
||
|
|
responses.append(content)
|
||
|
|
await asyncio.sleep(0.5)
|
||
|
|
|
||
|
|
# Verify we got responses for all questions
|
||
|
|
assert len(responses) == len(conversation_flow)
|
||
|
|
for response_content in responses:
|
||
|
|
assert len(response_content) > 30
|
||
|
|
|
||
|
|
@pytest.mark.asyncio
|
||
|
|
async def test_session_context_continuity(self, service_url: str):
|
||
|
|
"""Test that session context is maintained across requests"""
|
||
|
|
session_id = f"context_test_{int(time.time())}"
|
||
|
|
|
||
|
|
async with httpx.AsyncClient(timeout=60.0) as client:
|
||
|
|
# First message - establish context
|
||
|
|
request1 = {
|
||
|
|
"session_id": session_id,
|
||
|
|
"messages": [{"role": "user", "content": "I'm working on a safety system for automotive braking. What standard should I follow?"}]
|
||
|
|
}
|
||
|
|
|
||
|
|
response1 = await client.post(
|
||
|
|
f"{service_url}/api/chat",
|
||
|
|
json=request1,
|
||
|
|
headers={"Content-Type": "application/json"}
|
||
|
|
)
|
||
|
|
assert response1.status_code == 200
|
||
|
|
|
||
|
|
# Wait for processing
|
||
|
|
await asyncio.sleep(2)
|
||
|
|
|
||
|
|
# Follow-up question that depends on context
|
||
|
|
request2 = {
|
||
|
|
"session_id": session_id,
|
||
|
|
"messages": [{"role": "user", "content": "What are the specific testing requirements for this standard?"}]
|
||
|
|
}
|
||
|
|
|
||
|
|
response2 = await client.post(
|
||
|
|
f"{service_url}/api/chat",
|
||
|
|
json=request2,
|
||
|
|
headers={"Content-Type": "application/json"}
|
||
|
|
)
|
||
|
|
assert response2.status_code == 200
|
||
|
|
|
||
|
|
# Verify both responses are meaningful
|
||
|
|
content1 = ""
|
||
|
|
async for chunk in response1.aiter_text():
|
||
|
|
content1 += chunk
|
||
|
|
if len(content1) > 100:
|
||
|
|
break
|
||
|
|
|
||
|
|
content2 = ""
|
||
|
|
async for chunk in response2.aiter_text():
|
||
|
|
content2 += chunk
|
||
|
|
if len(content2) > 100:
|
||
|
|
break
|
||
|
|
|
||
|
|
assert len(content1) > 50
|
||
|
|
assert len(content2) > 50
|
||
|
|
|
||
|
|
|
||
|
|
class TestErrorRecoveryWorkflows:
|
||
|
|
"""Test error recovery and edge case workflows"""
|
||
|
|
|
||
|
|
@pytest.mark.asyncio
|
||
|
|
async def test_session_recovery_after_error(self, service_url: str):
|
||
|
|
"""Test that sessions can recover after encountering errors"""
|
||
|
|
session_id = f"error_recovery_{int(time.time())}"
|
||
|
|
|
||
|
|
async with httpx.AsyncClient(timeout=60.0) as client:
|
||
|
|
# Valid request
|
||
|
|
valid_request = {
|
||
|
|
"session_id": session_id,
|
||
|
|
"messages": [{"role": "user", "content": "What is ISO 9001?"}]
|
||
|
|
}
|
||
|
|
|
||
|
|
response = await client.post(
|
||
|
|
f"{service_url}/api/chat",
|
||
|
|
json=valid_request,
|
||
|
|
headers={"Content-Type": "application/json"}
|
||
|
|
)
|
||
|
|
assert response.status_code == 200
|
||
|
|
|
||
|
|
# Try an invalid request that might cause issues
|
||
|
|
invalid_request = {
|
||
|
|
"session_id": session_id,
|
||
|
|
"messages": [{"role": "user", "content": ""}] # Empty content
|
||
|
|
}
|
||
|
|
|
||
|
|
try:
|
||
|
|
await client.post(
|
||
|
|
f"{service_url}/api/chat",
|
||
|
|
json=invalid_request,
|
||
|
|
headers={"Content-Type": "application/json"}
|
||
|
|
)
|
||
|
|
except Exception:
|
||
|
|
pass # Expected to potentially fail
|
||
|
|
|
||
|
|
await asyncio.sleep(1)
|
||
|
|
|
||
|
|
# Another valid request to test recovery
|
||
|
|
recovery_request = {
|
||
|
|
"session_id": session_id,
|
||
|
|
"messages": [{"role": "user", "content": "Can you summarize what we discussed?"}]
|
||
|
|
}
|
||
|
|
|
||
|
|
recovery_response = await client.post(
|
||
|
|
f"{service_url}/api/chat",
|
||
|
|
json=recovery_request,
|
||
|
|
headers={"Content-Type": "application/json"}
|
||
|
|
)
|
||
|
|
|
||
|
|
# Session should still work
|
||
|
|
assert recovery_response.status_code == 200
|
||
|
|
|
||
|
|
@pytest.mark.asyncio
|
||
|
|
async def test_concurrent_sessions(self, service_url: str):
|
||
|
|
"""Test multiple concurrent sessions"""
|
||
|
|
base_time = int(time.time())
|
||
|
|
sessions = [f"concurrent_{base_time}_{i}" for i in range(3)]
|
||
|
|
|
||
|
|
async def test_session(session_id: str, question: str):
|
||
|
|
"""Test a single session"""
|
||
|
|
async with httpx.AsyncClient(timeout=60.0) as client:
|
||
|
|
request = {
|
||
|
|
"session_id": session_id,
|
||
|
|
"messages": [{"role": "user", "content": question}]
|
||
|
|
}
|
||
|
|
|
||
|
|
response = await client.post(
|
||
|
|
f"{service_url}/api/chat",
|
||
|
|
json=request,
|
||
|
|
headers={"Content-Type": "application/json"}
|
||
|
|
)
|
||
|
|
|
||
|
|
assert response.status_code == 200
|
||
|
|
return session_id
|
||
|
|
|
||
|
|
# Run concurrent sessions
|
||
|
|
questions = [
|
||
|
|
"What is ISO 27001?",
|
||
|
|
"What is NIST Cybersecurity Framework?",
|
||
|
|
"What is GDPR compliance?"
|
||
|
|
]
|
||
|
|
|
||
|
|
tasks = [
|
||
|
|
test_session(session_id, question)
|
||
|
|
for session_id, question in zip(sessions, questions)
|
||
|
|
]
|
||
|
|
|
||
|
|
results = await asyncio.gather(*tasks, return_exceptions=True)
|
||
|
|
|
||
|
|
# All sessions should complete successfully
|
||
|
|
assert len(results) == 3
|
||
|
|
for result in results:
|
||
|
|
assert not isinstance(result, Exception)
|
||
|
|
|
||
|
|
|
||
|
|
class TestPerformanceWorkflows:
|
||
|
|
"""Test performance-related workflows"""
|
||
|
|
|
||
|
|
@pytest.mark.asyncio
|
||
|
|
async def test_rapid_fire_requests(self, service_url: str):
|
||
|
|
"""Test rapid consecutive requests in same session"""
|
||
|
|
session_id = f"rapid_fire_{int(time.time())}"
|
||
|
|
|
||
|
|
questions = [
|
||
|
|
"Hello",
|
||
|
|
"What is ISO 14001?",
|
||
|
|
"Thank you",
|
||
|
|
"Goodbye"
|
||
|
|
]
|
||
|
|
|
||
|
|
async with httpx.AsyncClient(timeout=60.0) as client:
|
||
|
|
for i, question in enumerate(questions):
|
||
|
|
request = {
|
||
|
|
"session_id": session_id,
|
||
|
|
"messages": [{"role": "user", "content": question}]
|
||
|
|
}
|
||
|
|
|
||
|
|
response = await client.post(
|
||
|
|
f"{service_url}/api/chat",
|
||
|
|
json=request,
|
||
|
|
headers={"Content-Type": "application/json"}
|
||
|
|
)
|
||
|
|
|
||
|
|
assert response.status_code == 200
|
||
|
|
print(f"Rapid request {i+1} completed")
|
||
|
|
|
||
|
|
# Very short delay
|
||
|
|
await asyncio.sleep(0.1)
|
||
|
|
|
||
|
|
@pytest.mark.asyncio
|
||
|
|
async def test_large_context_workflow(self, service_url: str):
|
||
|
|
"""Test workflow with gradually increasing context"""
|
||
|
|
session_id = f"large_context_{int(time.time())}"
|
||
|
|
|
||
|
|
async with httpx.AsyncClient(timeout=60.0) as client:
|
||
|
|
# Build up context over multiple turns
|
||
|
|
conversation = [
|
||
|
|
"I need to understand automotive safety standards",
|
||
|
|
"Specifically, tell me about ISO 26262 functional safety",
|
||
|
|
"What are the different ASIL levels and their requirements?",
|
||
|
|
"How do I implement ASIL D for a braking system?",
|
||
|
|
"What testing and validation is required for ASIL D?",
|
||
|
|
"Can you provide a summary of everything we've discussed?"
|
||
|
|
]
|
||
|
|
|
||
|
|
for i, message in enumerate(conversation):
|
||
|
|
request = {
|
||
|
|
"session_id": session_id,
|
||
|
|
"messages": [{"role": "user", "content": message}]
|
||
|
|
}
|
||
|
|
|
||
|
|
response = await client.post(
|
||
|
|
f"{service_url}/api/chat",
|
||
|
|
json=request,
|
||
|
|
headers={"Content-Type": "application/json"}
|
||
|
|
)
|
||
|
|
|
||
|
|
assert response.status_code == 200
|
||
|
|
print(f"Context turn {i+1} completed")
|
||
|
|
|
||
|
|
# Allow time for processing
|
||
|
|
await asyncio.sleep(1)
|
||
|
|
|
||
|
|
|
||
|
|
class TestRealWorldScenarios:
|
||
|
|
"""Test realistic user scenarios"""
|
||
|
|
|
||
|
|
@pytest.mark.asyncio
|
||
|
|
async def test_compliance_officer_scenario(self, service_url: str):
|
||
|
|
"""Simulate a compliance officer's typical workflow"""
|
||
|
|
session_id = f"compliance_officer_{int(time.time())}"
|
||
|
|
|
||
|
|
# Typical compliance questions
|
||
|
|
scenario_questions = [
|
||
|
|
"I need to ensure our new product meets regulatory requirements. What standards apply to automotive safety systems?",
|
||
|
|
"Our system is classified as ASIL C. What does this mean for our development process?",
|
||
|
|
"What documentation do we need to prepare for safety assessment?",
|
||
|
|
"How often do we need to review and update our safety processes?"
|
||
|
|
]
|
||
|
|
|
||
|
|
async with httpx.AsyncClient(timeout=90.0) as client:
|
||
|
|
for i, question in enumerate(scenario_questions):
|
||
|
|
request = {
|
||
|
|
"session_id": session_id,
|
||
|
|
"messages": [{"role": "user", "content": question}]
|
||
|
|
}
|
||
|
|
|
||
|
|
response = await client.post(
|
||
|
|
f"{service_url}/api/ai-sdk/chat",
|
||
|
|
json=request,
|
||
|
|
headers={"Content-Type": "application/json"}
|
||
|
|
)
|
||
|
|
|
||
|
|
assert response.status_code == 200
|
||
|
|
|
||
|
|
# Allow realistic time between questions
|
||
|
|
await asyncio.sleep(2)
|
||
|
|
print(f"Compliance scenario step {i+1} completed")
|
||
|
|
|
||
|
|
@pytest.mark.asyncio
|
||
|
|
async def test_engineer_research_scenario(self, service_url: str):
|
||
|
|
"""Simulate an engineer researching technical details"""
|
||
|
|
session_id = f"engineer_research_{int(time.time())}"
|
||
|
|
|
||
|
|
research_flow = [
|
||
|
|
"I'm designing a safety-critical system. What's the difference between ISO 26262 and IEC 61508?",
|
||
|
|
"For automotive applications, which standard takes precedence?",
|
||
|
|
"What are the specific requirements for software development under ISO 26262?",
|
||
|
|
"Can you explain the V-model development process required by the standard?"
|
||
|
|
]
|
||
|
|
|
||
|
|
async with httpx.AsyncClient(timeout=90.0) as client:
|
||
|
|
for question in research_flow:
|
||
|
|
request = {
|
||
|
|
"session_id": session_id,
|
||
|
|
"messages": [{"role": "user", "content": question}]
|
||
|
|
}
|
||
|
|
|
||
|
|
response = await client.post(
|
||
|
|
f"{service_url}/api/chat",
|
||
|
|
json=request,
|
||
|
|
headers={"Content-Type": "application/json"}
|
||
|
|
)
|
||
|
|
|
||
|
|
assert response.status_code == 200
|
||
|
|
|
||
|
|
# Read some response to verify it's working
|
||
|
|
content = ""
|
||
|
|
async for chunk in response.aiter_text():
|
||
|
|
content += chunk
|
||
|
|
if len(content) > 150:
|
||
|
|
break
|
||
|
|
|
||
|
|
assert len(content) > 50
|
||
|
|
await asyncio.sleep(1.5)
|