78 lines
3.0 KiB
Python
78 lines
3.0 KiB
Python
"""Tests for the scheduled crawl Celery task.
|
|
|
|
These pin the draining contract: the task must consume every item from
|
|
CrawlService.run_crawl(), tally per-source errors without raising on them, and
|
|
let a genuine whole-crawl exception propagate rather than swallowing it.
|
|
|
|
Patches target app.shared.bootstrap.get_crawl_service — not
|
|
perception_tasks.get_crawl_service — because the task imports it inside its
|
|
own function body (see perception_tasks.py's docstring for why), so there is
|
|
no module-level name in perception_tasks to intercept.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
import pytest
|
|
|
|
|
|
def _fake_crawl_service(events):
|
|
"""Build a fake whose run_crawl() yields the given fixed event sequence."""
|
|
service = MagicMock()
|
|
service.run_crawl.return_value = iter(events)
|
|
return service
|
|
|
|
|
|
def test_task_drains_generator_and_summarizes_errors():
|
|
"""One source error among two must not stop the run or raise."""
|
|
events = [
|
|
{"event": "progress", "data": {"source": "CATARC", "stage": "fetching"}},
|
|
{"event": "error", "data": {"source": "CATARC", "message": "timeout"}},
|
|
{"event": "progress", "data": {"source": "EUR-Lex", "stage": "fetching"}},
|
|
{"event": "done", "data": {"total_new": 2, "total_updated": 1}},
|
|
]
|
|
with patch(
|
|
"app.shared.bootstrap.get_crawl_service",
|
|
return_value=_fake_crawl_service(events),
|
|
):
|
|
from app.infrastructure.tasks.perception_tasks import crawl_regulations_task
|
|
result = crawl_regulations_task()
|
|
|
|
assert result == {"new": 2, "updated": 1, "source_errors": 1}
|
|
|
|
|
|
def test_task_reports_zero_errors_on_a_clean_run():
|
|
"""A run with no source errors must report source_errors: 0."""
|
|
events = [
|
|
{"event": "progress", "data": {"source": "CATARC", "stage": "fetching"}},
|
|
{"event": "done", "data": {"total_new": 0, "total_updated": 0}},
|
|
]
|
|
with patch(
|
|
"app.shared.bootstrap.get_crawl_service",
|
|
return_value=_fake_crawl_service(events),
|
|
):
|
|
from app.infrastructure.tasks.perception_tasks import crawl_regulations_task
|
|
result = crawl_regulations_task()
|
|
|
|
assert result == {"new": 0, "updated": 0, "source_errors": 0}
|
|
|
|
|
|
def test_whole_crawl_exception_is_not_swallowed():
|
|
"""A failure below run_crawl's own error handling must propagate.
|
|
|
|
Per-source failures are already handled inside run_crawl and never raise;
|
|
an exception escaping the generator entirely means something unexpected
|
|
broke, and Celery's own failure handling — not a silent catch here — is
|
|
the intended backstop.
|
|
"""
|
|
broken_service = MagicMock()
|
|
broken_service.run_crawl.side_effect = RuntimeError("event store unreachable")
|
|
|
|
with patch(
|
|
"app.shared.bootstrap.get_crawl_service",
|
|
return_value=broken_service,
|
|
):
|
|
from app.infrastructure.tasks.perception_tasks import crawl_regulations_task
|
|
with pytest.raises(RuntimeError, match="event store unreachable"):
|
|
crawl_regulations_task()
|