"""Tests for compiler/sources/discover.py. All offline — HTTP calls are intercepted with httpx.MockTransport using canned fixture responses from tests/fixtures/http/discover/. """ from __future__ import annotations from pathlib import Path import httpx import pytest from compiler.sources.discover import ( SourceCandidate, candidates_to_yaml, discover, ) FIXTURE_DIR = Path(__file__).parent.parent / "fixtures" / "http" / "discover" def _load(name: str) -> bytes: return (FIXTURE_DIR / name).read_bytes() # ── Mock transport ──────────────────────────────────────────────────────────── class _DiscoverTransport(httpx.AsyncBaseTransport): """Routes requests to canned fixture responses.""" async def handle_async_request(self, request: httpx.Request) -> httpx.Response: url = str(request.url) if "wikipedia.org/w/api.php" in url and "list=search" in url: return httpx.Response(200, content=_load("wikipedia_search.json")) if "en.wikiquote.org" in url: html = b"wgPageName Marcus Aurelius" return httpx.Response(200, content=html) if "gutendex.com" in url: return httpx.Response(200, content=_load("gutendex_search.json")) if "openlibrary.org/search" in url: return httpx.Response(200, content=_load("open_library_search.json")) if "itunes.apple.com" in url: return httpx.Response(200, content=_load("itunes_search.json")) return httpx.Response(404, content=b"{}") def _mock_client() -> httpx.AsyncClient: return httpx.AsyncClient(transport=_DiscoverTransport()) # ── Unit tests ──────────────────────────────────────────────────────────────── @pytest.mark.asyncio async def test_discover_returns_candidates(): """discover() returns a non-empty list of SourceCandidates.""" async with _mock_client() as client: results = await discover("Marcus Aurelius", client) assert len(results) > 0 assert all(isinstance(c, SourceCandidate) for c in results) @pytest.mark.asyncio async def test_discover_finds_wikipedia(): async with _mock_client() as client: results = await discover("Marcus Aurelius", client) urls = [c.origin_url for c in results] assert any("wikipedia.org" in u for u in urls) @pytest.mark.asyncio async def test_discover_finds_gutenberg(): async with _mock_client() as client: results = await discover("Marcus Aurelius", client) public_domain = [c for c in results if c.rights == "public-domain"] assert len(public_domain) > 0 @pytest.mark.asyncio async def test_discover_finds_wikiquote(): async with _mock_client() as client: results = await discover("Marcus Aurelius", client) urls = [c.origin_url for c in results] assert any("wikiquote.org" in u for u in urls) @pytest.mark.asyncio async def test_discover_public_domain_sorted_first(): """public-domain sources should appear before fair-use-excerpt.""" async with _mock_client() as client: results = await discover("Marcus Aurelius", client) rights_order = [c.rights for c in results] # Find first PD and first fair-use index pd_indices = [i for i, r in enumerate(rights_order) if r == "public-domain"] fu_indices = [i for i, r in enumerate(rights_order) if r == "fair-use-excerpt"] if pd_indices and fu_indices: assert min(pd_indices) < max(fu_indices) @pytest.mark.asyncio async def test_discover_deduplicates_urls(): """No duplicate URLs in results.""" async with _mock_client() as client: results = await discover("Marcus Aurelius", client) urls = [c.origin_url for c in results] assert len(urls) == len(set(urls)) @pytest.mark.asyncio async def test_discover_all_candidates_have_required_fields(): async with _mock_client() as client: results = await discover("Marcus Aurelius", client) for c in results: assert c.source_type in ("book", "interview", "podcast", "article", "letters") assert c.title assert c.origin_url assert c.rights in ("public-domain", "fair-use-excerpt", "personal-use") assert isinstance(c.approved, bool) @pytest.mark.asyncio async def test_discover_candidates_not_approved_by_default(): async with _mock_client() as client: results = await discover("Marcus Aurelius", client) assert all(not c.approved for c in results) # ── candidates_to_yaml ──────────────────────────────────────────────────────── @pytest.mark.asyncio async def test_candidates_to_yaml_roundtrips(): """Serialized YAML is valid and contains expected fields.""" import yaml async with _mock_client() as client: results = await discover("Marcus Aurelius", client) yml = candidates_to_yaml("Marcus Aurelius", results) data = yaml.safe_load(yml) assert data["name"] == "Marcus Aurelius" assert data["slug"] == "marcus-aurelius" assert isinstance(data["sources"], list) assert len(data["sources"]) == len(results) @pytest.mark.asyncio async def test_candidates_to_yaml_sources_are_acquire_ready(): """Approved discovered sources should have the fields acquire_all expects.""" import yaml async with _mock_client() as client: results = await discover("Marcus Aurelius", client, include_podcasts=False) yml = candidates_to_yaml("Marcus Aurelius", results) data = yaml.safe_load(yml) gutenberg = next(s for s in data["sources"] if s["fetch_type"] == "gutenberg") assert gutenberg["id"] assert gutenberg["gutenberg_id"] wikipedia = next(s for s in data["sources"] if s["fetch_type"] == "wikipedia") assert wikipedia["id"] == "wikipedia-marcus-aurelius" assert wikipedia["wikipedia_title"] == "Marcus Aurelius" urls = [s for s in data["sources"] if s["fetch_type"] == "url"] assert urls assert all(s["url"].startswith("https://") for s in urls) def test_candidates_to_yaml_slug_format(): """Slug is lowercase with hyphens, no special chars.""" yml = candidates_to_yaml("Ayrton Senna", []) import yaml data = yaml.safe_load(yml) assert data["slug"] == "ayrton-senna" def test_source_candidate_as_yaml_dict(): c = SourceCandidate( source_type="book", title="Meditations", origin_url="https://example.com", rights="public-domain", notes="test", ) d = c.as_yaml_dict() assert d["id"] assert d["fetch_type"] == "url" assert d["url"] == "https://example.com" assert d["type"] == "book" assert d["rights"] == "public-domain" assert d["approved"] is False