"""Tests for compiler/stages/eval.py — offline via FakeProvider."""

from __future__ import annotations

import json
from pathlib import Path

import pytest
from compiler.stages.eval import (
    GROUNDEDNESS_FLOOR,
    STYLE_FLOOR,
    _check_regression,
    evaluate,
)
from runtime.providers.fake import FakeProvider
from shared.config import MODELS
from shared.package import PersonaManifest

FIXTURES = Path(__file__).parent.parent / "fixtures"

def _holdout_line(i: int) -> str:
    text = f"Passage {i}: virtue is the only good, and reason guides us forward into clarity."
    return json.dumps({"source_id": "test-src", "text": text})


HOLDOUT_CONTENT = "\n".join(_holdout_line(i) for i in range(5))


def _make_package(tmp_path: Path) -> tuple[PersonaManifest, Path]:
    pkg_dir = tmp_path / "test-persona"
    style_dir = pkg_dir / "style"
    eval_dir = pkg_dir / "eval"
    style_dir.mkdir(parents=True)
    eval_dir.mkdir()
    (style_dir / "holdout.jsonl").write_text(HOLDOUT_CONTENT)
    (eval_dir / "history.jsonl").write_text("")

    manifest = PersonaManifest(
        slug="test-persona",
        display_name="Test Persona",
        born=100,
        died=180,
        voice_policy="generated",
    )
    return manifest, pkg_dir


@pytest.fixture
def judge_provider():
    # Cycles: groundedness → style → misattribution, for each sample
    responses = [
        ['{"score": 0.85, "reasoning": "Direct citation."}'],
        ['{"score": 0.80, "reasoning": "Good voice match."}'],
        ['{"misattribution_count": 0, "examples": []}'],
    ] * 5  # up to 5 samples
    return FakeProvider(responses)


# ── evaluate ──────────────────────────────────────────────────────────────────


@pytest.mark.asyncio
async def test_evaluate_returns_result_dict(tmp_path, judge_provider):
    manifest, pkg_dir = _make_package(tmp_path)
    result = await evaluate(manifest, pkg_dir, judge_provider, n_samples=2)
    assert "groundedness" in result
    assert "style_match" in result
    assert "misattribution" in result
    assert "passed" in result


@pytest.mark.asyncio
async def test_evaluate_appends_history(tmp_path, judge_provider):
    manifest, pkg_dir = _make_package(tmp_path)
    await evaluate(manifest, pkg_dir, judge_provider, n_samples=2)
    history = (pkg_dir / "eval" / "history.jsonl").read_text().strip().splitlines()
    assert len(history) == 1
    record = json.loads(history[0])
    assert record["model"] == MODELS.fidelity_judge
    assert "timestamp" in record


@pytest.mark.asyncio
async def test_evaluate_scores_above_floor(tmp_path, judge_provider):
    manifest, pkg_dir = _make_package(tmp_path)
    result = await evaluate(manifest, pkg_dir, judge_provider, n_samples=2)
    assert result["groundedness"] >= GROUNDEDNESS_FLOOR - 0.01
    assert result["style_match"] >= STYLE_FLOOR - 0.01


@pytest.mark.asyncio
async def test_evaluate_missing_holdout_raises(tmp_path, judge_provider):
    pkg_dir = tmp_path / "no-holdout"
    pkg_dir.mkdir()
    manifest = PersonaManifest(slug="no-holdout", display_name="X")
    with pytest.raises(FileNotFoundError):
        await evaluate(manifest, pkg_dir, judge_provider, n_samples=1)


@pytest.mark.asyncio
async def test_evaluate_ingests_capture_eval_questions(tmp_path, judge_provider):
    manifest, pkg_dir = _make_package(tmp_path)
    eval_questions = pkg_dir / "eval" / "eval-questions.jsonl"
    eval_questions.write_text(
        json.dumps(
            {
                "id": "e001",
                "question": "What matters most?",
                "expect": "Remain aligned with virtue.",
                "kind": "core-belief",
            }
        )
        + "\n",
        encoding="utf-8",
    )

    result = await evaluate(manifest, pkg_dir, judge_provider, n_samples=1)

    assert result["capture_eval_questions"] == 1
    assert result["n_samples"] == 1


# ── _check_regression ────────────────────────────────────────────────────────


def test_check_regression_no_prior_passes(tmp_path):
    history = tmp_path / "history.jsonl"
    history.write_text(json.dumps({"groundedness": 0.8, "style_match": 0.7}) + "\n")
    _check_regression(history, 0.75, 0.65)  # must not raise (only 1 prior)


def test_check_regression_raises_on_groundedness_drop(tmp_path):
    history = tmp_path / "history.jsonl"
    history.write_text(
        json.dumps({"groundedness": 0.85, "style_match": 0.75}) + "\n"
        + json.dumps({"groundedness": 0.70, "style_match": 0.70}) + "\n"
    )
    with pytest.raises(RuntimeError, match="Groundedness regression"):
        _check_regression(history, 0.70, 0.70)


def test_check_regression_ok_within_threshold(tmp_path):
    history = tmp_path / "history.jsonl"
    history.write_text(
        json.dumps({"groundedness": 0.82, "style_match": 0.75}) + "\n"
        + json.dumps({"groundedness": 0.80, "style_match": 0.73}) + "\n"
    )
    # 0.02 drop is within 0.05 threshold
    _check_regression(history, 0.80, 0.73)