"""
Predictive signal generator.

For each high-activity member with committee data:
  1. Pull their committee assignments and last 3 years of trade history
  2. Fetch active legislation relevant to those committees (if API key available)
  3. Ask LLM: "What will this member likely trade in the next 45 days?"
  4. Store results in predicted_signals table

This is a separate pipeline from the reactive scorer — it runs per-member
and produces forward-looking watchlist entries that can appear in the report
BEFORE the member discloses.  When a predicted ticker later appears in a real
disclosure, predicted_signals.confirmed is set to 1 so we can track accuracy.

Usage:
  python analysis/predictor.py                # run for top members
  python analysis/predictor.py --dry-run      # print prompts, no API call
  python analysis/predictor.py --member "Nancy Pelosi"
"""

from __future__ import annotations
import sqlite3
import sys
import os
import json
import argparse
from datetime import datetime, timedelta

sys.path.insert(0, os.path.dirname(os.path.dirname(__file__)))
from config import (
    DB_PATH, PREDICTOR_MAX_MEMBERS, PREDICTOR_DECAY_DAYS,
    PREDICTOR_MIN_TRADES, PREDICTOR_LEGISLATION_DAYS,
)
from analysis.scorer import _COMMITTEE_SECTOR_MAP, _committee_score

TOKEN_LOG = os.path.join(os.path.dirname(os.path.dirname(__file__)), "reports", "token_usage.log")

SYSTEM_PROMPT = (
    "You are a political intelligence analyst specialising in US congressional trading patterns. "
    "Your task is to predict future stock trades based on a member's committee assignments, "
    "historical trading behaviour, and active legislation. Be specific about named bills, "
    "sectors, and companies. Acknowledge when you are speculating. "
    "Keep each field concise — 2-3 sentences maximum."
)

PREDICT_PROMPT_TEMPLATE = """\
Analyse the following US Congress member's trading history and committee assignments, \
then predict which stocks they are most likely to trade in the next 45 days.

MEMBER: {member_name} ({party}, {chamber})
COMMITTEES: {committees}

TRADE HISTORY (last 3 years — {trade_count} trades):
{trade_history}

{legislation_section}\
Based on:
1. The sectors this member repeatedly trades and their timing relative to legislation
2. Their committee assignments and what industries those committees oversee
3. Active legislation in their committees that could create an information advantage{legislation_note}

Provide a forward-looking watchlist of 3-5 stocks this member is most likely to buy \
or sell in the next 45 days.

Respond in the following JSON format exactly:

{{
  "behavioral_profile": "2-3 sentences: this member's trading style — which sectors they \
favour, how close to legislation they tend to trade, any notable patterns.",
  "predicted_trades": [
    {{
      "ticker": "XXXX",
      "company_name": "Full Company Name",
      "sector": "sector-slug",
      "predicted_action": "buy",
      "confidence": "LOW|MEDIUM|HIGH",
      "legislation_catalyst": "Specific bill or policy area driving this prediction, \
or 'General sector exposure' if no specific bill.",
      "historical_precedent": "Prior similar trade by this member if one exists, \
or 'No direct precedent' if not.",
      "rationale": "2-3 sentence explanation of why this member is likely to trade this."
    }}
  ]
}}

Respond with valid JSON only, no markdown fencing."""

_LEGISLATION_SECTION_TEMPLATE = """\
ACTIVE LEGISLATION in their committees (last {days} days):
{bill_lines}

"""

_NO_LEGISLATION_NOTE = (
    "\n   (No live bill data available — use your general knowledge of pending "
    "legislation in the 119th Congress that touches these committees.)"
)

def _format_trade_history(trades: list[dict]) -> str:
    if not trades:
        return "  (no trade history found)"
    lines = []
    for t in trades[:60]:  # cap at 60 rows to keep prompt manageable
        opt = " [option]" if t.get("is_option") else ""
        tier_map = {1: "$1K-15K", 2: "$15K-50K", 3: "$50K-100K",
                    4: "$100K-250K", 5: "$250K-500K", 6: "$500K+"}
        size = tier_map.get(t.get("amount_tier") or 0, "?")
        lines.append(
            f"  {t.get('tx_date', '?'):12}  {t.get('tx_type', '?'):4}  "
            f"{t.get('issuer_ticker', '?'):8}  {size:12}  "
            f"{(t.get('issuer_sector') or '?')[:30]}{opt}"
        )
    if len(trades) > 60:
        lines.append(f"  ... and {len(trades)-60} more trades")
    return "\n".join(lines)

def _format_legislation(bills: list[dict], days: int) -> str:
    """Format bill list into a prompt section. Returns empty string if no bills."""
    if not bills:
        return ""
    lines = []
    for b in bills[:25]:
        num   = f"{b.get('type', '')} {b.get('number', '')}".strip()
        title = b.get("title", "")[:120]
        date  = b.get("latest_action_date", "")
        action = b.get("latest_action", "")[:100]
        lines.append(f"  {num}: {title}")
        if date or action:
            lines.append(f"    → {date}  {action}")
    return _LEGISLATION_SECTION_TEMPLATE.format(
        days=days,
        bill_lines="\n".join(lines),
    )

def _build_prompt(member: dict, trades: list[dict], bills: list[dict]) -> str:
    committees_raw = []
    if member.get("committees"):
        try:
            committees_raw = json.loads(member["committees"])
        except Exception:
            pass
    committees_str = ", ".join(str(c) for c in committees_raw[:8]) or "unknown"

    legislation_section = _format_legislation(bills, PREDICTOR_LEGISLATION_DAYS)
    legislation_note = "" if bills else _NO_LEGISLATION_NOTE

    return PREDICT_PROMPT_TEMPLATE.format(
        member_name=f"{member['first_name']} {member['last_name']}",
        party=(member.get("party") or "?")[:1].upper(),
        chamber=member.get("chamber") or "?",
        committees=committees_str,
        trade_count=len(trades),
        trade_history=_format_trade_history(trades),
        legislation_section=legislation_section,
        legislation_note=legislation_note,
    )

def _log_tokens(label: str, usage: dict):
    os.makedirs(os.path.dirname(TOKEN_LOG), exist_ok=True)
    with open(TOKEN_LOG, "a") as f:
        total_cost_est = (
            usage.get("input_tokens", 0) * 3e-6
            + usage.get("output_tokens", 0) * 15e-6
            - usage.get("cache_read_tokens", 0) * 2.7e-6
        )
        f.write(
            f"{datetime.now().isoformat(timespec='seconds')} | predict:{label:<20} | "
            f"in={usage.get('input_tokens',0):>5} out={usage.get('output_tokens',0):>4} "
            f"cache_hit={usage.get('cache_read_tokens',0):>5} "
            f"est_cost=${total_cost_est:.4f}\n"
        )

def _committee_alignment(committees_json: str | None, sector: str) -> str:
    """
    Return 'direct', 'indirect', or 'none' based on how well the member's
    committees overlap with the predicted ticker's sector.
    Reuses scorer._committee_score (0-3 scale).
    """
    score = _committee_score(committees_json, sector or "")
    if score >= 3:
        return "direct"
    if score >= 1:
        return "indirect"
    return "none"

def _parse_response(text: str) -> dict:
    text = text.strip()
    if text.startswith("```"):
        lines = text.splitlines()
        text = "\n".join(lines[1:-1] if lines[-1].strip() == "```" else lines[1:])
    try:
        return json.loads(text)
    except json.JSONDecodeError:
        start = text.find("{")
        end = text.rfind("}") + 1
        if start >= 0 and end > start:
            return json.loads(text[start:end])
        raise

def _was_predicted_recently(conn: sqlite3.Connection, member_id: str, decay_days: int) -> bool:
    """True if this member was predicted within decay_days."""
    row = conn.execute(
        "SELECT MAX(generated_at) FROM predicted_signals WHERE member_id = ?",
        (member_id,),
    ).fetchone()
    if not row or not row[0]:
        return False
    try:
        dt = datetime.fromisoformat(row[0])
        return (datetime.now() - dt).days < decay_days
    except ValueError:
        return False

def _select_members(conn: sqlite3.Connection, max_members: int,
                    decay_days: int, min_trades: int) -> list[dict]:
    """
    Pick members to predict for this run.
    Priority: never-predicted → stalest-predicted → highest trade count.
    Only members with committee data and enough history.
    """
    rows = conn.execute(
        """
        SELECT m.bioguide_id, m.first_name, m.last_name, m.party, m.chamber,
               m.committees, COUNT(t.tx_id) AS trade_count,
               (SELECT MAX(ps.generated_at)
                FROM predicted_signals ps
                WHERE ps.member_id = m.bioguide_id) AS last_predicted
        FROM members m
        JOIN trades t ON t.member_id = m.bioguide_id
        WHERE m.committees IS NOT NULL
          AND m.committees != '[]'
          AND m.committees != ''
        GROUP BY m.bioguide_id
        HAVING COUNT(t.tx_id) >= ?
        ORDER BY last_predicted ASC NULLS FIRST, trade_count DESC
        """,
        (min_trades,),
    ).fetchall()

    now = datetime.now()
    selected = []
    for r in rows:
        rd = dict(r)
        lp = rd.get("last_predicted")
        if lp:
            try:
                age = (now - datetime.fromisoformat(lp)).days
                if age < decay_days:
                    continue
            except ValueError:
                pass
        selected.append(rd)
        if len(selected) >= max_members:
            break

    return selected

def predict_for_member(
    member: dict,
    bills: list[dict],
    dry_run: bool = False,
) -> list[dict] | None:
    """
    Run LLM prediction for one member. Returns list of predicted signal dicts or None.
    """
    conn = sqlite3.connect(DB_PATH)
    conn.row_factory = sqlite3.Row

    # Pull 3 years of trade history for this member
    cutoff_3yr = (datetime.now() - timedelta(days=3*365)).strftime("%Y-%m-%d")
    trades = conn.execute(
        """
        SELECT tx_date, tx_type, issuer_ticker, issuer_sector, amount_tier, is_option
        FROM trades
        WHERE member_id = ? AND tx_date >= ?
        ORDER BY tx_date DESC
        """,
        (member["bioguide_id"], cutoff_3yr),
    ).fetchall()
    trades = [dict(t) for t in trades]

    conn.close()

    member_name = f"{member['first_name']} {member['last_name']}"
    prompt = _build_prompt(member, trades, bills)

    if dry_run:
        print(f"\n{'='*60}")
        print(f"DRY RUN — predict for: {member_name}")
        print(f"{'='*60}")
        print(prompt[:3000])
        return None

    from analysis.llm_client import call, LLMError
    try:
        print(f"  {member_name}: calling LLM...", end=" ", flush=True)
        text, usage = call(prompt, system=SYSTEM_PROMPT, max_tokens=2048)
        _log_tokens(member_name[:20], usage)
        print(f"done (in={usage['input_tokens']} out={usage['output_tokens']})")
    except LLMError as e:
        print(f"  {member_name}: LLM error — {e}")
        return None
    except Exception as e:
        msg = str(e)
        if "429" in msg or "RESOURCE_EXHAUSTED" in msg or "quota" in msg.lower():
            print(f"  {member_name}: API quota exceeded — stopping predictor")
        else:
            print(f"  {member_name}: unexpected error — {e}")
        return None

    try:
        result = _parse_response(text)
    except Exception as e:
        print(f"  {member_name}: parse error — {e}")
        return None

    behavioral_profile = result.get("behavioral_profile", "")
    predicted_trades   = result.get("predicted_trades") or []
    if not isinstance(predicted_trades, list):
        print(f"  {member_name}: unexpected result shape")
        return None

    now_str = datetime.now().isoformat(timespec="seconds")
    conn2 = sqlite3.connect(DB_PATH)

    for pt in predicted_trades:
        ticker = (pt.get("ticker") or "").strip().upper()
        if not ticker:
            continue
        sector    = pt.get("sector") or ""
        alignment = _committee_alignment(member.get("committees"), sector)

        conn2.execute(
            """
            INSERT INTO predicted_signals (
                member_id, member_name, ticker, company_name, sector,
                predicted_action, confidence, legislation_catalyst,
                historical_precedent, rationale, behavioral_profile,
                committee_alignment, generated_at
            ) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)
            ON CONFLICT(member_id, ticker) DO UPDATE SET
                member_name          = excluded.member_name,
                company_name         = excluded.company_name,
                sector               = excluded.sector,
                predicted_action     = excluded.predicted_action,
                confidence           = excluded.confidence,
                legislation_catalyst = excluded.legislation_catalyst,
                historical_precedent = excluded.historical_precedent,
                rationale            = excluded.rationale,
                behavioral_profile   = excluded.behavioral_profile,
                committee_alignment  = excluded.committee_alignment,
                generated_at         = excluded.generated_at
            """,
            (
                member["bioguide_id"],
                member_name,
                ticker,
                pt.get("company_name") or "",
                sector,
                pt.get("predicted_action") or "buy",
                pt.get("confidence") or "LOW",
                pt.get("legislation_catalyst") or "",
                pt.get("historical_precedent") or "",
                pt.get("rationale") or "",
                behavioral_profile,
                alignment,
                now_str,
            ),
        )

    conn2.commit()
    conn2.close()

    return [{"ticker": (pt.get("ticker") or "").upper(),
             "confidence": pt.get("confidence", "?"),
             "action": pt.get("predicted_action", "?")} for pt in predicted_trades]

def run_predictions(
    max_members: int = PREDICTOR_MAX_MEMBERS,
    decay_days: int = PREDICTOR_DECAY_DAYS,
    min_trades: int = PREDICTOR_MIN_TRADES,
    member_filter: str | None = None,
    dry_run: bool = False,
) -> list[dict]:
    """
    Entry point for the daily pipeline.
    Returns list of summary dicts: {member_name, predictions: [{ticker, confidence, action}]}.
    """
    from tools.fetch_legislation import load as load_legislation, filter_for_committees

    conn = sqlite3.connect(DB_PATH)
    conn.row_factory = sqlite3.Row

    if member_filter:
        # Override member selection for CLI testing
        name_parts = member_filter.strip().split()
        rows = conn.execute(
            """
            SELECT m.bioguide_id, m.first_name, m.last_name, m.party, m.chamber,
                   m.committees, COUNT(t.tx_id) AS trade_count, NULL AS last_predicted
            FROM members m
            JOIN trades t ON t.member_id = m.bioguide_id
            WHERE (m.first_name LIKE ? OR m.last_name LIKE ? OR
                   (m.first_name || ' ' || m.last_name) LIKE ?)
            GROUP BY m.bioguide_id
            """,
            (f"%{name_parts[0]}%", f"%{name_parts[-1]}%",
             f"%{member_filter}%"),
        ).fetchall()
        members = [dict(r) for r in rows]
        if not members:
            print(f"  No member found matching '{member_filter}'")
            conn.close()
            return []
    else:
        members = _select_members(conn, max_members=max_members,
                                  decay_days=decay_days, min_trades=min_trades)

    conn.close()

    if not members:
        print("  No members eligible for prediction this run (all within decay window)")
        return []

    # Fetch legislation once and filter per member
    all_bills = load_legislation(days_back=PREDICTOR_LEGISLATION_DAYS)
    has_bills = bool(all_bills)
    if has_bills:
        print(f"  Loaded {len(all_bills)} active bills for cross-referencing")
    else:
        print("  No live bill data (set CONGRESS_GOV_API_KEY for real-time legislation)")

    results = []
    for member in members:
        member_name = f"{member['first_name']} {member['last_name']}"
        print(f"  Predicting for {member_name} ({member.get('trade_count', '?')} trades)...")

        # Filter bills for this member's committees
        committees = []
        if member.get("committees"):
            try:
                committees = json.loads(member["committees"])
            except Exception:
                pass
        member_bills = filter_for_committees(all_bills, committees) if all_bills else []

        preds = predict_for_member(member, member_bills, dry_run=dry_run)
        if preds:
            results.append({"member_name": member_name, "predictions": preds})

    return results

def backfill_committee_alignment():
    """
    Compute committee_alignment for existing rows that were stored before
    this column existed (value = 'none' by default migration).
    """
    conn = sqlite3.connect(DB_PATH)
    conn.row_factory = sqlite3.Row
    rows = conn.execute(
        """
        SELECT ps.id, ps.member_id, ps.sector, m.committees
        FROM predicted_signals ps
        JOIN members m ON m.bioguide_id = ps.member_id
        WHERE ps.committee_alignment = 'none' OR ps.committee_alignment IS NULL
        """
    ).fetchall()

    updated = 0
    for r in rows:
        alignment = _committee_alignment(r["committees"], r["sector"] or "")
        if alignment != "none":
            conn.execute(
                "UPDATE predicted_signals SET committee_alignment = ? WHERE id = ?",
                (alignment, r["id"]),
            )
            updated += 1

    conn.commit()
    conn.close()
    if updated:
        print(f"  Backfilled committee_alignment for {updated} predictions")
    return updated

def confirm_predictions():
    """
    Scan trades disclosed today and mark any matching predicted_signals as confirmed.
    Call this after each ingest run.
    """
    conn = sqlite3.connect(DB_PATH)
    today = datetime.now().strftime("%Y-%m-%d")

    # Find tickers disclosed today that we predicted
    confirmed = conn.execute(
        """
        UPDATE predicted_signals
        SET confirmed = 1, confirmed_at = ?
        WHERE confirmed = 0
          AND ticker IN (
              SELECT DISTINCT issuer_ticker FROM trades
              WHERE pub_date >= date('now', '-7 days')
          )
        """,
        (today,),
    ).rowcount

    conn.commit()
    conn.close()

    if confirmed:
        print(f"  Confirmed {confirmed} predicted signal(s) — trades now disclosed!")
    return confirmed

def load_predictions(days_back: int = 45) -> list[dict]:
    """Return recent predicted signals for report generation."""
    conn = sqlite3.connect(DB_PATH)
    conn.row_factory = sqlite3.Row
    cutoff = (datetime.now() - timedelta(days=days_back)).strftime("%Y-%m-%dT")

    rows = conn.execute(
        """
        SELECT * FROM predicted_signals
        WHERE generated_at >= ?
        ORDER BY
            confidence DESC,
            generated_at DESC
        """,
        (cutoff,),
    ).fetchall()
    conn.close()
    return [dict(r) for r in rows]

def load_consensus(days_back: int = 45, top_n: int = 30) -> list[dict]:
    """
    Aggregate predicted_signals by ticker into a ranked consensus view.

    Score per ticker:
      cluster_pts      = member_count × conf_avg
      small_cap_pts    = from signals.small_cap_score OR ticker_metadata.market_cap
      win_rate_pts     = avg win_rate of predicting members (0-2 pts)
      alignment_bonus  = +2 if any member has direct committee-sector alignment
      precedent_bonus  = +1 if any member has prior history trading this ticker
      ubiquity_penalty = breadth discount for tickers bought by 15+ members historically
                         (same _breadth_mult logic as reactive scorer)

    Filters: ETFs, invalid tickers (no yfinance data), malformed ticker formats.
    """
    _CONF = {"HIGH": 3.0, "MEDIUM": 2.0, "LOW": 1.0}

    # Known junk formats — LLM occasionally hallucinates these
    _INVALID_PREFIXES = ("$$", "$", "##")
    _INVALID_EXACT    = {"SPY", "QQQ", "VTI", "VOO", "IWM", "GLD", "SLV", "TLT",
                         "HYG", "LQD", "DIA", "MDY", "AGG", "BND", "XLF", "XLE",
                         "XLK", "XLV", "XLI", "XLU", "XLB", "XLRE", "XLY", "XLP",
                         "XLC", "VNQ", "ARKK", "ARKG", "ARKW"}

    conn = sqlite3.connect(DB_PATH)
    conn.row_factory = sqlite3.Row
    cutoff = (datetime.now() - timedelta(days=days_back)).strftime("%Y-%m-%dT")

    rows = conn.execute(
        """
        SELECT
            ps.ticker,
            MAX(ps.company_name)                        AS company_name,
            MAX(ps.sector)                              AS sector,
            COUNT(DISTINCT ps.member_id)                AS member_count,
            GROUP_CONCAT(DISTINCT ps.member_name)       AS member_names,
            GROUP_CONCAT(ps.confidence, '|')            AS confidences,
            GROUP_CONCAT(ps.predicted_action, '|')      AS actions,
            GROUP_CONCAT(ps.legislation_catalyst, '|||') AS catalysts,
            GROUP_CONCAT(ps.rationale, '|||')           AS rationales,
            GROUP_CONCAT(ps.historical_precedent, '|||') AS precedents,
            GROUP_CONCAT(COALESCE(ps.committee_alignment,'none'), '|') AS alignments,
            MAX(ps.confirmed)                           AS confirmed,
            MAX(ps.generated_at)                        AS last_generated,
            -- market cap: prefer reactive signals table, fall back to ticker_metadata
            COALESCE(s.small_cap_score, NULL)           AS small_cap_score,
            COALESCE(s.market_cap, tm.market_cap)       AS market_cap,
            s.total_score                               AS reactive_score,
            COALESCE(tm.is_etf, 0)                      AS is_etf,
            COALESCE(tm.is_valid, 1)                    AS is_valid,
            AVG(COALESCE(m.win_rate, 0.5))              AS avg_win_rate,
            tm.pe_ratio                                 AS pe_ratio,
            tm.analyst_rating                           AS analyst_rating,
            tm.price_vs_50d_pct                         AS price_vs_50d_pct
        FROM predicted_signals ps
        LEFT JOIN signals s         ON s.ticker        = ps.ticker
        LEFT JOIN ticker_metadata tm ON tm.ticker      = ps.ticker
        LEFT JOIN members m         ON m.bioguide_id   = ps.member_id
        WHERE ps.generated_at >= ?
          AND ps.predicted_action = 'buy'
        GROUP BY ps.ticker
        """,
        (cutoff,),
    ).fetchall()

    # Historical buyer breadth — for ubiquity discount
    breadth_rows = conn.execute(
        """
        SELECT issuer_ticker, COUNT(DISTINCT member_id) AS cnt
        FROM trades WHERE tx_type = 'buy' AND issuer_ticker != ''
        GROUP BY issuer_ticker
        """
    ).fetchall()
    historical_buyers: dict[str, int] = {r["issuer_ticker"]: r["cnt"] for r in breadth_rows}

    conn.close()

    def _breadth_mult(hist: int) -> float:
        if hist >= 20: return 0.40
        if hist >= 12: return 0.65
        if hist >= 6:  return 0.85
        return 1.0

    def _small_cap_pts(market_cap: float | None) -> float:
        if market_cap is None: return 0.0
        if market_cap > 100e9: return -2.5
        if market_cap > 10e9:  return -0.5
        if market_cap > 2e9:   return  2.0
        if market_cap > 500e6: return  4.5
        return 6.0

    results = []
    for r in rows:
        rd = dict(r)
        ticker = rd["ticker"]

        # --- Filter junk ---
        if any(ticker.startswith(p) for p in _INVALID_PREFIXES):
            continue
        if ticker in _INVALID_EXACT:
            continue
        if rd.get("is_etf"):
            continue
        if rd.get("is_valid") == 0:
            continue

        # --- Core scoring components ---
        conf_list = [c for c in (rd.get("confidences") or "").split("|") if c]
        conf_avg  = sum(_CONF.get(c, 1.0) for c in conf_list) / max(len(conf_list), 1)

        member_count  = rd["member_count"] or 1
        hist_buyers   = historical_buyers.get(ticker, 0)
        cluster_pts   = member_count * conf_avg * _breadth_mult(hist_buyers)

        market_cap    = rd.get("market_cap")
        # If small_cap_score already computed by reactive scorer, use it directly
        sc_score      = rd.get("small_cap_score")
        cap_pts       = float(sc_score) if sc_score is not None else _small_cap_pts(market_cap)

        avg_wr        = float(rd.get("avg_win_rate") or 0.5)
        win_rate_pts  = 2.0 if avg_wr > 0.6 else (1.0 if avg_wr >= 0.5 else 0.0)

        # --- Alignment bonus (+2 for any direct committee-sector match) ---
        alignments     = (rd.get("alignments") or "").split("|")
        alignment_bonus = 2.0 if "direct" in alignments else (0.5 if "indirect" in alignments else 0.0)

        # --- Historical precedent bonus (+1 if member has traded this before) ---
        precedents = [p.strip() for p in (rd.get("precedents") or "").split("|||")]
        has_precedent = any(
            p and p.lower() not in ("no direct precedent", "none", "n/a", "")
            for p in precedents
        )
        precedent_bonus = 1.0 if has_precedent else 0.0

        score = round(cluster_pts + cap_pts + win_rate_pts + alignment_bonus + precedent_bonus, 2)

        # --- Metadata for display ---
        rationale_list = [s.strip() for s in (rd.get("rationales") or "").split("|||") if s.strip()]
        catalyst_list  = [s.strip() for s in (rd.get("catalysts") or "").split("|||") if s.strip()]
        best_rationale = max(rationale_list, key=len) if rationale_list else ""
        best_catalyst  = max(catalyst_list,  key=len) if catalyst_list  else ""

        best_alignment = ("direct" if "direct" in alignments
                          else "indirect" if "indirect" in alignments else "none")

        if member_count >= 2:
            tier = "consensus"
        elif conf_avg >= 3.0:
            tier = "solo_high"
        else:
            tier = "solo_low"

        reactive_score = rd.get("reactive_score")
        cross_signal   = reactive_score is not None and reactive_score > 0

        results.append({
            "ticker":           ticker,
            "company_name":     rd["company_name"] or "",
            "sector":           rd["sector"] or "",
            "member_count":     member_count,
            "member_names":     rd.get("member_names") or "",
            "conf_avg":         round(conf_avg, 1),
            "conf_label":       "HIGH" if conf_avg >= 2.5 else ("MEDIUM" if conf_avg >= 1.5 else "LOW"),
            "tier":             tier,
            "small_cap_score":  cap_pts,
            "market_cap":       market_cap,
            "avg_win_rate":     round(avg_wr, 2),
            "alignment":        best_alignment,
            "alignment_bonus":  alignment_bonus,
            "has_precedent":    has_precedent,
            "precedent_bonus":  precedent_bonus,
            "hist_buyers":      hist_buyers,
            "reactive_score":   reactive_score,
            "cross_signal":     cross_signal,
            "confirmed":        bool(rd.get("confirmed")),
            "best_rationale":   best_rationale,
            "best_catalyst":    best_catalyst,
            "score":            score,
            "last_generated":   (rd.get("last_generated") or "")[:10],
            "actions":          rd.get("actions") or "",
            "pe_ratio":         rd.get("pe_ratio"),
            "analyst_rating":   rd.get("analyst_rating"),
            "price_vs_50d_pct": rd.get("price_vs_50d_pct"),
        })

    results.sort(key=lambda x: x["score"], reverse=True)
    return results[:top_n]

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Generate predictive trade signals via LLM")
    parser.add_argument("--dry-run", action="store_true", help="print prompts, no API call")
    parser.add_argument("--member", help="predict for a specific member name")
    parser.add_argument("--max", type=int, default=PREDICTOR_MAX_MEMBERS,
                        help=f"max members to process (default {PREDICTOR_MAX_MEMBERS})")
    args = parser.parse_args()

    results = run_predictions(
        max_members=args.max,
        member_filter=args.member,
        dry_run=args.dry_run,
    )

    if not args.dry_run and results:
        print(f"\n{'='*60}")
        for r in results:
            print(f"\n{r['member_name']}:")
            for p in r["predictions"]:
                print(f"  {p['ticker']:8}  {p['confidence']:6}  {p['action']}")


Perfect! Now let me check for any test fixtures or TODOs in the codebase.


Bash

curl -s "https://api.github.com/repos/jamesjlopez/congressional-trading/git/trees/main?recursive=1" | grep -o '"path":"[^"]*"' | grep -E '(test|fixture|example)' | head -20


(Bash completed with no output)


Let me search for TODO comments and check the run_daily.py to understand the full pipeline.


"""
CongressWatch daily runner.

Pipeline: ingest → committees → score → prices → win_rates → research → report → notify

Usage:
  python run_daily.py              # full run
  python run_daily.py --dry-run    # ingest + score + report, no LLM calls, no notifications
  python run_daily.py --no-ingest  # skip ingest (re-score + research + report only)
  python run_daily.py --force      # re-notify even if already notified today
"""

from __future__ import annotations
import argparse
import sqlite3
import sys
import os
from datetime import datetime, date

sys.path.insert(0, os.path.dirname(__file__))
from config import (DB_PATH, SCORE_NOTIFY_THRESHOLD, SCORE_RESEARCH_THRESHOLD,
                    LOOKBACK_DAYS, TIMELINESS_MULTIPLIERS,
                    DEEP_RESEARCH_TOP_N, DEEP_RESEARCH_DECAY_DAYS,
                    MAX_NOTIFICATIONS_PER_RUN, NOTIFY_TICKER_COOLDOWN_DAYS,
                    PREDICTOR_MAX_MEMBERS, PREDICTOR_DECAY_DAYS)

def _timeliness_adjusted_score(base_score: float, timeliness: str | None) -> float:
    """Apply timeliness multiplier for notify-gate purposes only. Score is not stored."""
    key = (timeliness or "fresh").split("|")[0].strip().lower()
    return base_score * TIMELINESS_MULTIPLIERS.get(key, 1.0)

def _send_signal_notification(sig: dict, research: dict | None, adjusted_score: float):
    from notify.ntfy_client import push
    from analysis.timeliness import assess_age

    ticker = sig["ticker"]
    score = sig["total_score"]
    buyers = sig["member_count"]
    timeliness = research.get("timeliness", assess_age(sig["last_seen"])) if research else assess_age(sig["last_seen"])
    conviction = (research.get("conviction") or "?").upper() if research else "?"

    tier_labels = {1:"$1K-15K", 2:"$15K-50K", 3:"$50K-100K",
                   4:"$100K-250K", 5:"$250K-500K", 6:"$500K+"}
    size_label = tier_labels.get(sig.get("max_tier", 0), "unknown")
    opt_flag = " | Options: YES" if sig.get("has_options") else ""

    tl_note = ""
    if adjusted_score < score:
        tl_note = f" (adj {adjusted_score:.1f} for {timeliness})"

    action_short = ""
    if research and research.get("recommended_action"):
        raw_action = research["recommended_action"]
        action_short = raw_action.split("—")[0].strip()[:40]

    summary_line = ""
    if research and research.get("company_summary"):
        summary_line = "\n\n" + research["company_summary"][:300]
    catalyst_line = ""
    if research and research.get("catalyst_hypothesis"):
        catalyst_line = "\n\nCatalyst: " + research["catalyst_hypothesis"][:200]
    action_line = ""
    if action_short:
        action_line = f"\n\n→ {action_short}"

    body = (
        f"{sig.get('issuer_name', ticker)} — {buyers} member{'s' if buyers>1 else ''} bought\n"
        f"Size: {size_label}{opt_flag}\n"
        f"Timeliness: {timeliness.upper()} | Conviction: {conviction} | Action: {action_short}"
        f"{summary_line}{catalyst_line}{action_line}"
    )

    push(
        title=f"CongressWatch: ${ticker} (score {score:.1f}{tl_note})",
        body=body,
        priority="high" if adjusted_score >= 9 else "default",
        tags=["chart_with_upwards_trend", "money_with_wings"],
    )
    print(f"  ntfy sent for {ticker}")

def _already_notified_today(ticker: str) -> bool:
    conn = sqlite3.connect(DB_PATH)
    row = conn.execute(
        "SELECT notified_at FROM signals WHERE ticker = ? AND notified = 1",
        (ticker,),
    ).fetchone()
    conn.close()
    if not row or not row[0]:
        return False
    try:
        return row[0][:10] == str(date.today())
    except Exception:
        return False

def _mark_notified(ticker: str):
    conn = sqlite3.connect(DB_PATH)
    conn.execute(
        "UPDATE signals SET notified = 1, notified_at = datetime('now') WHERE ticker = ?",
        (ticker,),
    )
    conn.commit()
    conn.close()

def run(dry_run: bool = False, no_ingest: bool = False, force_notify: bool = False):
    start = datetime.now()
    print(f"\n{'='*60}")
    print(f"  CongressWatch run — {start.strftime('%Y-%m-%d %H:%M')}")
    if dry_run:
        print("  MODE: dry-run (no LLM calls, no notifications)")
    print(f"{'='*60}\n")

    # 1. Ingest
    if not no_ingest:
        print("[ 1/9 ] Ingesting trades...")
        from ingest.capitol_trades import ingest
        summary = ingest(lookback_days=LOOKBACK_DAYS)
        print(f"        +{summary['new_trades']} new trades\n")
    else:
        print("[ 1/9 ] Skipping ingest (--no-ingest)\n")

    # 1b. Executive branch trades (OGE Form 278-T PDFs for POTUS/VP)
    if not no_ingest:
        print("[ 1b/9 ] OGE Form 278-T executive branch trades...")
        from ingest.presidential_oge import ingest_presidential_oge
        exec_summary = ingest_presidential_oge()
        if exec_summary.get("new_trades", 0) > 0:
            print(f"        +{exec_summary['new_trades']} new executive trades "
                  f"({exec_summary['pdfs_processed']} PDFs)\n")
        else:
            print(f"        no new trades (all known PDFs already processed)\n")

    # 2. Committees (weekly refresh if stale)
    print("[ 2/9 ] Committee assignments...")
    from ingest.committees import needs_refresh, refresh as refresh_committees
    if needs_refresh():
        summary = refresh_committees()
        print(f"        {summary['updated']} updated, {summary.get('no_data', 0)} with no data\n")
    else:
        print("        up to date (refreshed within 7 days)\n")

    # 3. Score (uses win_rates + market_caps from previous run)
    print("[ 3/9 ] Scoring signals...")
    from analysis.scorer import score_tickers
    signals = score_tickers()
    print(f"        {len(signals)} tickers scored\n")

    # 4. Prices + market caps
    print("[ 4/9 ] Fetching prices...")
    from tools.price_fetcher import fetch_and_store as fetch_prices
    price_summary = fetch_prices(min_score=SCORE_RESEARCH_THRESHOLD)
    print(f"        {price_summary['fetched']} updated, {price_summary['skipped']} already fresh\n")

    # 5. Win rates (uses today's price data; feeds into tomorrow's scoring)
    print("[ 5/9 ] Computing win rates...")
    from tools.win_rate import compute as compute_win_rates
    wr_summary = compute_win_rates()
    print(f"        {wr_summary['updated']} members updated, "
          f"{wr_summary['skipped']} skipped (insufficient data)\n")

    # 6. Research top signals (regular: all signals ≥ SCORE_RESEARCH_THRESHOLD)
    print("[ 6/9 ] Researching top signals...")
    research_map: dict[str, dict] = {}
    if not dry_run:
        from analysis.researcher import research_signal
        from datetime import timedelta

        cutoff = (datetime.now() - timedelta(days=LOOKBACK_DAYS)).strftime("%Y-%m-%d")
        conn = sqlite3.connect(DB_PATH)
        conn.row_factory = sqlite3.Row

        for sig in signals:
            if sig["total_score"] < SCORE_RESEARCH_THRESHOLD:
                break  # signals are sorted descending
            buyers = conn.execute(
                """
                SELECT DISTINCT t.member_id,
                    m.first_name || ' ' || m.last_name AS member_name,
                    m.party, m.chamber, m.committees,
                    MAX(t.amount_tier) AS amount_tier
                FROM trades t JOIN members m ON m.bioguide_id = t.member_id
                WHERE t.issuer_ticker = ? AND t.tx_type = 'buy' AND t.pub_date >= ?
                GROUP BY t.member_id
                """,
                (sig["ticker"], cutoff),
            ).fetchall()
            result = research_signal(sig, [dict(b) for b in buyers], dry_run=False)
            if result:
                research_map[sig["ticker"]] = result

        conn.close()
    else:
        print("        (skipped — dry-run mode)")
    print()

    # 6b. Deep research — top N scorers with risk_factors + recommended_action
    # Skips any ticker already deep-researched within DEEP_RESEARCH_DECAY_DAYS,
    # falling through to the next-ranked signal to spend the budget wisely.
    print(f"[ 6b/9 ] Deep research (top {DEEP_RESEARCH_TOP_N}, decay {DEEP_RESEARCH_DECAY_DAYS}d)...")
    if not dry_run:
        from analysis.researcher import research_top_n_deep
        notify_tier = [s for s in signals if s["total_score"] >= SCORE_NOTIFY_THRESHOLD]
        deep_results = research_top_n_deep(
            notify_tier, top_n=DEEP_RESEARCH_TOP_N, decay_days=DEEP_RESEARCH_DECAY_DAYS
        )
        for sig, result in deep_results:
            research_map[sig["ticker"]] = result  # update with richer deep result
            print(f"  {sig['ticker']}: deep research done — {result.get('recommended_action', '?')}")
        if not deep_results:
            print("  All top candidates within decay window — no deep research needed")
    else:
        print("        (skipped — dry-run mode)")
    print()

    # 6c. Predictive signals — behavioural analysis + forward watchlist per member
    print(f"[ 6c/9 ] Predictive signals (top {PREDICTOR_MAX_MEMBERS} members, decay {PREDICTOR_DECAY_DAYS}d)...")
    if not dry_run:
        from analysis.predictor import run_predictions, confirm_predictions, backfill_committee_alignment
        confirm_predictions()
        backfill_committee_alignment()
        pred_results = run_predictions(
            max_members=PREDICTOR_MAX_MEMBERS,
            decay_days=PREDICTOR_DECAY_DAYS,
        )
        for pr in pred_results:
            tickers = ", ".join(p["ticker"] for p in pr["predictions"])
            print(f"  {pr['member_name']}: {tickers}")
        if not pred_results:
            print("  All members within decay window — no predictions generated")

        # Fetch market caps + ETF/validity flags for predicted tickers
        print("  Fetching metadata for new predicted tickers...")
        from tools.price_fetcher import fetch_for_predicted
        meta = fetch_for_predicted()
        if meta["fetched"] or meta["invalid"]:
            print(f"  {meta['fetched']} metadata fetched, {meta['invalid']} filtered (ETF/invalid)")
    else:
        print("        (skipped — dry-run mode)")
    print()

    # 7. Generate report
    print("[ 7/9 ] Generating report...")
    from tools.generate_report import generate
    generate()
    print()

    # 7b. Update pending signal outcomes (30d/60d returns)
    print("[ 7b/9 ] Checking signal outcomes...")
    if not dry_run:
        from tools.outcome_tracker import update_pending_outcomes
        outcome_result = update_pending_outcomes()
        if outcome_result.get("updated"):
            print(f"        {outcome_result['updated']} outcomes updated")
        else:
            print("        no pending outcomes ready for scoring")
    else:
        print("        (skipped — dry-run mode)")
    print()

    # 8. Notify — apply timeliness, quality gates, and ticker cooldown before firing
    notified = skipped_tl = skipped_gate = skipped_cooldown = 0
    notify_candidates = [s for s in signals if s["total_score"] >= SCORE_NOTIFY_THRESHOLD]

    if not dry_run and notify_candidates:
        from analysis.notification_policy import (
            evaluate_notification_candidate,
            recent_ticker_notification,
        )

        print("[ 8/9 ] Sending notifications...")
        for sig in notify_candidates:
            if notified >= MAX_NOTIFICATIONS_PER_RUN:
                remaining = len(notify_candidates) - notified - skipped_tl - skipped_gate - skipped_cooldown
                if remaining > 0:
                    print(f"  Cap reached ({MAX_NOTIFICATIONS_PER_RUN}/run) — {remaining} remaining suppressed")
                break

            ticker = sig["ticker"]

            # Timeliness gate
            research = research_map.get(ticker)
            timeliness = (research or {}).get("timeliness") or "fresh"
            adj = _timeliness_adjusted_score(sig["total_score"], timeliness)
            if adj < SCORE_NOTIFY_THRESHOLD:
                tl_key = timeliness.split("|")[0].strip().lower()
                print(f"  {ticker}: {sig['total_score']:.1f} × {TIMELINESS_MULTIPLIERS.get(tl_key,1):.2f} = {adj:.1f} — below threshold ({tl_key})")
                skipped_tl += 1
                continue

            decision = evaluate_notification_candidate(sig, research, adj)
            if not decision.allowed:
                print(f"  {ticker}: {decision.reason} — skipping")
                skipped_gate += 1
                continue

            if not force_notify:
                in_cooldown, last_notified, days_left = recent_ticker_notification(ticker)
                if in_cooldown:
                    last_label = last_notified.isoformat(timespec="minutes") if last_notified else "recently"
                    print(
                        f"  {ticker}: notified {last_label}; "
                        f"{days_left}d left in {NOTIFY_TICKER_COOLDOWN_DAYS}d cooldown"
                    )
                    skipped_cooldown += 1
                    continue

            if not force_notify and _already_notified_today(ticker):
                print(f"  {ticker}: already notified today")
                continue

            _send_signal_notification(sig, research, adj)
            _mark_notified(ticker)

            # Record outcome for later tracking
            from tools.outcome_tracker import record_notification
            record_notification(
                ticker=ticker,
                score=adj,
                conviction=(research or {}).get("conviction", ""),
                recommended_action=(research or {}).get("recommended_action", ""),
                price=sig.get("price_current"),
                notified_at=datetime.now().isoformat(timespec="seconds"),
            )

            notified += 1
        print()

    elapsed = (datetime.now() - start).total_seconds()
    print(f"Run complete in {elapsed:.1f}s — {notified} notifications sent"
          + (f", {skipped_tl} suppressed by timeliness" if skipped_tl else "")
          + (f", {skipped_gate} filtered by quality gates" if skipped_gate else "")
          + (f", {skipped_cooldown} suppressed by cooldown" if skipped_cooldown else "") + ".")
    print(f"Report: reports/signals_latest.md\n")

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="CongressWatch daily runner")
    parser.add_argument("--dry-run",   action="store_true", help="no LLM calls, no notifications")
    parser.add_argument("--no-ingest", action="store_true", help="skip ingest step")
    parser.add_argument("--force",     action="store_true", help="re-notify even if notified today")
    args = parser.parse_args()
    run(dry_run=args.dry_run, no_ingest=args.no_ingest, force_notify=args.force)


Excellent! Now let me search for any TODO or FIXME comments in the Python source files.


curl -s "https://api.github.com/repos/jamesjlopez/congressional-trading/git/trees/main?recursive=1" | grep -o '"path":"[^"]*\.py"' | sort -u | head -30


Let me try a different approach to find files.


"""
Signal timeliness assessment.

Phase 1: age-based only (days since last disclosure pub_date).
Phase 2 (future): compare current price vs price at tx_date using a market data API.
"""

from datetime import datetime, date

def assess_age(last_pub_date: str) -> str:
    """
    Returns 'fresh', 'watch', or 'late' based on days since last disclosure.
      fresh : < 21 days  — thesis likely intact, meaningful runway
      watch : 21–45 days — still worth monitoring, may be partially priced in
      late  : > 45 days  — 45-day STOCK Act window fully elapsed, likely priced in
    """
    try:
        pub = datetime.strptime(last_pub_date[:10], "%Y-%m-%d").date()
    except (ValueError, TypeError):
        return "unknown"

    age_days = (date.today() - pub).days
    if age_days < 21:
        return "fresh"
    if age_days <= 45:
        return "watch"
    return "late"

def age_days(last_pub_date: str) -> int:
    try:
        pub = datetime.strptime(last_pub_date[:10], "%Y-%m-%d").date()
        return (date.today() - pub).days
    except (ValueError, TypeError):
        return -1


Great! Now let me check the notification client to see how notifications are formatted.


"""Push notifications via ntfy.sh."""
import requests
import sys
import os

sys.path.insert(0, os.path.dirname(os.path.dirname(__file__)))
from config import NTFY_BASE_URL, NTFY_TOPIC_SIGNALS, NTFY_TOPIC_STATUS

def push(title: str, body: str, topic: str = None, priority: str = "default", tags: list = None):
    topic = topic or NTFY_TOPIC_SIGNALS
    headers = {
        "Title": title,
        "Priority": priority,
    }
    if tags:
        headers["Tags"] = ",".join(tags)
    try:
        r = requests.post(f"{NTFY_BASE_URL}/{topic}", data=body.encode(), headers=headers, timeout=10)
        r.raise_for_status()
        return True
    except Exception as e:
        print(f"ntfy error: {e}", file=sys.stderr)
        return False

def waiting_on_user(message: str):
    """Notify that Claude Code needs user input to continue."""
    push(
        title="Claude Code needs your input",
        body=message,
        topic=NTFY_TOPIC_STATUS,
        priority="high",
        tags=["robot", "question"],
    )


Perfect! Now let me get the LLM client to see how it works.


"""
Abstracted LLM client.

Supports:
  - Anthropic (claude-sonnet-4-6) via ANTHROPIC_API_KEY
  - Google Gemini via GOOGLE_API_KEY

Provider selected by LLM_PROVIDER in config.py ("anthropic" or "google").
Returns plain text response. Raises LLMError on misconfiguration or API failure.
"""

import sys
import os

sys.path.insert(0, os.path.dirname(os.path.dirname(__file__)))
from config import LLM_PROVIDER, ANTHROPIC_API_KEY, GOOGLE_API_KEY

MODEL_ANTHROPIC = "claude-sonnet-4-6"
MODEL_GOOGLE    = "gemini-2.0-flash"

class LLMError(Exception):
    pass

def call(prompt: str, system: str = "", max_tokens: int = 1024) -> tuple[str, dict]:
    """
    Send a prompt, return (response_text, usage_dict).
    usage_dict keys: input_tokens, output_tokens, cache_read_tokens, cache_write_tokens
    """
    if LLM_PROVIDER == "anthropic":
        return _call_anthropic(prompt, system, max_tokens=max_tokens)
    if LLM_PROVIDER == "google":
        return _call_google(prompt, system, max_tokens=max_tokens)
    raise LLMError(f"Unknown LLM_PROVIDER: {LLM_PROVIDER!r}")

def _call_anthropic(prompt: str, system: str, max_tokens: int = 1024) -> tuple[str, dict]:
    if not ANTHROPIC_API_KEY:
        raise LLMError(
            "ANTHROPIC_API_KEY is not set. "
            "Export it: export ANTHROPIC_API_KEY=sk-ant-..."
        )
    try:
        import anthropic
    except ImportError:
        raise LLMError("anthropic package not installed. Run: pip install anthropic")

    client = anthropic.Anthropic(api_key=ANTHROPIC_API_KEY)
    kwargs = {"model": MODEL_ANTHROPIC, "max_tokens": max_tokens, "messages": [{"role": "user", "content": prompt}]}
    if system:
        kwargs["system"] = system

    msg = client.messages.create(**kwargs)
    text = msg.content[0].text
    usage = {
        "input_tokens":       msg.usage.input_tokens,
        "output_tokens":      msg.usage.output_tokens,
        "cache_read_tokens":  getattr(msg.usage, "cache_read_input_tokens", 0) or 0,
        "cache_write_tokens": getattr(msg.usage, "cache_creation_input_tokens", 0) or 0,
    }
    return text, usage

def _call_google(prompt: str, system: str, max_tokens: int = 1024) -> tuple[str, dict]:
    if not GOOGLE_API_KEY:
        raise LLMError(
            "GOOGLE_API_KEY is not set. "
            "Export it: export GOOGLE_API_KEY=..."
        )
    try:
        from google import genai
        from google.genai import types
    except ImportError:
        raise LLMError("google-genai package not installed. Run: pip install google-genai")

    client = genai.Client(api_key=GOOGLE_API_KEY)
    config = types.GenerateContentConfig(
        system_instruction=system or None,
        max_output_tokens=max_tokens,
    )
    resp = client.models.generate_content(
        model=MODEL_GOOGLE,
        contents=prompt,
        config=config,
    )
    usage = {
        "input_tokens":       getattr(resp.usage_metadata, "prompt_token_count", 0) or 0,
        "output_tokens":      getattr(resp.usage_metadata, "candidates_token_count", 0) or 0,
        "cache_read_tokens":  0,
        "cache_write_tokens": 0,
    }
    return resp.text, usage


Perfect! Now let me search for any hardcoded thresholds or TODOs by checking a few more key files.


# Copy this to .env and fill in your keys
# .env is gitignored — never commit it

ANTHROPIC_API_KEY=sk-ant-...
GOOGLE_API_KEY=
CONGRESS_GOV_API_KEY=    # free at https://api.congress.gov/sign-up/ — enables live bill data for predictive signals

# ntfy.sh push topics — change these to something unique if you use ntfy.sh cloud
NTFY_TOPIC_SIGNALS=
NTFY_TOPIC_STATUS=


Now let me check if there are test files or example data.


curl -s "https://api.github.com/repos/jamesjlopez/congressional-trading/contents/tests" 2>&1 | head -50


Excellent! Let me get the test file to see example signals.


from __future__ import annotations

from datetime import datetime, timedelta
import os
import sqlite3
import tempfile
import unittest

from analysis.notification_policy import (
    edge_factors,
    evaluate_notification_candidate,
    is_buy_recommendation,
    recent_ticker_notification,
)

def _signal(**overrides):
    sig = {
        "ticker": "EDGE",
        "member_count": 2,
        "max_tier": 4,
        "has_options": False,
        "committee_score": 3.0,
        "small_cap_score": 0.0,
        "sell_pressure_score": 0.0,
        "strategic_score": 3.0,
        "disruption_score": 1.0,
    }
    sig.update(overrides)
    return sig

def _research(**overrides):
    research = {
        "conviction": "HIGH",
        "recommended_action": "BUY NOW - catalyst remains timely",
        "risk_factors": "Valuation and execution risk.",
        "strategic_score": 3,
        "disruption_score": 1,
    }
    research.update(overrides)
    return research

class NotificationPolicyTest(unittest.TestCase):
    def test_allows_explicit_high_signal_buy(self):
        decision = evaluate_notification_candidate(_signal(), _research(), 11.0)

        self.assertTrue(decision.allowed)
        self.assertGreaterEqual(len(decision.edge_factors), 3)
        self.assertIn("multi-member cluster", decision.edge_factors)
        self.assertIn("direct committee alignment", decision.edge_factors)

    def test_rejects_monitor_or_not_buy_text(self):
        self.assertFalse(is_buy_recommendation("MONITOR - do not buy until earnings"))
        self.assertFalse(is_buy_recommendation("Do not buy despite the catalyst"))

        decision = evaluate_notification_candidate(
            _signal(),
            _research(recommended_action="MONITOR - wait before buying"),
            11.0,
        )

        self.assertFalse(decision.allowed)
        self.assertIn("not an explicit buy", decision.reason)

    def test_rejects_high_score_without_enough_edge_factors(self):
        sig = _signal(
            member_count=1,
            max_tier=2,
            has_options=False,
            committee_score=1.0,
            small_cap_score=0.0,
            strategic_score=1.0,
            disruption_score=1.0,
        )
        research = _research(strategic_score=1, disruption_score=1)

        decision = evaluate_notification_candidate(sig, research, 12.0)

        self.assertFalse(decision.allowed)
        self.assertIn("edge factors", decision.reason)
        self.assertEqual(edge_factors(sig, research), ())

    def test_rejects_meaningful_sell_pressure(self):
        decision = evaluate_notification_candidate(
            _signal(sell_pressure_score=-1.5),
            _research(),
            12.0,
        )

        self.assertFalse(decision.allowed)
        self.assertIn("sell pressure", decision.reason)

    def test_recent_ticker_notification_uses_rolling_cooldown(self):
        now = datetime(2026, 6, 5, 12, 0, 0)
        with tempfile.TemporaryDirectory() as tmp:
            db_path = os.path.join(tmp, "signals.db")
            conn = sqlite3.connect(db_path)
            conn.execute(
                """
                CREATE TABLE signal_outcomes (
                    ticker TEXT NOT NULL,
                    notified_at TEXT NOT NULL
                )
                """
            )
            conn.execute(
                "INSERT INTO signal_outcomes (ticker, notified_at) VALUES (?, ?)",
                ("AMD", (now - timedelta(days=3)).isoformat(timespec="seconds")),
            )
            conn.commit()
            conn.close()

            in_cooldown, latest, days_left = recent_ticker_notification(
                "AMD",
                db_path=db_path,
                cooldown_days=14,
                now=now,
            )
            stale, _, stale_days_left = recent_ticker_notification(
                "MSI",
                db_path=db_path,
                cooldown_days=14,
                now=now,
            )

        self.assertTrue(in_cooldown)
        self.assertEqual(latest, now - timedelta(days=3))
        self.assertEqual(days_left, 11)
        self.assertFalse(stale)
        self.assertEqual(stale_days_left, 0)

if __name__ == "__main__":
    unittest.main()


Perfect! Now I have all the key information. Let me compile my comprehensive summary.

## Comprehensive Analysis of Congressional-Trading Repository

Based on my thorough exploration of the codebase, here's the structured summary:

---

### **1. SCORING/SIGNAL SYSTEM — What Makes a Trade "High Conviction"**

**Core Algorithm** (`analysis/scorer.py`):
The system scores tickers on a 0-13 point scale based on aggregate congressional activity:

| Component | Range | Logic |
|-----------|-------|-------|
| **committee_score** | 0–3 | Member's committee has jurisdiction over company sector (Armed Services + defense contractor = 3) |
| **track_record_score** | 0–2 | Historical win rate; defaults to 1.0 until data available |
| **cluster_score** (with breadth discount) | 0–3 | 2 buyers=1pt, 3=2pts, 4+=3pts. Multiplied by breadth discount (0.4–1.0) to penalize mega-caps routinely held by 15+ members across history |
| **options_bonus** | 0 or 2 | Any options trade on ticker (+2 — implies timing knowledge) |
| **size_score** | 0–5 | Highest dollar tier: $1K-$15K=1, $15K-$50K=2, ..., $500K+=5 |
| **small_cap_bonus** | −2.5 to +6 | Market cap tier incentive: mega-cap >$100B = -2.5, large $10-100B = -0.5, mid $2-10B = +2.0, small $500M-$2B = +4.5, micro <$500M = +6.0 |
| **sell_pressure_penalty** | −4.0 to 0 | Penalty when sellers ≥ buyers (e.g., 1 of 3 sellers = -0.5, ratio ≥1.0 = -4.0) |
| **strategic_score** | 0–3 | LLM assessment: 0=no national security/infra nexus, 1=indirect, 2=defense/energy contractor with gov exposure, 3=direct military/dual-use |
| **disruption_score** | 0–3 | LLM assessment: 0=incremental, 1=meaningful differentiation, 2=significant market-shift, 3=paradigm-shifting |

**Score Thresholds** (`config.py`):
- `SCORE_RESEARCH_THRESHOLD = 5.0` → Triggers LLM research (shallow: company summary, news, catalyst, timeliness)
- `SCORE_NOTIFY_THRESHOLD = 9.0` → Eligible for push notifications (before quality gates)
- `SCORE_NOTIFY_THRESHOLD` signals also get **deep research** (adds: risk_factors, recommended_action)

**Key Subtlety**: The raw `total_score` is stored unchanged in the signals table for historical consistency. **Timeliness multipliers are applied only at notification-decision time** (never at scoring time).

---

### **2. SIGNAL NOTIFICATION — What Data Is Included**

**Notification Format** (from `run_daily.py:_send_signal_notification`):
```
Title: CongressWatch: $TICKER (score X.X)
Body:
  - Company name & member count (e.g., "1 member bought", "3 members bought")
  - Size tier ($1K-$15K, etc.)
  - Options flag if present
  - Timeliness (FRESH <21d | WATCH 21-45d | LATE >45d)
  - Conviction (LOW | MEDIUM | HIGH from deep research)
  - Recommended action (BUY NOW | MONITOR | AVOID)
  - Company summary (max 300 chars from LLM)
  - Catalyst hypothesis (max 200 chars from LLM)
  - Full recommended action text (max 40 chars)

Priority: HIGH if adjusted_score ≥ 9, else DEFAULT
Tags: chart_with_upwards_trend, money_with_wings
Topic: ntfy.sh (configurable, defaults to "congressWatchSignals")
```

**Notification Gating** (ALL gates must pass):
1. Adjusted score ≥ `SCORE_NOTIFY_THRESHOLD` (9.0 default) after timeliness multiplier
2. Deep research completed with `risk_factors` and `recommended_action` fields
3. Conviction ≥ `NOTIFY_MIN_CONVICTION` (HIGH by default; can be LOW/MEDIUM/HIGH)
4. `recommended_action` starts with explicit BUY/ADD/ACCUMULATE/INITIATE (not MONITOR/HOLD/PASS/AVOID)
5. Sell pressure not too conflicting: `sell_pressure_score ≥ -1.0` (default)
6. At least `NOTIFY_MIN_EDGE_FACTORS` (3 default) independent edge factors:
   - Multi-member cluster (≥2 buyers)
   - Large disclosed size (tier ≥4, i.e., $100K+)
   - Options activity
   - Direct committee alignment (score ≥2)
   - Undercovered small/mid-cap (score ≥2)
   - Strong LLM catalyst (strategic + disruption ≥4)
7. Ticker not notified within `NOTIFY_TICKER_COOLDOWN_DAYS` (14 days default)
8. `MAX_NOTIFICATIONS_PER_RUN = 1` — only single strongest candidate fires per run

**Timeliness Multipliers** (applied only at notification gate):
```python
TIMELINESS_MULTIPLIERS = {
    "fresh": 1.0,   # < 21 days since disclosure
    "watch": 0.85,  # 21–45 days
    "late": 0.70,   # > 45 days
}
```
Example: score 10.5 with "late" timeliness → 10.5 × 0.70 = 7.35 → below 9.0 threshold, not notified.

---

### **3. SHADOW/PREDICTIVE MODE — Forward-Looking Watchlist**

**NOT a traditional "shadow mode"** but rather a **predictive/anticipatory signal pipeline** (`analysis/predictor.py`):

**What it tracks**:
- Predicted upcoming trades BEFORE disclosure happens
- Stores in `predicted_signals` table (separate from reactive `signals`)
- Per-member behavioral profiles + predicted ticker watchlists

**How it works**:
1. Analyzes top members with ≥5 historical trades + committee data
2. Pulls 3 years of trade history + active legislation relevant to their committees
3. Calls LLM: "What will this member likely trade in next 45 days?"
4. Stores 3-5 predicted tickers per member with:
   - `predicted_action` (buy/sell)
   - `confidence` (LOW/MEDIUM/HIGH)
   - `legislation_catalyst` (specific bill or "General sector exposure")
   - `historical_precedent` (prior similar trades)
   - `behavioral_profile` (member's trading style)
   - `committee_alignment` (direct/indirect/none vs predicted sector)

**Backtesting/Confirmation Mechanism**:
- When real disclosures come in via ingest, `confirm_predictions()` marks matching predicted tickers as `confirmed=1`
- Tracks accuracy (confirmed vs total predictions)
- Feeds into consensus scoring: `load_consensus()` aggregates predictions by ticker and scores them

**Consensus Scoring** (for predicted-only signals):
Combines cluster size, confidence, small-cap bonus, win rate, committee alignment, and historical precedent to rank predicted tickers alongside reactive signals in the final report.

---

### **4. HISTORICAL TRADE DATA — Storage & Backtesting Support**

**Data Stored**:
- `trades` table: append-only source data (14k+ trades from Capitol Trades HTML scraper)
  - `tx_date` (when trade executed)
  - `pub_date` (when disclosure published — lag up to 45 days under STOCK Act)
  - `amount_tier` (1-6 derived from dollar range)
  - `price_at_trade` (attempted from historical yfinance lookups)
  - `is_option` (0/1 flag)

- `signals` table: rebuilt from scratch every scoring run
  - Aggregates trades per ticker within lookback window
  - Stores all score components + `return_from_tx_pct` and `return_from_pub_pct`

- `signal_outcomes` table: persistent tracking of notifications sent
  - Records score at notification time
  - Tracks 30d/60d post-notification returns (for win/loss assessment)
  - Marks outcome as "win"/"loss"/"pending"

- `predicted_signals` table: forward-looking watchlist
  - `confirmed` flag (1 when real disclosure matches prediction)
  - Tracks prediction accuracy over time

**Backtest Readiness**:
✓ Full historical trade data available (append-only `trades` table)
✓ Score reproduction is deterministic (all scoring logic hardcoded)
✓ Return calculations stored (`return_from_tx_pct`, `return_from_pub_pct`)
✓ Timeliness-adjusted scores can be recomputed (multipliers in config)
⚠ **Gap**: Win rate currently uses current price snapshots, not fixed-horizon returns (see backlog item)

---

### **5. GRADUATION CRITERIA — Shadow to Real Notification**

**There is no explicit "shadow mode" state transition.** Instead:

**Regular Scoring Pipeline** (always runs):
- All tickers with buy activity in lookback window get scored
- Signals ≥5.0 are researched
- Signals ≥9.0 are candidates for deep research and notification

**Notification Eligibility** (at decision time):
A signal "graduates" to notification when it satisfies ALL gates listed in section 2 above. The key transitions are:
1. Score rises above 5.0 → Basic LLM research triggered
2. Score rises above 9.0 → Eligible for deep research + notification gates
3. All 7 quality gates + cooldown check pass → Notification fires (max 1/run)

**No persistent "shadow state"** — signals are scored fresh each run based on new trades.

---

### **6. EXAMPLE SIGNALS & TEST FIXTURES**

**Test File** (`tests/test_notification_policy.py`):
```python
# Example high-conviction signal that passes all gates:
sig = {
    "ticker": "EDGE",
    "member_count": 2,        # multi-member cluster (edge factor ✓)
    "max_tier": 4,             # $100K-$250K (large size edge factor ✓)
    "committee_score": 3.0,    # direct alignment (edge factor ✓)
    "small_cap_score": 0.0,
    "sell_pressure_score": 0.0,
    "strategic_score": 3.0,    # LLM catalyst (edge factor ✓)
    "disruption_score": 1.0,
}
research = {
    "conviction": "HIGH",      # passes conviction gate ✓
    "recommended_action": "BUY NOW - catalyst remains timely",  # explicit buy ✓
    "risk_factors": "Valuation and execution risk.",  # has risk factors ✓
}
score = 11.0  # passes threshold gate ✓
# Result: ALLOWED (4 edge factors, all gates pass)
```

---

### **7. OPEN DECISIONS & GAPS IN THE CODE**

**From `backlog.md`**:

| Issue | Impact | Status |
|-------|--------|--------|
| **Committee-sector mapping too coarse** | Over-broad heuristic matches (e.g., Armed Services → any tech stock). Solution: per-trade LLM committee_score, fallback to heuristic. | Deferred (cost + prompt design concern) |
| **Win rate is snapshot, not realized** | Current win rate uses today's price, not fixed-horizon returns. Affects member credibility scoring. | Deferred (14k+ yfinance calls needed) |
| **Signal return uses first_tx_date, not per-member** | Multi-buyer signals compute return from earliest buyer's trade date, not each member's individual entry. | Moderate impact, deferred |
| **No sell signals** | Sell-count tracked but not acted on. Could fire separate EXIT alerts. | Feature request |
| **Options direction not distinguished** | Calls get +2 bonus same as puts. Puts should warrant separate alert (inverse signal). | Feature request |
| **No sector-wide clustering** | 5+ members buying same sector in one week not flagged (potential macro signal). | Feature request |

**Hardcoded Values to Review**:
- `_COMMITTEE_SECTOR_MAP` in `scorer.py` (hand-coded committee→sector mappings)
- `_TIER_LABELS` in `researcher.py` ($1K-$15K, etc. tier ranges)
- `RESEARCH_PROMPT_TEMPLATE` in `researcher.py` (strategic/disruption score guides 0-3)
- `PREDICT_PROMPT_TEMPLATE` in `predictor.py` (member behavior analysis prompt)

**TODOs & Future Work**:
- Phase 2 of timeliness: compare current price vs. price-at-tx_date (currently age-based only)
- LLM-derived committee_score per trade (not just heuristic)
- Sell signal alerts
- Options call/put differentiation
- Sector-level clustering alerts

---

### **8. ALGORITHM ASSESSMENT FOR BACKTESTING**

**Strengths for backtesting**:
- ✓ Fully deterministic scoring (no randomness, all rules explicit)
- ✓ Historical trade data complete (14k+ records with tx_date, pub_date, amount_tier)
- ✓ Return data stored (can reconstruct tx-date and pub-date returns)
- ✓ LLM components can be cached in `research_cache` (no re-calls on replays)
- ✓ Timeliness multipliers configurable (can test different decay profiles)

**Gaps for backtesting**:
- ⚠ Win rate requires fixed-horizon returns (e.g., 60-day post-trade); currently uses current price snapshots
- ⚠ Market cap data fetched once per run, cached; not available for all historical tickers
- ⚠ LLM research (strategic_score, disruption_score) cached but may reflect current knowledge, not knowledge-as-of-trade-date
- ⚠ Committee assignments refresh weekly; historical committee data for past years may not be available
- ⚠ `small_cap_bonus` is recalculated on second run after prices fetched; first-run results show 0 for all signals

**Recommended approach for backtest**:
1. Use pure scoring components (committee, cluster, size, track record)
2. Skip or mock LLM scores (strategic/disruption) or use cached values as-is
3. Reconstruct returns using historical price data for each signal's tx_date and pub_date
4. Test notification gate with varying threshold/conviction/edge-factor configs
5. Track outcome vs. 30/60d returns using `signal_outcomes` table

---

**Key Files for Reference**:
- `/analysis/scorer.py` — Core scoring algorithm (16KB, fully readable)
- `/analysis/notification_policy.py` — Notification quality gates (5.7KB)
- `/analysis/researcher.py` — LLM research pipeline & deep prompts (21.5KB)
- `/analysis/predictor.py` — Predictive/forward-looking signals (28.8KB)
- `/config.py` — All tunable thresholds & API keys
- `/run_daily.py` — Full pipeline orchestration
- `/db/schema.sql` — Data model