"""Tests for the OUTLIERS chapter — DoD: golden + edges + error path.

Self-contained: builds synthetic ``numeric`` blocks + a raw_numeric sample (no
DuckDB) so the suite is fast and deterministic. Verifies that the chapter emits
the univariate per-column table, a boxplots figure, the multivariate Isolation
Forest section and the outlier≠error interpretation; that the most contaminated
column is ranked first; that a profile with no numeric column yields None; that
None/empty never raises; that the glossary terms are registered; and that the
chapter renders into both PDF and PPTX without cutting its title.
"""

import math
import os
import re
import tempfile

from pypdf import PdfReader

from datascience.automatic_eda.chapters.outliers import (
    build_outliers, CHAPTER_VERSION, CHAPTER_TITLE, _TERM_DEFS,
)
from datascience.automatic_eda import model
from datascience.render_automatic_eda_pdf import render_automatic_eda_pdf
from datascience.render_automatic_eda_pptx import render_automatic_eda_pptx


def _percentile(sorted_vals, q):
    """Linear-interpolation percentile (q in 0..1) on an already-sorted list."""
    if not sorted_vals:
        return None
    if len(sorted_vals) == 1:
        return float(sorted_vals[0])
    pos = q * (len(sorted_vals) - 1)
    lo = int(math.floor(pos))
    hi = int(math.ceil(pos))
    if lo == hi:
        return float(sorted_vals[lo])
    frac = pos - lo
    return float(sorted_vals[lo] * (1 - frac) + sorted_vals[hi] * frac)


def _col_from_values(values, nbins=10):
    """Build a ``numeric`` sub-block shaped like describe_numeric's output from a
    concrete list of raw values, so the profile percentiles and the raw sample
    are consistent (the boxplot fences match the crudo)."""
    vals = [float(v) for v in values]
    s = sorted(vals)
    n = len(s)
    mean = sum(vals) / n
    var = sum((v - mean) ** 2 for v in vals) / n
    std = math.sqrt(var)
    median = _percentile(s, 0.5)
    p25 = _percentile(s, 0.25)
    p75 = _percentile(s, 0.75)
    mn, mx = s[0], s[-1]
    # z-score outlier count (population), what the profile's n_outliers carries.
    n_out = sum(1 for v in vals if std > 0 and abs((v - mean) / std) > 3.0)
    width = (mx - mn) / nbins if mx > mn else 1.0
    hist = [{"lo": mn + i * width, "hi": mn + (i + 1) * width, "count": 1}
            for i in range(nbins)]
    return {
        "min": mn, "max": mx, "mean": mean, "median": median, "std": std,
        "p25": p25, "p50": median, "p75": p75, "iqr": (p75 - p25),
        "n_outliers": n_out, "outlier_pct": 100.0 * n_out / n,
        "distribution_type": "right-skewed", "histogram": hist,
    }


def _fare_values():
    """A heavy-tailed column (most ~10-30, a few 200-512): clear Tukey/z outliers."""
    base = [7.0 + (i % 25) for i in range(120)]      # bulk 7..31
    tail = [180.0, 210.0, 263.0, 512.0]              # extreme upper tail
    return base + tail


def _age_values():
    """A roughly symmetric column with one extreme low value."""
    base = [22.0 + (i % 40) for i in range(120)]     # 22..61
    return base + [80.0, 0.5, 74.0, 1.0]


def _quiet_values():
    """A clean column with no atypical values."""
    return [50.0 + (i % 5) for i in range(124)]


def _profile_and_ctx(with_models=True, with_raw=True):
    fare = _fare_values()
    age = _age_values()
    quiet = _quiet_values()
    cols = [
        {"name": "Fare", "inferred_type": "numeric", "numeric": _col_from_values(fare)},
        {"name": "Age", "inferred_type": "numeric", "numeric": _col_from_values(age)},
        {"name": "Quiet", "inferred_type": "numeric", "numeric": _col_from_values(quiet)},
        {"name": "Sexo", "inferred_type": "categorical",
         "categorical": {"top": [{"value": "male", "count": 80}]}},
    ]
    profile = {"table": "titanic", "n_rows": len(fare), "n_cols": len(cols),
               "columns": cols}
    if with_models:
        profile["models"] = {
            "outliers": {
                "n_outliers": 4, "outlier_pct": 3.2,
                "outlier_rows": [
                    {"row_index": 123, "score": -0.21},
                    {"row_index": 121, "score": -0.15},
                ],
                "threshold": -0.02, "n_rows_used": 124, "n_features": 3,
            }
        }
    ctx = {}
    if with_raw:
        ctx["raw_numeric"] = {"Fare": fare, "Age": age, "Quiet": quiet}
    return profile, ctx


def _pdf_text(path: str) -> str:
    txt = "".join((pg.extract_text() or "") for pg in PdfReader(path).pages)
    return re.sub(r"\s+", " ", txt)


def _flatten(blocks):
    out = []
    for b in blocks:
        if getattr(b, "kind", "") == "group":
            out.extend(_flatten(getattr(b, "blocks", []) or []))
        else:
            out.append(b)
    return out


# --------------------------------------------------------------------------- #
# Golden.
# --------------------------------------------------------------------------- #
def test_golden_estructura_y_secciones():
    profile, ctx = _profile_and_ctx()
    ctx["glossary"] = model.GlossaryCollector()
    ch = build_outliers(profile, ctx)
    assert ch is not None
    assert ch.id == "outliers"
    assert ch.version == CHAPTER_VERSION

    flat = _flatten(ch.blocks)
    kinds = [b.kind for b in flat]
    # Title heading + univariate DataTable + boxplots Figure + multivariate
    # KVTable + interpretation Markdown.
    assert kinds[0] == "heading" and flat[0].text == CHAPTER_TITLE
    tables = [b for b in flat if b.kind == "data_table"]
    titles = [t.title for t in tables]
    assert any(t and "atípicos por columna" in t for t in titles)
    assert any(b.kind == "figure" for b in flat), "falta la figura de boxplots"
    assert any(b.kind == "kv_table" for b in flat), "falta el resumen multivariante"

    # The boxplots figure maker yields a real matplotlib figure (or its fallback).
    fig = next(b for b in flat if b.kind == "figure").make()
    assert fig is not None
    import matplotlib.pyplot as plt
    plt.close(fig)


def test_golden_fare_es_la_mas_contaminada():
    # The univariate table must rank Fare (heavy tail) first and report a
    # non-zero Tukey percentage for it.
    profile, ctx = _profile_and_ctx()
    ch = build_outliers(profile, ctx)
    table = next(b for b in _flatten(ch.blocks)
                 if b.kind == "data_table" and b.title
                 and "atípicos por columna" in b.title)
    first_col = table.rows[0][0]
    assert first_col == "Fare", f"esperaba Fare primera, fue {first_col}"
    # % Tukey column (index 2) of the first row must be > 0.
    pct_cell = table.rows[0][2]
    assert pct_cell not in ("—", "0%", "0.00%"), f"% Tukey de Fare vacío: {pct_cell}"
    # The z-score rule (detect_outliers) must actually run with raw_numeric: at
    # least one column reports a non-empty z count/percentage (regression guard
    # for the detect_outliers import path).
    z_pcts = [r[4] for r in table.rows]
    assert any(c not in ("—",) for c in z_pcts), f"columna z toda vacía: {z_pcts}"
    z_counts = [r[3] for r in table.rows]
    assert any(c not in ("—",) for c in z_counts), f"conteo z vacío: {z_counts}"


def test_golden_interpretacion_outlier_no_es_error():
    profile, ctx = _profile_and_ctx()
    ch = build_outliers(profile, ctx)
    md = " ".join(b.text for b in _flatten(ch.blocks) if b.kind == "markdown")
    assert "no es necesariamente un error" in md.lower()
    # Mentions the actionable options (winsorize / re-express).
    assert "winsoriz" in md.lower()
    assert "re-expres" in md.lower() or "logarítmic" in md.lower()


def test_golden_terminos_glosario_registrados():
    profile, ctx = _profile_and_ctx()
    gloss = model.GlossaryCollector()
    ctx["glossary"] = gloss
    build_outliers(profile, ctx)
    for key in _TERM_DEFS:
        assert gloss.has(key), f"término '{key}' no registrado en el glosario"
    # Terms are marked clickable in the body text.
    md = " ".join(b.text for b in _flatten(build_outliers(profile, ctx).blocks)
                  if b.kind == "markdown")
    assert "[[term:outlier]]" in md and "[[term:tukey_fence]]" in md


# --------------------------------------------------------------------------- #
# Multivariate.
# --------------------------------------------------------------------------- #
def test_multivariante_live_con_raw_y_dims():
    # With a raw sample the chapter runs Isolation Forest live (over the same
    # columns summarize_outlier_dims uses) and lists the anomalous rows with the
    # dimensions that make each one rare.
    profile, ctx = _profile_and_ctx(with_models=False, with_raw=True)
    ch = build_outliers(profile, ctx)
    flat = _flatten(ch.blocks)
    kv = next(b for b in flat if b.kind == "kv_table")
    flat_kv = " ".join(f"{k} {v}" for (k, v) in kv.rows)
    assert "Filas atípicas" in flat_kv
    # A non-zero number of anomalous rows is reported.
    n_cell = dict(kv.rows).get("Filas atípicas")
    assert n_cell not in (None, "—", "0"), f"sin filas atípicas: {n_cell}"
    # The anomalous-rows table carries the per-row dimension breakdown.
    tbls = [b for b in flat if b.kind == "data_table" and b.title
            and "más atípicas" in b.title]
    assert tbls, "falta la tabla de filas más atípicas"
    assert any("hacen rara" in h for h in tbls[0].header), \
        f"falta la columna de dimensiones: {tbls[0].header}"


def test_multivariante_precomputed_sin_raw():
    # Without a raw sample the chapter falls back to profile['models']['outliers']
    # (lite preset path); the precomputed n_outliers (4) surfaces in the KV table.
    profile, ctx = _profile_and_ctx(with_models=True, with_raw=False)
    ch = build_outliers(profile, ctx)
    kv = next(b for b in _flatten(ch.blocks) if b.kind == "kv_table")
    assert any("4" in str(v) for (k, v) in kv.rows)


def test_multivariante_ausente_degrada_a_nota():
    # No models and no raw sample → an honest note, never a crash.
    profile, ctx = _profile_and_ctx(with_models=False, with_raw=False)
    ch = build_outliers(profile, ctx)
    assert ch is not None
    notes = [b.text for b in _flatten(ch.blocks) if b.kind == "note"]
    assert any("Isolation Forest" in n for n in notes)


# --------------------------------------------------------------------------- #
# Edges / error path.
# --------------------------------------------------------------------------- #
def test_edge_sin_columnas_numericas_devuelve_none():
    prof = {"columns": [{"name": "c", "inferred_type": "categorical",
                         "categorical": {"top": [{"value": "x", "count": 3}]}}]}
    assert build_outliers(prof, {}) is None


def test_edge_solo_texto_sintetico_devuelve_none():
    # A text-only synthetic table (no numeric column) yields None (does not break).
    prof = {"table": "notas", "n_rows": 3, "n_cols": 1,
            "columns": [{"name": "comentario", "inferred_type": "text",
                         "text": {"n_docs": 3}}]}
    assert build_outliers(prof, {}) is None


def test_edge_profile_none_y_vacio_no_revienta():
    assert build_outliers(None, None) is None
    assert build_outliers({}, {}) is None
    assert build_outliers({"columns": []}, {}) is None


def test_edge_sin_raw_numeric_degrada_a_perfil():
    # Without raw_numeric the chapter still builds, using the profile z-score
    # counts; the univariate table exists and Tukey counts degrade to '—'.
    profile, ctx = _profile_and_ctx(with_models=True, with_raw=False)
    ch = build_outliers(profile, ctx)
    assert ch is not None
    table = next(b for b in _flatten(ch.blocks)
                 if b.kind == "data_table" and b.title
                 and "atípicos por columna" in b.title)
    # z column comes from the profile; Tukey count is unknown ('—').
    assert all(len(r) == 8 for r in table.rows)


# --------------------------------------------------------------------------- #
# Anti-cut render.
# --------------------------------------------------------------------------- #
def test_render_pdf_y_pptx_incluyen_el_capitulo():
    profile, ctx = _profile_and_ctx()
    # The renderers build the whole document; the chapter is reached via the
    # registry. Render the chapter standalone through a one-chapter document by
    # passing the profile directly (the renderers run the full chapter registry).
    with tempfile.TemporaryDirectory() as d:
        pdf = os.path.join(d, "out.pdf")
        res_pdf = render_automatic_eda_pdf(profile, pdf,
                                           {"write_manifest": False, "ctx": ctx})
        assert res_pdf["path"] == pdf
        txt = _pdf_text(pdf)
        assert CHAPTER_TITLE in txt, "el capítulo OUTLIERS no aparece en el PDF"
        assert "Fare" in txt
        pptx = os.path.join(d, "out.pptx")
        res_pptx = render_automatic_eda_pptx(profile, pptx,
                                             {"write_manifest": False, "ctx": ctx})
        assert res_pptx["path"] == pptx
        assert res_pptx["n_slides"] >= 1
