merge(eda): scatters de pares correlacionados + tipo de relacion en cap CORRELACION

python/functions/datascience/automatic_eda/chapters/correlacion.py

@@ -31,7 +31,7 @@ import math
 
 from .. import model
 
-CHAPTER_VERSION = "1.0.0"
+CHAPTER_VERSION = "1.1.0"
 CHAPTER_ID = "correlacion"
 CHAPTER_TITLE = "Correlación"
 
@@ -47,6 +47,13 @@ _MAX_MATRIX_LABELS = 16
 # How many pairs to show in each of the top-positive / top-negative tables.
 _TOP_N = 10
 
+# How many of the strongest numeric-numeric pairs to draw as scatter plots on
+# each sign (positive / negative). A scatter per pair carries a fitted line/curve
+# and a relationship-type label; keeping the count small keeps the chapter
+# readable on a phone / a slide. Only signed (Pearson/Spearman) pairs qualify —
+# Cramér's V / correlation ratio pairs are not numeric-numeric, so no scatter.
+_SCATTER_TOP_N = 3
+
 # Glossary terms this chapter explains. Each is registered in the shared
 # collector (ctx['glossary']) and marked clickable on its first appearance in the
 # body — the canonical two-step pattern (see ``cat_distr`` for the reference
@@ -314,6 +321,139 @@ def _fdr_text(corr: dict, mark_term: bool = False) -> str | None:
     return " ".join(parts)
 
 
+def _is_seq(values) -> bool:
+    """True for a non-empty list/tuple of values (a raw numeric column)."""
+    return isinstance(values, (list, tuple)) and len(values) > 0
+
+
+def _select_scatter_pairs(pairs: list, top_n: int = _SCATTER_TOP_N):
+    """Pick the strongest numeric-numeric pairs to draw as scatters.
+
+    Only signed (Pearson/Spearman) pairs are numeric-numeric and thus eligible
+    for a scatter with a fitted curve. Returns up to ``top_n`` of the strongest
+    positive pairs followed by up to ``top_n`` of the strongest negative ones,
+    each ranked by magnitude. Mixed-type metrics (Cramér's V, correlation ratio,
+    mutual information) are excluded — they have no x/y scatter interpretation.
+    """
+    positive = []
+    negative = []
+    for pair in pairs:
+        if not isinstance(pair, dict) or not _is_signed(pair):
+            continue
+        value = pair.get("value")
+        if not _is_num(value):
+            continue
+        if value > 0:
+            positive.append(pair)
+        elif value < 0:
+            negative.append(pair)
+    positive.sort(key=lambda p: abs(float(p.get("value", 0.0))), reverse=True)
+    negative.sort(key=lambda p: abs(float(p.get("value", 0.0))), reverse=True)
+    return positive[:top_n] + negative[:top_n]
+
+
+def _classification_note(a: str, b: str, cls: dict) -> str:
+    """Human-readable sentence describing the relationship of a pair.
+
+    Plain text (not baked into the figure image) so the type label is selectable
+    in the PDF / extractable by pdftotext, and sits right next to its scatter
+    inside the keep-together Group.
+    """
+    tipo = model._safe_str(cls.get("tipo")) or "sin forma clara"
+    bits = []
+    pearson = cls.get("pearson")
+    spearman = cls.get("spearman")
+    r2_lin = cls.get("r2_linear")
+    r2_poly = None
+    for key in ("r2_poly2", "r2_poly3"):
+        v = cls.get(key)
+        if _is_num(v) and (r2_poly is None or float(v) > r2_poly):
+            r2_poly = float(v)
+    if _is_num(pearson):
+        bits.append(f"Pearson r={float(pearson):+.2f}")
+    if _is_num(spearman):
+        bits.append(f"Spearman ρ={float(spearman):+.2f}")
+    if _is_num(r2_lin):
+        bits.append(f"R² lineal={float(r2_lin):.2f}")
+    if r2_poly is not None:
+        bits.append(f"R² polinómico={r2_poly:.2f}")
+    metrics = "; ".join(bits)
+    text = (f"Relación **{tipo}** entre «{a}» y «{b}»."
+            + (f" {metrics}." if metrics else ""))
+    return text
+
+
+def _scatter_blocks(pairs: list, raw_numeric):
+    """Build keep-together scatter Groups for the strongest num-num pairs.
+
+    Returns a list of blocks (a Heading plus one Group per pair), or an empty
+    list when there is no raw numeric data (e.g. the lite profile drops
+    ``ctx['raw_numeric']`` to skip live recomputation) or the relationship
+    helpers are unavailable. Never raises: any failure degrades to no scatters,
+    leaving the matrix + tables intact.
+    """
+    if not isinstance(raw_numeric, dict) or not raw_numeric:
+        return []
+    selected = _select_scatter_pairs(pairs)
+    if not selected:
+        return []
+
+    # The relationship helpers live in the datascience package. Import lazily so
+    # the chapter still builds (matrix + tables) when they are absent.
+    try:
+        from datascience.classify_relationship_type import (
+            classify_relationship_type,
+        )
+        from datascience.relationship_scatter_figure import (
+            relationship_scatter_figure,
+        )
+    except Exception:  # noqa: BLE001 — degrade, never break the chapter.
+        return []
+
+    groups = []
+    for pair in selected:
+        a = pair.get("a")
+        b = pair.get("b")
+        xs = raw_numeric.get(a)
+        ys = raw_numeric.get(b)
+        # Edge: a selected pair has no raw column (aggregated profile, renamed
+        # column, …) — skip just that pair, keep the rest.
+        if not _is_seq(xs) or not _is_seq(ys):
+            continue
+        try:
+            cls = classify_relationship_type(list(xs), list(ys)) or {}
+        except Exception:  # noqa: BLE001
+            continue
+        a_lbl = model._safe_str(a)
+        b_lbl = model._safe_str(b)
+
+        def _make(xs=xs, ys=ys, a_lbl=a_lbl, b_lbl=b_lbl, cls=cls):
+            return relationship_scatter_figure(
+                list(xs), list(ys), x_label=a_lbl, y_label=b_lbl,
+                classification=cls)
+
+        groups.append(model.Group(blocks=[
+            model.Heading(text=f"{a_lbl} ↔ {b_lbl}", level=2),
+            model.Figure(
+                make=_make,
+                caption=(f"Dispersión de «{a_lbl}» frente a «{b_lbl}» con la "
+                         "curva de ajuste del mejor modelo.")),
+            model.Markdown(text=_classification_note(a_lbl, b_lbl, cls)),
+        ]))
+
+    if not groups:
+        return []
+    intro = model.Markdown(text=(
+        "Para los pares numéricos más fuertes (positivos y negativos) se dibuja "
+        "la nube de puntos con su ajuste y se clasifica el **tipo de relación**: "
+        "**lineal** (una recta basta), **polinómica** (curva de grado 2/3 que "
+        "mejora claramente el ajuste lineal), **monótona no-lineal** (crece o "
+        "decrece siempre pero no en línea recta; Spearman ≫ Pearson) o "
+        "**débil/sin forma**."))
+    return [model.Heading(text="Relaciones más fuertes (scatter)", level=2),
+            intro] + groups
+
+
 def build_correlacion(profile: dict, ctx: dict):
     """Build the Correlation Chapter, or None if there are no pairs to show.
 
@@ -392,6 +532,18 @@ def build_correlacion(profile: dict, ctx: dict):
             "No se han hallado correlaciones negativas significativas entre "
             "columnas numéricas.")))
 
+    # 2.5) Scatter plots of the strongest numeric-numeric pairs, each with its
+    # fitted curve and a relationship-type label (lineal / polinómica / monótona
+    # / débil). Needs the raw numeric sample (ctx['raw_numeric'], row-aligned);
+    # when it is absent (aggregated/lite profile) the scatters are simply omitted
+    # and the matrix + tables above stand on their own.
+    raw_numeric = None
+    if isinstance(ctx, dict):
+        raw_numeric = ctx.get("raw_numeric") or profile.get("raw_numeric")
+    else:
+        raw_numeric = profile.get("raw_numeric")
+    blocks.extend(_scatter_blocks(pairs, raw_numeric))
+
     # 3) Spuriousness caveat for level-based correlations (Granger–Newbold).
     caveat = corr.get("levels_caveat")
     if isinstance(caveat, str) and caveat.strip():

python/functions/datascience/automatic_eda/chapters/correlacion_test.py

@@ -175,6 +175,105 @@ def test_anticorte_matriz_ancha_y_etiquetas_largas_no_se_cortan():
         assert "azufre" in _pdf_text(pdf)
 
 
+def _raw_numeric_for_profile(n: int = 80) -> dict:
+    """Row-aligned raw numeric sample matching the signed pairs of _profile().
+
+    Builds columns with a clear, deterministic shape so the relationship-type
+    classifier has something unambiguous to label:
+      - density vs alcohol: strong negative linear (the top-negative pair).
+      - alcohol vs quality: positive linear.
+      - ph, fixed_acidity, sulphates: filler columns for the remaining pairs.
+    """
+    import math as _m
+
+    alcohol = [8.0 + 0.05 * i for i in range(n)]
+    density = [1.0 - 0.002 * a for a in alcohol]           # neg linear vs alcohol
+    quality = [3.0 + 0.4 * a + (0.1 if i % 2 else -0.1)    # pos linear vs alcohol
+               for i, a in enumerate(alcohol)]
+    ph = [3.0 + 0.3 * _m.sin(i / 5.0) for i in range(n)]
+    fixed_acidity = [7.0 - 0.5 * p for p in ph]            # neg linear vs ph
+    sulphates = [0.5 + 0.01 * (i % 7) for i in range(n)]
+    return {
+        "alcohol": alcohol, "density": density, "quality": quality,
+        "ph": ph, "fixed_acidity": fixed_acidity, "sulphates": sulphates,
+    }
+
+
+def test_golden_scatters_de_pares_num_num_con_tipo_de_relacion():
+    """Con ctx['raw_numeric'], el capítulo añade scatters (Figure dentro de Group)
+    de los pares num-num más fuertes, cada uno con su etiqueta de tipo en texto."""
+    from datascience.automatic_eda.model import Group
+
+    ctx = {"raw_numeric": _raw_numeric_for_profile()}
+    ch = build_correlacion(_profile(), ctx)
+    assert ch is not None
+    groups = [b for b in ch.blocks if isinstance(b, Group)]
+    assert groups, "debe emitir al menos un Group con scatter"
+    # Cada Group lleva su figura (lazy) y una nota de texto con el tipo.
+    for g in groups:
+        gkinds = [b.kind for b in g.blocks]
+        assert "figure" in gkinds and "markdown" in gkinds
+    # La sección y la etiqueta de tipo aparecen como texto plano (extraíble).
+    headings = " ".join(b.text for b in ch.blocks if b.kind == "heading")
+    assert "Relaciones más fuertes" in headings
+    body = " ".join(b.text for g in groups for b in g.blocks
+                    if b.kind == "markdown")
+    assert any(t in body for t in
+               ("lineal", "polinómica", "monótona", "sin forma"))
+    # El par num-num más fuerte (density ↔ alcohol) tiene scatter; el par cat-cat
+    # (region ↔ type) NO — no es numérico.
+    assert "density" in body or "alcohol" in body
+    assert "region" not in body and "type" not in body
+
+
+def test_golden_pdf_muestra_scatters_con_etiqueta_de_tipo():
+    """En el PDF, el capítulo Correlación incluye los scatters y su etiqueta de
+    tipo en texto seleccionable (pdftotext la encuentra)."""
+    prof = _profile()
+    ctx = {"raw_numeric": _raw_numeric_for_profile()}
+    with tempfile.TemporaryDirectory() as d:
+        pdf = os.path.join(d, "corr_scatter.pdf")
+        rp = render_automatic_eda_pdf(prof, pdf, {"title": "EDA — wine",
+                                                  "ctx": ctx})
+        assert rp["path"] == pdf and rp["n_pages"] >= 1
+        txt = _pdf_text(pdf)
+        assert "Relaciones" in txt and "scatter" in txt.lower()
+        # Alguna etiqueta de tipo de relación, en texto.
+        assert any(t in txt for t in
+                   ("lineal", "polin", "monóton", "monoton", "sin forma"))
+
+
+def test_edge_sin_raw_numeric_omite_scatters_sin_lanzar():
+    """profile lite / ctx None: sin raw_numeric el capítulo omite los scatters
+    pero sigue emitiendo matriz + tablas (no lanza)."""
+    from datascience.automatic_eda.model import Group
+
+    for ctx in (None, {}, {"raw_numeric": None}, {"raw_numeric": {}}):
+        ch = build_correlacion(_profile(), ctx)
+        assert ch is not None
+        assert not [b for b in ch.blocks if isinstance(b, Group)]
+        # La matriz y al menos una tabla top siguen presentes.
+        assert any(b.kind == "figure" for b in ch.blocks)
+        assert any(b.kind == "data_table" for b in ch.blocks)
+
+
+def test_edge_par_sin_columna_cruda_se_omite_sin_lanzar():
+    """Si un par seleccionado no tiene su columna en raw_numeric, se omite ese
+    par (no lanza); los demás scatters se construyen igual."""
+    from datascience.automatic_eda.model import Group
+
+    raw = _raw_numeric_for_profile()
+    raw.pop("density", None)   # rompe el par density ↔ alcohol
+    ch = build_correlacion(_profile(), {"raw_numeric": raw})
+    assert ch is not None
+    groups = [b for b in ch.blocks if isinstance(b, Group)]
+    body = " ".join(b.text for g in groups for b in g.blocks
+                    if b.kind == "markdown")
+    # density desaparece de los scatters; otros pares (p.ej. ph↔fixed_acidity,
+    # alcohol↔quality) pueden seguir presentes sin error.
+    assert "density" not in body
+
+
 def test_glosario_engancha_metodos_y_fdr():
     """Mejora 4b: los métodos de correlación (Pearson, Spearman, Cramér's V,
     razón de correlación) y la corrección por comparaciones múltiples (FDR) se