Compare commits

..

2 Commits

Author SHA1 Message Date
egutierrez af1dd9bcc2 test(eda): tests del capítulo ANÁLISIS LLM (golden + edges + anti-cortes)
Suite self-contained (perfil sintético + un golden, sin DuckDB):
- golden: build_analisis_llm devuelve el Chapter y el documento entero renderiza
  a PDF y PPTX con resumen, análisis sugeridos, limpieza y una columna del
  diccionario presentes.
- orden: el capítulo queda inmediatamente después de `overview`.
- edges: profile sin bloque `llm` (o None/{}/malformado/llm vacío) -> None sin
  lanzar; fallback a ctx['llm'].
- anti-cortes: diccionario de 40 filas + sugerencia de limpieza de ~150 chars se
  reparten en varias páginas/slides sin perder ninguna fila ni palabra.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-30 15:01:26 +02:00
egutierrez fc5bc334c8 feat(eda): capítulo ANÁLISIS LLM para AutomaticEDA, junto al overview
Nuevo capítulo `analisis_llm` del motor AutomaticEDA. Consume el bloque `llm`
que `eda_llm_insights` (grupo eda) ya deja en el TableProfile —no llama al LLM
ni recalcula— y lo convierte en bloques del modelo de documento para que se
renderice sin cortarse en PDF ni PPTX:

- Resumen de la tabla y significado de una fila -> bloques Markdown (el
  renderer los envuelve a líneas completas, nunca pierde texto).
- Diccionario de datos y PII -> DataTable (el paginador parte por filas
  repitiendo cabecera y envuelve celdas largas dentro de su columna).
- Análisis sugeridos y limpieza sugerida -> listas de viñetas Markdown; cada
  entrada es una línea completa que el renderer envuelve, nunca trunca.

Lectura defensiva (.get) en todo; devuelve None si el profile no trae bloque
`llm` (p.ej. profile_table sin run_llm) para omitir el capítulo.

MUST-3.2 (report 2043): se mueve `analisis_llm` en CHAPTER_ORDER a la posición
inmediatamente posterior a `overview`, como pidió el usuario ("va junto al
overview").

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-30 15:01:26 +02:00
8 changed files with 412 additions and 701 deletions
@@ -0,0 +1,221 @@
"""LLM analysis chapter (ANÁLISIS LLM) — the interpretive layer, next to overview.
Third reference chapter for AutomaticEDA. Renders the ``llm`` block that the
``eda`` group function ``eda_llm_insights`` already produced and stored in the
``TableProfile`` — it does NOT call the LLM nor recompute anything. The block is
turned into clean, markdown-style document blocks so it reads as a real chapter
(table summary, row meaning, data dictionary, suggested analyses, cleaning
suggestions, PII findings) and, crucially, **nothing is ever cut** in PDF or
PPTX:
* Prose (summary, row meaning) → ``Markdown`` blocks the renderers wrap to whole
lines, so no word is lost no matter how long the text is.
* The data dictionary and PII findings → ``DataTable`` blocks the paginator
splits by rows (repeating the header) and whose long cells wrap inside their
column — wide, multi-row tables never overflow a page/slide.
* Cleaning suggestions and suggested analyses → ``Markdown`` bullet lists; each
item is a whole line the renderer wraps, never truncated mid-entry.
Position: this chapter is declared in ``chapters_registry.CHAPTER_ORDER`` right
after ``overview`` so the interpretation sits next to the table preview, as the
user asked ("va junto al overview").
Data source: the ``llm`` dict produced by ``eda_llm_insights`` (group ``eda``),
read from ``profile['llm']`` (or ``ctx['llm']`` as a fallback). Shape::
{
"summary": str, # what the table is, 2-3 sentences
"row_meaning": str, # what one row represents / granularity
"dictionary": [ {"column","description","business_meaning","unit"} ],
"pii": [ {"column","kind","severity"} ],
"cleaning": [str], # cleaning / transformation suggestions
"analyses": [str], # suggested questions / analyses / hypotheses
}
Contract: build_<id>(profile, ctx) -> Chapter | None ; CHAPTER_VERSION = "x.y.z".
Reads everything defensively (``.get``) and NEVER raises; returns ``None`` when
the profile carries no LLM block (e.g. ``profile_table`` ran without
``run_llm``), so the chapter is simply omitted from the document.
"""
from __future__ import annotations
from .. import model
CHAPTER_VERSION = "1.0.0"
CHAPTER_ID = "analisis_llm"
CHAPTER_TITLE = "Análisis LLM"
# Key under which eda_llm_insights stores its interpretive block in the profile.
LLM_KEY = "llm"
def _clean_text(value) -> str:
"""Coerce a value to a single trimmed line (collapse inner newlines).
Used for bullet items so each suggestion stays a single markdown bullet the
renderer wraps; never drops content, only normalizes whitespace.
"""
text = model._safe_str(value).strip()
if not text:
return ""
return " ".join(text.split())
def _para(value) -> str:
"""Coerce a value to trimmed prose, preserving paragraph breaks."""
text = model._safe_str(value).strip()
if not text:
return ""
# Keep blank-line paragraph breaks; collapse runs of spaces/tabs per line.
lines = [" ".join(ln.split()) for ln in text.splitlines()]
out: list = []
for ln in lines:
if ln or (out and out[-1] != ""):
out.append(ln)
return "\n".join(out).strip()
def _bullets(items) -> str:
"""Build a markdown bullet list from a sequence of strings.
Each item becomes one ``- ...`` line (a whole, wrappable unit). Empty items
and non-list inputs are handled gracefully; returns "" when there is nothing.
"""
if isinstance(items, str):
items = [items]
if not isinstance(items, (list, tuple)):
return ""
lines = []
for it in items:
text = _clean_text(it)
if text:
lines.append(f"- {text}")
return "\n".join(lines)
def _summary_blocks(llm: dict) -> list:
"""Heading + prose for the table summary, or [] if absent."""
text = _para(llm.get("summary"))
if not text:
return []
return [model.Heading(text="Resumen de la tabla", level=2),
model.Markdown(text=text)]
def _row_meaning_blocks(llm: dict) -> list:
"""Heading + prose for what one row represents, or [] if absent."""
text = _para(llm.get("row_meaning"))
if not text:
return []
return [model.Heading(text="Significado de una fila", level=2),
model.Markdown(text=text)]
def _dictionary_block(llm: dict):
"""DataTable for the data dictionary, or None if absent/empty.
Columns: Columna / Descripción / Significado de negocio / Unidad. The
paginator splits this by rows repeating the header and wraps long cells, so a
long dictionary (many columns) never gets cut.
"""
entries = llm.get("dictionary")
if not isinstance(entries, (list, tuple)) or not entries:
return None
header = ["Columna", "Descripción", "Significado de negocio", "Unidad"]
rows = []
for e in entries:
if not isinstance(e, dict):
# Be tolerant: a bare string still shows up as a description row.
rows.append(["", _clean_text(e), "", ""])
continue
rows.append([
_clean_text(e.get("column")) or "",
_clean_text(e.get("description")),
_clean_text(e.get("business_meaning")),
_clean_text(e.get("unit")),
])
if not rows:
return None
return model.DataTable(header=header, rows=rows, title="Diccionario de datos")
def _analyses_blocks(llm: dict) -> list:
"""Heading + bullet list of suggested analyses, or [] if absent."""
bullets = _bullets(llm.get("analyses"))
if not bullets:
return []
return [model.Heading(text="Análisis sugeridos", level=2),
model.Markdown(text=bullets)]
def _cleaning_blocks(llm: dict) -> list:
"""Heading + bullet list of cleaning suggestions, or [] if absent."""
bullets = _bullets(llm.get("cleaning"))
if not bullets:
return []
return [model.Heading(text="Limpieza sugerida", level=2),
model.Markdown(text=bullets)]
def _pii_block(llm: dict):
"""DataTable for PII/GDPR findings, or None if absent/empty."""
entries = llm.get("pii")
if not isinstance(entries, (list, tuple)) or not entries:
return None
header = ["Columna", "Tipo", "Severidad"]
rows = []
for e in entries:
if not isinstance(e, dict):
continue
rows.append([
_clean_text(e.get("column")) or "",
_clean_text(e.get("kind")),
_clean_text(e.get("severity")),
])
if not rows:
return None
return model.DataTable(
header=header, rows=rows, title="Datos personales (PII / RGPD)",
note="detección automática orientativa — revisar antes de tratar los datos")
def build_analisis_llm(profile: dict, ctx: dict):
"""Build the LLM analysis Chapter, or None if there is no LLM block.
Consumes ``profile['llm']`` (the block produced by ``eda_llm_insights``,
group ``eda``); falls back to ``ctx['llm']``. Returns ``None`` when no LLM
block is present or it carries no usable content, so the chapter is omitted
rather than rendering an empty section.
"""
profile = profile or {}
ctx = ctx or {}
llm = profile.get(LLM_KEY)
if not isinstance(llm, dict):
llm = ctx.get(LLM_KEY)
if not isinstance(llm, dict) or not llm:
return None
blocks: list = []
blocks += _summary_blocks(llm)
blocks += _row_meaning_blocks(llm)
dict_block = _dictionary_block(llm)
if dict_block is not None:
blocks.append(model.Heading(text="Diccionario de datos", level=2))
blocks.append(dict_block)
blocks += _analyses_blocks(llm)
blocks += _cleaning_blocks(llm)
pii_block = _pii_block(llm)
if pii_block is not None:
blocks.append(model.Heading(text="Datos personales (PII / RGPD)", level=2))
blocks.append(pii_block)
if not blocks:
return None # LLM block present but every field empty → omit chapter.
return model.Chapter(id=CHAPTER_ID, title=CHAPTER_TITLE,
version=CHAPTER_VERSION, blocks=blocks)
@@ -0,0 +1,190 @@
"""Tests for the ANÁLISIS LLM chapter — DoD: golden + edges + anti-cut.
Self-contained: builds a synthetic TableProfile carrying an ``llm`` block (the
shape ``eda_llm_insights`` produces) so the suite is fast and deterministic — no
DuckDB and no LLM call. Verifies:
* golden — ``build_analisis_llm`` yields the chapter and the full document
renders to PDF *and* PPTX with the summary, a suggested analysis, a cleaning
suggestion and a dictionary column all present;
* order — the chapter sits immediately after ``overview`` (user requirement);
* edges — a profile with no ``llm`` block (or None/empty/malformed) returns
``None`` and never raises;
* anti-cut — a long dictionary (40 rows) and a 150-char cleaning suggestion are
rendered to PDF and PPTX without losing a single row or word.
"""
import os
import re
import tempfile
from pypdf import PdfReader
from pptx import Presentation
from datascience.automatic_eda.chapters.analisis_llm import (
build_analisis_llm, CHAPTER_VERSION)
from datascience.automatic_eda.chapters_registry import build_document
from datascience.automatic_eda.model import Chapter, DataTable
from datascience.render_automatic_eda_pdf import render_automatic_eda_pdf
from datascience.render_automatic_eda_pptx import render_automatic_eda_pptx
def _profile() -> dict:
return {
"table": "ventas",
"source": "/data/ventas.csv",
"profiled_at": "2026-06-30T10:00:00+00:00",
"n_rows": 1000,
"n_cols": 2,
"quality_score": 92.5,
"columns": [
{"name": "precio", "inferred_type": "numeric", "null_pct": 0.0,
"null_count": 0,
"numeric": {"mean": 42.5, "median": 40.0, "min": 1.0,
"max": 100.0, "std": 12.3}},
{"name": "categoria", "inferred_type": "categorical",
"null_pct": 0.0, "null_count": 0,
"categorical": {"top": [{"value": "neumaticos", "count": 500}]}},
],
"llm": {
"summary": "Tabla de ventas por producto. Token SUMMARYTOKEN.",
"row_meaning": "Cada fila es una venta. Token ROWTOKEN.",
"dictionary": [
{"column": "precio", "description": "Precio unitario DESCTOKEN",
"business_meaning": "Ingreso por unidad", "unit": "EUR"},
{"column": "categoria", "description": "Familia de producto",
"business_meaning": "Segmento comercial", "unit": ""},
],
"pii": [{"column": "categoria", "kind": "ninguno", "severity": "low"}],
"cleaning": ["Quitar nulos de precio CLEANTOKEN",
"Normalizar mayusculas en categoria"],
"analyses": ["Estudiar relacion precio-categoria ANALYSISTOKEN",
"Detectar outliers de precio"],
},
}
def _pdf_text(path: str) -> str:
txt = "".join((pg.extract_text() or "") for pg in PdfReader(path).pages)
return re.sub(r"\s+", " ", txt)
def _pptx_text(path: str) -> str:
prs = Presentation(path)
parts = []
for sl in prs.slides:
for sh in sl.shapes:
if sh.has_text_frame:
parts.append(sh.text_frame.text)
if sh.has_table:
tb = sh.table
for r in range(len(tb.rows)):
for c in range(len(tb.columns)):
parts.append(tb.cell(r, c).text)
return re.sub(r"\s+", " ", " ".join(parts))
def test_golden_build_y_render_pdf_pptx():
prof = _profile()
ch = build_analisis_llm(prof, {})
assert ch is not None
assert ch.id == "analisis_llm"
assert ch.version == CHAPTER_VERSION
assert ch.blocks # non-empty.
with tempfile.TemporaryDirectory() as d:
out_pdf = os.path.join(d, "eda.pdf")
res = render_automatic_eda_pdf(prof, out_pdf, {"title": "EDA — ventas"})
assert res["path"] == out_pdf and os.path.exists(out_pdf)
ids = [c["id"] for c in res["chapters"]]
assert "analisis_llm" in ids
txt = _pdf_text(out_pdf)
# The user's required content: summary, suggested analyses, cleaning.
assert "SUMMARYTOKEN" in txt
assert "ANALYSISTOKEN" in txt
assert "CLEANTOKEN" in txt
assert "DESCTOKEN" in txt # data dictionary cell.
out_pptx = os.path.join(d, "eda.pptx")
res2 = render_automatic_eda_pptx(prof, out_pptx, {"title": "EDA — ventas"})
assert res2["path"] == out_pptx and os.path.exists(out_pptx)
ids2 = [c["id"] for c in res2["chapters"]]
assert "analisis_llm" in ids2
ptx = _pptx_text(out_pptx)
assert "SUMMARYTOKEN" in ptx
assert "ANALYSISTOKEN" in ptx
assert "CLEANTOKEN" in ptx
assert "DESCTOKEN" in ptx
def test_orden_capitulo_junto_a_overview():
chapters = build_document(_profile(), {})
ids = [c.id for c in chapters]
assert "overview" in ids and "analisis_llm" in ids
# User requirement: the LLM chapter sits right after overview.
assert ids.index("analisis_llm") == ids.index("overview") + 1
def test_edge_sin_llm_devuelve_none():
# No llm block at all.
prof = {k: v for k, v in _profile().items() if k != "llm"}
assert build_analisis_llm(prof, {}) is None
# None / empty / malformed never raise and yield None.
assert build_analisis_llm(None, None) is None
assert build_analisis_llm({}, {}) is None
assert build_analisis_llm({"llm": {}}, {}) is None
assert build_analisis_llm({"llm": "not-a-dict"}, {}) is None
# All-empty fields → omitted (no blocks).
empty = {"llm": {"summary": "", "dictionary": [], "cleaning": [],
"analyses": [], "pii": [], "row_meaning": ""}}
assert build_analisis_llm(empty, {}) is None
def test_edge_llm_via_ctx_fallback():
# The block may arrive in ctx instead of the profile.
prof = {k: v for k, v in _profile().items() if k != "llm"}
ctx = {"llm": {"summary": "Resumen via ctx CTXTOKEN."}}
ch = build_analisis_llm(prof, ctx)
assert ch is not None and ch.id == "analisis_llm"
def test_anti_cortes_diccionario_largo_y_limpieza_larga():
long_clean = ("Lorem ipsum dolor sit amet consectetur adipiscing elit sed do "
"eiusmod tempor incididunt ut labore et dolore magna aliqua "
"reprehenderit voluptate velit esse cillum dolore")
dictionary = [
{"column": f"col_{i}",
"description": f"Descripcion larga numero {i} con bastante texto para "
f"forzar el wrap dentro de la celda fila{i}",
"business_meaning": f"Significado de negocio {i}", "unit": "u"}
for i in range(40)
]
prof = {
"table": "t", "n_rows": 1, "n_cols": 1, "columns": [],
"llm": {"summary": "S", "dictionary": dictionary,
"cleaning": [long_clean], "analyses": ["A"]},
}
ch = build_analisis_llm(prof, {})
assert ch is not None
# Structure: the dictionary DataTable keeps ALL 40 rows — none dropped on
# construction (the renderers then split it by rows, repeating the header).
dts = [b for b in ch.blocks if isinstance(b, DataTable)]
assert any(len(dt.rows) == 40 for dt in dts)
with tempfile.TemporaryDirectory() as d:
out_pdf = os.path.join(d, "x.pdf")
render_automatic_eda_pdf([ch], out_pdf, {"write_manifest": False})
# 40 wide rows + a long cleaning line cannot fit one page → it spills,
# which is exactly the no-cut behaviour (paginate, never truncate).
assert len(PdfReader(out_pdf).pages) > 1
txt = _pdf_text(out_pdf)
# The long cleaning suggestion is wrapped word-by-word, not truncated.
for word in ("Lorem", "incididunt", "reprehenderit", "voluptate", "cillum"):
assert word in txt
out_pptx = os.path.join(d, "x.pptx")
res2 = render_automatic_eda_pptx([ch], out_pptx, {"write_manifest": False})
assert res2["n_slides"] > 1 # table + long text spill across slides.
ptx = _pptx_text(out_pptx)
for word in ("Lorem", "reprehenderit", "voluptate"):
assert word in ptx
@@ -1,289 +0,0 @@
"""Numeric distributions chapter (NUM DISTR) for AutomaticEDA.
For every numeric column the chapter draws, as a single indivisible figure, a
histogram with the **mean, median and ±1σ band drawn as reference lines** and a
**Tukey boxplot right below it** sharing the same X axis — exactly the user
requirement for this chapter. Each figure is emitted as a lazy ``Figure`` block
so the renderers rasterize and scale it to fit a whole page/slide and nothing is
ever cut; columns with many numerics simply flow across pages as small
multiples.
Data comes from the ``eda`` group profile and is never recomputed here:
- ``columns[i]['numeric']`` (the output of ``describe_numeric``) gives
``mean, median, std, min, max, p25, p75, iqr, n_outliers, outlier_pct,
distribution_type`` and the ``histogram`` bins ``[{lo, hi, count}]``.
- The boxplot five-number summary + Tukey 1.5·IQR fences are derived by the
pure registry function ``build_boxplot_stats`` (group ``eda``); this chapter
only consumes its output, it does not reimplement the statistics.
Contract: build_<id>(profile, ctx) -> Chapter | None ; CHAPTER_VERSION = "x.y.z".
Reads everything defensively (``.get``) and never raises: a column whose figure
cannot be built is degraded to a short note instead of aborting the chapter.
"""
from __future__ import annotations
from .. import model
# Pure registry function (group ``eda``) that derives the Tukey boxplot stats
# from a ``numeric`` sub-block. Imported defensively so the chapter still builds
# (degrading the boxplot to a note) if the function is somehow unavailable.
try:
from datascience.build_boxplot_stats import build_boxplot_stats
except Exception: # noqa: BLE001 — keep the chapter importable no matter what.
build_boxplot_stats = None # type: ignore[assignment]
CHAPTER_VERSION = "1.0.0"
CHAPTER_ID = "num_distr"
CHAPTER_TITLE = "Distribuciones numéricas"
# Plain-Spanish gloss for every label ``detect_distribution_type`` can emit, so a
# non-expert reader understands the shape and the suggested next step (MUST-4.3).
_DIST_GLOSS = {
"normal-ish": "aproximadamente simétrica (campana); media y mediana casi "
"coinciden.",
"right-skewed": "asimétrica a la derecha (cola larga hacia valores altos); "
"la media supera a la mediana — considera una transformación "
"logarítmica.",
"left-skewed": "asimétrica a la izquierda (cola larga hacia valores bajos); "
"la media queda por debajo de la mediana.",
"heavy-tail": "colas pesadas (curtosis alta): más valores extremos de lo "
"que esperaría una normal — vigila los outliers.",
"lognormal-ish": "compatible con lognormal (simétrica al tomar logaritmos); "
"la re-expresión log suele normalizarla.",
"multimodal": "varios picos: probablemente mezcla de subgrupos — conviene "
"segmentar antes de resumir con una sola media.",
"discrete": "pocos valores distintos (discreta/ordinal); el histograma "
"cuenta niveles, no un continuo.",
"too_few_samples": "muestra demasiado pequeña para clasificar la forma con "
"fiabilidad.",
"other": "forma no encuadrada en las categorías estándar.",
}
def _fmt_num(value, decimals: int = 3) -> str:
"""Compact, defensive number formatting shared with the other chapters."""
if value is None:
return ""
if isinstance(value, bool):
return str(value)
if isinstance(value, int):
return f"{value:,}".replace(",", ".")
if isinstance(value, float):
if value != value: # NaN
return "NaN"
if value in (float("inf"), float("-inf")):
return str(value)
text = f"{value:.{decimals}f}".rstrip("0").rstrip(".")
return text if text else "0"
return str(value)
def _numeric_columns(profile: dict) -> list:
"""Return the list of (name, numeric_dict) for columns with usable stats."""
out = []
for col in profile.get("columns") or []:
if not isinstance(col, dict):
continue
if col.get("inferred_type") != "numeric":
continue
num = col.get("numeric")
if not isinstance(num, dict) or not num:
continue
# A numeric block is renderable when it carries at least a center.
if num.get("mean") is None and num.get("median") is None:
continue
out.append((col.get("name") or "(columna)", num))
return out
def _make_hist_box(name: str, numeric: dict, box: dict):
"""Build the histogram (with mean/median/±σ lines) + boxplot figure.
Returned lazily to the renderer (a zero-arg callable via ``Figure.make``) so
matplotlib is only imported and the figure only drawn when a renderer needs
it. The two stacked axes share the X axis and are produced as a single
figure, which both renderers treat as one indivisible unit (scaled whole,
never cut).
"""
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
fig, (ax_h, ax_b) = plt.subplots(
2, 1, figsize=(6.4, 3.4), sharex=True,
gridspec_kw={"height_ratios": [3.2, 1.0], "hspace": 0.08})
# ---- Histogram from the precomputed equal-width bins {lo, hi, count}. ----
hist = numeric.get("histogram") or []
drew_bars = False
for b in hist:
if not isinstance(b, dict):
continue
lo = b.get("lo")
hi = b.get("hi")
count = b.get("count") or 0
if lo is None or hi is None:
continue
width = (hi - lo) if hi > lo else max(abs(lo) * 1e-3, 1e-6)
ax_h.bar(lo, count, width=width, align="edge", color="#9ec6df",
edgecolor="#5b8aa6", linewidth=0.4, zorder=2)
drew_bars = True
if not drew_bars:
ax_h.text(0.5, 0.5, "(sin histograma)", ha="center", va="center",
fontsize=9, color="#8a8a8a", transform=ax_h.transAxes)
mean = numeric.get("mean")
median = numeric.get("median")
std = numeric.get("std")
# ±1σ band first (behind the lines), then median (solid) and mean (dashed).
if mean is not None and std is not None and std > 0:
ax_h.axvspan(mean - std, mean + std, color="#f0c27b", alpha=0.22,
zorder=1, label="±1σ")
if median is not None:
ax_h.axvline(median, color="#2e8b57", linestyle="-", linewidth=1.6,
zorder=4, label=f"mediana = {_fmt_num(median)}")
if mean is not None:
ax_h.axvline(mean, color="#c0392b", linestyle="--", linewidth=1.6,
zorder=4, label=f"media = {_fmt_num(mean)}")
ax_h.set_ylabel("frecuencia", fontsize=8)
ax_h.tick_params(labelsize=7)
ax_h.legend(fontsize=6.5, loc="upper right", framealpha=0.85)
for spine in ("top", "right"):
ax_h.spines[spine].set_visible(False)
# ---- Tukey boxplot below, sharing the X axis (MUST-4.2). ----
if box:
stats = [{
"med": box.get("median"),
"q1": box.get("q1"),
"q3": box.get("q3"),
"whislo": box.get("whisker_lo"),
"whishi": box.get("whisker_hi"),
"fliers": [], # raw outlier values are not in the profile.
"label": "",
}]
bxp_kw = dict(
showfliers=False, widths=0.5, patch_artist=True,
boxprops={"facecolor": "#9ec6df", "edgecolor": "#5b8aa6"},
medianprops={"color": "#2e8b57", "linewidth": 1.6},
whiskerprops={"color": "#5b8aa6"},
capprops={"color": "#5b8aa6"})
try:
# ``orientation`` is the current API; older matplotlib uses ``vert``.
try:
ax_b.bxp(stats, orientation="horizontal", **bxp_kw)
except TypeError:
ax_b.bxp(stats, vert=False, **bxp_kw)
except Exception: # noqa: BLE001 — never let one axis kill the figure.
pass
# Mark the presence of out-of-fence points (the raw values are unknown).
if box.get("has_low_outliers") and box.get("min") is not None:
ax_b.plot([box["min"]], [1], marker="o", markersize=3.5,
color="#c0392b", zorder=5)
if box.get("has_high_outliers") and box.get("max") is not None:
ax_b.plot([box["max"]], [1], marker="o", markersize=3.5,
color="#c0392b", zorder=5)
else:
ax_b.text(0.5, 0.5, "(boxplot no disponible)", ha="center", va="center",
fontsize=8, color="#8a8a8a", transform=ax_b.transAxes)
ax_b.set_yticks([])
ax_b.set_xlabel(name, fontsize=8)
ax_b.tick_params(labelsize=7)
for spine in ("top", "right", "left"):
ax_b.spines[spine].set_visible(False)
fig.suptitle(name, fontsize=10, fontweight="bold", x=0.02, ha="left")
return fig
def _stats_note(name: str, numeric: dict, box: dict) -> str:
"""One compact line of the key numbers + a plain-Spanish shape gloss."""
bits = [
f"media {_fmt_num(numeric.get('mean'))}",
f"mediana {_fmt_num(numeric.get('median'))}",
f"σ {_fmt_num(numeric.get('std'))}",
f"min {_fmt_num(numeric.get('min'))}",
f"max {_fmt_num(numeric.get('max'))}",
f"IQR {_fmt_num(numeric.get('iqr'))}",
]
n_out = numeric.get("n_outliers")
out_pct = numeric.get("outlier_pct")
if n_out is not None:
pct = f" ({_fmt_num(out_pct, 2)}%)" if out_pct is not None else ""
bits.append(f"outliers {n_out}{pct}")
if box and (box.get("lower_fence") is not None):
bits.append(
f"vallas Tukey [{_fmt_num(box.get('lower_fence'))}, "
f"{_fmt_num(box.get('upper_fence'))}]")
line = " · ".join(bits)
dist = numeric.get("distribution_type")
gloss = _DIST_GLOSS.get(dist)
if dist and gloss:
line += f"\n\n**Forma ({dist}):** {gloss}"
return line
def _figure_maker(name: str, numeric: dict, box: dict):
"""Bind the per-column arguments so the lazy closure is loop-safe."""
def _make():
return _make_hist_box(name, numeric, box)
return _make
def build_num_distr(profile: dict, ctx: dict):
"""Build the numeric-distributions Chapter, or None if no numeric column.
Args:
profile: the ``eda`` group TableProfile dict.
ctx: presentation context (unused here beyond defensive handling).
Returns:
A ``model.Chapter`` with, per numeric column, a histogram+boxplot figure
and a stats note; or ``None`` when the dataset has no numeric column.
"""
profile = profile or {}
ctx = ctx or {}
numerics = _numeric_columns(profile)
if not numerics:
return None # chapter does not apply to a dataset with no numerics.
intro = (
"Para cada columna numérica se muestra su **histograma** con tres líneas "
"de referencia: la **media** (línea roja discontinua), la **mediana** "
"(línea verde continua) y la banda **±1σ** (zona sombreada). Debajo, "
"alineado al mismo eje, un **boxplot de Tukey**: la caja abarca del "
"primer al tercer cuartil (P25P75), la línea interior es la mediana y "
"los bigotes llegan hasta 1,5·IQR; los puntos rojos señalan que hay "
"valores más allá de las vallas. Comparar media y mediana revela la "
"asimetría de la distribución.")
blocks = [
model.Heading(text=CHAPTER_TITLE, level=1),
model.Markdown(text=intro),
]
for name, numeric in numerics:
box = {}
if build_boxplot_stats is not None:
try:
box = build_boxplot_stats(numeric) or {}
except Exception: # noqa: BLE001 — degrade, never raise.
box = {}
blocks.append(model.Heading(text=str(name), level=2))
blocks.append(model.Figure(
make=_figure_maker(name, numeric, box),
caption=f"Distribución de «{name}» — histograma (media/mediana/±σ) "
f"y boxplot."))
blocks.append(model.Markdown(text=_stats_note(name, numeric, box)))
return model.Chapter(id=CHAPTER_ID, title=CHAPTER_TITLE,
version=CHAPTER_VERSION, blocks=blocks)
@@ -1,151 +0,0 @@
"""Tests for the NUM DISTR chapter — DoD: golden + edges + anti-cut.
Self-contained: builds synthetic ``numeric`` blocks (no DuckDB) so the suite is
fast and deterministic. Verifies that the chapter emits, per numeric column, a
histogram+boxplot figure plus a stats note; that the mean/median/±σ requirement
and the boxplot are present; that a profile with no numeric column yields None;
that None/empty never raises; and that with many numeric columns and long text
both the PDF and the PPTX render without cutting anything (every column heading
survives in the rendered output).
"""
import os
import re
import tempfile
from pypdf import PdfReader
from datascience.automatic_eda.chapters.num_distr import (
build_num_distr, CHAPTER_VERSION, _DIST_GLOSS,
)
from datascience.automatic_eda import model
from datascience.render_automatic_eda_pdf import render_automatic_eda_pdf
from datascience.render_automatic_eda_pptx import render_automatic_eda_pptx
def _numeric_block(mean, median, std, mn, mx, dist="normal-ish",
n_outliers=0, nbins=10):
"""A synthetic ``numeric`` sub-block shaped like describe_numeric's output."""
width = (mx - mn) / nbins if mx > mn else 1.0
hist = [{"lo": mn + i * width, "hi": mn + (i + 1) * width,
"count": (i + 1) * 3} for i in range(nbins)]
p25 = mn + (mx - mn) * 0.25
p75 = mn + (mx - mn) * 0.75
return {
"min": mn, "max": mx, "mean": mean, "median": median, "std": std,
"p25": p25, "p50": median, "p75": p75, "iqr": p75 - p25,
"n_outliers": n_outliers, "outlier_pct": 100.0 * n_outliers / 300.0,
"distribution_type": dist, "histogram": hist,
}
def _profile(n_numeric=2, extra_categorical=True):
cols = []
presets = [
("precio", 42.5, 40.0, 12.3, 1.0, 100.0, "right-skewed", 5),
("alcohol", 10.4, 10.3, 1.1, 8.0, 14.9, "normal-ish", 0),
("sulfatos", 0.66, 0.62, 0.17, 0.33, 2.0, "heavy-tail", 9),
("calidad", 5.6, 6.0, 0.8, 3.0, 8.0, "discrete", 0),
]
for i in range(n_numeric):
name, mean, med, std, mn, mx, dist, no = presets[i % len(presets)]
if i >= len(presets):
name = f"{name}_{i}"
cols.append({"name": name, "inferred_type": "numeric",
"numeric": _numeric_block(mean, med, std, mn, mx, dist, no)})
if extra_categorical:
cols.append({"name": "categoria", "inferred_type": "categorical",
"categorical": {"top": [{"value": "tinto", "count": 200}]}})
return {"table": "vinos", "n_rows": 300, "n_cols": len(cols),
"columns": cols}
def _pdf_text(path: str) -> str:
txt = "".join((pg.extract_text() or "") for pg in PdfReader(path).pages)
return re.sub(r"\s+", " ", txt)
def test_golden_chapter_estructura_y_bloques():
ch = build_num_distr(_profile(n_numeric=2), {})
assert ch is not None
assert ch.id == "num_distr"
assert ch.version == CHAPTER_VERSION
kinds = [b.kind for b in ch.blocks]
# Heading + intro Markdown, then per column: Heading + Figure + Markdown.
assert kinds[0] == "heading"
assert kinds[1] == "markdown"
assert kinds.count("figure") == 2 # one figure per numeric column.
assert kinds.count("heading") == 1 + 2 # chapter title + one per column.
# Each figure has a lazy maker that produces a real matplotlib figure.
figs = [b for b in ch.blocks if b.kind == "figure"]
fig = figs[0].make()
assert fig is not None
# Two stacked axes: histogram + boxplot share the figure.
assert len(fig.axes) == 2
import matplotlib.pyplot as plt
plt.close(fig)
def test_golden_media_mediana_sigma_y_boxplot_presentes():
# The intro documents the three reference lines and the Tukey boxplot; the
# per-column note carries the actual mean/median/σ numbers and the shape.
ch = build_num_distr(_profile(n_numeric=1, extra_categorical=False), {})
md_texts = " ".join(b.text for b in ch.blocks if b.kind == "markdown")
assert "media" in md_texts and "mediana" in md_texts
assert "±1σ" in md_texts or "σ" in md_texts
assert "boxplot" in md_texts.lower()
assert "Tukey" in md_texts
# distribution_type gloss surfaced for the column (right-skewed preset).
assert _DIST_GLOSS["right-skewed"].split(";")[0][:20] in md_texts
def test_boxplot_stats_se_consumen_del_registry():
# The chapter must feed build_boxplot_stats (group eda) and the resulting
# box must carry the Tukey fences for the figure.
from datascience.build_boxplot_stats import build_boxplot_stats
box = build_boxplot_stats(
_numeric_block(42.5, 40.0, 12.3, 1.0, 100.0, "right-skewed", 5))
assert box
assert "lower_fence" in box and "upper_fence" in box
assert box["q1"] is not None and box["q3"] is not None
def test_edge_sin_columnas_numericas_devuelve_none():
prof = {"columns": [{"name": "c", "inferred_type": "categorical",
"categorical": {"top": []}}]}
assert build_num_distr(prof, {}) is None
def test_edge_profile_none_y_vacio_no_revienta():
assert build_num_distr(None, None) is None
assert build_num_distr({}, {}) is None
assert build_num_distr({"columns": []}, {}) is None
def test_anti_corte_muchas_columnas_pdf_y_pptx():
# 8 numeric columns + long note text: nothing may be cut. Every column
# heading must survive in both the PDF text and the PPTX deck.
ch = build_num_distr(_profile(n_numeric=8), {})
names = [b.text for b in ch.blocks if b.kind == "heading" and b.level == 2]
assert len(names) == 8
with tempfile.TemporaryDirectory() as d:
pdf = os.path.join(d, "num.pdf")
res_pdf = render_automatic_eda_pdf(_profile(n_numeric=8), pdf,
{"write_manifest": False})
assert res_pdf["path"] == pdf
txt = _pdf_text(pdf)
for name in names:
assert name in txt, f"columna '{name}' cortada/ausente en el PDF"
pptx = os.path.join(d, "num.pptx")
res_pptx = render_automatic_eda_pptx(_profile(n_numeric=8), pptx,
{"write_manifest": False})
assert res_pptx["path"] == pptx
assert res_pptx["n_slides"] >= 8 # at least one slide per column figure.
def test_distribution_gloss_cubre_todas_las_etiquetas():
# Every label detect_distribution_type can emit has a Spanish gloss.
for label in ("normal-ish", "right-skewed", "left-skewed", "heavy-tail",
"lognormal-ish", "multimodal", "discrete", "too_few_samples",
"other"):
assert label in _DIST_GLOSS and _DIST_GLOSS[label]
@@ -28,12 +28,12 @@ from . import model
CHAPTER_ORDER = [
"portada", # cover
"overview", # df.head + columns/types/nulls/examples + describe
"analisis_llm", # LLM interpretation — sits next to overview (user request)
"num_distr", # numeric distributions
"cat_distr", # categorical distributions
"calidad", # data quality
"correlacion", # correlations / associations
"modelos", # cheap models (PCA/KMeans/outliers)
"analisis_llm", # LLM interpretation
"timeseries", # time-series analysis
"geospatial", # geospatial
"agregacion", # aggregations / pivots
@@ -1,58 +0,0 @@
---
name: build_boxplot_stats
kind: function
lang: py
domain: datascience
version: "1.0.0"
purity: pure
signature: "def build_boxplot_stats(numeric: dict) -> dict"
description: "Deriva las estadisticas de un boxplot de Tukey desde el sub-bloque numeric de un ColumnProfile del grupo eda (salida de describe_numeric). Aplica la regla del 1.5*IQR a los percentiles p25/p50/p75 para obtener cuartiles, fences, bigotes reales y flags de outliers. Lectura defensiva con .get; NUNCA lanza. Si faltan los percentiles clave devuelve {} para que el caller omita el grafico."
tags: [eda, statistics, profiling, boxplot, tukey, iqr, datascience]
params:
- name: numeric
desc: "Sub-bloque numeric de un ColumnProfile del grupo eda (la salida de describe_numeric). Claves esperadas (todas pueden ser None): min, max, mean, median, mode, std, variance, cv, p1, p5, p25, p50, p75, p95, p99, iqr, skew, kurtosis, n_outliers, outlier_pct, zero_pct, negative_pct, distribution_type, histogram. Solo se usan p25, median/p50, p75, min, max y n_outliers."
output: "Dict con las cifras de un boxplot horizontal de Tukey: {q1=p25, median=median(o p50), q3=p75, iqr=q3-q1, lower_fence=q1-1.5*iqr, upper_fence=q3+1.5*iqr, whisker_lo=max(min,lower_fence), whisker_hi=min(max,upper_fence), min, max, has_low_outliers=min<lower_fence, has_high_outliers=max>upper_fence, n_outliers}. Numericos en float, flags en bool nativo, n_outliers en int. Si faltan p25/median(o p50)/p75 devuelve {} (dict vacio). Cuando min/max faltan, los bigotes caen a la fence correspondiente."
uses_functions: []
uses_types: []
returns: []
returns_optional: false
error_type: ""
imports: []
tested: true
tests: ["test_boxplot_tukey_basico", "test_percentiles_faltan_devuelve_vacio", "test_median_cae_a_p50", "test_whiskers_usan_fence_si_falta_min_max", "test_tipos_salida_float_bool_int"]
test_file_path: "python/functions/datascience/build_boxplot_stats_test.py"
file_path: "python/functions/datascience/build_boxplot_stats.py"
---
## Ejemplo
```python
import sys, os
sys.path.insert(0, os.path.join("python", "functions"))
from datascience.build_boxplot_stats import build_boxplot_stats
# Sub-bloque numeric tal y como lo produce describe_numeric:
numeric = {
"min": 1.0, "max": 100.0,
"p25": 10.0, "median": 25.0, "p75": 40.0,
"iqr": 30.0, "n_outliers": 3,
}
box = build_boxplot_stats(numeric)
print(box["lower_fence"], box["upper_fence"]) # -35.0 85.0
print(box["whisker_lo"], box["whisker_hi"]) # 1.0 85.0
print(box["has_low_outliers"], box["has_high_outliers"]) # False True
```
## Cuando usarla
- Usala al dibujar un boxplot horizontal bajo el histograma en el capitulo `num_distr` de `AutomaticEDA`: convierte el bloque `numeric` de un `ColumnProfile` en las cifras exactas que el renderer necesita (cuartiles, fences, extremos de los bigotes y flags de outliers).
- Cuando ya tengas los percentiles calculados (salida de `describe_numeric`) y solo necesites derivar la geometria del boxplot de Tukey sin volver a tocar los valores crudos.
- Cuando quieras decidir si una columna tiene cola alta/baja (`has_high_outliers` / `has_low_outliers`) antes de proponer una transformacion (log, winsorize).
## Gotchas
- Funcion pura, sin I/O y determinista. Lectura defensiva con `.get`: NUNCA lanza. Si faltan `p25`, `median`/`p50` o `p75` devuelve `{}` (dict vacio) — el caller debe omitir el boxplot.
- Los `n_outliers` que se propagan vienen del bloque z-score del profile (`detect_outliers`, threshold 3.0), NO de la regla IQR. Son informativos: el conteo de Tukey que esta funcion calcula son los **fences** (`lower_fence`/`upper_fence`), no un recuento de puntos.
- No recibe los valores crudos de la columna, solo deriva cifras desde los percentiles ya calculados. Por eso no puede contar cuantos puntos caen fuera de las fences, solo si los extremos (`min`/`max`) las superan.
- `iqr` se recalcula como `q3 - q1` aunque el bloque traiga `numeric['iqr']`: asi funciona aunque esa clave falte.
- Cuando `min`/`max` faltan, los bigotes caen a la fence correspondiente y los flags de outliers quedan en `False` (sin extremo real no se afirma cola).
@@ -1,94 +0,0 @@
"""build_boxplot_stats — Tukey boxplot statistics from an EDA `numeric` sub-block.
Pure function: no I/O, deterministic. Takes the `numeric` dict of a ColumnProfile
(group `eda`, the output of describe_numeric) and derives the figures needed to
draw a horizontal Tukey boxplot using the 1.5 * IQR rule.
It only derives numbers from already-computed percentiles; it never sees the raw
column values. Reading is defensive (.get throughout) and the function NEVER
raises: if the key percentiles (p25 / p50 / p75) are missing it returns {} so the
caller can simply skip the boxplot.
"""
def _num(value):
"""Coerce to float defensively; return None for None/bool/non-numeric."""
# bool is a subclass of int; a percentile value is never a real bool, so
# treat True/False as missing instead of silently coercing to 1.0/0.0.
if value is None or isinstance(value, bool):
return None
try:
return float(value)
except (TypeError, ValueError):
return None
def build_boxplot_stats(numeric: dict) -> dict:
"""Derive Tukey boxplot statistics from the `numeric` sub-block of a profile.
Reads the percentiles already computed by describe_numeric and applies the
classic 1.5 * IQR fence rule to obtain the whisker extremes and outlier
flags of a horizontal boxplot. No raw values are needed.
Args:
numeric: The `numeric` sub-block of an eda ColumnProfile (output of
describe_numeric). Every value may be None; read defensively.
Returns:
Dict with the boxplot figures
{q1, median, q3, iqr, lower_fence, upper_fence, whisker_lo, whisker_hi,
min, max, has_low_outliers, has_high_outliers, n_outliers}.
If p25, p50/median or p75 are missing (None) returns {} (empty dict) so
the caller omits the plot.
"""
if not isinstance(numeric, dict):
return {}
q1 = _num(numeric.get("p25"))
q3 = _num(numeric.get("p75"))
# Prefer the explicit median; fall back to p50 (they are the same quantile).
median = _num(numeric.get("median"))
if median is None:
median = _num(numeric.get("p50"))
# Without the three quartiles a boxplot cannot be drawn.
if q1 is None or q3 is None or median is None:
return {}
# Recompute the IQR from the quartiles rather than trusting numeric['iqr'],
# which may be missing even when the percentiles are present.
iqr = q3 - q1
lower_fence = q1 - 1.5 * iqr
upper_fence = q3 + 1.5 * iqr
mn = _num(numeric.get("min"))
mx = _num(numeric.get("max"))
# Whisker extremes: the real data range clamped to the fences. When the
# corresponding extreme is missing, fall back to the fence itself.
whisker_lo = max(mn, lower_fence) if mn is not None else lower_fence
whisker_hi = min(mx, upper_fence) if mx is not None else upper_fence
has_low_outliers = bool(mn is not None and mn < lower_fence)
has_high_outliers = bool(mx is not None and mx > upper_fence)
# Informative only: these outliers come from the z-score block of the
# profile, not from this IQR fence computation.
raw_n = numeric.get("n_outliers")
n_outliers = int(raw_n) if isinstance(raw_n, (int, float)) and not isinstance(raw_n, bool) else 0
return {
"q1": q1,
"median": median,
"q3": q3,
"iqr": iqr,
"lower_fence": lower_fence,
"upper_fence": upper_fence,
"whisker_lo": whisker_lo,
"whisker_hi": whisker_hi,
"min": mn,
"max": mx,
"has_low_outliers": has_low_outliers,
"has_high_outliers": has_high_outliers,
"n_outliers": n_outliers,
}
@@ -1,108 +0,0 @@
"""Tests para build_boxplot_stats."""
import os
import sys
sys.path.insert(0, os.path.dirname(__file__))
from build_boxplot_stats import build_boxplot_stats
# Keys that a non-empty result dict must always contain.
_EXPECTED_KEYS = {
"q1", "median", "q3", "iqr", "lower_fence", "upper_fence",
"whisker_lo", "whisker_hi", "min", "max",
"has_low_outliers", "has_high_outliers", "n_outliers",
}
def test_boxplot_tukey_basico():
"""Golden: bloque numeric con outlier alto claro -> fences IQR de Tukey."""
numeric = {
"min": 1.0, "max": 100.0,
"p25": 10.0, "median": 25.0, "p75": 40.0,
"iqr": 30.0, "n_outliers": 3,
}
box = build_boxplot_stats(numeric)
assert set(box.keys()) == _EXPECTED_KEYS
assert box["q1"] == 10.0
assert box["median"] == 25.0
assert box["q3"] == 40.0
# iqr recomputado desde los cuartiles.
assert box["iqr"] == 30.0
# lower = 10 - 1.5*30 = -35 ; upper = 40 + 1.5*30 = 85.
assert box["lower_fence"] == -35.0
assert box["upper_fence"] == 85.0
# whisker_lo = max(min=1, -35) = 1 ; whisker_hi = min(max=100, 85) = 85.
assert box["whisker_lo"] == 1.0
assert box["whisker_hi"] == 85.0
assert box["min"] == 1.0
assert box["max"] == 100.0
# Solo hay outliers altos (100 > 85), no bajos (1 no < -35).
assert box["has_low_outliers"] is False
assert box["has_high_outliers"] is True
# n_outliers se propaga del bloque z-score (informativo).
assert box["n_outliers"] == 3
def test_percentiles_faltan_devuelve_vacio():
"""Si falta p25/median/p75 -> {} (caller omite el boxplot)."""
# Falta p25.
assert build_boxplot_stats({"median": 25.0, "p75": 40.0}) == {}
# Falta p75.
assert build_boxplot_stats({"p25": 10.0, "median": 25.0}) == {}
# Falta median y p50.
assert build_boxplot_stats({"p25": 10.0, "p75": 40.0}) == {}
# numeric None / no dict tambien es vacio, nunca lanza.
assert build_boxplot_stats(None) == {}
assert build_boxplot_stats({}) == {}
def test_median_cae_a_p50():
"""median ausente cae a p50."""
numeric = {"min": 0.0, "max": 10.0, "p25": 2.0, "p50": 5.0, "p75": 8.0}
box = build_boxplot_stats(numeric)
assert box["median"] == 5.0
assert box["q1"] == 2.0
assert box["q3"] == 8.0
def test_whiskers_usan_fence_si_falta_min_max():
"""Sin min/max los bigotes caen a las fences y no hay outliers marcados."""
numeric = {"p25": 10.0, "median": 25.0, "p75": 40.0} # sin min ni max
box = build_boxplot_stats(numeric)
assert box["min"] is None
assert box["max"] is None
# iqr = 30, fences -35 / 85; los bigotes caen a las fences.
assert box["whisker_lo"] == box["lower_fence"] == -35.0
assert box["whisker_hi"] == box["upper_fence"] == 85.0
# Sin extremos reales, no se afirma que haya outliers.
assert box["has_low_outliers"] is False
assert box["has_high_outliers"] is False
# n_outliers ausente -> 0.
assert box["n_outliers"] == 0
def test_tipos_salida_float_bool_int():
"""Numericos en float, flags bool nativos, n_outliers int."""
numeric = {
"min": -50.0, "max": 200.0,
"p25": 10.0, "median": 25.0, "p75": 40.0,
"n_outliers": 7,
}
box = build_boxplot_stats(numeric)
for key in ("q1", "median", "q3", "iqr", "lower_fence", "upper_fence",
"whisker_lo", "whisker_hi", "min", "max"):
assert isinstance(box[key], float), f"{key} debe ser float"
assert isinstance(box["has_low_outliers"], bool)
assert isinstance(box["has_high_outliers"], bool)
assert isinstance(box["n_outliers"], int) and not isinstance(box["n_outliers"], bool)
# min=-50 < lower_fence=-35 -> outlier bajo ; max=200 > upper_fence=85 -> alto.
assert box["has_low_outliers"] is True
assert box["has_high_outliers"] is True
assert box["n_outliers"] == 7