merge: 4c cat_distr una hoja por columna (PDF+PPTX 1:1) + sin descripcion entropia redundante + page_break motor (verificado met)

This commit is contained in:
2026-06-30 19:53:57 +02:00
5 changed files with 486 additions and 95 deletions
@@ -1,19 +1,25 @@
"""Categorical distributions chapter (CAT DISTR).
Third reference chapter for AutomaticEDA. For every categorical column it shows,
fulfilling the user's request:
Third reference chapter for AutomaticEDA. Each categorical column gets **its own
page (PDF) / slide (PPTX)**: every column is wrapped in a keep-together
``model.Group`` with ``page_break_before=True`` (except the first, which may share
the intro's page), so its chart sits next to its tables and no column is split.
1. A short opening explanation of **Shannon entropy** (what it measures, its 0
and log2(k) bounds, the normalized 01 version) and the dataset row total used
as a comparison baseline.
2. Per column, a cardinality key/value table: distinct values, ``% distinct``
(distinct / total rows), total dataset rows, singleton values (frequency 1),
entropy with its theoretical maximum and the normalized ratio, mode, imbalance
and string-length stats.
3. A short note flagging problematic cardinality (id-like ≈100% distinct, or a
A short intro names the clickable **[[term:entropia]]entropía[[/term]]** term —
the full definition lives in the GLOSARIO chapter, so it is NOT repeated inline
here (one click jumps to the glossary entry). The intro also carries the dataset
row total used as a comparison baseline.
Per column the Group contains, in order:
1. A cardinality key/value table: distinct values, ``% distinct`` (distinct /
total rows), total dataset rows, singleton values (frequency 1), entropy with
its theoretical maximum and the normalized ratio, mode, imbalance and
string-length stats.
2. A short note flagging problematic cardinality (id-like ≈100% distinct, or a
single dominating category).
4. A ``top-k`` table (value / count / %).
5. A **donut pie chart** of the most common categories (top-k + an "Otros"
3. A ``top-k`` table (value / count / %).
4. A **donut pie chart** of the most common categories (top-k + an "Otros"
bucket), drawn lazily so the renderers scale it to fit entirely.
Data comes from the ``eda`` group: each ``columns[i]['categorical']`` is the
@@ -33,7 +39,7 @@ import math
from .. import model
CHAPTER_VERSION = "1.1.0"
CHAPTER_VERSION = "1.2.0"
CHAPTER_ID = "cat_distr"
CHAPTER_TITLE = "Distribuciones categóricas"
@@ -53,11 +59,17 @@ _TERM_ENTROPIA_DEF = (
# Cap the number of categorical columns rendered to keep the document bounded;
# the rest are summarized in a closing note (no silent truncation).
MAX_COLS = 40
# Rows shown in each top-k table and explicit slices in the pie.
TOP_TABLE_ROWS = 15
# Rows shown in each top-k table and explicit slices in the pie. Kept moderate so
# the whole column — cardinality table + top-k table + donut — fits on ONE
# page/slide with the chart next to its tables; the table note still reports
# "top N of M" so nothing is silently hidden. For id-like columns (≈100%
# distinct) the top-k table is dropped entirely (it would be a list of unique
# values — pure noise), which also frees the room the donut needs (see build).
TOP_TABLE_ROWS = 8
PIE_TOP_K = 6
# Truncate very long category labels in tables (the renderer also wraps).
LABEL_MAX = 48
# Truncate very long category labels in tables (the renderer also wraps). Kept
# tight so a column with long id-like values (names, tickets) still fits its page.
LABEL_MAX = 28
def _fmt_int(value) -> str:
@@ -267,45 +279,55 @@ def _normalize_card(card: dict) -> dict:
def _cardinality_block(card: dict):
"""KVTable with the cardinality / entropy metrics for one column."""
"""KVTable with the cardinality / entropy metrics for one column.
Related metrics are grouped onto a single row each (distinct/%/unique;
entropy bits/max/normalized; length min/mean/max) so the whole column —
table + chart — fits one page/slide without dropping any datum; the short
16:9 PPTX slide does not fit one metric per row plus a chart otherwise."""
n_singletons = card.get("n_singletons")
if n_singletons is not None and card.get("n_singletons_partial"):
singletons = f"{_fmt_int(n_singletons)} (en top mostrado)"
singletons = f"{_fmt_int(n_singletons)}"
elif n_singletons is not None:
singletons = _fmt_int(n_singletons)
else:
singletons = ""
entropy_ref = _fmt_num(card.get("entropy"))
emax = card.get("entropy_max")
if emax is not None:
entropy_ref = f"{entropy_ref} (máx {_fmt_num(emax)})"
# Distinct count · % distinct · unique (frequency 1) on one row.
distinct_combo = (f"{_fmt_int(card.get('n_distinct'))} · "
f"{_fmt_pct_value(card.get('pct_distinct'))} · "
f"{singletons} únicos")
# Entropy bits · theoretical max · normalized 01 on one row.
entropy_combo = (f"{_fmt_num(card.get('entropy'))} bits · "
f"máx {_fmt_num(card.get('entropy_max'))} · "
f"norm {_fmt_num(card.get('entropy_norm'))}")
mode = card.get("mode")
mode_pct = card.get("mode_pct")
mode_str = "" if mode is None else model._safe_str(mode)
mode_str = "" if mode is None else _truncate(mode, 32)
if mode is not None and mode_pct is not None:
mode_str = f"{mode_str} ({_fmt_pct_value(mode_pct)})"
rows = [
("Valores distintos", _fmt_int(card.get("n_distinct"))),
("% distintos", _fmt_pct_value(card.get("pct_distinct"))),
("Distintos · % · únicos", distinct_combo),
("Total filas (dataset)", _fmt_int(card.get("n_rows"))),
("Valores únicos (frecuencia 1)", singletons),
("Entropía (bits)", entropy_ref),
("Entropía normalizada (01)", _fmt_num(card.get("entropy_norm"))),
("Entropía (bits · máx · norm)", entropy_combo),
("Moda", mode_str),
]
imbalance = card.get("imbalance")
if imbalance is not None:
rows.append(("Desbalance", _fmt_num(imbalance)))
lm = card.get("len_min")
lmean = card.get("len_mean")
lmax = card.get("len_max")
# Imbalance and string length (both secondary) share one closing row.
extras = []
if imbalance is not None:
extras.append(f"desbalance {_fmt_num(imbalance)}")
if any(v is not None for v in (lm, lmean, lmax)):
rows.append((
"Longitud (mín/media/máx)",
f"{_fmt_num(lm)} / {_fmt_num(lmean)} / {_fmt_num(lmax)}"))
extras.append(
f"long. {_fmt_num(lm)}/{_fmt_num(lmean)}/{_fmt_num(lmax)}")
if extras:
rows.append(("Desbalance · longitud", " · ".join(extras)))
return model.KVTable(rows=rows, title="Cardinalidad")
@@ -315,7 +337,8 @@ def _flag_note(card: dict):
return model.Note(
"Casi todos los valores son distintos (≈100% distintos): la columna "
"se comporta como un identificador y aporta poco para agrupar o "
"comparar categorías.")
"comparar categorías. No se lista el top de categorías (serían "
"valores casi todos únicos).")
if card.get("dominated"):
mp = card.get("mode_pct")
mp_str = _fmt_pct_value(mp) if mp is not None else "muy alta"
@@ -335,7 +358,7 @@ def _topk_table(cat: dict):
if not isinstance(t, dict):
continue
rows.append([
model._safe_str(t.get("value")),
_truncate(t.get("value")),
_fmt_int(t.get("count")),
_pct_from_maybe_fraction(t.get("pct")),
])
@@ -353,20 +376,16 @@ def _topk_table(cat: dict):
def _intro_blocks(n_rows, mark_term: bool = False):
total = _fmt_int(n_rows)
# Mark the first appearance of the term as a clickable glossary jump when the
# term was registered (mark_term). The visible text is identical either way.
entropia = ("[[term:entropia]]**entropía de Shannon**[[/term]]" if mark_term
else "**entropía de Shannon**")
# term was registered (mark_term). The full definition of entropy lives in the
# GLOSARIO chapter, so the intro only names the clickable term here instead of
# repeating the long explanation (avoids the redundancy with the glossary).
entropia = ("[[term:entropia]]entropía[[/term]]" if mark_term
else "entropía")
text = (
f"La {entropia} mide cómo de repartidos están los valores de "
"una columna categórica, en bits. Vale 0 cuando una sola categoría "
"concentra todas las filas (xima previsibilidad) y alcanza su máximo, "
"log2(k) para k categorías distintas, cuando todas aparecen por igual "
"(máxima diversidad). La **entropía normalizada** (entropía dividida por "
"su máximo) la lleva al rango 01 para comparar columnas con distinto "
"número de categorías. Para cada columna se muestran los valores "
"distintos, el porcentaje que representan sobre el total de filas, los "
"valores únicos (que aparecen una sola vez), la tabla de las categorías "
"más frecuentes y un gráfico de tarta (donut) de las más comunes."
f"Cada columna categórica ocupa su propia página: sus métricas de "
f"cardinalidad —incluida la {entropia}—, una nota que señala cardinalidad "
"problemática, la tabla de las categorías más frecuentes y un gráfico de "
"tarta (donut) de las más comunes, todo junto."
)
if n_rows is not None:
text += f" El dataset tiene {total} filas en total como referencia."
@@ -398,24 +417,37 @@ def build_cat_distr(profile: dict, ctx: dict):
blocks = list(_intro_blocks(n_rows, mark_term=mark_term))
rendered = cat_cols[:MAX_COLS]
for col in rendered:
for idx, col in enumerate(rendered):
name = col.get("name") or "(columna)"
cat = col.get("categorical") or {}
card = _normalize_card(_cardinality(cat, n_rows))
blocks.append(model.Heading(text=str(name), level=2))
blocks.append(_cardinality_block(card))
# One Group per categorical column: heading + cardinality table + flag
# note + top-k table + donut figure are kept together and the renderer
# starts each on a fresh page/slide (page_break_before) so every column
# gets its own page with its chart next to its tables. The first column
# may share the intro's page (no forced break) to avoid a near-empty page.
col_blocks = [
model.Heading(text=str(name), level=2),
_cardinality_block(card),
]
note = _flag_note(card)
if note is not None:
blocks.append(note)
topk = _topk_table(cat)
if topk is not None:
blocks.append(topk)
blocks.append(model.Figure(
col_blocks.append(note)
# For id-like columns (≈100% distinct) the top-k is a list of unique
# values — pure noise; skip it (the flag note already explains why) and
# let the donut take that room so the whole column fits one page/slide.
if not card.get("id_like"):
topk = _topk_table(cat)
if topk is not None:
col_blocks.append(topk)
col_blocks.append(model.Figure(
make=_pie_make(cat.get("top") or [], card.get("n_distinct"),
str(name), n_rows),
caption=(f"Categorías más comunes de «{_truncate(name, 32)}» "
"(donut: top-k + «Otros»)")))
blocks.append(model.Group(blocks=col_blocks,
page_break_before=(idx > 0)))
if len(cat_cols) > len(rendered):
omitted = len(cat_cols) - len(rendered)
@@ -2,11 +2,14 @@
Self-contained: builds synthetic TableProfiles (no DuckDB) so the suite is fast
and deterministic. Verifies that ``build_cat_distr`` emits the blocks the user
asked for (entropy intro, distinct/total/%-distinct/unique metrics, top-k table
and a donut figure), that the chapter renders inside the full document to both
PDF and PPTX showing that content, that a profile with no categorical columns
yields ``None`` without raising, and that long labels / many columns are never
cut in either output.
asked for (distinct/total/%-distinct/unique metrics, top-k table and a donut
figure), that EACH categorical column is wrapped in its own keep-together
``Group`` that starts on a fresh page/slide (one column per page, chart next to
its tables), that the long entropy explanation is NOT repeated inline (it lives
in the glossary — only the clickable term is kept), that the chapter renders
inside the full document to both PDF and PPTX showing that content, that a
profile with no categorical columns yields ``None`` without raising, and that
long labels / many columns are never cut in either output.
"""
import os
@@ -17,7 +20,8 @@ from pypdf import PdfReader
from pptx import Presentation
from datascience.automatic_eda.model import (
DataTable, Figure, Heading, KVTable, Note,
DataTable, Figure, GlossaryCollector, Group, Heading, KVTable, Markdown,
Note,
)
from datascience.automatic_eda.chapters.cat_distr import (
CHAPTER_ID, CHAPTER_VERSION, build_cat_distr,
@@ -81,8 +85,20 @@ def _pptx_text(path: str) -> str:
return re.sub(r"\s+", " ", " ".join(parts))
def _kinds(chapter):
return [b.kind for b in chapter.blocks]
def _flatten(blocks):
"""Expand keep-together Groups so the per-column heading/table/figure are
inspectable as a flat block list (the chapter wraps each column in a Group)."""
out = []
for b in blocks:
if getattr(b, "kind", "") == "group":
out.extend(_flatten(getattr(b, "blocks", []) or []))
else:
out.append(b)
return out
def _column_groups(chapter):
return [b for b in chapter.blocks if isinstance(b, Group)]
def test_golden_build_cat_distr_emite_bloques_pedidos():
@@ -90,36 +106,101 @@ def test_golden_build_cat_distr_emite_bloques_pedidos():
assert ch is not None
assert ch.id == CHAPTER_ID
assert ch.version == CHAPTER_VERSION
kinds = _kinds(ch)
# Entropy intro present.
# Entropy intro present, but the long explanation is gone (it lives in the
# glossary now): only the term is named, no log2/normalizada walkthrough.
headings = [b.text for b in ch.blocks if isinstance(b, Heading)]
assert any("Entrop" in h for h in headings)
md = next(b for b in ch.blocks if b.kind == "markdown")
assert "entropía" in md.text.lower() and "log2" in md.text
# Cardinality metrics: distinct, total rows, %-distinct, unique values.
kv = next(b for b in ch.blocks if isinstance(b, KVTable))
md = next(b for b in ch.blocks if isinstance(b, Markdown))
assert "entropía" in md.text.lower()
assert "log2" not in md.text # redundant explanation removed.
assert "máxima diversidad" not in md.text
# Per-column blocks are wrapped in keep-together Groups: flatten to inspect.
flat = _flatten(ch.blocks)
kv = next(b for b in flat if isinstance(b, KVTable))
labels = [r[0] for r in kv.rows]
assert "Valores distintos" in labels
assert "% distintos" in labels
values = " ".join(str(r[1]) for r in kv.rows)
# Cardinality metrics: distinct count, %-distinct, unique values and total
# rows are present (grouped onto compact rows so the chart fits the page).
assert "Distintos · % · únicos" in labels
assert "Total filas (dataset)" in labels
assert "Valores únicos (frecuencia 1)" in labels
assert any("Entropía" in lbl for lbl in labels)
assert "únicos" in values and "%" in values
assert "bits" in values and "norm" in values # entropy + max + normalized.
# Top-k table + pie figure.
dt = next(b for b in ch.blocks if isinstance(b, DataTable))
dt = next(b for b in flat if isinstance(b, DataTable))
assert dt.header == ["Valor", "Conteo", "%"]
assert any("neumaticos" in str(cell) for row in dt.rows for cell in row)
assert any(isinstance(b, Figure) for b in ch.blocks)
# id-like column flagged with a Note.
assert any(isinstance(b, Note) and "identificador" in b.text
for b in ch.blocks)
assert any(isinstance(b, Figure) for b in flat)
# id-like column flagged with a Note that also explains the top-k is dropped.
idnote = next((b for b in flat
if isinstance(b, Note) and "identificador" in b.text), None)
assert idnote is not None
assert "No se lista el top" in idnote.text
def test_golden_render_pdf_muestra_categoricas():
def test_golden_idlike_omite_topk_y_conserva_donut():
# The id-like column (uuid, 100% distinct) must NOT carry a top-k DataTable
# (it would be a list of unique values), but must still keep its donut Figure
# and its cardinality table so it stays a full per-column page.
ch = build_cat_distr(_profile(), {})
groups = _column_groups(ch)
uuid_group = next(g for g in groups
if any(getattr(b, "text", "") == "uuid" for b in g.blocks))
kinds = [b.kind for b in uuid_group.blocks]
assert "data_table" not in kinds # top-k of unique values dropped.
assert "kv_table" in kinds # cardinality kept.
assert "figure" in kinds # donut kept (chart per column).
# A non-id-like column keeps its top-k table.
cat_group = next(g for g in groups
if any(getattr(b, "text", "") == "categoria"
for b in g.blocks))
assert "data_table" in [b.kind for b in cat_group.blocks]
def test_golden_una_pagina_por_columna_groups():
ch = build_cat_distr(_profile(), {})
groups = _column_groups(ch)
# Two categorical columns -> two column Groups (numeric column excluded).
assert len(groups) == 2
# Each Group carries one column: a heading + its cardinality table + figure.
for g in groups:
kinds = [b.kind for b in g.blocks]
assert kinds[0] == "heading"
assert "kv_table" in kinds
assert "figure" in kinds
# The first column may share the intro page (no forced break); every later
# column starts on a fresh page/slide so each column gets its own page.
assert groups[0].page_break_before is False
assert all(g.page_break_before is True for g in groups[1:])
def test_golden_entropia_clicable_y_definicion_en_glosario():
# With a glossary collector the intro marks the clickable term and the FULL
# definition (the long explanation removed from the intro) lands in the
# glossary, not inline — no data lost, just relocated.
gc = GlossaryCollector()
ch = build_cat_distr(_profile(), {"glossary": gc})
md = next(b for b in ch.blocks if isinstance(b, Markdown))
assert "[[term:entropia]]entropía[[/term]]" in md.text
assert gc.has("entropia")
entry = gc.get("entropia")
assert entry is not None
# The definition kept in the glossary still carries the detail removed inline.
assert "log2" in entry["definition"]
assert "normalizada" in entry["definition"].lower()
def test_golden_render_pdf_una_pagina_por_columna():
with tempfile.TemporaryDirectory() as d:
out = os.path.join(d, "eda.pdf")
res = render_automatic_eda_pdf(_profile(), out, {"title": "EDA"})
assert res["path"] == out and os.path.exists(out)
assert CHAPTER_ID in [c["id"] for c in res["chapters"]]
cat_meta = next(c for c in res["chapters"] if c["id"] == CHAPTER_ID)
# Two categorical columns, each on its own page -> >= 2 pages for the
# chapter (intro shares the first column's page).
assert cat_meta["n_pages"] >= 2
txt = _pdf_text(out)
assert "Entrop" in txt
assert "distintos" in txt
@@ -133,13 +214,91 @@ def test_golden_render_pptx_muestra_categoricas():
out = os.path.join(d, "eda.pptx")
res = render_automatic_eda_pptx(_profile(), out, {"title": "EDA"})
assert res["path"] == out and os.path.exists(out)
assert CHAPTER_ID in [c["id"] for c in res["chapters"]]
cat_meta = next(c for c in res["chapters"] if c["id"] == CHAPTER_ID)
assert cat_meta["n_slides"] >= 2 # one slide per categorical column.
txt = _pptx_text(out)
assert "Entrop" in txt
assert "categoria" in txt and "neumaticos" in txt
assert "distintos" in txt
def _profile_high_card() -> dict:
"""Profile with a high-cardinality NON-id-like categorical column whose top-k
of long values would split from its donut on a short 16:9 slide unless the
renderer trims the table — the exact case the adversarial check flagged
(Ticket / Cabin)."""
long_vals = [f"Valor largo de categoria numero {i:02d} con texto extra"
for i in range(40)]
top = [{"value": v, "count": 60 - i, "pct": (60 - i) / 5000.0}
for i, v in enumerate(long_vals)]
return {
"table": "t", "source": "t.csv", "n_rows": 5000, "n_cols": 3,
"quality_score": 80.0,
"columns": [
{"name": "precio", "inferred_type": "numeric", "null_pct": 0.0,
"numeric": {"mean": 1.0, "median": 1.0, "min": 0.0, "max": 2.0,
"std": 0.5}},
# 40 distinct over 5000 rows = 0.8% distinct -> NOT id-like, keeps
# its (long) top-k table; the tall table must not push the donut off.
{"name": "alta_card_col", "inferred_type": "categorical",
"null_pct": 0.0, "distinct_count": 40,
"categorical": {"top": top, "mode": long_vals[0], "n_distinct": 40,
"entropy": 5.2, "imbalance": 1.2, "len_min": 40,
"len_mean": 45, "len_max": 50}},
{"name": "baja_card_col", "inferred_type": "categorical",
"null_pct": 0.0, "distinct_count": 4,
"categorical": {
"top": [{"value": "norte", "count": 2000, "pct": 0.4},
{"value": "sur", "count": 1500, "pct": 0.3},
{"value": "este", "count": 1000, "pct": 0.2},
{"value": "oeste", "count": 500, "pct": 0.1}],
"mode": "norte", "n_distinct": 4, "entropy": 1.8}},
],
}
def test_golden_pptx_una_slide_por_columna_con_su_grafico():
"""Each categorical column occupies EXACTLY ONE cat_distr slide that carries
BOTH its cardinality table and its donut figure (picture) — i.e. the chart is
never separated from its table, even for a high-cardinality column."""
from pptx.enum.shapes import MSO_SHAPE_TYPE
prof = _profile_high_card()
cat_names = ["alta_card_col", "baja_card_col"]
with tempfile.TemporaryDirectory() as d:
out = os.path.join(d, "eda.pptx")
res = render_automatic_eda_pptx(prof, out, {"title": "EDA"})
assert res["path"] == out and os.path.exists(out)
prs = Presentation(out)
# Per column: the cat_distr slides whose text mentions it, and whether the
# owning slide also has the donut caption + an actual picture shape.
slides_with_col = {n: [] for n in cat_names}
owner_has_chart = {n: False for n in cat_names}
for i, sl in enumerate(prs.slides):
texts, has_pic = [], False
for sh in sl.shapes:
if sh.has_text_frame:
texts.append(sh.text_frame.text)
if sh.shape_type == MSO_SHAPE_TYPE.PICTURE:
has_pic = True
txt = re.sub(r"\s+", " ", " ".join(texts))
if "Distribuciones categ" not in txt: # footer stamp of the chapter.
continue
for n in cat_names:
if n in txt:
slides_with_col[n].append(i)
has_table = "Cardinalidad" in txt or "distintos" in txt
if has_pic and "donut" in txt and has_table:
owner_has_chart[n] = True
for n in cat_names:
# Exactly one slide carries the column (not split across slides).
assert len(slides_with_col[n]) == 1, (n, slides_with_col[n])
# That single slide also holds its table AND its donut picture.
assert owner_has_chart[n], (n, "tabla y donut no están en el mismo slide")
def test_edge_sin_categoricas_devuelve_none():
only_numeric = {
"n_rows": 10, "columns": [
@@ -170,11 +329,15 @@ def test_anti_corte_label_largo_y_muchas_columnas():
ch = build_cat_distr(profile, {})
assert ch is not None
# One Group per column, each forcing its own page (except the first).
groups = _column_groups(ch)
assert len(groups) == 30
assert sum(1 for g in groups if g.page_break_before) == 29
with tempfile.TemporaryDirectory() as d:
pdf = os.path.join(d, "anti.pdf")
res = render_automatic_eda_pdf(profile, pdf, {"write_manifest": False})
assert res["path"] == pdf
assert res["n_pages"] > 1 # many columns spilled across pages, OK.
assert res["n_pages"] > 1 # one page per column, OK.
txt = _pdf_text(pdf)
# Long label wrapped (not truncated): every word survives.
for word in ("Lorem", "incididunt", "reprehenderit", "voluptate"):