Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 13c82be780 |
@@ -89,35 +89,6 @@ _DEF_MAX_CARD = 20
|
||||
_DEF_MAX_MEASURES = 4
|
||||
_DEF_TOP_N = 12
|
||||
|
||||
# Glossary terms this chapter explains. Both appear in the always-rendered intro,
|
||||
# so they are registered and marked clickable whenever a collector is in ctx —
|
||||
# the canonical two-step pattern (see ``cat_distr``): ``glossary.add(key, label,
|
||||
# definition)`` + the inline span ``[[term:KEY]]texto[[/term]]`` in a Markdown
|
||||
# block. Mapping key -> (label, definition).
|
||||
_TERM_DEFS = {
|
||||
"groupby": (
|
||||
"Agrupación (split-apply-combine)",
|
||||
"Operación de agrupación (group by): parte la tabla en grupos según los "
|
||||
"valores de una columna categórica, aplica un cálculo (conteo, media, "
|
||||
"mediana…) dentro de cada grupo y combina los resultados en una tabla "
|
||||
"resumen. Es el patrón split-apply-combine."),
|
||||
"pivot_table": (
|
||||
"Tabla dinámica (pivot)",
|
||||
"Tabla dinámica que cruza dos variables categóricas — una en las filas y "
|
||||
"otra en las columnas — y rellena cada celda con un agregado (media, "
|
||||
"suma…) de una medida numérica. Resume de un vistazo cómo interactúan las "
|
||||
"dos categóricas sobre esa medida."),
|
||||
}
|
||||
|
||||
|
||||
def _term(mark: bool, key: str, text: str) -> str:
|
||||
"""Wrap ``text`` as a clickable glossary span when ``mark`` is True.
|
||||
|
||||
The visible text is identical with or without the marker (the renderers strip
|
||||
it), so wrapping never changes line layout — it only adds the link.
|
||||
"""
|
||||
return f"[[term:{key}]]{text}[[/term]]" if mark else text
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Formatting helpers (mirror the other chapters' defensive style).
|
||||
@@ -554,18 +525,13 @@ def _sections_live(profile: dict, ctx: dict, candidates: dict) -> list:
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Entry point.
|
||||
# --------------------------------------------------------------------------- #
|
||||
def _intro_blocks(gloss=None, mark_term: bool = False) -> list:
|
||||
if gloss is not None:
|
||||
for key, (label, definition) in _TERM_DEFS.items():
|
||||
gloss.add(key, label, definition)
|
||||
t_groupby = _term(mark_term, "groupby", "**por grupos** (split-apply-combine)")
|
||||
t_pivot = _term(mark_term, "pivot_table", "**tablas dinámicas** (pivot)")
|
||||
def _intro_blocks() -> list:
|
||||
text = (
|
||||
f"Este capítulo analiza la tabla {t_groupby}: "
|
||||
"Este capítulo analiza la tabla **por grupos** (split-apply-combine): "
|
||||
"elige las columnas categóricas más informativas — por su cardinalidad "
|
||||
"y relevancia, no todas contra todas, para no inflar comparaciones "
|
||||
"espurias — y resume las variables numéricas dentro de cada grupo "
|
||||
f"(conteo, media, mediana, desviación). Las {t_pivot} "
|
||||
"(conteo, media, mediana, desviación). Las **tablas dinámicas** (pivot) "
|
||||
"cruzan dos categóricas sobre una medida, y los **gráficos de barras** "
|
||||
"(siempre desde cero) comparan los grupos de un vistazo."
|
||||
)
|
||||
@@ -590,21 +556,13 @@ def build_agregacion(profile: dict, ctx: dict):
|
||||
if not isinstance(profile, dict):
|
||||
return None
|
||||
|
||||
# Shared glossary collector: groupby + pivot_table live in the always-present
|
||||
# intro, so they are registered + marked there. Degrades silently (mark_term
|
||||
# False) when no collector is in ctx (standalone render).
|
||||
glossary = ctx.get("glossary")
|
||||
gloss = glossary if isinstance(glossary, model.GlossaryCollector) else None
|
||||
mark_term = gloss is not None
|
||||
|
||||
# Pre-computed results take precedence (offline / tests / forward-compat).
|
||||
pre = ctx.get("aggregations")
|
||||
if _is_dict(pre) and (pre.get("groupby") or pre.get("pivots")):
|
||||
sections = _sections_from_precomputed(pre)
|
||||
if not sections:
|
||||
return None
|
||||
blocks = (_intro_blocks(gloss, mark_term) + sections
|
||||
+ _insights_section(ctx))
|
||||
blocks = _intro_blocks() + sections + _insights_section(ctx)
|
||||
return model.Chapter(id=CHAPTER_ID, title=CHAPTER_TITLE,
|
||||
version=CHAPTER_VERSION, blocks=blocks)
|
||||
|
||||
@@ -625,11 +583,10 @@ def build_agregacion(profile: dict, ctx: dict):
|
||||
"crudos. Pasa ctx['db_path'] + ctx['table'] (para el cálculo "
|
||||
"push-down en DuckDB) o ctx['aggregations'] ya precalculado. "
|
||||
f"Columnas categóricas candidatas: {keys or '—'}.")
|
||||
blocks = (_intro_blocks(gloss, mark_term) + [note]
|
||||
+ _insights_section(ctx))
|
||||
blocks = _intro_blocks() + [note] + _insights_section(ctx)
|
||||
return model.Chapter(id=CHAPTER_ID, title=CHAPTER_TITLE,
|
||||
version=CHAPTER_VERSION, blocks=blocks)
|
||||
|
||||
blocks = _intro_blocks(gloss, mark_term) + sections + _insights_section(ctx)
|
||||
blocks = _intro_blocks() + sections + _insights_section(ctx)
|
||||
return model.Chapter(id=CHAPTER_ID, title=CHAPTER_TITLE,
|
||||
version=CHAPTER_VERSION, blocks=blocks)
|
||||
|
||||
@@ -254,25 +254,3 @@ def test_anti_corte_muchos_grupos_y_texto_largo():
|
||||
# First, middle and last words of the long paragraph all present.
|
||||
for i in (0, 60, 119):
|
||||
assert f"palabra{i}" in txt
|
||||
|
||||
|
||||
def test_glosario_engancha_groupby_y_pivot():
|
||||
"""Mejora 4b: la agrupación (split-apply-combine) y la tabla dinámica (pivot)
|
||||
se registran en el colector compartido y se marcan clicables en el cuerpo.
|
||||
Sin colector en ctx, el capítulo degrada y no marca nada."""
|
||||
from datascience.automatic_eda.model import GlossaryCollector
|
||||
|
||||
g = GlossaryCollector()
|
||||
ctx = dict(_ctx_precomputed())
|
||||
ctx["glossary"] = g
|
||||
ch = build_agregacion(_profile(), ctx)
|
||||
assert ch is not None
|
||||
keys = {t["key"] for t in g.terms()}
|
||||
assert {"groupby", "pivot_table"} <= keys
|
||||
body = " ".join(b.text for b in ch.blocks if b.kind == "markdown")
|
||||
assert "[[term:groupby]]" in body and "[[term:pivot_table]]" in body
|
||||
|
||||
# Sin colector: degrada limpio (ningún marcador en el cuerpo).
|
||||
ch2 = build_agregacion(_profile(), _ctx_precomputed())
|
||||
body2 = " ".join(b.text for b in ch2.blocks if b.kind == "markdown")
|
||||
assert "[[term:" not in body2
|
||||
|
||||
@@ -47,53 +47,6 @@ _MAX_MATRIX_LABELS = 16
|
||||
# How many pairs to show in each of the top-positive / top-negative tables.
|
||||
_TOP_N = 10
|
||||
|
||||
# Glossary terms this chapter explains. Each is registered in the shared
|
||||
# collector (ctx['glossary']) and marked clickable on its first appearance in the
|
||||
# body — the canonical two-step pattern (see ``cat_distr`` for the reference
|
||||
# implementation): ``glossary.add(key, label, definition)`` + the inline span
|
||||
# ``[[term:KEY]]texto visible[[/term]]`` in a Markdown block. Mapping key ->
|
||||
# (label, definition). ``fdr`` is only registered when the FDR summary is present.
|
||||
_TERM_DEFS = {
|
||||
"pearson": (
|
||||
"Pearson (coeficiente r)",
|
||||
"Coeficiente de correlación lineal de Pearson (r) entre dos variables "
|
||||
"numéricas. Va de −1 (relación lineal inversa perfecta) a +1 (directa "
|
||||
"perfecta); 0 indica ausencia de relación lineal. Sólo capta relaciones "
|
||||
"lineales, por eso lleva signo."),
|
||||
"spearman": (
|
||||
"Spearman (correlación de rangos)",
|
||||
"Correlación de rangos de Spearman: el coeficiente de Pearson calculado "
|
||||
"sobre los puestos (rangos) de los valores en vez de sus magnitudes. Mide "
|
||||
"relaciones monótonas (no necesariamente lineales), va de −1 a +1 y es "
|
||||
"robusta frente a valores atípicos."),
|
||||
"cramers_v": (
|
||||
"Cramér's V",
|
||||
"Medida de asociación entre dos variables categóricas, derivada del "
|
||||
"estadístico chi-cuadrado y normalizada al rango 0–1 (0 = independientes, "
|
||||
"1 = asociación total). No tiene signo: sólo mide la intensidad."),
|
||||
"correlation_ratio": (
|
||||
"Razón de correlación (η)",
|
||||
"Razón de correlación (eta) entre una variable numérica y una "
|
||||
"categórica: la fracción de la varianza de la numérica explicada por los "
|
||||
"grupos de la categórica. Va de 0 (los grupos no explican nada) a 1 (la "
|
||||
"explican toda); no tiene signo."),
|
||||
"fdr": (
|
||||
"Comparaciones múltiples (FDR)",
|
||||
"Al evaluar muchos pares a la vez, algunos parecen significativos por "
|
||||
"puro azar. La corrección por tasa de falsos descubrimientos (FDR, "
|
||||
"Benjamini-Hochberg) ajusta los p-valores para controlar la proporción "
|
||||
"esperada de falsos positivos entre los pares declarados significativos."),
|
||||
}
|
||||
|
||||
|
||||
def _term(mark: bool, key: str, text: str) -> str:
|
||||
"""Wrap ``text`` as a clickable glossary span when ``mark`` is True.
|
||||
|
||||
The visible text is identical with or without the marker (the renderers strip
|
||||
the marker), so wrapping never changes line layout — it only adds the link.
|
||||
"""
|
||||
return f"[[term:{key}]]{text}[[/term]]" if mark else text
|
||||
|
||||
|
||||
def _is_num(v) -> bool:
|
||||
"""True for a real, finite int/float (not bool, not NaN/inf)."""
|
||||
@@ -292,7 +245,7 @@ def _methods_block(corr: dict):
|
||||
return model.KVTable(rows=rows, title="Métodos de asociación")
|
||||
|
||||
|
||||
def _fdr_text(corr: dict, mark_term: bool = False) -> str | None:
|
||||
def _fdr_text(corr: dict) -> str | None:
|
||||
"""One-line summary of the multiple-testing (FDR) correction, or None."""
|
||||
mt = corr.get("multiple_testing")
|
||||
if not isinstance(mt, dict) or not mt:
|
||||
@@ -301,8 +254,7 @@ def _fdr_text(corr: dict, mark_term: bool = False) -> str | None:
|
||||
alpha = mt.get("alpha")
|
||||
n_tests = mt.get("n_tests")
|
||||
n_rej = mt.get("n_rejected")
|
||||
multi = _term(mark_term, "fdr", "comparaciones múltiples")
|
||||
parts = [f"Corrección por {multi} ({method}"]
|
||||
parts = [f"Corrección por comparaciones múltiples ({method}"]
|
||||
if _is_num(alpha):
|
||||
parts[0] += f", α={float(alpha):g}"
|
||||
parts[0] += ")."
|
||||
@@ -337,31 +289,13 @@ def build_correlacion(profile: dict, ctx: dict):
|
||||
|
||||
blocks: list = []
|
||||
|
||||
# Register the always-present method terms in the shared glossary and mark
|
||||
# their first appearance clickable (the FDR term is registered lazily below,
|
||||
# only when the FDR summary is actually emitted). Degrades silently when no
|
||||
# collector is in ctx (standalone render) — mark_term stays False.
|
||||
glossary = ctx.get("glossary")
|
||||
gloss = glossary if isinstance(glossary, model.GlossaryCollector) else None
|
||||
mark_term = gloss is not None
|
||||
if gloss is not None:
|
||||
for key in ("pearson", "spearman", "cramers_v", "correlation_ratio"):
|
||||
label, definition = _TERM_DEFS[key]
|
||||
gloss.add(key, label, definition)
|
||||
|
||||
# Intro: what this chapter shows and how to read the sign. Build the marked
|
||||
# method names as locals first (avoids backslash-in-f-string for "Cramér's V").
|
||||
t_pearson = _term(mark_term, "pearson", "Pearson")
|
||||
t_spearman = _term(mark_term, "spearman", "Spearman")
|
||||
t_cramers = _term(mark_term, "cramers_v", "Cramér's V")
|
||||
t_corr_ratio = _term(mark_term, "correlation_ratio", "razón de correlación")
|
||||
# Intro: what this chapter shows and how to read the sign.
|
||||
blocks.append(model.Markdown(text=(
|
||||
"Asociación entre columnas. Cada par se evalúa con la métrica adecuada a "
|
||||
f"sus tipos ({t_pearson}/{t_spearman} entre numéricas — con **signo**; "
|
||||
f"{t_cramers} entre categóricas; {t_corr_ratio} num-categórica; "
|
||||
"información mutua como medida común no lineal). Sólo las correlaciones "
|
||||
"**num-num** tienen dirección: por eso los pares **negativos** son siempre "
|
||||
"num-num.")))
|
||||
"sus tipos (Pearson/Spearman entre numéricas — con **signo**; Cramér's V "
|
||||
"entre categóricas; razón de correlación num-categórica; información mutua "
|
||||
"como medida común no lineal). Sólo las correlaciones **num-num** tienen "
|
||||
"dirección: por eso los pares **negativos** son siempre num-num.")))
|
||||
|
||||
# 1) Association matrix (heatmap).
|
||||
labels, trimmed = _ordered_labels(pairs)
|
||||
@@ -403,13 +337,9 @@ def build_correlacion(profile: dict, ctx: dict):
|
||||
"no estacionarias y pueden ser espurias (Granger–Newbold). Compáralas "
|
||||
"sobre los retornos/diferencias antes de interpretarlas.")))
|
||||
|
||||
# 4) FDR summary + methods legend. Register the FDR term only when its
|
||||
# summary is emitted, so the glossary never lists an unreferenced entry.
|
||||
fdr_text = _fdr_text(corr, mark_term=mark_term)
|
||||
# 4) FDR summary + methods legend.
|
||||
fdr_text = _fdr_text(corr)
|
||||
if fdr_text:
|
||||
if gloss is not None:
|
||||
label, definition = _TERM_DEFS["fdr"]
|
||||
gloss.add("fdr", label, definition)
|
||||
blocks.append(model.Markdown(text=fdr_text))
|
||||
methods = _methods_block(corr)
|
||||
if methods is not None:
|
||||
|
||||
@@ -173,25 +173,3 @@ def test_anticorte_matriz_ancha_y_etiquetas_largas_no_se_cortan():
|
||||
assert rx["path"] == pptx and os.path.exists(pptx) and rx["n_slides"] >= 1
|
||||
# A short, unbreakable fragment of the long label survives the wrap.
|
||||
assert "azufre" in _pdf_text(pdf)
|
||||
|
||||
|
||||
def test_glosario_engancha_metodos_y_fdr():
|
||||
"""Mejora 4b: los métodos de correlación (Pearson, Spearman, Cramér's V,
|
||||
razón de correlación) y la corrección por comparaciones múltiples (FDR) se
|
||||
registran en el colector compartido y se marcan clicables en el cuerpo. Sin
|
||||
colector en ctx, el capítulo degrada y no marca nada."""
|
||||
from datascience.automatic_eda.model import GlossaryCollector
|
||||
|
||||
g = GlossaryCollector()
|
||||
ch = build_correlacion(_profile(), {"glossary": g})
|
||||
assert ch is not None
|
||||
keys = {t["key"] for t in g.terms()}
|
||||
assert {"pearson", "spearman", "cramers_v", "correlation_ratio", "fdr"} <= keys
|
||||
body = " ".join(b.text for b in ch.blocks if b.kind == "markdown")
|
||||
for k in ("pearson", "spearman", "cramers_v", "correlation_ratio", "fdr"):
|
||||
assert f"[[term:{k}]]" in body, k
|
||||
|
||||
# Sin colector: degrada limpio (ningún marcador en el cuerpo).
|
||||
ch2 = build_correlacion(_profile(), {})
|
||||
body2 = " ".join(b.text for b in ch2.blocks if b.kind == "markdown")
|
||||
assert "[[term:" not in body2
|
||||
|
||||
@@ -55,62 +55,6 @@ _CLUSTER_COLORS = [
|
||||
"#edc948", "#b07aa1", "#ff9da7", "#9c755f", "#bab0ac",
|
||||
]
|
||||
|
||||
# Glossary terms this chapter explains. Each is registered in the shared
|
||||
# collector (ctx['glossary']) and marked clickable on its first appearance — the
|
||||
# canonical two-step pattern (see ``cat_distr``): ``glossary.add(key, label,
|
||||
# definition)`` + the inline span ``[[term:KEY]]texto[[/term]]`` in a Markdown
|
||||
# block. A term is registered only when its section is actually rendered, so the
|
||||
# glossary never lists an entry no in-text appearance points to.
|
||||
_TERM_DEFS = {
|
||||
"zscore": (
|
||||
"Estandarización z-score",
|
||||
"Transformación que lleva cada columna numérica a media 0 y desviación "
|
||||
"típica 1: a cada valor le resta la media de su columna y lo divide por "
|
||||
"la desviación típica. Así variables con escalas muy distintas (euros "
|
||||
"frente a un ratio 0–1) pesan por igual en las distancias y la varianza."),
|
||||
"pca": (
|
||||
"PCA (componentes principales)",
|
||||
"El análisis de componentes principales resume muchas variables "
|
||||
"numéricas correlacionadas en pocos ejes nuevos (componentes), "
|
||||
"ortogonales entre sí y ordenados por la cantidad de varianza que "
|
||||
"capturan. Permite ver la estructura de los datos en 2D y saber cuántas "
|
||||
"dimensiones bastan para explicarlos."),
|
||||
"kmeans": (
|
||||
"KMeans (segmentación)",
|
||||
"Algoritmo de agrupamiento no supervisado que reparte las filas en k "
|
||||
"segmentos: asigna cada fila al centro (centroide) más cercano y recoloca "
|
||||
"los centroides de forma iterativa hasta minimizar la distancia interna "
|
||||
"de cada grupo. Aquí k se elige automáticamente."),
|
||||
"silhouette": (
|
||||
"Coeficiente de silueta (silhouette)",
|
||||
"Métrica de calidad de un agrupamiento, en el rango −1 a 1: para cada "
|
||||
"fila compara cómo de cerca está de su propio segmento frente al segmento "
|
||||
"vecino más próximo. Cuanto más alto el promedio, más compactos y "
|
||||
"separados están los segmentos."),
|
||||
"isolation_forest": (
|
||||
"Isolation Forest (anomalías)",
|
||||
"Algoritmo de detección de anomalías multivariante: construye árboles que "
|
||||
"parten el espacio con cortes aleatorios y mide cuántos cortes hacen "
|
||||
"falta para aislar cada fila. Las filas raras se aíslan con muy pocos "
|
||||
"cortes y se marcan como outliers según un umbral de contaminación."),
|
||||
}
|
||||
|
||||
|
||||
def _term(mark: bool, key: str, text: str) -> str:
|
||||
"""Wrap ``text`` as a clickable glossary span when ``mark`` is True.
|
||||
|
||||
The visible text is identical with or without the marker (the renderers strip
|
||||
it), so wrapping never changes line layout — it only adds the link.
|
||||
"""
|
||||
return f"[[term:{key}]]{text}[[/term]]" if mark else text
|
||||
|
||||
|
||||
def _register(gloss, key: str) -> None:
|
||||
"""Register term ``key`` in the collector (idempotent); no-op if gloss None."""
|
||||
if gloss is not None:
|
||||
label, definition = _TERM_DEFS[key]
|
||||
gloss.add(key, label, definition)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Formatting helpers (mirror the overview chapter's defensive style).
|
||||
@@ -308,37 +252,34 @@ def _make_cluster_scatter(projection: dict):
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Section builders. Each returns a list of blocks (possibly empty).
|
||||
# --------------------------------------------------------------------------- #
|
||||
def _normalization_intro(gloss=None, mark_term: bool = False) -> list:
|
||||
_register(gloss, "zscore")
|
||||
zscore = _term(mark_term, "zscore", "**estandarizan con z-score**")
|
||||
def _normalization_intro() -> list:
|
||||
text = (
|
||||
"Estos modelos son **no supervisados**: buscan estructura latente sin "
|
||||
"una variable objetivo. Antes de aplicarlos, todas las columnas "
|
||||
f"numéricas se {zscore} (cada valor menos la media, dividido por la "
|
||||
"desviación típica). Sin esta normalización, una variable con escala "
|
||||
"grande (p.ej. ingresos en euros) dominaría las distancias y la varianza "
|
||||
"frente a otra de escala pequeña (p.ej. un ratio entre 0 y 1), sesgando "
|
||||
"tanto el PCA como el KMeans. Tras la estandarización todas las variables "
|
||||
"pesan por igual."
|
||||
"numéricas se **estandarizan con z-score** (cada valor menos la media, "
|
||||
"dividido por la desviación típica). Sin esta normalización, una "
|
||||
"variable con escala grande (p.ej. ingresos en euros) dominaría las "
|
||||
"distancias y la varianza frente a otra de escala pequeña (p.ej. un "
|
||||
"ratio entre 0 y 1), sesgando tanto el PCA como el KMeans. Tras la "
|
||||
"estandarización todas las variables pesan por igual."
|
||||
)
|
||||
return [model.Heading(text="Modelos no supervisados", level=1),
|
||||
model.Markdown(text=text)]
|
||||
|
||||
|
||||
def _pca_section(pca: dict, gloss=None, mark_term: bool = False) -> list:
|
||||
def _pca_section(pca: dict) -> list:
|
||||
if not _is_dict(pca) or not pca.get("explained_variance_ratio"):
|
||||
return []
|
||||
_register(gloss, "pca")
|
||||
blocks = [model.Heading(text="PCA — varianza explicada", level=2)]
|
||||
|
||||
n_used = pca.get("n_rows_used")
|
||||
n_feat = pca.get("n_features")
|
||||
intro = (
|
||||
f"El {_term(mark_term, 'pca', 'PCA')} resume {_fmt_num(n_feat)} variables "
|
||||
"numéricas en componentes ortogonales ordenados por la varianza que "
|
||||
f"capturan ({_fmt_num(n_used)} filas usadas tras eliminar nulos). El "
|
||||
"gráfico de sedimentación (scree) muestra cuánta varianza aporta cada "
|
||||
"componente y su acumulado: un codo marca cuántos componentes bastan."
|
||||
f"El PCA resume {_fmt_num(n_feat)} variables numéricas en componentes "
|
||||
f"ortogonales ordenados por la varianza que capturan "
|
||||
f"({_fmt_num(n_used)} filas usadas tras eliminar nulos). El gráfico de "
|
||||
"sedimentación (scree) muestra cuánta varianza aporta cada componente y "
|
||||
"su acumulado: un codo marca cuántos componentes bastan."
|
||||
)
|
||||
blocks.append(model.Markdown(text=intro))
|
||||
|
||||
@@ -384,14 +325,11 @@ def _pca_section(pca: dict, gloss=None, mark_term: bool = False) -> list:
|
||||
return blocks
|
||||
|
||||
|
||||
def _kmeans_section(kmeans: dict, projection: dict, titles,
|
||||
gloss=None, mark_term: bool = False) -> list:
|
||||
def _kmeans_section(kmeans: dict, projection: dict, titles) -> list:
|
||||
has_km = _is_dict(kmeans) and kmeans.get("best_k")
|
||||
has_proj = _is_dict(projection) and projection.get("points")
|
||||
if not has_km and not has_proj:
|
||||
return []
|
||||
_register(gloss, "kmeans")
|
||||
_register(gloss, "silhouette")
|
||||
|
||||
blocks = [model.Heading(text="Segmentación (KMeans)", level=2)]
|
||||
|
||||
@@ -399,11 +337,9 @@ def _kmeans_section(kmeans: dict, projection: dict, titles,
|
||||
sil = (projection or {}).get("silhouette")
|
||||
if sil is None:
|
||||
sil = (kmeans or {}).get("silhouette")
|
||||
t_kmeans = _term(mark_term, "kmeans", "KMeans")
|
||||
t_sil = _term(mark_term, "silhouette", "*silhouette*")
|
||||
intro = (
|
||||
f"{t_kmeans} agrupa las filas en **{_fmt_num(best_k)} segmentos** "
|
||||
f"elegidos automáticamente maximizando el coeficiente de {t_sil} "
|
||||
f"KMeans agrupa las filas en **{_fmt_num(best_k)} segmentos** elegidos "
|
||||
"automáticamente maximizando el coeficiente de *silhouette* "
|
||||
f"(**{_fmt_num(sil)}**, rango −1 a 1: cuanto más alto, segmentos más "
|
||||
"compactos y separados). Los segmentos se proyectan sobre el plano de "
|
||||
"los dos primeros componentes principales para visualizarlos."
|
||||
@@ -458,18 +394,16 @@ def _kmeans_section(kmeans: dict, projection: dict, titles,
|
||||
return blocks
|
||||
|
||||
|
||||
def _outliers_section(outliers: dict, gloss=None, mark_term: bool = False) -> list:
|
||||
def _outliers_section(outliers: dict) -> list:
|
||||
if not _is_dict(outliers) or outliers.get("n_outliers") is None:
|
||||
return []
|
||||
if outliers.get("note") and not outliers.get("n_rows_used"):
|
||||
# insufficient data — nothing meaningful to show.
|
||||
return []
|
||||
_register(gloss, "isolation_forest")
|
||||
blocks = [model.Heading(text="Detección de anomalías (Isolation Forest)",
|
||||
level=2)]
|
||||
isof = _term(mark_term, "isolation_forest", "**Isolation Forest**")
|
||||
explain = (
|
||||
f"{isof} detecta filas anómalas de forma *multivariante*: "
|
||||
"**Isolation Forest** detecta filas anómalas de forma *multivariante*: "
|
||||
"construye árboles que parten el espacio con cortes aleatorios y mide "
|
||||
"cuántos cortes hacen falta para aislar cada fila. Las filas raras "
|
||||
"(combinaciones de valores poco frecuentes considerando **todas las "
|
||||
@@ -550,21 +484,15 @@ def build_modelos(profile: dict, ctx: dict):
|
||||
(kmeans and kmeans.get("best_k")) or (projection and projection.get("points"))
|
||||
) else None
|
||||
|
||||
# Shared glossary collector: terms are registered + marked clickable inside
|
||||
# each section, only when that section actually renders (no orphan entries).
|
||||
glossary = ctx.get("glossary")
|
||||
gloss = glossary if isinstance(glossary, model.GlossaryCollector) else None
|
||||
mark_term = gloss is not None
|
||||
|
||||
sections = []
|
||||
sections += _pca_section(pca, gloss, mark_term) if pca else []
|
||||
sections += _kmeans_section(kmeans, projection, titles, gloss, mark_term)
|
||||
sections += _outliers_section(outliers, gloss, mark_term) if outliers else []
|
||||
sections += _pca_section(pca) if pca else []
|
||||
sections += _kmeans_section(kmeans, projection, titles)
|
||||
sections += _outliers_section(outliers) if outliers else []
|
||||
sections += _normality_section(normality) if normality else []
|
||||
|
||||
if not sections:
|
||||
return None # models block present but nothing renderable.
|
||||
|
||||
blocks = _normalization_intro(gloss, mark_term) + sections
|
||||
blocks = _normalization_intro() + sections
|
||||
return model.Chapter(id=CHAPTER_ID, title=CHAPTER_TITLE,
|
||||
version=CHAPTER_VERSION, blocks=blocks)
|
||||
|
||||
@@ -257,26 +257,3 @@ def test_anticortes_tabla_normalidad_larga_no_corta():
|
||||
# Every column name survives (wrapped/split, never truncated).
|
||||
for i in (0, 19, 39):
|
||||
assert f"col_{i}" in txt
|
||||
|
||||
|
||||
def test_glosario_engancha_terminos_modelos():
|
||||
"""Mejora 4b: PCA, KMeans, silhouette, Isolation Forest y la estandarización
|
||||
z-score se registran en el colector compartido y se marcan clicables en el
|
||||
cuerpo. Sin colector en ctx, el capítulo degrada y no marca nada."""
|
||||
from datascience.automatic_eda.model import GlossaryCollector
|
||||
|
||||
g = GlossaryCollector()
|
||||
ctx = dict(_ctx_full())
|
||||
ctx["glossary"] = g
|
||||
ch = build_modelos(_profile(), ctx)
|
||||
assert ch is not None
|
||||
keys = {t["key"] for t in g.terms()}
|
||||
assert {"zscore", "pca", "kmeans", "silhouette", "isolation_forest"} <= keys
|
||||
body = " ".join(b.text for b in ch.blocks if b.kind == "markdown")
|
||||
for k in ("zscore", "pca", "kmeans", "silhouette", "isolation_forest"):
|
||||
assert f"[[term:{k}]]" in body, k
|
||||
|
||||
# Sin colector: degrada limpio (ningún marcador en el cuerpo).
|
||||
ch2 = build_modelos(_profile(), _ctx_full())
|
||||
body2 = " ".join(b.text for b in ch2.blocks if b.kind == "markdown")
|
||||
assert "[[term:" not in body2
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
"""Numeric distributions chapter (NUM DISTR) for AutomaticEDA.
|
||||
|
||||
For every numeric column the chapter draws, as a single indivisible figure, a
|
||||
histogram with the **mean, median and ±1σ band drawn as reference lines** and a
|
||||
**Tukey boxplot right below it** sharing the same X axis — exactly the user
|
||||
requirement for this chapter. Each figure is emitted as a lazy ``Figure`` block
|
||||
histogram with the **mean, median and ±1σ band drawn as reference lines** (the
|
||||
legend reports the numeric value of the mean, the median **and the standard
|
||||
deviation σ**) and a **Tukey boxplot right below it** sharing the same X axis —
|
||||
exactly the user requirement for this chapter. Each figure is emitted as a lazy ``Figure`` block
|
||||
so the renderers rasterize and scale it to fit a whole page/slide and nothing is
|
||||
ever cut; columns with many numerics simply flow across pages as small
|
||||
multiples.
|
||||
@@ -34,7 +35,7 @@ try:
|
||||
except Exception: # noqa: BLE001 — keep the chapter importable no matter what.
|
||||
build_boxplot_stats = None # type: ignore[assignment]
|
||||
|
||||
CHAPTER_VERSION = "1.1.0"
|
||||
CHAPTER_VERSION = "1.2.0"
|
||||
CHAPTER_ID = "num_distr"
|
||||
CHAPTER_TITLE = "Distribuciones numéricas"
|
||||
|
||||
@@ -140,9 +141,11 @@ def _make_hist_box(name: str, numeric: dict, box: dict):
|
||||
std = numeric.get("std")
|
||||
|
||||
# ±1σ band first (behind the lines), then median (solid) and mean (dashed).
|
||||
# The band's legend entry also reports the numeric value of the standard
|
||||
# deviation, so the reader sees mean, median AND σ at a glance.
|
||||
if mean is not None and std is not None and std > 0:
|
||||
ax_h.axvspan(mean - std, mean + std, color="#f0c27b", alpha=0.22,
|
||||
zorder=1, label="±1σ")
|
||||
zorder=1, label=f"±1σ (σ = {_fmt_num(std)})")
|
||||
if median is not None:
|
||||
ax_h.axvline(median, color="#2e8b57", linestyle="-", linewidth=1.6,
|
||||
zorder=4, label=f"mediana = {_fmt_num(median)}")
|
||||
@@ -152,7 +155,19 @@ def _make_hist_box(name: str, numeric: dict, box: dict):
|
||||
|
||||
ax_h.set_ylabel("frecuencia", fontsize=8)
|
||||
ax_h.tick_params(labelsize=7)
|
||||
ax_h.legend(fontsize=6.5, loc="upper right", framealpha=0.85)
|
||||
# Always surface σ in the legend: if the ±1σ band could not be drawn (no mean
|
||||
# or std<=0) but σ is still known, add a label-only proxy handle so the value
|
||||
# of the standard deviation is reported regardless of the band.
|
||||
handles, labels = ax_h.get_legend_handles_labels()
|
||||
if std is not None and not any("σ =" in lbl for lbl in labels):
|
||||
from matplotlib.lines import Line2D
|
||||
proxy = Line2D([], [], linestyle="none", marker="",
|
||||
label=f"σ = {_fmt_num(std)}")
|
||||
handles.append(proxy)
|
||||
labels.append(f"σ = {_fmt_num(std)}")
|
||||
if handles:
|
||||
ax_h.legend(handles, labels, fontsize=6.5, loc="upper right",
|
||||
framealpha=0.85)
|
||||
for spine in ("top", "right"):
|
||||
ax_h.spines[spine].set_visible(False)
|
||||
|
||||
|
||||
@@ -159,6 +159,50 @@ def test_anti_corte_muchas_columnas_pdf_y_pptx():
|
||||
assert res_pptx["n_slides"] >= 8 # at least one slide per column figure.
|
||||
|
||||
|
||||
def _hist_legend_texts(numeric, box=None):
|
||||
"""Build the per-column figure and return its histogram-legend label texts."""
|
||||
from datascience.automatic_eda.chapters.num_distr import _make_hist_box
|
||||
import matplotlib.pyplot as plt
|
||||
fig = _make_hist_box("col", numeric, box or {})
|
||||
ax_h = fig.axes[0] # the histogram is the top axis.
|
||||
leg = ax_h.get_legend()
|
||||
texts = [t.get_text() for t in leg.get_texts()] if leg else []
|
||||
plt.close(fig)
|
||||
return texts
|
||||
|
||||
|
||||
def test_golden_leyenda_histograma_reporta_valor_std():
|
||||
# The histogram legend must report the numeric value of the standard
|
||||
# deviation σ next to mean and median.
|
||||
numeric = _numeric_block(42.5, 40.0, 12.3, 1.0, 100.0, "right-skewed", 5)
|
||||
texts = _hist_legend_texts(numeric)
|
||||
joined = " ".join(texts)
|
||||
assert any("σ =" in t for t in texts), f"σ value missing in legend: {texts}"
|
||||
assert "12.3" in joined, f"std value 12.3 not in legend: {texts}"
|
||||
assert any("media =" in t for t in texts)
|
||||
assert any("mediana =" in t for t in texts)
|
||||
|
||||
|
||||
def test_edge_std_en_leyenda_aunque_no_haya_banda():
|
||||
# When the ±1σ band cannot be drawn (no mean) but σ is known, the legend
|
||||
# still surfaces the σ value via a label-only proxy handle.
|
||||
numeric = _numeric_block(42.5, 40.0, 7.5, 1.0, 100.0, "right-skewed", 0)
|
||||
numeric["mean"] = None # forces the band off; σ must still appear.
|
||||
texts = _hist_legend_texts(numeric)
|
||||
assert any("σ = 7.5" in t for t in texts), f"σ proxy missing: {texts}"
|
||||
|
||||
|
||||
def test_edge_sin_std_no_revienta_la_figura():
|
||||
# A numeric block without σ must not raise and simply omits the σ entry.
|
||||
import matplotlib.pyplot as plt
|
||||
numeric = _numeric_block(42.5, 40.0, 0.0, 1.0, 100.0, "discrete", 0)
|
||||
numeric["std"] = None
|
||||
texts = _hist_legend_texts(numeric)
|
||||
assert not any("σ =" in t for t in texts)
|
||||
# mean/median lines still produce their own legend entries.
|
||||
assert any("media =" in t for t in texts)
|
||||
|
||||
|
||||
def test_distribution_gloss_cubre_todas_las_etiquetas():
|
||||
# Every label detect_distribution_type can emit has a Spanish gloss.
|
||||
for label in ("normal-ish", "right-skewed", "left-skewed", "heavy-tail",
|
||||
|
||||
Reference in New Issue
Block a user