merge: capitulo AutomaticEDA num_distr (verificado met)
This commit is contained in:
@@ -0,0 +1,289 @@
|
|||||||
|
"""Numeric distributions chapter (NUM DISTR) for AutomaticEDA.
|
||||||
|
|
||||||
|
For every numeric column the chapter draws, as a single indivisible figure, a
|
||||||
|
histogram with the **mean, median and ±1σ band drawn as reference lines** and a
|
||||||
|
**Tukey boxplot right below it** sharing the same X axis — exactly the user
|
||||||
|
requirement for this chapter. Each figure is emitted as a lazy ``Figure`` block
|
||||||
|
so the renderers rasterize and scale it to fit a whole page/slide and nothing is
|
||||||
|
ever cut; columns with many numerics simply flow across pages as small
|
||||||
|
multiples.
|
||||||
|
|
||||||
|
Data comes from the ``eda`` group profile and is never recomputed here:
|
||||||
|
|
||||||
|
- ``columns[i]['numeric']`` (the output of ``describe_numeric``) gives
|
||||||
|
``mean, median, std, min, max, p25, p75, iqr, n_outliers, outlier_pct,
|
||||||
|
distribution_type`` and the ``histogram`` bins ``[{lo, hi, count}]``.
|
||||||
|
- The boxplot five-number summary + Tukey 1.5·IQR fences are derived by the
|
||||||
|
pure registry function ``build_boxplot_stats`` (group ``eda``); this chapter
|
||||||
|
only consumes its output, it does not reimplement the statistics.
|
||||||
|
|
||||||
|
Contract: build_<id>(profile, ctx) -> Chapter | None ; CHAPTER_VERSION = "x.y.z".
|
||||||
|
Reads everything defensively (``.get``) and never raises: a column whose figure
|
||||||
|
cannot be built is degraded to a short note instead of aborting the chapter.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from .. import model
|
||||||
|
|
||||||
|
# Pure registry function (group ``eda``) that derives the Tukey boxplot stats
|
||||||
|
# from a ``numeric`` sub-block. Imported defensively so the chapter still builds
|
||||||
|
# (degrading the boxplot to a note) if the function is somehow unavailable.
|
||||||
|
try:
|
||||||
|
from datascience.build_boxplot_stats import build_boxplot_stats
|
||||||
|
except Exception: # noqa: BLE001 — keep the chapter importable no matter what.
|
||||||
|
build_boxplot_stats = None # type: ignore[assignment]
|
||||||
|
|
||||||
|
CHAPTER_VERSION = "1.0.0"
|
||||||
|
CHAPTER_ID = "num_distr"
|
||||||
|
CHAPTER_TITLE = "Distribuciones numéricas"
|
||||||
|
|
||||||
|
# Plain-Spanish gloss for every label ``detect_distribution_type`` can emit, so a
|
||||||
|
# non-expert reader understands the shape and the suggested next step (MUST-4.3).
|
||||||
|
_DIST_GLOSS = {
|
||||||
|
"normal-ish": "aproximadamente simétrica (campana); media y mediana casi "
|
||||||
|
"coinciden.",
|
||||||
|
"right-skewed": "asimétrica a la derecha (cola larga hacia valores altos); "
|
||||||
|
"la media supera a la mediana — considera una transformación "
|
||||||
|
"logarítmica.",
|
||||||
|
"left-skewed": "asimétrica a la izquierda (cola larga hacia valores bajos); "
|
||||||
|
"la media queda por debajo de la mediana.",
|
||||||
|
"heavy-tail": "colas pesadas (curtosis alta): más valores extremos de lo "
|
||||||
|
"que esperaría una normal — vigila los outliers.",
|
||||||
|
"lognormal-ish": "compatible con lognormal (simétrica al tomar logaritmos); "
|
||||||
|
"la re-expresión log suele normalizarla.",
|
||||||
|
"multimodal": "varios picos: probablemente mezcla de subgrupos — conviene "
|
||||||
|
"segmentar antes de resumir con una sola media.",
|
||||||
|
"discrete": "pocos valores distintos (discreta/ordinal); el histograma "
|
||||||
|
"cuenta niveles, no un continuo.",
|
||||||
|
"too_few_samples": "muestra demasiado pequeña para clasificar la forma con "
|
||||||
|
"fiabilidad.",
|
||||||
|
"other": "forma no encuadrada en las categorías estándar.",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _fmt_num(value, decimals: int = 3) -> str:
|
||||||
|
"""Compact, defensive number formatting shared with the other chapters."""
|
||||||
|
if value is None:
|
||||||
|
return "—"
|
||||||
|
if isinstance(value, bool):
|
||||||
|
return str(value)
|
||||||
|
if isinstance(value, int):
|
||||||
|
return f"{value:,}".replace(",", ".")
|
||||||
|
if isinstance(value, float):
|
||||||
|
if value != value: # NaN
|
||||||
|
return "NaN"
|
||||||
|
if value in (float("inf"), float("-inf")):
|
||||||
|
return str(value)
|
||||||
|
text = f"{value:.{decimals}f}".rstrip("0").rstrip(".")
|
||||||
|
return text if text else "0"
|
||||||
|
return str(value)
|
||||||
|
|
||||||
|
|
||||||
|
def _numeric_columns(profile: dict) -> list:
|
||||||
|
"""Return the list of (name, numeric_dict) for columns with usable stats."""
|
||||||
|
out = []
|
||||||
|
for col in profile.get("columns") or []:
|
||||||
|
if not isinstance(col, dict):
|
||||||
|
continue
|
||||||
|
if col.get("inferred_type") != "numeric":
|
||||||
|
continue
|
||||||
|
num = col.get("numeric")
|
||||||
|
if not isinstance(num, dict) or not num:
|
||||||
|
continue
|
||||||
|
# A numeric block is renderable when it carries at least a center.
|
||||||
|
if num.get("mean") is None and num.get("median") is None:
|
||||||
|
continue
|
||||||
|
out.append((col.get("name") or "(columna)", num))
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def _make_hist_box(name: str, numeric: dict, box: dict):
|
||||||
|
"""Build the histogram (with mean/median/±σ lines) + boxplot figure.
|
||||||
|
|
||||||
|
Returned lazily to the renderer (a zero-arg callable via ``Figure.make``) so
|
||||||
|
matplotlib is only imported and the figure only drawn when a renderer needs
|
||||||
|
it. The two stacked axes share the X axis and are produced as a single
|
||||||
|
figure, which both renderers treat as one indivisible unit (scaled whole,
|
||||||
|
never cut).
|
||||||
|
"""
|
||||||
|
import matplotlib
|
||||||
|
|
||||||
|
matplotlib.use("Agg")
|
||||||
|
import matplotlib.pyplot as plt
|
||||||
|
|
||||||
|
fig, (ax_h, ax_b) = plt.subplots(
|
||||||
|
2, 1, figsize=(6.4, 3.4), sharex=True,
|
||||||
|
gridspec_kw={"height_ratios": [3.2, 1.0], "hspace": 0.08})
|
||||||
|
|
||||||
|
# ---- Histogram from the precomputed equal-width bins {lo, hi, count}. ----
|
||||||
|
hist = numeric.get("histogram") or []
|
||||||
|
drew_bars = False
|
||||||
|
for b in hist:
|
||||||
|
if not isinstance(b, dict):
|
||||||
|
continue
|
||||||
|
lo = b.get("lo")
|
||||||
|
hi = b.get("hi")
|
||||||
|
count = b.get("count") or 0
|
||||||
|
if lo is None or hi is None:
|
||||||
|
continue
|
||||||
|
width = (hi - lo) if hi > lo else max(abs(lo) * 1e-3, 1e-6)
|
||||||
|
ax_h.bar(lo, count, width=width, align="edge", color="#9ec6df",
|
||||||
|
edgecolor="#5b8aa6", linewidth=0.4, zorder=2)
|
||||||
|
drew_bars = True
|
||||||
|
if not drew_bars:
|
||||||
|
ax_h.text(0.5, 0.5, "(sin histograma)", ha="center", va="center",
|
||||||
|
fontsize=9, color="#8a8a8a", transform=ax_h.transAxes)
|
||||||
|
|
||||||
|
mean = numeric.get("mean")
|
||||||
|
median = numeric.get("median")
|
||||||
|
std = numeric.get("std")
|
||||||
|
|
||||||
|
# ±1σ band first (behind the lines), then median (solid) and mean (dashed).
|
||||||
|
if mean is not None and std is not None and std > 0:
|
||||||
|
ax_h.axvspan(mean - std, mean + std, color="#f0c27b", alpha=0.22,
|
||||||
|
zorder=1, label="±1σ")
|
||||||
|
if median is not None:
|
||||||
|
ax_h.axvline(median, color="#2e8b57", linestyle="-", linewidth=1.6,
|
||||||
|
zorder=4, label=f"mediana = {_fmt_num(median)}")
|
||||||
|
if mean is not None:
|
||||||
|
ax_h.axvline(mean, color="#c0392b", linestyle="--", linewidth=1.6,
|
||||||
|
zorder=4, label=f"media = {_fmt_num(mean)}")
|
||||||
|
|
||||||
|
ax_h.set_ylabel("frecuencia", fontsize=8)
|
||||||
|
ax_h.tick_params(labelsize=7)
|
||||||
|
ax_h.legend(fontsize=6.5, loc="upper right", framealpha=0.85)
|
||||||
|
for spine in ("top", "right"):
|
||||||
|
ax_h.spines[spine].set_visible(False)
|
||||||
|
|
||||||
|
# ---- Tukey boxplot below, sharing the X axis (MUST-4.2). ----
|
||||||
|
if box:
|
||||||
|
stats = [{
|
||||||
|
"med": box.get("median"),
|
||||||
|
"q1": box.get("q1"),
|
||||||
|
"q3": box.get("q3"),
|
||||||
|
"whislo": box.get("whisker_lo"),
|
||||||
|
"whishi": box.get("whisker_hi"),
|
||||||
|
"fliers": [], # raw outlier values are not in the profile.
|
||||||
|
"label": "",
|
||||||
|
}]
|
||||||
|
bxp_kw = dict(
|
||||||
|
showfliers=False, widths=0.5, patch_artist=True,
|
||||||
|
boxprops={"facecolor": "#9ec6df", "edgecolor": "#5b8aa6"},
|
||||||
|
medianprops={"color": "#2e8b57", "linewidth": 1.6},
|
||||||
|
whiskerprops={"color": "#5b8aa6"},
|
||||||
|
capprops={"color": "#5b8aa6"})
|
||||||
|
try:
|
||||||
|
# ``orientation`` is the current API; older matplotlib uses ``vert``.
|
||||||
|
try:
|
||||||
|
ax_b.bxp(stats, orientation="horizontal", **bxp_kw)
|
||||||
|
except TypeError:
|
||||||
|
ax_b.bxp(stats, vert=False, **bxp_kw)
|
||||||
|
except Exception: # noqa: BLE001 — never let one axis kill the figure.
|
||||||
|
pass
|
||||||
|
# Mark the presence of out-of-fence points (the raw values are unknown).
|
||||||
|
if box.get("has_low_outliers") and box.get("min") is not None:
|
||||||
|
ax_b.plot([box["min"]], [1], marker="o", markersize=3.5,
|
||||||
|
color="#c0392b", zorder=5)
|
||||||
|
if box.get("has_high_outliers") and box.get("max") is not None:
|
||||||
|
ax_b.plot([box["max"]], [1], marker="o", markersize=3.5,
|
||||||
|
color="#c0392b", zorder=5)
|
||||||
|
else:
|
||||||
|
ax_b.text(0.5, 0.5, "(boxplot no disponible)", ha="center", va="center",
|
||||||
|
fontsize=8, color="#8a8a8a", transform=ax_b.transAxes)
|
||||||
|
|
||||||
|
ax_b.set_yticks([])
|
||||||
|
ax_b.set_xlabel(name, fontsize=8)
|
||||||
|
ax_b.tick_params(labelsize=7)
|
||||||
|
for spine in ("top", "right", "left"):
|
||||||
|
ax_b.spines[spine].set_visible(False)
|
||||||
|
|
||||||
|
fig.suptitle(name, fontsize=10, fontweight="bold", x=0.02, ha="left")
|
||||||
|
return fig
|
||||||
|
|
||||||
|
|
||||||
|
def _stats_note(name: str, numeric: dict, box: dict) -> str:
|
||||||
|
"""One compact line of the key numbers + a plain-Spanish shape gloss."""
|
||||||
|
bits = [
|
||||||
|
f"media {_fmt_num(numeric.get('mean'))}",
|
||||||
|
f"mediana {_fmt_num(numeric.get('median'))}",
|
||||||
|
f"σ {_fmt_num(numeric.get('std'))}",
|
||||||
|
f"min {_fmt_num(numeric.get('min'))}",
|
||||||
|
f"max {_fmt_num(numeric.get('max'))}",
|
||||||
|
f"IQR {_fmt_num(numeric.get('iqr'))}",
|
||||||
|
]
|
||||||
|
n_out = numeric.get("n_outliers")
|
||||||
|
out_pct = numeric.get("outlier_pct")
|
||||||
|
if n_out is not None:
|
||||||
|
pct = f" ({_fmt_num(out_pct, 2)}%)" if out_pct is not None else ""
|
||||||
|
bits.append(f"outliers {n_out}{pct}")
|
||||||
|
if box and (box.get("lower_fence") is not None):
|
||||||
|
bits.append(
|
||||||
|
f"vallas Tukey [{_fmt_num(box.get('lower_fence'))}, "
|
||||||
|
f"{_fmt_num(box.get('upper_fence'))}]")
|
||||||
|
line = " · ".join(bits)
|
||||||
|
|
||||||
|
dist = numeric.get("distribution_type")
|
||||||
|
gloss = _DIST_GLOSS.get(dist)
|
||||||
|
if dist and gloss:
|
||||||
|
line += f"\n\n**Forma ({dist}):** {gloss}"
|
||||||
|
return line
|
||||||
|
|
||||||
|
|
||||||
|
def _figure_maker(name: str, numeric: dict, box: dict):
|
||||||
|
"""Bind the per-column arguments so the lazy closure is loop-safe."""
|
||||||
|
def _make():
|
||||||
|
return _make_hist_box(name, numeric, box)
|
||||||
|
|
||||||
|
return _make
|
||||||
|
|
||||||
|
|
||||||
|
def build_num_distr(profile: dict, ctx: dict):
|
||||||
|
"""Build the numeric-distributions Chapter, or None if no numeric column.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
profile: the ``eda`` group TableProfile dict.
|
||||||
|
ctx: presentation context (unused here beyond defensive handling).
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
A ``model.Chapter`` with, per numeric column, a histogram+boxplot figure
|
||||||
|
and a stats note; or ``None`` when the dataset has no numeric column.
|
||||||
|
"""
|
||||||
|
profile = profile or {}
|
||||||
|
ctx = ctx or {}
|
||||||
|
|
||||||
|
numerics = _numeric_columns(profile)
|
||||||
|
if not numerics:
|
||||||
|
return None # chapter does not apply to a dataset with no numerics.
|
||||||
|
|
||||||
|
intro = (
|
||||||
|
"Para cada columna numérica se muestra su **histograma** con tres líneas "
|
||||||
|
"de referencia: la **media** (línea roja discontinua), la **mediana** "
|
||||||
|
"(línea verde continua) y la banda **±1σ** (zona sombreada). Debajo, "
|
||||||
|
"alineado al mismo eje, un **boxplot de Tukey**: la caja abarca del "
|
||||||
|
"primer al tercer cuartil (P25–P75), la línea interior es la mediana y "
|
||||||
|
"los bigotes llegan hasta 1,5·IQR; los puntos rojos señalan que hay "
|
||||||
|
"valores más allá de las vallas. Comparar media y mediana revela la "
|
||||||
|
"asimetría de la distribución.")
|
||||||
|
|
||||||
|
blocks = [
|
||||||
|
model.Heading(text=CHAPTER_TITLE, level=1),
|
||||||
|
model.Markdown(text=intro),
|
||||||
|
]
|
||||||
|
|
||||||
|
for name, numeric in numerics:
|
||||||
|
box = {}
|
||||||
|
if build_boxplot_stats is not None:
|
||||||
|
try:
|
||||||
|
box = build_boxplot_stats(numeric) or {}
|
||||||
|
except Exception: # noqa: BLE001 — degrade, never raise.
|
||||||
|
box = {}
|
||||||
|
blocks.append(model.Heading(text=str(name), level=2))
|
||||||
|
blocks.append(model.Figure(
|
||||||
|
make=_figure_maker(name, numeric, box),
|
||||||
|
caption=f"Distribución de «{name}» — histograma (media/mediana/±σ) "
|
||||||
|
f"y boxplot."))
|
||||||
|
blocks.append(model.Markdown(text=_stats_note(name, numeric, box)))
|
||||||
|
|
||||||
|
return model.Chapter(id=CHAPTER_ID, title=CHAPTER_TITLE,
|
||||||
|
version=CHAPTER_VERSION, blocks=blocks)
|
||||||
@@ -0,0 +1,151 @@
|
|||||||
|
"""Tests for the NUM DISTR chapter — DoD: golden + edges + anti-cut.
|
||||||
|
|
||||||
|
Self-contained: builds synthetic ``numeric`` blocks (no DuckDB) so the suite is
|
||||||
|
fast and deterministic. Verifies that the chapter emits, per numeric column, a
|
||||||
|
histogram+boxplot figure plus a stats note; that the mean/median/±σ requirement
|
||||||
|
and the boxplot are present; that a profile with no numeric column yields None;
|
||||||
|
that None/empty never raises; and that with many numeric columns and long text
|
||||||
|
both the PDF and the PPTX render without cutting anything (every column heading
|
||||||
|
survives in the rendered output).
|
||||||
|
"""
|
||||||
|
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import tempfile
|
||||||
|
|
||||||
|
from pypdf import PdfReader
|
||||||
|
|
||||||
|
from datascience.automatic_eda.chapters.num_distr import (
|
||||||
|
build_num_distr, CHAPTER_VERSION, _DIST_GLOSS,
|
||||||
|
)
|
||||||
|
from datascience.automatic_eda import model
|
||||||
|
from datascience.render_automatic_eda_pdf import render_automatic_eda_pdf
|
||||||
|
from datascience.render_automatic_eda_pptx import render_automatic_eda_pptx
|
||||||
|
|
||||||
|
|
||||||
|
def _numeric_block(mean, median, std, mn, mx, dist="normal-ish",
|
||||||
|
n_outliers=0, nbins=10):
|
||||||
|
"""A synthetic ``numeric`` sub-block shaped like describe_numeric's output."""
|
||||||
|
width = (mx - mn) / nbins if mx > mn else 1.0
|
||||||
|
hist = [{"lo": mn + i * width, "hi": mn + (i + 1) * width,
|
||||||
|
"count": (i + 1) * 3} for i in range(nbins)]
|
||||||
|
p25 = mn + (mx - mn) * 0.25
|
||||||
|
p75 = mn + (mx - mn) * 0.75
|
||||||
|
return {
|
||||||
|
"min": mn, "max": mx, "mean": mean, "median": median, "std": std,
|
||||||
|
"p25": p25, "p50": median, "p75": p75, "iqr": p75 - p25,
|
||||||
|
"n_outliers": n_outliers, "outlier_pct": 100.0 * n_outliers / 300.0,
|
||||||
|
"distribution_type": dist, "histogram": hist,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _profile(n_numeric=2, extra_categorical=True):
|
||||||
|
cols = []
|
||||||
|
presets = [
|
||||||
|
("precio", 42.5, 40.0, 12.3, 1.0, 100.0, "right-skewed", 5),
|
||||||
|
("alcohol", 10.4, 10.3, 1.1, 8.0, 14.9, "normal-ish", 0),
|
||||||
|
("sulfatos", 0.66, 0.62, 0.17, 0.33, 2.0, "heavy-tail", 9),
|
||||||
|
("calidad", 5.6, 6.0, 0.8, 3.0, 8.0, "discrete", 0),
|
||||||
|
]
|
||||||
|
for i in range(n_numeric):
|
||||||
|
name, mean, med, std, mn, mx, dist, no = presets[i % len(presets)]
|
||||||
|
if i >= len(presets):
|
||||||
|
name = f"{name}_{i}"
|
||||||
|
cols.append({"name": name, "inferred_type": "numeric",
|
||||||
|
"numeric": _numeric_block(mean, med, std, mn, mx, dist, no)})
|
||||||
|
if extra_categorical:
|
||||||
|
cols.append({"name": "categoria", "inferred_type": "categorical",
|
||||||
|
"categorical": {"top": [{"value": "tinto", "count": 200}]}})
|
||||||
|
return {"table": "vinos", "n_rows": 300, "n_cols": len(cols),
|
||||||
|
"columns": cols}
|
||||||
|
|
||||||
|
|
||||||
|
def _pdf_text(path: str) -> str:
|
||||||
|
txt = "".join((pg.extract_text() or "") for pg in PdfReader(path).pages)
|
||||||
|
return re.sub(r"\s+", " ", txt)
|
||||||
|
|
||||||
|
|
||||||
|
def test_golden_chapter_estructura_y_bloques():
|
||||||
|
ch = build_num_distr(_profile(n_numeric=2), {})
|
||||||
|
assert ch is not None
|
||||||
|
assert ch.id == "num_distr"
|
||||||
|
assert ch.version == CHAPTER_VERSION
|
||||||
|
kinds = [b.kind for b in ch.blocks]
|
||||||
|
# Heading + intro Markdown, then per column: Heading + Figure + Markdown.
|
||||||
|
assert kinds[0] == "heading"
|
||||||
|
assert kinds[1] == "markdown"
|
||||||
|
assert kinds.count("figure") == 2 # one figure per numeric column.
|
||||||
|
assert kinds.count("heading") == 1 + 2 # chapter title + one per column.
|
||||||
|
# Each figure has a lazy maker that produces a real matplotlib figure.
|
||||||
|
figs = [b for b in ch.blocks if b.kind == "figure"]
|
||||||
|
fig = figs[0].make()
|
||||||
|
assert fig is not None
|
||||||
|
# Two stacked axes: histogram + boxplot share the figure.
|
||||||
|
assert len(fig.axes) == 2
|
||||||
|
import matplotlib.pyplot as plt
|
||||||
|
plt.close(fig)
|
||||||
|
|
||||||
|
|
||||||
|
def test_golden_media_mediana_sigma_y_boxplot_presentes():
|
||||||
|
# The intro documents the three reference lines and the Tukey boxplot; the
|
||||||
|
# per-column note carries the actual mean/median/σ numbers and the shape.
|
||||||
|
ch = build_num_distr(_profile(n_numeric=1, extra_categorical=False), {})
|
||||||
|
md_texts = " ".join(b.text for b in ch.blocks if b.kind == "markdown")
|
||||||
|
assert "media" in md_texts and "mediana" in md_texts
|
||||||
|
assert "±1σ" in md_texts or "σ" in md_texts
|
||||||
|
assert "boxplot" in md_texts.lower()
|
||||||
|
assert "Tukey" in md_texts
|
||||||
|
# distribution_type gloss surfaced for the column (right-skewed preset).
|
||||||
|
assert _DIST_GLOSS["right-skewed"].split(";")[0][:20] in md_texts
|
||||||
|
|
||||||
|
|
||||||
|
def test_boxplot_stats_se_consumen_del_registry():
|
||||||
|
# The chapter must feed build_boxplot_stats (group eda) and the resulting
|
||||||
|
# box must carry the Tukey fences for the figure.
|
||||||
|
from datascience.build_boxplot_stats import build_boxplot_stats
|
||||||
|
box = build_boxplot_stats(
|
||||||
|
_numeric_block(42.5, 40.0, 12.3, 1.0, 100.0, "right-skewed", 5))
|
||||||
|
assert box
|
||||||
|
assert "lower_fence" in box and "upper_fence" in box
|
||||||
|
assert box["q1"] is not None and box["q3"] is not None
|
||||||
|
|
||||||
|
|
||||||
|
def test_edge_sin_columnas_numericas_devuelve_none():
|
||||||
|
prof = {"columns": [{"name": "c", "inferred_type": "categorical",
|
||||||
|
"categorical": {"top": []}}]}
|
||||||
|
assert build_num_distr(prof, {}) is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_edge_profile_none_y_vacio_no_revienta():
|
||||||
|
assert build_num_distr(None, None) is None
|
||||||
|
assert build_num_distr({}, {}) is None
|
||||||
|
assert build_num_distr({"columns": []}, {}) is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_anti_corte_muchas_columnas_pdf_y_pptx():
|
||||||
|
# 8 numeric columns + long note text: nothing may be cut. Every column
|
||||||
|
# heading must survive in both the PDF text and the PPTX deck.
|
||||||
|
ch = build_num_distr(_profile(n_numeric=8), {})
|
||||||
|
names = [b.text for b in ch.blocks if b.kind == "heading" and b.level == 2]
|
||||||
|
assert len(names) == 8
|
||||||
|
with tempfile.TemporaryDirectory() as d:
|
||||||
|
pdf = os.path.join(d, "num.pdf")
|
||||||
|
res_pdf = render_automatic_eda_pdf(_profile(n_numeric=8), pdf,
|
||||||
|
{"write_manifest": False})
|
||||||
|
assert res_pdf["path"] == pdf
|
||||||
|
txt = _pdf_text(pdf)
|
||||||
|
for name in names:
|
||||||
|
assert name in txt, f"columna '{name}' cortada/ausente en el PDF"
|
||||||
|
pptx = os.path.join(d, "num.pptx")
|
||||||
|
res_pptx = render_automatic_eda_pptx(_profile(n_numeric=8), pptx,
|
||||||
|
{"write_manifest": False})
|
||||||
|
assert res_pptx["path"] == pptx
|
||||||
|
assert res_pptx["n_slides"] >= 8 # at least one slide per column figure.
|
||||||
|
|
||||||
|
|
||||||
|
def test_distribution_gloss_cubre_todas_las_etiquetas():
|
||||||
|
# Every label detect_distribution_type can emit has a Spanish gloss.
|
||||||
|
for label in ("normal-ish", "right-skewed", "left-skewed", "heavy-tail",
|
||||||
|
"lognormal-ish", "multimodal", "discrete", "too_few_samples",
|
||||||
|
"other"):
|
||||||
|
assert label in _DIST_GLOSS and _DIST_GLOSS[label]
|
||||||
@@ -0,0 +1,58 @@
|
|||||||
|
---
|
||||||
|
name: build_boxplot_stats
|
||||||
|
kind: function
|
||||||
|
lang: py
|
||||||
|
domain: datascience
|
||||||
|
version: "1.0.0"
|
||||||
|
purity: pure
|
||||||
|
signature: "def build_boxplot_stats(numeric: dict) -> dict"
|
||||||
|
description: "Deriva las estadisticas de un boxplot de Tukey desde el sub-bloque numeric de un ColumnProfile del grupo eda (salida de describe_numeric). Aplica la regla del 1.5*IQR a los percentiles p25/p50/p75 para obtener cuartiles, fences, bigotes reales y flags de outliers. Lectura defensiva con .get; NUNCA lanza. Si faltan los percentiles clave devuelve {} para que el caller omita el grafico."
|
||||||
|
tags: [eda, statistics, profiling, boxplot, tukey, iqr, datascience]
|
||||||
|
params:
|
||||||
|
- name: numeric
|
||||||
|
desc: "Sub-bloque numeric de un ColumnProfile del grupo eda (la salida de describe_numeric). Claves esperadas (todas pueden ser None): min, max, mean, median, mode, std, variance, cv, p1, p5, p25, p50, p75, p95, p99, iqr, skew, kurtosis, n_outliers, outlier_pct, zero_pct, negative_pct, distribution_type, histogram. Solo se usan p25, median/p50, p75, min, max y n_outliers."
|
||||||
|
output: "Dict con las cifras de un boxplot horizontal de Tukey: {q1=p25, median=median(o p50), q3=p75, iqr=q3-q1, lower_fence=q1-1.5*iqr, upper_fence=q3+1.5*iqr, whisker_lo=max(min,lower_fence), whisker_hi=min(max,upper_fence), min, max, has_low_outliers=min<lower_fence, has_high_outliers=max>upper_fence, n_outliers}. Numericos en float, flags en bool nativo, n_outliers en int. Si faltan p25/median(o p50)/p75 devuelve {} (dict vacio). Cuando min/max faltan, los bigotes caen a la fence correspondiente."
|
||||||
|
uses_functions: []
|
||||||
|
uses_types: []
|
||||||
|
returns: []
|
||||||
|
returns_optional: false
|
||||||
|
error_type: ""
|
||||||
|
imports: []
|
||||||
|
tested: true
|
||||||
|
tests: ["test_boxplot_tukey_basico", "test_percentiles_faltan_devuelve_vacio", "test_median_cae_a_p50", "test_whiskers_usan_fence_si_falta_min_max", "test_tipos_salida_float_bool_int"]
|
||||||
|
test_file_path: "python/functions/datascience/build_boxplot_stats_test.py"
|
||||||
|
file_path: "python/functions/datascience/build_boxplot_stats.py"
|
||||||
|
---
|
||||||
|
|
||||||
|
## Ejemplo
|
||||||
|
|
||||||
|
```python
|
||||||
|
import sys, os
|
||||||
|
sys.path.insert(0, os.path.join("python", "functions"))
|
||||||
|
from datascience.build_boxplot_stats import build_boxplot_stats
|
||||||
|
|
||||||
|
# Sub-bloque numeric tal y como lo produce describe_numeric:
|
||||||
|
numeric = {
|
||||||
|
"min": 1.0, "max": 100.0,
|
||||||
|
"p25": 10.0, "median": 25.0, "p75": 40.0,
|
||||||
|
"iqr": 30.0, "n_outliers": 3,
|
||||||
|
}
|
||||||
|
box = build_boxplot_stats(numeric)
|
||||||
|
print(box["lower_fence"], box["upper_fence"]) # -35.0 85.0
|
||||||
|
print(box["whisker_lo"], box["whisker_hi"]) # 1.0 85.0
|
||||||
|
print(box["has_low_outliers"], box["has_high_outliers"]) # False True
|
||||||
|
```
|
||||||
|
|
||||||
|
## Cuando usarla
|
||||||
|
|
||||||
|
- Usala al dibujar un boxplot horizontal bajo el histograma en el capitulo `num_distr` de `AutomaticEDA`: convierte el bloque `numeric` de un `ColumnProfile` en las cifras exactas que el renderer necesita (cuartiles, fences, extremos de los bigotes y flags de outliers).
|
||||||
|
- Cuando ya tengas los percentiles calculados (salida de `describe_numeric`) y solo necesites derivar la geometria del boxplot de Tukey sin volver a tocar los valores crudos.
|
||||||
|
- Cuando quieras decidir si una columna tiene cola alta/baja (`has_high_outliers` / `has_low_outliers`) antes de proponer una transformacion (log, winsorize).
|
||||||
|
|
||||||
|
## Gotchas
|
||||||
|
|
||||||
|
- Funcion pura, sin I/O y determinista. Lectura defensiva con `.get`: NUNCA lanza. Si faltan `p25`, `median`/`p50` o `p75` devuelve `{}` (dict vacio) — el caller debe omitir el boxplot.
|
||||||
|
- Los `n_outliers` que se propagan vienen del bloque z-score del profile (`detect_outliers`, threshold 3.0), NO de la regla IQR. Son informativos: el conteo de Tukey que esta funcion calcula son los **fences** (`lower_fence`/`upper_fence`), no un recuento de puntos.
|
||||||
|
- No recibe los valores crudos de la columna, solo deriva cifras desde los percentiles ya calculados. Por eso no puede contar cuantos puntos caen fuera de las fences, solo si los extremos (`min`/`max`) las superan.
|
||||||
|
- `iqr` se recalcula como `q3 - q1` aunque el bloque traiga `numeric['iqr']`: asi funciona aunque esa clave falte.
|
||||||
|
- Cuando `min`/`max` faltan, los bigotes caen a la fence correspondiente y los flags de outliers quedan en `False` (sin extremo real no se afirma cola).
|
||||||
@@ -0,0 +1,94 @@
|
|||||||
|
"""build_boxplot_stats — Tukey boxplot statistics from an EDA `numeric` sub-block.
|
||||||
|
|
||||||
|
Pure function: no I/O, deterministic. Takes the `numeric` dict of a ColumnProfile
|
||||||
|
(group `eda`, the output of describe_numeric) and derives the figures needed to
|
||||||
|
draw a horizontal Tukey boxplot using the 1.5 * IQR rule.
|
||||||
|
|
||||||
|
It only derives numbers from already-computed percentiles; it never sees the raw
|
||||||
|
column values. Reading is defensive (.get throughout) and the function NEVER
|
||||||
|
raises: if the key percentiles (p25 / p50 / p75) are missing it returns {} so the
|
||||||
|
caller can simply skip the boxplot.
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
def _num(value):
|
||||||
|
"""Coerce to float defensively; return None for None/bool/non-numeric."""
|
||||||
|
# bool is a subclass of int; a percentile value is never a real bool, so
|
||||||
|
# treat True/False as missing instead of silently coercing to 1.0/0.0.
|
||||||
|
if value is None or isinstance(value, bool):
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
return float(value)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def build_boxplot_stats(numeric: dict) -> dict:
|
||||||
|
"""Derive Tukey boxplot statistics from the `numeric` sub-block of a profile.
|
||||||
|
|
||||||
|
Reads the percentiles already computed by describe_numeric and applies the
|
||||||
|
classic 1.5 * IQR fence rule to obtain the whisker extremes and outlier
|
||||||
|
flags of a horizontal boxplot. No raw values are needed.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
numeric: The `numeric` sub-block of an eda ColumnProfile (output of
|
||||||
|
describe_numeric). Every value may be None; read defensively.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Dict with the boxplot figures
|
||||||
|
{q1, median, q3, iqr, lower_fence, upper_fence, whisker_lo, whisker_hi,
|
||||||
|
min, max, has_low_outliers, has_high_outliers, n_outliers}.
|
||||||
|
If p25, p50/median or p75 are missing (None) returns {} (empty dict) so
|
||||||
|
the caller omits the plot.
|
||||||
|
"""
|
||||||
|
if not isinstance(numeric, dict):
|
||||||
|
return {}
|
||||||
|
|
||||||
|
q1 = _num(numeric.get("p25"))
|
||||||
|
q3 = _num(numeric.get("p75"))
|
||||||
|
# Prefer the explicit median; fall back to p50 (they are the same quantile).
|
||||||
|
median = _num(numeric.get("median"))
|
||||||
|
if median is None:
|
||||||
|
median = _num(numeric.get("p50"))
|
||||||
|
|
||||||
|
# Without the three quartiles a boxplot cannot be drawn.
|
||||||
|
if q1 is None or q3 is None or median is None:
|
||||||
|
return {}
|
||||||
|
|
||||||
|
# Recompute the IQR from the quartiles rather than trusting numeric['iqr'],
|
||||||
|
# which may be missing even when the percentiles are present.
|
||||||
|
iqr = q3 - q1
|
||||||
|
lower_fence = q1 - 1.5 * iqr
|
||||||
|
upper_fence = q3 + 1.5 * iqr
|
||||||
|
|
||||||
|
mn = _num(numeric.get("min"))
|
||||||
|
mx = _num(numeric.get("max"))
|
||||||
|
|
||||||
|
# Whisker extremes: the real data range clamped to the fences. When the
|
||||||
|
# corresponding extreme is missing, fall back to the fence itself.
|
||||||
|
whisker_lo = max(mn, lower_fence) if mn is not None else lower_fence
|
||||||
|
whisker_hi = min(mx, upper_fence) if mx is not None else upper_fence
|
||||||
|
|
||||||
|
has_low_outliers = bool(mn is not None and mn < lower_fence)
|
||||||
|
has_high_outliers = bool(mx is not None and mx > upper_fence)
|
||||||
|
|
||||||
|
# Informative only: these outliers come from the z-score block of the
|
||||||
|
# profile, not from this IQR fence computation.
|
||||||
|
raw_n = numeric.get("n_outliers")
|
||||||
|
n_outliers = int(raw_n) if isinstance(raw_n, (int, float)) and not isinstance(raw_n, bool) else 0
|
||||||
|
|
||||||
|
return {
|
||||||
|
"q1": q1,
|
||||||
|
"median": median,
|
||||||
|
"q3": q3,
|
||||||
|
"iqr": iqr,
|
||||||
|
"lower_fence": lower_fence,
|
||||||
|
"upper_fence": upper_fence,
|
||||||
|
"whisker_lo": whisker_lo,
|
||||||
|
"whisker_hi": whisker_hi,
|
||||||
|
"min": mn,
|
||||||
|
"max": mx,
|
||||||
|
"has_low_outliers": has_low_outliers,
|
||||||
|
"has_high_outliers": has_high_outliers,
|
||||||
|
"n_outliers": n_outliers,
|
||||||
|
}
|
||||||
@@ -0,0 +1,108 @@
|
|||||||
|
"""Tests para build_boxplot_stats."""
|
||||||
|
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
|
||||||
|
sys.path.insert(0, os.path.dirname(__file__))
|
||||||
|
|
||||||
|
from build_boxplot_stats import build_boxplot_stats
|
||||||
|
|
||||||
|
# Keys that a non-empty result dict must always contain.
|
||||||
|
_EXPECTED_KEYS = {
|
||||||
|
"q1", "median", "q3", "iqr", "lower_fence", "upper_fence",
|
||||||
|
"whisker_lo", "whisker_hi", "min", "max",
|
||||||
|
"has_low_outliers", "has_high_outliers", "n_outliers",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_boxplot_tukey_basico():
|
||||||
|
"""Golden: bloque numeric con outlier alto claro -> fences IQR de Tukey."""
|
||||||
|
numeric = {
|
||||||
|
"min": 1.0, "max": 100.0,
|
||||||
|
"p25": 10.0, "median": 25.0, "p75": 40.0,
|
||||||
|
"iqr": 30.0, "n_outliers": 3,
|
||||||
|
}
|
||||||
|
box = build_boxplot_stats(numeric)
|
||||||
|
|
||||||
|
assert set(box.keys()) == _EXPECTED_KEYS
|
||||||
|
|
||||||
|
assert box["q1"] == 10.0
|
||||||
|
assert box["median"] == 25.0
|
||||||
|
assert box["q3"] == 40.0
|
||||||
|
# iqr recomputado desde los cuartiles.
|
||||||
|
assert box["iqr"] == 30.0
|
||||||
|
# lower = 10 - 1.5*30 = -35 ; upper = 40 + 1.5*30 = 85.
|
||||||
|
assert box["lower_fence"] == -35.0
|
||||||
|
assert box["upper_fence"] == 85.0
|
||||||
|
# whisker_lo = max(min=1, -35) = 1 ; whisker_hi = min(max=100, 85) = 85.
|
||||||
|
assert box["whisker_lo"] == 1.0
|
||||||
|
assert box["whisker_hi"] == 85.0
|
||||||
|
assert box["min"] == 1.0
|
||||||
|
assert box["max"] == 100.0
|
||||||
|
# Solo hay outliers altos (100 > 85), no bajos (1 no < -35).
|
||||||
|
assert box["has_low_outliers"] is False
|
||||||
|
assert box["has_high_outliers"] is True
|
||||||
|
# n_outliers se propaga del bloque z-score (informativo).
|
||||||
|
assert box["n_outliers"] == 3
|
||||||
|
|
||||||
|
|
||||||
|
def test_percentiles_faltan_devuelve_vacio():
|
||||||
|
"""Si falta p25/median/p75 -> {} (caller omite el boxplot)."""
|
||||||
|
# Falta p25.
|
||||||
|
assert build_boxplot_stats({"median": 25.0, "p75": 40.0}) == {}
|
||||||
|
# Falta p75.
|
||||||
|
assert build_boxplot_stats({"p25": 10.0, "median": 25.0}) == {}
|
||||||
|
# Falta median y p50.
|
||||||
|
assert build_boxplot_stats({"p25": 10.0, "p75": 40.0}) == {}
|
||||||
|
# numeric None / no dict tambien es vacio, nunca lanza.
|
||||||
|
assert build_boxplot_stats(None) == {}
|
||||||
|
assert build_boxplot_stats({}) == {}
|
||||||
|
|
||||||
|
|
||||||
|
def test_median_cae_a_p50():
|
||||||
|
"""median ausente cae a p50."""
|
||||||
|
numeric = {"min": 0.0, "max": 10.0, "p25": 2.0, "p50": 5.0, "p75": 8.0}
|
||||||
|
box = build_boxplot_stats(numeric)
|
||||||
|
assert box["median"] == 5.0
|
||||||
|
assert box["q1"] == 2.0
|
||||||
|
assert box["q3"] == 8.0
|
||||||
|
|
||||||
|
|
||||||
|
def test_whiskers_usan_fence_si_falta_min_max():
|
||||||
|
"""Sin min/max los bigotes caen a las fences y no hay outliers marcados."""
|
||||||
|
numeric = {"p25": 10.0, "median": 25.0, "p75": 40.0} # sin min ni max
|
||||||
|
box = build_boxplot_stats(numeric)
|
||||||
|
|
||||||
|
assert box["min"] is None
|
||||||
|
assert box["max"] is None
|
||||||
|
# iqr = 30, fences -35 / 85; los bigotes caen a las fences.
|
||||||
|
assert box["whisker_lo"] == box["lower_fence"] == -35.0
|
||||||
|
assert box["whisker_hi"] == box["upper_fence"] == 85.0
|
||||||
|
# Sin extremos reales, no se afirma que haya outliers.
|
||||||
|
assert box["has_low_outliers"] is False
|
||||||
|
assert box["has_high_outliers"] is False
|
||||||
|
# n_outliers ausente -> 0.
|
||||||
|
assert box["n_outliers"] == 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_tipos_salida_float_bool_int():
|
||||||
|
"""Numericos en float, flags bool nativos, n_outliers int."""
|
||||||
|
numeric = {
|
||||||
|
"min": -50.0, "max": 200.0,
|
||||||
|
"p25": 10.0, "median": 25.0, "p75": 40.0,
|
||||||
|
"n_outliers": 7,
|
||||||
|
}
|
||||||
|
box = build_boxplot_stats(numeric)
|
||||||
|
|
||||||
|
for key in ("q1", "median", "q3", "iqr", "lower_fence", "upper_fence",
|
||||||
|
"whisker_lo", "whisker_hi", "min", "max"):
|
||||||
|
assert isinstance(box[key], float), f"{key} debe ser float"
|
||||||
|
|
||||||
|
assert isinstance(box["has_low_outliers"], bool)
|
||||||
|
assert isinstance(box["has_high_outliers"], bool)
|
||||||
|
assert isinstance(box["n_outliers"], int) and not isinstance(box["n_outliers"], bool)
|
||||||
|
|
||||||
|
# min=-50 < lower_fence=-35 -> outlier bajo ; max=200 > upper_fence=85 -> alto.
|
||||||
|
assert box["has_low_outliers"] is True
|
||||||
|
assert box["has_high_outliers"] is True
|
||||||
|
assert box["n_outliers"] == 7
|
||||||
Reference in New Issue
Block a user