From 9cdde4a34127b0ac2a8bed4f4e939da8999a33b1 Mon Sep 17 00:00:00 2001 From: Egutierrez Date: Tue, 30 Jun 2026 14:30:31 +0200 Subject: [PATCH 01/53] =?UTF-8?q?feat(eda):=20n=C3=BAcleo=20AutomaticEDA?= =?UTF-8?q?=20=E2=80=94=20documento=20por=20cap=C3=ADtulos=20+=20renderers?= =?UTF-8?q?=20PDF/PPTX=20anti-corte?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduce la capa intermedia entre el contenido de un EDA y su formato de salida. Un documento es una lista de capítulos versionados; cada capítulo es un conjunto ordenado de bloques (heading, markdown, kv_table, data_table, figure, image, caption, note) independientes del formato. Núcleo (paquete de soporte python/functions/datascience/automatic_eda/): - model.py: dataclasses de bloques + Chapter, normalizadores defensivos (aceptan dataclass o dict, nunca lanzan), ENGINE_VERSION y el manifiesto por capítulo (automatic_eda_manifest.json). - text_layout.py: medición/wrapping por rejilla de caracteres compartida. - chapters_registry.py: CHAPTER_ORDER pre-declarado + build_document con auto-discovery de capítulos por convención (permite añadir capítulos en paralelo sin editar el registro). - render_pdf_impl.py: paginador A5 retrato móvil que MIDE cada bloque y nunca corta: texto a líneas completas, tablas largas partidas por filas repitiendo cabecera, figuras/imágenes escaladas para caber enteras. Pie versionado por capítulo. - render_pptx_impl.py: mismo principio sobre slides 16:9 (continúa en slide "(cont.)"; tablas repiten cabecera; figuras exportadas a PNG escaladas). - chapters/portada.py y chapters/overview.py: capítulos de referencia. Portada con nombre, rótulo Automatic-EDA, fuente, almacenamiento (inferido de source), fecha europea, filas×cols, descripción, granularidad y calidad con criterios. Overview con df.head (placeholder honesto si falta head_rows), diccionario de columnas (tipo/nulos/ejemplos) y describe numérico. Funciones públicas del registry (grupo eda, dict-no-throw): - render_automatic_eda_pdf / render_automatic_eda_pptx: aceptan capítulos o un TableProfile (construyen los capítulos con build_document) y escriben el manifiesto. Aditivas — no reemplazan render_eda_pdf. Tests self-contained (sin DuckDB) para ambos renderers: golden (portada + overview), partición de tablas largas repitiendo cabecera, no-corte de celdas y markdown largos, profile None/{} válido de 1 página/slide, y error path en directorio no escribible. 23 tests verdes (incluye los previos de render_eda_pdf, intactos). Dependencia nueva python-pptx>=1.0.2 declarada en python/pyproject.toml. Co-Authored-By: Claude Opus 4.8 (1M context) --- python/functions/datascience/__init__.py | 4 + .../datascience/automatic_eda/__init__.py | 57 ++ .../automatic_eda/chapters/__init__.py | 7 + .../automatic_eda/chapters/overview.py | 176 ++++++ .../automatic_eda/chapters/portada.py | 156 +++++ .../automatic_eda/chapters_registry.py | 89 +++ .../datascience/automatic_eda/model.py | 310 ++++++++++ .../automatic_eda/render_pdf_impl.py | 532 ++++++++++++++++++ .../automatic_eda/render_pptx_impl.py | 518 +++++++++++++++++ .../datascience/automatic_eda/text_layout.py | 107 ++++ .../datascience/render_automatic_eda_pdf.md | 107 ++++ .../datascience/render_automatic_eda_pdf.py | 83 +++ .../render_automatic_eda_pdf_test.py | 140 +++++ .../datascience/render_automatic_eda_pptx.md | 86 +++ .../datascience/render_automatic_eda_pptx.py | 76 +++ .../render_automatic_eda_pptx_test.py | 114 ++++ python/pyproject.toml | 1 + 17 files changed, 2563 insertions(+) create mode 100644 python/functions/datascience/automatic_eda/__init__.py create mode 100644 python/functions/datascience/automatic_eda/chapters/__init__.py create mode 100644 python/functions/datascience/automatic_eda/chapters/overview.py create mode 100644 python/functions/datascience/automatic_eda/chapters/portada.py create mode 100644 python/functions/datascience/automatic_eda/chapters_registry.py create mode 100644 python/functions/datascience/automatic_eda/model.py create mode 100644 python/functions/datascience/automatic_eda/render_pdf_impl.py create mode 100644 python/functions/datascience/automatic_eda/render_pptx_impl.py create mode 100644 python/functions/datascience/automatic_eda/text_layout.py create mode 100644 python/functions/datascience/render_automatic_eda_pdf.md create mode 100644 python/functions/datascience/render_automatic_eda_pdf.py create mode 100644 python/functions/datascience/render_automatic_eda_pdf_test.py create mode 100644 python/functions/datascience/render_automatic_eda_pptx.md create mode 100644 python/functions/datascience/render_automatic_eda_pptx.py create mode 100644 python/functions/datascience/render_automatic_eda_pptx_test.py diff --git a/python/functions/datascience/__init__.py b/python/functions/datascience/__init__.py index 65cefda7..afa5ac45 100644 --- a/python/functions/datascience/__init__.py +++ b/python/functions/datascience/__init__.py @@ -53,8 +53,12 @@ from .fdr_correction import fdr_correction from .suggest_reexpression import suggest_reexpression from .exploratory_caveats import exploratory_caveats from .render_eda_pdf import render_eda_pdf, render_eda_pdf_relational +from .render_automatic_eda_pdf import render_automatic_eda_pdf +from .render_automatic_eda_pptx import render_automatic_eda_pptx __all__ = [ + "render_automatic_eda_pdf", + "render_automatic_eda_pptx", "decode_qr_image", "adf_kpss_stationarity", "acf_pacf", diff --git a/python/functions/datascience/automatic_eda/__init__.py b/python/functions/datascience/automatic_eda/__init__.py new file mode 100644 index 00000000..95d6f374 --- /dev/null +++ b/python/functions/datascience/automatic_eda/__init__.py @@ -0,0 +1,57 @@ +"""AutomaticEDA — chapter-based, versioned EDA document with PDF + PPTX output. + +Public surface (support package for the registry functions +``render_automatic_eda_pdf`` and ``render_automatic_eda_pptx``): + +- Document model: ``Heading``, ``Markdown``, ``KVTable``, ``DataTable``, + ``Figure``, ``Image``, ``Caption``, ``Note``, ``Chapter``; normalizers + ``as_blocks`` / ``as_chapters``; ``ENGINE_VERSION`` / ``ENGINE_NAME``. +- ``build_document(profile, ctx)`` — assemble the ordered chapters of a profile. +- ``render_pdf(chapters, out_path, meta)`` / ``render_pptx(...)`` — the two + renderers (used by the public registry functions). +- ``merge_manifest(...)`` — write/update the per-chapter version manifest. +""" + +from __future__ import annotations + +from .model import ( # noqa: F401 + ENGINE_NAME, + ENGINE_VERSION, + Caption, + Chapter, + DataTable, + Figure, + Heading, + Image, + KVTable, + Markdown, + Note, + as_blocks, + as_chapters, + merge_manifest, +) +from .chapters_registry import CHAPTER_ORDER, build_chapter, build_document # noqa: F401 +from .render_pdf_impl import render_pdf # noqa: F401 +from .render_pptx_impl import render_pptx # noqa: F401 + +__all__ = [ + "ENGINE_NAME", + "ENGINE_VERSION", + "Heading", + "Markdown", + "KVTable", + "DataTable", + "Figure", + "Image", + "Caption", + "Note", + "Chapter", + "as_blocks", + "as_chapters", + "merge_manifest", + "CHAPTER_ORDER", + "build_chapter", + "build_document", + "render_pdf", + "render_pptx", +] diff --git a/python/functions/datascience/automatic_eda/chapters/__init__.py b/python/functions/datascience/automatic_eda/chapters/__init__.py new file mode 100644 index 00000000..e8c1fd18 --- /dev/null +++ b/python/functions/datascience/automatic_eda/chapters/__init__.py @@ -0,0 +1,7 @@ +"""AutomaticEDA chapters. + +Each chapter is a module ``.py`` exposing ``build_(profile, ctx) -> +Chapter | None`` and a ``CHAPTER_VERSION`` constant. The canonical document +order lives in :mod:`automatic_eda.chapters_registry`. Implemented today: +``portada`` and ``overview`` (the reference chapters other agents copy). +""" diff --git a/python/functions/datascience/automatic_eda/chapters/overview.py b/python/functions/datascience/automatic_eda/chapters/overview.py new file mode 100644 index 00000000..93b25b52 --- /dev/null +++ b/python/functions/datascience/automatic_eda/chapters/overview.py @@ -0,0 +1,176 @@ +"""Overview chapter — df.head, column dictionary and describe (reference). + +Second reference chapter for AutomaticEDA. Renders (across as many pages/slides +as needed, the renderers paginate): + +1. ``df.head`` — the first rows of the table. The current ``TableProfile`` does + NOT carry the raw head, so this is read from ``ctx['head_rows']`` / + ``profile['head_rows']`` (a list of row dicts). When absent the chapter shows + an honest placeholder documenting the missing key instead of inventing data. +2. Column dictionary — name / type / nulls / non-null examples. Examples come + from ``columns[i]['examples']`` when present; otherwise they are derived from + real non-null profile values (categorical top values, numeric min/median/max) + so the cell is never empty nor fabricated. +3. ``df.describe`` — mean / median / min / max / std for every numeric column. + +Contract: build_(profile, ctx) -> Chapter | None ; CHAPTER_VERSION = "x.y.z". +""" + +from __future__ import annotations + +from .. import model + +CHAPTER_VERSION = "1.0.0" +CHAPTER_ID = "overview" +CHAPTER_TITLE = "Overview" + +# Profile/ctx keys the calculation phase must add for a full head + examples. +HEAD_KEY = "head_rows" # list[dict] — df.head(n) +EXAMPLES_KEY = "examples" # per column: list of non-null sample values + + +def _fmt_num(value, decimals: int = 3) -> str: + if value is None: + return "—" + if isinstance(value, bool): + return str(value) + if isinstance(value, int): + return f"{value:,}".replace(",", ".") + if isinstance(value, float): + if value != value: # NaN + return "NaN" + if value in (float("inf"), float("-inf")): + return str(value) + text = f"{value:.{decimals}f}".rstrip("0").rstrip(".") + return text if text else "0" + return str(value) + + +def _fmt_pct(value, decimals: int = 1) -> str: + if value is None: + return "—" + try: + return f"{float(value) * 100:.{decimals}f}%" + except (TypeError, ValueError): + return str(value) + + +def _examples_for(col: dict) -> str: + """Build a short string of real non-null example values for a column.""" + explicit = col.get(EXAMPLES_KEY) + if isinstance(explicit, (list, tuple)) and explicit: + return ", ".join(model._safe_str(v) for v in explicit[:4]) + cat = col.get("categorical") or {} + top = cat.get("top") or [] + if top: + vals = [model._safe_str((t or {}).get("value")) for t in top[:4] + if isinstance(t, dict)] + vals = [v for v in vals if v] + if vals: + return ", ".join(vals) + num = col.get("numeric") or {} + if num: + bits = [] + for key in ("min", "median", "max"): + v = num.get(key) + if v is not None: + bits.append(_fmt_num(v)) + if bits: + return ", ".join(bits) + return "—" + + +def _head_block(profile: dict, ctx: dict): + """Return a DataTable for df.head, or a Note documenting the missing key.""" + head = ctx.get(HEAD_KEY) or profile.get(HEAD_KEY) + if isinstance(head, list) and head and isinstance(head[0], dict): + # Column order from the profile, then any extra keys present in rows. + cols = [c.get("name") for c in (profile.get("columns") or []) + if c.get("name")] + if not cols: + cols = list(head[0].keys()) + rows = [[model._safe_str(r.get(c)) for c in cols] for r in head[:10]] + return model.DataTable(header=cols, rows=rows, + note=f"primeras {len(rows)} filas") + return model.Note( + "df.head no disponible: el TableProfile no incluye 'head_rows'. La fase " + "de cálculo debe añadir profile['head_rows'] (lista de dicts fila) o " + "pasarlo en ctx['head_rows'] para mostrar las primeras filas.") + + +def _columns_block(profile: dict): + cols = profile.get("columns") or [] + header = ["Columna", "Tipo", "Nulos", "Ejemplos (no nulos)"] + rows = [] + for c in cols: + if not isinstance(c, dict): + continue + name = c.get("name") or "(col)" + ctype = c.get("inferred_type") or c.get("physical_type") or "—" + sem = c.get("semantic_type") + if sem: + ctype = f"{ctype} ({sem})" + null_pct = c.get("null_pct") + null_count = c.get("null_count") + if null_pct is not None: + nulls = _fmt_pct(null_pct) + if null_count is not None: + nulls += f" ({null_count})" + elif null_count is not None: + nulls = str(null_count) + else: + nulls = "—" + rows.append([name, ctype, nulls, _examples_for(c)]) + if not rows: + return None + return model.DataTable(header=header, rows=rows, title="Columnas") + + +def _describe_block(profile: dict): + cols = profile.get("columns") or [] + header = ["Columna", "mean", "median", "min", "max", "std"] + rows = [] + for c in cols: + if not isinstance(c, dict) or c.get("inferred_type") != "numeric": + continue + num = c.get("numeric") or {} + if not num: + continue + rows.append([ + c.get("name") or "(col)", + _fmt_num(num.get("mean")), + _fmt_num(num.get("median")), + _fmt_num(num.get("min")), + _fmt_num(num.get("max")), + _fmt_num(num.get("std")), + ]) + if not rows: + return None + return model.DataTable(header=header, rows=rows, title="Estadística (describe)") + + +def build_overview(profile: dict, ctx: dict): + """Build the Overview Chapter, or None if the profile has no columns.""" + profile = profile or {} + ctx = ctx or {} + cols = profile.get("columns") or [] + if not cols and not (ctx.get(HEAD_KEY) or profile.get(HEAD_KEY)): + return None + + blocks = [ + model.Heading(text="Primeras filas (df.head)", level=2), + _head_block(profile, ctx), + ] + cols_block = _columns_block(profile) + if cols_block is not None: + blocks.append(model.Heading( + text="Diccionario de columnas", level=2)) + blocks.append(cols_block) + desc_block = _describe_block(profile) + if desc_block is not None: + blocks.append(model.Heading( + text="Resumen estadístico numérico", level=2)) + blocks.append(desc_block) + + return model.Chapter(id=CHAPTER_ID, title=CHAPTER_TITLE, + version=CHAPTER_VERSION, blocks=blocks) diff --git a/python/functions/datascience/automatic_eda/chapters/portada.py b/python/functions/datascience/automatic_eda/chapters/portada.py new file mode 100644 index 00000000..3582d981 --- /dev/null +++ b/python/functions/datascience/automatic_eda/chapters/portada.py @@ -0,0 +1,156 @@ +"""Cover chapter (PORTADA) — the reference chapter for AutomaticEDA. + +Builds the document cover from a TableProfile plus an optional ``ctx`` of +presentation metadata. Reads everything defensively (``.get``) and degrades +honestly: a field that is neither in the profile nor in ``ctx`` is shown as a +placeholder rather than invented, leaving a hook for the LLM layer to fill it. + +Contract for chapter authors (see ``docs/capabilities/automatic_eda.md``): + build_(profile: dict, ctx: dict) -> Chapter | None + CHAPTER_VERSION = "x.y.z" +""" + +from __future__ import annotations + +import os +from datetime import datetime, timezone + +from .. import model + +CHAPTER_VERSION = "1.0.0" +CHAPTER_ID = "portada" +CHAPTER_TITLE = "Portada" + +# Default human description of what the table quality score measures. Chapters +# can override it via ctx["quality_criteria"]. +_DEFAULT_QUALITY_CRITERIA = ( + "media de los scores por columna (0–100): completitud (sin nulos/vacíos), " + "validez (tipo y rango coherentes) y consistencia (sin duplicados/constantes)." +) + + +def _storage_from_source(source: str) -> str: + """Infer the storage technology the dataset currently lives in. + + Heuristic on the profile ``source`` string (a path, DSN or backend name). + Returns a human label; falls back to the raw source when unknown. + """ + s = (source or "").strip().lower() + if not s: + return "—" + if s.endswith(".csv") or s.endswith(".tsv"): + return "CSV" + if s.endswith(".parquet") or s.endswith(".pq"): + return "Parquet" + if s.endswith(".json") or s.endswith(".ndjson"): + return "JSON" + if s.endswith(".xlsx") or s.endswith(".xls"): + return "Excel" + if s.endswith((".duckdb", ".ddb")) or s == "duckdb" or s.endswith(".db"): + return "DuckDB" + if s.startswith(("postgres://", "postgresql://")) or "postgres" in s: + return "PostgreSQL" + if s.startswith("bigquery") or "bigquery" in s or s.count(".") == 2 and " " not in s: + return "BigQuery" + if "sqlite" in s: + return "SQLite" + # Unknown: show the raw source so nothing is hidden. + return source + + +def _fmt_int(v) -> str: + if v is None: + return "—" + try: + return f"{int(v):,}".replace(",", ".") + except (TypeError, ValueError): + return str(v) + + +def _fmt_date_eu(value) -> str: + """Format a date/ISO string as European DD/MM/AAAA HH:mm (UI convention). + + Accepts a datetime, an ISO-8601 string (with or without microseconds/tz) or + any other string. Non-parseable strings are returned verbatim so nothing is + lost; None yields a placeholder. + """ + if value is None: + return "—" + if isinstance(value, datetime): + return value.strftime("%d/%m/%Y %H:%M") + s = str(value).strip() + if not s: + return "—" + try: + dt = datetime.fromisoformat(s.replace("Z", "+00:00")) + return dt.strftime("%d/%m/%Y %H:%M") + except (TypeError, ValueError): + # Try a couple of common forms before giving up. + for fmt in ("%Y-%m-%d %H:%M:%S UTC", "%Y-%m-%d %H:%M UTC", + "%Y-%m-%d %H:%M:%S", "%Y-%m-%d"): + try: + return datetime.strptime(s, fmt).strftime("%d/%m/%Y %H:%M") + except ValueError: + continue + return s + + +def build_portada(profile: dict, ctx: dict): + """Build the cover Chapter, or None if there is truly nothing to show.""" + profile = profile or {} + ctx = ctx or {} + + dataset_name = (ctx.get("dataset_name") or profile.get("table") + or "(dataset sin nombre)") + source = profile.get("source") or "" + # Where the dataset comes from (origin), distinct from where it is stored. + source_origin = ctx.get("source_origin") or source or "—" + storage = ctx.get("storage") or _storage_from_source(source) + + when = _fmt_date_eu( + ctx.get("generated_at") or profile.get("profiled_at") + or datetime.now(timezone.utc)) + + n_rows = profile.get("n_rows") + n_cols = profile.get("n_cols") + shape = f"{_fmt_int(n_rows)} filas × {_fmt_int(n_cols)} columnas" + + score = profile.get("quality_score") + quality_criteria = ctx.get("quality_criteria") or _DEFAULT_QUALITY_CRITERIA + quality_value = "—" if score is None else f"{score} / 100" + + # Granularity: ctx wins; else derive from key candidates; else be honest. + granularity = ctx.get("granularity") + if not granularity: + keys = profile.get("key_candidates") or [] + if keys: + granularity = ("Cada fila parece identificada por " + + ", ".join(str(k) for k in keys[:3]) + ".") + else: + granularity = ("Cada fila es… (granularidad no determinada — " + "pendiente de la capa de cálculo/LLM).") + + description = ctx.get("description") + if not description: + description = ("Descripción no provista — pendiente de la capa LLM " + "(`run_llm`) o de `ctx['description']`.") + + blocks = [ + model.Heading(text=str(dataset_name), level=1), + model.Markdown(text="**Automatic-EDA** · informe exploratorio automático"), + model.KVTable(rows=[ + ("Fuente", source_origin), + ("Almacenamiento", storage), + ("Generado", when), + ("Tamaño", shape), + ("Calidad", quality_value), + ("Criterios de calidad", quality_criteria), + ]), + model.Heading(text="Descripción", level=2), + model.Markdown(text=str(description)), + model.Heading(text="Granularidad", level=2), + model.Markdown(text=str(granularity)), + ] + + return model.Chapter(id=CHAPTER_ID, title=CHAPTER_TITLE, + version=CHAPTER_VERSION, blocks=blocks) diff --git a/python/functions/datascience/automatic_eda/chapters_registry.py b/python/functions/datascience/automatic_eda/chapters_registry.py new file mode 100644 index 00000000..1d6743f4 --- /dev/null +++ b/python/functions/datascience/automatic_eda/chapters_registry.py @@ -0,0 +1,89 @@ +"""Chapter registry — the canonical order of an AutomaticEDA document. + +``CHAPTER_ORDER`` declares every chapter the engine will *ever* place, in the +order they appear in the document. Each id maps by convention to a module +``automatic_eda/chapters/.py`` exposing ``build_(profile, ctx) -> +Chapter | None`` and a ``CHAPTER_VERSION`` constant. + +This pre-declared order is what lets many agents add chapters in parallel +without contention: an agent only creates its own ``chapters/.py`` module — +it never edits this file. ``build_document`` imports each chapter lazily; a +chapter whose module does not exist yet (not implemented) is simply skipped, so +the document is always renderable with whatever chapters are present today. + +``build_document`` never raises: a chapter that errors out is dropped with a +note, and a chapter that returns ``None`` (does not apply to this dataset, e.g. +time series on a dataset with no date column) is omitted. +""" + +from __future__ import annotations + +import importlib + +from . import model + +# Canonical document order. Implemented today: portada, overview. The rest are +# placeholders other agents will fill by creating chapters/.py — they will +# appear in this exact position automatically once their module exists. +CHAPTER_ORDER = [ + "portada", # cover + "overview", # df.head + columns/types/nulls/examples + describe + "num_distr", # numeric distributions + "cat_distr", # categorical distributions + "calidad", # data quality + "correlacion", # correlations / associations + "modelos", # cheap models (PCA/KMeans/outliers) + "analisis_llm", # LLM interpretation + "timeseries", # time-series analysis + "geospatial", # geospatial + "agregacion", # aggregations / pivots +] + + +def build_chapter(chapter_id: str, profile: dict, ctx: dict): + """Build a single chapter by id, or None if absent/not-applicable/error. + + Looks up ``automatic_eda.chapters.`` and calls its + ``build_(profile, ctx)``. Returns a normalized Chapter, or None + when the module is missing, the builder returns None, or anything raises. + """ + mod_name = f"{__package__}.chapters.{chapter_id}" + try: + mod = importlib.import_module(mod_name) + except Exception: # noqa: BLE001 — chapter not implemented yet → skip. + return None + builder = getattr(mod, f"build_{chapter_id}", None) + if builder is None: + return None + try: + result = builder(profile or {}, ctx or {}) + except Exception: # noqa: BLE001 — a broken chapter never aborts the doc. + return None + return model.as_chapter(result) + + +def build_document(profile: dict, ctx: dict = None) -> list: + """Build the full ordered list of chapters for a TableProfile. + + Args: + profile: the ``eda`` group TableProfile dict (may be None/empty). + ctx: optional context dict carrying presentation metadata not present in + the profile (dataset_name, source_origin, storage, generated_at, + description, granularity, quality_criteria, head_rows, ...). + + Returns: + list[Chapter] in canonical order, containing only the chapters that are + implemented and applicable. Never raises. + """ + if profile is None: + profile = {} + if not isinstance(profile, dict): + profile = {} + if ctx is None: + ctx = {} + chapters = [] + for cid in CHAPTER_ORDER: + ch = build_chapter(cid, profile, ctx) + if ch is not None and ch.blocks: + chapters.append(ch) + return chapters diff --git a/python/functions/datascience/automatic_eda/model.py b/python/functions/datascience/automatic_eda/model.py new file mode 100644 index 00000000..8a5c488d --- /dev/null +++ b/python/functions/datascience/automatic_eda/model.py @@ -0,0 +1,310 @@ +"""AutomaticEDA document model — format-independent blocks and chapters. + +This is the intermediate layer between *content* (what an EDA chapter wants to +say) and *output format* (PDF for mobile reading, PPTX for sharing). A document +is an ordered list of :class:`Chapter`. A chapter is ``{id, title, version, +blocks}``. A block is one of a small, closed set of presentation primitives +(heading, markdown, key/value table, data table, figure, image, caption, note). + +Neither renderer knows anything about the EDA profile: they only know how to lay +out blocks so that **nothing is ever cut** — long text wraps to whole lines, +long tables split by rows repeating the header, figures and images are scaled to +fit entirely. Each chapter declares its own ``version`` so every page/slide can +be stamped `` · v`` and tracked in a manifest for continuous, +per-chapter improvement. + +Reading is defensive throughout (the ``eda`` group "dict-no-throw" style): the +normalizers accept dataclass blocks *or* plain dicts, coerce anything unknown +into a readable :class:`Note` instead of raising, and the renderers degrade a +malformed block to text rather than crashing the whole document. +""" + +from __future__ import annotations + +import json +import os +from dataclasses import dataclass, field +from typing import Any, Callable, Optional + +# Global engine version. Bump when the document model or a renderer changes in a +# way that affects output. Individual chapters carry their own CHAPTER_VERSION. +ENGINE_VERSION = "1.0.0" +ENGINE_NAME = "AutomaticEDA" + + +# --------------------------------------------------------------------------- # +# Block primitives. Each carries a stable ``kind`` string so renderers can +# dispatch by kind (works for dataclass instances and for plain dicts alike). +# --------------------------------------------------------------------------- # +@dataclass +class Heading: + """A section heading. ``level`` 1 (largest) .. 3 (smallest).""" + + text: str = "" + level: int = 1 + kind: str = field(default="heading", init=False) + + +@dataclass +class Markdown: + """A block of light markdown text. + + Supported subset (everything else is rendered verbatim, never dropped): + ``#``/``##``/``###`` headings, ``-``/``*`` bullet lists, ``| a | b |`` + tables (consecutive pipe lines become a data table), blank lines as + paragraph breaks, and ``**bold**`` inline markers (markers are stripped, the + text is kept). Text is wrapped to whole lines so it is never cut mid-line. + """ + + text: str = "" + kind: str = field(default="markdown", init=False) + + +@dataclass +class KVTable: + """A two-column key/value table. ``rows`` is a list of ``(label, value)``.""" + + rows: list = field(default_factory=list) + title: Optional[str] = None + kind: str = field(default="kv_table", init=False) + + +@dataclass +class DataTable: + """A tabular block with a header row. + + If it does not fit in the remaining page/slide space it is split by rows, + **repeating the header** on each continuation. Long cell text wraps inside + its column (the row grows taller) so no cell content is ever lost. + """ + + header: list = field(default_factory=list) + rows: list = field(default_factory=list) # list[list[Any]] + title: Optional[str] = None + note: Optional[str] = None + kind: str = field(default="data_table", init=False) + + +@dataclass +class Figure: + """A matplotlib figure, scaled to fit entirely (never cropped). + + Provide either an already-built ``fig`` (a ``matplotlib.figure.Figure``) or + a zero-arg ``make`` callable that returns one (lazy: only built when the + renderer needs it). ``height_in`` is an optional hint for the target height + on the page; renderers clamp it to the available space preserving aspect. + """ + + fig: Any = None + make: Optional[Callable[[], Any]] = None + caption: Optional[str] = None + height_in: Optional[float] = None + kind: str = field(default="figure", init=False) + + +@dataclass +class Image: + """A raster image (PNG/JPG) by path, scaled to fit entirely.""" + + path: str = "" + caption: Optional[str] = None + height_in: Optional[float] = None + kind: str = field(default="image", init=False) + + +@dataclass +class Caption: + """Small auxiliary text rendered under a figure/table.""" + + text: str = "" + kind: str = field(default="caption", init=False) + + +@dataclass +class Note: + """Small auxiliary note (italic). Also the fallback for unknown content.""" + + text: str = "" + kind: str = field(default="note", init=False) + + +@dataclass +class Chapter: + """An ordered set of blocks with an id, a title and a generation version.""" + + id: str = "" + title: str = "" + version: str = "1.0.0" + blocks: list = field(default_factory=list) + + +# --------------------------------------------------------------------------- # +# Defensive normalizers — accept dataclasses OR plain dicts, never raise. +# --------------------------------------------------------------------------- # +_BLOCK_BY_KIND = { + "heading": Heading, + "markdown": Markdown, + "kv_table": KVTable, + "data_table": DataTable, + "figure": Figure, + "image": Image, + "caption": Caption, + "note": Note, +} + + +def as_block(obj: Any): + """Coerce a value into a block dataclass. Unknown values become a Note.""" + if isinstance(obj, (Heading, Markdown, KVTable, DataTable, Figure, Image, + Caption, Note)): + return obj + if isinstance(obj, dict): + kind = obj.get("kind") + cls = _BLOCK_BY_KIND.get(kind) + if cls is None: + return Note(text=_safe_str(obj)) + # Build only with fields the dataclass accepts (ignore extras). + try: + if cls is Heading: + return Heading(text=_safe_str(obj.get("text")), + level=int(obj.get("level", 1) or 1)) + if cls is Markdown: + return Markdown(text=_safe_str(obj.get("text"))) + if cls is KVTable: + return KVTable(rows=list(obj.get("rows") or []), + title=obj.get("title")) + if cls is DataTable: + return DataTable(header=list(obj.get("header") or []), + rows=list(obj.get("rows") or []), + title=obj.get("title"), note=obj.get("note")) + if cls is Figure: + return Figure(fig=obj.get("fig"), make=obj.get("make"), + caption=obj.get("caption"), + height_in=obj.get("height_in")) + if cls is Image: + return Image(path=_safe_str(obj.get("path")), + caption=obj.get("caption"), + height_in=obj.get("height_in")) + if cls is Caption: + return Caption(text=_safe_str(obj.get("text"))) + if cls is Note: + return Note(text=_safe_str(obj.get("text"))) + except Exception: # noqa: BLE001 — never raise on a malformed block. + return Note(text=_safe_str(obj)) + return Note(text=_safe_str(obj)) + + +def as_blocks(seq: Any) -> list: + """Normalize an arbitrary sequence into a list of block dataclasses.""" + if seq is None: + return [] + if not isinstance(seq, (list, tuple)): + return [as_block(seq)] + return [as_block(b) for b in seq] + + +def as_chapter(obj: Any) -> Optional[Chapter]: + """Coerce a value into a Chapter (or None). Accepts a dict or a Chapter.""" + if obj is None: + return None + if isinstance(obj, Chapter): + obj.blocks = as_blocks(obj.blocks) + return obj + if isinstance(obj, dict): + return Chapter( + id=_safe_str(obj.get("id")), + title=_safe_str(obj.get("title")) or _safe_str(obj.get("id")), + version=_safe_str(obj.get("version")) or "1.0.0", + blocks=as_blocks(obj.get("blocks")), + ) + return None + + +def as_chapters(seq: Any) -> list: + """Normalize a sequence of chapters, dropping anything that can't coerce.""" + if seq is None: + return [] + if isinstance(seq, Chapter): + return [as_chapter(seq)] + if not isinstance(seq, (list, tuple)): + return [] + out = [] + for c in seq: + ch = as_chapter(c) + if ch is not None: + out.append(ch) + return out + + +def _safe_str(v: Any) -> str: + """str() that never raises and maps None to ''.""" + if v is None: + return "" + try: + return str(v) + except Exception: # noqa: BLE001 + return "" + + +# --------------------------------------------------------------------------- # +# Manifest — per-chapter versions and page/slide counts for tracking. +# --------------------------------------------------------------------------- # +def merge_manifest(manifest_path: str, renderer: str, chapters_meta: list, + generated_at: str, + engine_version: str = ENGINE_VERSION) -> dict: + """Read-modify-write the AutomaticEDA manifest, merging one renderer's run. + + The manifest lives next to the outputs as ``automatic_eda_manifest.json`` + and records, per chapter, its version plus the page count (PDF) and slide + count (PPTX). Calling either renderer creates or updates it. Never raises: + on any error returns the in-memory manifest without writing. + + Args: + manifest_path: path to the JSON manifest to create or update. + renderer: "pdf" or "pptx" — selects which count key is written. + chapters_meta: list of ``{"id", "version", "n_pages"|"n_slides"}``. + generated_at: ISO-ish timestamp string for this run. + engine_version: AutomaticEDA engine version. + + Returns: + The merged manifest dict (also written to disk on success). + """ + data: dict = {} + try: + if manifest_path and os.path.exists(manifest_path): + with open(manifest_path, "r", encoding="utf-8") as fh: + loaded = json.load(fh) + if isinstance(loaded, dict): + data = loaded + except Exception: # noqa: BLE001 — a corrupt manifest is overwritten. + data = {} + + data["engine"] = ENGINE_NAME + data["engine_version"] = engine_version + data["generated_at"] = generated_at + chapters = data.get("chapters") + if not isinstance(chapters, dict): + chapters = {} + count_key = "n_slides" if renderer == "pptx" else "n_pages" + for cm in chapters_meta or []: + if not isinstance(cm, dict): + continue + cid = cm.get("id") + if not cid: + continue + entry = chapters.get(cid) + if not isinstance(entry, dict): + entry = {} + entry["version"] = cm.get("version") or entry.get("version") or "1.0.0" + entry[count_key] = cm.get(count_key, cm.get("n_pages", cm.get("n_slides"))) + chapters[cid] = entry + data["chapters"] = chapters + + try: + parent = os.path.dirname(os.path.abspath(manifest_path)) + os.makedirs(parent, exist_ok=True) + with open(manifest_path, "w", encoding="utf-8") as fh: + json.dump(data, fh, ensure_ascii=False, indent=2, default=str) + except Exception: # noqa: BLE001 — never raise from the manifest writer. + pass + return data diff --git a/python/functions/datascience/automatic_eda/render_pdf_impl.py b/python/functions/datascience/automatic_eda/render_pdf_impl.py new file mode 100644 index 00000000..b7961b0c --- /dev/null +++ b/python/functions/datascience/automatic_eda/render_pdf_impl.py @@ -0,0 +1,532 @@ +"""AutomaticEDA PDF renderer — A5 portrait, mobile-first, never cuts content. + +A flow paginator: it measures each block (using the deterministic character grid +from :mod:`text_layout`) and places it top-to-bottom on the current page. When a +unit does not fit in the remaining space it moves whole to the next page — +text by whole lines (never mid-line, never mid-word), data tables by rows +**repeating the header**, figures/images scaled to fit entirely (never cropped). + +Each chapter starts on a fresh page and every page is stamped in the footer with +`` · v`` plus the engine version and a running page number, so +output is versioned per chapter for continuous improvement. + +dict-no-throw: a failure inside one block is caught and noted; the PDF is always +produced and at least one page is guaranteed. Engine: matplotlib ``PdfPages``. +""" + +from __future__ import annotations + +import io +import os + +import matplotlib + +matplotlib.use("Agg") + +import matplotlib.image as mpimg # noqa: E402 +import matplotlib.pyplot as plt # noqa: E402 +from matplotlib.backends.backend_pdf import PdfPages # noqa: E402 +from matplotlib.patches import Rectangle # noqa: E402 + +from . import model # noqa: E402 +from . import text_layout as tl # noqa: E402 + +# A5 portrait, inches. +_W, _H = 5.83, 8.27 +_ML, _MR, _MT, _MB = 0.5, 0.42, 0.55, 0.5 +_FOOTER_H = 0.34 +_USABLE_W = _W - _ML - _MR +_CONTENT_TOP = _MT +_CONTENT_BOTTOM = _H - _MB - _FOOTER_H + +# Palette / type (inherits the Tufte-ish mobile look of render_eda_pdf). +_INK = "#1b1b1b" +_ACCENT = "#2a6f97" +_MUTED = "#8a8a8a" +_RULE = "#cccccc" +_HEAD_BG = "#eef3f6" + +_RC = { + "font.size": 10, + "font.family": "sans-serif", + "figure.facecolor": "white", + "savefig.facecolor": "white", + "pdf.fonttype": 42, # embed TrueType — text stays selectable on mobile. +} + +# Font sizes (pt) and derived line heights (in). +_FS_H1, _FS_H2, _FS_H3 = 17, 13, 11 +_FS_BODY, _FS_CELL, _FS_NOTE = 10.5, 9.0, 9.0 +_GAP = 0.12 # vertical gap after a block, inches. +_CELL_PAD = 0.06 # horizontal padding inside a table cell, inches. +_ROW_VPAD = 0.05 # vertical padding inside a table row, inches. + + +class _PdfState: + """Mutable layout cursor for the running PDF document.""" + + def __init__(self, pdf, title: str): + self.pdf = pdf + self.title = title + self.fig = None + self.y = _CONTENT_TOP # inches from the top of the page. + self.page = 0 # global page counter. + self.chapter = None # current Chapter (for the footer). + self.chapter_pages = 0 # pages produced for the current chapter. + + +# --------------------------------------------------------------------------- # +# Coordinate helpers (inches-from-top → matplotlib figure fraction). +# --------------------------------------------------------------------------- # +def _yf(y_in: float) -> float: + return 1.0 - (y_in / _H) + + +def _xf(x_in: float) -> float: + return x_in / _W + + +def _new_page(st: _PdfState) -> None: + """Close the current page (if any) and open a fresh one with a footer.""" + _flush_page(st) + st.fig = plt.figure(figsize=(_W, _H)) + st.y = _CONTENT_TOP + st.page += 1 + st.chapter_pages += 1 + _draw_footer(st) + + +def _flush_page(st: _PdfState) -> None: + if st.fig is not None: + st.pdf.savefig(st.fig) + plt.close(st.fig) + st.fig = None + + +def _draw_footer(st: _PdfState) -> None: + ch = st.chapter + left = "" + if ch is not None: + left = f"{ch.title} · v{ch.version}" + right = f"{model.ENGINE_NAME} v{model.ENGINE_VERSION} · p.{st.page}" + yb = (_MB * 0.45) / _H + st.fig.text(_xf(_ML), yb, left, fontsize=7.5, color=_MUTED, + ha="left", va="center") + st.fig.text(_xf(_W - _MR), yb, right, fontsize=7.5, color=_MUTED, + ha="right", va="center") + # A thin rule above the footer. + st.fig.add_artist(Rectangle( + (_xf(_ML), (_MB + _FOOTER_H * 0.5) / _H), + _xf(_W - _MR) - _xf(_ML), 0.0008, + transform=st.fig.transFigure, color=_RULE, lw=0.6)) + + +def _remaining(st: _PdfState) -> float: + return _CONTENT_BOTTOM - st.y + + +def _ensure_space(st: _PdfState, height: float) -> None: + """Open a new page if ``height`` does not fit in the remaining space.""" + if _remaining(st) < height: + _new_page(st) + + +# --------------------------------------------------------------------------- # +# Block placers. Each advances st.y and paginates as needed. +# --------------------------------------------------------------------------- # +def _place_heading(st: _PdfState, block) -> None: + level = max(1, min(3, int(getattr(block, "level", 1) or 1))) + fs = {1: _FS_H1, 2: _FS_H2, 3: _FS_H3}[level] + text = tl.strip_inline_md(getattr(block, "text", "")) + max_chars = tl.chars_per_line(_USABLE_W, fs) + lines = tl.wrap(text, max_chars) + lh = tl.line_height_in(fs, leading=1.2) + block_h = lh * len(lines) + 0.06 + # Keep at least the heading + a couple of body lines together when possible. + _ensure_space(st, min(block_h + tl.line_height_in(_FS_BODY) * 2, + _CONTENT_BOTTOM - _CONTENT_TOP)) + for ln in lines: + _ensure_space(st, lh) + st.fig.text(_xf(_ML), _yf(st.y), ln, fontsize=fs, fontweight="bold", + color=_INK, ha="left", va="top") + st.y += lh + if level == 1: + # Accent underline under a top-level heading. + st.fig.add_artist(Rectangle( + (_xf(_ML), _yf(st.y + 0.02)), _xf(_ML + 1.4) - _xf(_ML), 0.0016, + transform=st.fig.transFigure, color=_ACCENT, lw=0)) + st.y += 0.10 + st.y += _GAP + + +def _place_text_lines(st: _PdfState, lines: list, fs: float, color: str, + style: str = "normal", indent: float = 0.0) -> None: + lh = tl.line_height_in(fs) + for ln in lines: + _ensure_space(st, lh) + st.fig.text(_xf(_ML + indent), _yf(st.y), ln, fontsize=fs, color=color, + ha="left", va="top", style=style) + st.y += lh + + +def _place_markdown(st: _PdfState, block) -> None: + raw = getattr(block, "text", "") or "" + md_lines = str(raw).split("\n") + i = 0 + n = len(md_lines) + while i < n: + line = md_lines[i] + stripped = line.strip() + # Consecutive pipe-table lines → a DataTable. + if stripped.startswith("|") and stripped.endswith("|"): + j = i + tbl_lines = [] + while j < n and md_lines[j].strip().startswith("|") \ + and md_lines[j].strip().endswith("|"): + tbl_lines.append(md_lines[j]) + j += 1 + parsed = tl.parse_md_table(tbl_lines) + if parsed: + header, rows = parsed + _place_data_table(st, model.DataTable(header=header, rows=rows)) + i = j + continue + if stripped == "": + st.y += tl.line_height_in(_FS_BODY) * 0.5 + i += 1 + continue + if stripped.startswith("### "): + _place_heading(st, model.Heading(stripped[4:], level=3)) + i += 1 + continue + if stripped.startswith("## "): + _place_heading(st, model.Heading(stripped[3:], level=2)) + i += 1 + continue + if stripped.startswith("# "): + _place_heading(st, model.Heading(stripped[2:], level=1)) + i += 1 + continue + if stripped.startswith("- ") or stripped.startswith("* "): + content = tl.strip_inline_md(stripped[2:]) + bullet_chars = tl.chars_per_line(_USABLE_W - 0.22, _FS_BODY) + wrapped = tl.wrap(content, bullet_chars) + first = True + for w in wrapped: + prefix = "• " if first else " " + _place_text_lines(st, [prefix + w], _FS_BODY, _INK, + indent=0.0) + first = False + i += 1 + continue + # Plain paragraph (gather following plain lines into one paragraph). + para = [tl.strip_inline_md(stripped)] + j = i + 1 + while j < n: + nxt = md_lines[j].strip() + if nxt == "" or nxt.startswith(("|", "#", "- ", "* ")): + break + para.append(tl.strip_inline_md(nxt)) + j += 1 + text = " ".join(para) + max_chars = tl.chars_per_line(_USABLE_W, _FS_BODY) + _place_text_lines(st, tl.wrap(text, max_chars), _FS_BODY, _INK) + i = j + st.y += _GAP + + +def _place_kv_table(st: _PdfState, block) -> None: + title = getattr(block, "title", None) + if title: + _place_heading(st, model.Heading(title, level=2)) + rows = getattr(block, "rows", []) or [] + key_w = 1.9 # inches reserved for the label column. + val_chars = tl.chars_per_line(_USABLE_W - key_w - 0.1, _FS_BODY) + lh = tl.line_height_in(_FS_BODY) + for row in rows: + try: + label, value = row[0], row[1] + except Exception: # noqa: BLE001 + label, value = str(row), "" + v_lines = tl.wrap(model._safe_str(value), val_chars) + row_h = lh * len(v_lines) + _ROW_VPAD + _ensure_space(st, row_h) + y0 = st.y + st.fig.text(_xf(_ML), _yf(y0), tl.strip_inline_md(model._safe_str(label)), + fontsize=_FS_BODY, color=_MUTED, ha="left", va="top") + for k, vl in enumerate(v_lines): + st.fig.text(_xf(_ML + key_w), _yf(y0 + k * lh), vl, + fontsize=_FS_BODY, color=_INK, ha="left", va="top") + st.y = y0 + row_h + st.y += _GAP + + +def _col_widths(header: list, rows: list, fs: float) -> list: + """Distribute usable width across columns proportional to content length.""" + ncol = len(header) if header else (len(rows[0]) if rows else 1) + ncol = max(1, ncol) + natural = [3] * ncol + for c in range(ncol): + if header and c < len(header): + natural[c] = max(natural[c], len(model._safe_str(header[c]))) + for r in rows: + if c < len(r): + natural[c] = max(natural[c], len(model._safe_str(r[c]))) + # Clamp so one very long column does not starve the others. + clamped = [min(max(w, 4), 40) for w in natural] + total = float(sum(clamped)) or 1.0 + widths = [_USABLE_W * w / total for w in clamped] + # Enforce a minimum readable column width. + min_w = 0.45 + widths = [max(w, min_w) for w in widths] + # Renormalize if the minimums pushed us over the usable width. + s = sum(widths) + if s > _USABLE_W: + widths = [w * _USABLE_W / s for w in widths] + return widths + + +def _wrap_row(cells: list, widths: list, fs: float) -> list: + """Wrap each cell to its column width → list of line-lists per cell.""" + out = [] + for c, w in enumerate(widths): + text = model._safe_str(cells[c]) if c < len(cells) else "" + max_chars = tl.chars_per_line(w - _CELL_PAD * 2, fs) + out.append(tl.wrap(text, max_chars)) + return out + + +def _draw_table_row(st: _PdfState, cells_lines: list, widths: list, fs: float, + y0: float, header: bool) -> float: + lh = tl.line_height_in(fs) + nlines = max((len(c) for c in cells_lines), default=1) + row_h = lh * nlines + _ROW_VPAD * 2 + if header: + st.fig.add_artist(Rectangle( + (_xf(_ML), _yf(y0 + row_h)), _xf(_ML + _USABLE_W) - _xf(_ML), + _yf(y0) - _yf(y0 + row_h), transform=st.fig.transFigure, + color=_HEAD_BG, lw=0, zorder=0)) + x = _ML + for c, lines in enumerate(cells_lines): + for k, ln in enumerate(lines): + st.fig.text(_xf(x + _CELL_PAD), _yf(y0 + _ROW_VPAD + k * lh), ln, + fontsize=fs, color=_INK, + fontweight="bold" if header else "normal", + ha="left", va="top", zorder=2) + x += widths[c] + # Bottom rule of the row. + st.fig.add_artist(Rectangle( + (_xf(_ML), _yf(y0 + row_h)), _xf(_ML + _USABLE_W) - _xf(_ML), 0.0006, + transform=st.fig.transFigure, color=_RULE, lw=0, zorder=1)) + return row_h + + +def _place_data_table(st: _PdfState, block) -> None: + title = getattr(block, "title", None) + if title: + _place_heading(st, model.Heading(title, level=2)) + header = list(getattr(block, "header", []) or []) + rows = list(getattr(block, "rows", []) or []) + fs = _FS_CELL + widths = _col_widths(header, rows, fs) + header_lines = _wrap_row(header, widths, fs) if header else None + lh = tl.line_height_in(fs) + + def header_h() -> float: + if not header_lines: + return 0.0 + return lh * max((len(c) for c in header_lines), default=1) + _ROW_VPAD * 2 + + def draw_header() -> None: + if header_lines: + st.y += _draw_table_row(st, header_lines, widths, fs, st.y, + header=True) + + # Ensure header + first row fit, else start on a new page. + first_row_h = 0.0 + if rows: + first_lines = _wrap_row(rows[0], widths, fs) + first_row_h = lh * max((len(c) for c in first_lines), default=1) \ + + _ROW_VPAD * 2 + _ensure_space(st, header_h() + max(first_row_h, lh)) + draw_header() + for r in rows: + cells_lines = _wrap_row(r, widths, fs) + row_h = lh * max((len(c) for c in cells_lines), default=1) \ + + _ROW_VPAD * 2 + if _remaining(st) < row_h: + _new_page(st) + draw_header() # repeat header on the continuation page. + st.y += _draw_table_row(st, cells_lines, widths, fs, st.y, header=False) + note = getattr(block, "note", None) + if note: + _place_text_lines(st, tl.wrap(model._safe_str(note), + tl.chars_per_line(_USABLE_W, _FS_NOTE)), + _FS_NOTE, _MUTED, style="italic") + st.y += _GAP + + +def _resolve_figure(block): + fig = getattr(block, "fig", None) + if fig is not None: + return fig, False + make = getattr(block, "make", None) + if callable(make): + try: + return make(), True + except Exception: # noqa: BLE001 + return None, False + return None, False + + +def _png_from_figure(fig) -> bytes: + buf = io.BytesIO() + fig.savefig(buf, format="png", dpi=150, bbox_inches="tight") + buf.seek(0) + return buf.read() + + +def _place_image_array(st: _PdfState, arr, caption) -> None: + h_px, w_px = arr.shape[0], arr.shape[1] + aspect = (h_px / w_px) if w_px else 1.0 + max_h = _CONTENT_BOTTOM - _CONTENT_TOP + target_w = _USABLE_W + target_h = target_w * aspect + if target_h > max_h: + target_h = max_h + target_w = target_h / aspect if aspect else _USABLE_W + cap_h = tl.line_height_in(_FS_NOTE) + 0.04 if caption else 0.0 + # Move whole image to next page if it does not fit in remaining space. + if _remaining(st) < target_h + cap_h: + if (max_h) >= target_h + cap_h: + _new_page(st) + else: + # Taller than a full page even at min — already clamped to max_h. + _new_page(st) + left_frac = _xf(_ML + (_USABLE_W - target_w) / 2.0) + bottom_frac = _yf(st.y + target_h) + ax = st.fig.add_axes([left_frac, bottom_frac, target_w / _W, target_h / _H]) + ax.imshow(arr) + ax.axis("off") + st.y += target_h + 0.04 + if caption: + _place_text_lines(st, tl.wrap(model._safe_str(caption), + tl.chars_per_line(_USABLE_W, _FS_NOTE)), + _FS_NOTE, _MUTED, style="italic") + st.y += _GAP + + +def _place_figure(st: _PdfState, block) -> None: + fig, owned = _resolve_figure(block) + if fig is None: + _place_text_lines(st, ["(figura no disponible)"], _FS_NOTE, _MUTED, + style="italic") + st.y += _GAP + return + try: + png = _png_from_figure(fig) + finally: + if owned: + try: + plt.close(fig) + except Exception: # noqa: BLE001 + pass + arr = mpimg.imread(io.BytesIO(png)) + _place_image_array(st, arr, getattr(block, "caption", None)) + + +def _place_image(st: _PdfState, block) -> None: + path = getattr(block, "path", "") + if not path or not os.path.exists(path): + _place_text_lines(st, [f"(imagen no encontrada: {path})"], _FS_NOTE, + _MUTED, style="italic") + st.y += _GAP + return + arr = mpimg.imread(path) + _place_image_array(st, arr, getattr(block, "caption", None)) + + +def _place_caption(st: _PdfState, block) -> None: + _place_text_lines(st, tl.wrap(getattr(block, "text", ""), + tl.chars_per_line(_USABLE_W, _FS_NOTE)), + _FS_NOTE, _MUTED, style="italic") + st.y += _GAP + + +def _place_note(st: _PdfState, block) -> None: + _place_text_lines(st, tl.wrap(getattr(block, "text", ""), + tl.chars_per_line(_USABLE_W, _FS_NOTE)), + _FS_NOTE, _MUTED, style="italic") + st.y += _GAP + + +_PLACERS = { + "heading": _place_heading, + "markdown": _place_markdown, + "kv_table": _place_kv_table, + "data_table": _place_data_table, + "figure": _place_figure, + "image": _place_image, + "caption": _place_caption, + "note": _place_note, +} + + +def render_pdf(chapters: list, out_path: str, meta: dict = None) -> dict: + """Render a list of Chapters into an A5-portrait, mobile-readable PDF. + + Never raises. Returns ``{path, n_pages, chapters, note}`` where ``chapters`` + is a list of ``{id, version, n_pages}`` for the manifest. On a fatal write + error ``path`` is None and ``note`` explains why. + """ + meta = meta or {} + chapters = model.as_chapters(chapters) + notes = [] + + try: + parent = os.path.dirname(os.path.abspath(out_path)) + os.makedirs(parent, exist_ok=True) + except OSError as e: + return {"path": None, "n_pages": 0, "chapters": [], + "note": f"no se pudo crear el directorio destino: {e}"} + + title = meta.get("title") or model.ENGINE_NAME + chapters_meta = [] + try: + with plt.rc_context(_RC): + with PdfPages(out_path) as pdf: + st = _PdfState(pdf, title) + for ch in chapters: + st.chapter = ch + st.chapter_pages = 0 + _new_page(st) # each chapter starts on a fresh page. + for block in ch.blocks: + placer = _PLACERS.get(getattr(block, "kind", ""), + _place_note) + try: + placer(st, block) + except Exception as e: # noqa: BLE001 + notes.append( + f"bloque '{getattr(block, 'kind', '?')}' del " + f"capítulo '{ch.id}' omitido: {e}") + chapters_meta.append({"id": ch.id, "version": ch.version, + "n_pages": st.chapter_pages}) + _flush_page(st) + if st.page == 0: + # No chapters at all → guarantee one valid page. + st.chapter = model.Chapter(id="vacio", title=title, + version=model.ENGINE_VERSION) + _new_page(st) + _place_note(st, model.Note( + "(documento vacío — sin capítulos aplicables)")) + _flush_page(st) + n_pages = st.page + except Exception as e: # noqa: BLE001 + return {"path": None, "n_pages": 0, "chapters": [], + "note": f"fallo al escribir el PDF: {e}"} + + note = f"{n_pages} páginas" + if notes: + note += " · " + "; ".join(notes) + return {"path": out_path, "n_pages": n_pages, "chapters": chapters_meta, + "note": note} diff --git a/python/functions/datascience/automatic_eda/render_pptx_impl.py b/python/functions/datascience/automatic_eda/render_pptx_impl.py new file mode 100644 index 00000000..5494d604 --- /dev/null +++ b/python/functions/datascience/automatic_eda/render_pptx_impl.py @@ -0,0 +1,518 @@ +"""AutomaticEDA PPTX renderer — 16:9 slides, never cuts content. + +Same flow principle as the PDF renderer but onto PowerPoint slides: measure each +block and place it top-to-bottom; when it does not fit in the remaining slide +space, continue on a new slide titled `` (cont.)``. Data tables split by +rows **repeating the header**; figures/images are scaled to fit entirely. Every +slide carries a footer `` · v`` plus the engine version. + +dict-no-throw: a failure inside one block is caught and noted; the deck is always +produced with at least one slide. Engine: ``python-pptx`` (added dependency). +""" + +from __future__ import annotations + +import io +import os + +from . import model +from . import text_layout as tl + +try: + from pptx import Presentation + from pptx.util import Inches, Pt, Emu + from pptx.dml.color import RGBColor + from pptx.enum.text import PP_ALIGN + _PPTX_OK = True + _PPTX_ERR = "" +except Exception as _e: # noqa: BLE001 — surfaced as a dict-no-throw note. + _PPTX_OK = False + _PPTX_ERR = str(_e) + +# 16:9 widescreen, inches. +_W, _H = 13.333, 7.5 +_ML, _MR = 0.7, 0.7 +_TITLE_TOP, _TITLE_H = 0.28, 0.7 +_CONTENT_TOP = 1.12 +_FOOTER_H = 0.4 +_CONTENT_BOTTOM = _H - _FOOTER_H - 0.15 +_USABLE_W = _W - _ML - _MR + +_INK = (0x1B, 0x1B, 0x1B) +_ACCENT = (0x2A, 0x6F, 0x97) +_MUTED = (0x8A, 0x8A, 0x8A) +_HEAD_BG = (0xEE, 0xF3, 0xF6) +_WHITE = (0xFF, 0xFF, 0xFF) + +_FS_TITLE = 26 +_FS_H1, _FS_H2, _FS_H3 = 20, 16, 13 +_FS_BODY, _FS_CELL, _FS_NOTE = 14, 11, 11 +_GAP = 0.12 + + +class _PptxState: + def __init__(self, prs, title: str): + self.prs = prs + self.title = title + self.slide = None + self.y = _CONTENT_TOP + self.chapter = None + self.slide_no = 0 + self.chapter_slides = 0 + + +def _rgb(c): + return RGBColor(*c) + + +def _new_slide(st: _PptxState, cont: bool = False) -> None: + blank = st.prs.slide_layouts[6] + st.slide = st.prs.slides.add_slide(blank) + st.y = _CONTENT_TOP + st.slide_no += 1 + st.chapter_slides += 1 + _draw_title(st, cont) + _draw_footer(st) + + +def _draw_title(st: _PptxState, cont: bool) -> None: + ch = st.chapter + title = ch.title if ch is not None else st.title + if cont: + title = f"{title} (cont.)" + box = st.slide.shapes.add_textbox( + Inches(_ML), Inches(_TITLE_TOP), Inches(_USABLE_W), Inches(_TITLE_H)) + tf = box.text_frame + tf.word_wrap = True + p = tf.paragraphs[0] + run = p.add_run() + run.text = title + run.font.size = Pt(_FS_TITLE) + run.font.bold = True + run.font.color.rgb = _rgb(_INK) + + +def _draw_footer(st: _PptxState) -> None: + ch = st.chapter + left = f"{ch.title} · v{ch.version}" if ch is not None else "" + right = f"{model.ENGINE_NAME} v{model.ENGINE_VERSION} · {st.slide_no}" + box = st.slide.shapes.add_textbox( + Inches(_ML), Inches(_H - _FOOTER_H), Inches(_USABLE_W), + Inches(_FOOTER_H * 0.7)) + tf = box.text_frame + tf.word_wrap = False + p = tf.paragraphs[0] + r = p.add_run() + r.text = left + r.font.size = Pt(9) + r.font.color.rgb = _rgb(_MUTED) + # Right-aligned engine stamp on a second textbox. + box2 = st.slide.shapes.add_textbox( + Inches(_ML), Inches(_H - _FOOTER_H), Inches(_USABLE_W), + Inches(_FOOTER_H * 0.7)) + tf2 = box2.text_frame + p2 = tf2.paragraphs[0] + p2.alignment = PP_ALIGN.RIGHT + r2 = p2.add_run() + r2.text = right + r2.font.size = Pt(9) + r2.font.color.rgb = _rgb(_MUTED) + + +def _remaining(st: _PptxState) -> float: + return _CONTENT_BOTTOM - st.y + + +def _ensure(st: _PptxState, height: float) -> None: + if _remaining(st) < height: + _new_slide(st, cont=True) + + +def _add_text(st: _PptxState, lines: list, fs: float, color, bold=False, + italic=False, indent=0.0, bullet=False) -> None: + lh = tl.line_height_in(fs) + height = lh * len(lines) + 0.05 + _ensure(st, height) + box = st.slide.shapes.add_textbox( + Inches(_ML + indent), Inches(st.y), Inches(_USABLE_W - indent), + Inches(height)) + tf = box.text_frame + tf.word_wrap = True + first = True + for ln in lines: + p = tf.paragraphs[0] if first else tf.add_paragraph() + first = False + run = p.add_run() + run.text = ("• " + ln) if bullet else ln + run.font.size = Pt(fs) + run.font.bold = bold + run.font.italic = italic + run.font.color.rgb = _rgb(color) + st.y += height + + +def _place_heading(st: _PptxState, block) -> None: + level = max(1, min(3, int(getattr(block, "level", 1) or 1))) + fs = {1: _FS_H1, 2: _FS_H2, 3: _FS_H3}[level] + text = tl.strip_inline_md(getattr(block, "text", "")) + lines = tl.wrap(text, tl.chars_per_line(_USABLE_W, fs)) + _add_text(st, lines, fs, _INK, bold=True) + st.y += 0.04 + + +def _place_markdown(st: _PptxState, block) -> None: + raw = str(getattr(block, "text", "") or "") + md_lines = raw.split("\n") + i, n = 0, len(md_lines) + while i < n: + stripped = md_lines[i].strip() + if stripped.startswith("|") and stripped.endswith("|"): + j = i + tbl = [] + while j < n and md_lines[j].strip().startswith("|") \ + and md_lines[j].strip().endswith("|"): + tbl.append(md_lines[j]) + j += 1 + parsed = tl.parse_md_table(tbl) + if parsed: + header, rows = parsed + _place_data_table(st, model.DataTable(header=header, rows=rows)) + i = j + continue + if stripped == "": + st.y += tl.line_height_in(_FS_BODY) * 0.4 + i += 1 + continue + if stripped.startswith("### "): + _place_heading(st, model.Heading(stripped[4:], level=3)) + i += 1 + continue + if stripped.startswith("## "): + _place_heading(st, model.Heading(stripped[3:], level=2)) + i += 1 + continue + if stripped.startswith("# "): + _place_heading(st, model.Heading(stripped[2:], level=1)) + i += 1 + continue + if stripped.startswith("- ") or stripped.startswith("* "): + content = tl.strip_inline_md(stripped[2:]) + lines = tl.wrap(content, tl.chars_per_line(_USABLE_W - 0.3, _FS_BODY)) + _add_text(st, lines, _FS_BODY, _INK, bullet=True) + i += 1 + continue + para = [tl.strip_inline_md(stripped)] + j = i + 1 + while j < n: + nxt = md_lines[j].strip() + if nxt == "" or nxt.startswith(("|", "#", "- ", "* ")): + break + para.append(tl.strip_inline_md(nxt)) + j += 1 + text = " ".join(para) + _add_text(st, tl.wrap(text, tl.chars_per_line(_USABLE_W, _FS_BODY)), + _FS_BODY, _INK) + i = j + st.y += _GAP + + +def _place_kv_table(st: _PptxState, block) -> None: + title = getattr(block, "title", None) + if title: + _place_heading(st, model.Heading(title, level=2)) + rows = getattr(block, "rows", []) or [] + data_rows = [] + for row in rows: + try: + label, value = row[0], row[1] + except Exception: # noqa: BLE001 + label, value = str(row), "" + data_rows.append([model._safe_str(label), model._safe_str(value)]) + _place_data_table(st, model.DataTable(header=["Campo", "Valor"], + rows=data_rows), shaded_header=True, + key_value=True) + + +def _col_widths(header, rows): + ncol = len(header) if header else (len(rows[0]) if rows else 1) + ncol = max(1, ncol) + natural = [3] * ncol + for c in range(ncol): + if header and c < len(header): + natural[c] = max(natural[c], len(model._safe_str(header[c]))) + for r in rows: + if c < len(r): + natural[c] = max(natural[c], len(model._safe_str(r[c]))) + clamped = [min(max(w, 4), 44) for w in natural] + total = float(sum(clamped)) or 1.0 + return [_USABLE_W * w / total for w in clamped] + + +def _row_height_in(cells, widths, fs) -> float: + lh = tl.line_height_in(fs) + maxlines = 1 + for c, w in enumerate(widths): + text = model._safe_str(cells[c]) if c < len(cells) else "" + lines = tl.wrap(text, tl.chars_per_line(w - 0.12, fs)) + maxlines = max(maxlines, len(lines)) + return lh * maxlines + 0.10 + + +def _emit_table(st: _PptxState, header, chunk, widths, fs) -> None: + nrows = len(chunk) + (1 if header else 0) + ncol = len(widths) + # Pre-measure total height to size the shape (pptx still auto-grows rows). + heights = [] + if header: + heights.append(_row_height_in(header, widths, fs)) + for r in chunk: + heights.append(_row_height_in(r, widths, fs)) + total_h = sum(heights) + gtable = st.slide.shapes.add_table( + nrows, ncol, Inches(_ML), Inches(st.y), Inches(_USABLE_W), + Inches(total_h)).table + gtable.first_row = bool(header) + gtable.horz_banding = False + for c in range(ncol): + gtable.columns[c].width = Emu(int(Inches(widths[c]))) + ridx = 0 + if header: + for c in range(ncol): + cell = gtable.cell(0, c) + cell.text = model._safe_str(header[c]) if c < len(header) else "" + _style_cell(cell, fs, _INK, bold=True, fill=_HEAD_BG) + ridx = 1 + for r in chunk: + for c in range(ncol): + cell = gtable.cell(ridx, c) + cell.text = model._safe_str(r[c]) if c < len(r) else "" + _style_cell(cell, fs, _INK, bold=False, fill=_WHITE) + ridx += 1 + st.y += total_h + _GAP + + +def _style_cell(cell, fs, color, bold, fill) -> None: + cell.fill.solid() + cell.fill.fore_color.rgb = _rgb(fill) + cell.margin_left = Inches(0.05) + cell.margin_right = Inches(0.05) + cell.margin_top = Inches(0.02) + cell.margin_bottom = Inches(0.02) + for p in cell.text_frame.paragraphs: + for run in p.runs: + run.font.size = Pt(fs) + run.font.bold = bold + run.font.color.rgb = _rgb(color) + + +def _place_data_table(st: _PptxState, block, shaded_header=True, + key_value=False) -> None: + title = getattr(block, "title", None) + if title: + _place_heading(st, model.Heading(title, level=2)) + header = list(getattr(block, "header", []) or []) + rows = list(getattr(block, "rows", []) or []) + fs = _FS_CELL + widths = _col_widths(header, rows) + header_h = _row_height_in(header, widths, fs) if header else 0.0 + + idx = 0 + n = len(rows) + if n == 0: + # Header-only table still rendered (one slide). + _ensure(st, header_h + 0.2) + _emit_table(st, header, [], widths, fs) + return + while idx < n: + # Greedily fill the current slide with as many rows as fit. + if _remaining(st) < header_h + _row_height_in(rows[idx], widths, fs): + _new_slide(st, cont=True) + avail = _remaining(st) - header_h + chunk = [] + used = 0.0 + while idx < n: + rh = _row_height_in(rows[idx], widths, fs) + if used + rh > avail and chunk: + break + chunk.append(rows[idx]) + used += rh + idx += 1 + _emit_table(st, header, chunk, widths, fs) + note = getattr(block, "note", None) + if note: + _add_text(st, tl.wrap(model._safe_str(note), + tl.chars_per_line(_USABLE_W, _FS_NOTE)), _FS_NOTE, _MUTED, + italic=True) + + +def _img_size_px(data: bytes): + try: + from PIL import Image + with Image.open(io.BytesIO(data)) as im: + return im.size # (w, h) + except Exception: # noqa: BLE001 + return (1200, 800) + + +def _resolve_png(block): + fig = getattr(block, "fig", None) + make = getattr(block, "make", None) + f = fig + owned = False + if f is None and callable(make): + try: + f = make() + owned = True + except Exception: # noqa: BLE001 + f = None + if f is None: + return None + try: + import matplotlib.pyplot as plt + buf = io.BytesIO() + f.savefig(buf, format="png", dpi=150, bbox_inches="tight") + buf.seek(0) + return buf.read() + except Exception: # noqa: BLE001 + return None + finally: + if owned: + try: + import matplotlib.pyplot as plt + plt.close(f) + except Exception: # noqa: BLE001 + pass + + +def _place_picture_bytes(st: _PptxState, data: bytes, caption) -> None: + w_px, h_px = _img_size_px(data) + aspect = (h_px / w_px) if w_px else 0.66 + max_h = _CONTENT_BOTTOM - _CONTENT_TOP + target_w = _USABLE_W + target_h = target_w * aspect + if target_h > max_h: + target_h = max_h + target_w = target_h / aspect if aspect else _USABLE_W + cap_h = tl.line_height_in(_FS_NOTE) + 0.05 if caption else 0.0 + if _remaining(st) < target_h + cap_h: + _new_slide(st, cont=True) + left = _ML + (_USABLE_W - target_w) / 2.0 + st.slide.shapes.add_picture(io.BytesIO(data), Inches(left), Inches(st.y), + width=Inches(target_w), height=Inches(target_h)) + st.y += target_h + 0.05 + if caption: + _add_text(st, tl.wrap(model._safe_str(caption), + tl.chars_per_line(_USABLE_W, _FS_NOTE)), _FS_NOTE, _MUTED, + italic=True) + st.y += _GAP + + +def _place_figure(st: _PptxState, block) -> None: + png = _resolve_png(block) + if png is None: + _add_text(st, ["(figura no disponible)"], _FS_NOTE, _MUTED, italic=True) + st.y += _GAP + return + _place_picture_bytes(st, png, getattr(block, "caption", None)) + + +def _place_image(st: _PptxState, block) -> None: + path = getattr(block, "path", "") + if not path or not os.path.exists(path): + _add_text(st, [f"(imagen no encontrada: {path})"], _FS_NOTE, _MUTED, + italic=True) + st.y += _GAP + return + try: + with open(path, "rb") as fh: + data = fh.read() + except Exception as e: # noqa: BLE001 + _add_text(st, [f"(no se pudo leer la imagen: {e})"], _FS_NOTE, _MUTED, + italic=True) + st.y += _GAP + return + _place_picture_bytes(st, data, getattr(block, "caption", None)) + + +def _place_caption(st: _PptxState, block) -> None: + _add_text(st, tl.wrap(getattr(block, "text", ""), + tl.chars_per_line(_USABLE_W, _FS_NOTE)), _FS_NOTE, _MUTED, + italic=True) + st.y += _GAP + + +def _place_note(st: _PptxState, block) -> None: + _place_caption(st, block) + + +_PLACERS = { + "heading": _place_heading, + "markdown": _place_markdown, + "kv_table": _place_kv_table, + "data_table": _place_data_table, + "figure": _place_figure, + "image": _place_image, + "caption": _place_caption, + "note": _place_note, +} + + +def render_pptx(chapters: list, out_path: str, meta: dict = None) -> dict: + """Render a list of Chapters into a 16:9 PPTX deck. Never raises. + + Returns ``{path, n_slides, chapters, note}`` where ``chapters`` is a list of + ``{id, version, n_slides}`` for the manifest. On a fatal error ``path`` is + None and ``note`` explains why (e.g. python-pptx not installed). + """ + meta = meta or {} + if not _PPTX_OK: + return {"path": None, "n_slides": 0, "chapters": [], + "note": f"python-pptx no disponible: {_PPTX_ERR}"} + + chapters = model.as_chapters(chapters) + notes = [] + try: + parent = os.path.dirname(os.path.abspath(out_path)) + os.makedirs(parent, exist_ok=True) + except OSError as e: + return {"path": None, "n_slides": 0, "chapters": [], + "note": f"no se pudo crear el directorio destino: {e}"} + + title = meta.get("title") or model.ENGINE_NAME + chapters_meta = [] + try: + prs = Presentation() + prs.slide_width = Inches(_W) + prs.slide_height = Inches(_H) + st = _PptxState(prs, title) + for ch in chapters: + st.chapter = ch + st.chapter_slides = 0 + _new_slide(st, cont=False) + for block in ch.blocks: + placer = _PLACERS.get(getattr(block, "kind", ""), _place_note) + try: + placer(st, block) + except Exception as e: # noqa: BLE001 + notes.append( + f"bloque '{getattr(block, 'kind', '?')}' del capítulo " + f"'{ch.id}' omitido: {e}") + chapters_meta.append({"id": ch.id, "version": ch.version, + "n_slides": st.chapter_slides}) + if st.slide_no == 0: + st.chapter = model.Chapter(id="vacio", title=title, + version=model.ENGINE_VERSION) + _new_slide(st, cont=False) + _place_note(st, model.Note( + "(documento vacío — sin capítulos aplicables)")) + prs.save(out_path) + n_slides = st.slide_no + except Exception as e: # noqa: BLE001 + return {"path": None, "n_slides": 0, "chapters": [], + "note": f"fallo al escribir el PPTX: {e}"} + + note = f"{n_slides} slides" + if notes: + note += " · " + "; ".join(notes) + return {"path": out_path, "n_slides": n_slides, "chapters": chapters_meta, + "note": note} diff --git a/python/functions/datascience/automatic_eda/text_layout.py b/python/functions/datascience/automatic_eda/text_layout.py new file mode 100644 index 00000000..dae00904 --- /dev/null +++ b/python/functions/datascience/automatic_eda/text_layout.py @@ -0,0 +1,107 @@ +"""Shared text-measurement helpers for the AutomaticEDA renderers. + +Both renderers flow content top-to-bottom and must know, *before* placing a +block, how much vertical space it will take — that is what guarantees nothing is +cut: a unit either fits in the remaining space or moves to the next page/slide +whole. Measuring proportional text exactly in matplotlib/pptx is impractical, so +we use a deterministic character-grid estimate (chars-per-line from an average +glyph width) which slightly over-estimates and is therefore safe: it never +claims something fits when it would overflow. + +Wrapping is word-aware (``textwrap``) and additionally hard-splits any single +token longer than the line so a 200-character value still wraps instead of +overflowing — that is wrapping, not loss: every character is still rendered. +""" + +from __future__ import annotations + +import textwrap + + +def avg_char_width_in(fontsize_pt: float) -> float: + """Approximate average glyph width in inches for a sans-serif font. + + ~0.5 of the point size is a conservative mean advance width for proportional + sans fonts; dividing by 72 converts points to inches. + """ + return 0.5 * fontsize_pt / 72.0 + + +def line_height_in(fontsize_pt: float, leading: float = 1.32) -> float: + """Line height in inches for a given font size and leading.""" + return leading * fontsize_pt / 72.0 + + +def chars_per_line(width_in: float, fontsize_pt: float) -> int: + """How many average glyphs fit in ``width_in`` at ``fontsize_pt``.""" + cw = avg_char_width_in(fontsize_pt) + if cw <= 0: + return 80 + n = int(width_in / cw) + return max(1, n) + + +def wrap(text: str, max_chars: int) -> list: + """Word-wrap ``text`` to lines of at most ``max_chars``, never losing chars. + + Long tokens (no spaces) are hard-split so they cannot overflow. Existing + newlines are honored as hard breaks. Empty input yields a single empty line + so callers can still reserve a row. + """ + if max_chars < 1: + max_chars = 1 + s = "" if text is None else str(text) + out: list = [] + for raw_line in s.split("\n"): + if raw_line == "": + out.append("") + continue + # textwrap with break_long_words so no token overflows the column. + wrapped = textwrap.wrap( + raw_line, width=max_chars, break_long_words=True, + break_on_hyphens=False, replace_whitespace=True, + drop_whitespace=True, + ) + if not wrapped: + out.append("") + else: + out.extend(wrapped) + return out or [""] + + +def strip_inline_md(text: str) -> str: + """Strip a tiny subset of inline markdown markers, keeping the text. + + Removes ``**bold**`` / ``__bold__`` / ``*em*`` / `` `code` `` markers so the + content is preserved without trying to style spans (which the line-grid + layout cannot do). Nothing is dropped except the markers themselves. + """ + if not text: + return "" + s = str(text) + for marker in ("**", "__", "`"): + s = s.replace(marker, "") + return s + + +def parse_md_table(lines: list): + """Parse consecutive ``| a | b |`` lines into ``(header, rows)`` or None. + + Accepts an optional separator row (``|---|---|``) right after the header, + which is ignored. Returns None if the lines are not a pipe table. + """ + cells_rows = [] + for ln in lines: + s = ln.strip() + if not (s.startswith("|") and s.endswith("|")): + return None + parts = [c.strip() for c in s.strip("|").split("|")] + cells_rows.append(parts) + if not cells_rows: + return None + header = cells_rows[0] + body = cells_rows[1:] + # Drop a markdown separator row (all cells are dashes/colons). + if body and all(set(c) <= set("-: ") and "-" in c for c in body[0]): + body = body[1:] + return header, body diff --git a/python/functions/datascience/render_automatic_eda_pdf.md b/python/functions/datascience/render_automatic_eda_pdf.md new file mode 100644 index 00000000..cf92cf09 --- /dev/null +++ b/python/functions/datascience/render_automatic_eda_pdf.md @@ -0,0 +1,107 @@ +--- +name: render_automatic_eda_pdf +kind: function +lang: py +domain: datascience +version: "1.0.0" +purity: impure +signature: "def render_automatic_eda_pdf(chapters_or_profile, out_path: str, meta: dict = None) -> dict" +description: "Renderiza un documento AutomaticEDA por CAPÍTULOS (modelo de bloques independiente del formato) en un PDF A5 retrato pensado para LEER EN EL MÓVIL. Acepta una lista de capítulos del modelo o directamente un TableProfile del grupo eda (en cuyo caso construye los capítulos canónicos con build_document). El paginador MIDE cada bloque y NUNCA corta nada: el texto se envuelve a líneas completas, las tablas largas se parten por filas REPITIENDO la cabecera, figuras e imágenes se escalan para caber enteras. Cada capítulo empieza en página nueva con pie 'Capítulo · vX.Y.Z' y se escribe un manifiesto automatic_eda_manifest.json junto a la salida para seguimiento por capítulo. dict-no-throw: nunca lanza, devuelve {path, n_pages, chapters, manifest_path, note}. Motor matplotlib PdfPages. Aditivo: NO reemplaza render_eda_pdf." +tags: [eda, pdf, render, report, mobile, automatic-eda, chapters, versioned, no-cut, pagination, matplotlib, datascience, python] +uses_functions: [] +uses_types: [] +returns: [] +returns_optional: false +error_type: "error_go_core" +imports: [os, matplotlib, "datascience.automatic_eda"] +params: + - name: chapters_or_profile + desc: "una lista de capítulos del modelo AutomaticEDA (dataclasses Chapter o dicts {id,title,version,blocks}) O un TableProfile dict del grupo eda. Si es un TableProfile, los capítulos canónicos se construyen con build_document(profile, meta['ctx']). Un capítulo es {id,title,version,blocks}; un bloque es uno de: heading, markdown, kv_table, data_table, figure, image, caption, note. Lectura defensiva: cualquier cosa no reconocida se degrada a Note, nunca lanza." + - name: out_path + desc: "ruta del archivo PDF de salida. Los directorios padre se crean si faltan. Si está en un directorio no escribible (p.ej. /proc/...) devuelve {path:None, note:} sin lanzar." + - name: meta + desc: "dict opcional. Claves: title (título de portada/pie), ctx (contexto de presentación pasado a los builders de capítulo cuando se da un profile: dataset_name, source_origin, storage, generated_at, description, granularity, quality_criteria, head_rows...), manifest_path (override; por defecto automatic_eda_manifest.json junto a out_path), write_manifest (False para no escribirlo), generated_at." +output: "dict (nunca lanza): {path: str|None, n_pages: int, chapters: list[{id,version,n_pages}], manifest_path: str|None, note: str}. En éxito path es la ruta escrita, n_pages el total de páginas, chapters el desglose por capítulo para el manifiesto. En error fatal path es None y note explica la causa." +tested: true +tests: ["test_golden_profile_genera_pdf_portada_y_overview", "test_edge_tabla_larga_parte_repitiendo_cabecera", "test_edge_celda_larga_no_se_corta", "test_no_corta_texto_markdown", "test_edge_profile_none_y_vacio_un_pagina", "test_error_path_directorio_no_escribible_no_revienta"] +test_file_path: "python/functions/datascience/render_automatic_eda_pdf_test.py" +file_path: "python/functions/datascience/render_automatic_eda_pdf.py" +--- + +## Ejemplo + +```python +from datascience import render_automatic_eda_pdf + +# Caso 1: directamente desde un TableProfile del grupo eda. +# profile = profile_table(db, "ventas", backend="duckdb")["profile"] +profile = { + "table": "ventas", "source": "/data/ventas.csv", + "n_rows": 1000, "n_cols": 2, "quality_score": 92.5, + "columns": [ + {"name": "precio", "inferred_type": "numeric", "null_pct": 0.01, + "null_count": 10, + "numeric": {"mean": 42.5, "median": 40.0, "min": 1.0, "max": 100.0, + "std": 12.3}}, + {"name": "categoria", "inferred_type": "categorical", "null_pct": 0.0, + "categorical": {"top": [{"value": "neumaticos", "count": 500}, + {"value": "aceite", "count": 300}]}}, + ], +} +res = render_automatic_eda_pdf( + profile, "reports/ventas_aeda.pdf", + {"title": "EDA — ventas", + "ctx": {"dataset_name": "Ventas", "source_origin": "ERP export", + "description": "Líneas de venta del ERP.", + "granularity": "Cada fila es una línea de venta."}}) +print(res["n_pages"], res["chapters"], res["manifest_path"]) +# -> 3 [{'id':'portada','version':'1.0.0','n_pages':1}, +# {'id':'overview','version':'1.0.0','n_pages':2}] reports/automatic_eda_manifest.json + +# Caso 2: desde capítulos construidos a mano (modelo de bloques). +from datascience.automatic_eda.model import Chapter, Heading, DataTable +ch = Chapter(id="resumen", title="Resumen", version="1.0.0", blocks=[ + Heading("Tabla", 1), + DataTable(header=["col", "valor"], rows=[["a", "1"], ["b", "2"]]), +]) +render_automatic_eda_pdf([ch], "reports/manual.pdf") +``` + +## Cuando usarla + +Cuando quieras el **PDF móvil del nuevo motor AutomaticEDA por capítulos** (portada ++ overview + los capítulos que existan): después de `profile_table(...)`, pásale el +`profile` y obtienes un PDF A5 retrato versionado por capítulo, con manifiesto. Úsala +como capa de presentación PDF del grupo `eda` cuando necesites **garantía de no-corte** +(texto, tablas e imágenes nunca recortados) y **versionado por capítulo** para mejora +continua. Es el reemplazo evolutivo de `render_eda_pdf`: comparte estética Tufte/móvil +pero separa contenido (capítulos/bloques) de formato (renderer), de modo que el mismo +documento se emite también como PPTX (`render_automatic_eda_pptx`). Para añadir un +capítulo nuevo, ver `docs/capabilities/automatic_eda.md`. + +## Gotchas + +- **Impura**: escribe el PDF en `out_path` (crea los directorios padre) y, salvo + `meta['write_manifest']=False`, un `automatic_eda_manifest.json` junto a la salida. + Backend headless `Agg` de matplotlib (corre en agentes/CI sin display). +- **Nunca lanza** (dict-no-throw): un bloque o capítulo que falle se omite y se anota + en `note`; el PDF se genera igual. Un profile `None`/`{}` produce un PDF de 1 página + válido. `out_path` no escribible → `{path: None, note: }`. +- **No corta nada**: el paginador mide cada bloque con una rejilla de caracteres + (sobre-estima ligeramente, nunca afirma que algo cabe cuando se desbordaría). El + texto se envuelve a líneas completas (sin cortar a media palabra), las tablas largas + se parten por filas **repitiendo la cabecera**, las celdas con texto largo se + envuelven dentro de su columna (la fila crece), y figuras/imágenes se escalan para + caber enteras (nunca se recortan). +- **Tablas muy anchas**: con muchas columnas (>10) cada columna se estrecha y su texto + se envuelve en varias líneas (sigue sin perderse). El reparto por columnas-en-grupos + para tablas muy anchas es una mejora pendiente (ver capability page). +- **head_rows / examples**: el capítulo Overview muestra `df.head` desde + `ctx['head_rows']`/`profile['head_rows']` y ejemplos no-nulos desde + `columns[i]['examples']`; si el profile no los trae (hoy no los trae), degrada con un + placeholder honesto y deriva los ejemplos de los valores reales del perfil (top + categóricos, min/median/max numéricos). Documentado en el contrato. +- **Registro en el package**: el `## Ejemplo` usa `from datascience import + render_automatic_eda_pdf` (añadido al `__init__.py`); el test importa el módulo + directo para no depender de ese registro. +- **Fechas en UI europeas**: la portada formatea la fecha como `DD/MM/AAAA HH:mm`. diff --git a/python/functions/datascience/render_automatic_eda_pdf.py b/python/functions/datascience/render_automatic_eda_pdf.py new file mode 100644 index 00000000..67ab10e0 --- /dev/null +++ b/python/functions/datascience/render_automatic_eda_pdf.py @@ -0,0 +1,83 @@ +"""render_automatic_eda_pdf — chapter-based EDA report as an A5-portrait PDF. + +Public ``eda``-group entry point of the AutomaticEDA engine. Takes either a list +of chapters (the format-independent document model) or an ``eda`` TableProfile +dict (in which case the canonical chapters are built with ``build_document``), +and renders a mobile-first PDF whose paginator MEASURES every block and never +cuts text, tables or images: text wraps to whole lines, long tables split by +rows repeating the header, figures/images scale to fit entirely. Each chapter +starts on a fresh page stamped `` · v`` in the footer, and a +per-chapter manifest (``automatic_eda_manifest.json``) is written next to the +output for version tracking. + +dict-no-throw: never raises. Returns ``{path, n_pages, chapters, manifest_path, +note}``; on a fatal write error ``path`` is None and ``note`` explains why. + +Additive: this does NOT replace ``render_eda_pdf`` (still used by +``profile_table(emit_pdf=True)``). It is the new engine that will, in the next +phase, let every EDA emit both a PDF and a PPTX from the same chapter model. +""" + +from __future__ import annotations + +import os + +from datascience.automatic_eda import build_document, merge_manifest, render_pdf +from datascience.automatic_eda.model import as_chapter, as_chapters + + +def _coerce_chapters(chapters_or_profile, meta: dict) -> list: + """Accept chapters OR an eda profile and return a list of Chapter.""" + arg = chapters_or_profile + if isinstance(arg, (list, tuple)): + return as_chapters(list(arg)) + if isinstance(arg, dict): + # A single chapter dict has 'blocks'; a profile has columns/table/rows. + if "blocks" in arg and "columns" not in arg: + ch = as_chapter(arg) + return [ch] if ch is not None else [] + # Treat as an eda TableProfile. + return build_document(arg, (meta or {}).get("ctx")) + return [] + + +def render_automatic_eda_pdf(chapters_or_profile, out_path: str, + meta: dict = None) -> dict: + """Render an AutomaticEDA document into a mobile-readable PDF. + + Args: + chapters_or_profile: either a list of chapters (``Chapter`` dataclasses + or dicts following the document model) or an ``eda`` TableProfile + dict — in the latter case the canonical chapters are built via + ``build_document(profile, meta['ctx'])``. + out_path: filesystem path for the PDF (parent dirs are created). + meta: optional dict. Recognised keys: ``title`` (cover/footer title), + ``ctx`` (presentation context passed to chapter builders when a + profile is given), ``manifest_path`` (override; defaults to + ``automatic_eda_manifest.json`` beside ``out_path``), + ``write_manifest`` (set False to skip), ``generated_at``. + + Returns: + dict (never raises): ``{path, n_pages, chapters, manifest_path, note}``. + """ + meta = dict(meta or {}) + chapters = _coerce_chapters(chapters_or_profile, meta) + result = render_pdf(chapters, out_path, meta) + + manifest_path = None + if meta.get("write_manifest", True) and result.get("path"): + manifest_path = meta.get("manifest_path") + if not manifest_path: + manifest_path = os.path.join( + os.path.dirname(os.path.abspath(out_path)), + "automatic_eda_manifest.json") + generated_at = meta.get("generated_at") or _now_iso() + merge_manifest(manifest_path, "pdf", result.get("chapters") or [], + generated_at) + result["manifest_path"] = manifest_path + return result + + +def _now_iso() -> str: + from datetime import datetime, timezone + return datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC") diff --git a/python/functions/datascience/render_automatic_eda_pdf_test.py b/python/functions/datascience/render_automatic_eda_pdf_test.py new file mode 100644 index 00000000..c5e42612 --- /dev/null +++ b/python/functions/datascience/render_automatic_eda_pdf_test.py @@ -0,0 +1,140 @@ +"""Tests for render_automatic_eda_pdf — DoD: golden + edges + error path. + +Self-contained: builds a synthetic TableProfile (no DuckDB) so the suite is fast +and deterministic. Verifies the cover/overview reference chapters render, that +long tables split by rows repeating the header without losing any cell text, +that an empty/None profile still yields a valid 1-page PDF, and that an +unwritable destination returns ``{path: None}`` without raising. +""" + +import os +import re +import tempfile + +from pypdf import PdfReader + +from datascience.render_automatic_eda_pdf import render_automatic_eda_pdf +from datascience.automatic_eda.model import Chapter, DataTable, Heading, Markdown + + +def _profile() -> dict: + return { + "table": "ventas", + "source": "/data/ventas.csv", + "profiled_at": "2026-06-30T10:00:00+00:00", + "n_rows": 1000, + "n_cols": 3, + "quality_score": 92.5, + "key_candidates": ["id"], + "type_breakdown": {"numeric": 2, "categorical": 1}, + "columns": [ + {"name": "id", "inferred_type": "numeric", "null_pct": 0.0, + "null_count": 0, + "numeric": {"mean": 500.0, "median": 500.0, "min": 1.0, + "max": 1000.0, "std": 288.7}}, + {"name": "precio", "inferred_type": "numeric", "null_pct": 0.01, + "null_count": 10, + "numeric": {"mean": 42.5, "median": 40.0, "min": 1.0, + "max": 100.0, "std": 12.3}}, + {"name": "categoria", "inferred_type": "categorical", + "null_pct": 0.0, "null_count": 0, + "categorical": {"top": [{"value": "neumaticos", "count": 500}, + {"value": "aceite", "count": 300}]}}, + ], + } + + +def _pdf_text(path: str) -> str: + txt = "".join((pg.extract_text() or "") for pg in PdfReader(path).pages) + return re.sub(r"\s+", " ", txt) + + +def test_golden_profile_genera_pdf_portada_y_overview(): + with tempfile.TemporaryDirectory() as d: + out = os.path.join(d, "eda.pdf") + res = render_automatic_eda_pdf(_profile(), out, {"title": "EDA — ventas"}) + assert res["path"] == out + assert os.path.exists(out) + assert res["n_pages"] >= 2 # portada + overview (1+ each). + ids = [c["id"] for c in res["chapters"]] + assert "portada" in ids and "overview" in ids + # Manifest written next to the output with both chapters versioned. + assert res["manifest_path"] and os.path.exists(res["manifest_path"]) + txt = _pdf_text(out) + # Cover fields. + assert "Automatic-EDA" in txt + assert "CSV" in txt # storage inferred from .csv source. + assert "Calidad" in txt and "92.5" in txt + assert "Fuente" in txt + # Overview content: column dictionary + describe. + assert "precio" in txt and "categoria" in txt + assert "median" in txt + + +def test_edge_tabla_larga_parte_repitiendo_cabecera(): + # 60 rows over 6 wide columns: the table must split across pages and repeat + # the header on every continuation page (headers wide enough not to wrap). + header = ["ALPHA", "BETA", "GAMMA", "DELTA", "EPSILON", "ZETA"] + rows = [[f"r{r}c{c}" for c in range(6)] for r in range(60)] + ch = Chapter(id="edge", title="Edge", version="1.0.0", + blocks=[Heading("Tabla", 1), + DataTable(header=header, rows=rows)]) + with tempfile.TemporaryDirectory() as d: + out = os.path.join(d, "edge.pdf") + res = render_automatic_eda_pdf([ch], out, {"write_manifest": False}) + assert res["path"] == out + reader = PdfReader(out) + n_pages = len(reader.pages) + assert n_pages > 1 # table spilled to several pages. + pages_with_header = sum( + 1 for pg in reader.pages if "ALPHA" in (pg.extract_text() or "")) + assert pages_with_header == n_pages # header repeated on every page. + + +def test_edge_celda_larga_no_se_corta(): + # A single cell with ~150 chars must wrap inside its column (the row grows), + # never truncated: all of its words survive in the rendered PDF. + long_cell = ("Lorem ipsum dolor sit amet consectetur adipiscing elit sed do " + "eiusmod tempor incididunt ut labore et dolore magna aliqua " + "reprehenderit voluptate") + header = ["clave", "descripcion"] + rows = [["k1", long_cell], ["k2", "corto"]] + ch = Chapter(id="edge2", title="Edge2", version="1.0.0", + blocks=[DataTable(header=header, rows=rows)]) + with tempfile.TemporaryDirectory() as d: + out = os.path.join(d, "edge2.pdf") + render_automatic_eda_pdf([ch], out, {"write_manifest": False}) + txt = _pdf_text(out) + # Every word of the long cell present (wrapped, not truncated). + for word in ("Lorem", "incididunt", "reprehenderit", "voluptate"): + assert word in txt + + +def test_no_corta_texto_markdown(): + para = " ".join(f"palabra{i}" for i in range(120)) + ch = Chapter(id="md", title="MD", version="1.0.0", + blocks=[Markdown(text=para)]) + with tempfile.TemporaryDirectory() as d: + out = os.path.join(d, "md.pdf") + render_automatic_eda_pdf([ch], out, {"write_manifest": False}) + txt = _pdf_text(out) + for i in (0, 60, 119): # first, middle, last words all present. + assert f"palabra{i}" in txt + + +def test_edge_profile_none_y_vacio_un_pagina(): + with tempfile.TemporaryDirectory() as d: + for arg, name in ((None, "none"), ({}, "empty")): + out = os.path.join(d, f"{name}.pdf") + res = render_automatic_eda_pdf(arg, out, {"write_manifest": False}) + assert res["path"] == out + assert os.path.exists(out) + assert res["n_pages"] == 1 + + +def test_error_path_directorio_no_escribible_no_revienta(): + res = render_automatic_eda_pdf(_profile(), "/proc/nope/x.pdf", + {"write_manifest": False}) + assert res["path"] is None + assert res["n_pages"] == 0 + assert res["note"] diff --git a/python/functions/datascience/render_automatic_eda_pptx.md b/python/functions/datascience/render_automatic_eda_pptx.md new file mode 100644 index 00000000..233609e7 --- /dev/null +++ b/python/functions/datascience/render_automatic_eda_pptx.md @@ -0,0 +1,86 @@ +--- +name: render_automatic_eda_pptx +kind: function +lang: py +domain: datascience +version: "1.0.0" +purity: impure +signature: "def render_automatic_eda_pptx(chapters_or_profile, out_path: str, meta: dict = None) -> dict" +description: "Renderiza un documento AutomaticEDA por CAPÍTULOS (modelo de bloques independiente del formato) en una presentación PPTX 16:9 pensada para COMPARTIR. Acepta una lista de capítulos del modelo o directamente un TableProfile del grupo eda (construye los capítulos canónicos con build_document). Mismo principio anti-corte que el renderer PDF: cada bloque se mide y, si no cabe en la slide, continúa en una slide ' (cont.)'; las tablas largas se parten por filas REPITIENDO la cabecera; las figuras matplotlib se exportan a PNG e insertan escaladas para caber enteras. Cada slide lleva pie 'Capítulo · vX.Y.Z' y se escribe automatic_eda_manifest.json junto a la salida. dict-no-throw: nunca lanza, devuelve {path, n_slides, chapters, manifest_path, note}. Motor python-pptx (dependencia declarada en python/pyproject.toml)." +tags: [eda, pptx, render, report, share, automatic-eda, chapters, versioned, no-cut, slides, python-pptx, datascience, python] +uses_functions: [] +uses_types: [] +returns: [] +returns_optional: false +error_type: "error_go_core" +imports: [os, "python-pptx", "datascience.automatic_eda"] +params: + - name: chapters_or_profile + desc: "una lista de capítulos del modelo AutomaticEDA (dataclasses Chapter o dicts {id,title,version,blocks}) O un TableProfile dict del grupo eda. Si es un TableProfile, los capítulos canónicos se construyen con build_document(profile, meta['ctx']). Bloques soportados: heading, markdown, kv_table, data_table, figure, image, caption, note. Lectura defensiva: lo no reconocido se degrada a Note, nunca lanza." + - name: out_path + desc: "ruta del archivo PPTX de salida. Los directorios padre se crean si faltan. Directorio no escribible → {path:None, note:} sin lanzar." + - name: meta + desc: "dict opcional. Claves: title (título), ctx (contexto de presentación para los builders de capítulo cuando se da un profile), manifest_path (override; por defecto automatic_eda_manifest.json junto a out_path), write_manifest (False para no escribirlo), generated_at." +output: "dict (nunca lanza): {path: str|None, n_slides: int, chapters: list[{id,version,n_slides}], manifest_path: str|None, note: str}. En error fatal (incluida python-pptx no instalada) path es None y note explica la causa." +tested: true +tests: ["test_golden_profile_genera_pptx_portada_y_overview", "test_edge_tabla_larga_parte_repitiendo_cabecera_sin_cortar", "test_edge_profile_none_y_vacio_un_slide", "test_error_path_directorio_no_escribible_no_revienta"] +test_file_path: "python/functions/datascience/render_automatic_eda_pptx_test.py" +file_path: "python/functions/datascience/render_automatic_eda_pptx.py" +--- + +## Ejemplo + +```python +from datascience import render_automatic_eda_pptx + +# Desde un TableProfile del grupo eda (mismo modelo que el renderer PDF). +profile = { + "table": "ventas", "source": "/data/ventas.csv", + "n_rows": 1000, "n_cols": 2, "quality_score": 92.5, + "columns": [ + {"name": "precio", "inferred_type": "numeric", "null_pct": 0.01, + "numeric": {"mean": 42.5, "median": 40.0, "min": 1.0, "max": 100.0, + "std": 12.3}}, + {"name": "categoria", "inferred_type": "categorical", "null_pct": 0.0, + "categorical": {"top": [{"value": "neumaticos", "count": 500}]}}, + ], +} +res = render_automatic_eda_pptx( + profile, "reports/ventas_aeda.pptx", + {"title": "EDA — ventas", + "ctx": {"dataset_name": "Ventas", "source_origin": "ERP export"}}) +print(res["n_slides"], res["chapters"], res["manifest_path"]) +# -> 3 [{'id':'portada','version':'1.0.0','n_slides':1}, +# {'id':'overview','version':'1.0.0','n_slides':2}] reports/automatic_eda_manifest.json +``` + +## Cuando usarla + +Cuando quieras **compartir el EDA como una presentación** (no para móvil sino para +enseñar a alguien): mismo documento por capítulos que el PDF, emitido como PPTX 16:9. +Úsala junto a `render_automatic_eda_pdf` para que cada EDA tenga sus dos salidas (PDF +móvil + PPTX para compartir) desde el mismo modelo de capítulos. Garantiza no-corte: +ningún texto, tabla ni imagen se recorta — lo que no cabe en una slide continúa en otra +`(cont.)` con la cabecera repetida en las tablas. Para añadir capítulos nuevos al +documento, ver `docs/capabilities/automatic_eda.md`. + +## Gotchas + +- **Impura**: escribe el PPTX en `out_path` y, salvo `meta['write_manifest']=False`, el + manifiesto `automatic_eda_manifest.json` junto a la salida. +- **Dependencia python-pptx**: declarada en `python/pyproject.toml` + (`python-pptx>=1.0.2`). Si no está instalada, devuelve `{path: None, note: + 'python-pptx no disponible: ...'}` sin lanzar. Instalar: + `uv pip install --python python/.venv/bin/python3 python-pptx`. +- **Nunca lanza** (dict-no-throw): un bloque que falle se omite y se anota en `note`; el + deck se genera igual. Un profile `None`/`{}` produce un deck de 1 slide válido. +- **No corta nada**: cada bloque se mide; si no cabe en la slide actual, abre una slide + `(cont.)`. Las tablas largas se parten por filas **repitiendo la cabecera** (las filas + restantes pasan a la siguiente slide). Las figuras matplotlib se exportan a PNG en + memoria y se insertan escaladas para caber enteras (nunca recortadas). +- **Figuras**: un bloque `figure` puede traer una `matplotlib.figure.Figure` ya + construida o un callable `make` (se construye perezosamente). Se cierra tras + rasterizar. Las imágenes (`image`) por ruta se escalan manteniendo el aspecto. +- **Tablas anchas**: con muchas columnas el ancho por columna se reduce y el texto se + envuelve dentro de la celda (sigue sin perderse). El reparto por grupos de columnas + para tablas muy anchas es mejora pendiente. diff --git a/python/functions/datascience/render_automatic_eda_pptx.py b/python/functions/datascience/render_automatic_eda_pptx.py new file mode 100644 index 00000000..9ad36d4f --- /dev/null +++ b/python/functions/datascience/render_automatic_eda_pptx.py @@ -0,0 +1,76 @@ +"""render_automatic_eda_pptx — chapter-based EDA report as a 16:9 PPTX deck. + +Public ``eda``-group entry point that renders an AutomaticEDA document (a list +of chapters, or an ``eda`` TableProfile from which the canonical chapters are +built) into a PowerPoint deck for sharing. Same anti-cut principle as the PDF +renderer: every block is measured and, when it does not fit, continues on a new +slide titled `` (cont.)``; data tables split by rows repeating the +header; matplotlib figures are exported to PNG and inserted scaled to fit +entirely. Each slide is stamped `` · v`` and a per-chapter +manifest (``automatic_eda_manifest.json``) is written next to the output. + +dict-no-throw: never raises. Returns ``{path, n_slides, chapters, +manifest_path, note}``; on a fatal error ``path`` is None and ``note`` explains +why (e.g. python-pptx not installed). + +Engine: ``python-pptx`` (added dependency; declared in python/pyproject.toml). +""" + +from __future__ import annotations + +import os + +from datascience.automatic_eda import build_document, merge_manifest, render_pptx +from datascience.automatic_eda.model import as_chapter, as_chapters + + +def _coerce_chapters(chapters_or_profile, meta: dict) -> list: + """Accept chapters OR an eda profile and return a list of Chapter.""" + arg = chapters_or_profile + if isinstance(arg, (list, tuple)): + return as_chapters(list(arg)) + if isinstance(arg, dict): + if "blocks" in arg and "columns" not in arg: + ch = as_chapter(arg) + return [ch] if ch is not None else [] + return build_document(arg, (meta or {}).get("ctx")) + return [] + + +def render_automatic_eda_pptx(chapters_or_profile, out_path: str, + meta: dict = None) -> dict: + """Render an AutomaticEDA document into a shareable PPTX deck. + + Args: + chapters_or_profile: a list of chapters (``Chapter`` dataclasses or + dicts) or an ``eda`` TableProfile dict (chapters built via + ``build_document(profile, meta['ctx'])``). + out_path: filesystem path for the PPTX (parent dirs are created). + meta: optional dict. Recognised keys: ``title``, ``ctx``, + ``manifest_path`` (defaults to ``automatic_eda_manifest.json`` beside + ``out_path``), ``write_manifest`` (False to skip), ``generated_at``. + + Returns: + dict (never raises): ``{path, n_slides, chapters, manifest_path, note}``. + """ + meta = dict(meta or {}) + chapters = _coerce_chapters(chapters_or_profile, meta) + result = render_pptx(chapters, out_path, meta) + + manifest_path = None + if meta.get("write_manifest", True) and result.get("path"): + manifest_path = meta.get("manifest_path") + if not manifest_path: + manifest_path = os.path.join( + os.path.dirname(os.path.abspath(out_path)), + "automatic_eda_manifest.json") + generated_at = meta.get("generated_at") or _now_iso() + merge_manifest(manifest_path, "pptx", result.get("chapters") or [], + generated_at) + result["manifest_path"] = manifest_path + return result + + +def _now_iso() -> str: + from datetime import datetime, timezone + return datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC") diff --git a/python/functions/datascience/render_automatic_eda_pptx_test.py b/python/functions/datascience/render_automatic_eda_pptx_test.py new file mode 100644 index 00000000..0cde8da6 --- /dev/null +++ b/python/functions/datascience/render_automatic_eda_pptx_test.py @@ -0,0 +1,114 @@ +"""Tests for render_automatic_eda_pptx — DoD: golden + edges + error path. + +Self-contained synthetic TableProfile (no DuckDB). Verifies the cover/overview +chapters render to slides, that long tables split across slides repeating the +header without losing cell text, that an empty/None profile yields a valid +1-slide deck, and that an unwritable destination returns ``{path: None}``. +""" + +import os +import tempfile + +from pptx import Presentation + +from datascience.render_automatic_eda_pptx import render_automatic_eda_pptx +from datascience.automatic_eda.model import Chapter, DataTable, Heading + + +def _profile() -> dict: + return { + "table": "ventas", + "source": "/data/ventas.csv", + "profiled_at": "2026-06-30T10:00:00+00:00", + "n_rows": 1000, + "n_cols": 2, + "quality_score": 92.5, + "columns": [ + {"name": "precio", "inferred_type": "numeric", "null_pct": 0.01, + "null_count": 10, + "numeric": {"mean": 42.5, "median": 40.0, "min": 1.0, + "max": 100.0, "std": 12.3}}, + {"name": "categoria", "inferred_type": "categorical", + "null_pct": 0.0, "null_count": 0, + "categorical": {"top": [{"value": "neumaticos", "count": 500}, + {"value": "aceite", "count": 300}]}}, + ], + } + + +def _slide_texts(path: str) -> list: + prs = Presentation(path) + out = [] + for sl in prs.slides: + parts = [] + for sh in sl.shapes: + if sh.has_text_frame: + parts.append(sh.text_frame.text) + if sh.has_table: + tb = sh.table + for r in range(len(tb.rows)): + for c in range(len(tb.columns)): + parts.append(tb.cell(r, c).text) + out.append(" ".join(parts)) + return out + + +def test_golden_profile_genera_pptx_portada_y_overview(): + with tempfile.TemporaryDirectory() as d: + out = os.path.join(d, "eda.pptx") + res = render_automatic_eda_pptx(_profile(), out, {"title": "EDA — ventas"}) + assert res["path"] == out + assert os.path.exists(out) + assert res["n_slides"] >= 2 + ids = [c["id"] for c in res["chapters"]] + assert "portada" in ids and "overview" in ids + assert res["manifest_path"] and os.path.exists(res["manifest_path"]) + joined = " ".join(_slide_texts(out)) + assert "Automatic-EDA" in joined + assert "CSV" in joined + assert "92.5" in joined + assert "precio" in joined and "categoria" in joined + assert "median" in joined + + +def test_edge_tabla_larga_parte_repitiendo_cabecera_sin_cortar(): + long_cell = ("Lorem ipsum dolor sit amet consectetur adipiscing elit sed do " + "eiusmod tempor incididunt reprehenderit voluptate") + header = ["ALPHA", "BETA", "GAMMA", "DELTA"] + rows = [[f"r{r}c{c}" for c in range(4)] for r in range(50)] + rows[0][1] = long_cell + ch = Chapter(id="edge", title="Edge", version="1.0.0", + blocks=[Heading("Tabla", 1), + DataTable(header=header, rows=rows)]) + with tempfile.TemporaryDirectory() as d: + out = os.path.join(d, "edge.pptx") + res = render_automatic_eda_pptx([ch], out, {"write_manifest": False}) + assert res["path"] == out + texts = _slide_texts(out) + assert res["n_slides"] > 1 # table spilled to several slides. + # Header repeated: every slide that carries table rows shows "ALPHA". + slides_with_header = sum(1 for t in texts if "ALPHA" in t) + assert slides_with_header >= 2 + joined = " ".join(texts) + assert "Lorem ipsum dolor" in joined and "reprehenderit voluptate" in joined + # No row lost: every data cell r0..r49 col0 present. + for r in (0, 25, 49): + assert f"r{r}c0" in joined + + +def test_edge_profile_none_y_vacio_un_slide(): + with tempfile.TemporaryDirectory() as d: + for arg, name in ((None, "none"), ({}, "empty")): + out = os.path.join(d, f"{name}.pptx") + res = render_automatic_eda_pptx(arg, out, {"write_manifest": False}) + assert res["path"] == out + assert os.path.exists(out) + assert res["n_slides"] == 1 + + +def test_error_path_directorio_no_escribible_no_revienta(): + res = render_automatic_eda_pptx(_profile(), "/proc/nope/x.pptx", + {"write_manifest": False}) + assert res["path"] is None + assert res["n_slides"] == 0 + assert res["note"] diff --git a/python/pyproject.toml b/python/pyproject.toml index f75f2afa..9553fbe8 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -28,6 +28,7 @@ dependencies = [ "pypdf>=6.10.0", "pyproj>=3.7.2", "python-docx>=1.2.0", + "python-pptx>=1.0.2", "pyyaml>=6.0.3", "qrcode[pil]>=8.2", "rapidfuzz>=3.14.5", From cb7a7fc1fdaa97070a57ce45dca42e80f6710c34 Mon Sep 17 00:00:00 2001 From: Egutierrez Date: Tue, 30 Jun 2026 14:30:31 +0200 Subject: [PATCH 02/53] =?UTF-8?q?docs(eda):=20contrato=20de=20cap=C3=ADtul?= =?UTF-8?q?os=20AutomaticEDA=20+=20capability=20page?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Añade docs/automatic_eda_contract.md: documento autoritativo y autosuficiente para que otros agentes escriban capítulos en paralelo (NUM DISTR, CAT DISTR, CALIDAD, CORRELACIÓN, MODELOS, ANÁLISIS LLM, TIMESERIES, GEOSPATIAL, AGREGACIÓN). Cubre el modelo de bloques/capítulo exacto, la firma build_(profile, ctx) -> Chapter|None, la declaración de CHAPTER_VERSION, dónde colocar el módulo, cómo se registra el orden del documento, qué claves del profile consume cada capítulo, las claves nuevas que la fase de cálculo debe añadir (head_rows, columns[].examples) y un ejemplo completo del capítulo de referencia OVERVIEW. Enlaza las dos funciones nuevas y el contrato desde docs/capabilities/eda.md y actualiza el recuento del grupo eda en el índice de capabilities. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/automatic_eda_contract.md | 299 +++++++++++++++++++++++++++++++++ docs/capabilities/INDEX.md | 2 +- docs/capabilities/eda.md | 4 + 3 files changed, 304 insertions(+), 1 deletion(-) create mode 100644 docs/automatic_eda_contract.md diff --git a/docs/automatic_eda_contract.md b/docs/automatic_eda_contract.md new file mode 100644 index 00000000..63e55213 --- /dev/null +++ b/docs/automatic_eda_contract.md @@ -0,0 +1,299 @@ +# AutomaticEDA — contrato de capítulos + +Documento autoritativo para **escribir capítulos** del informe AutomaticEDA. Léelo +entero antes de añadir un capítulo: define el modelo de bloques, la firma del builder, +el versionado, dónde colocar el módulo, cómo se registra en el orden del documento, qué +claves del `profile` consume cada capítulo y un ejemplo completo de capítulo de +referencia (OVERVIEW). + +AutomaticEDA es la capa intermedia entre **contenido** (lo que un capítulo quiere +decir) y **formato de salida** (PDF móvil + PPTX para compartir). Un mismo documento por +capítulos se renderiza a los dos formatos con garantía de **no-corte**: el texto se +envuelve a líneas completas, las tablas largas se parten por filas repitiendo la +cabecera, y figuras/imágenes se escalan para caber enteras. + +- Código del motor: `python/functions/datascience/automatic_eda/` (paquete de soporte). +- Funciones públicas del registry (grupo `eda`): `render_automatic_eda_pdf`, + `render_automatic_eda_pptx`. +- Sustituye evolutivamente a `render_eda_pdf` **de forma aditiva** (ese sigue activo en + `profile_table(emit_pdf=True)`). + +--- + +## 1. Modelo de documento + +``` +Document = list[Chapter] +Chapter = { id: str, title: str, version: str, blocks: list[Block] } +Block = Heading | Markdown | KVTable | DataTable | Figure | Image | Caption | Note +``` + +Importa el modelo desde `datascience.automatic_eda.model` (o +`from datascience.automatic_eda import ...`). Todos los bloques son dataclasses; los +renderers también aceptan **dicts** con la clave `kind` (lectura defensiva: lo no +reconocido se degrada a `Note`, nunca lanza). + +### Bloques + +| Bloque | Construcción | Qué hace en el render | +|---|---|---| +| `Heading(text, level=1)` | título de sección, `level` 1 (grande) … 3 (chico) | una o varias líneas en negrita; nivel 1 lleva subrayado de acento | +| `Markdown(text)` | texto markdown ligero | ver subset abajo; **nunca corta a media línea** | +| `KVTable(rows, title=None)` | `rows = [(clave, valor), ...]` | tabla de 2 columnas etiqueta/valor; el valor se envuelve | +| `DataTable(header, rows, title=None, note=None)` | `header=[...]`, `rows=[[...],...]` | tabla con cabecera; **se parte por filas repitiendo cabecera**; las celdas largas se envuelven dentro de su columna | +| `Figure(fig=None, make=None, caption=None, height_in=None)` | una `matplotlib.figure.Figure` ya construida (`fig`) o un callable `make()->Figure` (perezoso) | se rasteriza y escala para caber entera (nunca recortada) | +| `Image(path, caption=None, height_in=None)` | ruta a PNG/JPG | se escala para caber entera | +| `Caption(text)` / `Note(text)` | texto auxiliar pequeño | pie/nota en gris; `Note` es además el fallback de lo desconocido | + +### Subset de markdown soportado (`Markdown`) + +`#`/`##`/`###` → headings; `-`/`*` → viñetas; líneas `| a | b |` consecutivas → una +`DataTable`; línea en blanco → separación de párrafo; `**bold**`/`__bold__`/`` `code` `` +→ se quitan los marcadores y se conserva el texto. Todo lo demás se renderiza tal cual. +Garantía: ningún carácter se pierde; lo que no cabe se envuelve o pasa de página/slide. + +--- + +## 2. Firma del builder de capítulo (OBLIGATORIA) + +Cada capítulo es un módulo `python/functions/datascience/automatic_eda/chapters/.py` +que expone **dos** símbolos: + +```python +CHAPTER_VERSION = "1.0.0" # semver de generación del capítulo (ver §4) + +def build_(profile: dict, ctx: dict) -> "Chapter | None": + """Construye el capítulo desde el TableProfile y el contexto de presentación. + + Devuelve None si el capítulo NO aplica a este dataset (p.ej. timeseries sin + columna fecha). Lee SIEMPRE defensivamente con .get y NUNCA lanza. + """ +``` + +- El nombre de la función es exactamente `build_` donde `` es el del módulo y + el de `CHAPTER_ORDER` (§3). Ej.: `chapters/num_distr.py` → `build_num_distr`. +- Devuelve un `model.Chapter(id, title, version=CHAPTER_VERSION, blocks=[...])` o `None`. +- Un capítulo que devuelve `None` o cuyos `blocks` quedan vacíos se omite del documento. + +--- + +## 3. Registro y orden del documento + +El orden canónico está **pre-declarado** en +`python/functions/datascience/automatic_eda/chapters_registry.py`: + +```python +CHAPTER_ORDER = [ + "portada", "overview", "num_distr", "cat_distr", "calidad", "correlacion", + "modelos", "analisis_llm", "timeseries", "geospatial", "agregacion", +] +``` + +`build_document(profile, ctx)` recorre este orden, importa perezosamente +`chapters/.py` y llama `build_`. **Para añadir un capítulo NO se edita +`chapters_registry.py`**: basta crear el módulo `chapters/.py` (con su `` ya en +`CHAPTER_ORDER`) y aparecerá automáticamente en su posición. Esto permite que muchos +agentes trabajen **en paralelo** sin contención: cada uno toca solo su archivo. + +Si tu capítulo usa un `` que aún no está en `CHAPTER_ORDER`, añádelo en la posición +correcta (única edición compartida; coordínala con el orquestador). + +`build_document` nunca lanza: un capítulo cuyo módulo no existe se salta, y uno que falla +o devuelve `None` se omite. + +--- + +## 4. Versionado por capítulo + manifiesto + +- `CHAPTER_VERSION` (semver) identifica la **generación** del capítulo. Bumpéalo cuando + cambies qué/cómo emite el capítulo (no en cada corrida). Se estampa en el pie de cada + página/slide: ` · v`. +- `ENGINE_VERSION` (en `model.py`) versiona el motor global. +- Al renderizar se escribe `automatic_eda_manifest.json` junto a la salida: + +```json +{ + "engine": "AutomaticEDA", + "engine_version": "1.0.0", + "generated_at": "2026-06-30 12:20:56 UTC", + "chapters": { + "portada": { "version": "1.0.0", "n_pages": 1, "n_slides": 1 }, + "overview": { "version": "1.0.0", "n_pages": 2, "n_slides": 2 } + } +} +``` + +Llamar a uno o ambos renderers crea/actualiza el manifiesto (read-modify-write +defensivo). Esto habilita el **seguimiento y la mejora continua por capítulo**. + +--- + +## 5. `ctx` — contexto de presentación + +`ctx` lleva metadatos que **no están** en el `TableProfile` (lo aporta el caller via +`meta['ctx']`). Claves convencionales (todas opcionales): + +| Clave | Uso | +|---|---| +| `dataset_name` | nombre del dataset (portada). Default: `profile['table']` | +| `source_origin` | de dónde viene el dataset (portada). Default: `profile['source']` | +| `storage` | tecnología de almacenamiento (portada). Default: inferido de `source` | +| `generated_at` | fecha de generación (portada/manifiesto). Default: `profiled_at`/ahora | +| `description` | frase de descripción del dataset (portada) | +| `granularity` | "Cada fila es…" (portada). Default: derivado de `key_candidates` | +| `quality_criteria` | criterios del score de calidad (portada) | +| `head_rows` | `list[dict]` con `df.head` (overview). Ver §7 | + +Un capítulo puede definir y consumir sus propias claves `ctx` — documenta cuáles en su +docstring. + +--- + +## 6. Claves del `profile` que consume cada capítulo + +El `TableProfile` lo produce `profile_table(...)["profile"]` (grupo `eda`). Claves de +nivel superior: `table, source, profiled_at, n_rows, n_cols, size_bytes, duplicate_rows, +duplicate_pct, null_cell_pct, constant_cols, all_null_cols, quality_score, +type_breakdown, key_candidates, columns[], correlations, llm, models, series, caveats`. + +Cada `columns[i]`: `name, inferred_type, semantic_type, physical_type, distinct_count, +unique_pct, null_count, null_pct, empty_count, empty_pct, flags, quality_score, +numeric{min,max,mean,median,std,variance,cv,iqr,skew,kurtosis,p1..p99,mode,n_outliers, +outlier_pct,zero_pct,negative_pct,distribution_type,histogram[{lo,hi,count}]}, +categorical{top[{value,count,pct}],mode,n_distinct,entropy,imbalance,len_min/mean/max}, +reexpression, series{...}`. + +| Capítulo | Claves del profile que consume | +|---|---| +| `portada` | `table, source, profiled_at, n_rows, n_cols, quality_score, key_candidates` + `ctx` | +| `overview` | `columns[].{name,inferred_type,semantic_type,physical_type,null_pct,null_count,categorical.top,numeric.{min,median,max,mean,std}}`, `head_rows` (ver §7) | +| `num_distr` (pendiente) | `columns[] numeric.{histogram,mean,median,std,outlier_pct,...}` | +| `cat_distr` (pendiente) | `columns[] categorical.{top,entropy,imbalance}` | +| `calidad` (pendiente) | `quality_score`, `columns[].{quality_score,flags,issues}`, `duplicate_*`, `null_cell_pct`, `constant_cols`, `all_null_cols` | +| `correlacion` (pendiente) | `correlations.pairs[{a,b,value,method}]`, `correlations.levels_caveat` | +| `modelos` (pendiente) | `models.{pca,kmeans,outliers,normality}` | +| `analisis_llm` (pendiente) | `llm` | +| `timeseries` (pendiente) | `series{col:{stationarity,acf_pacf,stl,levels_*}}` | +| `geospatial` (pendiente) | columnas con `semantic_type` geográfico (lat/lon) | +| `agregacion` (pendiente) | `columns[]` + agregados que la fase de cálculo añada | + +--- + +## 7. Claves nuevas del profile que la fase de cálculo debe añadir + +El `TableProfile` actual **no** trae estas claves; el capítulo OVERVIEW las consume y, si +faltan, degrada honestamente (placeholder + derivación de valores reales). Para un +overview completo, la fase de cálculo (otro agente) debe añadir: + +- `profile['head_rows']`: `list[dict]` con las primeras N filas (`df.head`), una por + dict `{columna: valor}`. Mientras tanto OVERVIEW muestra un placeholder. +- `columns[i]['examples']`: `list` de hasta N valores **no nulos** crudos de la columna. + Mientras tanto OVERVIEW deriva ejemplos de `categorical.top[].value` (categóricas) y de + `numeric.{min,median,max}` (numéricas) — son valores reales, no inventados. + +Sugerencia de implementación (no obligatoria en esta fase): una función del registry que +muestree `head_rows`/`examples` desde DuckDB y las inyecte en el profile antes de +renderizar (delegar a `fn-constructor`, tag `eda`). + +--- + +## 8. Ejemplo COMPLETO de capítulo de referencia (OVERVIEW) + +Copia este patrón. Archivo real: +`python/functions/datascience/automatic_eda/chapters/overview.py`. + +```python +from .. import model + +CHAPTER_VERSION = "1.0.0" +CHAPTER_ID = "overview" +CHAPTER_TITLE = "Overview" + +def _fmt_num(v, d=3): + # ... formateo defensivo (None -> "—", floats compactos) ... + ... + +def _examples_for(col: dict) -> str: + # 1) col['examples'] si existe; 2) categorical.top[].value; + # 3) numeric.{min,median,max}. Nunca celda vacía ni inventada. + ... + +def build_overview(profile: dict, ctx: dict): + profile = profile or {} + ctx = ctx or {} + cols = profile.get("columns") or [] + if not cols and not (ctx.get("head_rows") or profile.get("head_rows")): + return None # no aplica. + + blocks = [ + model.Heading(text="Primeras filas (df.head)", level=2), + _head_block(profile, ctx), # DataTable(df.head) o Note si falta head_rows. + ] + cols_block = _columns_block(profile) # DataTable: nombre/tipo/nulos/ejemplos. + if cols_block is not None: + blocks.append(model.Heading(text="Diccionario de columnas", level=2)) + blocks.append(cols_block) + desc_block = _describe_block(profile) # DataTable: mean/median/min/max/std. + if desc_block is not None: + blocks.append(model.Heading(text="Resumen estadístico numérico", level=2)) + blocks.append(desc_block) + + return model.Chapter(id=CHAPTER_ID, title=CHAPTER_TITLE, + version=CHAPTER_VERSION, blocks=blocks) +``` + +Puntos clave que todo capítulo debe respetar: + +1. **Lectura defensiva**: `profile.get(...)`, `or []`, comprobar `isinstance` — nunca + asumir que una clave existe ni lanzar. +2. **`None` si no aplica**: devuelve `None` (o `blocks` vacíos) cuando el dataset no tiene + lo que el capítulo necesita. +3. **No inventar**: si falta un dato (p.ej. `df.head`), muestra un placeholder honesto o + deriva de valores reales del perfil; deja el hueco documentado. +4. **Tablas vía `DataTable`**: deja que el renderer las parta y repita cabecera; no + pre-pagines tú. +5. **Figuras vía `Figure(make=...)`**: pásalas perezosas; las dibuja y escala el renderer. + +--- + +## 9. Cómo se prueba un capítulo + +```python +from datascience.automatic_eda import build_document, render_pdf, render_pptx +chapters = build_document(profile, ctx={"dataset_name": "..."}) +render_pdf(chapters, "reports/x.pdf", {"title": "EDA"}) +render_pptx(chapters, "reports/x.pptx", {"title": "EDA"}) +``` + +O directo desde las funciones públicas con el profile entero (construyen los capítulos): + +```python +from datascience import render_automatic_eda_pdf, render_automatic_eda_pptx +render_automatic_eda_pdf(profile, "reports/x.pdf", {"ctx": {...}}) +render_automatic_eda_pptx(profile, "reports/x.pptx", {"ctx": {...}}) +``` + +Añade un test self-contained por capítulo (perfil sintético, sin DuckDB) que verifique +sus bloques presentes y el no-corte (texto largo intacto en la salida). Patrón: +`render_automatic_eda_pdf_test.py`. + +--- + +## 10. Integración futura con `profile_table` (siguiente fase) + +`profile_table(emit_pdf=True)` usa hoy `render_eda_pdf` (intacto). En la siguiente fase +se añadirá `emit_automatic=True` (o se migrará `emit_pdf`) para que cada EDA emita +**siempre** PDF + PPTX del motor AutomaticEDA desde el mismo profile: + +```python +# Bosquejo de la integración aditiva (NO activar si rompe los tests actuales): +if emit_automatic: + ctx = {"dataset_name": table, "source_origin": db_path, ...} + render_automatic_eda_pdf(prof, os.path.join(report_dir, f"aeda_{table}_{ts}.pdf"), + {"title": f"EDA — {table}", "ctx": ctx}) + render_automatic_eda_pptx(prof, os.path.join(report_dir, f"aeda_{table}_{ts}.pptx"), + {"title": f"EDA — {table}", "ctx": ctx}) +``` + +Hasta entonces los renderers se invocan directamente sobre el `profile` que +`profile_table` ya devuelve. diff --git a/docs/capabilities/INDEX.md b/docs/capabilities/INDEX.md index 98e0141e..dbea6af4 100644 --- a/docs/capabilities/INDEX.md +++ b/docs/capabilities/INDEX.md @@ -68,7 +68,7 @@ Indice de grupos de capacidades del registry. Cada grupo agrupa >=3 funciones qu | [consent](consent.md) | 3 | CMP / IAB TCF / data brokers: detectar el CMP de un sitio (Didomi/OneTrust/Sourcepoint/Quantcast), leer `__tcfapi` para contar vendors y propositos, aceptar el banner (selectores + fallback LLM con haiku que localiza Aceptar/Ver socios), y descargar la GVL de IAB para nominar cada broker y que datos recopila. Nacio de `projects/databrokers/` | | [onlyoffice](onlyoffice.md) | 3 | Operar ONLYOFFICE Desktop Editors (binario onlyoffice-desktopeditors) en Linux/X11 desde terminal via instancia aislada (slot HOME=/tmp/oo_): abrir un archivo en ventana propia, cerrar+reabrir para mostrar datos editados en disco (no hay reload nativo, Issue #2313), y matar el proceso del slot. Solo gestiona la ventana, NO edita ni crea archivos. Requiere X11 + wmctrl + xdotool. No confundir con el Document Server (web/Docker) | | [email](email.md) | 21 | Gestionar cuentas de correo por IMAP+SMTP directo (Python stdlib, sin browser ni MCP Gmail): conectar/listar/buscar/leer (imap_*), mutar estado (mark_seen/move/delete/save_draft) por UID, y construir+enviar (email_build_html/smtp_send). Auth user+app-password (NO OAuth; Outlook fuera). Credenciales desde pass, resueltas por la capa app. Complementa al browser (interactivo) — no lo reemplaza | -| [eda](eda.md) | 27 | Exploratory Data Analysis por tabla y base con motor DuckDB + PostgreSQL push-down: perfil base SQL (SUMMARIZE + distinct exacto), estadística numérica/categórica, tipo semántico regex, calidad, correlación/asociación (Pearson/Spearman/Cramér's V/Theil's U/η/MI), relaciones inter-tabla (FK containment + join graph mermaid), modelos baratos (PCA/KMeans/IsolationForest/normalidad/tendencia), capa LLM (dictionary/PII/limpieza/análisis) y generación de notebook. Orquestadores `profile_table` (backend duckdb/postgres, flags run_models/run_llm) y `profile_database` | +| [eda](eda.md) | 29 | Exploratory Data Analysis por tabla y base con motor DuckDB + PostgreSQL push-down: perfil base SQL (SUMMARIZE + distinct exacto), estadística numérica/categórica, tipo semántico regex, calidad, correlación/asociación (Pearson/Spearman/Cramér's V/Theil's U/η/MI), relaciones inter-tabla (FK containment + join graph mermaid), modelos baratos (PCA/KMeans/IsolationForest/normalidad/tendencia), capa LLM (dictionary/PII/limpieza/análisis) y generación de notebook. Orquestadores `profile_table` (backend duckdb/postgres, flags run_models/run_llm) y `profile_database` | | [seo](seo.md) | 3 | SEO orientado a datos sobre Google Search Console: autenticar con service account (`gsc_auth`), extraer Search Analytics paginado (`pull_gsc_search_analytics`) y el pipeline de ingesta a DuckDB + espejo Postgres para Metabase (`ingest_gsc_search_analytics`). Cadena de ingesta del proyecto `seo_analytics`; alimenta dashboards de striking distance, CTR opportunities y content decay | | [local-hub](local-hub.md) | 4 | Exponer los procesos locales como subdominios `*.localhost` (via Caddy, sin DNS) y reunirlos en una pantalla principal Glance con estado en vivo, refrescada a diario por dag_engine. Descubre servicios (manifiesto + registry), renderiza Caddyfile + config Glance (puras), y el pipeline `refresh_local_hub` regenera+recarga. Fuente de verdad: `apps/local_hub/local_services.yaml` | | [comfyui-judge](comfyui-judge.md) | 4 | Panel multi-juez de calidad de imagen: estético LAION-V2 (`comfyui_score_aesthetic`, 0-10) + fidelidad CLIP prompt↔imagen (`comfyui_score_clip_alignment`, 0-1) + crítica LLM-vision (`comfyui_critique_image_llm`, good/bad). Agregados por voto mayoría en `comfyui_judge_image`. Gate objetivo para tests/DoD y el bucle de mejora de skills ComfyUI; degrada con gracia si un juez cae. Jueces estético/fidelidad por subproceso al venv ComfyUI (torch+open_clip), crítico via claude-direct | diff --git a/docs/capabilities/eda.md b/docs/capabilities/eda.md index 946fec5f..d569acd6 100644 --- a/docs/capabilities/eda.md +++ b/docs/capabilities/eda.md @@ -71,6 +71,10 @@ Orquestadores one-shot: | `eda_llm_insights_py_datascience` | impure | 1 call LLM sobre el perfil agregado (no filas crudas): data dictionary, resumen, granularidad de fila, PII/RGPD, limpieza, análisis sugeridos. | | `build_eda_notebook_py_datascience` | impure | Genera un `.ipynb` (nbformat v4) que perfila la tabla, listo para lanzar en Jupyter colaborativo. | | `render_eda_pdf_py_datascience` | impure | Renderiza el `TableProfile` a un PDF multipágina **vertical (A5), legible en móvil** (estilo Tufte: histogramas como small multiples, top-k, heatmap de asociación). 4ª salida del workflow, junto a JSON/Markdown/notebook. | +| `render_automatic_eda_pdf_py_datascience` | impure | Motor **AutomaticEDA**: documento por CAPÍTULOS (modelo de bloques independiente del formato) → PDF A5 móvil que **nunca corta** texto/tablas/imágenes (tablas largas se parten repitiendo cabecera) + manifiesto versionado por capítulo. Acepta el `TableProfile` o capítulos del modelo. Aditivo, no reemplaza `render_eda_pdf`. | +| `render_automatic_eda_pptx_py_datascience` | impure | Motor **AutomaticEDA** → PPTX 16:9 para **compartir** desde el mismo documento por capítulos; mismo principio anti-corte (continúa en slide `(cont.)`). Motor `python-pptx`. | + +> **AutomaticEDA** (núcleo nuevo, fase de capítulos): separa contenido (capítulos/bloques) de formato (PDF móvil + PPTX). Para escribir un capítulo nuevo (NUM DISTR, CAT DISTR, CALIDAD, CORRELACIÓN, MODELOS, ANÁLISIS LLM, TIMESERIES, GEOSPATIAL, AGREGACIÓN) lee el contrato: **`docs/automatic_eda_contract.md`**. Código del motor en `python/functions/datascience/automatic_eda/`; capítulos de referencia: `portada`, `overview`. ### Orquestadores (pipelines) | ID | Qué hace | From a3f75d61ec6f12729a85221f3deae2067603e45a Mon Sep 17 00:00:00 2001 From: Egutierrez Date: Tue, 30 Jun 2026 14:43:51 +0200 Subject: [PATCH 03/53] chore: avance acumulado de sesiones previas (reorg dev/issues + ajustes) Reorganizacion de dev/issues en subcarpetas (completed/, cpp/, gamedev/, kanban/, trading/, imagegen/, matrix/) y cambios acumulados en cmd/fn/pyrunner, .claude/commands y settings. Trabajo de otro LLM/sesion, commiteado a peticion del usuario para desbloquear el working tree. Excluido logs/ardour_mcp_server.log (ruido). --- .claude/commands/issue.md | 5 +- .claude/settings.local.json | 4 +- cmd/fn/pyrunner.go | 84 ++++++----- cmd/fn/pyrunner_test.go | 141 ++++++++++++++++++ .../0051-extraction-pipeline-followups.md | 2 +- ...4-deploy-server-registry-first-refactor.md | 2 +- ...0055-docker-tui-registry-first-refactor.md | 2 +- .../0056-audit-python-nested-imports.md | 2 +- dev/issues/0057-audit-go-symbol-naming.md | 2 +- .../0060-fn-doctor-secrets-subcommand.md | 2 +- .../0061-notify-telegram-integration.md | 2 +- dev/issues/0071a-extract-claude-chat-panel.md | 3 +- dev/issues/0071b-extract-jobs-queue-panel.md | 3 +- .../0071f-extract-subprocess-streamer.md | 2 +- dev/issues/0071g-extract-app-db-init.md | 2 +- ...-gradle_run-no-detecta-android-sdk-path.md | 2 +- dev/issues/0077-fn-run-bash-output-mudo.md | 2 +- .../0100-issues-frontmatter-migration.md | 2 +- .../0104-fix-issue-type-aware-dispatch.md | 2 +- dev/issues/0107e-version-pinning-codegen.md | 2 +- .../0109k-skill-tree-dashboard-panel.md | 7 +- dev/issues/0109m-issues-api-service.md | 8 +- dev/issues/0121-e2e-checks-coverage-masa.md | 2 +- dev/issues/0121a-design-e2e-batch.md | 2 +- dev/issues/0121c-apply-e2e-proposals.md | 4 +- dev/issues/0122-fn-revisor-auto-apply.md | 2 +- dev/issues/0124-dag-engine-cleanup.md | 2 +- dev/issues/0125-deploy-server-db-flag.md | 2 +- dev/issues/0128-kanban-files-attachments.md | 2 +- dev/issues/0130a-kanban-cpp-v2-parser.md | 7 +- dev/issues/0130b-kanban-cpp-v2-backend.md | 3 +- dev/issues/0134-mesh-protocol-spec.md | 4 +- dev/issues/0144-agent-per-machine-llm.md | 4 +- .../0146-add-pc-oneshot-mesh-scaling.md | 4 +- dev/issues/0147-matrix-client-pc-scaffold.md | 4 +- .../0148-matrix-client-pc-rooms-timeline.md | 4 +- dev/issues/0149-matrix-client-pc-composer.md | 4 +- dev/issues/0150-matrix-client-pc-e2ee.md | 4 +- .../0151-matrix-client-pc-livekit-calls.md | 4 +- .../0152-matrix-client-pc-mini-webapps.md | 4 +- .../0154-matrix-client-android-scaffold.md | 4 +- ...55-matrix-client-android-rooms-timeline.md | 4 +- .../0156-matrix-client-android-composer.md | 4 +- dev/issues/0157-matrix-client-android-e2ee.md | 4 +- ...158-matrix-client-android-livekit-calls.md | 4 +- .../0159-matrix-client-android-push-fcm.md | 4 +- ...0160-matrix-client-android-mini-webapps.md | 4 +- ...atrix-client-android-foreground-service.md | 4 +- .../0162-matrix-enable-mas-delegated-auth.md | 4 +- dev/issues/0163-matrix-custom-admin-panel.md | 4 +- .../0179-dev-console-recursive-issue-scan.md | 45 ++++++ dev/issues/0180-ausente-issues-queue-dag.md | 59 ++++++++ .../0059-nested-app-md-tracking.md | 2 +- .../0067-odr-osint-prereqs-roadmap.md | 2 +- .../0068-e2e-validation-loop-fn4-fn5.md | 0 ...onomous-agent-loop-self-iterating-tasks.md | 0 .../0072l-gamedev-scripting-optional.md | 0 ...call-standardization-and-usage-tracking.md | 0 ...ude-md-delegation-and-capability-groups.md | 0 .../0087-capability-discovery-acceleration.md | 2 +- .../0096-standardize-app-locations.md | 2 +- .../0101-dev-console-binary.md | 2 +- .../0103-taxonomy-and-slash-commands.md | 2 +- ...105-service-frontmatter-standardization.md | 2 +- .../0109g-skill-tree-embedded-terminal.md | 2 +- .../completed/0114-dod-evidence-schema.md | 2 +- .../0128-agents-and-robots-http-api-sse.md | 2 +- ...0153-matrix-client-pc-agent-integration.md | 2 +- .../0164-agents-cryptohelper-init-hang.md | 2 +- .../0166-matrix-livekit-turn-deploy.md | 2 +- ...71-project-subrepo-manifest-and-reclone.md | 2 +- ...173-eda-correctitud-estadistica-critica.md | 2 +- ...4-eda-series-periodo-estacional-niveles.md | 2 +- .../0175-eda-relational-fk-inference-views.md | 2 +- ...76-eda-render-models-series-caveats-pdf.md | 2 +- ...77-eda-tipos-id-secuencial-eta-cuadrado.md | 2 +- .../{ => cpp}/0027-cpp-gl-compute-pingpong.md | 0 .../{ => cpp}/0030-cpp-audio-reactive.md | 0 .../{ => cpp}/0033-cpp-http-ws-inspector.md | 2 +- dev/issues/{ => cpp}/0035-cpp-map-tiles.md | 0 .../{ => cpp}/0036-cpp-image-canvas-webcam.md | 0 ...071-extract-reusable-cpp-panels-roadmap.md | 0 .../0072a-gamedev-smoke-sdl3-sokol-imgui.md | 5 +- .../{ => cpp}/0082-compile-sd-cpp-binary.md | 2 +- .../0095-dag-engine-cpp-imgui-frontend.md | 0 .../{ => cpp}/0110-cpp-http-client-helper.md | 2 +- dev/issues/{ => cpp}/0130-kanban-cpp-v2.md | 0 .../{ => cpp}/0130c-kanban-cpp-v2-frontend.md | 3 +- .../{ => cpp}/0131-cpp-module-chat-panel.md | 0 .../0132-cpp-module-terminal-panel.md | 2 +- .../{ => cpp}/0133-cpp-data-table-10m-rows.md | 0 .../0072-gamedev-stack-roadmap.md | 0 .../0072b-gamedev-runtime-core.md | 3 +- .../0072c-gamedev-asset-pipeline.md | 3 +- .../0072d-gamedev-wasm-build-size-budget.md | 4 +- .../0072e-gamedev-crypto-bridge-web3.md | 4 +- ...-gamedev-crypto-onchain-assets-payments.md | 3 +- .../0072g-gamedev-android-build.md | 4 +- .../{ => gamedev}/0072h-gamedev-ios-build.md | 4 +- .../{ => gamedev}/0072i-gamedev-editor-app.md | 4 +- .../0072j-gamedev-physics-box2d.md | 8 +- .../0072k-gamedev-demo-platformer.md | 6 +- .../0083-imagegen-spike02-cross-validation.md | 2 +- .../0084-imagegen-studio-go-app.md | 0 .../0058-kanban-uses-functions-sync.md | 2 +- .../{ => kanban}/0063-kanban-stickers.md | 0 .../0089-kanban-column-max-time.md | 2 +- .../0090-kanban-column-random-pick.md | 2 +- .../0091-kanban-sidebar-drag-zones.md | 2 +- .../{ => kanban}/0092-kanban-done-archive.md | 2 +- .../{ => kanban}/0093-kanban-daily-report.md | 2 +- .../0094-kanban-daily-summary-pdf.md | 0 .../0178-kanban-requester-empty-nav.md} | 4 +- .../0165-matrix-media-store-luks.md | 0 .../0088-trading-skill-management-roadmap.md | 0 .../0088a-trading-project-scaffolding.md | 2 +- ...8b-trading-market-data-capability-group.md | 2 +- ...ding-broker-interface-and-paper-adapter.md | 2 +- .../0088d-trading-portfolio-tracker-app.md | 2 +- ...0088e-trading-strategy-capability-group.md | 2 +- ...g-risk-capability-group-and-kill-switch.md | 2 +- .../0088g-trading-backtester-app.md | 2 +- .../0088h-trading-live-runner-service.md | 2 +- .../0088i-trading-journal-app.md | 2 +- .../0088j-trading-reactive-loop-wiring.md | 2 +- 125 files changed, 421 insertions(+), 203 deletions(-) create mode 100644 cmd/fn/pyrunner_test.go create mode 100644 dev/issues/0179-dev-console-recursive-issue-scan.md create mode 100644 dev/issues/0180-ausente-issues-queue-dag.md rename dev/issues/{ => completed}/0059-nested-app-md-tracking.md (99%) rename dev/issues/{ => completed}/0067-odr-osint-prereqs-roadmap.md (99%) rename dev/issues/{ => completed}/0068-e2e-validation-loop-fn4-fn5.md (100%) rename dev/issues/{ => completed}/0069-autonomous-agent-loop-self-iterating-tasks.md (100%) rename dev/issues/{ => completed}/0072l-gamedev-scripting-optional.md (100%) rename dev/issues/{ => completed}/0085-registry-call-standardization-and-usage-tracking.md (100%) rename dev/issues/{ => completed}/0086-claude-md-delegation-and-capability-groups.md (100%) rename dev/issues/{ => completed}/0087-capability-discovery-acceleration.md (99%) rename dev/issues/{ => completed}/0096-standardize-app-locations.md (99%) rename dev/issues/{ => completed}/0101-dev-console-binary.md (99%) rename dev/issues/{ => completed}/0103-taxonomy-and-slash-commands.md (99%) rename dev/issues/{ => completed}/0105-service-frontmatter-standardization.md (99%) rename dev/issues/{ => completed}/0109g-skill-tree-embedded-terminal.md (99%) rename dev/issues/{ => completed}/0153-matrix-client-pc-agent-integration.md (99%) rename dev/issues/{ => completed}/0164-agents-cryptohelper-init-hang.md (99%) rename dev/issues/{ => completed}/0171-project-subrepo-manifest-and-reclone.md (99%) rename dev/issues/{ => completed}/0173-eda-correctitud-estadistica-critica.md (99%) rename dev/issues/{ => completed}/0174-eda-series-periodo-estacional-niveles.md (99%) rename dev/issues/{ => completed}/0175-eda-relational-fk-inference-views.md (99%) rename dev/issues/{ => completed}/0176-eda-render-models-series-caveats-pdf.md (99%) rename dev/issues/{ => completed}/0177-eda-tipos-id-secuencial-eta-cuadrado.md (99%) rename dev/issues/{ => cpp}/0027-cpp-gl-compute-pingpong.md (100%) rename dev/issues/{ => cpp}/0030-cpp-audio-reactive.md (100%) rename dev/issues/{ => cpp}/0033-cpp-http-ws-inspector.md (99%) rename dev/issues/{ => cpp}/0035-cpp-map-tiles.md (100%) rename dev/issues/{ => cpp}/0036-cpp-image-canvas-webcam.md (100%) rename dev/issues/{ => cpp}/0071-extract-reusable-cpp-panels-roadmap.md (100%) rename dev/issues/{ => cpp}/0072a-gamedev-smoke-sdl3-sokol-imgui.md (99%) rename dev/issues/{ => cpp}/0082-compile-sd-cpp-binary.md (99%) rename dev/issues/{ => cpp}/0095-dag-engine-cpp-imgui-frontend.md (100%) rename dev/issues/{ => cpp}/0110-cpp-http-client-helper.md (98%) rename dev/issues/{ => cpp}/0130-kanban-cpp-v2.md (100%) rename dev/issues/{ => cpp}/0130c-kanban-cpp-v2-frontend.md (99%) rename dev/issues/{ => cpp}/0131-cpp-module-chat-panel.md (100%) rename dev/issues/{ => cpp}/0132-cpp-module-terminal-panel.md (98%) rename dev/issues/{ => cpp}/0133-cpp-data-table-10m-rows.md (100%) rename dev/issues/{ => gamedev}/0072-gamedev-stack-roadmap.md (100%) rename dev/issues/{ => gamedev}/0072b-gamedev-runtime-core.md (99%) rename dev/issues/{ => gamedev}/0072c-gamedev-asset-pipeline.md (99%) rename dev/issues/{ => gamedev}/0072d-gamedev-wasm-build-size-budget.md (99%) rename dev/issues/{ => gamedev}/0072e-gamedev-crypto-bridge-web3.md (99%) rename dev/issues/{ => gamedev}/0072f-gamedev-crypto-onchain-assets-payments.md (99%) rename dev/issues/{ => gamedev}/0072g-gamedev-android-build.md (99%) rename dev/issues/{ => gamedev}/0072h-gamedev-ios-build.md (99%) rename dev/issues/{ => gamedev}/0072i-gamedev-editor-app.md (99%) rename dev/issues/{ => gamedev}/0072j-gamedev-physics-box2d.md (99%) rename dev/issues/{ => gamedev}/0072k-gamedev-demo-platformer.md (99%) rename dev/issues/{ => imagegen}/0083-imagegen-spike02-cross-validation.md (99%) rename dev/issues/{ => imagegen}/0084-imagegen-studio-go-app.md (100%) rename dev/issues/{ => kanban}/0058-kanban-uses-functions-sync.md (99%) rename dev/issues/{ => kanban}/0063-kanban-stickers.md (100%) rename dev/issues/{ => kanban}/0089-kanban-column-max-time.md (99%) rename dev/issues/{ => kanban}/0090-kanban-column-random-pick.md (99%) rename dev/issues/{ => kanban}/0091-kanban-sidebar-drag-zones.md (99%) rename dev/issues/{ => kanban}/0092-kanban-done-archive.md (99%) rename dev/issues/{ => kanban}/0093-kanban-daily-report.md (99%) rename dev/issues/{ => kanban}/0094-kanban-daily-summary-pdf.md (100%) rename dev/issues/{0088-kanban-requester-empty-nav.md => kanban/0178-kanban-requester-empty-nav.md} (98%) rename dev/issues/{ => matrix}/0165-matrix-media-store-luks.md (100%) rename dev/issues/{ => trading}/0088-trading-skill-management-roadmap.md (100%) rename dev/issues/{ => trading}/0088a-trading-project-scaffolding.md (98%) rename dev/issues/{ => trading}/0088b-trading-market-data-capability-group.md (99%) rename dev/issues/{ => trading}/0088c-trading-broker-interface-and-paper-adapter.md (98%) rename dev/issues/{ => trading}/0088d-trading-portfolio-tracker-app.md (99%) rename dev/issues/{ => trading}/0088e-trading-strategy-capability-group.md (99%) rename dev/issues/{ => trading}/0088f-trading-risk-capability-group-and-kill-switch.md (99%) rename dev/issues/{ => trading}/0088g-trading-backtester-app.md (97%) rename dev/issues/{ => trading}/0088h-trading-live-runner-service.md (97%) rename dev/issues/{ => trading}/0088i-trading-journal-app.md (98%) rename dev/issues/{ => trading}/0088j-trading-reactive-loop-wiring.md (98%) diff --git a/.claude/commands/issue.md b/.claude/commands/issue.md index 5aa0a386..b6f182b8 100644 --- a/.claude/commands/issue.md +++ b/.claude/commands/issue.md @@ -31,12 +31,13 @@ Diferencia con `dev/flows/`: **Fase 1 (manual via Claude):** -El agente lee `dev/issues/*.md`, parsea frontmatter YAML con `yaml.safe_load`, aplica el filtro, imprime tabla. +El agente lee `dev/issues/**/*.md` (recursivo: incluye subcarpetas por dominio como `dev/issues/kanban/`, `dev/issues/cpp/`, ... excluyendo `completed/`), parsea frontmatter YAML con `yaml.safe_load`, aplica el filtro, imprime tabla. ```python import yaml, pathlib, re issues = [] -for f in pathlib.Path("dev/issues").glob("*.md"): +for f in pathlib.Path("dev/issues").glob("**/*.md"): + if f.parent.name == "completed": continue if f.name in {"README.md", "template.md"}: continue txt = f.read_text() m = re.match(r"^---\n(.*?)\n---", txt, re.S) diff --git a/.claude/settings.local.json b/.claude/settings.local.json index 325a41ee..aed36146 100644 --- a/.claude/settings.local.json +++ b/.claude/settings.local.json @@ -9,7 +9,9 @@ "enabledMcpjsonServers": [ "registry", "jupyter", - "orchestrator" + "orchestrator", + "godot", + "ardour" ], "hooks": { "PreToolUse": [ diff --git a/cmd/fn/pyrunner.go b/cmd/fn/pyrunner.go index c0cdda06..ab43e046 100644 --- a/cmd/fn/pyrunner.go +++ b/cmd/fn/pyrunner.go @@ -18,6 +18,7 @@ type pyParam struct { Default string // empty if required IsKwargs bool // **kwargs IsRegistry bool // type is a registry type (needs factory) + KwOnly bool // declared after a bare "*" or "*args" — must be passed by keyword } // pyFactory links a registry type to the function that creates it. @@ -45,12 +46,21 @@ func parsePySignature(sig string) []pyParam { // Split by comma, respecting nested brackets parts := splitParams(raw) var params []pyParam + kwOnly := false for _, part := range parts { part = strings.TrimSpace(part) if part == "" || part == "self" || part == "cls" { continue } + // A bare "*" (PEP 3102) or "*args" var-positional marks the start of + // keyword-only params. Neither maps cleanly to positional CLI args, so + // skip the marker itself and flag every following param as keyword-only. + if part == "*" || (strings.HasPrefix(part, "*") && !strings.HasPrefix(part, "**")) { + kwOnly = true + continue + } p := parseSingleParam(part) + p.KwOnly = kwOnly params = append(params, p) } return params @@ -189,11 +199,19 @@ func generatePyRunner(fn *registry.Function, db *registry.DB, registryRoot strin // Classify params var factoryImports []string // import lines for factories var factorySetup []string // code to create factory objects - var argLines []string // code to parse CLI args - var callArgs []string // arguments to pass to the function + var bodyLines []string // code that fills _call_args / _call_kwargs cliArgIdx := 0 + // emitCall appends one param to _call_args (positional) or _call_kwargs + // (keyword-only). indent prefixes the line (for params read inside an `if`). + emitCall := func(p pyParam, indent string) string { + if p.KwOnly { + return fmt.Sprintf("%s_call_kwargs[%q] = %s", indent, p.Name, p.Name) + } + return fmt.Sprintf("%s_call_args.append(%s)", indent, p.Name) + } + for _, p := range params { if p.IsKwargs { // Skip **kwargs for now — can't auto-resolve from CLI @@ -235,27 +253,35 @@ func generatePyRunner(fn *registry.Function, db *registry.DB, registryRoot strin fmt.Sprintf("%s = %s(%s)", p.Name, factory.FuncName, strings.Join(factoryArgs, ", "))) - callArgs = append(callArgs, p.Name) + // Factory objects are always present (required). + bodyLines = append(bodyLines, emitCall(p, "")) } else { - // Primitive type — from CLI args + // Primitive type — from CLI args. if p.Default != "" { - // Optional param with default - argLines = append(argLines, - fmt.Sprintf("%s = _args[%d] if len(_args) > %d else %s", - p.Name, cliArgIdx, cliArgIdx, convertDefault(p.Type, p.Default))) - argLines = append(argLines, - convertArg(p.Name, p.Type, true)) + // Optional: only pass when the CLI arg is present. When absent we + // DON'T replicate the signature default (it may reference a module + // constant that doesn't exist in this runner) — we simply omit the + // argument so the function applies its own native default. + bodyLines = append(bodyLines, + fmt.Sprintf("if len(_args) > %d:", cliArgIdx)) + bodyLines = append(bodyLines, + fmt.Sprintf(" %s = _args[%d]", p.Name, cliArgIdx)) + if conv := convertArg(p.Name, p.Type, true); conv != "" { + bodyLines = append(bodyLines, " "+conv) + } + bodyLines = append(bodyLines, emitCall(p, " ")) } else { - // Required param - argLines = append(argLines, + // Required param. + bodyLines = append(bodyLines, fmt.Sprintf("if len(_args) <= %d: sys.exit('error: missing required arg: %s (%s)')", cliArgIdx, p.Name, p.Type)) - argLines = append(argLines, + bodyLines = append(bodyLines, fmt.Sprintf("%s = _args[%d]", p.Name, cliArgIdx)) - argLines = append(argLines, - convertArg(p.Name, p.Type, false)) + if conv := convertArg(p.Name, p.Type, false); conv != "" { + bodyLines = append(bodyLines, conv) + } + bodyLines = append(bodyLines, emitCall(p, "")) } - callArgs = append(callArgs, p.Name) cliArgIdx++ } } @@ -289,18 +315,18 @@ func generatePyRunner(fn *registry.Function, db *registry.DB, registryRoot strin sb.WriteString("\n") } - // Arg parsing - if len(argLines) > 0 { - sb.WriteString("# --- parse CLI args ---\n") - for _, line := range argLines { - sb.WriteString(line + "\n") - } - sb.WriteString("\n") + // Arg parsing — build the positional/keyword argument collections. + sb.WriteString("# --- parse CLI args ---\n") + sb.WriteString("_call_args = []\n") + sb.WriteString("_call_kwargs = {}\n") + for _, line := range bodyLines { + sb.WriteString(line + "\n") } + sb.WriteString("\n") // Call sb.WriteString("# --- execute ---\n") - sb.WriteString(fmt.Sprintf("_result = %s(%s)\n", fn.Name, strings.Join(callArgs, ", "))) + sb.WriteString(fmt.Sprintf("_result = %s(*_call_args, **_call_kwargs)\n", fn.Name)) sb.WriteString("\n") // Output @@ -365,16 +391,6 @@ func convertArg(name, typ string, _ bool) string { } } -// convertDefault ensures the default value is valid Python for the given type. -func convertDefault(_, def string) string { - // Most defaults from the signature are already valid Python - // Just handle the None case for Optional types - if def == "None" || def == "" { - return "None" - } - return def -} - // pythonList creates a Python list literal from strings: ["a", "b", "c"] func pythonList(items []string) string { quoted := make([]string, len(items)) diff --git a/cmd/fn/pyrunner_test.go b/cmd/fn/pyrunner_test.go new file mode 100644 index 00000000..b90e7234 --- /dev/null +++ b/cmd/fn/pyrunner_test.go @@ -0,0 +1,141 @@ +package main + +import ( + "os" + "os/exec" + "strings" + "testing" + + "fn-registry/registry" +) + +// Signature with a bare "*" (PEP 3102) separating positional from keyword-only +// params. This is the shape that used to make fn run emit "* = _args[3]". +const kwOnlySig = "def add_event_dav(summary: str, start: str, end: str = '', *, location: str = '', all_day: bool = False) -> dict" + +func TestParsePySignatureBareStarKeywordOnly(t *testing.T) { + params := parsePySignature(kwOnlySig) + + // The bare "*" marker must never surface as a real parameter. + for _, p := range params { + if p.Name == "*" { + t.Fatalf("bare '*' leaked as a param: %+v", params) + } + } + + want := map[string]bool{ // name -> expected KwOnly + "summary": false, + "start": false, + "end": false, + "location": true, + "all_day": true, + } + if len(params) != len(want) { + t.Fatalf("got %d params, want %d: %+v", len(params), len(want), params) + } + for _, p := range params { + kw, ok := want[p.Name] + if !ok { + t.Errorf("unexpected param %q", p.Name) + continue + } + if p.KwOnly != kw { + t.Errorf("param %q KwOnly=%v, want %v", p.Name, p.KwOnly, kw) + } + } +} + +func TestGeneratePyRunnerKeywordOnlyValid(t *testing.T) { + fn := ®istry.Function{ + Name: "add_event_dav", + Lang: "py", + FilePath: "python/functions/pipelines/add_event_dav.py", + Signature: kwOnlySig, + } + + // All params are primitive, so no factory lookup happens and db is unused. + script, err := generatePyRunner(fn, nil, "") + if err != nil { + t.Fatalf("generatePyRunner: %v", err) + } + + if strings.Contains(script, "* = _args") { + t.Fatalf("runner emitted invalid syntax '* = _args':\n%s", script) + } + + // The signature default DEFAULT_BASE_URL (a module constant) must NOT be + // replicated into the runner — that NameErrors at runtime. + if strings.Contains(script, "DEFAULT_BASE_URL") { + t.Errorf("runner replicated non-literal default DEFAULT_BASE_URL:\n%s", script) + } + + // Required positionals are appended; keyword-only optionals go to kwargs. + for _, want := range []string{ + "_call_args.append(summary)", + "_call_args.append(start)", + `_call_kwargs["location"] = location`, + `_call_kwargs["all_day"] = all_day`, + "_result = add_event_dav(*_call_args, **_call_kwargs)", + } { + if !strings.Contains(script, want) { + t.Errorf("missing %q in generated runner:\n%s", want, script) + } + } + + // The generated runner must itself be valid Python (compile, don't run). + mustCompilePython(t, script) +} + +// mustCompilePython checks the script parses as valid Python via py_compile. +func mustCompilePython(t *testing.T, script string) { + t.Helper() + f, err := os.CreateTemp(t.TempDir(), "runner_*.py") + if err != nil { + t.Fatalf("temp file: %v", err) + } + if _, err := f.WriteString(script); err != nil { + t.Fatalf("write: %v", err) + } + f.Close() + py := pythonBinForTest() + out, err := exec.Command(py, "-m", "py_compile", f.Name()).CombinedOutput() + if err != nil { + t.Fatalf("generated runner is not valid Python (%s): %v\n%s", py, err, out) + } +} + +// pythonBinForTest prefers the project venv, falling back to python3 on PATH. +func pythonBinForTest() string { + for _, c := range []string{"../../python/.venv/bin/python3", "python3"} { + if c == "python3" { + return c + } + if _, err := os.Stat(c); err == nil { + return c + } + } + return "python3" +} + +// A "*args" var-positional marker must behave like the bare "*": skipped, and +// everything after it treated as keyword-only. +func TestParsePySignatureVarargsKeywordOnly(t *testing.T) { + sig := "def f(a: str, *args, b: int = 0) -> dict" + params := parsePySignature(sig) + + for _, p := range params { + if strings.HasPrefix(p.Name, "*") { + t.Fatalf("'*args' marker leaked as a param: %+v", params) + } + } + if len(params) != 2 { + t.Fatalf("got %d params, want 2: %+v", len(params), params) + } + got := map[string]bool{} + for _, p := range params { + got[p.Name] = p.KwOnly + } + if got["a"] != false || got["b"] != true { + t.Errorf("KwOnly mismatch: a=%v (want false), b=%v (want true)", got["a"], got["b"]) + } +} diff --git a/dev/issues/0051-extraction-pipeline-followups.md b/dev/issues/0051-extraction-pipeline-followups.md index d0fc03be..71565878 100644 --- a/dev/issues/0051-extraction-pipeline-followups.md +++ b/dev/issues/0051-extraction-pipeline-followups.md @@ -11,7 +11,7 @@ blocks: [] related: [] created: 2026-05-17 updated: 2026-05-17 -tags: [] +tags: [ausente-ready] --- # 0051 — Funciones pendientes del pipeline de extraccion (NER+RE+OpenIE) diff --git a/dev/issues/0054-deploy-server-registry-first-refactor.md b/dev/issues/0054-deploy-server-registry-first-refactor.md index 43af6c71..480734f7 100644 --- a/dev/issues/0054-deploy-server-registry-first-refactor.md +++ b/dev/issues/0054-deploy-server-registry-first-refactor.md @@ -13,7 +13,7 @@ blocks: [] related: [] created: 2026-05-17 updated: 2026-05-17 -tags: [] +tags: [ausente-ready] --- # 0054 — deploy_server: refactor registry-first (SSH/systemd/rsync/health/docker-compose) diff --git a/dev/issues/0055-docker-tui-registry-first-refactor.md b/dev/issues/0055-docker-tui-registry-first-refactor.md index b1d4bf5d..510b3570 100644 --- a/dev/issues/0055-docker-tui-registry-first-refactor.md +++ b/dev/issues/0055-docker-tui-registry-first-refactor.md @@ -12,7 +12,7 @@ blocks: [] related: [] created: 2026-05-17 updated: 2026-05-17 -tags: [] +tags: [ausente-ready] --- # 0055 — docker_tui: refactor para usar funciones docker_* del registry diff --git a/dev/issues/0056-audit-python-nested-imports.md b/dev/issues/0056-audit-python-nested-imports.md index bedfdb97..beba18ee 100644 --- a/dev/issues/0056-audit-python-nested-imports.md +++ b/dev/issues/0056-audit-python-nested-imports.md @@ -12,7 +12,7 @@ blocks: [] related: [] created: 2026-05-17 updated: 2026-05-17 -tags: [] +tags: [ausente-ready] --- # 0056 — audit_uses_functions: detectar imports Python anidados (`from pkg.subpkg import X`) diff --git a/dev/issues/0057-audit-go-symbol-naming.md b/dev/issues/0057-audit-go-symbol-naming.md index 6f7d9b11..9ffca427 100644 --- a/dev/issues/0057-audit-go-symbol-naming.md +++ b/dev/issues/0057-audit-go-symbol-naming.md @@ -12,7 +12,7 @@ blocks: [] related: [] created: 2026-05-17 updated: 2026-05-17 -tags: [] +tags: [ausente-ready] --- # 0057 — audit_uses_functions: mejorar deteccion de simbolos Go con abreviaturas diff --git a/dev/issues/0060-fn-doctor-secrets-subcommand.md b/dev/issues/0060-fn-doctor-secrets-subcommand.md index a14df57d..0f5ac6b5 100644 --- a/dev/issues/0060-fn-doctor-secrets-subcommand.md +++ b/dev/issues/0060-fn-doctor-secrets-subcommand.md @@ -11,7 +11,7 @@ blocks: [] related: [] created: 2026-05-17 updated: 2026-05-17 -tags: [] +tags: [ausente-ready] --- # 0060 — `fn doctor secrets`: scan de secrets en TODOS los repos diff --git a/dev/issues/0061-notify-telegram-integration.md b/dev/issues/0061-notify-telegram-integration.md index de1a5d76..9c3989d5 100644 --- a/dev/issues/0061-notify-telegram-integration.md +++ b/dev/issues/0061-notify-telegram-integration.md @@ -12,7 +12,7 @@ blocks: [] related: [] created: 2026-05-17 updated: 2026-05-17 -tags: [] +tags: [ausente-ready] --- # 0061 — Integrar `notify_telegram` en deploy_server + bucle reactivo diff --git a/dev/issues/0071a-extract-claude-chat-panel.md b/dev/issues/0071a-extract-claude-chat-panel.md index 471166e6..13da1a65 100644 --- a/dev/issues/0071a-extract-claude-chat-panel.md +++ b/dev/issues/0071a-extract-claude-chat-panel.md @@ -7,8 +7,7 @@ domain: - registry-quality scope: registry-only priority: alta -depends: - - "0071f" +depends: ["0071f"] blocks: [] related: [] created: 2026-05-10 diff --git a/dev/issues/0071b-extract-jobs-queue-panel.md b/dev/issues/0071b-extract-jobs-queue-panel.md index 3eb61073..7b69c06c 100644 --- a/dev/issues/0071b-extract-jobs-queue-panel.md +++ b/dev/issues/0071b-extract-jobs-queue-panel.md @@ -7,8 +7,7 @@ domain: - registry-quality scope: registry-only priority: media -depends: - - "0071f" +depends: ["0071f"] blocks: [] related: [] created: 2026-05-10 diff --git a/dev/issues/0071f-extract-subprocess-streamer.md b/dev/issues/0071f-extract-subprocess-streamer.md index 81b1c6d3..7ac44b28 100644 --- a/dev/issues/0071f-extract-subprocess-streamer.md +++ b/dev/issues/0071f-extract-subprocess-streamer.md @@ -12,7 +12,7 @@ blocks: [] related: [] created: 2026-05-10 updated: 2026-05-17 -tags: [] +tags: [ausente-ready] --- ## Contexto diff --git a/dev/issues/0071g-extract-app-db-init.md b/dev/issues/0071g-extract-app-db-init.md index 43a03c2a..e90d5cf4 100644 --- a/dev/issues/0071g-extract-app-db-init.md +++ b/dev/issues/0071g-extract-app-db-init.md @@ -12,7 +12,7 @@ blocks: [] related: [] created: 2026-05-10 updated: 2026-05-17 -tags: [] +tags: [ausente-ready] --- ## Contexto diff --git a/dev/issues/0076-gradle_run-no-detecta-android-sdk-path.md b/dev/issues/0076-gradle_run-no-detecta-android-sdk-path.md index 250d04c6..b6b7b510 100644 --- a/dev/issues/0076-gradle_run-no-detecta-android-sdk-path.md +++ b/dev/issues/0076-gradle_run-no-detecta-android-sdk-path.md @@ -12,7 +12,7 @@ blocks: [] related: [] created: 2026-05-10 updated: 2026-05-17 -tags: [] +tags: [ausente-ready] --- ## Sintoma diff --git a/dev/issues/0077-fn-run-bash-output-mudo.md b/dev/issues/0077-fn-run-bash-output-mudo.md index 383e4b2a..3e348a11 100644 --- a/dev/issues/0077-fn-run-bash-output-mudo.md +++ b/dev/issues/0077-fn-run-bash-output-mudo.md @@ -12,7 +12,7 @@ blocks: [] related: [] created: 2026-05-10 updated: 2026-05-17 -tags: [] +tags: [ausente-ready] --- ## Sintoma diff --git a/dev/issues/0100-issues-frontmatter-migration.md b/dev/issues/0100-issues-frontmatter-migration.md index 7daada4f..f6ed6d48 100644 --- a/dev/issues/0100-issues-frontmatter-migration.md +++ b/dev/issues/0100-issues-frontmatter-migration.md @@ -12,7 +12,7 @@ blocks: [] related: [] created: 2026-05-17 updated: 2026-05-17 -tags: [] +tags: [ausente-ready] --- # 0100 — Migrar frontmatter inline a YAML canonico en dev/issues/ diff --git a/dev/issues/0104-fix-issue-type-aware-dispatch.md b/dev/issues/0104-fix-issue-type-aware-dispatch.md index 823f6a43..8dc2f11e 100644 --- a/dev/issues/0104-fix-issue-type-aware-dispatch.md +++ b/dev/issues/0104-fix-issue-type-aware-dispatch.md @@ -16,7 +16,7 @@ related: - "0103" created: 2026-05-17 updated: 2026-05-17 -tags: [slash-command, dispatch, type-aware] +tags: [slash-command, dispatch, type-aware, ausente-ready] --- # 0104 — `/fix-issue` type-aware dispatch diff --git a/dev/issues/0107e-version-pinning-codegen.md b/dev/issues/0107e-version-pinning-codegen.md index 3f973aa9..695d012b 100644 --- a/dev/issues/0107e-version-pinning-codegen.md +++ b/dev/issues/0107e-version-pinning-codegen.md @@ -16,7 +16,7 @@ related: - "0107" created: 2026-05-17 updated: 2026-05-17 -tags: [modules, versioning, codegen, fail-loud] +tags: [modules, versioning, codegen, fail-loud, ausente-ready] --- # 0107e — Version pinning + codegen fail-loud diff --git a/dev/issues/0109k-skill-tree-dashboard-panel.md b/dev/issues/0109k-skill-tree-dashboard-panel.md index 29302dd0..a9c0c89c 100644 --- a/dev/issues/0109k-skill-tree-dashboard-panel.md +++ b/dev/issues/0109k-skill-tree-dashboard-panel.md @@ -15,12 +15,7 @@ related: - "0109" created: 2026-05-17 updated: 2026-05-17 -tags: - - skill-tree - - cpp - - imgui - - dashboard - - gamification +tags: [ausente-ready, skill-tree, cpp, imgui, dashboard, gamification] --- # 0109k — Dashboard panel diff --git a/dev/issues/0109m-issues-api-service.md b/dev/issues/0109m-issues-api-service.md index 5d374935..5826e8f7 100644 --- a/dev/issues/0109m-issues-api-service.md +++ b/dev/issues/0109m-issues-api-service.md @@ -16,13 +16,7 @@ related: - "0106" created: 2026-05-18 updated: 2026-05-18 -tags: - - service - - go - - http - - issues - - flows - - api +tags: [ausente-ready, service, go, http, issues, flows, api] --- # 0109m — issues_api service diff --git a/dev/issues/0121-e2e-checks-coverage-masa.md b/dev/issues/0121-e2e-checks-coverage-masa.md index 6434aa46..4a0b9370 100644 --- a/dev/issues/0121-e2e-checks-coverage-masa.md +++ b/dev/issues/0121-e2e-checks-coverage-masa.md @@ -16,7 +16,7 @@ related: - "0068" created: 2026-05-18 updated: 2026-05-19 -tags: [e2e_checks, recopilador, batch, coverage, epic] +tags: [e2e_checks, recopilador, batch, coverage, epic, ausente-ready] --- # Sub-issues diff --git a/dev/issues/0121a-design-e2e-batch.md b/dev/issues/0121a-design-e2e-batch.md index bfa8b70e..61c66ac8 100644 --- a/dev/issues/0121a-design-e2e-batch.md +++ b/dev/issues/0121a-design-e2e-batch.md @@ -16,7 +16,7 @@ related: - "0068" created: 2026-05-19 updated: 2026-05-19 -tags: [e2e_checks, recopilador, batch, design] +tags: [e2e_checks, recopilador, batch, design, ausente-ready] --- # 0121a — Design-e2e batch diff --git a/dev/issues/0121c-apply-e2e-proposals.md b/dev/issues/0121c-apply-e2e-proposals.md index 696eac88..bff1854f 100644 --- a/dev/issues/0121c-apply-e2e-proposals.md +++ b/dev/issues/0121c-apply-e2e-proposals.md @@ -7,9 +7,7 @@ domain: - registry-quality scope: registry priority: media -depends: - - "0121a" - - "0121b" +depends: ["0121a"] blocks: - "0122" related: diff --git a/dev/issues/0122-fn-revisor-auto-apply.md b/dev/issues/0122-fn-revisor-auto-apply.md index b99470c1..25438504 100644 --- a/dev/issues/0122-fn-revisor-auto-apply.md +++ b/dev/issues/0122-fn-revisor-auto-apply.md @@ -17,7 +17,7 @@ related: - "0086" created: 2026-05-18 updated: 2026-05-18 -tags: [revisor, mejorador, proposals, auto-apply, autonomous] +tags: [revisor, mejorador, proposals, auto-apply, autonomous, ausente-ready] --- # 0122 — fn-revisor + ampliar filtro auto-aplicable del orquestador diff --git a/dev/issues/0124-dag-engine-cleanup.md b/dev/issues/0124-dag-engine-cleanup.md index 64f52685..c198e92a 100644 --- a/dev/issues/0124-dag-engine-cleanup.md +++ b/dev/issues/0124-dag-engine-cleanup.md @@ -13,7 +13,7 @@ related: - "0121a" created: 2026-05-19 updated: 2026-05-19 -tags: [dag_engine, cleanup, technical-debt] +tags: [dag_engine, cleanup, technical-debt, ausente-ready] --- # 0124 — dag_engine cleanup diff --git a/dev/issues/0125-deploy-server-db-flag.md b/dev/issues/0125-deploy-server-db-flag.md index ef7144aa..10f53886 100644 --- a/dev/issues/0125-deploy-server-db-flag.md +++ b/dev/issues/0125-deploy-server-db-flag.md @@ -13,7 +13,7 @@ related: - "0121a" created: 2026-05-19 updated: 2026-05-19 -tags: [deploy_server, cli, idempotency] +tags: [deploy_server, cli, idempotency, ausente-ready] --- # 0125 — deploy_server `--db` flag diff --git a/dev/issues/0128-kanban-files-attachments.md b/dev/issues/0128-kanban-files-attachments.md index b2654640..130eedd0 100644 --- a/dev/issues/0128-kanban-files-attachments.md +++ b/dev/issues/0128-kanban-files-attachments.md @@ -1,7 +1,7 @@ --- id: "0128" title: "kanban: adjuntar archivos (drag&drop desc/chat + tab Archivos)" -status: in_progress +status: in-progress type: feature domain: - apps-tools diff --git a/dev/issues/0130a-kanban-cpp-v2-parser.md b/dev/issues/0130a-kanban-cpp-v2-parser.md index 97d3c93f..3172b9e6 100644 --- a/dev/issues/0130a-kanban-cpp-v2-parser.md +++ b/dev/issues/0130a-kanban-cpp-v2-parser.md @@ -13,12 +13,7 @@ blocks: - 0130b related: - "0130" -tags: - - registry - - go - - parser - - frontmatter - - fsnotify +tags: [registry, go, parser, frontmatter, fsnotify, ausente-ready] flow: "0130" created: "2026-05-22" updated: "2026-05-22" diff --git a/dev/issues/0130b-kanban-cpp-v2-backend.md b/dev/issues/0130b-kanban-cpp-v2-backend.md index e8bcecd8..32f1c6b5 100644 --- a/dev/issues/0130b-kanban-cpp-v2-backend.md +++ b/dev/issues/0130b-kanban-cpp-v2-backend.md @@ -8,8 +8,7 @@ domain: - dev-ux scope: app-scoped priority: alta -depends: - - "0130a" +depends: ["0130a"] blocks: - "0130c" related: diff --git a/dev/issues/0134-mesh-protocol-spec.md b/dev/issues/0134-mesh-protocol-spec.md index 1fded110..791614fa 100644 --- a/dev/issues/0134-mesh-protocol-spec.md +++ b/dev/issues/0134-mesh-protocol-spec.md @@ -1,14 +1,14 @@ --- id: "0134" title: "Mesh protocol spec: capability manifests, ed25519 envelopes, enrollment, audit chain" -status: pending +status: pendiente type: spec domain: - infra - cybersecurity - protocols scope: cross-app -priority: high +priority: alta depends: [] blocks: - "0135" diff --git a/dev/issues/0144-agent-per-machine-llm.md b/dev/issues/0144-agent-per-machine-llm.md index a6d824fa..a084b5d9 100644 --- a/dev/issues/0144-agent-per-machine-llm.md +++ b/dev/issues/0144-agent-per-machine-llm.md @@ -1,7 +1,7 @@ --- id: "0144" title: "Agent LLM per machine (user + sudo) con tool registry y mesh dispatch" -status: pending +status: pendiente type: spec domain: - agents @@ -9,7 +9,7 @@ domain: - infra - cybersecurity scope: multi-app -priority: high +priority: alta depends: - "0134" - "0140" diff --git a/dev/issues/0146-add-pc-oneshot-mesh-scaling.md b/dev/issues/0146-add-pc-oneshot-mesh-scaling.md index f2e766d6..c6368810 100644 --- a/dev/issues/0146-add-pc-oneshot-mesh-scaling.md +++ b/dev/issues/0146-add-pc-oneshot-mesh-scaling.md @@ -1,8 +1,8 @@ --- id: "0146" title: "add-pc one-shot: añade PC al mesh + agente LLM en <2min desde movil" -status: pending -priority: high +status: pendiente +priority: alta created: 2026-05-24 related_flows: ["0009"] related_issues: ["0134", "0144", "0145"] diff --git a/dev/issues/0147-matrix-client-pc-scaffold.md b/dev/issues/0147-matrix-client-pc-scaffold.md index 45e5c7c6..066a2cc8 100644 --- a/dev/issues/0147-matrix-client-pc-scaffold.md +++ b/dev/issues/0147-matrix-client-pc-scaffold.md @@ -1,8 +1,8 @@ --- id: "0147" title: "matrix-client-pc scaffold: Wails + React+Mantine + login MAS" -status: pending -priority: high +status: pendiente +priority: alta created: 2026-05-24 related_flows: ["0010"] related_issues: ["0148", "0162"] diff --git a/dev/issues/0148-matrix-client-pc-rooms-timeline.md b/dev/issues/0148-matrix-client-pc-rooms-timeline.md index 555a8b77..dcc0b6a4 100644 --- a/dev/issues/0148-matrix-client-pc-rooms-timeline.md +++ b/dev/issues/0148-matrix-client-pc-rooms-timeline.md @@ -1,8 +1,8 @@ --- id: "0148" title: "matrix-client-pc rooms list + timeline con sync incremental" -status: pending -priority: high +status: pendiente +priority: alta created: 2026-05-24 related_flows: ["0010"] related_issues: ["0147", "0149"] diff --git a/dev/issues/0149-matrix-client-pc-composer.md b/dev/issues/0149-matrix-client-pc-composer.md index 8797bf19..40277506 100644 --- a/dev/issues/0149-matrix-client-pc-composer.md +++ b/dev/issues/0149-matrix-client-pc-composer.md @@ -1,8 +1,8 @@ --- id: "0149" title: "matrix-client-pc composer: markdown, reply, edit, reactions, media" -status: pending -priority: high +status: pendiente +priority: alta created: 2026-05-24 related_flows: ["0010"] related_issues: ["0148", "0150"] diff --git a/dev/issues/0150-matrix-client-pc-e2ee.md b/dev/issues/0150-matrix-client-pc-e2ee.md index cf7c4644..e5322605 100644 --- a/dev/issues/0150-matrix-client-pc-e2ee.md +++ b/dev/issues/0150-matrix-client-pc-e2ee.md @@ -1,8 +1,8 @@ --- id: "0150" title: "matrix-client-pc E2EE: cross-signing, SAS verification, recovery" -status: pending -priority: critical +status: pendiente +priority: alta created: 2026-05-24 related_flows: ["0010"] related_issues: ["0149", "0151"] diff --git a/dev/issues/0151-matrix-client-pc-livekit-calls.md b/dev/issues/0151-matrix-client-pc-livekit-calls.md index 579dadf1..e0f0cf59 100644 --- a/dev/issues/0151-matrix-client-pc-livekit-calls.md +++ b/dev/issues/0151-matrix-client-pc-livekit-calls.md @@ -1,8 +1,8 @@ --- id: "0151" title: "matrix-client-pc calls LiveKit: 1:1 + grupales, mic/cam/screen" -status: pending -priority: high +status: pendiente +priority: alta created: 2026-05-24 related_flows: ["0010"] related_issues: ["0150", "0152"] diff --git a/dev/issues/0152-matrix-client-pc-mini-webapps.md b/dev/issues/0152-matrix-client-pc-mini-webapps.md index 57b98498..a4ff803e 100644 --- a/dev/issues/0152-matrix-client-pc-mini-webapps.md +++ b/dev/issues/0152-matrix-client-pc-mini-webapps.md @@ -1,8 +1,8 @@ --- id: "0152" title: "matrix-client-pc mini-webapps embebidas: Matrix Widget API v2" -status: pending -priority: high +status: pendiente +priority: alta created: 2026-05-24 related_flows: ["0010"] related_issues: ["0151", "0153"] diff --git a/dev/issues/0154-matrix-client-android-scaffold.md b/dev/issues/0154-matrix-client-android-scaffold.md index 074f4df4..9a35c577 100644 --- a/dev/issues/0154-matrix-client-android-scaffold.md +++ b/dev/issues/0154-matrix-client-android-scaffold.md @@ -1,8 +1,8 @@ --- id: "0154" title: "matrix-client-android scaffold: Kotlin + Compose + login MAS" -status: pending -priority: high +status: pendiente +priority: alta created: 2026-05-24 related_flows: ["0011"] related_issues: ["0155", "0162"] diff --git a/dev/issues/0155-matrix-client-android-rooms-timeline.md b/dev/issues/0155-matrix-client-android-rooms-timeline.md index 727e6dee..c5dc3c33 100644 --- a/dev/issues/0155-matrix-client-android-rooms-timeline.md +++ b/dev/issues/0155-matrix-client-android-rooms-timeline.md @@ -1,8 +1,8 @@ --- id: "0155" title: "matrix-client-android rooms list + timeline Compose" -status: pending -priority: high +status: pendiente +priority: alta created: 2026-05-24 related_flows: ["0011"] related_issues: ["0154", "0156"] diff --git a/dev/issues/0156-matrix-client-android-composer.md b/dev/issues/0156-matrix-client-android-composer.md index 99548a66..411d24ae 100644 --- a/dev/issues/0156-matrix-client-android-composer.md +++ b/dev/issues/0156-matrix-client-android-composer.md @@ -1,8 +1,8 @@ --- id: "0156" title: "matrix-client-android composer: markdown, replies, edits, reactions, media" -status: pending -priority: high +status: pendiente +priority: alta created: 2026-05-24 related_flows: ["0011"] related_issues: ["0155", "0157"] diff --git a/dev/issues/0157-matrix-client-android-e2ee.md b/dev/issues/0157-matrix-client-android-e2ee.md index 32e75e73..801e1151 100644 --- a/dev/issues/0157-matrix-client-android-e2ee.md +++ b/dev/issues/0157-matrix-client-android-e2ee.md @@ -1,8 +1,8 @@ --- id: "0157" title: "matrix-client-android E2EE rust-sdk: cross-signing, SAS, recovery" -status: pending -priority: critical +status: pendiente +priority: alta created: 2026-05-24 related_flows: ["0011"] related_issues: ["0156", "0158"] diff --git a/dev/issues/0158-matrix-client-android-livekit-calls.md b/dev/issues/0158-matrix-client-android-livekit-calls.md index 8166155b..77739f64 100644 --- a/dev/issues/0158-matrix-client-android-livekit-calls.md +++ b/dev/issues/0158-matrix-client-android-livekit-calls.md @@ -1,8 +1,8 @@ --- id: "0158" title: "matrix-client-android calls LiveKit nativo: mic/cam/screen + PiP" -status: pending -priority: high +status: pendiente +priority: alta created: 2026-05-24 related_flows: ["0011"] related_issues: ["0157", "0159", "0161"] diff --git a/dev/issues/0159-matrix-client-android-push-fcm.md b/dev/issues/0159-matrix-client-android-push-fcm.md index 9984457d..190f649f 100644 --- a/dev/issues/0159-matrix-client-android-push-fcm.md +++ b/dev/issues/0159-matrix-client-android-push-fcm.md @@ -1,8 +1,8 @@ --- id: "0159" title: "matrix-client-android push FCM via sygnal + Firebase setup" -status: pending -priority: high +status: pendiente +priority: alta created: 2026-05-24 related_flows: ["0011"] related_issues: ["0158", "0160"] diff --git a/dev/issues/0160-matrix-client-android-mini-webapps.md b/dev/issues/0160-matrix-client-android-mini-webapps.md index e3ff733d..48f5e965 100644 --- a/dev/issues/0160-matrix-client-android-mini-webapps.md +++ b/dev/issues/0160-matrix-client-android-mini-webapps.md @@ -1,8 +1,8 @@ --- id: "0160" title: "matrix-client-android mini-webapps: WebView + Widget API v2 bridge" -status: pending -priority: medium +status: pendiente +priority: media created: 2026-05-24 related_flows: ["0011"] related_issues: ["0159", "0161"] diff --git a/dev/issues/0161-matrix-client-android-foreground-service.md b/dev/issues/0161-matrix-client-android-foreground-service.md index 97952f2d..bda12c68 100644 --- a/dev/issues/0161-matrix-client-android-foreground-service.md +++ b/dev/issues/0161-matrix-client-android-foreground-service.md @@ -1,8 +1,8 @@ --- id: "0161" title: "matrix-client-android foreground service: calls + lifecycle + lockscreen" -status: pending -priority: high +status: pendiente +priority: alta created: 2026-05-24 related_flows: ["0011"] related_issues: ["0158", "0160"] diff --git a/dev/issues/0162-matrix-enable-mas-delegated-auth.md b/dev/issues/0162-matrix-enable-mas-delegated-auth.md index 035128e0..ed907fe1 100644 --- a/dev/issues/0162-matrix-enable-mas-delegated-auth.md +++ b/dev/issues/0162-matrix-enable-mas-delegated-auth.md @@ -1,8 +1,8 @@ --- id: "0162" title: "Matrix: migrar Synapse a MAS como unico auth provider (MSC3861)" -status: pending -priority: critical +status: pendiente +priority: alta created: 2026-05-24 related_flows: ["0010", "0011"] related_issues: ["0147", "0154", "0163"] diff --git a/dev/issues/0163-matrix-custom-admin-panel.md b/dev/issues/0163-matrix-custom-admin-panel.md index cfa35a40..08a5cc96 100644 --- a/dev/issues/0163-matrix-custom-admin-panel.md +++ b/dev/issues/0163-matrix-custom-admin-panel.md @@ -1,8 +1,8 @@ --- id: "0163" title: "Matrix admin panel propio: users, rooms, devices, sessions (sustituye synapse-admin)" -status: pending -priority: medium +status: pendiente +priority: media created: 2026-05-24 related_flows: ["0010", "0011"] related_issues: ["0162", "0147"] diff --git a/dev/issues/0179-dev-console-recursive-issue-scan.md b/dev/issues/0179-dev-console-recursive-issue-scan.md new file mode 100644 index 00000000..86ebac51 --- /dev/null +++ b/dev/issues/0179-dev-console-recursive-issue-scan.md @@ -0,0 +1,45 @@ +--- +id: "0179" +title: "dev_console: escaneo recursivo de dev/issues/ (subcarpetas por dominio)" +status: in-progress +type: bugfix +domain: + - meta +scope: app-scoped +priority: media +depends: [] +blocks: [] +related: [] +created: 2026-06-30 +updated: 2026-06-30 +tags: [ausente-ready] +--- +# 0179 — dev_console: escaneo recursivo de dev/issues/ + +## Contexto + +Los issues activos se reorganizaron en subcarpetas por dominio dentro de `dev/issues/` (`kanban/`, `trading/`, `gamedev/`, `cpp/`, `matrix/`, `imagegen/`) para descongestionar el listado plano. El skill `/issue` ya se actualizó a glob recursivo (`dev/issues/**/*.md`, excluyendo `completed/`). Falta alinear el binario `dev_console`, que carga los issues con `LoadAllIssues(root)` / `LoadOpenIssues(root)` en `apps/dev_console/` y hoy no recorre subcarpetas — por lo que no ve los 49 issues movidos. + +## Objetivo + +Que `dev_console issue list/board/work` y los flujos que dependen de `LoadAllIssues`/`LoadOpenIssues` recorran `dev/issues/` de forma recursiva, excluyendo `dev/issues/completed/`, manteniendo el resto del comportamiento idéntico. + +## Tareas + +- [ ] Localizar la implementación de `LoadAllIssues` / `LoadOpenIssues` en `apps/dev_console/` (probable `parser.go` o equivalente). +- [ ] Cambiar el escaneo a `filepath.WalkDir` (o glob recursivo) bajo `dev/issues/`, saltando el directorio `completed/`. +- [ ] Mantener el orden de salida estable (ordenar por `id`). +- [ ] Recompilar el binario en el sub-repo de `dev_console` siguiendo TBD (`issue/0179-...`). + +## Definition of Done + +| Escenario | Tipo | Comando / evidencia | Resultado esperado | +|---|---|---|---| +| Golden: lista incluye subcarpetas | e2e | `./apps/dev_console/dev_console issue list` | Aparecen issues de `cpp/`, `kanban/`, `trading/`, etc. (>= 49 que antes faltaban) | +| Edge: excluye completed/ | e2e | `dev_console issue list` | Ningún issue con `status: completado` de `completed/` aparece en el listado activo | +| Edge: conteo total coincide con /issue | e2e | comparar conteo con el glob recursivo de `/issue` | Mismo total de activos | +| Error: dev/issues vacío o ausente | unit | run en dir sin `dev/issues/` | Error claro, no panic | + +## Notas + +Hermano del cambio ya hecho en `.claude/commands/issue.md` (glob `**/*.md`). Hasta cerrar este issue, usar `/issue` (no `dev_console`) para vistas completas del backlog. diff --git a/dev/issues/0180-ausente-issues-queue-dag.md b/dev/issues/0180-ausente-issues-queue-dag.md new file mode 100644 index 00000000..5a57ce44 --- /dev/null +++ b/dev/issues/0180-ausente-issues-queue-dag.md @@ -0,0 +1,59 @@ +--- +id: "0180" +title: "Modo ausente sobre la cola de issues: parametrizar /ausente + DAG dag_engine + validación" +status: pendiente +type: infra +domain: + - meta +scope: multi-app +priority: alta +depends: ["0179"] +blocks: [] +related: [] +created: 2026-06-30 +updated: 2026-06-30 +tags: [] +--- +# 0180 — Modo ausente sobre la cola de issues (parametrizar /ausente + DAG + validación) + +## Contexto + +Modelo de colaboración acordado (ver memoria `modelo-colaboracion-ausente`): durante la jornada de oficina (L–J 10–14 / 15–19, V 10–16) y la noche (01–09), Claude trabaja en `/ausente` la cola de issues `ausente-ready` (39 issues hoy), sin supervisión. La curación del backlog ya está hecha (triage, taxonomía, deps de series formalizadas, tag `ausente-ready`). + +Faltan 3 piezas para automatizarlo de forma segura. + +## Problemas a resolver + +1. **`/ausente` está acoplado al roadmap ComfyUI.** El skill (`.claude/commands/ausente.md`) hardcodea su backlog a funciones ComfyUI (secciones "Configuración" y "Backlog del roadmap ComfyUI"). Hay que **parametrizar la fuente de tareas** para que pueda tomar la cola de issues: la siguiente tarea = primer issue de `/issue list -t ausente-ready` cuyas `depends` estén todas en `completed/`, re-cruzando deps en cada ciclo (un issue se libera cuando su dep se cierra). +2. **Lanzamiento headless desde dag_engine.** `dag_engine` ejecuta steps (command/script/function), no abre una sesión Claude interactiva. Hay que resolver cómo un step arranca una sesión `role=orchestrator` en modo `/ausente` (candidatos: `launch_claude_agent_kitty_bash_infra` con DISPLAY, o `spawn_fleet_agent_bash_infra` si hay sesión tmux fleet) con el prompt autónomo + presupuesto. +3. **Presupuesto conservador aplicado.** Tope: 1–2 ejecutores concurrentes, solo issues S/M, ~1M tokens por franja, parada al llegar. Materializar el tope de tokens (hoy `orchestration.md` solo fija fan-out=6). + +## Schedule objetivo (cuando se active) + +- Inicio de franjas de oficina: `0 10 * * 1-5` (10:00 L–V) y `0 15 * * 1-4` (15:00 L–J, tras comida). +- Nocturno: `0 1 * * *` (01:00 diario). +- El modo, una vez lanzado, itera con `ScheduleWakeup` hasta que el humano vuelve (para al recibir prompt humano). + +Borrador del DAG: `apps/dag_engine/dags/ausente-issues-queue.yaml` (creado como DRAFT sin schedule activo). + +## Definition of Done + +| Escenario | Tipo | Comando / evidencia | Resultado esperado | +|---|---|---|---| +| Golden: corrida manual | e2e | lanzar `/ausente` con backlog=issues sobre 1 issue S de la cola | Coge el issue, lo implementa en worktree/sub-repo aislado, cierra DoD verde (golden+edge+error), push, bitácora actualizada | +| Edge: dep no satisfecha | e2e | cola con un issue cuya `depends` sigue activa | NO lo coge; pasa al siguiente arrancable | +| Edge: flota llena | e2e | 2 ejecutores activos (tope conservador) | Encola el resto, no lanza el 3.º | +| Error: presupuesto agotado | e2e | tope de tokens alcanzado | Para limpio, deja bitácora con lo pendiente, no deja agentes huérfanos | +| Vida útil | observabilidad | tras activar cron, 1 semana | Issues cerrados/semana > 0, 0 merges rotos a master, bitácora legible | + +## Plan + +1. Cerrar `0179` (dev_console recursivo) — dependencia. +2. Parametrizar `/ausente` (fuente de backlog = issues ausente-ready | roadmap; pasar la fuente al invocar). +3. Resolver el step de lanzamiento headless + presupuesto de tokens. +4. **Validación manual** (golden + edges) antes de activar el cron. +5. Activar schedule en el DAG + `systemctl --user restart dag_engine.service` con `--scheduler`. + +## Notas + +Este issue NO es `ausente-ready` a propósito: requiere decisiones de diseño humanas (mecanismo de lanzamiento, forma del presupuesto) y toca el propio sistema que orquesta el modo ausente. Se hace JUNTOS, no desatendido. diff --git a/dev/issues/0059-nested-app-md-tracking.md b/dev/issues/completed/0059-nested-app-md-tracking.md similarity index 99% rename from dev/issues/0059-nested-app-md-tracking.md rename to dev/issues/completed/0059-nested-app-md-tracking.md index 94da5346..60aa1986 100644 --- a/dev/issues/0059-nested-app-md-tracking.md +++ b/dev/issues/completed/0059-nested-app-md-tracking.md @@ -1,7 +1,7 @@ --- id: "0059" title: "Resolver doble tracking de `apps/*/app.md` (fn_registry + sub-repo)" -status: pendiente +status: completado type: infra domain: - registry-quality diff --git a/dev/issues/0067-odr-osint-prereqs-roadmap.md b/dev/issues/completed/0067-odr-osint-prereqs-roadmap.md similarity index 99% rename from dev/issues/0067-odr-osint-prereqs-roadmap.md rename to dev/issues/completed/0067-odr-osint-prereqs-roadmap.md index 65dc38d0..12216b44 100644 --- a/dev/issues/0067-odr-osint-prereqs-roadmap.md +++ b/dev/issues/completed/0067-odr-osint-prereqs-roadmap.md @@ -1,7 +1,7 @@ --- id: "55" title: "Roadmap de prereqs — issues de osint_graph que odr_console necesita antes/durante MVP" -status: pendiente +status: deferred type: epic domain: - osint diff --git a/dev/issues/0068-e2e-validation-loop-fn4-fn5.md b/dev/issues/completed/0068-e2e-validation-loop-fn4-fn5.md similarity index 100% rename from dev/issues/0068-e2e-validation-loop-fn4-fn5.md rename to dev/issues/completed/0068-e2e-validation-loop-fn4-fn5.md diff --git a/dev/issues/0069-autonomous-agent-loop-self-iterating-tasks.md b/dev/issues/completed/0069-autonomous-agent-loop-self-iterating-tasks.md similarity index 100% rename from dev/issues/0069-autonomous-agent-loop-self-iterating-tasks.md rename to dev/issues/completed/0069-autonomous-agent-loop-self-iterating-tasks.md diff --git a/dev/issues/0072l-gamedev-scripting-optional.md b/dev/issues/completed/0072l-gamedev-scripting-optional.md similarity index 100% rename from dev/issues/0072l-gamedev-scripting-optional.md rename to dev/issues/completed/0072l-gamedev-scripting-optional.md diff --git a/dev/issues/0085-registry-call-standardization-and-usage-tracking.md b/dev/issues/completed/0085-registry-call-standardization-and-usage-tracking.md similarity index 100% rename from dev/issues/0085-registry-call-standardization-and-usage-tracking.md rename to dev/issues/completed/0085-registry-call-standardization-and-usage-tracking.md diff --git a/dev/issues/0086-claude-md-delegation-and-capability-groups.md b/dev/issues/completed/0086-claude-md-delegation-and-capability-groups.md similarity index 100% rename from dev/issues/0086-claude-md-delegation-and-capability-groups.md rename to dev/issues/completed/0086-claude-md-delegation-and-capability-groups.md diff --git a/dev/issues/0087-capability-discovery-acceleration.md b/dev/issues/completed/0087-capability-discovery-acceleration.md similarity index 99% rename from dev/issues/0087-capability-discovery-acceleration.md rename to dev/issues/completed/0087-capability-discovery-acceleration.md index b1318dd1..4530327f 100644 --- a/dev/issues/0087-capability-discovery-acceleration.md +++ b/dev/issues/completed/0087-capability-discovery-acceleration.md @@ -1,7 +1,7 @@ --- id: "0087" title: "Capability Discovery Acceleration" -status: pendiente +status: completado type: feature domain: - meta diff --git a/dev/issues/0096-standardize-app-locations.md b/dev/issues/completed/0096-standardize-app-locations.md similarity index 99% rename from dev/issues/0096-standardize-app-locations.md rename to dev/issues/completed/0096-standardize-app-locations.md index 22c15c19..20493015 100644 --- a/dev/issues/0096-standardize-app-locations.md +++ b/dev/issues/completed/0096-standardize-app-locations.md @@ -1,7 +1,7 @@ --- id: "0096" title: "Estandarizar ubicacion de apps: fuera de carpetas por lenguaje" -status: pendiente +status: completado type: feature domain: - apps-infra diff --git a/dev/issues/0101-dev-console-binary.md b/dev/issues/completed/0101-dev-console-binary.md similarity index 99% rename from dev/issues/0101-dev-console-binary.md rename to dev/issues/completed/0101-dev-console-binary.md index 5c8126f4..d912e24d 100644 --- a/dev/issues/0101-dev-console-binary.md +++ b/dev/issues/completed/0101-dev-console-binary.md @@ -1,7 +1,7 @@ --- id: "0101" title: "dev_console Go binario: /issue /flow /work unificados" -status: pendiente +status: completado type: app domain: - meta diff --git a/dev/issues/0103-taxonomy-and-slash-commands.md b/dev/issues/completed/0103-taxonomy-and-slash-commands.md similarity index 99% rename from dev/issues/0103-taxonomy-and-slash-commands.md rename to dev/issues/completed/0103-taxonomy-and-slash-commands.md index 3f871235..71d8a35d 100644 --- a/dev/issues/0103-taxonomy-and-slash-commands.md +++ b/dev/issues/completed/0103-taxonomy-and-slash-commands.md @@ -1,7 +1,7 @@ --- id: "0103" title: "Taxonomia + slash commands /issue /flow /work" -status: pendiente +status: completado type: feature domain: - meta diff --git a/dev/issues/0105-service-frontmatter-standardization.md b/dev/issues/completed/0105-service-frontmatter-standardization.md similarity index 99% rename from dev/issues/0105-service-frontmatter-standardization.md rename to dev/issues/completed/0105-service-frontmatter-standardization.md index 212a499e..780146c0 100644 --- a/dev/issues/0105-service-frontmatter-standardization.md +++ b/dev/issues/completed/0105-service-frontmatter-standardization.md @@ -1,7 +1,7 @@ --- id: "0105" title: "Estandarizar bloque service: en app.md + indexer + fn doctor services-spec" -status: in-progress +status: completado type: feature domain: - meta diff --git a/dev/issues/0109g-skill-tree-embedded-terminal.md b/dev/issues/completed/0109g-skill-tree-embedded-terminal.md similarity index 99% rename from dev/issues/0109g-skill-tree-embedded-terminal.md rename to dev/issues/completed/0109g-skill-tree-embedded-terminal.md index 0e052d79..41b52dc2 100644 --- a/dev/issues/0109g-skill-tree-embedded-terminal.md +++ b/dev/issues/completed/0109g-skill-tree-embedded-terminal.md @@ -1,7 +1,7 @@ --- id: "0109g" title: "skill_tree: panel terminal embebida (claude TUI dentro de la app)" -status: pendiente +status: deferred type: feature domain: - meta diff --git a/dev/issues/completed/0114-dod-evidence-schema.md b/dev/issues/completed/0114-dod-evidence-schema.md index 2b588cd8..710893d5 100644 --- a/dev/issues/completed/0114-dod-evidence-schema.md +++ b/dev/issues/completed/0114-dod-evidence-schema.md @@ -19,7 +19,7 @@ related: - "0102" created: 2026-05-18 updated: 2026-05-18 -tags: [dod, evidence, frontmatter, taxonomy, validator] +tags: [dod, evidence, frontmatter, taxonomy, validator, ausente-ready] flow: "0008" --- diff --git a/dev/issues/completed/0128-agents-and-robots-http-api-sse.md b/dev/issues/completed/0128-agents-and-robots-http-api-sse.md index 969aa381..a5ad4681 100644 --- a/dev/issues/completed/0128-agents-and-robots-http-api-sse.md +++ b/dev/issues/completed/0128-agents-and-robots-http-api-sse.md @@ -15,7 +15,7 @@ blocks: related: [] created: 2026-05-22 updated: 2026-05-22 -tags: [agents_and_robots, http, sse, apikey, traefik, systemd] +tags: [agents_and_robots, http, sse, apikey, traefik, systemd, ausente-ready] dod_evidence_schema: - id: build_ok kind: cmd diff --git a/dev/issues/0153-matrix-client-pc-agent-integration.md b/dev/issues/completed/0153-matrix-client-pc-agent-integration.md similarity index 99% rename from dev/issues/0153-matrix-client-pc-agent-integration.md rename to dev/issues/completed/0153-matrix-client-pc-agent-integration.md index d76bcd43..d38c0d1f 100644 --- a/dev/issues/0153-matrix-client-pc-agent-integration.md +++ b/dev/issues/completed/0153-matrix-client-pc-agent-integration.md @@ -1,7 +1,7 @@ --- id: "0153" title: "matrix-client-pc agent integration: paneles para rooms operados por agentes" -status: pending +status: deferred priority: medium created: 2026-05-24 related_flows: ["0010", "0009"] diff --git a/dev/issues/0164-agents-cryptohelper-init-hang.md b/dev/issues/completed/0164-agents-cryptohelper-init-hang.md similarity index 99% rename from dev/issues/0164-agents-cryptohelper-init-hang.md rename to dev/issues/completed/0164-agents-cryptohelper-init-hang.md index e87db499..852c0c4a 100644 --- a/dev/issues/0164-agents-cryptohelper-init-hang.md +++ b/dev/issues/completed/0164-agents-cryptohelper-init-hang.md @@ -1,7 +1,7 @@ --- id: "0164" title: "Bots agents_and_robots: cryptohelper.Init() cuelga al habilitar encryption=true" -status: pending +status: deferred priority: high created: 2026-05-24 related_flows: ["0009"] diff --git a/dev/issues/completed/0166-matrix-livekit-turn-deploy.md b/dev/issues/completed/0166-matrix-livekit-turn-deploy.md index b222dc1b..0d7b498b 100644 --- a/dev/issues/completed/0166-matrix-livekit-turn-deploy.md +++ b/dev/issues/completed/0166-matrix-livekit-turn-deploy.md @@ -12,7 +12,7 @@ blocks: [] related: ["0167", "0168"] created: 2026-05-24 updated: 2026-05-24 -tags: [matrix, livekit, webrtc, turn, nat] +tags: [matrix, livekit, webrtc, turn, nat, ausente-ready] --- # 0166 — Desplegar TURN para LiveKit (coturn o integrado) diff --git a/dev/issues/0171-project-subrepo-manifest-and-reclone.md b/dev/issues/completed/0171-project-subrepo-manifest-and-reclone.md similarity index 99% rename from dev/issues/0171-project-subrepo-manifest-and-reclone.md rename to dev/issues/completed/0171-project-subrepo-manifest-and-reclone.md index 24924afe..53137229 100644 --- a/dev/issues/0171-project-subrepo-manifest-and-reclone.md +++ b/dev/issues/completed/0171-project-subrepo-manifest-and-reclone.md @@ -1,7 +1,7 @@ --- id: "0171" title: "Manifest de sub-repos por project + re-clonado y auditoría de cobertura en Gitea" -status: pendiente +status: completado type: enhancement domain: - registry-quality diff --git a/dev/issues/0173-eda-correctitud-estadistica-critica.md b/dev/issues/completed/0173-eda-correctitud-estadistica-critica.md similarity index 99% rename from dev/issues/0173-eda-correctitud-estadistica-critica.md rename to dev/issues/completed/0173-eda-correctitud-estadistica-critica.md index 90ab0072..03640d6f 100644 --- a/dev/issues/0173-eda-correctitud-estadistica-critica.md +++ b/dev/issues/completed/0173-eda-correctitud-estadistica-critica.md @@ -1,7 +1,7 @@ --- id: "0173" title: "EDA: bugs críticos de correctitud estadística (outlier_pct ×100, distribution_type por-skew)" -status: resuelto +status: completado type: bugfix domain: - registry-quality diff --git a/dev/issues/0174-eda-series-periodo-estacional-niveles.md b/dev/issues/completed/0174-eda-series-periodo-estacional-niveles.md similarity index 99% rename from dev/issues/0174-eda-series-periodo-estacional-niveles.md rename to dev/issues/completed/0174-eda-series-periodo-estacional-niveles.md index 65afe5d6..138573ef 100644 --- a/dev/issues/0174-eda-series-periodo-estacional-niveles.md +++ b/dev/issues/completed/0174-eda-series-periodo-estacional-niveles.md @@ -1,7 +1,7 @@ --- id: "0174" title: "EDA series temporales: período estacional roto + correlación de niveles + to_returns ciego" -status: resuelto +status: completado type: bugfix domain: - registry-quality diff --git a/dev/issues/0175-eda-relational-fk-inference-views.md b/dev/issues/completed/0175-eda-relational-fk-inference-views.md similarity index 99% rename from dev/issues/0175-eda-relational-fk-inference-views.md rename to dev/issues/completed/0175-eda-relational-fk-inference-views.md index 19bfd2d6..866e2514 100644 --- a/dev/issues/0175-eda-relational-fk-inference-views.md +++ b/dev/issues/completed/0175-eda-relational-fk-inference-views.md @@ -1,7 +1,7 @@ --- id: "0175" title: "EDA relational: precisión de FK inference (falsos positivos) + filtrar VIEWs + test ATTACH" -status: resuelto +status: completado type: bugfix domain: - registry-quality diff --git a/dev/issues/0176-eda-render-models-series-caveats-pdf.md b/dev/issues/completed/0176-eda-render-models-series-caveats-pdf.md similarity index 99% rename from dev/issues/0176-eda-render-models-series-caveats-pdf.md rename to dev/issues/completed/0176-eda-render-models-series-caveats-pdf.md index 52f655af..61b0ac22 100644 --- a/dev/issues/0176-eda-render-models-series-caveats-pdf.md +++ b/dev/issues/completed/0176-eda-render-models-series-caveats-pdf.md @@ -1,7 +1,7 @@ --- id: "0176" title: "EDA render: models/series/caveats en markdown+PDF + PDF para profile_database" -status: resuelto +status: completado type: feature domain: - registry-quality diff --git a/dev/issues/0177-eda-tipos-id-secuencial-eta-cuadrado.md b/dev/issues/completed/0177-eda-tipos-id-secuencial-eta-cuadrado.md similarity index 99% rename from dev/issues/0177-eda-tipos-id-secuencial-eta-cuadrado.md rename to dev/issues/completed/0177-eda-tipos-id-secuencial-eta-cuadrado.md index 6694c7eb..9f070680 100644 --- a/dev/issues/0177-eda-tipos-id-secuencial-eta-cuadrado.md +++ b/dev/issues/completed/0177-eda-tipos-id-secuencial-eta-cuadrado.md @@ -1,7 +1,7 @@ --- id: "0177" title: "EDA tipos: id secuencial fuera de correlación/PCA + η² espurio por cardinalidad + re-expresión no-continuas" -status: resuelto +status: completado type: bugfix domain: - registry-quality diff --git a/dev/issues/0027-cpp-gl-compute-pingpong.md b/dev/issues/cpp/0027-cpp-gl-compute-pingpong.md similarity index 100% rename from dev/issues/0027-cpp-gl-compute-pingpong.md rename to dev/issues/cpp/0027-cpp-gl-compute-pingpong.md diff --git a/dev/issues/0030-cpp-audio-reactive.md b/dev/issues/cpp/0030-cpp-audio-reactive.md similarity index 100% rename from dev/issues/0030-cpp-audio-reactive.md rename to dev/issues/cpp/0030-cpp-audio-reactive.md diff --git a/dev/issues/0033-cpp-http-ws-inspector.md b/dev/issues/cpp/0033-cpp-http-ws-inspector.md similarity index 99% rename from dev/issues/0033-cpp-http-ws-inspector.md rename to dev/issues/cpp/0033-cpp-http-ws-inspector.md index efecd4bc..b94c9910 100644 --- a/dev/issues/0033-cpp-http-ws-inspector.md +++ b/dev/issues/cpp/0033-cpp-http-ws-inspector.md @@ -12,7 +12,7 @@ blocks: [] related: [] created: 2026-05-17 updated: 2026-05-17 -tags: [] +tags: [ausente-ready] --- # 0033 — C++ http_inspector + websocket_client diff --git a/dev/issues/0035-cpp-map-tiles.md b/dev/issues/cpp/0035-cpp-map-tiles.md similarity index 100% rename from dev/issues/0035-cpp-map-tiles.md rename to dev/issues/cpp/0035-cpp-map-tiles.md diff --git a/dev/issues/0036-cpp-image-canvas-webcam.md b/dev/issues/cpp/0036-cpp-image-canvas-webcam.md similarity index 100% rename from dev/issues/0036-cpp-image-canvas-webcam.md rename to dev/issues/cpp/0036-cpp-image-canvas-webcam.md diff --git a/dev/issues/0071-extract-reusable-cpp-panels-roadmap.md b/dev/issues/cpp/0071-extract-reusable-cpp-panels-roadmap.md similarity index 100% rename from dev/issues/0071-extract-reusable-cpp-panels-roadmap.md rename to dev/issues/cpp/0071-extract-reusable-cpp-panels-roadmap.md diff --git a/dev/issues/0072a-gamedev-smoke-sdl3-sokol-imgui.md b/dev/issues/cpp/0072a-gamedev-smoke-sdl3-sokol-imgui.md similarity index 99% rename from dev/issues/0072a-gamedev-smoke-sdl3-sokol-imgui.md rename to dev/issues/cpp/0072a-gamedev-smoke-sdl3-sokol-imgui.md index d743ceed..8eeedf62 100644 --- a/dev/issues/0072a-gamedev-smoke-sdl3-sokol-imgui.md +++ b/dev/issues/cpp/0072a-gamedev-smoke-sdl3-sokol-imgui.md @@ -13,10 +13,7 @@ blocks: [] related: [] created: 2026-05-10 updated: 2026-05-17 -tags: - - gamedev - - cpp - - wasm +tags: [ausente-ready, gamedev, cpp, wasm] --- ## Objetivo diff --git a/dev/issues/0082-compile-sd-cpp-binary.md b/dev/issues/cpp/0082-compile-sd-cpp-binary.md similarity index 99% rename from dev/issues/0082-compile-sd-cpp-binary.md rename to dev/issues/cpp/0082-compile-sd-cpp-binary.md index 67c506ba..781556cb 100644 --- a/dev/issues/0082-compile-sd-cpp-binary.md +++ b/dev/issues/cpp/0082-compile-sd-cpp-binary.md @@ -13,7 +13,7 @@ blocks: [] related: [] created: 2026-05-13 updated: 2026-05-17 -tags: [] +tags: [ausente-ready] --- ## Objetivo diff --git a/dev/issues/0095-dag-engine-cpp-imgui-frontend.md b/dev/issues/cpp/0095-dag-engine-cpp-imgui-frontend.md similarity index 100% rename from dev/issues/0095-dag-engine-cpp-imgui-frontend.md rename to dev/issues/cpp/0095-dag-engine-cpp-imgui-frontend.md diff --git a/dev/issues/0110-cpp-http-client-helper.md b/dev/issues/cpp/0110-cpp-http-client-helper.md similarity index 98% rename from dev/issues/0110-cpp-http-client-helper.md rename to dev/issues/cpp/0110-cpp-http-client-helper.md index df4bd17d..de83a59d 100644 --- a/dev/issues/0110-cpp-http-client-helper.md +++ b/dev/issues/cpp/0110-cpp-http-client-helper.md @@ -15,7 +15,7 @@ related: - "0106" created: 2026-05-18 updated: 2026-05-18 -tags: [http, cpp, registry-gap, curl, helper] +tags: [http, cpp, registry-gap, curl, helper, ausente-ready] --- # 0110 — Helper HTTP cliente C++ en el registry diff --git a/dev/issues/0130-kanban-cpp-v2.md b/dev/issues/cpp/0130-kanban-cpp-v2.md similarity index 100% rename from dev/issues/0130-kanban-cpp-v2.md rename to dev/issues/cpp/0130-kanban-cpp-v2.md diff --git a/dev/issues/0130c-kanban-cpp-v2-frontend.md b/dev/issues/cpp/0130c-kanban-cpp-v2-frontend.md similarity index 99% rename from dev/issues/0130c-kanban-cpp-v2-frontend.md rename to dev/issues/cpp/0130c-kanban-cpp-v2-frontend.md index 793bd1a5..3cb5a246 100644 --- a/dev/issues/0130c-kanban-cpp-v2-frontend.md +++ b/dev/issues/cpp/0130c-kanban-cpp-v2-frontend.md @@ -8,8 +8,7 @@ domain: - dev-ux scope: app-scoped priority: alta -depends: - - "0130b" +depends: ["0130b"] blocks: [] related: - "0130" diff --git a/dev/issues/0131-cpp-module-chat-panel.md b/dev/issues/cpp/0131-cpp-module-chat-panel.md similarity index 100% rename from dev/issues/0131-cpp-module-chat-panel.md rename to dev/issues/cpp/0131-cpp-module-chat-panel.md diff --git a/dev/issues/0132-cpp-module-terminal-panel.md b/dev/issues/cpp/0132-cpp-module-terminal-panel.md similarity index 98% rename from dev/issues/0132-cpp-module-terminal-panel.md rename to dev/issues/cpp/0132-cpp-module-terminal-panel.md index 169da72b..8953892f 100644 --- a/dev/issues/0132-cpp-module-terminal-panel.md +++ b/dev/issues/cpp/0132-cpp-module-terminal-panel.md @@ -16,7 +16,7 @@ related: - "0131" created: 2026-05-22 updated: 2026-05-22 -tags: [cpp, imgui, terminal, pty, module] +tags: [cpp, imgui, terminal, pty, module, ausente-ready] flow: "" --- diff --git a/dev/issues/0133-cpp-data-table-10m-rows.md b/dev/issues/cpp/0133-cpp-data-table-10m-rows.md similarity index 100% rename from dev/issues/0133-cpp-data-table-10m-rows.md rename to dev/issues/cpp/0133-cpp-data-table-10m-rows.md diff --git a/dev/issues/0072-gamedev-stack-roadmap.md b/dev/issues/gamedev/0072-gamedev-stack-roadmap.md similarity index 100% rename from dev/issues/0072-gamedev-stack-roadmap.md rename to dev/issues/gamedev/0072-gamedev-stack-roadmap.md diff --git a/dev/issues/0072b-gamedev-runtime-core.md b/dev/issues/gamedev/0072b-gamedev-runtime-core.md similarity index 99% rename from dev/issues/0072b-gamedev-runtime-core.md rename to dev/issues/gamedev/0072b-gamedev-runtime-core.md index abdcc5c5..ac689902 100644 --- a/dev/issues/0072b-gamedev-runtime-core.md +++ b/dev/issues/gamedev/0072b-gamedev-runtime-core.md @@ -7,8 +7,7 @@ domain: - gamedev scope: multi-app priority: alta -depends: - - "0072a" +depends: ["0072a"] blocks: [] related: [] created: 2026-05-10 diff --git a/dev/issues/0072c-gamedev-asset-pipeline.md b/dev/issues/gamedev/0072c-gamedev-asset-pipeline.md similarity index 99% rename from dev/issues/0072c-gamedev-asset-pipeline.md rename to dev/issues/gamedev/0072c-gamedev-asset-pipeline.md index 4dfcfacd..dcc3084f 100644 --- a/dev/issues/0072c-gamedev-asset-pipeline.md +++ b/dev/issues/gamedev/0072c-gamedev-asset-pipeline.md @@ -7,8 +7,7 @@ domain: - gamedev scope: multi-app priority: alta -depends: - - "0072b" +depends: ["0072b"] blocks: [] related: [] created: 2026-05-10 diff --git a/dev/issues/0072d-gamedev-wasm-build-size-budget.md b/dev/issues/gamedev/0072d-gamedev-wasm-build-size-budget.md similarity index 99% rename from dev/issues/0072d-gamedev-wasm-build-size-budget.md rename to dev/issues/gamedev/0072d-gamedev-wasm-build-size-budget.md index 93a5bde1..492fb659 100644 --- a/dev/issues/0072d-gamedev-wasm-build-size-budget.md +++ b/dev/issues/gamedev/0072d-gamedev-wasm-build-size-budget.md @@ -7,9 +7,7 @@ domain: - gamedev scope: multi-app priority: alta -depends: - - "0072a" - - "0072b" +depends: ["0072a", "0072b"] blocks: [] related: [] created: 2026-05-10 diff --git a/dev/issues/0072e-gamedev-crypto-bridge-web3.md b/dev/issues/gamedev/0072e-gamedev-crypto-bridge-web3.md similarity index 99% rename from dev/issues/0072e-gamedev-crypto-bridge-web3.md rename to dev/issues/gamedev/0072e-gamedev-crypto-bridge-web3.md index 28a877b6..8be5bfe6 100644 --- a/dev/issues/0072e-gamedev-crypto-bridge-web3.md +++ b/dev/issues/gamedev/0072e-gamedev-crypto-bridge-web3.md @@ -7,9 +7,7 @@ domain: - gamedev scope: multi-app priority: alta -depends: - - "0072a" - - "0072d" +depends: ["0072a", "0072d"] blocks: [] related: [] created: 2026-05-10 diff --git a/dev/issues/0072f-gamedev-crypto-onchain-assets-payments.md b/dev/issues/gamedev/0072f-gamedev-crypto-onchain-assets-payments.md similarity index 99% rename from dev/issues/0072f-gamedev-crypto-onchain-assets-payments.md rename to dev/issues/gamedev/0072f-gamedev-crypto-onchain-assets-payments.md index da34fefc..d06f8c9d 100644 --- a/dev/issues/0072f-gamedev-crypto-onchain-assets-payments.md +++ b/dev/issues/gamedev/0072f-gamedev-crypto-onchain-assets-payments.md @@ -7,8 +7,7 @@ domain: - gamedev scope: multi-app priority: media -depends: - - "0072e" +depends: ["0072e"] blocks: [] related: [] created: 2026-05-10 diff --git a/dev/issues/0072g-gamedev-android-build.md b/dev/issues/gamedev/0072g-gamedev-android-build.md similarity index 99% rename from dev/issues/0072g-gamedev-android-build.md rename to dev/issues/gamedev/0072g-gamedev-android-build.md index eb0a0345..f5beed5f 100644 --- a/dev/issues/0072g-gamedev-android-build.md +++ b/dev/issues/gamedev/0072g-gamedev-android-build.md @@ -7,9 +7,7 @@ domain: - gamedev scope: multi-app priority: media -depends: - - "0072b" - - "0072c" +depends: ["0072b", "0072c"] blocks: [] related: [] created: 2026-05-10 diff --git a/dev/issues/0072h-gamedev-ios-build.md b/dev/issues/gamedev/0072h-gamedev-ios-build.md similarity index 99% rename from dev/issues/0072h-gamedev-ios-build.md rename to dev/issues/gamedev/0072h-gamedev-ios-build.md index 0091cbc4..e23c3432 100644 --- a/dev/issues/0072h-gamedev-ios-build.md +++ b/dev/issues/gamedev/0072h-gamedev-ios-build.md @@ -7,9 +7,7 @@ domain: - gamedev scope: multi-app priority: media -depends: - - "0072b" - - "0072c" +depends: ["0072b", "0072c"] blocks: [] related: [] created: 2026-05-10 diff --git a/dev/issues/0072i-gamedev-editor-app.md b/dev/issues/gamedev/0072i-gamedev-editor-app.md similarity index 99% rename from dev/issues/0072i-gamedev-editor-app.md rename to dev/issues/gamedev/0072i-gamedev-editor-app.md index dd02ca40..ab7ba9dd 100644 --- a/dev/issues/0072i-gamedev-editor-app.md +++ b/dev/issues/gamedev/0072i-gamedev-editor-app.md @@ -7,9 +7,7 @@ domain: - gamedev scope: app-scoped priority: media -depends: - - "0072b" - - "0072c" +depends: ["0072b", "0072c"] blocks: [] related: [] created: 2026-05-10 diff --git a/dev/issues/0072j-gamedev-physics-box2d.md b/dev/issues/gamedev/0072j-gamedev-physics-box2d.md similarity index 99% rename from dev/issues/0072j-gamedev-physics-box2d.md rename to dev/issues/gamedev/0072j-gamedev-physics-box2d.md index 74ffaa96..1ddce59a 100644 --- a/dev/issues/0072j-gamedev-physics-box2d.md +++ b/dev/issues/gamedev/0072j-gamedev-physics-box2d.md @@ -7,16 +7,12 @@ domain: - gamedev scope: multi-app priority: media -depends: - - "0072b" +depends: ["0072b"] blocks: [] related: [] created: 2026-05-10 updated: 2026-05-17 -tags: - - gamedev - - cpp - - physics +tags: [gamedev, cpp, physics] --- ## Objetivo diff --git a/dev/issues/0072k-gamedev-demo-platformer.md b/dev/issues/gamedev/0072k-gamedev-demo-platformer.md similarity index 99% rename from dev/issues/0072k-gamedev-demo-platformer.md rename to dev/issues/gamedev/0072k-gamedev-demo-platformer.md index 3d4cc46c..b3425074 100644 --- a/dev/issues/0072k-gamedev-demo-platformer.md +++ b/dev/issues/gamedev/0072k-gamedev-demo-platformer.md @@ -7,11 +7,7 @@ domain: - gamedev scope: multi-app priority: alta -depends: - - "0072b" - - "0072c" - - "0072d" - - "0072j" +depends: ["0072b", "0072c", "0072d", "0072j"] blocks: [] related: [] created: 2026-05-10 diff --git a/dev/issues/0083-imagegen-spike02-cross-validation.md b/dev/issues/imagegen/0083-imagegen-spike02-cross-validation.md similarity index 99% rename from dev/issues/0083-imagegen-spike02-cross-validation.md rename to dev/issues/imagegen/0083-imagegen-spike02-cross-validation.md index 27a48b13..fc835f64 100644 --- a/dev/issues/0083-imagegen-spike02-cross-validation.md +++ b/dev/issues/imagegen/0083-imagegen-spike02-cross-validation.md @@ -12,7 +12,7 @@ blocks: [] related: [] created: 2026-05-13 updated: 2026-05-17 -tags: [] +tags: [ausente-ready] --- ## Objetivo diff --git a/dev/issues/0084-imagegen-studio-go-app.md b/dev/issues/imagegen/0084-imagegen-studio-go-app.md similarity index 100% rename from dev/issues/0084-imagegen-studio-go-app.md rename to dev/issues/imagegen/0084-imagegen-studio-go-app.md diff --git a/dev/issues/0058-kanban-uses-functions-sync.md b/dev/issues/kanban/0058-kanban-uses-functions-sync.md similarity index 99% rename from dev/issues/0058-kanban-uses-functions-sync.md rename to dev/issues/kanban/0058-kanban-uses-functions-sync.md index bf9b486e..0c18ee23 100644 --- a/dev/issues/0058-kanban-uses-functions-sync.md +++ b/dev/issues/kanban/0058-kanban-uses-functions-sync.md @@ -13,7 +13,7 @@ blocks: [] related: [] created: 2026-05-17 updated: 2026-05-17 -tags: [] +tags: [ausente-ready] --- # 0058 — kanban: sync uses_functions cuando termine WIP en curso diff --git a/dev/issues/0063-kanban-stickers.md b/dev/issues/kanban/0063-kanban-stickers.md similarity index 100% rename from dev/issues/0063-kanban-stickers.md rename to dev/issues/kanban/0063-kanban-stickers.md diff --git a/dev/issues/0089-kanban-column-max-time.md b/dev/issues/kanban/0089-kanban-column-max-time.md similarity index 99% rename from dev/issues/0089-kanban-column-max-time.md rename to dev/issues/kanban/0089-kanban-column-max-time.md index fbfef633..8a933a09 100644 --- a/dev/issues/0089-kanban-column-max-time.md +++ b/dev/issues/kanban/0089-kanban-column-max-time.md @@ -12,7 +12,7 @@ blocks: [] related: [] created: 2026-05-17 updated: 2026-05-17 -tags: [] +tags: [ausente-ready] --- ## Problema diff --git a/dev/issues/0090-kanban-column-random-pick.md b/dev/issues/kanban/0090-kanban-column-random-pick.md similarity index 99% rename from dev/issues/0090-kanban-column-random-pick.md rename to dev/issues/kanban/0090-kanban-column-random-pick.md index 9cf7caa7..b92aedfb 100644 --- a/dev/issues/0090-kanban-column-random-pick.md +++ b/dev/issues/kanban/0090-kanban-column-random-pick.md @@ -12,7 +12,7 @@ blocks: [] related: [] created: 2026-05-17 updated: 2026-05-17 -tags: [] +tags: [ausente-ready] --- ## Problema diff --git a/dev/issues/0091-kanban-sidebar-drag-zones.md b/dev/issues/kanban/0091-kanban-sidebar-drag-zones.md similarity index 99% rename from dev/issues/0091-kanban-sidebar-drag-zones.md rename to dev/issues/kanban/0091-kanban-sidebar-drag-zones.md index 9e47ba43..abf3e07d 100644 --- a/dev/issues/0091-kanban-sidebar-drag-zones.md +++ b/dev/issues/kanban/0091-kanban-sidebar-drag-zones.md @@ -12,7 +12,7 @@ blocks: [] related: [] created: 2026-05-17 updated: 2026-05-17 -tags: [] +tags: [ausente-ready] --- ## Problema diff --git a/dev/issues/0092-kanban-done-archive.md b/dev/issues/kanban/0092-kanban-done-archive.md similarity index 99% rename from dev/issues/0092-kanban-done-archive.md rename to dev/issues/kanban/0092-kanban-done-archive.md index 743c3317..bd2c2296 100644 --- a/dev/issues/0092-kanban-done-archive.md +++ b/dev/issues/kanban/0092-kanban-done-archive.md @@ -12,7 +12,7 @@ blocks: [] related: [] created: 2026-05-17 updated: 2026-05-17 -tags: [] +tags: [ausente-ready] --- ## Problema diff --git a/dev/issues/0093-kanban-daily-report.md b/dev/issues/kanban/0093-kanban-daily-report.md similarity index 99% rename from dev/issues/0093-kanban-daily-report.md rename to dev/issues/kanban/0093-kanban-daily-report.md index 2c58bf42..d63b5527 100644 --- a/dev/issues/0093-kanban-daily-report.md +++ b/dev/issues/kanban/0093-kanban-daily-report.md @@ -12,7 +12,7 @@ blocks: [] related: [] created: 2026-05-17 updated: 2026-05-17 -tags: [] +tags: [ausente-ready] --- ## Problema diff --git a/dev/issues/0094-kanban-daily-summary-pdf.md b/dev/issues/kanban/0094-kanban-daily-summary-pdf.md similarity index 100% rename from dev/issues/0094-kanban-daily-summary-pdf.md rename to dev/issues/kanban/0094-kanban-daily-summary-pdf.md diff --git a/dev/issues/0088-kanban-requester-empty-nav.md b/dev/issues/kanban/0178-kanban-requester-empty-nav.md similarity index 98% rename from dev/issues/0088-kanban-requester-empty-nav.md rename to dev/issues/kanban/0178-kanban-requester-empty-nav.md index d0e781b2..38fbc653 100644 --- a/dev/issues/0088-kanban-requester-empty-nav.md +++ b/dev/issues/kanban/0178-kanban-requester-empty-nav.md @@ -1,5 +1,5 @@ --- -id: "0088" +id: "0178" title: "kanban: requester input vacío + navegación con teclado" status: pendiente type: feature @@ -12,7 +12,7 @@ blocks: [] related: [] created: 2026-05-17 updated: 2026-05-17 -tags: [] +tags: [ausente-ready] --- ## Problema diff --git a/dev/issues/0165-matrix-media-store-luks.md b/dev/issues/matrix/0165-matrix-media-store-luks.md similarity index 100% rename from dev/issues/0165-matrix-media-store-luks.md rename to dev/issues/matrix/0165-matrix-media-store-luks.md diff --git a/dev/issues/0088-trading-skill-management-roadmap.md b/dev/issues/trading/0088-trading-skill-management-roadmap.md similarity index 100% rename from dev/issues/0088-trading-skill-management-roadmap.md rename to dev/issues/trading/0088-trading-skill-management-roadmap.md diff --git a/dev/issues/0088a-trading-project-scaffolding.md b/dev/issues/trading/0088a-trading-project-scaffolding.md similarity index 98% rename from dev/issues/0088a-trading-project-scaffolding.md rename to dev/issues/trading/0088a-trading-project-scaffolding.md index cc5a31bc..771da8cf 100644 --- a/dev/issues/0088a-trading-project-scaffolding.md +++ b/dev/issues/trading/0088a-trading-project-scaffolding.md @@ -12,7 +12,7 @@ blocks: [] related: [] created: 2026-05-17 updated: 2026-05-17 -tags: [] +tags: [ausente-ready] --- # 0088a — Trading: project scaffolding diff --git a/dev/issues/0088b-trading-market-data-capability-group.md b/dev/issues/trading/0088b-trading-market-data-capability-group.md similarity index 99% rename from dev/issues/0088b-trading-market-data-capability-group.md rename to dev/issues/trading/0088b-trading-market-data-capability-group.md index e386006a..98febaa2 100644 --- a/dev/issues/0088b-trading-market-data-capability-group.md +++ b/dev/issues/trading/0088b-trading-market-data-capability-group.md @@ -8,7 +8,7 @@ domain: - meta scope: multi-app priority: alta -depends: [] +depends: ["0088a"] blocks: [] related: [] created: 2026-05-17 diff --git a/dev/issues/0088c-trading-broker-interface-and-paper-adapter.md b/dev/issues/trading/0088c-trading-broker-interface-and-paper-adapter.md similarity index 98% rename from dev/issues/0088c-trading-broker-interface-and-paper-adapter.md rename to dev/issues/trading/0088c-trading-broker-interface-and-paper-adapter.md index cfe1609d..67469203 100644 --- a/dev/issues/0088c-trading-broker-interface-and-paper-adapter.md +++ b/dev/issues/trading/0088c-trading-broker-interface-and-paper-adapter.md @@ -7,7 +7,7 @@ domain: - trading scope: multi-app priority: alta -depends: [] +depends: ["0088a", "0088d"] blocks: [] related: [] created: 2026-05-17 diff --git a/dev/issues/0088d-trading-portfolio-tracker-app.md b/dev/issues/trading/0088d-trading-portfolio-tracker-app.md similarity index 99% rename from dev/issues/0088d-trading-portfolio-tracker-app.md rename to dev/issues/trading/0088d-trading-portfolio-tracker-app.md index 7e8e3bc7..363f4bee 100644 --- a/dev/issues/0088d-trading-portfolio-tracker-app.md +++ b/dev/issues/trading/0088d-trading-portfolio-tracker-app.md @@ -7,7 +7,7 @@ domain: - trading scope: app-scoped priority: alta -depends: [] +depends: ["0088a"] blocks: [] related: [] created: 2026-05-17 diff --git a/dev/issues/0088e-trading-strategy-capability-group.md b/dev/issues/trading/0088e-trading-strategy-capability-group.md similarity index 99% rename from dev/issues/0088e-trading-strategy-capability-group.md rename to dev/issues/trading/0088e-trading-strategy-capability-group.md index c06de7e5..b8707e74 100644 --- a/dev/issues/0088e-trading-strategy-capability-group.md +++ b/dev/issues/trading/0088e-trading-strategy-capability-group.md @@ -8,7 +8,7 @@ domain: - meta scope: multi-app priority: alta -depends: [] +depends: ["0088a"] blocks: [] related: [] created: 2026-05-17 diff --git a/dev/issues/0088f-trading-risk-capability-group-and-kill-switch.md b/dev/issues/trading/0088f-trading-risk-capability-group-and-kill-switch.md similarity index 99% rename from dev/issues/0088f-trading-risk-capability-group-and-kill-switch.md rename to dev/issues/trading/0088f-trading-risk-capability-group-and-kill-switch.md index f5f4454b..87d20524 100644 --- a/dev/issues/0088f-trading-risk-capability-group-and-kill-switch.md +++ b/dev/issues/trading/0088f-trading-risk-capability-group-and-kill-switch.md @@ -8,7 +8,7 @@ domain: - meta scope: multi-app priority: alta -depends: [] +depends: ["0088a"] blocks: [] related: [] created: 2026-05-17 diff --git a/dev/issues/0088g-trading-backtester-app.md b/dev/issues/trading/0088g-trading-backtester-app.md similarity index 97% rename from dev/issues/0088g-trading-backtester-app.md rename to dev/issues/trading/0088g-trading-backtester-app.md index 81994a47..5ab7ead1 100644 --- a/dev/issues/0088g-trading-backtester-app.md +++ b/dev/issues/trading/0088g-trading-backtester-app.md @@ -7,7 +7,7 @@ domain: - trading scope: app-scoped priority: alta -depends: [] +depends: ["0088b", "0088c", "0088d", "0088e", "0088f"] blocks: [] related: [] created: 2026-05-17 diff --git a/dev/issues/0088h-trading-live-runner-service.md b/dev/issues/trading/0088h-trading-live-runner-service.md similarity index 97% rename from dev/issues/0088h-trading-live-runner-service.md rename to dev/issues/trading/0088h-trading-live-runner-service.md index f9d84d8d..a7fc1849 100644 --- a/dev/issues/0088h-trading-live-runner-service.md +++ b/dev/issues/trading/0088h-trading-live-runner-service.md @@ -7,7 +7,7 @@ domain: - trading scope: multi-app priority: alta -depends: [] +depends: ["0088b", "0088c", "0088d", "0088e", "0088f", "0088g"] blocks: [] related: [] created: 2026-05-17 diff --git a/dev/issues/0088i-trading-journal-app.md b/dev/issues/trading/0088i-trading-journal-app.md similarity index 98% rename from dev/issues/0088i-trading-journal-app.md rename to dev/issues/trading/0088i-trading-journal-app.md index 7e5aec48..dfdb1e7e 100644 --- a/dev/issues/0088i-trading-journal-app.md +++ b/dev/issues/trading/0088i-trading-journal-app.md @@ -7,7 +7,7 @@ domain: - trading scope: app-scoped priority: alta -depends: [] +depends: ["0088a", "0088d"] blocks: [] related: [] created: 2026-05-17 diff --git a/dev/issues/0088j-trading-reactive-loop-wiring.md b/dev/issues/trading/0088j-trading-reactive-loop-wiring.md similarity index 98% rename from dev/issues/0088j-trading-reactive-loop-wiring.md rename to dev/issues/trading/0088j-trading-reactive-loop-wiring.md index 4ebe1c33..12d204d9 100644 --- a/dev/issues/0088j-trading-reactive-loop-wiring.md +++ b/dev/issues/trading/0088j-trading-reactive-loop-wiring.md @@ -8,7 +8,7 @@ domain: - frontend scope: multi-app priority: alta -depends: [] +depends: ["0088d", "0088g", "0088h", "0088i"] blocks: [] related: [] created: 2026-05-17 From fcf5a4c6a31ab8cf81c4b956bbc48c87a93bb437 Mon Sep 17 00:00:00 2001 From: Egutierrez Date: Tue, 30 Jun 2026 14:54:49 +0200 Subject: [PATCH 04/53] =?UTF-8?q?feat(eda):=20build=5Fboxplot=5Fstats=20?= =?UTF-8?q?=E2=80=94=20estad=C3=ADsticas=20de=20boxplot=20Tukey=20desde=20?= =?UTF-8?q?sub-bloque=20numeric?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 (1M context) --- .../datascience/build_boxplot_stats.md | 58 ++++++++++ .../datascience/build_boxplot_stats.py | 94 +++++++++++++++ .../datascience/build_boxplot_stats_test.py | 108 ++++++++++++++++++ 3 files changed, 260 insertions(+) create mode 100644 python/functions/datascience/build_boxplot_stats.md create mode 100644 python/functions/datascience/build_boxplot_stats.py create mode 100644 python/functions/datascience/build_boxplot_stats_test.py diff --git a/python/functions/datascience/build_boxplot_stats.md b/python/functions/datascience/build_boxplot_stats.md new file mode 100644 index 00000000..05dd5fab --- /dev/null +++ b/python/functions/datascience/build_boxplot_stats.md @@ -0,0 +1,58 @@ +--- +name: build_boxplot_stats +kind: function +lang: py +domain: datascience +version: "1.0.0" +purity: pure +signature: "def build_boxplot_stats(numeric: dict) -> dict" +description: "Deriva las estadisticas de un boxplot de Tukey desde el sub-bloque numeric de un ColumnProfile del grupo eda (salida de describe_numeric). Aplica la regla del 1.5*IQR a los percentiles p25/p50/p75 para obtener cuartiles, fences, bigotes reales y flags de outliers. Lectura defensiva con .get; NUNCA lanza. Si faltan los percentiles clave devuelve {} para que el caller omita el grafico." +tags: [eda, statistics, profiling, boxplot, tukey, iqr, datascience] +params: + - name: numeric + desc: "Sub-bloque numeric de un ColumnProfile del grupo eda (la salida de describe_numeric). Claves esperadas (todas pueden ser None): min, max, mean, median, mode, std, variance, cv, p1, p5, p25, p50, p75, p95, p99, iqr, skew, kurtosis, n_outliers, outlier_pct, zero_pct, negative_pct, distribution_type, histogram. Solo se usan p25, median/p50, p75, min, max y n_outliers." +output: "Dict con las cifras de un boxplot horizontal de Tukey: {q1=p25, median=median(o p50), q3=p75, iqr=q3-q1, lower_fence=q1-1.5*iqr, upper_fence=q3+1.5*iqr, whisker_lo=max(min,lower_fence), whisker_hi=min(max,upper_fence), min, max, has_low_outliers=minupper_fence, n_outliers}. Numericos en float, flags en bool nativo, n_outliers en int. Si faltan p25/median(o p50)/p75 devuelve {} (dict vacio). Cuando min/max faltan, los bigotes caen a la fence correspondiente." +uses_functions: [] +uses_types: [] +returns: [] +returns_optional: false +error_type: "" +imports: [] +tested: true +tests: ["test_boxplot_tukey_basico", "test_percentiles_faltan_devuelve_vacio", "test_median_cae_a_p50", "test_whiskers_usan_fence_si_falta_min_max", "test_tipos_salida_float_bool_int"] +test_file_path: "python/functions/datascience/build_boxplot_stats_test.py" +file_path: "python/functions/datascience/build_boxplot_stats.py" +--- + +## Ejemplo + +```python +import sys, os +sys.path.insert(0, os.path.join("python", "functions")) +from datascience.build_boxplot_stats import build_boxplot_stats + +# Sub-bloque numeric tal y como lo produce describe_numeric: +numeric = { + "min": 1.0, "max": 100.0, + "p25": 10.0, "median": 25.0, "p75": 40.0, + "iqr": 30.0, "n_outliers": 3, +} +box = build_boxplot_stats(numeric) +print(box["lower_fence"], box["upper_fence"]) # -35.0 85.0 +print(box["whisker_lo"], box["whisker_hi"]) # 1.0 85.0 +print(box["has_low_outliers"], box["has_high_outliers"]) # False True +``` + +## Cuando usarla + +- Usala al dibujar un boxplot horizontal bajo el histograma en el capitulo `num_distr` de `AutomaticEDA`: convierte el bloque `numeric` de un `ColumnProfile` en las cifras exactas que el renderer necesita (cuartiles, fences, extremos de los bigotes y flags de outliers). +- Cuando ya tengas los percentiles calculados (salida de `describe_numeric`) y solo necesites derivar la geometria del boxplot de Tukey sin volver a tocar los valores crudos. +- Cuando quieras decidir si una columna tiene cola alta/baja (`has_high_outliers` / `has_low_outliers`) antes de proponer una transformacion (log, winsorize). + +## Gotchas + +- Funcion pura, sin I/O y determinista. Lectura defensiva con `.get`: NUNCA lanza. Si faltan `p25`, `median`/`p50` o `p75` devuelve `{}` (dict vacio) — el caller debe omitir el boxplot. +- Los `n_outliers` que se propagan vienen del bloque z-score del profile (`detect_outliers`, threshold 3.0), NO de la regla IQR. Son informativos: el conteo de Tukey que esta funcion calcula son los **fences** (`lower_fence`/`upper_fence`), no un recuento de puntos. +- No recibe los valores crudos de la columna, solo deriva cifras desde los percentiles ya calculados. Por eso no puede contar cuantos puntos caen fuera de las fences, solo si los extremos (`min`/`max`) las superan. +- `iqr` se recalcula como `q3 - q1` aunque el bloque traiga `numeric['iqr']`: asi funciona aunque esa clave falte. +- Cuando `min`/`max` faltan, los bigotes caen a la fence correspondiente y los flags de outliers quedan en `False` (sin extremo real no se afirma cola). diff --git a/python/functions/datascience/build_boxplot_stats.py b/python/functions/datascience/build_boxplot_stats.py new file mode 100644 index 00000000..6fbcdc86 --- /dev/null +++ b/python/functions/datascience/build_boxplot_stats.py @@ -0,0 +1,94 @@ +"""build_boxplot_stats — Tukey boxplot statistics from an EDA `numeric` sub-block. + +Pure function: no I/O, deterministic. Takes the `numeric` dict of a ColumnProfile +(group `eda`, the output of describe_numeric) and derives the figures needed to +draw a horizontal Tukey boxplot using the 1.5 * IQR rule. + +It only derives numbers from already-computed percentiles; it never sees the raw +column values. Reading is defensive (.get throughout) and the function NEVER +raises: if the key percentiles (p25 / p50 / p75) are missing it returns {} so the +caller can simply skip the boxplot. +""" + + +def _num(value): + """Coerce to float defensively; return None for None/bool/non-numeric.""" + # bool is a subclass of int; a percentile value is never a real bool, so + # treat True/False as missing instead of silently coercing to 1.0/0.0. + if value is None or isinstance(value, bool): + return None + try: + return float(value) + except (TypeError, ValueError): + return None + + +def build_boxplot_stats(numeric: dict) -> dict: + """Derive Tukey boxplot statistics from the `numeric` sub-block of a profile. + + Reads the percentiles already computed by describe_numeric and applies the + classic 1.5 * IQR fence rule to obtain the whisker extremes and outlier + flags of a horizontal boxplot. No raw values are needed. + + Args: + numeric: The `numeric` sub-block of an eda ColumnProfile (output of + describe_numeric). Every value may be None; read defensively. + + Returns: + Dict with the boxplot figures + {q1, median, q3, iqr, lower_fence, upper_fence, whisker_lo, whisker_hi, + min, max, has_low_outliers, has_high_outliers, n_outliers}. + If p25, p50/median or p75 are missing (None) returns {} (empty dict) so + the caller omits the plot. + """ + if not isinstance(numeric, dict): + return {} + + q1 = _num(numeric.get("p25")) + q3 = _num(numeric.get("p75")) + # Prefer the explicit median; fall back to p50 (they are the same quantile). + median = _num(numeric.get("median")) + if median is None: + median = _num(numeric.get("p50")) + + # Without the three quartiles a boxplot cannot be drawn. + if q1 is None or q3 is None or median is None: + return {} + + # Recompute the IQR from the quartiles rather than trusting numeric['iqr'], + # which may be missing even when the percentiles are present. + iqr = q3 - q1 + lower_fence = q1 - 1.5 * iqr + upper_fence = q3 + 1.5 * iqr + + mn = _num(numeric.get("min")) + mx = _num(numeric.get("max")) + + # Whisker extremes: the real data range clamped to the fences. When the + # corresponding extreme is missing, fall back to the fence itself. + whisker_lo = max(mn, lower_fence) if mn is not None else lower_fence + whisker_hi = min(mx, upper_fence) if mx is not None else upper_fence + + has_low_outliers = bool(mn is not None and mn < lower_fence) + has_high_outliers = bool(mx is not None and mx > upper_fence) + + # Informative only: these outliers come from the z-score block of the + # profile, not from this IQR fence computation. + raw_n = numeric.get("n_outliers") + n_outliers = int(raw_n) if isinstance(raw_n, (int, float)) and not isinstance(raw_n, bool) else 0 + + return { + "q1": q1, + "median": median, + "q3": q3, + "iqr": iqr, + "lower_fence": lower_fence, + "upper_fence": upper_fence, + "whisker_lo": whisker_lo, + "whisker_hi": whisker_hi, + "min": mn, + "max": mx, + "has_low_outliers": has_low_outliers, + "has_high_outliers": has_high_outliers, + "n_outliers": n_outliers, + } diff --git a/python/functions/datascience/build_boxplot_stats_test.py b/python/functions/datascience/build_boxplot_stats_test.py new file mode 100644 index 00000000..dbf541e3 --- /dev/null +++ b/python/functions/datascience/build_boxplot_stats_test.py @@ -0,0 +1,108 @@ +"""Tests para build_boxplot_stats.""" + +import os +import sys + +sys.path.insert(0, os.path.dirname(__file__)) + +from build_boxplot_stats import build_boxplot_stats + +# Keys that a non-empty result dict must always contain. +_EXPECTED_KEYS = { + "q1", "median", "q3", "iqr", "lower_fence", "upper_fence", + "whisker_lo", "whisker_hi", "min", "max", + "has_low_outliers", "has_high_outliers", "n_outliers", +} + + +def test_boxplot_tukey_basico(): + """Golden: bloque numeric con outlier alto claro -> fences IQR de Tukey.""" + numeric = { + "min": 1.0, "max": 100.0, + "p25": 10.0, "median": 25.0, "p75": 40.0, + "iqr": 30.0, "n_outliers": 3, + } + box = build_boxplot_stats(numeric) + + assert set(box.keys()) == _EXPECTED_KEYS + + assert box["q1"] == 10.0 + assert box["median"] == 25.0 + assert box["q3"] == 40.0 + # iqr recomputado desde los cuartiles. + assert box["iqr"] == 30.0 + # lower = 10 - 1.5*30 = -35 ; upper = 40 + 1.5*30 = 85. + assert box["lower_fence"] == -35.0 + assert box["upper_fence"] == 85.0 + # whisker_lo = max(min=1, -35) = 1 ; whisker_hi = min(max=100, 85) = 85. + assert box["whisker_lo"] == 1.0 + assert box["whisker_hi"] == 85.0 + assert box["min"] == 1.0 + assert box["max"] == 100.0 + # Solo hay outliers altos (100 > 85), no bajos (1 no < -35). + assert box["has_low_outliers"] is False + assert box["has_high_outliers"] is True + # n_outliers se propaga del bloque z-score (informativo). + assert box["n_outliers"] == 3 + + +def test_percentiles_faltan_devuelve_vacio(): + """Si falta p25/median/p75 -> {} (caller omite el boxplot).""" + # Falta p25. + assert build_boxplot_stats({"median": 25.0, "p75": 40.0}) == {} + # Falta p75. + assert build_boxplot_stats({"p25": 10.0, "median": 25.0}) == {} + # Falta median y p50. + assert build_boxplot_stats({"p25": 10.0, "p75": 40.0}) == {} + # numeric None / no dict tambien es vacio, nunca lanza. + assert build_boxplot_stats(None) == {} + assert build_boxplot_stats({}) == {} + + +def test_median_cae_a_p50(): + """median ausente cae a p50.""" + numeric = {"min": 0.0, "max": 10.0, "p25": 2.0, "p50": 5.0, "p75": 8.0} + box = build_boxplot_stats(numeric) + assert box["median"] == 5.0 + assert box["q1"] == 2.0 + assert box["q3"] == 8.0 + + +def test_whiskers_usan_fence_si_falta_min_max(): + """Sin min/max los bigotes caen a las fences y no hay outliers marcados.""" + numeric = {"p25": 10.0, "median": 25.0, "p75": 40.0} # sin min ni max + box = build_boxplot_stats(numeric) + + assert box["min"] is None + assert box["max"] is None + # iqr = 30, fences -35 / 85; los bigotes caen a las fences. + assert box["whisker_lo"] == box["lower_fence"] == -35.0 + assert box["whisker_hi"] == box["upper_fence"] == 85.0 + # Sin extremos reales, no se afirma que haya outliers. + assert box["has_low_outliers"] is False + assert box["has_high_outliers"] is False + # n_outliers ausente -> 0. + assert box["n_outliers"] == 0 + + +def test_tipos_salida_float_bool_int(): + """Numericos en float, flags bool nativos, n_outliers int.""" + numeric = { + "min": -50.0, "max": 200.0, + "p25": 10.0, "median": 25.0, "p75": 40.0, + "n_outliers": 7, + } + box = build_boxplot_stats(numeric) + + for key in ("q1", "median", "q3", "iqr", "lower_fence", "upper_fence", + "whisker_lo", "whisker_hi", "min", "max"): + assert isinstance(box[key], float), f"{key} debe ser float" + + assert isinstance(box["has_low_outliers"], bool) + assert isinstance(box["has_high_outliers"], bool) + assert isinstance(box["n_outliers"], int) and not isinstance(box["n_outliers"], bool) + + # min=-50 < lower_fence=-35 -> outlier bajo ; max=200 > upper_fence=85 -> alto. + assert box["has_low_outliers"] is True + assert box["has_high_outliers"] is True + assert box["n_outliers"] == 7 From 4de071f2f9da4afbd8d9384fb2310808358ccc67 Mon Sep 17 00:00:00 2001 From: Egutierrez Date: Tue, 30 Jun 2026 14:57:27 +0200 Subject: [PATCH 05/53] feat(eda): project_clusters_2d + describe_clusters_llm para el capitulo MODELOS project_clusters_2d (pura): PCA(2)+KMeans sobre el MISMO subset estandarizado, devolviendo proyeccion 2D y labels alineados por fila + centroides en espacio PCA + perfiles de cluster desestandarizados. Es la pieza que garantiza la alineacion points<->labels que pca_explained y kmeans_segments no cubren (estandarizan por separado y kmeans descarta los labels). Habilita el scatter PCA coloreado por cluster (MUST-8.1). describe_clusters_llm (impura): micro-analisis LLM de los clusters en una sola llamada a ask_llm (grupo claude-direct), devuelve titulo + descripcion por cluster con degradacion dict-no-throw a titulos genericos si el LLM no responde (MUST-8.2). Ambas re-exportadas en datascience/__init__.py. Tests: 6/6 y 9/9 (sin red). Co-Authored-By: Claude Opus 4.8 (1M context) --- python/functions/datascience/__init__.py | 4 + .../datascience/describe_clusters_llm.md | 97 +++++++ .../datascience/describe_clusters_llm.py | 240 ++++++++++++++++++ .../datascience/describe_clusters_llm_test.py | 160 ++++++++++++ .../datascience/project_clusters_2d.md | 95 +++++++ .../datascience/project_clusters_2d.py | 208 +++++++++++++++ .../datascience/project_clusters_2d_test.py | 127 +++++++++ 7 files changed, 931 insertions(+) create mode 100644 python/functions/datascience/describe_clusters_llm.md create mode 100644 python/functions/datascience/describe_clusters_llm.py create mode 100644 python/functions/datascience/describe_clusters_llm_test.py create mode 100644 python/functions/datascience/project_clusters_2d.md create mode 100644 python/functions/datascience/project_clusters_2d.py create mode 100644 python/functions/datascience/project_clusters_2d_test.py diff --git a/python/functions/datascience/__init__.py b/python/functions/datascience/__init__.py index afa5ac45..9fc8c206 100644 --- a/python/functions/datascience/__init__.py +++ b/python/functions/datascience/__init__.py @@ -42,6 +42,8 @@ from .isolation_forest_outliers import isolation_forest_outliers from .normality_tests import normality_tests from .trend_slope import trend_slope from .run_eda_models import run_eda_models +from .project_clusters_2d import project_clusters_2d +from .describe_clusters_llm import describe_clusters_llm from .eda_llm_insights import eda_llm_insights from .build_eda_notebook import build_eda_notebook from .decode_qr_image import decode_qr_image @@ -86,6 +88,8 @@ __all__ = [ "normality_tests", "trend_slope", "run_eda_models", + "project_clusters_2d", + "describe_clusters_llm", "eda_llm_insights", "build_eda_notebook", "describe_numeric", diff --git a/python/functions/datascience/describe_clusters_llm.md b/python/functions/datascience/describe_clusters_llm.md new file mode 100644 index 00000000..9b99e730 --- /dev/null +++ b/python/functions/datascience/describe_clusters_llm.md @@ -0,0 +1,97 @@ +--- +name: describe_clusters_llm +kind: function +lang: py +domain: datascience +version: "1.0.0" +purity: impure +signature: "def describe_clusters_llm(cluster_profiles: list, feature_names: list, model: str = \"claude-haiku-4-5-20251001\") -> dict" +description: "Micro-analisis LLM de clusters de KMeans (grupo eda). Toma los perfiles AGREGADOS de cada cluster (los que produce project_clusters_2d: tamano, centroide en escala original, features distintivas y centroide en z-score) y, con UNA sola llamada al LLM, pide por cada cluster un TITULO corto + una descripcion de 1-2 frases en espanol. Clave de coste/privacidad: NO envia filas crudas, solo el resumen agregado de cada grupo (tamano, % del total y la media de las features distintivas con su signo respecto a la media global). Reusa ask_llm del grupo claude-direct (API directa con token OAuth de Claude). Impura, dict-no-throw: nunca lanza, degrada a titulos genericos 'Cluster N' si el LLM no responde o el parseo falla." +tags: [eda, clustering, llm, claude-direct, datascience, kmeans] +params: + - name: cluster_profiles + desc: "Lista de perfiles de cluster con la forma que produce project_clusters_2d: cada uno {cluster:int, size:int, pct:float, centroid_original:{feature: media en escala original}, distinctive:[features distintivas], centroid_z:{feature: z-score}}. Solo se le envia al LLM un resumen agregado; nunca filas crudas. Lista vacia o no-lista -> clusters=[] sin llamar al LLM." + - name: feature_names + desc: "Nombres de las features del dataset. Se incluyen como contexto en el prompt para que el LLM pueda nombrar los clusters; no es obligatorio que coincida con las features distintivas de cada perfil." + - name: model + desc: "id del modelo Anthropic a usar. Default 'claude-haiku-4-5-20251001' (haiku, coste bajo, ~2-3s). Para titulos/descripciones mas finas, pasar p.ej. 'claude-opus-4-8'." +output: "dict dict-no-throw: {clusters:[{cluster:int, title:str, description:str}], model:str, note:str}. note=='' si todo fue bien. Si el LLM no respondio (note='LLM no disponible') o el parseo fallo (note='parse fallido'), clusters trae titulos genericos 'Cluster N' con description vacia. Si cluster_profiles esta vacio o no es lista: {clusters:[], model, note:'sin clusters'}. NUNCA lanza." +uses_functions: [ask_llm_py_core] +uses_types: [] +returns: [] +returns_optional: false +error_type: "error_go_core" +imports: [] +tested: true +tests: ["test_parse_clusters_json_valid_array", "test_parse_clusters_json_wrapped_in_junk_text", "test_parse_clusters_json_non_json_returns_none", "test_parse_clusters_json_fills_missing_cluster_by_index", "test_describe_clusters_llm_ok_with_monkeypatched_llm", "test_describe_clusters_llm_degrades_on_empty_response", "test_describe_clusters_llm_degrades_on_unparseable_response", "test_describe_clusters_llm_empty_list_skips_llm", "test_describe_clusters_llm_non_list_input_skips_llm"] +test_file_path: "python/functions/datascience/describe_clusters_llm_test.py" +file_path: "python/functions/datascience/describe_clusters_llm.py" +--- + +## Ejemplo + +```python +import sys, os +sys.path.insert(0, os.path.join("python", "functions")) + +from datascience.describe_clusters_llm import describe_clusters_llm + +# Perfiles agregados producidos por project_clusters_2d (no hay filas crudas). +cluster_profiles = [ + { + "cluster": 0, "size": 60, "pct": 60.0, + "centroid_original": {"acidez": 8.5, "alcohol": 9.2}, + "distinctive": ["acidez", "alcohol"], + "centroid_z": {"acidez": 1.4, "alcohol": -0.9}, + }, + { + "cluster": 1, "size": 40, "pct": 40.0, + "centroid_original": {"acidez": 5.1, "alcohol": 13.0}, + "distinctive": ["alcohol"], + "centroid_z": {"acidez": -0.7, "alcohol": 1.6}, + }, +] +feature_names = ["acidez", "alcohol", "azucar"] + +out = describe_clusters_llm(cluster_profiles, feature_names) # haiku por defecto +# out = describe_clusters_llm(cluster_profiles, feature_names, model="claude-opus-4-8") + +if not out["note"]: + for c in out["clusters"]: + print(f"Cluster {c['cluster']}: {c['title']}") + print(" ", c["description"]) +else: + # Degradacion: titulos genericos "Cluster N". + print("LLM no usado:", out["note"]) + for c in out["clusters"]: + print(c["cluster"], c["title"]) +``` + +## Cuando usarla + +Cuando ya has clusterizado un dataset (KMeans + `project_clusters_2d`) y quieres +poner NOMBRE y descripcion legible a cada grupo en vez de dejar "Cluster 0/1/2". +Es el paso interpretativo que sigue al perfilado de clusters: `project_clusters_2d` +calcula tamano, centroides y features distintivas, y `describe_clusters_llm` los +traduce a un titulo corto + 1-2 frases por cluster. Usala al cerrar un EDA con +segmentacion para el resumen final o el report. Una sola llamada al LLM describe +todos los clusters a la vez (barato). + +## Gotchas + +- **Impura: hace 1 llamada de red al LLM.** No es determinista ni gratis. Latencia + tipica ~2-3s con haiku. +- **Requiere token OAuth de Claude** en `~/.claude/.credentials.json` (via `ask_llm` + / grupo `claude-direct`). Sin token / sin red, NO lanza: degrada a titulos + genericos `Cluster N` con `note="LLM no disponible"`. +- **NO envia filas crudas al LLM**, solo el resumen AGREGADO de cada cluster + (tamano, % del total y la media de las features distintivas con su signo respecto + a la media global). Privacidad y coste minimos por diseno — pero requiere que los + perfiles vengan ya calculados por `project_clusters_2d`. +- **Modelo `haiku` por defecto** para coste bajo; sube a `claude-opus-4-8` si + necesitas titulos/descripciones mas finas (mas caro y lento). +- **dict-no-throw**: si el modelo no devuelve un JSON array parseable, retorna + titulos genericos con `note="parse fallido"`. Comprueba siempre `out["note"]` + antes de fiarte de los titulos. +- El LLM puede sobre-interpretar: el system prompt le pide ser sobrio y no inventar + causas, pero revisa los titulos antes de publicarlos en un report. diff --git a/python/functions/datascience/describe_clusters_llm.py b/python/functions/datascience/describe_clusters_llm.py new file mode 100644 index 00000000..306176bb --- /dev/null +++ b/python/functions/datascience/describe_clusters_llm.py @@ -0,0 +1,240 @@ +"""describe_clusters_llm — micro-analisis LLM de clusters de KMeans (grupo `eda`). + +Toma los PERFILES AGREGADOS de cada cluster (los que produce `project_clusters_2d`: +tamano, centroide en escala original, features distintivas y centroide en z-score) +y, con UNA sola llamada al LLM, pide por cada cluster un TITULO corto + una +descripcion de 1-2 frases, en espanol. + +Clave de coste y privacidad: NO se envian filas crudas al LLM. Solo viaja el +perfil AGREGADO de cada grupo (tamano, % del total y la media de las features +distintivas con su signo respecto a la media global). El coste es minimo y ningun +dato fila-a-fila sale del proceso. + +Reusa `ask_llm` del registry (grupo claude-direct, API directa con el token OAuth +de Claude en ~/.claude/.credentials.json, arranque 0). Impura: una llamada de red. +Estilo dict-no-throw: NUNCA lanza; ante cualquier fallo (red, LLM caido, parseo) +degrada a titulos genericos "Cluster N" + una nota explicando el motivo. +""" + +import json + +from core.ask_llm import ask_llm + +_SYSTEM = ( + "Eres un analista de datos. Recibes los PERFILES AGREGADOS de los clusters de " + "un KMeans (por cada grupo: su tamano y la media de sus features distintivas, " + "con el signo respecto a la media global; nunca filas crudas) y los describes " + "de forma sobria y util. Para cada cluster generas un titulo corto y " + "descriptivo (por ejemplo 'Vinos de alta acidez y baja graduacion') y una " + "descripcion de 1-2 frases. NO inventes causas ni sobre-interpretes: limitate a " + "lo que dicen los numeros. Responde en espanol. Responde SIEMPRE y SOLO con un " + "unico JSON array valido, sin texto alrededor y sin fences de markdown, con " + 'EXACTAMENTE la forma [{"cluster": , "title": "", ' + '"description": "<1-2 frases>"}], un objeto por cluster.' +) + + +def _fmt_num(value) -> str: + """Formatea un numero de forma compacta para el prompt (None -> '?').""" + if value is None: + return "?" + if isinstance(value, bool): + return str(value) + if isinstance(value, float): + if value == int(value): + return str(int(value)) + return f"{value:.4g}" + return str(value) + + +def _cluster_id(profile: dict, index: int) -> int: + """Devuelve el id del cluster del perfil, o el indice si no es un int valido.""" + raw = (profile or {}).get("cluster") + if isinstance(raw, bool): + return index + if isinstance(raw, int): + return raw + try: + return int(raw) + except (TypeError, ValueError): + return index + + +def _build_prompt(cluster_profiles: list, feature_names: list) -> str: + """Construye un resumen textual compacto de los perfiles para el LLM. + + Funcion interna PURA: no toca red ni disco, es testeable sin credenciales. + Por cada cluster incluye su numero, tamano (size + pct%) y, para cada feature + distintiva, el valor del centroide en escala original mas si esta por encima o + por debajo de la media (signo del z-score en centroid_z). Pasa AGREGADOS, nunca + dato crudo de filas. + + Args: + cluster_profiles: lista de perfiles de cluster (forma de project_clusters_2d). + feature_names: nombres de las features del dataset (solo contexto). + + Returns: + El texto del prompt. + """ + cluster_profiles = cluster_profiles or [] + feature_names = feature_names if isinstance(feature_names, list) else [] + + lines = [ + "Perfiles AGREGADOS de clusters de KMeans. No hay filas crudas, solo medias por grupo.", + f"Numero de clusters: {len(cluster_profiles)}", + ] + if feature_names: + lines.append("Features del dataset: " + ", ".join(str(f) for f in feature_names)) + lines.append("") + + for i, prof in enumerate(cluster_profiles): + prof = prof or {} + cid = _cluster_id(prof, i) + size = prof.get("size") + pct = prof.get("pct") + pct_str = f"{pct:.1f}%" if isinstance(pct, (int, float)) and not isinstance(pct, bool) else "?" + lines.append(f"Cluster {cid}: tamano={_fmt_num(size)} ({pct_str} del total)") + + distinctive = prof.get("distinctive") or [] + centroid_o = prof.get("centroid_original") or {} + centroid_z = prof.get("centroid_z") or {} + + if distinctive: + lines.append(" Features distintivas (media del grupo):") + for feat in distinctive: + val = centroid_o.get(feat) + z = centroid_z.get(feat) + direction = "" + if isinstance(z, (int, float)) and not isinstance(z, bool): + if z > 0: + direction = "por encima de la media" + elif z < 0: + direction = "por debajo de la media" + else: + direction = "en la media" + if direction: + lines.append(f" - {feat}: {_fmt_num(val)} ({direction})") + else: + lines.append(f" - {feat}: {_fmt_num(val)}") + else: + lines.append(" (sin features distintivas marcadas)") + lines.append("") + + lines.append( + "Devuelve SOLO el JSON array descrito en las instrucciones del sistema, " + "sin texto antes ni despues." + ) + return "\n".join(lines) + + +def _parse_clusters_json(text: str, n: int): + """Extrae y normaliza el array JSON de la respuesta del LLM. + + Funcion interna testeable sin red. Localiza el primer '[' y el ultimo ']' del + texto (tolerando texto basura alrededor o fences de markdown), hace json.loads + y normaliza cada entrada a {cluster:int, title:str, description:str}, rellenando + el cluster por indice si falta. NUNCA lanza: ante cualquier fallo devuelve None + (senal de degradacion para el caller). + + Args: + text: respuesta cruda del LLM. + n: numero de perfiles esperados (referencia; la longitud real la marca el array). + + Returns: + Lista normalizada de dicts, o None si no se pudo parsear un array valido. + """ + if not text or not isinstance(text, str): + return None + + start = text.find("[") + end = text.rfind("]") + if start == -1 or end == -1 or end <= start: + return None + + try: + data = json.loads(text[start : end + 1]) + except (ValueError, TypeError): + return None + + if not isinstance(data, list): + return None + + out = [] + for i, item in enumerate(data): + if not isinstance(item, dict): + out.append({"cluster": i, "title": f"Cluster {i}", "description": ""}) + continue + + raw_cluster = item.get("cluster") + if isinstance(raw_cluster, bool): + cluster = i + elif isinstance(raw_cluster, int): + cluster = raw_cluster + else: + try: + cluster = int(raw_cluster) + except (TypeError, ValueError): + cluster = i + + title = item.get("title") + title = str(title) if title is not None else f"Cluster {cluster}" + + desc = item.get("description") + desc = str(desc) if desc is not None else "" + + out.append({"cluster": cluster, "title": title, "description": desc}) + + return out + + +def _generic_clusters(cluster_profiles: list) -> list: + """Titulos genericos por cluster para la degradacion (sin LLM).""" + out = [] + for i, prof in enumerate(cluster_profiles): + cid = _cluster_id(prof or {}, i) + out.append({"cluster": cid, "title": f"Cluster {cid}", "description": ""}) + return out + + +def describe_clusters_llm( + cluster_profiles: list, + feature_names: list, + model: str = "claude-haiku-4-5-20251001", +) -> dict: + """Describe los clusters de un KMeans con UNA sola llamada al LLM. + + Args: + cluster_profiles: lista de perfiles de cluster (la forma que produce + project_clusters_2d): cada uno {"cluster": int, "size": int, + "pct": float, "centroid_original": {feature: media}, + "distinctive": [features], "centroid_z": {feature: z}}. Solo se le + envia al LLM el resumen agregado, nunca filas crudas. + feature_names: nombres de las features del dataset (contexto para el LLM). + model: id del modelo Anthropic. Default claude-haiku-4-5-20251001 + (haiku, coste bajo). + + Returns: + dict dict-no-throw: {"clusters": [{cluster:int, title:str, description:str}], + "model": str, "note": str}. note == "" si todo fue bien; si el LLM no + respondio o el parseo fallo, clusters trae titulos genericos "Cluster N" y + note explica el motivo ("LLM no disponible" / "parse fallido"). Si + cluster_profiles esta vacio o no es lista, devuelve clusters=[] sin llamar + al LLM (note "sin clusters"). NUNCA lanza. + """ + if not isinstance(cluster_profiles, list) or not cluster_profiles: + return {"clusters": [], "model": model, "note": "sin clusters"} + + n = len(cluster_profiles) + prompt = _build_prompt(cluster_profiles, feature_names) + + try: + text = ask_llm(prompt, model=model, system=_SYSTEM, echo=False) + except Exception: # noqa: BLE001 — degradacion: cualquier fallo de red/LLM. + text = "" + + parsed = _parse_clusters_json(text, n) + if parsed: + return {"clusters": parsed, "model": model, "note": ""} + + note = "LLM no disponible" if not text else "parse fallido" + return {"clusters": _generic_clusters(cluster_profiles), "model": model, "note": note} diff --git a/python/functions/datascience/describe_clusters_llm_test.py b/python/functions/datascience/describe_clusters_llm_test.py new file mode 100644 index 00000000..c7ee3fae --- /dev/null +++ b/python/functions/datascience/describe_clusters_llm_test.py @@ -0,0 +1,160 @@ +"""Tests para describe_clusters_llm. + +NO acceden a red ni a credenciales: _parse_clusters_json es testeable aislada y la +unica via que llamaria al LLM (describe_clusters_llm) se prueba monkeypatcheando +ask_llm con respuestas simuladas. Cubre golden (LLM ok), edge (cluster faltante, +array envuelto en basura, lista vacia / input no-lista) y error (LLM caido, texto +no parseable) — todos sin tocar la red. +""" + +import importlib +import json + +from datascience.describe_clusters_llm import ( + _parse_clusters_json, + describe_clusters_llm, +) + +# Perfiles de ejemplo con la forma que produce project_clusters_2d. +_PROFILES = [ + { + "cluster": 0, + "size": 60, + "pct": 60.0, + "centroid_original": {"acidez": 8.5, "alcohol": 9.2}, + "distinctive": ["acidez", "alcohol"], + "centroid_z": {"acidez": 1.4, "alcohol": -0.9}, + }, + { + "cluster": 1, + "size": 40, + "pct": 40.0, + "centroid_original": {"acidez": 5.1, "alcohol": 13.0}, + "distinctive": ["alcohol"], + "centroid_z": {"acidez": -0.7, "alcohol": 1.6}, + }, +] +_FEATURES = ["acidez", "alcohol", "azucar"] + + +def _patch_ask_llm(monkeypatch, returner): + """Monkeypatchea ask_llm en el modulo bajo prueba con un callable simulado.""" + mod = importlib.import_module("datascience.describe_clusters_llm") + monkeypatch.setattr( + mod, "ask_llm", lambda prompt, model="x", system="", echo=True: returner + ) + + +# --- _parse_clusters_json (parser puro, sin red) --- + + +def test_parse_clusters_json_valid_array(): + text = json.dumps( + [ + {"cluster": 0, "title": "A", "description": "desc a"}, + {"cluster": 1, "title": "B", "description": "desc b"}, + ] + ) + parsed = _parse_clusters_json(text, 2) + assert parsed == [ + {"cluster": 0, "title": "A", "description": "desc a"}, + {"cluster": 1, "title": "B", "description": "desc b"}, + ] + + +def test_parse_clusters_json_wrapped_in_junk_text(): + payload = [{"cluster": 0, "title": "Solo uno", "description": "d"}] + text = "Claro, aqui tienes el resultado:\n" + json.dumps(payload) + "\nEspero que sirva." + parsed = _parse_clusters_json(text, 1) + assert parsed[0]["title"] == "Solo uno" + assert parsed[0]["cluster"] == 0 + + +def test_parse_clusters_json_non_json_returns_none(): + # Texto sin array JSON -> degradacion (None) sin lanzar. + assert _parse_clusters_json("no hay json aqui", 2) is None + assert _parse_clusters_json("", 2) is None + assert _parse_clusters_json("{solo un objeto}", 2) is None + + +def test_parse_clusters_json_fills_missing_cluster_by_index(): + text = json.dumps( + [ + {"title": "A", "description": "d"}, + {"title": "B", "description": "e"}, + ] + ) + parsed = _parse_clusters_json(text, 2) + assert parsed[0]["cluster"] == 0 + assert parsed[1]["cluster"] == 1 + assert parsed[0]["title"] == "A" + + +# --- describe_clusters_llm (con ask_llm monkeypatcheado, sin red) --- + + +def test_describe_clusters_llm_ok_with_monkeypatched_llm(monkeypatch): + fake = json.dumps( + [ + { + "cluster": 0, + "title": "Vinos de alta acidez", + "description": "Acidez por encima de la media y graduacion baja.", + }, + { + "cluster": 1, + "title": "Vinos de alta graduacion", + "description": "Alcohol claramente por encima de la media.", + }, + ] + ) + _patch_ask_llm(monkeypatch, fake) + + out = describe_clusters_llm(_PROFILES, _FEATURES) + assert out["note"] == "" + assert out["model"] == "claude-haiku-4-5-20251001" + assert len(out["clusters"]) == 2 + assert out["clusters"][0]["title"] == "Vinos de alta acidez" + assert set(out["clusters"][0].keys()) == {"cluster", "title", "description"} + + +def test_describe_clusters_llm_degrades_on_empty_response(monkeypatch): + # ask_llm devuelve "" (error/red caida) -> titulos genericos + note. + _patch_ask_llm(monkeypatch, "") + + out = describe_clusters_llm(_PROFILES, _FEATURES) + assert out["clusters"][0]["title"] == "Cluster 0" + assert out["clusters"][1]["title"] == "Cluster 1" + assert out["clusters"][0]["description"] == "" + assert out["note"] == "LLM no disponible" + assert out["model"] == "claude-haiku-4-5-20251001" + + +def test_describe_clusters_llm_degrades_on_unparseable_response(monkeypatch): + _patch_ask_llm(monkeypatch, "lo siento, no puedo ayudarte con eso") + + out = describe_clusters_llm(_PROFILES, _FEATURES) + assert out["clusters"][0]["title"] == "Cluster 0" + assert out["clusters"][1]["title"] == "Cluster 1" + assert out["note"] == "parse fallido" + + +def test_describe_clusters_llm_empty_list_skips_llm(monkeypatch): + # Con lista vacia NO debe llamarse al LLM en absoluto. + def boom(*args, **kwargs): + raise AssertionError("ask_llm no debe llamarse con lista vacia") + + mod = importlib.import_module("datascience.describe_clusters_llm") + monkeypatch.setattr(mod, "ask_llm", boom) + + out = describe_clusters_llm([], _FEATURES) + assert out["clusters"] == [] + assert out["note"] == "sin clusters" + + +def test_describe_clusters_llm_non_list_input_skips_llm(): + # Input no-lista (None) -> clusters vacio sin tocar la red. + out = describe_clusters_llm(None, _FEATURES) + assert out["clusters"] == [] + assert out["note"] == "sin clusters" + assert out["model"] == "claude-haiku-4-5-20251001" diff --git a/python/functions/datascience/project_clusters_2d.md b/python/functions/datascience/project_clusters_2d.md new file mode 100644 index 00000000..f0013253 --- /dev/null +++ b/python/functions/datascience/project_clusters_2d.md @@ -0,0 +1,95 @@ +--- +name: project_clusters_2d +kind: function +lang: py +domain: datascience +version: "1.0.0" +purity: pure +signature: "def project_clusters_2d(columns: dict, k_min: int = 2, k_max: int = 8, max_points: int = 2000) -> dict" +description: "PCA a 2D + KMeans sobre el MISMO subset numerico estandarizado, devolviendo proyeccion 2D y labels de cluster ALINEADOS por fila para pintar un scatter PCA coloreado por cluster. Estandariza una sola vez, elige k por silhouette y proyecta centroides al espacio PCA. Determinista." +tags: [eda, models, clustering, pca, kmeans, scatter, dimensionality-reduction, datascience, sklearn] +params: + - name: columns + desc: "Mapa {nombre_columna: [valores numericos]}. Listas alineadas por fila (misma longitud). Columnas no numericas o con <2 valores distintos se descartan; None/NaN descartan la fila completa (listwise)." + - name: k_min + desc: "Numero minimo de clusters a probar por silhouette (default 2). El minimo de filas validas requerido es max(3, k_min*2)." + - name: k_max + desc: "Numero maximo de clusters a probar (default 8). Se acota a min(k_max, n_filas_validas-1)." + - name: max_points + desc: "Tope de puntos devueltos en points/labels (default 2000). Si n_used lo supera, points y labels se submuestrean CONJUNTAMENTE con paso determinista para seguir alineados; el fit usa siempre todas las filas." +output: "dict con points (proyeccion 2D, posiblemente submuestreada a max_points), labels (cluster de cada point, alineado con points), centers_2d (centroides en espacio PCA, len==best_k), best_k, silhouette, explained_2d ([var PC1, var PC2]), cluster_sizes (sobre n_used total), cluster_profiles (lista de {cluster, size, pct, centroid_original, distinctive top-3 por |z|, centroid_z}), feature_names, n_used (filas del fit antes de muestreo) y note (\"\" si ok). Con <2 columnas numericas o max_points` (paso + determinista `[::ceil(n_used/max_points)]`); `n_used`, `centers_2d`, + `cluster_sizes` y `cluster_profiles` se calculan SIEMPRE sobre todas las filas. + Cuando hay submuestreo, `note` lo indica. +- `centroid_z` y `distinctive` estan en z-score (espacio escalado); + `centroid_original` esta en las unidades originales (via + `scaler.inverse_transform`). No mezcles ambos al interpretar. +- `centers_2d` esta en el espacio PCA (coordenadas del scatter), no en unidades + originales: pintalo sobre el mismo eje que `points`. +- Silhouette baja con best_k alto sugiere que no hay estructura de cluster real; + el scatter puede no mostrar grupos separados. + +## Notas + +Pieza de composicion que `pca_explained` + `kmeans_segments` no cubren: ambas +estandarizan internamente por separado (cada una su propio `StandardScaler`) y +`kmeans_segments` no expone los labels por fila, por lo que no se pueden cruzar +con la `projection` de `pca_explained`. Esta funcion usa `sklearn` directo +(StandardScaler una sola vez compartido por PCA y KMeans) para garantizar la +alineacion `points[i] <-> labels[i]` y proyectar los centroides KMeans al +espacio PCA. Coercion y listwise deletion siguen el estilo de `pca_explained` +(None/NaN -> fila descartada, columnas no parseables o constantes descartadas). +Degrada con gracia: con <2 columnas numericas o dict: + """Proyecta a 2D (PCA) y clusteriza (KMeans) el mismo subset estandarizado. + + PCA a 2D y KMeans se ajustan sobre la MISMA matriz estandarizada, por lo que + `points` (proyeccion 2D) y `labels` (cluster por fila) quedan alineados por + indice. El k se elige automaticamente por silhouette en el rango + [k_min, min(k_max, n_rows-1)], igual criterio que `kmeans_segments`. + Determinista: StandardScaler + PCA(random_state=0) + KMeans(random_state=0, + n_init=10). + + Args: + columns: mapa {nombre_columna: [valores numericos]}. Listas alineadas por + fila (misma longitud). Columnas no numericas o con menos de 2 valores + distintos se descartan. None/NaN marcan filas a descartar listwise + (una fila se elimina si cualquier feature falta). + k_min: numero minimo de clusters a probar (default 2). + k_max: numero maximo de clusters a probar (default 8). Se acota a + min(k_max, n_rows_validas-1). + max_points: tope de puntos devueltos en `points`/`labels`. Si las filas + usadas superan este tope, se submuestrea points y labels CONJUNTAMENTE + con paso determinista para mantenerlos alineados. El fit (best_k, + silhouette, centroides, perfiles) usa SIEMPRE todas las filas. + + Returns: + dict con points (proyeccion 2D, posiblemente submuestreada a max_points), + labels (cluster de cada point, alineado con points), centers_2d + (centroides en espacio PCA, len == best_k), best_k, silhouette, + explained_2d (varianza de PC1 y PC2), cluster_sizes (sobre n_used total), + cluster_profiles (ver abajo), feature_names, n_used (filas del fit antes + de muestreo) y note ("" si ok). Cada entrada de cluster_profiles: + {cluster, size, pct, centroid_original (medias en escala original), + centroid_z (z del centroide), distinctive (top 3 features por |z|)}. + Con <2 columnas numericas o dict: + return { + "best_k": 0, + "points": [], + "labels": [], + "centers_2d": [], + "cluster_profiles": [], + "feature_names": names, + "n_used": int(n_used), + "note": "datos insuficientes", + } + + try: + if not isinstance(columns, dict) or not columns: + return insufficient([], 0) + + # 1. Coerce a numerico, descartando columnas no parseables o constantes. + numeric_cols: dict[str, list] = {} + for name, values in columns.items(): + if not isinstance(values, (list, tuple)): + continue + coerced: list[float] = [] + usable = True + for v in values: + if v is None: + coerced.append(math.nan) + continue + try: + coerced.append(float(v)) + except (TypeError, ValueError): + usable = False + break + if not usable: + continue + # Menos de 2 valores distintos no aporta varianza -> descartar. + distinct = {x for x in coerced if not math.isnan(x)} + if len(distinct) < 2: + continue + numeric_cols[name] = coerced + + feature_names = list(numeric_cols.keys()) + if len(feature_names) < 2: + return insufficient(feature_names, 0) + + # 2. Matriz alineada por fila + listwise deletion (cualquier NaN -> fuera). + matrix = np.array( + [numeric_cols[n] for n in feature_names], dtype=float + ).T + valid_mask = ~np.isnan(matrix).any(axis=1) + data = matrix[valid_mask] + + n_used = int(data.shape[0]) + min_rows = max(3, k_min * 2) + if n_used < min_rows: + return insufficient(feature_names, n_used) + + # 3. Estandarizar UNA sola vez (guardamos el scaler para desestandarizar). + scaler = StandardScaler() + X_scaled = scaler.fit_transform(data) + + # 4. PCA a 2D sobre la matriz escalada. + pca = PCA(n_components=2, random_state=0) + pca.fit(X_scaled) + proj = pca.transform(X_scaled) + + # 5. KMeans con seleccion automatica de k por silhouette (mismo X_scaled). + upper_k = min(k_max, n_used - 1) + if upper_k < k_min: + return insufficient(feature_names, n_used) + + best = None # (silhouette, k, model, labels) + for k in range(k_min, upper_k + 1): + model = KMeans(n_clusters=k, n_init=10, random_state=0) + labels_k = model.fit_predict(X_scaled) + if len(set(labels_k)) < 2: + sil = -1.0 + else: + sil = float(silhouette_score(X_scaled, labels_k)) + if best is None or sil > best[0]: + best = (sil, k, model, labels_k) + + best_sil, best_k, best_model, labels = best + + # 6. Centroides KMeans (espacio escalado) proyectados al espacio PCA. + centers_2d = pca.transform(best_model.cluster_centers_) + + # 7. Perfiles por cluster sobre TODAS las filas usadas. + centroids_original = scaler.inverse_transform(best_model.cluster_centers_) + cluster_sizes: list[int] = [] + cluster_profiles: list[dict] = [] + for c in range(best_k): + size = int(np.sum(labels == c)) + cluster_sizes.append(size) + z_vec = best_model.cluster_centers_[c] + orig_vec = centroids_original[c] + centroid_z = { + feature_names[j]: float(z_vec[j]) for j in range(len(feature_names)) + } + centroid_original = { + feature_names[j]: float(orig_vec[j]) + for j in range(len(feature_names)) + } + order = np.argsort(np.abs(z_vec))[::-1] + distinctive = [feature_names[int(j)] for j in order[:3]] + cluster_profiles.append( + { + "cluster": int(c), + "size": size, + "pct": float(size / n_used) if n_used else 0.0, + "centroid_original": centroid_original, + "distinctive": distinctive, + "centroid_z": centroid_z, + } + ) + + # 8. Muestreo determinista CONJUNTO de points + labels (mantiene alineacion). + note = "" + if n_used > max_points and max_points > 0: + step = math.ceil(n_used / max_points) + proj_out = proj[::step] + labels_out = labels[::step] + note = f"submuestreado a {len(proj_out)} de {n_used} puntos para visualizacion" + else: + proj_out = proj + labels_out = labels + + points = [[float(row[0]), float(row[1])] for row in proj_out] + labels_list = [int(v) for v in labels_out] + centers_list = [[float(row[0]), float(row[1])] for row in centers_2d] + explained_2d = [float(x) for x in pca.explained_variance_ratio_] + + return { + "points": points, + "labels": labels_list, + "centers_2d": centers_list, + "best_k": int(best_k), + "silhouette": float(best_sil), + "explained_2d": explained_2d, + "cluster_sizes": cluster_sizes, + "cluster_profiles": cluster_profiles, + "feature_names": feature_names, + "n_used": n_used, + "note": note, + } + except Exception: + # Lectura defensiva: nunca propagar excepciones al caller del EDA. + return insufficient(feature_names, 0) diff --git a/python/functions/datascience/project_clusters_2d_test.py b/python/functions/datascience/project_clusters_2d_test.py new file mode 100644 index 00000000..10cf5edd --- /dev/null +++ b/python/functions/datascience/project_clusters_2d_test.py @@ -0,0 +1,127 @@ +"""Tests para project_clusters_2d.""" + +import numpy as np + +from project_clusters_2d import project_clusters_2d + + +def _three_blobs(seed: int = 0, per_blob: int = 50, n_features: int = 4): + """Genera 3 gaussianas bien separadas en n_features dims, alineadas por fila. + + Devuelve un dict {col: [valores]} con las columnas alineadas por fila. + """ + rng = np.random.default_rng(seed) + base_centers = [ + np.full(n_features, 0.0), + np.full(n_features, 12.0), + np.array([0.0, 12.0, 0.0, 12.0][:n_features] + [0.0] * max(0, n_features - 4)), + ] + rows: list[np.ndarray] = [] + for center in base_centers: + pts = rng.normal(loc=center, scale=0.4, size=(per_blob, n_features)) + rows.extend(pts) + mat = np.array(rows) + return {f"f{j}": [float(v) for v in mat[:, j]] for j in range(n_features)} + + +def test_golden_three_blobs_aligned_projection_and_clusters(): + columns = _three_blobs(seed=0, per_blob=50, n_features=4) + result = project_clusters_2d(columns, k_min=2, k_max=8) + + n_used = result["n_used"] + assert n_used == 150 + assert result["note"] == "" + + best_k = result["best_k"] + assert 2 <= best_k <= 4 + + # points y labels alineados por fila. + assert len(result["points"]) == len(result["labels"]) + assert len(result["points"]) == n_used # sin submuestreo (150 < 2000) + + # Cada punto es un par (x, y). + assert all(len(p) == 2 for p in result["points"]) + + # Labels dentro del rango [0, best_k). + assert all(0 <= lbl < best_k for lbl in result["labels"]) + + # Centroides 2D: uno por cluster. + assert len(result["centers_2d"]) == best_k + assert all(len(c) == 2 for c in result["centers_2d"]) + + # Varianza explicada de los 2 componentes. + assert len(result["explained_2d"]) == 2 + + # cluster_sizes cubre todas las filas usadas. + assert sum(result["cluster_sizes"]) == n_used + assert len(result["cluster_sizes"]) == best_k + + # cluster_profiles: una entrada por cluster, con centroid_original poblado. + assert len(result["cluster_profiles"]) == best_k + for prof in result["cluster_profiles"]: + assert set(prof["centroid_original"].keys()) == set(result["feature_names"]) + assert set(prof["centroid_z"].keys()) == set(result["feature_names"]) + assert 1 <= len(prof["distinctive"]) <= 3 + assert prof["size"] >= 0 + assert 0.0 <= prof["pct"] <= 1.0 + + +def test_edge_subsampling_keeps_points_labels_aligned(): + # max_points pequeño fuerza submuestreo conjunto de points + labels. + columns = _three_blobs(seed=1, per_blob=50, n_features=3) + result = project_clusters_2d(columns, k_min=2, k_max=6, max_points=40) + + n_used = result["n_used"] + assert n_used == 150 # el fit usa todas las filas + + # points y labels submuestreados pero siempre con la misma longitud. + assert len(result["points"]) == len(result["labels"]) + assert len(result["points"]) <= 40 + + # centers/sizes/profiles se calculan sobre TODOS los puntos. + assert sum(result["cluster_sizes"]) == n_used + assert len(result["centers_2d"]) == result["best_k"] + assert result["note"] != "" # senala el submuestreo + + +def test_edge_single_numeric_column_insufficient(): + columns = {"x": [float(i) for i in range(50)]} + result = project_clusters_2d(columns, k_min=2, k_max=8) + + assert result["best_k"] == 0 + assert result["note"] == "datos insuficientes" + assert result["points"] == [] + assert result["labels"] == [] + assert result["centers_2d"] == [] + assert result["cluster_profiles"] == [] + + +def test_edge_too_few_rows_insufficient(): + # Solo 2 filas validas, min_rows = max(3, k_min*2) = 4 -> insuficiente. + columns = {"x": [1.0, 5.0], "y": [2.0, 9.0]} + result = project_clusters_2d(columns, k_min=2, k_max=8) + + assert result["best_k"] == 0 + assert result["note"] == "datos insuficientes" + + +def test_edge_non_numeric_column_dropped_without_error(): + # La columna de strings se descarta; quedan 3 numericas -> funciona. + columns = _three_blobs(seed=2, per_blob=50, n_features=3) + columns["label"] = ["a"] * len(columns["f0"]) + result = project_clusters_2d(columns, k_min=2, k_max=6) + + assert result["best_k"] >= 2 + assert "label" not in result["feature_names"] + assert set(result["feature_names"]) == {"f0", "f1", "f2"} + assert len(result["points"]) == len(result["labels"]) + + +def test_edge_constant_column_dropped(): + # Una columna constante (0 varianza) se descarta por <2 valores distintos. + columns = _three_blobs(seed=3, per_blob=50, n_features=3) + columns["const"] = [7.0] * len(columns["f0"]) + result = project_clusters_2d(columns, k_min=2, k_max=6) + + assert "const" not in result["feature_names"] + assert result["best_k"] >= 2 From 81e8597d2154af0816f3045d66b670ee89a501a7 Mon Sep 17 00:00:00 2001 From: Egutierrez Date: Tue, 30 Jun 2026 14:57:43 +0200 Subject: [PATCH 06/53] feat(eda): capitulo MODELOS de AutomaticEDA (markdown, scatter PCA+clusters, micro-LLM) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implementa chapters/modelos.py (build_modelos / CHAPTER_VERSION) consumiendo profile['models'] {pca,kmeans,outliers,normality} de run_eda_models. Render markdown estructurado con bloques anti-corte: - Intro de normalizacion z-score: por que se estandariza antes de PCA/KMeans (MUST-8.3). - PCA: scree plot (varianza explicada + acumulada, un solo eje Y) + tablas de varianza y cargas principales (SHOULD-8.4). - Segmentacion KMeans: scatter PCA coloreado por cluster con centroides, en su propia pagina/slide (MUST-8.1); tabla de tamaños; micro-analisis LLM por cluster con titulo, cada entrada indivisible (MUST-8.2). - Isolation Forest: explicacion de la deteccion multivariante de outliers y del umbral + conteos (MUST-8.3). - Normalidad: tabla por columna (Jarque-Bera / D'Agostino / Shapiro), pagina sola. El scatter coloreado y los titulos LLM no estan en el TableProfile, asi que el capitulo los toma de ctx (cluster_projection precomputado, o raw_numeric para calcular project_clusters_2d en vivo, o cluster_titles/run_cluster_llm para el micro-analisis), igual que overview lee head_rows; degrada honesto con una Note cuando faltan. Devuelve None si el profile no trae bloque models renderizable. Tests self-contained (sin DuckDB/sklearn/LLM/red): golden PDF+PPTX, edges (profile None/vacio/insuficiente, kmeans sin proyeccion), anti-corte (tabla de normalidad de 40 columnas parte repitiendo cabecera sin perder ninguna). 8/8. Suite del nucleo render_automatic_eda_pdf/pptx sigue verde. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../automatic_eda/chapters/modelos.py | 498 ++++++++++++++++++ .../automatic_eda/chapters/modelos_test.py | 259 +++++++++ 2 files changed, 757 insertions(+) create mode 100644 python/functions/datascience/automatic_eda/chapters/modelos.py create mode 100644 python/functions/datascience/automatic_eda/chapters/modelos_test.py diff --git a/python/functions/datascience/automatic_eda/chapters/modelos.py b/python/functions/datascience/automatic_eda/chapters/modelos.py new file mode 100644 index 00000000..ffc43346 --- /dev/null +++ b/python/functions/datascience/automatic_eda/chapters/modelos.py @@ -0,0 +1,498 @@ +"""Models chapter (MODELOS) — cheap unsupervised models, rendered as markdown. + +Builds the *Modelos* chapter of an AutomaticEDA document from the ``models`` +block of a TableProfile (``run_eda_models`` output: ``{pca, kmeans, outliers, +normality}``). It renders, as structured markdown/tables/figures that the core +paginator never cuts: + +1. **Normalization note** — every multivariate model below standardizes the + columns with z-score first; the chapter explains why (different scales would + otherwise dominate distance/variance). +2. **PCA** — a scree plot (explained + cumulative variance, single Y axis) plus + variance and top-loadings tables. +3. **KMeans segments** — a PCA scatter **coloured by cluster** (its own + page/slide), the cluster-size table, and a per-cluster LLM micro-analysis + with a title for each segment. +4. **Isolation Forest outliers** — a short explanation of how anomalous rows are + isolated multivariately and how the threshold is chosen, plus the counts. +5. **Normality** — per-column Jarque-Bera / D'Agostino / Shapiro verdicts. + +The raw numeric data needed to colour the cluster scatter is **not** in the +TableProfile, so — exactly like ``overview`` reads ``head_rows`` from ``ctx`` — +this chapter looks for the cluster projection / raw numeric columns in ``ctx`` +(or in ``profile``) and degrades honestly when they are absent: it falls back to +the uncoloured ``pca.projection`` with a note, or omits the scatter entirely. + +ctx keys this chapter consumes (all optional): + cluster_projection : dict — a pre-computed ``project_clusters_2d`` result + (``points``/``labels``/``centers_2d``/``cluster_profiles``/...). Used + directly when present (forward-compatible with the calculation phase). + raw_numeric : dict — ``{col: [values]}`` raw numeric columns; when present + and ``cluster_projection`` is not, the chapter calls + ``project_clusters_2d`` live to build points + aligned labels. + cluster_titles : list — pre-computed ``[{cluster, title, description}]`` + (a ``describe_clusters_llm`` ``clusters`` list). Used for the per-cluster + micro-analysis without an LLM call (offline/tests). + run_cluster_llm : bool — when True and ``cluster_titles`` is absent, call + ``describe_clusters_llm`` live on the cluster profiles. + cluster_llm_model : str — model id for the live LLM call. + +Contract: build_(profile, ctx) -> Chapter | None ; CHAPTER_VERSION = "x.y.z". +""" + +from __future__ import annotations + +from .. import model + +CHAPTER_VERSION = "1.0.0" +CHAPTER_ID = "modelos" +CHAPTER_TITLE = "Modelos" + +# Tableau-10 palette (matplotlib's default cycle) — used both for the matplotlib +# scatter and to keep the legend/colours stable per cluster index. +_CLUSTER_COLORS = [ + "#4e79a7", "#f28e2b", "#e15759", "#76b7b2", "#59a14f", + "#edc948", "#b07aa1", "#ff9da7", "#9c755f", "#bab0ac", +] + + +# --------------------------------------------------------------------------- # +# Formatting helpers (mirror the overview chapter's defensive style). +# --------------------------------------------------------------------------- # +def _fmt_num(value, decimals: int = 3) -> str: + if value is None: + return "—" + if isinstance(value, bool): + return "sí" if value else "no" + if isinstance(value, int): + return f"{value:,}".replace(",", ".") + if isinstance(value, float): + if value != value: # NaN + return "NaN" + if value in (float("inf"), float("-inf")): + return str(value) + text = f"{value:.{decimals}f}".rstrip("0").rstrip(".") + return text if text else "0" + return model._safe_str(value) + + +def _fmt_pct_ratio(value, decimals: int = 1) -> str: + """Format a 0..1 ratio as a percentage.""" + if value is None: + return "—" + try: + return f"{float(value) * 100:.{decimals}f}%" + except (TypeError, ValueError): + return model._safe_str(value) + + +def _fmt_pct_already(value, decimals: int = 2) -> str: + """Format a value that is *already* a 0..100 percentage.""" + if value is None: + return "—" + try: + return f"{float(value):.{decimals}f}%" + except (TypeError, ValueError): + return model._safe_str(value) + + +def _is_dict(v) -> bool: + return isinstance(v, dict) + + +# --------------------------------------------------------------------------- # +# Cluster projection: prefer a pre-computed result, else compute it live, else +# fall back to the uncoloured PCA projection. +# --------------------------------------------------------------------------- # +def _resolve_cluster_projection(profile: dict, ctx: dict): + """Return (projection_dict_or_None, source_label). + + Order: ctx/profile['cluster_projection'] (pre-computed) → live + project_clusters_2d on ctx/profile['raw_numeric'] → None. + """ + pre = ctx.get("cluster_projection") or profile.get("cluster_projection") + models = profile.get("models") if _is_dict(profile.get("models")) else {} + if not pre and _is_dict(models): + pre = models.get("cluster_projection") + if _is_dict(pre) and pre.get("points"): + return pre, "precomputed" + + raw = ctx.get("raw_numeric") or profile.get("raw_numeric") + if _is_dict(raw) and raw: + try: + # Import the submodule's function explicitly (avoid the package + # attribute shadowing the function with the same-named module). + from datascience.project_clusters_2d import project_clusters_2d + proj = project_clusters_2d(raw) + if _is_dict(proj) and proj.get("points"): + return proj, "live" + except Exception: # noqa: BLE001 — never break the chapter. + return None, "none" + return None, "none" + + +def _cluster_titles(profile: dict, ctx: dict, projection: dict): + """Return a list of {cluster, title, description} for the segments. + + Order: ctx['cluster_titles'] (pre-computed) → live describe_clusters_llm when + ctx['run_cluster_llm'] and we have cluster_profiles → derived titles from the + distinctive features → None. + """ + pre = ctx.get("cluster_titles") + if isinstance(pre, list) and pre: + return [c for c in pre if _is_dict(c)] + + profiles = (projection or {}).get("cluster_profiles") or [] + feats = (projection or {}).get("feature_names") or [] + if ctx.get("run_cluster_llm") and profiles: + try: + from datascience.describe_clusters_llm import describe_clusters_llm + out = describe_clusters_llm( + profiles, feats, + model=ctx.get("cluster_llm_model", "claude-haiku-4-5-20251001")) + clusters = (out or {}).get("clusters") + if isinstance(clusters, list) and clusters: + return [c for c in clusters if _is_dict(c)] + except Exception: # noqa: BLE001 + pass + + # Derived fallback: name each cluster by its distinctive features. + if profiles: + derived = [] + for p in profiles: + if not _is_dict(p): + continue + cid = p.get("cluster", len(derived)) + dist = p.get("distinctive") or [] + label = ", ".join(model._safe_str(d) for d in dist[:2]) if dist else "" + title = f"Segmento {cid}" + (f" — {label}" if label else "") + derived.append({"cluster": cid, "title": title, "description": ""}) + if derived: + return derived + return None + + +# --------------------------------------------------------------------------- # +# Figure builders (lazy: matplotlib only imported when the renderer draws them). +# --------------------------------------------------------------------------- # +def _make_scree(pca: dict): + """Return a zero-arg callable drawing the PCA scree plot, or None.""" + evr = pca.get("explained_variance_ratio") or [] + cum = pca.get("cumulative") or [] + if not evr: + return None + + def _draw(): + import matplotlib + matplotlib.use("Agg") + import matplotlib.pyplot as plt + + comps = list(range(1, len(evr) + 1)) + fig, ax = plt.subplots(figsize=(7.0, 4.2)) + ax.bar(comps, evr, color="#4e79a7", alpha=0.85, + label="Varianza explicada") + if cum: + ax.plot(comps[:len(cum)], cum, color="#e15759", marker="o", + linewidth=1.8, label="Acumulada") + ax.set_xlabel("Componente principal") + ax.set_ylabel("Proporción de varianza") + ax.set_xticks(comps) + ax.set_ylim(0, 1.0) + ax.grid(axis="y", color="#dddddd", linewidth=0.6) + ax.legend(loc="best", fontsize=8, frameon=False) + ax.set_title("Varianza explicada por componente (PCA)", fontsize=10) + fig.tight_layout() + return fig + + return _draw + + +def _make_cluster_scatter(projection: dict): + """Return a zero-arg callable drawing the cluster scatter, or None.""" + points = projection.get("points") or [] + labels = projection.get("labels") or [] + if not points or len(points) != len(labels): + return None + centers = projection.get("centers_2d") or [] + explained = projection.get("explained_2d") or [] + + def _draw(): + import matplotlib + matplotlib.use("Agg") + import matplotlib.pyplot as plt + + fig, ax = plt.subplots(figsize=(7.0, 5.2)) + uniq = sorted(set(int(l) for l in labels)) + for cl in uniq: + xs = [p[0] for p, l in zip(points, labels) if int(l) == cl] + ys = [p[1] for p, l in zip(points, labels) if int(l) == cl] + color = _CLUSTER_COLORS[cl % len(_CLUSTER_COLORS)] + ax.scatter(xs, ys, s=14, c=color, alpha=0.7, linewidths=0, + label=f"Cluster {cl} (n={len(xs)})") + for cl, c in enumerate(centers): + color = _CLUSTER_COLORS[cl % len(_CLUSTER_COLORS)] + ax.scatter([c[0]], [c[1]], s=180, c=color, marker="X", + edgecolors="black", linewidths=1.2, zorder=5) + xlab, ylab = "PC1", "PC2" + if len(explained) >= 2: + xlab = f"PC1 ({_fmt_pct_ratio(explained[0])} var.)" + ylab = f"PC2 ({_fmt_pct_ratio(explained[1])} var.)" + ax.set_xlabel(xlab) + ax.set_ylabel(ylab) + ax.set_title("Segmentos KMeans proyectados sobre el plano PCA", + fontsize=10) + ax.grid(color="#eeeeee", linewidth=0.5) + ax.legend(loc="best", fontsize=8, frameon=True, framealpha=0.9) + fig.tight_layout() + return fig + + return _draw + + +# --------------------------------------------------------------------------- # +# Section builders. Each returns a list of blocks (possibly empty). +# --------------------------------------------------------------------------- # +def _normalization_intro() -> list: + text = ( + "Estos modelos son **no supervisados**: buscan estructura latente sin " + "una variable objetivo. Antes de aplicarlos, todas las columnas " + "numéricas se **estandarizan con z-score** (cada valor menos la media, " + "dividido por la desviación típica). Sin esta normalización, una " + "variable con escala grande (p.ej. ingresos en euros) dominaría las " + "distancias y la varianza frente a otra de escala pequeña (p.ej. un " + "ratio entre 0 y 1), sesgando tanto el PCA como el KMeans. Tras la " + "estandarización todas las variables pesan por igual." + ) + return [model.Heading(text="Modelos no supervisados", level=1), + model.Markdown(text=text)] + + +def _pca_section(pca: dict) -> list: + if not _is_dict(pca) or not pca.get("explained_variance_ratio"): + return [] + blocks = [model.Heading(text="PCA — varianza explicada", level=2)] + + n_used = pca.get("n_rows_used") + n_feat = pca.get("n_features") + intro = ( + f"El PCA resume {_fmt_num(n_feat)} variables numéricas en componentes " + f"ortogonales ordenados por la varianza que capturan " + f"({_fmt_num(n_used)} filas usadas tras eliminar nulos). El gráfico de " + "sedimentación (scree) muestra cuánta varianza aporta cada componente y " + "su acumulado: un codo marca cuántos componentes bastan." + ) + blocks.append(model.Markdown(text=intro)) + + scree = _make_scree(pca) + if scree is not None: + blocks.append(model.Figure( + make=scree, caption="Varianza explicada y acumulada por componente.")) + + evr = pca.get("explained_variance_ratio") or [] + cum = pca.get("cumulative") or [] + rows = [] + for i, v in enumerate(evr): + acc = cum[i] if i < len(cum) else None + rows.append([f"PC{i + 1}", _fmt_pct_ratio(v), _fmt_pct_ratio(acc)]) + if rows: + blocks.append(model.DataTable( + header=["Componente", "Varianza", "Acumulada"], rows=rows, + title="Varianza por componente")) + + # Top loadings: keep the strongest features per component (capped). + loadings = pca.get("top_loadings") or [] + if loadings: + per_comp: dict = {} + for ld in loadings: + if not _is_dict(ld): + continue + comp = ld.get("component") + per_comp.setdefault(comp, []) + if len(per_comp[comp]) < 4: + per_comp[comp].append(ld) + rows = [] + for comp in sorted(per_comp.keys(), key=lambda x: (x is None, x)): + for ld in per_comp[comp]: + rows.append([f"PC{int(comp) + 1}" if comp is not None else "—", + model._safe_str(ld.get("feature")), + _fmt_num(ld.get("loading"))]) + if rows: + blocks.append(model.DataTable( + header=["Componente", "Variable", "Carga"], rows=rows, + title="Cargas principales (top por componente)", + note="Cargas con mayor valor absoluto: qué variables definen " + "cada eje.")) + return blocks + + +def _kmeans_section(kmeans: dict, projection: dict, titles) -> list: + has_km = _is_dict(kmeans) and kmeans.get("best_k") + has_proj = _is_dict(projection) and projection.get("points") + if not has_km and not has_proj: + return [] + + blocks = [model.Heading(text="Segmentación (KMeans)", level=2)] + + best_k = (projection or {}).get("best_k") or (kmeans or {}).get("best_k") + sil = (projection or {}).get("silhouette") + if sil is None: + sil = (kmeans or {}).get("silhouette") + intro = ( + f"KMeans agrupa las filas en **{_fmt_num(best_k)} segmentos** elegidos " + "automáticamente maximizando el coeficiente de *silhouette* " + f"(**{_fmt_num(sil)}**, rango −1 a 1: cuanto más alto, segmentos más " + "compactos y separados). Los segmentos se proyectan sobre el plano de " + "los dos primeros componentes principales para visualizarlos." + ) + blocks.append(model.Markdown(text=intro)) + + if has_proj: + scatter = _make_cluster_scatter(projection) + if scatter is not None: + blocks.append(model.Figure( + make=scatter, + caption="Cada punto es una fila coloreada por su segmento " + "KMeans; las «X» son los centroides.")) + else: + blocks.append(model.Note( + "Proyección de clusters no dibujable (puntos y etiquetas " + "desalineados).")) + else: + # We have kmeans stats but no aligned points+labels to colour by. + blocks.append(model.Note( + "Scatter coloreado por segmento no disponible: el perfil no incluye " + "la proyección con etiquetas alineadas (pásala en " + "ctx['cluster_projection'] o las columnas crudas en " + "ctx['raw_numeric'] para colorear el plano PCA).")) + + # Cluster sizes table. + sizes = (projection or {}).get("cluster_sizes") or (kmeans or {}).get("cluster_sizes") or [] + total = sum(s for s in sizes if isinstance(s, (int, float))) or 0 + if sizes: + rows = [] + for i, s in enumerate(sizes): + pct = (s / total) if total else None + rows.append([f"Cluster {i}", _fmt_num(s), _fmt_pct_ratio(pct)]) + blocks.append(model.DataTable( + header=["Segmento", "Tamaño", "% del total"], rows=rows, + title="Tamaño de cada segmento")) + + # Per-cluster LLM micro-analysis (each entry kept indivisible as one block). + if titles: + blocks.append(model.Heading(text="Interpretación de los segmentos", + level=3)) + for t in titles: + if not _is_dict(t): + continue + cid = t.get("cluster") + title = model._safe_str(t.get("title")) or f"Cluster {cid}" + desc = model._safe_str(t.get("description")) + line = f"**Cluster {cid} — {title}.**" + if desc: + line += " " + desc + blocks.append(model.Markdown(text=line)) + return blocks + + +def _outliers_section(outliers: dict) -> list: + if not _is_dict(outliers) or outliers.get("n_outliers") is None: + return [] + if outliers.get("note") and not outliers.get("n_rows_used"): + # insufficient data — nothing meaningful to show. + return [] + blocks = [model.Heading(text="Detección de anomalías (Isolation Forest)", + level=2)] + explain = ( + "**Isolation Forest** detecta filas anómalas de forma *multivariante*: " + "construye árboles que parten el espacio con cortes aleatorios y mide " + "cuántos cortes hacen falta para aislar cada fila. Las filas raras " + "(combinaciones de valores poco frecuentes considerando **todas las " + "columnas a la vez**, no una sola) se aíslan con muy pocos cortes y " + "obtienen un score bajo. El **umbral** de decisión separa las filas " + "normales de las anómalas según la contaminación esperada del modelo: " + "una fila es outlier cuando su score queda por debajo de ese umbral." + ) + blocks.append(model.Markdown(text=explain)) + blocks.append(model.KVTable(rows=[ + ("Filas analizadas", _fmt_num(outliers.get("n_rows_used"))), + ("Outliers detectados", _fmt_num(outliers.get("n_outliers"))), + ("% outliers", _fmt_pct_already(outliers.get("outlier_pct"))), + ("Umbral de decisión", _fmt_num(outliers.get("threshold"), 4)), + ], title="Anomalías multivariantes")) + return blocks + + +def _normality_section(normality: dict) -> list: + if not _is_dict(normality) or not normality: + return [] + header = ["Columna", "Jarque-Bera (p)", "D'Agostino (p)", "Shapiro (p)", + "¿Normal?"] + rows = [] + for col, res in normality.items(): + if not _is_dict(res): + continue + jb = res.get("jarque_bera") if _is_dict(res.get("jarque_bera")) else {} + da = res.get("dagostino") if _is_dict(res.get("dagostino")) else {} + sh = res.get("shapiro") if _is_dict(res.get("shapiro")) else {} + is_norm = res.get("is_normal") + if res.get("note") and is_norm is None and not jb: + rows.append([model._safe_str(col), "—", "—", "—", + model._safe_str(res.get("note"))]) + continue + rows.append([ + model._safe_str(col), + _fmt_num(jb.get("p"), 4) if jb else "—", + _fmt_num(da.get("p"), 4) if da else "—", + _fmt_num(sh.get("p"), 4) if sh else "—", + "sí" if is_norm else ("no" if is_norm is not None else "—"), + ]) + if not rows: + return [] + return [ + model.Heading(text="Normalidad de las variables", level=2), + model.Markdown(text=( + "Tests de hipótesis de normalidad por columna (hipótesis nula: la " + "muestra proviene de una distribución normal). Se marca **normal** " + "cuando el p-valor supera 0,05 (no se rechaza la nula). Pocas " + "variables reales son estrictamente normales; esto orienta qué " + "transformaciones o tests robustos aplicar después.")), + model.DataTable(header=header, rows=rows, + title="Pruebas de normalidad"), + ] + + +# --------------------------------------------------------------------------- # +# Entry point. +# --------------------------------------------------------------------------- # +def build_modelos(profile: dict, ctx: dict): + """Build the MODELOS Chapter, or None if there are no models to show.""" + profile = profile or {} + ctx = ctx or {} + if not isinstance(profile, dict): + return None + models = profile.get("models") + if not _is_dict(models): + return None + + pca = models.get("pca") if _is_dict(models.get("pca")) else None + kmeans = models.get("kmeans") if _is_dict(models.get("kmeans")) else None + outliers = models.get("outliers") if _is_dict(models.get("outliers")) else None + normality = models.get("normality") if _is_dict(models.get("normality")) else None + + projection, _src = _resolve_cluster_projection(profile, ctx) + titles = _cluster_titles(profile, ctx, projection) if ( + (kmeans and kmeans.get("best_k")) or (projection and projection.get("points")) + ) else None + + sections = [] + sections += _pca_section(pca) if pca else [] + sections += _kmeans_section(kmeans, projection, titles) + sections += _outliers_section(outliers) if outliers else [] + sections += _normality_section(normality) if normality else [] + + if not sections: + return None # models block present but nothing renderable. + + blocks = _normalization_intro() + sections + return model.Chapter(id=CHAPTER_ID, title=CHAPTER_TITLE, + version=CHAPTER_VERSION, blocks=blocks) diff --git a/python/functions/datascience/automatic_eda/chapters/modelos_test.py b/python/functions/datascience/automatic_eda/chapters/modelos_test.py new file mode 100644 index 00000000..9d2597a5 --- /dev/null +++ b/python/functions/datascience/automatic_eda/chapters/modelos_test.py @@ -0,0 +1,259 @@ +"""Tests for the MODELOS chapter — DoD: golden + edges + anti-cut. + +Self-contained: builds a synthetic TableProfile with a ``models`` block (no +DuckDB, no sklearn, no LLM, no network). The cluster scatter is fed a synthetic +pre-computed ``cluster_projection`` via ``ctx`` and the per-cluster titles via +``ctx['cluster_titles']`` so the suite is fast and deterministic. The live paths +(``project_clusters_2d`` / ``describe_clusters_llm``) are exercised against the +real wine dataset in the work report, not here. + +Verifies: the chapter renders to PDF *and* PPTX showing the user-required pieces +(markdown text, PCA scree, cluster scatter, per-cluster LLM micro-analysis, +outlier + normalization explanations); that an inapplicable profile yields None +without raising; and that a long normality table is split without losing any +column (anti-cut). +""" + +import os +import re +import tempfile + +from pypdf import PdfReader +from pptx import Presentation + +from datascience.automatic_eda.chapters.modelos import build_modelos +from datascience.automatic_eda.model import Figure, DataTable, Markdown +from datascience.render_automatic_eda_pdf import render_automatic_eda_pdf +from datascience.render_automatic_eda_pptx import render_automatic_eda_pptx + + +# --------------------------------------------------------------------------- # +# Synthetic fixtures. +# --------------------------------------------------------------------------- # +def _models_block(n_norm_cols: int = 4) -> dict: + feats = ["fixed_acidity", "alcohol", "ph", "sulphates"] + normality = {} + for i in range(n_norm_cols): + normality[f"col_{i}"] = { + "n": 500, + "jarque_bera": {"stat": 12.3, "p": 0.002 + i * 0.0001, "normal": False}, + "dagostino": {"stat": 9.1, "p": 0.01, "normal": False}, + "shapiro": {"stat": 0.98, "p": 0.04, "normal": False}, + "is_normal": False, + } + return { + "n_numeric_cols": 4, + "pca": { + "n_components": 2, "n_rows_used": 1599, "n_features": 4, + "explained_variance_ratio": [0.41, 0.22], + "cumulative": [0.41, 0.63], + "top_loadings": [ + {"component": 0, "feature": "alcohol", "loading": 0.62}, + {"component": 0, "feature": "fixed_acidity", "loading": -0.48}, + {"component": 1, "feature": "ph", "loading": 0.71}, + {"component": 1, "feature": "sulphates", "loading": 0.33}, + ], + "projection": [[0.1, 0.2], [0.3, -0.1]], + }, + "kmeans": { + "best_k": 3, "silhouette": 0.27, + "scores_by_k": [{"k": 2, "silhouette": 0.21}, {"k": 3, "silhouette": 0.27}], + "cluster_sizes": [700, 500, 399], + "centers": [[0.1, 0.2, 0.3, 0.4]], + "n_rows_used": 1599, "n_features": 4, + }, + "outliers": { + "n_outliers": 80, "outlier_pct": 5.0, "threshold": -0.0123, + "n_rows_used": 1599, + }, + "normality": normality, + "note": "", + "_feats": feats, + } + + +def _cluster_projection() -> dict: + # 30 points across 3 clusters, aligned points<->labels. + points, labels = [], [] + centers = [(-2.0, -2.0), (2.0, 0.0), (0.0, 2.5)] + for cl, (cx, cy) in enumerate(centers): + for j in range(10): + points.append([cx + (j - 5) * 0.05, cy + (j - 5) * 0.05]) + labels.append(cl) + return { + "points": points, "labels": labels, + "centers_2d": [list(c) for c in centers], + "best_k": 3, "silhouette": 0.27, + "explained_2d": [0.41, 0.22], + "cluster_sizes": [10, 10, 10], + "cluster_profiles": [ + {"cluster": 0, "size": 10, "pct": 0.33, + "centroid_original": {"alcohol": 9.5, "ph": 3.5}, + "distinctive": ["alcohol", "ph"], "centroid_z": {"alcohol": -1.2}}, + {"cluster": 1, "size": 10, "pct": 0.33, + "centroid_original": {"alcohol": 12.0, "ph": 3.1}, + "distinctive": ["alcohol"], "centroid_z": {"alcohol": 1.4}}, + {"cluster": 2, "size": 10, "pct": 0.33, + "centroid_original": {"alcohol": 10.5, "ph": 3.8}, + "distinctive": ["ph"], "centroid_z": {"ph": 1.6}}, + ], + "feature_names": ["alcohol", "ph", "fixed_acidity", "sulphates"], + "n_used": 1599, "note": "", + } + + +def _ctx_full() -> dict: + return { + "cluster_projection": _cluster_projection(), + "cluster_titles": [ + {"cluster": 0, "title": "Vinos suaves de baja graduación", + "description": "Alcohol bajo y pH alto; perfil ligero."}, + {"cluster": 1, "title": "Vinos potentes", + "description": "Alta graduación alcohólica."}, + {"cluster": 2, "title": "Vinos de pH elevado", + "description": "Acidez baja relativa al resto."}, + ], + } + + +def _profile() -> dict: + return {"table": "wine", "n_rows": 1599, "n_cols": 12, + "models": _models_block()} + + +def _pdf_text(path: str) -> str: + txt = "".join((pg.extract_text() or "") for pg in PdfReader(path).pages) + return re.sub(r"\s+", " ", txt) + + +def _pptx_text(path: str) -> str: + prs = Presentation(path) + out = [] + for slide in prs.slides: + for shape in slide.shapes: + if shape.has_text_frame: + out.append(shape.text_frame.text) + return re.sub(r"\s+", " ", " ".join(out)) + + +# --------------------------------------------------------------------------- # +# Golden. +# --------------------------------------------------------------------------- # +def test_golden_build_modelos_bloques_requeridos(): + ch = build_modelos(_profile(), _ctx_full()) + assert ch is not None + assert ch.id == "modelos" and ch.version + # Both figures present: scree plot + cluster scatter. + n_figures = sum(1 for b in ch.blocks if isinstance(b, Figure)) + assert n_figures >= 2 + # Tables present (variance, loadings, sizes, normality). + assert sum(1 for b in ch.blocks if isinstance(b, DataTable)) >= 3 + # Markdown carries the required explanations. + md = " ".join(b.text for b in ch.blocks if isinstance(b, Markdown)) + assert "z-score" in md # normalization explained + assert "Isolation Forest" in md # outlier generation explained + assert "silhouette" in md # kmeans + # Per-cluster micro-analysis titles present. + assert "Vinos potentes" in md + assert "Cluster 1" in md + + +def test_golden_render_pdf_muestra_lo_exigido(): + with tempfile.TemporaryDirectory() as d: + out = os.path.join(d, "modelos.pdf") + res = render_automatic_eda_pdf( + _profile(), out, {"title": "EDA — wine", "ctx": _ctx_full()}) + assert res["path"] == out and os.path.exists(out) + ids = [c["id"] for c in res["chapters"]] + assert "modelos" in ids + txt = _pdf_text(out) + for needle in ("Modelos no supervisados", "z-score", "PCA", + "Segmentación", "Isolation Forest", "Normalidad", + "Vinos potentes"): + assert needle in txt, f"falta en PDF: {needle}" + + +def test_golden_render_pptx_muestra_lo_exigido(): + with tempfile.TemporaryDirectory() as d: + out = os.path.join(d, "modelos.pptx") + res = render_automatic_eda_pptx( + _profile(), out, {"title": "EDA — wine", "ctx": _ctx_full()}) + assert res["path"] == out and os.path.exists(out) + assert res["n_slides"] >= 1 + txt = _pptx_text(out) + for needle in ("Modelos no supervisados", "z-score", "Isolation Forest", + "Vinos potentes"): + assert needle in txt, f"falta en PPTX: {needle}" + + +# --------------------------------------------------------------------------- # +# Edges. +# --------------------------------------------------------------------------- # +def test_edge_profile_none_o_vacio_devuelve_none(): + assert build_modelos(None, {}) is None + assert build_modelos({}, {}) is None + assert build_modelos({"n_rows": 5}, None) is None # no 'models' key + + +def test_edge_models_insuficiente_devuelve_none(): + prof = {"table": "tiny", "models": { + "n_numeric_cols": 1, + "pca": {"n_components": 0, "explained_variance_ratio": [], + "note": "datos insuficientes"}, + "kmeans": {"best_k": 0, "note": "datos insuficientes"}, + "outliers": {"n_outliers": 0, "note": "datos insuficientes"}, + "normality": None, + "note": "insuficientes columnas numericas para modelos multivariantes", + }} + assert build_modelos(prof, {}) is None + + +def test_edge_solo_normalidad_si_genera_capitulo(): + # A single numeric column: only normality applies. Chapter must still build. + prof = {"table": "one", "models": { + "n_numeric_cols": 1, "pca": None, "kmeans": None, "outliers": None, + "normality": {"x": {"n": 500, "jarque_bera": {"stat": 1.0, "p": 0.2, + "normal": True}, "dagostino": {"stat": 1.0, "p": 0.3, + "normal": True}, "shapiro": {"stat": 0.99, "p": 0.4, + "normal": True}, "is_normal": True}}, + }} + ch = build_modelos(prof, {}) + assert ch is not None + md = " ".join(b.text for b in ch.blocks if isinstance(b, Markdown)) + assert "z-score" in md # normalization intro still present + + +def test_edge_kmeans_sin_proyeccion_degrada_sin_romper(): + # kmeans stats present but no cluster_projection / raw_numeric to colour by. + prof = _profile() + ch = build_modelos(prof, {}) # no ctx projection + assert ch is not None + # No scatter figure for clusters, but a Note explaining the degradation. + notes = [b.text for b in ch.blocks if b.kind == "note"] + assert any("ctx['raw_numeric']" in n or "cluster_projection" in n + for n in notes) + # PDF still renders fine. + with tempfile.TemporaryDirectory() as d: + out = os.path.join(d, "deg.pdf") + res = render_automatic_eda_pdf(prof, out, {"write_manifest": False}) + assert res["path"] == out and os.path.exists(out) + + +# --------------------------------------------------------------------------- # +# Anti-cut. +# --------------------------------------------------------------------------- # +def test_anticortes_tabla_normalidad_larga_no_corta(): + # 40 numeric columns → the normality DataTable must split across pages, + # repeating the header, without losing any column name. + prof = {"table": "wide", "models": _models_block(n_norm_cols=40)} + with tempfile.TemporaryDirectory() as d: + out = os.path.join(d, "wide.pdf") + render_automatic_eda_pdf(prof, out, {"write_manifest": False, + "ctx": _ctx_full()}) + reader = PdfReader(out) + n_pages = len(reader.pages) + assert n_pages > 1 + txt = "".join((pg.extract_text() or "") for pg in reader.pages) + # Every column name survives (wrapped/split, never truncated). + for i in (0, 19, 39): + assert f"col_{i}" in txt From c1a4a8371736198c1f405565b365ce344e8e9adf Mon Sep 17 00:00:00 2001 From: Egutierrez Date: Tue, 30 Jun 2026 14:58:03 +0200 Subject: [PATCH 07/53] =?UTF-8?q?feat(eda):=20cap=C3=ADtulo=20num=5Fdistr?= =?UTF-8?q?=20=E2=80=94=20histograma=20con=20media/mediana/=C2=B1=CF=83=20?= =?UTF-8?q?+=20boxplot=20Tukey?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Capítulo NUM DISTR del motor AutomaticEDA. Por cada columna numérica emite, como una sola Figure indivisible de dos ejes compartiendo X, un histograma con la media (línea roja discontinua), la mediana (línea verde continua) y la banda ±1σ dibujadas como referencias, y un boxplot de Tukey debajo (caja P25–P75, bigotes a 1,5·IQR, marca de valores fuera de las vallas). Una nota por columna traduce el distribution_type a lenguaje llano (MUST-4.1/4.2/4.3 del report 2043). Consume el profile del grupo eda sin recalcular: el histograma usa los bins {lo,hi,count} de describe_numeric y las vallas del boxplot las deriva la función pura build_boxplot_stats_py_datascience. Lectura defensiva: sin columna numérica devuelve None; profile None/{} no lanza. Test self-contained: golden + edges + anti-corte (8 columnas no cortan en PDF ni PPTX). --- .../automatic_eda/chapters/num_distr.py | 289 ++++++++++++++++++ .../automatic_eda/chapters/num_distr_test.py | 151 +++++++++ 2 files changed, 440 insertions(+) create mode 100644 python/functions/datascience/automatic_eda/chapters/num_distr.py create mode 100644 python/functions/datascience/automatic_eda/chapters/num_distr_test.py diff --git a/python/functions/datascience/automatic_eda/chapters/num_distr.py b/python/functions/datascience/automatic_eda/chapters/num_distr.py new file mode 100644 index 00000000..6c105dc6 --- /dev/null +++ b/python/functions/datascience/automatic_eda/chapters/num_distr.py @@ -0,0 +1,289 @@ +"""Numeric distributions chapter (NUM DISTR) for AutomaticEDA. + +For every numeric column the chapter draws, as a single indivisible figure, a +histogram with the **mean, median and ±1σ band drawn as reference lines** and a +**Tukey boxplot right below it** sharing the same X axis — exactly the user +requirement for this chapter. Each figure is emitted as a lazy ``Figure`` block +so the renderers rasterize and scale it to fit a whole page/slide and nothing is +ever cut; columns with many numerics simply flow across pages as small +multiples. + +Data comes from the ``eda`` group profile and is never recomputed here: + +- ``columns[i]['numeric']`` (the output of ``describe_numeric``) gives + ``mean, median, std, min, max, p25, p75, iqr, n_outliers, outlier_pct, + distribution_type`` and the ``histogram`` bins ``[{lo, hi, count}]``. +- The boxplot five-number summary + Tukey 1.5·IQR fences are derived by the + pure registry function ``build_boxplot_stats`` (group ``eda``); this chapter + only consumes its output, it does not reimplement the statistics. + +Contract: build_(profile, ctx) -> Chapter | None ; CHAPTER_VERSION = "x.y.z". +Reads everything defensively (``.get``) and never raises: a column whose figure +cannot be built is degraded to a short note instead of aborting the chapter. +""" + +from __future__ import annotations + +from .. import model + +# Pure registry function (group ``eda``) that derives the Tukey boxplot stats +# from a ``numeric`` sub-block. Imported defensively so the chapter still builds +# (degrading the boxplot to a note) if the function is somehow unavailable. +try: + from datascience.build_boxplot_stats import build_boxplot_stats +except Exception: # noqa: BLE001 — keep the chapter importable no matter what. + build_boxplot_stats = None # type: ignore[assignment] + +CHAPTER_VERSION = "1.0.0" +CHAPTER_ID = "num_distr" +CHAPTER_TITLE = "Distribuciones numéricas" + +# Plain-Spanish gloss for every label ``detect_distribution_type`` can emit, so a +# non-expert reader understands the shape and the suggested next step (MUST-4.3). +_DIST_GLOSS = { + "normal-ish": "aproximadamente simétrica (campana); media y mediana casi " + "coinciden.", + "right-skewed": "asimétrica a la derecha (cola larga hacia valores altos); " + "la media supera a la mediana — considera una transformación " + "logarítmica.", + "left-skewed": "asimétrica a la izquierda (cola larga hacia valores bajos); " + "la media queda por debajo de la mediana.", + "heavy-tail": "colas pesadas (curtosis alta): más valores extremos de lo " + "que esperaría una normal — vigila los outliers.", + "lognormal-ish": "compatible con lognormal (simétrica al tomar logaritmos); " + "la re-expresión log suele normalizarla.", + "multimodal": "varios picos: probablemente mezcla de subgrupos — conviene " + "segmentar antes de resumir con una sola media.", + "discrete": "pocos valores distintos (discreta/ordinal); el histograma " + "cuenta niveles, no un continuo.", + "too_few_samples": "muestra demasiado pequeña para clasificar la forma con " + "fiabilidad.", + "other": "forma no encuadrada en las categorías estándar.", +} + + +def _fmt_num(value, decimals: int = 3) -> str: + """Compact, defensive number formatting shared with the other chapters.""" + if value is None: + return "—" + if isinstance(value, bool): + return str(value) + if isinstance(value, int): + return f"{value:,}".replace(",", ".") + if isinstance(value, float): + if value != value: # NaN + return "NaN" + if value in (float("inf"), float("-inf")): + return str(value) + text = f"{value:.{decimals}f}".rstrip("0").rstrip(".") + return text if text else "0" + return str(value) + + +def _numeric_columns(profile: dict) -> list: + """Return the list of (name, numeric_dict) for columns with usable stats.""" + out = [] + for col in profile.get("columns") or []: + if not isinstance(col, dict): + continue + if col.get("inferred_type") != "numeric": + continue + num = col.get("numeric") + if not isinstance(num, dict) or not num: + continue + # A numeric block is renderable when it carries at least a center. + if num.get("mean") is None and num.get("median") is None: + continue + out.append((col.get("name") or "(columna)", num)) + return out + + +def _make_hist_box(name: str, numeric: dict, box: dict): + """Build the histogram (with mean/median/±σ lines) + boxplot figure. + + Returned lazily to the renderer (a zero-arg callable via ``Figure.make``) so + matplotlib is only imported and the figure only drawn when a renderer needs + it. The two stacked axes share the X axis and are produced as a single + figure, which both renderers treat as one indivisible unit (scaled whole, + never cut). + """ + import matplotlib + + matplotlib.use("Agg") + import matplotlib.pyplot as plt + + fig, (ax_h, ax_b) = plt.subplots( + 2, 1, figsize=(6.4, 3.4), sharex=True, + gridspec_kw={"height_ratios": [3.2, 1.0], "hspace": 0.08}) + + # ---- Histogram from the precomputed equal-width bins {lo, hi, count}. ---- + hist = numeric.get("histogram") or [] + drew_bars = False + for b in hist: + if not isinstance(b, dict): + continue + lo = b.get("lo") + hi = b.get("hi") + count = b.get("count") or 0 + if lo is None or hi is None: + continue + width = (hi - lo) if hi > lo else max(abs(lo) * 1e-3, 1e-6) + ax_h.bar(lo, count, width=width, align="edge", color="#9ec6df", + edgecolor="#5b8aa6", linewidth=0.4, zorder=2) + drew_bars = True + if not drew_bars: + ax_h.text(0.5, 0.5, "(sin histograma)", ha="center", va="center", + fontsize=9, color="#8a8a8a", transform=ax_h.transAxes) + + mean = numeric.get("mean") + median = numeric.get("median") + std = numeric.get("std") + + # ±1σ band first (behind the lines), then median (solid) and mean (dashed). + if mean is not None and std is not None and std > 0: + ax_h.axvspan(mean - std, mean + std, color="#f0c27b", alpha=0.22, + zorder=1, label="±1σ") + if median is not None: + ax_h.axvline(median, color="#2e8b57", linestyle="-", linewidth=1.6, + zorder=4, label=f"mediana = {_fmt_num(median)}") + if mean is not None: + ax_h.axvline(mean, color="#c0392b", linestyle="--", linewidth=1.6, + zorder=4, label=f"media = {_fmt_num(mean)}") + + ax_h.set_ylabel("frecuencia", fontsize=8) + ax_h.tick_params(labelsize=7) + ax_h.legend(fontsize=6.5, loc="upper right", framealpha=0.85) + for spine in ("top", "right"): + ax_h.spines[spine].set_visible(False) + + # ---- Tukey boxplot below, sharing the X axis (MUST-4.2). ---- + if box: + stats = [{ + "med": box.get("median"), + "q1": box.get("q1"), + "q3": box.get("q3"), + "whislo": box.get("whisker_lo"), + "whishi": box.get("whisker_hi"), + "fliers": [], # raw outlier values are not in the profile. + "label": "", + }] + bxp_kw = dict( + showfliers=False, widths=0.5, patch_artist=True, + boxprops={"facecolor": "#9ec6df", "edgecolor": "#5b8aa6"}, + medianprops={"color": "#2e8b57", "linewidth": 1.6}, + whiskerprops={"color": "#5b8aa6"}, + capprops={"color": "#5b8aa6"}) + try: + # ``orientation`` is the current API; older matplotlib uses ``vert``. + try: + ax_b.bxp(stats, orientation="horizontal", **bxp_kw) + except TypeError: + ax_b.bxp(stats, vert=False, **bxp_kw) + except Exception: # noqa: BLE001 — never let one axis kill the figure. + pass + # Mark the presence of out-of-fence points (the raw values are unknown). + if box.get("has_low_outliers") and box.get("min") is not None: + ax_b.plot([box["min"]], [1], marker="o", markersize=3.5, + color="#c0392b", zorder=5) + if box.get("has_high_outliers") and box.get("max") is not None: + ax_b.plot([box["max"]], [1], marker="o", markersize=3.5, + color="#c0392b", zorder=5) + else: + ax_b.text(0.5, 0.5, "(boxplot no disponible)", ha="center", va="center", + fontsize=8, color="#8a8a8a", transform=ax_b.transAxes) + + ax_b.set_yticks([]) + ax_b.set_xlabel(name, fontsize=8) + ax_b.tick_params(labelsize=7) + for spine in ("top", "right", "left"): + ax_b.spines[spine].set_visible(False) + + fig.suptitle(name, fontsize=10, fontweight="bold", x=0.02, ha="left") + return fig + + +def _stats_note(name: str, numeric: dict, box: dict) -> str: + """One compact line of the key numbers + a plain-Spanish shape gloss.""" + bits = [ + f"media {_fmt_num(numeric.get('mean'))}", + f"mediana {_fmt_num(numeric.get('median'))}", + f"σ {_fmt_num(numeric.get('std'))}", + f"min {_fmt_num(numeric.get('min'))}", + f"max {_fmt_num(numeric.get('max'))}", + f"IQR {_fmt_num(numeric.get('iqr'))}", + ] + n_out = numeric.get("n_outliers") + out_pct = numeric.get("outlier_pct") + if n_out is not None: + pct = f" ({_fmt_num(out_pct, 2)}%)" if out_pct is not None else "" + bits.append(f"outliers {n_out}{pct}") + if box and (box.get("lower_fence") is not None): + bits.append( + f"vallas Tukey [{_fmt_num(box.get('lower_fence'))}, " + f"{_fmt_num(box.get('upper_fence'))}]") + line = " · ".join(bits) + + dist = numeric.get("distribution_type") + gloss = _DIST_GLOSS.get(dist) + if dist and gloss: + line += f"\n\n**Forma ({dist}):** {gloss}" + return line + + +def _figure_maker(name: str, numeric: dict, box: dict): + """Bind the per-column arguments so the lazy closure is loop-safe.""" + def _make(): + return _make_hist_box(name, numeric, box) + + return _make + + +def build_num_distr(profile: dict, ctx: dict): + """Build the numeric-distributions Chapter, or None if no numeric column. + + Args: + profile: the ``eda`` group TableProfile dict. + ctx: presentation context (unused here beyond defensive handling). + + Returns: + A ``model.Chapter`` with, per numeric column, a histogram+boxplot figure + and a stats note; or ``None`` when the dataset has no numeric column. + """ + profile = profile or {} + ctx = ctx or {} + + numerics = _numeric_columns(profile) + if not numerics: + return None # chapter does not apply to a dataset with no numerics. + + intro = ( + "Para cada columna numérica se muestra su **histograma** con tres líneas " + "de referencia: la **media** (línea roja discontinua), la **mediana** " + "(línea verde continua) y la banda **±1σ** (zona sombreada). Debajo, " + "alineado al mismo eje, un **boxplot de Tukey**: la caja abarca del " + "primer al tercer cuartil (P25–P75), la línea interior es la mediana y " + "los bigotes llegan hasta 1,5·IQR; los puntos rojos señalan que hay " + "valores más allá de las vallas. Comparar media y mediana revela la " + "asimetría de la distribución.") + + blocks = [ + model.Heading(text=CHAPTER_TITLE, level=1), + model.Markdown(text=intro), + ] + + for name, numeric in numerics: + box = {} + if build_boxplot_stats is not None: + try: + box = build_boxplot_stats(numeric) or {} + except Exception: # noqa: BLE001 — degrade, never raise. + box = {} + blocks.append(model.Heading(text=str(name), level=2)) + blocks.append(model.Figure( + make=_figure_maker(name, numeric, box), + caption=f"Distribución de «{name}» — histograma (media/mediana/±σ) " + f"y boxplot.")) + blocks.append(model.Markdown(text=_stats_note(name, numeric, box))) + + return model.Chapter(id=CHAPTER_ID, title=CHAPTER_TITLE, + version=CHAPTER_VERSION, blocks=blocks) diff --git a/python/functions/datascience/automatic_eda/chapters/num_distr_test.py b/python/functions/datascience/automatic_eda/chapters/num_distr_test.py new file mode 100644 index 00000000..a9b459ed --- /dev/null +++ b/python/functions/datascience/automatic_eda/chapters/num_distr_test.py @@ -0,0 +1,151 @@ +"""Tests for the NUM DISTR chapter — DoD: golden + edges + anti-cut. + +Self-contained: builds synthetic ``numeric`` blocks (no DuckDB) so the suite is +fast and deterministic. Verifies that the chapter emits, per numeric column, a +histogram+boxplot figure plus a stats note; that the mean/median/±σ requirement +and the boxplot are present; that a profile with no numeric column yields None; +that None/empty never raises; and that with many numeric columns and long text +both the PDF and the PPTX render without cutting anything (every column heading +survives in the rendered output). +""" + +import os +import re +import tempfile + +from pypdf import PdfReader + +from datascience.automatic_eda.chapters.num_distr import ( + build_num_distr, CHAPTER_VERSION, _DIST_GLOSS, +) +from datascience.automatic_eda import model +from datascience.render_automatic_eda_pdf import render_automatic_eda_pdf +from datascience.render_automatic_eda_pptx import render_automatic_eda_pptx + + +def _numeric_block(mean, median, std, mn, mx, dist="normal-ish", + n_outliers=0, nbins=10): + """A synthetic ``numeric`` sub-block shaped like describe_numeric's output.""" + width = (mx - mn) / nbins if mx > mn else 1.0 + hist = [{"lo": mn + i * width, "hi": mn + (i + 1) * width, + "count": (i + 1) * 3} for i in range(nbins)] + p25 = mn + (mx - mn) * 0.25 + p75 = mn + (mx - mn) * 0.75 + return { + "min": mn, "max": mx, "mean": mean, "median": median, "std": std, + "p25": p25, "p50": median, "p75": p75, "iqr": p75 - p25, + "n_outliers": n_outliers, "outlier_pct": 100.0 * n_outliers / 300.0, + "distribution_type": dist, "histogram": hist, + } + + +def _profile(n_numeric=2, extra_categorical=True): + cols = [] + presets = [ + ("precio", 42.5, 40.0, 12.3, 1.0, 100.0, "right-skewed", 5), + ("alcohol", 10.4, 10.3, 1.1, 8.0, 14.9, "normal-ish", 0), + ("sulfatos", 0.66, 0.62, 0.17, 0.33, 2.0, "heavy-tail", 9), + ("calidad", 5.6, 6.0, 0.8, 3.0, 8.0, "discrete", 0), + ] + for i in range(n_numeric): + name, mean, med, std, mn, mx, dist, no = presets[i % len(presets)] + if i >= len(presets): + name = f"{name}_{i}" + cols.append({"name": name, "inferred_type": "numeric", + "numeric": _numeric_block(mean, med, std, mn, mx, dist, no)}) + if extra_categorical: + cols.append({"name": "categoria", "inferred_type": "categorical", + "categorical": {"top": [{"value": "tinto", "count": 200}]}}) + return {"table": "vinos", "n_rows": 300, "n_cols": len(cols), + "columns": cols} + + +def _pdf_text(path: str) -> str: + txt = "".join((pg.extract_text() or "") for pg in PdfReader(path).pages) + return re.sub(r"\s+", " ", txt) + + +def test_golden_chapter_estructura_y_bloques(): + ch = build_num_distr(_profile(n_numeric=2), {}) + assert ch is not None + assert ch.id == "num_distr" + assert ch.version == CHAPTER_VERSION + kinds = [b.kind for b in ch.blocks] + # Heading + intro Markdown, then per column: Heading + Figure + Markdown. + assert kinds[0] == "heading" + assert kinds[1] == "markdown" + assert kinds.count("figure") == 2 # one figure per numeric column. + assert kinds.count("heading") == 1 + 2 # chapter title + one per column. + # Each figure has a lazy maker that produces a real matplotlib figure. + figs = [b for b in ch.blocks if b.kind == "figure"] + fig = figs[0].make() + assert fig is not None + # Two stacked axes: histogram + boxplot share the figure. + assert len(fig.axes) == 2 + import matplotlib.pyplot as plt + plt.close(fig) + + +def test_golden_media_mediana_sigma_y_boxplot_presentes(): + # The intro documents the three reference lines and the Tukey boxplot; the + # per-column note carries the actual mean/median/σ numbers and the shape. + ch = build_num_distr(_profile(n_numeric=1, extra_categorical=False), {}) + md_texts = " ".join(b.text for b in ch.blocks if b.kind == "markdown") + assert "media" in md_texts and "mediana" in md_texts + assert "±1σ" in md_texts or "σ" in md_texts + assert "boxplot" in md_texts.lower() + assert "Tukey" in md_texts + # distribution_type gloss surfaced for the column (right-skewed preset). + assert _DIST_GLOSS["right-skewed"].split(";")[0][:20] in md_texts + + +def test_boxplot_stats_se_consumen_del_registry(): + # The chapter must feed build_boxplot_stats (group eda) and the resulting + # box must carry the Tukey fences for the figure. + from datascience.build_boxplot_stats import build_boxplot_stats + box = build_boxplot_stats( + _numeric_block(42.5, 40.0, 12.3, 1.0, 100.0, "right-skewed", 5)) + assert box + assert "lower_fence" in box and "upper_fence" in box + assert box["q1"] is not None and box["q3"] is not None + + +def test_edge_sin_columnas_numericas_devuelve_none(): + prof = {"columns": [{"name": "c", "inferred_type": "categorical", + "categorical": {"top": []}}]} + assert build_num_distr(prof, {}) is None + + +def test_edge_profile_none_y_vacio_no_revienta(): + assert build_num_distr(None, None) is None + assert build_num_distr({}, {}) is None + assert build_num_distr({"columns": []}, {}) is None + + +def test_anti_corte_muchas_columnas_pdf_y_pptx(): + # 8 numeric columns + long note text: nothing may be cut. Every column + # heading must survive in both the PDF text and the PPTX deck. + ch = build_num_distr(_profile(n_numeric=8), {}) + names = [b.text for b in ch.blocks if b.kind == "heading" and b.level == 2] + assert len(names) == 8 + with tempfile.TemporaryDirectory() as d: + pdf = os.path.join(d, "num.pdf") + res_pdf = render_automatic_eda_pdf(_profile(n_numeric=8), pdf, + {"write_manifest": False}) + assert res_pdf["path"] == pdf + txt = _pdf_text(pdf) + for name in names: + assert name in txt, f"columna '{name}' cortada/ausente en el PDF" + pptx = os.path.join(d, "num.pptx") + res_pptx = render_automatic_eda_pptx(_profile(n_numeric=8), pptx, + {"write_manifest": False}) + assert res_pptx["path"] == pptx + assert res_pptx["n_slides"] >= 8 # at least one slide per column figure. + + +def test_distribution_gloss_cubre_todas_las_etiquetas(): + # Every label detect_distribution_type can emit has a Spanish gloss. + for label in ("normal-ish", "right-skewed", "left-skewed", "heavy-tail", + "lognormal-ish", "multimodal", "discrete", "too_few_samples", + "other"): + assert label in _DIST_GLOSS and _DIST_GLOSS[label] From d412522db9d74a667eba7f3562dd69d6180fa93f Mon Sep 17 00:00:00 2001 From: Egutierrez Date: Tue, 30 Jun 2026 14:59:10 +0200 Subject: [PATCH 08/53] =?UTF-8?q?feat(eda):=20cap=C3=ADtulo=20CALIDAD=20de?= =?UTF-8?q?l=20AutomaticEDA=20(criterios=20+=20scores=20+=20problemas=20ES?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Añade el capítulo de calidad de datos al motor AutomaticEDA, siguiendo el contrato de capítulos (build_calidad(profile, ctx) -> Chapter | None, CHAPTER_VERSION). El capítulo responde lo que pidió el usuario, en español y en formato de tabla: - Intro "Cómo se calcula la calidad": explica los tres criterios y sus pesos (completitud 50%, validez 30%, consistencia 20%) antes de cualquier número, más una KVTable de resumen a nivel tabla (calidad global y agregados). - Tabla "Scores por columna": score total más su desglose en completitud / validez / consistencia, ordenada de peor a mejor. - Tabla "Problemas detectados": los issues en español por columna, separados de los flags de tipo. Cuando no hay problemas, una nota honesta. Registry-first: el desglose y los issues NO se recalculan aquí; se consumen de la función pura del registry column_quality_score (grupo eda), que ya deriva {score, completeness, validity, consistency, issues} del ColumnProfile. El capítulo es render-only y compone bloques del modelo; los renderers paginan las tablas (parten por filas repitiendo cabecera) y envuelven celdas largas, de modo que nada se corta en PDF ni en PPTX. La lista de issues por celda se acota a 160 caracteres con "(+N más)" para que una fila nunca crezca más que una página. Test self-contained (sin DuckDB): golden con desglose + issues ES, edges (None/{}/sin columnas -> None; perfil limpio -> nota), y anti-cortes (perfil de 22 columnas con nombres largos renderizado a PDF y PPTX: el nombre completo sobrevive al envolverse, sin marcador de truncado). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../automatic_eda/chapters/calidad.py | 266 ++++++++++++++++++ .../automatic_eda/chapters/calidad_test.py | 194 +++++++++++++ 2 files changed, 460 insertions(+) create mode 100644 python/functions/datascience/automatic_eda/chapters/calidad.py create mode 100644 python/functions/datascience/automatic_eda/chapters/calidad_test.py diff --git a/python/functions/datascience/automatic_eda/chapters/calidad.py b/python/functions/datascience/automatic_eda/chapters/calidad.py new file mode 100644 index 00000000..dcedcf6f --- /dev/null +++ b/python/functions/datascience/automatic_eda/chapters/calidad.py @@ -0,0 +1,266 @@ +"""Data-quality chapter (CALIDAD) for AutomaticEDA. + +Builds the quality chapter from a ``TableProfile`` of the ``eda`` group. The +chapter answers, in Spanish and as tables, the three things the user asked for: + +1. **En qué se basa la calidad** — an intro paragraph explaining the criteria and + their weights (completeness, validity, consistency) before any number, plus a + table-level summary (global score and aggregates). +2. **Scores por columna** — a table with, per column, the total quality score and + its breakdown into completeness / validity / consistency. +3. **Problemas en español** — a second table listing, per column, the readable + issues in Spanish (kept separate from the type ``flags``). + +The breakdown and the issues are NOT recomputed here: they come from the registry +function ``column_quality_score`` (group ``eda``), which already derives +``{score, completeness, validity, consistency, issues}`` from the ColumnProfile. +This chapter is render-only — it consumes that function and lays the result out +as model blocks; the renderers paginate tables (splitting by rows, repeating the +header) and wrap long cells so nothing is ever cut. + +Contract: build_(profile, ctx) -> Chapter | None ; CHAPTER_VERSION = "x.y.z". +""" + +from __future__ import annotations + +from .. import model + +# Reuse the registry's pure quality function (group ``eda``). Import defensively: +# if the package cannot be imported for any reason the chapter degrades to the +# per-column ``quality_score`` already present in the profile instead of failing. +try: # pragma: no cover - import wiring + from ...column_quality_score import column_quality_score as _column_quality_score +except Exception: # noqa: BLE001 - never let an import error abort the document. + _column_quality_score = None + +CHAPTER_VERSION = "1.0.0" +CHAPTER_ID = "calidad" +CHAPTER_TITLE = "Calidad" + +# Weights mirror column_quality_score: completeness 0.5, validity 0.3, +# consistency 0.2. Kept here only to render the human explanation; the actual +# numbers always come from the function so the two never drift in computation. +_CRITERIA_INTRO = ( + "La calidad de cada columna es un score de 0 a 100 que combina tres " + "criterios, cada uno con un peso:\n\n" + "- **Completitud (peso 50%)**: proporción de valores presentes (sin nulos " + "ni vacíos). Una columna con muchos nulos baja de score.\n" + "- **Validez (peso 30%)**: los valores son coherentes con su tipo y rango " + "esperado (penaliza outliers y semánticas declaradas que no coinciden).\n" + "- **Consistencia (peso 20%)**: la columna aporta información útil (penaliza " + "columnas constantes o identificadores de cardinalidad muy alta).\n\n" + "Score = 100 × (0,5·completitud + 0,3·validez + 0,2·consistencia). " + "Los problemas detectados por columna se listan en español más abajo." +) + +# Cap for the joined issues cell so a single row never grows taller than a page; +# the remainder is summarized as "(+N más)" instead of being silently dropped. +_ISSUES_MAXLEN = 160 + + +def _fmt_score(value) -> str: + """Format a 0-100 score as ``NN / 100`` (or a placeholder).""" + if value is None: + return "—" + try: + num = float(value) + except (TypeError, ValueError): + return str(value) + if num != num: # NaN + return "—" + text = f"{num:.1f}".rstrip("0").rstrip(".") + return f"{text} / 100" + + +def _fmt_unit_pct(value) -> str: + """Format a 0-1 fraction as a percentage (``95%``).""" + if value is None: + return "—" + try: + return f"{float(value) * 100:.0f}%" + except (TypeError, ValueError): + return str(value) + + +def _quality_of(col: dict) -> dict: + """Return ``{score, completeness, validity, consistency, issues}`` for a column. + + Uses the registry ``column_quality_score`` when available; otherwise falls + back to the per-column ``quality_score`` already in the profile (number only, + empty breakdown/issues). Never raises. + """ + if not isinstance(col, dict): + col = {} + if _column_quality_score is not None: + try: + res = _column_quality_score(col) + if isinstance(res, dict): + return res + except Exception: # noqa: BLE001 - degrade instead of aborting. + pass + # Fallback: only the final score is available pre-computed in the profile. + return { + "score": col.get("quality_score"), + "completeness": None, + "validity": None, + "consistency": None, + "issues": [], + } + + +def _join_issues(issues) -> str: + """Join Spanish issue strings into one cell, truncating overly long lists. + + The renderer wraps cell text, but a column with many long issues could make a + single row taller than a whole page; cap the length and append ``(+N más)`` + so the count of hidden issues is honest rather than silently lost. + """ + if not isinstance(issues, (list, tuple)) or not issues: + return "" + parts = [model._safe_str(i).strip() for i in issues] + parts = [p for p in parts if p] + if not parts: + return "" + out = [] + used = 0 + for idx, part in enumerate(parts): + extra = len(part) + (2 if out else 0) + if used + extra > _ISSUES_MAXLEN and out: + remaining = len(parts) - idx + out.append(f"(+{remaining} más)") + return "; ".join(out) + out.append(part) + used += extra + return "; ".join(out) + + +def _columns_with_quality(profile: dict): + """Yield ``(col, quality_dict)`` for every column dict in the profile.""" + cols = profile.get("columns") or [] + for c in cols: + if isinstance(c, dict): + yield c, _quality_of(c) + + +def _summary_block(profile: dict, evaluated: list): + """Table-level KVTable: global score and quality aggregates.""" + rows = [] + score = profile.get("quality_score") + rows.append(("Calidad global", _fmt_score(score))) + rows.append(("Columnas evaluadas", str(len(evaluated)))) + + comps = [q.get("completeness") for _, q in evaluated + if isinstance(q.get("completeness"), (int, float))] + vals = [q.get("validity") for _, q in evaluated + if isinstance(q.get("validity"), (int, float))] + cons = [q.get("consistency") for _, q in evaluated + if isinstance(q.get("consistency"), (int, float))] + if comps: + rows.append(("Completitud media", _fmt_unit_pct(sum(comps) / len(comps)))) + if vals: + rows.append(("Validez media", _fmt_unit_pct(sum(vals) / len(vals)))) + if cons: + rows.append(("Consistencia media", _fmt_unit_pct(sum(cons) / len(cons)))) + + n_problem = sum(1 for _, q in evaluated if q.get("issues")) + rows.append(("Columnas con problemas", str(n_problem))) + + # Extra table-wide quality signals already in the profile, when present. + dup_pct = profile.get("duplicate_pct") + if dup_pct is not None: + rows.append(("Filas duplicadas", _fmt_unit_pct_or_pct(dup_pct))) + null_cell_pct = profile.get("null_cell_pct") + if null_cell_pct is not None: + rows.append(("Celdas nulas (global)", _fmt_unit_pct_or_pct(null_cell_pct))) + constant_cols = profile.get("constant_cols") + if isinstance(constant_cols, (list, tuple)) and constant_cols: + rows.append(("Columnas constantes", str(len(constant_cols)))) + all_null_cols = profile.get("all_null_cols") + if isinstance(all_null_cols, (list, tuple)) and all_null_cols: + rows.append(("Columnas 100% nulas", str(len(all_null_cols)))) + + return model.KVTable(rows=rows, title="Resumen de calidad") + + +def _fmt_unit_pct_or_pct(value) -> str: + """Format a value that may be a 0-1 fraction or an already-0-100 percentage.""" + try: + num = float(value) + except (TypeError, ValueError): + return model._safe_str(value) + if num != num: # NaN + return "—" + pct = num * 100 if num <= 1.0 else num + text = f"{pct:.1f}".rstrip("0").rstrip(".") + return f"{text}%" + + +def _scores_block(evaluated: list): + """DataTable with per-column score and its three-criteria breakdown.""" + header = ["Columna", "Calidad", "Completitud", "Validez", "Consistencia"] + rows = [] + # Worst columns first so the reader sees the problems at the top. + ordered = sorted( + evaluated, + key=lambda cq: (cq[1].get("score") + if isinstance(cq[1].get("score"), (int, float)) else 101.0), + ) + for col, q in ordered: + rows.append([ + col.get("name") or "(col)", + _fmt_score(q.get("score")), + _fmt_unit_pct(q.get("completeness")), + _fmt_unit_pct(q.get("validity")), + _fmt_unit_pct(q.get("consistency")), + ]) + if not rows: + return None + return model.DataTable(header=header, rows=rows, + title="Scores de calidad por columna", + note="0 = peor, 100 = mejor; ordenado de peor a mejor") + + +def _issues_block(evaluated: list): + """DataTable listing Spanish issues per column, or a Note when there are none.""" + header = ["Columna", "Problemas detectados (español)"] + rows = [] + for col, q in evaluated: + joined = _join_issues(q.get("issues")) + if joined: + rows.append([col.get("name") or "(col)", joined]) + if not rows: + return model.Note( + "No se detectaron problemas de calidad en las columnas evaluadas.") + return model.DataTable(header=header, rows=rows, + title="Problemas de calidad por columna") + + +def build_calidad(profile: dict, ctx: dict): + """Build the data-quality Chapter, or None if the profile has no columns. + + Reads everything defensively; returns ``None`` when there are no columns to + score (the chapter does not apply), and never raises on a malformed profile. + """ + profile = profile or {} + if not isinstance(profile, dict): + profile = {} + ctx = ctx or {} + + evaluated = list(_columns_with_quality(profile)) + if not evaluated: + return None # no columns to score -> chapter does not apply. + + blocks = [ + model.Heading(text="Cómo se calcula la calidad", level=2), + model.Markdown(text=_CRITERIA_INTRO), + _summary_block(profile, evaluated), + model.Heading(text="Scores por columna", level=2), + ] + scores = _scores_block(evaluated) + if scores is not None: + blocks.append(scores) + blocks.append(model.Heading(text="Problemas detectados", level=2)) + blocks.append(_issues_block(evaluated)) + + return model.Chapter(id=CHAPTER_ID, title=CHAPTER_TITLE, + version=CHAPTER_VERSION, blocks=blocks) diff --git a/python/functions/datascience/automatic_eda/chapters/calidad_test.py b/python/functions/datascience/automatic_eda/chapters/calidad_test.py new file mode 100644 index 00000000..3e6bf5f6 --- /dev/null +++ b/python/functions/datascience/automatic_eda/chapters/calidad_test.py @@ -0,0 +1,194 @@ +"""Tests for the CALIDAD chapter — DoD: golden + edges + anti-cut. + +Self-contained: builds synthetic TableProfiles (no DuckDB) so the suite is fast +and deterministic. Verifies that the chapter explains the quality criteria, shows +per-column scores with the completeness/validity/consistency breakdown, lists the +issues in Spanish (separate from the type flags), returns None when it does not +apply, and that a wide profile with long names renders to PDF and PPTX without +cutting any cell text (long content wraps, it is never truncated). +""" + +import os +import re +import tempfile + +from pypdf import PdfReader +from pptx import Presentation + +from datascience.automatic_eda.chapters.calidad import ( + build_calidad, + CHAPTER_VERSION, +) +from datascience.automatic_eda import build_document, render_pdf, render_pptx + + +def _profile() -> dict: + """A small profile with one column per quality problem (nulls, outliers, + constant, high-cardinality id) plus one clean column.""" + return { + "table": "demo", + "quality_score": 72.5, + "duplicate_pct": 0.04, + "null_cell_pct": 0.11, + "constant_cols": ["flag_const"], + "all_null_cols": [], + "columns": [ + {"name": "edad", "inferred_type": "integer", "null_pct": 0.2, + "numeric": {"outlier_pct": 0.15, "min": 0, "max": 99}, + "quality_score": 60}, + {"name": "nombre", "inferred_type": "text", "null_pct": 0.0, + "unique_pct": 0.98, "quality_score": 80}, + {"name": "flag_const", "inferred_type": "text", "null_pct": 0.0, + "flags": ["constant"], "quality_score": 50}, + {"name": "limpia", "inferred_type": "float", "null_pct": 0.0, + "numeric": {"outlier_pct": 0.0}, "quality_score": 100}, + ], + } + + +def _tables(chapter): + return [b for b in chapter.blocks if getattr(b, "kind", None) == "data_table"] + + +def _scores_table(chapter): + for t in _tables(chapter): + if "Scores" in (t.title or ""): + return t + return None + + +def _issues_table(chapter): + for t in _tables(chapter): + if "Problemas" in (t.title or ""): + return t + return None + + +# --------------------------------------------------------------------------- # +# Golden +# --------------------------------------------------------------------------- # +def test_golden_chapter_estructura_y_version(): + ch = build_calidad(_profile(), {}) + assert ch is not None + assert ch.id == "calidad" + assert ch.version == CHAPTER_VERSION + kinds = [b.kind for b in ch.blocks] + # intro heading + markdown criteria + summary kv + scores table + issues table + assert "markdown" in kinds and "kv_table" in kinds and "data_table" in kinds + + +def test_golden_intro_explica_criterios_y_pesos(): + ch = build_calidad(_profile(), {}) + intro = [b for b in ch.blocks if b.kind == "markdown"][0].text + for needle in ("Completitud", "Validez", "Consistencia", + "50%", "30%", "20%"): + assert needle in intro, f"falta {needle!r} en la intro de criterios" + + +def test_golden_scores_incluyen_desglose_por_criterio(): + ch = build_calidad(_profile(), {}) + scores = _scores_table(ch) + assert scores is not None + assert scores.header == ["Columna", "Calidad", "Completitud", + "Validez", "Consistencia"] + # 4 columns scored, none dropped. + assert len(scores.rows) == 4 + names = {r[0] for r in scores.rows} + assert names == {"edad", "nombre", "flag_const", "limpia"} + + +def test_golden_issues_en_espanol_separados_de_flags(): + ch = build_calidad(_profile(), {}) + issues = _issues_table(ch) + assert issues is not None + flat = " | ".join(" ".join(r) for r in issues.rows) + assert "nulos" in flat # completeness issue (ES) + assert "outliers" in flat # validity issue (ES) + assert "columna constante" in flat + assert "posible id de alta cardinalidad" in flat + # The raw type flag string must NOT leak as a "problem". + assert "constant" not in flat or "columna constante" in flat + + +# --------------------------------------------------------------------------- # +# Edges +# --------------------------------------------------------------------------- # +def test_edge_none_vacio_sin_columnas_devuelve_none(): + assert build_calidad(None, None) is None + assert build_calidad({}, {}) is None + assert build_calidad({"columns": []}, {}) is None + assert build_calidad("not a dict", {}) is None + + +def test_edge_perfil_limpio_sin_problemas_usa_nota(): + prof = { + "quality_score": 100, + "columns": [ + {"name": "a", "inferred_type": "float", "null_pct": 0.0, + "numeric": {"outlier_pct": 0.0}}, + {"name": "b", "inferred_type": "float", "null_pct": 0.0, + "numeric": {"outlier_pct": 0.0}}, + ], + } + ch = build_calidad(prof, {}) + assert ch is not None + assert _issues_table(ch) is None # no issues table + notes = [b for b in ch.blocks if b.kind == "note"] + assert notes and "No se detectaron problemas" in notes[0].text + + +# --------------------------------------------------------------------------- # +# Anti-cut: a wide profile with long names renders without truncation +# --------------------------------------------------------------------------- # +def _wide_profile(ncols: int = 22) -> dict: + cols = [ + {"name": "identificador_unico_de_transaccion_con_nombre_muy_largo", + "inferred_type": "text", "null_pct": 0.0, "unique_pct": 0.99}, + {"name": "columna_constante_sin_ninguna_variacion_de_valor", + "inferred_type": "text", "null_pct": 0.0, "flags": ["constant"]}, + ] + for k in range(ncols - 2): + cols.append({ + "name": f"metrica_numerica_de_negocio_{k:02d}_con_nombre_largo", + "inferred_type": "float", "null_pct": 0.1 + (k % 3) * 0.05, + "numeric": {"outlier_pct": 0.08, "min": 0, "max": 1000}, + }) + return {"table": "ancha", "quality_score": 70.0, "columns": cols} + + +def test_anticut_pdf_y_pptx_no_truncan_nombres_largos(): + prof = _wide_profile(22) + full = build_document(prof, {"dataset_name": "ancha"}) + assert any(c.id == "calidad" for c in full) + # Render ONLY the calidad chapter so the anti-cut assertions are scoped to + # this chapter (other chapters, e.g. portada, legitimately contain '…'). + chapters = [c for c in full if c.id == "calidad"] + long_name = "metrica_numerica_de_negocio_00_con_nombre_largo" + with tempfile.TemporaryDirectory() as d: + pdf = os.path.join(d, "q.pdf") + pptx = os.path.join(d, "q.pptx") + rp = render_pdf(chapters, pdf, {"title": "EDA"}) + rx = render_pptx(chapters, pptx, {"title": "EDA"}) + assert os.path.exists(pdf) and os.path.exists(pptx) + # The wide table forces pagination across several pages/slides. + assert (rp or {}).get("n_pages", 0) >= 2 + + # PDF: the long name survives whole once wraps (spaces/newlines) removed, + # and there is no truncation marker. + pdf_txt = "".join((pg.extract_text() or "") for pg in PdfReader(pdf).pages) + assert "…" not in pdf_txt and "..." not in pdf_txt + norm = re.sub(r"\s+", "", pdf_txt) + assert long_name in norm, "el nombre largo se cortó en el PDF" + + # PPTX: long name present in some cell, untruncated. + allt = [] + for s in Presentation(pptx).slides: + for sh in s.shapes: + if sh.has_text_frame: + allt.append(sh.text_frame.text) + if sh.has_table: + for row in sh.table.rows: + for c in row.cells: + allt.append(c.text) + joined = re.sub(r"\s+", "", "\n".join(allt)) + assert long_name in joined, "el nombre largo se cortó en el PPTX" From 03f3dca82368311410d35e0d1307e84cf5c87661 Mon Sep 17 00:00:00 2001 From: Egutierrez Date: Tue, 30 Jun 2026 14:59:50 +0200 Subject: [PATCH 09/53] =?UTF-8?q?feat(eda):=20cap=C3=ADtulo=20CORRELACION?= =?UTF-8?q?=20de=20AutomaticEDA=20(matriz=20+=20top=20pares=20=C2=B1)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implementa chapters/correlacion.py siguiendo el contrato de capítulos: build_correlacion(profile, ctx) -> Chapter|None, CHAPTER_VERSION="1.0.0". Consume profile['correlations'] (salida de association_matrix del grupo eda, sin recalcular estadística) y emite, como bloques del modelo: - Matriz de asociación (Figure/heatmap perezoso, RdBu_r, con signo en num-num y magnitud en métricas mixtas; etiquetas ordenadas por conectividad y recortadas a las 16 más conectadas para legibilidad). - TOP de pares POSITIVOS y TOP de pares NEGATIVOS en dos DataTable separadas (los negativos son por construcción num-num, único método con signo), con método, valor, p-valor corregido (FDR) y significancia. - Resumen FDR (multiple_testing) + leyenda de métodos. - Aviso de espuriedad por niveles no estacionarios (Granger-Newbold) cuando el profile lo marca. Lectura defensiva en todo (None si no hay pares; nunca lanza). Anti-cortes: sólo bloques del modelo, el paginador parte tablas repitiendo cabecera y escala la figura entera. Test self-contained (5 casos): golden a nivel de bloques + golden render PDF/PPTX, edge sin pares -> None, edge sólo positivos -> nota honesta, y anti-corte con matriz ancha + etiquetas largas (dato íntegro a nivel de bloque, ambos renderers sin reventar). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../automatic_eda/chapters/correlacion.py | 352 ++++++++++++++++++ .../chapters/correlacion_test.py | 175 +++++++++ 2 files changed, 527 insertions(+) create mode 100644 python/functions/datascience/automatic_eda/chapters/correlacion.py create mode 100644 python/functions/datascience/automatic_eda/chapters/correlacion_test.py diff --git a/python/functions/datascience/automatic_eda/chapters/correlacion.py b/python/functions/datascience/automatic_eda/chapters/correlacion.py new file mode 100644 index 00000000..22b6eb0c --- /dev/null +++ b/python/functions/datascience/automatic_eda/chapters/correlacion.py @@ -0,0 +1,352 @@ +"""Correlation chapter — association matrix plus top positive/negative pairs. + +Builds the CORRELACION chapter of an AutomaticEDA document from a TableProfile. +It renders exactly what the user asked for: + +1. A correlation/association **matrix** (heatmap) reconstructed from the evaluated + pairs, signed for numeric-numeric pairs (Pearson/Spearman, ``[-1, 1]``) and as + magnitude for the mixed-type metrics (Cramér's V, correlation ratio, mutual + information, ``[0, 1]``). Labels are ordered by total connectivity so strong + associations cluster together instead of being scattered alphabetically. +2. The **TOP positive** pairs and the **TOP negative** pairs as two separate + tables. Only numeric-numeric metrics carry a sign, so negative pairs are by + construction Pearson/Spearman; positive pairs may use any method. +3. The methods legend and the multiple-testing (FDR) summary, so the reader sees + how many pairs survive the correction. +4. A spuriousness caveat when the profile flags level-based correlations on + non-stationary series (Granger–Newbold). + +All data comes from ``profile['correlations']`` — the output of the ``eda`` group +function ``association_matrix`` (optionally enriched by ``profile_table``). The +chapter never recomputes any statistic; it only lays the existing values out as +format-independent blocks. The renderers paginate tables (repeating the header) +and scale the heatmap to fit entirely, so nothing is ever cut. + +Contract: build_(profile, ctx) -> Chapter | None ; CHAPTER_VERSION = "x.y.z". +""" + +from __future__ import annotations + +import math + +from .. import model + +CHAPTER_VERSION = "1.0.0" +CHAPTER_ID = "correlacion" +CHAPTER_TITLE = "Correlación" + +# Methods whose value carries a sign (direction). Everything else is a magnitude +# in [0, 1] and therefore only ever contributes to the positive side. +_SIGNED_METHODS = ("pearson", "spearman") + +# Cap the heatmap to the most-connected variables so it stays legible on a phone +# screen / a slide. The renderer would scale a bigger matrix to fit, but the +# cells become unreadable; we instead show the top-N and say so. +_MAX_MATRIX_LABELS = 16 + +# How many pairs to show in each of the top-positive / top-negative tables. +_TOP_N = 10 + + +def _is_num(v) -> bool: + """True for a real, finite int/float (not bool, not NaN/inf).""" + return ( + isinstance(v, (int, float)) + and not isinstance(v, bool) + and not (isinstance(v, float) and (math.isnan(v) or math.isinf(v))) + ) + + +def _fmt_val(value, decimals: int = 2) -> str: + """Format an association value compactly, signed, with a fixed width feel.""" + if not _is_num(value): + return "—" + text = f"{float(value):+.{decimals}f}" + # Strip a trailing -0.00 / +0.00 into a clean 0.00 for readability. + if text in ("+0.00", "-0.00"): + return "0.00" + return text + + +def _fmt_p(value) -> str: + """Format an adjusted p-value; tiny values collapse to a '<' threshold.""" + if not _is_num(value): + return "—" + p = float(value) + if p < 0.001: + return "<0.001" + return f"{p:.3f}" + + +def _is_signed(pair: dict) -> bool: + """True if the pair's method reports a directional (signed) value.""" + method = str(pair.get("method") or "").lower() + return any(m in method for m in _SIGNED_METHODS) + + +def _significant(pair: dict) -> bool: + """True if the pair is significant after FDR (or has no test to correct).""" + if pair.get("significant") is True: + return True + # Pairs without an applicable test (p_value None) are not penalised: they are + # admitted on magnitude alone upstream, so treat missing as "not rejected". + return pair.get("p_value") is None and pair.get("significant") is None + + +def _label(pair: dict) -> str: + """Human label for a pair, e.g. 'alcohol ↔ density'.""" + return f"{model._safe_str(pair.get('a'))} ↔ {model._safe_str(pair.get('b'))}" + + +def _split_top(pairs: list, top_n: int = _TOP_N): + """Split evaluated pairs into ranked top-positive and top-negative lists. + + Positive: any pair with a positive value, ranked by value descending. + Negative: only signed (numeric-numeric) pairs with a negative value, ranked + by value ascending (most negative first). Non-finite values are dropped. + """ + positive = [] + negative = [] + for pair in pairs: + if not isinstance(pair, dict): + continue + value = pair.get("value") + if not _is_num(value): + continue + if value > 0: + positive.append(pair) + elif value < 0 and _is_signed(pair): + negative.append(pair) + positive.sort(key=lambda p: float(p.get("value", 0.0)), reverse=True) + negative.sort(key=lambda p: float(p.get("value", 0.0))) + return positive[:top_n], negative[:top_n] + + +def _top_table(pairs: list, title: str): + """Build a DataTable for a list of pairs, or None if there are none.""" + if not pairs: + return None + header = ["Par", "Método", "Valor", "p (FDR)", "Sig."] + rows = [] + for pair in pairs: + method = model._safe_str(pair.get("method")) or "—" + rows.append([ + _label(pair), + method, + _fmt_val(pair.get("value")), + _fmt_p(pair.get("p_value_adjusted")), + "sí" if _significant(pair) else "no", + ]) + return model.DataTable(header=header, rows=rows, title=title) + + +def _ordered_labels(pairs: list): + """Pick and order the matrix labels by total connectivity (descending). + + Returns the list of variable names to place on the axes, capped at + ``_MAX_MATRIX_LABELS`` (the most-connected ones), plus a boolean saying + whether the cap trimmed anything. + """ + strength = {} + for pair in pairs: + if not isinstance(pair, dict): + continue + value = pair.get("value") + if not _is_num(value): + continue + mag = abs(float(value)) + for key in ("a", "b"): + name = pair.get(key) + if name is None: + continue + strength[name] = strength.get(name, 0.0) + mag + if not strength: + return [], False + ordered = sorted(strength, key=lambda n: strength[n], reverse=True) + trimmed = len(ordered) > _MAX_MATRIX_LABELS + return ordered[:_MAX_MATRIX_LABELS], trimmed + + +def _matrix_figure(pairs: list, labels: list): + """Return a Figure (lazy) with the signed association heatmap, or None. + + The matplotlib figure is built lazily inside ``make`` so importing this + module never requires matplotlib and a malformed plot degrades to nothing + instead of aborting the chapter. + """ + if len(labels) < 2: + return None + + index = {name: i for i, name in enumerate(labels)} + + def make(): + import numpy as np + from matplotlib.figure import Figure + + n = len(labels) + grid = np.full((n, n), np.nan, dtype=float) + for i in range(n): + grid[i, i] = 1.0 + for pair in pairs: + if not isinstance(pair, dict): + continue + a = pair.get("a") + b = pair.get("b") + value = pair.get("value") + if a not in index or b not in index or not _is_num(value): + continue + v = float(value) + # Mixed-type magnitudes are non-negative; keep them as-is on [0, 1]. + ia, ib = index[a], index[b] + grid[ia, ib] = v + grid[ib, ia] = v + + import matplotlib + + masked = np.ma.masked_invalid(grid) + fig = Figure(figsize=(6.2, 5.6)) + ax = fig.add_subplot(111) + cmap = matplotlib.colormaps["RdBu_r"].copy() + cmap.set_bad(color="#eeeeee") + im = ax.imshow(masked, cmap=cmap, vmin=-1.0, vmax=1.0, aspect="auto") + ax.set_xticks(range(n)) + ax.set_yticks(range(n)) + short = [str(s)[:14] for s in labels] + ax.set_xticks(range(n)) + ax.set_xticklabels(short, rotation=90, fontsize=7) + ax.set_yticklabels(short, fontsize=7) + # Annotate cells only when the matrix is small enough to stay legible. + if n <= 8: + for i in range(n): + for j in range(n): + cell = grid[i, j] + if _is_num(cell): + ax.text(j, i, f"{cell:+.2f}".replace("+", "") if cell < 0 + else f"{cell:.2f}", + ha="center", va="center", fontsize=6, + color="#222222") + fig.colorbar(im, ax=ax, fraction=0.046, pad=0.04, + label="asociación (signo en num-num)") + fig.tight_layout() + return fig + + return model.Figure(make=make, + caption="Matriz de asociación. Azul = positiva, rojo = " + "negativa (sólo num-num lleva signo); gris = par " + "no evaluado.") + + +def _methods_block(corr: dict): + """Build a KVTable with the legend of the methods actually present.""" + legend = corr.get("methods_legend") + if not isinstance(legend, dict) or not legend: + return None + rows = [(model._safe_str(k), model._safe_str(v)) for k, v in legend.items()] + return model.KVTable(rows=rows, title="Métodos de asociación") + + +def _fdr_text(corr: dict) -> str | None: + """One-line summary of the multiple-testing (FDR) correction, or None.""" + mt = corr.get("multiple_testing") + if not isinstance(mt, dict) or not mt: + return None + method = model._safe_str(mt.get("method")).upper() or "FDR" + alpha = mt.get("alpha") + n_tests = mt.get("n_tests") + n_rej = mt.get("n_rejected") + parts = [f"Corrección por comparaciones múltiples ({method}"] + if _is_num(alpha): + parts[0] += f", α={float(alpha):g}" + parts[0] += ")." + if _is_num(n_tests): + rej = n_rej if _is_num(n_rej) else "—" + parts.append( + f"De {int(n_tests)} pares con test, {rej} siguen siendo " + f"significativos tras la corrección.") + return " ".join(parts) + + +def build_correlacion(profile: dict, ctx: dict): + """Build the Correlation Chapter, or None if there are no pairs to show. + + Reads ``profile['correlations']`` (the ``association_matrix`` output). Returns + ``None`` when the dataset has fewer than two associable columns (no evaluated + pairs), so the chapter is omitted instead of showing an empty section. Never + raises: every access is defensive. + + ctx keys consumed: none specific (presentation metadata is inherited from the + document). The chapter reads everything it needs from the profile. + """ + profile = profile or {} + ctx = ctx or {} + + corr = profile.get("correlations") + if not isinstance(corr, dict): + return None + pairs = corr.get("pairs") + if not isinstance(pairs, list) or not pairs: + return None + + blocks: list = [] + + # Intro: what this chapter shows and how to read the sign. + blocks.append(model.Markdown(text=( + "Asociación entre columnas. Cada par se evalúa con la métrica adecuada a " + "sus tipos (Pearson/Spearman entre numéricas — con **signo**; Cramér's V " + "entre categóricas; razón de correlación num-categórica; información mutua " + "como medida común no lineal). Sólo las correlaciones **num-num** tienen " + "dirección: por eso los pares **negativos** son siempre num-num."))) + + # 1) Association matrix (heatmap). + labels, trimmed = _ordered_labels(pairs) + fig = _matrix_figure(pairs, labels) + if fig is not None: + blocks.append(model.Heading(text="Matriz de asociación", level=2)) + blocks.append(fig) + if trimmed: + blocks.append(model.Note(text=( + f"Se muestran las {len(labels)} variables más conectadas de la " + "matriz para mantenerla legible; el resto de pares siguen en las " + "tablas de abajo."))) + + # 2) Top positive / top negative pairs. + positive, negative = _split_top(pairs, _TOP_N) + pos_table = _top_table(positive, f"Top {len(positive)} positivas") + neg_table = _top_table(negative, f"Top {len(negative)} negativas") + if pos_table is not None: + blocks.append(model.Heading(text="Pares más correlacionados (positivos)", + level=2)) + blocks.append(pos_table) + if neg_table is not None: + blocks.append(model.Heading(text="Pares más correlacionados (negativos)", + level=2)) + blocks.append(neg_table) + elif pos_table is not None: + # No signed-negative pairs at all: say so honestly rather than omit. + blocks.append(model.Note(text=( + "No se han hallado correlaciones negativas significativas entre " + "columnas numéricas."))) + + # 3) Spuriousness caveat for level-based correlations (Granger–Newbold). + caveat = corr.get("levels_caveat") + if isinstance(caveat, str) and caveat.strip(): + blocks.append(model.Note(text=caveat.strip())) + elif corr.get("levels_possible_spurious"): + blocks.append(model.Note(text=( + "Aviso: algunas correlaciones se calcularon sobre niveles de series " + "no estacionarias y pueden ser espurias (Granger–Newbold). Compáralas " + "sobre los retornos/diferencias antes de interpretarlas."))) + + # 4) FDR summary + methods legend. + fdr_text = _fdr_text(corr) + if fdr_text: + blocks.append(model.Markdown(text=fdr_text)) + methods = _methods_block(corr) + if methods is not None: + blocks.append(model.Heading(text="Métodos y leyenda", level=2)) + blocks.append(methods) + + if not blocks: + return None + return model.Chapter(id=CHAPTER_ID, title=CHAPTER_TITLE, + version=CHAPTER_VERSION, blocks=blocks) diff --git a/python/functions/datascience/automatic_eda/chapters/correlacion_test.py b/python/functions/datascience/automatic_eda/chapters/correlacion_test.py new file mode 100644 index 00000000..88ddc726 --- /dev/null +++ b/python/functions/datascience/automatic_eda/chapters/correlacion_test.py @@ -0,0 +1,175 @@ +"""Tests for the CORRELACION chapter — DoD: golden + edges + error/anti-cut. + +Self-contained: builds a synthetic TableProfile carrying a ``correlations`` block +shaped exactly like ``association_matrix`` output (no DuckDB), so the suite is +fast and deterministic. Verifies that the chapter emits the association-matrix +figure plus separate top-positive / top-negative tables with the right pairs, +that it returns None when the profile has no pairs, that a None/empty profile +does not raise, and that a wide matrix with long labels renders to PDF *and* PPTX +without cutting anything. +""" + +import os +import re +import tempfile + +from pypdf import PdfReader + +from datascience.automatic_eda.chapters.correlacion import ( + CHAPTER_VERSION, + build_correlacion, +) +from datascience.automatic_eda.model import DataTable, Figure +from datascience.render_automatic_eda_pdf import render_automatic_eda_pdf +from datascience.render_automatic_eda_pptx import render_automatic_eda_pptx + + +def _pair(a, b, value, method, padj, sig, p=0.0001): + return { + "a": a, "b": b, "a_type": "numeric", "b_type": "numeric", + "method": method, "value": value, "extra": {"mi": abs(value) * 0.5}, + "p_value": p, "p_value_adjusted": padj, "significant": sig, + } + + +def _profile() -> dict: + """Synthetic wine-like profile with signed and unsigned associations.""" + pairs = [ + _pair("alcohol", "quality", 0.48, "pearson/spearman", 0.0005, True), + _pair("density", "alcohol", -0.78, "pearson/spearman", 0.0001, True), + _pair("ph", "fixed_acidity", -0.68, "pearson/spearman", 0.0002, True), + _pair("sulphates", "quality", 0.25, "pearson/spearman", 0.03, True), + # Unsigned mixed-type metrics: only ever positive, never in the neg table. + {"a": "region", "b": "type", "a_type": "categorical", + "b_type": "categorical", "method": "cramers_v", "value": 0.55, + "extra": {"mi": 0.3}, "p_value": 0.001, "p_value_adjusted": 0.004, + "significant": True}, + ] + return { + "table": "wine", + "source": "/data/wine.csv", + "n_rows": 1599, + "n_cols": 12, + "correlations": { + "pairs": pairs, + "strong": [p for p in pairs if abs(p["value"]) >= 0.5], + "methods_legend": { + "pearson": "num-num lineal (Pearson r), [-1, 1]", + "cramers_v": "cat-cat simétrica (Cramér's V), [0, 1]", + }, + "multiple_testing": {"method": "bh", "alpha": 0.05, + "n_tests": 5, "n_rejected": 5}, + }, + } + + +def _pdf_text(path: str) -> str: + txt = "".join((pg.extract_text() or "") for pg in PdfReader(path).pages) + return re.sub(r"\s+", " ", txt) + + +def test_golden_chapter_tiene_matriz_y_top_positivos_y_negativos(): + ch = build_correlacion(_profile(), {}) + assert ch is not None + assert ch.id == "correlacion" + assert ch.version == CHAPTER_VERSION + kinds = [b.kind for b in ch.blocks] + assert "figure" in kinds # association matrix heatmap. + figs = [b for b in ch.blocks if isinstance(b, Figure)] + assert figs and figs[0].make is not None # lazy figure. + + tables = [b for b in ch.blocks if isinstance(b, DataTable)] + assert len(tables) >= 2 # top positive + top negative. + flat = " ".join(str(c) for t in tables for r in t.rows for c in r) + # Strongest positive present and signed +, strongest negative present and -. + assert "alcohol" in flat and "quality" in flat + assert "+0.48" in flat + assert "density" in flat and "-0.78" in flat + + +def test_golden_render_pdf_y_pptx_muestran_lo_exigido(): + prof = _profile() + with tempfile.TemporaryDirectory() as d: + pdf = os.path.join(d, "corr.pdf") + pptx = os.path.join(d, "corr.pptx") + rp = render_automatic_eda_pdf(prof, pdf, {"title": "EDA — wine"}) + rx = render_automatic_eda_pptx(prof, pptx, {"title": "EDA — wine"}) + assert rp["path"] == pdf and rp["n_pages"] >= 1 + assert rx["path"] == pptx and rx["n_slides"] >= 1 + assert "correlacion" in [c["id"] for c in rp["chapters"]] + assert "correlacion" in [c["id"] for c in rx["chapters"]] + txt = _pdf_text(pdf) + # The requirement: matrix + top positive/negative pairs, all visible. + assert "Correlaci" in txt # chapter title (accents may vary in extract). + assert "density" in txt and "alcohol" in txt and "quality" in txt + assert "0.78" in txt and "0.48" in txt + # Both signs surfaced as separate sections. + assert "positiv" in txt.lower() and "negativ" in txt.lower() + + +def test_edge_sin_pares_devuelve_none(): + # No correlations key, empty pairs, and wrong types all yield None, not error. + assert build_correlacion({"table": "x"}, {}) is None + assert build_correlacion({"correlations": {}}, {}) is None + assert build_correlacion({"correlations": {"pairs": []}}, {}) is None + assert build_correlacion({"correlations": {"pairs": "nope"}}, {}) is None + assert build_correlacion(None, None) is None + assert build_correlacion({}, {}) is None + + +def test_edge_solo_positivos_emite_nota_sin_tabla_negativa(): + prof = { + "correlations": { + "pairs": [ + _pair("a", "b", 0.6, "pearson/spearman", 0.001, True), + {"a": "c", "b": "d", "a_type": "categorical", + "b_type": "categorical", "method": "cramers_v", "value": 0.7, + "extra": {"mi": 0.4}, "p_value": 0.001, + "p_value_adjusted": 0.003, "significant": True}, + ], + }, + } + ch = build_correlacion(prof, {}) + assert ch is not None + tables = [b for b in ch.blocks if isinstance(b, DataTable)] + assert len(tables) == 1 # only the positive table. + notes = " ".join(b.text for b in ch.blocks if b.kind == "note") + assert "negativas" in notes # honest "no negative correlations" note. + + +def test_anticorte_matriz_ancha_y_etiquetas_largas_no_se_cortan(): + # 20 numeric vars with long names -> matrix trimmed to top-N + both renderers + # must lay the chapter out without raising and keep a long label intact. + long_a = "concentracion_de_dioxido_de_azufre_libre" + long_b = "concentracion_de_dioxido_de_azufre_total" + pairs = [_pair(long_a, long_b, -0.72, "pearson/spearman", 0.0001, True)] + for i in range(20): + pairs.append(_pair(f"variable_numerica_larga_{i:02d}", + f"variable_numerica_larga_{(i + 1) % 20:02d}", + 0.55 - i * 0.02, "pearson/spearman", 0.01, True)) + prof = {"correlations": {"pairs": pairs, + "multiple_testing": {"method": "bh", "alpha": 0.05, + "n_tests": len(pairs), + "n_rejected": len(pairs)}}} + ch = build_correlacion(prof, {}) + assert ch is not None + # A "showing top-N most connected" note appears when the matrix is trimmed. + notes = " ".join(b.text for b in ch.blocks if b.kind == "note") + assert "más conectadas" in notes + # Anti-cut guarantee at the block level: the long pair reaches the renderer + # whole (the block never truncates); the renderer then wraps the cell inside + # its column. Both long labels are present, intact, in a table cell. + tables = [b for b in ch.blocks if isinstance(b, DataTable)] + cells = [str(c) for t in tables for r in t.rows for c in r] + assert any(long_a in c and long_b in c for c in cells) + with tempfile.TemporaryDirectory() as d: + pdf = os.path.join(d, "wide.pdf") + pptx = os.path.join(d, "wide.pptx") + rp = render_automatic_eda_pdf(prof, pdf, {"write_manifest": False}) + rx = render_automatic_eda_pptx(prof, pptx, {"write_manifest": False}) + # Both renderers lay the wide chapter out without raising and produce a + # non-empty document (nothing dropped, just wrapped/scaled to fit). + assert rp["path"] == pdf and os.path.exists(pdf) and rp["n_pages"] >= 1 + assert rx["path"] == pptx and os.path.exists(pptx) and rx["n_slides"] >= 1 + # A short, unbreakable fragment of the long label survives the wrap. + assert "azufre" in _pdf_text(pdf) From fc5bc334c8907934a6726dde345ba56cbeb5e92e Mon Sep 17 00:00:00 2001 From: Egutierrez Date: Tue, 30 Jun 2026 15:01:26 +0200 Subject: [PATCH 10/53] =?UTF-8?q?feat(eda):=20cap=C3=ADtulo=20AN=C3=81LISI?= =?UTF-8?q?S=20LLM=20para=20AutomaticEDA,=20junto=20al=20overview?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Nuevo capítulo `analisis_llm` del motor AutomaticEDA. Consume el bloque `llm` que `eda_llm_insights` (grupo eda) ya deja en el TableProfile —no llama al LLM ni recalcula— y lo convierte en bloques del modelo de documento para que se renderice sin cortarse en PDF ni PPTX: - Resumen de la tabla y significado de una fila -> bloques Markdown (el renderer los envuelve a líneas completas, nunca pierde texto). - Diccionario de datos y PII -> DataTable (el paginador parte por filas repitiendo cabecera y envuelve celdas largas dentro de su columna). - Análisis sugeridos y limpieza sugerida -> listas de viñetas Markdown; cada entrada es una línea completa que el renderer envuelve, nunca trunca. Lectura defensiva (.get) en todo; devuelve None si el profile no trae bloque `llm` (p.ej. profile_table sin run_llm) para omitir el capítulo. MUST-3.2 (report 2043): se mueve `analisis_llm` en CHAPTER_ORDER a la posición inmediatamente posterior a `overview`, como pidió el usuario ("va junto al overview"). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../automatic_eda/chapters/analisis_llm.py | 221 ++++++++++++++++++ .../automatic_eda/chapters_registry.py | 2 +- 2 files changed, 222 insertions(+), 1 deletion(-) create mode 100644 python/functions/datascience/automatic_eda/chapters/analisis_llm.py diff --git a/python/functions/datascience/automatic_eda/chapters/analisis_llm.py b/python/functions/datascience/automatic_eda/chapters/analisis_llm.py new file mode 100644 index 00000000..e182e6a0 --- /dev/null +++ b/python/functions/datascience/automatic_eda/chapters/analisis_llm.py @@ -0,0 +1,221 @@ +"""LLM analysis chapter (ANÁLISIS LLM) — the interpretive layer, next to overview. + +Third reference chapter for AutomaticEDA. Renders the ``llm`` block that the +``eda`` group function ``eda_llm_insights`` already produced and stored in the +``TableProfile`` — it does NOT call the LLM nor recompute anything. The block is +turned into clean, markdown-style document blocks so it reads as a real chapter +(table summary, row meaning, data dictionary, suggested analyses, cleaning +suggestions, PII findings) and, crucially, **nothing is ever cut** in PDF or +PPTX: + +* Prose (summary, row meaning) → ``Markdown`` blocks the renderers wrap to whole + lines, so no word is lost no matter how long the text is. +* The data dictionary and PII findings → ``DataTable`` blocks the paginator + splits by rows (repeating the header) and whose long cells wrap inside their + column — wide, multi-row tables never overflow a page/slide. +* Cleaning suggestions and suggested analyses → ``Markdown`` bullet lists; each + item is a whole line the renderer wraps, never truncated mid-entry. + +Position: this chapter is declared in ``chapters_registry.CHAPTER_ORDER`` right +after ``overview`` so the interpretation sits next to the table preview, as the +user asked ("va junto al overview"). + +Data source: the ``llm`` dict produced by ``eda_llm_insights`` (group ``eda``), +read from ``profile['llm']`` (or ``ctx['llm']`` as a fallback). Shape:: + + { + "summary": str, # what the table is, 2-3 sentences + "row_meaning": str, # what one row represents / granularity + "dictionary": [ {"column","description","business_meaning","unit"} ], + "pii": [ {"column","kind","severity"} ], + "cleaning": [str], # cleaning / transformation suggestions + "analyses": [str], # suggested questions / analyses / hypotheses + } + +Contract: build_(profile, ctx) -> Chapter | None ; CHAPTER_VERSION = "x.y.z". +Reads everything defensively (``.get``) and NEVER raises; returns ``None`` when +the profile carries no LLM block (e.g. ``profile_table`` ran without +``run_llm``), so the chapter is simply omitted from the document. +""" + +from __future__ import annotations + +from .. import model + +CHAPTER_VERSION = "1.0.0" +CHAPTER_ID = "analisis_llm" +CHAPTER_TITLE = "Análisis LLM" + +# Key under which eda_llm_insights stores its interpretive block in the profile. +LLM_KEY = "llm" + + +def _clean_text(value) -> str: + """Coerce a value to a single trimmed line (collapse inner newlines). + + Used for bullet items so each suggestion stays a single markdown bullet the + renderer wraps; never drops content, only normalizes whitespace. + """ + text = model._safe_str(value).strip() + if not text: + return "" + return " ".join(text.split()) + + +def _para(value) -> str: + """Coerce a value to trimmed prose, preserving paragraph breaks.""" + text = model._safe_str(value).strip() + if not text: + return "" + # Keep blank-line paragraph breaks; collapse runs of spaces/tabs per line. + lines = [" ".join(ln.split()) for ln in text.splitlines()] + out: list = [] + for ln in lines: + if ln or (out and out[-1] != ""): + out.append(ln) + return "\n".join(out).strip() + + +def _bullets(items) -> str: + """Build a markdown bullet list from a sequence of strings. + + Each item becomes one ``- ...`` line (a whole, wrappable unit). Empty items + and non-list inputs are handled gracefully; returns "" when there is nothing. + """ + if isinstance(items, str): + items = [items] + if not isinstance(items, (list, tuple)): + return "" + lines = [] + for it in items: + text = _clean_text(it) + if text: + lines.append(f"- {text}") + return "\n".join(lines) + + +def _summary_blocks(llm: dict) -> list: + """Heading + prose for the table summary, or [] if absent.""" + text = _para(llm.get("summary")) + if not text: + return [] + return [model.Heading(text="Resumen de la tabla", level=2), + model.Markdown(text=text)] + + +def _row_meaning_blocks(llm: dict) -> list: + """Heading + prose for what one row represents, or [] if absent.""" + text = _para(llm.get("row_meaning")) + if not text: + return [] + return [model.Heading(text="Significado de una fila", level=2), + model.Markdown(text=text)] + + +def _dictionary_block(llm: dict): + """DataTable for the data dictionary, or None if absent/empty. + + Columns: Columna / Descripción / Significado de negocio / Unidad. The + paginator splits this by rows repeating the header and wraps long cells, so a + long dictionary (many columns) never gets cut. + """ + entries = llm.get("dictionary") + if not isinstance(entries, (list, tuple)) or not entries: + return None + header = ["Columna", "Descripción", "Significado de negocio", "Unidad"] + rows = [] + for e in entries: + if not isinstance(e, dict): + # Be tolerant: a bare string still shows up as a description row. + rows.append(["—", _clean_text(e), "", ""]) + continue + rows.append([ + _clean_text(e.get("column")) or "—", + _clean_text(e.get("description")), + _clean_text(e.get("business_meaning")), + _clean_text(e.get("unit")), + ]) + if not rows: + return None + return model.DataTable(header=header, rows=rows, title="Diccionario de datos") + + +def _analyses_blocks(llm: dict) -> list: + """Heading + bullet list of suggested analyses, or [] if absent.""" + bullets = _bullets(llm.get("analyses")) + if not bullets: + return [] + return [model.Heading(text="Análisis sugeridos", level=2), + model.Markdown(text=bullets)] + + +def _cleaning_blocks(llm: dict) -> list: + """Heading + bullet list of cleaning suggestions, or [] if absent.""" + bullets = _bullets(llm.get("cleaning")) + if not bullets: + return [] + return [model.Heading(text="Limpieza sugerida", level=2), + model.Markdown(text=bullets)] + + +def _pii_block(llm: dict): + """DataTable for PII/GDPR findings, or None if absent/empty.""" + entries = llm.get("pii") + if not isinstance(entries, (list, tuple)) or not entries: + return None + header = ["Columna", "Tipo", "Severidad"] + rows = [] + for e in entries: + if not isinstance(e, dict): + continue + rows.append([ + _clean_text(e.get("column")) or "—", + _clean_text(e.get("kind")), + _clean_text(e.get("severity")), + ]) + if not rows: + return None + return model.DataTable( + header=header, rows=rows, title="Datos personales (PII / RGPD)", + note="detección automática orientativa — revisar antes de tratar los datos") + + +def build_analisis_llm(profile: dict, ctx: dict): + """Build the LLM analysis Chapter, or None if there is no LLM block. + + Consumes ``profile['llm']`` (the block produced by ``eda_llm_insights``, + group ``eda``); falls back to ``ctx['llm']``. Returns ``None`` when no LLM + block is present or it carries no usable content, so the chapter is omitted + rather than rendering an empty section. + """ + profile = profile or {} + ctx = ctx or {} + + llm = profile.get(LLM_KEY) + if not isinstance(llm, dict): + llm = ctx.get(LLM_KEY) + if not isinstance(llm, dict) or not llm: + return None + + blocks: list = [] + blocks += _summary_blocks(llm) + blocks += _row_meaning_blocks(llm) + + dict_block = _dictionary_block(llm) + if dict_block is not None: + blocks.append(model.Heading(text="Diccionario de datos", level=2)) + blocks.append(dict_block) + + blocks += _analyses_blocks(llm) + blocks += _cleaning_blocks(llm) + + pii_block = _pii_block(llm) + if pii_block is not None: + blocks.append(model.Heading(text="Datos personales (PII / RGPD)", level=2)) + blocks.append(pii_block) + + if not blocks: + return None # LLM block present but every field empty → omit chapter. + + return model.Chapter(id=CHAPTER_ID, title=CHAPTER_TITLE, + version=CHAPTER_VERSION, blocks=blocks) diff --git a/python/functions/datascience/automatic_eda/chapters_registry.py b/python/functions/datascience/automatic_eda/chapters_registry.py index 1d6743f4..6dd73237 100644 --- a/python/functions/datascience/automatic_eda/chapters_registry.py +++ b/python/functions/datascience/automatic_eda/chapters_registry.py @@ -28,12 +28,12 @@ from . import model CHAPTER_ORDER = [ "portada", # cover "overview", # df.head + columns/types/nulls/examples + describe + "analisis_llm", # LLM interpretation — sits next to overview (user request) "num_distr", # numeric distributions "cat_distr", # categorical distributions "calidad", # data quality "correlacion", # correlations / associations "modelos", # cheap models (PCA/KMeans/outliers) - "analisis_llm", # LLM interpretation "timeseries", # time-series analysis "geospatial", # geospatial "agregacion", # aggregations / pivots From af1dd9bcc2649a9f4f7b7f5606d3dbce19696c11 Mon Sep 17 00:00:00 2001 From: Egutierrez Date: Tue, 30 Jun 2026 15:01:26 +0200 Subject: [PATCH 11/53] =?UTF-8?q?test(eda):=20tests=20del=20cap=C3=ADtulo?= =?UTF-8?q?=20AN=C3=81LISIS=20LLM=20(golden=20+=20edges=20+=20anti-cortes)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Suite self-contained (perfil sintético + un golden, sin DuckDB): - golden: build_analisis_llm devuelve el Chapter y el documento entero renderiza a PDF y PPTX con resumen, análisis sugeridos, limpieza y una columna del diccionario presentes. - orden: el capítulo queda inmediatamente después de `overview`. - edges: profile sin bloque `llm` (o None/{}/malformado/llm vacío) -> None sin lanzar; fallback a ctx['llm']. - anti-cortes: diccionario de 40 filas + sugerencia de limpieza de ~150 chars se reparten en varias páginas/slides sin perder ninguna fila ni palabra. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../chapters/analisis_llm_test.py | 190 ++++++++++++++++++ 1 file changed, 190 insertions(+) create mode 100644 python/functions/datascience/automatic_eda/chapters/analisis_llm_test.py diff --git a/python/functions/datascience/automatic_eda/chapters/analisis_llm_test.py b/python/functions/datascience/automatic_eda/chapters/analisis_llm_test.py new file mode 100644 index 00000000..2b32470a --- /dev/null +++ b/python/functions/datascience/automatic_eda/chapters/analisis_llm_test.py @@ -0,0 +1,190 @@ +"""Tests for the ANÁLISIS LLM chapter — DoD: golden + edges + anti-cut. + +Self-contained: builds a synthetic TableProfile carrying an ``llm`` block (the +shape ``eda_llm_insights`` produces) so the suite is fast and deterministic — no +DuckDB and no LLM call. Verifies: + +* golden — ``build_analisis_llm`` yields the chapter and the full document + renders to PDF *and* PPTX with the summary, a suggested analysis, a cleaning + suggestion and a dictionary column all present; +* order — the chapter sits immediately after ``overview`` (user requirement); +* edges — a profile with no ``llm`` block (or None/empty/malformed) returns + ``None`` and never raises; +* anti-cut — a long dictionary (40 rows) and a 150-char cleaning suggestion are + rendered to PDF and PPTX without losing a single row or word. +""" + +import os +import re +import tempfile + +from pypdf import PdfReader +from pptx import Presentation + +from datascience.automatic_eda.chapters.analisis_llm import ( + build_analisis_llm, CHAPTER_VERSION) +from datascience.automatic_eda.chapters_registry import build_document +from datascience.automatic_eda.model import Chapter, DataTable +from datascience.render_automatic_eda_pdf import render_automatic_eda_pdf +from datascience.render_automatic_eda_pptx import render_automatic_eda_pptx + + +def _profile() -> dict: + return { + "table": "ventas", + "source": "/data/ventas.csv", + "profiled_at": "2026-06-30T10:00:00+00:00", + "n_rows": 1000, + "n_cols": 2, + "quality_score": 92.5, + "columns": [ + {"name": "precio", "inferred_type": "numeric", "null_pct": 0.0, + "null_count": 0, + "numeric": {"mean": 42.5, "median": 40.0, "min": 1.0, + "max": 100.0, "std": 12.3}}, + {"name": "categoria", "inferred_type": "categorical", + "null_pct": 0.0, "null_count": 0, + "categorical": {"top": [{"value": "neumaticos", "count": 500}]}}, + ], + "llm": { + "summary": "Tabla de ventas por producto. Token SUMMARYTOKEN.", + "row_meaning": "Cada fila es una venta. Token ROWTOKEN.", + "dictionary": [ + {"column": "precio", "description": "Precio unitario DESCTOKEN", + "business_meaning": "Ingreso por unidad", "unit": "EUR"}, + {"column": "categoria", "description": "Familia de producto", + "business_meaning": "Segmento comercial", "unit": ""}, + ], + "pii": [{"column": "categoria", "kind": "ninguno", "severity": "low"}], + "cleaning": ["Quitar nulos de precio CLEANTOKEN", + "Normalizar mayusculas en categoria"], + "analyses": ["Estudiar relacion precio-categoria ANALYSISTOKEN", + "Detectar outliers de precio"], + }, + } + + +def _pdf_text(path: str) -> str: + txt = "".join((pg.extract_text() or "") for pg in PdfReader(path).pages) + return re.sub(r"\s+", " ", txt) + + +def _pptx_text(path: str) -> str: + prs = Presentation(path) + parts = [] + for sl in prs.slides: + for sh in sl.shapes: + if sh.has_text_frame: + parts.append(sh.text_frame.text) + if sh.has_table: + tb = sh.table + for r in range(len(tb.rows)): + for c in range(len(tb.columns)): + parts.append(tb.cell(r, c).text) + return re.sub(r"\s+", " ", " ".join(parts)) + + +def test_golden_build_y_render_pdf_pptx(): + prof = _profile() + ch = build_analisis_llm(prof, {}) + assert ch is not None + assert ch.id == "analisis_llm" + assert ch.version == CHAPTER_VERSION + assert ch.blocks # non-empty. + + with tempfile.TemporaryDirectory() as d: + out_pdf = os.path.join(d, "eda.pdf") + res = render_automatic_eda_pdf(prof, out_pdf, {"title": "EDA — ventas"}) + assert res["path"] == out_pdf and os.path.exists(out_pdf) + ids = [c["id"] for c in res["chapters"]] + assert "analisis_llm" in ids + txt = _pdf_text(out_pdf) + # The user's required content: summary, suggested analyses, cleaning. + assert "SUMMARYTOKEN" in txt + assert "ANALYSISTOKEN" in txt + assert "CLEANTOKEN" in txt + assert "DESCTOKEN" in txt # data dictionary cell. + + out_pptx = os.path.join(d, "eda.pptx") + res2 = render_automatic_eda_pptx(prof, out_pptx, {"title": "EDA — ventas"}) + assert res2["path"] == out_pptx and os.path.exists(out_pptx) + ids2 = [c["id"] for c in res2["chapters"]] + assert "analisis_llm" in ids2 + ptx = _pptx_text(out_pptx) + assert "SUMMARYTOKEN" in ptx + assert "ANALYSISTOKEN" in ptx + assert "CLEANTOKEN" in ptx + assert "DESCTOKEN" in ptx + + +def test_orden_capitulo_junto_a_overview(): + chapters = build_document(_profile(), {}) + ids = [c.id for c in chapters] + assert "overview" in ids and "analisis_llm" in ids + # User requirement: the LLM chapter sits right after overview. + assert ids.index("analisis_llm") == ids.index("overview") + 1 + + +def test_edge_sin_llm_devuelve_none(): + # No llm block at all. + prof = {k: v for k, v in _profile().items() if k != "llm"} + assert build_analisis_llm(prof, {}) is None + # None / empty / malformed never raise and yield None. + assert build_analisis_llm(None, None) is None + assert build_analisis_llm({}, {}) is None + assert build_analisis_llm({"llm": {}}, {}) is None + assert build_analisis_llm({"llm": "not-a-dict"}, {}) is None + # All-empty fields → omitted (no blocks). + empty = {"llm": {"summary": "", "dictionary": [], "cleaning": [], + "analyses": [], "pii": [], "row_meaning": ""}} + assert build_analisis_llm(empty, {}) is None + + +def test_edge_llm_via_ctx_fallback(): + # The block may arrive in ctx instead of the profile. + prof = {k: v for k, v in _profile().items() if k != "llm"} + ctx = {"llm": {"summary": "Resumen via ctx CTXTOKEN."}} + ch = build_analisis_llm(prof, ctx) + assert ch is not None and ch.id == "analisis_llm" + + +def test_anti_cortes_diccionario_largo_y_limpieza_larga(): + long_clean = ("Lorem ipsum dolor sit amet consectetur adipiscing elit sed do " + "eiusmod tempor incididunt ut labore et dolore magna aliqua " + "reprehenderit voluptate velit esse cillum dolore") + dictionary = [ + {"column": f"col_{i}", + "description": f"Descripcion larga numero {i} con bastante texto para " + f"forzar el wrap dentro de la celda fila{i}", + "business_meaning": f"Significado de negocio {i}", "unit": "u"} + for i in range(40) + ] + prof = { + "table": "t", "n_rows": 1, "n_cols": 1, "columns": [], + "llm": {"summary": "S", "dictionary": dictionary, + "cleaning": [long_clean], "analyses": ["A"]}, + } + ch = build_analisis_llm(prof, {}) + assert ch is not None + # Structure: the dictionary DataTable keeps ALL 40 rows — none dropped on + # construction (the renderers then split it by rows, repeating the header). + dts = [b for b in ch.blocks if isinstance(b, DataTable)] + assert any(len(dt.rows) == 40 for dt in dts) + + with tempfile.TemporaryDirectory() as d: + out_pdf = os.path.join(d, "x.pdf") + render_automatic_eda_pdf([ch], out_pdf, {"write_manifest": False}) + # 40 wide rows + a long cleaning line cannot fit one page → it spills, + # which is exactly the no-cut behaviour (paginate, never truncate). + assert len(PdfReader(out_pdf).pages) > 1 + txt = _pdf_text(out_pdf) + # The long cleaning suggestion is wrapped word-by-word, not truncated. + for word in ("Lorem", "incididunt", "reprehenderit", "voluptate", "cillum"): + assert word in txt + + out_pptx = os.path.join(d, "x.pptx") + res2 = render_automatic_eda_pptx([ch], out_pptx, {"write_manifest": False}) + assert res2["n_slides"] > 1 # table + long text spill across slides. + ptx = _pptx_text(out_pptx) + for word in ("Lorem", "reprehenderit", "voluptate"): + assert word in ptx From 649de07d6bc3842d0218e9b8bb698a75ef094e5a Mon Sep 17 00:00:00 2001 From: Egutierrez Date: Tue, 30 Jun 2026 15:04:10 +0200 Subject: [PATCH 12/53] =?UTF-8?q?feat(eda):=20cap=C3=ADtulo=20AutomaticEDA?= =?UTF-8?q?=20CAT=20DISTR=20+=20funciones=20cardinalidad/pie?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Capítulo cat_distr del motor AutomaticEDA: distribuciones categóricas con explicación de entropía de Shannon, métricas de cardinalidad por columna (valores distintos, % distintos, total de filas, valores únicos, entropía y su máximo log2(k) + normalizada), tabla top-k y un donut de las categorías más comunes (top-k + «Otros»). Marca columnas id-like y dominadas. Delegadas a fn-constructor (grupo eda): - categorical_cardinality_block: deriva métricas de cardinalidad/entropía. - categorical_top_pie_figure: figura donut top-k + «Otros», leyenda lateral. Defensivo (dict-no-throw): None si no hay columnas categóricas; normaliza mode_pct a escala 0-100 (summarize_categorical lo emite como fracción). Tablas vía DataTable y figura perezosa: el paginador del núcleo garantiza no-corte en PDF y PPTX. Tests: golden + edge (sin categóricas) + anti-corte (label largo / muchas columnas) en ambos renderers. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../automatic_eda/chapters/cat_distr.py | 402 ++++++++++++++++++ .../automatic_eda/chapters/cat_distr_test.py | 186 ++++++++ .../categorical_cardinality_block.md | 115 +++++ .../categorical_cardinality_block.py | 132 ++++++ .../categorical_cardinality_block_test.py | 216 ++++++++++ .../datascience/categorical_top_pie_figure.md | 108 +++++ .../datascience/categorical_top_pie_figure.py | 230 ++++++++++ .../categorical_top_pie_figure_test.py | 104 +++++ 8 files changed, 1493 insertions(+) create mode 100644 python/functions/datascience/automatic_eda/chapters/cat_distr.py create mode 100644 python/functions/datascience/automatic_eda/chapters/cat_distr_test.py create mode 100644 python/functions/datascience/categorical_cardinality_block.md create mode 100644 python/functions/datascience/categorical_cardinality_block.py create mode 100644 python/functions/datascience/categorical_cardinality_block_test.py create mode 100644 python/functions/datascience/categorical_top_pie_figure.md create mode 100644 python/functions/datascience/categorical_top_pie_figure.py create mode 100644 python/functions/datascience/categorical_top_pie_figure_test.py diff --git a/python/functions/datascience/automatic_eda/chapters/cat_distr.py b/python/functions/datascience/automatic_eda/chapters/cat_distr.py new file mode 100644 index 00000000..c593a6b7 --- /dev/null +++ b/python/functions/datascience/automatic_eda/chapters/cat_distr.py @@ -0,0 +1,402 @@ +"""Categorical distributions chapter (CAT DISTR). + +Third reference chapter for AutomaticEDA. For every categorical column it shows, +fulfilling the user's request: + +1. A short opening explanation of **Shannon entropy** (what it measures, its 0 + and log2(k) bounds, the normalized 0–1 version) and the dataset row total used + as a comparison baseline. +2. Per column, a cardinality key/value table: distinct values, ``% distinct`` + (distinct / total rows), total dataset rows, singleton values (frequency 1), + entropy with its theoretical maximum and the normalized ratio, mode, imbalance + and string-length stats. +3. A short note flagging problematic cardinality (id-like ≈100% distinct, or a + single dominating category). +4. A ``top-k`` table (value / count / %). +5. A **donut pie chart** of the most common categories (top-k + an "Otros" + bucket), drawn lazily so the renderers scale it to fit entirely. + +Data comes from the ``eda`` group: each ``columns[i]['categorical']`` is the +output of ``summarize_categorical`` (``top[{value,count,pct}]``, ``mode``, +``n_distinct``, ``entropy``, ``imbalance``, ``len_min/mean/max``). The derived +cardinality metrics and the pie figure are delegated to two registry functions +(``categorical_cardinality_block`` and ``categorical_top_pie_figure``); both are +imported lazily and degrade to a minimal inline fallback so this chapter never +raises even if they are unavailable. + +Contract: build_(profile, ctx) -> Chapter | None ; CHAPTER_VERSION = "x.y.z". +""" + +from __future__ import annotations + +import math + +from .. import model + +CHAPTER_VERSION = "1.0.0" +CHAPTER_ID = "cat_distr" +CHAPTER_TITLE = "Distribuciones categóricas" + +# Cap the number of categorical columns rendered to keep the document bounded; +# the rest are summarized in a closing note (no silent truncation). +MAX_COLS = 40 +# Rows shown in each top-k table and explicit slices in the pie. +TOP_TABLE_ROWS = 15 +PIE_TOP_K = 6 +# Truncate very long category labels in tables (the renderer also wraps). +LABEL_MAX = 48 + + +def _fmt_int(value) -> str: + if value is None: + return "—" + try: + return f"{int(value):,}".replace(",", ".") + except (TypeError, ValueError): + return str(value) + + +def _fmt_num(value, decimals: int = 3) -> str: + if value is None: + return "—" + if isinstance(value, bool): + return str(value) + if isinstance(value, int): + return f"{value:,}".replace(",", ".") + if isinstance(value, float): + if value != value: # NaN + return "NaN" + if value in (float("inf"), float("-inf")): + return str(value) + text = f"{value:.{decimals}f}".rstrip("0").rstrip(".") + return text if text else "0" + return str(value) + + +def _fmt_pct_value(value, decimals: int = 1) -> str: + """Format an already-in-percent value (0–100). None -> placeholder.""" + if value is None: + return "—" + try: + return f"{float(value):.{decimals}f}%" + except (TypeError, ValueError): + return str(value) + + +def _pct_from_maybe_fraction(value, decimals: int = 1) -> str: + """Format a percentage that may arrive as a 0–1 fraction or a 0–100 number.""" + if value is None: + return "—" + try: + v = float(value) + except (TypeError, ValueError): + return str(value) + if v <= 1.0: + v *= 100.0 + return f"{v:.{decimals}f}%" + + +def _truncate(text: str, limit: int = LABEL_MAX) -> str: + s = model._safe_str(text) + if len(s) <= limit: + return s + return s[: max(1, limit - 1)].rstrip() + "…" + + +def _is_categorical(col: dict) -> bool: + """A column is treated as categorical when it carries a non-empty top list + and is not a pure numeric column (numeric columns may still expose a top).""" + if not isinstance(col, dict): + return False + cat = col.get("categorical") + if not (isinstance(cat, dict) and cat.get("top")): + return False + if col.get("inferred_type") == "numeric": + return False + return True + + +def _cardinality(cat: dict, n_rows) -> dict: + """Derive cardinality metrics for a column, via the registry function when + available, otherwise a minimal inline fallback. Never raises.""" + try: + from datascience.categorical_cardinality_block import ( + categorical_cardinality_block, + ) + + out = categorical_cardinality_block(cat=cat, n_rows=n_rows) + if isinstance(out, dict): + return out + except Exception: # noqa: BLE001 — fall back to the inline derivation. + pass + return _fallback_cardinality(cat, n_rows) + + +def _fallback_cardinality(cat: dict, n_rows) -> dict: + cat = cat or {} + top = cat.get("top") or [] + n_distinct = cat.get("n_distinct") + entropy = cat.get("entropy") + try: + nr = int(n_rows) if n_rows is not None else None + except (TypeError, ValueError): + nr = None + pct_distinct = None + if isinstance(n_distinct, (int, float)) and nr: + pct_distinct = float(n_distinct) / nr * 100.0 + entropy_max = None + if isinstance(n_distinct, (int, float)): + entropy_max = math.log2(n_distinct) if n_distinct > 1 else 0.0 + entropy_norm = None + if isinstance(entropy, (int, float)) and entropy_max: + entropy_norm = max(0.0, min(1.0, float(entropy) / entropy_max)) + mode_pct = cat.get("mode_pct") + if mode_pct is None and top and isinstance(top[0], dict): + mode_pct = top[0].get("pct") + # Normalize to a 0–100 scale: summarize_categorical emits a 0–1 fraction. + if isinstance(mode_pct, (int, float)) and not isinstance(mode_pct, bool): + mode_pct = float(mode_pct) * 100.0 if mode_pct <= 1.0 else float(mode_pct) + else: + mode_pct = None + n_singletons = None + if top: + n_singletons = sum( + 1 for t in top if isinstance(t, dict) and t.get("count") == 1) + return { + "n_distinct": n_distinct, + "n_rows": nr, + "pct_distinct": pct_distinct, + "entropy": entropy, + "entropy_max": entropy_max, + "entropy_norm": entropy_norm, + "mode": cat.get("mode"), + "mode_pct": mode_pct, + "imbalance": cat.get("imbalance"), + "n_singletons": n_singletons, + "n_singletons_partial": ( + isinstance(n_distinct, (int, float)) and n_distinct > len(top)), + "len_min": cat.get("len_min"), + "len_mean": cat.get("len_mean"), + "len_max": cat.get("len_max"), + "id_like": pct_distinct is not None and pct_distinct >= 99.0, + "dominated": mode_pct is not None and mode_pct >= 90.0, + } + + +def _pie_make(top, n_distinct, title, n_rows): + """Return a zero-arg callable that builds the donut figure lazily.""" + + def make(): + try: + from datascience.categorical_top_pie_figure import ( + categorical_top_pie_figure, + ) + + return categorical_top_pie_figure( + top=top, n_distinct=n_distinct or 0, title=title, + top_k=PIE_TOP_K, n_rows=n_rows) + except Exception: # noqa: BLE001 — minimal local fallback figure. + return _fallback_pie(top, title) + + return make + + +def _fallback_pie(top, title): + """Minimal donut figure used only if the registry function is unavailable.""" + import matplotlib + + matplotlib.use("Agg") + from matplotlib.figure import Figure + + fig = Figure(figsize=(5.0, 3.2)) + ax = fig.add_subplot(111) + items = [t for t in (top or []) + if isinstance(t, dict) and isinstance(t.get("count"), (int, float))] + items = sorted(items, key=lambda t: t.get("count") or 0, reverse=True) + head = items[:PIE_TOP_K] + rest = items[PIE_TOP_K:] + labels = [_truncate(t.get("value"), 20) for t in head] + sizes = [float(t.get("count") or 0) for t in head] + if rest: + labels.append(f"Otros ({len(rest)})") + sizes.append(sum(float(t.get("count") or 0) for t in rest)) + if not sizes or sum(sizes) <= 0: + ax.text(0.5, 0.5, "sin datos categóricos", ha="center", va="center") + ax.axis("off") + return fig + ax.pie(sizes, labels=None, wedgeprops={"width": 0.42}, + autopct=lambda p: f"{p:.0f}%" if p >= 4 else "") + ax.legend(labels, loc="center left", bbox_to_anchor=(1.0, 0.5), + fontsize=7, frameon=False) + ax.set_title(_truncate(title, 40)) + fig.tight_layout() + return fig + + +def _normalize_card(card: dict) -> dict: + """Make the cardinality dict robust regardless of the upstream scale. + + ``summarize_categorical`` emits ``mode_pct`` as a 0–1 fraction; bring it to a + 0–100 scale and recompute the ``dominated`` flag here so the chapter is + correct whether it consumed the registry function or the inline fallback. + """ + card = dict(card or {}) + mp = card.get("mode_pct") + if isinstance(mp, (int, float)) and not isinstance(mp, bool): + mp = float(mp) * 100.0 if mp <= 1.0 else float(mp) + else: + mp = None + card["mode_pct"] = mp + card["dominated"] = mp is not None and mp >= 90.0 + pd = card.get("pct_distinct") + card["id_like"] = isinstance(pd, (int, float)) and pd >= 99.0 + return card + + +def _cardinality_block(card: dict): + """KVTable with the cardinality / entropy metrics for one column.""" + n_singletons = card.get("n_singletons") + if n_singletons is not None and card.get("n_singletons_partial"): + singletons = f"≥{_fmt_int(n_singletons)} (en top mostrado)" + elif n_singletons is not None: + singletons = _fmt_int(n_singletons) + else: + singletons = "—" + + entropy_ref = _fmt_num(card.get("entropy")) + emax = card.get("entropy_max") + if emax is not None: + entropy_ref = f"{entropy_ref} (máx {_fmt_num(emax)})" + + mode = card.get("mode") + mode_pct = card.get("mode_pct") + mode_str = "—" if mode is None else model._safe_str(mode) + if mode is not None and mode_pct is not None: + mode_str = f"{mode_str} ({_fmt_pct_value(mode_pct)})" + + rows = [ + ("Valores distintos", _fmt_int(card.get("n_distinct"))), + ("% distintos", _fmt_pct_value(card.get("pct_distinct"))), + ("Total filas (dataset)", _fmt_int(card.get("n_rows"))), + ("Valores únicos (frecuencia 1)", singletons), + ("Entropía (bits)", entropy_ref), + ("Entropía normalizada (0–1)", _fmt_num(card.get("entropy_norm"))), + ("Moda", mode_str), + ] + imbalance = card.get("imbalance") + if imbalance is not None: + rows.append(("Desbalance", _fmt_num(imbalance))) + lm = card.get("len_min") + lmean = card.get("len_mean") + lmax = card.get("len_max") + if any(v is not None for v in (lm, lmean, lmax)): + rows.append(( + "Longitud (mín/media/máx)", + f"{_fmt_num(lm)} / {_fmt_num(lmean)} / {_fmt_num(lmax)}")) + return model.KVTable(rows=rows, title="Cardinalidad") + + +def _flag_note(card: dict): + """Return a Note flagging problematic cardinality, or None.""" + if card.get("id_like"): + return model.Note( + "Casi todos los valores son distintos (≈100% distintos): la columna " + "se comporta como un identificador y aporta poco para agrupar o " + "comparar categorías.") + if card.get("dominated"): + mp = card.get("mode_pct") + mp_str = _fmt_pct_value(mp) if mp is not None else "muy alta" + return model.Note( + f"Una sola categoría domina la columna (moda {mp_str}): la " + "distribución está muy desbalanceada.") + return None + + +def _topk_table(cat: dict): + """DataTable value / count / % for the top categories.""" + top = cat.get("top") or [] + n_distinct = cat.get("n_distinct") + header = ["Valor", "Conteo", "%"] + rows = [] + for t in top[:TOP_TABLE_ROWS]: + if not isinstance(t, dict): + continue + rows.append([ + model._safe_str(t.get("value")), + _fmt_int(t.get("count")), + _pct_from_maybe_fraction(t.get("pct")), + ]) + if not rows: + return None + shown = len(rows) + if isinstance(n_distinct, (int, float)) and n_distinct > shown: + note = f"top {shown} de {_fmt_int(n_distinct)} categorías distintas" + else: + note = f"{shown} categorías" + return model.DataTable(header=header, rows=rows, title="Top categorías", + note=note) + + +def _intro_blocks(n_rows): + total = _fmt_int(n_rows) + text = ( + "La **entropía de Shannon** mide cómo de repartidos están los valores de " + "una columna categórica, en bits. Vale 0 cuando una sola categoría " + "concentra todas las filas (máxima previsibilidad) y alcanza su máximo, " + "log2(k) para k categorías distintas, cuando todas aparecen por igual " + "(máxima diversidad). La **entropía normalizada** (entropía dividida por " + "su máximo) la lleva al rango 0–1 para comparar columnas con distinto " + "número de categorías. Para cada columna se muestran los valores " + "distintos, el porcentaje que representan sobre el total de filas, los " + "valores únicos (que aparecen una sola vez), la tabla de las categorías " + "más frecuentes y un gráfico de tarta (donut) de las más comunes." + ) + if n_rows is not None: + text += f" El dataset tiene {total} filas en total como referencia." + return [ + model.Heading(text="Entropía y cardinalidad", level=2), + model.Markdown(text=text), + ] + + +def build_cat_distr(profile: dict, ctx: dict): + """Build the categorical-distributions Chapter, or None if the dataset has + no categorical columns.""" + profile = profile or {} + ctx = ctx or {} + cols = profile.get("columns") or [] + cat_cols = [c for c in cols if _is_categorical(c)] + if not cat_cols: + return None + + n_rows = profile.get("n_rows") + blocks = list(_intro_blocks(n_rows)) + + rendered = cat_cols[:MAX_COLS] + for col in rendered: + name = col.get("name") or "(columna)" + cat = col.get("categorical") or {} + card = _normalize_card(_cardinality(cat, n_rows)) + + blocks.append(model.Heading(text=str(name), level=2)) + blocks.append(_cardinality_block(card)) + note = _flag_note(card) + if note is not None: + blocks.append(note) + topk = _topk_table(cat) + if topk is not None: + blocks.append(topk) + blocks.append(model.Figure( + make=_pie_make(cat.get("top") or [], card.get("n_distinct"), + str(name), n_rows), + caption=(f"Categorías más comunes de «{_truncate(name, 32)}» " + "(donut: top-k + «Otros»)"))) + + if len(cat_cols) > len(rendered): + omitted = len(cat_cols) - len(rendered) + blocks.append(model.Note( + f"Se muestran las primeras {len(rendered)} columnas categóricas; " + f"quedan {omitted} sin mostrar para mantener acotado el informe.")) + + return model.Chapter(id=CHAPTER_ID, title=CHAPTER_TITLE, + version=CHAPTER_VERSION, blocks=blocks) diff --git a/python/functions/datascience/automatic_eda/chapters/cat_distr_test.py b/python/functions/datascience/automatic_eda/chapters/cat_distr_test.py new file mode 100644 index 00000000..a061c67d --- /dev/null +++ b/python/functions/datascience/automatic_eda/chapters/cat_distr_test.py @@ -0,0 +1,186 @@ +"""Tests for the CAT DISTR chapter — DoD: golden + edges + anti-cut. + +Self-contained: builds synthetic TableProfiles (no DuckDB) so the suite is fast +and deterministic. Verifies that ``build_cat_distr`` emits the blocks the user +asked for (entropy intro, distinct/total/%-distinct/unique metrics, top-k table +and a donut figure), that the chapter renders inside the full document to both +PDF and PPTX showing that content, that a profile with no categorical columns +yields ``None`` without raising, and that long labels / many columns are never +cut in either output. +""" + +import os +import re +import tempfile + +from pypdf import PdfReader +from pptx import Presentation + +from datascience.automatic_eda.model import ( + DataTable, Figure, Heading, KVTable, Note, +) +from datascience.automatic_eda.chapters.cat_distr import ( + CHAPTER_ID, CHAPTER_VERSION, build_cat_distr, +) +from datascience.render_automatic_eda_pdf import render_automatic_eda_pdf +from datascience.render_automatic_eda_pptx import render_automatic_eda_pptx + + +def _profile() -> dict: + return { + "table": "productos", + "source": "/data/productos.csv", + "profiled_at": "2026-06-30T10:00:00+00:00", + "n_rows": 1000, + "n_cols": 3, + "quality_score": 90.0, + "columns": [ + {"name": "precio", "inferred_type": "numeric", "null_pct": 0.0, + "null_count": 0, + "numeric": {"mean": 42.5, "median": 40.0, "min": 1.0, + "max": 100.0, "std": 12.3}}, + {"name": "categoria", "inferred_type": "categorical", + "null_pct": 0.0, "null_count": 0, "distinct_count": 8, + "categorical": { + "top": [ + {"value": "neumaticos", "count": 500, "pct": 0.5}, + {"value": "aceite", "count": 300, "pct": 0.3}, + {"value": "filtros", "count": 120, "pct": 0.12}, + {"value": "frenos", "count": 80, "pct": 0.08}, + ], + "mode": "neumaticos", "n_distinct": 8, "entropy": 1.6, + "imbalance": 6.25, "len_min": 6, "len_mean": 7.5, + "len_max": 10}}, + {"name": "uuid", "inferred_type": "categorical", + "null_pct": 0.0, "null_count": 0, "distinct_count": 1000, + "categorical": { + "top": [{"value": f"id-{i}", "count": 1} for i in range(5)], + "mode": "id-0", "n_distinct": 1000, "entropy": 9.97, + "imbalance": 1.0}}, + ], + } + + +def _pdf_text(path: str) -> str: + txt = "".join((pg.extract_text() or "") for pg in PdfReader(path).pages) + return re.sub(r"\s+", " ", txt) + + +def _pptx_text(path: str) -> str: + prs = Presentation(path) + parts = [] + for sl in prs.slides: + for sh in sl.shapes: + if sh.has_text_frame: + parts.append(sh.text_frame.text) + if sh.has_table: + tb = sh.table + for r in range(len(tb.rows)): + for c in range(len(tb.columns)): + parts.append(tb.cell(r, c).text) + return re.sub(r"\s+", " ", " ".join(parts)) + + +def _kinds(chapter): + return [b.kind for b in chapter.blocks] + + +def test_golden_build_cat_distr_emite_bloques_pedidos(): + ch = build_cat_distr(_profile(), {}) + assert ch is not None + assert ch.id == CHAPTER_ID + assert ch.version == CHAPTER_VERSION + kinds = _kinds(ch) + # Entropy intro present. + headings = [b.text for b in ch.blocks if isinstance(b, Heading)] + assert any("Entrop" in h for h in headings) + md = next(b for b in ch.blocks if b.kind == "markdown") + assert "entropía" in md.text.lower() and "log2" in md.text + # Cardinality metrics: distinct, total rows, %-distinct, unique values. + kv = next(b for b in ch.blocks if isinstance(b, KVTable)) + labels = [r[0] for r in kv.rows] + assert "Valores distintos" in labels + assert "% distintos" in labels + assert "Total filas (dataset)" in labels + assert "Valores únicos (frecuencia 1)" in labels + assert any("Entropía" in lbl for lbl in labels) + # Top-k table + pie figure. + dt = next(b for b in ch.blocks if isinstance(b, DataTable)) + assert dt.header == ["Valor", "Conteo", "%"] + assert any("neumaticos" in str(cell) for row in dt.rows for cell in row) + assert any(isinstance(b, Figure) for b in ch.blocks) + # id-like column flagged with a Note. + assert any(isinstance(b, Note) and "identificador" in b.text + for b in ch.blocks) + + +def test_golden_render_pdf_muestra_categoricas(): + with tempfile.TemporaryDirectory() as d: + out = os.path.join(d, "eda.pdf") + res = render_automatic_eda_pdf(_profile(), out, {"title": "EDA"}) + assert res["path"] == out and os.path.exists(out) + assert CHAPTER_ID in [c["id"] for c in res["chapters"]] + txt = _pdf_text(out) + assert "Entrop" in txt + assert "distintos" in txt + assert "categoria" in txt and "neumaticos" in txt + assert "donut" in txt # figure caption rendered as text. + assert "identificador" in txt # id-like note rendered. + + +def test_golden_render_pptx_muestra_categoricas(): + with tempfile.TemporaryDirectory() as d: + out = os.path.join(d, "eda.pptx") + res = render_automatic_eda_pptx(_profile(), out, {"title": "EDA"}) + assert res["path"] == out and os.path.exists(out) + assert CHAPTER_ID in [c["id"] for c in res["chapters"]] + txt = _pptx_text(out) + assert "Entrop" in txt + assert "categoria" in txt and "neumaticos" in txt + assert "distintos" in txt + + +def test_edge_sin_categoricas_devuelve_none(): + only_numeric = { + "n_rows": 10, "columns": [ + {"name": "x", "inferred_type": "numeric", + "numeric": {"mean": 1.0}}]} + assert build_cat_distr(only_numeric, {}) is None + # None / empty / no-columns never raise and yield None. + assert build_cat_distr(None, None) is None + assert build_cat_distr({}, {}) is None + assert build_cat_distr({"columns": []}, {}) is None + + +def test_anti_corte_label_largo_y_muchas_columnas(): + long_label = ("Lorem ipsum dolor sit amet consectetur adipiscing elit sed " + "do eiusmod tempor incididunt ut labore reprehenderit voluptate") + cols = [] + for i in range(30): + cols.append({ + "name": f"cat_{i}", "inferred_type": "categorical", + "distinct_count": 3, + "categorical": { + "top": [{"value": long_label, "count": 60}, + {"value": "b", "count": 30}, + {"value": "c", "count": 10}], + "mode": long_label, "n_distinct": 3, "entropy": 1.2}}) + profile = {"table": "t", "source": "t.csv", "n_rows": 100, + "n_cols": len(cols), "columns": cols} + + ch = build_cat_distr(profile, {}) + assert ch is not None + with tempfile.TemporaryDirectory() as d: + pdf = os.path.join(d, "anti.pdf") + res = render_automatic_eda_pdf(profile, pdf, {"write_manifest": False}) + assert res["path"] == pdf + assert res["n_pages"] > 1 # many columns spilled across pages, OK. + txt = _pdf_text(pdf) + # Long label wrapped (not truncated): every word survives. + for word in ("Lorem", "incididunt", "reprehenderit", "voluptate"): + assert word in txt + # PPTX path must not raise either. + pptx = os.path.join(d, "anti.pptx") + res2 = render_automatic_eda_pptx(profile, pptx, + {"write_manifest": False}) + assert res2["path"] == pptx and os.path.exists(pptx) diff --git a/python/functions/datascience/categorical_cardinality_block.md b/python/functions/datascience/categorical_cardinality_block.md new file mode 100644 index 00000000..b0584d0b --- /dev/null +++ b/python/functions/datascience/categorical_cardinality_block.md @@ -0,0 +1,115 @@ +--- +id: categorical_cardinality_block_py_datascience +name: categorical_cardinality_block +kind: function +lang: py +domain: datascience +version: "1.0.0" +purity: pure +signature: "def categorical_cardinality_block(cat: dict, n_rows: int) -> dict" +description: "Deriva métricas de cardinalidad listas para renderizar a partir de la salida de summarize_categorical para UNA columna categórica más el número total de filas. Calcula pct_distinct, entropy_max=log2(n_distinct), entropy_norm (recortada a [0,1]), n_singletons (sobre el top visible) y los flags id_like / dominated. NO recalcula la entropía ni reimplementa summarize_categorical: la consume. Estilo dict-no-throw del grupo eda — nunca lanza." +tags: [eda, categorical, cardinality, entropy, profiling, datascience, pure] +uses_functions: [] +uses_types: [] +returns: [] +returns_optional: false +error_type: "" +imports: [math] +example: | + from categorical_cardinality_block import categorical_cardinality_block + cat = {"top": [{"value": "a", "count": 5, "pct": 0.5}], "mode": "a", + "mode_pct": 0.5, "n_distinct": 4, "entropy": 1.685, "imbalance": 5.0, + "len_min": 1, "len_mean": 1.0, "len_max": 1} + block = categorical_cardinality_block(cat, n_rows=10) +tested: true +tests: + - "test_normal_case" + - "test_empty_cat_does_not_raise" + - "test_none_cat_does_not_raise" + - "test_n_rows_zero_no_zero_division" + - "test_id_like_when_distinct_near_rows" + - "test_dominated_when_mode_pct_high" + - "test_mode_pct_fallback_from_top_fraction" + - "test_n_singletons_partial_when_top_truncated" + - "test_single_distinct_value_entropy_norm_none" +test_file_path: "python/functions/datascience/categorical_cardinality_block_test.py" +file_path: "python/functions/datascience/categorical_cardinality_block.py" +params: + - name: cat + desc: "Dict producido por summarize_categorical para UNA columna categórica. Claves leídas (todas opcionales, lectura defensiva): top (list de {value,count,pct}), mode, mode_pct (puede faltar), n_distinct, entropy (Shannon en bits), imbalance, len_min, len_mean, len_max. None o no-dict se tratan como {}." + - name: n_rows + desc: "Número total de filas del dataset. Usado para pct_distinct. Si es 0 o None, pct_distinct sale None (sin ZeroDivisionError)." +output: "Dict con exactamente 16 claves, todas siempre presentes: n_distinct, n_rows, pct_distinct, entropy, entropy_max, entropy_norm, mode, mode_pct, imbalance, n_singletons, n_singletons_partial, len_min, len_mean, len_max, id_like, dominated. Valores None/False cuando no son derivables; la función nunca lanza. pct_distinct en escala 0-100. entropy_max=log2(n_distinct) (0.0 si n_distinct in {0,1}). entropy_norm=entropy/entropy_max recortada a [0,1]. n_singletons = nº de elementos de top con count==1 (None si top vacío). n_singletons_partial=True si n_distinct>len(top). id_like=pct_distinct>=99. dominated=mode_pct>=90." +--- + +## Ejemplo + +```python +from categorical_cardinality_block import categorical_cardinality_block + +# Salida típica de summarize_categorical para una columna, con n_rows del dataset. +cat = { + "top": [ + {"value": "a", "count": 5, "pct": 0.5}, + {"value": "b", "count": 3, "pct": 0.3}, + {"value": "c", "count": 1, "pct": 0.1}, + {"value": "d", "count": 1, "pct": 0.1}, + ], + "mode": "a", + "mode_pct": 0.5, + "n_distinct": 4, + "entropy": 1.685, # Shannon en bits (<= log2(4) = 2.0) + "imbalance": 5.0, + "len_min": 1, "len_mean": 1.0, "len_max": 1, +} + +categorical_cardinality_block(cat, n_rows=10) +# { +# "n_distinct": 4, "n_rows": 10, +# "pct_distinct": 40.0, # 4 / 10 * 100 +# "entropy": 1.685, +# "entropy_max": 2.0, # log2(4) +# "entropy_norm": 0.8425, # 1.685 / 2.0, recortado a [0,1] +# "mode": "a", "mode_pct": 0.5, +# "imbalance": 5.0, +# "n_singletons": 2, # c y d con count == 1 +# "n_singletons_partial": False, # top cubre los 4 distintos +# "len_min": 1, "len_mean": 1.0, "len_max": 1, +# "id_like": False, # pct_distinct 40 < 99 +# "dominated": False, # mode_pct 0.5 < 90 +# } +``` + +## Cuando usarla + +Úsala justo después de `summarize_categorical`, cuando vayas a renderizar el +bloque de cardinalidad de una columna categórica en un EDA: necesitas el ratio +de valores distintos (`pct_distinct`), la entropía normalizada al rango `[0,1]` +para comparar columnas con cardinalidades distintas, el conteo de singletons, y +las banderas heurísticas `id_like` (la columna parece un identificador) y +`dominated` (una sola categoría domina). Pásale el dict crudo de +`summarize_categorical` para esa columna y el `n_rows` total del dataset. No +reimplementa nada: solo deriva métricas de presentación a partir de lo ya +calculado. + +## Gotchas + +- **`mode_pct` se pasa tal cual viene en `cat`.** `summarize_categorical` + produce `mode_pct` como **fracción** (0–1), no como porcentaje. El flag + `dominated` compara `mode_pct >= 90.0`, así que con la salida cruda de + `summarize_categorical` (fracciones) `dominated` no se dispara: aliméntalo con + `mode_pct` en escala 0–100 si quieres usar esa bandera. Solo el camino de + *fallback* (cuando `cat` no trae `mode_pct` y se deriva de `top[0]['pct']`) + normaliza una fracción `<= 1` multiplicándola por 100. +- **`n_singletons` solo cubre el `top` visible.** Si `summarize_categorical` se + llamó con `top_k` pequeño, hay valores fuera del top; en ese caso + `n_singletons_partial` es `True` para avisar de que el conteo es parcial. +- **`pct_distinct` es `None` si `n_rows` es 0 o `None`** (no lanza + `ZeroDivisionError`); por tanto `id_like` queda `False` en ese caso. +- **`entropy_norm` es `None` cuando `entropy_max <= 0`** (columna constante, + `n_distinct in {0,1}`): no hay división por cero y no se inventa un 0/1. +- **No recalcula la entropía.** Si `cat['entropy']` es incoherente con + `n_distinct`, `entropy_norm` se recorta a `[0,1]` pero el valor de entrada no + se corrige. +- **`bool` no cuenta como número.** Un `True`/`False` en una clave numérica de + `cat` se trata como ausente (`None`), por la guarda defensiva. diff --git a/python/functions/datascience/categorical_cardinality_block.py b/python/functions/datascience/categorical_cardinality_block.py new file mode 100644 index 00000000..e5227abe --- /dev/null +++ b/python/functions/datascience/categorical_cardinality_block.py @@ -0,0 +1,132 @@ +"""Pure EDA helper: cardinality metrics block from a `summarize_categorical` output. + +Part of the `eda` capability group. Consumes the per-column dict produced by +``summarize_categorical`` (for a single categorical/text column) plus the total +row count of the dataset and derives render-ready cardinality metrics: distinct +ratio, normalized entropy, singleton count, and the ``id_like`` / ``dominated`` +flags. + +It does NOT recompute the entropy nor reimplement ``summarize_categorical`` — it +only reads that function's output. Dict-no-throw style of the `eda` group: it +never raises. Missing or malformed inputs yield ``None``/``False``/``0`` for the +affected keys, never an exception. Stdlib only (``math.log2``). +""" + +from math import log2 + + +def _num(value): + """Return ``value`` unchanged if it is a real (non-bool) number, else ``None``. + + ``bool`` is rejected on purpose: in Python ``True`` is an ``int`` but it is + never a meaningful count/ratio here. + """ + if isinstance(value, bool): + return None + if isinstance(value, (int, float)): + return value + return None + + +def categorical_cardinality_block(cat: dict, n_rows: int) -> dict: + """Derive cardinality metrics for one categorical column. + + Args: + cat: The per-column dict produced by ``summarize_categorical`` for a + single categorical/text column. Expected (all optional, read + defensively) keys: ``top`` (list of ``{value, count, pct}``), + ``mode``, ``mode_pct``, ``n_distinct``, ``entropy`` (Shannon, bits), + ``imbalance``, ``len_min``, ``len_mean``, ``len_max``. ``None`` or a + non-dict is treated as ``{}``. + n_rows: Total number of rows in the dataset (used for ``pct_distinct``). + + Returns: + Dict with exactly these keys, every one always present: + ``n_distinct``, ``n_rows``, ``pct_distinct``, ``entropy``, + ``entropy_max``, ``entropy_norm``, ``mode``, ``mode_pct``, + ``imbalance``, ``n_singletons``, ``n_singletons_partial``, ``len_min``, + ``len_mean``, ``len_max``, ``id_like``, ``dominated``. Values are + ``None``/``False`` when not derivable; the function never raises. + """ + cat = cat if isinstance(cat, dict) else {} + + # --- passthroughs (numeric-validated, type preserved) --- + n_distinct = _num(cat.get("n_distinct")) + n_rows_out = _num(n_rows) + entropy = _num(cat.get("entropy")) + imbalance = _num(cat.get("imbalance")) + len_min = _num(cat.get("len_min")) + len_mean = _num(cat.get("len_mean")) + len_max = _num(cat.get("len_max")) + mode = cat.get("mode") # any value (or None); passthrough as-is + + # --- pct_distinct --- + if n_distinct is None or n_rows_out is None or n_rows_out == 0: + pct_distinct = None + else: + pct_distinct = n_distinct / n_rows_out * 100.0 + + # --- entropy_max = log2(n_distinct) --- + if n_distinct is None: + entropy_max = None + elif n_distinct > 1: + entropy_max = log2(n_distinct) + else: # n_distinct in {0, 1} + entropy_max = 0.0 + + # --- entropy_norm = entropy / entropy_max, clipped to [0, 1] --- + if entropy_max is not None and entropy_max > 0 and entropy is not None: + entropy_norm = entropy / entropy_max + entropy_norm = max(0.0, min(1.0, entropy_norm)) + else: + entropy_norm = None + + # --- mode_pct: prefer cat['mode_pct']; else derive from top[0].pct --- + mode_pct = _num(cat.get("mode_pct")) + top = cat.get("top") + has_top = isinstance(top, (list, tuple)) and len(top) > 0 + if mode_pct is None and has_top: + first = top[0] + if isinstance(first, dict): + first_pct = _num(first.get("pct")) + if first_pct is not None: + # Normalize to 0-100: a fraction (<= 1) becomes a percentage. + mode_pct = first_pct * 100.0 if first_pct <= 1 else first_pct + + # --- singletons (count == 1) within the visible top --- + if has_top: + n_singletons = sum( + 1 + for item in top + if isinstance(item, dict) and _num(item.get("count")) == 1 + ) + else: + n_singletons = None + + # The singleton count only covers the visible top; there may be more + # distinct values (and thus more singletons) outside it. + top_len = len(top) if isinstance(top, (list, tuple)) else 0 + n_singletons_partial = bool(n_distinct is not None and n_distinct > top_len) + + # --- derived flags --- + id_like = pct_distinct is not None and pct_distinct >= 99.0 + dominated = mode_pct is not None and mode_pct >= 90.0 + + return { + "n_distinct": n_distinct, + "n_rows": n_rows_out, + "pct_distinct": pct_distinct, + "entropy": entropy, + "entropy_max": entropy_max, + "entropy_norm": entropy_norm, + "mode": mode, + "mode_pct": mode_pct, + "imbalance": imbalance, + "n_singletons": n_singletons, + "n_singletons_partial": n_singletons_partial, + "len_min": len_min, + "len_mean": len_mean, + "len_max": len_max, + "id_like": id_like, + "dominated": dominated, + } diff --git a/python/functions/datascience/categorical_cardinality_block_test.py b/python/functions/datascience/categorical_cardinality_block_test.py new file mode 100644 index 00000000..616015ed --- /dev/null +++ b/python/functions/datascience/categorical_cardinality_block_test.py @@ -0,0 +1,216 @@ +"""Tests para categorical_cardinality_block.""" + +import sys +import os +from math import log2 + +sys.path.insert(0, os.path.dirname(__file__)) + +from categorical_cardinality_block import categorical_cardinality_block + + +# Output contract: every call returns exactly these 16 keys. +EXPECTED_KEYS = { + "n_distinct", + "n_rows", + "pct_distinct", + "entropy", + "entropy_max", + "entropy_norm", + "mode", + "mode_pct", + "imbalance", + "n_singletons", + "n_singletons_partial", + "len_min", + "len_mean", + "len_max", + "id_like", + "dominated", +} + + +def _sample_cat(): + """A realistic summarize_categorical output for one column.""" + return { + "top": [ + {"value": "a", "count": 5, "pct": 0.5}, + {"value": "b", "count": 3, "pct": 0.3}, + {"value": "c", "count": 1, "pct": 0.1}, + {"value": "d", "count": 1, "pct": 0.1}, + ], + "mode": "a", + "mode_pct": 0.5, + "n_distinct": 4, + "entropy": 1.685, # <= log2(4) = 2.0 + "imbalance": 5.0, + "len_min": 1, + "len_mean": 1.0, + "len_max": 1, + } + + +def test_normal_case(): + """Caso normal: pct_distinct, entropy_max=log2(n_distinct), entropy_norm in [0,1], n_singletons.""" + cat = _sample_cat() + result = categorical_cardinality_block(cat, n_rows=10) + + assert set(result.keys()) == EXPECTED_KEYS + + # passthroughs + assert result["n_distinct"] == 4 + assert result["n_rows"] == 10 + assert result["entropy"] == 1.685 + assert result["imbalance"] == 5.0 + assert result["mode"] == "a" + assert result["mode_pct"] == 0.5 # passthrough, not normalized + assert result["len_min"] == 1 + assert result["len_max"] == 1 + + # pct_distinct = 4 / 10 * 100 + assert abs(result["pct_distinct"] - 40.0) < 1e-12 + + # entropy_max = log2(4) = 2.0 + assert abs(result["entropy_max"] - log2(4)) < 1e-12 + assert abs(result["entropy_max"] - 2.0) < 1e-12 + + # entropy_norm = 1.685 / 2.0 = 0.8425, within [0, 1] + assert abs(result["entropy_norm"] - 1.685 / 2.0) < 1e-12 + assert 0.0 <= result["entropy_norm"] <= 1.0 + + # singletons: c and d have count == 1 + assert result["n_singletons"] == 2 + # top covers all distinct values (4 == 4) + assert result["n_singletons_partial"] is False + + # neither id-like (40%) nor dominated (mode_pct 0.5) + assert result["id_like"] is False + assert result["dominated"] is False + + +def test_empty_cat_does_not_raise(): + """Caso cat={}: no lanza, claves derivadas None y flags False.""" + result = categorical_cardinality_block({}, n_rows=100) + + assert set(result.keys()) == EXPECTED_KEYS + for key in ( + "n_distinct", + "pct_distinct", + "entropy", + "entropy_max", + "entropy_norm", + "mode", + "mode_pct", + "imbalance", + "n_singletons", + "len_min", + "len_mean", + "len_max", + ): + assert result[key] is None + assert result["n_singletons_partial"] is False + assert result["id_like"] is False + assert result["dominated"] is False + # n_rows is a passthrough of the argument, still coherent. + assert result["n_rows"] == 100 + + +def test_none_cat_does_not_raise(): + """Caso cat=None: tratado como {}, mismas garantias que el dict vacio.""" + result = categorical_cardinality_block(None, n_rows=None) + assert set(result.keys()) == EXPECTED_KEYS + assert result["n_distinct"] is None + assert result["pct_distinct"] is None + assert result["entropy_max"] is None + assert result["entropy_norm"] is None + assert result["id_like"] is False + assert result["dominated"] is False + + +def test_n_rows_zero_no_zero_division(): + """Caso n_rows=0: pct_distinct None sin ZeroDivisionError.""" + cat = _sample_cat() + result = categorical_cardinality_block(cat, n_rows=0) + assert result["pct_distinct"] is None + # n_distinct still passes through. + assert result["n_distinct"] == 4 + assert result["id_like"] is False + + +def test_id_like_when_distinct_near_rows(): + """id_like True cuando n_distinct ~ n_rows (pct_distinct >= 99).""" + cat = {"n_distinct": 99, "entropy": 6.6, "top": [], "mode": None} + result = categorical_cardinality_block(cat, n_rows=100) + assert abs(result["pct_distinct"] - 99.0) < 1e-12 + assert result["id_like"] is True + + # exact identity column: 100 / 100 = 100% + cat_full = {"n_distinct": 100, "top": []} + result_full = categorical_cardinality_block(cat_full, n_rows=100) + assert result_full["id_like"] is True + + +def test_dominated_when_mode_pct_high(): + """dominated True cuando mode_pct alto (>= 90).""" + cat = { + "n_distinct": 3, + "entropy": 0.3, + "mode": "x", + "mode_pct": 95.0, + "top": [ + {"value": "x", "count": 95, "pct": 0.95}, + {"value": "y", "count": 3, "pct": 0.03}, + {"value": "z", "count": 2, "pct": 0.02}, + ], + "imbalance": 47.5, + } + result = categorical_cardinality_block(cat, n_rows=100) + assert result["mode_pct"] == 95.0 + assert result["dominated"] is True + + +def test_mode_pct_fallback_from_top_fraction(): + """Sin mode_pct: deriva del pct del primer top, fraccion <=1 escala a 0-100.""" + cat = { + "n_distinct": 3, + "top": [ + {"value": "x", "count": 95, "pct": 0.95}, + {"value": "y", "count": 5, "pct": 0.05}, + ], + } + result = categorical_cardinality_block(cat, n_rows=100) + # 0.95 (fraction) -> 95.0 (percentage) + assert abs(result["mode_pct"] - 95.0) < 1e-12 + assert result["dominated"] is True + + +def test_n_singletons_partial_when_top_truncated(): + """n_distinct > len(top): n_singletons cubre solo el top visible, partial True.""" + cat = { + "n_distinct": 10, + "top": [ + {"value": "a", "count": 4, "pct": 0.4}, + {"value": "b", "count": 1, "pct": 0.1}, + {"value": "c", "count": 1, "pct": 0.1}, + ], + "entropy": 2.5, + } + result = categorical_cardinality_block(cat, n_rows=12) + assert result["n_singletons"] == 2 # only b, c visible + assert result["n_singletons_partial"] is True + + +def test_single_distinct_value_entropy_norm_none(): + """n_distinct=1: entropy_max=0.0 -> entropy_norm None (no division by zero).""" + cat = { + "n_distinct": 1, + "entropy": 0.0, + "mode": "only", + "mode_pct": 1.0, + "top": [{"value": "only", "count": 7, "pct": 1.0}], + "imbalance": 1.0, + } + result = categorical_cardinality_block(cat, n_rows=7) + assert result["entropy_max"] == 0.0 + assert result["entropy_norm"] is None + assert result["n_singletons"] == 0 diff --git a/python/functions/datascience/categorical_top_pie_figure.md b/python/functions/datascience/categorical_top_pie_figure.md new file mode 100644 index 00000000..8bbe067c --- /dev/null +++ b/python/functions/datascience/categorical_top_pie_figure.md @@ -0,0 +1,108 @@ +--- +id: categorical_top_pie_figure_py_datascience +name: categorical_top_pie_figure +kind: function +lang: py +domain: datascience +version: "1.0.0" +purity: impure +signature: "def categorical_top_pie_figure(top: list, n_distinct: int = 0, title: str = \"\", top_k: int = 6, n_rows=None) -> \"matplotlib.figure.Figure\"" +description: "Construye una figura matplotlib tipo donut (pie con agujero central) de las top_k categorías más frecuentes de una columna categórica, agregando el resto en un sector gris \"Otros (N categorías)\". Consume el bloque `top` de summarize_categorical y devuelve un matplotlib.figure.Figure listo para rasterizar por el renderer del informe EDA. Backend Agg sin pyplot global; defensivo ante top vacío/None." +tags: [eda, categorical, pie, donut, matplotlib, figure, visualization, datascience, impure] +uses_functions: [] +uses_types: [] +returns: [] +returns_optional: false +error_type: "error_go_core" +imports: [matplotlib] +example: | + from categorical_top_pie_figure import categorical_top_pie_figure + top = [ + {"value": "rojo", "count": 40, "pct": 0.4}, + {"value": "azul", "count": 30, "pct": 0.3}, + {"value": "verde", "count": 20, "pct": 0.2}, + ] + fig = categorical_top_pie_figure(top, n_distinct=12, title="color", top_k=6, n_rows=100) +tested: true +tests: + - "test_returns_figure" + - "test_ten_items_topk_six_yields_seven_wedges" + - "test_empty_top_does_not_raise_and_returns_figure" + - "test_long_value_truncated_in_legend" + - "test_none_value_and_none_count_are_handled" + - "test_n_rows_adds_exact_others_slice" +test_file_path: "python/functions/datascience/categorical_top_pie_figure_test.py" +file_path: "python/functions/datascience/categorical_top_pie_figure.py" +params: + - name: top + desc: "Lista de dicts {value, count, pct} ordenada de mayor a menor por count (salida del bloque `top` de summarize_categorical). Puede venir vacía o con dicts incompletos: items no-dict, sin count, con count None o count <= 0 se descartan. value None se admite (sin etiqueta)." + - name: n_distinct + desc: "Nº total de categorías distintas de la columna. Etiqueta el sector agregado como \"Otros (n_distinct - top_k)\" (mínimo 0). Si no supera el nº de sectores mostrados, se usa el overflow real de `top` como nº de categorías agregadas. Default 0." + - name: title + desc: "Título de la figura (nombre de la columna). Se trunca a ~48 chars con elipsis si es muy largo. Default \"\" (sin título)." + - name: top_k + desc: "Nº máximo de sectores explícitos. Default 6. El sector \"Otros\" no cuenta contra este límite. Con top_k <= 0 se muestra al menos la categoría mayor." + - name: n_rows + desc: "Opcional. Total de filas del dataset. Si se da y la suma de counts mostrados < n_rows, el sector \"Otros\" usa (n_rows - suma_mostrada) como count para que los ángulos sean exactos respecto al total real. Si se omite, \"Otros\" usa la suma de counts fuera del top_k mostrado (solo cuando top trae más de top_k items). Default None." +output: "Un matplotlib.figure.Figure (figsize 6.4x4.0, dpi 150) con un Axes donut (wedgeprops width 0.42) más una leyenda lateral con value truncado a 20 chars + count; el sector \"Otros\" en gris. Anotación central con el total n. Si no hay counts válidos, devuelve igualmente una Figure con un texto centrado \"sin datos categóricos\" (nunca lanza). El caller rasteriza/cierra la figura; la función no la muestra ni la guarda." +--- + +## Ejemplo + +```python +from categorical_top_pie_figure import categorical_top_pie_figure + +# `top` es la salida del bloque "top" de summarize_categorical (ya ordenado desc). +top = [ + {"value": "rojo", "count": 40, "pct": 0.40}, + {"value": "azul", "count": 30, "pct": 0.30}, + {"value": "verde", "count": 20, "pct": 0.20}, + {"value": "amarillo", "count": 5, "pct": 0.05}, +] + +fig = categorical_top_pie_figure( + top, + n_distinct=12, # 12 categorías distintas en total + title="color_producto", + top_k=6, # hasta 6 sectores explícitos + n_rows=100, # "Otros" = 100 - 95 = 5, sobre 8 categorías agregadas +) + +# El renderer del informe lo rasteriza; aquí solo persistimos para inspección. +fig.savefig("/tmp/donut_color.png") +``` + +## Cuando usarla + +Úsala dentro de un informe EDA cuando quieras visualizar la composición de una +columna categórica de un vistazo: cuántas filas caen en las categorías +dominantes frente a la cola larga. Pásale directamente el bloque `top` de +`summarize_categorical` (ya ordenado de mayor a menor) más `n_distinct` para que +el sector "Otros" indique cuántas categorías quedan agrupadas. Es la pareja +"composición" del gráfico de barras top-k: el donut comunica proporciones del +total, las barras comunican magnitudes comparables. + +## Gotchas + +- **Impura por matplotlib.** Toca la maquinaria de render. Usa el backend `Agg` + y la API orientada a objetos `Figure`/`add_subplot` — NUNCA `pyplot.*` aquí, + para no tocar el estado global ni filtrar figuras entre llamadas. `pyplot` NO + es thread-safe; esta función evita ese riesgo construyendo el `Figure` + directamente, así que es segura de llamar en bucle desde el renderer. +- **El caller cierra la figura.** La función devuelve el `Figure` pero no lo + muestra ni lo guarda. Quien la consume debe rasterizarla y luego liberarla + (`fig.clf()` / `matplotlib.pyplot.close(fig)` si se usó pyplot en el caller) + para no acumular memoria en lotes grandes de columnas. +- **Pie engaña con muchos sectores.** Por eso `top_k` por defecto es 6 y el + resto se agrega en "Otros": donuts con 15+ sectores son ilegibles. Para + cardinalidad muy alta el donut solo muestra la cabeza de la distribución; la + cola vive en el sector gris. +- **Ángulos exactos solo con `n_rows`.** Sin `n_rows`, el sector "Otros" se + calcula con el overflow presente en `top`; si `top` ya viene recortado a + `top_k` por el productor, no habrá "Otros" aunque existan más categorías. Pasa + `n_rows` (total de filas del dataset) para ángulos correctos respecto al total + real. +- **Defensiva, nunca lanza.** `top=[]`, `value=None`, `count=None` o counts no + numéricos se manejan sin error: en el peor caso devuelve una `Figure` con + "sin datos categóricos". No envuelvas la llamada en try/except por miedo a un + raise — no lo hay. diff --git a/python/functions/datascience/categorical_top_pie_figure.py b/python/functions/datascience/categorical_top_pie_figure.py new file mode 100644 index 00000000..6e7776f8 --- /dev/null +++ b/python/functions/datascience/categorical_top_pie_figure.py @@ -0,0 +1,230 @@ +"""Impure EDA helper: donut figure of the most common categories (`eda` group). + +Builds a matplotlib donut (pie with a central hole) of the ``top_k`` most +frequent categories of a categorical column, folding everything else into a +single "Otros (N categorías)" slice. Returns a ready-to-rasterize +``matplotlib.figure.Figure``; it never shows nor saves it. + +Impure because it touches matplotlib's rendering machinery. It uses the headless +Agg backend and the object-oriented ``Figure`` API (no ``pyplot``) so it leaks no +global state and is safe to call repeatedly from a report renderer. +""" + +import matplotlib + +matplotlib.use("Agg") + +from matplotlib.figure import Figure # noqa: E402 + + +# Gray reserved for the aggregated "Otros" slice. +_OTHER_COLOR = "#9e9e9e" +# Muted gray for secondary text (title fallback, center annotation, no-data). +_MUTED_TEXT = "#5f6b7a" +# Pleasant, colour-blind-friendly qualitative palette for the explicit slices. +_PALETTE = [ + "#4C72B0", + "#DD8452", + "#55A868", + "#C44E52", + "#8172B3", + "#937860", + "#DA8BC3", + "#8C8C8C", + "#CCB974", + "#64B5CD", +] + + +def _truncate(text, width: int = 20) -> str: + """Truncate ``text`` to ``width`` chars, appending an ellipsis if cut.""" + s = "" if text is None else str(text) + if len(s) <= width: + return s + if width <= 1: + return s[:width] + return s[: width - 1] + "…" + + +def categorical_top_pie_figure( + top: list, + n_distinct: int = 0, + title: str = "", + top_k: int = 6, + n_rows=None, +) -> "matplotlib.figure.Figure": + """Build a donut figure of the most common categories of a column. + + Renders the ``top_k`` most frequent categories as explicit donut slices and + aggregates every remaining category into a single gray "Otros (N + categorías)" slice. Category names are not painted on the wedges; they are + listed in a lateral legend (truncated value + count) to avoid overlap on + narrow (mobile) figures. + + The function is fully defensive: empty input, missing/``None`` values or + counts never raise. When there is nothing valid to draw it still returns a + ``Figure`` carrying a centered "sin datos categóricos" message. + + Args: + top: List of ``{value, count, pct}`` dicts, already sorted by ``count`` + descending (the ``top`` block of ``summarize_categorical``). May be + empty or carry incomplete/``None`` entries; non-dict items, items + without a positive numeric ``count`` and ``None`` counts are skipped. + n_distinct: Total number of distinct categories in the column. Used to + label the aggregated slice as "Otros (n_distinct - top_k)" (floored + at 0). Ignored when it does not exceed the number of shown slices. + title: Figure title (the column name). Truncated when too long. + top_k: Maximum number of explicit slices. Default 6. The "Otros" slice + does not count against this limit. + n_rows: Optional total row count of the dataset. When given and the sum + of shown counts is below ``n_rows``, the "Otros" slice uses + ``n_rows - sum_shown`` as its count so the wedge angles are exact + with respect to the real total. When omitted, "Otros" uses the sum + of the counts that fall outside the shown ``top_k`` (only when + ``top`` carries more than ``top_k`` items). + + Returns: + A ``matplotlib.figure.Figure`` with a single donut Axes plus a lateral + legend. The caller is responsible for rasterizing/closing it. + """ + fig = Figure(figsize=(6.4, 4.0), dpi=150) + ax = fig.add_subplot(111) + + safe_title = _truncate(title, 48) + + # --- Defensive parse: keep only well-formed {value, count} with count > 0. + cleaned = [] + if isinstance(top, list): + for item in top: + if not isinstance(item, dict): + continue + count = item.get("count") + if count is None: + continue + try: + count = float(count) + except (TypeError, ValueError): + continue + if count <= 0: + continue + cleaned.append((item.get("value"), count)) + + if not cleaned: + ax.axis("off") + ax.text( + 0.5, + 0.5, + "sin datos categóricos", + ha="center", + va="center", + fontsize=12, + color=_MUTED_TEXT, + transform=ax.transAxes, + ) + if safe_title: + ax.set_title(safe_title, fontsize=12, loc="center", pad=8) + fig.tight_layout() + return fig + + # --- Split into shown slices and the aggregated remainder. + shown = cleaned[: max(int(top_k), 0)] + if not shown: # top_k <= 0 — show at least the largest category. + shown = cleaned[:1] + + sum_shown = sum(c for _, c in shown) + overflow_count = sum(c for _, c in cleaned[len(shown):]) + + # How many categories are folded into "Otros". + try: + nd = int(n_distinct) + except (TypeError, ValueError): + nd = 0 + others_categories = max(nd - len(shown), 0) + # If n_distinct is unknown/too small, fall back to the overflow we actually + # have in `top` beyond the shown slices. + overflow_items = len(cleaned) - len(shown) + if others_categories == 0 and overflow_items > 0: + others_categories = overflow_items + + # Count attributed to the "Otros" slice for exact angles. + others_count = 0.0 + if n_rows is not None: + try: + total_rows = float(n_rows) + except (TypeError, ValueError): + total_rows = None + if total_rows is not None and total_rows > sum_shown: + others_count = total_rows - sum_shown + if others_count <= 0: + others_count = overflow_count + + labels = [v for v, _ in shown] + values = [c for _, c in shown] + colors = [_PALETTE[i % len(_PALETTE)] for i in range(len(shown))] + + has_others = others_count > 0 and others_categories > 0 + if has_others: + values.append(others_count) + labels.append("Otros") + colors.append(_OTHER_COLOR) + + total = sum(values) + + def _autopct(pct: float) -> str: + # Hide tiny labels to avoid crowding the wedges. + return f"{pct:.0f}%" if pct >= 5 else "" + + wedges, _texts, autotexts = ax.pie( + values, + colors=colors, + startangle=90, + counterclock=False, + wedgeprops={"width": 0.42, "edgecolor": "white", "linewidth": 1.0}, + autopct=_autopct, + pctdistance=0.79, + textprops={"fontsize": 8}, + ) + for at in autotexts: + at.set_color("white") + at.set_fontweight("bold") + ax.set_aspect("equal") + + # --- Lateral legend: truncated value + count (+ "(N categorías)" for Otros). + legend_labels = [] + for idx, (lab, val) in enumerate(zip(labels, values)): + if has_others and idx == len(labels) - 1: + legend_labels.append( + f"Otros ({others_categories} categorías) — {int(round(val))}" + ) + else: + legend_labels.append(f"{_truncate(lab, 20)} — {int(round(val))}") + + ax.legend( + wedges, + legend_labels, + title="Categorías", + loc="center left", + bbox_to_anchor=(1.02, 0.5), + fontsize=8, + title_fontsize=9, + frameon=False, + ) + + if safe_title: + ax.set_title(safe_title, fontsize=13, loc="left", pad=10) + + # Center annotation: total count covered by the donut. + ax.text( + 0, + 0, + f"n={int(round(total))}", + ha="center", + va="center", + fontsize=11, + color=_MUTED_TEXT, + fontweight="bold", + ) + + # Leave room on the right for the legend (avoid clipping it). + fig.subplots_adjust(left=0.02, right=0.62, top=0.88, bottom=0.06) + return fig diff --git a/python/functions/datascience/categorical_top_pie_figure_test.py b/python/functions/datascience/categorical_top_pie_figure_test.py new file mode 100644 index 00000000..4d0da0ad --- /dev/null +++ b/python/functions/datascience/categorical_top_pie_figure_test.py @@ -0,0 +1,104 @@ +"""Tests para categorical_top_pie_figure (donut de categorías top, grupo eda). + +Usa el backend Agg sin pyplot; no muestra ni guarda figuras. Cada test cierra +explícitamente la Figure construida (matplotlib.pyplot.close) para no acumular +estado entre tests. +""" + +import matplotlib + +matplotlib.use("Agg") + +import matplotlib.pyplot as plt # noqa: E402 +from matplotlib.figure import Figure # noqa: E402 + +from categorical_top_pie_figure import categorical_top_pie_figure + + +def _make_top(n): + """n items {value, count, pct} ordenados desc por count.""" + return [ + {"value": f"cat_{i}", "count": n - i, "pct": (n - i) / sum(range(1, n + 1))} + for i in range(n) + ] + + +def _wedges(ax): + """Devuelve los wedges (sectores) de un Axes con un pie.""" + from matplotlib.patches import Wedge + + return [p for p in ax.patches if isinstance(p, Wedge)] + + +def test_returns_figure(): + fig = categorical_top_pie_figure(_make_top(3), n_distinct=3, title="col") + assert isinstance(fig, Figure) + plt.close(fig) + + +def test_ten_items_topk_six_yields_seven_wedges(): + top = _make_top(10) + fig = categorical_top_pie_figure(top, n_distinct=10, title="muchas", top_k=6) + ax = fig.axes[0] + wedges = _wedges(ax) + # 6 categorías explícitas + 1 sector "Otros". + assert len(wedges) == 7 + plt.close(fig) + + +def test_empty_top_does_not_raise_and_returns_figure(): + fig = categorical_top_pie_figure([], n_distinct=0, title="vacía") + assert isinstance(fig, Figure) + # Sin datos: no debe haber sectores de pie. + assert len(_wedges(fig.axes[0])) == 0 + plt.close(fig) + + +def test_long_value_truncated_in_legend(): + long_value = "una_categoria_con_un_nombre_larguisimo_que_excede_el_limite" + top = [ + {"value": long_value, "count": 10, "pct": 0.5}, + {"value": "corta", "count": 10, "pct": 0.5}, + ] + fig = categorical_top_pie_figure(top, n_distinct=2, title="col", top_k=6) + ax = fig.axes[0] + legend = ax.get_legend() + assert legend is not None + texts = [t.get_text() for t in legend.get_texts()] + # El valor largo aparece truncado con elipsis y NO en su forma completa. + assert any("…" in t for t in texts) + assert long_value not in " ".join(texts) + plt.close(fig) + + +def test_none_value_and_none_count_are_handled(): + top = [ + {"value": None, "count": 5, "pct": 0.5}, + {"value": "b", "count": None, "pct": 0.0}, # count None -> se descarta + {"value": "c", "count": 5, "pct": 0.5}, + ] + fig = categorical_top_pie_figure(top, n_distinct=2, title="con nones", top_k=6) + assert isinstance(fig, Figure) + # Solo 2 items válidos, sin overflow -> 2 wedges, sin "Otros". + assert len(_wedges(fig.axes[0])) == 2 + plt.close(fig) + + +def test_n_rows_adds_exact_others_slice(): + # 3 categorías mostradas suman 30, dataset real 100 -> "Otros" = 70. + top = _make_top(3) # counts 3,2,1 -> reescalamos abajo + top = [ + {"value": "a", "count": 15, "pct": 0.15}, + {"value": "b", "count": 10, "pct": 0.10}, + {"value": "c", "count": 5, "pct": 0.05}, + ] + fig = categorical_top_pie_figure( + top, n_distinct=20, title="col", top_k=3, n_rows=100 + ) + ax = fig.axes[0] + # 3 explícitas + Otros. + assert len(_wedges(ax)) == 4 + legend_texts = [t.get_text() for t in ax.get_legend().get_texts()] + # El sector Otros refleja n_distinct - top_k = 17 categorías y count 70. + assert any("Otros (17 categorías)" in t and "70" in t for t in legend_texts) + plt.close(fig) From cd658cc7036836304b67791b4d753e1c85fdef9d Mon Sep 17 00:00:00 2001 From: Egutierrez Date: Tue, 30 Jun 2026 15:29:33 +0200 Subject: [PATCH 13/53] =?UTF-8?q?feat(eda):=20primitivas=20geoespaciales?= =?UTF-8?q?=20del=20grupo=20eda=20(detecci=C3=B3n=20lat/lon=20+=20extensi?= =?UTF-8?q?=C3=B3n=20+=20scatter)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tres funciones puras nuevas del dominio datascience (tags eda + geospatial) que sostienen el capítulo GEOSPATIAL del AutomaticEDA, delegadas a fn-constructor: - detect_latlon_columns: identifica el par (lat, lon) por nombre de columna + rango de valores ([-90,90] / [-180,180]) desde profile['columns']. Devuelve {lat_col, lon_col, confidence, reason}. 9 tests. - analyze_geo_extent: bbox, centroide, span haversine, conteo por zona/país (lookup offline con bounding boxes embebidos, KISS sin geopandas) y hemisferios. 7 tests. - build_geo_scatter: prepara los puntos del scatter en orden [lon, lat] con downsampling determinista por paso fijo + aspect equirectangular 1/cos(lat) clampado. 6 tests. Registradas en datascience/__init__.py. Todas pure, params_schema completo, .md autosuficiente (Ejemplo + Cuando usarla + Gotchas). Co-Authored-By: Claude Opus 4.8 (1M context) --- python/functions/datascience/__init__.py | 6 + .../datascience/analyze_geo_extent.md | 61 +++++ .../datascience/analyze_geo_extent.py | 209 ++++++++++++++++++ .../datascience/analyze_geo_extent_test.py | 126 +++++++++++ .../datascience/build_geo_scatter.md | 68 ++++++ .../datascience/build_geo_scatter.py | 153 +++++++++++++ .../datascience/build_geo_scatter_test.py | 140 ++++++++++++ .../datascience/detect_latlon_columns.md | 67 ++++++ .../datascience/detect_latlon_columns.py | 198 +++++++++++++++++ .../datascience/detect_latlon_columns_test.py | 141 ++++++++++++ 10 files changed, 1169 insertions(+) create mode 100644 python/functions/datascience/analyze_geo_extent.md create mode 100644 python/functions/datascience/analyze_geo_extent.py create mode 100644 python/functions/datascience/analyze_geo_extent_test.py create mode 100644 python/functions/datascience/build_geo_scatter.md create mode 100644 python/functions/datascience/build_geo_scatter.py create mode 100644 python/functions/datascience/build_geo_scatter_test.py create mode 100644 python/functions/datascience/detect_latlon_columns.md create mode 100644 python/functions/datascience/detect_latlon_columns.py create mode 100644 python/functions/datascience/detect_latlon_columns_test.py diff --git a/python/functions/datascience/__init__.py b/python/functions/datascience/__init__.py index 9fc8c206..60f5260d 100644 --- a/python/functions/datascience/__init__.py +++ b/python/functions/datascience/__init__.py @@ -44,6 +44,9 @@ from .trend_slope import trend_slope from .run_eda_models import run_eda_models from .project_clusters_2d import project_clusters_2d from .describe_clusters_llm import describe_clusters_llm +from .detect_latlon_columns import detect_latlon_columns +from .analyze_geo_extent import analyze_geo_extent +from .build_geo_scatter import build_geo_scatter from .eda_llm_insights import eda_llm_insights from .build_eda_notebook import build_eda_notebook from .decode_qr_image import decode_qr_image @@ -90,6 +93,9 @@ __all__ = [ "run_eda_models", "project_clusters_2d", "describe_clusters_llm", + "detect_latlon_columns", + "analyze_geo_extent", + "build_geo_scatter", "eda_llm_insights", "build_eda_notebook", "describe_numeric", diff --git a/python/functions/datascience/analyze_geo_extent.md b/python/functions/datascience/analyze_geo_extent.md new file mode 100644 index 00000000..c46b6711 --- /dev/null +++ b/python/functions/datascience/analyze_geo_extent.md @@ -0,0 +1,61 @@ +--- +name: analyze_geo_extent +kind: function +lang: py +domain: datascience +version: "1.0.0" +purity: pure +signature: "def analyze_geo_extent(lats: list, lons: list) -> dict" +description: "Calcula la extension geografica de una nube de coordenadas (lat/lon) y asigna cada punto a un pais/region mediante un lookup OFFLINE contra una tabla de bounding boxes embebida como constante. Devuelve bounding box, centroide, span de la diagonal (haversine), conteo por region (top-8 + Otros), reparto por hemisferios y una frase resumen en ES. Lectura defensiva: descarta pares None/NaN/fuera de rango y NUNCA lanza. Solo stdlib (math); sin geopandas/shapely. Las cajas de paises son rectangulos aproximados, no reverse-geocoding exacto." +tags: [eda, geospatial, geo, coordinates, bounding-box, haversine, datascience] +params: + - name: lats + desc: "Lista de latitudes en grados, rango valido [-90, 90]. Se empareja por indice con lons (gana la longitud minima comun si difieren). Cada valor puede ser None/NaN/no-numerico/fuera de rango: se lee defensivo y se descarta el par." + - name: lons + desc: "Lista de longitudes en grados, rango valido [-180, 180]. Paralela a lats, emparejada por indice. Valores None/NaN/no-numericos/fuera de rango se descartan junto con su par." +output: "Dict con el resumen geografico: {n_points=pares validos usados, bbox={lat_min,lat_max,lon_min,lon_max} o None, centroid={lat,lon}=media de lat/lon validos o None, span_km=distancia haversine (radio 6371 km) de la diagonal SO->NE del bbox, by_region=[{region,count}] descendente por count limitado a top-8 con el resto agregado en 'Otros', hemisphere={north,south,east,west} (ecuador->norte, meridiano 0->este), note=frase ES resumen}. Si no hay pares validos devuelve la forma cero: n_points 0, bbox None, centroid None, span_km 0.0, by_region [], hemisphere a ceros y note 'sin coordenadas validas'. Puntos que no caen en ninguna caja -> region 'Oceano/Otros'." +uses_functions: [] +uses_types: [] +returns: [] +returns_optional: false +error_type: "" +imports: [math] +tested: true +tests: ["test_nube_en_espana", "test_dos_paises_distintos", "test_listas_vacias", "test_pares_invalidos_filtrados", "test_longitudes_desbalanceadas", "test_span_km_haversine_par_conocido", "test_no_lanza_con_entradas_raras"] +test_file_path: "python/functions/datascience/analyze_geo_extent_test.py" +file_path: "python/functions/datascience/analyze_geo_extent.py" +--- + +## Ejemplo + +```python +import sys, os +sys.path.insert(0, os.path.join("python", "functions")) +from datascience.analyze_geo_extent import analyze_geo_extent + +# Nube de puntos alrededor de Madrid + un punto en Paris. +lats = [40.4, 40.0, 41.0, 48.8] +lons = [-3.7, -3.5, -4.0, 2.3] +res = analyze_geo_extent(lats, lons) + +print(res["n_points"]) # 4 +print(res["by_region"]) # [{'region': 'España', 'count': 3}, {'region': 'Francia', 'count': 1}] +print(round(res["span_km"], 1)) # diagonal SO->NE del bbox en km +print(res["hemisphere"]) # {'north': 4, 'south': 0, 'east': 1, 'west': 3} +print(res["note"]) # los puntos se concentran en España (3 de 4) +``` + +## Cuando usarla + +- Usala en el perfilado EDA (grupo `eda`) cuando una tabla tenga columnas de latitud y longitud y quieras un resumen geografico rapido: donde se concentran los puntos, cuanto territorio cubren y a que paises/regiones caen, sin montar geopandas ni un reverse-geocoder. +- Cuando necesites un capitulo `geospatial` del `AutomaticEDA`: alimenta el bbox + centroide para centrar un mapa, el `span_km` para elegir el zoom, y `by_region` para una tabla de conteos por pais. +- Cuando quieras detectar datos sucios de coordenadas (mezcla de hemisferios inesperada, puntos en `Oceano/Otros`, span enorme) antes de seguir el analisis. + +## Gotchas + +- Funcion pura, sin I/O ni red y determinista: mismas entradas -> misma salida. Lectura defensiva, NUNCA lanza; pares con None/NaN o fuera de rango ([-90,90] lat, [-180,180] lon) se descartan en silencio. +- El lookup de region es una **aproximacion rectangular**: cada pais/region es un bounding box, NO su frontera real. Un punto en el mar cerca de una costa, o en una esquina del rectangulo, puede asignarse a un pais vecino. No es reverse-geocoding exacto — para precision real hace falta un shapefile (fuera de scope por KISS). +- Cajas solapadas se resuelven por orden: gana la PRIMERA que contiene el punto. Los paises se listan antes que los continentes (fallback), y entre vecinos el mas estrecho/occidental va primero (Portugal antes que España, Chile antes que Argentina, EEUU contiguo antes que Canada). Un punto que no cae en ninguna caja -> `Oceano/Otros`. +- La tabla cubre ~24 paises grandes + 6 regiones continentales; paises pequeños o no listados caen a su continente o a `Oceano/Otros`. No incluye territorios insulares lejanos (Canarias, Hawaii, etc.). +- `span_km` es la diagonal del bounding box (esquina SO a NE), no la dispersion real de la nube ni el area; con un solo punto valido el bbox es degenerado y `span_km` es 0.0. +- El ecuador (`lat == 0`) cuenta como hemisferio norte y el meridiano 0 (`lon == 0`) como este, por convencion `>= 0`. diff --git a/python/functions/datascience/analyze_geo_extent.py b/python/functions/datascience/analyze_geo_extent.py new file mode 100644 index 00000000..1e17b95e --- /dev/null +++ b/python/functions/datascience/analyze_geo_extent.py @@ -0,0 +1,209 @@ +"""analyze_geo_extent — geographic extent of a cloud of coordinates (EDA `geospatial`). + +Pure function: no I/O, no network, deterministic. Given two parallel lists of +latitudes and longitudes it derives the bounding box, centroid, diagonal span +(haversine), per-region counts and hemisphere split of the points, and assigns +each point to a country/region via an OFFLINE lookup against a table of +rectangular bounding boxes embedded as a constant (`_REGION_BBOXES`). + +It never reads files, never hits the network and depends only on `math`. The +country boxes are deliberately coarse rectangles (a KISS approximation, NOT a +reverse-geocoder). Reading is defensive throughout and the function NEVER +raises: invalid pairs (None / NaN / out of range) are silently discarded and an +empty cloud yields a zeroed result the caller can skip. +""" + +import math + +# Earth mean radius in km used by the haversine formula. +_EARTH_RADIUS_KM = 6371.0 + +# How many distinct regions to surface in `by_region` before collapsing the +# remainder into a single "Otros" bucket. +_TOP_REGIONS = 8 + +# Offline region lookup: (name, lat_min, lat_max, lon_min, lon_max). +# +# Specific countries are listed FIRST and continental fallbacks LAST: each point +# is assigned to the FIRST box that contains it, so the more specific country box +# wins over the broad continent box. Boxes are coarse rectangles approximating +# the mainland extent of each region; overlapping neighbours are ordered so the +# narrower/more-western country claims its coastal points (e.g. Portugal before +# Spain, Chile before Argentina, the contiguous US before Canada). +_REGION_BBOXES = ( + # --- countries (specific) --- + ("Portugal", 36.9, 42.2, -9.6, -6.2), + ("España", 36.0, 43.8, -9.4, 3.4), + ("Francia", 41.3, 51.1, -5.2, 9.6), + ("Reino Unido", 49.9, 58.7, -8.6, 1.8), + ("Irlanda", 51.4, 55.4, -10.6, -5.9), + ("Países Bajos", 50.7, 53.6, 3.3, 7.2), + ("Bélgica", 49.5, 51.5, 2.5, 6.4), + ("Suiza", 45.8, 47.8, 5.9, 10.5), + ("Alemania", 47.3, 55.1, 5.9, 15.0), + ("Italia", 36.6, 47.1, 6.6, 18.5), + ("Marruecos", 27.7, 35.9, -13.2, -1.0), + ("Egipto", 22.0, 31.7, 25.0, 35.0), + ("Sudáfrica", -34.8, -22.1, 16.5, 32.9), + ("China", 18.0, 53.6, 73.5, 135.1), + ("Japón", 24.0, 45.6, 122.9, 145.9), + ("India", 6.7, 35.5, 68.1, 97.4), + ("Australia", -43.7, -10.0, 112.9, 153.7), + ("México", 14.5, 32.7, -118.4, -86.7), + ("Estados Unidos", 24.4, 49.4, -125.0, -66.9), + ("Canadá", 41.7, 83.1, -141.0, -52.6), + ("Chile", -55.9, -17.5, -75.6, -66.4), + ("Argentina", -55.1, -21.8, -73.6, -53.6), + ("Brasil", -33.8, 5.3, -74.0, -34.8), + ("Rusia", 41.2, 77.0, 19.6, 180.0), + # --- continental fallbacks (broad) --- + ("Europa", 34.0, 72.0, -25.0, 45.0), + ("África", -35.0, 37.5, -18.0, 52.0), + ("Asia", 5.0, 78.0, 26.0, 180.0), + ("América del Norte", 7.0, 84.0, -168.0, -52.0), + ("América del Sur", -56.0, 13.0, -82.0, -34.0), + ("Oceanía", -50.0, 0.0, 110.0, 180.0), +) + + +def _coord(value, limit): + """Coerce a coordinate to a valid float in [-limit, limit] or None. + + bool is a subclass of int but never a real coordinate, so True/False are + treated as missing. NaN and out-of-range values are rejected. + """ + if value is None or isinstance(value, bool): + return None + try: + f = float(value) + except (TypeError, ValueError): + return None + # NaN is the only value that is not equal to itself. + if f != f or f < -limit or f > limit: + return None + return f + + +def _haversine_km(lat1, lon1, lat2, lon2): + """Great-circle distance in km between two (lat, lon) points in degrees.""" + rlat1, rlat2 = math.radians(lat1), math.radians(lat2) + dlat = math.radians(lat2 - lat1) + dlon = math.radians(lon2 - lon1) + a = math.sin(dlat / 2.0) ** 2 + math.cos(rlat1) * math.cos(rlat2) * math.sin(dlon / 2.0) ** 2 + return 2.0 * _EARTH_RADIUS_KM * math.asin(min(1.0, math.sqrt(a))) + + +def _region_of(lat, lon): + """Return the name of the first embedded box containing (lat, lon).""" + for name, lat_min, lat_max, lon_min, lon_max in _REGION_BBOXES: + if lat_min <= lat <= lat_max and lon_min <= lon <= lon_max: + return name + return "Océano/Otros" + + +def _empty_result(): + """Result shape when there are no valid coordinate pairs.""" + return { + "n_points": 0, + "bbox": None, + "centroid": None, + "span_km": 0.0, + "by_region": [], + "hemisphere": {"north": 0, "south": 0, "east": 0, "west": 0}, + "note": "sin coordenadas validas", + } + + +def analyze_geo_extent(lats: list, lons: list) -> dict: + """Summarise the geographic extent of a cloud of lat/lon coordinates. + + Pairs `lats[i]` with `lons[i]` by index (over the common length when the two + lists differ in size), discards any pair where either value is None / NaN or + outside [-90, 90] (lat) / [-180, 180] (lon), and derives the bounding box, + centroid, diagonal span, per-region counts and hemisphere split. Each valid + point is matched to a country/region by an offline lookup against coarse + rectangular bounding boxes (`_REGION_BBOXES`). + + Args: + lats: List of latitudes in degrees ([-90, 90]); read defensively. + lons: List of longitudes in degrees ([-180, 180]); read defensively. + Paired with `lats` by index; the shorter length wins when they differ. + + Returns: + Dict with the geographic summary: + {n_points, bbox={lat_min,lat_max,lon_min,lon_max}, centroid={lat,lon}, + span_km (haversine of the SW->NE bbox diagonal), by_region=[{region,count}] + (descending, top-8 with the rest folded into "Otros"), + hemisphere={north,south,east,west}, note (Spanish summary phrase)}. + With no valid pairs returns the zeroed shape: n_points 0, bbox None, + centroid None, span_km 0.0, empty by_region, zeroed hemisphere and the + note "sin coordenadas validas". Never raises. + """ + if not isinstance(lats, (list, tuple)) or not isinstance(lons, (list, tuple)): + return _empty_result() + + valid = [] + # zip already stops at the shorter list -> unbalanced lengths are handled. + for raw_lat, raw_lon in zip(lats, lons): + lat = _coord(raw_lat, 90.0) + lon = _coord(raw_lon, 180.0) + if lat is None or lon is None: + continue + valid.append((lat, lon)) + + if not valid: + return _empty_result() + + n = len(valid) + lat_vals = [p[0] for p in valid] + lon_vals = [p[1] for p in valid] + + lat_min, lat_max = min(lat_vals), max(lat_vals) + lon_min, lon_max = min(lon_vals), max(lon_vals) + + centroid_lat = sum(lat_vals) / n + centroid_lon = sum(lon_vals) / n + + # Diagonal span: SW corner (lat_min, lon_min) to NE corner (lat_max, lon_max). + span_km = _haversine_km(lat_min, lon_min, lat_max, lon_max) + + # Hemisphere split: the equator/prime-meridian go to north/east respectively. + north = sum(1 for lat in lat_vals if lat >= 0.0) + south = n - north + east = sum(1 for lon in lon_vals if lon >= 0.0) + west = n - east + + # Count points per region (offline bbox lookup). + counts = {} + for lat, lon in valid: + region = _region_of(lat, lon) + counts[region] = counts.get(region, 0) + 1 + + # Descending by count, then by name for a deterministic tie-break. + ranked = sorted(counts.items(), key=lambda kv: (-kv[1], kv[0])) + by_region = [{"region": name, "count": count} for name, count in ranked[:_TOP_REGIONS]] + rest = sum(count for _, count in ranked[_TOP_REGIONS:]) + if rest > 0: + by_region.append({"region": "Otros", "count": rest}) + + top_region, top_count = ranked[0] + note = ( + "los puntos se concentran en {region} ({count} de {n})".format( + region=top_region, count=top_count, n=n + ) + ) + + return { + "n_points": n, + "bbox": { + "lat_min": lat_min, + "lat_max": lat_max, + "lon_min": lon_min, + "lon_max": lon_max, + }, + "centroid": {"lat": centroid_lat, "lon": centroid_lon}, + "span_km": span_km, + "by_region": by_region, + "hemisphere": {"north": north, "south": south, "east": east, "west": west}, + "note": note, + } diff --git a/python/functions/datascience/analyze_geo_extent_test.py b/python/functions/datascience/analyze_geo_extent_test.py new file mode 100644 index 00000000..2ef91bee --- /dev/null +++ b/python/functions/datascience/analyze_geo_extent_test.py @@ -0,0 +1,126 @@ +"""Tests para analyze_geo_extent.""" + +import math +import os +import sys + +sys.path.insert(0, os.path.dirname(__file__)) + +from analyze_geo_extent import analyze_geo_extent, _haversine_km + +# Keys that a non-empty result dict must always contain. +_EXPECTED_KEYS = { + "n_points", "bbox", "centroid", "span_km", + "by_region", "hemisphere", "note", +} + + +def test_nube_en_espana(): + """Golden: nube de puntos alrededor de Madrid -> region top = España.""" + # Cuatro puntos en torno a Madrid (lat ~40, lon ~-3.7), con algo de spread. + lats = [40.4, 40.0, 41.0, 39.5] + lons = [-3.7, -3.5, -4.0, -3.2] + res = analyze_geo_extent(lats, lons) + + assert set(res.keys()) == _EXPECTED_KEYS + assert res["n_points"] == 4 + + # Todos caen en España -> by_region una sola entrada. + assert res["by_region"][0]["region"] == "España" + assert res["by_region"][0]["count"] == 4 + + # Centroide coherente: media de lat y lon. + assert math.isclose(res["centroid"]["lat"], sum(lats) / 4, rel_tol=1e-9) + assert math.isclose(res["centroid"]["lon"], sum(lons) / 4, rel_tol=1e-9) + + # bbox correcto. + assert res["bbox"]["lat_min"] == 39.5 + assert res["bbox"]["lat_max"] == 41.0 + assert res["bbox"]["lon_min"] == -4.0 + assert res["bbox"]["lon_max"] == -3.2 + + # Hay spread -> diagonal > 0. + assert res["span_km"] > 0.0 + + # Hemisferio norte (lat>0) y oeste (lon<0). + assert res["hemisphere"]["north"] == 4 + assert res["hemisphere"]["south"] == 0 + assert res["hemisphere"]["east"] == 0 + assert res["hemisphere"]["west"] == 4 + + assert "España" in res["note"] + + +def test_dos_paises_distintos(): + """Golden: puntos en España y Francia -> by_region con 2 entradas.""" + # Madrid (España) x2 y Paris (Francia) x1. + lats = [40.4, 40.0, 48.8] + lons = [-3.7, -3.5, 2.3] + res = analyze_geo_extent(lats, lons) + + assert res["n_points"] == 3 + regions = {entry["region"]: entry["count"] for entry in res["by_region"]} + assert regions == {"España": 2, "Francia": 1} + # Orden descendente por count: España (2) antes que Francia (1). + assert res["by_region"][0]["region"] == "España" + assert res["by_region"][0]["count"] == 2 + + # Madrid y Paris ambos hemisferio norte; Paris lon>0 -> 1 east, 2 west. + assert res["hemisphere"]["north"] == 3 + assert res["hemisphere"]["east"] == 1 + assert res["hemisphere"]["west"] == 2 + + +def test_listas_vacias(): + """Edge: listas vacias -> n_points 0, bbox None, sin lanzar.""" + res = analyze_geo_extent([], []) + assert res["n_points"] == 0 + assert res["bbox"] is None + assert res["centroid"] is None + assert res["span_km"] == 0.0 + assert res["by_region"] == [] + assert res["hemisphere"] == {"north": 0, "south": 0, "east": 0, "west": 0} + assert res["note"] == "sin coordenadas validas" + + +def test_pares_invalidos_filtrados(): + """Edge: None / NaN / fuera de rango se descartan, no lanza.""" + nan = float("nan") + lats = [40.4, None, nan, 91.0, -200.0, 40.0] + lons = [-3.7, -3.5, -3.0, 2.0, 5.0, -3.5] + # Validos: indices 0 y 5 (lat 91 fuera de rango, lon -200 fuera de rango, + # None y NaN descartados). + res = analyze_geo_extent(lats, lons) + assert res["n_points"] == 2 + assert res["by_region"][0]["region"] == "España" + assert res["by_region"][0]["count"] == 2 + + +def test_longitudes_desbalanceadas(): + """Edge: len(lats) != len(lons) usa el minimo comun sin lanzar.""" + lats = [40.4, 40.0, 41.0, 39.5] # 4 elementos + lons = [-3.7, -3.5] # 2 elementos + res = analyze_geo_extent(lats, lons) + # Solo se emparejan los 2 primeros. + assert res["n_points"] == 2 + assert res["bbox"]["lat_min"] == 40.0 + assert res["bbox"]["lat_max"] == 40.4 + + +def test_span_km_haversine_par_conocido(): + """Edge: span_km coincide con haversine de la diagonal del bbox.""" + # Dos puntos: (0, 0) y (0, 1). bbox diagonal = mismos dos puntos. + res = analyze_geo_extent([0.0, 0.0], [0.0, 1.0]) + # 1 grado de longitud en el ecuador ~ 111.19 km. + expected = _haversine_km(0.0, 0.0, 0.0, 1.0) + assert math.isclose(res["span_km"], expected, rel_tol=1e-9) + assert math.isclose(res["span_km"], 111.19, abs_tol=0.5) + + +def test_no_lanza_con_entradas_raras(): + """Edge: tipos no-lista o None devuelven la forma vacia sin lanzar.""" + assert analyze_geo_extent(None, None)["n_points"] == 0 + assert analyze_geo_extent("foo", "bar")["n_points"] == 0 + # Strings dentro de las listas se descartan como invalidos. + res = analyze_geo_extent(["x", 40.0], [None, -3.5]) + assert res["n_points"] == 1 diff --git a/python/functions/datascience/build_geo_scatter.md b/python/functions/datascience/build_geo_scatter.md new file mode 100644 index 00000000..a90e4648 --- /dev/null +++ b/python/functions/datascience/build_geo_scatter.md @@ -0,0 +1,68 @@ +--- +name: build_geo_scatter +kind: function +lang: py +domain: datascience +version: "1.0.0" +purity: pure +signature: "def build_geo_scatter(lats: list, lons: list, max_points: int = 2000) -> dict" +description: "Prepara los datos de un scatter geografico en proyeccion equirectangular para el grupo eda. Empareja lats/lons por indice, descarta pares None/NaN/inf/bool o fuera de rango (lat en [-90,90], lon en [-180,180]) y aplica downsampling DETERMINISTA por paso fijo (pairs[::step]) cuando hay mas pares validos que max_points, para no saturar el PDF/PPTX en moviles. Devuelve los puntos en orden [lon, lat] listos para ax.scatter, el bbox, el aspect 1/cos(centroid_lat) clampado a [0.3,5.0] y un pad sugerido (~5% del rango con suelo minimo). Lectura defensiva; NUNCA lanza ni dibuja: el capitulo se encarga de matplotlib." +tags: [eda, geospatial, datascience, scatter, map, downsample, equirectangular, profiling] +params: + - name: lats + desc: "Lista (o tupla) de latitudes en grados, paralela a lons. Se empareja por indice. Un valor None, NaN, infinito, bool o fuera de [-90,90] descarta ese par. Lectura defensiva." + - name: lons + desc: "Lista (o tupla) de longitudes en grados, paralela a lats. Un valor None, NaN, infinito, bool o fuera de [-180,180] descarta ese par." + - name: max_points + desc: "Tope de puntos a devolver (default 2000). Si los pares validos superan el tope, se hace downsampling determinista por paso fijo step=ceil(n_total/max_points) tomando pairs[::step] (NO aleatorio, reproducible). Un valor no entero o <=0 desactiva el downsampling." +output: "Dict listo para dibujar: {points: [[lon, lat], ...] en orden x=lon/y=lat para ax.scatter; n_total: pares validos antes del downsample (int); n_shown: puntos devueltos tras el downsample (int); downsampled: bool (n_shown orden [x=lon, y=lat] +print(geo["bbox"]) # {'lat_min': 39.0, 'lat_max': 41.0, 'lon_min': -4.0, 'lon_max': -3.0} +print(round(geo["aspect"], 3)) # 1.308 -> ensancha el eje x en latitudes medias +print(geo["pad"]) # {'lon': 0.05, 'lat': 0.1} -> margen ~5% + +# El capitulo dibuja con matplotlib (esta funcion NO dibuja): +# xs = [p[0] for p in geo["points"]]; ys = [p[1] for p in geo["points"]] +# ax.scatter(xs, ys); ax.set_aspect(geo["aspect"]) +# ax.set_xlim(geo["bbox"]["lon_min"] - geo["pad"]["lon"], geo["bbox"]["lon_max"] + geo["pad"]["lon"]) +# ax.set_ylim(geo["bbox"]["lat_min"] - geo["pad"]["lat"], geo["bbox"]["lat_max"] + geo["pad"]["lat"]) +``` + +## Cuando usarla + +- Usala antes de dibujar un scatter geografico (mapa de puntos en proyeccion equirectangular) en el capitulo geospatial de `AutomaticEDA`: limpia los pares de coordenadas, los reduce a un tamano razonable para el PDF/PPTX y te da bbox, aspect y pad listos para fijar los ejes. +- Cuando tengas dos columnas de lat/lon ya extraidas y quieras un punto de entrada determinista (mismo dataset -> mismo dibujo) que no sature el documento en moviles. +- Cuando necesites el aspect correcto para que un grado de longitud no se vea estirado respecto a uno de latitud (integridad visual, Tufte) sin calcularlo a mano. + +## Gotchas + +- Funcion pura, sin I/O y determinista. NO dibuja: solo PREPARA los datos; el capitulo se encarga de matplotlib. Lectura defensiva: pares con None/NaN/inf/bool o coordenadas fuera de rango se descartan en silencio y NUNCA lanza. +- El downsampling es DETERMINISTA por paso fijo (`step = ceil(n_total / max_points)`, `pairs[::step]`), NO aleatorio: la misma entrada produce siempre la misma salida (reproducible en tests). El primer punto mostrado es siempre el primer par valido. No es un muestreo uniforme aleatorio — es un barrido regular del orden de entrada. +- `points` va en orden `[lon, lat]` (x, y), no `[lat, lon]`: pasalo directo a `ax.scatter(xs, ys)` sin invertir. Confundir el orden espeja el mapa. +- `aspect = 1/cos(centroid_lat)` se clampa a `[0.3, 5.0]`. En latitudes altas `cos -> 0` y el valor real explota: por encima de ~78 grados el aspect queda fijado en 5.0. Si el centroide cae justo en un polo (`+-90`) se usa el clamp en vez de dividir por cero. +- `pad` es ~5% del rango de cada eje con un suelo minimo de `0.01` grados: con un solo punto o todos iguales (rango 0) el pad cae al suelo para que el punto no quede en una linea. En el caso sin puntos validos el pad es `{lon:0.0, lat:0.0}` y `bbox` es `None`. +- `bbox`, `aspect` y `pad` se calculan sobre los puntos YA mostrados (tras el downsample), de modo que los ejes encajan exactamente con lo que se dibuja. diff --git a/python/functions/datascience/build_geo_scatter.py b/python/functions/datascience/build_geo_scatter.py new file mode 100644 index 00000000..858d815c --- /dev/null +++ b/python/functions/datascience/build_geo_scatter.py @@ -0,0 +1,153 @@ +"""build_geo_scatter — prepare points for a geographic scatter (EDA `geospatial`). + +Pure function: no I/O, deterministic. Takes two parallel lists of latitudes and +longitudes and returns the data a caller needs to draw a geographic scatter in an +equirectangular projection: cleaned points in [lon, lat] order, a bounding box, a +projection aspect ratio and a suggested axis padding. + +It NEVER draws anything (no matplotlib) — the chapter that consumes this output is +responsible for the rendering. Reading is defensive throughout and the function +NEVER raises: malformed pairs (None, NaN, infinity or out-of-range coordinates) +are silently dropped and an empty/valid result is always returned. + +To keep the rendered PDF/PPTX light on phones, when the number of valid pairs +exceeds `max_points` the points are down-sampled DETERMINISTICALLY by a fixed +step (`pairs[::step]`), never randomly, so the result is reproducible. +""" + +import math + +# Minimum axis padding (in degrees) so a single point or a zero-range cloud is +# never drawn glued to the axis border (it would collapse to a line). +_MIN_PAD = 0.01 + +# Aspect ratio clamp. 1/cos(lat) blows up near the poles; clamp keeps the render +# sane (Tufte: do not let the projection stretch the cloud out of proportion). +_ASPECT_MIN = 0.3 +_ASPECT_MAX = 5.0 + + +def _coord(value): + """Coerce to a finite float defensively; return None for invalid coordinates. + + bool is a subclass of int, but a real latitude/longitude is never a bool, so + True/False are treated as missing instead of coercing to 1.0/0.0. NaN and + +/-infinity are never valid coordinates either. + """ + if value is None or isinstance(value, bool): + return None + try: + coord = float(value) + except (TypeError, ValueError): + return None + if math.isnan(coord) or math.isinf(coord): + return None + return coord + + +def build_geo_scatter(lats: list, lons: list, max_points: int = 2000) -> dict: + """Prepare the data for a geographic scatter in equirectangular projection. + + Pairs `lats` and `lons` by index, drops invalid pairs, optionally + down-samples deterministically, and derives the geometry (bbox, aspect, pad) + a caller needs to draw the cloud. No raw rendering is performed. + + Args: + lats: List (or tuple) of latitudes in degrees. Paired by index with + `lons`. A value that is None, NaN, infinite, bool or outside + [-90, 90] discards that pair. Read defensively. + lons: List (or tuple) of longitudes in degrees, parallel to `lats`. A + value outside [-180, 180] (or None/NaN/inf/bool) discards that pair. + max_points: Cap on the number of points returned. When the number of + valid pairs exceeds this cap, the points are down-sampled by a fixed + step `ceil(n_total / max_points)` taking `pairs[::step]` — DETERMINISTIC, + not random, so the output is reproducible. A non-positive or non-int + value disables down-sampling. + + Returns: + Dict ready for a caller's ax.scatter: + {points: [[lon, lat], ...] (x=lon, y=lat order), n_total: valid pairs + before down-sampling, n_shown: points returned, downsampled: bool, + bbox: {lat_min, lat_max, lon_min, lon_max} or None, aspect: 1/cos(centroid + lat) clamped to [0.3, 5.0], pad: {lon, lat} ~5% of each range with a small + floor}. When there are no valid pairs returns points=[], n_total=0, + n_shown=0, downsampled=False, bbox=None, aspect=1.0, pad={lon:0.0, lat:0.0}. + """ + pairs = [] # each item is (lon, lat) — already in [x, y] order + if isinstance(lats, (list, tuple)) and isinstance(lons, (list, tuple)): + n = min(len(lats), len(lons)) + for i in range(n): + lat = _coord(lats[i]) + lon = _coord(lons[i]) + if lat is None or lon is None: + continue + if lat < -90.0 or lat > 90.0: + continue + if lon < -180.0 or lon > 180.0: + continue + pairs.append((lon, lat)) + + n_total = len(pairs) + if n_total == 0: + return { + "points": [], + "n_total": 0, + "n_shown": 0, + "downsampled": False, + "bbox": None, + "aspect": 1.0, + "pad": {"lon": 0.0, "lat": 0.0}, + } + + # Deterministic down-sampling by a fixed step. Reproducible: same input -> + # same output, no randomness. + if ( + isinstance(max_points, int) + and not isinstance(max_points, bool) + and max_points > 0 + and n_total > max_points + ): + step = math.ceil(n_total / max_points) + sampled = pairs[::step] + else: + sampled = pairs + + points = [[lon, lat] for (lon, lat) in sampled] + n_shown = len(points) + downsampled = n_shown < n_total + + lons_s = [p[0] for p in sampled] + lats_s = [p[1] for p in sampled] + lon_min, lon_max = min(lons_s), max(lons_s) + lat_min, lat_max = min(lats_s), max(lats_s) + bbox = { + "lat_min": lat_min, + "lat_max": lat_max, + "lon_min": lon_min, + "lon_max": lon_max, + } + + # Aspect for an equirectangular projection: stretch the x axis by 1/cos(lat) + # at the cloud centroid so a degree of longitude reads at its real width. + centroid_lat = sum(lats_s) / len(lats_s) + cos_lat = math.cos(math.radians(centroid_lat)) + if cos_lat < 1e-12: # centroid at (or numerically at) a pole + aspect = _ASPECT_MAX + else: + aspect = 1.0 / cos_lat + aspect = max(_ASPECT_MIN, min(_ASPECT_MAX, aspect)) + + # Padding ~5% of each range, with a small floor so a zero-range cloud (single + # point / all identical) still gets a non-zero margin. + pad_lon = max(0.05 * (lon_max - lon_min), _MIN_PAD) + pad_lat = max(0.05 * (lat_max - lat_min), _MIN_PAD) + + return { + "points": points, + "n_total": n_total, + "n_shown": n_shown, + "downsampled": downsampled, + "bbox": bbox, + "aspect": aspect, + "pad": {"lon": pad_lon, "lat": pad_lat}, + } diff --git a/python/functions/datascience/build_geo_scatter_test.py b/python/functions/datascience/build_geo_scatter_test.py new file mode 100644 index 00000000..a9f318dd --- /dev/null +++ b/python/functions/datascience/build_geo_scatter_test.py @@ -0,0 +1,140 @@ +"""Tests para build_geo_scatter.""" + +import math +import os +import sys + +sys.path.insert(0, os.path.dirname(__file__)) + +from build_geo_scatter import build_geo_scatter + +# Keys that a non-empty result dict must always contain. +_EXPECTED_KEYS = { + "points", "n_total", "n_shown", "downsampled", "bbox", "aspect", "pad", +} + + +def test_geo_scatter_nube_espana(): + """Golden: nube en Espana -> points en orden [lon, lat], bbox, aspect>1, pad 5%.""" + # Cuatro puntos alrededor de Madrid (lat ~40, lon negativo). + lats = [40.0, 41.0, 39.0, 40.5] + lons = [-3.7, -3.0, -4.0, -3.5] + r = build_geo_scatter(lats, lons) + + assert set(r.keys()) == _EXPECTED_KEYS + + # points en orden [x=lon, y=lat]: primer elemento lon (negativo), segundo lat (~40). + assert r["points"] == [[-3.7, 40.0], [-3.0, 41.0], [-4.0, 39.0], [-3.5, 40.5]] + for lon, lat in r["points"]: + assert lon < 0.0 # longitudes de Espana son negativas + assert 36.0 < lat < 44.0 # latitudes peninsulares + + # Sin downsampling: 4 < 2000. + assert r["n_total"] == 4 + assert r["n_shown"] == 4 + assert r["downsampled"] is False + + # bbox correcto. + assert r["bbox"] == { + "lat_min": 39.0, "lat_max": 41.0, + "lon_min": -4.0, "lon_max": -3.0, + } + + # aspect = 1/cos(centroid_lat); centroid = 40.125 -> ~1.31 > 1. + centroid_lat = (40.0 + 41.0 + 39.0 + 40.5) / 4.0 + expected_aspect = 1.0 / math.cos(math.radians(centroid_lat)) + assert r["aspect"] > 1.0 + assert abs(r["aspect"] - expected_aspect) < 1e-9 + assert abs(r["aspect"] - 1.305) < 0.02 # cos(40) ~ 0.77 + + # pad 5% del rango (lon_range=1.0 -> 0.05 ; lat_range=2.0 -> 0.1). + assert abs(r["pad"]["lon"] - 0.05) < 1e-9 + assert abs(r["pad"]["lat"] - 0.10) < 1e-9 + + +def test_downsampling_determinista_y_reproducible(): + """Golden: 5000 puntos, max_points=2000 -> n_shown<=2000, downsampled, reproducible.""" + lats = [40.0 + (i % 100) * 0.01 for i in range(5000)] + lons = [-3.0 - (i % 100) * 0.01 for i in range(5000)] + + r1 = build_geo_scatter(lats, lons, max_points=2000) + + assert r1["n_total"] == 5000 + assert r1["n_shown"] <= 2000 + assert r1["downsampled"] is True + # step = ceil(5000/2000) = 3 -> len(pairs[::3]) = 1667. + assert r1["n_shown"] == 1667 + + # Determinista: dos llamadas con la misma entrada dan exactamente lo mismo. + r2 = build_geo_scatter(lats, lons, max_points=2000) + assert r1 == r2 + assert r1["points"] == r2["points"] + + # El primer punto del downsample es el primer par valido (step parte de 0). + assert r1["points"][0] == [lons[0], lats[0]] + + +def test_listas_vacias_no_lanza(): + """Edge: listas vacias / None -> points [] sin lanzar.""" + r = build_geo_scatter([], []) + assert r["points"] == [] + assert r["n_total"] == 0 + assert r["n_shown"] == 0 + assert r["downsampled"] is False + assert r["bbox"] is None + assert r["aspect"] == 1.0 + assert r["pad"] == {"lon": 0.0, "lat": 0.0} + + # None como entrada tampoco lanza. + assert build_geo_scatter(None, None)["points"] == [] + assert build_geo_scatter([40.0], None)["n_total"] == 0 + assert build_geo_scatter(None, [-3.0])["n_total"] == 0 + + +def test_un_solo_punto_pad_minimo_y_aspect_finito(): + """Edge: un solo punto -> pad minimo no cero, bbox degenerado, aspect finito.""" + r = build_geo_scatter([40.0], [-3.7]) + + assert r["n_total"] == 1 + assert r["n_shown"] == 1 + assert r["points"] == [[-3.7, 40.0]] + assert r["downsampled"] is False + assert r["bbox"] == { + "lat_min": 40.0, "lat_max": 40.0, + "lon_min": -3.7, "lon_max": -3.7, + } + # rango 0 -> pad cae al floor minimo (no cero). + assert r["pad"]["lon"] == 0.01 + assert r["pad"]["lat"] == 0.01 + # aspect finito y dentro del clamp. + assert math.isfinite(r["aspect"]) + assert 0.3 <= r["aspect"] <= 5.0 + + +def test_filtra_none_nan_y_fuera_de_rango(): + """Edge: pares con None/NaN/fuera de rango se descartan por indice.""" + nan = float("nan") + inf = float("inf") + # i=0 i=1 i=2 i=3 i=4 i=5 i=6 + lats = [40.0, None, nan, 200.0, 41.0, 39.0, inf] + lons = [-3.0, -3.5, -3.6, -3.7, 999.0, -4.0, -2.0] + r = build_geo_scatter(lats, lons) + + # Validos solo i=0 (40,-3.0) e i=5 (39,-4.0): + # i=1 lat None, i=2 lat NaN, i=3 lat 200 fuera de rango, + # i=4 lon 999 fuera de rango, i=6 lat inf. + assert r["n_total"] == 2 + assert r["points"] == [[-3.0, 40.0], [-4.0, 39.0]] + assert r["bbox"] == { + "lat_min": 39.0, "lat_max": 40.0, + "lon_min": -4.0, "lon_max": -3.0, + } + + +def test_latitud_alta_aspect_clamped(): + """Edge: latitudes ~85 -> aspect clamped <= 5.0.""" + r = build_geo_scatter([85.0, 85.0, 84.0], [10.0, 11.0, 9.0]) + # cos(~84.7) ~ 0.093 -> 1/0.093 ~ 10.7 -> clamp a 5.0. + assert r["aspect"] <= 5.0 + assert r["aspect"] == 5.0 + assert math.isfinite(r["aspect"]) diff --git a/python/functions/datascience/detect_latlon_columns.md b/python/functions/datascience/detect_latlon_columns.md new file mode 100644 index 00000000..0fb08e20 --- /dev/null +++ b/python/functions/datascience/detect_latlon_columns.md @@ -0,0 +1,67 @@ +--- +name: detect_latlon_columns +id: detect_latlon_columns_py_datascience +kind: function +lang: py +domain: datascience +version: "1.0.0" +purity: pure +signature: "def detect_latlon_columns(columns: list, samples: dict | None = None) -> dict" +description: "Detecta un par (latitud, longitud) entre las columnas de un TableProfile del grupo eda combinando heuristica de nombre (latitude/longitude/lat/lon/lng + x/y debiles) con validacion de rango obligatoria (latitud en [-90,90], longitud en [-180,180]). Lee defensivamente con .get; NUNCA lanza. Usa el sub-bloque numeric.min/max o, si falta, la lista de samples opcional. Devuelve SIEMPRE un dict {lat_col, lon_col, confidence, reason}; si no hay par valido, las columnas van a None y confidence a 0.0." +tags: [eda, geospatial, profiling, latlon, coordinates, detection, datascience] +params: + - name: columns + desc: "Lista de dicts ColumnProfile (el campo `columns` de un TableProfile del grupo eda). Cada dict se lee con .get; solo `name` (str) es obligatorio. Se consultan `inferred_type` (p.ej. 'numeric') y el sub-dict `numeric` con `min`/`max` (floats) para validar el rango. Entradas no-dict o sin name se ignoran sin lanzar." + - name: samples + desc: "Opcional {nombre_columna: [valores...]} para validar el rango cuando una columna no trae numeric.min/max. Los valores nulos se ignoran; si algun valor no nulo no es numerico la columna no se considera coordenada. Si es None u omitido, solo se usa el bloque numeric." +output: "Dict SIEMPRE presente con la forma {lat_col: str|None, lon_col: str|None, confidence: float en [0,1], reason: str en espanol}. En exito, lat_col y lon_col nombran columnas distintas; confidence ~1.0 para par con nombre fuerte (latitude/longitude/lat/lon/lng) + rango valido y ~0.7 para par debil (x/y) + rango. En fallo, ambas columnas None, confidence 0.0 y reason explica por que (sin columnas, nombre sin match, rango fuera de bounds, falta uno de los dos ejes...)." +uses_functions: [] +uses_types: [] +returns: [] +returns_optional: false +error_type: "" +imports: [] +tested: true +tests: ["test_par_latitude_longitude_fuerte", "test_par_lat_lon_abreviado", "test_par_x_y_debil_con_rango_valido", "test_nombre_lat_lon_pero_rango_fuera_no_detecta", "test_par_fuerte_prevalece_sobre_debil", "test_entradas_vacias_o_invalidas_no_lanzan", "test_solo_latitud_sin_longitud_no_detecta", "test_deteccion_por_samples_cuando_falta_numeric", "test_samples_fuera_de_rango_descarta"] +test_file_path: "python/functions/datascience/detect_latlon_columns_test.py" +file_path: "python/functions/datascience/detect_latlon_columns.py" +--- + +## Ejemplo + +```python +import sys, os +sys.path.insert(0, os.path.join("python", "functions")) +from datascience.detect_latlon_columns import detect_latlon_columns + +# Columnas tal y como vienen en profile['columns'] de un TableProfile del grupo eda: +columns = [ + {"name": "id", "inferred_type": "numeric", "numeric": {"min": 1, "max": 9999}}, + {"name": "latitude", "inferred_type": "numeric", "numeric": {"min": -45.0, "max": 45.0}}, + {"name": "longitude", "inferred_type": "numeric", "numeric": {"min": -120.0, "max": 120.0}}, +] +res = detect_latlon_columns(columns) +print(res["lat_col"], res["lon_col"], res["confidence"]) +# latitude longitude 1.0 + +# Sin bloque numeric, validando el rango con samples: +cols2 = [{"name": "lat"}, {"name": "lon"}] +samples = {"lat": [10.5, 20.0, 30.25], "lon": [-40.0, 50.5, 60.0]} +print(detect_latlon_columns(cols2, samples)["lat_col"]) # lat +``` + +## Cuando usarla + +- Usala al perfilar una tabla en `AutomaticEDA` para decidir si tiene geometria de puntos: cuando `detect_latlon_columns` devuelve un par con `confidence` alta, el capitulo geospatial puede dibujar un mapa, calcular un bounding box o proponer un cluster espacial. +- Antes de un analisis geoespacial (alpha shape, convex hull, joins por proximidad) para localizar automaticamente que columnas son la latitud y la longitud sin pedirlo al usuario. +- Cuando recibas un `TableProfile` del grupo `eda` y quieras enrutar columnas a sub-analisis por tipo semantico: este es el detector del par lat/lon, complementario a `infer_semantic_type`. + +## Gotchas + +- Funcion pura, sin I/O y determinista. Lectura defensiva con `.get`: NUNCA lanza. Cualquier input malformado (None, no-lista, entradas no-dict, claves ausentes) devuelve el dict de fallo con `lat_col`/`lon_col` en None y `confidence` 0.0. +- **El nombre solo no basta**: una columna `latitude` cuyo rango se sale de `[-90, 90]` se descarta (no es coordenada real). Igual para `longitude` fuera de `[-180, 180]`. La validacion de rango es obligatoria. +- El rango de latitud `[-90, 90]` es un subconjunto del de longitud `[-180, 180]`, por eso el nombre es necesario para desambiguar cual eje es cual; una columna numerica en `[-90, 90]` sin nombre que sugiera lat/lon no se detecta. +- Los nombres genericos `x`/`y` (y `x_coord`/`y_coord`) son candidatos **debiles**: solo forman par si el rango encaja y existe la otra mitad (un `x`/`lon` para la `y`, un `y`/`lat` para la `x`). Un `y` suelto sin pareja devuelve None. +- Requiere AMBOS ejes para considerar exito. Si solo encuentra latitud o solo longitud, devuelve el dict de fallo (no media coordenada). +- `samples` solo se consulta cuando falta `numeric.min`/`numeric.max`. Si una columna trae el bloque numeric, ese manda aunque pases samples para ella. +- El matching de nombre es por subcadena normalizada (se quitan `_`, `-` y espacios), asi que nombres como `plate` (contiene "lat") podrian marcarse como candidatos por nombre — pero solo pasarian si su rango cae en `[-90, 90]` y hay una longitud pareja, filtro que en la practica descarta los falsos positivos. diff --git a/python/functions/datascience/detect_latlon_columns.py b/python/functions/datascience/detect_latlon_columns.py new file mode 100644 index 00000000..a9323985 --- /dev/null +++ b/python/functions/datascience/detect_latlon_columns.py @@ -0,0 +1,198 @@ +"""detect_latlon_columns — detect a (latitude, longitude) column pair in an EDA profile. + +Pure function: no I/O, deterministic. Takes the `columns` list of a TableProfile +(group `eda`) and decides whether two of its columns form a geographic coordinate +pair (latitude + longitude), combining a name heuristic with a value-range check. + +The detection is intentionally conservative: a name hint alone is never enough. A +column is only accepted as latitude/longitude if its numeric range fits inside the +valid coordinate bounds ([-90, 90] for latitude, [-180, 180] for longitude). When +the `numeric` sub-block is absent the optional `samples` argument is used instead. + +Reading is fully defensive (.get throughout) and the function NEVER raises: any +malformed input (None, non-list, non-dict entries, missing keys) simply yields a +no-pair result {"lat_col": None, "lon_col": None, "confidence": 0.0, "reason": ...}. +""" + +import re + +# Collapse the separators a column name may use (snake_case, kebab-case, spaces) +# so that "y_coord", "y-coord" and "y coord" all normalize to the same token. +_SEP_RE = re.compile(r"[\s_\-]+") + +# Name-match strengths: a strong, unambiguous coordinate name vs a weak generic +# axis name (x / y) that only counts when the range also fits and a partner exists. +_STRONG = 0.6 +_WEAK = 0.3 +_RANGE_BONUS = 0.4 # added once the mandatory range validation passes + + +def _normalize(name): + """Lowercase a column name and strip separator chars (_, -, whitespace).""" + if not isinstance(name, str): + return "" + return _SEP_RE.sub("", name.strip().lower()) + + +def _num(value): + """Coerce to float defensively; return None for None/bool/non-numeric.""" + # bool is a subclass of int; a coordinate value is never a real bool, so treat + # True/False as missing instead of silently coercing to 1.0/0.0. + if value is None or isinstance(value, bool): + return None + try: + return float(value) + except (TypeError, ValueError): + return None + + +def _lat_name_strength(nn): + """Strength of a normalized name as a latitude candidate (0=no match).""" + if not nn: + return 0.0 + # "lat", "latitude", "latitud" all contain the "lat" stem. + if "lat" in nn: + return _STRONG + # Weak generic axis name: only useful when paired with an x/lon partner. + if nn in ("y", "ycoord", "ycoordinate", "ycoordinates"): + return _WEAK + return 0.0 + + +def _lon_name_strength(nn): + """Strength of a normalized name as a longitude candidate (0=no match).""" + if not nn: + return 0.0 + # "lon", "long", "longitude", "longitud" share the "lon" stem; "lng" is separate. + if "lon" in nn or "lng" in nn: + return _STRONG + if nn in ("x", "xcoord", "xcoordinate", "xcoordinates"): + return _WEAK + return 0.0 + + +def _col_range(col, sample_values): + """Return (min, max) floats for a column, or (None, None) if not numeric. + + Prefers the `numeric` sub-block min/max (the output of describe_numeric); falls + back to the provided sample list. A column is only treated as numeric when both + extremes are derivable: from the numeric block, or from samples whose every + non-null value coerces to a number. + """ + if isinstance(col, dict): + numeric = col.get("numeric") + if isinstance(numeric, dict): + mn = _num(numeric.get("min")) + mx = _num(numeric.get("max")) + if mn is not None and mx is not None: + return mn, mx + # Fall back to samples when the numeric block is missing or incomplete. + if isinstance(sample_values, (list, tuple)): + non_null = [v for v in sample_values if v is not None] + if non_null: + coerced = [_num(v) for v in non_null] + # Any non-numeric sample means we cannot trust the column as numeric. + if all(c is not None for c in coerced): + return min(coerced), max(coerced) + return None, None + + +def _no_pair(reason): + """Canonical empty result: no coordinate pair detected.""" + return {"lat_col": None, "lon_col": None, "confidence": 0.0, "reason": reason} + + +def detect_latlon_columns(columns: list, samples: dict | None = None) -> dict: + """Detect a (latitude, longitude) column pair from an eda TableProfile. + + Combines a name heuristic (latitude/longitude/lat/lon/lng + weak x/y) with a + mandatory range validation: the chosen latitude must sit in [-90, 90] and the + longitude in [-180, 180]. A name hint whose range does not fit is discarded. + Both sides are required for success; if only one is found, no pair is returned. + + Args: + columns: List of ColumnProfile dicts (the `columns` of a TableProfile). + Each dict is read defensively with .get; only `name` is required. + `numeric.min` / `numeric.max` (and optionally `inferred_type`) are used + for the range check when present. + samples: Optional {column_name: [values...]} used to validate the range + when a column lacks `numeric.min`/`numeric.max`. If None/omitted, only + the `numeric` sub-block is consulted. + + Returns: + Always a dict {"lat_col": str|None, "lon_col": str|None, + "confidence": float, "reason": str}. On success lat_col and lon_col name + the detected pair (distinct columns) and confidence is in [0, 1]: a pair + validated by a strong name on both sides scores ~1.0, a weak x/y pair ~0.7. + On failure both columns are None and confidence is 0.0. + """ + if not isinstance(columns, (list, tuple)) or len(columns) == 0: + return _no_pair("sin columnas que inspeccionar") + + sample_map = samples if isinstance(samples, dict) else {} + + # (column_name, confidence) for each side. Confidence already includes the + # range bonus because membership in the list implies the range was validated. + lat_candidates = [] + lon_candidates = [] + + for col in columns: + if not isinstance(col, dict): + continue + name = col.get("name") + if not isinstance(name, str) or not name: + continue + + nn = _normalize(name) + lat_strength = _lat_name_strength(nn) + lon_strength = _lon_name_strength(nn) + if lat_strength == 0.0 and lon_strength == 0.0: + continue # name gives no coordinate hint; skip. + + mn, mx = _col_range(col, sample_map.get(name)) + is_numeric = mn is not None and mx is not None + if not is_numeric: + continue # range cannot be validated -> not a coordinate. + + if lat_strength > 0.0 and mn >= -90.0 and mx <= 90.0: + lat_candidates.append((name, lat_strength + _RANGE_BONUS)) + if lon_strength > 0.0 and mn >= -180.0 and mx <= 180.0: + lon_candidates.append((name, lon_strength + _RANGE_BONUS)) + + if not lat_candidates and not lon_candidates: + return _no_pair("ninguna columna sugiere latitud ni longitud por nombre+rango") + if not lat_candidates: + return _no_pair("no se encontro columna de latitud valida (nombre+rango en [-90,90])") + if not lon_candidates: + return _no_pair("no se encontro columna de longitud valida (nombre+rango en [-180,180])") + + # Pick the distinct pair with the highest combined confidence. First match wins + # on ties to keep the result deterministic by input order. + best = None # (combined, lat_name, lon_name, lat_c, lon_c) + for lat_name, lat_c in lat_candidates: + for lon_name, lon_c in lon_candidates: + if lat_name == lon_name: + continue # a column cannot be both axes of the same pair. + combined = (lat_c + lon_c) / 2.0 + if best is None or combined > best[0]: + best = (combined, lat_name, lon_name, lat_c, lon_c) + + if best is None: + return _no_pair("solo una columna sirve para ambos ejes; no hay par lat/lon distinto") + + combined, lat_name, lon_name, lat_c, lon_c = best + confidence = max(0.0, min(1.0, combined)) + + lat_label = "fuerte" if lat_c >= 0.9 else "debil" + lon_label = "fuerte" if lon_c >= 0.9 else "debil" + reason = ( + f"par lat='{lat_name}' (nombre {lat_label}) / lon='{lon_name}' " + f"(nombre {lon_label}) con rango valido" + ) + + return { + "lat_col": lat_name, + "lon_col": lon_name, + "confidence": confidence, + "reason": reason, + } diff --git a/python/functions/datascience/detect_latlon_columns_test.py b/python/functions/datascience/detect_latlon_columns_test.py new file mode 100644 index 00000000..6a64f708 --- /dev/null +++ b/python/functions/datascience/detect_latlon_columns_test.py @@ -0,0 +1,141 @@ +"""Tests para detect_latlon_columns.""" + +import os +import sys + +sys.path.insert(0, os.path.dirname(__file__)) + +from detect_latlon_columns import detect_latlon_columns + +# Keys that every result dict (success or failure) must expose. +_EXPECTED_KEYS = {"lat_col", "lon_col", "confidence", "reason"} + + +def _col(name, mn=None, mx=None, inferred="numeric"): + """Build a minimal ColumnProfile-like dict for the tests.""" + col = {"name": name, "inferred_type": inferred} + if mn is not None or mx is not None: + col["numeric"] = {"min": mn, "max": mx} + return col + + +def test_par_latitude_longitude_fuerte(): + """Golden: nombres latitude/longitude con rango valido -> par con confianza alta.""" + columns = [ + _col("id", mn=1, mx=9999, inferred="numeric"), + _col("latitude", mn=-45.0, mx=45.0), + _col("longitude", mn=-120.0, mx=120.0), + ] + res = detect_latlon_columns(columns) + + assert set(res.keys()) == _EXPECTED_KEYS + assert res["lat_col"] == "latitude" + assert res["lon_col"] == "longitude" + # Nombre fuerte (0.6) + rango (0.4) en ambos lados -> 1.0. + assert abs(res["confidence"] - 1.0) < 1e-9 + assert "rango valido" in res["reason"] + + +def test_par_lat_lon_abreviado(): + """Golden: nombres abreviados lat/lon tambien se detectan como fuertes.""" + columns = [ + _col("lat", mn=40.0, mx=43.0), + _col("lon", mn=-4.0, mx=-1.0), + _col("precio", mn=0.0, mx=500.0), + ] + res = detect_latlon_columns(columns) + assert res["lat_col"] == "lat" + assert res["lon_col"] == "lon" + assert abs(res["confidence"] - 1.0) < 1e-9 + + +def test_par_x_y_debil_con_rango_valido(): + """Edge: x/y genericos solo cuentan como par debil cuando el rango encaja.""" + columns = [ + _col("y_coord", mn=-10.0, mx=10.0), # debil latitud + _col("x_coord", mn=-150.0, mx=150.0), # debil longitud + ] + res = detect_latlon_columns(columns) + assert res["lat_col"] == "y_coord" + assert res["lon_col"] == "x_coord" + # Nombre debil (0.3) + rango (0.4) -> 0.7 en ambos lados. + assert abs(res["confidence"] - 0.7) < 1e-9 + + +def test_nombre_lat_lon_pero_rango_fuera_no_detecta(): + """Edge: nombre lat/lon con rango fuera de bounds -> NO es coordenada.""" + columns = [ + _col("latitude", mn=-200.0, mx=200.0), # fuera de [-90, 90] + _col("longitude", mn=-120.0, mx=120.0), # valido, pero sin par lat + ] + res = detect_latlon_columns(columns) + assert res["lat_col"] is None + assert res["lon_col"] is None + assert res["confidence"] == 0.0 + assert isinstance(res["reason"], str) and res["reason"] + + +def test_par_fuerte_prevalece_sobre_debil(): + """Edge: con candidatos fuertes y debiles, gana el par de mayor confianza.""" + columns = [ + _col("latitude", mn=-45.0, mx=45.0), # fuerte lat + _col("y", mn=-30.0, mx=30.0), # debil lat + _col("longitude", mn=-120.0, mx=120.0), # fuerte lon + _col("x", mn=-100.0, mx=100.0), # debil lon + ] + res = detect_latlon_columns(columns) + assert res["lat_col"] == "latitude" + assert res["lon_col"] == "longitude" + assert abs(res["confidence"] - 1.0) < 1e-9 + + +def test_entradas_vacias_o_invalidas_no_lanzan(): + """Edge: sin columnas / vacio / no-lista / entradas no-dict -> dict None sin lanzar.""" + for bad in ([], None, "no soy lista", 42, [1, 2, 3], [{}], [{"foo": "bar"}]): + res = detect_latlon_columns(bad) + assert set(res.keys()) == _EXPECTED_KEYS + assert res["lat_col"] is None + assert res["lon_col"] is None + assert res["confidence"] == 0.0 + assert isinstance(res["reason"], str) + + +def test_solo_latitud_sin_longitud_no_detecta(): + """Edge: solo hay latitud valida, falta la longitud -> sin par.""" + columns = [ + _col("latitude", mn=-45.0, mx=45.0), + _col("temperatura", mn=-5.0, mx=40.0), + ] + res = detect_latlon_columns(columns) + assert res["lat_col"] is None + assert res["lon_col"] is None + assert res["confidence"] == 0.0 + + +def test_deteccion_por_samples_cuando_falta_numeric(): + """Edge: sin bloque numeric, el rango se valida con samples.""" + columns = [ + {"name": "lat"}, # sin numeric ni inferred_type + {"name": "lon"}, + ] + samples = { + "lat": [10.5, 20.0, None, 30.25], # todos dentro de [-90, 90] + "lon": [-40.0, 50.5, 60.0], # todos dentro de [-180, 180] + } + res = detect_latlon_columns(columns, samples) + assert res["lat_col"] == "lat" + assert res["lon_col"] == "lon" + assert abs(res["confidence"] - 1.0) < 1e-9 + + +def test_samples_fuera_de_rango_descarta(): + """Edge: samples fuera de bounds invalidan la columna pese al nombre fuerte.""" + columns = [{"name": "lat"}, {"name": "lon"}] + samples = { + "lat": [10.0, 95.0], # 95 > 90 -> latitud invalida + "lon": [-40.0, 50.0], + } + res = detect_latlon_columns(columns, samples) + assert res["lat_col"] is None + assert res["lon_col"] is None + assert res["confidence"] == 0.0 From 00cd5274bc0a6bffffcbdeeb3c15ee775e526cbe Mon Sep 17 00:00:00 2001 From: Egutierrez Date: Tue, 30 Jun 2026 15:29:33 +0200 Subject: [PATCH 14/53] =?UTF-8?q?feat(eda):=20cap=C3=ADtulo=20GEOSPATIAL?= =?UTF-8?q?=20del=20AutomaticEDA=20(scatter=20geogr=C3=A1fico=20+=20zona/p?= =?UTF-8?q?a=C3=ADs)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Capítulo nuevo chapters/geospatial.py (CHAPTER_VERSION 1.0.0). Cuando el dataset tiene un par de coordenadas, dibuja un scatter geográfico en proyección equirectangular (la escala respeta la latitud para no estirar la longitud) y analiza la extensión: bounding box, centroide, span, conteo por zona/país, hemisferios y una interpretación. Cuando NO hay coordenadas, build_geospatial devuelve None y el capítulo se omite. Sigue el contrato de capítulos (firma build_(profile, ctx) -> Chapter|None, lectura defensiva, nunca lanza) y el patrón de modelos/num_distr: delega el cálculo a las primitivas puras del registry (detect_latlon_columns, analyze_geo_extent, build_geo_scatter) y solo dibuja la figura matplotlib de forma perezosa. Las coordenadas crudas llegan por ctx['geo_points'] o ctx['raw_numeric'] (como modelos lee raw_numeric); sin ellas, degrada con un bounding box aproximado de numeric.min/max y una nota honesta. Anti-cortes: usa DataTable/KVTable/Figure/Markdown del modelo, que el paginador parte sin cortar. Test self-contained con golden + 6 edges + anti-cut (nombres largos + 2100 puntos en varias regiones renderizan a PDF y PPTX sin truncar). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../automatic_eda/chapters/geospatial.py | 477 ++++++++++++++++++ .../automatic_eda/chapters/geospatial_test.py | 245 +++++++++ 2 files changed, 722 insertions(+) create mode 100644 python/functions/datascience/automatic_eda/chapters/geospatial.py create mode 100644 python/functions/datascience/automatic_eda/chapters/geospatial_test.py diff --git a/python/functions/datascience/automatic_eda/chapters/geospatial.py b/python/functions/datascience/automatic_eda/chapters/geospatial.py new file mode 100644 index 00000000..d60830cd --- /dev/null +++ b/python/functions/datascience/automatic_eda/chapters/geospatial.py @@ -0,0 +1,477 @@ +"""Geospatial chapter (GEOSPATIAL) for AutomaticEDA. + +When the dataset carries a coordinate pair (latitude/longitude), this chapter +draws the points on a **geographic scatter** in an equirectangular projection +(scaled so degrees of longitude are not stretched at the data's latitude) and +analyses the **zone / country** the points fall in: bounding box, centroid, +geographic span, and a per-region count. When there is **no** coordinate pair the +chapter returns ``None`` — exactly the user requirement. + +Detection and the heavy lifting are delegated to pure ``eda``-group registry +functions, never reimplemented here: + +- ``detect_latlon_columns`` — finds the (lat, lon) column pair by name + value + range from the ``profile['columns']`` metadata. +- ``analyze_geo_extent`` — bbox, centroid, haversine span, per-region counts and + hemisphere from the raw coordinate arrays. +- ``build_geo_scatter`` — deterministically down-sampled points + bbox + the + aspect ratio for the equirectangular projection. This chapter only draws the + matplotlib figure from that prepared data (same split as ``num_distr`` does + with ``build_boxplot_stats``). + +The raw coordinate arrays are **not** in a standard TableProfile (it stores only +per-column aggregates), so — exactly like ``modelos`` reads ``raw_numeric`` from +``ctx`` — this chapter looks for the coordinates in ``ctx`` (or ``profile``) and +degrades honestly when they are absent: it still detects the columns and shows an +approximate bounding box derived from the per-column ``numeric.min/max``, with a +note that the raw points are needed for the map. + +ctx keys this chapter consumes (all optional): + geo_points : dict — ``{"lats": [...], "lons": [...]}`` raw coordinate arrays. + Used directly when present (forward-compatible with a calculation phase + that samples them from the table). + raw_numeric : dict — ``{col: [values]}`` raw numeric columns; when present + and ``geo_points`` is not, the detected lat/lon columns are read from it. + run_geo_llm : bool — when True, call ``ask_llm`` for a one-line narrative of + where the points concentrate (otherwise a derived note is used). + geo_llm_model : str — model id for the optional live LLM call. + +Contract: build_(profile, ctx) -> Chapter | None ; CHAPTER_VERSION = "x.y.z". +Reads everything defensively (``.get``) and never raises. +""" + +from __future__ import annotations + +import math + +from .. import model + +# Pure registry functions (group ``eda``) delegated to. Imported defensively so +# the chapter stays importable (degrading gracefully) if one is unavailable. +try: + from datascience.detect_latlon_columns import detect_latlon_columns +except Exception: # noqa: BLE001 — keep the chapter importable no matter what. + detect_latlon_columns = None # type: ignore[assignment] +try: + from datascience.analyze_geo_extent import analyze_geo_extent +except Exception: # noqa: BLE001 + analyze_geo_extent = None # type: ignore[assignment] +try: + from datascience.build_geo_scatter import build_geo_scatter +except Exception: # noqa: BLE001 + build_geo_scatter = None # type: ignore[assignment] + +CHAPTER_VERSION = "1.0.0" +CHAPTER_ID = "geospatial" +CHAPTER_TITLE = "Análisis geoespacial" + + +# --------------------------------------------------------------------------- # +# Formatting helpers (mirror the other chapters' defensive style). +# --------------------------------------------------------------------------- # +def _fmt_num(value, decimals: int = 4) -> str: + if value is None: + return "—" + if isinstance(value, bool): + return "sí" if value else "no" + if isinstance(value, int): + return f"{value:,}".replace(",", ".") + if isinstance(value, float): + if value != value: # NaN + return "NaN" + if value in (float("inf"), float("-inf")): + return str(value) + text = f"{value:.{decimals}f}".rstrip("0").rstrip(".") + return text if text else "0" + return model._safe_str(value) + + +def _fmt_coord(value, decimals: int = 4) -> str: + """Format a coordinate degree value, defensively.""" + try: + return f"{float(value):.{decimals}f}°" + except (TypeError, ValueError): + return model._safe_str(value) + + +def _fmt_km(value) -> str: + if value is None: + return "—" + try: + v = float(value) + except (TypeError, ValueError): + return model._safe_str(value) + if v >= 100: + return f"{v:,.0f} km".replace(",", ".") + return f"{v:.1f} km" + + +def _is_dict(v) -> bool: + return isinstance(v, dict) + + +def _clean_floats(seq) -> list: + """Return a list of floats from an arbitrary sequence (drop None/NaN).""" + out = [] + if not isinstance(seq, (list, tuple)): + return out + for v in seq: + try: + f = float(v) + except (TypeError, ValueError): + out.append(None) + continue + out.append(f if f == f else None) # NaN -> None + return out + + +# --------------------------------------------------------------------------- # +# Resolve the (lat, lon) columns and the raw coordinate arrays. +# --------------------------------------------------------------------------- # +def _detect_columns(profile: dict) -> dict: + """Detect the lat/lon column pair from the profile metadata, or {}.""" + cols = profile.get("columns") + if not isinstance(cols, list) or not cols or detect_latlon_columns is None: + return {} + try: + det = detect_latlon_columns(cols) + except Exception: # noqa: BLE001 — never break the chapter. + return {} + return det if _is_dict(det) else {} + + +def _resolve_coords(profile: dict, ctx: dict, detected: dict): + """Return (lats, lons, source_label). + + Order: ctx/profile['geo_points'] (explicit arrays) → ctx/profile + ['raw_numeric'] keyed by the detected lat/lon column names → (None, None). + """ + gp = ctx.get("geo_points") or profile.get("geo_points") + if _is_dict(gp): + lats = gp.get("lats") + if lats is None: + lats = gp.get("lat") + lons = gp.get("lons") + if lons is None: + lons = gp.get("lon") + if lats and lons: + return list(lats), list(lons), "geo_points" + + lat_col = (detected or {}).get("lat_col") + lon_col = (detected or {}).get("lon_col") + if lat_col and lon_col: + raw = ctx.get("raw_numeric") or profile.get("raw_numeric") + if _is_dict(raw): + lats = raw.get(lat_col) + lons = raw.get(lon_col) + if lats and lons: + return list(lats), list(lons), "raw_numeric" + return None, None, "none" + + +def _column_by_name(profile: dict, name): + if not name: + return None + for col in profile.get("columns") or []: + if isinstance(col, dict) and col.get("name") == name: + return col + return None + + +def _bbox_from_profile(profile: dict, detected: dict): + """Approximate bbox from the per-column numeric.min/max (no raw points).""" + lat_c = _column_by_name(profile, (detected or {}).get("lat_col")) + lon_c = _column_by_name(profile, (detected or {}).get("lon_col")) + lat_n = lat_c.get("numeric") if _is_dict(lat_c) else None + lon_n = lon_c.get("numeric") if _is_dict(lon_c) else None + if not _is_dict(lat_n) or not _is_dict(lon_n): + return None + try: + return { + "lat_min": float(lat_n.get("min")), + "lat_max": float(lat_n.get("max")), + "lon_min": float(lon_n.get("min")), + "lon_max": float(lon_n.get("max")), + } + except (TypeError, ValueError): + return None + + +# --------------------------------------------------------------------------- # +# Figure builder (lazy: matplotlib only imported when the renderer draws it). +# --------------------------------------------------------------------------- # +def _make_geo_scatter(scatter: dict, lat_col: str, lon_col: str): + """Return a zero-arg callable drawing the geographic scatter, or None.""" + points = scatter.get("points") or [] + if not points: + return None + bbox = scatter.get("bbox") if _is_dict(scatter.get("bbox")) else {} + aspect = scatter.get("aspect") or 1.0 + pad = scatter.get("pad") if _is_dict(scatter.get("pad")) else {} + n_total = scatter.get("n_total") + n_shown = scatter.get("n_shown") + + def _draw(): + import matplotlib + matplotlib.use("Agg") + import matplotlib.pyplot as plt + + xs = [p[0] for p in points if isinstance(p, (list, tuple)) and len(p) >= 2] + ys = [p[1] for p in points if isinstance(p, (list, tuple)) and len(p) >= 2] + + fig, ax = plt.subplots(figsize=(6.6, 5.0)) + # More points -> smaller markers + lower alpha so dense clouds read as + # density without saturating the page with ink (Tufte). + n = max(len(xs), 1) + size = 18 if n <= 200 else (8 if n <= 1000 else 4) + alpha = 0.75 if n <= 200 else (0.5 if n <= 1000 else 0.35) + ax.scatter(xs, ys, s=size, c="#2a6f97", alpha=alpha, linewidths=0, + zorder=3) + + # Bounding box rectangle for orientation. + if bbox: + try: + lo_x, hi_x = float(bbox["lon_min"]), float(bbox["lon_max"]) + lo_y, hi_y = float(bbox["lat_min"]), float(bbox["lat_max"]) + ax.plot([lo_x, hi_x, hi_x, lo_x, lo_x], + [lo_y, lo_y, hi_y, hi_y, lo_y], + color="#e15759", linewidth=1.0, linestyle="--", + alpha=0.8, zorder=4, label="Bounding box") + px = float(pad.get("lon", 0.0) or 0.0) + py = float(pad.get("lat", 0.0) or 0.0) + ax.set_xlim(lo_x - px, hi_x + px) + ax.set_ylim(lo_y - py, hi_y + py) + except (TypeError, ValueError, KeyError): + pass + + # Equirectangular: scale Y/X so longitude is not stretched at this + # latitude (integridad de proyección, Tufte). aspect = 1/cos(lat). + try: + ax.set_aspect(float(aspect)) + except (TypeError, ValueError): + pass + + ax.set_xlabel(f"Longitud ({lon_col})", fontsize=8) + ax.set_ylabel(f"Latitud ({lat_col})", fontsize=8) + ax.tick_params(labelsize=7) + ax.grid(color="#e6e6e6", linewidth=0.5, zorder=0) + title = "Distribución geográfica de las coordenadas" + if n_shown is not None and n_total is not None and n_shown < n_total: + title += f"\n(mostrando {n_shown:,} de {n_total:,} puntos)".replace(",", ".") + ax.set_title(title, fontsize=10) + ax.legend(loc="best", fontsize=7, frameon=True, framealpha=0.9) + fig.tight_layout() + return fig + + return _draw + + +# --------------------------------------------------------------------------- # +# Section builders. +# --------------------------------------------------------------------------- # +def _intro_block(detected: dict, lat_col: str, lon_col: str) -> list: + conf = (detected or {}).get("confidence") + reason = model._safe_str((detected or {}).get("reason")) + conf_txt = "" + if conf is not None: + try: + conf_txt = f" (confianza {float(conf) * 100:.0f}%)" + except (TypeError, ValueError): + conf_txt = "" + text = ( + "Este dataset contiene **coordenadas geográficas**: se identificó el par " + f"**latitud = «{lat_col}»** y **longitud = «{lon_col}»**{conf_txt}. La " + "detección combina el nombre de la columna y el rango de sus valores " + "(latitud en [−90, 90], longitud en [−180, 180])." + ) + if reason: + text += f"\n\n*Criterio de detección:* {reason}." + return [model.Heading(text=CHAPTER_TITLE, level=1), + model.Markdown(text=text)] + + +def _extent_blocks(extent: dict) -> list: + """KVTable with bbox/centroid/span + DataTable with the per-region counts.""" + if not _is_dict(extent) or not extent.get("n_points"): + return [] + blocks = [] + bbox = extent.get("bbox") if _is_dict(extent.get("bbox")) else {} + centroid = extent.get("centroid") if _is_dict(extent.get("centroid")) else {} + hemi = extent.get("hemisphere") if _is_dict(extent.get("hemisphere")) else {} + + rows = [("Puntos con coordenadas", _fmt_num(extent.get("n_points")))] + if bbox: + rows.append(("Latitud (mín. / máx.)", + f"{_fmt_coord(bbox.get('lat_min'))} a " + f"{_fmt_coord(bbox.get('lat_max'))}")) + rows.append(("Longitud (mín. / máx.)", + f"{_fmt_coord(bbox.get('lon_min'))} a " + f"{_fmt_coord(bbox.get('lon_max'))}")) + if centroid: + rows.append(("Centroide", + f"{_fmt_coord(centroid.get('lat'))}, " + f"{_fmt_coord(centroid.get('lon'))}")) + if extent.get("span_km") is not None: + rows.append(("Extensión (diagonal)", _fmt_km(extent.get("span_km")))) + if hemi: + n, s = hemi.get("north"), hemi.get("south") + e, w = hemi.get("east"), hemi.get("west") + rows.append(("Hemisferios", + f"N {_fmt_num(n)} / S {_fmt_num(s)} · " + f"E {_fmt_num(e)} / O {_fmt_num(w)}")) + blocks.append(model.KVTable(rows=rows, title="Extensión geográfica")) + + by_region = extent.get("by_region") + if isinstance(by_region, list) and by_region: + total = sum(r.get("count", 0) for r in by_region if _is_dict(r)) or 0 + rrows = [] + for r in by_region: + if not _is_dict(r): + continue + cnt = r.get("count", 0) + pct = (cnt / total) if total else None + pct_txt = f"{pct * 100:.1f}%" if pct is not None else "—" + rrows.append([model._safe_str(r.get("region")), _fmt_num(cnt), + pct_txt]) + if rrows: + blocks.append(model.DataTable( + header=["Zona / país", "Puntos", "% del total"], rows=rrows, + title="Distribución por zona", + note="Asignación aproximada por bounding box de cada región " + "(no es reverse-geocoding exacto de fronteras).")) + return blocks + + +def _narrative_block(profile: dict, ctx: dict, extent: dict) -> list: + """A one-line narrative of where the points concentrate. + + Uses the derived ``note`` from analyze_geo_extent by default; optionally + calls an LLM (ctx['run_geo_llm']) for a richer one-liner. + """ + note = model._safe_str((extent or {}).get("note")) + if ctx.get("run_geo_llm"): + by_region = (extent or {}).get("by_region") or [] + bbox = (extent or {}).get("bbox") or {} + try: + from core.ask_llm import ask_llm + prompt = ( + "Eres un analista de datos. En UNA frase en español, describe " + "dónde se concentran geográficamente estos puntos. Sé concreto " + "y no inventes precisión que los datos no tienen.\n" + f"Conteo por zona: {by_region}\nBounding box: {bbox}." + ) + out = ask_llm(prompt, + model=ctx.get("geo_llm_model", + "claude-haiku-4-5-20251001"), + echo=False) + if out and isinstance(out, str) and out.strip(): + note = out.strip() + except Exception: # noqa: BLE001 — degrade to the derived note. + pass + if not note: + return [] + return [model.Markdown(text=f"**Interpretación.** {note}")] + + +def _no_points_block(profile: dict, detected: dict) -> list: + """Degrade honestly when the raw coordinate arrays are not available.""" + blocks = [] + bbox = _bbox_from_profile(profile, detected) + if bbox: + rows = [ + ("Latitud (mín. / máx.)", + f"{_fmt_coord(bbox.get('lat_min'))} a " + f"{_fmt_coord(bbox.get('lat_max'))}"), + ("Longitud (mín. / máx.)", + f"{_fmt_coord(bbox.get('lon_min'))} a " + f"{_fmt_coord(bbox.get('lon_max'))}"), + ] + blocks.append(model.KVTable( + rows=rows, title="Extensión geográfica (aproximada)")) + blocks.append(model.Note( + "No se incluyeron las coordenadas crudas en el contexto, por lo que el " + "mapa y el análisis por zona no se han dibujado. El bounding box " + "mostrado se deriva de los mínimos y máximos por columna. Para el " + "scatter geográfico completo, pasa los arrays en " + "ctx['geo_points'] = {'lats': [...], 'lons': [...]} o las columnas en " + "ctx['raw_numeric'].")) + return blocks + + +# --------------------------------------------------------------------------- # +# Entry point. +# --------------------------------------------------------------------------- # +def build_geospatial(profile: dict, ctx: dict): + """Build the GEOSPATIAL Chapter, or None if the dataset has no coordinates. + + Args: + profile: the ``eda`` group TableProfile dict. + ctx: presentation context; may carry ``geo_points``/``raw_numeric`` with + the raw coordinate arrays and the ``run_geo_llm`` flag. + + Returns: + A ``model.Chapter`` with the geographic scatter + zone/country analysis, + or ``None`` when no latitude/longitude column pair is detected. + """ + profile = profile or {} + ctx = ctx or {} + if not isinstance(profile, dict): + return None + + detected = _detect_columns(profile) + lats, lons, source = _resolve_coords(profile, ctx, detected) + + has_detection = bool((detected or {}).get("lat_col") and + (detected or {}).get("lon_col")) + has_points = bool(lats and lons) + if not has_detection and not has_points: + return None # chapter does not apply: no coordinates in this dataset. + + # Labels for axes / intro. When only raw arrays were given (no detection), + # fall back to generic names. + lat_col = (detected or {}).get("lat_col") or "lat" + lon_col = (detected or {}).get("lon_col") or "lon" + + blocks = _intro_block(detected, lat_col, lon_col) + + if has_points: + clean_lats = _clean_floats(lats) + clean_lons = _clean_floats(lons) + + # Zone / country analysis. + extent = {} + if analyze_geo_extent is not None: + try: + extent = analyze_geo_extent(clean_lats, clean_lons) or {} + except Exception: # noqa: BLE001 + extent = {} + + # The geographic scatter figure (its own page/slide). + scatter = {} + if build_geo_scatter is not None: + try: + scatter = build_geo_scatter(clean_lats, clean_lons) or {} + except Exception: # noqa: BLE001 + scatter = {} + maker = _make_geo_scatter(scatter, lat_col, lon_col) if scatter else None + if maker is not None: + blocks.append(model.Figure( + make=maker, + caption="Cada punto es una observación situada por sus " + "coordenadas; el recuadro rojo es el bounding box. La " + "escala respeta la latitud (proyección equirectangular).")) + else: + blocks.append(model.Note( + "No se pudo construir el scatter geográfico a partir de las " + "coordenadas proporcionadas.")) + + blocks += _extent_blocks(extent) + blocks += _narrative_block(profile, ctx, extent) + else: + # Columns detected but no raw points available — degrade honestly. + blocks += _no_points_block(profile, detected) + + if not blocks: + return None + return model.Chapter(id=CHAPTER_ID, title=CHAPTER_TITLE, + version=CHAPTER_VERSION, blocks=blocks) diff --git a/python/functions/datascience/automatic_eda/chapters/geospatial_test.py b/python/functions/datascience/automatic_eda/chapters/geospatial_test.py new file mode 100644 index 00000000..434eae4e --- /dev/null +++ b/python/functions/datascience/automatic_eda/chapters/geospatial_test.py @@ -0,0 +1,245 @@ +"""Tests for the GEOSPATIAL chapter — DoD: golden + edges + anti-cut. + +Self-contained: builds synthetic TableProfiles (no DuckDB) so the suite is fast +and deterministic. The raw coordinate arrays are passed through ``ctx`` exactly +as the chapter's contract documents (``ctx['geo_points']`` / ``ctx['raw_numeric']``). + +Verifies that the chapter detects the lat/lon pair, draws the geographic scatter +figure, analyses the zone/country (bounding box + per-region counts), returns +None when there are no coordinates, degrades honestly when the raw points are +absent, and that a profile with long column names + many points + several +regions renders to PDF and PPTX without cutting any text (long content wraps, it +is never truncated). +""" + +import os +import re +import tempfile + +from pypdf import PdfReader +from pptx import Presentation + +from datascience.automatic_eda.chapters.geospatial import ( + build_geospatial, + CHAPTER_VERSION, +) +from datascience.automatic_eda import build_document, render_pdf, render_pptx + + +# --------------------------------------------------------------------------- # +# Synthetic data helpers +# --------------------------------------------------------------------------- # +def _grid(lat0: float, lon0: float, n: int, spread: float = 1.0): + """A small deterministic cloud of n points around (lat0, lon0).""" + lats, lons = [], [] + for i in range(n): + # deterministic pseudo-spread, no randomness. + f = (i % 11) / 11.0 - 0.5 + g = (i % 7) / 7.0 - 0.5 + lats.append(lat0 + f * spread) + lons.append(lon0 + g * spread) + return lats, lons + + +def _profile_with_coords(lat_name="lat", lon_name="lon", lats=None, lons=None): + """A profile carrying a lat/lon column pair with valid ranges.""" + lats = lats if lats is not None else [40.4, 41.0, 39.8, 40.1] + lons = lons if lons is not None else [-3.7, -3.6, -4.0, -3.9] + return { + "table": "lugares", + "columns": [ + {"name": lat_name, "inferred_type": "numeric", + "numeric": {"min": min(lats), "max": max(lats), + "mean": sum(lats) / len(lats)}}, + {"name": lon_name, "inferred_type": "numeric", + "numeric": {"min": min(lons), "max": max(lons), + "mean": sum(lons) / len(lons)}}, + {"name": "valor", "inferred_type": "numeric", + "numeric": {"min": 0, "max": 100, "mean": 50}}, + ], + } + + +def _ctx_points(lats, lons): + return {"geo_points": {"lats": lats, "lons": lons}} + + +def _kinds(chapter): + return [getattr(b, "kind", None) for b in chapter.blocks] + + +def _tables(chapter): + return [b for b in chapter.blocks if getattr(b, "kind", None) == "data_table"] + + +def _figures(chapter): + return [b for b in chapter.blocks if getattr(b, "kind", None) == "figure"] + + +# --------------------------------------------------------------------------- # +# Golden +# --------------------------------------------------------------------------- # +def test_golden_estructura_y_version(): + lats, lons = [40.4, 41.0, 39.8, 40.1], [-3.7, -3.6, -4.0, -3.9] + ch = build_geospatial(_profile_with_coords(lats=lats, lons=lons), + _ctx_points(lats, lons)) + assert ch is not None + assert ch.id == "geospatial" + assert ch.version == CHAPTER_VERSION + kinds = _kinds(ch) + # intro heading + markdown + scatter figure + extent kv + per-region table. + assert "heading" in kinds + assert "markdown" in kinds + assert "figure" in kinds, "falta el scatter geográfico" + assert "kv_table" in kinds, "falta la tabla de extensión" + + +def test_golden_detecta_columnas_y_nombra_ejes(): + lats, lons = _grid(40.4, -3.7, 30, spread=0.8) + prof = _profile_with_coords("latitude", "longitude", lats, lons) + ch = build_geospatial(prof, _ctx_points(lats, lons)) + intro = [b for b in ch.blocks if b.kind == "markdown"][0].text + assert "latitude" in intro and "longitude" in intro + + +def test_golden_figura_es_perezosa_y_dibujable(): + lats, lons = _grid(40.4, -3.7, 50, spread=0.6) + ch = build_geospatial(_profile_with_coords(lats=lats, lons=lons), + _ctx_points(lats, lons)) + fig_block = _figures(ch)[0] + assert fig_block.make is not None and fig_block.fig is None # lazy + fig = fig_block.make() # must draw without raising + assert fig is not None + import matplotlib.pyplot as plt + plt.close(fig) + + +def test_golden_analisis_por_zona_espana(): + lats, lons = _grid(40.4, -3.7, 40, spread=0.5) # Madrid area + ch = build_geospatial(_profile_with_coords(lats=lats, lons=lons), + _ctx_points(lats, lons)) + tables = _tables(ch) + region_tbl = [t for t in tables if "zona" in (t.title or "").lower()] + assert region_tbl, "falta la tabla por zona/país" + flat = " ".join(" ".join(str(c) for c in r) for r in region_tbl[0].rows) + # Spain-area points must resolve to a Spain/European region, not empty. + assert region_tbl[0].rows + assert any(c for c in (region_tbl[0].rows[0])) + + +def test_golden_raw_numeric_source(): + """Coordinates can also come from ctx['raw_numeric'] keyed by detected cols.""" + lats, lons = _grid(48.85, 2.35, 25, spread=0.4) # Paris area + prof = _profile_with_coords("lat", "lon", lats, lons) + ctx = {"raw_numeric": {"lat": lats, "lon": lons}} + ch = build_geospatial(prof, ctx) + assert ch is not None + assert _figures(ch), "el scatter debe construirse desde raw_numeric" + + +# --------------------------------------------------------------------------- # +# Edges +# --------------------------------------------------------------------------- # +def test_edge_sin_coordenadas_devuelve_none(): + prof = { + "table": "ventas", + "columns": [ + {"name": "precio", "inferred_type": "numeric", + "numeric": {"min": 0, "max": 1000}}, + {"name": "categoria", "inferred_type": "text"}, + ], + } + assert build_geospatial(prof, {}) is None + + +def test_edge_none_y_vacio_no_rompen(): + assert build_geospatial(None, None) is None + assert build_geospatial({}, {}) is None + assert build_geospatial({"columns": []}, {}) is None + assert build_geospatial("not a dict", {}) is None + + +def test_edge_nombre_lat_pero_rango_invalido_no_aplica(): + """A column named 'lat' whose values are out of [-90,90] is NOT a coordinate.""" + prof = { + "table": "x", + "columns": [ + {"name": "lat", "inferred_type": "numeric", + "numeric": {"min": 1000, "max": 9999}}, + {"name": "lon", "inferred_type": "numeric", + "numeric": {"min": 1000, "max": 9999}}, + ], + } + assert build_geospatial(prof, {}) is None + + +def test_edge_columnas_detectadas_sin_puntos_degrada(): + """Detected lat/lon but no raw arrays -> honest note + approx bbox, no crash.""" + prof = _profile_with_coords(lats=[40.0, 41.0], lons=[-3.0, -4.0]) + ch = build_geospatial(prof, {}) # no geo_points / raw_numeric + assert ch is not None + assert not _figures(ch), "sin puntos no debe dibujarse el scatter" + notes = [b for b in ch.blocks if b.kind == "note"] + assert notes and "coordenadas crudas" in notes[0].text + + +def test_edge_coordenadas_con_nan_se_filtran(): + lats = [40.4, float("nan"), 41.0, None, 39.8] + lons = [-3.7, -3.6, float("nan"), -3.9, -4.0] + ch = build_geospatial(_profile_with_coords(lats=[39.8, 41.0], + lons=[-4.0, -3.6]), + _ctx_points(lats, lons)) + assert ch is not None # must not raise on NaN/None + + +# --------------------------------------------------------------------------- # +# Anti-cut: long names + many points + several regions render without truncation +# --------------------------------------------------------------------------- # +def _multiregion_points(per: int = 700): + """Points spread across Spain, France and the USA to fill the region table.""" + lats, lons = [], [] + for (la, lo) in ((40.4, -3.7), (48.85, 2.35), (39.0, -98.0)): + gl, gn = _grid(la, lo, per, spread=2.0) + lats += gl + lons += gn + return lats, lons + + +def test_anticut_pdf_y_pptx_no_truncan(): + lat_name = "latitud_geografica_del_punto_de_observacion_registrado" + lon_name = "longitud_geografica_del_punto_de_observacion_registrado" + lats, lons = _multiregion_points(700) + prof = _profile_with_coords(lat_name, lon_name, lats, lons) + ctx = {"geo_points": {"lats": lats, "lons": lons}} + + full = build_document(prof, ctx) + assert any(c.id == "geospatial" for c in full) + chapters = [c for c in full if c.id == "geospatial"] + + with tempfile.TemporaryDirectory() as d: + pdf = os.path.join(d, "g.pdf") + pptx = os.path.join(d, "g.pptx") + rp = render_pdf(chapters, pdf, {"title": "EDA"}) + rx = render_pptx(chapters, pptx, {"title": "EDA"}) + assert os.path.exists(pdf) and os.path.exists(pptx) + assert (rp or {}).get("n_pages", 0) >= 1 + + # PDF: the long lat column name survives whole (wraps, not cut) and there + # is no truncation marker in this chapter. + pdf_txt = "".join((pg.extract_text() or "") for pg in PdfReader(pdf).pages) + assert "…" not in pdf_txt and "..." not in pdf_txt + norm = re.sub(r"\s+", "", pdf_txt) + assert lat_name in norm, "el nombre largo de la columna se cortó en el PDF" + + # PPTX: long name present in some shape/cell, untruncated. + allt = [] + for s in Presentation(pptx).slides: + for sh in s.shapes: + if sh.has_text_frame: + allt.append(sh.text_frame.text) + if sh.has_table: + for row in sh.table.rows: + for c in row.cells: + allt.append(c.text) + joined = re.sub(r"\s+", "", "\n".join(allt)) + assert lat_name in joined, "el nombre largo de la columna se cortó en el PPTX" From 96da9e3015c3eee89a9ea5fd2dd613d28009e5b2 Mon Sep 17 00:00:00 2001 From: Egutierrez Date: Tue, 30 Jun 2026 15:33:55 +0200 Subject: [PATCH 15/53] =?UTF-8?q?feat(eda):=20funciones=20de=20agregaci?= =?UTF-8?q?=C3=B3n/OLAP=20para=20AutomaticEDA=20(groupby/pivot=20push-down?= =?UTF-8?q?=20+=20selecci=C3=B3n=20LLM)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cuatro funciones nuevas del grupo eda que nutren el capítulo AGREGACION: - select_groupby_keys (pure): elige categóricas agrupables + numéricas medida desde el TableProfile. - groupby_stats_duckdb (impure): GROUP BY push-down en DuckDB (count/mean/median/std/min/max por grupo). - pivot_table_duckdb (impure): pivot A×B push-down, limitado a top filas/cols para no cortar. - suggest_aggregations_llm (impure): el LLM elige las agregaciones interesantes con fallback determinista. Co-Authored-By: Claude Opus 4.8 (1M context) --- python/functions/datascience/__init__.py | 6 + .../datascience/groupby_stats_duckdb.md | 87 ++++ .../datascience/groupby_stats_duckdb.py | 184 ++++++++ .../datascience/groupby_stats_duckdb_test.py | 106 +++++ .../datascience/pivot_table_duckdb.md | 92 ++++ .../datascience/pivot_table_duckdb.py | 176 ++++++++ .../datascience/pivot_table_duckdb_test.py | 115 +++++ .../datascience/select_groupby_keys.md | 158 +++++++ .../datascience/select_groupby_keys.py | 310 ++++++++++++++ .../datascience/select_groupby_keys_test.py | 213 +++++++++ .../datascience/suggest_aggregations_llm.md | 96 +++++ .../datascience/suggest_aggregations_llm.py | 405 ++++++++++++++++++ .../suggest_aggregations_llm_test.py | 198 +++++++++ 13 files changed, 2146 insertions(+) create mode 100644 python/functions/datascience/groupby_stats_duckdb.md create mode 100644 python/functions/datascience/groupby_stats_duckdb.py create mode 100644 python/functions/datascience/groupby_stats_duckdb_test.py create mode 100644 python/functions/datascience/pivot_table_duckdb.md create mode 100644 python/functions/datascience/pivot_table_duckdb.py create mode 100644 python/functions/datascience/pivot_table_duckdb_test.py create mode 100644 python/functions/datascience/select_groupby_keys.md create mode 100644 python/functions/datascience/select_groupby_keys.py create mode 100644 python/functions/datascience/select_groupby_keys_test.py create mode 100644 python/functions/datascience/suggest_aggregations_llm.md create mode 100644 python/functions/datascience/suggest_aggregations_llm.py create mode 100644 python/functions/datascience/suggest_aggregations_llm_test.py diff --git a/python/functions/datascience/__init__.py b/python/functions/datascience/__init__.py index 9fc8c206..f2746d11 100644 --- a/python/functions/datascience/__init__.py +++ b/python/functions/datascience/__init__.py @@ -25,6 +25,7 @@ from .describe_numeric import describe_numeric from .summarize_categorical import summarize_categorical from .infer_semantic_type import infer_semantic_type from .column_quality_score import column_quality_score +from .select_groupby_keys import select_groupby_keys from .render_eda_markdown import render_eda_markdown from .detect_distribution_type import detect_distribution_type from .spearman_corr import spearman_corr @@ -36,6 +37,8 @@ from .infer_fk_containment_duckdb import infer_fk_containment_duckdb from .build_join_graph import build_join_graph from .association_matrix import association_matrix from .correlation_matrix_duckdb import correlation_matrix_duckdb +from .pivot_table_duckdb import pivot_table_duckdb +from .groupby_stats_duckdb import groupby_stats_duckdb from .pca_explained import pca_explained from .kmeans_segments import kmeans_segments from .isolation_forest_outliers import isolation_forest_outliers @@ -82,6 +85,8 @@ __all__ = [ "build_join_graph", "association_matrix", "correlation_matrix_duckdb", + "pivot_table_duckdb", + "groupby_stats_duckdb", "pca_explained", "kmeans_segments", "isolation_forest_outliers", @@ -96,6 +101,7 @@ __all__ = [ "summarize_categorical", "infer_semantic_type", "column_quality_score", + "select_groupby_keys", "render_eda_markdown", "detect_distribution_type", "pull_gsc_search_analytics", diff --git a/python/functions/datascience/groupby_stats_duckdb.md b/python/functions/datascience/groupby_stats_duckdb.md new file mode 100644 index 00000000..faf22222 --- /dev/null +++ b/python/functions/datascience/groupby_stats_duckdb.md @@ -0,0 +1,87 @@ +--- +name: groupby_stats_duckdb +kind: function +lang: py +domain: datascience +version: "1.0.0" +purity: impure +signature: "def groupby_stats_duckdb(db_path: str, table: str, group_by: str, measures: list, aggs: list = None, top_n: int = 15) -> dict" +description: "Agregaciones GROUP BY con push-down SQL en DuckDB: para cada measure numerica calcula mean/median/std/min/max por grupo (split-apply-combine en el motor), trayendo solo una fila por grupo. Nucleo de un capitulo de agregacion/OLAP de un EDA. count = tamanio del grupo, independiente de measures." +tags: [eda, groupby, aggregation, olap, duckdb, datascience, push-down, split-apply-combine] +uses_functions: [duckdb_query_readonly_py_infra] +uses_types: [] +returns: [] +returns_optional: false +error_type: "error_go_core" +imports: [] +params: + - name: db_path + desc: "Ruta al archivo DuckDB. Debe existir; el modo read_only NO crea la base. Path inexistente -> {status:'error'} sin lanzar." + - name: table + desc: "Nombre de la tabla. Se interpola citado con dobles comillas (soporta nombres con espacios; las comillas internas se escapan)." + - name: group_by + desc: "Columna por la que agrupar. Se interpola citada. Sus valores distintos son las claves de los grupos." + - name: measures + desc: "Lista de columnas numericas a agregar. Lista vacia es valida: cada grupo trae solo su tamanio `n` y `stats` vacio." + - name: aggs + desc: "Lista de agregaciones. None (default) = ['count','mean','median','std','min','max']. Validas: count (tamanio del grupo, va a `n`), mean->avg, median, std->stddev_samp, min, max (estas cinco por measure). Agg desconocido -> error." + - name: top_n + desc: "Maximo de grupos a devolver, ordenados por tamanio de grupo descendente (default 15). Internamente se piden top_n+1 para detectar truncado." +output: "dict. En exito {status:'ok', group_by, measures:[...], aggs:[...], n_groups:int, truncated:bool, groups:[{key:, n:int, stats:{:{mean,median,std,min,max}}}], note:str}. Las estadisticas son float o None (p.ej. std de un grupo de 1 fila -> NULL -> None). En error {status:'error', error:str} (no lanza)." +tested: true +tests: ["agrega por grupo con valores conocidos", "db inexistente devuelve error sin lanzar", "measures vacias agrega solo count", "columna con espacio agrupa bien"] +test_file_path: "python/functions/datascience/groupby_stats_duckdb_test.py" +file_path: "python/functions/datascience/groupby_stats_duckdb.py" +--- + +## Ejemplo + +```python +import duckdb +from datascience import groupby_stats_duckdb + +# Cargar el titanic en una tabla DuckDB de prueba. +db = "/tmp/titanic.duckdb" +con = duckdb.connect(db) +con.execute( + "CREATE TABLE titanic AS " + "SELECT * FROM read_csv_auto('https://raw.githubusercontent.com/" + "datasciencedojo/datasets/master/titanic.csv')" +) +con.close() + +# Agrupar por sexo midiendo edad y tarifa. +res = groupby_stats_duckdb(db, "titanic", "Sex", ["Age", "Fare"]) +print(res["status"]) # ok +print(res["n_groups"]) # 2 (male, female) +for g in res["groups"]: + print(g["key"], g["n"], round(g["stats"]["Fare"]["mean"], 2)) +# female 314 44.48 +# male 577 25.52 +``` + +## Cuando usarla + +Cuando en un EDA necesitas el clasico split-apply-combine: "para cada categoria de X, +¿cuanto vale en media/mediana/desviacion/min/max la metrica Y?". Es el nucleo de un +capitulo de agregacion/OLAP. Usala antes de pintar barras o boxplots por grupo, para +detectar segmentos con comportamiento distinto, o para resumir una tabla grande sin +traer las filas a RAM: todo el GROUP BY ocurre push-down en el motor de DuckDB y solo +viaja una fila por grupo. `top_n` te deja quedarte con los grupos mas poblados. + +## Gotchas + +- Funcion impura: lee un archivo DuckDB del disco (read_only, nunca lo modifica). La + tabla debe existir ya en el `.db` (no carga CSV; para eso crea la tabla antes). +- Identificadores (tabla, group_by, measures) se interpolan citados con dobles comillas + y escapando las internas: soporta nombres con espacios y evita inyeccion. No pases + expresiones SQL como group_by/measure — solo nombres de columna. +- `count` es el tamanio del grupo (`COUNT(*)`), independiente de las measures: se + refleja en el campo `n` de cada grupo, NO como clave dentro de `stats`. Las claves de + `stats[measure]` son las measure-aggs efectivas (mean/median/std/min/max menos count). +- `std` usa `stddev_samp` (muestral, n-1): un grupo con una sola fila da `NULL` -> `None`. + Las measures pueden contener NULLs; cada agregada los ignora segun la semantica de DuckDB. +- `truncated:True` indica que habia mas grupos que `top_n` (se devolvieron los `top_n` + mayores por tamanio). Sube `top_n` si necesitas todos los grupos. +- Si `measures` esta vacio, cada grupo trae solo `n` y `stats == {}` (valido, util para + un simple conteo por categoria). diff --git a/python/functions/datascience/groupby_stats_duckdb.py b/python/functions/datascience/groupby_stats_duckdb.py new file mode 100644 index 00000000..9a2bfd1e --- /dev/null +++ b/python/functions/datascience/groupby_stats_duckdb.py @@ -0,0 +1,184 @@ +"""groupby_stats_duckdb — agregaciones GROUP BY con push-down SQL en DuckDB. + +Funcion impura: lee de disco a traves de DuckDB (via la primitiva read-only +`duckdb_query_readonly` del grupo `duckdb`). Pertenece al grupo de capacidad `eda`. + +Ejecuta un `GROUP BY ` en el motor de DuckDB (split-apply-combine con +push-down) calculando, para cada columna numerica de `measures`, las agregaciones +pedidas (mean/median/std/min/max). Solo trae al cliente una fila por grupo, nunca +las filas crudas: apto para tablas grandes. Es el nucleo de un capitulo de +agregacion/OLAP de un EDA. + +Estilo dict-no-throw del grupo duckdb: nunca lanza; captura cualquier error y +devuelve {status:'error', error:str}. +""" + +from infra import duckdb_query_readonly + +# Mapeo agg -> funcion agregada SQL de DuckDB. `count` se trata aparte: es +# COUNT(*) (tamanio del grupo), independiente de las measures. +_AGG_SQL = { + "mean": "avg", + "median": "median", + "std": "stddev_samp", + "min": "min", + "max": "max", +} + +# Aggs por defecto cuando aggs=None. count primero (tamanio del grupo) + las +# cinco estadisticas por measure. +_DEFAULT_AGGS = ["count", "mean", "median", "std", "min", "max"] + + +def _quote_ident(ident: str) -> str: + """Cita un identificador SQL con dobles comillas, escapando las internas. + + Soporta nombres con espacios o caracteres especiales y evita inyeccion: dentro + de un identificador entrecomillado el unico caracter peligroso es la propia + comilla doble, que se duplica ("") segun el estandar SQL. DuckDB no admite + parametros posicionales para nombres de tabla/columna, asi que esta es la via + segura de interpolarlos. + """ + return '"' + str(ident).replace('"', '""') + '"' + + +def groupby_stats_duckdb( + db_path: str, + table: str, + group_by: str, + measures: list, + aggs: list = None, + top_n: int = 15, +) -> dict: + """GROUP BY con agregaciones por measure, todo push-down en DuckDB. + + Args: + db_path: ruta al archivo DuckDB. Debe existir; el modo read_only NO crea la + base. Un path inexistente devuelve {status:'error', ...} sin lanzar. + table: nombre de la tabla. Se interpola citado con dobles comillas (soporta + nombres con espacios). + group_by: columna por la que agrupar. Se interpola citada. + measures: lista de columnas numericas a agregar. Lista vacia es valida: + cada grupo trae solo su tamanio `n` y `stats` vacio. + aggs: lista de agregaciones a calcular. None (default) = + ["count", "mean", "median", "std", "min", "max"]. Valores validos: + count (tamanio del grupo, va a `n`), mean, median, std, min, max + (estas cinco se calculan por cada measure). Un agg desconocido devuelve + error. + top_n: numero maximo de grupos a devolver, ordenados por tamanio de grupo + descendente (default 15). Se pide top_n+1 internamente para detectar si + habia mas grupos y marcar `truncated`. + + Returns: + dict. En exito: + {status:'ok', + group_by:str, + measures:[...], + aggs:[...], # las efectivas (incluye count si se pidio) + n_groups:int, # nº de grupos devueltos (<= top_n) + truncated:bool, # True si habia mas de top_n grupos + groups:[{key:, n:int, + stats:{:{mean,median,std,min,max}}}, ...], + note:str} + Las estadisticas son float o None (p.ej. stddev_samp de un grupo de una + sola fila -> NULL -> None). En error (sin lanzar): {status:'error', error:str}. + """ + try: + # 1. Validar entradas. + if not isinstance(table, str) or table == "": + return {"status": "error", "error": "table must be a non-empty string"} + if not isinstance(group_by, str) or group_by == "": + return {"status": "error", "error": "group_by must be a non-empty string"} + + if measures is None: + measures = [] + if not isinstance(measures, list): + return {"status": "error", "error": "measures must be a list"} + for m in measures: + if not isinstance(m, str) or m == "": + return { + "status": "error", + "error": f"invalid measure identifier: {m!r}", + } + + if aggs is None: + aggs = list(_DEFAULT_AGGS) + if not isinstance(aggs, list) or len(aggs) == 0: + return { + "status": "error", + "error": "aggs must be a non-empty list or None", + } + for a in aggs: + if a != "count" and a not in _AGG_SQL: + return { + "status": "error", + "error": f"unknown agg {a!r}; valid: count, " + + ", ".join(_AGG_SQL), + } + + if not isinstance(top_n, int) or isinstance(top_n, bool) or top_n < 1: + return {"status": "error", "error": "top_n must be a positive int"} + + # 2. Aggs por measure = todas menos count (count es el tamanio del grupo, + # se mapea siempre a la columna `n`). + measure_aggs = [a for a in aggs if a != "count"] + + # 3. Construir el SELECT. grp y n primero; luego un termino por measure x agg + # con alias posicional (m{idx}_{agg}) para no chocar con nombres de columna + # que lleven espacios o caracteres raros. + select_terms = [f"{_quote_ident(group_by)} AS grp", "COUNT(*) AS n"] + agg_index = [] # (measure_name, agg_name, alias) + for mi, m in enumerate(measures): + for a in measure_aggs: + alias = f"m{mi}_{a}" + fn = _AGG_SQL[a] + select_terms.append(f"{fn}({_quote_ident(m)}) AS {alias}") + agg_index.append((m, a, alias)) + + # Pedimos top_n+1 grupos para detectar truncado (habia mas que top_n). + sql = ( + f"SELECT {', '.join(select_terms)} " + f"FROM {_quote_ident(table)} " + f"GROUP BY {_quote_ident(group_by)} " + f"ORDER BY n DESC " + f"LIMIT {top_n + 1}" + ) + + # 4. Ejecutar push-down. sandbox=True (default) basta: la tabla ya existe en + # el .db, no necesitamos read_csv/read_blob ni acceso al filesystem. + result = duckdb_query_readonly(db_path, sql, max_rows=top_n + 1) + if result.get("status") != "ok": + return { + "status": "error", + "error": "groupby query failed: " + + str(result.get("error", "unknown")), + } + + rows = result.get("rows", []) + truncated = len(rows) > top_n + if truncated: + rows = rows[:top_n] + + # 5. Reconstruir la estructura por grupo. + groups = [] + for row in rows: + stats = {m: {} for m in measures} + for (m, a, alias) in agg_index: + stats[m][a] = row.get(alias) + groups.append( + {"key": row.get("grp"), "n": row.get("n"), "stats": stats} + ) + + return { + "status": "ok", + "group_by": group_by, + "measures": list(measures), + "aggs": list(aggs), + "n_groups": len(groups), + "truncated": truncated, + "groups": groups, + "note": f"GROUP BY {group_by}: top {len(groups)} grupos por tamanio sobre " + f"{len(measures)} measure(s)", + } + except Exception as e: # noqa: BLE001 + return {"status": "error", "error": str(e)} diff --git a/python/functions/datascience/groupby_stats_duckdb_test.py b/python/functions/datascience/groupby_stats_duckdb_test.py new file mode 100644 index 00000000..e0857b3d --- /dev/null +++ b/python/functions/datascience/groupby_stats_duckdb_test.py @@ -0,0 +1,106 @@ +"""Tests para groupby_stats_duckdb.""" + +import os +import sys + +import duckdb + +# Permitir importar funciones del registry (from infra import ..., from datascience import ...). +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..")) +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "functions")) + +from datascience.groupby_stats_duckdb import groupby_stats_duckdb + + +def _make_db(tmp_path, rows): + """Crea una DuckDB con tabla t(g VARCHAR, x DOUBLE) e inserta `rows`.""" + db = os.path.join(str(tmp_path), "t.duckdb") + con = duckdb.connect(db) + con.execute("CREATE TABLE t(g VARCHAR, x DOUBLE)") + con.executemany("INSERT INTO t VALUES (?, ?)", rows) + con.close() + return db + + +def test_agrega_por_grupo_con_valores_conocidos(tmp_path): + # Grupo a: [10, 20, 30] -> n=3, mean=20, min=10, max=30, median=20, std=10. + # Grupo b: [5, 15] -> n=2, mean=10, median=10. + # Grupo c: [100] -> n=1, mean=100, std=None (1 sola fila). + rows = [ + ("a", 10.0), ("a", 20.0), ("a", 30.0), + ("b", 5.0), ("b", 15.0), + ("c", 100.0), + ] + db = _make_db(tmp_path, rows) + res = groupby_stats_duckdb(db, "t", "g", ["x"]) + assert res["status"] == "ok", res + assert res["n_groups"] == 3 + assert res["truncated"] is False + assert res["aggs"] == ["count", "mean", "median", "std", "min", "max"] + + by_key = {g["key"]: g for g in res["groups"]} + assert set(by_key) == {"a", "b", "c"} + + # Grupo a: comprobacion manual de mean/min/max/median/std. + sa = by_key["a"]["stats"]["x"] + assert by_key["a"]["n"] == 3 + assert abs(sa["mean"] - 20.0) < 1e-9 + assert abs(sa["min"] - 10.0) < 1e-9 + assert abs(sa["max"] - 30.0) < 1e-9 + assert abs(sa["median"] - 20.0) < 1e-9 + assert "std" in sa and sa["std"] is not None + assert abs(sa["std"] - 10.0) < 1e-9 # stddev_samp([10,20,30]) = 10 + + # Grupo b: mean y median conocidas. + sb = by_key["b"]["stats"]["x"] + assert by_key["b"]["n"] == 2 + assert abs(sb["mean"] - 10.0) < 1e-9 + assert abs(sb["median"] - 10.0) < 1e-9 + assert "median" in sb and "std" in sb + + # Grupo c: una sola fila -> std None (stddev_samp NULL), mean/min/max definidos. + sc = by_key["c"]["stats"]["x"] + assert by_key["c"]["n"] == 1 + assert abs(sc["mean"] - 100.0) < 1e-9 + assert sc["std"] is None + + +def test_db_inexistente_devuelve_error_sin_lanzar(tmp_path): + db = os.path.join(str(tmp_path), "no_existe.duckdb") + res = groupby_stats_duckdb(db, "t", "g", ["x"]) + assert res["status"] == "error", res + assert isinstance(res["error"], str) and res["error"] + + +def test_measures_vacias_agrega_solo_count(tmp_path): + rows = [("a", 1.0), ("a", 2.0), ("b", 3.0)] + db = _make_db(tmp_path, rows) + res = groupby_stats_duckdb(db, "t", "g", []) + assert res["status"] == "ok", res + by_key = {g["key"]: g for g in res["groups"]} + assert by_key["a"]["n"] == 2 + assert by_key["b"]["n"] == 1 + # Sin measures, stats por grupo es un dict vacio (valido). + assert by_key["a"]["stats"] == {} + assert by_key["b"]["stats"] == {} + + +def test_columna_con_espacio_agrupa_bien(tmp_path): + # Tabla con nombres de columna con espacios -> prueba el quoting con dobles + # comillas tanto en group_by como en la measure. + db = os.path.join(str(tmp_path), "space.duckdb") + con = duckdb.connect(db) + con.execute('CREATE TABLE t("my col" VARCHAR, "the val" DOUBLE)') + con.executemany( + 'INSERT INTO t VALUES (?, ?)', + [("x", 1.0), ("x", 3.0), ("y", 10.0)], + ) + con.close() + + res = groupby_stats_duckdb(db, "t", "my col", ["the val"]) + assert res["status"] == "ok", res + by_key = {g["key"]: g for g in res["groups"]} + assert by_key["x"]["n"] == 2 + assert abs(by_key["x"]["stats"]["the val"]["mean"] - 2.0) < 1e-9 + assert by_key["y"]["n"] == 1 + assert abs(by_key["y"]["stats"]["the val"]["mean"] - 10.0) < 1e-9 diff --git a/python/functions/datascience/pivot_table_duckdb.md b/python/functions/datascience/pivot_table_duckdb.md new file mode 100644 index 00000000..a14d55eb --- /dev/null +++ b/python/functions/datascience/pivot_table_duckdb.md @@ -0,0 +1,92 @@ +--- +name: pivot_table_duckdb +kind: function +lang: py +domain: datascience +version: "1.0.0" +purity: impure +signature: "def pivot_table_duckdb(db_path: str, table: str, index: str, columns: str, value: str, agg: str = 'mean', top_rows: int = 10, top_cols: int = 8) -> dict" +description: "Pivot table (index x columns -> agg(value)) calculada con push-down SQL en DuckDB (GROUP BY en el motor, sin traer filas a RAM) y recortada a las top_rows filas y top_cols columnas con mas observaciones para que quepa entera en un PDF movil / slide PPTX sin cortarse. Version push-down para tablas grandes de la funcion pura `pivot` (que pivota list[dict] en memoria)." +tags: [eda, pivot, duckdb, aggregate, datascience, push-down, report] +uses_functions: [duckdb_query_readonly_py_infra] +uses_types: [] +returns: [] +returns_optional: false +error_type: "error_go_core" +imports: [] +params: + - name: db_path + desc: "Ruta al archivo DuckDB. Debe existir; el modo read_only NO crea la base." + - name: table + desc: "Nombre de la tabla a pivotar. Se interpola citado con dobles comillas (DuckDB no admite parametros para identificadores)." + - name: index + desc: "Columna cuyos valores forman las filas de la pivot (eje vertical)." + - name: columns + desc: "Columna cuyos valores forman las columnas de la pivot (eje horizontal)." + - name: value + desc: "Columna numerica a agregar en cada celda. Ignorada cuando agg='count'." + - name: agg + desc: "Funcion de agregacion: mean, sum, count, min, max, median. mean->avg(), count->COUNT(*). Otro valor devuelve {status:'error'}." + - name: top_rows + desc: "Numero maximo de filas a conservar, elegidas por mayor numero de observaciones (suma de COUNT(*) por valor de index). Default 10." + - name: top_cols + desc: "Numero maximo de columnas a conservar, elegidas por mayor numero de observaciones (suma de COUNT(*) por valor de columns). Default 8." +output: "dict. En exito {status:'ok', index, columns, value, agg, row_labels:[...], col_labels:[...], matrix:[[...]], truncated_rows:bool, truncated_cols:bool, note:str}. matrix tiene len(row_labels) filas y cada fila len(col_labels) celdas (valor agregado o None si la combinacion no existe). truncated_* indica si hubo mas filas/columnas que el top. En error {status:'error', error:str} (no lanza)." +tested: true +tests: ["pivot mean labels y celda conocida", "pivot trunca a top rows y top cols", "pivot count no necesita value real", "pivot db inexistente devuelve error sin lanzar", "pivot agg invalido devuelve error"] +test_file_path: "python/functions/datascience/pivot_table_duckdb_test.py" +file_path: "python/functions/datascience/pivot_table_duckdb.py" +--- + +## Ejemplo + +```python +import duckdb +from datascience import pivot_table_duckdb + +# Tabla DuckDB de prueba estilo titanic: sex x pclass -> mean(fare). +db = "/tmp/pivot_demo.duckdb" +con = duckdb.connect(db) +con.execute( + "CREATE TABLE titanic AS SELECT * FROM (VALUES " + "('male',1,211.3),('female',1,151.5),('male',3,7.9)," + "('female',3,16.7),('male',1,52.0),('female',2,41.6)" + ") t(sex, pclass, fare)" +) +con.close() + +res = pivot_table_duckdb(db, "titanic", index="sex", columns="pclass", value="fare", agg="mean") +print(res["status"]) # ok +print(res["row_labels"]) # ['female', 'male'] (orden por nº de observaciones desc; empate -> etiqueta) +print(res["col_labels"]) # [1, 3, 2] (pclass=1 tiene 3 obs, pclass=3 -> 2, pclass=2 -> 1) +print(res["matrix"]) # [[151.5, 16.7, 41.6], [131.65, 7.9, None]] (male/pclass=2 no existe -> None) +``` + +## Cuando usarla + +Cuando quieres una pivot table (`index` x `columns` -> `agg(value)`) de una tabla +DuckDB con MUCHAS filas y necesitas que el resultado quepa entero en un informe: un +PDF abierto en el movil o un slide PPTX, donde una matriz de 50x30 se cortaria. La +agregacion se hace push-down en el motor (no traes las filas a RAM) y el resultado se +limita a las `top_rows` x `top_cols` combinaciones con mas observaciones. Encaja en el +flujo `eda` para resumir el cruce de dos categoricas (sexo x clase, region x producto) +contra una metrica. Para pivotar un `list[dict]` ya cargado en memoria usa la funcion +pura `pivot_py_datascience`; esta es la version push-down sobre disco. + +## Gotchas + +- Funcion impura: lee un archivo DuckDB del disco (read_only, nunca lo modifica). +- Recorta a `top_rows` x `top_cols` por numero de observaciones (suma de `COUNT(*)`), + NO por magnitud del valor agregado. Si habia mas filas/columnas, `truncated_rows` / + `truncated_cols` quedan en True y esas combinaciones NO aparecen en la matriz. +- Las celdas sin datos (combinacion `index` x `columns` que no existe en la tabla) se + rellenan con `None`, no con 0: distinguir "cero medido" de "sin observaciones". +- `agg='count'` cuenta filas por celda con `COUNT(*)` e ignora `value` (puedes pasar + cualquier nombre de columna). Para el resto de aggs, `value` debe ser una columna + numerica real o la query fallara con `{status:'error'}`. +- `agg` solo admite mean, sum, count, min, max, median; cualquier otro valor devuelve + `{status:'error'}` sin tocar la base. +- Orden de `row_labels` / `col_labels`: por numero de observaciones descendente, con + desempate estable por etiqueta. No es orden alfabetico ni el de aparicion. +- La query se ejecuta con `sandbox=False` en `duckdb_query_readonly` (uso interno + confiable: el SQL lo construye esta funcion, no un cliente externo). diff --git a/python/functions/datascience/pivot_table_duckdb.py b/python/functions/datascience/pivot_table_duckdb.py new file mode 100644 index 00000000..cee20464 --- /dev/null +++ b/python/functions/datascience/pivot_table_duckdb.py @@ -0,0 +1,176 @@ +"""pivot_table_duckdb — pivot table (index x columns -> agg(value)) con push-down SQL. + +Funcion impura: lee de disco a traves de DuckDB reusando la primitiva read-only del +grupo `duckdb` (`duckdb_query_readonly`). Pertenece al grupo de capacidad `eda` +(exploratory data analysis). + +A diferencia de la funcion pura `pivot` (que pivota un `list[dict]` ya cargado en +memoria), esta version empuja la agregacion al motor de DuckDB (push-down): el +GROUP BY lo resuelve DuckDB y solo se traen los valores agregados, nunca las filas +crudas. Esto la hace apta para tablas grandes. + +Ademas reduce el resultado a las `top_rows` filas y `top_cols` columnas con mas +observaciones, de modo que la pivot quepa entera en un PDF movil / slide PPTX sin +cortarse. Marca `truncated_rows`/`truncated_cols` cuando hubo recorte. + +Estilo dict-no-throw del grupo duckdb: nunca lanza; captura cualquier error y +devuelve {status:'error', error:str}. +""" + +from collections import defaultdict + +from infra import duckdb_query_readonly + +# Funciones de agregacion permitidas y su nombre en SQL DuckDB. +# mean -> avg; el resto mapea directo. count se trata aparte (COUNT(*), sin value). +_AGG_SQL = { + "mean": "avg", + "sum": "sum", + "count": "count", + "min": "min", + "max": "max", + "median": "median", +} + + +def _quote_ident(ident: str) -> str: + """Cita un identificador SQL con dobles comillas, escapando `"` -> `""`. + + DuckDB no admite parametros posicionales para nombres de tabla/columna, asi que + hay que interpolarlos. El quoting con `"` y el doblado de comillas internas evita + que un nombre rompa la sentencia (mismo patron que correlation_matrix_duckdb). + """ + return '"' + str(ident).replace('"', '""') + '"' + + +def pivot_table_duckdb( + db_path: str, + table: str, + index: str, + columns: str, + value: str, + agg: str = "mean", + top_rows: int = 10, + top_cols: int = 8, +) -> dict: + """Pivot table push-down en DuckDB, recortada a top_rows x top_cols. + + Construye una pivot (filas = valores de `index`, columnas = valores de `columns`, + celda = `agg(value)`) agregando en el motor de DuckDB, y la reduce a las filas y + columnas con mas observaciones para que quepa en un PDF / slide. + + Args: + db_path: ruta al archivo DuckDB. Debe existir (read_only NO crea la base). + table: nombre de la tabla a pivotar. + index: columna cuyos valores forman las filas de la pivot. + columns: columna cuyos valores forman las columnas de la pivot. + value: columna numerica a agregar. Ignorada cuando agg="count". + agg: funcion de agregacion. Una de: "mean", "sum", "count", "min", "max", + "median". mean se traduce a avg(); count a COUNT(*). + top_rows: numero maximo de filas a conservar, elegidas por mayor numero de + observaciones (suma de COUNT(*) por valor de index). Default 10. + top_cols: numero maximo de columnas a conservar, elegidas por mayor numero de + observaciones (suma de COUNT(*) por valor de columns). Default 8. + + Returns: + dict. En exito: + {status:'ok', + index, columns, value, agg, + row_labels:[...], # valores de index, en orden de freq desc + col_labels:[...], # valores de columns, en orden de freq desc + matrix:[[...], ...], # len == len(row_labels); cada fila + # len == len(col_labels); celda = agg o None + truncated_rows:bool, truncated_cols:bool, + note:str} + En error (sin lanzar): {status:'error', error:str}. + """ + try: + if not isinstance(agg, str) or agg not in _AGG_SQL: + return { + "status": "error", + "error": "invalid agg " + + repr(agg) + + "; allowed: " + + ", ".join(sorted(_AGG_SQL)), + } + + # Paso 1 (push-down): agregar (index, columns) -> agg(value) + COUNT(*). + if agg == "count": + agg_expr = "COUNT(*)" + else: + agg_expr = f"{_AGG_SQL[agg]}({_quote_ident(value)})" + + sql = ( + f"SELECT {_quote_ident(index)} AS r, " + f"{_quote_ident(columns)} AS c, " + f"{agg_expr} AS v, " + f"COUNT(*) AS n " + f"FROM {_quote_ident(table)} " + f"GROUP BY {_quote_ident(index)}, {_quote_ident(columns)}" + ) + + # max_rows alto: queremos todos los grupos (index x columns) para elegir el + # top con criterio global. sandbox=False igual que correlation_matrix_duckdb, + # porque db_path es una ruta interna de confianza. + result = duckdb_query_readonly( + db_path, sql, max_rows=1_000_000, sandbox=False + ) + if result.get("status") != "ok": + return { + "status": "error", + "error": "pivot query failed: " + + str(result.get("error", "unknown")), + } + + # Paso 2 (en python): contar observaciones por fila y por columna, y guardar + # el valor agregado de cada celda (r, c). + row_obs: dict = defaultdict(int) + col_obs: dict = defaultdict(int) + cell: dict = {} + for row in result.get("rows", []): + r = row.get("r") + c = row.get("c") + n = row.get("n") or 0 + row_obs[r] += n + col_obs[c] += n + cell[(r, c)] = row.get("v") + + def _top(obs: dict, limit: int): + # Orden: mas observaciones primero; desempate estable por etiqueta. + ranked = sorted(obs.items(), key=lambda kv: (-kv[1], str(kv[0]))) + selected = [label for label, _ in ranked[:limit]] + return selected, len(ranked) > limit + + row_labels, truncated_rows = _top(row_obs, top_rows) + col_labels, truncated_cols = _top(col_obs, top_cols) + + # Paso 3: materializar la matriz; None donde la combinacion no existe. + matrix = [ + [cell.get((r, c)) for c in col_labels] for r in row_labels + ] + + note = ( + f"pivot {agg}({value}) reducida a {len(row_labels)}x{len(col_labels)} " + "(top por observaciones) para caber en PDF/slide" + ) + if agg == "count": + note = ( + f"pivot count(*) reducida a {len(row_labels)}x{len(col_labels)} " + "(top por observaciones) para caber en PDF/slide" + ) + + return { + "status": "ok", + "index": index, + "columns": columns, + "value": value, + "agg": agg, + "row_labels": row_labels, + "col_labels": col_labels, + "matrix": matrix, + "truncated_rows": truncated_rows, + "truncated_cols": truncated_cols, + "note": note, + } + except Exception as e: # noqa: BLE001 + return {"status": "error", "error": str(e)} diff --git a/python/functions/datascience/pivot_table_duckdb_test.py b/python/functions/datascience/pivot_table_duckdb_test.py new file mode 100644 index 00000000..53f85e2e --- /dev/null +++ b/python/functions/datascience/pivot_table_duckdb_test.py @@ -0,0 +1,115 @@ +"""Tests para pivot_table_duckdb.""" + +import os +import sys + +import duckdb + +# Permitir importar funciones del registry (from infra import ..., from datascience import ...). +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..")) +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "functions")) + +from datascience.pivot_table_duckdb import pivot_table_duckdb + + +def _make_db(tmp_name: str) -> str: + """Crea una DuckDB con dos categoricas (a, b) y un valor numerico conocido. + + Filas: + a='x', b='y', val=10 + a='x', b='y', val=20 -> mean(x,y) = 15, count(x,y) = 2 + a='x', b='z', val=5 -> mean(x,z) = 5 + a='w', b='y', val=100 -> mean(w,y) = 100 + Observaciones por a: x=3, w=1. Por b: y=3, z=1. + La combinacion (w, z) no existe -> celda None. + """ + db = os.path.join("/tmp", tmp_name) + if os.path.exists(db): + os.remove(db) + con = duckdb.connect(db) + con.execute("CREATE TABLE t (a VARCHAR, b VARCHAR, val DOUBLE)") + con.execute( + "INSERT INTO t VALUES " + "('x','y',10),('x','y',20),('x','z',5),('w','y',100)" + ) + con.close() + return db + + +def test_pivot_mean_labels_y_celda_conocida(): + db = _make_db("pivot_test_mean.duckdb") + res = pivot_table_duckdb(db, "t", index="a", columns="b", value="val", agg="mean") + assert res["status"] == "ok", res + # Filas ordenadas por observaciones desc: x (3) antes que w (1). + assert res["row_labels"] == ["x", "w"], res["row_labels"] + # Columnas ordenadas por observaciones desc: y (3) antes que z (1). + assert res["col_labels"] == ["y", "z"], res["col_labels"] + # matrix[0][0] = mean(a='x', b='y') = (10 + 20) / 2 = 15. + assert abs(res["matrix"][0][0] - 15.0) < 1e-9, res["matrix"] + # matrix[0][1] = mean(a='x', b='z') = 5. + assert abs(res["matrix"][0][1] - 5.0) < 1e-9, res["matrix"] + # matrix[1][0] = mean(a='w', b='y') = 100. + assert abs(res["matrix"][1][0] - 100.0) < 1e-9, res["matrix"] + # (w, z) no existe -> None. + assert res["matrix"][1][1] is None, res["matrix"] + # Sin truncado con los defaults (top_rows=10, top_cols=8). + assert res["truncated_rows"] is False + assert res["truncated_cols"] is False + # La matriz es rectangular consistente con las etiquetas. + assert len(res["matrix"]) == len(res["row_labels"]) + for fila in res["matrix"]: + assert len(fila) == len(res["col_labels"]) + + +def test_pivot_trunca_a_top_rows_y_top_cols(): + db = _make_db("pivot_test_trunc.duckdb") + res = pivot_table_duckdb( + db, "t", index="a", columns="b", value="val", agg="mean", + top_rows=1, top_cols=1, + ) + assert res["status"] == "ok", res + # Solo la fila/columna mas frecuente sobrevive. + assert res["row_labels"] == ["x"], res["row_labels"] + assert res["col_labels"] == ["y"], res["col_labels"] + assert res["matrix"] == [[15.0]], res["matrix"] + # Habia mas de 1 fila y mas de 1 columna -> truncado en ambos ejes. + assert res["truncated_rows"] is True + assert res["truncated_cols"] is True + + +def test_pivot_count_no_necesita_value_real(): + db = _make_db("pivot_test_count.duckdb") + # value apunta a una columna real pero count(*) la ignora; tambien valdria un + # nombre cualquiera. Verificamos que count funciona igualmente. + res = pivot_table_duckdb( + db, "t", index="a", columns="b", value="val", agg="count" + ) + assert res["status"] == "ok", res + assert res["row_labels"] == ["x", "w"] + assert res["col_labels"] == ["y", "z"] + # count(a='x', b='y') = 2 observaciones. + assert res["matrix"][0][0] == 2, res["matrix"] + # count(a='x', b='z') = 1. + assert res["matrix"][0][1] == 1, res["matrix"] + # count(a='w', b='y') = 1. + assert res["matrix"][1][0] == 1, res["matrix"] + # (w, z) no existe -> None. + assert res["matrix"][1][1] is None, res["matrix"] + + +def test_pivot_db_inexistente_devuelve_error_sin_lanzar(): + res = pivot_table_duckdb( + "/nonexistent/path/does_not_exist.duckdb", + "t", index="a", columns="b", value="val", agg="mean", + ) + assert res["status"] == "error", res + assert isinstance(res["error"], str) + + +def test_pivot_agg_invalido_devuelve_error(): + db = _make_db("pivot_test_badagg.duckdb") + res = pivot_table_duckdb( + db, "t", index="a", columns="b", value="val", agg="stddev" + ) + assert res["status"] == "error", res + assert "invalid agg" in res["error"] diff --git a/python/functions/datascience/select_groupby_keys.md b/python/functions/datascience/select_groupby_keys.md new file mode 100644 index 00000000..d97332eb --- /dev/null +++ b/python/functions/datascience/select_groupby_keys.md @@ -0,0 +1,158 @@ +--- +id: select_groupby_keys_py_datascience +name: select_groupby_keys +kind: function +lang: py +domain: datascience +version: "1.0.0" +purity: pure +signature: "def select_groupby_keys(profile: dict, max_keys: int = 3, max_card: int = 20, max_measures: int = 4) -> dict" +description: "Elige deterministicamente las columnas categoricas mas interesantes para GROUP BY, las numericas medida y pares pivote a partir de un TableProfile del grupo eda. Respaldo cuantitativo para el capitulo de agregacion/OLAP de un EDA. Funcion pura, no muta el input, nunca lanza." +tags: [eda, aggregation, groupby, olap, profiling, datascience] +uses_functions: [] +uses_types: [] +returns: [] +returns_optional: false +error_type: "" +imports: [] +example: | + from datascience import select_groupby_keys + profile = { + "n_rows": 891, + "key_candidates": ["passenger_id"], + "columns": [ + {"name": "sex", "inferred_type": "categorical", "distinct_count": 2, + "unique_pct": 0.002, "null_pct": 0.0, "flags": [], + "categorical": {"imbalance": 1.8}, "numeric": None}, + {"name": "pclass", "inferred_type": "categorical", "distinct_count": 3, + "unique_pct": 0.003, "null_pct": 0.0, "flags": [], + "categorical": {"imbalance": 2.5}, "numeric": None}, + {"name": "fare", "inferred_type": "numeric", "distinct_count": 200, + "unique_pct": 0.2, "null_pct": 0.0, "flags": [], + "numeric": {"std": 49.7, "cv": 1.54}, "categorical": None}, + ], + } + select_groupby_keys(profile) + # {"group_keys": [{"col": "sex", ...}, {"col": "pclass", ...}], + # "measures": ["fare"], + # "pivots": [{"index": "sex", "columns": "pclass", "value": "fare"}], + # "note": "2 clave(s) de grupo: sex, pclass; 1 medida(s): fare; 1 pivot(s)."} +tested: true +tests: + - "test_titanic_picks_good_cats_excludes_id_and_constant" + - "test_titanic_measures_exclude_id_constant_and_keep_numerics" + - "test_titanic_generates_one_pivot" + - "test_empty_profile_returns_all_empty_and_does_not_crash" + - "test_none_profile_does_not_crash" + - "test_only_numerics_yields_empty_group_keys_and_no_pivots" + - "test_high_cardinality_and_max_card_are_excluded" + - "test_max_keys_limits_group_keys" + - "test_three_keys_cap_pivots_to_two" + - "test_does_not_mutate_input" +test_file_path: "python/functions/datascience/select_groupby_keys_test.py" +file_path: "python/functions/datascience/select_groupby_keys.py" +params: + - name: profile + desc: > + TableProfile dict del grupo eda (p.ej. salida de summarize_table_duckdb). + Se lee de forma defensiva (.get / or [] / isinstance). Claves usadas: + columns (list[ColumnProfile]), key_candidates (list de nombres de columna + o dicts {name}), n_rows. Cada ColumnProfile usa: name, inferred_type + ("numeric"|"categorical"|"datetime"|"text"|"boolean"), distinct_count, + unique_pct (0..1), null_pct (0..1), flags (list[str], reconoce + "possible_id"/"high_cardinality"/"constant"), numeric ({std, cv, ...}|None) + y categorical ({imbalance, mode_pct, ...}|None). + - name: max_keys + desc: "Numero maximo de claves de grupo (group_keys) a devolver. Default 3." + - name: max_card + desc: > + Cardinalidad maxima (distinct_count) que una columna categorica puede + tener para seguir siendo candidata a clave de grupo. Default 20. + - name: max_measures + desc: "Numero maximo de columnas medida (nombres) a devolver. Default 4." +output: > + dict con group_keys (list de {col, cardinality, score} ordenada por score + desc), measures (list[str] de nombres de columnas numericas ordenadas por + dispersion), pivots (list de {index, columns, value}, hasta 2 pares + categorica x categorica con la primera measure como valor) y note (str, + resumen corto en espanol de lo elegido). Ante profile vacio/None devuelve + todas las listas vacias y una note descriptiva; nunca lanza. +--- + +## Ejemplo + +```python +from datascience import select_groupby_keys + +# TableProfile estilo titanic: 2 categoricas buenas, 1 numerica medida, +# 1 id secuencial (descartado) y un key_candidate declarado. +profile = { + "n_rows": 891, + "key_candidates": ["passenger_id"], + "columns": [ + {"name": "sex", "inferred_type": "categorical", "distinct_count": 2, + "unique_pct": 0.002, "null_pct": 0.0, "flags": [], + "categorical": {"imbalance": 1.8}, "numeric": None}, + {"name": "pclass", "inferred_type": "categorical", "distinct_count": 3, + "unique_pct": 0.003, "null_pct": 0.0, "flags": [], + "categorical": {"imbalance": 2.5}, "numeric": None}, + {"name": "fare", "inferred_type": "numeric", "distinct_count": 200, + "unique_pct": 0.2, "null_pct": 0.0, "flags": [], + "numeric": {"std": 49.7, "cv": 1.54}, "categorical": None}, + {"name": "passenger_id", "inferred_type": "numeric", "distinct_count": 891, + "unique_pct": 1.0, "null_pct": 0.0, "flags": ["possible_id"], + "numeric": {"std": 257.4, "cv": 0.58}, "categorical": None}, + ], +} + +select_groupby_keys(profile) +# { +# "group_keys": [ +# {"col": "sex", "cardinality": 2, "score": 0.5556}, +# {"col": "pclass", "cardinality": 3, "score": 0.4}, +# ], +# "measures": ["fare"], # passenger_id excluido (id secuencial) +# "pivots": [{"index": "sex", "columns": "pclass", "value": "fare"}], +# "note": "2 clave(s) de grupo: sex, pclass; 1 medida(s): fare; 1 pivot(s).", +# } +``` + +## Cuando usarla + +Cuando hayas perfilado una tabla con el grupo `eda` (p.ej. +`summarize_table_duckdb`) y necesites decidir, sin mirar los datos, por qué +columnas merece la pena agrupar (GROUP BY) y qué métricas numéricas agregar: +el respaldo cuantitativo del capítulo de agregación/OLAP de un AutomaticEDA, o +para proponer pivotes en un dashboard. Es la capa de selección sobre el +TableProfile crudo: lee el perfil, ordena candidatos de forma determinista y +no toca los datos. + +## Notas + +Función pura, sin I/O ni dependencias externas (solo stdlib), no muta +`profile`. Lectura defensiva total (`.get`, `or []`, `isinstance`): un `{}` o +`None` produce `{"group_keys": [], "measures": [], "pivots": [], "note": ...}` +y nunca lanza. + +Criterios de selección (deterministas): + +- **group_keys** — candidatas con `inferred_type` en `("categorical","boolean")`. + Se descartan las que estén en `key_candidates`, con flag + `possible_id`/`high_cardinality`/`constant`, con `distinct_count` fuera de + `[2, max_card]`, o all-null (`null_pct >= 0.999`). `score = card_score * + balance_score`: `card_score` mantiene un plateau para cardinalidad moderada + (2..12) y decae hacia `max_card`; `balance_score = 1/imbalance` usando + `categorical.imbalance` si está, aproximando con `mode_pct` si no, o un valor + neutro (0.5) en último caso. Devuelve hasta `max_keys`, ordenadas por score + desc (empates por orden de columna). +- **measures** — candidatas con `inferred_type` en + `("numeric","integer","float")`. Se descartan id-like (flag `possible_id` y + `unique_pct >= 0.99`) y constantes (`numeric.std` == 0 o None). Se rankean por + dispersión informativa: `abs(cv)` si está, si no `abs(std)`. Devuelve hasta + `max_measures` **nombres** (strings). +- **pivots** — hasta 2 pares `(group_keys[i].col, group_keys[j].col)` con i dict: + """Select GROUP BY keys, measures and pivot pairs from a TableProfile. + + Reads everything defensively (``.get(...)``, ``or []``, ``isinstance``) and + never raises. With an empty/None profile it returns every list empty. + + Selection rules (deterministic): + + - **group_keys** (categorical columns to group by): candidates have + ``inferred_type`` in ``("categorical", "boolean")``. Discarded if they are + in ``profile['key_candidates']``, carry a ``possible_id`` / + ``high_cardinality`` / ``constant`` flag, have ``distinct_count`` outside + ``[2, max_card]``, or are all-null (``null_pct >= 0.999``). Each survivor + gets ``score = card_score * balance_score`` where ``card_score`` keeps a + plateau for moderate cardinality (2..12) and decays towards ``max_card``, + and ``balance_score = 1 / imbalance`` (``categorical.imbalance`` if + present, else approximated from ``mode_pct``, else a neutral default). + The top ``max_keys`` by score (desc, ties by column order) are returned. + + - **measures** (numeric columns to aggregate): candidates have + ``inferred_type`` in ``("numeric", "integer", "float")``. Discarded if + id-like (``possible_id`` flag *and* ``unique_pct >= 0.99``) or constant + (``numeric.std`` is ``0`` or ``None``). Ranked by informative dispersion: + ``abs(cv)`` when available, else ``abs(std)``. The top ``max_measures`` + **names** are returned. + + - **pivots**: up to 2 ``(group_keys[i].col, group_keys[j].col)`` pairs with + ``i < j``, using the first measure as the aggregated value. Empty when + fewer than 2 group keys were selected. + + Args: + profile: TableProfile dict of the ``eda`` group. Relevant keys: + ``columns`` (list[ColumnProfile]), ``key_candidates`` (list of + column names or ``{name}`` dicts), ``n_rows``. Each ColumnProfile + uses: ``name``, ``inferred_type``, ``distinct_count``, + ``unique_pct`` (0..1), ``null_pct`` (0..1), ``flags`` (list[str]), + ``numeric`` ({std, cv, ...}|None), ``categorical`` + ({imbalance, mode_pct, ...}|None). + max_keys: Maximum number of group-by keys to return. Default 3. + max_card: Maximum cardinality (``distinct_count``) a categorical column + may have to still qualify as a group key. Default 20. + max_measures: Maximum number of measure names to return. Default 4. + + Returns: + dict with: + group_keys (list[{col, cardinality, score}], ordered by score desc), + measures (list[str], numeric column names ordered by dispersion), + pivots (list[{index, columns, value}], up to 2 pairs), + note (str, short summary of what was chosen). + """ + if not isinstance(profile, dict): + profile = {} + + try: + max_keys = int(max_keys) + except (TypeError, ValueError): + max_keys = 3 + try: + max_card = int(max_card) + except (TypeError, ValueError): + max_card = 20 + try: + max_measures = int(max_measures) + except (TypeError, ValueError): + max_measures = 4 + max_keys = max(max_keys, 0) + max_card = max(max_card, 2) + max_measures = max(max_measures, 0) + + columns = profile.get("columns") or [] + if not isinstance(columns, (list, tuple)): + columns = [] + + key_names = _key_candidate_names(profile.get("key_candidates")) + + group_keys = _select_group_keys(columns, key_names, max_keys, max_card) + measures = _select_measures(columns, max_measures) + pivots = _select_pivots(group_keys, measures) + + return { + "group_keys": group_keys, + "measures": measures, + "pivots": pivots, + "note": _build_note(group_keys, measures, pivots), + } + + +# --------------------------------------------------------------------------- +# group_keys +# --------------------------------------------------------------------------- + +_GROUP_TYPES = ("categorical", "boolean") +_DISQUALIFYING_FLAGS = frozenset({"possible_id", "high_cardinality", "constant"}) +_CARD_PLATEAU_HI = 12 # cardinalities 2..12 are all "moderate" (best). + + +def _select_group_keys(columns, key_names, max_keys, max_card) -> list: + """Rank categorical/boolean columns suitable for GROUP BY.""" + scored = [] + for idx, col in enumerate(columns): + if not isinstance(col, dict): + continue + if (col.get("inferred_type") or "") not in _GROUP_TYPES: + continue + + name = col.get("name") + if name is None: + continue + if name in key_names: + continue + + flags = _as_set(col.get("flags")) + if flags & _DISQUALIFYING_FLAGS: + continue + + if _num(col.get("null_pct"), 0.0) >= 0.999: + continue + + card = _num(col.get("distinct_count"), 0.0) + if card < 2 or card > max_card: + continue + card_i = int(card) + + score = _card_score(card_i, max_card) * _balance_score(col.get("categorical")) + scored.append((round(score, 6), idx, name, card_i)) + + # Deterministic: higher score first, ties broken by original column order. + scored.sort(key=lambda t: (-t[0], t[1])) + + out = [] + for score, _idx, name, card_i in scored[:max_keys]: + out.append({"col": name, "cardinality": card_i, "score": score}) + return out + + +def _card_score(card: int, max_card: int) -> float: + """Prefer moderate cardinality; plateau at 2..12, decay towards max_card.""" + if card <= 1: + return 0.0 + if card <= _CARD_PLATEAU_HI: + return 1.0 + denom = max(max_card - _CARD_PLATEAU_HI, 1) + over = card - _CARD_PLATEAU_HI + return max(0.1, 1.0 - over / denom) + + +def _balance_score(categorical) -> float: + """1.0 for a perfectly balanced category, decaying as imbalance grows. + + Uses ``categorical.imbalance`` (max_count/min_count, >= 1) when available; + otherwise approximates from ``mode_pct`` (top-class dominance); otherwise a + neutral default so the column is still selectable. + """ + if isinstance(categorical, dict): + imbalance = categorical.get("imbalance") + if isinstance(imbalance, (int, float)) and imbalance >= 1.0: + return 1.0 / float(imbalance) + mode_pct = categorical.get("mode_pct") + if isinstance(mode_pct, (int, float)): + return _clamp(1.0 - float(mode_pct), 0.0, 1.0) + return 0.5 + + +# --------------------------------------------------------------------------- +# measures +# --------------------------------------------------------------------------- + +_NUMERIC_TYPES = ("numeric", "integer", "float") + + +def _select_measures(columns, max_measures) -> list: + """Rank numeric columns by informative dispersion (cv, else std).""" + scored = [] + for idx, col in enumerate(columns): + if not isinstance(col, dict): + continue + if (col.get("inferred_type") or "") not in _NUMERIC_TYPES: + continue + + name = col.get("name") + if name is None: + continue + + flags = _as_set(col.get("flags")) + unique_pct = _num(col.get("unique_pct"), 0.0) + if "possible_id" in flags and unique_pct >= 0.99: + continue # sequential id, not a measure. + + numeric = col.get("numeric") + std = numeric.get("std") if isinstance(numeric, dict) else None + if not isinstance(std, (int, float)) or std == 0: + continue # constant or unknown spread -> not informative. + + cv = numeric.get("cv") if isinstance(numeric, dict) else None + if isinstance(cv, (int, float)): + dispersion = abs(float(cv)) + else: + dispersion = abs(float(std)) + + scored.append((dispersion, idx, name)) + + # Higher dispersion first, ties broken by original column order. + scored.sort(key=lambda t: (-t[0], t[1])) + return [name for _disp, _idx, name in scored[:max_measures]] + + +# --------------------------------------------------------------------------- +# pivots +# --------------------------------------------------------------------------- + + +def _select_pivots(group_keys, measures) -> list: + """Up to 2 (cat_a, cat_b) pairs from the chosen group keys.""" + if not isinstance(group_keys, list) or len(group_keys) < 2: + return [] + value = measures[0] if measures else None + pairs = [] + n = len(group_keys) + for i in range(n): + for j in range(i + 1, n): + pairs.append({ + "index": group_keys[i].get("col"), + "columns": group_keys[j].get("col"), + "value": value, + }) + if len(pairs) >= 2: + return pairs + return pairs + + +# --------------------------------------------------------------------------- +# helpers +# --------------------------------------------------------------------------- + + +def _build_note(group_keys, measures, pivots) -> str: + """One-line Spanish summary of the selection.""" + parts = [] + if group_keys: + cols = ", ".join(str(g.get("col")) for g in group_keys) + parts.append(f"{len(group_keys)} clave(s) de grupo: {cols}") + else: + parts.append("sin categóricas agrupables") + if measures: + parts.append(f"{len(measures)} medida(s): " + ", ".join(str(m) for m in measures)) + else: + parts.append("sin medidas numéricas") + if pivots: + parts.append(f"{len(pivots)} pivot(s)") + return "; ".join(parts) + "." + + +def _key_candidate_names(key_candidates) -> set: + """Normalize ``key_candidates`` (strings or ``{name}`` dicts) to a name set.""" + names = set() + if not isinstance(key_candidates, (list, tuple)): + return names + for entry in key_candidates: + if isinstance(entry, str): + names.add(entry) + elif isinstance(entry, dict): + nm = entry.get("name") or entry.get("col") + if nm is not None: + names.add(nm) + return names + + +def _as_set(flags) -> set: + """Coerce a flags value into a set, tolerating None / non-iterables.""" + if isinstance(flags, (list, tuple, set)): + return set(flags) + return set() + + +def _num(value, default: float) -> float: + """Best-effort float conversion with a fallback default.""" + if value is None: + return default + try: + return float(value) + except (TypeError, ValueError): + return default + + +def _clamp(x: float, lo: float, hi: float) -> float: + """Recorta x al rango [lo, hi].""" + if x < lo: + return lo + if x > hi: + return hi + return x diff --git a/python/functions/datascience/select_groupby_keys_test.py b/python/functions/datascience/select_groupby_keys_test.py new file mode 100644 index 00000000..0aaf167d --- /dev/null +++ b/python/functions/datascience/select_groupby_keys_test.py @@ -0,0 +1,213 @@ +"""Tests para select_groupby_keys (grupo eda, dominio datascience).""" + +import os +import sys + +sys.path.insert(0, os.path.dirname(__file__)) + +from select_groupby_keys import select_groupby_keys + + +def _cat_col(name, card, *, imbalance=2.0, flags=None, null_pct=0.0): + """ColumnProfile categorico minimo con bloque categorical.""" + return { + "name": name, + "inferred_type": "categorical", + "distinct_count": card, + "unique_pct": card / 1000.0, + "null_pct": null_pct, + "flags": flags or [], + "numeric": None, + "categorical": {"imbalance": imbalance, "mode_pct": 0.5, "n_distinct": card}, + } + + +def _num_col(name, *, std, cv, flags=None, unique_pct=0.1): + """ColumnProfile numerico minimo con bloque numeric.""" + return { + "name": name, + "inferred_type": "numeric", + "distinct_count": 200, + "unique_pct": unique_pct, + "null_pct": 0.0, + "flags": flags or [], + "numeric": {"std": std, "cv": cv}, + "categorical": None, + } + + +def _titanic_like_profile() -> dict: + """Perfil estilo titanic: 2 categoricas buenas, 2 numericas, 1 id, 1 constante.""" + return { + "n_rows": 891, + "key_candidates": ["passenger_id"], + "columns": [ + _cat_col("sex", 2, imbalance=1.8), + _cat_col("pclass", 3, imbalance=2.5), + _num_col("age", std=14.5, cv=0.49), + _num_col("fare", std=49.7, cv=1.54), + # id secuencial: flag possible_id + unique_pct alto. + { + "name": "passenger_id", + "inferred_type": "numeric", + "distinct_count": 891, + "unique_pct": 1.0, + "null_pct": 0.0, + "flags": ["possible_id"], + "numeric": {"std": 257.4, "cv": 0.58}, + "categorical": None, + }, + # columna constante: flag constant + std 0. + { + "name": "embarked_const", + "inferred_type": "categorical", + "distinct_count": 1, + "unique_pct": 0.001, + "null_pct": 0.0, + "flags": ["constant"], + "numeric": None, + "categorical": {"imbalance": 1.0}, + }, + ], + } + + +def test_titanic_picks_good_cats_excludes_id_and_constant(): + out = select_groupby_keys(_titanic_like_profile()) + + # Elige las dos categoricas buenas. + chosen_cols = {g["col"] for g in out["group_keys"]} + assert chosen_cols == {"sex", "pclass"} + + # Excluye la constante y el key_candidate. + assert "embarked_const" not in chosen_cols + assert "passenger_id" not in chosen_cols + + # Cada group key trae col, cardinality y score. + for g in out["group_keys"]: + assert set(g.keys()) == {"col", "cardinality", "score"} + assert isinstance(g["score"], float) + by_col = {g["col"]: g for g in out["group_keys"]} + assert by_col["sex"]["cardinality"] == 2 + assert by_col["pclass"]["cardinality"] == 3 + + # Ordenadas por score descendente. + scores = [g["score"] for g in out["group_keys"]] + assert scores == sorted(scores, reverse=True) + + +def test_titanic_measures_exclude_id_constant_and_keep_numerics(): + out = select_groupby_keys(_titanic_like_profile()) + + # Solo nombres (strings) de numericas informativas, sin el id secuencial. + assert all(isinstance(m, str) for m in out["measures"]) + assert "passenger_id" not in out["measures"] + assert set(out["measures"]) == {"age", "fare"} + + # fare tiene mayor cv (1.54 > 0.49) -> primero. + assert out["measures"][0] == "fare" + + +def test_titanic_generates_one_pivot(): + out = select_groupby_keys(_titanic_like_profile()) + + # Con 2 group keys -> exactamente 1 pivot. + assert len(out["pivots"]) == 1 + pivot = out["pivots"][0] + assert set(pivot.keys()) == {"index", "columns", "value"} + assert {pivot["index"], pivot["columns"]} == {"sex", "pclass"} + # El valor es la primera measure (fare). + assert pivot["value"] == "fare" + + +def test_empty_profile_returns_all_empty_and_does_not_crash(): + out = select_groupby_keys({}) + assert out["group_keys"] == [] + assert out["measures"] == [] + assert out["pivots"] == [] + assert isinstance(out["note"], str) + + +def test_none_profile_does_not_crash(): + out = select_groupby_keys(None) + assert out == { + "group_keys": [], + "measures": [], + "pivots": [], + "note": out["note"], + } + assert isinstance(out["note"], str) + + +def test_only_numerics_yields_empty_group_keys_and_no_pivots(): + profile = { + "n_rows": 500, + "key_candidates": [], + "columns": [ + _num_col("price", std=12.0, cv=0.6), + _num_col("weight", std=3.0, cv=0.2), + ], + } + out = select_groupby_keys(profile) + assert out["group_keys"] == [] + assert out["pivots"] == [] + # Las numericas si se eligen como measures. + assert set(out["measures"]) == {"price", "weight"} + assert out["measures"][0] == "price" # mayor cv. + + +def test_high_cardinality_and_max_card_are_excluded(): + profile = { + "n_rows": 1000, + "key_candidates": [], + "columns": [ + _cat_col("city", 50, flags=["high_cardinality"]), # flag -> fuera. + _cat_col("zone", 35), # card 35 > max_card 20 -> fuera. + _cat_col("region", 5), # valida. + ], + } + out = select_groupby_keys(profile, max_card=20) + assert {g["col"] for g in out["group_keys"]} == {"region"} + + +def test_max_keys_limits_group_keys(): + profile = { + "n_rows": 1000, + "key_candidates": [], + "columns": [ + _cat_col("a", 4, imbalance=1.0), + _cat_col("b", 5, imbalance=1.2), + _cat_col("c", 6, imbalance=1.5), + _cat_col("d", 7, imbalance=2.0), + ], + } + out = select_groupby_keys(profile, max_keys=2) + assert len(out["group_keys"]) == 2 + # Hasta 2 pivots con >=2 keys (aqui exactamente 1 par posible entre 2 keys). + assert len(out["pivots"]) == 1 + + +def test_three_keys_cap_pivots_to_two(): + profile = { + "n_rows": 1000, + "key_candidates": [], + "columns": [ + _cat_col("a", 4, imbalance=1.0), + _cat_col("b", 5, imbalance=1.1), + _cat_col("c", 6, imbalance=1.2), + _num_col("m", std=10.0, cv=0.5), + ], + } + out = select_groupby_keys(profile, max_keys=3) + assert len(out["group_keys"]) == 3 + # 3 keys -> 3 pares posibles, capado a 2. + assert len(out["pivots"]) == 2 + for p in out["pivots"]: + assert p["value"] == "m" + + +def test_does_not_mutate_input(): + profile = _titanic_like_profile() + before = repr(profile) + select_groupby_keys(profile) + assert repr(profile) == before diff --git a/python/functions/datascience/suggest_aggregations_llm.md b/python/functions/datascience/suggest_aggregations_llm.md new file mode 100644 index 00000000..2b4a79fd --- /dev/null +++ b/python/functions/datascience/suggest_aggregations_llm.md @@ -0,0 +1,96 @@ +--- +name: suggest_aggregations_llm +kind: function +lang: py +domain: datascience +version: "1.0.0" +purity: impure +signature: "def suggest_aggregations_llm(profile: dict, candidates: dict, max_aggs: int = 4, model: str = \"claude-haiku-4-5-20251001\") -> dict" +description: "MUST-11.1 del capitulo AGREGACION del AutomaticEDA (grupo eda). Dado el TableProfile de una tabla y los candidatos cuantitativos de select_groupby_keys ({group_keys:[{col,cardinality,score}], measures:[str], pivots:[{index,columns,value}]}), con UNA sola llamada al LLM elige y ordena las K agregaciones (GROUP BY categorica x medidas numericas) y los pivots MAS INFORMATIVOS para un analisis de grupos, con una razon corta cada uno, evitando la explosion combinatoria (no todo contra todo). Privacidad/coste: NO envia filas crudas, solo el resumen AGREGADO de los candidatos (tabla, columnas categoricas con cardinalidad/score, medidas, pivots). Reusa ask_llm del grupo claude-direct (API directa con token OAuth de Claude). Impura, dict-no-throw: NUNCA lanza y SIEMPRE devuelve algo usable; si el LLM falla, el JSON no parsea o no hay seleccion valida, cae a un fallback determinista construido desde los candidatos (source='fallback'). Toda columna que el LLM invente se descarta." +tags: [eda, claude-direct, llm, aggregation, groupby, pivot, datascience, automatic-eda] +params: + - name: profile + desc: "TableProfile del grupo eda. Solo se usa profile['table'] para nombrar la tabla en el prompt; puede ir vacio o sin esa clave (se usa '(tabla sin nombre)')." + - name: candidates + desc: "Salida de select_groupby_keys: {group_keys:[{col, cardinality, score}], measures:[str], pivots:[{index, columns, value}]}. group_keys = columnas categoricas candidatas para GROUP BY; measures = columnas numericas a agregar (sum/avg); pivots = cruces index x columns -> value sugeridos. Cualquier columna que el LLM elija debe existir aqui o se descarta. None o no-dict se trata como vacio." + - name: max_aggs + desc: "Tope de agregaciones a devolver. Default 4. Valores <1 o no-int se normalizan a 4. Limita tanto la seleccion del LLM como el fallback determinista, para evitar la explosion combinatoria." + - name: model + desc: "id del modelo Anthropic a usar en la unica llamada. Default 'claude-haiku-4-5-20251001' (haiku, coste bajo, ~2-3s). Para razones mas finas, pasar p.ej. 'claude-opus-4-8'." +output: "dict dict-no-throw: {status:'ok', source:'llm'|'fallback', aggregations:[{group_by:str, measures:[str], why:str}], pivots:[{index:str, columns:str, value:str|None, why:str}], note:str}. source=='llm' si el LLM produjo al menos una agregacion valida (columnas existentes en candidates); en cualquier otro caso (LLM caido, JSON invalido, seleccion vacia, sin candidatos) source=='fallback' y aggregations/pivots se derivan de candidates con why='selección cuantitativa (sin LLM)'. NUNCA lanza." +uses_functions: [ask_llm_py_core, select_groupby_keys_py_datascience] +uses_types: [] +returns: [] +returns_optional: false +error_type: "error_go_core" +imports: [] +tested: true +tests: ["test_extract_json_object", "test_extract_json_wrapped_in_fences_and_junk", "test_extract_json_non_json_returns_none", "test_validate_aggregations_drops_invalid_columns", "test_llm_path_uses_selection", "test_llm_path_respects_max_aggs", "test_llm_invented_column_is_discarded", "test_fallback_on_empty_llm_response", "test_fallback_on_unparseable_response", "test_fallback_respects_max_aggs", "test_fallback_when_llm_raises", "test_no_candidates_returns_empty_fallback", "test_non_dict_candidates_does_not_raise"] +test_file_path: "python/functions/datascience/suggest_aggregations_llm_test.py" +file_path: "python/functions/datascience/suggest_aggregations_llm.py" +--- + +## Ejemplo + +```python +import sys, os +sys.path.insert(0, os.path.join("python", "functions")) + +from datascience.suggest_aggregations_llm import suggest_aggregations_llm + +profile = {"table": "ventas"} + +# candidates = salida de select_groupby_keys (aqui literal de ejemplo). +candidates = { + "group_keys": [ + {"col": "categoria", "cardinality": 8, "score": 0.91}, + {"col": "region", "cardinality": 5, "score": 0.74}, + {"col": "canal", "cardinality": 3, "score": 0.60}, + ], + "measures": ["importe", "unidades"], + "pivots": [ + {"index": "categoria", "columns": "region", "value": "importe"}, + ], +} + +out = suggest_aggregations_llm(profile, candidates, max_aggs=4) # haiku por defecto + +print("fuente:", out["source"]) # "llm" o "fallback" si no hay red +for agg in out["aggregations"]: + print(f"GROUP BY {agg['group_by']} -> {agg['measures']} ({agg['why']})") +for piv in out["pivots"]: + print(f"pivot {piv['index']} x {piv['columns']} = {piv['value']} ({piv['why']})") +``` + +## Cuando usarla + +Justo despues de `select_groupby_keys` en el capitulo AGREGACION del AutomaticEDA: +cuando ya tienes los candidatos cuantitativos (columnas categoricas con cardinalidad, +medidas numericas y pivots posibles) y quieres que un LLM se quede con las K +agregaciones y pivots MAS INFORMATIVOS en vez de generar "todo contra todo". Usala para +priorizar el plan de analisis de grupos antes de materializar las tablas con +`aggregate_by_group` / pivots, manteniendo el coste y el ruido bajos. Si no hay red o +credenciales, sigue funcionando con un fallback determinista, asi que es seguro +ponerla en un pipeline. + +## Gotchas + +- **Impura: hace 1 llamada de red al LLM.** No es determinista ni gratis. Latencia + tipica ~2-3s con haiku. Una sola llamada cubre toda la seleccion. +- **Requiere token OAuth de Claude** en `~/.claude/.credentials.json` (via `ask_llm` / + grupo `claude-direct`). Sin token / sin red NO lanza: cae al **fallback + determinista** (`source="fallback"`) construido desde `candidates` + (group_keys x measures hasta `max_aggs`, pivots tal cual) con + `why="selección cuantitativa (sin LLM)"`. Comprueba `out["source"]` para saber si la + seleccion vino del LLM o del fallback. +- **NO envia filas crudas al LLM**, solo el resumen AGREGADO de los candidatos. Esto + exige que `candidates` venga ya calculado por `select_groupby_keys` (cardinalidades, + scores, medidas, pivots). +- **Valida columnas inventadas**: si el LLM propone un `group_by`/`measure`/`index`/ + `columns` que no esta en `candidates`, esa entrada se descarta (las medidas se + recortan a las validas). Si tras validar no queda ninguna agregacion, cae al + fallback completo. +- **`max_aggs` acota la explosion combinatoria** tanto en el camino LLM como en el + fallback. Subirlo demasiado reintroduce el ruido que esta funcion evita. +- **Modelo `haiku` por defecto** para coste bajo; sube a `claude-opus-4-8` si necesitas + razones (`why`) mas finas (mas caro y lento). diff --git a/python/functions/datascience/suggest_aggregations_llm.py b/python/functions/datascience/suggest_aggregations_llm.py new file mode 100644 index 00000000..b7fc4ac5 --- /dev/null +++ b/python/functions/datascience/suggest_aggregations_llm.py @@ -0,0 +1,405 @@ +"""suggest_aggregations_llm — el LLM elige las agregaciones mas informativas (grupo `eda`). + +MUST-11.1 del capitulo AGREGACION del AutomaticEDA. Dado el `TableProfile` de una +tabla y los CANDIDATOS cuantitativos que produce `select_groupby_keys` +(`{group_keys:[{col,cardinality,score}], measures:[str], pivots:[{index,columns,value}]}`), +con UNA sola llamada al LLM elige y ordena las K agregaciones (GROUP BY categorica x +medidas numericas) y los pivots MAS INFORMATIVOS para un analisis de grupos, con una +razon corta cada uno. El objetivo es evitar la explosion combinatoria: en vez de +"todo contra todo", el LLM se queda con lo que mas informa. + +Privacidad y coste: NO se envian filas crudas al LLM. El prompt solo lleva el resumen +AGREGADO de los candidatos (nombre de la tabla, columnas categoricas con su +cardinalidad/score, medidas y pivots posibles). Una sola llamada barata. + +Reusa `ask_llm` del registry (grupo claude-direct, API directa con el token OAuth de +Claude en ~/.claude/.credentials.json, arranque 0). Impura: una llamada de red. + +Estilo dict-no-throw con FALLBACK DETERMINISTA: la funcion NUNCA lanza y SIEMPRE +devuelve algo usable. Si `ask_llm` falla (devuelve ""), el JSON no parsea, o el LLM no +produce ninguna seleccion valida, se construye la respuesta directamente desde los +candidatos (group_keys x measures hasta max_aggs, pivots tal cual) con +`source="fallback"`. Ademas, toda columna que el LLM invente (no presente en los +candidatos) se descarta. +""" + +import json + +from core.ask_llm import ask_llm + +_SYSTEM = ( + "Eres un analista de datos conciso. Te dan los CANDIDATOS AGREGADOS de una tabla " + "(columnas categoricas para GROUP BY con su cardinalidad, medidas numericas y " + "pivots posibles) y eliges las agregaciones y pivots MAS INFORMATIVOS para " + "entender los grupos, evitando la explosion combinatoria (no todo contra todo). " + "No recibes filas crudas. Responde en espanol. Responde SIEMPRE y SOLO con un " + "unico objeto JSON valido, sin texto alrededor ni fences de markdown, con la forma " + '{"aggregations": [{"group_by": "", "measures": ["", ...], ' + '"why": ""}], "pivots": [{"index": "", "columns": "", ' + '"value": "", "why": ""}]}. Usa SOLO nombres de columna ' + "que aparezcan en los candidatos; no inventes nombres." +) + + +def _fmt_num(value) -> str: + """Formatea un numero de forma compacta para el prompt (None -> '?').""" + if value is None: + return "?" + if isinstance(value, bool): + return str(value) + if isinstance(value, float): + if value == int(value): + return str(int(value)) + return f"{value:.4g}" + return str(value) + + +def _candidate_view(candidates: dict): + """Extrae las vistas utiles de los candidatos. Funcion interna PURA. + + Devuelve la tupla (group_cols, measures, measure_set, pivots, group_keys): + - group_cols: set de nombres de columna categorica validas (de group_keys[].col). + - measures: lista de medidas numericas (str) preservando orden. + - measure_set: set de las medidas para validar pertenencia rapido. + - pivots: lista de pivots candidatos (dicts) tal cual vienen. + - group_keys: lista de dicts {col, cardinality, score} ya filtrada a entradas validas. + + Tolera estructuras incompletas o de tipo incorrecto sin lanzar. + """ + candidates = candidates if isinstance(candidates, dict) else {} + + gk_raw = candidates.get("group_keys") + group_keys = [] + if isinstance(gk_raw, list): + for gk in gk_raw: + if isinstance(gk, dict) and isinstance(gk.get("col"), str): + group_keys.append(gk) + group_cols = {gk["col"] for gk in group_keys} + + m_raw = candidates.get("measures") + measures = [m for m in m_raw if isinstance(m, str)] if isinstance(m_raw, list) else [] + measure_set = set(measures) + + p_raw = candidates.get("pivots") + pivots = p_raw if isinstance(p_raw, list) else [] + + return group_cols, measures, measure_set, pivots, group_keys + + +def _sorted_group_cols(group_keys: list) -> list: + """Nombres de columna categorica ordenados por score descendente. PURA.""" + + def _score(gk): + s = gk.get("score") + if isinstance(s, (int, float)) and not isinstance(s, bool): + return s + return 0.0 + + return [gk["col"] for gk in sorted(group_keys, key=_score, reverse=True)] + + +def _build_prompt(profile: dict, candidates: dict, max_aggs: int) -> str: + """Construye el prompt compacto SOLO con agregados. Funcion interna PURA. + + No toca red ni disco: testeable sin credenciales. Incluye el nombre de la tabla, + las columnas categoricas candidatas (con cardinalidad y score), las medidas + numericas y los pivots candidatos. Nunca filas crudas. + + Args: + profile: TableProfile (se usa solo profile['table'] para nombrar la tabla). + candidates: salida de select_groupby_keys. + max_aggs: tope de agregaciones a pedir. + + Returns: + El texto del prompt. + """ + profile = profile if isinstance(profile, dict) else {} + candidates = candidates if isinstance(candidates, dict) else {} + + table = profile.get("table") + table = str(table) if table is not None else "(tabla sin nombre)" + + lines = [ + f"Tabla: {table}", + ( + "Tarea: elegir las agregaciones (GROUP BY categorica x medidas numericas) y " + "los pivots MAS INFORMATIVOS para un analisis de grupos. Evita la explosion " + "combinatoria: NO combines todo contra todo, prioriza lo que mas informa." + ), + f"Devuelve a lo sumo {max_aggs} agregaciones.", + "", + "Columnas categoricas candidatas para GROUP BY (col: cardinalidad, score):", + ] + + group_keys = candidates.get("group_keys") or [] + for gk in group_keys: + if not isinstance(gk, dict) or not isinstance(gk.get("col"), str): + continue + lines.append( + f" - {gk['col']}: cardinalidad={_fmt_num(gk.get('cardinality'))}, " + f"score={_fmt_num(gk.get('score'))}" + ) + + measures = candidates.get("measures") or [] + lines.append("") + lines.append("Medidas numericas disponibles (para sum/avg por grupo):") + lines.append(" " + ", ".join(str(m) for m in measures if isinstance(m, str))) + + pivots = candidates.get("pivots") or [] + if pivots: + lines.append("") + lines.append("Pivots candidatos (index x columns -> value):") + for p in pivots: + if not isinstance(p, dict): + continue + lines.append( + f" - index={p.get('index')}, columns={p.get('columns')}, " + f"value={p.get('value')}" + ) + + lines.append("") + lines.append( + "Usa SOLO columnas de las listas anteriores; no inventes nombres. Responde " + "SOLO con el JSON descrito en las instrucciones del sistema." + ) + return "\n".join(lines) + + +def _extract_json(text: str): + """Extrae el primer bloque JSON (objeto o array) de la respuesta. PURA. + + Localiza el bloque que empieza antes (el primer '{' o el primer '[') y, para ese + delimitador, hace json.loads del rango hasta su ultimo cierre. Tolera texto basura + alrededor y fences ```json. NUNCA lanza: ante cualquier fallo devuelve None. + + Args: + text: respuesta cruda del LLM. + + Returns: + El objeto/lista deserializado, o None si no se pudo parsear. + """ + if not text or not isinstance(text, str): + return None + + opens = [] + i_obj = text.find("{") + if i_obj != -1: + opens.append((i_obj, "{", "}")) + i_arr = text.find("[") + if i_arr != -1: + opens.append((i_arr, "[", "]")) + opens.sort() + + for _, open_c, close_c in opens: + start = text.find(open_c) + end = text.rfind(close_c) + if start != -1 and end != -1 and end > start: + try: + return json.loads(text[start : end + 1]) + except (ValueError, TypeError): + continue + return None + + +def _validate_aggregations(raw_aggs, group_cols: set, measure_set: set, max_aggs: int) -> list: + """Filtra las agregaciones del LLM a las que usan SOLO columnas candidatas. PURA. + + Descarta cualquier agregacion cuyo group_by no este en group_cols o que no tenga + al menos una medida valida. Recorta las medidas a las presentes en measure_set. + Limita el resultado a max_aggs entradas. + """ + out = [] + if not isinstance(raw_aggs, list): + return out + for item in raw_aggs: + if not isinstance(item, dict): + continue + gb = item.get("group_by") + if not isinstance(gb, str) or gb not in group_cols: + continue # columna inventada -> se descarta + raw_measures = item.get("measures") + if isinstance(raw_measures, str): + raw_measures = [raw_measures] + if not isinstance(raw_measures, list): + continue + measures = [m for m in raw_measures if isinstance(m, str) and m in measure_set] + if not measures: + continue # sin medidas validas -> agregacion inutil + why = item.get("why") + why = str(why) if why is not None else "" + out.append({"group_by": gb, "measures": measures, "why": why}) + if len(out) >= max_aggs: + break + return out + + +def _validate_pivots(raw_pivots, group_cols: set, measure_set: set) -> list: + """Filtra los pivots del LLM a los que usan SOLO columnas candidatas. PURA. + + Descarta el pivot si index o columns no son columnas categoricas validas. Si el + value no es una medida valida, lo deja en None (un pivot de conteo sigue siendo util). + """ + out = [] + if not isinstance(raw_pivots, list): + return out + for item in raw_pivots: + if not isinstance(item, dict): + continue + idx = item.get("index") + cols = item.get("columns") + if not (isinstance(idx, str) and idx in group_cols): + continue + if not (isinstance(cols, str) and cols in group_cols): + continue + val = item.get("value") + if not (isinstance(val, str) and val in measure_set): + val = None + why = item.get("why") + why = str(why) if why is not None else "" + out.append({"index": idx, "columns": cols, "value": val, "why": why}) + return out + + +def _fallback_aggregations(group_cols_sorted: list, measures: list, max_aggs: int) -> list: + """Agregaciones deterministas: cada columna categorica x todas las medidas. PURA.""" + out = [] + for col in group_cols_sorted: + out.append( + { + "group_by": col, + "measures": list(measures), + "why": "selección cuantitativa (sin LLM)", + } + ) + if len(out) >= max_aggs: + break + return out + + +def _fallback_pivots(cand_pivots: list) -> list: + """Normaliza los pivots candidatos a la forma de salida (tal cual + why). PURA.""" + out = [] + if not isinstance(cand_pivots, list): + return out + for p in cand_pivots: + if not isinstance(p, dict): + continue + idx = p.get("index") + cols = p.get("columns") + if not (isinstance(idx, str) and isinstance(cols, str)): + continue + val = p.get("value") + if not isinstance(val, str): + val = None + out.append( + { + "index": idx, + "columns": cols, + "value": val, + "why": "selección cuantitativa (sin LLM)", + } + ) + return out + + +def suggest_aggregations_llm( + profile: dict, + candidates: dict, + max_aggs: int = 4, + model: str = "claude-haiku-4-5-20251001", +) -> dict: + """Elige las agregaciones y pivots mas informativos con UNA llamada al LLM. + + MUST-11.1 del capitulo AGREGACION del AutomaticEDA. Toma el perfil de la tabla y + los candidatos cuantitativos (salida de select_groupby_keys) y deja que el LLM + seleccione/ordene las K agregaciones (GROUP BY categorica x medidas) y los pivots + mas utiles, con una razon corta cada uno, evitando la explosion combinatoria. + + Privacidad/coste: solo viaja al LLM el resumen AGREGADO de los candidatos, nunca + filas crudas. Una sola llamada barata. + + dict-no-throw con fallback determinista: NUNCA lanza. Si el LLM falla, el JSON no + parsea, o no produce seleccion valida -> construye la respuesta desde los candidatos + (group_keys x measures hasta max_aggs, pivots tal cual) con source="fallback". Las + columnas que el LLM invente (no presentes en los candidatos) se descartan. + + Args: + profile: TableProfile del grupo eda. Solo se usa profile['table'] para nombrar + la tabla en el prompt; puede ir vacio. + candidates: salida de select_groupby_keys, con la forma + {group_keys:[{col,cardinality,score}], measures:[str], + pivots:[{index,columns,value}]}. + max_aggs: tope de agregaciones a devolver. Default 4. Valores <1 o no-int se + normalizan a 4. + model: id del modelo Anthropic. Default 'claude-haiku-4-5-20251001' (haiku, + coste bajo, ~2-3s). + + Returns: + dict {status:"ok", source:"llm"|"fallback", + aggregations:[{group_by:str, measures:[str], why:str}], + pivots:[{index:str, columns:str, value:str|None, why:str}], note:str}. + source=="llm" si el LLM produjo al menos una agregacion valida; en cualquier + otro caso "fallback". NUNCA lanza. + """ + if not isinstance(candidates, dict): + candidates = {} + if isinstance(max_aggs, bool) or not isinstance(max_aggs, int) or max_aggs < 1: + max_aggs = 4 + + group_cols, measures, measure_set, cand_pivots, group_keys = _candidate_view(candidates) + group_cols_sorted = _sorted_group_cols(group_keys) + + # Sin material suficiente para agregar: no merece la pena llamar al LLM. + if not group_cols or not measures: + return { + "status": "ok", + "source": "fallback", + "aggregations": [], + "pivots": _fallback_pivots(cand_pivots), + "note": "sin candidatos suficientes para agregar", + } + + prompt = _build_prompt(profile, candidates, max_aggs) + try: + text = ask_llm(prompt, model=model, system=_SYSTEM, echo=False) + except Exception: # noqa: BLE001 — degradacion: cualquier fallo de red/LLM. + text = "" + + parsed = _extract_json(text) + if parsed is not None: + if isinstance(parsed, dict): + raw_aggs = parsed.get("aggregations") + raw_pivots = parsed.get("pivots") + elif isinstance(parsed, list): + raw_aggs = parsed + raw_pivots = None + else: + raw_aggs = None + raw_pivots = None + + aggs = _validate_aggregations(raw_aggs, group_cols, measure_set, max_aggs) + if aggs: + pivots = _validate_pivots(raw_pivots, group_cols, measure_set) + if not pivots: + pivots = _fallback_pivots(cand_pivots) + return { + "status": "ok", + "source": "llm", + "aggregations": aggs, + "pivots": pivots, + "note": f"{len(aggs)} agregaciones y {len(pivots)} pivots seleccionados por el LLM", + } + + # Fallback determinista. + note = ( + "LLM no disponible; selección cuantitativa determinista" + if not text + else "LLM sin selección válida; selección cuantitativa determinista" + ) + return { + "status": "ok", + "source": "fallback", + "aggregations": _fallback_aggregations(group_cols_sorted, measures, max_aggs), + "pivots": _fallback_pivots(cand_pivots), + "note": note, + } diff --git a/python/functions/datascience/suggest_aggregations_llm_test.py b/python/functions/datascience/suggest_aggregations_llm_test.py new file mode 100644 index 00000000..29a4f2a9 --- /dev/null +++ b/python/functions/datascience/suggest_aggregations_llm_test.py @@ -0,0 +1,198 @@ +"""Tests para suggest_aggregations_llm. + +NO acceden a red ni a credenciales: las funciones internas (_build_prompt, +_extract_json, _validate_*, _fallback_*) son puras y testeables aisladas; la unica +via que llamaria al LLM (suggest_aggregations_llm) se prueba reemplazando el simbolo +`ask_llm` del modulo bajo prueba con una funcion simulada. Los candidatos van +literales en el test: NO se importa select_groupby_keys. + +Cubre golden (LLM ok con columnas validas), edge (max_aggs respetado, sin candidatos) +y error (LLM caido -> fallback, JSON invalido -> fallback, columna inventada -> se +descarta). Todos sin tocar la red. +""" + +import json + +import datascience.suggest_aggregations_llm as M +from datascience.suggest_aggregations_llm import ( + _extract_json, + _validate_aggregations, + suggest_aggregations_llm, +) + +# Candidatos de ejemplo con la forma que produce select_groupby_keys (literales). +_CANDIDATES = { + "group_keys": [ + {"col": "categoria", "cardinality": 8, "score": 0.91}, + {"col": "region", "cardinality": 5, "score": 0.74}, + {"col": "canal", "cardinality": 3, "score": 0.60}, + ], + "measures": ["importe", "unidades"], + "pivots": [ + {"index": "categoria", "columns": "region", "value": "importe"}, + ], +} +_PROFILE = {"table": "ventas"} + + +def _fake_returner(text): + """Devuelve un ask_llm simulado que ignora args y retorna `text`.""" + + def _fake(prompt, model="x", system="", echo=True, **kwargs): + return text + + return _fake + + +# --- _extract_json (parser puro, sin red) --- + + +def test_extract_json_object(): + obj = {"aggregations": [{"group_by": "categoria", "measures": ["importe"], "why": "x"}]} + assert _extract_json(json.dumps(obj)) == obj + + +def test_extract_json_wrapped_in_fences_and_junk(): + obj = {"aggregations": [], "pivots": []} + text = "Claro, aqui tienes:\n```json\n" + json.dumps(obj) + "\n```\nFin." + assert _extract_json(text) == obj + + +def test_extract_json_non_json_returns_none(): + assert _extract_json("no hay json aqui") is None + assert _extract_json("") is None + assert _extract_json(None) is None + + +# --- _validate_aggregations (puro) --- + + +def test_validate_aggregations_drops_invalid_columns(): + group_cols = {"categoria", "region"} + measure_set = {"importe", "unidades"} + raw = [ + {"group_by": "categoria", "measures": ["importe", "inventada"], "why": "ok"}, + {"group_by": "no_existe", "measures": ["importe"], "why": "mala"}, + {"group_by": "region", "measures": ["solo_inventada"], "why": "sin medidas"}, + ] + out = _validate_aggregations(raw, group_cols, measure_set, max_aggs=4) + # Solo sobrevive la primera, con las medidas recortadas a las validas. + assert out == [{"group_by": "categoria", "measures": ["importe"], "why": "ok"}] + + +# --- suggest_aggregations_llm: camino LLM (golden) --- + + +def test_llm_path_uses_selection(monkeypatch): + llm_obj = { + "aggregations": [ + {"group_by": "categoria", "measures": ["importe"], "why": "ventas por familia"}, + {"group_by": "region", "measures": ["importe", "unidades"], "why": "reparto geografico"}, + ], + "pivots": [ + {"index": "categoria", "columns": "region", "value": "importe", "why": "cruce clave"}, + ], + } + monkeypatch.setattr(M, "ask_llm", _fake_returner(json.dumps(llm_obj))) + + out = suggest_aggregations_llm(_PROFILE, _CANDIDATES) + assert out["status"] == "ok" + assert out["source"] == "llm" + assert out["aggregations"] == llm_obj["aggregations"] + assert out["pivots"][0]["index"] == "categoria" + assert out["pivots"][0]["why"] == "cruce clave" + + +def test_llm_path_respects_max_aggs(monkeypatch): + llm_obj = { + "aggregations": [ + {"group_by": "categoria", "measures": ["importe"], "why": "a"}, + {"group_by": "region", "measures": ["importe"], "why": "b"}, + {"group_by": "canal", "measures": ["unidades"], "why": "c"}, + ], + "pivots": [], + } + monkeypatch.setattr(M, "ask_llm", _fake_returner(json.dumps(llm_obj))) + + out = suggest_aggregations_llm(_PROFILE, _CANDIDATES, max_aggs=2) + assert out["source"] == "llm" + assert len(out["aggregations"]) == 2 + + +def test_llm_invented_column_is_discarded(monkeypatch): + # El LLM mezcla una agregacion valida con otra de columna inexistente. + llm_obj = { + "aggregations": [ + {"group_by": "categoria", "measures": ["importe"], "why": "valida"}, + {"group_by": "columna_fantasma", "measures": ["importe"], "why": "inventada"}, + ], + "pivots": [ + {"index": "fantasma", "columns": "region", "value": "importe", "why": "mala"}, + ], + } + monkeypatch.setattr(M, "ask_llm", _fake_returner(json.dumps(llm_obj))) + + out = suggest_aggregations_llm(_PROFILE, _CANDIDATES) + assert out["source"] == "llm" + # La agregacion inventada se descarta; queda solo la valida. + assert [a["group_by"] for a in out["aggregations"]] == ["categoria"] + # El pivot con index fantasma se descarta -> cae a los pivots de candidates. + assert all(p["index"] in {"categoria", "region", "canal"} for p in out["pivots"]) + + +# --- suggest_aggregations_llm: fallback determinista (error paths) --- + + +def test_fallback_on_empty_llm_response(monkeypatch): + monkeypatch.setattr(M, "ask_llm", _fake_returner("")) + + out = suggest_aggregations_llm(_PROFILE, _CANDIDATES, max_aggs=4) + assert out["status"] == "ok" + assert out["source"] == "fallback" + # Las agregaciones se derivan de candidates (una por group_key, con todas las medidas). + assert out["aggregations"][0]["group_by"] in {"categoria", "region", "canal"} + assert out["aggregations"][0]["measures"] == ["importe", "unidades"] + assert out["aggregations"][0]["why"] == "selección cuantitativa (sin LLM)" + # Pivots tal cual de candidates. + assert out["pivots"][0]["index"] == "categoria" + + +def test_fallback_on_unparseable_response(monkeypatch): + monkeypatch.setattr(M, "ask_llm", _fake_returner("esto no es JSON {roto")) + + out = suggest_aggregations_llm(_PROFILE, _CANDIDATES) + assert out["source"] == "fallback" + assert len(out["aggregations"]) >= 1 + + +def test_fallback_respects_max_aggs(monkeypatch): + monkeypatch.setattr(M, "ask_llm", _fake_returner("")) + + out = suggest_aggregations_llm(_PROFILE, _CANDIDATES, max_aggs=2) + assert out["source"] == "fallback" + assert len(out["aggregations"]) == 2 + + +def test_fallback_when_llm_raises(monkeypatch): + def _boom(*args, **kwargs): + raise RuntimeError("sin red") + + monkeypatch.setattr(M, "ask_llm", _boom) + + out = suggest_aggregations_llm(_PROFILE, _CANDIDATES) + assert out["source"] == "fallback" + assert out["aggregations"] # no vacio, no lanza + + +def test_no_candidates_returns_empty_fallback(): + # Sin red porque ni siquiera se llama al LLM (no hay material). + out = suggest_aggregations_llm(_PROFILE, {"group_keys": [], "measures": [], "pivots": []}) + assert out["status"] == "ok" + assert out["source"] == "fallback" + assert out["aggregations"] == [] + + +def test_non_dict_candidates_does_not_raise(): + out = suggest_aggregations_llm(_PROFILE, None) + assert out["status"] == "ok" + assert out["aggregations"] == [] From fd59530751298abc0e2eb569c542850d453aea62 Mon Sep 17 00:00:00 2001 From: Egutierrez Date: Tue, 30 Jun 2026 15:33:55 +0200 Subject: [PATCH 16/53] =?UTF-8?q?feat(eda):=20cap=C3=ADtulo=20AGREGACION?= =?UTF-8?q?=20del=20AutomaticEDA=20(groupby=20+=20pivot=20+=20barras)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Capítulo nuevo (siempre presente cuando hay categóricas agrupables) que analiza la tabla por grupos: stats de numéricas por grupo, tablas dinámicas (pivot) y gráficos de barras desde cero. Obtiene los datos por ctx['aggregations'] precomputado o en vivo vía push-down (ctx['db_path']+table), siguiendo el patrón de chapters/modelos.py. Degrada a None cuando no hay categóricas; emite los bloques del modelo (DataTable, Markdown, Figure) para que el paginador del núcleo no corte nada en PDF ni PPTX. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../automatic_eda/chapters/agregacion.py | 592 ++++++++++++++++++ .../automatic_eda/chapters/agregacion_test.py | 256 ++++++++ 2 files changed, 848 insertions(+) create mode 100644 python/functions/datascience/automatic_eda/chapters/agregacion.py create mode 100644 python/functions/datascience/automatic_eda/chapters/agregacion_test.py diff --git a/python/functions/datascience/automatic_eda/chapters/agregacion.py b/python/functions/datascience/automatic_eda/chapters/agregacion.py new file mode 100644 index 00000000..7b5e03e6 --- /dev/null +++ b/python/functions/datascience/automatic_eda/chapters/agregacion.py @@ -0,0 +1,592 @@ +"""Aggregation chapter (AGREGACION) — group analysis / OLAP of the EDA. + +This chapter is the group-by / pivot ("OLAP") section of an AutomaticEDA report +and is meant to be present **whenever the dataset has at least one low-cardinality +categorical column to group by**. For the most interesting categoricals (chosen +by their cardinality/relevance, optionally with an LLM) it renders, as blocks the +core paginator never cuts: + +1. **Per-group statistics** (split-apply-combine) — for each interesting + categorical key, the count of rows per group and, for each numeric measure, + its mean/median/std/min/max. One compact summary table (mean of every measure + per group) plus a per-measure detail table. +2. **Bar charts** — a vertical bar chart of a measure's mean per group, bars from + zero (Tufte Lie-Factor = 1). +3. **Pivot tables** — categorical A x categorical B -> aggregate of a measure, + limited to the top rows/cols so it fits a mobile page/slide, with a grouped + bar chart of the same pivot. + +The raw data needed to aggregate is **not** in the TableProfile, so — exactly +like ``modelos`` reads its cluster projection from ``ctx`` — this chapter gets +the aggregation results in one of two ways and degrades honestly when neither is +available: + +ctx keys this chapter consumes (all optional): + aggregations : dict — pre-computed results, used directly (offline / tests / + forward-compatible with a calculation phase). Shape:: + + {"groupby": [{"group_by": str, "measures": [str], "why": str, + "result": }], + "pivots": [{"index": str, "columns": str, "value": str, "agg": str, + "why": str, "result": }]} + + db_path, table : str — when ``aggregations`` is absent, the chapter selects + the interesting keys (``select_groupby_keys``), optionally asks an LLM + which to show (``suggest_aggregations_llm`` when ``run_agg_llm`` is True) + and computes the group-by/pivot results live via the push-down registry + functions ``groupby_stats_duckdb`` / ``pivot_table_duckdb``. + run_agg_llm : bool — when True (and ``db_path``/``table`` present), let the + LLM pick the interesting aggregations; otherwise the deterministic + quantitative selection is used. + agg_llm_model : str — model id for the optional LLM selection. + agg_max_keys, agg_max_card, agg_max_measures, agg_top_n : int — limits. + agg_insights : list — optional pre-computed micro-analysis entries + (``[{"title": str, "text": str}]``) rendered as an interpretation section. + +Contract: build_(profile, ctx) -> Chapter | None ; CHAPTER_VERSION = "x.y.z". +Reads everything defensively (``.get``) and never raises: anything missing +degrades to a note instead of aborting the chapter; the chapter returns ``None`` +only when the dataset has no categorical column to group by. +""" + +from __future__ import annotations + +from .. import model + +# Pure/impure registry functions (group ``eda``) this chapter composes. Imported +# defensively so the chapter still builds (degrading the affected part to a note) +# if a function is somehow unavailable / not indexed yet. +try: + from datascience.select_groupby_keys import select_groupby_keys +except Exception: # noqa: BLE001 — keep the chapter importable no matter what. + select_groupby_keys = None # type: ignore[assignment] +try: + from datascience.groupby_stats_duckdb import groupby_stats_duckdb +except Exception: # noqa: BLE001 + groupby_stats_duckdb = None # type: ignore[assignment] +try: + from datascience.pivot_table_duckdb import pivot_table_duckdb +except Exception: # noqa: BLE001 + pivot_table_duckdb = None # type: ignore[assignment] +try: + from datascience.suggest_aggregations_llm import suggest_aggregations_llm +except Exception: # noqa: BLE001 + suggest_aggregations_llm = None # type: ignore[assignment] + +CHAPTER_VERSION = "1.0.0" +CHAPTER_ID = "agregacion" +CHAPTER_TITLE = "Agregación por grupos" + +# Tableau-10 palette — stable colours for the pivot's grouped-bar series. +_SERIES_COLORS = [ + "#4e79a7", "#f28e2b", "#e15759", "#76b7b2", "#59a14f", + "#edc948", "#b07aa1", "#ff9da7", "#9c755f", "#bab0ac", +] + +# Defaults for the live selection/aggregation (overridable via ctx). +_DEF_MAX_KEYS = 3 +_DEF_MAX_CARD = 20 +_DEF_MAX_MEASURES = 4 +_DEF_TOP_N = 12 + + +# --------------------------------------------------------------------------- # +# Formatting helpers (mirror the other chapters' defensive style). +# --------------------------------------------------------------------------- # +def _fmt_num(value, decimals: int = 3) -> str: + if value is None: + return "—" + if isinstance(value, bool): + return "sí" if value else "no" + if isinstance(value, int): + return f"{value:,}".replace(",", ".") + if isinstance(value, float): + if value != value: # NaN + return "NaN" + if value in (float("inf"), float("-inf")): + return str(value) + text = f"{value:.{decimals}f}".rstrip("0").rstrip(".") + return text if text else "0" + return model._safe_str(value) + + +def _is_dict(v) -> bool: + return isinstance(v, dict) + + +def _measure_mean(group: dict, measure: str): + """Pull the mean of one measure out of a groupby-result group entry.""" + stats = group.get("stats") if _is_dict(group.get("stats")) else {} + ms = stats.get(measure) if _is_dict(stats.get(measure)) else {} + return ms.get("mean") + + +# --------------------------------------------------------------------------- # +# Plan + data resolution. Either a pre-computed ctx['aggregations'] is used +# verbatim, or the plan is selected and the results are computed live. +# --------------------------------------------------------------------------- # +def _resolve_candidates(profile: dict, ctx: dict) -> dict: + """Return {group_keys, measures, pivots, note} of interesting columns.""" + pre = ctx.get("agg_candidates") + if _is_dict(pre) and pre.get("group_keys") is not None: + return pre + if select_groupby_keys is not None: + try: + out = select_groupby_keys( + profile, + max_keys=int(ctx.get("agg_max_keys", _DEF_MAX_KEYS)), + max_card=int(ctx.get("agg_max_card", _DEF_MAX_CARD)), + max_measures=int(ctx.get("agg_max_measures", _DEF_MAX_MEASURES)), + ) + if _is_dict(out): + return out + except Exception: # noqa: BLE001 — fall through to the inline fallback. + pass + return _inline_candidates(profile, ctx) + + +def _inline_candidates(profile: dict, ctx: dict) -> dict: + """Minimal defensive selection when select_groupby_keys is unavailable.""" + max_card = int(ctx.get("agg_max_card", _DEF_MAX_CARD)) + max_keys = int(ctx.get("agg_max_keys", _DEF_MAX_KEYS)) + max_measures = int(ctx.get("agg_max_measures", _DEF_MAX_MEASURES)) + keys = profile.get("key_candidates") or [] + group_keys, measures = [], [] + for col in profile.get("columns") or []: + if not _is_dict(col): + continue + name = col.get("name") + it = col.get("inferred_type") + flags = col.get("flags") or [] + dc = col.get("distinct_count") + if it in ("categorical", "boolean") and name not in keys: + if ("possible_id" not in flags and "high_cardinality" not in flags + and "constant" not in flags + and isinstance(dc, int) and 2 <= dc <= max_card): + group_keys.append({"col": name, "cardinality": dc, "score": 0.0}) + elif it == "numeric": + num = col.get("numeric") or {} + if num.get("std") not in (None, 0) and not ( + "possible_id" in flags and (col.get("unique_pct") or 0) >= 0.99): + measures.append(name) + group_keys = group_keys[:max_keys] + measures = measures[:max_measures] + pivots = [] + if len(group_keys) >= 2: + pivots.append({"index": group_keys[0]["col"], + "columns": group_keys[1]["col"], + "value": measures[0] if measures else None}) + return {"group_keys": group_keys, "measures": measures, "pivots": pivots, + "note": "selección cuantitativa básica"} + + +def _resolve_plan(profile: dict, ctx: dict, candidates: dict) -> dict: + """Return {aggregations:[{group_by,measures,why}], pivots:[...], source}.""" + group_keys = candidates.get("group_keys") or [] + measures = candidates.get("measures") or [] + + if ctx.get("run_agg_llm") and suggest_aggregations_llm is not None: + try: + plan = suggest_aggregations_llm( + profile, candidates, + max_aggs=int(ctx.get("agg_max_keys", _DEF_MAX_KEYS)), + model=ctx.get("agg_llm_model", "claude-haiku-4-5-20251001")) + if _is_dict(plan) and plan.get("aggregations"): + return {"aggregations": plan.get("aggregations") or [], + "pivots": plan.get("pivots") or [], + "source": plan.get("source", "llm")} + except Exception: # noqa: BLE001 — fall back to the quantitative plan. + pass + + aggregations = [{ + "group_by": gk.get("col"), + "measures": measures, + "why": f"categórica de {_fmt_num(gk.get('cardinality'))} niveles", + } for gk in group_keys if _is_dict(gk) and gk.get("col")] + pivots = [] + for pv in candidates.get("pivots") or []: + if _is_dict(pv) and pv.get("index") and pv.get("columns"): + pivots.append({"index": pv.get("index"), "columns": pv.get("columns"), + "value": pv.get("value") or (measures[0] if measures else None), + "agg": "mean", "why": "cruce de dos categóricas"}) + return {"aggregations": aggregations, "pivots": pivots, "source": "quantitative"} + + +def _live_groupby(ctx: dict, group_by: str, measures: list, top_n: int): + """Compute one group-by result live via the push-down registry function.""" + db_path = ctx.get("db_path") + table = ctx.get("table") + if not db_path or not table or groupby_stats_duckdb is None: + return None + try: + out = groupby_stats_duckdb(db_path, table, group_by, list(measures or []), + top_n=top_n) + if _is_dict(out) and out.get("status") == "ok": + return out + except Exception: # noqa: BLE001 + return None + return None + + +def _live_pivot(ctx: dict, index: str, columns: str, value, agg: str): + """Compute one pivot live via the push-down registry function.""" + db_path = ctx.get("db_path") + table = ctx.get("table") + if not db_path or not table or pivot_table_duckdb is None or not value: + return None + try: + out = pivot_table_duckdb(db_path, table, index, columns, value, + agg=agg or "mean") + if _is_dict(out) and out.get("status") == "ok": + return out + except Exception: # noqa: BLE001 + return None + return None + + +# --------------------------------------------------------------------------- # +# Figure builders (lazy: matplotlib only imported when the renderer draws them). +# --------------------------------------------------------------------------- # +def _make_group_bars(group_by: str, measure: str, groups: list): + """Vertical bars: mean of ``measure`` per group, bars from zero.""" + labels, values = [], [] + for g in groups: + if not _is_dict(g): + continue + mean = _measure_mean(g, measure) + if mean is None: + continue + labels.append(model._safe_str(g.get("key"))) + values.append(float(mean)) + if not labels: + return None + + def _draw(): + import matplotlib + matplotlib.use("Agg") + import matplotlib.pyplot as plt + + fig, ax = plt.subplots(figsize=(6.6, 3.6)) + xs = list(range(len(labels))) + ax.bar(xs, values, color="#4e79a7", alpha=0.9, edgecolor="#2f4d6e", + linewidth=0.4) + ax.set_xticks(xs) + short = [(s[:18] + "…") if len(s) > 19 else s for s in labels] + rot = 30 if max((len(s) for s in short), default=0) > 6 else 0 + ax.set_xticklabels(short, rotation=rot, ha="right" if rot else "center", + fontsize=7) + ax.set_ylabel(f"media de {measure}", fontsize=8) + ax.set_xlabel(group_by, fontsize=8) + ax.set_title(f"Media de «{measure}» por «{group_by}»", fontsize=10) + ax.grid(axis="y", color="#dddddd", linewidth=0.6) + for spine in ("top", "right"): + ax.spines[spine].set_visible(False) + # Value labels above each bar. + vmax = max(values) if values else 0 + for x, v in zip(xs, values): + ax.text(x, v + (abs(vmax) * 0.01 if vmax else 0.01), + _fmt_num(v, 2), ha="center", va="bottom", fontsize=6.5) + fig.tight_layout() + return fig + + return _draw + + +def _make_pivot_bars(pivot: dict): + """Grouped bars of a pivot: x = row_labels, one series per col_label.""" + row_labels = pivot.get("row_labels") or [] + col_labels = pivot.get("col_labels") or [] + matrix = pivot.get("matrix") or [] + if not row_labels or not col_labels or not matrix: + return None + + def _draw(): + import matplotlib + matplotlib.use("Agg") + import matplotlib.pyplot as plt + + n_rows = len(row_labels) + n_cols = len(col_labels) + fig, ax = plt.subplots(figsize=(6.8, 3.8)) + total_w = 0.8 + bar_w = total_w / max(n_cols, 1) + base = list(range(n_rows)) + for j, clabel in enumerate(col_labels): + offs = [b - total_w / 2 + bar_w * (j + 0.5) for b in base] + vals = [] + for i in range(n_rows): + cell = matrix[i][j] if (i < len(matrix) and j < len(matrix[i])) else None + vals.append(float(cell) if isinstance(cell, (int, float)) else 0.0) + color = _SERIES_COLORS[j % len(_SERIES_COLORS)] + ax.bar(offs, vals, width=bar_w, color=color, alpha=0.9, + label=model._safe_str(clabel)) + ax.set_xticks(base) + short = [(s[:16] + "…") if len(s) > 17 else s + for s in (model._safe_str(r) for r in row_labels)] + rot = 30 if max((len(s) for s in short), default=0) > 6 else 0 + ax.set_xticklabels(short, rotation=rot, ha="right" if rot else "center", + fontsize=7) + ax.set_xlabel(model._safe_str(pivot.get("index")), fontsize=8) + ax.set_ylabel(f"{pivot.get('agg','mean')} de {pivot.get('value')}", + fontsize=8) + ax.set_title(f"{pivot.get('index')} × {pivot.get('columns')}", fontsize=10) + ax.grid(axis="y", color="#dddddd", linewidth=0.6) + ax.legend(title=model._safe_str(pivot.get("columns")), fontsize=6.5, + title_fontsize=7, frameon=True, framealpha=0.9, loc="best") + for spine in ("top", "right"): + ax.spines[spine].set_visible(False) + fig.tight_layout() + return fig + + return _draw + + +def _group_bars_maker(group_by: str, measure: str, groups: list): + """Bind per-aggregation args so the lazy closure is loop-safe.""" + def _make(): + return _make_group_bars(group_by, measure, groups)() + return _make + + +def _pivot_bars_maker(pivot: dict): + def _make(): + return _make_pivot_bars(pivot)() + return _make + + +# --------------------------------------------------------------------------- # +# Section builders. Each returns a list of blocks (possibly empty). +# --------------------------------------------------------------------------- # +def _groupby_section(group_by: str, measures: list, result: dict, why: str) -> list: + """Build the blocks for one group-by aggregation, or [] if unusable.""" + if not _is_dict(result) or not result.get("groups"): + return [] + groups = [g for g in result.get("groups") or [] if _is_dict(g)] + if not groups: + return [] + eff_measures = result.get("measures") or measures or [] + + blocks = [model.Heading(text=f"Agrupado por «{group_by}»", level=2)] + intro = f"**{why}.** " if why else "" + intro += (f"{_fmt_num(result.get('n_groups') or len(groups))} grupos" + f"{' (top por tamaño)' if result.get('truncated') else ''}.") + blocks.append(model.Markdown(text=intro)) + + # Summary table: one row per group, count + mean of every measure. + header = ["Grupo", "n"] + [f"{m} (media)" for m in eff_measures] + rows = [] + for g in groups: + row = [model._safe_str(g.get("key")), _fmt_num(g.get("n"))] + for m in eff_measures: + row.append(_fmt_num(_measure_mean(g, m), 2)) + rows.append(row) + blocks.append(model.DataTable( + header=header, rows=rows, title=f"Resumen por «{group_by}»", + note="Conteo de filas y media de cada medida por grupo.")) + + if not eff_measures: + return blocks + + # Primary measure: a bar chart + a detail table (mean/median/std/min/max). + primary = eff_measures[0] + bars = _make_group_bars(group_by, primary, groups) + if bars is not None: + blocks.append(model.Figure( + make=_group_bars_maker(group_by, primary, groups), + caption=f"Media de «{primary}» por «{group_by}» (barras desde cero).")) + + det_header = ["Grupo", "n", "media", "mediana", "σ", "mín", "máx"] + det_rows = [] + for g in groups: + stats = g.get("stats") if _is_dict(g.get("stats")) else {} + ms = stats.get(primary) if _is_dict(stats.get(primary)) else {} + det_rows.append([ + model._safe_str(g.get("key")), _fmt_num(g.get("n")), + _fmt_num(ms.get("mean"), 2), _fmt_num(ms.get("median"), 2), + _fmt_num(ms.get("std"), 2), _fmt_num(ms.get("min"), 2), + _fmt_num(ms.get("max"), 2), + ]) + blocks.append(model.DataTable( + header=det_header, rows=det_rows, + title=f"Detalle de «{primary}» por «{group_by}»")) + return blocks + + +def _pivot_section(pivot_spec: dict, result: dict) -> list: + """Build the blocks for one pivot table, or [] if unusable.""" + if not _is_dict(result) or not result.get("row_labels"): + return [] + row_labels = result.get("row_labels") or [] + col_labels = result.get("col_labels") or [] + matrix = result.get("matrix") or [] + if not row_labels or not col_labels or not matrix: + return [] + + index = result.get("index") or pivot_spec.get("index") + columns = result.get("columns") or pivot_spec.get("columns") + value = result.get("value") or pivot_spec.get("value") + agg = result.get("agg") or pivot_spec.get("agg") or "mean" + why = pivot_spec.get("why") or "" + + blocks = [model.Heading(text=f"Pivot: «{index}» × «{columns}»", level=2)] + intro = f"**{why}.** " if why else "" + intro += (f"{agg} de «{value}» cruzando «{index}» (filas) y «{columns}» " + f"(columnas).") + if result.get("truncated_rows") or result.get("truncated_cols"): + intro += " Limitado a las filas/columnas más frecuentes." + blocks.append(model.Markdown(text=intro)) + + header = [model._safe_str(index)] + [model._safe_str(c) for c in col_labels] + rows = [] + for i, rlabel in enumerate(row_labels): + row = [model._safe_str(rlabel)] + cells = matrix[i] if i < len(matrix) else [] + for j in range(len(col_labels)): + cell = cells[j] if j < len(cells) else None + row.append(_fmt_num(cell, 2)) + rows.append(row) + blocks.append(model.DataTable( + header=header, rows=rows, + title=f"{agg} de «{value}»", + note=f"Cada celda es {agg} de «{value}» para esa combinación.")) + + fig_pivot = {"row_labels": row_labels, "col_labels": col_labels, + "matrix": matrix, "index": index, "columns": columns, + "value": value, "agg": agg} + if _make_pivot_bars(fig_pivot) is not None: + blocks.append(model.Figure( + make=_pivot_bars_maker(fig_pivot), + caption=f"{agg} de «{value}» por «{index}» y «{columns}» " + f"(barras agrupadas).")) + return blocks + + +def _insights_section(ctx: dict) -> list: + """Optional pre-computed micro-analysis of the aggregations (SHOULD-11.4).""" + entries = ctx.get("agg_insights") + if not isinstance(entries, list) or not entries: + return [] + blocks = [model.Heading(text="Interpretación de los grupos", level=2)] + for e in entries: + if not _is_dict(e): + continue + title = model._safe_str(e.get("title")) + text = model._safe_str(e.get("text")) + line = (f"**{title}.** " if title else "") + text + if line.strip(): + blocks.append(model.Markdown(text=line)) + return blocks if len(blocks) > 1 else [] + + +# --------------------------------------------------------------------------- # +# Pre-computed path: ctx['aggregations'] already carries the results. +# --------------------------------------------------------------------------- # +def _sections_from_precomputed(agg: dict) -> list: + sections = [] + for entry in agg.get("groupby") or []: + if not _is_dict(entry): + continue + sections += _groupby_section( + entry.get("group_by"), entry.get("measures") or [], + entry.get("result") or {}, entry.get("why") or "") + for entry in agg.get("pivots") or []: + if not _is_dict(entry): + continue + sections += _pivot_section(entry, entry.get("result") or {}) + return sections + + +# --------------------------------------------------------------------------- # +# Live path: select keys, pick a plan, compute results via push-down functions. +# --------------------------------------------------------------------------- # +def _sections_live(profile: dict, ctx: dict, candidates: dict) -> list: + top_n = int(ctx.get("agg_top_n", _DEF_TOP_N)) + plan = _resolve_plan(profile, ctx, candidates) + sections = [] + for agg in plan.get("aggregations") or []: + if not _is_dict(agg) or not agg.get("group_by"): + continue + result = _live_groupby(ctx, agg.get("group_by"), + agg.get("measures") or [], top_n) + if result is not None: + sections += _groupby_section(agg.get("group_by"), + agg.get("measures") or [], result, + agg.get("why") or "") + for pv in plan.get("pivots") or []: + if not _is_dict(pv) or not pv.get("index") or not pv.get("columns"): + continue + result = _live_pivot(ctx, pv.get("index"), pv.get("columns"), + pv.get("value"), pv.get("agg") or "mean") + if result is not None: + sections += _pivot_section(pv, result) + return sections + + +# --------------------------------------------------------------------------- # +# Entry point. +# --------------------------------------------------------------------------- # +def _intro_blocks() -> list: + text = ( + "Este capítulo analiza la tabla **por grupos** (split-apply-combine): " + "elige las columnas categóricas más informativas — por su cardinalidad " + "y relevancia, no todas contra todas, para no inflar comparaciones " + "espurias — y resume las variables numéricas dentro de cada grupo " + "(conteo, media, mediana, desviación). Las **tablas dinámicas** (pivot) " + "cruzan dos categóricas sobre una medida, y los **gráficos de barras** " + "(siempre desde cero) comparan los grupos de un vistazo." + ) + return [model.Heading(text=CHAPTER_TITLE, level=1), + model.Markdown(text=text)] + + +def build_agregacion(profile: dict, ctx: dict): + """Build the AGREGACION Chapter, or None if the dataset can't be grouped. + + Args: + profile: the ``eda`` group TableProfile dict. + ctx: presentation context (see module docstring for the keys consumed). + + Returns: + A ``model.Chapter`` with per-group stats, pivots and bar charts; or + ``None`` when the dataset has no low-cardinality categorical column to + group by (the chapter does not apply). + """ + profile = profile or {} + ctx = ctx or {} + if not isinstance(profile, dict): + return None + + # Pre-computed results take precedence (offline / tests / forward-compat). + pre = ctx.get("aggregations") + if _is_dict(pre) and (pre.get("groupby") or pre.get("pivots")): + sections = _sections_from_precomputed(pre) + if not sections: + return None + blocks = _intro_blocks() + sections + _insights_section(ctx) + return model.Chapter(id=CHAPTER_ID, title=CHAPTER_TITLE, + version=CHAPTER_VERSION, blocks=blocks) + + # Live path: needs at least one categorical key to group by. + candidates = _resolve_candidates(profile, ctx) + if not _is_dict(candidates) or not (candidates.get("group_keys")): + return None # chapter does not apply: nothing to group by. + + sections = _sections_live(profile, ctx, candidates) + if not sections: + # Applies (there are categorical keys) but no aggregation data is + # reachable: emit an honest note instead of fabricating numbers. + keys = ", ".join(model._safe_str((k or {}).get("col")) + for k in candidates.get("group_keys") or [] + if _is_dict(k)) + note = model.Note( + "No se pudo calcular la agregación: el capítulo necesita los datos " + "crudos. Pasa ctx['db_path'] + ctx['table'] (para el cálculo " + "push-down en DuckDB) o ctx['aggregations'] ya precalculado. " + f"Columnas categóricas candidatas: {keys or '—'}.") + blocks = _intro_blocks() + [note] + _insights_section(ctx) + return model.Chapter(id=CHAPTER_ID, title=CHAPTER_TITLE, + version=CHAPTER_VERSION, blocks=blocks) + + blocks = _intro_blocks() + sections + _insights_section(ctx) + return model.Chapter(id=CHAPTER_ID, title=CHAPTER_TITLE, + version=CHAPTER_VERSION, blocks=blocks) diff --git a/python/functions/datascience/automatic_eda/chapters/agregacion_test.py b/python/functions/datascience/automatic_eda/chapters/agregacion_test.py new file mode 100644 index 00000000..e35005be --- /dev/null +++ b/python/functions/datascience/automatic_eda/chapters/agregacion_test.py @@ -0,0 +1,256 @@ +"""Tests for the AGREGACION chapter — DoD: golden + edges + error/no-cut path. + +Self-contained and deterministic: no DuckDB and no LLM. The aggregation results +are passed pre-computed via ``ctx['aggregations']`` (the same shape the push-down +registry functions ``groupby_stats_duckdb`` / ``pivot_table_duckdb`` produce), so +the chapter's rendering logic is exercised without touching disk or the network. +Live push-down + LLM selection are covered separately by the golden script. + +Verifies: +- Golden: a profile with categoricals + numerics builds a Chapter with per-group + stats tables, a pivot table and bar-chart figures, and it renders to PDF AND + PPTX showing the group keys, values and pivot — nothing cut. +- Edges: a dataset with no low-cardinality categorical returns None; an empty + profile returns None; a profile that *could* be grouped but has no reachable + data degrades to an honest note instead of raising. +- No-cut: many groups (30) + a long interpretation paragraph survive intact in + the rendered PDF (table split by rows, text wrapped whole). +""" + +import os +import re +import tempfile + +from pptx import Presentation +from pypdf import PdfReader + +from datascience.automatic_eda.chapters.agregacion import build_agregacion +from datascience.automatic_eda.model import Chapter +from datascience.render_automatic_eda_pdf import render_automatic_eda_pdf +from datascience.render_automatic_eda_pptx import render_automatic_eda_pptx + + +# --------------------------------------------------------------------------- # +# Synthetic fixtures. +# --------------------------------------------------------------------------- # +def _profile() -> dict: + """A titanic-like profile: 2 categoricals + 2 numeric measures + 1 id.""" + return { + "table": "titanic", + "source": "/data/titanic.csv", + "n_rows": 891, + "n_cols": 5, + "key_candidates": ["passenger_id"], + "columns": [ + {"name": "passenger_id", "inferred_type": "numeric", + "unique_pct": 1.0, "flags": ["possible_id"], + "numeric": {"mean": 446.0, "std": 257.0}}, + {"name": "sex", "inferred_type": "categorical", "distinct_count": 2, + "flags": [], "categorical": {"n_distinct": 2, "imbalance": 0.1, + "top": [{"value": "male", "count": 577}]}}, + {"name": "pclass", "inferred_type": "categorical", "distinct_count": 3, + "flags": [], "categorical": {"n_distinct": 3, "imbalance": 0.2}}, + {"name": "fare", "inferred_type": "numeric", "flags": [], + "numeric": {"mean": 32.2, "std": 49.7, "cv": 1.54}}, + {"name": "age", "inferred_type": "numeric", "flags": [], + "numeric": {"mean": 29.7, "std": 14.5, "cv": 0.49}}, + ], + } + + +def _groupby_result(group_by: str, keys_n: list) -> dict: + """A groupby_stats_duckdb-shaped result for `fare` and `age`.""" + groups = [] + for i, (key, n) in enumerate(keys_n): + groups.append({ + "key": key, "n": n, + "stats": { + "fare": {"mean": 20.0 + i * 15, "median": 10.0 + i * 8, + "std": 40.0 + i, "min": 0.0, "max": 512.3}, + "age": {"mean": 28.0 + i, "median": 27.0 + i, "std": 14.0, + "min": 0.42, "max": 80.0}, + }, + }) + return {"status": "ok", "group_by": group_by, "measures": ["fare", "age"], + "aggs": ["count", "mean", "median", "std", "min", "max"], + "n_groups": len(groups), "truncated": False, "groups": groups} + + +def _pivot_result() -> dict: + return {"status": "ok", "index": "sex", "columns": "pclass", "value": "fare", + "agg": "mean", "row_labels": ["male", "female"], + "col_labels": ["1", "2", "3"], + "matrix": [[62.0, 19.0, 12.0], [110.0, 22.0, 15.0]], + "truncated_rows": False, "truncated_cols": False} + + +def _ctx_precomputed() -> dict: + return { + "aggregations": { + "groupby": [ + {"group_by": "sex", "measures": ["fare", "age"], + "why": "sexo del pasajero", + "result": _groupby_result("sex", [("male", 577), ("female", 314)])}, + {"group_by": "pclass", "measures": ["fare", "age"], + "why": "clase del billete", + "result": _groupby_result( + "pclass", [("3", 491), ("1", 216), ("2", 184)])}, + ], + "pivots": [ + {"index": "sex", "columns": "pclass", "value": "fare", + "agg": "mean", "why": "tarifa por sexo y clase", + "result": _pivot_result()}, + ], + }, + "agg_insights": [ + {"title": "Tarifa por sexo", + "text": "Las mujeres pagaron de media casi el doble que los hombres."}, + ], + } + + +def _pdf_text(path: str) -> str: + txt = "".join((pg.extract_text() or "") for pg in PdfReader(path).pages) + return re.sub(r"\s+", " ", txt) + + +def _pptx_text(path: str) -> str: + prs = Presentation(path) + parts = [] + for sl in prs.slides: + for sh in sl.shapes: + if sh.has_text_frame: + parts.append(sh.text_frame.text) + if sh.has_table: + tb = sh.table + for r in range(len(tb.rows)): + for c in range(len(tb.columns)): + parts.append(tb.cell(r, c).text) + return re.sub(r"\s+", " ", " ".join(parts)) + + +# --------------------------------------------------------------------------- # +# Golden: builds a Chapter and renders to both formats. +# --------------------------------------------------------------------------- # +def test_golden_chapter_blocks_present(): + ch = build_agregacion(_profile(), _ctx_precomputed()) + assert isinstance(ch, Chapter) + assert ch.id == "agregacion" + kinds = [b.kind for b in ch.blocks] + assert "heading" in kinds + assert kinds.count("data_table") >= 3 # 2 group summaries + pivot (+details) + assert "figure" in kinds # at least one bar chart. + # Headings mention the group keys and the pivot. + htext = " ".join(b.text for b in ch.blocks if b.kind == "heading") + assert "sex" in htext and "pclass" in htext and "Pivot" in htext + + +def test_golden_render_pdf(): + ch = build_agregacion(_profile(), _ctx_precomputed()) + with tempfile.TemporaryDirectory() as d: + out = os.path.join(d, "agg.pdf") + res = render_automatic_eda_pdf([ch], out, {"write_manifest": False}) + assert res["path"] == out and os.path.exists(out) + assert res["n_pages"] >= 1 + txt = _pdf_text(out) + assert "Agregación por grupos" in txt + assert "male" in txt and "female" in txt # group + pivot labels. + assert "Pivot" in txt + assert "mediana" in txt # per-measure detail. + assert "casi el doble" in txt # interpretation kept. + + +def test_golden_render_pptx(): + ch = build_agregacion(_profile(), _ctx_precomputed()) + with tempfile.TemporaryDirectory() as d: + out = os.path.join(d, "agg.pptx") + res = render_automatic_eda_pptx([ch], out, {"write_manifest": False}) + assert res["path"] == out and os.path.exists(out) + assert res["n_slides"] >= 1 + txt = _pptx_text(out) + assert "male" in txt and "pclass" in txt + assert "Pivot" in txt or "sex" in txt + + +# --------------------------------------------------------------------------- # +# Edges. +# --------------------------------------------------------------------------- # +def test_edge_no_categorical_returns_none(): + # Only numerics + an id: nothing to group by -> chapter does not apply. + prof = { + "table": "t", "n_rows": 100, "key_candidates": ["id"], + "columns": [ + {"name": "id", "inferred_type": "numeric", "unique_pct": 1.0, + "flags": ["possible_id"], "numeric": {"std": 10.0}}, + {"name": "x", "inferred_type": "numeric", "flags": [], + "numeric": {"mean": 1.0, "std": 2.0}}, + ], + } + assert build_agregacion(prof, {}) is None + + +def test_edge_empty_profile_returns_none(): + assert build_agregacion({}, {}) is None + assert build_agregacion(None, None) is None + + +def test_edge_high_cardinality_only_returns_none(): + # The single categorical is id-like (high cardinality) -> not groupable. + prof = { + "table": "t", "n_rows": 100, "key_candidates": ["uuid"], + "columns": [ + {"name": "uuid", "inferred_type": "categorical", "distinct_count": 100, + "flags": ["high_cardinality", "possible_id"]}, + {"name": "x", "inferred_type": "numeric", "flags": [], + "numeric": {"mean": 1.0, "std": 2.0}}, + ], + } + assert build_agregacion(prof, {}) is None + + +def test_live_without_data_degrades_to_note(): + # Has a categorical to group by but no db_path / no precomputed results: + # must NOT raise and must emit an honest note (chapter still applies). + prof = { + "table": "t", "n_rows": 100, "key_candidates": [], + "columns": [ + {"name": "grp", "inferred_type": "categorical", "distinct_count": 3, + "flags": [], "categorical": {"n_distinct": 3}}, + {"name": "v", "inferred_type": "numeric", "flags": [], + "numeric": {"mean": 1.0, "std": 2.0}}, + ], + } + ch = build_agregacion(prof, {}) + assert isinstance(ch, Chapter) + notes = [b.text for b in ch.blocks if b.kind == "note"] + assert any("datos crudos" in n for n in notes) + + +# --------------------------------------------------------------------------- # +# No-cut: many groups + long text survive intact in the PDF. +# --------------------------------------------------------------------------- # +def test_anti_corte_muchos_grupos_y_texto_largo(): + keys_n = [(f"grupo_{i:02d}", 30 - (i % 5)) for i in range(30)] + long_text = " ".join(f"palabra{i}" for i in range(120)) + ctx = { + "aggregations": { + "groupby": [ + {"group_by": "cat", "measures": ["fare"], "why": "muchos niveles", + "result": _groupby_result("cat", keys_n)}, + ], + "pivots": [], + }, + "agg_insights": [{"title": "Nota larga", "text": long_text}], + } + ch = build_agregacion(_profile(), ctx) + with tempfile.TemporaryDirectory() as d: + out = os.path.join(d, "big.pdf") + res = render_automatic_eda_pdf([ch], out, {"write_manifest": False}) + assert res["path"] == out + assert res["n_pages"] > 1 # 30-row table + figure spill across pages. + txt = _pdf_text(out) + # First and last group labels both survive (table not truncated). + assert "grupo_00" in txt and "grupo_29" in txt + # First, middle and last words of the long paragraph all present. + for i in (0, 60, 119): + assert f"palabra{i}" in txt From a69d14d38e769707763d833cb6a17075b393725c Mon Sep 17 00:00:00 2001 From: Egutierrez Date: Tue, 30 Jun 2026 15:35:42 +0200 Subject: [PATCH 17/53] =?UTF-8?q?feat(eda):=20cap=C3=ADtulo=20TIMESERIES?= =?UTF-8?q?=20del=20AutomaticEDA=20(evoluci=C3=B3n=20+=20an=C3=A1lisis=20d?= =?UTF-8?q?e=20serie)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Capítulo nuevo build_timeseries(profile, ctx) -> Chapter|None del motor AutomaticEDA. Cuando la tabla tiene columna de fecha/datetime, grafica la evolución de cada columna numérica por periodo (valor agregado + conteo de filas) y los paneles de descomposición STL y autocorrelación (ACF), con el análisis de la serie: estacionariedad (ADF+KPSS), autocorrelación (Ljung-Box), fuerzas de tendencia/estacionalidad (Hyndman) y la transformación sugerida (retornos o diferencias) para evitar correlaciones espurias. Sin columna temporal devuelve None. Consolida series OHLC casi idénticas en un único gráfico conservando el análisis de cada columna. La serie cruda llega por ctx['timeseries_raw'] (mismo patrón que modelos con raw_numeric); las figuras son perezosas (Figure.make) y el paginador del núcleo garantiza no-corte en PDF y PPTX. CHAPTER_VERSION 1.0.0. Cubre los MUST del diseño (report 2043): MUST-9.1 (línea valor-vs-tiempo + conteo por periodo), MUST-9.2 (paneles STL + ACF), MUST-9.3 (perfil datetime + consolidación OHLC). Funciones nuevas del registry (grupo eda), delegadas a fn-constructor, no inline: - detect_time_column (pure): detecta la columna temporal y las numéricas - profile_datetime (pure): rango/frecuencia/regularidad/huecos de la fecha - resample_timeseries (pure): agrega la serie por periodo + conteo - extract_timeseries_raw (impure): lee la serie cruda ordenada de DuckDB/PG Verificación: 69 tests verdes (capítulo 9 + funciones 28 + núcleo/renderers); golden real sobre seattle-weather (estacional) y aapl (OHLC) con PDF+PPTX sin cortar nada (cols_cortadas=[]). Co-Authored-By: Claude Opus 4.8 (1M context) --- python/functions/datascience/__init__.py | 8 + .../automatic_eda/chapters/timeseries.py | 613 ++++++++++++++++++ .../automatic_eda/chapters/timeseries_test.py | 244 +++++++ .../datascience/detect_time_column.md | 68 ++ .../datascience/detect_time_column.py | 112 ++++ .../datascience/detect_time_column_test.py | 102 +++ .../datascience/extract_timeseries_raw.md | 92 +++ .../datascience/extract_timeseries_raw.py | 122 ++++ .../extract_timeseries_raw_test.py | 109 ++++ .../functions/datascience/profile_datetime.md | 79 +++ .../functions/datascience/profile_datetime.py | 183 ++++++ .../datascience/profile_datetime_test.py | 127 ++++ .../datascience/resample_timeseries.md | 72 ++ .../datascience/resample_timeseries.py | 275 ++++++++ .../datascience/resample_timeseries_test.py | 118 ++++ 15 files changed, 2324 insertions(+) create mode 100644 python/functions/datascience/automatic_eda/chapters/timeseries.py create mode 100644 python/functions/datascience/automatic_eda/chapters/timeseries_test.py create mode 100644 python/functions/datascience/detect_time_column.md create mode 100644 python/functions/datascience/detect_time_column.py create mode 100644 python/functions/datascience/detect_time_column_test.py create mode 100644 python/functions/datascience/extract_timeseries_raw.md create mode 100644 python/functions/datascience/extract_timeseries_raw.py create mode 100644 python/functions/datascience/extract_timeseries_raw_test.py create mode 100644 python/functions/datascience/profile_datetime.md create mode 100644 python/functions/datascience/profile_datetime.py create mode 100644 python/functions/datascience/profile_datetime_test.py create mode 100644 python/functions/datascience/resample_timeseries.md create mode 100644 python/functions/datascience/resample_timeseries.py create mode 100644 python/functions/datascience/resample_timeseries_test.py diff --git a/python/functions/datascience/__init__.py b/python/functions/datascience/__init__.py index 9fc8c206..9c45fd23 100644 --- a/python/functions/datascience/__init__.py +++ b/python/functions/datascience/__init__.py @@ -57,8 +57,16 @@ from .exploratory_caveats import exploratory_caveats from .render_eda_pdf import render_eda_pdf, render_eda_pdf_relational from .render_automatic_eda_pdf import render_automatic_eda_pdf from .render_automatic_eda_pptx import render_automatic_eda_pptx +from .detect_time_column import detect_time_column +from .extract_timeseries_raw import extract_timeseries_raw +from .profile_datetime import profile_datetime +from .resample_timeseries import resample_timeseries __all__ = [ + "detect_time_column", + "extract_timeseries_raw", + "profile_datetime", + "resample_timeseries", "render_automatic_eda_pdf", "render_automatic_eda_pptx", "decode_qr_image", diff --git a/python/functions/datascience/automatic_eda/chapters/timeseries.py b/python/functions/datascience/automatic_eda/chapters/timeseries.py new file mode 100644 index 00000000..150eb03e --- /dev/null +++ b/python/functions/datascience/automatic_eda/chapters/timeseries.py @@ -0,0 +1,613 @@ +"""Time-series chapter (TIMESERIES) for AutomaticEDA. + +This chapter applies **only when the table has a date/datetime column**. When it +does, it draws — exactly the user requirement — the evolution of the data over +time (the value of each numeric column aggregated per period *and* the count of +rows per period) plus the statistical analysis of the series (stationarity, +autocorrelation, trend and seasonality). When there is no temporal column +``build_timeseries`` returns ``None``. + +Data sources, read defensively and never recomputed here: + +- ``profile['columns']`` — to detect the time column and the numeric columns. + Delegated to the pure registry function ``detect_time_column`` (group ``eda``). +- ``profile['series'][col]`` — the per-column time-series analysis already + produced by ``profile_table(run_series=True)``: ``stationarity`` (ADF+KPSS), + ``acf_pacf`` (ACF/PACF + Ljung-Box), ``stl`` (trend/seasonal/resid + + Hyndman strengths) and the levels/returns suggestion. +- ``ctx['timeseries_raw']`` (or ``profile['timeseries_raw']``) — the *raw* ordered + series ``{time_col, t:[iso...], series:{col:[float|None]}}`` needed to draw the + value-vs-time line and the per-period row count. Exactly like ``modelos`` reads + ``raw_numeric`` from ``ctx``, this chapter looks for the raw series there and + degrades honestly when it is absent (it still renders the textual analysis). + +The raw series is aggregated per period with the pure registry function +``resample_timeseries`` and the datetime header is built with ``profile_datetime`` +(both group ``eda``). Every figure is emitted as a lazy ``Figure`` so the +renderers rasterize and scale it to fit a whole page/slide; tables go through +``DataTable``/``KVTable`` so the paginator splits them repeating the header. No +content is ever cut. + +ctx keys this chapter consumes (all optional): + timeseries_raw : dict — ``{time_col, t:[...], series:{col:[...]}}`` raw + ordered series used to draw the value-vs-time line and the row-count + panel. When absent the chapter omits those figures (with a note) and + renders only the analysis available in ``profile['series']``. + +Contract: build_(profile, ctx) -> Chapter | None ; CHAPTER_VERSION = "x.y.z". +Reads everything defensively (``.get``) and never raises. +""" + +from __future__ import annotations + +from .. import model + +# Pure/impure registry functions (group ``eda``) consumed by this chapter, +# imported defensively so the chapter still builds (degrading the affected +# section to a note) if any of them is somehow unavailable. +try: + from datascience.detect_time_column import detect_time_column +except Exception: # noqa: BLE001 — keep the chapter importable no matter what. + detect_time_column = None # type: ignore[assignment] +try: + from datascience.profile_datetime import profile_datetime +except Exception: # noqa: BLE001 + profile_datetime = None # type: ignore[assignment] +try: + from datascience.resample_timeseries import resample_timeseries +except Exception: # noqa: BLE001 + resample_timeseries = None # type: ignore[assignment] + +CHAPTER_VERSION = "1.0.0" +CHAPTER_ID = "timeseries" +CHAPTER_TITLE = "Series temporales" + +# Plain-Spanish gloss for the stationarity verdict of adf_kpss_stationarity. +_VERDICT_GLOSS = { + "stationary": "estacionaria: media y varianza estables en el tiempo; se " + "puede modelar directamente.", + "non_stationary": "no estacionaria: tiene tendencia o varianza cambiante " + "(raíz unitaria). Correlacionar o modelar sus niveles " + "produce relaciones espurias (Granger-Newbold); conviene " + "diferenciar o pasar a retornos.", + "inconclusive": "resultado no concluyente (ADF y KPSS discrepan): tratar con " + "cautela, probablemente cerca de la no estacionariedad.", +} + +# OHLC-style name fragments used to collapse near-identical financial series. +_OHLC_HINTS = ("open", "high", "low", "close", "adj", "price", "vwap") + + +def _fmt_num(value, decimals: int = 3) -> str: + """Compact, defensive number formatting shared with the other chapters.""" + if value is None: + return "—" + if isinstance(value, bool): + return "sí" if value else "no" + if isinstance(value, int): + return f"{value:,}".replace(",", ".") + if isinstance(value, float): + if value != value: # NaN + return "NaN" + if value in (float("inf"), float("-inf")): + return str(value) + text = f"{value:.{decimals}f}".rstrip("0").rstrip(".") + return text if text else "0" + return model._safe_str(value) + + +def _is_dict(v) -> bool: + return isinstance(v, dict) + + +# --------------------------------------------------------------------------- # +# Detection: which column is the time axis and which numeric columns to chart. +# --------------------------------------------------------------------------- # +def _detect(cols: list) -> dict: + """Return ``{time_col, numeric_cols, ...}`` via the registry function. + + Falls back to an inline scan (datetime inferred_type / datetime semantic + types) when ``detect_time_column`` is unavailable, so the chapter still works. + """ + if detect_time_column is not None: + try: + res = detect_time_column(cols) + if _is_dict(res): + return res + except Exception: # noqa: BLE001 — degrade to the inline scan. + pass + time_col = None + numeric_cols = [] + for c in cols or []: + if not _is_dict(c): + continue + it = c.get("inferred_type") + sem = c.get("semantic_type") + if time_col is None and ( + it == "datetime" or sem in ("datetime_iso", "date_eu")): + time_col = c.get("name") + if it == "numeric": + numeric_cols.append(c.get("name")) + return {"time_col": time_col, "numeric_cols": numeric_cols, + "time_semantic": "", "reason": "inline fallback"} + + +def _raw_series_for(raw: dict, col: str): + """Return (t_list, v_list) for a column from the raw bundle, or (None, None).""" + if not _is_dict(raw): + return None, None + t = raw.get("t") + series = raw.get("series") if _is_dict(raw.get("series")) else {} + v = series.get(col) + if isinstance(t, list) and isinstance(v, list) and t and len(t) == len(v): + return t, v + return None, None + + +def _ohlc_groups(numeric_cols: list, raw: dict) -> dict: + """Map each numeric column to a representative to collapse OHLC duplicates. + + When several numeric columns are near-identical financial level series + (open/high/low/close/adj close), charting each one repeats the same figure + four times. We keep the first OHLC-looking column as the representative for + the *figures* and list the collapsed ones in a note; the textual analysis is + still produced for every column. Detection is by name only (cheap, no extra + data dependency) and conservative: only collapses when >=2 OHLC-like names + are present. + """ + ohlc = [c for c in numeric_cols + if isinstance(c, str) and any(h in c.lower() for h in _OHLC_HINTS)] + if len(ohlc) < 2: + return {} + representative = ohlc[0] + return {c: representative for c in ohlc if c != representative} + + +# --------------------------------------------------------------------------- # +# Datetime header (MUST-9.3): range / frequency / regularity / gaps. +# --------------------------------------------------------------------------- # +def _datetime_header(time_col: str, raw: dict) -> list: + """Build the datetime profile header from the raw time axis, when present.""" + blocks: list = [] + t, _ = (raw.get("t"), None) if _is_dict(raw) else (None, None) + if not (isinstance(t, list) and t and profile_datetime is not None): + return blocks + try: + dt = profile_datetime(t) + except Exception: # noqa: BLE001 + return blocks + if not _is_dict(dt): + return blocks + + freq_gloss = { + "daily": "diaria", "weekly": "semanal", "monthly": "mensual", + "quarterly": "trimestral", "yearly": "anual", + "irregular": "irregular", "unknown": "indeterminada", + } + rows = [ + ("Columna de fecha", model._safe_str(time_col)), + ("Rango", f"{model._safe_str(dt.get('min'))} → " + f"{model._safe_str(dt.get('max'))}"), + ("Observaciones", _fmt_num(dt.get("n"))), + ("Fechas distintas", _fmt_num(dt.get("n_distinct"))), + ("Frecuencia", freq_gloss.get(dt.get("freq"), model._safe_str(dt.get("freq")))), + ("Regular", "sí" if dt.get("is_regular") else "no"), + ] + span = dt.get("span_days") + if span is not None: + rows.append(("Duración (días)", _fmt_num(span, 1))) + n_gaps = dt.get("n_gaps") + if n_gaps is not None: + rows.append(("Huecos en la rejilla", _fmt_num(n_gaps))) + blocks.append(model.KVTable(rows=rows, title="Perfil temporal")) + note = dt.get("note") + if note: + blocks.append(model.Note(model._safe_str(note))) + return blocks + + +# --------------------------------------------------------------------------- # +# Figure builders (lazy: matplotlib only imported when the renderer draws them). +# --------------------------------------------------------------------------- # +def _parse_dates(labels: list): + """Parse a list of ISO-ish strings/dates to datetime, dropping unparseable. + + Returns (dates, kept_index) so callers can align the values list. + """ + from datetime import date, datetime + + out = [] + keep = [] + for i, lab in enumerate(labels): + if isinstance(lab, datetime): + out.append(lab) + keep.append(i) + continue + if isinstance(lab, date): + out.append(datetime(lab.year, lab.month, lab.day)) + keep.append(i) + continue + s = model._safe_str(lab).strip() + if not s: + continue + s2 = s.replace("T", " ") + parsed = None + for fmt in ("%Y-%m-%d %H:%M:%S", "%Y-%m-%d %H:%M", "%Y-%m-%d"): + try: + parsed = datetime.strptime(s2[:len(fmt) + 4] if False else s2, fmt) + break + except ValueError: + continue + if parsed is None: + try: + parsed = datetime.fromisoformat(s.replace("T", " ")) + except ValueError: + continue + out.append(parsed) + keep.append(i) + return out, keep + + +def _make_evolution_figure(name: str, rs: dict): + """Lazy callable: value-vs-time line + per-period row-count panel (MUST-9.1).""" + def _draw(): + import matplotlib + matplotlib.use("Agg") + import matplotlib.pyplot as plt + import matplotlib.dates as mdates + + t_labels = rs.get("t") or [] + v = rs.get("v") or [] + counts = rs.get("count") or [] + dates, keep = _parse_dates(t_labels) + vv = [v[i] if i < len(v) else None for i in keep] + cc = [counts[i] if i < len(counts) else 0 for i in keep] + + fig, (ax_v, ax_c) = plt.subplots( + 2, 1, figsize=(7.0, 4.6), sharex=True, + gridspec_kw={"height_ratios": [3.0, 1.2], "hspace": 0.12}) + + # Top: value aggregated per period (line; gaps where the value is None). + xs = [d for d, val in zip(dates, vv) if val is not None] + ys = [val for val in vv if val is not None] + if xs and ys: + ax_v.plot(xs, ys, color="#4e79a7", linewidth=1.4, zorder=3) + ax_v.fill_between(xs, ys, min(ys), color="#9ec6df", alpha=0.18, + zorder=1) + else: + ax_v.text(0.5, 0.5, "(sin valores numéricos)", ha="center", + va="center", fontsize=9, color="#8a8a8a", + transform=ax_v.transAxes) + ax_v.set_ylabel(name, fontsize=8) + ax_v.tick_params(labelsize=7) + ax_v.grid(axis="y", color="#eeeeee", linewidth=0.6) + for spine in ("top", "right"): + ax_v.spines[spine].set_visible(False) + + # Bottom: number of observations per period (density / gaps). + if dates and cc: + # Bar width ~ median spacing so bars do not overlap nor leave gaps. + width = 1.0 + if len(dates) > 1: + deltas = sorted((dates[i + 1] - dates[i]).days + for i in range(len(dates) - 1)) + width = max(deltas[len(deltas) // 2] * 0.8, 1.0) + ax_c.bar(dates, cc, width=width, color="#59a14f", alpha=0.75, + align="center") + ax_c.set_ylabel("nº filas", fontsize=8) + ax_c.tick_params(labelsize=7) + ax_c.grid(axis="y", color="#eeeeee", linewidth=0.6) + for spine in ("top", "right"): + ax_c.spines[spine].set_visible(False) + + ax_c.xaxis.set_major_locator(mdates.AutoDateLocator()) + ax_c.xaxis.set_major_formatter(mdates.ConciseDateFormatter( + ax_c.xaxis.get_major_locator())) + freq = rs.get("freq") + suptitle = f"{name} — evolución temporal" + if freq: + suptitle += f" (agregado {freq})" + fig.suptitle(suptitle, fontsize=10, fontweight="bold", x=0.02, ha="left") + return fig + + return _draw + + +def _make_stl_figure(stl: dict): + """Lazy callable: the STL trend/seasonal/resid panels, or None if no values. + + ``stl_decompose`` only carries the component *values* for short series; for + long ones it returns just summary stats (``note``). In that case there is + nothing to plot and we return None (the caller renders the strengths as text). + """ + def _component_values(comp): + if _is_dict(comp): + vals = comp.get("values") + if isinstance(vals, list) and vals: + return [x for x in vals] + return None + + trend = _component_values(stl.get("trend")) + seasonal = _component_values(stl.get("seasonal")) + resid = _component_values(stl.get("resid")) + if not any([trend, seasonal, resid]): + return None + + def _draw(): + import matplotlib + matplotlib.use("Agg") + import matplotlib.pyplot as plt + + panels = [("Tendencia", trend, "#4e79a7"), + ("Estacional", seasonal, "#59a14f"), + ("Resto", resid, "#e15759")] + panels = [(lbl, vals, col) for lbl, vals, col in panels if vals] + fig, axes = plt.subplots(len(panels), 1, figsize=(7.0, 1.4 * len(panels) + 0.6), + sharex=True) + if len(panels) == 1: + axes = [axes] + for ax, (lbl, vals, col) in zip(axes, panels): + ax.plot(range(len(vals)), vals, color=col, linewidth=1.2) + ax.set_ylabel(lbl, fontsize=8) + ax.tick_params(labelsize=7) + ax.grid(axis="y", color="#eeeeee", linewidth=0.6) + for spine in ("top", "right"): + ax.spines[spine].set_visible(False) + axes[-1].set_xlabel("índice temporal", fontsize=8) + fig.suptitle("Descomposición STL", fontsize=10, fontweight="bold", + x=0.02, ha="left") + fig.tight_layout(rect=(0, 0, 1, 0.96)) + return fig + + return _draw + + +def _make_acf_figure(acf_pacf: dict): + """Lazy callable: the ACF stem plot with ±1.96/√n bands, or None.""" + acf = acf_pacf.get("acf") + n = acf_pacf.get("n") + if not (isinstance(acf, list) and len(acf) > 1 and isinstance(n, int) and n > 0): + return None + + def _draw(): + import math + import matplotlib + matplotlib.use("Agg") + import matplotlib.pyplot as plt + + lags = list(range(len(acf))) + fig, ax = plt.subplots(figsize=(7.0, 3.2)) + ax.vlines(lags, 0, acf, color="#4e79a7", linewidth=1.4) + ax.plot(lags, acf, "o", color="#4e79a7", markersize=3) + band = 1.96 / math.sqrt(n) + ax.axhspan(-band, band, color="#cccccc", alpha=0.3, + label="banda ±1.96/√n (ruido blanco)") + ax.axhline(0, color="#888888", linewidth=0.8) + ax.set_xlabel("retardo (lag)", fontsize=8) + ax.set_ylabel("ACF", fontsize=8) + ax.tick_params(labelsize=7) + ax.legend(fontsize=7, loc="upper right", framealpha=0.85) + ax.set_title("Autocorrelación (ACF): lags fuera de la banda = " + "correlación significativa", fontsize=9) + fig.tight_layout() + return fig + + return _draw + + +# --------------------------------------------------------------------------- # +# Per-column textual analysis from profile['series'][col]. +# --------------------------------------------------------------------------- # +def _analysis_markdown(sblock: dict) -> str: + """One markdown block summarizing stationarity / autocorrelation / STL.""" + parts: list = [] + + stat = sblock.get("stationarity") if _is_dict(sblock.get("stationarity")) else {} + verdict = stat.get("verdict") + if verdict: + adf = stat.get("adf") if _is_dict(stat.get("adf")) else {} + kpss = stat.get("kpss") if _is_dict(stat.get("kpss")) else {} + line = (f"**Estacionariedad:** {_VERDICT_GLOSS.get(verdict, verdict)} " + f"(ADF p={_fmt_num(adf.get('p_value'), 4)}, " + f"KPSS p={_fmt_num(kpss.get('p_value'), 4)}).") + warning = stat.get("warning") + if warning: + line += f" ⚠ {model._safe_str(warning)}" + parts.append(line) + + acf = sblock.get("acf_pacf") if _is_dict(sblock.get("acf_pacf")) else {} + if acf: + is_auto = acf.get("is_autocorrelated") + lb = acf.get("ljung_box") if _is_dict(acf.get("ljung_box")) else {} + sig = acf.get("significant_acf_lags") or [] + if is_auto is True: + ac_line = ("**Autocorrelación:** la serie está autocorrelada " + "(Ljung-Box rechaza independencia, " + f"p={_fmt_num(lb.get('p_value'), 4)}): los valores dependen " + "de su pasado, no es ruido blanco.") + if sig: + shown = ", ".join(str(x) for x in sig[:8]) + more = "…" if len(sig) > 8 else "" + ac_line += f" Lags significativos: {shown}{more}." + elif is_auto is False: + ac_line = ("**Autocorrelación:** no se detecta autocorrelación " + "significativa (compatible con ruido blanco, Ljung-Box " + f"p={_fmt_num(lb.get('p_value'), 4)}).") + else: + ac_line = "**Autocorrelación:** no evaluable (datos insuficientes)." + parts.append(ac_line) + + stl = sblock.get("stl") if _is_dict(sblock.get("stl")) else {} + if stl: + ts = stl.get("trend_strength") + ss = stl.get("seasonal_strength") + if ts is not None or ss is not None: + parts.append( + "**Descomposición STL:** fuerza de tendencia " + f"{_fmt_num(ts, 2)} y fuerza estacional {_fmt_num(ss, 2)} " + "(escala 0–1 de Hyndman: cuanto más alto, más marcada la " + "componente).") + elif stl.get("note"): + parts.append(f"**Descomposición STL:** {model._safe_str(stl.get('note'))}") + + if sblock.get("levels_suggested"): + reason = sblock.get("levels_reason") + kind = sblock.get("levels_kind") + tr = sblock.get("to_returns") if _is_dict(sblock.get("to_returns")) else None + line = "**Transformación sugerida:** " + line += "pasar a retornos" if kind == "returns" else "diferenciar la serie" + if reason: + line += f" — {model._safe_str(reason)}" + if tr and tr.get("mean") is not None: + line += (f" (retornos: media {_fmt_num(tr.get('mean'), 5)}, " + f"σ {_fmt_num(tr.get('std'), 5)}).") + parts.append(line) + + return "\n\n".join(parts) + + +# --------------------------------------------------------------------------- # +# Per-column section. +# --------------------------------------------------------------------------- # +def _column_section(name: str, sblock: dict, raw: dict, collapsed_into) -> list: + """Blocks for one numeric column: evolution figure + STL + ACF + analysis.""" + blocks = [model.Heading(text=model._safe_str(name), level=2)] + + # --- Value-vs-time line + per-period row count (MUST-9.1). --- + drew_evolution = False + if collapsed_into is None: # skip the figure for collapsed OHLC duplicates. + t, v = _raw_series_for(raw, name) + if t is not None and resample_timeseries is not None: + try: + rs = resample_timeseries(t, v) + except Exception: # noqa: BLE001 + rs = None + if _is_dict(rs) and rs.get("t"): + blocks.append(model.Figure( + make=_make_evolution_figure(name, rs), + caption=f"Evolución de «{name}» por periodo y nº de " + f"observaciones (conteo de filas).")) + drew_evolution = True + else: + blocks.append(model.Note( + f"Serie casi idéntica a «{collapsed_into}» (grupo OHLC): se omite el " + "gráfico para no repetirlo; el análisis estadístico se mantiene.")) + + if not drew_evolution and collapsed_into is None: + blocks.append(model.Note( + "Gráfico de evolución temporal no disponible: falta la serie cruda " + "(pásala en ctx['timeseries_raw'] = {time_col, t, series}). Se " + "muestra solo el análisis estadístico.")) + + # --- STL panels (MUST-9.2). --- + stl = sblock.get("stl") if _is_dict(sblock.get("stl")) else {} + if collapsed_into is None and stl: + stl_fig = _make_stl_figure(stl) + if stl_fig is not None: + blocks.append(model.Figure( + make=stl_fig, + caption=f"Descomposición STL de «{name}»: tendencia, componente " + f"estacional y resto.")) + + # --- ACF figure (autocorrelation structure). --- + acf = sblock.get("acf_pacf") if _is_dict(sblock.get("acf_pacf")) else {} + if collapsed_into is None and acf: + acf_fig = _make_acf_figure(acf) + if acf_fig is not None: + blocks.append(model.Figure( + make=acf_fig, + caption=f"Función de autocorrelación de «{name}».")) + + # --- Textual analysis (always, even for collapsed duplicates). --- + analysis = _analysis_markdown(sblock) + if analysis: + blocks.append(model.Markdown(text=analysis)) + return blocks + + +# --------------------------------------------------------------------------- # +# Entry point. +# --------------------------------------------------------------------------- # +def build_timeseries(profile: dict, ctx: dict): + """Build the TIMESERIES Chapter, or ``None`` if the table has no date column. + + Args: + profile: the ``eda`` group TableProfile dict. + ctx: presentation context; ``ctx['timeseries_raw']`` (optional) carries + the raw ordered series used to draw the value-vs-time line and the + per-period row count. + + Returns: + A ``model.Chapter`` with, per numeric column, the value-vs-time evolution + + row-count figure, the STL panels, the ACF figure and the statistical + analysis; or ``None`` when there is no temporal column (the chapter does + not apply). + """ + profile = profile or {} + if not _is_dict(profile): + profile = {} + ctx = ctx or {} + cols = profile.get("columns") or [] + + det = _detect(cols) + time_col = det.get("time_col") + if not time_col: + return None # no date/datetime column -> chapter does not apply. + + numeric_cols = det.get("numeric_cols") or [] + series_map = profile.get("series") if _is_dict(profile.get("series")) else {} + raw = ctx.get("timeseries_raw") or profile.get("timeseries_raw") + raw = raw if _is_dict(raw) else {} + + # Which columns can the chapter say anything about: those with a series + # analysis block and/or a raw series to chart. Preserve the profile order. + chartable = [] + for name in numeric_cols: + has_analysis = _is_dict(series_map.get(name)) + has_raw, _ = _raw_series_for(raw, name) + if has_analysis or has_raw is not None: + chartable.append(name) + if not chartable: + # A date column exists but nothing numeric to chart/analyse: still a + # valid (small) chapter — show just the datetime header if we have it. + header = _datetime_header(time_col, raw) + if not header: + return None + intro = ( + f"La tabla tiene una columna temporal («{time_col}») pero no hay " + "columnas numéricas con serie analizable.") + blocks = [model.Heading(text=CHAPTER_TITLE, level=1), + model.Markdown(text=intro)] + header + return model.Chapter(id=CHAPTER_ID, title=CHAPTER_TITLE, + version=CHAPTER_VERSION, blocks=blocks) + + collapsed = _ohlc_groups(chartable, raw) + + intro = ( + "Este capítulo analiza la evolución de la tabla en el tiempo usando la " + f"columna de fecha «{time_col}». Para cada columna numérica se muestra su " + "**evolución por periodo** (valor agregado) junto al **número de filas por " + "periodo** (densidad de observaciones), su **descomposición STL** " + "(tendencia / estacionalidad / resto) y la **función de autocorrelación**; " + "debajo, el análisis de la serie: estacionariedad (ADF + KPSS), " + "autocorrelación (Ljung-Box) y, cuando procede, la transformación " + "sugerida (retornos o diferencias) para evitar correlaciones espurias.") + + blocks = [model.Heading(text=CHAPTER_TITLE, level=1), + model.Markdown(text=intro)] + blocks += _datetime_header(time_col, raw) + + if collapsed: + reps = sorted(set(collapsed.values())) + collapsed_names = ", ".join(sorted(collapsed.keys())) + blocks.append(model.Note( + f"Series OHLC casi idénticas detectadas ({collapsed_names}): se " + f"grafican consolidadas en «{', '.join(reps)}» para no repetir el " + "mismo gráfico; cada columna conserva su análisis estadístico.")) + + for name in chartable: + sblock = series_map.get(name) if _is_dict(series_map.get(name)) else {} + blocks += _column_section(name, sblock, raw, collapsed.get(name)) + + return model.Chapter(id=CHAPTER_ID, title=CHAPTER_TITLE, + version=CHAPTER_VERSION, blocks=blocks) diff --git a/python/functions/datascience/automatic_eda/chapters/timeseries_test.py b/python/functions/datascience/automatic_eda/chapters/timeseries_test.py new file mode 100644 index 00000000..2e18c022 --- /dev/null +++ b/python/functions/datascience/automatic_eda/chapters/timeseries_test.py @@ -0,0 +1,244 @@ +"""Tests for the TIMESERIES chapter — DoD: golden + edges + anti-cut. + +Self-contained: builds synthetic ``series`` blocks (shaped like +``profile_table(run_series=True)`` output) and a raw ``timeseries_raw`` bundle, +with no DuckDB, so the suite is fast and deterministic. Verifies that the chapter: + +- returns ``None`` when there is no date/datetime column (the user requirement); +- never raises on ``None``/empty/garbage input; +- with a date column + raw series emits, per numeric column, the value-vs-time + + row-count evolution figure, the STL panels, the ACF figure and the textual + analysis (stationarity / autocorrelation / suggested transform); +- collapses near-identical OHLC series into one chart while keeping every + column's analysis; +- renders without cutting anything in both PDF and PPTX (every column heading + survives in the rendered output). +""" + +import math +import os +import re +import tempfile + +from pypdf import PdfReader + +from datascience.automatic_eda.chapters.timeseries import ( + build_timeseries, CHAPTER_VERSION, _VERDICT_GLOSS, +) +from datascience.render_automatic_eda_pdf import render_automatic_eda_pdf +from datascience.render_automatic_eda_pptx import render_automatic_eda_pptx + + +# --------------------------------------------------------------------------- # +# Synthetic fixtures shaped like the real profile_table(run_series=True) output. +# --------------------------------------------------------------------------- # +def _dates(n: int) -> list: + """n consecutive daily ISO date strings starting 2021-01-01.""" + from datetime import date, timedelta + + start = date(2021, 1, 1) + return [(start + timedelta(days=i)).isoformat() for i in range(n)] + + +def _series_block(n=120, verdict="non_stationary", autocorr=True, levels=True, + with_stl_values=True): + """A synthetic ``series`` block like _build_series_block produces.""" + trend = [float(i) for i in range(n)] + seasonal = [math.sin(i / 6.0) for i in range(n)] + resid = [0.1 * ((-1) ** i) for i in range(n)] + acf = [1.0] + [max(0.0, 0.9 - 0.05 * k) for k in range(1, 21)] + block = { + "order_col": "fecha", + "ordered": True, + "n": n, + "stationarity": { + "n": n, "verdict": verdict, + "adf": {"p_value": 0.42, "stationary": False}, + "kpss": {"p_value": 0.01, "stationary": False}, + "warning": ("serie no estacionaria: riesgo de correlación espuria" + if verdict != "stationary" else None), + }, + "acf_pacf": { + "n": n, "nlags": 20, "acf": acf, + "significant_acf_lags": [1, 2, 3, 4, 5], + "ljung_box": {"stat": 123.4, "p_value": 0.0 if autocorr else 0.7, + "lags": 20}, + "is_autocorrelated": autocorr, + }, + "period_source": "datetime_freq", + "stl": { + "n": n, "period": 7, "period_inferred": False, "robust": False, + "trend": {"values": trend} if with_stl_values else { + "note": "serie larga: solo estadisticos", "mean": 60.0}, + "seasonal": {"values": seasonal} if with_stl_values else {"mean": 0.0}, + "resid": {"values": resid} if with_stl_values else {"mean": 0.0}, + "trend_strength": 0.95, "seasonal_strength": 0.42, + }, + } + if levels: + block["levels_suggested"] = True + block["levels_kind"] = "returns" + block["levels_reason"] = ("columna financiera no estacionaria: usar " + "retornos evita correlación espuria.") + block["to_returns"] = {"method": "log", "mean": 0.001, "std": 0.02} + else: + block["levels_suggested"] = False + return block + + +def _profile(numeric_names=("precio",), n=120, with_stl_values=True): + cols = [{"name": "fecha", "inferred_type": "datetime", + "semantic_type": "datetime_iso"}] + series_map = {} + for nm in numeric_names: + cols.append({"name": nm, "inferred_type": "numeric", + "numeric": {"min": 1.0, "max": 200.0, "mean": 100.0, + "median": 95.0, "std": 40.0}}) + series_map[nm] = _series_block(n=n, with_stl_values=with_stl_values) + return {"table": "cotizaciones", "n_rows": n, "n_cols": len(cols), + "columns": cols, "series": series_map} + + +def _ctx_raw(numeric_names=("precio",), n=120): + t = _dates(n) + series = {} + for j, nm in enumerate(numeric_names): + series[nm] = [float(100 + i + 5 * j) for i in range(n)] + return {"timeseries_raw": {"time_col": "fecha", "t": t, "series": series}} + + +def _pdf_text(path: str) -> str: + txt = "".join((pg.extract_text() or "") for pg in PdfReader(path).pages) + return re.sub(r"\s+", " ", txt) + + +# --------------------------------------------------------------------------- # +# Golden. +# --------------------------------------------------------------------------- # +def test_golden_estructura_y_figuras(): + ch = build_timeseries(_profile(("precio",)), _ctx_raw(("precio",))) + assert ch is not None + assert ch.id == "timeseries" + assert ch.version == CHAPTER_VERSION + kinds = [b.kind for b in ch.blocks] + assert kinds[0] == "heading" # chapter title + assert kinds[1] == "markdown" # intro + assert "kv_table" in kinds # datetime profile header (MUST-9.3) + # Per column: evolution figure + STL figure + ACF figure + analysis markdown. + figs = [b for b in ch.blocks if b.kind == "figure"] + assert len(figs) >= 3, "evolución + STL + ACF esperadas" + # Lazy makers must produce real matplotlib figures. + import matplotlib.pyplot as plt + for f in figs: + fig = f.make() + assert fig is not None + plt.close(fig) + + +def test_golden_evolucion_tiene_dos_paneles_valor_y_conteo(): + # MUST-9.1: the evolution figure has a value panel + a row-count panel. + ch = build_timeseries(_profile(("precio",)), _ctx_raw(("precio",))) + figs = [b for b in ch.blocks if b.kind == "figure"] + import matplotlib.pyplot as plt + fig = figs[0].make() # first figure is the evolution one. + assert len(fig.axes) == 2, "panel de valor + panel de conteo de filas" + plt.close(fig) + + +def test_golden_analisis_textual_presente(): + ch = build_timeseries(_profile(("precio",)), _ctx_raw(("precio",))) + md = " ".join(b.text for b in ch.blocks if b.kind == "markdown") + assert "Estacionariedad" in md + assert "Autocorrelación" in md + assert "STL" in md + # Verdict gloss surfaced for the non-stationary preset. + assert _VERDICT_GLOSS["non_stationary"].split(":")[0] in md + # Levels/returns suggestion surfaced. + assert "retornos" in md.lower() + + +# --------------------------------------------------------------------------- # +# Edges. +# --------------------------------------------------------------------------- # +def test_edge_sin_columna_fecha_devuelve_none(): + prof = {"columns": [ + {"name": "precio", "inferred_type": "numeric", "numeric": {"mean": 1.0}}, + {"name": "ciudad", "inferred_type": "categorical", + "categorical": {"top": []}}, + ], "series": {"precio": _series_block()}} + assert build_timeseries(prof, {}) is None + + +def test_edge_none_y_vacio_no_revienta(): + assert build_timeseries(None, None) is None + assert build_timeseries({}, {}) is None + assert build_timeseries({"columns": []}, {}) is None + # Date column but nothing numeric/series and no raw -> None (nothing to say). + assert build_timeseries( + {"columns": [{"name": "fecha", "inferred_type": "datetime"}]}, {}) is None + + +def test_edge_sin_raw_degrada_pero_mantiene_analisis(): + # No ctx['timeseries_raw']: the chapter must still build (STL/ACF/analysis + # from the profile) and note that the evolution chart is unavailable. + ch = build_timeseries(_profile(("precio",)), {}) + assert ch is not None + notes = " ".join(b.text for b in ch.blocks if b.kind == "note") + assert "evolución temporal no disponible" in notes + md = " ".join(b.text for b in ch.blocks if b.kind == "markdown") + assert "Estacionariedad" in md + + +def test_edge_stl_solo_estadisticos_no_dibuja_panel_pero_no_revienta(): + # Long series: STL carries only stats (no 'values') -> no STL figure, but the + # strengths still surface in the textual analysis. + ch = build_timeseries(_profile(("precio",), with_stl_values=False), + _ctx_raw(("precio",))) + assert ch is not None + md = " ".join(b.text for b in ch.blocks if b.kind == "markdown") + assert "STL" in md + + +# --------------------------------------------------------------------------- # +# OHLC consolidation (MUST-9.3). +# --------------------------------------------------------------------------- # +def test_ohlc_consolidacion(): + names = ("Open", "High", "Low", "Close") + ch = build_timeseries(_profile(names), _ctx_raw(names)) + assert ch is not None + notes = " ".join(b.text for b in ch.blocks if b.kind == "note") + assert "OHLC" in notes + # Only the representative draws the evolution figure; the other 3 are collapsed + # so there are fewer evolution figures than columns. + captions = [b.caption or "" for b in ch.blocks if b.kind == "figure"] + evo = [c for c in captions if "Evolución" in c] + assert len(evo) < len(names), "las series OHLC deben consolidarse" + # Every column still has its analysis markdown (one heading per column). + headings = [b.text for b in ch.blocks if b.kind == "heading" and b.level == 2] + for nm in names: + assert nm in headings + + +# --------------------------------------------------------------------------- # +# Anti-cut: PDF + PPTX. +# --------------------------------------------------------------------------- # +def test_anti_corte_pdf_y_pptx(): + names = tuple(f"serie_{i}" for i in range(6)) + prof = _profile(names, n=90) + ctx = _ctx_raw(names, n=90) + ch = build_timeseries(prof, ctx) + col_headings = [b.text for b in ch.blocks if b.kind == "heading" and b.level == 2] + assert len(col_headings) == 6 + with tempfile.TemporaryDirectory() as d: + pdf = os.path.join(d, "ts.pdf") + res_pdf = render_automatic_eda_pdf( + prof, pdf, {"ctx": ctx, "write_manifest": False}) + assert res_pdf["path"] == pdf + txt = _pdf_text(pdf) + for nm in col_headings: + assert nm in txt, f"columna '{nm}' cortada/ausente en el PDF" + pptx = os.path.join(d, "ts.pptx") + res_pptx = render_automatic_eda_pptx( + prof, pptx, {"ctx": ctx, "write_manifest": False}) + assert res_pptx["path"] == pptx + assert res_pptx["n_slides"] >= 6 diff --git a/python/functions/datascience/detect_time_column.md b/python/functions/datascience/detect_time_column.md new file mode 100644 index 00000000..60d75994 --- /dev/null +++ b/python/functions/datascience/detect_time_column.md @@ -0,0 +1,68 @@ +--- +name: detect_time_column +kind: function +lang: py +domain: datascience +version: "1.0.0" +purity: pure +signature: "def detect_time_column(columns: list) -> dict" +description: "Detecta, a partir de la lista de ColumnProfile de un TableProfile del grupo eda, cual es la columna de orden temporal y que columnas numericas hay para graficar una serie en el tiempo. Una columna es temporal si inferred_type=='datetime' o semantic_type in {datetime_iso, date_eu}; time_col es la primera temporal en orden. Es la pieza que usa el capitulo TIMESERIES del AutomaticEDA para decidir si aplica. Lectura defensiva dict-no-throw: nunca lanza, siempre devuelve las mismas claves." +tags: [eda, timeseries, datetime, profiling, column-detection, automatic-eda, datascience, python] +uses_functions: [] +uses_types: [] +returns: [] +returns_optional: false +error_type: "" +imports: [] +params: + - name: columns + desc: "lista de ColumnProfile dict de un TableProfile del grupo eda. Cada elemento suele tener name, inferred_type, semantic_type y numeric. Elementos que no sean dict se ignoran; None/no-lista/vacia -> dict 'no aplica'." +output: "dict SIEMPRE con: time_col (str|None, columna temporal elegida = primera temporal), time_semantic (str, semantic_type de la temporal o ''), numeric_cols (list[str], columnas con inferred_type=='numeric' en orden), n_datetime_cols (int), datetime_cols (list[str], todas las temporales en orden de aparicion), reason (str en espanol explicando la eleccion). Nunca lanza excepcion." +tested: true +tests: ["test_golden_datetime_y_numericas", "test_deteccion_por_semantic_type_date_eu", "test_sin_columna_temporal", "test_columns_none_no_revienta", "test_columns_vacia_no_revienta", "test_columns_no_lista_no_revienta", "test_elementos_basura_se_ignoran", "test_varias_datetime_elige_la_primera"] +test_file_path: "python/functions/datascience/detect_time_column_test.py" +file_path: "python/functions/datascience/detect_time_column.py" +--- + +## Ejemplo + +```python +from datascience import detect_time_column + +columns = [ + {"name": "fecha", "inferred_type": "datetime", "semantic_type": "datetime_iso"}, + {"name": "ventas", "inferred_type": "numeric"}, + {"name": "unidades", "inferred_type": "numeric"}, + {"name": "region", "inferred_type": "text"}, +] +res = detect_time_column(columns) +res["time_col"] # -> "fecha" +res["numeric_cols"] # -> ["ventas", "unidades"] +res["n_datetime_cols"] # -> 1 + +# Sin columna temporal: el capitulo TIMESERIES no aplica. +detect_time_column([{"name": "id", "inferred_type": "numeric"}])["time_col"] # -> None +``` + +## Cuando usarla + +Cuando el capitulo TIMESERIES del AutomaticEDA recibe un TableProfile y necesita +decidir si la tabla admite analisis de serie temporal: si `time_col` es None no +hay eje de tiempo y el capitulo se salta; si hay `time_col` y `numeric_cols`, +úsalas como eje X (orden cronologico) y series Y. Tambien sirve para enrutar el +resto del pipeline (acf_pacf / stl_decompose / adf_kpss_stationarity) sobre las +columnas numericas detectadas. + +## Gotchas + +- Es pura y stdlib-only (sin numpy ni DuckDB): segura de llamar en cualquier paso. +- `time_col` se elige por ORDEN de aparicion en la lista, no por "mejor candidata". + Si hay varias columnas datetime y quieres otra, filtra `datetime_cols` tu mismo. +- Solo mira metadatos del perfil (`inferred_type`/`semantic_type`); no parsea ni + valida los valores reales de la columna. La calidad de la deteccion depende de + que el profiler (summarize_table_duckdb / infer_semantic_type) haya inferido bien. +- Las claves del semantic_type son exactamente las del profiler: `datetime_iso` + (ISO 8601) y `date_eu` (DD/MM/AAAA). Otros formatos de fecha no se detectan por + semantic_type salvo que `inferred_type` ya sea `"datetime"`. +- `numeric_cols` se basa en `inferred_type == "numeric"` (no en "integer"/"float"); + si tu profiler usa otra etiqueta, normalizala antes. diff --git a/python/functions/datascience/detect_time_column.py b/python/functions/datascience/detect_time_column.py new file mode 100644 index 00000000..aa817720 --- /dev/null +++ b/python/functions/datascience/detect_time_column.py @@ -0,0 +1,112 @@ +"""Detecta la columna temporal y las columnas numericas de un TableProfile (grupo eda). + +Funcion pura y determinista: a partir de la lista de columnas de un TableProfile +producido por el grupo de capacidad `eda` (cada elemento es un ColumnProfile dict), +decide cual es la columna de orden temporal y que columnas numericas hay disponibles +para graficar una serie en el tiempo. Es la pieza que usa el capitulo TIMESERIES del +AutomaticEDA para decidir si la tabla admite analisis de serie temporal. + +Lectura 100% defensiva al estilo "dict-no-throw" del grupo eda: nunca lanza +excepcion, siempre devuelve el mismo conjunto de claves. +""" + +# semantic_type que el profiler (infer_semantic_type) emite para fechas/datetimes. +_DATETIME_SEMANTICS = ("datetime_iso", "date_eu") + + +def detect_time_column(columns: list) -> dict: + """Detecta la columna temporal y las numericas de una lista de ColumnProfile. + + Recorre los ColumnProfile de un TableProfile y clasifica cada columna como + temporal o numerica leyendo de forma defensiva sus claves. Una columna es + temporal si su ``inferred_type == "datetime"`` o si su ``semantic_type`` esta + en {``"datetime_iso"``, ``"date_eu"``}. La columna temporal elegida + (``time_col``) es la PRIMERA temporal en el orden de la lista. Las numericas + (``numeric_cols``) son las de ``inferred_type == "numeric"``, en orden. + + Funcion pura: no hace I/O, no muta el input, es determinista. + + Args: + columns: lista de ColumnProfile dict del grupo eda. Cada elemento suele + tener claves como ``name``, ``inferred_type``, ``semantic_type`` y + ``numeric``. Los elementos que no sean dict se ignoran. Si ``columns`` + es None, no es lista o esta vacia, se devuelve el dict "no aplica". + + Returns: + Siempre un dict con las mismas claves:: + + { + "time_col": str | None, # columna temporal elegida (None si no hay) + "time_semantic": str, # semantic_type de la temporal ("" si no aplica) + "numeric_cols": [str, ...], # columnas con inferred_type == "numeric" + "n_datetime_cols": int, # nº de columnas temporales detectadas + "datetime_cols": [str, ...],# todas las temporales, en orden de aparicion + "reason": str, # frase corta (en espanol) que explica la eleccion + } + """ + # Caso "no aplica": entrada invalida o vacia. + if not isinstance(columns, list) or not columns: + return { + "time_col": None, + "time_semantic": "", + "numeric_cols": [], + "n_datetime_cols": 0, + "datetime_cols": [], + "reason": "no se detecto columna de fecha/datetime", + } + + datetime_cols: list[str] = [] + datetime_semantics: list[str] = [] + numeric_cols: list[str] = [] + + for col in columns: + # Ignora elementos que no sean dict sin fallar. + if not isinstance(col, dict): + continue + + name = col.get("name") + if name is None: + name = "" + else: + name = str(name) + + inferred_type = col.get("inferred_type") or "" + semantic_type = col.get("semantic_type") or "" + + is_datetime = inferred_type == "datetime" or semantic_type in _DATETIME_SEMANTICS + if is_datetime: + datetime_cols.append(name) + datetime_semantics.append(semantic_type) + + if inferred_type == "numeric": + numeric_cols.append(name) + + if not datetime_cols: + return { + "time_col": None, + "time_semantic": "", + "numeric_cols": numeric_cols, + "n_datetime_cols": 0, + "datetime_cols": [], + "reason": "no se detecto columna de fecha/datetime", + } + + time_col = datetime_cols[0] + time_semantic = datetime_semantics[0] + + if len(datetime_cols) == 1: + reason = f"columna temporal '{time_col}' detectada" + else: + reason = ( + f"{len(datetime_cols)} columnas temporales; se elige la primera " + f"'{time_col}'" + ) + + return { + "time_col": time_col, + "time_semantic": time_semantic, + "numeric_cols": numeric_cols, + "n_datetime_cols": len(datetime_cols), + "datetime_cols": datetime_cols, + "reason": reason, + } diff --git a/python/functions/datascience/detect_time_column_test.py b/python/functions/datascience/detect_time_column_test.py new file mode 100644 index 00000000..0b391d2a --- /dev/null +++ b/python/functions/datascience/detect_time_column_test.py @@ -0,0 +1,102 @@ +"""Tests para detect_time_column (grupo eda). Self-contained, sin DuckDB.""" + +from detect_time_column import detect_time_column + + +def test_golden_datetime_y_numericas(): + columns = [ + {"name": "fecha", "inferred_type": "datetime", "semantic_type": "datetime_iso"}, + {"name": "ventas", "inferred_type": "numeric"}, + {"name": "unidades", "inferred_type": "numeric"}, + {"name": "region", "inferred_type": "text"}, + ] + res = detect_time_column(columns) + assert res["time_col"] == "fecha" + assert res["time_semantic"] == "datetime_iso" + assert res["numeric_cols"] == ["ventas", "unidades"] + assert res["n_datetime_cols"] == 1 + assert res["datetime_cols"] == ["fecha"] + assert isinstance(res["reason"], str) and res["reason"] + + +def test_deteccion_por_semantic_type_date_eu(): + # inferred_type no es datetime, pero semantic_type date_eu => temporal. + columns = [ + {"name": "id", "inferred_type": "numeric"}, + {"name": "dia", "inferred_type": "text", "semantic_type": "date_eu"}, + {"name": "importe", "inferred_type": "numeric"}, + ] + res = detect_time_column(columns) + assert res["time_col"] == "dia" + assert res["time_semantic"] == "date_eu" + assert res["numeric_cols"] == ["id", "importe"] + assert res["n_datetime_cols"] == 1 + assert res["datetime_cols"] == ["dia"] + + +def test_sin_columna_temporal(): + columns = [ + {"name": "id", "inferred_type": "numeric"}, + {"name": "nombre", "inferred_type": "text"}, + {"name": "activo", "inferred_type": "boolean"}, + ] + res = detect_time_column(columns) + assert res["time_col"] is None + assert res["time_semantic"] == "" + assert res["numeric_cols"] == ["id"] + assert res["n_datetime_cols"] == 0 + assert res["datetime_cols"] == [] + assert res["reason"] == "no se detecto columna de fecha/datetime" + + +def test_columns_none_no_revienta(): + res = detect_time_column(None) + assert res["time_col"] is None + assert res["time_semantic"] == "" + assert res["numeric_cols"] == [] + assert res["n_datetime_cols"] == 0 + assert res["datetime_cols"] == [] + assert res["reason"] == "no se detecto columna de fecha/datetime" + + +def test_columns_vacia_no_revienta(): + res = detect_time_column([]) + assert res["time_col"] is None + assert res["numeric_cols"] == [] + assert res["n_datetime_cols"] == 0 + + +def test_columns_no_lista_no_revienta(): + # Un dict (no lista) tambien debe caer en el caso "no aplica". + res = detect_time_column({"name": "fecha", "inferred_type": "datetime"}) + assert res["time_col"] is None + assert res["numeric_cols"] == [] + + +def test_elementos_basura_se_ignoran(): + columns = [ + None, + "no soy un dict", + 42, + {"name": "ts", "inferred_type": "datetime"}, + {"name": "valor", "inferred_type": "numeric"}, + ] + res = detect_time_column(columns) + assert res["time_col"] == "ts" + assert res["numeric_cols"] == ["valor"] + assert res["n_datetime_cols"] == 1 + + +def test_varias_datetime_elige_la_primera(): + columns = [ + {"name": "created_at", "inferred_type": "datetime", "semantic_type": "datetime_iso"}, + {"name": "metric", "inferred_type": "numeric"}, + {"name": "updated_at", "inferred_type": "datetime", "semantic_type": "datetime_iso"}, + {"name": "fecha_baja", "inferred_type": "text", "semantic_type": "date_eu"}, + ] + res = detect_time_column(columns) + assert res["time_col"] == "created_at" + assert res["time_semantic"] == "datetime_iso" + assert res["n_datetime_cols"] == 3 + assert res["datetime_cols"] == ["created_at", "updated_at", "fecha_baja"] + assert res["numeric_cols"] == ["metric"] diff --git a/python/functions/datascience/extract_timeseries_raw.md b/python/functions/datascience/extract_timeseries_raw.md new file mode 100644 index 00000000..5ce92f91 --- /dev/null +++ b/python/functions/datascience/extract_timeseries_raw.md @@ -0,0 +1,92 @@ +--- +name: extract_timeseries_raw +kind: function +lang: py +domain: datascience +version: "1.0.0" +purity: impure +signature: "def extract_timeseries_raw(query_fn, table: str, time_col: str, value_cols: list, max_rows: int = 5000) -> dict" +description: "Extrae la serie temporal CRUDA (fechas + una o varias columnas numericas) de una tabla, ordenada cronologicamente, para alimentar el render del capitulo TIMESERIES de AutomaticEDA (linea valor-vs-tiempo + conteo por periodo). Recibe un lector read-only inyectado `query_fn(sql) -> dict` (mismo contrato que duckdb_query_readonly / pg_query / el `_q` de profile_table) y NO abre ninguna conexion por su cuenta. Construye UNA sola query con identificadores escapados, ORDER BY por la columna temporal y LIMIT. Devuelve dict dict-no-throw: t (fechas ISO string), series (lista paralela float|None por columna) y n. El capitulo no toca la BD: recibe esto en ctx['timeseries_raw']. Reutilizable tambien por profile_table en una fase futura." +tags: [eda, timeseries, datascience, automatic-eda, extraction, read-only, duckdb, postgres, python] +uses_functions: [] +uses_types: [] +returns: [] +returns_optional: false +error_type: "error_go_core" +imports: [datetime] +params: + - name: query_fn + desc: "callable lector read-only del backend activo. Recibe un string SQL y devuelve un dict {'status':'ok','rows':[{col:val,...},...]} (mismo contrato que duckdb_query_readonly o el `_q` de profile_table). NO se abre ninguna conexion dentro de la funcion: toda la lectura pasa por query_fn. Si es None -> error." + - name: table + desc: "nombre de la tabla de la que extraer la serie. Se escapa con comillas dobles en la query." + - name: time_col + desc: "nombre de la columna de orden temporal. Se usa en ORDER BY (cronologico ascendente) y se filtra IS NOT NULL. Sus valores se devuelven en `t` como string ISO." + - name: value_cols + desc: "lista de nombres de columnas numericas a extraer. Cada una produce una entrada en `series` con una lista paralela a `t`. Vacia o None -> status error." + - name: max_rows + desc: "limite de filas a leer (clausula LIMIT). Default 5000. Protege el render frente a tablas enormes." +output: "dict (nunca lanza). En exito: {'status':'ok','time_col':str,'t':[str,...] (fechas ISO en orden),'series':{col:[float|None,...],...} (paralela a t por value_col, None si el valor no es convertible a float),'n':int}. En error (sin lanzar): {'status':'error','error':str,'time_col':str,'t':[],'series':{},'n':0}. Errores: query_fn None, value_cols vacia, table/time_col vacios, o query_fn devuelve status!='ok' (se propaga su error)." +tested: true +tests: ["test_golden_t_y_series_alineadas", "test_valor_no_convertible_da_none", "test_value_cols_vacia_status_error", "test_query_fn_status_error_propaga", "test_query_fn_none_da_error_sin_reventar", "test_sql_contiene_order_by_y_limit"] +test_file_path: "python/functions/datascience/extract_timeseries_raw_test.py" +file_path: "python/functions/datascience/extract_timeseries_raw.py" +--- + +## Ejemplo + +```python +import sys, os +sys.path.insert(0, os.path.join("python", "functions")) +from datascience import extract_timeseries_raw +from infra import duckdb_query_readonly + +# El lector read-only se inyecta como closure (igual que el `_q` de profile_table). +db = "data/ventas.duckdb" +def _q(sql): + return duckdb_query_readonly(db, sql) + +res = extract_timeseries_raw(_q, "ventas_diarias", "fecha", ["importe", "unidades"]) +# res == { +# "status": "ok", +# "time_col": "fecha", +# "t": ["2024-01-01", "2024-01-02", ...], +# "series": {"importe": [1234.5, 980.0, ...], "unidades": [12.0, 9.0, ...]}, +# "n": 365, +# } + +# Se entrega al capitulo TIMESERIES sin que este toque la BD: +ctx = {"timeseries_raw": res} +``` + +## Cuando usarla + +Cuando el capitulo TIMESERIES de AutomaticEDA necesita pintar una serie +valor-vs-tiempo (o conteo por periodo) y NO debe abrir la base de datos por su +cuenta: extraes aqui las fechas + columnas numericas ordenadas y se las pasas en +`ctx['timeseries_raw']`. Usala tambien siempre que quieras la secuencia cruda +ordenada cronologicamente de una o varias columnas para alimentar otros +contrastes de serie (ADF/KPSS, ACF/PACF, STL) reutilizando un unico lector +read-only inyectado, en vez de hacer N muestreos a mano. + +## Gotchas + +- **Impura**: lee de la base de datos a traves de `query_fn`. No abre conexiones + por su cuenta — depende por completo del lector inyectado. Sigue el estilo + dict-no-throw del grupo `eda`: nunca lanza; ante cualquier fallo devuelve + `{"status":"error","error":...}` con `t=[]`, `series={}`, `n=0`. +- **`error_type` en el frontmatter es `error_go_core` por convencion del registry** + (toda funcion impura debe declararlo y el indexer lo exige), pero el codigo + NO lanza esa excepcion: degrada al dict de error. Es metadata, no comportamiento. +- **No loguear los datos crudos**: `t`/`series` pueden contener datos sensibles + (igual que un HAR). No volcar el dict completo a logs ni a telemetria; en + trazas usa solo `n` y los nombres de columna. +- **Alineacion por fila**: `series[col][i]` corresponde a `t[i]`. Un valor no + convertible a float se guarda como `None` (no se descarta la fila) para no + romper la alineacion temporal. +- **Orden**: el orden cronologico depende del `ORDER BY "time_col"` del backend. + Si `time_col` esta guardada como texto con formato no lexicograficamente + ordenable (p.ej. `DD/MM/YYYY`), el orden no sera el real — normaliza la columna + a date/timestamp antes, o pasa una columna ya ordenable. +- **`max_rows`**: con LIMIT, si la tabla supera `max_rows` obtienes solo el primer + tramo cronologico, no un muestreo uniforme. Sube `max_rows` si necesitas el rango + completo. diff --git a/python/functions/datascience/extract_timeseries_raw.py b/python/functions/datascience/extract_timeseries_raw.py new file mode 100644 index 00000000..455c90f4 --- /dev/null +++ b/python/functions/datascience/extract_timeseries_raw.py @@ -0,0 +1,122 @@ +"""extract_timeseries_raw — extrae la serie temporal CRUDA de una tabla. + +Lector read-only inyectado: recibe `query_fn(sql) -> dict` con el mismo contrato +que duckdb_query_readonly / pg_query (y que el `_q` de profile_table): +`{"status": "ok", "rows": [{col: val, ...}, ...]}`. Esta funcion NO abre ninguna +conexion por su cuenta — solo usa `query_fn`. Construye UNA sola query ordenada +por la columna temporal y devuelve las fechas (`t`) mas cada columna numerica en +listas paralelas (`series`), listas para alimentar el render del capitulo +TIMESERIES de AutomaticEDA (linea valor-vs-tiempo + conteo por periodo) sin que +el capitulo toque la base de datos: recibe esto en `ctx['timeseries_raw']`. + +Estilo dict-no-throw del grupo `eda`: nunca lanza; captura cualquier excepcion y +degrada a `{"status": "error", "error": str, ...}`. +""" + +from datetime import date, datetime + + +def _to_float(value): + """Convierte un valor a float de forma defensiva. None si no es convertible.""" + if value is None: + return None + if isinstance(value, bool): + # Un bool es subclase de int en Python; no es un valor de serie valido. + return None + if isinstance(value, (int, float)): + return float(value) + s = str(value).strip() + if not s: + return None + try: + return float(s) + except (TypeError, ValueError): + return None + + +def _to_iso(value): + """Convierte un valor temporal a string ISO conservando el orden de la query. + + date/datetime -> isoformat(); cualquier otro valor (string, etc.) -> str(). + None se preserva como None. + """ + if value is None: + return None + if isinstance(value, (datetime, date)): + return value.isoformat() + return str(value) + + +def extract_timeseries_raw(query_fn, table, time_col, value_cols, max_rows=5000): + """Extrae la serie temporal cruda (fechas + columnas numericas) de una tabla. + + Args: + query_fn: callable lector read-only del backend activo. Recibe un string + SQL y devuelve un dict {"status": "ok", "rows": [{col: val, ...}]} + (mismo contrato que duckdb_query_readonly / el `_q` de profile_table). + No se abre ninguna conexion aqui: toda la lectura pasa por query_fn. + table: nombre de la tabla. + time_col: nombre de la columna de orden temporal. + value_cols: lista de nombres de columnas numericas a extraer. + max_rows: limite de filas (LIMIT). Default 5000. + + Returns: + dict (nunca lanza): + { + "status": "ok" | "error", + "error": str, # solo si status == "error" + "time_col": str, + "t": [str, ...], # time_col como ISO string, en orden + "series": {col: [float|None, ...], ...}, # paralela a t por columna + "n": int # nº de filas devueltas + } + """ + base = {"status": "ok", "time_col": time_col, "t": [], "series": {}, "n": 0} + try: + if query_fn is None: + return {**base, "status": "error", "error": "query_fn es None"} + if not value_cols: + return {**base, "status": "error", "error": "value_cols vacío"} + if not table or not time_col: + return { + **base, + "status": "error", + "error": "table y time_col son obligatorios", + } + + # Identificadores escapados con comillas dobles (como hace profile_table) + # para tolerar nombres con mayusculas/espacios/palabras reservadas. + cols_sql = ", ".join(f'"{c}"' for c in value_cols) + sql = ( + f'SELECT "{time_col}", {cols_sql} FROM "{table}" ' + f'WHERE "{time_col}" IS NOT NULL ' + f'ORDER BY "{time_col}" ' + f"LIMIT {int(max_rows)}" + ) + + q = query_fn(sql) + if not isinstance(q, dict) or q.get("status") != "ok": + err = ( + q.get("error", "query_fn fallo") + if isinstance(q, dict) + else "query_fn no devolvio un dict" + ) + return {**base, "status": "error", "error": err} + + rows = q.get("rows", []) or [] + t = [] + series = {c: [] for c in value_cols} + for row in rows: + t.append(_to_iso(row.get(time_col))) + for c in value_cols: + series[c].append(_to_float(row.get(c))) + + return { + "status": "ok", + "time_col": time_col, + "t": t, + "series": series, + "n": len(t), + } + except Exception as e: # noqa: BLE001 - dict-no-throw: degradar, nunca lanzar + return {**base, "status": "error", "error": str(e)} diff --git a/python/functions/datascience/extract_timeseries_raw_test.py b/python/functions/datascience/extract_timeseries_raw_test.py new file mode 100644 index 00000000..bcfd1bd5 --- /dev/null +++ b/python/functions/datascience/extract_timeseries_raw_test.py @@ -0,0 +1,109 @@ +"""Tests para extract_timeseries_raw. + +No usa DuckDB real: inyecta un query_fn FAKE (closure) que devuelve filas +predefinidas y, opcionalmente, captura el SQL recibido para verificar la query +generada (ORDER BY por la columna temporal + LIMIT). Asi el test es +autocontenido y no depende de ningun backend. +""" + +import os +import sys + +sys.path.insert(0, os.path.dirname(__file__)) + +from extract_timeseries_raw import extract_timeseries_raw + + +def _fake_query(rows, captured=None, status="ok", error=None): + """Crea un query_fn FAKE. + + `captured` (lista opcional) recibe el SQL ejecutado para poder inspeccionarlo. + `status`/`error` permiten simular un fallo del backend. + """ + + def _q(sql): + if captured is not None: + captured.append(sql) + if status != "ok": + return {"status": "error", "error": error or "boom"} + return {"status": "ok", "rows": rows} + + return _q + + +def test_golden_t_y_series_alineadas(): + """Golden: t y series alineadas, floats convertidos, n correcto.""" + rows = [ + {"fecha": "2024-01-01", "ventas": "10", "stock": 5}, + {"fecha": "2024-01-02", "ventas": "20.5", "stock": 7}, + {"fecha": "2024-01-03", "ventas": 30, "stock": 9}, + ] + res = extract_timeseries_raw(_fake_query(rows), "t", "fecha", ["ventas", "stock"]) + assert res["status"] == "ok" + assert res["n"] == 3 + assert res["time_col"] == "fecha" + assert res["t"] == ["2024-01-01", "2024-01-02", "2024-01-03"] + assert res["series"]["ventas"] == [10.0, 20.5, 30.0] + assert res["series"]["stock"] == [5.0, 7.0, 9.0] + + +def test_valor_no_convertible_da_none(): + """Valor no convertible a float -> None en la serie (alineacion preservada).""" + rows = [ + {"fecha": "2024-01-01", "ventas": "abc"}, + {"fecha": "2024-01-02", "ventas": None}, + {"fecha": "2024-01-03", "ventas": "12.5"}, + ] + res = extract_timeseries_raw(_fake_query(rows), "t", "fecha", ["ventas"]) + assert res["status"] == "ok" + assert res["series"]["ventas"] == [None, None, 12.5] + assert res["n"] == 3 + + +def test_value_cols_vacia_status_error(): + """value_cols vacia -> status error con t/series/n vacios.""" + res = extract_timeseries_raw(_fake_query([]), "t", "fecha", []) + assert res["status"] == "error" + assert "value_cols" in res["error"] + assert res["t"] == [] + assert res["series"] == {} + assert res["n"] == 0 + + +def test_query_fn_status_error_propaga(): + """query_fn que devuelve status != ok -> se propaga como error.""" + res = extract_timeseries_raw( + _fake_query([], status="error", error="db locked"), + "t", + "fecha", + ["ventas"], + ) + assert res["status"] == "error" + assert "db locked" in res["error"] + assert res["n"] == 0 + + +def test_query_fn_none_da_error_sin_reventar(): + """query_fn None -> error degradado, sin excepcion.""" + res = extract_timeseries_raw(None, "t", "fecha", ["ventas"]) + assert res["status"] == "error" + assert res["t"] == [] + assert res["n"] == 0 + + +def test_sql_contiene_order_by_y_limit(): + """La query generada ordena por time_col y aplica el LIMIT sobre la tabla.""" + captured = [] + rows = [{"fecha": "2024-01-01", "ventas": 1}] + extract_timeseries_raw( + _fake_query(rows, captured), + "ventas_tbl", + "fecha", + ["ventas"], + max_rows=123, + ) + assert len(captured) == 1 + sql = captured[0] + assert 'ORDER BY "fecha"' in sql + assert "LIMIT 123" in sql + assert 'FROM "ventas_tbl"' in sql diff --git a/python/functions/datascience/profile_datetime.md b/python/functions/datascience/profile_datetime.md new file mode 100644 index 00000000..0f0ced85 --- /dev/null +++ b/python/functions/datascience/profile_datetime.md @@ -0,0 +1,79 @@ +--- +name: profile_datetime +kind: function +lang: py +domain: datascience +version: "1.0.0" +purity: pure +signature: "def profile_datetime(values: list) -> dict" +description: "Perfil minimo de una columna fecha/datetime para la cabecera del capitulo TIMESERIES de AutomaticEDA. Acepta datetime.date, datetime.datetime y strings ISO mezclados, parsea defensivamente e ignora lo no parseable (nunca lanza). Devuelve rango (min/max ISO), n, n_distinct, span_days, frecuencia inferida (daily/weekly/monthly/quarterly/yearly/irregular/unknown) a partir del paso mediano entre fechas distintas, is_regular (pasos ~constantes), n_gaps (huecos en la rejilla) y median_step_days. Solo stdlib (datetime + statistics)." +tags: [statistics, timeseries, datetime, profiling, frequency, eda, automatic_eda, python] +uses_functions: [] +uses_types: [] +returns: [] +returns_optional: false +error_type: "" +imports: [datetime, statistics] +params: + - name: values + desc: "lista de valores fecha. Acepta datetime.date, datetime.datetime y strings ISO ('2021-06-28', '2021-06-28T00:00:00', '2021-06-28 12:00:00'). None, vacios y no parseables se ignoran; tz-aware se normaliza a naive. Si values es None o no iterable se trata como lista vacia." +output: "dict SIEMPRE presente con: 'min'/'max' (ISO date YYYY-MM-DD o None), 'n' (valores parseables), 'n_distinct' (fechas unicas), 'span_days' (float o None), 'freq' (daily|weekly|monthly|quarterly|yearly|irregular|unknown), 'is_regular' (bool), 'n_gaps' (int), 'median_step_days' (float o None) y 'note' (str). Con <2 valores o una sola fecha distinta: freq='unknown', is_regular=False, n_gaps=0, median_step_days=None y nota. Nunca lanza." +tested: true +tests: ["test_serie_diaria_regular_golden", "test_serie_mensual_freq_monthly", "test_serie_con_hueco_cuenta_gaps", "test_strings_iso_mezclados_con_datetime", "test_lista_vacia_y_none_devuelve_unknown", "test_valores_no_parseables_ignorados", "test_span_days_correcto", "test_una_sola_fecha_es_coherente"] +test_file_path: "python/functions/datascience/profile_datetime_test.py" +file_path: "python/functions/datascience/profile_datetime.py" +--- + +## Ejemplo + +```python +from datascience import profile_datetime +from datetime import date, datetime, timedelta + +# Serie diaria regular de 30 dias +fechas = [date(2021, 1, 1) + timedelta(days=i) for i in range(30)] +res = profile_datetime(fechas) +res["freq"] # -> "daily" +res["is_regular"] # -> True +res["n_gaps"] # -> 0 +res["min"], res["max"] # -> ("2021-01-01", "2021-01-30") +res["span_days"] # -> 29.0 + +# Acepta strings ISO mezclados con objetos datetime/date; ignora lo no parseable +profile_datetime(["2021-06-28", datetime(2021, 6, 29, 12), "basura", None])["n"] # -> 2 + +# Columna vacia o sin fechas validas +profile_datetime([])["freq"] # -> "unknown" + note "datos insuficientes" +``` + +## Cuando usarla + +Cuando construyes la cabecera del capitulo TIMESERIES de un EDA y necesitas +caracterizar la columna de fecha antes de modelar: que rango cubre, cada cuanto +llegan los datos (frecuencia), si la cadencia es regular y si hay huecos en la +rejilla temporal. Es el complemento de fecha al perfil numerico/categorico del +TableProfile (cierra el `datetime{}=None` pendiente). Pasale la columna de fechas +en bruto (tal cual venga de la BD: dates, datetimes o strings ISO) y usa `freq` + +`is_regular` + `n_gaps` para decidir si conviene resamplear, rellenar huecos o +desestacionalizar mas adelante. + +## Gotchas + +- Es pura y stdlib-only, pero la inferencia de `freq` es heuristica por bandas + sobre el **paso mediano entre fechas distintas** (se deduplica antes de medir). + Cualquier paso fuera de las bandas conocidas (incluido sub-diario, p.ej. datos + horarios) cae en `"irregular"`: no hay banda hourly. +- El analisis de frecuencia/regularidad/huecos necesita **>=2 fechas distintas**. + Con 0-1 valores parseables o una sola fecha unica, `freq="unknown"`, + `median_step_days=None` y `n_gaps=0`, pero `min`/`max`/`span_days` siguen siendo + coherentes si hay al menos una fecha. +- `min`/`max` se reportan como ISO **date** (`YYYY-MM-DD`); la hora se conserva + internamente para calcular `span_days` y `median_step_days` (que pueden ser + fraccionarios con datetimes sub-diarios) pero no aparece en min/max. +- Los datetime con zona horaria se normalizan a naive (se descarta el tzinfo) para + poder mezclarlos con fechas naive sin que las restas lancen; esto puede desplazar + la fecha en datetimes con offset grande. Para EDA es despreciable. +- `is_regular` usa tolerancia ±25% sobre el paso mediano y umbral del 80% de los + pasos dentro de banda; series de "primero de mes" (deltas 28-31) salen regulares. +- `n_gaps` solo se calcula cuando `freq` es una rejilla regular conocida; con + `freq` `"irregular"` o `"unknown"` siempre es 0. diff --git a/python/functions/datascience/profile_datetime.py b/python/functions/datascience/profile_datetime.py new file mode 100644 index 00000000..15c21ffa --- /dev/null +++ b/python/functions/datascience/profile_datetime.py @@ -0,0 +1,183 @@ +"""Perfil minimo de una columna fecha/datetime para la cabecera TIMESERIES (grupo eda). + +Funcion pura y determinista que resume una columna temporal: rango (min/max), +numero de fechas distintas, frecuencia inferida (daily/weekly/monthly/quarterly/ +yearly/irregular), regularidad de los pasos, huecos respecto a la rejilla inferida +y paso mediano entre fechas consecutivas. Cierra el `datetime{}=None` que hoy deja +pendiente el TableProfile de AutomaticEDA. + +Acepta valores heterogeneos (``datetime.date``, ``datetime.datetime`` y strings +ISO como ``"2021-06-28"``, ``"2021-06-28T00:00:00"`` o ``"2021-06-28 12:00:00"``), +parsea de forma defensiva, ignora lo que no se puede parsear y NUNCA lanza. + +Solo usa stdlib (``datetime`` + ``statistics``). +""" + +from __future__ import annotations + +import statistics +from datetime import date, datetime + + +def _parse_one(v) -> datetime | None: + """Parsea un valor a ``datetime`` naive, o devuelve None si no es una fecha. + + Acepta ``datetime.datetime``, ``datetime.date`` y strings ISO. Cualquier + datetime con zona horaria se normaliza a naive (se descarta el tzinfo) para + poder mezclarlo con fechas naive sin que las restas lancen ``TypeError``. + """ + if v is None or isinstance(v, bool): + return None + # datetime es subclase de date: comprobar datetime primero. + if isinstance(v, datetime): + return v.replace(tzinfo=None) + if isinstance(v, date): + return datetime(v.year, v.month, v.day) + if isinstance(v, str): + s = v.strip() + if not s: + return None + try: + dt = datetime.fromisoformat(s) + except ValueError: + return None + return dt.replace(tzinfo=None) + return None + + +def _infer_freq(median_step_days: float) -> str: + """Clasifica la frecuencia a partir del paso mediano (en dias) entre fechas. + + Bandas con tolerancia: ~1 dia -> daily, ~7 -> weekly, 28-31 -> monthly, + 89-92 -> quarterly, 360-366 -> yearly. Cualquier paso fuera de las bandas + (incluido sub-diario) -> irregular. + """ + m = median_step_days + if 0.5 <= m <= 1.5: + return "daily" + if 6.0 <= m <= 8.0: + return "weekly" + if 28.0 <= m <= 31.0: + return "monthly" + if 89.0 <= m <= 92.0: + return "quarterly" + if 360.0 <= m <= 366.0: + return "yearly" + return "irregular" + + +def profile_datetime(values: list) -> dict: + """Perfila una columna de fechas para la cabecera del capitulo TIMESERIES. + + Funcion pura y determinista: no hace I/O, no muta el input y nunca lanza. + + El analisis de frecuencia, regularidad y huecos se hace sobre las **fechas + distintas ordenadas** (se deduplica antes de calcular los pasos): los valores + repetidos generarian pasos de 0 dias que distorsionarian el mediano y la + inferencia. ``n`` cuenta los valores parseables (con duplicados) y + ``n_distinct`` las fechas unicas. + + Args: + values: lista de valores fecha. Acepta ``datetime.date``, + ``datetime.datetime`` y strings ISO (``"2021-06-28"``, + ``"2021-06-28T00:00:00"``, ``"2021-06-28 12:00:00"``). Los valores + None, vacios o no parseables se ignoran. Si ``values`` es None o no + iterable se trata como lista vacia. + + Returns: + Siempre un dict con esta forma:: + + { + "min": str | None, # fecha minima ISO date (YYYY-MM-DD) + "max": str | None, # fecha maxima ISO date + "n": int, # nº de valores fecha parseables + "n_distinct": int, # nº de fechas distintas + "span_days": float | None, # (max - min) en dias + "freq": str, # daily|weekly|monthly|quarterly| + # yearly|irregular|unknown + "is_regular": bool, # pasos ~constantes (tolerancia ±25%) + "n_gaps": int, # saltos > ~1.5x el paso mediano + "median_step_days": float | None, # paso mediano entre fechas + "note": str # "" o nota corta + } + + Con menos de 2 valores parseables (o una sola fecha distinta) devuelve + ``freq="unknown"``, ``is_regular=False``, ``n_gaps=0``, + ``median_step_days=None`` y la nota correspondiente, manteniendo min/max + y span_days coherentes cuando hay al menos una fecha. + """ + base = { + "min": None, + "max": None, + "n": 0, + "n_distinct": 0, + "span_days": None, + "freq": "unknown", + "is_regular": False, + "n_gaps": 0, + "median_step_days": None, + "note": "", + } + + if values is None: + values = [] + try: + iterator = list(values) + except TypeError: + iterator = [] + + parsed: list[datetime] = [] + for v in iterator: + dt = _parse_one(v) + if dt is not None: + parsed.append(dt) + + n = len(parsed) + base["n"] = n + + if n == 0: + base["note"] = "datos insuficientes" + return base + + distinct = sorted(set(parsed)) + n_distinct = len(distinct) + dt_min = min(parsed) + dt_max = max(parsed) + + base["n_distinct"] = n_distinct + base["min"] = dt_min.date().isoformat() + base["max"] = dt_max.date().isoformat() + base["span_days"] = round((dt_max - dt_min).total_seconds() / 86400.0, 6) + + # Sin al menos dos fechas distintas no hay pasos que medir. + if n_distinct < 2: + base["note"] = "datos insuficientes" if n < 2 else "una sola fecha distinta" + return base + + steps = [ + (distinct[i + 1] - distinct[i]).total_seconds() / 86400.0 + for i in range(n_distinct - 1) + ] + median_step = float(statistics.median(steps)) + base["median_step_days"] = round(median_step, 6) + + freq = _infer_freq(median_step) + base["freq"] = freq + + # Regularidad: >=80% de los pasos dentro de ±25% del paso mediano. + if median_step > 0: + tol = 0.25 * median_step + within = sum(1 for s in steps if abs(s - median_step) <= tol) + base["is_regular"] = (within / len(steps)) >= 0.8 + else: + base["is_regular"] = False + + # Huecos: pasos que superan ~1.5x el mediano. Solo tiene sentido cuando la + # frecuencia es una rejilla regular conocida (no irregular/unknown). + if freq not in ("unknown", "irregular") and median_step > 0: + threshold = 1.5 * median_step + base["n_gaps"] = sum(1 for s in steps if s > threshold) + else: + base["n_gaps"] = 0 + + return base diff --git a/python/functions/datascience/profile_datetime_test.py b/python/functions/datascience/profile_datetime_test.py new file mode 100644 index 00000000..533f701d --- /dev/null +++ b/python/functions/datascience/profile_datetime_test.py @@ -0,0 +1,127 @@ +"""Tests para profile_datetime.""" + +from datetime import date, datetime, timedelta + +from profile_datetime import profile_datetime + + +def test_serie_diaria_regular_golden(): + # 30 dias consecutivos: frecuencia diaria, regular, sin huecos. + fechas = [date(2021, 1, 1) + timedelta(days=i) for i in range(30)] + res = profile_datetime(fechas) + assert res["n"] == 30 + assert res["n_distinct"] == 30 + assert res["min"] == "2021-01-01" + assert res["max"] == "2021-01-30" + assert res["span_days"] == 29.0 + assert res["freq"] == "daily" + assert res["is_regular"] is True + assert res["n_gaps"] == 0 + assert res["median_step_days"] == 1.0 + assert res["note"] == "" + + +def test_serie_mensual_freq_monthly(): + # Primero de mes durante 14 meses: paso mediano ~30/31 dias -> monthly. + fechas = [] + y, m = 2021, 1 + for _ in range(14): + fechas.append(date(y, m, 1)) + m += 1 + if m > 12: + m = 1 + y += 1 + res = profile_datetime(fechas) + assert res["n"] == 14 + assert res["freq"] == "monthly" + assert res["min"] == "2021-01-01" + assert res["max"] == "2022-02-01" + assert 28.0 <= res["median_step_days"] <= 31.0 + + +def test_serie_con_hueco_cuenta_gaps(): + # Serie diaria con un hueco de 3 dias (faltan i=7,8,9) -> n_gaps >= 1. + fechas = [ + date(2021, 1, 1) + timedelta(days=i) + for i in range(20) + if i not in (7, 8, 9) + ] + res = profile_datetime(fechas) + assert res["freq"] == "daily" + assert res["n_gaps"] >= 1 + assert res["median_step_days"] == 1.0 + + +def test_strings_iso_mezclados_con_datetime(): + # Mezcla de strings ISO (varios formatos) y objetos datetime/date. + valores = [ + "2021-06-28", + datetime(2021, 6, 29, 12, 0, 0), + "2021-06-30T00:00:00", + date(2021, 7, 1), + ] + res = profile_datetime(valores) + assert res["n"] == 4 + assert res["n_distinct"] == 4 + assert res["min"] == "2021-06-28" + assert res["max"] == "2021-07-01" + assert res["freq"] == "daily" + assert res["note"] == "" + + +def test_lista_vacia_y_none_devuelve_unknown(): + for entrada in ([], None): + res = profile_datetime(entrada) + assert res["n"] == 0 + assert res["n_distinct"] == 0 + assert res["min"] is None + assert res["max"] is None + assert res["span_days"] is None + assert res["freq"] == "unknown" + assert res["is_regular"] is False + assert res["n_gaps"] == 0 + assert res["median_step_days"] is None + assert res["note"] == "datos insuficientes" + + +def test_valores_no_parseables_ignorados(): + # Strings basura, None, ints y un date valido mezclados: ignora lo no fecha. + valores = [ + "no es una fecha", + None, + "2021-01-01", + "2021-01-02", + 12345, + "tampoco", + date(2021, 1, 3), + "", + ] + res = profile_datetime(valores) + assert res["n"] == 3 # solo 3 fechas parseables + assert res["n_distinct"] == 3 + assert res["freq"] == "daily" + assert res["min"] == "2021-01-01" + assert res["max"] == "2021-01-03" + + +def test_span_days_correcto(): + # Dos fechas a un anio de distancia: span 365 dias -> yearly. + res = profile_datetime([date(2020, 1, 1), date(2020, 12, 31)]) + assert res["n"] == 2 + assert res["n_distinct"] == 2 + assert res["span_days"] == 365.0 + assert res["median_step_days"] == 365.0 + assert res["freq"] == "yearly" + + +def test_una_sola_fecha_es_coherente(): + # Un unico valor: min == max, span 0, freq unknown, nota datos insuficientes. + res = profile_datetime(["2021-06-28"]) + assert res["n"] == 1 + assert res["n_distinct"] == 1 + assert res["min"] == "2021-06-28" + assert res["max"] == "2021-06-28" + assert res["span_days"] == 0.0 + assert res["freq"] == "unknown" + assert res["median_step_days"] is None + assert res["note"] == "datos insuficientes" diff --git a/python/functions/datascience/resample_timeseries.md b/python/functions/datascience/resample_timeseries.md new file mode 100644 index 00000000..554920d2 --- /dev/null +++ b/python/functions/datascience/resample_timeseries.md @@ -0,0 +1,72 @@ +--- +name: resample_timeseries +kind: function +lang: py +domain: datascience +version: "1.0.0" +purity: pure +signature: "def resample_timeseries(t: list, v: list, freq: str = \"auto\", agg: str = \"mean\", max_points: int = 400) -> dict" +description: "Agrega una serie temporal por periodo para graficar su evolucion y el CONTEO de observaciones por bucket. Nucleo del capitulo TIMESERIES de AutomaticEDA (grupo eda): recibe las fechas y los valores YA leidos (pura, sin tocar ninguna base de datos), empareja t[i] con v[i] por indice, parsea fechas defensivamente, trunca cada fecha al inicio de su bucket (daily/weekly/monthly/quarterly/yearly), y agrega los valores numericos validos por bucket mientras cuenta TODAS las observaciones con fecha valida (densidad temporal, incluida la fila cuyo valor es None). freq='auto' infiere del delta mediano entre fechas. Si hay mas buckets que max_points hace downsampling uniforme conservando primero y ultimo. Estilo dict-no-throw: NUNCA lanza; entrada vacia o longitudes incompatibles devuelve listas vacias + note='datos insuficientes'." +tags: [eda, timeseries, resample, aggregate, profiling, datascience, time] +params: + - name: t + desc: "Lista de fechas paralela a v. Acepta strings ISO ('YYYY-MM-DD' o 'YYYY-MM-DDTHH:MM:SS', con 'Z' opcional), datetime.date o datetime.datetime. Se parsea defensivamente; los pares cuya fecha no parsea se descartan junto con su valor." + - name: v + desc: "Lista de valores numericos (float/int) paralela a t. Puede contener None o valores no numericos: se ignoran en la agregacion pero la fila sigue contando en 'count' si su fecha es valida. bool, NaN e Inf se tratan como no numericos." + - name: freq + desc: "Granularidad del bucket: 'auto' (infiere del delta mediano en dias entre fechas: <=3 daily, <=16 weekly, <=75 monthly, <=200 quarterly, mayor yearly) o explicita en {daily, weekly, monthly, quarterly, yearly}. Una frecuencia desconocida cae a 'auto'." + - name: agg + desc: "Agregacion por bucket sobre los valores numericos validos: 'mean' | 'sum' | 'median' | 'last' (valor de la observacion cronologicamente mas reciente del bucket) | 'min' | 'max'. Una agregacion desconocida cae a 'mean'." + - name: max_points + desc: "Tope de buckets en la salida. Si n_buckets > max_points hace downsampling uniforme (1 de cada k buckets equiespaciados, conservando el primero y el ultimo) para no saturar el grafico del PDF/PPTX. max_points<=0 desactiva el limite." +output: "Dict siempre con las mismas claves: t (lista de etiquetas ISO 'YYYY-MM-DD' por bucket, orden cronologico), v (lista paralela del valor agregado por bucket segun agg; None si el bucket no tiene ningun valor numerico valido), count (lista paralela del nº de observaciones con fecha valida por bucket), freq (frecuencia efectivamente usada), agg (agregacion usada), n_in (nº de pares (t,v) con fecha valida que entraron), n_buckets (nº de buckets antes del downsample), downsampled (bool, True si se aplico downsampling), note ('' o 'datos insuficientes' cuando no hay pares validos / longitudes incompatibles / listas vacias). Numericos de v en float, count en int." +uses_functions: [] +uses_types: [] +returns: [] +returns_optional: false +error_type: "" +imports: [] +tested: true +tests: ["test_daily_a_mensual_mean", "test_agg_sum_y_last", "test_count_cuenta_observacion_con_valor_none", "test_downsampling_respeta_max_points_y_extremos", "test_freq_auto_infiere_mensual", "test_edge_listas_vacias_o_desiguales"] +test_file_path: "python/functions/datascience/resample_timeseries_test.py" +file_path: "python/functions/datascience/resample_timeseries.py" +--- + +## Ejemplo + +```python +import sys, os +sys.path.insert(0, os.path.join("python", "functions")) +from datascience.resample_timeseries import resample_timeseries + +# Serie diaria agregada a buckets mensuales: media del valor + conteo de filas. +t = ["2020-01-01", "2020-01-15", "2020-02-01", "2020-02-10", "2020-02-20"] +v = [10.0, 20.0, 30.0, 40.0, 50.0] + +r = resample_timeseries(t, v, freq="monthly", agg="mean") +print(r["t"]) # ['2020-01-01', '2020-02-01'] +print(r["v"]) # [15.0, 40.0] +print(r["count"]) # [2, 3] <- densidad: nº de observaciones por mes +print(r["freq"], r["downsampled"]) # monthly False + +# freq='auto' infiere la granularidad del delta mediano entre fechas. +mensual = [f"2022-{m:02d}-01" for m in range(1, 13)] +print(resample_timeseries(mensual, list(range(1, 13)))["freq"]) # monthly +``` + +## Cuando usarla + +- Usala en el capitulo TIMESERIES de `AutomaticEDA` para construir, a partir de una columna temporal (`detect_time_column`) y una columna numerica, la doble serie que el renderer dibuja: la EVOLUCION del valor agregado por periodo y el CONTEO de observaciones por periodo. +- Cuando ya tengas las fechas y los valores leidos en memoria (de DuckDB, polars, CSV, etc.) y solo necesites agregarlos por dia/semana/mes/trimestre/año sin volver a tocar la base de datos — esta funcion es pura y recibe los datos por parametro. +- Cuando quieras un downsampling controlado para que una serie muy larga (miles de fechas) quepa en un grafico de un PDF/PPTX sin saturarlo, conservando el primer y el ultimo punto. +- Cuando no sepas la cadencia de la serie: pasa `freq="auto"` y deja que la infiera del delta mediano. + +## Gotchas + +- Funcion pura, sin I/O y determinista. NUNCA lanza: ante entrada invalida (listas vacias, longitudes distintas o todas las fechas no parseables) devuelve listas vacias + `note="datos insuficientes"`. +- `count` cuenta OBSERVACIONES con fecha valida en el bucket (densidad temporal), aunque su valor numerico sea `None`/no numerico. `v` agrega SOLO los valores numericos validos del bucket; si no hay ninguno, `v` del bucket es `None` mientras `count` sigue reflejando las filas. No confundas `count` (filas) con el nº de valores agregados. +- `bool`, `NaN` e `Inf` se tratan como NO numericos (se ignoran en `v`). Un string que no parsea a numero tambien se ignora en `v` pero su fila cuenta si la fecha es valida. +- El truncado de bucket usa el inicio del periodo: semana = lunes ISO (`weekday()==0`), mes = dia 1, trimestre = primer dia del trimestre (ene/abr/jul/oct), año = 1 de enero. La etiqueta de cada bucket es esa fecha de inicio en ISO `YYYY-MM-DD`, no un rango. +- El downsampling (`n_buckets > max_points`) reduce la salida a `<= max_points` puntos equiespaciados conservando primero y ultimo, pero `n_buckets` SIEMPRE reporta el conteo real previo al recorte. Si necesitas todos los buckets, sube `max_points` o ponlo `<=0`. +- Las fechas con hora se truncan a su `date()` antes de agrupar: la granularidad minima es el dia (no hay buckets horarios). +- `freq` desconocida o no-string cae a `"auto"`; `agg` desconocida cae a `"mean"`. El campo devuelto refleja la opcion efectivamente usada. diff --git a/python/functions/datascience/resample_timeseries.py b/python/functions/datascience/resample_timeseries.py new file mode 100644 index 00000000..c3e49a26 --- /dev/null +++ b/python/functions/datascience/resample_timeseries.py @@ -0,0 +1,275 @@ +"""Agrega una serie temporal por periodo para el capitulo TIMESERIES (grupo eda). + +Funcion pura y determinista: recibe las fechas y los valores YA leidos (nunca +toca una base de datos ni hace I/O) y los agrega por bucket temporal para poder +graficar la evolucion de la serie y, en paralelo, el CONTEO de observaciones por +periodo (densidad temporal). + +Estilo "dict-no-throw" del grupo eda: NUNCA lanza excepcion, siempre devuelve el +mismo conjunto de claves. Lectura y parseo de fechas 100% defensivos. Solo usa la +libreria estandar (``datetime``, ``statistics``, ``re``). +""" + +from __future__ import annotations + +import datetime +import re +import statistics + +# Frecuencias soportadas, de mas fina a mas gruesa. +_FREQS = ("daily", "weekly", "monthly", "quarterly", "yearly") + +# Agregaciones soportadas. +_AGGS = ("mean", "sum", "median", "last", "min", "max") + +# Acepta el inicio de una fecha ISO con cualquier separador posterior +# (incluido un caracter raro entre la fecha y la hora). +_DATE_RE = re.compile(r"(\d{4})-(\d{2})-(\d{2})") + + +def _to_date(x) -> "datetime.date | None": + """Parsea defensivamente un valor a ``datetime.date``; devuelve None si falla.""" + if x is None: + return None + # datetime es subclase de date: comprobarlo primero. + if isinstance(x, datetime.datetime): + return x.date() + if isinstance(x, datetime.date): + return x + s = str(x).strip() + if not s: + return None + # Camino feliz: ISO completo (con o sin hora, con o sin 'Z' final). + try: + s2 = s[:-1] if s.endswith("Z") else s + return datetime.datetime.fromisoformat(s2).date() + except ValueError: + pass + # Fallback robusto: extrae el prefijo YYYY-MM-DD con cualquier separador. + m = _DATE_RE.match(s) + if m: + try: + return datetime.date(int(m.group(1)), int(m.group(2)), int(m.group(3))) + except ValueError: + return None + return None + + +def _to_number(x) -> "float | None": + """Convierte a float si es numerico finito; devuelve None en otro caso.""" + if x is None: + return None + if isinstance(x, bool): + # bool es subclase de int: lo tratamos como no-numerico para una serie. + return None + try: + f = float(x) + except (TypeError, ValueError): + return None + # Descarta NaN / Inf (no agregables de forma estable). + if f != f or f in (float("inf"), float("-inf")): + return None + return f + + +def _infer_freq(dates_sorted: list) -> str: + """Infiere la frecuencia desde el delta mediano (en dias) entre fechas.""" + if len(dates_sorted) < 2: + return "daily" + diffs = [ + (dates_sorted[i + 1] - dates_sorted[i]).days + for i in range(len(dates_sorted) - 1) + ] + diffs = [d for d in diffs if d > 0] # ignora duplicados del mismo dia + if not diffs: + return "daily" + med = statistics.median(diffs) + if med <= 3: + return "daily" + if med <= 16: + return "weekly" + if med <= 75: + return "monthly" + if med <= 200: + return "quarterly" + return "yearly" + + +def _bucket_start(d: "datetime.date", freq: str) -> "datetime.date": + """Trunca una fecha al inicio de su bucket segun la frecuencia.""" + if freq == "weekly": + return d - datetime.timedelta(days=d.weekday()) # lunes ISO + if freq == "monthly": + return datetime.date(d.year, d.month, 1) + if freq == "quarterly": + first_month = ((d.month - 1) // 3) * 3 + 1 + return datetime.date(d.year, first_month, 1) + if freq == "yearly": + return datetime.date(d.year, 1, 1) + return d # daily (o cualquier otra cosa): la propia fecha + + +def _downsample_indices(n: int, max_points: int) -> list: + """Indices equiespaciados conservando primero y ultimo (<= max_points).""" + if max_points <= 0 or max_points >= n: + return list(range(n)) + if max_points == 1: + return [0] + idx = sorted({round(i * (n - 1) / (max_points - 1)) for i in range(max_points)}) + return idx + + +def _empty(freq_req: str, agg: str) -> dict: + """Resultado canonico cuando no hay datos suficientes.""" + eff_freq = freq_req if freq_req in _FREQS else "auto" + return { + "t": [], + "v": [], + "count": [], + "freq": eff_freq, + "agg": agg if agg in _AGGS else "mean", + "n_in": 0, + "n_buckets": 0, + "downsampled": False, + "note": "datos insuficientes", + } + + +def resample_timeseries( + t: list, + v: list, + freq: str = "auto", + agg: str = "mean", + max_points: int = 400, +) -> dict: + """Agrega una serie temporal por periodo (buckets) para graficarla. + + Empareja ``t[i]`` con ``v[i]`` por indice, descarta los pares cuya fecha no + parsea, trunca cada fecha al inicio de su bucket segun ``freq`` y agrupa. Por + cada bucket devuelve el valor agregado (``agg`` sobre los valores numericos + validos) y el CONTEO de observaciones con fecha valida (densidad temporal), + independientemente de si su valor numerico es ``None``. + + Funcion pura: no hace I/O, no muta los inputs, es determinista, NUNCA lanza. + + Args: + t: lista de fechas paralela a ``v``. Acepta strings ISO + (``"YYYY-MM-DD"`` o ``"YYYY-MM-DDTHH:MM:SS"``, con ``Z`` opcional), + ``datetime.date`` o ``datetime.datetime``. Se parsea defensivamente; + las fechas que no parsean se descartan junto con su valor. + v: lista de valores numericos (float/int). Puede contener ``None`` o + valores no numericos: estos se ignoran en la agregacion, pero la fila + sigue contando en ``count`` (siempre que su fecha sea valida). + freq: ``"auto"`` (infiere del delta mediano entre fechas) o uno de + ``"daily"``, ``"weekly"``, ``"monthly"``, ``"quarterly"``, + ``"yearly"``. Una frecuencia desconocida cae a ``"auto"``. + agg: agregacion por bucket: ``"mean"``, ``"sum"``, ``"median"``, + ``"last"`` (valor de la observacion cronologicamente mas reciente), + ``"min"`` o ``"max"``. Una agregacion desconocida cae a ``"mean"``. + max_points: si tras agregar hay mas buckets que este limite, se hace + downsampling uniforme (1 de cada k buckets equiespaciados, + conservando el primero y el ultimo) para no saturar el grafico. + + Returns: + Siempre un dict con las mismas claves:: + + { + "t": [str, ...], # etiqueta ISO YYYY-MM-DD de cada bucket + "v": [float|None, ...], # valor agregado por bucket (None si vacio) + "count": [int, ...], # nº de observaciones con fecha valida + "freq": str, # frecuencia efectivamente usada + "agg": str, # agregacion usada + "n_in": int, # nº de pares (t,v) con fecha valida + "n_buckets": int, # nº de buckets antes del downsample + "downsampled": bool, # True si se aplico downsampling + "note": str, # "" o nota (p.ej. "datos insuficientes") + } + """ + agg = agg if agg in _AGGS else "mean" + freq_req = freq if isinstance(freq, str) else "auto" + + # Validacion de entrada: deben ser listas de igual longitud y no vacias. + if ( + not isinstance(t, list) + or not isinstance(v, list) + or len(t) == 0 + or len(t) != len(v) + ): + return _empty(freq_req, agg) + + # Empareja por indice y descarta fechas no parseables. + parsed: list = [] # (date, original_index, number_or_None) + for i, (ti, vi) in enumerate(zip(t, v)): + d = _to_date(ti) + if d is None: + continue + parsed.append((d, i, _to_number(vi))) + + n_in = len(parsed) + if n_in == 0: + return _empty(freq_req, agg) + + # Resuelve la frecuencia efectiva. + if freq_req in _FREQS: + eff_freq = freq_req + else: + dates_sorted = sorted(d for d, _, _ in parsed) + eff_freq = _infer_freq(dates_sorted) + + # Agrupa por bucket. + buckets: dict = {} + for d, idx, num in parsed: + b = _bucket_start(d, eff_freq) + slot = buckets.get(b) + if slot is None: + slot = {"count": 0, "vals": [], "last_key": None, "last_val": None} + buckets[b] = slot + slot["count"] += 1 + if num is not None: + slot["vals"].append(num) + key = (d, idx) + if slot["last_key"] is None or key > slot["last_key"]: + slot["last_key"] = key + slot["last_val"] = num + + ordered = sorted(buckets.items(), key=lambda kv: kv[0]) + n_buckets = len(ordered) + + def _aggregate(vals: list, last_val) -> "float | None": + if not vals: + return None + if agg == "sum": + return float(sum(vals)) + if agg == "median": + return float(statistics.median(vals)) + if agg == "last": + return float(last_val) if last_val is not None else None + if agg == "min": + return float(min(vals)) + if agg == "max": + return float(max(vals)) + return float(statistics.fmean(vals)) # mean (default) + + t_out = [b.isoformat() for b, _ in ordered] + v_out = [_aggregate(s["vals"], s["last_val"]) for _, s in ordered] + c_out = [s["count"] for _, s in ordered] + + downsampled = False + if n_buckets > max_points > 0: + keep = _downsample_indices(n_buckets, max_points) + t_out = [t_out[i] for i in keep] + v_out = [v_out[i] for i in keep] + c_out = [c_out[i] for i in keep] + downsampled = True + + return { + "t": t_out, + "v": v_out, + "count": c_out, + "freq": eff_freq, + "agg": agg, + "n_in": n_in, + "n_buckets": n_buckets, + "downsampled": downsampled, + "note": "", + } diff --git a/python/functions/datascience/resample_timeseries_test.py b/python/functions/datascience/resample_timeseries_test.py new file mode 100644 index 00000000..58472865 --- /dev/null +++ b/python/functions/datascience/resample_timeseries_test.py @@ -0,0 +1,118 @@ +"""Tests para resample_timeseries (grupo eda).""" + +import datetime +import os +import sys + +sys.path.insert(0, os.path.dirname(__file__)) + +from resample_timeseries import resample_timeseries + + +def test_daily_a_mensual_mean(): + # Serie diaria agregada a buckets mensuales con agg="mean". + t = [ + "2020-01-01", "2020-01-15", + "2020-02-01", "2020-02-10", "2020-02-20", + ] + v = [10.0, 20.0, 30.0, 40.0, 50.0] + r = resample_timeseries(t, v, freq="monthly", agg="mean") + + assert r["t"] == ["2020-01-01", "2020-02-01"] + assert r["v"] == [15.0, 40.0] # (10+20)/2 ; (30+40+50)/3 + assert r["count"] == [2, 3] + assert r["freq"] == "monthly" + assert r["agg"] == "mean" + assert r["n_in"] == 5 + assert r["n_buckets"] == 2 + assert r["downsampled"] is False + assert r["note"] == "" + + +def test_agg_sum_y_last(): + t = [ + "2020-01-01", "2020-01-15", + "2020-02-01", "2020-02-10", "2020-02-20", + ] + v = [10.0, 20.0, 30.0, 40.0, 50.0] + + r_sum = resample_timeseries(t, v, freq="monthly", agg="sum") + assert r_sum["v"] == [30.0, 120.0] + assert r_sum["agg"] == "sum" + + # last = valor de la observacion cronologicamente mas reciente del bucket, + # aunque el orden de entrada este desordenado. + t2 = ["2020-02-20", "2020-02-01", "2020-02-10", "2020-01-15", "2020-01-01"] + v2 = [50.0, 30.0, 40.0, 20.0, 10.0] + r_last = resample_timeseries(t2, v2, freq="monthly", agg="last") + assert r_last["t"] == ["2020-01-01", "2020-02-01"] + assert r_last["v"] == [20.0, 50.0] # Jan->2020-01-15=20 ; Feb->2020-02-20=50 + assert r_last["agg"] == "last" + + +def test_count_cuenta_observacion_con_valor_none(): + # Un bucket con un valor None: count cuenta la fila, v ignora el None. + t = ["2020-03-05", "2020-03-06", "2020-03-20"] + v = [None, 7.0, 9.0] + r = resample_timeseries(t, v, freq="monthly", agg="mean") + + assert r["t"] == ["2020-03-01"] + assert r["count"] == [3] # 3 filas con fecha valida + assert r["v"] == [8.0] # media de los validos: (7+9)/2 + assert r["n_in"] == 3 + + # Bucket entero sin ningun valor numerico valido -> v = None, count sigue. + r2 = resample_timeseries( + ["2020-04-01", "2020-04-02"], [None, "n/a"], freq="monthly" + ) + assert r2["t"] == ["2020-04-01"] + assert r2["count"] == [2] + assert r2["v"] == [None] + + +def test_downsampling_respeta_max_points_y_extremos(): + base = datetime.date(2021, 1, 1) + t = [(base + datetime.timedelta(days=i)).isoformat() for i in range(500)] + v = [float(i) for i in range(500)] + r = resample_timeseries(t, v, freq="daily", agg="mean", max_points=400) + + assert r["n_buckets"] == 500 + assert r["downsampled"] is True + assert len(r["t"]) <= 400 + assert len(r["t"]) == len(r["v"]) == len(r["count"]) + # Primero y ultimo bucket conservados. + assert r["t"][0] == "2021-01-01" + assert r["t"][-1] == (base + datetime.timedelta(days=499)).isoformat() + + +def test_freq_auto_infiere_mensual(): + # Fechas separadas ~1 mes -> auto infiere "monthly". + t = [f"2022-{m:02d}-01" for m in range(1, 13)] + v = [float(m) for m in range(1, 13)] + r = resample_timeseries(t, v, freq="auto", agg="mean") + + assert r["freq"] == "monthly" + assert r["n_buckets"] == 12 + assert r["count"] == [1] * 12 + + # Fechas diarias consecutivas -> auto infiere "daily". + base = datetime.date(2023, 1, 1) + td = [(base + datetime.timedelta(days=i)).isoformat() for i in range(20)] + rd = resample_timeseries(td, [float(i) for i in range(20)], freq="auto") + assert rd["freq"] == "daily" + + +def test_edge_listas_vacias_o_desiguales(): + vacio = resample_timeseries([], []) + assert vacio["t"] == [] and vacio["v"] == [] and vacio["count"] == [] + assert vacio["note"] == "datos insuficientes" + assert vacio["n_in"] == 0 and vacio["n_buckets"] == 0 + + desigual = resample_timeseries(["2020-01-01", "2020-01-02"], [1.0]) + assert desigual["note"] == "datos insuficientes" + assert desigual["t"] == [] + + # Todas las fechas invalidas -> tambien insuficiente. + invalidas = resample_timeseries(["no-fecha", "tampoco"], [1.0, 2.0]) + assert invalidas["note"] == "datos insuficientes" + assert invalidas["n_in"] == 0 From f5b30b23dc5af0c96caafc4f3dde1e75d6efc8b1 Mon Sep 17 00:00:00 2001 From: Egutierrez Date: Tue, 30 Jun 2026 16:08:16 +0200 Subject: [PATCH 18/53] feat(eda): negrita inline real (**bold**) en renderers AutomaticEDA MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit El render de Markdown del motor AutomaticEDA quitaba los marcadores **negrita** sin aplicar estilo. Ahora los spans **bold**/__bold__ se renderizan en negrita real, de forma aditiva y sin romper el anti-corte: - text_layout.py: parse_inline_bold() tokeniza spans preservando el texto visible (== strip_inline_md) y wrap_rich() envuelve por palabras a max_chars conservando el flag de negrita por segmento (la anchura visible no cambia, así que la paginación es idéntica). - render_pdf_impl.py: _place_rich_lines() dibuja cada segmento con su fontweight avanzando x por el mismo grid de caracteres que usa el wrap (párrafos+bullets). - render_pptx_impl.py: _add_rich_text() usa runs nativos de python-pptx con font.bold por segmento (negrita real de PowerPoint). - bold_render_test.py: helpers puros (no-overflow, bold preservado, marcadores desbalanceados) + e2e que abre el .pptx y confirma un run con font.bold True. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../automatic_eda/bold_render_test.py | 113 ++++++++++++++ .../automatic_eda/render_pdf_impl.py | 50 +++++-- .../automatic_eda/render_pptx_impl.py | 51 ++++++- .../datascience/automatic_eda/text_layout.py | 138 ++++++++++++++++++ 4 files changed, 334 insertions(+), 18 deletions(-) create mode 100644 python/functions/datascience/automatic_eda/bold_render_test.py diff --git a/python/functions/datascience/automatic_eda/bold_render_test.py b/python/functions/datascience/automatic_eda/bold_render_test.py new file mode 100644 index 00000000..dcd98628 --- /dev/null +++ b/python/functions/datascience/automatic_eda/bold_render_test.py @@ -0,0 +1,113 @@ +"""Tests for inline-bold rendering (**bold**) in the AutomaticEDA engine. + +Covers the pure helpers (parse_inline_bold / wrap_rich) and an end-to-end PPTX +check that a ``**bold**`` span is rendered with NATIVE PowerPoint bold +(``run.font.bold is True``) while no line overflows the wrap width (no-cut). +""" + +import os +import sys + +import pytest + +# Make the engine importable as a package (datascience.automatic_eda). +_HERE = os.path.dirname(os.path.abspath(__file__)) +_FUNCTIONS = os.path.abspath(os.path.join(_HERE, "..", "..", "..")) # python/functions +if _FUNCTIONS not in sys.path: + sys.path.insert(0, _FUNCTIONS) + +from datascience.automatic_eda import model # noqa: E402 +from datascience.automatic_eda import text_layout as tl # noqa: E402 +from datascience.automatic_eda import render_pptx # noqa: E402 + + +# --------------------------------------------------------------------------- # +# Pure helpers. +# --------------------------------------------------------------------------- # +def test_parse_inline_bold_marks_spans_and_preserves_visible_text(): + src = "**Estacionariedad:** serie no estacionaria con `code` y normal." + segs = tl.parse_inline_bold(src) + # Visible text equals strip_inline_md (no characters lost, markers removed). + visible = "".join(s for s, _ in segs) + assert visible == tl.strip_inline_md(src) + # The span "Estacionariedad:" is flagged bold; the rest is not. + bold_text = "".join(s for s, b in segs if b) + assert "Estacionariedad:" in bold_text + assert "serie no estacionaria" not in bold_text + + +def test_parse_inline_bold_handles_unbalanced_markers(): + # An unbalanced ** must not crash and must be stripped (matches strip_inline_md). + segs = tl.parse_inline_bold("texto **sin cierre aqui") + visible = "".join(s for s, _ in segs) + assert visible == "texto sin cierre aqui" + assert not any(b for _, b in segs) # nothing rendered bold. + + +def test_wrap_rich_never_overflows_and_keeps_bold(): + text = ("**Segmento premium.** Clientes de alto gasto y baja frecuencia con " + "ticket medio elevado y recurrencia anual estable a lo largo del año.") + max_chars = 30 + lines = tl.wrap_rich(text, max_chars) + # No visible line exceeds max_chars (no-cut: the renderer measures these). + for ln in lines: + visible = "".join(s for s, _ in ln) + assert len(visible) <= max_chars, f"línea desborda: {visible!r}" + # At least one segment is bold and it is the span content. + bold_segs = [s for ln in lines for s, b in ln if b] + assert any("Segmento premium." in s for s in bold_segs) + + +def test_wrap_rich_hard_splits_long_token(): + long = "x" * 50 + lines = tl.wrap_rich(f"**{long}**", 20) + for ln in lines: + assert len("".join(s for s, _ in ln)) <= 20 + # The whole long token is preserved across the split lines. + joined = "".join(s for ln in lines for s, _ in ln) + assert joined == long + + +# --------------------------------------------------------------------------- # +# End-to-end: PPTX renders **bold** as a real bold run. +# --------------------------------------------------------------------------- # +def _has_pptx(): + try: + import pptx # noqa: F401 + return True + except Exception: # noqa: BLE001 + return False + + +@pytest.mark.skipif(not _has_pptx(), reason="python-pptx no instalado") +def test_pptx_renders_bold_span_as_native_bold_run(tmp_path): + from pptx import Presentation + + doc = [model.Chapter( + id="t", title="Negrita", version="1.0.0", + blocks=[model.Markdown( + text="Frase con **PALABRACLAVE** resaltada y texto normal después.")], + )] + out = str(tmp_path / "bold.pptx") + res = render_pptx(doc, out, {"title": "T"}) + assert res.get("path") == out + assert os.path.exists(out) + + prs = Presentation(out) + bold_texts = [] + all_text = [] + for slide in prs.slides: + for shape in slide.shapes: + if not shape.has_text_frame: + continue + for para in shape.text_frame.paragraphs: + for run in para.runs: + all_text.append(run.text) + if run.font.bold: + bold_texts.append(run.text) + # The bold span text appears in a run with font.bold True (native bold). + assert any("PALABRACLAVE" in t for t in bold_texts), \ + f"no se encontró run bold con el span; bold={bold_texts}" + # And the surrounding plain text is NOT bold (markers did not bleed). + assert any("resaltada" in t for t in all_text) + assert not any("resaltada" in t for t in bold_texts) diff --git a/python/functions/datascience/automatic_eda/render_pdf_impl.py b/python/functions/datascience/automatic_eda/render_pdf_impl.py index b7961b0c..fe8702ce 100644 --- a/python/functions/datascience/automatic_eda/render_pdf_impl.py +++ b/python/functions/datascience/automatic_eda/render_pdf_impl.py @@ -169,6 +169,38 @@ def _place_text_lines(st: _PdfState, lines: list, fs: float, color: str, st.y += lh +def _place_rich_lines(st: _PdfState, rich_lines: list, fs: float, color: str, + indent: float = 0.0, prefixes=None) -> None: + """Draw pre-wrapped lines of styled segments (bold spans rendered bold). + + Each line is ``[(text, is_bold), ...]``. Segments are placed left-to-right, + advancing x by the deterministic character grid (same metric the wrapper + used), so a bold span is rendered with ``fontweight='bold'`` without + changing the line's measured width — the no-cut guarantee is preserved. + ``prefixes`` is an optional ``(first_line, other_lines)`` pair (e.g. a + bullet) drawn before the segments. + """ + lh = tl.line_height_in(fs) + cw = tl.avg_char_width_in(fs) + for idx, segs in enumerate(rich_lines): + _ensure_space(st, lh) + x = _ML + indent + if prefixes is not None: + prefix = prefixes[0] if idx == 0 else prefixes[1] + if prefix: + st.fig.text(_xf(x), _yf(st.y), prefix, fontsize=fs, color=color, + ha="left", va="top") + x += cw * len(prefix) + for seg_text, is_bold in segs: + if seg_text == "": + continue + st.fig.text(_xf(x), _yf(st.y), seg_text, fontsize=fs, color=color, + ha="left", va="top", + fontweight="bold" if is_bold else "normal") + x += cw * len(seg_text) + st.y += lh + + def _place_markdown(st: _PdfState, block) -> None: raw = getattr(block, "text", "") or "" md_lines = str(raw).split("\n") @@ -208,29 +240,25 @@ def _place_markdown(st: _PdfState, block) -> None: i += 1 continue if stripped.startswith("- ") or stripped.startswith("* "): - content = tl.strip_inline_md(stripped[2:]) + content = stripped[2:] # keep inline markers for bold rendering. bullet_chars = tl.chars_per_line(_USABLE_W - 0.22, _FS_BODY) - wrapped = tl.wrap(content, bullet_chars) - first = True - for w in wrapped: - prefix = "• " if first else " " - _place_text_lines(st, [prefix + w], _FS_BODY, _INK, - indent=0.0) - first = False + rich = tl.wrap_rich(content, bullet_chars) + _place_rich_lines(st, rich, _FS_BODY, _INK, + prefixes=("• ", " ")) i += 1 continue # Plain paragraph (gather following plain lines into one paragraph). - para = [tl.strip_inline_md(stripped)] + para = [stripped] # keep inline markers; wrap_rich renders **bold**. j = i + 1 while j < n: nxt = md_lines[j].strip() if nxt == "" or nxt.startswith(("|", "#", "- ", "* ")): break - para.append(tl.strip_inline_md(nxt)) + para.append(nxt) j += 1 text = " ".join(para) max_chars = tl.chars_per_line(_USABLE_W, _FS_BODY) - _place_text_lines(st, tl.wrap(text, max_chars), _FS_BODY, _INK) + _place_rich_lines(st, tl.wrap_rich(text, max_chars), _FS_BODY, _INK) i = j st.y += _GAP diff --git a/python/functions/datascience/automatic_eda/render_pptx_impl.py b/python/functions/datascience/automatic_eda/render_pptx_impl.py index 5494d604..db7d201a 100644 --- a/python/functions/datascience/automatic_eda/render_pptx_impl.py +++ b/python/functions/datascience/automatic_eda/render_pptx_impl.py @@ -151,6 +151,42 @@ def _add_text(st: _PptxState, lines: list, fs: float, color, bold=False, st.y += height +def _add_rich_text(st: _PptxState, rich_lines: list, fs: float, color, + indent=0.0, bullet=False) -> None: + """Add pre-wrapped lines of styled segments as one paragraph per line. + + Each line is ``[(text, is_bold), ...]``; every segment becomes its own run + so ``**bold**`` spans render with native PowerPoint bold (``run.font.bold``) + without affecting the measured height (one paragraph per pre-wrapped line). + """ + lh = tl.line_height_in(fs) + height = lh * len(rich_lines) + 0.05 + _ensure(st, height) + box = st.slide.shapes.add_textbox( + Inches(_ML + indent), Inches(st.y), Inches(_USABLE_W - indent), + Inches(height)) + tf = box.text_frame + tf.word_wrap = True + first = True + for segs in rich_lines: + p = tf.paragraphs[0] if first else tf.add_paragraph() + first = False + if bullet: + r0 = p.add_run() + r0.text = "• " + r0.font.size = Pt(fs) + r0.font.color.rgb = _rgb(color) + for seg_text, is_bold in segs: + if seg_text == "": + continue + run = p.add_run() + run.text = seg_text + run.font.size = Pt(fs) + run.font.bold = bool(is_bold) + run.font.color.rgb = _rgb(color) + st.y += height + + def _place_heading(st: _PptxState, block) -> None: level = max(1, min(3, int(getattr(block, "level", 1) or 1))) fs = {1: _FS_H1, 2: _FS_H2, 3: _FS_H3}[level] @@ -196,22 +232,23 @@ def _place_markdown(st: _PptxState, block) -> None: i += 1 continue if stripped.startswith("- ") or stripped.startswith("* "): - content = tl.strip_inline_md(stripped[2:]) - lines = tl.wrap(content, tl.chars_per_line(_USABLE_W - 0.3, _FS_BODY)) - _add_text(st, lines, _FS_BODY, _INK, bullet=True) + content = stripped[2:] # keep inline markers for bold rendering. + rich = tl.wrap_rich(content, + tl.chars_per_line(_USABLE_W - 0.3, _FS_BODY)) + _add_rich_text(st, rich, _FS_BODY, _INK, bullet=True) i += 1 continue - para = [tl.strip_inline_md(stripped)] + para = [stripped] # keep inline markers; wrap_rich renders **bold**. j = i + 1 while j < n: nxt = md_lines[j].strip() if nxt == "" or nxt.startswith(("|", "#", "- ", "* ")): break - para.append(tl.strip_inline_md(nxt)) + para.append(nxt) j += 1 text = " ".join(para) - _add_text(st, tl.wrap(text, tl.chars_per_line(_USABLE_W, _FS_BODY)), - _FS_BODY, _INK) + _add_rich_text(st, tl.wrap_rich(text, tl.chars_per_line(_USABLE_W, _FS_BODY)), + _FS_BODY, _INK) i = j st.y += _GAP diff --git a/python/functions/datascience/automatic_eda/text_layout.py b/python/functions/datascience/automatic_eda/text_layout.py index dae00904..0d07d140 100644 --- a/python/functions/datascience/automatic_eda/text_layout.py +++ b/python/functions/datascience/automatic_eda/text_layout.py @@ -15,8 +15,15 @@ overflowing — that is wrapping, not loss: every character is still rendered. from __future__ import annotations +import re import textwrap +# Inline span markers: ``**bold**`` / ``__bold__`` (rendered bold) and +# `` `code` `` (markers removed, not styled). Matched non-greedily so the +# shortest balanced pair wins. Unbalanced leftovers are stripped afterwards so +# the visible text matches ``strip_inline_md`` exactly. +_INLINE_SPAN_RE = re.compile(r"(\*\*.+?\*\*|__.+?__|`.+?`)") + def avg_char_width_in(fontsize_pt: float) -> float: """Approximate average glyph width in inches for a sans-serif font. @@ -84,6 +91,137 @@ def strip_inline_md(text: str) -> str: return s +def _strip_leftover_markers(s: str) -> str: + """Drop any unbalanced inline markers from a plain (non-span) fragment. + + Keeps the visible text identical to :func:`strip_inline_md` even when a + ``**`` / ``__`` / `` ` `` has no matching closing marker. + """ + for marker in ("**", "__", "`"): + s = s.replace(marker, "") + return s + + +def parse_inline_bold(text: str): + """Split ``text`` into ``[(fragment, is_bold), ...]`` preserving order. + + ``**...**`` and ``__...__`` spans become bold fragments (markers removed); + `` `code` `` keeps its text without the backticks and is not bold; any other + text is emitted verbatim with unbalanced markers stripped. The concatenation + of all fragment texts equals :func:`strip_inline_md` of the input — so the + *visible* characters (and therefore line wrapping) are unchanged; only the + bold flag is added. Adjacent fragments of the same weight are merged. + """ + s = "" if text is None else str(text) + if not s: + return [] + out = [] + + def _emit(fragment: str, bold: bool) -> None: + if fragment == "": + return + if out and out[-1][1] == bold: + out[-1] = (out[-1][0] + fragment, bold) + else: + out.append((fragment, bold)) + + pos = 0 + for m in _INLINE_SPAN_RE.finditer(s): + if m.start() > pos: + _emit(_strip_leftover_markers(s[pos:m.start()]), False) + tok = m.group(0) + if tok.startswith("**") and tok.endswith("**"): + _emit(tok[2:-2], True) + elif tok.startswith("__") and tok.endswith("__"): + _emit(tok[2:-2], True) + else: # `code` + _emit(tok[1:-1], False) + pos = m.end() + if pos < len(s): + _emit(_strip_leftover_markers(s[pos:]), False) + return out + + +def _hard_split(word: str, max_chars: int): + """Split a single long token into <= max_chars chunks (never loses chars).""" + return [word[i:i + max_chars] for i in range(0, len(word), max_chars)] or [""] + + +def wrap_rich(text: str, max_chars: int): + """Word-wrap ``text`` to ``max_chars`` while preserving inline bold spans. + + Returns ``list[list[(fragment, is_bold)]]`` — one inner list of styled + fragments per output line; concatenating an inner list's fragment texts is + the visible line. Wrapping is word-aware and hard-splits over-long tokens, so + no line exceeds ``max_chars`` (the renderers measure these very lines, so the + no-cut guarantee holds). Bold spans never widen a line: only the bold flag is + carried, the visible width is identical to :func:`wrap`. + """ + if max_chars < 1: + max_chars = 1 + spans = parse_inline_bold(text) + if not spans: + return [[("", False)]] + + # Flatten to (word, is_bold) tokens, honoring hard newlines as line breaks. + # A token list of None marks a forced line break. + tokens = [] # each: (word, bold) or ("\n", None) + for frag, bold in spans: + parts = frag.split("\n") + for pi, part in enumerate(parts): + if pi > 0: + tokens.append(("\n", None)) + for word in part.split(" "): + if word == "": + continue + tokens.append((word, bold)) + + lines = [] # list[list[(seg, bold)]] + cur = [] # list[(word, bold)] + cur_len = 0 + + def _flush(): + nonlocal cur, cur_len + # Merge adjacent same-weight words (with separating spaces) into segments. + merged = [] + for k, (word, bold) in enumerate(cur): + piece = word if k == 0 else " " + word + if merged and merged[-1][1] == bold: + merged[-1] = (merged[-1][0] + piece, bold) + else: + merged.append((piece, bold)) + lines.append(merged or [("", False)]) + cur = [] + cur_len = 0 + + for word, bold in tokens: + if bold is None: # forced newline + _flush() + continue + if len(word) > max_chars: + if cur: + _flush() + chunks = _hard_split(word, max_chars) + for ci, chunk in enumerate(chunks): + if ci < len(chunks) - 1: + lines.append([(chunk, bold)]) + else: + cur = [(chunk, bold)] + cur_len = len(chunk) + continue + add = len(word) if cur_len == 0 else cur_len + 1 + len(word) + if cur_len != 0 and add > max_chars: + _flush() + cur = [(word, bold)] + cur_len = len(word) + else: + cur.append((word, bold)) + cur_len = add + if cur: + _flush() + return lines or [[("", False)]] + + def parse_md_table(lines: list): """Parse consecutive ``| a | b |`` lines into ``(header, rows)`` or None. From f3d427d9e406db3c72d32fac6a78a42b89f9ad16 Mon Sep 17 00:00:00 2001 From: Egutierrez Date: Tue, 30 Jun 2026 16:08:41 +0200 Subject: [PATCH 19/53] =?UTF-8?q?feat(eda):=20wiring=20AutomaticEDA=20?= =?UTF-8?q?=E2=80=94=20build=5Feda=5Frender=5Fctx=20+=20pipeline=20render?= =?UTF-8?q?=5Fautomatic=5Feda=20+=20profile=5Ftable(emit=5Fautomatic)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Conecta el motor AutomaticEDA con los datos crudos para que los 4 capítulos dependientes de ctx (modelos, timeseries, geospatial, agregacion) salgan POBLADOS en vez de degradar a una nota. - build_eda_render_ctx (datascience, impure, dict-no-throw): dado db_path+table y el TableProfile agregado, construye el ctx con los datos crudos que el perfil no incluye: raw_numeric {col:[float|None]} alineado por fila (modelos / geospatial), timeseries_raw {time_col,t,series} vía extract_timeseries_raw, geo_points {lats,lons} desde el par lat/lon detectado, y db_path/table para el groupby/pivot push-down de agregacion. Muestrea con LIMIT (no trae la tabla entera a RAM). Compone detect_time_column / extract_timeseries_raw / detect_latlon_columns / duckdb_query_readonly (imports lazy para evitar ciclo). - render_automatic_eda (pipeline): one-shot perfil -> ctx -> PDF + PPTX con los 11 capítulos poblados; devuelve rutas + manifest de versiones por capítulo. - profile_table: flag aditivo emit_automatic=True emite el AutomaticEDA PDF+PPTX además del flujo legacy (emit_pdf/render_eda_pdf intacto). Nuevas claves de retorno aeda_pdf_path / aeda_pptx_path / aeda_manifest_path. Co-Authored-By: Claude Opus 4.8 (1M context) --- python/functions/datascience/__init__.py | 2 + .../datascience/build_eda_render_ctx.md | 114 ++++++++++ .../datascience/build_eda_render_ctx.py | 200 ++++++++++++++++++ .../datascience/build_eda_render_ctx_test.py | 153 ++++++++++++++ python/functions/pipelines/profile_table.md | 9 +- python/functions/pipelines/profile_table.py | 52 +++++ .../pipelines/render_automatic_eda.md | 91 ++++++++ .../pipelines/render_automatic_eda.py | 157 ++++++++++++++ .../pipelines/render_automatic_eda_test.py | 91 ++++++++ 9 files changed, 867 insertions(+), 2 deletions(-) create mode 100644 python/functions/datascience/build_eda_render_ctx.md create mode 100644 python/functions/datascience/build_eda_render_ctx.py create mode 100644 python/functions/datascience/build_eda_render_ctx_test.py create mode 100644 python/functions/pipelines/render_automatic_eda.md create mode 100644 python/functions/pipelines/render_automatic_eda.py create mode 100644 python/functions/pipelines/render_automatic_eda_test.py diff --git a/python/functions/datascience/__init__.py b/python/functions/datascience/__init__.py index bdf596e7..a1e6331f 100644 --- a/python/functions/datascience/__init__.py +++ b/python/functions/datascience/__init__.py @@ -65,12 +65,14 @@ from .render_automatic_eda_pdf import render_automatic_eda_pdf from .render_automatic_eda_pptx import render_automatic_eda_pptx from .detect_time_column import detect_time_column from .extract_timeseries_raw import extract_timeseries_raw +from .build_eda_render_ctx import build_eda_render_ctx from .profile_datetime import profile_datetime from .resample_timeseries import resample_timeseries __all__ = [ "detect_time_column", "extract_timeseries_raw", + "build_eda_render_ctx", "profile_datetime", "resample_timeseries", "render_automatic_eda_pdf", diff --git a/python/functions/datascience/build_eda_render_ctx.md b/python/functions/datascience/build_eda_render_ctx.md new file mode 100644 index 00000000..4f098383 --- /dev/null +++ b/python/functions/datascience/build_eda_render_ctx.md @@ -0,0 +1,114 @@ +--- +name: build_eda_render_ctx +kind: function +lang: py +domain: datascience +version: "1.0.0" +purity: impure +signature: "def build_eda_render_ctx(db_path: str, table: str, profile: dict, backend: str = 'duckdb', sample: int = 5000, base_ctx: dict = None) -> dict" +description: "Constructor del `ctx` de datos crudos del motor AutomaticEDA. Dado un db_path+table (DuckDB o Postgres) y el TableProfile AGREGADO ya calculado por profile_table, produce el dict ctx que los renderers (render_automatic_eda_pdf/_pptx -> build_document(profile, ctx)) pasan a los capitulos que necesitan DATOS CRUDOS no presentes en el perfil agregado: modelos (project_clusters_2d en vivo), timeseries, geospatial y agregacion (groupby/pivot push-down). NO trae tablas enteras a RAM: muestrea con LIMIT sample y delega el push-down de la serie en extract_timeseries_raw. Construye el lector read-only query_fn(sql)->dict igual que profile_table (closure sobre duckdb_query_readonly / pg_query). Estilo dict-no-throw del grupo eda: NUNCA lanza; si una pieza falla, degrada esa clave a ausente/[] y sigue. Devuelve el ctx dict directamente (NO un wrapper {status,...}); se pasa tal cual como meta={'ctx': }. Claves de datos que produce: raw_numeric (muestra cruda alineada por fila), timeseries_raw (fechas+series), geo_points (lats/lons) y db_path+table para el push-down de agregacion. Respeta base_ctx: parte de una copia y solo AÑADE las claves de datos; las de presentacion (dataset_name, source_origin, ...) no se pisan." +tags: [eda, datascience, automatic-eda, render, ctx, extraction, read-only, duckdb, postgres, python] +uses_functions: [detect_time_column_py_datascience, extract_timeseries_raw_py_datascience, detect_latlon_columns_py_datascience, duckdb_query_readonly_py_infra, pg_query_py_infra] +uses_types: [] +returns: [] +returns_optional: false +error_type: "error_go_core" +imports: [] +params: + - name: db_path + desc: "ruta al archivo DuckDB, o DSN PostgreSQL si backend='postgres'. Se guarda tal cual en ctx['db_path'] (el capitulo agregacion lo usa para el groupby/pivot push-down via DuckDB) y se inyecta en el closure query_fn. No se valida aqui: si la base no existe, las queries devuelven status error y las claves de datos se omiten." + - name: table + desc: "nombre de la tabla. Se escapa con comillas dobles en las queries (raw_numeric y timeseries) y se guarda en ctx['table']." + - name: profile + desc: "TableProfile AGREGADO producido por profile_table. Solo se lee su clave `columns` (lista de ColumnProfile dict con name / inferred_type / numeric.{min,max} / semantic_type). Lectura defensiva: si no es dict o no tiene columns, se trata como []. NO se traen las filas crudas de aqui — se muestrean de la base." + - name: backend + desc: "'duckdb' (default) o 'postgres'. Selecciona el lector read-only del registry (duckdb_query_readonly / pg_query). Cualquier otro valor devuelve el base_ctx tal cual, SIN añadir claves de datos (ni siquiera db_path/table)." + - name: sample + desc: "maximo de filas a muestrear (clausula LIMIT) tanto para raw_numeric (una sola query SELECT de las numericas) como para timeseries_raw (max_rows de extract_timeseries_raw). Default 5000. Acota memoria y tiempo de render." + - name: base_ctx + desc: "dict opcional con claves de PRESENTACION ya preparadas (dataset_name, source_origin, ...). Se parte de una copia y NO se pisan sus claves; solo se añaden las de datos. Default None -> {}." +output: "El dict `ctx` directamente (NO un wrapper {status,...}); se pasa tal cual como meta={'ctx': } a render_automatic_eda_pdf/pptx. Nunca lanza. Para backends validos contiene SIEMPRE db_path + table, y opcionalmente: raw_numeric {col:[float|None,...]} (muestra cruda alineada por fila; omitida si no hay numericas o falla la query), timeseries_raw {time_col, t:[iso...], series:{col:[float|None,...]}} (solo si hay columna temporal + numericas y trae filas), geo_points {lats:[...], lons:[...]} (solo si se detecta par lat/lon y ambas estan en raw_numeric). Ante fallo global devuelve al menos {**base_ctx, 'db_path': db_path, 'table': table}. Backend desconocido -> base_ctx tal cual sin claves de datos." +tested: true +tests: ["test_db_path_y_table_en_ctx", "test_raw_numeric_con_columnas_numericas", "test_timeseries_raw_con_fecha", "test_geo_points_con_latlon", "test_sin_fecha_no_hay_timeseries", "test_base_ctx_preservado"] +test_file_path: "python/functions/datascience/build_eda_render_ctx_test.py" +file_path: "python/functions/datascience/build_eda_render_ctx.py" +--- + +## Ejemplo + +```python +import sys, os +sys.path.insert(0, os.path.join("python", "functions")) +from datascience import build_eda_render_ctx, render_automatic_eda_pdf +from datascience import profile_table # opcional: para obtener el TableProfile + +# 1) Perfil agregado de la tabla (push-down, sin RAM). +prof = profile_table("data/ventas.duckdb", "ventas_geo", write_report=False)["profile"] + +# 2) ctx de datos crudos para los capitulos (muestrea con LIMIT, no carga todo). +ctx = build_eda_render_ctx( + "data/ventas.duckdb", "ventas_geo", prof, + backend="duckdb", sample=5000, + base_ctx={"dataset_name": "Ventas con geolocalizacion"}, +) +# ctx == { +# "dataset_name": "Ventas con geolocalizacion", # preservado del base_ctx +# "db_path": "data/ventas.duckdb", "table": "ventas_geo", +# "raw_numeric": {"ventas": [1200.5, ...], "lat": [40.41, ...], "lon": [-3.70, ...]}, +# "timeseries_raw": {"time_col": "fecha", "t": ["2024-01-01", ...], "series": {...}}, +# "geo_points": {"lats": [40.41, ...], "lons": [-3.70, ...]}, +# } + +# 3) Se entrega tal cual a los renderers via meta={"ctx": ctx}. +render_automatic_eda_pdf(prof, "reports/eda.pdf", meta={"ctx": ctx}) +``` + +## Cuando usarla + +Justo antes de renderizar un AutomaticEDA (PDF o PPTX), cuando ya tienes el +TableProfile AGREGADO de `profile_table` pero los capitulos de modelos, +timeseries, geospatial y agregacion necesitan DATOS CRUDOS que el perfil +agregado no lleva (la muestra numerica alineada por fila, la serie cronologica, +el par lat/lon, y el db_path/table para el push-down del groupby/pivot). Es el +puente entre el perfil agregado y `build_document(profile, ctx)`: una sola +llamada produce el `ctx` completo muestreando con `LIMIT` en vez de cargar la +tabla entera en memoria. + +## Gotchas + +- **Impura**: lee de la base de datos a traves de `query_fn` (closure sobre + `duckdb_query_readonly` / `pg_query`). No abre conexiones fuera de esos + wrappers del registry. Estilo dict-no-throw del grupo `eda`: NUNCA lanza; ante + cualquier fallo (query, deteccion, render de una clave) degrada esa clave a + ausente/`[]` y sigue. Ante un fallo global devuelve al menos + `{**base_ctx, "db_path": db_path, "table": table}`. +- **`error_type` en el frontmatter es `error_go_core` por convencion del + registry** (toda funcion impura debe declararlo y el indexer lo exige), pero el + codigo NO lanza esa excepcion: degrada al ctx parcial. Es metadata, no + comportamiento. +- **Devuelve el ctx dict directamente, NO un wrapper `{status,...}`**: a + diferencia de `extract_timeseries_raw` / `profile_table`, esta funcion es el + ultimo eslabon antes del render y su salida se pasa tal cual como + `meta={"ctx": }`. No envuelvas su retorno. +- **Backend desconocido**: con un `backend` que no sea `duckdb` ni `postgres` + devuelve el `base_ctx` tal cual, SIN claves de datos (ni siquiera + `db_path`/`table`). Comprueba el backend antes si dependes de esas claves. +- **Alineacion por fila de `raw_numeric`**: `raw_numeric[col]` tiene una entrada + por fila muestreada (un valor no convertible a float queda como `None`, no se + descarta la fila) porque `project_clusters_2d` descarta filas listwise: todas + las columnas deben tener la MISMA longitud. `geo_points` se construye desde + `raw_numeric` para heredar esa alineacion. +- **`geo_points` exige lat/lon en `raw_numeric`**: el par lat/lon solo se adjunta + si ambas columnas se detectaron (nombre+rango) Y figuran en `raw_numeric` + (es decir, son numericas en el perfil). Si la tabla guarda lat/lon como texto + no promovido a numeric, no apareceran; el capitulo geospatial sabe degradar. +- **`timeseries_raw` depende del orden del backend**: hereda el `ORDER BY + "time_col"` de `extract_timeseries_raw`. Si la columna temporal esta guardada + como texto no ordenable lexicograficamente (p.ej. `DD/MM/YYYY`), el orden no + sera el cronologico real — normaliza la columna a date/timestamp antes. +- **`LIMIT sample`**: con tablas grandes obtienes el primer tramo (raw_numeric + por orden fisico, timeseries por orden cronologico), no un muestreo uniforme. + Sube `sample` si necesitas mas cobertura. +- **No loguear los datos crudos**: `raw_numeric` / `timeseries_raw` / + `geo_points` pueden contener datos sensibles. En trazas usa solo conteos y + nombres de columna, no el ctx completo. diff --git a/python/functions/datascience/build_eda_render_ctx.py b/python/functions/datascience/build_eda_render_ctx.py new file mode 100644 index 00000000..efcda2cb --- /dev/null +++ b/python/functions/datascience/build_eda_render_ctx.py @@ -0,0 +1,200 @@ +"""build_eda_render_ctx — constructor del `ctx` de datos crudos del motor AutomaticEDA. + +Funcion impura (lee de la base de datos) del grupo de capacidad `eda`. Dado un +``db_path`` + ``table`` (DuckDB o PostgreSQL) y el ``TableProfile`` AGREGADO ya +calculado por ``profile_table``, produce el dict ``ctx`` que los renderers +(``render_automatic_eda_pdf`` / ``render_automatic_eda_pptx`` -> +``build_document(profile, ctx)``) pasan a los capitulos que necesitan DATOS +CRUDOS no presentes en el perfil agregado: modelos (``project_clusters_2d`` en +vivo), timeseries, geospatial y agregacion (groupby/pivot push-down). + +NO trae tablas enteras a RAM: muestrea con ``LIMIT sample`` y, para la serie +temporal, delega el push-down en ``extract_timeseries_raw`` (una sola query +ordenada). El lector read-only ``query_fn(sql) -> dict`` se construye igual que +en ``profile_table`` (un closure sobre ``duckdb_query_readonly`` / ``pg_query``) +y nunca abre conexiones fuera de esos wrappers. + +Estilo dict-no-throw del grupo `eda`: la funcion NUNCA lanza. Si una pieza falla +(query, deteccion, render de una clave), esa clave se degrada a ausente / lista +vacia y el resto del ctx se construye igual. Ante un fallo global devuelve al +menos ``{**base_ctx, "db_path": db_path, "table": table}``. + +Claves de DATOS que produce (las consumen los capitulos): + - ``raw_numeric`` : {col: [float|None, ...]} muestra cruda de las columnas + numericas, ALINEADA POR FILA (una entrada por fila aunque + sea None). La leen modelos (clustering 2D en vivo) y + geospatial (lat/lon salen de aqui). + - ``timeseries_raw`` : {time_col, t: [iso...], series: {col: [float|None, ...]}}. + La lee el capitulo TIMESERIES. + - ``geo_points`` : {lats: [...], lons: [...]} listas alineadas (ya floats). + La lee el capitulo GEOSPATIAL. + - ``db_path``, ``table`` : los usa el capitulo AGREGACION para el groupby/pivot + push-down via DuckDB. + +Las claves de PRESENTACION que traiga ``base_ctx`` (dataset_name, source_origin, +...) NO se pisan: esta funcion solo AÑADE las claves de datos sobre una copia. +""" + + +def _to_float(value): + """Convierte un valor a float de forma defensiva. None si no es convertible. + + Un bool es subclase de int en Python pero nunca es un valor numerico de + serie/coordenada valido, asi que se trata como None (mismo criterio que + extract_timeseries_raw / detect_latlon_columns). + """ + if value is None or isinstance(value, bool): + return None + if isinstance(value, (int, float)): + return float(value) + s = str(value).strip() + if not s: + return None + try: + return float(s) + except (TypeError, ValueError): + return None + + +def build_eda_render_ctx(db_path, table, profile, backend="duckdb", sample=5000, base_ctx=None): + """Construye el ctx de datos crudos para los renderers de AutomaticEDA. + + Args: + db_path: ruta al archivo DuckDB, o DSN PostgreSQL si backend="postgres". + Se guarda tal cual en ctx["db_path"] (el capitulo agregacion lo usa + para el push-down). + table: nombre de la tabla. Se escapa con comillas dobles en las queries y + se guarda en ctx["table"]. + profile: TableProfile agregado producido por profile_table. Solo se lee + su clave ``columns`` (lista de ColumnProfile dict con name / + inferred_type / numeric.{min,max} / semantic_type). Lectura + defensiva: si no es dict o no tiene columns, se trata como []. + backend: "duckdb" (default) o "postgres". Selecciona el lector read-only + (duckdb_query_readonly / pg_query). Cualquier otro valor devuelve el + base_ctx tal cual, sin añadir claves de datos. + sample: maximo de filas a muestrear (clausula LIMIT) tanto para + raw_numeric como para timeseries_raw. Default 5000. + base_ctx: dict opcional con claves de presentacion ya preparadas + (dataset_name, source_origin, ...). Se parte de una copia y NO se + pisan sus claves; solo se añaden las de datos. Default None -> {}. + + Returns: + El dict ``ctx`` directamente (NO un wrapper {status,...}): se pasa tal + cual como ``meta={"ctx": }`` a render_automatic_eda_pdf/pptx. + Nunca lanza. Claves que puede contener: raw_numeric, timeseries_raw, + geo_points (omitidas si no aplican o fallan), y siempre db_path + table + para backends validos. + """ + # Copia de base_ctx: nunca mutamos el dict del caller. Las claves de + # presentacion que ya traiga se conservan; las de datos se añaden encima. + ctx = dict(base_ctx) if isinstance(base_ctx, dict) else {} + + try: + # 1) Lector read-only del backend activo, construido EXACTAMENTE como en + # profile_table (closure sobre el wrapper del registry). Imports perezosos + # dentro de la funcion: este modulo vive en el paquete `datascience`, asi + # que importar sus hermanas a nivel de modulo crearia un ciclo al cargar + # el __init__ del paquete. Lazy import rompe el ciclo y respeta el + # contrato (imports explicitos, sin `import *`). + if backend == "duckdb": + from infra import duckdb_query_readonly + + def query_fn(sql): + return duckdb_query_readonly(db_path, sql) + + elif backend == "postgres": + from infra import pg_query + + def query_fn(sql): + return pg_query(db_path, sql) + + else: + # Backend desconocido: devolver base_ctx tal cual, sin claves de datos. + return ctx + + # 7) db_path + table SIEMPRE (para backends validos): el capitulo + # agregacion los necesita para el groupby/pivot push-down via DuckDB. + ctx["db_path"] = db_path + ctx["table"] = table + + # 2) Columnas del perfil agregado (lectura defensiva). + cols = profile.get("columns") if isinstance(profile, dict) else None + cols = cols or [] + + # 3) Deteccion temporal/numerica con la funcion PURA del registry. + from datascience import detect_time_column + + det = detect_time_column(cols) + time_col = det.get("time_col") + numeric_cols = det.get("numeric_cols") or [] + + # 4) raw_numeric: muestra de las columnas numericas CRUDAS, ALINEADAS POR + # FILA en UNA sola query. Cada columna queda con una entrada por fila + # (None si no parsea) para no desalinear filas: project_clusters_2d + # descarta filas listwise, asi que las listas deben tener igual longitud. + raw_numeric = {} + if numeric_cols: + try: + cols_sql = ", ".join(f'"{c}"' for c in numeric_cols) + sql = f'SELECT {cols_sql} FROM "{table}" LIMIT {int(sample)}' + q = query_fn(sql) + if isinstance(q, dict) and q.get("status") == "ok": + rows = q.get("rows", []) or [] + raw_numeric = {c: [] for c in numeric_cols} + for row in rows: + for c in numeric_cols: + raw_numeric[c].append(_to_float(row.get(c))) + except Exception: # noqa: BLE001 - dict-no-throw: degradar la clave + raw_numeric = {} + if raw_numeric: + ctx["raw_numeric"] = raw_numeric + + # 5) timeseries_raw: SOLO si hay columna temporal y numericas. Se delega + # el push-down en la funcion impura extract_timeseries_raw (una sola query + # ordenada cronologicamente). Solo se adjunta si trae filas. + if time_col and numeric_cols: + try: + from datascience import extract_timeseries_raw + + ts = extract_timeseries_raw( + query_fn, table, time_col, numeric_cols, max_rows=sample + ) + if ( + isinstance(ts, dict) + and ts.get("status") == "ok" + and (ts.get("n") or 0) > 0 + ): + ctx["timeseries_raw"] = { + "time_col": ts["time_col"], + "t": ts["t"], + "series": ts["series"], + } + except Exception: # noqa: BLE001 - dict-no-throw: omitir la clave + pass + + # 6) geo_points: detecta el par lat/lon con la funcion PURA del registry. + # Solo se adjunta si AMBAS columnas estan en raw_numeric (ya floats, + # alineadas por fila). Si no hay par o no estan, se omite: el capitulo + # geospatial sabe degradar. + try: + from datascience import detect_latlon_columns + + geo = detect_latlon_columns(cols) + lat_col = geo.get("lat_col") + lon_col = geo.get("lon_col") + if lat_col and lon_col and lat_col in raw_numeric and lon_col in raw_numeric: + ctx["geo_points"] = { + "lats": raw_numeric[lat_col], + "lons": raw_numeric[lon_col], + } + except Exception: # noqa: BLE001 - dict-no-throw: omitir la clave + pass + + return ctx + except Exception: # noqa: BLE001 - dict-no-throw global: nunca reventar. + # Fallback minimo: copia de base_ctx + db_path/table para que el capitulo + # agregacion siga teniendo lo imprescindible. + out = dict(base_ctx) if isinstance(base_ctx, dict) else {} + out["db_path"] = db_path + out["table"] = table + return out diff --git a/python/functions/datascience/build_eda_render_ctx_test.py b/python/functions/datascience/build_eda_render_ctx_test.py new file mode 100644 index 00000000..bf8aee7a --- /dev/null +++ b/python/functions/datascience/build_eda_render_ctx_test.py @@ -0,0 +1,153 @@ +"""Tests para build_eda_render_ctx. + +Self-contained: crea un DuckDB temporal pequeño con una columna fecha, varias +numericas y un par lat/lon, construye un TableProfile minimo a mano (con la forma +de columnas del grupo `eda`: name / inferred_type / numeric.{min,max} / +semantic_type) y verifica que el ctx producido contiene las claves de datos que +consumen los capitulos del AutomaticEDA. +""" + +import os +import sys + +# El test importa funciones del registry como una app del registry: inserta el +# directorio raiz `python/functions` en sys.path y luego `from datascience import`. +_FUNCTIONS_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if _FUNCTIONS_ROOT not in sys.path: + sys.path.insert(0, _FUNCTIONS_ROOT) + +import duckdb # noqa: E402 + +from datascience import build_eda_render_ctx # noqa: E402 + +_TABLE = "ventas_geo" +# Filas: fecha creciente, 2 columnas numericas (ventas, unidades) y un par lat/lon +# (Madrid -> lat ~40, lon ~-3, dentro de [-90,90] y [-180,180]). +_ROWS = [ + ("2024-01-01", 1200.5, 12, 40.41, -3.70), + ("2024-01-02", 980.0, 9, 41.38, 2.17), + ("2024-01-03", 1500.25, 15, 37.39, -5.99), + ("2024-01-04", 1100.0, 11, 39.47, -0.38), + ("2024-01-05", 1750.75, 18, 43.26, -2.93), +] + + +def _make_db(tmp_path): + """Crea un DuckDB temporal con la tabla de prueba y devuelve su ruta.""" + db_path = os.path.join(str(tmp_path), "eda_ctx.duckdb") + con = duckdb.connect(db_path) + try: + con.execute( + f'CREATE TABLE "{_TABLE}" ' + "(fecha DATE, ventas DOUBLE, unidades INTEGER, lat DOUBLE, lon DOUBLE)" + ) + con.executemany( + f'INSERT INTO "{_TABLE}" VALUES (?, ?, ?, ?, ?)', _ROWS + ) + finally: + con.close() + return db_path + + +def _profile_with_date(): + """TableProfile minimo con columna fecha + numericas + lat/lon.""" + return { + "columns": [ + {"name": "fecha", "inferred_type": "datetime", "semantic_type": "datetime_iso"}, + { + "name": "ventas", + "inferred_type": "numeric", + "semantic_type": "decimal", + "numeric": {"min": 980.0, "max": 1750.75}, + }, + { + "name": "unidades", + "inferred_type": "numeric", + "semantic_type": "integer", + "numeric": {"min": 9, "max": 18}, + }, + { + "name": "lat", + "inferred_type": "numeric", + "semantic_type": "decimal", + "numeric": {"min": 37.39, "max": 43.26}, + }, + { + "name": "lon", + "inferred_type": "numeric", + "semantic_type": "decimal", + "numeric": {"min": -5.99, "max": 2.17}, + }, + ] + } + + +def _profile_without_date(): + """Mismo perfil pero SIN columna temporal (solo numericas).""" + prof = _profile_with_date() + prof["columns"] = [c for c in prof["columns"] if c["name"] != "fecha"] + return prof + + +def test_db_path_y_table_en_ctx(tmp_path): + db_path = _make_db(tmp_path) + ctx = build_eda_render_ctx(db_path, _TABLE, _profile_with_date()) + assert ctx["db_path"] == db_path + assert ctx["table"] == _TABLE + + +def test_raw_numeric_con_columnas_numericas(tmp_path): + db_path = _make_db(tmp_path) + ctx = build_eda_render_ctx(db_path, _TABLE, _profile_with_date()) + raw = ctx["raw_numeric"] + # Las 4 columnas numericas (ventas, unidades, lat, lon), listas no vacias y + # alineadas por fila (misma longitud == nº de filas). + for col in ("ventas", "unidades", "lat", "lon"): + assert col in raw + assert len(raw[col]) == len(_ROWS) + assert raw["ventas"][0] == 1200.5 + assert raw["unidades"][0] == 12.0 # int promovido a float + + +def test_timeseries_raw_con_fecha(tmp_path): + db_path = _make_db(tmp_path) + ctx = build_eda_render_ctx(db_path, _TABLE, _profile_with_date()) + ts = ctx["timeseries_raw"] + assert ts["time_col"] == "fecha" + assert len(ts["t"]) == len(_ROWS) # fechas ISO no vacias + # Las numericas aparecen como series paralelas a t. + for col in ("ventas", "unidades", "lat", "lon"): + assert col in ts["series"] + assert len(ts["series"][col]) == len(_ROWS) + + +def test_geo_points_con_latlon(tmp_path): + db_path = _make_db(tmp_path) + ctx = build_eda_render_ctx(db_path, _TABLE, _profile_with_date()) + geo = ctx["geo_points"] + assert len(geo["lats"]) == len(_ROWS) + assert len(geo["lons"]) == len(_ROWS) + # Listas alineadas, ya floats, leidas de raw_numeric. + assert geo["lats"][0] == 40.41 + assert geo["lons"][0] == -3.70 + + +def test_sin_fecha_no_hay_timeseries(tmp_path): + db_path = _make_db(tmp_path) + ctx = build_eda_render_ctx(db_path, _TABLE, _profile_without_date()) + assert "timeseries_raw" not in ctx + # raw_numeric y geo_points siguen presentes (no dependen de la fecha). + assert "raw_numeric" in ctx + assert "geo_points" in ctx + + +def test_base_ctx_preservado(tmp_path): + db_path = _make_db(tmp_path) + base = {"dataset_name": "ventas_geo_demo", "source_origin": "test"} + ctx = build_eda_render_ctx(db_path, _TABLE, _profile_with_date(), base_ctx=base) + # Las claves de presentacion del base_ctx no se pisan. + assert ctx["dataset_name"] == "ventas_geo_demo" + assert ctx["source_origin"] == "test" + # Y las de datos se añaden encima. + assert ctx["db_path"] == db_path + assert "raw_numeric" in ctx diff --git a/python/functions/pipelines/profile_table.md b/python/functions/pipelines/profile_table.md index eb8ce6d2..2808965a 100644 --- a/python/functions/pipelines/profile_table.md +++ b/python/functions/pipelines/profile_table.md @@ -5,7 +5,7 @@ lang: py domain: pipelines purity: impure version: "1.0.0" -signature: "def profile_table(db_path: str, table: str, backend: str = \"duckdb\", sample: int = 5000, run_models: bool = False, run_llm: bool = False, run_series: bool = False, emit_pdf: bool = False, report_dir: str = \"reports\", write_report: bool = True) -> dict" +signature: "def profile_table(db_path: str, table: str, backend: str = \"duckdb\", sample: int = 5000, run_models: bool = False, run_llm: bool = False, run_series: bool = False, emit_pdf: bool = False, emit_automatic: bool = False, report_dir: str = \"reports\", write_report: bool = True) -> dict" description: "Orquestador one-shot del grupo de capacidad eda: perfila UNA tabla (DuckDB o PostgreSQL) end-to-end componiendo las funciones del grupo (perfil base SQL + muestreo read-only + inferencia semantica + promocion de tipo + estadistica numerica/categorica + score de calidad + correlaciones con correccion FDR + re-expresion de Tukey + avisos exploratorios) y, opcional, modelos baratos (run_models), interpretacion LLM (run_llm) y analisis de serie temporal por columna (run_series: estacionariedad ADF+KPSS, ACF/PACF, STL, retornos). Emite el TableProfile completo mas (opcional) report markdown + JSON sidecar + PDF movil (emit_pdf). Es la composicion canonica para hazme un EDA de esta tabla." tags: [eda, duckdb, postgres, profiling, data-quality, pipeline, dataops, timeseries] uses_functions: @@ -26,6 +26,9 @@ uses_functions: - exploratory_caveats_py_datascience - render_eda_markdown_py_datascience - render_eda_pdf_py_datascience + - build_eda_render_ctx_py_datascience + - render_automatic_eda_pdf_py_datascience + - render_automatic_eda_pptx_py_datascience - duckdb_query_readonly_py_infra - pg_query_py_infra uses_types: [] @@ -55,11 +58,13 @@ params: desc: "Si True (default False) calcula por columna numerica un bloque de serie temporal (estacionariedad ADF+KPSS, ACF/PACF, STL y, si parece de niveles, retornos). Ordena por la primera columna datetime si existe; si no, por el orden fisico. Guardado en col['series'] y agregado en prof['series']." - name: emit_pdf desc: "Si True (default False) renderiza un PDF multipagina vertical (legible en movil) del perfil junto al report markdown y devuelve su ruta en pdf_path." + - name: emit_automatic + desc: "Si True (default False) emite ADEMAS el informe AutomaticEDA completo en PDF (A5 movil) Y PPTX (16:9) con los 11 capitulos del motor; construye el ctx de datos crudos con build_eda_render_ctx para que modelos/timeseries/geospatial/agregacion salgan poblados. Aditivo: no sustituye a emit_pdf. Rutas en aeda_pdf_path / aeda_pptx_path / aeda_manifest_path." - name: report_dir desc: "Directorio donde escribir los reports si write_report (y el PDF si emit_pdf). Default 'reports'. Se crea si no existe." - name: write_report desc: "Si True (default) escribe report markdown + JSON sidecar timestamped en report_dir; si False no toca disco y los paths markdown/json del retorno son None (emit_pdf es independiente)." -output: "dict {status:'ok', profile:, report_md_path:str|None, report_json_path:str|None, pdf_path:str|None} o {status:'error', error:str} (dict-no-throw)." +output: "dict {status:'ok', profile:, report_md_path:str|None, report_json_path:str|None, pdf_path:str|None, aeda_pdf_path:str|None, aeda_pptx_path:str|None, aeda_manifest_path:str|None (estos tres solo con emit_automatic)} o {status:'error', error:str} (dict-no-throw)." --- ## Ejemplo diff --git a/python/functions/pipelines/profile_table.py b/python/functions/pipelines/profile_table.py index 8838a81e..8a0077af 100644 --- a/python/functions/pipelines/profile_table.py +++ b/python/functions/pipelines/profile_table.py @@ -32,11 +32,14 @@ from datascience import ( acf_pacf, adf_kpss_stationarity, association_matrix, + build_eda_render_ctx, column_quality_score, describe_numeric, eda_llm_insights, exploratory_caveats, infer_semantic_type, + render_automatic_eda_pdf, + render_automatic_eda_pptx, render_eda_markdown, render_eda_pdf, run_eda_models, @@ -385,6 +388,7 @@ def profile_table( run_llm: bool = False, run_series: bool = False, emit_pdf: bool = False, + emit_automatic: bool = False, report_dir: str = "reports", write_report: bool = True, ) -> dict: @@ -412,6 +416,15 @@ def profile_table( emit_pdf: si True (default False) renderiza un PDF multipagina vertical (legible en movil) del perfil junto al report markdown y devuelve su ruta en pdf_path. + emit_automatic: si True (default False) emite ademas el informe + AutomaticEDA COMPLETO en sus dos formatos (PDF A5 movil + PPTX 16:9) + con los 11 capitulos del motor por capitulos. Construye el contexto + de datos crudos con build_eda_render_ctx (raw_numeric para modelos/ + geo, timeseries_raw para series, geo_points para el mapa, db_path/ + table para la agregacion push-down) para que los capitulos modelos/ + timeseries/geospatial/agregacion salgan poblados, no degradados. Es + ADITIVO: no sustituye a emit_pdf (render_eda_pdf). Sus rutas vuelven + en aeda_pdf_path / aeda_pptx_path / aeda_manifest_path. report_dir: directorio donde escribir los reports si write_report. Default "reports". Se crea si no existe. write_report: si True (default), escribe un report markdown + un JSON @@ -727,12 +740,51 @@ def profile_table( except Exception: # noqa: BLE001 pdf_path = None + # Informe AutomaticEDA completo (PDF + PPTX por capitulos). Aditivo: + # convive con emit_pdf (render_eda_pdf) sin sustituirlo. Construye el ctx + # con los datos crudos para que modelos/timeseries/geospatial/agregacion + # salgan poblados; degrada por clave si build_eda_render_ctx falla. + aeda_pdf_path = None + aeda_pptx_path = None + aeda_manifest_path = None + if emit_automatic: + try: + os.makedirs(report_dir, exist_ok=True) + base_ctx = { + "dataset_name": table, + "source_origin": db_path, + "storage": "DuckDB" if backend == "duckdb" else ( + "PostgreSQL" if backend == "postgres" else backend), + } + if run_llm: + base_ctx.update({"run_cluster_llm": True, + "run_geo_llm": True, "run_agg_llm": True}) + ctx = build_eda_render_ctx( + db_path, table, prof, backend=backend, sample=sample, + base_ctx=base_ctx) + meta = {"title": f"EDA — {table}", "ctx": ctx} + aeda_pdf_target = os.path.join(report_dir, + f"aeda_{table}_{ts}.pdf") + aeda_pptx_target = os.path.join(report_dir, + f"aeda_{table}_{ts}.pptx") + rpdf = render_automatic_eda_pdf(prof, aeda_pdf_target, meta) or {} + rpptx = render_automatic_eda_pptx( + prof, aeda_pptx_target, meta) or {} + aeda_pdf_path = rpdf.get("path") + aeda_pptx_path = rpptx.get("path") + aeda_manifest_path = rpdf.get("manifest_path") + except Exception: # noqa: BLE001 + pass + return { "status": "ok", "profile": prof, "report_md_path": report_md_path, "report_json_path": report_json_path, "pdf_path": pdf_path, + "aeda_pdf_path": aeda_pdf_path, + "aeda_pptx_path": aeda_pptx_path, + "aeda_manifest_path": aeda_manifest_path, } except Exception as e: # noqa: BLE001 return {"status": "error", "error": str(e)} diff --git a/python/functions/pipelines/render_automatic_eda.md b/python/functions/pipelines/render_automatic_eda.md new file mode 100644 index 00000000..b157dfd2 --- /dev/null +++ b/python/functions/pipelines/render_automatic_eda.md @@ -0,0 +1,91 @@ +--- +name: render_automatic_eda +kind: pipeline +lang: py +domain: pipelines +purity: impure +version: "1.0.0" +signature: "def render_automatic_eda(db_path: str, table: str, backend: str = \"duckdb\", sample: int = 5000, run_models: bool = True, run_series: bool = True, run_llm: bool = False, out_dir: str = \"reports\", basename: str = None, ctx_extra: dict = None) -> dict" +description: "Informe AutomaticEDA COMPLETO one-shot de una tabla DuckDB/PostgreSQL: perfila con profile_table, construye el ctx con los datos crudos (build_eda_render_ctx: raw_numeric para modelos/geo, timeseries_raw para series, geo_points para el mapa, db_path/table para la agregacion push-down) y emite PDF (A5 movil) Y PPTX (16:9) del mismo documento por capitulos, con los 11 capitulos POBLADOS de verdad (clusters pintados sobre el PCA, evolucion temporal, mapa geografico y tablas de agregacion), no degradados. Devuelve las rutas de PDF/PPTX y el manifiesto de versiones por capitulo." +tags: [eda, duckdb, postgres, profiling, pipeline, dataops, report, pdf, pptx] +uses_functions: + - profile_table_py_pipelines + - build_eda_render_ctx_py_datascience + - render_automatic_eda_pdf_py_datascience + - render_automatic_eda_pptx_py_datascience +uses_types: [] +returns: [] +returns_optional: false +error_type: error_go_core +imports: [] +tested: true +tests: + - "render end-to-end sobre DuckDB sintetico con categoricas + fecha + lat/lon emite PDF y PPTX con paginas/slides" +test_file_path: "python/functions/pipelines/render_automatic_eda_test.py" +file_path: "python/functions/pipelines/render_automatic_eda.py" +params: + - name: db_path + desc: "Ruta al archivo DuckDB (read-only, debe existir) o DSN PostgreSQL si backend='postgres'." + - name: table + desc: "Nombre de la tabla a perfilar e informar." + - name: backend + desc: "'duckdb' (default) o 'postgres'. Selecciona el motor de perfilado y muestreo." + - name: sample + desc: "Maximo de filas/valores muestreados por columna para el perfil y para los datos crudos del ctx (LIMIT). Default 5000." + - name: run_models + desc: "Si True (default) corre los modelos baratos (PCA/KMeans/IsolationForest/normalidad); necesario para que el capitulo modelos pinte los clusters sobre el plano PCA." + - name: run_series + desc: "Si True (default) calcula el analisis de serie temporal por columna numerica; necesario para el analisis del capitulo timeseries (la grafica de evolucion sale de los datos crudos del ctx aunque sea False)." + - name: run_llm + desc: "Si True (default False) hace la interpretacion LLM del perfil y ACTIVA la narrativa LLM de los capitulos modelos/geospatial/agregacion (titulos de segmento, descripcion de zona, seleccion de agregaciones). Con False usan su derivacion cuantitativa sin red." + - name: out_dir + desc: "Directorio de salida (se crea si no existe). Default 'reports'." + - name: basename + desc: "Nombre base de los archivos sin extension. Default 'aeda__'." + - name: ctx_extra + desc: "Dict opcional con claves de presentacion/contexto extra que se mezclan en el ctx (dataset_name, description, source_origin, ...); no pisan las claves de datos calculadas por build_eda_render_ctx." +output: "dict {status:'ok', pdf_path:str, pptx_path:str, manifest_path:str|None, n_pages:int, n_slides:int, pdf_note:str, pptx_note:str, profile:} o {status:'error', error:str} (dict-no-throw)." +--- + +## Ejemplo + +```python +from pipelines.render_automatic_eda import render_automatic_eda + +# Tabla DuckDB con categoricas + fecha + numericas: informe completo a reports/. +r = render_automatic_eda("/tmp/ventas.duckdb", "ventas", + run_models=True, run_series=True, out_dir="reports") +print(r["status"], r["pdf_path"], r["pptx_path"], r["n_pages"], r["n_slides"]) +# ok reports/aeda_ventas_20260630-120500.pdf reports/aeda_ventas_20260630-120500.pptx 14 16 + +# Con narrativa LLM (titulos de segmento, descripcion geografica, etc.): +r = render_automatic_eda("/tmp/ventas.duckdb", "ventas", run_llm=True) +``` + +## Cuando usarla + +Cuando quieras el informe AutomaticEDA COMPLETO (PDF + PPTX) de una tabla en una +sola llamada, con los capitulos de modelos, series, geoespacial y agregacion ya +poblados (no degradados). Es el reemplazo de "perfila + monta el ctx a mano + +llama a los dos renderers": este pipeline orquesta `profile_table` -> +`build_eda_render_ctx` -> `render_automatic_eda_pdf`/`_pptx`. Usalo como +entregable para compartir un EDA, o como el motor detras de `profile_table( +emit_automatic=True)` y del skill `/eda`. + +## Gotchas + +- Impura: ESCRIBE el PDF, el PPTX y `automatic_eda_manifest.json` en `out_dir`. +- `db_path` debe existir: DuckDB read-only no crea la base. +- `run_models=True` y `run_series=True` por defecto encarecen el perfil (PCA/ + KMeans/IsolationForest + ADF/KPSS/STL por columna). Para un informe mas barato + ponlos a False: los capitulos modelos/timeseries se omiten o se reducen, pero + el resto del informe sale igual. +- `run_llm=True` hace llamadas de red (interpretacion del perfil + narrativa por + capitulo). Sin red, dejalo en False: los capitulos siguen completos con su + derivacion cuantitativa (titulos de segmento derivados, nota geografica + derivada, seleccion de agregaciones cuantitativa). +- El PPTX requiere `python-pptx`; si no esta instalado, `pptx_path` sera None y + `pptx_note` lo explica (el PDF se emite igual). +- Los datos crudos del ctx se muestrean con `sample` (LIMIT), no se trae la tabla + entera a RAM; con tablas enormes sube `sample` si quieres mas representatividad + (coste: mas memoria). diff --git a/python/functions/pipelines/render_automatic_eda.py b/python/functions/pipelines/render_automatic_eda.py new file mode 100644 index 00000000..c0b58065 --- /dev/null +++ b/python/functions/pipelines/render_automatic_eda.py @@ -0,0 +1,157 @@ +"""render_automatic_eda — EDA completo one-shot: perfil → ctx → PDF + PPTX. + +Pipeline impuro del grupo de capacidad `eda`. Dada UNA tabla DuckDB (o +PostgreSQL), produce el informe AutomaticEDA COMPLETO en sus dos formatos a la +vez (PDF móvil A5 + PPTX 16:9) con los 11 capítulos POBLADOS, en una sola +llamada. Compone, sin reimplementar su lógica, cuatro funciones del registry: + + - profile_table : perfila la tabla end-to-end (TableProfile agregado), + opcionalmente con modelos baratos y análisis de serie. + - build_eda_render_ctx : construye el `ctx` con los DATOS CRUDOS que el + TableProfile agregado no incluye (raw_numeric para + modelos/geo, timeseries_raw para series, geo_points + para el mapa, db_path/table para la agregación + push-down). Sin él, esos capítulos degradan. + - render_automatic_eda_pdf : renderiza el documento por capítulos a PDF. + - render_automatic_eda_pptx : renderiza el mismo documento a PPTX. + +El TableProfile agregado basta para portada/overview/distribuciones/calidad/ +correlación, pero los capítulos `modelos`, `timeseries`, `geospatial` y +`agregacion` necesitan datos crudos (los clusters proyectados sobre el PCA, la +serie valor-vs-tiempo, los puntos lat/lon, las filas para el groupby/pivot). +`build_eda_render_ctx` los muestrea (LIMIT + push-down, sin traer la tabla +entera a RAM) y los entrega en `ctx`; este pipeline los pasa como `meta['ctx']` +a ambos renderers para que el informe salga completo. + +Estilo dict-no-throw del grupo `eda`: nunca lanza; captura cualquier error y +degrada a `{"status": "error", "error": str}`. +""" + +import os +from datetime import datetime, timezone + +from datascience import ( + build_eda_render_ctx, + render_automatic_eda_pdf, + render_automatic_eda_pptx, +) +from pipelines.profile_table import profile_table + +# Tokens de almacenamiento por backend (para la portada del informe). +_STORAGE = {"duckdb": "DuckDB", "postgres": "PostgreSQL"} + + +def render_automatic_eda( + db_path: str, + table: str, + backend: str = "duckdb", + sample: int = 5000, + run_models: bool = True, + run_series: bool = True, + run_llm: bool = False, + out_dir: str = "reports", + basename: str = None, + ctx_extra: dict = None, +) -> dict: + """Perfila una tabla y emite el informe AutomaticEDA completo (PDF + PPTX). + + Args: + db_path: ruta al archivo DuckDB, o DSN PostgreSQL si backend="postgres". + table: nombre de la tabla a perfilar. + backend: "duckdb" (default) o "postgres". + sample: máximo de filas/valores muestreados por columna para el perfil + y para los datos crudos del ctx (LIMIT). Default 5000. + run_models: si True (default) corre los modelos baratos + (PCA/KMeans/IsolationForest/normalidad). Necesario para que el + capítulo `modelos` pinte los clusters sobre el plano PCA. + run_series: si True (default) calcula el análisis de serie temporal por + columna numérica. Necesario para el análisis del capítulo + `timeseries` (la gráfica de evolución sale de los datos crudos del + ctx aunque run_series sea False). + run_llm: si True (default False) hace la interpretación LLM del perfil y + ACTIVA además la narrativa LLM de los capítulos modelos/geospatial/ + agregacion (títulos de segmento, descripción de la zona, selección de + agregaciones). Con False esos capítulos usan su derivación + cuantitativa (siguen completos, sin llamadas de red). + out_dir: directorio de salida (se crea si no existe). Default "reports". + basename: nombre base de los archivos sin extensión. Default + "aeda_
_". + ctx_extra: dict opcional con claves de presentación/contexto extra que se + mezclan en el ctx (p.ej. dataset_name, description, source_origin). + No pisan las claves de datos calculadas por build_eda_render_ctx. + + Returns: + dict (nunca lanza). En éxito:: + + {"status": "ok", "pdf_path": str, "pptx_path": str, + "manifest_path": str|None, "n_pages": int, "n_slides": int, + "pdf_note": str, "pptx_note": str, "profile": } + + En error: {"status": "error", "error": str}. + """ + try: + # 1) Perfil base + modelos/serie opcionales. No escribe report propio + # (write_report=False): este pipeline emite su propio par PDF/PPTX. + pres = profile_table( + db_path, + table, + backend=backend, + sample=sample, + run_models=run_models, + run_llm=run_llm, + run_series=run_series, + emit_pdf=False, + write_report=False, + ) + if pres.get("status") != "ok": + return {"status": "error", + "error": f"profile_table falló: {pres.get('error')}"} + prof = pres.get("profile") or {} + + # 2) Contexto de presentación + datos crudos para los 4 capítulos que los + # necesitan. Las claves de presentación van en base_ctx; build_eda_render_ctx + # añade raw_numeric / timeseries_raw / geo_points / db_path / table. + base_ctx = { + "dataset_name": table, + "source_origin": db_path, + "storage": _STORAGE.get(backend, backend), + } + if run_llm: + # Activa la narrativa LLM de los capítulos que la soportan. + base_ctx.update({ + "run_cluster_llm": True, + "run_geo_llm": True, + "run_agg_llm": True, + }) + if ctx_extra: + base_ctx.update(ctx_extra) + + ctx = build_eda_render_ctx( + db_path, table, prof, backend=backend, sample=sample, + base_ctx=base_ctx, + ) + + # 3) Render a ambos formatos desde el MISMO documento por capítulos. + os.makedirs(out_dir, exist_ok=True) + ts = datetime.now(timezone.utc).strftime("%Y%m%d-%H%M%S") + base = basename or f"aeda_{table}_{ts}" + pdf_path = os.path.join(out_dir, base + ".pdf") + pptx_path = os.path.join(out_dir, base + ".pptx") + meta = {"title": f"EDA — {table}", "ctx": ctx} + + rpdf = render_automatic_eda_pdf(prof, pdf_path, meta) or {} + rpptx = render_automatic_eda_pptx(prof, pptx_path, meta) or {} + + return { + "status": "ok", + "pdf_path": rpdf.get("path"), + "pptx_path": rpptx.get("path"), + "manifest_path": rpdf.get("manifest_path"), + "n_pages": rpdf.get("n_pages"), + "n_slides": rpptx.get("n_slides"), + "pdf_note": rpdf.get("note"), + "pptx_note": rpptx.get("note"), + "profile": prof, + } + except Exception as e: # noqa: BLE001 — dict-no-throw: degradar, nunca lanzar. + return {"status": "error", "error": str(e)} diff --git a/python/functions/pipelines/render_automatic_eda_test.py b/python/functions/pipelines/render_automatic_eda_test.py new file mode 100644 index 00000000..a463e4f7 --- /dev/null +++ b/python/functions/pipelines/render_automatic_eda_test.py @@ -0,0 +1,91 @@ +"""Test del pipeline render_automatic_eda — EDA completo a PDF + PPTX. + +Self-contained: crea un DuckDB temporal pequeño con categóricas + fecha + lat/lon ++ varias numéricas, corre el pipeline (sin LLM) y verifica que emite PDF y PPTX +con páginas/slides, manifest, y que los capítulos dependientes de ctx quedan +POBLADOS (sin la nota de degradación). +""" + +import os +import sys + +_HERE = os.path.dirname(os.path.abspath(__file__)) +_FUNCTIONS = os.path.abspath(os.path.join(_HERE, "..", "..")) # python/functions +if _FUNCTIONS not in sys.path: + sys.path.insert(0, _FUNCTIONS) + +import duckdb # noqa: E402 + +from pipelines.render_automatic_eda import render_automatic_eda # noqa: E402 + + +def _make_db(path): + con = duckdb.connect(path) + con.execute( + "CREATE TABLE sales (d DATE, region VARCHAR, channel VARCHAR, " + "amount DOUBLE, units INTEGER, lat DOUBLE, lon DOUBLE)" + ) + from datetime import date, timedelta + + regions = ["norte", "sur", "este"] + channels = ["web", "tienda"] + centers = {"norte": (43.0, -3.0), "sur": (37.0, -5.0), "este": (39.5, -0.4)} + rows = [] + d0 = date(2024, 1, 1) + for i in range(180): + r = regions[i % 3] + ch = channels[i % 2] + clat, clon = centers[r] + rows.append(( + d0 + timedelta(days=i), r, ch, + round(100 + (i % 7) * 13.5 + (5 if ch == "web" else 0), 2), + 10 + (i % 5), + round(clat + (i % 3) * 0.1, 4), + round(clon + (i % 4) * 0.1, 4), + )) + con.executemany("INSERT INTO sales VALUES (?,?,?,?,?,?,?)", rows) + con.close() + + +def test_pipeline_emits_pdf_and_pptx_with_chapters(tmp_path): + db = str(tmp_path / "sales.duckdb") + _make_db(db) + out = str(tmp_path / "out") + + r = render_automatic_eda(db, "sales", run_models=True, run_series=True, + run_llm=False, out_dir=out, basename="test_sales") + assert r["status"] == "ok", r.get("error") + + # Both formats produced. + assert r["pdf_path"] and os.path.exists(r["pdf_path"]) + assert r["pptx_path"] and os.path.exists(r["pptx_path"]) + assert (r["n_pages"] or 0) > 0 + assert (r["n_slides"] or 0) > 0 + # Per-chapter manifest written next to the output. + assert r["manifest_path"] and os.path.exists(r["manifest_path"]) + + +def test_pipeline_chapters_populated_not_degraded(tmp_path): + """The 4 ctx-dependent chapters build with real data (no degradation note).""" + import json + + db = str(tmp_path / "sales.duckdb") + _make_db(db) + out = str(tmp_path / "out") + r = render_automatic_eda(db, "sales", run_models=True, run_series=True, + run_llm=False, out_dir=out, basename="t2") + assert r["status"] == "ok" + + # The manifest lists the ctx-dependent chapters as actually rendered. + with open(r["manifest_path"], encoding="utf-8") as fh: + man = json.load(fh) + chapters = man.get("chapters") or {} + for cid in ("modelos", "timeseries", "geospatial", "agregacion"): + assert cid in chapters, f"capítulo {cid} ausente del manifest: {list(chapters)}" + + +def test_pipeline_bad_db_degrades_without_raising(tmp_path): + r = render_automatic_eda(str(tmp_path / "nope.duckdb"), "ghost", + out_dir=str(tmp_path / "o")) + assert r["status"] == "error" + assert "error" in r From 437409641ca6a595e8b1507d50c3d3c43cf600e6 Mon Sep 17 00:00:00 2001 From: Egutierrez Date: Tue, 30 Jun 2026 16:08:50 +0200 Subject: [PATCH 20/53] docs(eda): el skill /eda emite SIEMPRE PDF + PPTX con AutomaticEDA MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Actualiza el flujo del comando para que un EDA completo emita el informe AutomaticEDA en sus dos formatos (PDF A5 móvil + PPTX 16:9) con los 11 capítulos poblados, vía render_automatic_eda (o profile_table(emit_automatic=True)). El PDF legacy (emit_pdf/render_eda_pdf) queda como salida independiente opcional. Co-Authored-By: Claude Opus 4.8 (1M context) --- .claude/commands/eda.md | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/.claude/commands/eda.md b/.claude/commands/eda.md index 8ce31ef8..860c340f 100644 --- a/.claude/commands/eda.md +++ b/.claude/commands/eda.md @@ -25,9 +25,10 @@ Página madre del grupo: `docs/capabilities/eda.md` (léela primero para cargar - `--models` → `run_models=True` (PCA/KMeans/IsolationForest/normalidad). - `--llm` → `run_llm=True` (1 call LLM sobre el perfil agregado). - `--series` → `run_series=True` (estacionariedad ADF+KPSS, ACF/PACF, STL, retornos por columna numérica). - - `--pdf` → `emit_pdf=True` (PDF A5 vertical legible en móvil). + - `--pdf` → `emit_pdf=True` (PDF A5 legacy de `render_eda_pdf`, legible en móvil). + - `--legacy-only` → emite SOLO el PDF legacy (sin AutomaticEDA), para casos en que solo se quiera el PDF rápido. -Por defecto, para un EDA "completo" cuando el usuario no especifica, activa `run_models`, `run_series` y `emit_pdf`; deja `run_llm` para cuando lo pida o cuando interese la interpretación semántica (es la única parte que gasta tokens del modelo). +Por defecto, **un EDA completo emite SIEMPRE el informe AutomaticEDA en sus dos formatos: PDF (A5 móvil) Y PPTX (16:9 para compartir)** con los 11 capítulos poblados (portada, overview, distribuciones, calidad, correlaciones, modelos, series, geoespacial, agregación, interpretación LLM). Usa el pipeline `render_automatic_eda` (o `profile_table(emit_automatic=True)`), que activa `run_models` y `run_series` para que los capítulos de modelos/series/geoespacial/agregación salgan poblados. Deja `run_llm` para cuando el usuario lo pida o interese la interpretación semántica + narrativa por capítulo (es la única parte que gasta tokens del modelo). ## Reglas duras @@ -35,7 +36,7 @@ Por defecto, para un EDA "completo" cuando el usuario no especifica, activa `run 2. **CSV/Parquet/Excel** entran cargándolos antes a DuckDB (`read_csv_auto`/`read_parquet`/`read_xlsx`) — DuckDB es el motor por defecto. No traigas la tabla entera a RAM. 3. **Secretos**: si la fuente es un DSN PostgreSQL con credenciales, NO las imprimas en los reports ni en el notebook; resuélvelas vía `resolve_pg_dsn`/`pass` cuando aplique. 4. **El report es un artefacto local**: vive en `reports/` (gitignored), no se sube a Gitea ni se versiona. Compartir = pasar la ruta (regla `reports.md`). -5. **Entrega las 4 salidas**: JSON sidecar + Markdown + **PDF móvil** + **notebook Jupyter colaborativo ejecutado en vivo**. +5. **Entrega las salidas**: el informe **AutomaticEDA PDF + PPTX** (siempre, con `render_automatic_eda` / `emit_automatic=True`) + (opcional) JSON sidecar + Markdown + PDF legacy + **notebook Jupyter colaborativo ejecutado en vivo**. Comparte las rutas de PDF y PPTX. ## Paso 1 — Perfilar y escribir los reports @@ -43,18 +44,26 @@ Una tabla (caso normal): ```bash PYTHONPATH=python/functions python/.venv/bin/python3 - <<'PYEOF' -from pipelines.profile_table import profile_table -r = profile_table( +from pipelines.render_automatic_eda import render_automatic_eda +# Informe AutomaticEDA COMPLETO one-shot: perfil + ctx (datos crudos) + PDF + PPTX +# con los 11 capítulos poblados (clusters pintados, evolución temporal, mapa, +# tablas de agregación). run_llm=True añade la narrativa LLM por capítulo. +r = render_automatic_eda( "/ruta/datos.duckdb", "ventas", - run_models=True, run_series=True, emit_pdf=True, run_llm=False, + run_models=True, run_series=True, run_llm=False, out_dir="reports", ) print("status:", r["status"]) -print("md: ", r["report_md_path"]) -print("json: ", r["report_json_path"]) -print("pdf: ", r["pdf_path"]) +print("pdf: ", r["pdf_path"], "(", r["n_pages"], "págs )") +print("pptx: ", r["pptx_path"], "(", r["n_slides"], "slides )") +print("manifest:", r["manifest_path"]) PYEOF ``` +Si además quieres el report Markdown + JSON sidecar y/o el PDF legacy junto al +AutomaticEDA, usa `profile_table(emit_automatic=True, emit_pdf=True, write_report=True)`: +emite todo a la vez (`report_md_path`, `report_json_path`, `pdf_path` legacy, +`aeda_pdf_path`, `aeda_pptx_path`, `aeda_manifest_path`). + Una base entera (todas las tablas + relaciones FK): ```bash @@ -90,6 +99,7 @@ Sigue la memoria `eda-workflow-registry` y la regla `notebook_collaboration.md`: ## Notas - El `TableProfile` lleva ahora, además del perfilado base y las correlaciones con FDR: `series` (por columna numérica, con `run_series`), `reexpression` por columna numérica (escalera de Tukey) y `caveats` (siempre, avisos exploratorios). El Markdown y el PDF renderizan estas secciones automáticamente cuando están presentes. -- El PDF (`emit_pdf`) está pensado para leerse en el móvil (A5 vertical, tipografía grande, gráficos Tufte). Se escribe junto al Markdown en `reports/`. +- El informe **AutomaticEDA** (`render_automatic_eda` / `emit_automatic=True`) emite el MISMO documento por capítulos a **PDF (A5 móvil)** y **PPTX (16:9)** con garantía de no-corte (texto envuelto, tablas partidas repitiendo cabecera, figuras escaladas) y negrita real (`**texto**`). Escribe `automatic_eda_manifest.json` con la versión de cada capítulo. Los capítulos modelos/series/geoespacial/agregación se pueblan con los datos crudos que `build_eda_render_ctx` muestrea de la base (no se traen tablas enteras a RAM). +- El PDF legacy (`emit_pdf`, `render_eda_pdf`) sigue disponible y es independiente del AutomaticEDA (A5 vertical, gráficos Tufte). Se escribe junto al Markdown en `reports/`. - `run_series` ordena por la primera columna datetime si existe; si no, por el orden físico de filas. Necesita ≥8 puntos válidos por columna. - Fuentes: DuckDB (CSV/Parquet/Excel cargados antes) y PostgreSQL (`backend="postgres"`). `profile_database` (multi-tabla + FK) es solo DuckDB por ahora. From d1a3d58a6bfc609c02c019e7a884ec09028072ad Mon Sep 17 00:00:00 2001 From: Egutierrez Date: Tue, 30 Jun 2026 17:35:19 +0200 Subject: [PATCH 21/53] =?UTF-8?q?feat(eda):=20motor=20AutomaticEDA=20fase?= =?UTF-8?q?=204a=20=E2=80=94=20render=20fixes=20+=20keep-together=20+=20gl?= =?UTF-8?q?osario=20clicable?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mejoras transversales del motor de render (no del contenido de capítulos): 1. Fix negrita pisa texto (PDF): _place_rich_lines mide el ancho REAL de cada span con las métricas de fuente del renderer (peso correcto) en vez del grid de ancho medio; negrita y normal en la misma línea ya no se solapan. 2. Zebra striping: filas pares sombreadas (#f6f8fa) en DataTable (PDF + PPTX), coherente al partir tablas largas (índice de fila lógico, no por página). 3. Keep-together: bloque Group nuevo; el renderer mide el grupo entero y lo mueve completo a la página/slide siguiente si no cabe, y encoge la figura (height_in) para dejar sitio a su título y texto. num_distr lo usa. 4. Caption siempre visible en toda figura PPTX (fallback al heading); la figura reserva el alto de su caption para que ambos quepan en el mismo slide. 5. Portada construida al final (con resumen agregado del análisis vía ctx['document_summary']) pero colocada primera por build_document. 6. Glosario: capítulo nuevo (último) + GlossaryCollector en ctx; los capítulos registran términos y marcan apariciones con [[term:key]]...[[/term]]. Links clicables reales: PDF (PyMuPDF, link GOTO) y PPTX (slide-jump nativo). Enganchado "entropía" en cat_distr como ejemplo end-to-end. Funciones reutilizables delegadas a fn-constructor (tag eda): - add_pdf_internal_links_py_datascience (PyMuPDF) - pptx_link_run_to_slide_py_datascience (slide-jump) Contrato docs/automatic_eda_contract.md actualizado (§1/§3/§5 + §11 nueva) con la API de glosario, keep-together y zebra para la siguiente fase. PyMuPDF declarado en pyproject. Suite verde (90 tests); golden titanic verificado. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/automatic_eda_contract.md | 126 +++++- python/functions/datascience/__init__.py | 2 + .../datascience/add_pdf_internal_links.md | 85 ++++ .../datascience/add_pdf_internal_links.py | 132 ++++++ .../add_pdf_internal_links_test.py | 77 ++++ .../datascience/automatic_eda/__init__.py | 6 + .../automatic_eda/chapters/cat_distr.py | 33 +- .../automatic_eda/chapters/glosario.py | 47 ++ .../automatic_eda/chapters/num_distr.py | 19 +- .../automatic_eda/chapters/num_distr_test.py | 24 +- .../automatic_eda/chapters/portada.py | 52 ++- .../automatic_eda/chapters_registry.py | 78 +++- .../datascience/automatic_eda/model.py | 107 ++++- .../automatic_eda/render_features_test.py | 354 +++++++++++++++ .../automatic_eda/render_pdf_impl.py | 409 ++++++++++++++++-- .../automatic_eda/render_pptx_impl.py | 334 ++++++++++++-- .../datascience/automatic_eda/text_layout.py | 129 ++++++ .../datascience/pptx_link_run_to_slide.md | 85 ++++ .../datascience/pptx_link_run_to_slide.py | 50 +++ .../pptx_link_run_to_slide_test.py | 73 ++++ python/pyproject.toml | 1 + 21 files changed, 2116 insertions(+), 107 deletions(-) create mode 100644 python/functions/datascience/add_pdf_internal_links.md create mode 100644 python/functions/datascience/add_pdf_internal_links.py create mode 100644 python/functions/datascience/add_pdf_internal_links_test.py create mode 100644 python/functions/datascience/automatic_eda/chapters/glosario.py create mode 100644 python/functions/datascience/automatic_eda/render_features_test.py create mode 100644 python/functions/datascience/pptx_link_run_to_slide.md create mode 100644 python/functions/datascience/pptx_link_run_to_slide.py create mode 100644 python/functions/datascience/pptx_link_run_to_slide_test.py diff --git a/docs/automatic_eda_contract.md b/docs/automatic_eda_contract.md index 63e55213..efd96fa9 100644 --- a/docs/automatic_eda_contract.md +++ b/docs/automatic_eda_contract.md @@ -25,7 +25,8 @@ cabecera, y figuras/imágenes se escalan para caber enteras. ``` Document = list[Chapter] Chapter = { id: str, title: str, version: str, blocks: list[Block] } -Block = Heading | Markdown | KVTable | DataTable | Figure | Image | Caption | Note +Block = Heading | Markdown | KVTable | DataTable | Figure | Image | Caption + | Note | Group | GlossaryEntry ``` Importa el modelo desde `datascience.automatic_eda.model` (o @@ -44,6 +45,10 @@ reconocido se degrada a `Note`, nunca lanza). | `Figure(fig=None, make=None, caption=None, height_in=None)` | una `matplotlib.figure.Figure` ya construida (`fig`) o un callable `make()->Figure` (perezoso) | se rasteriza y escala para caber entera (nunca recortada) | | `Image(path, caption=None, height_in=None)` | ruta a PNG/JPG | se escala para caber entera | | `Caption(text)` / `Note(text)` | texto auxiliar pequeño | pie/nota en gris; `Note` es además el fallback de lo desconocido | +| `Group(blocks, title=None)` | unidad **keep-together**: sus bloques se mantienen juntos | el renderer mide el grupo entero y lo mueve completo a la página/slide siguiente si no cabe; encoge la figura para dejar sitio al título+texto. Ver §11 | +| `GlossaryEntry(key, label, definition)` | una entrada del glosario (destino clicable) | la genera el capítulo `glosario`; registra su posición como destino de los términos marcados. Ver §11 | + +`Figure`/`Image` aceptan `height_in` (hint): el renderer **clampa** la figura a esa altura máxima (lo usa `Group` para encoger la figura). Toda figura escala dejando sitio a su caption en la misma página/slide; en PPTX el caption es **siempre** visible (si no se da `caption`, cae al último heading o a "Figura"). ### Subset de markdown soportado (`Markdown`) @@ -84,8 +89,9 @@ El orden canónico está **pre-declarado** en ```python CHAPTER_ORDER = [ - "portada", "overview", "num_distr", "cat_distr", "calidad", "correlacion", - "modelos", "analisis_llm", "timeseries", "geospatial", "agregacion", + "portada", "overview", "analisis_llm", "num_distr", "cat_distr", "calidad", + "correlacion", "modelos", "timeseries", "geospatial", "agregacion", + "glosario", ] ``` @@ -95,6 +101,15 @@ CHAPTER_ORDER = [ `CHAPTER_ORDER`) y aparecerá automáticamente en su posición. Esto permite que muchos agentes trabajen **en paralelo** sin contención: cada uno toca solo su archivo. +**Dos capítulos tienen posición especial** (los gestiona `build_document`, no toques esto): + +- `portada`: se **construye el último** (después del cuerpo) para poder resumir el + análisis, pero se **coloca el primero**. Recibe `ctx['document_summary']` (ver §5) con + un resumen agregado del resto. Decisión del usuario: la portada refleja hallazgos. +- `glosario`: se construye y se **coloca el último**. Lee los términos que los demás + capítulos registraron en `ctx['glossary']` (ver §11). Si no se registró ninguno, el + capítulo devuelve `None` y desaparece. + Si tu capítulo usa un `` que aún no está en `CHAPTER_ORDER`, añádelo en la posición correcta (única edición compartida; coordínala con el orquestador). @@ -143,6 +158,8 @@ defensivo). Esto habilita el **seguimiento y la mejora continua por capítulo**. | `granularity` | "Cada fila es…" (portada). Default: derivado de `key_candidates` | | `quality_criteria` | criterios del score de calidad (portada) | | `head_rows` | `list[dict]` con `df.head` (overview). Ver §7 | +| `glossary` | `GlossaryCollector` compartido — los capítulos registran términos en él. Lo crea `build_document`; ver §11 | +| `document_summary` | dict con el resumen agregado del cuerpo (n_rows, n_cols, quality_score, n_numeric, n_categorical, chapter_titles, …). Lo calcula `build_document` y lo consume la portada | Un capítulo puede definir y consumir sus propias claves `ctx` — documenta cuáles en su docstring. @@ -279,6 +296,109 @@ sus bloques presentes y el no-corte (texto largo intacto en la salida). Patrón: --- +## 11. Glosario, keep-together y zebra (motor, fase 4a) + +Tres capacidades transversales del motor que **todos** los capítulos pueden usar. La 6.1 +(glosario) requiere que el capítulo coopere (registrar + marcar términos); la 6.2 +(keep-together) es opt-in por capítulo (envolver bloques en `Group`); la 6.3 (zebra) es +automática (no hay nada que hacer). + +### 11.1 Glosario con términos clicables + +El glosario es un capítulo nuevo (`chapters/glosario.py`) que se renderiza **siempre el +último** y lista cada término técnico que algún capítulo haya registrado. Cada aparición +del término en el texto se vuelve un **clic real** que salta a su entrada: en PDF como +*link annotation* interno (post-proceso con PyMuPDF, porque `PdfPages` no soporta +hyperlinks internos), en PPTX como *slide-jump* nativo (`ppaction://hlinksldjump`). + +**API exacta para un capítulo (dos pasos):** + +1. **Registrar el término** en el colector compartido `ctx['glossary']` (un + `model.GlossaryCollector`, creado por `build_document` y pasado a todos los capítulos): + + ```python + glossary = ctx.get("glossary") + if isinstance(glossary, model.GlossaryCollector): + glossary.add("entropia", "Entropía (de Shannon)", "Medida, en bits, de …") + ``` + + `add(key, label, definition)` es idempotente (la primera definición de cada `key` gana). + `key` debe ser `[A-Za-z0-9_]+`. Si no hay colector en `ctx` (renderizado suelto), el + capítulo simplemente no marca términos — degrada sin romper. + +2. **Marcar cada aparición** en el texto de un bloque `Markdown` con el span inline + `[[term:KEY]]texto visible[[/term]]`. El texto visible puede llevar `**negrita**`. El + marcador no altera el texto visible (se elimina como cualquier marcador inline); solo + añade el destino clicable. + + ```python + # En cat_distr (ejemplo real ya implementado): + "La [[term:entropia]]**entropía de Shannon**[[/term]] mide cómo de repartidos…" + ``` + +Eso es todo: el capítulo `glosario` recoge los términos (orden alfabético por `label`), +emite un `GlossaryEntry` por término, y los renderers cablean los enlaces automáticamente. +Si ningún capítulo registró términos, el glosario no aparece. + +**Helpers de `text_layout` (no reimplementar):** `parse_inline_rich(text)` → +`[(texto, is_bold, term_key), …]`; `wrap_rich_terms(text, max_chars)` → líneas de esos +spans sin corte. `strip_inline_md` ya elimina los marcadores `[[term:…]]`/`[[/term]]`. +(Las funciones previas `parse_inline_bold` / `wrap_rich` siguen existiendo, sin términos.) + +**Funciones del registry que cablean los enlaces** (grupo `eda`, ya invocadas por los +renderers; degradan en silencio si faltan): `add_pdf_internal_links_py_datascience` +(PyMuPDF, link GOTO) y `pptx_link_run_to_slide_py_datascience` (salto a slide nativo). +Dependencia: `pymupdf` (declarada en `python/pyproject.toml`). + +**Trabajo de la siguiente fase — enganchar más términos.** El mecanismo está hecho y +probado de extremo a extremo con `entropia` (en `cat_distr`). Cada capítulo debe registrar +y marcar SUS términos con el mismo patrón de dos pasos. Candidatos por capítulo: + +| Capítulo | Términos a enganchar (key sugerida) | +|---|---| +| `cat_distr` | `entropia` ✅ (hecho) | +| `calidad` | `completitud`, `validez`, `consistencia` | +| `correlacion` | `cramers_v`, `fdr` (comparaciones múltiples), método de correlación usado | +| `modelos` | `pca`, `silhouette`, `isolation_forest` | +| `timeseries` | `estacionariedad`, `acf_pacf`, `stl` | +| `num_distr` | `iqr`, `curtosis`, `outlier` (vallas de Tukey) | + +Define la definición de cada término en su capítulo (constante local, como +`_TERM_ENTROPIA_DEF` en `cat_distr`) y márcalo en su primera aparición. + +### 11.2 Keep-together: gráfico junto a su título y texto (`Group`) + +Para que un encabezado no quede en una página/slide y su figura en la siguiente, envuelve +los bloques de una misma idea en un `model.Group`: + +```python +blocks.append(model.Group(blocks=[ + model.Heading(text=str(name), level=2), + model.Figure(make=_figura_perezosa(...), caption="…"), + model.Markdown(text="explicación…"), +])) +``` + +El renderer **mide el grupo entero** antes de dibujar nada: si no cabe en lo que queda de +página/slide pero cabe en una entera, lo mueve **completo** a la siguiente; y **encoge la +figura** (vía `height_in`) lo justo para que el título + texto + figura quepan juntos. Si +el grupo es más alto que una página entera, empieza en una nueva y fluye (degradación +honesta, nunca corta). Ejemplo real implementado: `num_distr` envuelve cada columna +(heading + figura histograma/boxplot + nota) en un `Group`. + +Recomendado para `agregacion` y cualquier capítulo donde una figura deba ir pegada a su +título/explicación. Coste: si un capítulo inspecciona `chapter.blocks` en sus tests, ahora +encontrará `Group`s — aplana con un helper recursivo (ver `num_distr_test.py::_flatten`). + +### 11.3 Zebra striping en tablas (automático) + +Todo `DataTable` se renderiza con **filas pares sombreadas** (gris muy suave `#f6f8fa`) y +cabecera con su fondo propio. Es automático en PDF y PPTX; el patrón se mantiene coherente +cuando una tabla larga se parte y repite cabecera (el índice de fila es lógico, no por +página). No hay nada que hacer en los capítulos. + +--- + ## 10. Integración futura con `profile_table` (siguiente fase) `profile_table(emit_pdf=True)` usa hoy `render_eda_pdf` (intacto). En la siguiente fase diff --git a/python/functions/datascience/__init__.py b/python/functions/datascience/__init__.py index a1e6331f..6302642f 100644 --- a/python/functions/datascience/__init__.py +++ b/python/functions/datascience/__init__.py @@ -68,11 +68,13 @@ from .extract_timeseries_raw import extract_timeseries_raw from .build_eda_render_ctx import build_eda_render_ctx from .profile_datetime import profile_datetime from .resample_timeseries import resample_timeseries +from .add_pdf_internal_links import add_pdf_internal_links __all__ = [ "detect_time_column", "extract_timeseries_raw", "build_eda_render_ctx", + "add_pdf_internal_links", "profile_datetime", "resample_timeseries", "render_automatic_eda_pdf", diff --git a/python/functions/datascience/add_pdf_internal_links.md b/python/functions/datascience/add_pdf_internal_links.md new file mode 100644 index 00000000..c1a3873f --- /dev/null +++ b/python/functions/datascience/add_pdf_internal_links.md @@ -0,0 +1,85 @@ +--- +name: add_pdf_internal_links +kind: function +lang: py +domain: datascience +version: "1.0.0" +purity: impure +signature: "def add_pdf_internal_links(pdf_path: str, links: list) -> dict" +description: "Postprocesa un PDF YA escrito insertando link annotations internos de tipo GOTO ('ir a') con PyMuPDF (import fitz). Pensado para PDFs generados por matplotlib PdfPages, que NO soporta hyperlinks internos: tras escribir el PDF se reabre y, por cada entrada de `links`, se añade una anotacion clicable desde un rectangulo de una pagina origen (src_page + src_rect en puntos top-left) hasta un punto de una pagina destino (dst_page + dst_point). Caso de uso tipico del grupo eda: hacer clicables los terminos de un AutomaticEDA que apuntan a su entrada en el glosario al final del documento. Estilo dict-no-throw: NUNCA lanza; valida cada link y SALTA (n_skipped++) los malformados o fuera de rango en vez de fallar. Guarda de forma segura escribiendo a un temporal en el mismo directorio y haciendo os.replace atomico (evita corromper el original). Devuelve {status:ok,n_links,n_skipped} o {status:error,error}; si pymupdf no esta disponible o el archivo no existe devuelve status error." +tags: [eda, datascience, pdf, links, glossary, pymupdf, fitz, postprocess, python] +uses_functions: [] +uses_types: [] +returns: [] +returns_optional: false +error_type: "error_go_core" +imports: [] +params: + - name: pdf_path + desc: "ruta al PDF existente (str no vacio). Se reescribe IN SITU (in-place) tras añadir los links: se guarda a un temporal `..tmp_links` en el mismo directorio y se reemplaza atomicamente con os.replace. Si no es str o no existe el archivo -> {status:error}." + - name: links + desc: "lista de dicts, uno por link a insertar. Cada dict: src_page (int 0-based de la pagina origen), src_rect ([x0,y0,x1,y1] del rectangulo clicable en PUNTOS PDF 1/72\" con origen ARRIBA-IZQUIERDA), dst_page (int 0-based de la pagina destino), dst_point ([x,y] punto destino, mismos puntos top-left). Las entradas que no son dict, con page fuera de rango [0,page_count), src_rect que no tenga 4 numeros o dst_point que no tenga 2 numeros se SALTAN (n_skipped++), no lanzan. None se trata como lista vacia." +output: "dict (NUNCA lanza): en exito {\"status\":\"ok\",\"n_links\":int,\"n_skipped\":int} con n_links = anotaciones GOTO insertadas y n_skipped = entradas invalidas saltadas. En fallo {\"status\":\"error\",\"error\":str}: pymupdf no disponible, pdf_path no es str / no existe, links no es lista, o cualquier excepcion global (el PDF original queda intacto porque el replace solo ocurre tras un save correcto)." +tested: true +tests: ["test_add_goto_link_basico", "test_links_invalidos_se_saltan", "test_archivo_inexistente_devuelve_error"] +test_file_path: "python/functions/datascience/add_pdf_internal_links_test.py" +file_path: "python/functions/datascience/add_pdf_internal_links.py" +--- + +## Ejemplo + +```python +import sys, os +sys.path.insert(0, os.path.join("python", "functions")) +from datascience import add_pdf_internal_links + +# Tienes un PDF ya escrito por matplotlib PdfPages (sin hyperlinks internos). +# Quieres que el texto "Margen bruto" de la pagina 0 (rectangulo en puntos +# top-left) salte a su entrada del glosario en la ultima pagina (indice 7). +res = add_pdf_internal_links( + "reports/eda.pdf", + [ + {"src_page": 0, "src_rect": [72, 120, 180, 134], "dst_page": 7, "dst_point": [72, 200]}, + {"src_page": 0, "src_rect": [72, 140, 180, 154], "dst_page": 7, "dst_point": [72, 260]}, + ], +) +# res == {"status": "ok", "n_links": 2, "n_skipped": 0} +``` + +## Cuando usarla + +Justo DESPUES de escribir un PDF con matplotlib `PdfPages` (o cualquier motor +que no genere hyperlinks internos) cuando necesitas que ciertos terminos o +referencias sean clicables y salten a otra pagina del mismo documento — el caso +canonico es enlazar los terminos de un AutomaticEDA con su entrada de glosario +al final. Es un paso de postproceso: primero generas el PDF y calculas en que +rectangulo quedo cada termino (en puntos PDF), luego pasas esa lista a esta +funcion para inyectar las anotaciones GOTO. + +## Gotchas + +- **Impura — reescribe el archivo IN SITU.** El PDF en `pdf_path` se reemplaza + por la version con los links. El guardado es seguro: escribe a un temporal + `..tmp_links` en el MISMO directorio y hace `os.replace` atomico tras + cerrar el documento, asi un fallo a mitad no corrompe el original. Aun asi, + conserva una copia si el PDF es valioso. +- **Sistema de coordenadas: puntos top-left, igual que matplotlib.** PyMuPDF y + matplotlib (PdfPages) usan ambos PUNTOS PDF (1/72") con el origen ARRIBA- + IZQUIERDA, asi que los rectangulos/puntos COINCIDEN: el `src_rect` que calcules + con la geometria de la figura matplotlib se pasa tal cual, sin invertir el eje + Y. (Ojo: el espacio de datos de matplotlib SI tiene el origen abajo; lo que + coincide es el espacio de la PAGINA en puntos.) +- **Indices de pagina 0-based.** `src_page` / `dst_page` son indices base 0 + (la primera pagina es 0). Fuera del rango `[0, page_count)` el link se SALTA + (cuenta en `n_skipped`), no lanza. +- **dict-no-throw, validacion por-link.** Las entradas malformadas (no dict, + page fuera de rango, `src_rect` sin 4 numeros, `dst_point` sin 2 numeros) se + saltan individualmente e incrementan `n_skipped`; el resto de links validos se + insertan igual. La funcion solo devuelve `{status:error}` ante fallos globales + (pymupdf ausente, archivo inexistente, `links` no es lista). +- **`error_type: error_go_core` es metadata del registry, no comportamiento.** + Toda funcion impura debe declararlo y el indexer lo exige, pero el codigo NUNCA + lanza esa excepcion: degrada al dict de estado. +- **Requiere PyMuPDF (`import fitz`).** Si no esta instalado devuelve + `{"status":"error","error":"pymupdf no disponible: ..."}`. En el registry el + venv `python/.venv` ya lo trae. diff --git a/python/functions/datascience/add_pdf_internal_links.py b/python/functions/datascience/add_pdf_internal_links.py new file mode 100644 index 00000000..7084e778 --- /dev/null +++ b/python/functions/datascience/add_pdf_internal_links.py @@ -0,0 +1,132 @@ +"""Postprocesa un PDF existente insertando link annotations internos (GOTO). + +Motor: PyMuPDF (``import fitz``). Pensado para PDFs generados por matplotlib +``PdfPages``, que no soporta hyperlinks internos: tras escribir el PDF, esta +funcion lo reabre y le añade anotaciones "ir a" (GOTO) desde un rectangulo de +una pagina origen hasta un punto de una pagina destino. Util para hacer +clicables terminos que apuntan a su entrada en un glosario al final del +documento. + +Estilo dict-no-throw del grupo `eda`: NUNCA lanza; devuelve un dict de estado. +""" + +import os + + +def add_pdf_internal_links(pdf_path: str, links: list) -> dict: + """Añade link annotations internos (GOTO) a un PDF ya escrito. + + Postprocesa un PDF (p.ej. generado por matplotlib PdfPages, que NO soporta + hyperlinks internos) insertando, por cada entrada de ``links``, una + anotacion de tipo "ir a" desde un rectangulo de una pagina origen hasta un + punto de una pagina destino. Sirve para hacer clicables terminos que apuntan + a su entrada en un glosario al final del documento. + + Args: + pdf_path: ruta al PDF existente (se reescribe in situ). + links: lista de dicts, cada uno: + { + "src_page": int, # indice 0-based de la pagina origen + "src_rect": [x0,y0,x1,y1], # rectangulo clicable, en PUNTOS PDF + # (1/72") con origen ARRIBA-IZQUIERDA + "dst_page": int, # indice 0-based de la pagina destino + "dst_point": [x, y], # punto destino, mismos puntos top-left + } + + Returns: + dict (NUNCA lanza): {"status":"ok","n_links":int,"n_skipped":int} + o {"status":"error","error":str}. Si pymupdf no esta disponible o el + archivo no existe -> {"status":"error", ...}. + """ + try: + try: + import fitz # PyMuPDF + except Exception as exc: # ImportError u otro fallo de carga + return {"status": "error", "error": f"pymupdf no disponible: {exc}"} + + if not isinstance(pdf_path, str) or not pdf_path: + return {"status": "error", "error": "pdf_path debe ser una ruta no vacia"} + if not os.path.isfile(pdf_path): + return {"status": "error", "error": f"el archivo no existe: {pdf_path}"} + + if links is None: + links = [] + if not isinstance(links, (list, tuple)): + return {"status": "error", "error": "links debe ser una lista de dicts"} + + doc = fitz.open(pdf_path) + try: + n_pages = doc.page_count + n_ok = 0 + n_skipped = 0 + + for link in links: + if not isinstance(link, dict): + n_skipped += 1 + continue + + src_page = link.get("src_page") + dst_page = link.get("dst_page") + src_rect = link.get("src_rect") + dst_point = link.get("dst_point") + + # src_page / dst_page: enteros 0-based en rango. + if not _is_int(src_page) or not _is_int(dst_page): + n_skipped += 1 + continue + if not (0 <= src_page < n_pages) or not (0 <= dst_page < n_pages): + n_skipped += 1 + continue + + # src_rect: 4 numeros. + if not _is_num_seq(src_rect, 4): + n_skipped += 1 + continue + # dst_point: 2 numeros. + if not _is_num_seq(dst_point, 2): + n_skipped += 1 + continue + + try: + doc[int(src_page)].insert_link( + { + "kind": fitz.LINK_GOTO, + "from": fitz.Rect(*[float(v) for v in src_rect]), + "page": int(dst_page), + "to": fitz.Point(*[float(v) for v in dst_point]), + } + ) + n_ok += 1 + except Exception: + n_skipped += 1 + continue + + # Guardado seguro: escribir a temporal en el mismo directorio y + # reemplazar atomicamente (evita corromper el PDF original). + directory = os.path.dirname(os.path.abspath(pdf_path)) or "." + base = os.path.basename(pdf_path) + tmp_path = os.path.join(directory, f".{base}.tmp_links") + doc.save(tmp_path) + finally: + doc.close() + + os.replace(tmp_path, pdf_path) + + return {"status": "ok", "n_links": n_ok, "n_skipped": n_skipped} + except Exception as exc: # degrada cualquier fallo a dict de error + return {"status": "error", "error": str(exc)} + + +def _is_int(value) -> bool: + """True si value es un entero (no bool).""" + return isinstance(value, int) and not isinstance(value, bool) + + +def _is_num_seq(value, length: int) -> bool: + """True si value es una secuencia de `length` numeros (int/float, no bool).""" + if not isinstance(value, (list, tuple)) or len(value) != length: + return False + for v in value: + if isinstance(v, bool) or not isinstance(v, (int, float)): + return False + return True diff --git a/python/functions/datascience/add_pdf_internal_links_test.py b/python/functions/datascience/add_pdf_internal_links_test.py new file mode 100644 index 00000000..79baabb2 --- /dev/null +++ b/python/functions/datascience/add_pdf_internal_links_test.py @@ -0,0 +1,77 @@ +"""Tests para add_pdf_internal_links.""" + +import os +import sys + +import pytest + +sys.path.insert(0, os.path.dirname(__file__)) + +from add_pdf_internal_links import add_pdf_internal_links + + +def test_add_goto_link_basico(tmp_path): + """Golden: un PDF de 2 paginas recibe un link GOTO de la pag 0 a la pag 1.""" + fitz = pytest.importorskip("fitz") + + # 1) PDF temporal de 2 paginas A5 (~419x595 puntos). + pdf = str(tmp_path / "doc.pdf") + doc = fitz.open() + doc.new_page(width=419, height=595) + doc.new_page(width=419, height=595) + doc.save(pdf) + doc.close() + + # 2) Insertar un link interno desde la pag 0 hacia la pag 1. + res = add_pdf_internal_links( + pdf, + [{"src_page": 0, "src_rect": [50, 50, 200, 70], "dst_page": 1, "dst_point": [40, 40]}], + ) + assert res["status"] == "ok" + assert res["n_links"] == 1 + assert res["n_skipped"] == 0 + + # 3) Reabrir y verificar que la pag 0 tiene un link GOTO a la pag 1. + doc = fitz.open(pdf) + try: + links = doc[0].get_links() + goto = [l for l in links if l.get("kind") == fitz.LINK_GOTO and l.get("page") == 1] + assert len(goto) >= 1 + finally: + doc.close() + + +def test_links_invalidos_se_saltan(tmp_path): + """Edge: entradas malformadas o fuera de rango incrementan n_skipped, no lanzan.""" + fitz = pytest.importorskip("fitz") + + pdf = str(tmp_path / "doc.pdf") + doc = fitz.open() + doc.new_page(width=419, height=595) + doc.new_page(width=419, height=595) + doc.save(pdf) + doc.close() + + res = add_pdf_internal_links( + pdf, + [ + # valido + {"src_page": 0, "src_rect": [10, 10, 90, 30], "dst_page": 1, "dst_point": [20, 20]}, + # dst_page fuera de rango + {"src_page": 0, "src_rect": [10, 40, 90, 60], "dst_page": 9, "dst_point": [20, 20]}, + # src_rect con 3 numeros + {"src_page": 0, "src_rect": [10, 70, 90], "dst_page": 1, "dst_point": [20, 20]}, + # no es dict + "no-soy-un-dict", + ], + ) + assert res["status"] == "ok" + assert res["n_links"] == 1 + assert res["n_skipped"] == 3 + + +def test_archivo_inexistente_devuelve_error(): + """Error path: pdf_path inexistente -> status error sin lanzar.""" + res = add_pdf_internal_links("/ruta/que/no/existe_xyz.pdf", []) + assert res["status"] == "error" + assert "error" in res diff --git a/python/functions/datascience/automatic_eda/__init__.py b/python/functions/datascience/automatic_eda/__init__.py index 95d6f374..f9a6f2e3 100644 --- a/python/functions/datascience/automatic_eda/__init__.py +++ b/python/functions/datascience/automatic_eda/__init__.py @@ -21,6 +21,9 @@ from .model import ( # noqa: F401 Chapter, DataTable, Figure, + GlossaryCollector, + GlossaryEntry, + Group, Heading, Image, KVTable, @@ -45,6 +48,9 @@ __all__ = [ "Image", "Caption", "Note", + "Group", + "GlossaryEntry", + "GlossaryCollector", "Chapter", "as_blocks", "as_chapters", diff --git a/python/functions/datascience/automatic_eda/chapters/cat_distr.py b/python/functions/datascience/automatic_eda/chapters/cat_distr.py index c593a6b7..6421a574 100644 --- a/python/functions/datascience/automatic_eda/chapters/cat_distr.py +++ b/python/functions/datascience/automatic_eda/chapters/cat_distr.py @@ -33,10 +33,23 @@ import math from .. import model -CHAPTER_VERSION = "1.0.0" +CHAPTER_VERSION = "1.1.0" CHAPTER_ID = "cat_distr" CHAPTER_TITLE = "Distribuciones categóricas" +# Glossary term this chapter explains. Registered in the shared collector and +# marked clickable on its first appearance (end-to-end glossary example — +# mejora 6). Other chapters hook their own terms the same way (see the contract). +_TERM_ENTROPIA_KEY = "entropia" +_TERM_ENTROPIA_LABEL = "Entropía (de Shannon)" +_TERM_ENTROPIA_DEF = ( + "Medida, en bits, de cómo de repartidos están los valores de una columna " + "categórica. Vale 0 cuando una sola categoría concentra todas las filas " + "(máxima previsibilidad) y alcanza su máximo, log2(k) para k categorías " + "distintas, cuando todas aparecen por igual (máxima diversidad). La entropía " + "normalizada (entropía dividida por su máximo) la lleva al rango 0–1 para " + "comparar columnas con distinto número de categorías.") + # Cap the number of categorical columns rendered to keep the document bounded; # the rest are summarized in a closing note (no silent truncation). MAX_COLS = 40 @@ -337,10 +350,14 @@ def _topk_table(cat: dict): note=note) -def _intro_blocks(n_rows): +def _intro_blocks(n_rows, mark_term: bool = False): total = _fmt_int(n_rows) + # Mark the first appearance of the term as a clickable glossary jump when the + # term was registered (mark_term). The visible text is identical either way. + entropia = ("[[term:entropia]]**entropía de Shannon**[[/term]]" if mark_term + else "**entropía de Shannon**") text = ( - "La **entropía de Shannon** mide cómo de repartidos están los valores de " + f"La {entropia} mide cómo de repartidos están los valores de " "una columna categórica, en bits. Vale 0 cuando una sola categoría " "concentra todas las filas (máxima previsibilidad) y alcanza su máximo, " "log2(k) para k categorías distintas, cuando todas aparecen por igual " @@ -370,7 +387,15 @@ def build_cat_distr(profile: dict, ctx: dict): return None n_rows = profile.get("n_rows") - blocks = list(_intro_blocks(n_rows)) + # Register "entropía" in the shared glossary collector (if present) and mark + # its first appearance clickable. End-to-end glossary example (mejora 6). + glossary = ctx.get("glossary") + mark_term = False + if isinstance(glossary, model.GlossaryCollector): + glossary.add(_TERM_ENTROPIA_KEY, _TERM_ENTROPIA_LABEL, + _TERM_ENTROPIA_DEF) + mark_term = True + blocks = list(_intro_blocks(n_rows, mark_term=mark_term)) rendered = cat_cols[:MAX_COLS] for col in rendered: diff --git a/python/functions/datascience/automatic_eda/chapters/glosario.py b/python/functions/datascience/automatic_eda/chapters/glosario.py new file mode 100644 index 00000000..fe7098fc --- /dev/null +++ b/python/functions/datascience/automatic_eda/chapters/glosario.py @@ -0,0 +1,47 @@ +"""Glossary chapter (GLOSARIO) — always the last chapter, clickable terms. + +Renders one entry per glossary term that the other chapters registered during +the document build through ``ctx['glossary'].add(key, label, definition)`` (see +``GlossaryCollector`` in ``model.py``). Each entry is a clickable destination: +every in-text appearance a chapter marked with ``[[term:key]]texto[[/term]]`` +becomes a real jump to its entry here — PDF link annotations (PyMuPDF) and PPTX +native slide jumps, both wired by the renderers. + +Returns ``None`` when no term was registered (there is nothing to show), so the +chapter simply disappears from documents that did not mark any term. + +Contract: build_(profile, ctx) -> Chapter | None ; CHAPTER_VERSION = "x.y.z". +""" + +from __future__ import annotations + +from .. import model + +CHAPTER_VERSION = "1.0.0" +CHAPTER_ID = "glosario" +CHAPTER_TITLE = "Glosario" + + +def build_glosario(profile: dict, ctx: dict): + """Build the glossary Chapter from the shared collector, or None if empty.""" + ctx = ctx or {} + glossary = ctx.get("glossary") + if not isinstance(glossary, model.GlossaryCollector) or not glossary: + return None + + blocks = [ + model.Heading(text="Glosario de términos", level=1), + model.Markdown(text=( + "Definición de los términos técnicos que aparecen en el informe. " + "Cada término va resaltado en el texto y, al pulsarlo, salta a su " + "definición en esta sección.")), + ] + # One clickable destination per term, alphabetically by visible label. + for term in glossary.terms(by="label"): + blocks.append(model.GlossaryEntry( + key=model._safe_str(term.get("key")), + label=model._safe_str(term.get("label")), + definition=model._safe_str(term.get("definition")))) + + return model.Chapter(id=CHAPTER_ID, title=CHAPTER_TITLE, + version=CHAPTER_VERSION, blocks=blocks) diff --git a/python/functions/datascience/automatic_eda/chapters/num_distr.py b/python/functions/datascience/automatic_eda/chapters/num_distr.py index 6c105dc6..67a47779 100644 --- a/python/functions/datascience/automatic_eda/chapters/num_distr.py +++ b/python/functions/datascience/automatic_eda/chapters/num_distr.py @@ -34,7 +34,7 @@ try: except Exception: # noqa: BLE001 — keep the chapter importable no matter what. build_boxplot_stats = None # type: ignore[assignment] -CHAPTER_VERSION = "1.0.0" +CHAPTER_VERSION = "1.1.0" CHAPTER_ID = "num_distr" CHAPTER_TITLE = "Distribuciones numéricas" @@ -278,12 +278,17 @@ def build_num_distr(profile: dict, ctx: dict): box = build_boxplot_stats(numeric) or {} except Exception: # noqa: BLE001 — degrade, never raise. box = {} - blocks.append(model.Heading(text=str(name), level=2)) - blocks.append(model.Figure( - make=_figure_maker(name, numeric, box), - caption=f"Distribución de «{name}» — histograma (media/mediana/±σ) " - f"y boxplot.")) - blocks.append(model.Markdown(text=_stats_note(name, numeric, box))) + # Keep the column heading, its figure and its stats note together on the + # same page/slide (mejora 3 — keep-together): the renderers measure the + # whole Group and move it whole when it would not fit. + blocks.append(model.Group(blocks=[ + model.Heading(text=str(name), level=2), + model.Figure( + make=_figure_maker(name, numeric, box), + caption=f"Distribución de «{name}» — histograma " + f"(media/mediana/±σ) y boxplot."), + model.Markdown(text=_stats_note(name, numeric, box)), + ])) return model.Chapter(id=CHAPTER_ID, title=CHAPTER_TITLE, version=CHAPTER_VERSION, blocks=blocks) diff --git a/python/functions/datascience/automatic_eda/chapters/num_distr_test.py b/python/functions/datascience/automatic_eda/chapters/num_distr_test.py index a9b459ed..71793ad1 100644 --- a/python/functions/datascience/automatic_eda/chapters/num_distr_test.py +++ b/python/functions/datascience/automatic_eda/chapters/num_distr_test.py @@ -65,19 +65,33 @@ def _pdf_text(path: str) -> str: return re.sub(r"\s+", " ", txt) +def _flatten(blocks): + """Expand keep-together Groups so the per-column heading/figure/markdown are + inspectable as a flat block list (the chapter wraps each column in a Group).""" + out = [] + for b in blocks: + if getattr(b, "kind", "") == "group": + out.extend(_flatten(getattr(b, "blocks", []) or [])) + else: + out.append(b) + return out + + def test_golden_chapter_estructura_y_bloques(): ch = build_num_distr(_profile(n_numeric=2), {}) assert ch is not None assert ch.id == "num_distr" assert ch.version == CHAPTER_VERSION - kinds = [b.kind for b in ch.blocks] + # Per-column blocks are wrapped in keep-together Groups: flatten to inspect. + flat = _flatten(ch.blocks) + kinds = [b.kind for b in flat] # Heading + intro Markdown, then per column: Heading + Figure + Markdown. assert kinds[0] == "heading" assert kinds[1] == "markdown" assert kinds.count("figure") == 2 # one figure per numeric column. assert kinds.count("heading") == 1 + 2 # chapter title + one per column. # Each figure has a lazy maker that produces a real matplotlib figure. - figs = [b for b in ch.blocks if b.kind == "figure"] + figs = [b for b in flat if b.kind == "figure"] fig = figs[0].make() assert fig is not None # Two stacked axes: histogram + boxplot share the figure. @@ -90,7 +104,8 @@ def test_golden_media_mediana_sigma_y_boxplot_presentes(): # The intro documents the three reference lines and the Tukey boxplot; the # per-column note carries the actual mean/median/σ numbers and the shape. ch = build_num_distr(_profile(n_numeric=1, extra_categorical=False), {}) - md_texts = " ".join(b.text for b in ch.blocks if b.kind == "markdown") + md_texts = " ".join(b.text for b in _flatten(ch.blocks) + if b.kind == "markdown") assert "media" in md_texts and "mediana" in md_texts assert "±1σ" in md_texts or "σ" in md_texts assert "boxplot" in md_texts.lower() @@ -126,7 +141,8 @@ def test_anti_corte_muchas_columnas_pdf_y_pptx(): # 8 numeric columns + long note text: nothing may be cut. Every column # heading must survive in both the PDF text and the PPTX deck. ch = build_num_distr(_profile(n_numeric=8), {}) - names = [b.text for b in ch.blocks if b.kind == "heading" and b.level == 2] + names = [b.text for b in _flatten(ch.blocks) + if b.kind == "heading" and b.level == 2] assert len(names) == 8 with tempfile.TemporaryDirectory() as d: pdf = os.path.join(d, "num.pdf") diff --git a/python/functions/datascience/automatic_eda/chapters/portada.py b/python/functions/datascience/automatic_eda/chapters/portada.py index 3582d981..c1bb43ab 100644 --- a/python/functions/datascience/automatic_eda/chapters/portada.py +++ b/python/functions/datascience/automatic_eda/chapters/portada.py @@ -17,7 +17,7 @@ from datetime import datetime, timezone from .. import model -CHAPTER_VERSION = "1.0.0" +CHAPTER_VERSION = "1.1.0" CHAPTER_ID = "portada" CHAPTER_TITLE = "Portada" @@ -67,6 +67,53 @@ def _fmt_int(v) -> str: return str(v) +def _fmt_pct(value) -> str: + """Format a percentage that may arrive as a 0–1 fraction or a 0–100 number.""" + if value is None: + return "—" + try: + v = float(value) + except (TypeError, ValueError): + return str(value) + if 0 < v <= 1.0: + v *= 100.0 + return f"{v:.1f}%" + + +def _summary_blocks(summary) -> list: + """Mini-summary of the rest of the analysis, shown on the cover (mejora 5). + + The cover is built AFTER the body (``build_document`` passes the aggregated + ``ctx['document_summary']``), so it can reflect what the analysis found: + shape, column types, quality flags and which chapters were included. Returns + an empty list when there is no summary (the cover degrades to its metadata + table only).""" + if not isinstance(summary, dict) or not summary: + return [] + rows = [] + n_num = summary.get("n_numeric") + n_cat = summary.get("n_categorical") + if n_num is not None or n_cat is not None: + rows.append(("Columnas numéricas / categóricas", + f"{_fmt_int(n_num)} / {_fmt_int(n_cat)}")) + if summary.get("duplicate_pct") is not None: + rows.append(("Filas duplicadas", _fmt_pct(summary.get("duplicate_pct")))) + if summary.get("null_cell_pct") is not None: + rows.append(("Celdas nulas", _fmt_pct(summary.get("null_cell_pct")))) + titles = summary.get("chapter_titles") or [] + if titles: + rows.append(("Capítulos del informe", _fmt_int(len(titles)))) + + blocks = [model.Heading(text="Resumen del análisis", level=2)] + if rows: + blocks.append(model.KVTable(rows=rows)) + if titles: + bullets = "\n".join(f"- {model._safe_str(t)}" for t in titles) + blocks.append(model.Markdown( + text="Este informe incluye los siguientes capítulos:\n" + bullets)) + return blocks + + def _fmt_date_eu(value) -> str: """Format a date/ISO string as European DD/MM/AAAA HH:mm (UI convention). @@ -152,5 +199,8 @@ def build_portada(profile: dict, ctx: dict): model.Markdown(text=str(granularity)), ] + # Mini-summary of the rest of the analysis (built last, shown on the cover). + blocks.extend(_summary_blocks(ctx.get("document_summary"))) + return model.Chapter(id=CHAPTER_ID, title=CHAPTER_TITLE, version=CHAPTER_VERSION, blocks=blocks) diff --git a/python/functions/datascience/automatic_eda/chapters_registry.py b/python/functions/datascience/automatic_eda/chapters_registry.py index 6dd73237..d4dc329d 100644 --- a/python/functions/datascience/automatic_eda/chapters_registry.py +++ b/python/functions/datascience/automatic_eda/chapters_registry.py @@ -26,7 +26,7 @@ from . import model # placeholders other agents will fill by creating chapters/.py — they will # appear in this exact position automatically once their module exists. CHAPTER_ORDER = [ - "portada", # cover + "portada", # cover — BUILT LAST, PLACED FIRST (see build_document). "overview", # df.head + columns/types/nulls/examples + describe "analisis_llm", # LLM interpretation — sits next to overview (user request) "num_distr", # numeric distributions @@ -37,8 +37,15 @@ CHAPTER_ORDER = [ "timeseries", # time-series analysis "geospatial", # geospatial "agregacion", # aggregations / pivots + "glosario", # glossary — ALWAYS LAST; clickable term destinations. ] +# Chapters whose position is special-cased by build_document: portada is built +# last (so it can summarize the rest) but placed first; glosario is built and +# placed last (it reads the terms every other chapter registered). +_PORTADA = "portada" +_GLOSARIO = "glosario" + def build_chapter(chapter_id: str, profile: dict, ctx: dict): """Build a single chapter by id, or None if absent/not-applicable/error. @@ -75,15 +82,72 @@ def build_document(profile: dict, ctx: dict = None) -> list: list[Chapter] in canonical order, containing only the chapters that are implemented and applicable. Never raises. """ - if profile is None: - profile = {} if not isinstance(profile, dict): profile = {} - if ctx is None: - ctx = {} - chapters = [] + # Copy ctx so the shared collector / summary we add do not leak to the caller. + ctx = dict(ctx) if isinstance(ctx, dict) else {} + + # A single glossary collector is shared by every chapter via ctx['glossary']. + # Chapters call ctx['glossary'].add(key, label, definition) and mark in-text + # appearances with [[term:key]]…[[/term]]; the glosario chapter renders the + # registered terms and the renderers wire the clickable links. + glossary = ctx.get("glossary") + if not isinstance(glossary, model.GlossaryCollector): + glossary = model.GlossaryCollector() + ctx["glossary"] = glossary + + # 1) Body: every chapter except portada (built last) and glosario (placed + # last), in canonical order. This also fills the glossary collector. + body = [] for cid in CHAPTER_ORDER: + if cid in (_PORTADA, _GLOSARIO): + continue ch = build_chapter(cid, profile, ctx) if ch is not None and ch.blocks: - chapters.append(ch) + body.append(ch) + + # 2) Aggregated summary of the rest, for the cover (user decision: the cover + # is BUILT after the body so it can reflect what the analysis found). + ctx["document_summary"] = _summarize_document(profile, body) + + # 3) Build the cover last, place it FIRST. + portada = build_chapter(_PORTADA, profile, ctx) + # 4) Build the glossary last (reads the terms the body registered), place LAST. + glosario = build_chapter(_GLOSARIO, profile, ctx) + + chapters = [] + if portada is not None and portada.blocks: + chapters.append(portada) + chapters.extend(body) + if glosario is not None and glosario.blocks: + chapters.append(glosario) return chapters + + +def _summarize_document(profile: dict, body: list) -> dict: + """Aggregate a tiny findings summary of the body for the cover. Never raises. + + Returns a dict with dataset shape, quality, column-type counts and the list + of chapters actually included — enough for the cover to show a mini-summary + of the analysis without re-deriving anything.""" + try: + cols = profile.get("columns") or [] + n_num = sum(1 for c in cols if isinstance(c, dict) + and c.get("inferred_type") == "numeric") + n_cat = sum(1 for c in cols if isinstance(c, dict) + and isinstance(c.get("categorical"), dict) + and c.get("categorical", {}).get("top") + and c.get("inferred_type") != "numeric") + return { + "n_chapters": len(body), + "chapter_titles": [getattr(c, "title", "") for c in body], + "n_rows": profile.get("n_rows"), + "n_cols": profile.get("n_cols"), + "quality_score": profile.get("quality_score"), + "n_numeric": n_num, + "n_categorical": n_cat, + "duplicate_pct": profile.get("duplicate_pct"), + "null_cell_pct": profile.get("null_cell_pct"), + } + except Exception: # noqa: BLE001 — the summary is best-effort. + return {"n_chapters": len(body) if isinstance(body, list) else 0} diff --git a/python/functions/datascience/automatic_eda/model.py b/python/functions/datascience/automatic_eda/model.py index 8a5c488d..53c41377 100644 --- a/python/functions/datascience/automatic_eda/model.py +++ b/python/functions/datascience/automatic_eda/model.py @@ -128,6 +128,39 @@ class Note: kind: str = field(default="note", init=False) +@dataclass +class Group: + """A keep-together unit: its blocks render on the SAME page/slide. + + Renderers measure the whole group first; if it does not fit in the remaining + space they move it *whole* to the next page (PDF) or slide (PPTX) before + drawing anything — so a heading never gets stranded apart from the figure and + text it introduces. If the group is taller than a full page even on its own, + it starts on a fresh page and flows (honest degradation, never cut). Use it to + bind ``Heading`` + ``Markdown`` + ``Figure`` of one idea together (see the + DISTR NUM / AGREGACION chapters). + """ + + blocks: list = field(default_factory=list) + title: Optional[str] = None + kind: str = field(default="group", init=False) + + +@dataclass +class GlossaryEntry: + """One glossary term: a clickable destination at the end of the document. + + Rendered as the term ``label`` (heading) plus its ``definition`` (markdown). + The renderers register its page/slide position as the link target so every + in-text appearance of the same ``key`` becomes a real clickable jump (PDF link + annotation via PyMuPDF; PPTX internal slide jump).""" + + key: str = "" + label: str = "" + definition: str = "" + kind: str = field(default="glossary_entry", init=False) + + @dataclass class Chapter: """An ordered set of blocks with an id, a title and a generation version.""" @@ -150,13 +183,17 @@ _BLOCK_BY_KIND = { "image": Image, "caption": Caption, "note": Note, + "group": Group, + "glossary_entry": GlossaryEntry, } def as_block(obj: Any): """Coerce a value into a block dataclass. Unknown values become a Note.""" if isinstance(obj, (Heading, Markdown, KVTable, DataTable, Figure, Image, - Caption, Note)): + Caption, Note, Group, GlossaryEntry)): + if isinstance(obj, Group): + obj.blocks = as_blocks(obj.blocks) return obj if isinstance(obj, dict): kind = obj.get("kind") @@ -189,6 +226,13 @@ def as_block(obj: Any): return Caption(text=_safe_str(obj.get("text"))) if cls is Note: return Note(text=_safe_str(obj.get("text"))) + if cls is Group: + return Group(blocks=as_blocks(obj.get("blocks")), + title=obj.get("title")) + if cls is GlossaryEntry: + return GlossaryEntry(key=_safe_str(obj.get("key")), + label=_safe_str(obj.get("label")), + definition=_safe_str(obj.get("definition"))) except Exception: # noqa: BLE001 — never raise on a malformed block. return Note(text=_safe_str(obj)) return Note(text=_safe_str(obj)) @@ -246,6 +290,67 @@ def _safe_str(v: Any) -> str: return "" +# --------------------------------------------------------------------------- # +# Glossary collector — chapters register the terms they use; the glosario +# chapter renders them at the end and the renderers wire the clickable links. +# --------------------------------------------------------------------------- # +class GlossaryCollector: + """Accumulates glossary terms registered by chapters during document build. + + A single instance is created by :func:`build_document` and passed to every + chapter via ``ctx['glossary']``. A chapter calls ``add(key, label, + definition)`` to declare a term it explains (e.g. ``"entropia"`` → + "Entropía"), and marks each in-text appearance with the inline span + ``[[term:key]]texto visible[[/term]]`` (see ``text_layout.parse_inline_rich``). + The ``glosario`` chapter reads ``terms()`` to emit one :class:`GlossaryEntry` + per term; the renderers turn every marked appearance into a real click that + jumps to that entry. First registration of a key wins (idempotent); never + raises.""" + + def __init__(self): + self._terms: dict = {} + self._order: list = [] + + def add(self, key: Any, label: Any = None, definition: Any = "") -> str: + """Register a term and return its normalized key (''. if invalid).""" + try: + k = _safe_str(key).strip() + if not k: + return "" + if k not in self._terms: + self._terms[k] = { + "key": k, + "label": _safe_str(label).strip() or k, + "definition": _safe_str(definition), + } + self._order.append(k) + return k + except Exception: # noqa: BLE001 — collecting a term never breaks a build. + return "" + + def has(self, key: Any) -> bool: + return _safe_str(key).strip() in self._terms + + def get(self, key: Any) -> Optional[dict]: + return self._terms.get(_safe_str(key).strip()) + + def terms(self, by: str = "label") -> list: + """Return the registered terms as dicts. + + ``by='label'`` (default) sorts alphabetically by visible label; + ``by='order'`` keeps first-appearance order.""" + if by == "order": + return [self._terms[k] for k in self._order] + return sorted(self._terms.values(), + key=lambda t: _safe_str(t.get("label")).lower()) + + def __len__(self) -> int: + return len(self._terms) + + def __bool__(self) -> bool: + return bool(self._terms) + + # --------------------------------------------------------------------------- # # Manifest — per-chapter versions and page/slide counts for tracking. # --------------------------------------------------------------------------- # diff --git a/python/functions/datascience/automatic_eda/render_features_test.py b/python/functions/datascience/automatic_eda/render_features_test.py new file mode 100644 index 00000000..40d247ba --- /dev/null +++ b/python/functions/datascience/automatic_eda/render_features_test.py @@ -0,0 +1,354 @@ +"""Tests for the AutomaticEDA engine features added in phase 4a. + +Covers, with executable evidence, the six render-engine improvements: + +1. Bold no longer overlaps the following text in the PDF (real width measured). +2. Zebra striping on data tables (PDF Rectangle fills + PPTX cell fills). +3. Keep-together: a Group moves whole to the next page/slide (heading never gets + stranded from its figure). +4. Every PPTX figure carries a visible caption/title (fallback to the heading). +5. Cover is built last but placed first and reflects an aggregated summary. +6. Glossary is the last chapter; the term "entropía" is a real clickable link in + the PDF (PyMuPDF GOTO annotation) and in the PPTX (native slide-jump run). + +Self-contained: synthetic profiles, no DuckDB. Heavy renderer checks (fitz/pptx) +skip cleanly when the optional engine is missing. +""" + +import os +import sys + +import pytest + +_HERE = os.path.dirname(os.path.abspath(__file__)) +_FUNCTIONS = os.path.abspath(os.path.join(_HERE, "..", "..", "..")) # python/functions +if _FUNCTIONS not in sys.path: + sys.path.insert(0, _FUNCTIONS) + +import matplotlib # noqa: E402 + +matplotlib.use("Agg") +import matplotlib.colors as mcolors # noqa: E402 +import matplotlib.pyplot as plt # noqa: E402 +from matplotlib.patches import Rectangle # noqa: E402 + +from datascience.automatic_eda import model # noqa: E402 +from datascience.automatic_eda import render_pdf_impl as RP # noqa: E402 +from datascience.automatic_eda import render_pptx_impl as RX # noqa: E402 +from datascience.automatic_eda import build_document # noqa: E402 +from datascience.render_automatic_eda_pdf import render_automatic_eda_pdf # noqa: E402 +from datascience.render_automatic_eda_pptx import render_automatic_eda_pptx # noqa: E402 + + +class _FakePdf: + """Stand-in for PdfPages so the placers can call _new_page in unit tests.""" + + def savefig(self, fig): # noqa: D401 + pass + + +def _small_fig(): + fig = plt.figure(figsize=(4.0, 1.5)) + ax = fig.add_subplot(111) + ax.plot([0, 1, 2], [1, 3, 2]) + return fig + + +def _profile_with_cat_and_num(): + """A tiny profile that triggers cat_distr (→ entropía term) and num_distr.""" + return { + "table": "ventas", "n_rows": 120, "n_cols": 2, "quality_score": 91, + "duplicate_pct": 1.5, "null_cell_pct": 0.8, + "columns": [ + {"name": "region", "inferred_type": "categorical", + "categorical": { + "top": [{"value": "norte", "count": 50, "pct": 0.42}, + {"value": "sur", "count": 40, "pct": 0.33}, + {"value": "este", "count": 30, "pct": 0.25}], + "mode": "norte", "n_distinct": 3, "entropy": 1.55, + "imbalance": 0.1}}, + {"name": "importe", "inferred_type": "numeric", + "numeric": {"mean": 50.0, "median": 48.0, "std": 10.0, + "min": 10, "max": 99, "iqr": 15, + "histogram": [{"lo": 0, "hi": 50, "count": 40}, + {"lo": 50, "hi": 100, "count": 80}]}}, + ], + } + + +# --------------------------------------------------------------------------- # +# 1) Bold does not overlap the following text (PDF). +# --------------------------------------------------------------------------- # +def test_pdf_bold_span_does_not_overlap_following_text(): + fig = plt.figure(figsize=(RP._W, RP._H)) + st = RP._PdfState(_FakePdf(), "t") + st.fig = fig + st.page = 1 + # A wide bold token immediately followed by normal text on the SAME line. + rich = [[("PALABRAMUYANCHAENNEGRITA", True, None), + (" texto normal justo después", False, None)]] + RP._place_rich_lines(st, rich, RP._FS_BODY, RP._INK) + + renderer = fig.canvas.get_renderer() + boxes = sorted((t.get_window_extent(renderer) for t in fig.texts), + key=lambda b: b.x0) + assert len(boxes) == 2, "se esperaban dos spans dibujados" + # The bold span ends before the normal span starts (no overlap). 1px slack. + assert boxes[0].x1 <= boxes[1].x0 + 1.0, \ + "la negrita se solapa con el texto siguiente" + plt.close(fig) + + +# --------------------------------------------------------------------------- # +# 2) Zebra striping. +# --------------------------------------------------------------------------- # +def _facecolor_eq(artist, hexcolor) -> bool: + want = mcolors.to_rgba(hexcolor) + got = artist.get_facecolor() + return all(abs(a - b) < 0.02 for a, b in zip(got[:3], want[:3])) + + +def test_pdf_table_has_zebra_striping(): + fig = plt.figure(figsize=(RP._W, RP._H)) + st = RP._PdfState(_FakePdf(), "t") + st.fig = fig + st.page = 1 + st.chapter = model.Chapter(id="c", title="C", version="1.0.0") + dt = model.DataTable(header=["A", "B"], + rows=[["1", "x"], ["2", "y"], ["3", "z"], ["4", "w"]]) + RP._place_data_table(st, dt) + zebra = [a for a in fig.findobj(Rectangle) if _facecolor_eq(a, RP._ZEBRA)] + # 4 data rows → even rows (1-based 2 and 4) shaded = 2 zebra rectangles. + assert len(zebra) == 2, f"esperadas 2 filas zebra, hay {len(zebra)}" + plt.close(fig) + + +def test_pptx_table_has_zebra_striping(tmp_path): + pptx = pytest.importorskip("pptx") + from pptx import Presentation + from pptx.dml.color import RGBColor + + doc = [model.Chapter(id="c", title="Tabla", version="1.0.0", blocks=[ + model.DataTable(header=["A", "B"], + rows=[["1", "x"], ["2", "y"], ["3", "z"], ["4", "w"]])])] + out = str(tmp_path / "zebra.pptx") + assert render_automatic_eda_pptx(doc, out, {"write_manifest": False})["path"] + + prs = Presentation(out) + table = None + for slide in prs.slides: + for sh in slide.shapes: + if sh.has_table: + table = sh.table + break + assert table is not None, "no se encontró la tabla en el deck" + zebra = RGBColor(0xF6, 0xF8, 0xFA) + white = RGBColor(0xFF, 0xFF, 0xFF) + # Row 0 = header; data rows follow. Even data rows (table rows 2, 4) shaded. + assert table.cell(1, 0).fill.fore_color.rgb == white + assert table.cell(2, 0).fill.fore_color.rgb == zebra + assert table.cell(4, 0).fill.fore_color.rgb == zebra + + +# --------------------------------------------------------------------------- # +# 3) Keep-together (Group): heading + figure never split. +# --------------------------------------------------------------------------- # +def test_pdf_group_moves_whole_to_next_page_when_it_does_not_fit(): + fig = plt.figure(figsize=(RP._W, RP._H)) + st = RP._PdfState(_FakePdf(), "t") + st.fig = fig + st.page = 1 + st.chapter = model.Chapter(id="c", title="C", version="1.0.0") + grp = model.Group(blocks=[ + model.Heading(text="Sección con figura", level=2), + model.Figure(make=_small_fig, caption="cap"), + model.Markdown(text="Descripción breve de la figura."), + ]) + # Only ~0.4in left: the group does not fit here but fits on a fresh page. + st.y = RP._CONTENT_BOTTOM - 0.4 + page_before = st.page + RP._place_group(st, grp) + # Exactly one page break: the whole group (heading+figure+text) stays + # together on the new page — no second break inside it. + assert st.page == page_before + 1 + plt.close(st.fig) + + +def test_pdf_group_does_not_break_when_it_fits(): + fig = plt.figure(figsize=(RP._W, RP._H)) + st = RP._PdfState(_FakePdf(), "t") + st.fig = fig + st.page = 1 + st.chapter = model.Chapter(id="c", title="C", version="1.0.0") + grp = model.Group(blocks=[ + model.Heading(text="Cabe entera", level=2), + model.Figure(make=_small_fig, caption="cap"), + ]) + st.y = RP._CONTENT_TOP # empty page → fits, must not break. + page_before = st.page + RP._place_group(st, grp) + assert st.page == page_before + plt.close(st.fig) + + +def test_pptx_group_moves_whole_to_next_slide(tmp_path): + pytest.importorskip("pptx") + from pptx import Presentation + from pptx.util import Inches + + prs = Presentation() + prs.slide_width = Inches(RX._W) + prs.slide_height = Inches(RX._H) + st = RX._PptxState(prs, "t") + st.chapter = model.Chapter(id="c", title="C", version="1.0.0") + RX._new_slide(st, cont=False) + grp = model.Group(blocks=[ + model.Heading(text="Sección con figura", level=2), + model.Figure(make=_small_fig, caption="cap"), + model.Markdown(text="Descripción breve."), + ]) + st.y = RX._CONTENT_BOTTOM - 0.4 # does not fit here. + slide_before = st.slide_no + RX._place_group(st, grp) + assert st.slide_no == slide_before + 1 # one jump; group kept together. + + +# --------------------------------------------------------------------------- # +# 4) Every PPTX figure carries a visible caption/title. +# --------------------------------------------------------------------------- # +def test_pptx_figure_without_caption_gets_heading_title(tmp_path): + pytest.importorskip("pptx") + from pptx import Presentation + from pptx.enum.shapes import MSO_SHAPE_TYPE + + doc = [model.Chapter(id="c", title="Cap", version="1.0.0", blocks=[ + model.Heading(text="Mi sección gráfica", level=2), + model.Figure(make=_small_fig), # NO caption provided. + ])] + out = str(tmp_path / "cap.pptx") + assert render_automatic_eda_pptx(doc, out, {"write_manifest": False})["path"] + + prs = Presentation(out) + for slide in prs.slides: + has_pic = any(sh.shape_type == MSO_SHAPE_TYPE.PICTURE + for sh in slide.shapes) + if not has_pic: + continue + italic = [r.text for sh in slide.shapes if sh.has_text_frame + for p in sh.text_frame.paragraphs for r in p.runs + if r.font.italic and r.text.strip()] + assert italic, "la figura no lleva caption visible en su slide" + assert any("Mi sección gráfica" in t for t in italic), \ + "el caption no cayó al título de la sección" + return + pytest.fail("no se encontró ningún slide con imagen") + + +def test_pptx_no_figure_slide_is_ever_untitled(tmp_path): + """Invariant: across many figures (incl. tall ones), NO slide with an image + lacks a visible caption — the caption never spills to the next slide.""" + pytest.importorskip("pptx") + from pptx import Presentation + from pptx.enum.shapes import MSO_SHAPE_TYPE + + def _tall_fig(): + fig = plt.figure(figsize=(5.0, 4.6)) # nearly square → fills the slide. + fig.add_subplot(111).bar([1, 2, 3], [4, 5, 6]) + return fig + + blocks = [] + for i in range(6): + blocks.append(model.Heading(text=f"Gráfico {i}", level=2)) + blocks.append(model.Figure( + make=_tall_fig, + caption=("Una descripción de la figura deliberadamente larga para " + "que el caption ocupe más de una línea al envolverse en el " + f"ancho del slide — figura número {i} del bloque."))) + doc = [model.Chapter(id="c", title="Muchas figuras", version="1.0.0", + blocks=blocks)] + out = str(tmp_path / "many.pptx") + assert render_automatic_eda_pptx(doc, out, {"write_manifest": False})["path"] + + prs = Presentation(out) + missing = [] + pics = 0 + for i, slide in enumerate(prs.slides): + if not any(sh.shape_type == MSO_SHAPE_TYPE.PICTURE + for sh in slide.shapes): + continue + pics += 1 + italic = [r.text for sh in slide.shapes if sh.has_text_frame + for p in sh.text_frame.paragraphs for r in p.runs + if r.font.italic and r.text.strip()] + if not italic: + missing.append(i) + assert pics >= 6, f"esperadas >=6 figuras, hay {pics}" + assert not missing, f"slides con imagen sin caption: {missing}" + + +# --------------------------------------------------------------------------- # +# 5) Cover built last, placed first, with an aggregated summary. +# --------------------------------------------------------------------------- # +def test_cover_first_glossary_last_with_summary(): + chs = build_document(_profile_with_cat_and_num(), ctx={"dataset_name": "v"}) + ids = [c.id for c in chs] + assert ids[0] == "portada", f"la portada no es la primera: {ids}" + assert ids[-1] == "glosario", f"el glosario no es el último: {ids}" + cover = chs[0] + headings = [b.text for b in cover.blocks if b.kind == "heading"] + assert any("Resumen" in h for h in headings), \ + "la portada no incluye el resumen agregado" + # The summary reflects the body chapters (e.g. the numeric/categorical ones). + cover_text = " ".join( + b.text for b in cover.blocks if getattr(b, "kind", "") == "markdown") + assert "Distribuciones" in cover_text, \ + "el resumen de portada no menciona los capítulos del cuerpo" + + +# --------------------------------------------------------------------------- # +# 6) Glossary clickable in PDF (PyMuPDF GOTO) and PPTX (native slide jump). +# --------------------------------------------------------------------------- # +def test_pdf_glossary_term_is_clickable(tmp_path): + fitz = pytest.importorskip("fitz") + out = str(tmp_path / "glos.pdf") + res = render_automatic_eda_pdf(_profile_with_cat_and_num(), out, + {"ctx": {"dataset_name": "v"}, + "write_manifest": False}) + assert res["path"] == out and os.path.exists(out) + + doc = fitz.open(out) + goto = [(pno, l) for pno in range(doc.page_count) + for l in doc[pno].get_links() if l.get("kind") == fitz.LINK_GOTO] + doc.close() + assert goto, "no hay ningún enlace interno (entropía → glosario) en el PDF" + # Destination must be a real page in the document (the glossary page). + assert all(0 <= l.get("page", -1) for _p, l in goto) + + +def test_pptx_glossary_term_is_clickable(tmp_path): + pytest.importorskip("pptx") + from pptx import Presentation + from pptx.oxml.ns import qn + + out = str(tmp_path / "glos.pptx") + res = render_automatic_eda_pptx(_profile_with_cat_and_num(), out, + {"ctx": {"dataset_name": "v"}, + "write_manifest": False}) + assert res["path"] == out and os.path.exists(out) + + prs = Presentation(out) + found = False + for slide in prs.slides: + for sh in slide.shapes: + if not sh.has_text_frame: + continue + for p in sh.text_frame.paragraphs: + for r in p.runs: + rpr = r._r.find(qn("a:rPr")) + if rpr is None: + continue + hl = rpr.find(qn("a:hlinkClick")) + if hl is not None and \ + hl.get("action") == "ppaction://hlinksldjump": + found = True + assert found, "ningún término tiene hyperlink de salto a slide en el PPTX" diff --git a/python/functions/datascience/automatic_eda/render_pdf_impl.py b/python/functions/datascience/automatic_eda/render_pdf_impl.py index fe8702ce..ffe9a349 100644 --- a/python/functions/datascience/automatic_eda/render_pdf_impl.py +++ b/python/functions/datascience/automatic_eda/render_pdf_impl.py @@ -60,6 +60,8 @@ _FS_BODY, _FS_CELL, _FS_NOTE = 10.5, 9.0, 9.0 _GAP = 0.12 # vertical gap after a block, inches. _CELL_PAD = 0.06 # horizontal padding inside a table cell, inches. _ROW_VPAD = 0.05 # vertical padding inside a table row, inches. +_ZEBRA = "#f6f8fa" # very light grey for zebra-striped (even) table rows. +_LINK = "#2a6f97" # accent colour for clickable glossary terms. class _PdfState: @@ -73,6 +75,11 @@ class _PdfState: self.page = 0 # global page counter. self.chapter = None # current Chapter (for the footer). self.chapter_pages = 0 # pages produced for the current chapter. + self.last_heading = "" # text of the most recent heading. + # Glossary wiring (mejora 6). Pages are 0-based; rects/points are in PDF + # points (1/72") with a top-left origin — same convention as PyMuPDF. + self.term_sources = [] # [{key, page, rect:[x0,y0,x1,y1]}] + self.term_dests = {} # key -> {page, point:[x,y]} # --------------------------------------------------------------------------- # @@ -121,6 +128,35 @@ def _draw_footer(st: _PdfState) -> None: transform=st.fig.transFigure, color=_RULE, lw=0.6)) +def _text_width_in(st: _PdfState, s: str, fs: float, bold: bool) -> float: + """Real rendered width (inches) of ``s`` at ``fs`` with the given weight. + + Measured with the Agg renderer's own font metrics (the same TrueType the PDF + backend embeds), so a **bold** span advances the cursor by its ACTUAL width — + fixing the bug where bold text overlapped the following normal text because + the cursor advanced by the normal-weight average-glyph estimate. Falls back to + the deterministic character grid if the renderer is unavailable, so it never + raises. + """ + if not s: + return 0.0 + try: + from matplotlib.font_manager import FontProperties + renderer = st.fig.canvas.get_renderer() + prop = FontProperties(family="sans-serif", size=fs, + weight="bold" if bold else "normal") + w_px, _h, _d = renderer.get_text_width_height_descent(s, prop, False) + return w_px / float(st.fig.dpi) + except Exception: # noqa: BLE001 — fall back to the conservative grid metric. + return tl.avg_char_width_in(fs) * len(s) + + +def _pt_rect(x0_in: float, y_top_in: float, x1_in: float, + y_bottom_in: float) -> list: + """An inches box (top-left origin) → a PDF-points rect for PyMuPDF links.""" + return [x0_in * 72.0, y_top_in * 72.0, x1_in * 72.0, y_bottom_in * 72.0] + + def _remaining(st: _PdfState) -> float: return _CONTENT_BOTTOM - st.y @@ -138,6 +174,7 @@ def _place_heading(st: _PdfState, block) -> None: level = max(1, min(3, int(getattr(block, "level", 1) or 1))) fs = {1: _FS_H1, 2: _FS_H2, 3: _FS_H3}[level] text = tl.strip_inline_md(getattr(block, "text", "")) + st.last_heading = text or st.last_heading max_chars = tl.chars_per_line(_USABLE_W, fs) lines = tl.wrap(text, max_chars) lh = tl.line_height_in(fs, leading=1.2) @@ -171,17 +208,19 @@ def _place_text_lines(st: _PdfState, lines: list, fs: float, color: str, def _place_rich_lines(st: _PdfState, rich_lines: list, fs: float, color: str, indent: float = 0.0, prefixes=None) -> None: - """Draw pre-wrapped lines of styled segments (bold spans rendered bold). + """Draw pre-wrapped lines of styled segments (bold + clickable term spans). - Each line is ``[(text, is_bold), ...]``. Segments are placed left-to-right, - advancing x by the deterministic character grid (same metric the wrapper - used), so a bold span is rendered with ``fontweight='bold'`` without - changing the line's measured width — the no-cut guarantee is preserved. + Each line is a list of ``(text, is_bold)`` or ``(text, is_bold, term_key)`` + segments. Segments are placed left-to-right, advancing x by the segment's + REAL rendered width (measured with the renderer's font metrics for the actual + weight) — this is what stops a bold span from overlapping the following text: + the cursor no longer advances by the normal-weight estimate. A segment with a + ``term_key`` is drawn in the accent colour and its rectangle is recorded in + ``st.term_sources`` so it becomes a clickable jump to the glossary entry. ``prefixes`` is an optional ``(first_line, other_lines)`` pair (e.g. a bullet) drawn before the segments. """ lh = tl.line_height_in(fs) - cw = tl.avg_char_width_in(fs) for idx, segs in enumerate(rich_lines): _ensure_space(st, lh) x = _ML + indent @@ -190,14 +229,23 @@ def _place_rich_lines(st: _PdfState, rich_lines: list, fs: float, color: str, if prefix: st.fig.text(_xf(x), _yf(st.y), prefix, fontsize=fs, color=color, ha="left", va="top") - x += cw * len(prefix) - for seg_text, is_bold in segs: + x += _text_width_in(st, prefix, fs, False) + for seg in segs: + if len(seg) == 3: + seg_text, is_bold, term = seg + else: + seg_text, is_bold, term = seg[0], seg[1], None if seg_text == "": continue - st.fig.text(_xf(x), _yf(st.y), seg_text, fontsize=fs, color=color, - ha="left", va="top", + w = _text_width_in(st, seg_text, fs, bool(is_bold)) + st.fig.text(_xf(x), _yf(st.y), seg_text, fontsize=fs, + color=(_LINK if term else color), ha="left", va="top", fontweight="bold" if is_bold else "normal") - x += cw * len(seg_text) + if term: + st.term_sources.append({ + "key": term, "page": st.page - 1, + "rect": _pt_rect(x, st.y, x + w, st.y + lh)}) + x += w st.y += lh @@ -242,7 +290,7 @@ def _place_markdown(st: _PdfState, block) -> None: if stripped.startswith("- ") or stripped.startswith("* "): content = stripped[2:] # keep inline markers for bold rendering. bullet_chars = tl.chars_per_line(_USABLE_W - 0.22, _FS_BODY) - rich = tl.wrap_rich(content, bullet_chars) + rich = tl.wrap_rich_terms(content, bullet_chars) _place_rich_lines(st, rich, _FS_BODY, _INK, prefixes=("• ", " ")) i += 1 @@ -258,7 +306,8 @@ def _place_markdown(st: _PdfState, block) -> None: j += 1 text = " ".join(para) max_chars = tl.chars_per_line(_USABLE_W, _FS_BODY) - _place_rich_lines(st, tl.wrap_rich(text, max_chars), _FS_BODY, _INK) + _place_rich_lines(st, tl.wrap_rich_terms(text, max_chars), _FS_BODY, + _INK) i = j st.y += _GAP @@ -325,15 +374,18 @@ def _wrap_row(cells: list, widths: list, fs: float) -> list: def _draw_table_row(st: _PdfState, cells_lines: list, widths: list, fs: float, - y0: float, header: bool) -> float: + y0: float, header: bool, zebra: bool = False) -> float: lh = tl.line_height_in(fs) nlines = max((len(c) for c in cells_lines), default=1) row_h = lh * nlines + _ROW_VPAD * 2 - if header: + # Background: header band, or a faint zebra fill for even data rows. Drawn + # below the text/rule (zorder 0) so striping never hides cell content. + bg = _HEAD_BG if header else (_ZEBRA if zebra else None) + if bg is not None: st.fig.add_artist(Rectangle( (_xf(_ML), _yf(y0 + row_h)), _xf(_ML + _USABLE_W) - _xf(_ML), _yf(y0) - _yf(y0 + row_h), transform=st.fig.transFigure, - color=_HEAD_BG, lw=0, zorder=0)) + color=bg, lw=0, zorder=0)) x = _ML for c, lines in enumerate(cells_lines): for k, ln in enumerate(lines): @@ -378,14 +430,18 @@ def _place_data_table(st: _PdfState, block) -> None: + _ROW_VPAD * 2 _ensure_space(st, header_h() + max(first_row_h, lh)) draw_header() - for r in rows: + # ``data_idx`` is the LOGICAL row index (not reset across page breaks) so the + # zebra pattern stays coherent when a long table splits and repeats the + # header: even rows (1-based) are shaded → 0-based odd indices. + for data_idx, r in enumerate(rows): cells_lines = _wrap_row(r, widths, fs) row_h = lh * max((len(c) for c in cells_lines), default=1) \ + _ROW_VPAD * 2 if _remaining(st) < row_h: _new_page(st) draw_header() # repeat header on the continuation page. - st.y += _draw_table_row(st, cells_lines, widths, fs, st.y, header=False) + st.y += _draw_table_row(st, cells_lines, widths, fs, st.y, + header=False, zebra=(data_idx % 2 == 1)) note = getattr(block, "note", None) if note: _place_text_lines(st, tl.wrap(model._safe_str(note), @@ -414,53 +470,98 @@ def _png_from_figure(fig) -> bytes: return buf.read() -def _place_image_array(st: _PdfState, arr, caption) -> None: +def _figure_png_cached(block): + """Rasterize a Figure to PNG bytes ONCE and cache (bytes, aspect). + + Measuring (keep-together) and drawing must agree on the REAL aspect ratio: + ``bbox_inches='tight'`` changes it vs ``figsize``, so we rasterize once and + reuse the bytes for both. Cached on the block; never raises.""" + cached = getattr(block, "_aeda_png", None) + if cached is not None: + return cached + fig, owned = _resolve_figure(block) + data = None + if fig is not None: + try: + data = _png_from_figure(fig) + finally: + if owned: + try: + plt.close(fig) + except Exception: # noqa: BLE001 + pass + aspect = 0.66 + if data is not None: + try: + arr = mpimg.imread(io.BytesIO(data)) + aspect = (arr.shape[0] / arr.shape[1]) if arr.shape[1] else 0.66 + except Exception: # noqa: BLE001 + aspect = 0.66 + try: + block._aeda_png = (data, aspect) + return block._aeda_png + except Exception: # noqa: BLE001 — block may reject attributes; degrade. + return (data, aspect) + + +def _image_aspect(block) -> float: + """Real aspect (h/w) of an Image block by path, for measurement.""" + path = getattr(block, "path", "") + if path and os.path.exists(path): + try: + arr = mpimg.imread(path) + return (arr.shape[0] / arr.shape[1]) if arr.shape[1] else 0.66 + except Exception: # noqa: BLE001 + pass + return 0.66 + + +def _place_image_array(st: _PdfState, arr, caption, max_h_in=None) -> None: h_px, w_px = arr.shape[0], arr.shape[1] aspect = (h_px / w_px) if w_px else 1.0 + # Reserve the caption's REAL (possibly multi-line) height FIRST, then scale + # the image to (max_h - cap_reserve) so figure + caption always fit the same + # page. cap_reserve adds a cushion so the caption never spills to next page. + cap_lines = (tl.wrap(model._safe_str(caption), + tl.chars_per_line(_USABLE_W, _FS_NOTE)) + if caption else []) + cap_real = tl.line_height_in(_FS_NOTE) * len(cap_lines) if caption else 0.0 + cap_reserve = (cap_real + 0.04 + 0.08) if caption else 0.0 max_h = _CONTENT_BOTTOM - _CONTENT_TOP + # height_in hint (model.Figure/Image): cap the height so a figure in a + # keep-together Group shrinks to leave room for its heading and text. + if isinstance(max_h_in, (int, float)) and max_h_in > 0: + max_h = min(max_h, float(max_h_in)) + max_img_h = max(max_h - cap_reserve, 0.6) target_w = _USABLE_W target_h = target_w * aspect - if target_h > max_h: - target_h = max_h + if target_h > max_img_h: + target_h = max_img_h target_w = target_h / aspect if aspect else _USABLE_W - cap_h = tl.line_height_in(_FS_NOTE) + 0.04 if caption else 0.0 # Move whole image to next page if it does not fit in remaining space. - if _remaining(st) < target_h + cap_h: - if (max_h) >= target_h + cap_h: - _new_page(st) - else: - # Taller than a full page even at min — already clamped to max_h. - _new_page(st) + if _remaining(st) < target_h + cap_reserve: + _new_page(st) left_frac = _xf(_ML + (_USABLE_W - target_w) / 2.0) bottom_frac = _yf(st.y + target_h) ax = st.fig.add_axes([left_frac, bottom_frac, target_w / _W, target_h / _H]) ax.imshow(arr) ax.axis("off") st.y += target_h + 0.04 - if caption: - _place_text_lines(st, tl.wrap(model._safe_str(caption), - tl.chars_per_line(_USABLE_W, _FS_NOTE)), - _FS_NOTE, _MUTED, style="italic") + if cap_lines: + _place_text_lines(st, cap_lines, _FS_NOTE, _MUTED, style="italic") st.y += _GAP def _place_figure(st: _PdfState, block) -> None: - fig, owned = _resolve_figure(block) - if fig is None: + png, _aspect = _figure_png_cached(block) + if png is None: _place_text_lines(st, ["(figura no disponible)"], _FS_NOTE, _MUTED, style="italic") st.y += _GAP return - try: - png = _png_from_figure(fig) - finally: - if owned: - try: - plt.close(fig) - except Exception: # noqa: BLE001 - pass arr = mpimg.imread(io.BytesIO(png)) - _place_image_array(st, arr, getattr(block, "caption", None)) + _place_image_array(st, arr, getattr(block, "caption", None), + max_h_in=getattr(block, "height_in", None)) def _place_image(st: _PdfState, block) -> None: @@ -471,7 +572,8 @@ def _place_image(st: _PdfState, block) -> None: st.y += _GAP return arr = mpimg.imread(path) - _place_image_array(st, arr, getattr(block, "caption", None)) + _place_image_array(st, arr, getattr(block, "caption", None), + max_h_in=getattr(block, "height_in", None)) def _place_caption(st: _PdfState, block) -> None: @@ -488,6 +590,189 @@ def _place_note(st: _PdfState, block) -> None: st.y += _GAP +# --------------------------------------------------------------------------- # +# Block measurement (mejora 3 — keep-together). These estimate a block's height +# WITHOUT drawing it, so a Group can decide to move whole to the next page before +# anything is drawn. Over-estimating is safe: it only triggers an earlier page +# break, never a content cut (the placers keep their own no-cut pagination). +# --------------------------------------------------------------------------- # +def _measure_heading_text(text: str, level: int) -> float: + level = max(1, min(3, int(level or 1))) + fs = {1: _FS_H1, 2: _FS_H2, 3: _FS_H3}[level] + lines = tl.wrap(tl.strip_inline_md(text), tl.chars_per_line(_USABLE_W, fs)) + h = tl.line_height_in(fs, leading=1.2) * len(lines) + 0.06 + if level == 1: + h += 0.10 + return h + _GAP + + +def _measure_markdown(block) -> float: + raw = str(getattr(block, "text", "") or "") + md_lines = raw.split("\n") + h = 0.0 + i, n = 0, len(md_lines) + while i < n: + stripped = md_lines[i].strip() + if stripped.startswith("|") and stripped.endswith("|"): + j = i + while j < n and md_lines[j].strip().startswith("|") \ + and md_lines[j].strip().endswith("|"): + j += 1 + h += (tl.line_height_in(_FS_CELL) + _ROW_VPAD * 2) * (j - i) + _GAP + i = j + continue + if stripped == "": + h += tl.line_height_in(_FS_BODY) * 0.5 + i += 1 + continue + if stripped.startswith("### "): + h += _measure_heading_text(stripped[4:], 3) + i += 1 + continue + if stripped.startswith("## "): + h += _measure_heading_text(stripped[3:], 2) + i += 1 + continue + if stripped.startswith("# "): + h += _measure_heading_text(stripped[2:], 1) + i += 1 + continue + if stripped.startswith("- ") or stripped.startswith("* "): + lines = tl.wrap_rich_terms( + stripped[2:], tl.chars_per_line(_USABLE_W - 0.22, _FS_BODY)) + h += tl.line_height_in(_FS_BODY) * len(lines) + i += 1 + continue + para = [stripped] + j = i + 1 + while j < n: + nxt = md_lines[j].strip() + if nxt == "" or nxt.startswith(("|", "#", "- ", "* ")): + break + para.append(nxt) + j += 1 + lines = tl.wrap_rich_terms(" ".join(para), + tl.chars_per_line(_USABLE_W, _FS_BODY)) + h += tl.line_height_in(_FS_BODY) * len(lines) + i = j + return h + _GAP + + +def _measure_figure_like(block) -> float: + max_h = _CONTENT_BOTTOM - _CONTENT_TOP + hint = getattr(block, "height_in", None) + if isinstance(hint, (int, float)) and hint > 0: + target_h = min(float(hint), max_h) + else: + # Real rasterized aspect (cached) so measuring matches drawing. + if getattr(block, "kind", "") == "image": + aspect = _image_aspect(block) + else: + _data, aspect = _figure_png_cached(block) + target_h = min(_USABLE_W * aspect, max_h) + cap = getattr(block, "caption", None) + cap_h = tl.line_height_in(_FS_NOTE) + 0.04 if cap else 0.0 + return target_h + 0.04 + cap_h + _GAP + + +def _measure_block(st: _PdfState, block) -> float: + kind = getattr(block, "kind", "") + try: + if kind == "heading": + return _measure_heading_text(getattr(block, "text", ""), + getattr(block, "level", 1)) + if kind == "markdown": + return _measure_markdown(block) + if kind in ("figure", "image"): + return _measure_figure_like(block) + if kind in ("caption", "note"): + lines = tl.wrap(getattr(block, "text", ""), + tl.chars_per_line(_USABLE_W, _FS_NOTE)) + return tl.line_height_in(_FS_NOTE) * len(lines) + _GAP + if kind == "kv_table": + rows = getattr(block, "rows", []) or [] + return (tl.line_height_in(_FS_BODY) + _ROW_VPAD) * (len(rows) + 1) \ + + _GAP + if kind == "data_table": + rows = getattr(block, "rows", []) or [] + return (tl.line_height_in(_FS_CELL) + _ROW_VPAD * 2) \ + * (len(rows) + 1) + _GAP + if kind == "group": + return sum(_measure_block(st, b) + for b in (getattr(block, "blocks", []) or [])) + except Exception: # noqa: BLE001 — a measurement never aborts rendering. + pass + return tl.line_height_in(_FS_BODY) + + +def _shrink_group_figures(st: _PdfState, blocks: list, avail_full: float) -> None: + """Cap each figure's height (via height_in) so the whole group fits a page. + + The figure shrinks just enough to leave room for its heading, text and + caption — keep-together puts the chart on the SAME page as its title and + description instead of pushing it to the next page.""" + fig_blocks = [b for b in blocks + if getattr(b, "kind", "") in ("figure", "image")] + if not fig_blocks: + return + nonfig_h = sum(_measure_block(st, b) for b in blocks + if getattr(b, "kind", "") not in ("figure", "image")) + fig_overhead = tl.line_height_in(_FS_NOTE) + 0.04 + 0.04 + _GAP + budget = avail_full - nonfig_h - 0.08 * len(fig_blocks) + if budget <= 0.8: + return + per = budget / len(fig_blocks) - fig_overhead + if per <= 0.6: + return + for fb in fig_blocks: + cur = getattr(fb, "height_in", None) + fb.height_in = (min(float(cur), per) + if isinstance(cur, (int, float)) and cur > 0 else per) + + +def _place_group(st: _PdfState, block) -> None: + """Render a keep-together Group: move it whole to the next page if needed.""" + blocks = getattr(block, "blocks", []) or [] + if not blocks: + return + avail_full = _CONTENT_BOTTOM - _CONTENT_TOP + _shrink_group_figures(st, blocks, avail_full) + total = sum(_measure_block(st, b) for b in blocks) + if total <= avail_full: + # Fits on one page: keep it together by moving whole when it won't fit. + if total > _remaining(st): + _new_page(st) + elif st.y > _CONTENT_TOP + 1e-6: + # Taller than a full page: at least start it on a fresh page, then flow. + _new_page(st) + for b in blocks: + placer = _PLACERS.get(getattr(b, "kind", ""), _place_note) + try: + placer(st, b) + except Exception: # noqa: BLE001 — a bad block never aborts the group. + pass + + +def _place_glossary_entry(st: _PdfState, block) -> None: + """Render one glossary term and register it as a clickable link target.""" + key = getattr(block, "key", "") + label = getattr(block, "label", "") or key + definition = getattr(block, "definition", "") + # Reserve the term + its first definition line together, then anchor the + # destination at the resolved page/position before drawing. + _ensure_space(st, tl.line_height_in(_FS_H3, leading=1.2) + + tl.line_height_in(_FS_BODY) * 2) + if key: + st.term_dests[key] = {"page": st.page - 1, + "point": [_ML * 72.0, st.y * 72.0]} + _place_heading(st, model.Heading(text=str(label), level=3)) + if definition: + _place_text_lines(st, tl.wrap(model._safe_str(definition), + tl.chars_per_line(_USABLE_W, _FS_BODY)), + _FS_BODY, _INK) + st.y += _GAP * 0.5 + + _PLACERS = { "heading": _place_heading, "markdown": _place_markdown, @@ -497,6 +782,8 @@ _PLACERS = { "image": _place_image, "caption": _place_caption, "note": _place_note, + "group": _place_group, + "glossary_entry": _place_glossary_entry, } @@ -553,8 +840,42 @@ def render_pdf(chapters: list, out_path: str, meta: dict = None) -> dict: return {"path": None, "n_pages": 0, "chapters": [], "note": f"fallo al escribir el PDF: {e}"} + # Mejora 6 — wire clickable glossary links now the PDF is closed on disk. + # PdfPages cannot emit internal hyperlinks, so we post-process with PyMuPDF + # (delegated registry function). Degrades silently if it is unavailable. + n_links = _wire_glossary_links(st, out_path, notes) + note = f"{n_pages} páginas" + if n_links: + note += f" · {n_links} enlaces de glosario" if notes: note += " · " + "; ".join(notes) return {"path": out_path, "n_pages": n_pages, "chapters": chapters_meta, "note": note} + + +def _wire_glossary_links(st: _PdfState, out_path: str, notes: list) -> int: + """Build {source rect → glossary dest} links and apply them via PyMuPDF. + + Returns the number of links applied (0 if there is nothing to wire or the + post-processor is unavailable). Never raises.""" + try: + links = [] + for src in st.term_sources: + dest = st.term_dests.get(src.get("key")) + if not dest: + continue + links.append({ + "src_page": src["page"], "src_rect": src["rect"], + "dst_page": dest["page"], "dst_point": dest["point"]}) + if not links: + return 0 + from datascience.add_pdf_internal_links import add_pdf_internal_links + res = add_pdf_internal_links(out_path, links) + if isinstance(res, dict) and res.get("status") == "ok": + return int(res.get("n_links") or 0) + if isinstance(res, dict) and res.get("error"): + notes.append(f"glosario sin enlaces: {res.get('error')}") + except Exception as e: # noqa: BLE001 — links are best-effort. + notes.append(f"glosario sin enlaces: {e}") + return 0 diff --git a/python/functions/datascience/automatic_eda/render_pptx_impl.py b/python/functions/datascience/automatic_eda/render_pptx_impl.py index db7d201a..5e3ba331 100644 --- a/python/functions/datascience/automatic_eda/render_pptx_impl.py +++ b/python/functions/datascience/automatic_eda/render_pptx_impl.py @@ -43,6 +43,8 @@ _ACCENT = (0x2A, 0x6F, 0x97) _MUTED = (0x8A, 0x8A, 0x8A) _HEAD_BG = (0xEE, 0xF3, 0xF6) _WHITE = (0xFF, 0xFF, 0xFF) +_ZEBRA = (0xF6, 0xF8, 0xFA) # faint grey for even (zebra) data rows. +_LINK = (0x2A, 0x6F, 0x97) # accent colour for clickable glossary terms. _FS_TITLE = 26 _FS_H1, _FS_H2, _FS_H3 = 20, 16, 13 @@ -59,6 +61,10 @@ class _PptxState: self.chapter = None self.slide_no = 0 self.chapter_slides = 0 + self.last_heading = "" # text of the most recent heading. + # Glossary wiring (mejora 6): runs to link and per-term target slide. + self.term_runs = [] # [(key, run)] + self.term_anchor_slide = {} # key -> Slide (glossary entry) def _rgb(c): @@ -155,9 +161,13 @@ def _add_rich_text(st: _PptxState, rich_lines: list, fs: float, color, indent=0.0, bullet=False) -> None: """Add pre-wrapped lines of styled segments as one paragraph per line. - Each line is ``[(text, is_bold), ...]``; every segment becomes its own run - so ``**bold**`` spans render with native PowerPoint bold (``run.font.bold``) - without affecting the measured height (one paragraph per pre-wrapped line). + Each line is a list of ``(text, is_bold)`` or ``(text, is_bold, term_key)`` + segments; every segment becomes its own run so ``**bold**`` spans render with + native PowerPoint bold (``run.font.bold``) without affecting the measured + height (one paragraph per pre-wrapped line). A segment carrying a + ``term_key`` is drawn in the accent colour and its run is recorded in + ``st.term_runs`` so it later becomes a native hyperlink jumping to the + glossary slide of that term. """ lh = tl.line_height_in(fs) height = lh * len(rich_lines) + 0.05 @@ -176,14 +186,20 @@ def _add_rich_text(st: _PptxState, rich_lines: list, fs: float, color, r0.text = "• " r0.font.size = Pt(fs) r0.font.color.rgb = _rgb(color) - for seg_text, is_bold in segs: + for seg in segs: + if len(seg) == 3: + seg_text, is_bold, term = seg + else: + seg_text, is_bold, term = seg[0], seg[1], None if seg_text == "": continue run = p.add_run() run.text = seg_text run.font.size = Pt(fs) run.font.bold = bool(is_bold) - run.font.color.rgb = _rgb(color) + run.font.color.rgb = _rgb(_LINK if term else color) + if term: + st.term_runs.append((term, run, st.slide)) st.y += height @@ -191,6 +207,7 @@ def _place_heading(st: _PptxState, block) -> None: level = max(1, min(3, int(getattr(block, "level", 1) or 1))) fs = {1: _FS_H1, 2: _FS_H2, 3: _FS_H3}[level] text = tl.strip_inline_md(getattr(block, "text", "")) + st.last_heading = text or st.last_heading lines = tl.wrap(text, tl.chars_per_line(_USABLE_W, fs)) _add_text(st, lines, fs, _INK, bold=True) st.y += 0.04 @@ -233,12 +250,12 @@ def _place_markdown(st: _PptxState, block) -> None: continue if stripped.startswith("- ") or stripped.startswith("* "): content = stripped[2:] # keep inline markers for bold rendering. - rich = tl.wrap_rich(content, - tl.chars_per_line(_USABLE_W - 0.3, _FS_BODY)) + rich = tl.wrap_rich_terms(content, + tl.chars_per_line(_USABLE_W - 0.3, _FS_BODY)) _add_rich_text(st, rich, _FS_BODY, _INK, bullet=True) i += 1 continue - para = [stripped] # keep inline markers; wrap_rich renders **bold**. + para = [stripped] # keep inline markers; wrap_rich_terms renders **bold**. j = i + 1 while j < n: nxt = md_lines[j].strip() @@ -247,8 +264,8 @@ def _place_markdown(st: _PptxState, block) -> None: para.append(nxt) j += 1 text = " ".join(para) - _add_rich_text(st, tl.wrap_rich(text, tl.chars_per_line(_USABLE_W, _FS_BODY)), - _FS_BODY, _INK) + _add_rich_text(st, tl.wrap_rich_terms( + text, tl.chars_per_line(_USABLE_W, _FS_BODY)), _FS_BODY, _INK) i = j st.y += _GAP @@ -295,7 +312,8 @@ def _row_height_in(cells, widths, fs) -> float: return lh * maxlines + 0.10 -def _emit_table(st: _PptxState, header, chunk, widths, fs) -> None: +def _emit_table(st: _PptxState, header, chunk, widths, fs, + start_index: int = 0) -> None: nrows = len(chunk) + (1 if header else 0) ncol = len(widths) # Pre-measure total height to size the shape (pptx still auto-grows rows). @@ -319,11 +337,14 @@ def _emit_table(st: _PptxState, header, chunk, widths, fs) -> None: cell.text = model._safe_str(header[c]) if c < len(header) else "" _style_cell(cell, fs, _INK, bold=True, fill=_HEAD_BG) ridx = 1 - for r in chunk: + # Zebra striping: shade even data rows (1-based) using the GLOBAL row index + # (start_index offset) so the pattern stays coherent across split chunks. + for k, r in enumerate(chunk): + fill = _ZEBRA if (start_index + k) % 2 == 1 else _WHITE for c in range(ncol): cell = gtable.cell(ridx, c) cell.text = model._safe_str(r[c]) if c < len(r) else "" - _style_cell(cell, fs, _INK, bold=False, fill=_WHITE) + _style_cell(cell, fs, _INK, bold=False, fill=fill) ridx += 1 st.y += total_h + _GAP @@ -367,6 +388,7 @@ def _place_data_table(st: _PptxState, block, shaded_header=True, avail = _remaining(st) - header_h chunk = [] used = 0.0 + chunk_start = idx # global index of the first row in this chunk (zebra). while idx < n: rh = _row_height_in(rows[idx], widths, fs) if used + rh > avail and chunk: @@ -374,7 +396,7 @@ def _place_data_table(st: _PptxState, block, shaded_header=True, chunk.append(rows[idx]) used += rh idx += 1 - _emit_table(st, header, chunk, widths, fs) + _emit_table(st, header, chunk, widths, fs, start_index=chunk_start) note = getattr(block, "note", None) if note: _add_text(st, tl.wrap(model._safe_str(note), @@ -421,54 +443,97 @@ def _resolve_png(block): pass -def _place_picture_bytes(st: _PptxState, data: bytes, caption) -> None: +def _figure_bytes_cached(block): + """Rasterize a figure/image to PNG bytes ONCE and cache (bytes, aspect). + + Measuring (keep-together) and drawing must agree on the real aspect ratio — + ``bbox_inches='tight'`` changes it vs ``figsize``, so we rasterize once and + reuse the bytes for both. Cached on the block; never raises.""" + cached = getattr(block, "_aeda_png", None) + if cached is not None: + return cached + kind = getattr(block, "kind", "") + data = None + if kind == "image": + path = getattr(block, "path", "") + if path and os.path.exists(path): + try: + with open(path, "rb") as fh: + data = fh.read() + except Exception: # noqa: BLE001 + data = None + else: + data = _resolve_png(block) + aspect = 0.66 + if data is not None: + w_px, h_px = _img_size_px(data) + aspect = (h_px / w_px) if w_px else 0.66 + try: + block._aeda_png = (data, aspect) + return block._aeda_png + except Exception: # noqa: BLE001 — block may reject attributes; degrade. + return (data, aspect) + + +def _place_picture_bytes(st: _PptxState, data: bytes, caption, + max_h_in=None) -> None: + # Mejora 4 — every figure on a slide carries a visible caption/title. If the + # block has no caption, fall back to the current section heading, then to a + # generic label, so no image is ever shown untitled. + caption = (model._safe_str(caption).strip() + or model._safe_str(st.last_heading).strip() or "Figura") w_px, h_px = _img_size_px(data) aspect = (h_px / w_px) if w_px else 0.66 + # Reserve the caption's REAL (possibly multi-line) height FIRST, then scale + # the image to (max_h - cap_reserve): a figure never fills the whole slide, + # so its caption always fits on the SAME slide and no image is untitled. + # cap_real = what _add_text consumes; cap_reserve adds the post-image gap and + # a small cushion so the caption never spills to the next slide. + cap_lines = tl.wrap(caption, tl.chars_per_line(_USABLE_W, _FS_NOTE)) + cap_real = tl.line_height_in(_FS_NOTE) * len(cap_lines) + 0.05 + cap_reserve = cap_real + 0.05 + 0.10 max_h = _CONTENT_BOTTOM - _CONTENT_TOP + # height_in hint (model.Figure/Image): cap the target height so a figure in a + # keep-together Group shrinks to leave room for its heading and text. + if isinstance(max_h_in, (int, float)) and max_h_in > 0: + max_h = min(max_h, float(max_h_in)) + max_img_h = max(max_h - cap_reserve, 0.6) target_w = _USABLE_W target_h = target_w * aspect - if target_h > max_h: - target_h = max_h + if target_h > max_img_h: + target_h = max_img_h target_w = target_h / aspect if aspect else _USABLE_W - cap_h = tl.line_height_in(_FS_NOTE) + 0.05 if caption else 0.0 - if _remaining(st) < target_h + cap_h: + # Keep the image and its caption together on the same slide. + if _remaining(st) < target_h + cap_reserve: _new_slide(st, cont=True) left = _ML + (_USABLE_W - target_w) / 2.0 st.slide.shapes.add_picture(io.BytesIO(data), Inches(left), Inches(st.y), width=Inches(target_w), height=Inches(target_h)) st.y += target_h + 0.05 - if caption: - _add_text(st, tl.wrap(model._safe_str(caption), - tl.chars_per_line(_USABLE_W, _FS_NOTE)), _FS_NOTE, _MUTED, - italic=True) + _add_text(st, cap_lines, _FS_NOTE, _MUTED, italic=True) st.y += _GAP def _place_figure(st: _PptxState, block) -> None: - png = _resolve_png(block) + png, _aspect = _figure_bytes_cached(block) if png is None: _add_text(st, ["(figura no disponible)"], _FS_NOTE, _MUTED, italic=True) st.y += _GAP return - _place_picture_bytes(st, png, getattr(block, "caption", None)) + _place_picture_bytes(st, png, getattr(block, "caption", None), + max_h_in=getattr(block, "height_in", None)) def _place_image(st: _PptxState, block) -> None: - path = getattr(block, "path", "") - if not path or not os.path.exists(path): + data, _aspect = _figure_bytes_cached(block) + if data is None: + path = getattr(block, "path", "") _add_text(st, [f"(imagen no encontrada: {path})"], _FS_NOTE, _MUTED, italic=True) st.y += _GAP return - try: - with open(path, "rb") as fh: - data = fh.read() - except Exception as e: # noqa: BLE001 - _add_text(st, [f"(no se pudo leer la imagen: {e})"], _FS_NOTE, _MUTED, - italic=True) - st.y += _GAP - return - _place_picture_bytes(st, data, getattr(block, "caption", None)) + _place_picture_bytes(st, data, getattr(block, "caption", None), + max_h_in=getattr(block, "height_in", None)) def _place_caption(st: _PptxState, block) -> None: @@ -482,6 +547,170 @@ def _place_note(st: _PptxState, block) -> None: _place_caption(st, block) +# --------------------------------------------------------------------------- # +# Block measurement (mejora 3 — keep-together). Estimate a block's slide height +# WITHOUT drawing it so a Group can move whole to the next slide before drawing. +# Over-estimating only triggers an earlier slide break, never a content cut. +# --------------------------------------------------------------------------- # +def _measure_heading_text(text: str, level: int) -> float: + level = max(1, min(3, int(level or 1))) + fs = {1: _FS_H1, 2: _FS_H2, 3: _FS_H3}[level] + lines = tl.wrap(tl.strip_inline_md(text), tl.chars_per_line(_USABLE_W, fs)) + return tl.line_height_in(fs) * len(lines) + 0.05 + 0.04 + + +def _measure_markdown(block) -> float: + raw = str(getattr(block, "text", "") or "") + md_lines = raw.split("\n") + h = 0.0 + i, n = 0, len(md_lines) + while i < n: + stripped = md_lines[i].strip() + if stripped.startswith("|") and stripped.endswith("|"): + j = i + while j < n and md_lines[j].strip().startswith("|") \ + and md_lines[j].strip().endswith("|"): + j += 1 + h += (tl.line_height_in(_FS_CELL) + 0.10) * (j - i) + _GAP + i = j + continue + if stripped == "": + h += tl.line_height_in(_FS_BODY) * 0.4 + i += 1 + continue + if stripped.startswith("### "): + h += _measure_heading_text(stripped[4:], 3) + i += 1 + continue + if stripped.startswith("## "): + h += _measure_heading_text(stripped[3:], 2) + i += 1 + continue + if stripped.startswith("# "): + h += _measure_heading_text(stripped[2:], 1) + i += 1 + continue + if stripped.startswith("- ") or stripped.startswith("* "): + lines = tl.wrap_rich_terms( + stripped[2:], tl.chars_per_line(_USABLE_W - 0.3, _FS_BODY)) + h += tl.line_height_in(_FS_BODY) * len(lines) + 0.05 + i += 1 + continue + para = [stripped] + j = i + 1 + while j < n: + nxt = md_lines[j].strip() + if nxt == "" or nxt.startswith(("|", "#", "- ", "* ")): + break + para.append(nxt) + j += 1 + lines = tl.wrap_rich_terms(" ".join(para), + tl.chars_per_line(_USABLE_W, _FS_BODY)) + h += tl.line_height_in(_FS_BODY) * len(lines) + 0.05 + i = j + return h + _GAP + + +def _measure_figure_like(block) -> float: + max_h = _CONTENT_BOTTOM - _CONTENT_TOP + hint = getattr(block, "height_in", None) + if isinstance(hint, (int, float)) and hint > 0: + max_h = min(max_h, float(hint)) + # Use the REAL rasterized aspect (cached) so measuring matches drawing — this + # is what keeps a figure together with its heading instead of splitting. + _data, aspect = _figure_bytes_cached(block) + target_h = min(_USABLE_W * aspect, max_h) + # Caption is always emitted now (mejora 4), so always reserve its line. + cap_h = tl.line_height_in(_FS_NOTE) + 0.05 + return target_h + 0.05 + cap_h + _GAP + + +def _measure_block(st: _PptxState, block) -> float: + kind = getattr(block, "kind", "") + try: + if kind == "heading": + return _measure_heading_text(getattr(block, "text", ""), + getattr(block, "level", 1)) + if kind == "markdown": + return _measure_markdown(block) + if kind in ("figure", "image"): + return _measure_figure_like(block) + if kind in ("caption", "note"): + lines = tl.wrap(getattr(block, "text", ""), + tl.chars_per_line(_USABLE_W, _FS_NOTE)) + return tl.line_height_in(_FS_NOTE) * len(lines) + 0.05 + _GAP + if kind in ("kv_table", "data_table"): + rows = getattr(block, "rows", []) or [] + return (tl.line_height_in(_FS_CELL) + 0.10) * (len(rows) + 1) + _GAP + if kind == "group": + return sum(_measure_block(st, b) + for b in (getattr(block, "blocks", []) or [])) + except Exception: # noqa: BLE001 — a measurement never aborts rendering. + pass + return tl.line_height_in(_FS_BODY) + + +def _shrink_group_figures(st: _PptxState, blocks: list, avail_full: float) -> None: + """Cap each figure's height (via height_in) so the whole group fits a slide. + + The figure shrinks just enough to leave room for its heading, text and + caption — that is how keep-together puts a chart on the SAME slide as its + title and description instead of pushing it to the next slide.""" + fig_blocks = [b for b in blocks + if getattr(b, "kind", "") in ("figure", "image")] + if not fig_blocks: + return + nonfig_h = sum(_measure_block(st, b) for b in blocks + if getattr(b, "kind", "") not in ("figure", "image")) + fig_overhead = tl.line_height_in(_FS_NOTE) + 0.05 + 0.05 + _GAP + budget = avail_full - nonfig_h - 0.10 * len(fig_blocks) + if budget <= 1.0: + return # not enough room to keep together; let it flow (degrade). + per = budget / len(fig_blocks) - fig_overhead + if per <= 0.8: + return + for fb in fig_blocks: + cur = getattr(fb, "height_in", None) + fb.height_in = (min(float(cur), per) + if isinstance(cur, (int, float)) and cur > 0 else per) + + +def _place_group(st: _PptxState, block) -> None: + """Render a keep-together Group: move it whole to the next slide if needed.""" + blocks = getattr(block, "blocks", []) or [] + if not blocks: + return + avail_full = _CONTENT_BOTTOM - _CONTENT_TOP + _shrink_group_figures(st, blocks, avail_full) + total = sum(_measure_block(st, b) for b in blocks) + if total <= avail_full: + if total > _remaining(st): + _new_slide(st, cont=True) + elif st.y > _CONTENT_TOP + 1e-6: + _new_slide(st, cont=True) + for b in blocks: + placer = _PLACERS.get(getattr(b, "kind", ""), _place_note) + try: + placer(st, b) + except Exception: # noqa: BLE001 — a bad block never aborts the group. + pass + + +def _place_glossary_entry(st: _PptxState, block) -> None: + """Render one glossary term and register its slide as the link target.""" + key = getattr(block, "key", "") + label = getattr(block, "label", "") or key + definition = getattr(block, "definition", "") + _ensure(st, tl.line_height_in(_FS_H3) + tl.line_height_in(_FS_BODY) * 2) + if key: + st.term_anchor_slide[key] = st.slide + _place_heading(st, model.Heading(text=str(label), level=3)) + if definition: + _add_text(st, tl.wrap(model._safe_str(definition), + tl.chars_per_line(_USABLE_W, _FS_BODY)), _FS_BODY, _INK) + st.y += _GAP + + _PLACERS = { "heading": _place_heading, "markdown": _place_markdown, @@ -491,6 +720,8 @@ _PLACERS = { "image": _place_image, "caption": _place_caption, "note": _place_note, + "group": _place_group, + "glossary_entry": _place_glossary_entry, } @@ -542,6 +773,9 @@ def render_pptx(chapters: list, out_path: str, meta: dict = None) -> dict: _new_slide(st, cont=False) _place_note(st, model.Note( "(documento vacío — sin capítulos aplicables)")) + # Mejora 6 — wire clickable glossary terms to their entry slide (native + # PowerPoint slide-jump). Delegated registry function; degrades silently. + n_links = _wire_glossary_links(st, notes) prs.save(out_path) n_slides = st.slide_no except Exception as e: # noqa: BLE001 @@ -549,7 +783,35 @@ def render_pptx(chapters: list, out_path: str, meta: dict = None) -> dict: "note": f"fallo al escribir el PPTX: {e}"} note = f"{n_slides} slides" + if n_links: + note += f" · {n_links} enlaces de glosario" if notes: note += " · " + "; ".join(notes) return {"path": out_path, "n_slides": n_slides, "chapters": chapters_meta, "note": note} + + +def _wire_glossary_links(st: _PptxState, notes: list) -> int: + """Turn each recorded term run into a native jump to its glossary slide. + + Returns the number of links applied. A term whose only appearance is inside + its own glossary entry (source slide == target slide) is skipped. Never + raises.""" + if not st.term_runs or not st.term_anchor_slide: + return 0 + linked = 0 + try: + from datascience.pptx_link_run_to_slide import pptx_link_run_to_slide + except Exception as e: # noqa: BLE001 + notes.append(f"glosario sin enlaces: {e}") + return 0 + for key, run, src_slide in st.term_runs: + tgt = st.term_anchor_slide.get(key) + if tgt is None or tgt is src_slide: + continue + try: + if pptx_link_run_to_slide(run, src_slide, tgt): + linked += 1 + except Exception: # noqa: BLE001 — links are best-effort. + pass + return linked diff --git a/python/functions/datascience/automatic_eda/text_layout.py b/python/functions/datascience/automatic_eda/text_layout.py index 0d07d140..059e12a2 100644 --- a/python/functions/datascience/automatic_eda/text_layout.py +++ b/python/functions/datascience/automatic_eda/text_layout.py @@ -24,6 +24,13 @@ import textwrap # the visible text matches ``strip_inline_md`` exactly. _INLINE_SPAN_RE = re.compile(r"(\*\*.+?\*\*|__.+?__|`.+?`)") +# Glossary term span: ``[[term:key]]texto visible[[/term]]``. The visible text +# (which may itself contain ``**bold**``) is kept and tagged with ``key`` so the +# renderers can turn each appearance into a clickable jump to the glossary entry. +_TERM_SPAN_RE = re.compile(r"\[\[term:([A-Za-z0-9_]+)\]\](.*?)\[\[/term\]\]", + re.S) +_TERM_OPEN_RE = re.compile(r"\[\[term:[A-Za-z0-9_]+\]\]") + def avg_char_width_in(fontsize_pt: float) -> float: """Approximate average glyph width in inches for a sans-serif font. @@ -86,11 +93,21 @@ def strip_inline_md(text: str) -> str: if not text: return "" s = str(text) + # Drop glossary term markers, keeping the visible inner text. + s = _TERM_SPAN_RE.sub(lambda m: m.group(2), s) + s = _TERM_OPEN_RE.sub("", s) # leftover unbalanced open marker. + s = s.replace("[[/term]]", "") # leftover unbalanced close marker. for marker in ("**", "__", "`"): s = s.replace(marker, "") return s +def _strip_term_markers(s: str) -> str: + """Remove any (balanced or leftover) glossary term markers, keeping text.""" + s = _TERM_OPEN_RE.sub("", s) + return s.replace("[[/term]]", "") + + def _strip_leftover_markers(s: str) -> str: """Drop any unbalanced inline markers from a plain (non-span) fragment. @@ -222,6 +239,118 @@ def wrap_rich(text: str, max_chars: int): return lines or [[("", False)]] +def parse_inline_rich(text: str): + """Split ``text`` into ``[(fragment, is_bold, term_key), ...]``. + + Extends :func:`parse_inline_bold` with glossary term spans + ``[[term:key]]visible[[/term]]``: the inner ``visible`` text is parsed for + ``**bold**`` as usual and every resulting fragment carries ``term_key`` so the + renderers can make it clickable. Text outside a term span gets ``term_key = + None``. Unbalanced term markers are stripped (kept identical to + :func:`strip_inline_md`). The concatenation of all fragment texts equals + ``strip_inline_md(text)`` — visible characters and wrapping are unchanged; only + the bold flag and the term key are added. Adjacent fragments with the same + (bold, term) are merged. + """ + s = "" if text is None else str(text) + if not s: + return [] + out = [] + + def _emit(fragment: str, bold: bool, term) -> None: + if fragment == "": + return + if out and out[-1][1] == bold and out[-1][2] == term: + out[-1] = (out[-1][0] + fragment, bold, term) + else: + out.append((fragment, bold, term)) + + def _emit_bolded(segment: str, term) -> None: + # Reuse the bold parser on a term-marker-free segment. + for frag, bold in parse_inline_bold(_strip_term_markers(segment)): + _emit(frag, bold, term) + + pos = 0 + for m in _TERM_SPAN_RE.finditer(s): + if m.start() > pos: + _emit_bolded(s[pos:m.start()], None) + _emit_bolded(m.group(2), m.group(1)) + pos = m.end() + if pos < len(s): + _emit_bolded(s[pos:], None) + return out + + +def wrap_rich_terms(text: str, max_chars: int): + """Like :func:`wrap_rich` but preserving glossary term keys per fragment. + + Returns ``list[list[(fragment, is_bold, term_key)]]`` — one inner list per + output line. Wrapping is word-aware and hard-splits over-long tokens so no + line exceeds ``max_chars`` (the renderers measure these very lines). Term and + bold flags never widen a line: the visible width matches :func:`wrap`. + """ + if max_chars < 1: + max_chars = 1 + spans = parse_inline_rich(text) + if not spans: + return [[("", False, None)]] + + tokens = [] # each: (word, bold, term) or ("\n", None, None) + for frag, bold, term in spans: + parts = frag.split("\n") + for pi, part in enumerate(parts): + if pi > 0: + tokens.append(("\n", None, None)) + for word in part.split(" "): + if word == "": + continue + tokens.append((word, bold, term)) + + lines = [] + cur = [] + cur_len = 0 + + def _flush(): + nonlocal cur, cur_len + merged = [] + for k, (word, bold, term) in enumerate(cur): + piece = word if k == 0 else " " + word + if merged and merged[-1][1] == bold and merged[-1][2] == term: + merged[-1] = (merged[-1][0] + piece, bold, term) + else: + merged.append((piece, bold, term)) + lines.append(merged or [("", False, None)]) + cur = [] + cur_len = 0 + + for word, bold, term in tokens: + if bold is None: # forced newline + _flush() + continue + if len(word) > max_chars: + if cur: + _flush() + chunks = _hard_split(word, max_chars) + for ci, chunk in enumerate(chunks): + if ci < len(chunks) - 1: + lines.append([(chunk, bold, term)]) + else: + cur = [(chunk, bold, term)] + cur_len = len(chunk) + continue + add = len(word) if cur_len == 0 else cur_len + 1 + len(word) + if cur_len != 0 and add > max_chars: + _flush() + cur = [(word, bold, term)] + cur_len = len(word) + else: + cur.append((word, bold, term)) + cur_len = add + if cur: + _flush() + return lines or [[("", False, None)]] + + def parse_md_table(lines: list): """Parse consecutive ``| a | b |`` lines into ``(header, rows)`` or None. diff --git a/python/functions/datascience/pptx_link_run_to_slide.md b/python/functions/datascience/pptx_link_run_to_slide.md new file mode 100644 index 00000000..33298afd --- /dev/null +++ b/python/functions/datascience/pptx_link_run_to_slide.md @@ -0,0 +1,85 @@ +--- +name: pptx_link_run_to_slide +kind: function +lang: py +domain: datascience +version: "1.0.0" +purity: impure +signature: "def pptx_link_run_to_slide(run, source_slide, target_slide) -> bool" +description: "Convierte un run de texto de python-pptx en un hyperlink INTERNO 'ir a la diapositiva'. python-pptx soporta run.hyperlink.address para URLs externas pero NO para saltar a otra slide del mismo deck; esta función crea ese salto manipulando el XML: añade una relación slide->slide (RT.SLIDE) y un con action='ppaction://hlinksldjump' y el r:id de la relación, insertado como primer hijo del del run (orden del schema CT_TextCharacterProperties). Idempotente (elimina un hlinkClick previo antes de insertar). Al pulsar el texto en PowerPoint o visores compatibles se navega a target_slide. Motor python-pptx. No lanza nunca: cualquier excepción -> return False." +tags: [eda, pptx, hyperlink, slide-jump, navigation, glossary, automatic-eda, python-pptx, xml, datascience, python] +uses_functions: [] +uses_types: [] +returns: [] +returns_optional: false +error_type: "error_go_core" +imports: ["python-pptx"] +params: + - name: run + desc: "el pptx.text.text._Run cuyo texto se vuelve clicable. Debe pertenecer a un run real (expone ._r, el elemento ). Un objeto sin ._r hace que la función devuelva False sin lanzar." + - name: source_slide + desc: "la Slide que contiene el run. Su part recibe la relación slide->slide (relate_to con RELATIONSHIP_TYPE.SLIDE); el r:id resultante se referencia en el hlinkClick." + - name: target_slide + desc: "la Slide de destino del salto. Debe pertenecer al MISMO Presentation que source_slide para que la relación interna sea válida." +output: "bool. True si se aplicó el hyperlink interno (relación creada + insertado en el rPr del run); False si algo lo impidió (run inválido, slides de presentaciones distintas, etc.). Nunca lanza." +tested: true +tests: ["test_golden_run_se_vuelve_salto_a_otra_slide", "test_idempotente_reaplica_sin_duplicar_hlinkclick", "test_error_path_run_invalido_devuelve_false_sin_lanzar"] +test_file_path: "python/functions/datascience/pptx_link_run_to_slide_test.py" +file_path: "python/functions/datascience/pptx_link_run_to_slide.py" +--- + +## Ejemplo + +```python +from pptx import Presentation +from pptx.util import Inches +from pptx.oxml.ns import qn + +from datascience.pptx_link_run_to_slide import pptx_link_run_to_slide + +prs = Presentation() +blank = prs.slide_layouts[6] # layout en blanco +slide0 = prs.slides.add_slide(blank) +slide1 = prs.slides.add_slide(blank) # destino del salto (p.ej. el glosario) + +box = slide0.shapes.add_textbox(Inches(1), Inches(1), Inches(4), Inches(1)) +run = box.text_frame.paragraphs[0].add_run() +run.text = "ir al glosario" + +ok = pptx_link_run_to_slide(run, slide0, slide1) +print(ok) # -> True + +# El run quedó con +hlink = run._r.get_or_add_rPr().find(qn("a:hlinkClick")) +print(hlink.get("action")) # -> ppaction://hlinksldjump +prs.save("deck_con_salto.pptx") +``` + +## Cuando usarla + +Cuando construyas un deck PPTX con **navegación interna** y quieras que un texto salte a +otra diapositiva al pulsarlo: un **glosario clicable** (cada término enlaza a su slide de +definición), un **índice/tabla de contenidos navegable**, botones "volver a la portada", o +referencias cruzadas entre capítulos. Es la pieza que `python-pptx` no cubre de fábrica — +úsala sobre los runs ya creados por renderers como `render_automatic_eda_pptx` del grupo +`eda` para enriquecer el deck con saltos sin reescribir el XML a mano cada vez. + +## Gotchas + +- **Impura**: muta el XML del run y crea una relación nueva en el part de `source_slide`. +- **Solo navega en visores que respetan `ppaction://hlinksldjump`**: PowerPoint y la + mayoría de visores compatibles lo siguen; algunos visores web/ligeros lo ignoran (el + texto se ve igual pero no salta). +- **Mismo Presentation**: `source_slide` y `target_slide` deben pertenecer al mismo deck. + Si son de presentaciones distintas, la relación interna no es válida y el salto no + funcionará (la función puede devolver True por crear la relación, pero el resultado en + el visor no será el esperado). +- **El `` vive en el `` del run**, no como hijo directo del ``. + Para localizarlo: `run._r.get_or_add_rPr().find(qn("a:hlinkClick"))` (un `find` sobre + `run._r` devuelve `None` porque solo mira hijos directos del ``). +- **Idempotente**: si el run ya tenía un `hlinkClick` (p.ej. una URL externa o un salto + previo), se elimina antes de insertar el nuevo — un run tiene como mucho un click-link. +- **Nunca lanza**: cualquier excepción (run sin `._r`, slides incompatibles, etc.) se + traga y devuelve `False`. Comprobar el booleano si el salto es crítico. +- **Dependencia python-pptx**: declarada en `python/pyproject.toml`. Tests con + `~/fn_registry/python/.venv/bin/python3` (tiene `python-pptx` instalado). diff --git a/python/functions/datascience/pptx_link_run_to_slide.py b/python/functions/datascience/pptx_link_run_to_slide.py new file mode 100644 index 00000000..bc3cfd1c --- /dev/null +++ b/python/functions/datascience/pptx_link_run_to_slide.py @@ -0,0 +1,50 @@ +"""Convierte un run de texto de python-pptx en un hyperlink interno "ir a la diapositiva". + +python-pptx expone ``run.hyperlink.address`` para URLs externas, pero NO ofrece una +API pública para saltar a otra diapositiva del mismo deck. Esta función crea ese salto +interno manipulando el XML: añade una relación ``slide -> slide`` y un +```` con la acción ``ppaction://hlinksldjump`` en el run, de modo que al +pulsar el texto en PowerPoint (o en visores que respetan esa acción) se navega a la +diapositiva de destino. +""" + +from pptx.opc.constants import RELATIONSHIP_TYPE as RT +from pptx.oxml.ns import qn + + +def pptx_link_run_to_slide(run, source_slide, target_slide) -> bool: + """Convierte un run de texto en un hyperlink interno "ir a la diapositiva". + + Añade una relación ``slide -> slide`` desde la slide origen al part de la slide + destino y crea un ```` con ``action="ppaction://hlinksldjump"`` como + primer hijo del ```` del run (orden válido del schema + ``CT_TextCharacterProperties``). La operación es idempotente: un ``hlinkClick`` + previo en el mismo run se elimina antes de insertar el nuevo. + + Args: + run: el ``pptx.text.text._Run`` cuyo texto se vuelve clicable. + source_slide: la ``Slide`` que contiene el run. + target_slide: la ``Slide`` de destino del salto. + + Returns: + True si se aplicó el hyperlink; False si algo impidió aplicarlo (no lanza). + """ + try: + rId = source_slide.part.relate_to(target_slide.part, RT.SLIDE) + rPr = run._r.get_or_add_rPr() + # Elimina un hlinkClick previo si lo hubiera (idempotente). + for existing in rPr.findall(qn("a:hlinkClick")): + rPr.remove(existing) + hlink = rPr.makeelement( + qn("a:hlinkClick"), + { + qn("r:id"): rId, + "action": "ppaction://hlinksldjump", + }, + ) + # a:hlinkClick debe ir como primer hijo de rPr + # (orden del schema CT_TextCharacterProperties). + rPr.insert(0, hlink) + return True + except Exception: + return False diff --git a/python/functions/datascience/pptx_link_run_to_slide_test.py b/python/functions/datascience/pptx_link_run_to_slide_test.py new file mode 100644 index 00000000..ccfc65dd --- /dev/null +++ b/python/functions/datascience/pptx_link_run_to_slide_test.py @@ -0,0 +1,73 @@ +"""Tests for pptx_link_run_to_slide — salto interno run -> diapositiva. + +Self-contained: construye una Presentation en memoria con dos slides en blanco, +un textbox con un run en la slide 0, y verifica que la función inyecta un +```` con ``action="ppaction://hlinksldjump"`` y un ``r:id`` que +resuelve al part de la slide 1. +""" + +import pytest + +pytest.importorskip("pptx") + +from pptx import Presentation # noqa: E402 +from pptx.oxml.ns import qn # noqa: E402 +from pptx.util import Inches # noqa: E402 + +from datascience.pptx_link_run_to_slide import pptx_link_run_to_slide # noqa: E402 + + +def _two_slide_deck_with_run(): + prs = Presentation() + blank = prs.slide_layouts[6] # layout en blanco + slide0 = prs.slides.add_slide(blank) + slide1 = prs.slides.add_slide(blank) + + box = slide0.shapes.add_textbox(Inches(1), Inches(1), Inches(4), Inches(1)) + tf = box.text_frame + para = tf.paragraphs[0] + run = para.add_run() + run.text = "ir al glosario" + return prs, slide0, slide1, run + + +def test_golden_run_se_vuelve_salto_a_otra_slide(): + prs, slide0, slide1, run = _two_slide_deck_with_run() + + ok = pptx_link_run_to_slide(run, slide0, slide1) + assert ok is True + + # El hlinkClick es hijo del rPr del run (orden del schema + # CT_TextCharacterProperties), no hijo directo del . + rPr = run._r.get_or_add_rPr() + hlink = rPr.find(qn("a:hlinkClick")) + assert hlink is not None + assert hlink.get("action") == "ppaction://hlinksldjump" + + rId = hlink.get(qn("r:id")) + assert rId, "el hlinkClick debe llevar un r:id no vacío" + + # El rId debe existir en las relaciones de la slide origen y apuntar + # al part de la slide destino. + rels = slide0.part.rels + assert rId in rels + assert rels[rId].target_part is slide1.part + + +def test_idempotente_reaplica_sin_duplicar_hlinkclick(): + prs, slide0, slide1, run = _two_slide_deck_with_run() + + assert pptx_link_run_to_slide(run, slide0, slide1) is True + assert pptx_link_run_to_slide(run, slide0, slide1) is True + + rPr = run._r.get_or_add_rPr() + hlinks = rPr.findall(qn("a:hlinkClick")) + assert len(hlinks) == 1 + + +def test_error_path_run_invalido_devuelve_false_sin_lanzar(): + prs, slide0, slide1, _run = _two_slide_deck_with_run() + + # Un objeto sin ._r ni soporte de relación -> la función no lanza, devuelve False. + ok = pptx_link_run_to_slide(object(), slide0, slide1) + assert ok is False diff --git a/python/pyproject.toml b/python/pyproject.toml index 9553fbe8..052f7280 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -25,6 +25,7 @@ dependencies = [ "polars>=1.40.1", "pymeshlab>=2025.7.post1", "pymssql>=2.3.13", + "pymupdf>=1.28.0", "pypdf>=6.10.0", "pyproj>=3.7.2", "python-docx>=1.2.0", From b1d205203adaa0474daba19d3ed70febf683dde2 Mon Sep 17 00:00:00 2001 From: Egutierrez Date: Tue, 30 Jun 2026 17:56:24 +0200 Subject: [PATCH 22/53] feat(eda): poblar head_rows real en el capitulo OVERVIEW (df.head) El capitulo OVERVIEW del motor AutomaticEDA mostraba "df.head no disponible" porque ninguna fase de calculo poblaba las primeras filas crudas de la tabla. - build_eda_render_ctx: nuevo bloque que muestrea SELECT * LIMIT head_n (param nuevo head_n=10) y lo expone en ctx["head_rows"] como lista de dicts fila. Estilo dict-no-throw: si la query falla, se omite la clave. - profile_table: puebla prof["head_rows"] reusando _sample_rows (SELECT de las columnas LIMIT 10) tras recalcular el type_breakdown. Asi el report JSON sidecar tambien lo lleva y el capitulo lo recoge via profile aunque no se construya el ctx. - overview.py: la nota del DataTable de df.head ahora indica el total de filas del dataset cuando se conoce ("primeras 10 filas de 891"). Bump CHAPTER_VERSION 1.0.0 -> 1.1.0. - overview_test.py (nuevo): golden (head via profile y via ctx, render PDF + PPTX muestran las filas reales, placeholder ausente), edge (sin head_rows degrada a nota honesta sin romper, None/vacio devuelven None). Verificado end-to-end con titanic: render_automatic_eda emite PDF + PPTX con df.head visible (Braund/Cumings/Heikkinen + columnas) y sin el placeholder. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../automatic_eda/chapters/overview.py | 12 +- .../automatic_eda/chapters/overview_test.py | 187 ++++++++++++++++++ .../datascience/build_eda_render_ctx.py | 32 ++- python/functions/pipelines/profile_table.py | 15 ++ 4 files changed, 239 insertions(+), 7 deletions(-) create mode 100644 python/functions/datascience/automatic_eda/chapters/overview_test.py diff --git a/python/functions/datascience/automatic_eda/chapters/overview.py b/python/functions/datascience/automatic_eda/chapters/overview.py index 93b25b52..f3dc8b53 100644 --- a/python/functions/datascience/automatic_eda/chapters/overview.py +++ b/python/functions/datascience/automatic_eda/chapters/overview.py @@ -20,7 +20,7 @@ from __future__ import annotations from .. import model -CHAPTER_VERSION = "1.0.0" +CHAPTER_VERSION = "1.1.0" CHAPTER_ID = "overview" CHAPTER_TITLE = "Overview" @@ -90,8 +90,14 @@ def _head_block(profile: dict, ctx: dict): if not cols: cols = list(head[0].keys()) rows = [[model._safe_str(r.get(c)) for c in cols] for r in head[:10]] - return model.DataTable(header=cols, rows=rows, - note=f"primeras {len(rows)} filas") + # Honest note: how many rows are shown and, when known, out of how many + # rows the dataset has (so "primeras 10 filas de 891" gives context). + note = f"primeras {len(rows)} filas" + n_rows = profile.get("n_rows") + if isinstance(n_rows, int) and not isinstance(n_rows, bool) \ + and n_rows > len(rows): + note += f" de {n_rows:,}".replace(",", ".") + return model.DataTable(header=cols, rows=rows, note=note) return model.Note( "df.head no disponible: el TableProfile no incluye 'head_rows'. La fase " "de cálculo debe añadir profile['head_rows'] (lista de dicts fila) o " diff --git a/python/functions/datascience/automatic_eda/chapters/overview_test.py b/python/functions/datascience/automatic_eda/chapters/overview_test.py new file mode 100644 index 00000000..b66263a1 --- /dev/null +++ b/python/functions/datascience/automatic_eda/chapters/overview_test.py @@ -0,0 +1,187 @@ +"""Tests for the OVERVIEW chapter — DoD: golden + edges + degradation. + +Self-contained: builds synthetic TableProfiles (no DuckDB) so the suite is fast +and deterministic. Verifies that ``build_overview`` renders the raw first rows +(``df.head``) as a DataTable when ``head_rows`` is present — both when it arrives +via ``profile['head_rows']`` (populated by ``profile_table``) and via +``ctx['head_rows']`` (populated by ``build_eda_render_ctx``) — that the chapter +also renders the column dictionary and the numeric describe, that the full +document renders to PDF and PPTX showing the head values, and that a profile with +NO head data degrades to an honest note instead of raising or inventing rows. +""" + +import os +import re +import tempfile + +from pypdf import PdfReader +from pptx import Presentation + +from datascience.automatic_eda.model import DataTable, Note +from datascience.automatic_eda.chapters.overview import ( + CHAPTER_ID, CHAPTER_VERSION, build_overview, +) +from datascience.render_automatic_eda_pdf import render_automatic_eda_pdf +from datascience.render_automatic_eda_pptx import render_automatic_eda_pptx + + +def _columns() -> list: + return [ + {"name": "PassengerId", "inferred_type": "numeric", "null_pct": 0.0, + "null_count": 0, "numeric": {"mean": 2.0, "median": 2.0, "min": 1.0, + "max": 3.0, "std": 1.0}}, + {"name": "Survived", "inferred_type": "numeric", "null_pct": 0.0, + "null_count": 0, "numeric": {"mean": 0.33, "median": 0.0, "min": 0.0, + "max": 1.0, "std": 0.58}}, + {"name": "Pclass", "inferred_type": "numeric", "null_pct": 0.0, + "null_count": 0, "numeric": {"mean": 2.33, "median": 3.0, "min": 1.0, + "max": 3.0, "std": 1.15}}, + {"name": "Name", "inferred_type": "categorical", "null_pct": 0.0, + "null_count": 0, "distinct_count": 3}, + {"name": "Sex", "inferred_type": "categorical", "null_pct": 0.0, + "null_count": 0, "distinct_count": 2, + "categorical": {"top": [{"value": "male", "count": 2}, + {"value": "female", "count": 1}]}}, + ] + + +def _head_rows() -> list: + return [ + {"PassengerId": 1, "Survived": 0, "Pclass": 3, + "Name": "Braund Owen", "Sex": "male"}, + {"PassengerId": 2, "Survived": 1, "Pclass": 1, + "Name": "Cumings Florence", "Sex": "female"}, + {"PassengerId": 3, "Survived": 1, "Pclass": 3, + "Name": "Heikkinen Laina", "Sex": "female"}, + ] + + +def _profile(with_head: bool = True) -> dict: + prof = { + "table": "titanic", + "source": "/data/titanic.csv", + "profiled_at": "2026-06-30T10:00:00+00:00", + "n_rows": 891, + "n_cols": 5, + "quality_score": 88.0, + "columns": _columns(), + } + if with_head: + prof["head_rows"] = _head_rows() + return prof + + +def _pdf_text(path: str) -> str: + txt = "".join((pg.extract_text() or "") for pg in PdfReader(path).pages) + return re.sub(r"\s+", " ", txt) + + +def _pptx_text(path: str) -> str: + prs = Presentation(path) + parts = [] + for sl in prs.slides: + for sh in sl.shapes: + if sh.has_text_frame: + parts.append(sh.text_frame.text) + if sh.has_table: + tb = sh.table + for r in range(len(tb.rows)): + for c in range(len(tb.columns)): + parts.append(tb.cell(r, c).text) + return re.sub(r"\s+", " ", " ".join(parts)) + + +def _flatten(blocks): + """Recursively flatten Group blocks into a flat list (none here today).""" + out = [] + for b in blocks: + inner = getattr(b, "blocks", None) + if inner is not None and getattr(b, "kind", None) == "group": + out.extend(_flatten(inner)) + else: + out.append(b) + return out + + +def test_golden_build_overview_muestra_head_desde_profile(): + ch = build_overview(_profile(), {}) + assert ch is not None + assert ch.id == CHAPTER_ID + assert ch.version == CHAPTER_VERSION + blocks = _flatten(ch.blocks) + # The first DataTable is df.head: its header is the column names and the + # real first rows are present (not a placeholder note). + tables = [b for b in blocks if isinstance(b, DataTable)] + assert tables, "overview must emit at least the df.head DataTable" + head_tbl = tables[0] + assert head_tbl.header == ["PassengerId", "Survived", "Pclass", + "Name", "Sex"] + assert len(head_tbl.rows) == 3 + flat = [str(c) for row in head_tbl.rows for c in row] + assert "Braund Owen" in flat and "Cumings Florence" in flat + # Honest note carries how many rows shown out of the dataset total. + assert head_tbl.note is not None + assert "primeras 3 filas" in head_tbl.note and "891" in head_tbl.note + # No "df.head no disponible" placeholder when head_rows is present. + assert not any(isinstance(b, Note) and "no disponible" in b.text + for b in blocks) + + +def test_golden_head_desde_ctx_tambien_funciona(): + # head_rows absent in profile but present in ctx (build_eda_render_ctx path). + prof = _profile(with_head=False) + ch = build_overview(prof, {"head_rows": _head_rows()}) + assert ch is not None + tables = [b for b in _flatten(ch.blocks) if isinstance(b, DataTable)] + flat = [str(c) for row in tables[0].rows for c in row] + assert "Braund Owen" in flat + + +def test_golden_render_pdf_muestra_head(): + with tempfile.TemporaryDirectory() as d: + out = os.path.join(d, "eda.pdf") + res = render_automatic_eda_pdf(_profile(), out, {"title": "EDA"}) + assert res["path"] == out and os.path.exists(out) + assert CHAPTER_ID in [c["id"] for c in res["chapters"]] + txt = _pdf_text(out) + assert "Braund" in txt and "male" in txt + assert "primeras" in txt # head note rendered. + assert "df.head" in txt # chapter heading rendered. + assert "no disponible" not in txt # placeholder NOT shown. + + +def test_golden_render_pptx_muestra_head(): + with tempfile.TemporaryDirectory() as d: + out = os.path.join(d, "eda.pptx") + res = render_automatic_eda_pptx(_profile(), out, {"title": "EDA"}) + assert res["path"] == out and os.path.exists(out) + assert CHAPTER_ID in [c["id"] for c in res["chapters"]] + txt = _pptx_text(out) + assert "Braund" in txt and "Cumings" in txt + + +def test_edge_sin_head_rows_degrada_a_nota_honesta(): + # No head data anywhere: chapter still builds (columns exist), shows the + # honest placeholder note, and never invents rows nor raises. + prof = _profile(with_head=False) + ch = build_overview(prof, {}) + assert ch is not None + blocks = _flatten(ch.blocks) + assert any(isinstance(b, Note) and "no disponible" in b.text + for b in blocks) + # The first DataTable now is the column dictionary, not df.head rows. + tables = [b for b in blocks if isinstance(b, DataTable)] + assert all("Braund" not in str(c) + for tbl in tables for row in tbl.rows for c in row) + + +def test_edge_none_y_vacio_no_rompen(): + # Nothing to render at all -> None, no raise. + assert build_overview(None, None) is None + assert build_overview({}, {}) is None + assert build_overview({"columns": []}, {}) is None + # Only head_rows (no columns) still yields a chapter with the head table. + ch = build_overview({"columns": []}, {"head_rows": _head_rows()}) + assert ch is not None + tables = [b for b in _flatten(ch.blocks) if isinstance(b, DataTable)] + assert tables and len(tables[0].rows) == 3 diff --git a/python/functions/datascience/build_eda_render_ctx.py b/python/functions/datascience/build_eda_render_ctx.py index efcda2cb..68959abf 100644 --- a/python/functions/datascience/build_eda_render_ctx.py +++ b/python/functions/datascience/build_eda_render_ctx.py @@ -20,6 +20,10 @@ vacia y el resto del ctx se construye igual. Ante un fallo global devuelve al menos ``{**base_ctx, "db_path": db_path, "table": table}``. Claves de DATOS que produce (las consumen los capitulos): + - ``head_rows`` : [ {col: valor, ...}, ... ] primeras filas CRUDAS de la + tabla (``SELECT * LIMIT head_n``), una entrada por fila. + La lee el capitulo OVERVIEW para mostrar df.head real en + lugar del placeholder "df.head no disponible". - ``raw_numeric`` : {col: [float|None, ...]} muestra cruda de las columnas numericas, ALINEADA POR FILA (una entrada por fila aunque sea None). La leen modelos (clustering 2D en vivo) y @@ -56,7 +60,7 @@ def _to_float(value): return None -def build_eda_render_ctx(db_path, table, profile, backend="duckdb", sample=5000, base_ctx=None): +def build_eda_render_ctx(db_path, table, profile, backend="duckdb", sample=5000, base_ctx=None, head_n=10): """Construye el ctx de datos crudos para los renderers de AutomaticEDA. Args: @@ -77,13 +81,15 @@ def build_eda_render_ctx(db_path, table, profile, backend="duckdb", sample=5000, base_ctx: dict opcional con claves de presentacion ya preparadas (dataset_name, source_origin, ...). Se parte de una copia y NO se pisan sus claves; solo se añaden las de datos. Default None -> {}. + head_n: numero de filas crudas a muestrear para ``ctx["head_rows"]`` + (df.head del capitulo OVERVIEW). Default 10. <=0 omite la clave. Returns: El dict ``ctx`` directamente (NO un wrapper {status,...}): se pasa tal cual como ``meta={"ctx": }`` a render_automatic_eda_pdf/pptx. - Nunca lanza. Claves que puede contener: raw_numeric, timeseries_raw, - geo_points (omitidas si no aplican o fallan), y siempre db_path + table - para backends validos. + Nunca lanza. Claves que puede contener: head_rows, raw_numeric, + timeseries_raw, geo_points (omitidas si no aplican o fallan), y siempre + db_path + table para backends validos. """ # Copia de base_ctx: nunca mutamos el dict del caller. Las claves de # presentacion que ya traiga se conservan; las de datos se añaden encima. @@ -117,6 +123,24 @@ def build_eda_render_ctx(db_path, table, profile, backend="duckdb", sample=5000, ctx["db_path"] = db_path ctx["table"] = table + # 1.5) head_rows: primeras filas CRUDAS de la tabla (SELECT * LIMIT n) + # para que el capitulo OVERVIEW muestre df.head real en vez del + # placeholder. Una sola query, dict-no-throw: si falla, se omite la + # clave (el capitulo degrada a su nota honesta). No se pisa una clave + # head_rows que ya viniera en base_ctx (presentacion). + if head_n and int(head_n) > 0 and "head_rows" not in ctx: + try: + hq = query_fn(f'SELECT * FROM "{table}" LIMIT {int(head_n)}') + if isinstance(hq, dict) and hq.get("status") == "ok": + hrows = [ + dict(r) for r in (hq.get("rows") or []) + if isinstance(r, dict) + ] + if hrows: + ctx["head_rows"] = hrows + except Exception: # noqa: BLE001 - dict-no-throw: omitir la clave + pass + # 2) Columnas del perfil agregado (lectura defensiva). cols = profile.get("columns") if isinstance(profile, dict) else None cols = cols or [] diff --git a/python/functions/pipelines/profile_table.py b/python/functions/pipelines/profile_table.py index 8a0077af..84912d11 100644 --- a/python/functions/pipelines/profile_table.py +++ b/python/functions/pipelines/profile_table.py @@ -536,6 +536,21 @@ def profile_table( type_breakdown[it] += 1 prof["type_breakdown"] = type_breakdown + # 8.1) Primeras filas crudas (df.head) para el capitulo OVERVIEW del motor + # AutomaticEDA: una muestra SELECT col1,col2,... LIMIT 10 alineada por fila. + # Se reusa _sample_rows (mismo lector read-only). Estilo dict-no-throw: si + # falla, head_rows queda None y el capitulo degrada a su nota honesta. El + # capitulo lo recoge via profile["head_rows"]; build_eda_render_ctx ademas + # lo replica en ctx["head_rows"] cuando se construye el contexto de render. + try: + head_names = [c.get("name") for c in cols if c.get("name")] + head_rows = _sample_rows(_q, table, head_names, 10) + prof["head_rows"] = [ + dict(r) for r in head_rows if isinstance(r, dict) + ] or None + except Exception: # noqa: BLE001 + prof["head_rows"] = None + # 8.5) Matriz de correlacion/asociacion sobre una muestra de filas # alineadas. Elige la metrica por par de tipos (Pearson/Spearman, # Cramer's V/Theil's U, correlation ratio, MI) via association_matrix. From 7fb00defdf585512cf76942573efacc9b8353d5c Mon Sep 17 00:00:00 2001 From: Egutierrez Date: Tue, 30 Jun 2026 17:53:44 +0200 Subject: [PATCH 23/53] fix(fleetclaude): reusar contexto dentro de la flota tmux en vez de abrir kitty nueva MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Lanzar `fleetclaude` estando ya dentro de una flota tmux viva abría una ventana kitty nueva (y creaba un perfil/socket nuevo fleetN+1) en vez de mostrar la flota en el pane actual. Causa: con $TMUX definido el launcher saltaba el `exec tmux attach` y caía a la rama `setsid kitty`. Cambio: cuando se invoca sin --new desde dentro de una flota fleetview viva (el socket actual, derivado de $TMUX, tiene una sesión homónima con window 'console'), se trae la TUI al contexto/pane actual (`fleetview show`, o `tmux select-window` de la window console como fallback sin binario) y se retorna 0 antes de las ramas kitty/wt.exe. Nuevo flag --new para forzar el comportamiento clásico (flota+ventana nueva) aun dentro de tmux; pasar --session con un nombre distinto al perfil actual equivale a --new implícito. Fuera de tmux el comportamiento es intacto (exec tmux attach reutiliza la terminal). Fix incidental: `local left_pane="" right_pane=""` (antes `local left_pane right_pane` reventaba con "unbound variable" bajo `set -u` al reutilizar una sesión existente, p. ej. con --reuse/--session sobre una flota viva). Verificación e2e con sockets aislados fctest* (sin tocar la flota del humano): golden (reuse, exit 0, kitty invariante), --new y --session-distinto (no reuse, ruta ventana-nueva), fuera de tmux (salta reuse, ruta attach). bash -n limpio. Docs: launch_fleetclaude.md (signature, params --new, ejemplo, cuando usarla, gotchas, growth log v1.7.0) + /fleet show en .claude/commands/fleet.md. Co-Authored-By: Claude Opus 4.8 (1M context) --- .claude/commands/fleet.md | 29 +++++- bash/functions/infra/launch_fleetclaude.md | 101 +++++++++++++++------ bash/functions/infra/launch_fleetclaude.sh | 63 ++++++++++++- 3 files changed, 160 insertions(+), 33 deletions(-) diff --git a/.claude/commands/fleet.md b/.claude/commands/fleet.md index b9a46356..b7a12f48 100644 --- a/.claude/commands/fleet.md +++ b/.claude/commands/fleet.md @@ -1,6 +1,6 @@ --- -description: Muestra la flota de Claudes vivos (sessionId + objetivo + estado) y, con argumento, salta con foco a esa conversación dentro de la sesión tmux fleet. -argument-hint: "[texto|sessionId|PID para saltar — vacío = listar la flota]" +description: Muestra la flota de Claudes vivos (sessionId + objetivo + estado) y, con argumento, salta con foco a esa conversación dentro de la sesión tmux fleet. `/fleet show` trae la TUI al contexto tmux actual. +argument-hint: "[show | texto|sessionId|PID para saltar — vacío = listar la flota]" --- # /fleet — ver y navegar la flota de Claudes @@ -33,9 +33,32 @@ cd "${FN_REGISTRY_ROOT:-$HOME/fn_registry}/apps/fleetview" && go build -o fleetv - la sesión actual / orquestador si la puedes identificar (su `session_id` coincide con el de quien invoca). 4. Si la lista está vacía, indícalo y sugiere que el perfil fleet podría no estar activo (revisar `$FLEET_SOCKET` y que la sesión tmux exista). +### `show` → traer la TUI al contexto tmux actual + +Si `$ARGUMENTS` es exactamente `show` (alias `open`/`attach`), el usuario quiere +volver a ver el panel FleetView en el contexto/pane actual sin abrir ninguna +ventana ni arrancar una flota nueva. Ejecuta: + +```bash +"${FN_REGISTRY_ROOT:-$HOME/fn_registry}/apps/fleetview/fleetview" show +``` + +Comportamiento (decidido por la app, no abre terminal externa): + +- **dentro de tmux con la flota viva** → `select-window` de la window `console` + del socket fleet (trae la TUI al frente; no abre nada). +- **fuera de tmux** → `attach` a la sesión fleet en la terminal actual (la reutiliza). +- **sin flota viva** → error claro, exit 1, no abre nada (sugiere arrancarla con + `fleetclaude`). + +Es el equivalente del comportamiento de `fleetclaude` sin args invocado dentro de +una flota viva (reuse de contexto): úsalo cuando ya tengas una flota corriendo y +solo quieras recuperar la vista del panel. Para abrir una flota NUEVA aparte, usa +`fleetclaude --new` (no este comando). + ### Con argumentos → saltar con foco -El usuario quiere que la interfaz tmux salte a una conversación concreta. `$ARGUMENTS` es el query: texto del objetivo, prefijo de `sessionId`, o PID. +El usuario quiere que la interfaz tmux salte a una conversación concreta. `$ARGUMENTS` es el query: texto del objetivo, prefijo de `sessionId`, o PID (cualquier valor que no sea `show`). 1. Ejecuta: ```bash diff --git a/bash/functions/infra/launch_fleetclaude.md b/bash/functions/infra/launch_fleetclaude.md index bc733ac8..652d9ffc 100644 --- a/bash/functions/infra/launch_fleetclaude.md +++ b/bash/functions/infra/launch_fleetclaude.md @@ -3,10 +3,10 @@ name: launch_fleetclaude kind: function lang: bash domain: infra -version: "1.6.0" +version: "1.7.0" purity: impure -signature: "launch_fleetclaude [--cwd ] [--bin ] [--session ] [--reuse] [--cols ]" -description: "Entrypoint de FleetView: abre una ventana de terminal con una sesion tmux (socket aislado por perfil) de dos panes (TUI fleetview a la izquierda, claude --dangerously-skip-permissions a la derecha) para centralizar la flota de Claudes. La terminal se AUTO-DETECTA sin config por PC: kitty si esta instalado y hay display ($DISPLAY/$WAYLAND_DISPLAY), si no Windows Terminal (wt.exe) en WSL adjuntando via wsl.exe. El pane de la TUI corre dentro del bucle supervisor supervise_fleetview_tui, que la relanza si muere (crash/panic/kill), asi el panel de control NUNCA se pierde. Soporta PERFILES multiples: sin --session/--reuse cada invocacion abre un perfil nuevo (fleet, fleet2, fleet3, ...) con su propia flota; inyecta FLEET_SOCKET/FLEET_SESSION a la TUI para que cada panel vea solo sus Claudes. Instala atajos alt+flechas/alt+enter/alt+n que controlan la TUI desde cualquier pane, y fija el ancho del sidebar con hooks." +signature: "launch_fleetclaude [--cwd ] [--bin ] [--session ] [--reuse] [--new] [--cols ]" +description: "Entrypoint de FleetView: abre una ventana de terminal con una sesion tmux (socket aislado por perfil) de dos panes (TUI fleetview a la izquierda, claude --dangerously-skip-permissions a la derecha) para centralizar la flota de Claudes. REUSO DE CONTEXTO: si se invoca DENTRO de una flota tmux viva (su window 'console') sin --new, NO abre ventana ni crea un perfil nuevo; trae la TUI al pane/contexto actual (equivale a 'fleetview show'). El flag --new fuerza una flota+ventana nueva aunque estes en tmux. La terminal se AUTO-DETECTA sin config por PC: kitty si esta instalado y hay display ($DISPLAY/$WAYLAND_DISPLAY), si no Windows Terminal (wt.exe) en WSL adjuntando via wsl.exe. El pane de la TUI corre dentro del bucle supervisor supervise_fleetview_tui, que la relanza si muere (crash/panic/kill), asi el panel de control NUNCA se pierde. Soporta PERFILES multiples: fuera de tmux, o con --new, cada invocacion abre un perfil nuevo (fleet, fleet2, fleet3, ...) con su propia flota; inyecta FLEET_SOCKET/FLEET_SESSION a la TUI para que cada panel vea solo sus Claudes. Instala atajos alt+flechas/alt+enter/alt+n que controlan la TUI desde cualquier pane, y fija el ancho del sidebar con hooks." tags: [claude-fleet, infra, kitty, tmux, claude, fleetview, launcher, wsl, windows-terminal] params: - name: --cwd @@ -14,12 +14,14 @@ params: - name: --bin desc: "Ruta al binario de la TUI fleetview que corre en el pane izquierdo. Opcional. Default: /apps/fleetview/fleetview. Si no es ejecutable, el pane izquierdo muestra un mensaje de como compilarla y deja una shell viva." - name: --session - desc: "Fija el perfil (socket+sesion tmux comparten nombre) por nombre exacto; reutiliza el existente si ya vive (idempotente sobre ese nombre). Opcional. Sin esta opcion, el perfil se elige automaticamente (primer nombre libre de la secuencia fleet, fleet2, ...)." + desc: "Fija el perfil (socket+sesion tmux comparten nombre) por nombre exacto; reutiliza el existente si ya vive (idempotente sobre ese nombre). Opcional. Sin esta opcion, el perfil se elige automaticamente (primer nombre libre de la secuencia fleet, fleet2, ...). Invocado DENTRO de tmux con un nombre DISTINTO al de la flota actual equivale a --new (pides otra flota: ventana nueva, sin reuse de contexto)." - name: --reuse desc: "Reattach al perfil principal 'fleet' en vez de abrir uno nuevo. Opcional. Recupera el comportamiento idempotente clasico (volver a invocar NO duplica la flota, reusa la existente)." + - name: --new + desc: "Fuerza una flota NUEVA en una ventana NUEVA (kitty/wt.exe) incluso estando dentro de una flota tmux. Opcional. Es la via explicita para abrir una FleetView aparte; sin este flag, invocado dentro de una flota viva se reusa el contexto actual (no abre ventana ni crea perfil)." - name: --cols desc: "Ancho en columnas del pane izquierdo (la TUI). Opcional. Default: 40." -output: "Crea/reutiliza una sesion tmux detached con dos panes y lanza una ventana de terminal 'FleetView' adjunta a ella (kitty o Windows Terminal segun auto-deteccion), desacoplada del shell padre. Imprime el estado por stdout. Sin valor de retorno; exit 0 en exito." +output: "Caso reuse de contexto (dentro de una flota tmux viva, sin --new): trae la TUI al pane/contexto actual con select-window de la window 'console' (o 'fleetview show' si el binario existe) y retorna 0, sin abrir nada. Caso ventana-nueva (fuera de tmux, o con --new): crea/reutiliza una sesion tmux detached con dos panes y lanza una ventana de terminal 'FleetView' adjunta (kitty o Windows Terminal segun auto-deteccion), desacoplada del shell padre. Imprime el estado por stdout. Sin valor de retorno; exit 0 en exito, !=0 con mensaje claro si no hay terminal ni contexto que reusar." uses_functions: - supervise_fleetview_tui_bash_infra uses_types: [] @@ -36,32 +38,44 @@ file_path: "bash/functions/infra/launch_fleetclaude.sh" ## Ejemplo ```bash -# Via fn run (resuelve por nombre o ID): -fn run launch_fleetclaude +# DENTRO de una flota tmux viva (p. ej. en el pane del orquestador): reusa el +# contexto, trae la TUI al pane actual. NO abre ventana ni crea perfil nuevo. +fleetclaude -# Perfil nuevo automatico (fleet la 1a vez; fleet2, fleet3, ... si ya hay uno): -launch_fleetclaude +# FUERA de tmux: perfil nuevo automatico (fleet la 1a vez; fleet2, ... si ya hay +# uno) en una ventana de terminal nueva, reutilizando la terminal actual (attach): +fleetclaude + +# Forzar una flota+ventana NUEVA aunque estes dentro de una flota tmux: +fleetclaude --new # Reattach a la flota principal 'fleet' (comportamiento idempotente clasico): -launch_fleetclaude --reuse +fleetclaude --reuse # Perfil con nombre fijo y ancho de pane personalizado: -launch_fleetclaude --session trabajo --cols 50 +fleetclaude --session trabajo --cols 50 + +# Via fn run (resuelve por nombre o ID): +fn run launch_fleetclaude ``` -Tras invocarlo aparece una ventana de terminal titulada `FleetView ()` con dos -panes lado a lado: a la izquierda la TUI `fleetview`, a la derecha una sesion de -`claude --dangerously-skip-permissions`. Cada perfil es un socket+sesion tmux -aislados con su propia flota: puedes tener varias FleetView abiertas a la vez. -Por defecto, volver a invocarlo abre un perfil NUEVO (no reusa); usa `--reuse` -o `--session ` para volver a una flota concreta. +Dentro de una flota viva, `fleetclaude` sin args reusa el contexto (la window +`console` pasa al frente). Fuera de tmux (o con `--new`) aparece una ventana de +terminal titulada `FleetView ()` con dos panes lado a lado: a la izquierda +la TUI `fleetview`, a la derecha una sesion de `claude --dangerously-skip-permissions`. +Cada perfil es un socket+sesion tmux aislados con su propia flota: puedes tener +varias FleetView abiertas a la vez con `--new`. ## Cuando usarla Usala cuando quieras un unico punto de entrada a la flota de Claudes en vez de N ventanas kitty sueltas: lanzas `fleetclaude` y tienes la TUI de control y un Claude listo para trabajar en la misma ventana. Tipico al empezar la jornada o -al retomar el trabajo en el repo `fn_registry`. +al retomar el trabajo en el repo `fn_registry`. Si **ya estas dentro de una +flota** (en el pane del orquestador) y solo quieres volver a ver la TUI, lanza +`fleetclaude` sin args: trae el panel al contexto actual sin abrir otra ventana +ni arrancar una flota duplicada. Usa `--new` solo cuando quieras DELIBERADAMENTE +una segunda flota aparte. ## Gotchas @@ -87,10 +101,27 @@ al retomar el trabajo en el repo `fn_registry`. funciona en un PC con kitty y en otro WSL sin kitty, cada uno elige su terminal. Causa raiz del sintoma "se lanza la flota pero no se ve": kitty no instalado en WSL hacia que la sesion tmux se creara sin ventana que la mostrara. -- **Dentro de tmux abre ventana nueva**: si invocas `fleetclaude` desde dentro de - una sesion tmux (`$TMUX` definido), NO hace `attach` anidado (rompe / avisa de - nesting); cae a la ruta ventana-nueva (auto-deteccion de terminal). Fuera de - tmux y con TTY, reutiliza la terminal actual con `exec tmux attach`. +- **Dentro de una flota tmux viva: reuse de contexto (no ventana nueva)**: si + invocas `fleetclaude` sin `--new` desde dentro de una flota fleetview viva + (`$TMUX` definido y el socket actual tiene una sesion homonima con window + `console`), NO abre ventana ni crea un perfil `fleetN+1`: trae la TUI al pane + actual (`fleetview show`, o `tmux -L select-window -t :console` + si el binario no esta compilado) y retorna 0. El perfil de la flota actual se + deriva de `$TMUX` (basename del socket = nombre `-L`), senal fiable aunque + `$FLEET_SOCKET` venga vacio (ver `detect_fleet_context`). **`--new`** fuerza el + comportamiento clasico (flota+ventana nueva); pasar `--session ` distinto + al perfil actual equivale a `--new` implicito. Fuera de tmux y con TTY, reutiliza + la terminal actual con `exec tmux attach` (nunca `attach` anidado dentro de + tmux). Sin TTY ni contexto que reusar (atajo de escritorio/cron) cae a la ruta + ventana-nueva. Antes de este fix (v1.6.0 y anteriores) cualquier `fleetclaude` + dentro de tmux abria una kitty nueva y un socket `fleetN+1` — el sintoma que + acumulaba 6+ sockets `fleet*`. +- **`local x` unbound bajo `set -u`**: el archivo corre con `set -euo pipefail`. + `local left_pane right_pane` dejaba esas vars *unbound* (no vacias), asi que la + rama "reutilizar sesion existente" (`--reuse`/`--session `) reventaba con + `left_pane: unbound variable` al evaluar `[[ -z "$left_pane" ]]`. Se inicializan + explicitamente a `""` (`local left_pane="" right_pane=""`). Si tocas estas vars, + no vuelvas a declararlas sin valor. - **kitty detached (setsid)**: la ventana kitty se lanza con `setsid ... &` para sobrevivir al cierre de la terminal que la invoco. La ventana de Windows Terminal (wt.exe) ya es un proceso Windows independiente del arbol Linux, asi @@ -128,15 +159,29 @@ al retomar el trabajo en el repo `fn_registry`. - **Ancho del sidebar via hooks**: `client-resized` y `window-layout-changed` re-fijan el pane 0 (TUI) a `--cols` columnas, porque el `attach` de kitty y el conmutar de Claude redistribuyen el espacio. -- **tmux siempre; terminal (kitty/wt.exe) solo sin TTY**: `tmux` es obligatorio - (aborta != 0 si falta). Una terminal nueva (kitty o Windows Terminal) solo se - necesita en la ruta sin-TTY (dentro de tmux, atajo de escritorio, cron, script), - donde abre una ventana nueva. Invocado desde una terminal interactiva fuera de - tmux (el caso normal del alias `fleetclaude`), reutiliza la terminal actual con - `exec tmux attach` y no necesita ni kitty ni wt.exe. +- **tmux siempre; terminal (kitty/wt.exe) solo en la ruta ventana-nueva**: `tmux` + es obligatorio (aborta != 0 si falta). Una terminal nueva (kitty o Windows + Terminal) solo se necesita en la ruta ventana-nueva: `--new`, o sin TTY ni flota + viva que reusar (atajo de escritorio, cron, script). Dentro de una flota viva sin + `--new` se reusa el contexto (ni kitty ni wt.exe). Invocado desde una terminal + interactiva fuera de tmux (el caso normal del alias `fleetclaude`), reutiliza la + terminal actual con `exec tmux attach` y tampoco necesita kitty ni wt.exe. ## Capability growth log +- v1.7.0 (2026-06-30) — **reuse de contexto dentro de la flota + flag `--new`**. + Invocado sin `--new` desde dentro de una flota tmux viva (su window `console`), + `fleetclaude` ya NO abre una kitty nueva ni crea un perfil `fleetN+1`: trae la + TUI al pane/contexto actual (`fleetview show`, o `tmux -L select-window + -t :console` como fallback sin binario) y retorna 0. El perfil actual se + deriva de `$TMUX` (basename del socket); pasar `--session ` distinto al + actual equivale a `--new` implicito. Nuevo flag `--new` para forzar la ruta + clasica (flota+ventana nueva) aun dentro de tmux. Fuera de tmux el comportamiento + es intacto (`exec tmux attach` reutiliza la terminal). Arregla el sintoma de que + lanzar `fleetclaude` dentro de una flota abria ventana kitty + socket nuevo + (`fleet7`, `fleet8`, ...). Fix incidental: `local left_pane="" right_pane=""` + (antes `local left_pane right_pane` reventaba con `unbound variable` bajo + `set -u` al reutilizar una sesion existente). - v1.6.0 (2026-06-29) — **auto-deteccion de terminal (kitty ↔ Windows Terminal)**. La ruta ventana-nueva ya no asume kitty: elige terminal segun el host. kitty si esta instalado y hay display (`$DISPLAY`/`$WAYLAND_DISPLAY`); si no, en WSL abre diff --git a/bash/functions/infra/launch_fleetclaude.sh b/bash/functions/infra/launch_fleetclaude.sh index 853171e1..8850a6a0 100644 --- a/bash/functions/infra/launch_fleetclaude.sh +++ b/bash/functions/infra/launch_fleetclaude.sh @@ -23,6 +23,7 @@ launch_fleetclaude() { local cols=52 local explicit_session=0 # 1 si el usuario pasó --session a mano local reuse=0 # 1 si el usuario pidió --reuse (reattach al perfil principal) + local want_new=0 # 1 si el usuario pidió --new (forzar flota+ventana nueva) local T="" # socket tmux aislado; se fija al resolver el perfil # ----------------------------------------------------------------------- @@ -46,6 +47,9 @@ launch_fleetclaude() { --reuse) reuse=1 ;; + --new) + want_new=1 + ;; --cols) shift cols="${1:-40}" @@ -62,6 +66,11 @@ Claudes). Sin --session ni --reuse, cada invocacion abre un perfil NUEVO: usa el primer nombre libre de la secuencia fleet, fleet2, fleet3, ... Asi puedes tener varias FleetView abiertas a la vez, cada una con su flota independiente. +REUSO DE CONTEXTO: si ya estas DENTRO de una flota tmux viva (p. ej. en el pane +del orquestador), 'fleetclaude' sin args NO abre una ventana ni crea un perfil +nuevo: trae la TUI al contexto/pane actual (equivale a 'fleetview show'). Para +abrir explicitamente una flota aparte en una ventana nueva, usa --new. + Opciones: --cwd Directorio de trabajo de los panes. Default: raiz del repo fn_registry (derivada dinamicamente). @@ -69,13 +78,21 @@ Opciones: Default: /apps/fleetview/fleetview --session Fija el perfil (socket+sesion) por nombre exacto; reutiliza el existente si ya esta vivo. Sin esta opcion, perfil auto. + Si se invoca DENTRO de tmux con un nombre DISTINTO al de la + flota actual, equivale a --new (pides otra flota). --reuse Reattach al perfil principal 'fleet' en vez de abrir uno nuevo (vuelve al comportamiento idempotente clasico). + --new Fuerza una flota NUEVA en una ventana NUEVA (kitty/wt.exe), + incluso dentro de tmux. Es la via explicita para tener una + FleetView aparte; sin este flag, dentro de tmux se reusa el + contexto actual. --cols Ancho (columnas) del pane izquierdo. Default: 40. -h, --help Muestra esta ayuda. Ejemplos: - launch_fleetclaude # perfil nuevo (fleet, luego fleet2, ...) + launch_fleetclaude # dentro de la flota: reusa el contexto; + # fuera de tmux: perfil nuevo (fleet, ...) + launch_fleetclaude --new # flota+ventana nueva aunque estes en tmux launch_fleetclaude --reuse # reattach a la flota principal 'fleet' launch_fleetclaude --session trabajo # perfil con nombre fijo 'trabajo' launch_fleetclaude --cwd ~/fn_registry --cols 50 @@ -127,6 +144,45 @@ USAGE return 1 fi + # ----------------------------------------------------------------------- + # REUSO DE CONTEXTO (sin --new): si ya estamos DENTRO de una flota tmux + # viva, 'fleetclaude' sin args NO abre una ventana/terminal nueva ni crea + # un perfil fleetN+1 — trae la TUI al contexto/pane actual, igual que + # 'fleetview show'. El flag --new fuerza el comportamiento clasico (flota + # nueva en ventana nueva); --reuse mantiene su semantica historica. + # + # El perfil de la flota actual se deriva de $TMUX (el basename del socket + # es el nombre -L; senal fiable aunque $FLEET_SOCKET venga vacio, ver + # detect_fleet_context). Si se paso --session con un nombre DISTINTO al + # actual, es pedir OTRA flota -> se trata como --new implicito (no reusa). + # "Flota viva" = el socket tiene una sesion homonima con una window + # 'console' (la firma de una FleetView), no un tmux cualquiera. + # ----------------------------------------------------------------------- + if [[ "$want_new" -eq 0 && "$reuse" -eq 0 && -n "${TMUX:-}" ]]; then + local current_socket target_socket + current_socket="$(basename "${TMUX%%,*}")" + target_socket="$current_socket" + [[ "$explicit_session" -eq 1 ]] && target_socket="$session" + + if [[ "$target_socket" == "$current_socket" ]] \ + && tmux -L "$current_socket" has-session -t "$current_socket" 2>/dev/null \ + && tmux -L "$current_socket" list-windows -t "$current_socket" \ + -F '#{window_name}' 2>/dev/null | grep -qx console; then + # Traer la TUI al contexto actual sin abrir nada nuevo. Preferimos + # el binario (centraliza la politica en la app: 'fleetview show'); + # si no esta compilado, caemos a 'select-window' directo, que es lo + # que 'show' hace por dentro dentro de tmux (cero dependencia). + if [[ -x "$bin" ]] \ + && FLEET_SOCKET="$current_socket" FLEET_SESSION="$current_socket" \ + "$bin" show 2>/dev/null; then + return 0 + fi + tmux -L "$current_socket" select-window -t "$current_socket":console + echo "launch_fleetclaude: flota '$current_socket' viva; TUI traida al contexto actual (sin ventana nueva)." + return 0 + fi + fi + # ----------------------------------------------------------------------- # Resolver el PERFIL (socket+sesion tmux comparten nombre). # @@ -200,7 +256,10 @@ USAGE # indice 1 y cualquier referencia a console.0 falla con # "can't find pane: 0". Los pane ID son estables e inmunes al base-index. # ----------------------------------------------------------------------- - local left_pane right_pane + # Inicializadas a "" (no solo declaradas): bajo `set -u` una `local x` sin + # valor queda *unbound*, y al reutilizar una sesion existente el `[[ -z + # "$left_pane" ]]` de mas abajo reventaba con "unbound variable". + local left_pane="" right_pane="" if $T has-session -t "$session" 2>/dev/null; then echo "launch_fleetclaude: la sesion tmux '$session' ya existe; reutilizandola." else From 13c82be7805715ce2b3d24e9521d8c56bd224373 Mon Sep 17 00:00:00 2001 From: Egutierrez Date: Tue, 30 Jun 2026 18:01:12 +0200 Subject: [PATCH 24/53] =?UTF-8?q?feat(eda):=20NUM=20DISTR=20muestra=20el?= =?UTF-8?q?=20valor=20de=20=CF=83=20(std)=20en=20la=20leyenda=20del=20hist?= =?UTF-8?q?ograma?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit La leyenda de cada histograma del capítulo de distribuciones numéricas ya reporta el valor de la media y la mediana; ahora también reporta el valor de la desviación estándar σ. La entrada de leyenda de la banda ±1σ pasa a incluir el número (±1σ (σ = X)) y, cuando la banda no puede dibujarse (sin media o std<=0) pero σ es conocido, se añade una entrada de leyenda mediante un handle proxy sin trazo, de modo que el valor de σ se reporta siempre. No se altera el boxplot de Tukey ni el keep-together (Group) por columna. Se añaden tests de la leyenda: golden (σ con valor junto a media y mediana), edge sin banda (proxy) y edge sin std (no revienta). Bump 1.1.0 -> 1.2.0. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../automatic_eda/chapters/num_distr.py | 27 +++++++++--- .../automatic_eda/chapters/num_distr_test.py | 44 +++++++++++++++++++ 2 files changed, 65 insertions(+), 6 deletions(-) diff --git a/python/functions/datascience/automatic_eda/chapters/num_distr.py b/python/functions/datascience/automatic_eda/chapters/num_distr.py index 67a47779..5890b123 100644 --- a/python/functions/datascience/automatic_eda/chapters/num_distr.py +++ b/python/functions/datascience/automatic_eda/chapters/num_distr.py @@ -1,9 +1,10 @@ """Numeric distributions chapter (NUM DISTR) for AutomaticEDA. For every numeric column the chapter draws, as a single indivisible figure, a -histogram with the **mean, median and ±1σ band drawn as reference lines** and a -**Tukey boxplot right below it** sharing the same X axis — exactly the user -requirement for this chapter. Each figure is emitted as a lazy ``Figure`` block +histogram with the **mean, median and ±1σ band drawn as reference lines** (the +legend reports the numeric value of the mean, the median **and the standard +deviation σ**) and a **Tukey boxplot right below it** sharing the same X axis — +exactly the user requirement for this chapter. Each figure is emitted as a lazy ``Figure`` block so the renderers rasterize and scale it to fit a whole page/slide and nothing is ever cut; columns with many numerics simply flow across pages as small multiples. @@ -34,7 +35,7 @@ try: except Exception: # noqa: BLE001 — keep the chapter importable no matter what. build_boxplot_stats = None # type: ignore[assignment] -CHAPTER_VERSION = "1.1.0" +CHAPTER_VERSION = "1.2.0" CHAPTER_ID = "num_distr" CHAPTER_TITLE = "Distribuciones numéricas" @@ -140,9 +141,11 @@ def _make_hist_box(name: str, numeric: dict, box: dict): std = numeric.get("std") # ±1σ band first (behind the lines), then median (solid) and mean (dashed). + # The band's legend entry also reports the numeric value of the standard + # deviation, so the reader sees mean, median AND σ at a glance. if mean is not None and std is not None and std > 0: ax_h.axvspan(mean - std, mean + std, color="#f0c27b", alpha=0.22, - zorder=1, label="±1σ") + zorder=1, label=f"±1σ (σ = {_fmt_num(std)})") if median is not None: ax_h.axvline(median, color="#2e8b57", linestyle="-", linewidth=1.6, zorder=4, label=f"mediana = {_fmt_num(median)}") @@ -152,7 +155,19 @@ def _make_hist_box(name: str, numeric: dict, box: dict): ax_h.set_ylabel("frecuencia", fontsize=8) ax_h.tick_params(labelsize=7) - ax_h.legend(fontsize=6.5, loc="upper right", framealpha=0.85) + # Always surface σ in the legend: if the ±1σ band could not be drawn (no mean + # or std<=0) but σ is still known, add a label-only proxy handle so the value + # of the standard deviation is reported regardless of the band. + handles, labels = ax_h.get_legend_handles_labels() + if std is not None and not any("σ =" in lbl for lbl in labels): + from matplotlib.lines import Line2D + proxy = Line2D([], [], linestyle="none", marker="", + label=f"σ = {_fmt_num(std)}") + handles.append(proxy) + labels.append(f"σ = {_fmt_num(std)}") + if handles: + ax_h.legend(handles, labels, fontsize=6.5, loc="upper right", + framealpha=0.85) for spine in ("top", "right"): ax_h.spines[spine].set_visible(False) diff --git a/python/functions/datascience/automatic_eda/chapters/num_distr_test.py b/python/functions/datascience/automatic_eda/chapters/num_distr_test.py index 71793ad1..280cff17 100644 --- a/python/functions/datascience/automatic_eda/chapters/num_distr_test.py +++ b/python/functions/datascience/automatic_eda/chapters/num_distr_test.py @@ -159,6 +159,50 @@ def test_anti_corte_muchas_columnas_pdf_y_pptx(): assert res_pptx["n_slides"] >= 8 # at least one slide per column figure. +def _hist_legend_texts(numeric, box=None): + """Build the per-column figure and return its histogram-legend label texts.""" + from datascience.automatic_eda.chapters.num_distr import _make_hist_box + import matplotlib.pyplot as plt + fig = _make_hist_box("col", numeric, box or {}) + ax_h = fig.axes[0] # the histogram is the top axis. + leg = ax_h.get_legend() + texts = [t.get_text() for t in leg.get_texts()] if leg else [] + plt.close(fig) + return texts + + +def test_golden_leyenda_histograma_reporta_valor_std(): + # The histogram legend must report the numeric value of the standard + # deviation σ next to mean and median. + numeric = _numeric_block(42.5, 40.0, 12.3, 1.0, 100.0, "right-skewed", 5) + texts = _hist_legend_texts(numeric) + joined = " ".join(texts) + assert any("σ =" in t for t in texts), f"σ value missing in legend: {texts}" + assert "12.3" in joined, f"std value 12.3 not in legend: {texts}" + assert any("media =" in t for t in texts) + assert any("mediana =" in t for t in texts) + + +def test_edge_std_en_leyenda_aunque_no_haya_banda(): + # When the ±1σ band cannot be drawn (no mean) but σ is known, the legend + # still surfaces the σ value via a label-only proxy handle. + numeric = _numeric_block(42.5, 40.0, 7.5, 1.0, 100.0, "right-skewed", 0) + numeric["mean"] = None # forces the band off; σ must still appear. + texts = _hist_legend_texts(numeric) + assert any("σ = 7.5" in t for t in texts), f"σ proxy missing: {texts}" + + +def test_edge_sin_std_no_revienta_la_figura(): + # A numeric block without σ must not raise and simply omits the σ entry. + import matplotlib.pyplot as plt + numeric = _numeric_block(42.5, 40.0, 0.0, 1.0, 100.0, "discrete", 0) + numeric["std"] = None + texts = _hist_legend_texts(numeric) + assert not any("σ =" in t for t in texts) + # mean/median lines still produce their own legend entries. + assert any("media =" in t for t in texts) + + def test_distribution_gloss_cubre_todas_las_etiquetas(): # Every label detect_distribution_type can emit has a Spanish gloss. for label in ("normal-ish", "right-skewed", "left-skewed", "heavy-tail", From a421f13d2ebc6cae2404a699ed469da375b22233 Mon Sep 17 00:00:00 2001 From: Egutierrez Date: Tue, 30 Jun 2026 18:02:31 +0200 Subject: [PATCH 25/53] feat(eda): engancha glosario clicable en correlacion/modelos/agregacion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fase 4b — extiende el glosario clicable de AutomaticEDA (mecanismo ya probado end-to-end con `entropia` en cat_distr) a tres capítulos más, siguiendo el contrato sección 11 (glossary.add(key,label,def) + span [[term:KEY]]texto[[/term]]): - correlacion: Pearson, Spearman, Cramér's V, razón de correlación (η) y la corrección por comparaciones múltiples (FDR). Los métodos se marcan en el intro (siempre presente); FDR se registra y marca solo cuando se emite su resumen, para no dejar entradas de glosario sin aparición que las referencie. - modelos: PCA, KMeans, coeficiente de silueta (silhouette), Isolation Forest y la estandarización z-score. Cada término se registra dentro de la sección que lo usa (tras su early-return), de modo que un término solo entra al glosario cuando su sección realmente se renderiza. - agregacion: agrupación (split-apply-combine / groupby) y tabla dinámica (pivot), ambos en el intro siempre presente. Solo se añaden los enganches de glosario: ningún cambio en la lógica de datos. El texto visible es idéntico con o sin marcador (los renderers lo eliminan), así que el layout de línea no cambia. Sin colector en ctx (render suelto) los capítulos degradan y no marcan nada. Tests: un test de glosario por capítulo verifica registro + marcado y la degradación sin colector. Suite AutomaticEDA + render pipeline: 87 passed. Golden titanic (run_models+series+llm): los 12 términos aparecen como entradas del glosario en PDF (16 link annotations GOTO) y PPTX (15 saltos hlinksldjump). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../automatic_eda/chapters/agregacion.py | 55 ++++++++- .../automatic_eda/chapters/agregacion_test.py | 22 ++++ .../automatic_eda/chapters/correlacion.py | 88 +++++++++++-- .../chapters/correlacion_test.py | 22 ++++ .../automatic_eda/chapters/modelos.py | 116 ++++++++++++++---- .../automatic_eda/chapters/modelos_test.py | 23 ++++ 6 files changed, 289 insertions(+), 37 deletions(-) diff --git a/python/functions/datascience/automatic_eda/chapters/agregacion.py b/python/functions/datascience/automatic_eda/chapters/agregacion.py index 7b5e03e6..c6eafcf8 100644 --- a/python/functions/datascience/automatic_eda/chapters/agregacion.py +++ b/python/functions/datascience/automatic_eda/chapters/agregacion.py @@ -89,6 +89,35 @@ _DEF_MAX_CARD = 20 _DEF_MAX_MEASURES = 4 _DEF_TOP_N = 12 +# Glossary terms this chapter explains. Both appear in the always-rendered intro, +# so they are registered and marked clickable whenever a collector is in ctx — +# the canonical two-step pattern (see ``cat_distr``): ``glossary.add(key, label, +# definition)`` + the inline span ``[[term:KEY]]texto[[/term]]`` in a Markdown +# block. Mapping key -> (label, definition). +_TERM_DEFS = { + "groupby": ( + "Agrupación (split-apply-combine)", + "Operación de agrupación (group by): parte la tabla en grupos según los " + "valores de una columna categórica, aplica un cálculo (conteo, media, " + "mediana…) dentro de cada grupo y combina los resultados en una tabla " + "resumen. Es el patrón split-apply-combine."), + "pivot_table": ( + "Tabla dinámica (pivot)", + "Tabla dinámica que cruza dos variables categóricas — una en las filas y " + "otra en las columnas — y rellena cada celda con un agregado (media, " + "suma…) de una medida numérica. Resume de un vistazo cómo interactúan las " + "dos categóricas sobre esa medida."), +} + + +def _term(mark: bool, key: str, text: str) -> str: + """Wrap ``text`` as a clickable glossary span when ``mark`` is True. + + The visible text is identical with or without the marker (the renderers strip + it), so wrapping never changes line layout — it only adds the link. + """ + return f"[[term:{key}]]{text}[[/term]]" if mark else text + # --------------------------------------------------------------------------- # # Formatting helpers (mirror the other chapters' defensive style). @@ -525,13 +554,18 @@ def _sections_live(profile: dict, ctx: dict, candidates: dict) -> list: # --------------------------------------------------------------------------- # # Entry point. # --------------------------------------------------------------------------- # -def _intro_blocks() -> list: +def _intro_blocks(gloss=None, mark_term: bool = False) -> list: + if gloss is not None: + for key, (label, definition) in _TERM_DEFS.items(): + gloss.add(key, label, definition) + t_groupby = _term(mark_term, "groupby", "**por grupos** (split-apply-combine)") + t_pivot = _term(mark_term, "pivot_table", "**tablas dinámicas** (pivot)") text = ( - "Este capítulo analiza la tabla **por grupos** (split-apply-combine): " + f"Este capítulo analiza la tabla {t_groupby}: " "elige las columnas categóricas más informativas — por su cardinalidad " "y relevancia, no todas contra todas, para no inflar comparaciones " "espurias — y resume las variables numéricas dentro de cada grupo " - "(conteo, media, mediana, desviación). Las **tablas dinámicas** (pivot) " + f"(conteo, media, mediana, desviación). Las {t_pivot} " "cruzan dos categóricas sobre una medida, y los **gráficos de barras** " "(siempre desde cero) comparan los grupos de un vistazo." ) @@ -556,13 +590,21 @@ def build_agregacion(profile: dict, ctx: dict): if not isinstance(profile, dict): return None + # Shared glossary collector: groupby + pivot_table live in the always-present + # intro, so they are registered + marked there. Degrades silently (mark_term + # False) when no collector is in ctx (standalone render). + glossary = ctx.get("glossary") + gloss = glossary if isinstance(glossary, model.GlossaryCollector) else None + mark_term = gloss is not None + # Pre-computed results take precedence (offline / tests / forward-compat). pre = ctx.get("aggregations") if _is_dict(pre) and (pre.get("groupby") or pre.get("pivots")): sections = _sections_from_precomputed(pre) if not sections: return None - blocks = _intro_blocks() + sections + _insights_section(ctx) + blocks = (_intro_blocks(gloss, mark_term) + sections + + _insights_section(ctx)) return model.Chapter(id=CHAPTER_ID, title=CHAPTER_TITLE, version=CHAPTER_VERSION, blocks=blocks) @@ -583,10 +625,11 @@ def build_agregacion(profile: dict, ctx: dict): "crudos. Pasa ctx['db_path'] + ctx['table'] (para el cálculo " "push-down en DuckDB) o ctx['aggregations'] ya precalculado. " f"Columnas categóricas candidatas: {keys or '—'}.") - blocks = _intro_blocks() + [note] + _insights_section(ctx) + blocks = (_intro_blocks(gloss, mark_term) + [note] + + _insights_section(ctx)) return model.Chapter(id=CHAPTER_ID, title=CHAPTER_TITLE, version=CHAPTER_VERSION, blocks=blocks) - blocks = _intro_blocks() + sections + _insights_section(ctx) + blocks = _intro_blocks(gloss, mark_term) + sections + _insights_section(ctx) return model.Chapter(id=CHAPTER_ID, title=CHAPTER_TITLE, version=CHAPTER_VERSION, blocks=blocks) diff --git a/python/functions/datascience/automatic_eda/chapters/agregacion_test.py b/python/functions/datascience/automatic_eda/chapters/agregacion_test.py index e35005be..a04ad1ca 100644 --- a/python/functions/datascience/automatic_eda/chapters/agregacion_test.py +++ b/python/functions/datascience/automatic_eda/chapters/agregacion_test.py @@ -254,3 +254,25 @@ def test_anti_corte_muchos_grupos_y_texto_largo(): # First, middle and last words of the long paragraph all present. for i in (0, 60, 119): assert f"palabra{i}" in txt + + +def test_glosario_engancha_groupby_y_pivot(): + """Mejora 4b: la agrupación (split-apply-combine) y la tabla dinámica (pivot) + se registran en el colector compartido y se marcan clicables en el cuerpo. + Sin colector en ctx, el capítulo degrada y no marca nada.""" + from datascience.automatic_eda.model import GlossaryCollector + + g = GlossaryCollector() + ctx = dict(_ctx_precomputed()) + ctx["glossary"] = g + ch = build_agregacion(_profile(), ctx) + assert ch is not None + keys = {t["key"] for t in g.terms()} + assert {"groupby", "pivot_table"} <= keys + body = " ".join(b.text for b in ch.blocks if b.kind == "markdown") + assert "[[term:groupby]]" in body and "[[term:pivot_table]]" in body + + # Sin colector: degrada limpio (ningún marcador en el cuerpo). + ch2 = build_agregacion(_profile(), _ctx_precomputed()) + body2 = " ".join(b.text for b in ch2.blocks if b.kind == "markdown") + assert "[[term:" not in body2 diff --git a/python/functions/datascience/automatic_eda/chapters/correlacion.py b/python/functions/datascience/automatic_eda/chapters/correlacion.py index 22b6eb0c..0c906cc9 100644 --- a/python/functions/datascience/automatic_eda/chapters/correlacion.py +++ b/python/functions/datascience/automatic_eda/chapters/correlacion.py @@ -47,6 +47,53 @@ _MAX_MATRIX_LABELS = 16 # How many pairs to show in each of the top-positive / top-negative tables. _TOP_N = 10 +# Glossary terms this chapter explains. Each is registered in the shared +# collector (ctx['glossary']) and marked clickable on its first appearance in the +# body — the canonical two-step pattern (see ``cat_distr`` for the reference +# implementation): ``glossary.add(key, label, definition)`` + the inline span +# ``[[term:KEY]]texto visible[[/term]]`` in a Markdown block. Mapping key -> +# (label, definition). ``fdr`` is only registered when the FDR summary is present. +_TERM_DEFS = { + "pearson": ( + "Pearson (coeficiente r)", + "Coeficiente de correlación lineal de Pearson (r) entre dos variables " + "numéricas. Va de −1 (relación lineal inversa perfecta) a +1 (directa " + "perfecta); 0 indica ausencia de relación lineal. Sólo capta relaciones " + "lineales, por eso lleva signo."), + "spearman": ( + "Spearman (correlación de rangos)", + "Correlación de rangos de Spearman: el coeficiente de Pearson calculado " + "sobre los puestos (rangos) de los valores en vez de sus magnitudes. Mide " + "relaciones monótonas (no necesariamente lineales), va de −1 a +1 y es " + "robusta frente a valores atípicos."), + "cramers_v": ( + "Cramér's V", + "Medida de asociación entre dos variables categóricas, derivada del " + "estadístico chi-cuadrado y normalizada al rango 0–1 (0 = independientes, " + "1 = asociación total). No tiene signo: sólo mide la intensidad."), + "correlation_ratio": ( + "Razón de correlación (η)", + "Razón de correlación (eta) entre una variable numérica y una " + "categórica: la fracción de la varianza de la numérica explicada por los " + "grupos de la categórica. Va de 0 (los grupos no explican nada) a 1 (la " + "explican toda); no tiene signo."), + "fdr": ( + "Comparaciones múltiples (FDR)", + "Al evaluar muchos pares a la vez, algunos parecen significativos por " + "puro azar. La corrección por tasa de falsos descubrimientos (FDR, " + "Benjamini-Hochberg) ajusta los p-valores para controlar la proporción " + "esperada de falsos positivos entre los pares declarados significativos."), +} + + +def _term(mark: bool, key: str, text: str) -> str: + """Wrap ``text`` as a clickable glossary span when ``mark`` is True. + + The visible text is identical with or without the marker (the renderers strip + the marker), so wrapping never changes line layout — it only adds the link. + """ + return f"[[term:{key}]]{text}[[/term]]" if mark else text + def _is_num(v) -> bool: """True for a real, finite int/float (not bool, not NaN/inf).""" @@ -245,7 +292,7 @@ def _methods_block(corr: dict): return model.KVTable(rows=rows, title="Métodos de asociación") -def _fdr_text(corr: dict) -> str | None: +def _fdr_text(corr: dict, mark_term: bool = False) -> str | None: """One-line summary of the multiple-testing (FDR) correction, or None.""" mt = corr.get("multiple_testing") if not isinstance(mt, dict) or not mt: @@ -254,7 +301,8 @@ def _fdr_text(corr: dict) -> str | None: alpha = mt.get("alpha") n_tests = mt.get("n_tests") n_rej = mt.get("n_rejected") - parts = [f"Corrección por comparaciones múltiples ({method}"] + multi = _term(mark_term, "fdr", "comparaciones múltiples") + parts = [f"Corrección por {multi} ({method}"] if _is_num(alpha): parts[0] += f", α={float(alpha):g}" parts[0] += ")." @@ -289,13 +337,31 @@ def build_correlacion(profile: dict, ctx: dict): blocks: list = [] - # Intro: what this chapter shows and how to read the sign. + # Register the always-present method terms in the shared glossary and mark + # their first appearance clickable (the FDR term is registered lazily below, + # only when the FDR summary is actually emitted). Degrades silently when no + # collector is in ctx (standalone render) — mark_term stays False. + glossary = ctx.get("glossary") + gloss = glossary if isinstance(glossary, model.GlossaryCollector) else None + mark_term = gloss is not None + if gloss is not None: + for key in ("pearson", "spearman", "cramers_v", "correlation_ratio"): + label, definition = _TERM_DEFS[key] + gloss.add(key, label, definition) + + # Intro: what this chapter shows and how to read the sign. Build the marked + # method names as locals first (avoids backslash-in-f-string for "Cramér's V"). + t_pearson = _term(mark_term, "pearson", "Pearson") + t_spearman = _term(mark_term, "spearman", "Spearman") + t_cramers = _term(mark_term, "cramers_v", "Cramér's V") + t_corr_ratio = _term(mark_term, "correlation_ratio", "razón de correlación") blocks.append(model.Markdown(text=( "Asociación entre columnas. Cada par se evalúa con la métrica adecuada a " - "sus tipos (Pearson/Spearman entre numéricas — con **signo**; Cramér's V " - "entre categóricas; razón de correlación num-categórica; información mutua " - "como medida común no lineal). Sólo las correlaciones **num-num** tienen " - "dirección: por eso los pares **negativos** son siempre num-num."))) + f"sus tipos ({t_pearson}/{t_spearman} entre numéricas — con **signo**; " + f"{t_cramers} entre categóricas; {t_corr_ratio} num-categórica; " + "información mutua como medida común no lineal). Sólo las correlaciones " + "**num-num** tienen dirección: por eso los pares **negativos** son siempre " + "num-num."))) # 1) Association matrix (heatmap). labels, trimmed = _ordered_labels(pairs) @@ -337,9 +403,13 @@ def build_correlacion(profile: dict, ctx: dict): "no estacionarias y pueden ser espurias (Granger–Newbold). Compáralas " "sobre los retornos/diferencias antes de interpretarlas."))) - # 4) FDR summary + methods legend. - fdr_text = _fdr_text(corr) + # 4) FDR summary + methods legend. Register the FDR term only when its + # summary is emitted, so the glossary never lists an unreferenced entry. + fdr_text = _fdr_text(corr, mark_term=mark_term) if fdr_text: + if gloss is not None: + label, definition = _TERM_DEFS["fdr"] + gloss.add("fdr", label, definition) blocks.append(model.Markdown(text=fdr_text)) methods = _methods_block(corr) if methods is not None: diff --git a/python/functions/datascience/automatic_eda/chapters/correlacion_test.py b/python/functions/datascience/automatic_eda/chapters/correlacion_test.py index 88ddc726..b4291e65 100644 --- a/python/functions/datascience/automatic_eda/chapters/correlacion_test.py +++ b/python/functions/datascience/automatic_eda/chapters/correlacion_test.py @@ -173,3 +173,25 @@ def test_anticorte_matriz_ancha_y_etiquetas_largas_no_se_cortan(): assert rx["path"] == pptx and os.path.exists(pptx) and rx["n_slides"] >= 1 # A short, unbreakable fragment of the long label survives the wrap. assert "azufre" in _pdf_text(pdf) + + +def test_glosario_engancha_metodos_y_fdr(): + """Mejora 4b: los métodos de correlación (Pearson, Spearman, Cramér's V, + razón de correlación) y la corrección por comparaciones múltiples (FDR) se + registran en el colector compartido y se marcan clicables en el cuerpo. Sin + colector en ctx, el capítulo degrada y no marca nada.""" + from datascience.automatic_eda.model import GlossaryCollector + + g = GlossaryCollector() + ch = build_correlacion(_profile(), {"glossary": g}) + assert ch is not None + keys = {t["key"] for t in g.terms()} + assert {"pearson", "spearman", "cramers_v", "correlation_ratio", "fdr"} <= keys + body = " ".join(b.text for b in ch.blocks if b.kind == "markdown") + for k in ("pearson", "spearman", "cramers_v", "correlation_ratio", "fdr"): + assert f"[[term:{k}]]" in body, k + + # Sin colector: degrada limpio (ningún marcador en el cuerpo). + ch2 = build_correlacion(_profile(), {}) + body2 = " ".join(b.text for b in ch2.blocks if b.kind == "markdown") + assert "[[term:" not in body2 diff --git a/python/functions/datascience/automatic_eda/chapters/modelos.py b/python/functions/datascience/automatic_eda/chapters/modelos.py index ffc43346..1ddf78ee 100644 --- a/python/functions/datascience/automatic_eda/chapters/modelos.py +++ b/python/functions/datascience/automatic_eda/chapters/modelos.py @@ -55,6 +55,62 @@ _CLUSTER_COLORS = [ "#edc948", "#b07aa1", "#ff9da7", "#9c755f", "#bab0ac", ] +# Glossary terms this chapter explains. Each is registered in the shared +# collector (ctx['glossary']) and marked clickable on its first appearance — the +# canonical two-step pattern (see ``cat_distr``): ``glossary.add(key, label, +# definition)`` + the inline span ``[[term:KEY]]texto[[/term]]`` in a Markdown +# block. A term is registered only when its section is actually rendered, so the +# glossary never lists an entry no in-text appearance points to. +_TERM_DEFS = { + "zscore": ( + "Estandarización z-score", + "Transformación que lleva cada columna numérica a media 0 y desviación " + "típica 1: a cada valor le resta la media de su columna y lo divide por " + "la desviación típica. Así variables con escalas muy distintas (euros " + "frente a un ratio 0–1) pesan por igual en las distancias y la varianza."), + "pca": ( + "PCA (componentes principales)", + "El análisis de componentes principales resume muchas variables " + "numéricas correlacionadas en pocos ejes nuevos (componentes), " + "ortogonales entre sí y ordenados por la cantidad de varianza que " + "capturan. Permite ver la estructura de los datos en 2D y saber cuántas " + "dimensiones bastan para explicarlos."), + "kmeans": ( + "KMeans (segmentación)", + "Algoritmo de agrupamiento no supervisado que reparte las filas en k " + "segmentos: asigna cada fila al centro (centroide) más cercano y recoloca " + "los centroides de forma iterativa hasta minimizar la distancia interna " + "de cada grupo. Aquí k se elige automáticamente."), + "silhouette": ( + "Coeficiente de silueta (silhouette)", + "Métrica de calidad de un agrupamiento, en el rango −1 a 1: para cada " + "fila compara cómo de cerca está de su propio segmento frente al segmento " + "vecino más próximo. Cuanto más alto el promedio, más compactos y " + "separados están los segmentos."), + "isolation_forest": ( + "Isolation Forest (anomalías)", + "Algoritmo de detección de anomalías multivariante: construye árboles que " + "parten el espacio con cortes aleatorios y mide cuántos cortes hacen " + "falta para aislar cada fila. Las filas raras se aíslan con muy pocos " + "cortes y se marcan como outliers según un umbral de contaminación."), +} + + +def _term(mark: bool, key: str, text: str) -> str: + """Wrap ``text`` as a clickable glossary span when ``mark`` is True. + + The visible text is identical with or without the marker (the renderers strip + it), so wrapping never changes line layout — it only adds the link. + """ + return f"[[term:{key}]]{text}[[/term]]" if mark else text + + +def _register(gloss, key: str) -> None: + """Register term ``key`` in the collector (idempotent); no-op if gloss None.""" + if gloss is not None: + label, definition = _TERM_DEFS[key] + gloss.add(key, label, definition) + # --------------------------------------------------------------------------- # # Formatting helpers (mirror the overview chapter's defensive style). @@ -252,34 +308,37 @@ def _make_cluster_scatter(projection: dict): # --------------------------------------------------------------------------- # # Section builders. Each returns a list of blocks (possibly empty). # --------------------------------------------------------------------------- # -def _normalization_intro() -> list: +def _normalization_intro(gloss=None, mark_term: bool = False) -> list: + _register(gloss, "zscore") + zscore = _term(mark_term, "zscore", "**estandarizan con z-score**") text = ( "Estos modelos son **no supervisados**: buscan estructura latente sin " "una variable objetivo. Antes de aplicarlos, todas las columnas " - "numéricas se **estandarizan con z-score** (cada valor menos la media, " - "dividido por la desviación típica). Sin esta normalización, una " - "variable con escala grande (p.ej. ingresos en euros) dominaría las " - "distancias y la varianza frente a otra de escala pequeña (p.ej. un " - "ratio entre 0 y 1), sesgando tanto el PCA como el KMeans. Tras la " - "estandarización todas las variables pesan por igual." + f"numéricas se {zscore} (cada valor menos la media, dividido por la " + "desviación típica). Sin esta normalización, una variable con escala " + "grande (p.ej. ingresos en euros) dominaría las distancias y la varianza " + "frente a otra de escala pequeña (p.ej. un ratio entre 0 y 1), sesgando " + "tanto el PCA como el KMeans. Tras la estandarización todas las variables " + "pesan por igual." ) return [model.Heading(text="Modelos no supervisados", level=1), model.Markdown(text=text)] -def _pca_section(pca: dict) -> list: +def _pca_section(pca: dict, gloss=None, mark_term: bool = False) -> list: if not _is_dict(pca) or not pca.get("explained_variance_ratio"): return [] + _register(gloss, "pca") blocks = [model.Heading(text="PCA — varianza explicada", level=2)] n_used = pca.get("n_rows_used") n_feat = pca.get("n_features") intro = ( - f"El PCA resume {_fmt_num(n_feat)} variables numéricas en componentes " - f"ortogonales ordenados por la varianza que capturan " - f"({_fmt_num(n_used)} filas usadas tras eliminar nulos). El gráfico de " - "sedimentación (scree) muestra cuánta varianza aporta cada componente y " - "su acumulado: un codo marca cuántos componentes bastan." + f"El {_term(mark_term, 'pca', 'PCA')} resume {_fmt_num(n_feat)} variables " + "numéricas en componentes ortogonales ordenados por la varianza que " + f"capturan ({_fmt_num(n_used)} filas usadas tras eliminar nulos). El " + "gráfico de sedimentación (scree) muestra cuánta varianza aporta cada " + "componente y su acumulado: un codo marca cuántos componentes bastan." ) blocks.append(model.Markdown(text=intro)) @@ -325,11 +384,14 @@ def _pca_section(pca: dict) -> list: return blocks -def _kmeans_section(kmeans: dict, projection: dict, titles) -> list: +def _kmeans_section(kmeans: dict, projection: dict, titles, + gloss=None, mark_term: bool = False) -> list: has_km = _is_dict(kmeans) and kmeans.get("best_k") has_proj = _is_dict(projection) and projection.get("points") if not has_km and not has_proj: return [] + _register(gloss, "kmeans") + _register(gloss, "silhouette") blocks = [model.Heading(text="Segmentación (KMeans)", level=2)] @@ -337,9 +399,11 @@ def _kmeans_section(kmeans: dict, projection: dict, titles) -> list: sil = (projection or {}).get("silhouette") if sil is None: sil = (kmeans or {}).get("silhouette") + t_kmeans = _term(mark_term, "kmeans", "KMeans") + t_sil = _term(mark_term, "silhouette", "*silhouette*") intro = ( - f"KMeans agrupa las filas en **{_fmt_num(best_k)} segmentos** elegidos " - "automáticamente maximizando el coeficiente de *silhouette* " + f"{t_kmeans} agrupa las filas en **{_fmt_num(best_k)} segmentos** " + f"elegidos automáticamente maximizando el coeficiente de {t_sil} " f"(**{_fmt_num(sil)}**, rango −1 a 1: cuanto más alto, segmentos más " "compactos y separados). Los segmentos se proyectan sobre el plano de " "los dos primeros componentes principales para visualizarlos." @@ -394,16 +458,18 @@ def _kmeans_section(kmeans: dict, projection: dict, titles) -> list: return blocks -def _outliers_section(outliers: dict) -> list: +def _outliers_section(outliers: dict, gloss=None, mark_term: bool = False) -> list: if not _is_dict(outliers) or outliers.get("n_outliers") is None: return [] if outliers.get("note") and not outliers.get("n_rows_used"): # insufficient data — nothing meaningful to show. return [] + _register(gloss, "isolation_forest") blocks = [model.Heading(text="Detección de anomalías (Isolation Forest)", level=2)] + isof = _term(mark_term, "isolation_forest", "**Isolation Forest**") explain = ( - "**Isolation Forest** detecta filas anómalas de forma *multivariante*: " + f"{isof} detecta filas anómalas de forma *multivariante*: " "construye árboles que parten el espacio con cortes aleatorios y mide " "cuántos cortes hacen falta para aislar cada fila. Las filas raras " "(combinaciones de valores poco frecuentes considerando **todas las " @@ -484,15 +550,21 @@ def build_modelos(profile: dict, ctx: dict): (kmeans and kmeans.get("best_k")) or (projection and projection.get("points")) ) else None + # Shared glossary collector: terms are registered + marked clickable inside + # each section, only when that section actually renders (no orphan entries). + glossary = ctx.get("glossary") + gloss = glossary if isinstance(glossary, model.GlossaryCollector) else None + mark_term = gloss is not None + sections = [] - sections += _pca_section(pca) if pca else [] - sections += _kmeans_section(kmeans, projection, titles) - sections += _outliers_section(outliers) if outliers else [] + sections += _pca_section(pca, gloss, mark_term) if pca else [] + sections += _kmeans_section(kmeans, projection, titles, gloss, mark_term) + sections += _outliers_section(outliers, gloss, mark_term) if outliers else [] sections += _normality_section(normality) if normality else [] if not sections: return None # models block present but nothing renderable. - blocks = _normalization_intro() + sections + blocks = _normalization_intro(gloss, mark_term) + sections return model.Chapter(id=CHAPTER_ID, title=CHAPTER_TITLE, version=CHAPTER_VERSION, blocks=blocks) diff --git a/python/functions/datascience/automatic_eda/chapters/modelos_test.py b/python/functions/datascience/automatic_eda/chapters/modelos_test.py index 9d2597a5..98e21eba 100644 --- a/python/functions/datascience/automatic_eda/chapters/modelos_test.py +++ b/python/functions/datascience/automatic_eda/chapters/modelos_test.py @@ -257,3 +257,26 @@ def test_anticortes_tabla_normalidad_larga_no_corta(): # Every column name survives (wrapped/split, never truncated). for i in (0, 19, 39): assert f"col_{i}" in txt + + +def test_glosario_engancha_terminos_modelos(): + """Mejora 4b: PCA, KMeans, silhouette, Isolation Forest y la estandarización + z-score se registran en el colector compartido y se marcan clicables en el + cuerpo. Sin colector en ctx, el capítulo degrada y no marca nada.""" + from datascience.automatic_eda.model import GlossaryCollector + + g = GlossaryCollector() + ctx = dict(_ctx_full()) + ctx["glossary"] = g + ch = build_modelos(_profile(), ctx) + assert ch is not None + keys = {t["key"] for t in g.terms()} + assert {"zscore", "pca", "kmeans", "silhouette", "isolation_forest"} <= keys + body = " ".join(b.text for b in ch.blocks if b.kind == "markdown") + for k in ("zscore", "pca", "kmeans", "silhouette", "isolation_forest"): + assert f"[[term:{k}]]" in body, k + + # Sin colector: degrada limpio (ningún marcador en el cuerpo). + ch2 = build_modelos(_profile(), _ctx_full()) + body2 = " ".join(b.text for b in ch2.blocks if b.kind == "markdown") + assert "[[term:" not in body2 From 048781df3f4834ccc9b39ff9a3a38255d4a8dd8e Mon Sep 17 00:00:00 2001 From: Egutierrez Date: Tue, 30 Jun 2026 18:04:05 +0200 Subject: [PATCH 26/53] =?UTF-8?q?feat(eda):=20portada=20=E2=80=94=20tama?= =?UTF-8?q?=C3=B1o=20grande=20+=20descripci=C3=B3n/granularidad=20reales?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit El capítulo PORTADA ahora muestra SIEMPRE el tamaño del dataset (N filas × M columnas) en grande, como heading junto al nombre y agrupado con él (Group keep-together), en lugar de enterrarlo en la tabla de metadatos. La Descripción y la Granularidad ya no salen vacías ni con placeholders: se resuelven por cascada — ctx explícito > bloque LLM (profile['llm'].summary / row_meaning de eda_llm_insights) > derivación del propio perfil (forma, mezcla de tipos y score de calidad para la descripción; columnas key_candidates o la forma de la tabla para una frase 'Cada fila es…'). Las derivaciones son honestas (declaran que vienen del perfil) y nunca inventan significado de negocio. Añade chapters/portada_test.py: golden (tamaño grande + textos del LLM, sin fila 'Tamaño' duplicada), fallbacks sin LLM (keys / forma), prioridad de ctx, edge de perfil vacío sin lanzar, y render a PDF + PPTX. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../automatic_eda/chapters/portada.py | 134 ++++++++++-- .../automatic_eda/chapters/portada_test.py | 197 ++++++++++++++++++ 2 files changed, 316 insertions(+), 15 deletions(-) create mode 100644 python/functions/datascience/automatic_eda/chapters/portada_test.py diff --git a/python/functions/datascience/automatic_eda/chapters/portada.py b/python/functions/datascience/automatic_eda/chapters/portada.py index c1bb43ab..409322f7 100644 --- a/python/functions/datascience/automatic_eda/chapters/portada.py +++ b/python/functions/datascience/automatic_eda/chapters/portada.py @@ -2,8 +2,17 @@ Builds the document cover from a TableProfile plus an optional ``ctx`` of presentation metadata. Reads everything defensively (``.get``) and degrades -honestly: a field that is neither in the profile nor in ``ctx`` is shown as a -placeholder rather than invented, leaving a hook for the LLM layer to fill it. +honestly. + +The dataset size (N rows x M columns) is always shown big, as a heading right +under the dataset name (kept together in a ``Group``), not buried in the +metadata table. The Description and Granularity are resolved through a cascade +so they are never empty: an explicit ``ctx`` value wins; otherwise the LLM block +(``profile['llm']`` from ``eda_llm_insights``) provides ``summary`` / +``row_meaning``; otherwise a short summary is derived from the profile itself +(shape, column-type mix, quality score) and a "Cada fila es…" sentence from the +key-candidate columns or the table shape. Nothing is invented: the derived +fallbacks state that they come from the profile. Contract for chapter authors (see ``docs/capabilities/automatic_eda.md``): build_(profile: dict, ctx: dict) -> Chapter | None @@ -17,10 +26,15 @@ from datetime import datetime, timezone from .. import model -CHAPTER_VERSION = "1.1.0" +CHAPTER_VERSION = "1.2.0" CHAPTER_ID = "portada" CHAPTER_TITLE = "Portada" +# Key under which eda_llm_insights stores its interpretive block in the profile. +# The cover reads ``summary`` (what the table is) and ``row_meaning`` (what one +# row represents) from it when the LLM layer ran (``run_llm``). +_LLM_KEY = "llm" + # Default human description of what the table quality score measures. Chapters # can override it via ctx["quality_criteria"]. _DEFAULT_QUALITY_CRITERIA = ( @@ -142,6 +156,88 @@ def _fmt_date_eu(value) -> str: return s +def _llm_block(profile: dict, ctx: dict) -> dict: + """Return the interpretive LLM block (``eda_llm_insights`` output), or {}. + + It is stored under ``profile['llm']`` by ``profile_table(run_llm=True)`` and + may also be forwarded in ``ctx['llm']``. Read defensively: anything that is + not a dict degrades to an empty dict so the cover never raises. + """ + block = profile.get(_LLM_KEY) + if not isinstance(block, dict): + block = ctx.get(_LLM_KEY) + return block if isinstance(block, dict) else {} + + +def _count_column_types(profile: dict, ctx: dict): + """Best-effort (n_numeric, n_categorical) for the dataset. + + Prefers the aggregated ``ctx['document_summary']`` (computed by the engine + over the whole body); falls back to counting the profile columns directly so + the cover still has the numbers when no summary was passed. + """ + summary = ctx.get("document_summary") + if isinstance(summary, dict): + n_num = summary.get("n_numeric") + n_cat = summary.get("n_categorical") + if n_num is not None or n_cat is not None: + return n_num, n_cat + cols = profile.get("columns") or [] + n_num = sum(1 for c in cols if isinstance(c, dict) + and c.get("inferred_type") == "numeric") + n_cat = sum(1 for c in cols if isinstance(c, dict) + and isinstance(c.get("categorical"), dict) + and c.get("categorical", {}).get("top") + and c.get("inferred_type") != "numeric") + return n_num, n_cat + + +def _derive_description(profile: dict, ctx: dict) -> str: + """A short, honest description of the dataset from the profile. + + Used only when no explicit ``ctx['description']`` and no LLM ``summary`` are + available. Summarizes shape, column-type mix and quality score; never empty, + never invents business meaning (it states the description was derived).""" + n_rows = profile.get("n_rows") + n_cols = profile.get("n_cols") + n_num, n_cat = _count_column_types(profile, ctx) + head = f"Conjunto de datos con {_fmt_int(n_rows)} filas y {_fmt_int(n_cols)} columnas" + type_bits = [] + if n_num: + type_bits.append(f"{_fmt_int(n_num)} numéricas") + if n_cat: + type_bits.append(f"{_fmt_int(n_cat)} categóricas") + if type_bits: + head += " (" + ", ".join(type_bits) + ")" + parts = [head + "."] + score = profile.get("quality_score") + if score is not None: + parts.append(f"Calidad media estimada: {score}/100.") + parts.append( + "Resumen derivado del perfil; active la interpretación LLM (`run_llm`) " + "para una descripción de negocio más rica.") + return " ".join(parts) + + +def _derive_granularity(profile: dict, dataset_name: str) -> str: + """A ``Cada fila es…`` granularity sentence from the profile. + + Prefers the key-candidate columns (a row is identified by them); when no key + is detected, falls back to the table shape so the line is always meaningful + and starts with ``Cada fila es`` as the user requested.""" + keys = profile.get("key_candidates") or [] + if keys: + shown = ", ".join(str(k) for k in keys[:3]) + more = "" if len(keys) <= 3 else f" (y {len(keys) - 3} más)" + return (f"Cada fila es un registro identificado por {shown}{more}, " + "candidata(s) a clave por ser únicas y sin nulos.") + n_rows = profile.get("n_rows") + tail = f" El dataset tiene {_fmt_int(n_rows)} filas en total." if n_rows else "" + return (f"Cada fila es un registro de «{dataset_name}». No se detectó una " + "columna identificadora única, así que la granularidad se infiere " + "de la forma de la tabla." + tail) + + def build_portada(profile: dict, ctx: dict): """Build the cover Chapter, or None if there is truly nothing to show.""" profile = profile or {} @@ -166,30 +262,38 @@ def build_portada(profile: dict, ctx: dict): quality_criteria = ctx.get("quality_criteria") or _DEFAULT_QUALITY_CRITERIA quality_value = "—" if score is None else f"{score} / 100" - # Granularity: ctx wins; else derive from key candidates; else be honest. + llm = _llm_block(profile, ctx) + + # Granularity: explicit ctx wins; then the LLM "row_meaning"; then the key + # candidates; finally a shape-based fallback. Always a real "Cada fila es…". granularity = ctx.get("granularity") if not granularity: - keys = profile.get("key_candidates") or [] - if keys: - granularity = ("Cada fila parece identificada por " - + ", ".join(str(k) for k in keys[:3]) + ".") - else: - granularity = ("Cada fila es… (granularidad no determinada — " - "pendiente de la capa de cálculo/LLM).") + granularity = (llm.get("row_meaning") or "").strip() or None + if not granularity: + granularity = _derive_granularity(profile, str(dataset_name)) + # Description: explicit ctx wins; then the LLM "summary"; finally a short + # profile-derived summary. Never the old empty placeholder. description = ctx.get("description") if not description: - description = ("Descripción no provista — pendiente de la capa LLM " - "(`run_llm`) o de `ctx['description']`.") + description = (llm.get("summary") or "").strip() or None + if not description: + description = _derive_description(profile, ctx) - blocks = [ + # Title + dataset size shown together and BIG (Heading) at the top, kept on + # the same page (Group). The size is no longer buried in the metadata table. + cover = [ model.Heading(text=str(dataset_name), level=1), model.Markdown(text="**Automatic-EDA** · informe exploratorio automático"), + model.Heading(text=shape, level=2), + ] + + blocks = [ + model.Group(blocks=cover), model.KVTable(rows=[ ("Fuente", source_origin), ("Almacenamiento", storage), ("Generado", when), - ("Tamaño", shape), ("Calidad", quality_value), ("Criterios de calidad", quality_criteria), ]), diff --git a/python/functions/datascience/automatic_eda/chapters/portada_test.py b/python/functions/datascience/automatic_eda/chapters/portada_test.py new file mode 100644 index 00000000..ae9df818 --- /dev/null +++ b/python/functions/datascience/automatic_eda/chapters/portada_test.py @@ -0,0 +1,197 @@ +"""Tests for the PORTADA (cover) chapter — DoD: golden + edges + render. + +Self-contained: builds synthetic TableProfiles (no DuckDB) so the suite is fast +and deterministic. Verifies the Fase 4b improvements: + +1. The dataset size (N rows x M columns) is always shown BIG — as a level-2 + heading kept together with the dataset name in a ``Group`` — and is no longer + a row of the metadata table. +2. Description and Granularity are resolved through a real cascade and are never + the old empty placeholders: an explicit ``ctx`` value wins; otherwise the LLM + block (``profile['llm']``) provides ``summary`` / ``row_meaning``; otherwise a + short summary is derived from the profile and a "Cada fila es…" sentence from + the key-candidate columns or the table shape. +3. The chapter degrades without raising on empty/None input. +4. It renders inside the full document to both PDF and PPTX showing that content. +""" + +import os +import re +import tempfile + +from pypdf import PdfReader +from pptx import Presentation + +from datascience.automatic_eda.model import Group, Heading, KVTable, Markdown +from datascience.automatic_eda.chapters.portada import ( + CHAPTER_ID, CHAPTER_VERSION, build_portada, +) +from datascience.render_automatic_eda_pdf import render_automatic_eda_pdf +from datascience.render_automatic_eda_pptx import render_automatic_eda_pptx + + +def _profile(with_llm: bool = True, with_keys: bool = True) -> dict: + prof = { + "table": "titanic", + "source": "/data/titanic.csv", + "profiled_at": "2026-06-30T10:00:00+00:00", + "n_rows": 891, + "n_cols": 12, + "quality_score": 78.0, + "columns": [ + {"name": "PassengerId", "inferred_type": "numeric", + "null_pct": 0.0, "numeric": {"mean": 446.0, "min": 1.0, + "max": 891.0, "std": 257.0}}, + {"name": "Survived", "inferred_type": "numeric", + "null_pct": 0.0, "numeric": {"mean": 0.38, "min": 0.0, + "max": 1.0, "std": 0.49}}, + {"name": "Sex", "inferred_type": "categorical", "null_pct": 0.0, + "categorical": {"top": [{"value": "male", "count": 577, "pct": 0.65}, + {"value": "female", "count": 314, + "pct": 0.35}], + "mode": "male", "n_distinct": 2, "entropy": 0.93}}, + ], + } + if with_keys: + prof["key_candidates"] = ["PassengerId"] + if with_llm: + prof["llm"] = { + "summary": "Pasajeros del Titanic con su supervivencia y datos de viaje.", + "row_meaning": "Cada fila es un pasajero del Titanic.", + "dictionary": [], "pii": [], "cleaning": [], "analyses": [], + } + return prof + + +def _pdf_text(path: str) -> str: + txt = "".join((pg.extract_text() or "") for pg in PdfReader(path).pages) + return re.sub(r"\s+", " ", txt) + + +def _pptx_text(path: str) -> str: + prs = Presentation(path) + parts = [] + for sl in prs.slides: + for sh in sl.shapes: + if sh.has_text_frame: + parts.append(sh.text_frame.text) + if sh.has_table: + tb = sh.table + for r in range(len(tb.rows)): + for c in range(len(tb.columns)): + parts.append(tb.cell(r, c).text) + return re.sub(r"\s+", " ", " ".join(parts)) + + +def _markdown_after(blocks, heading_text): + """Return the Markdown block that follows a Heading whose text matches.""" + for i, b in enumerate(blocks): + if isinstance(b, Heading) and heading_text.lower() in b.text.lower(): + for nb in blocks[i + 1:]: + if isinstance(nb, Markdown): + return nb + return None + + +def test_golden_tamano_grande_y_textos_llm(): + ch = build_portada(_profile(), {}) + assert ch is not None + assert ch.id == CHAPTER_ID + assert ch.version == CHAPTER_VERSION + + # 1) Title + size kept together in a Group; size is a BIG level-2 heading. + group = next(b for b in ch.blocks if isinstance(b, Group)) + inner = group.blocks + assert isinstance(inner[0], Heading) and inner[0].level == 1 + assert inner[0].text == "titanic" + size_h = next(b for b in inner if isinstance(b, Heading) and b.level == 2) + assert "891" in size_h.text and "12" in size_h.text + assert "filas" in size_h.text and "columnas" in size_h.text + + # 2) Size is no longer a row of the metadata table. + kv = next(b for b in ch.blocks if isinstance(b, KVTable)) + labels = [r[0] for r in kv.rows] + assert "Tamaño" not in labels + assert "Fuente" in labels and "Calidad" in labels + + # 3) Description and Granularity come from the LLM block. + desc = _markdown_after(ch.blocks, "Descripción") + gran = _markdown_after(ch.blocks, "Granularidad") + assert desc is not None and "Titanic" in desc.text + assert gran is not None and gran.text.startswith("Cada fila es") + assert "pasajero" in gran.text.lower() + + +def test_fallback_sin_llm_usa_keys_y_perfil(): + # No LLM block: description derived from the profile, granularity from keys. + ch = build_portada(_profile(with_llm=False, with_keys=True), {}) + desc = _markdown_after(ch.blocks, "Descripción") + gran = _markdown_after(ch.blocks, "Granularidad") + # Description is the derived summary, never the old "pendiente" placeholder. + assert "pendiente" not in desc.text.lower() + assert "891" in desc.text and "columnas" in desc.text + assert "numéricas" in desc.text or "categóricas" in desc.text + # Granularity mentions the key candidate and starts with "Cada fila es". + assert gran.text.startswith("Cada fila es") + assert "PassengerId" in gran.text + assert "…" not in gran.text # the old ellipsis placeholder is gone. + + +def test_fallback_sin_llm_sin_keys_usa_forma(): + ch = build_portada(_profile(with_llm=False, with_keys=False), {}) + gran = _markdown_after(ch.blocks, "Granularidad") + assert gran.text.startswith("Cada fila es") + assert "titanic" in gran.text.lower() + assert "pendiente" not in gran.text.lower() + + +def test_ctx_explicito_gana_sobre_llm(): + ctx = {"description": "Descripción manual.", + "granularity": "Cada fila es una unidad manual."} + ch = build_portada(_profile(), ctx) + desc = _markdown_after(ch.blocks, "Descripción") + gran = _markdown_after(ch.blocks, "Granularidad") + assert desc.text == "Descripción manual." + assert gran.text == "Cada fila es una unidad manual." + + +def test_edge_perfil_vacio_no_lanza(): + # Empty / None never raise; the cover still shows a size and real texts. + for prof, ctx in (({}, {}), (None, None)): + ch = build_portada(prof, ctx) + assert ch is not None + group = next(b for b in ch.blocks if isinstance(b, Group)) + size_h = next(b for b in group.blocks + if isinstance(b, Heading) and b.level == 2) + assert "filas" in size_h.text and "columnas" in size_h.text + desc = _markdown_after(ch.blocks, "Descripción") + gran = _markdown_after(ch.blocks, "Granularidad") + assert desc.text and "pendiente" not in desc.text.lower() + assert gran.text.startswith("Cada fila es") + + +def test_golden_render_pdf_muestra_portada(): + prof = _profile() + with tempfile.TemporaryDirectory() as d: + out = os.path.join(d, "eda.pdf") + res = render_automatic_eda_pdf(prof, out, {"title": "EDA"}) + assert res["path"] == out and os.path.exists(out) + assert CHAPTER_ID in [c["id"] for c in res["chapters"]] + txt = _pdf_text(out) + assert "titanic" in txt.lower() + assert "891" in txt and "filas" in txt and "columnas" in txt + assert "Titanic" in txt # LLM summary in the Description. + assert "Cada fila es" in txt # granularity sentence. + + +def test_golden_render_pptx_muestra_portada(): + prof = _profile() + with tempfile.TemporaryDirectory() as d: + out = os.path.join(d, "eda.pptx") + res = render_automatic_eda_pptx(prof, out, {"title": "EDA"}) + assert res["path"] == out and os.path.exists(out) + assert CHAPTER_ID in [c["id"] for c in res["chapters"]] + txt = _pptx_text(out) + assert "titanic" in txt.lower() + assert "891" in txt and "columnas" in txt + assert "Cada fila es" in txt From 7045f37554a58991d1baa2ec8ff98995b6393bb3 Mon Sep 17 00:00:00 2001 From: Egutierrez Date: Tue, 30 Jun 2026 18:07:12 +0200 Subject: [PATCH 27/53] =?UTF-8?q?fix(eda):=20quita=20r=C3=B3tulos=20duplic?= =?UTF-8?q?ados=20en=20cap=C3=ADtulo=20AN=C3=81LISIS=20LLM?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit El capítulo etiquetaba dos secciones por partida doble: un Heading de nivel 2 más el 'title' del propio DataTable, imprimiendo 'Diccionario de datos' y 'Datos personales (PII / RGPD)' dos veces seguidas en PDF y PPTX. Se elimina el 'title' de ambos DataTable y se conserva el Heading único (el patrón canónico OVERVIEW del contrato §8: el rótulo lo da el Heading, la tabla solo repite su cabecera de columnas al paginar). El DataTable de PII mantiene su 'note' orientativa. La columna del diccionario ya lee 'Significado de negocio'. CHAPTER_VERSION 1.0.0 -> 1.1.0. Test nuevo test_sin_rotulos_duplicados_y_significado_de_negocio fija: tablas sin title, cabecera exacta 'Significado de negocio', y cada rótulo una sola vez en el PDF. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../automatic_eda/chapters/analisis_llm.py | 22 ++++++++-- .../chapters/analisis_llm_test.py | 41 ++++++++++++++++++- 2 files changed, 58 insertions(+), 5 deletions(-) diff --git a/python/functions/datascience/automatic_eda/chapters/analisis_llm.py b/python/functions/datascience/automatic_eda/chapters/analisis_llm.py index e182e6a0..f1a8511c 100644 --- a/python/functions/datascience/automatic_eda/chapters/analisis_llm.py +++ b/python/functions/datascience/automatic_eda/chapters/analisis_llm.py @@ -42,7 +42,11 @@ from __future__ import annotations from .. import model -CHAPTER_VERSION = "1.0.0" +# 1.1.0: drop the duplicated section labels — the dictionary and PII DataTables +# no longer carry a ``title`` (the section Heading labels them once, per the +# OVERVIEW pattern in the contract). The data-dictionary column already reads +# "Significado de negocio". +CHAPTER_VERSION = "1.1.0" CHAPTER_ID = "analisis_llm" CHAPTER_TITLE = "Análisis LLM" @@ -118,6 +122,11 @@ def _dictionary_block(llm: dict): Columns: Columna / Descripción / Significado de negocio / Unidad. The paginator splits this by rows repeating the header and wraps long cells, so a long dictionary (many columns) never gets cut. + + The block carries **no** ``title``: the section is labelled once by the + ``Heading`` that ``build_analisis_llm`` appends right before it (the canonical + OVERVIEW pattern, contract §8). Giving the table its own ``title`` too would + print "Diccionario de datos" twice in a row. """ entries = llm.get("dictionary") if not isinstance(entries, (list, tuple)) or not entries: @@ -137,7 +146,7 @@ def _dictionary_block(llm: dict): ]) if not rows: return None - return model.DataTable(header=header, rows=rows, title="Diccionario de datos") + return model.DataTable(header=header, rows=rows) def _analyses_blocks(llm: dict) -> list: @@ -159,7 +168,12 @@ def _cleaning_blocks(llm: dict) -> list: def _pii_block(llm: dict): - """DataTable for PII/GDPR findings, or None if absent/empty.""" + """DataTable for PII/GDPR findings, or None if absent/empty. + + Like the dictionary block, it carries **no** ``title`` (the ``Heading`` in + ``build_analisis_llm`` labels the section once); it keeps its ``note`` with + the orientative-detection caveat, which the renderers print under the table. + """ entries = llm.get("pii") if not isinstance(entries, (list, tuple)) or not entries: return None @@ -176,7 +190,7 @@ def _pii_block(llm: dict): if not rows: return None return model.DataTable( - header=header, rows=rows, title="Datos personales (PII / RGPD)", + header=header, rows=rows, note="detección automática orientativa — revisar antes de tratar los datos") diff --git a/python/functions/datascience/automatic_eda/chapters/analisis_llm_test.py b/python/functions/datascience/automatic_eda/chapters/analisis_llm_test.py index 2b32470a..56b884e5 100644 --- a/python/functions/datascience/automatic_eda/chapters/analisis_llm_test.py +++ b/python/functions/datascience/automatic_eda/chapters/analisis_llm_test.py @@ -24,7 +24,7 @@ from pptx import Presentation from datascience.automatic_eda.chapters.analisis_llm import ( build_analisis_llm, CHAPTER_VERSION) from datascience.automatic_eda.chapters_registry import build_document -from datascience.automatic_eda.model import Chapter, DataTable +from datascience.automatic_eda.model import Chapter, DataTable, Heading from datascience.render_automatic_eda_pdf import render_automatic_eda_pdf from datascience.render_automatic_eda_pptx import render_automatic_eda_pptx @@ -117,6 +117,45 @@ def test_golden_build_y_render_pdf_pptx(): assert "DESCTOKEN" in ptx +def test_sin_rotulos_duplicados_y_significado_de_negocio(): + """The dictionary / PII sections must be labelled ONCE. + + Regression for the duplicated 'Diccionario de datos' and 'Datos personales + (PII / RGPD)' headings (each section used to print its label twice: a Heading + plus the DataTable's own title). The fix drops the DataTable title and keeps + a single Heading — the OVERVIEW pattern. The data-dictionary column header is + also pinned to the exact text 'Significado de negocio'. + """ + ch = build_analisis_llm(_profile(), {}) + assert ch is not None + + # Structure: section labels come from Headings; tables carry no title. + headings = [b.text for b in ch.blocks if isinstance(b, Heading)] + assert headings.count("Diccionario de datos") == 1 + assert headings.count("Datos personales (PII / RGPD)") == 1 + for b in ch.blocks: + if isinstance(b, DataTable): + assert not b.title, f"DataTable should not duplicate the label: {b.title!r}" + + # The data dictionary's third column reads exactly 'Significado de negocio'. + dicts = [b for b in ch.blocks if isinstance(b, DataTable) and "Descripción" in b.header] + assert dicts, "expected the data-dictionary DataTable" + assert dicts[0].header == ["Columna", "Descripción", "Significado de negocio", "Unidad"] + + # The PII table keeps its orientative-detection note. + pii = [b for b in ch.blocks if isinstance(b, DataTable) and b.header == ["Columna", "Tipo", "Severidad"]] + assert pii and pii[0].note and "orientativa" in pii[0].note + + # Render: each label appears exactly once across the whole document (the only + # 'Diccionario de datos' / 'Datos personales' producer is this chapter). + with tempfile.TemporaryDirectory() as d: + out_pdf = os.path.join(d, "eda.pdf") + render_automatic_eda_pdf(_profile(), out_pdf, {"title": "EDA — ventas"}) + txt = _pdf_text(out_pdf) + assert txt.count("Diccionario de datos") == 1 + assert txt.count("Datos personales") == 1 + + def test_orden_capitulo_junto_a_overview(): chapters = build_document(_profile(), {}) ids = [c.id for c in chapters] From a2074a0167df7921653964fca1ba1da229ecb3ef Mon Sep 17 00:00:00 2001 From: Egutierrez Date: Tue, 30 Jun 2026 18:10:23 +0200 Subject: [PATCH 28/53] =?UTF-8?q?feat(eda):=20nueva=20f=C3=B3rmula=20de=20?= =?UTF-8?q?calidad=20de=20datos=20(report=202046)=20+=20cap=C3=ADtulo=20ca?= =?UTF-8?q?lidad?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implementa el modelo de calidad del report 2046 en el grupo eda. Score de columna: 0.6·completeness + 0.4·validity con renormalización por aplicabilidad (si la validez no es medible —texto libre o columna 100% nula— el score se basa solo en completeness). Validez = conformidad real al tipo: nativo numérico/fecha/bool = 1.0; texto promovido a número/fecha = parse rate (validity_rate); texto con semantic_type = match_rate; texto libre = no aplica. Outliers, columnas constantes e identificadores salen del score a un bloque de observaciones analíticas (no son defectos de calidad). Se elimina el doble conteo de la falta de datos (mostly_null ya no castiga validez) y el bug de escala de outliers (que además ya no entran en el score). Score de dataset: 100·(0.85·cell_quality + 0.15·row_uniqueness) en vez de la media simple. Se pobla duplicate_rows/duplicate_pct push-down en summarize_table_duckdb (COUNT sobre DISTINCT *, sin RAM) para habilitar la unicidad de registro; renormaliza a solo cell_quality si no se puede calcular. Capítulo calidad (v2.0.0): intro de dos dimensiones (60/40) que declara que los outliers no bajan el score; tabla de scores Columna|Calidad|Completitud|Validez (sin Consistencia, n/a cuando no aplica); DOS tablas separadas (Problemas de calidad vs Observaciones analíticas); resumen con Unicidad de registro; glosario clicable de completitud, validez, unicidad de registro y calidad de datos. Verificado: 123 tests verdes (automatic_eda + render_automatic_eda + column_quality_score + summarize_table_duckdb + profile_table). Golden EDA de titanic (run_models+run_llm) con score recomputado a mano, outliers separados en observaciones y glosario clicable (5 links GOTO en el PDF). column_quality_score v2.0.0, summarize_table_duckdb v1.1.0, profile_table v1.1.0. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../automatic_eda/chapters/calidad.py | 259 +++++++++++----- .../automatic_eda/chapters/calidad_test.py | 159 ++++++---- .../datascience/column_quality_score.md | 122 +++++--- .../datascience/column_quality_score.py | 278 ++++++++++++------ .../datascience/column_quality_score_test.py | 194 ++++++++---- .../datascience/summarize_table_duckdb.md | 3 +- .../datascience/summarize_table_duckdb.py | 19 +- .../summarize_table_duckdb_test.py | 24 ++ python/functions/pipelines/profile_table.md | 11 +- python/functions/pipelines/profile_table.py | 38 ++- 10 files changed, 779 insertions(+), 328 deletions(-) diff --git a/python/functions/datascience/automatic_eda/chapters/calidad.py b/python/functions/datascience/automatic_eda/chapters/calidad.py index dcedcf6f..93ef9423 100644 --- a/python/functions/datascience/automatic_eda/chapters/calidad.py +++ b/python/functions/datascience/automatic_eda/chapters/calidad.py @@ -1,22 +1,26 @@ """Data-quality chapter (CALIDAD) for AutomaticEDA. Builds the quality chapter from a ``TableProfile`` of the ``eda`` group. The -chapter answers, in Spanish and as tables, the three things the user asked for: +chapter implements the quality model of report 2046: -1. **En qué se basa la calidad** — an intro paragraph explaining the criteria and - their weights (completeness, validity, consistency) before any number, plus a - table-level summary (global score and aggregates). +1. **En qué se basa la calidad** — an intro paragraph explaining the two scored + dimensions and their weights (completitud 60%, validez 40%) plus the + table-level row uniqueness, BEFORE any number, and stating explicitly that + outliers are reported as observations and do **not** lower the score. The + criteria terms (calidad de datos, completitud, validez, unicidad de registro) + are hooked into the shared glossary as clickable jumps. 2. **Scores por columna** — a table with, per column, the total quality score and - its breakdown into completeness / validity / consistency. -3. **Problemas en español** — a second table listing, per column, the readable - issues in Spanish (kept separate from the type ``flags``). + its breakdown into completeness / validity (no consistency dimension). +3. **Problemas de calidad** — a table listing ONLY real quality defects + (nulls, empty cells, values not conforming to their type/semantics). +4. **Observaciones analíticas** — a SEPARATE table for outliers, constant + columns, high-cardinality ids and strong skew, with an explicit note that + these do not affect the score. -The breakdown and the issues are NOT recomputed here: they come from the registry -function ``column_quality_score`` (group ``eda``), which already derives -``{score, completeness, validity, consistency, issues}`` from the ColumnProfile. -This chapter is render-only — it consumes that function and lays the result out -as model blocks; the renderers paginate tables (splitting by rows, repeating the -header) and wrap long cells so nothing is ever cut. +The breakdown, issues and observations are NOT recomputed here: they come from +the registry function ``column_quality_score`` (group ``eda``), which derives +``{score, completeness, validity, dimensions, applicable, issues, +observations}`` from the ColumnProfile. This chapter is render-only. Contract: build_(profile, ctx) -> Chapter | None ; CHAPTER_VERSION = "x.y.z". """ @@ -33,28 +37,47 @@ try: # pragma: no cover - import wiring except Exception: # noqa: BLE001 - never let an import error abort the document. _column_quality_score = None -CHAPTER_VERSION = "1.0.0" +CHAPTER_VERSION = "2.0.0" CHAPTER_ID = "calidad" CHAPTER_TITLE = "Calidad" -# Weights mirror column_quality_score: completeness 0.5, validity 0.3, -# consistency 0.2. Kept here only to render the human explanation; the actual -# numbers always come from the function so the two never drift in computation. -_CRITERIA_INTRO = ( - "La calidad de cada columna es un score de 0 a 100 que combina tres " - "criterios, cada uno con un peso:\n\n" - "- **Completitud (peso 50%)**: proporción de valores presentes (sin nulos " - "ni vacíos). Una columna con muchos nulos baja de score.\n" - "- **Validez (peso 30%)**: los valores son coherentes con su tipo y rango " - "esperado (penaliza outliers y semánticas declaradas que no coinciden).\n" - "- **Consistencia (peso 20%)**: la columna aporta información útil (penaliza " - "columnas constantes o identificadores de cardinalidad muy alta).\n\n" - "Score = 100 × (0,5·completitud + 0,3·validez + 0,2·consistencia). " - "Los problemas detectados por columna se listan en español más abajo." -) +# Glossary terms this chapter explains (report 2046 §6). Registered in the shared +# collector and marked clickable on their first appearance (contract §11.1). +_TERMS = { + "calidad_datos": ( + "Calidad de datos (score 0-100)", + "Mide hasta qué punto los datos están presentes y son utilizables tal " + "cual, no si son «buenos para el análisis». Se compone solo de " + "dimensiones medibles automáticamente desde el perfil de la tabla, sin " + "fuente externa de verdad: completitud (60%), validez (40%, cuando es " + "medible) y, a nivel de tabla, unicidad de registro. Los valores " + "atípicos NO bajan la calidad: se listan aparte como observaciones.", + ), + "completitud": ( + "Completitud", + "Proporción de valores realmente presentes en una columna (1 − % de " + "nulos; en texto, las celdas vacías también cuentan como faltantes). Los " + "nulos y vacíos bajan el score porque falta información que debería " + "estar. Pesa el 60% del score de columna.", + ), + "validez": ( + "Validez", + "Proporción de valores que encajan con su tipo o formato esperado: un " + "número que parsea, una fecha legible, un email con forma de email. Los " + "valores que no parsean a su tipo bajan el score. Si la columna es texto " + "libre sin formato esperado, la validez no se puede medir y el score se " + "basa solo en la completitud. Pesa el 40% del score cuando es medible.", + ), + "unicidad_registro": ( + "Unicidad de registro", + "A nivel de tabla, las filas duplicadas restan calidad al conjunto " + "(1 − % de filas duplicadas). Es distinta de que una columna no-clave " + "repita valores, que no es un defecto de calidad.", + ), +} -# Cap for the joined issues cell so a single row never grows taller than a page; -# the remainder is summarized as "(+N más)" instead of being silently dropped. +# Cap for the joined cell so a single row never grows taller than a page; the +# remainder is summarized as "(+N más)" instead of being silently dropped. _ISSUES_MAXLEN = 160 @@ -82,12 +105,19 @@ def _fmt_unit_pct(value) -> str: return str(value) +def _fmt_validity(value) -> str: + """Validity is ``None`` when not applicable: show ``n/a`` not a fake 0%.""" + if value is None: + return "n/a" + return _fmt_unit_pct(value) + + def _quality_of(col: dict) -> dict: - """Return ``{score, completeness, validity, consistency, issues}`` for a column. + """Return the quality dict for a column. Uses the registry ``column_quality_score`` when available; otherwise falls back to the per-column ``quality_score`` already in the profile (number only, - empty breakdown/issues). Never raises. + empty breakdown/issues/observations). Never raises. """ if not isinstance(col, dict): col = {} @@ -98,26 +128,25 @@ def _quality_of(col: dict) -> dict: return res except Exception: # noqa: BLE001 - degrade instead of aborting. pass - # Fallback: only the final score is available pre-computed in the profile. return { "score": col.get("quality_score"), "completeness": None, "validity": None, - "consistency": None, "issues": [], + "observations": [], } -def _join_issues(issues) -> str: - """Join Spanish issue strings into one cell, truncating overly long lists. +def _join_cells(items) -> str: + """Join Spanish strings into one cell, truncating overly long lists. - The renderer wraps cell text, but a column with many long issues could make a - single row taller than a whole page; cap the length and append ``(+N más)`` - so the count of hidden issues is honest rather than silently lost. + The renderer wraps cell text, but a column with many long entries could make + a single row taller than a whole page; cap the length and append ``(+N más)`` + so the count of hidden entries is honest rather than silently lost. """ - if not isinstance(issues, (list, tuple)) or not issues: + if not isinstance(items, (list, tuple)) or not items: return "" - parts = [model._safe_str(i).strip() for i in issues] + parts = [model._safe_str(i).strip() for i in items] parts = [p for p in parts if p] if not parts: return "" @@ -142,6 +171,33 @@ def _columns_with_quality(profile: dict): yield c, _quality_of(c) +def _fmt_unit_pct_or_pct(value) -> str: + """Format a value that may be a 0-1 fraction or an already-0-100 percentage.""" + try: + num = float(value) + except (TypeError, ValueError): + return model._safe_str(value) + if num != num: # NaN + return "—" + pct = num * 100 if num <= 1.0 else num + text = f"{pct:.1f}".rstrip("0").rstrip(".") + return f"{text}%" + + +def _row_uniqueness(profile: dict): + """Return row uniqueness (1 - duplicate_pct) in [0,1], or None if unknown.""" + dup = profile.get("duplicate_pct") + if dup is None: + return None + try: + d = float(dup) + except (TypeError, ValueError): + return None + if d > 1.0: # tolerate a 0-100 scale + d = d / 100.0 + return max(0.0, min(1.0, 1.0 - d)) + + def _summary_block(profile: dict, evaluated: list): """Table-level KVTable: global score and quality aggregates.""" rows = [] @@ -153,14 +209,15 @@ def _summary_block(profile: dict, evaluated: list): if isinstance(q.get("completeness"), (int, float))] vals = [q.get("validity") for _, q in evaluated if isinstance(q.get("validity"), (int, float))] - cons = [q.get("consistency") for _, q in evaluated - if isinstance(q.get("consistency"), (int, float))] if comps: rows.append(("Completitud media", _fmt_unit_pct(sum(comps) / len(comps)))) if vals: - rows.append(("Validez media", _fmt_unit_pct(sum(vals) / len(vals)))) - if cons: - rows.append(("Consistencia media", _fmt_unit_pct(sum(cons) / len(cons)))) + rows.append(("Validez media (donde aplica)", + _fmt_unit_pct(sum(vals) / len(vals)))) + + ru = _row_uniqueness(profile) + if ru is not None: + rows.append(("Unicidad de registro", _fmt_unit_pct(ru))) n_problem = sum(1 for _, q in evaluated if q.get("issues")) rows.append(("Columnas con problemas", str(n_problem))) @@ -182,22 +239,9 @@ def _summary_block(profile: dict, evaluated: list): return model.KVTable(rows=rows, title="Resumen de calidad") -def _fmt_unit_pct_or_pct(value) -> str: - """Format a value that may be a 0-1 fraction or an already-0-100 percentage.""" - try: - num = float(value) - except (TypeError, ValueError): - return model._safe_str(value) - if num != num: # NaN - return "—" - pct = num * 100 if num <= 1.0 else num - text = f"{pct:.1f}".rstrip("0").rstrip(".") - return f"{text}%" - - def _scores_block(evaluated: list): - """DataTable with per-column score and its three-criteria breakdown.""" - header = ["Columna", "Calidad", "Completitud", "Validez", "Consistencia"] + """DataTable with per-column score and its completeness/validity breakdown.""" + header = ["Columna", "Calidad", "Completitud", "Validez"] rows = [] # Worst columns first so the reader sees the problems at the top. ordered = sorted( @@ -210,22 +254,22 @@ def _scores_block(evaluated: list): col.get("name") or "(col)", _fmt_score(q.get("score")), _fmt_unit_pct(q.get("completeness")), - _fmt_unit_pct(q.get("validity")), - _fmt_unit_pct(q.get("consistency")), + _fmt_validity(q.get("validity")), ]) if not rows: return None return model.DataTable(header=header, rows=rows, title="Scores de calidad por columna", - note="0 = peor, 100 = mejor; ordenado de peor a mejor") + note="0 = peor, 100 = mejor; «n/a» = dimensión no " + "medible; ordenado de peor a mejor") def _issues_block(evaluated: list): - """DataTable listing Spanish issues per column, or a Note when there are none.""" - header = ["Columna", "Problemas detectados (español)"] + """DataTable listing ONLY real quality defects per column, or a Note.""" + header = ["Columna", "Problemas de calidad (español)"] rows = [] for col, q in evaluated: - joined = _join_issues(q.get("issues")) + joined = _join_cells(q.get("issues")) if joined: rows.append([col.get("name") or "(col)", joined]) if not rows: @@ -235,6 +279,63 @@ def _issues_block(evaluated: list): title="Problemas de calidad por columna") +def _observations_block(evaluated: list): + """DataTable listing analytical observations per column, or None. + + Observations (outliers, constant columns, ids, strong skew) are NOT quality + defects: they do not affect the score. Returned as a separate table from the + issues so the report never presents a legitimate outlier as a problem. + """ + header = ["Columna", "Observaciones analíticas"] + rows = [] + for col, q in evaluated: + joined = _join_cells(q.get("observations")) + if joined: + rows.append([col.get("name") or "(col)", joined]) + if not rows: + return None + return model.DataTable( + header=header, rows=rows, + title="Observaciones analíticas por columna", + note="No son defectos de calidad y NO afectan al score; orientan el " + "análisis (atípicos, columnas constantes, identificadores).") + + +def _term(key: str, label: str, mark: bool) -> str: + """Render a term as a clickable glossary span when marking is enabled.""" + if mark: + return f"[[term:{key}]]**{label}**[[/term]]" + return f"**{label}**" + + +def _criteria_intro(mark: bool) -> str: + """Intro paragraph explaining the two scored dimensions and the principle.""" + calidad = _term("calidad_datos", "calidad de datos", mark) + completitud = _term("completitud", "Completitud (peso 60%)", mark) + validez = _term("validez", "Validez (peso 40%, cuando es medible)", mark) + unicidad = _term("unicidad_registro", "unicidad de registro", mark) + return ( + f"La {calidad} de cada columna es un score de 0 a 100 que combina solo " + "dimensiones medibles desde el perfil de la tabla, sin fuente externa " + "de verdad:\n\n" + f"- {completitud}: proporción de valores presentes (1 − % de nulos; en " + "texto, las celdas vacías cuentan como faltantes). Los nulos y vacíos " + "bajan el score.\n" + f"- {validez}: proporción de valores que encajan con su tipo o formato " + "(un número que parsea, una fecha legible, un email con forma de email). " + "Si una columna es texto libre sin formato esperado, la validez no se " + "mide y el score se basa solo en la completitud.\n\n" + f"Score de columna = 100 × (0,6·completitud + 0,4·validez), " + "renormalizado cuando la validez no aplica. A nivel de tabla se añade " + f"la {unicidad} (1 − % de filas duplicadas).\n\n" + "**Los valores atípicos (outliers) NO bajan la calidad.** Un valor " + "extremo puede ser real y correcto; detectar atípicos es parte del " + "análisis de la distribución, no un juicio de corrección. Por eso, junto " + "con las columnas constantes y los identificadores, se listan aparte " + "como **observaciones analíticas** que no afectan al score." + ) + + def build_calidad(profile: dict, ctx: dict): """Build the data-quality Chapter, or None if the profile has no columns. @@ -250,17 +351,35 @@ def build_calidad(profile: dict, ctx: dict): if not evaluated: return None # no columns to score -> chapter does not apply. + # Register the criteria terms in the shared glossary (if present) and mark + # their first appearance clickable. Contract §11.1. + glossary = ctx.get("glossary") + mark = False + if isinstance(glossary, model.GlossaryCollector): + for key, (label, definition) in _TERMS.items(): + glossary.add(key, label, definition) + mark = True + blocks = [ model.Heading(text="Cómo se calcula la calidad", level=2), - model.Markdown(text=_CRITERIA_INTRO), + model.Markdown(text=_criteria_intro(mark)), _summary_block(profile, evaluated), model.Heading(text="Scores por columna", level=2), ] scores = _scores_block(evaluated) if scores is not None: blocks.append(scores) - blocks.append(model.Heading(text="Problemas detectados", level=2)) + + blocks.append(model.Heading(text="Problemas de calidad", level=2)) blocks.append(_issues_block(evaluated)) + observations = _observations_block(evaluated) + if observations is not None: + blocks.append(model.Heading(text="Observaciones analíticas", level=2)) + blocks.append(model.Note( + "Las observaciones siguientes NO son defectos de calidad y no " + "afectan al score: son señales para orientar el análisis.")) + blocks.append(observations) + return model.Chapter(id=CHAPTER_ID, title=CHAPTER_TITLE, version=CHAPTER_VERSION, blocks=blocks) diff --git a/python/functions/datascience/automatic_eda/chapters/calidad_test.py b/python/functions/datascience/automatic_eda/chapters/calidad_test.py index 3e6bf5f6..5dc623ee 100644 --- a/python/functions/datascience/automatic_eda/chapters/calidad_test.py +++ b/python/functions/datascience/automatic_eda/chapters/calidad_test.py @@ -1,11 +1,12 @@ -"""Tests for the CALIDAD chapter — DoD: golden + edges + anti-cut. +"""Tests for the CALIDAD chapter — DoD: golden + edges + anti-cut + glossary. Self-contained: builds synthetic TableProfiles (no DuckDB) so the suite is fast -and deterministic. Verifies that the chapter explains the quality criteria, shows -per-column scores with the completeness/validity/consistency breakdown, lists the -issues in Spanish (separate from the type flags), returns None when it does not -apply, and that a wide profile with long names renders to PDF and PPTX without -cutting any cell text (long content wraps, it is never truncated). +and deterministic. Verifies the report-2046 quality model: the chapter explains +the two scored dimensions (completitud 60% / validez 40%), shows per-column +scores without a consistency column, keeps quality DEFECTS (issues) separate +from analytical OBSERVATIONS (outliers, constant, ids), hooks the criteria terms +into the glossary, returns None when it does not apply, and renders a wide +profile to PDF and PPTX without cutting any cell text. """ import os @@ -20,28 +21,30 @@ from datascience.automatic_eda.chapters.calidad import ( CHAPTER_VERSION, ) from datascience.automatic_eda import build_document, render_pdf, render_pptx +from datascience.automatic_eda import model def _profile() -> dict: """A small profile with one column per quality problem (nulls, outliers, - constant, high-cardinality id) plus one clean column.""" + constant, high-cardinality id) plus one clean column. ``outlier_pct`` is in + the 0-100 scale that describe_numeric actually emits.""" return { "table": "demo", - "quality_score": 72.5, + "quality_score": 82.0, "duplicate_pct": 0.04, "null_cell_pct": 0.11, "constant_cols": ["flag_const"], "all_null_cols": [], "columns": [ - {"name": "edad", "inferred_type": "integer", "null_pct": 0.2, - "numeric": {"outlier_pct": 0.15, "min": 0, "max": 99}, - "quality_score": 60}, + {"name": "edad", "inferred_type": "numeric", "null_pct": 0.2, + "n_rows": 100, "unique_pct": 0.5, + "numeric": {"outlier_pct": 15.0, "min": 0, "max": 99}}, {"name": "nombre", "inferred_type": "text", "null_pct": 0.0, - "unique_pct": 0.98, "quality_score": 80}, + "unique_pct": 0.98, "flags": ["possible_id"]}, {"name": "flag_const", "inferred_type": "text", "null_pct": 0.0, - "flags": ["constant"], "quality_score": 50}, - {"name": "limpia", "inferred_type": "float", "null_pct": 0.0, - "numeric": {"outlier_pct": 0.0}, "quality_score": 100}, + "unique_pct": 0.01, "flags": ["constant"]}, + {"name": "limpia", "inferred_type": "numeric", "null_pct": 0.0, + "unique_pct": 0.5, "numeric": {"outlier_pct": 0.0}}, ], } @@ -50,16 +53,9 @@ def _tables(chapter): return [b for b in chapter.blocks if getattr(b, "kind", None) == "data_table"] -def _scores_table(chapter): +def _table_by_title(chapter, needle): for t in _tables(chapter): - if "Scores" in (t.title or ""): - return t - return None - - -def _issues_table(chapter): - for t in _tables(chapter): - if "Problemas" in (t.title or ""): + if needle in (t.title or ""): return t return None @@ -73,41 +69,84 @@ def test_golden_chapter_estructura_y_version(): assert ch.id == "calidad" assert ch.version == CHAPTER_VERSION kinds = [b.kind for b in ch.blocks] - # intro heading + markdown criteria + summary kv + scores table + issues table assert "markdown" in kinds and "kv_table" in kinds and "data_table" in kinds -def test_golden_intro_explica_criterios_y_pesos(): +def test_golden_intro_explica_dos_dimensiones_y_pesos(): ch = build_calidad(_profile(), {}) intro = [b for b in ch.blocks if b.kind == "markdown"][0].text - for needle in ("Completitud", "Validez", "Consistencia", - "50%", "30%", "20%"): + for needle in ("Completitud", "Validez", "60%", "40%", + "unicidad de registro"): assert needle in intro, f"falta {needle!r} en la intro de criterios" + # El principio: los outliers NO bajan la calidad. + assert "atípicos" in intro and "NO bajan" in intro + # Ya no se menciona la dimensión consistencia eliminada. + assert "20%" not in intro -def test_golden_scores_incluyen_desglose_por_criterio(): +def test_golden_scores_sin_columna_consistencia(): ch = build_calidad(_profile(), {}) - scores = _scores_table(ch) + scores = _table_by_title(ch, "Scores") assert scores is not None - assert scores.header == ["Columna", "Calidad", "Completitud", - "Validez", "Consistencia"] - # 4 columns scored, none dropped. + assert scores.header == ["Columna", "Calidad", "Completitud", "Validez"] + assert "Consistencia" not in scores.header assert len(scores.rows) == 4 names = {r[0] for r in scores.rows} assert names == {"edad", "nombre", "flag_const", "limpia"} -def test_golden_issues_en_espanol_separados_de_flags(): +def test_golden_outliers_en_observaciones_no_en_problemas(): ch = build_calidad(_profile(), {}) - issues = _issues_table(ch) - assert issues is not None - flat = " | ".join(" ".join(r) for r in issues.rows) - assert "nulos" in flat # completeness issue (ES) - assert "outliers" in flat # validity issue (ES) - assert "columna constante" in flat - assert "posible id de alta cardinalidad" in flat - # The raw type flag string must NOT leak as a "problem". - assert "constant" not in flat or "columna constante" in flat + problemas = _table_by_title(ch, "Problemas de calidad") + observaciones = _table_by_title(ch, "Observaciones") + assert problemas is not None + assert observaciones is not None + + problemas_txt = " | ".join(" ".join(r) for r in problemas.rows) + observaciones_txt = " | ".join(" ".join(r) for r in observaciones.rows) + + # Los nulos SÍ son problema de calidad. + assert "nulos" in problemas_txt + # Los outliers NO aparecen como problema... + assert "atípic" not in problemas_txt and "outlier" not in problemas_txt + # ...sino como observación analítica. + assert "atípic" in observaciones_txt + # Constante e id: observaciones, no problemas. + assert "constante" in observaciones_txt + assert "identificador" in observaciones_txt + assert "constante" not in problemas_txt + + +def test_golden_score_columna_limpia_es_100(): + """Columna sin nulos, numérica nativa: score 100 aunque tenga (o no) outliers.""" + ch = build_calidad(_profile(), {}) + scores = _table_by_title(ch, "Scores") + by_name = {r[0]: r for r in scores.rows} + assert by_name["limpia"][1] == "100 / 100" + # edad: 20% nulos -> 100*(0.6*0.8 + 0.4*1.0) = 88; los outliers no bajan nada. + assert by_name["edad"][1] == "88 / 100" + + +# --------------------------------------------------------------------------- # +# Glosario (contrato §11.1) +# --------------------------------------------------------------------------- # +def test_glosario_registra_los_cuatro_terminos_y_marca_clicable(): + glossary = model.GlossaryCollector() + ch = build_calidad(_profile(), {"glossary": glossary}) + for key in ("calidad_datos", "completitud", "validez", "unicidad_registro"): + assert glossary.has(key), f"término {key!r} no registrado en el glosario" + intro = [b for b in ch.blocks if b.kind == "markdown"][0].text + # Con colector presente, la primera aparición se marca clicable. + assert "[[term:completitud]]" in intro + assert "[[term:validez]]" in intro + assert "[[term:calidad_datos]]" in intro + assert "[[term:unicidad_registro]]" in intro + + +def test_sin_glosario_no_marca_terminos(): + ch = build_calidad(_profile(), {}) # ctx sin glossary + intro = [b for b in ch.blocks if b.kind == "markdown"][0].text + assert "[[term:" not in intro # --------------------------------------------------------------------------- # @@ -124,17 +163,17 @@ def test_edge_perfil_limpio_sin_problemas_usa_nota(): prof = { "quality_score": 100, "columns": [ - {"name": "a", "inferred_type": "float", "null_pct": 0.0, - "numeric": {"outlier_pct": 0.0}}, - {"name": "b", "inferred_type": "float", "null_pct": 0.0, - "numeric": {"outlier_pct": 0.0}}, + {"name": "a", "inferred_type": "numeric", "null_pct": 0.0, + "unique_pct": 0.5, "numeric": {"outlier_pct": 0.0}}, + {"name": "b", "inferred_type": "numeric", "null_pct": 0.0, + "unique_pct": 0.5, "numeric": {"outlier_pct": 0.0}}, ], } ch = build_calidad(prof, {}) assert ch is not None - assert _issues_table(ch) is None # no issues table + assert _table_by_title(ch, "Problemas de calidad") is None # no issues table notes = [b for b in ch.blocks if b.kind == "note"] - assert notes and "No se detectaron problemas" in notes[0].text + assert any("No se detectaron problemas" in n.text for n in notes) # --------------------------------------------------------------------------- # @@ -143,44 +182,42 @@ def test_edge_perfil_limpio_sin_problemas_usa_nota(): def _wide_profile(ncols: int = 22) -> dict: cols = [ {"name": "identificador_unico_de_transaccion_con_nombre_muy_largo", - "inferred_type": "text", "null_pct": 0.0, "unique_pct": 0.99}, + "inferred_type": "text", "null_pct": 0.0, "unique_pct": 0.99, + "flags": ["possible_id"]}, {"name": "columna_constante_sin_ninguna_variacion_de_valor", - "inferred_type": "text", "null_pct": 0.0, "flags": ["constant"]}, + "inferred_type": "text", "null_pct": 0.0, "unique_pct": 0.01, + "flags": ["constant"]}, ] for k in range(ncols - 2): cols.append({ "name": f"metrica_numerica_de_negocio_{k:02d}_con_nombre_largo", - "inferred_type": "float", "null_pct": 0.1 + (k % 3) * 0.05, - "numeric": {"outlier_pct": 0.08, "min": 0, "max": 1000}, + "inferred_type": "numeric", "null_pct": 0.1 + (k % 3) * 0.05, + "unique_pct": 0.5, + "numeric": {"outlier_pct": 8.0, "min": 0, "max": 1000}, }) - return {"table": "ancha", "quality_score": 70.0, "columns": cols} + return {"table": "ancha", "quality_score": 70.0, "duplicate_pct": 0.0, + "columns": cols} def test_anticut_pdf_y_pptx_no_truncan_nombres_largos(): prof = _wide_profile(22) full = build_document(prof, {"dataset_name": "ancha"}) assert any(c.id == "calidad" for c in full) - # Render ONLY the calidad chapter so the anti-cut assertions are scoped to - # this chapter (other chapters, e.g. portada, legitimately contain '…'). chapters = [c for c in full if c.id == "calidad"] long_name = "metrica_numerica_de_negocio_00_con_nombre_largo" with tempfile.TemporaryDirectory() as d: pdf = os.path.join(d, "q.pdf") pptx = os.path.join(d, "q.pptx") rp = render_pdf(chapters, pdf, {"title": "EDA"}) - rx = render_pptx(chapters, pptx, {"title": "EDA"}) + render_pptx(chapters, pptx, {"title": "EDA"}) assert os.path.exists(pdf) and os.path.exists(pptx) - # The wide table forces pagination across several pages/slides. assert (rp or {}).get("n_pages", 0) >= 2 - # PDF: the long name survives whole once wraps (spaces/newlines) removed, - # and there is no truncation marker. pdf_txt = "".join((pg.extract_text() or "") for pg in PdfReader(pdf).pages) assert "…" not in pdf_txt and "..." not in pdf_txt norm = re.sub(r"\s+", "", pdf_txt) assert long_name in norm, "el nombre largo se cortó en el PDF" - # PPTX: long name present in some cell, untruncated. allt = [] for s in Presentation(pptx).slides: for sh in s.shapes: diff --git a/python/functions/datascience/column_quality_score.md b/python/functions/datascience/column_quality_score.md index 2e5fa7af..0a33f60d 100644 --- a/python/functions/datascience/column_quality_score.md +++ b/python/functions/datascience/column_quality_score.md @@ -4,10 +4,10 @@ name: column_quality_score kind: function lang: py domain: datascience -version: "1.0.0" +version: "2.0.0" purity: pure signature: "def column_quality_score(col: dict) -> dict" -description: "Calcula un score de calidad de datos 0-100 para un ColumnProfile del grupo eda, con desglose completeness/validity/consistency y lista de issues legibles. Funcion pura, no muta el input." +description: "Calcula un score de calidad de datos 0-100 para un ColumnProfile del grupo eda. Combina completeness (0.6) y validity (0.4) con renormalizacion por aplicabilidad; los outliers, columnas constantes e ids NO bajan el score (van a observations). Devuelve desglose por dimension, issues (defectos) y observations (señales analiticas). Funcion pura, no muta el input." tags: [eda, data-quality, profiling, scoring, datascience] uses_functions: [] uses_types: [] @@ -17,20 +17,26 @@ error_type: "" imports: [] example: | from datascience import column_quality_score - col = {"name": "precio", "inferred_type": "float", "null_pct": 0.2, - "unique_pct": 0.4, "flags": [], "numeric": {"outlier_pct": 0.08}} + col = {"name": "precio", "inferred_type": "numeric", "null_pct": 0.2, + "unique_pct": 0.4, "flags": [], "numeric": {"outlier_pct": 8.0}} column_quality_score(col) - # {"score": 86.8, "completeness": 0.8, "validity": 0.92, - # "consistency": 1.0, "issues": ["20% nulos", "8% outliers"]} + # {"score": 88.0, "completeness": 0.8, "validity": 1.0, + # "applicable": ["completeness", "validity"], "issues": ["20% nulos"], + # "observations": ["8% de valores atípicos (z-score>3): ..."]} tested: true tests: - "test_clean_column_high_score" - - "test_half_null_lowers_completeness_and_score" - - "test_constant_column_flags_issue" + - "test_weights_60_40_native_type" + - "test_outliers_do_not_penalize_score" + - "test_nulls_lower_score_more_than_outliers" + - "test_validity_from_parse_rate_lowers_score" + - "test_validity_from_match_rate" + - "test_free_text_renormalizes_to_completeness_only" + - "test_all_null_column_scores_zero" + - "test_constant_column_scores_full_and_is_observation" + - "test_high_cardinality_id_scores_full_and_is_observation" + - "test_mostly_null_no_double_counts_validity" - "test_empty_dict_does_not_crash" - - "test_outliers_penalize_validity" - - "test_mostly_null_flag_halves_validity" - - "test_high_cardinality_text_flagged_as_id" - "test_none_values_treated_defensively" - "test_does_not_mutate_input" test_file_path: "python/functions/datascience/column_quality_score_test.py" @@ -38,16 +44,22 @@ file_path: "python/functions/datascience/column_quality_score.py" params: - name: col desc: > - ColumnProfile dict del grupo eda (p.ej. salida de summarize_table_duckdb). - Se leen sus claves de forma defensiva con .get(...) y se toleran valores - None. Claves usadas: null_pct (0-1), inferred_type, semantic_type, - unique_pct (0-1), flags (list[str], reconoce "constant"/"mostly_null"), - numeric ({outlier_pct: 0-1, ...}|None) y match_rate (opcional, 0-1). + ColumnProfile dict del grupo eda (p.ej. salida de summarize_table_duckdb / + profile_table). Se leen sus claves de forma defensiva con .get(...) y se + toleran valores None. Claves usadas: null_pct (0-1), n_rows, empty_count + (texto), inferred_type, semantic_type, validity_rate (0-1, lo expone + profile_table al promocionar texto a numero/fecha), match_rate (0-1), + unique_pct (0-1), flags (list[str], reconoce + "constant"/"possible_id"/"high_cardinality") y numeric ({outlier_pct: 0-100, + skew, ...}|None). output: > - dict con score (float 0-100, redondeado a 1 decimal), completeness (0-1), - validity (0-1), consistency (0-1) e issues (list[str] de descripciones - legibles de los problemas detectados). score = round(100 * (0.5*completeness - + 0.3*validity + 0.2*consistency), 1). + dict con score (float 0-100, 1 decimal), completeness (0-1), validity (0-1 o + None si no aplicable), dimensions ({completeness, validity}), applicable + (list[str] de dimensiones que entraron en el score), issues (list[str] SOLO de + defectos de calidad: nulos, vacios, valores no conformes) y observations + (list[str] de señales analiticas que NO bajan el score: outliers, columna + constante, posible id, asimetria). score = round(100 * (0.6*completeness + + 0.4*validity) / pesos_aplicables, 1), renormalizado cuando validity no aplica. --- ## Ejemplo @@ -59,51 +71,71 @@ from datascience import column_quality_score col = { "name": "precio", "physical_type": "DOUBLE", - "inferred_type": "float", + "inferred_type": "numeric", "semantic_type": "", - "count": 800, "n_rows": 1000, "null_count": 200, "null_pct": 0.20, "distinct_count": 400, "unique_pct": 0.40, "flags": [], - "numeric": {"outlier_pct": 0.08}, + "numeric": {"outlier_pct": 8.0, "skew": 0.3}, "categorical": None, "datetime": None, } column_quality_score(col) # { -# "score": 86.8, -# "completeness": 0.8, # 1 - 0.20 -# "validity": 0.92, # 1 - min(0.08, 0.3) -# "consistency": 1.0, -# "issues": ["20% nulos", "8% outliers"], +# "score": 88.0, # 100 * (0.6*0.8 + 0.4*1.0) +# "completeness": 0.8, # 1 - 0.20 +# "validity": 1.0, # numerica nativa: el tipo es conforme +# "dimensions": {"completeness": 0.8, "validity": 1.0}, +# "applicable": ["completeness", "validity"], +# "issues": ["20% nulos"], # SOLO defectos de calidad +# "observations": ["8% de valores atípicos (z-score>3): ..."], # NO bajan score # } ``` ## Cuando usarla Cuando hayas perfilado una tabla con el grupo `eda` (p.ej. -`summarize_table_duckdb`) y necesites un numero 0-100 por columna para -ordenar/priorizar limpieza de datos, pintar semaforos de calidad en un -dashboard, o decidir que columnas descartar antes de modelar. Es la capa de -scoring sobre el ColumnProfile crudo: lee el perfil, no toca los datos. +`summarize_table_duckdb` / `profile_table`) y necesites un numero 0-100 por +columna para ordenar/priorizar limpieza de datos, pintar semaforos de calidad, +o decidir que columnas descartar antes de modelar. Separa los **defectos de +calidad reales** (`issues`: nulos, vacios, valores que no parsean a su tipo) de +las **observaciones analiticas** (`observations`: outliers, columnas constantes, +ids), que se reportan pero no penalizan. Es la capa de scoring sobre el +ColumnProfile crudo: lee el perfil, no toca los datos. -## Notas +## Gotchas -Funcion pura, sin I/O ni dependencias externas, no muta `col`. Lee todas las -claves con `.get(...)` y tolera que vengan en `None` (un ColumnProfile recien -salido de `summarize_table_duckdb` trae muchas claves a `None`), por lo que -nunca falla por claves ausentes — un `{}` produce un resultado bien definido. +Funcion pura, sin I/O, no muta `col`. Aun asi conviene saber: -Pesos del score: completeness 0.5, validity 0.3, consistency 0.2. +- **Los outliers NO bajan el score.** Un valor extremo puede ser real y correcto + (un cliente que compra mucho); detectar atipicos es analisis de la + distribucion, no un juicio de correccion. Salen en `observations`, no en + `issues`. Mismo trato para columnas constantes e identificadores de alta + cardinalidad: son observaciones, no defectos. +- **`validity` puede ser `None`** (no aplicable): texto libre sin `semantic_type` + ni `validity_rate`, o columna 100% nula. En ese caso el score se renormaliza a + solo `completeness` (la columna no se premia ni castiga por algo no medible). +- **`outlier_pct` se interpreta en escala 0-100** (la que emite + `describe_numeric`, z-score>3). Pasar una fraccion 0-1 produce un texto de + observacion con el % equivocado, pero NUNCA afecta al score. +- **`validity_rate` lo puebla `profile_table`** al promocionar una columna de + texto a numero/fecha (fraccion que parsea). Si no esta presente y el tipo es + nativo numerico/fecha/bool, `validity = 1.0`. +- Sin doble conteo: la falta de datos cuenta solo en `completeness` (el antiguo + castigo de `mostly_null` sobre `validity` se elimino). -- **completeness** = `1 - null_pct` (None -> 0 nulls -> 1.0). -- **validity**: parte de 1.0 y penaliza `min(outlier_pct, 0.3)` en columnas - numericas, `0.5 * (1 - match_rate)` si hay `semantic_type` declarado con - `match_rate` bajo disponible, y multiplica por 0.5 si el flag `mostly_null` - esta presente. -- **consistency**: 1.0 salvo flag `constant` (-> 0.3, columna poco informativa) - o texto con `unique_pct > 0.9` (-> 0.6, posible id de alta cardinalidad). +## Capability growth log + +- v2.0.0 (2026-06-30) — nueva formula de calidad (report 2046): pesos 60/40 + (completeness/validity) con renormalizacion por aplicabilidad; se elimina la + dimension `consistency`-como-informatividad y el doble castigo de + `mostly_null`; los outliers/constantes/ids salen del score a `observations`; + validity mide conformidad real (parse rate / match rate / tipo nativo). Salida + ampliada con `dimensions`, `applicable` y `observations`. +- v1.0.0 — version inicial: pesos 50/30/20 (completeness/validity/consistency), + los outliers penalizaban validity (con bug de escala) y consistency penalizaba + informatividad. diff --git a/python/functions/datascience/column_quality_score.py b/python/functions/datascience/column_quality_score.py index d16e9849..11d83988 100644 --- a/python/functions/datascience/column_quality_score.py +++ b/python/functions/datascience/column_quality_score.py @@ -1,34 +1,78 @@ """Score de calidad de datos (0-100) para un ColumnProfile del grupo eda. Funcion pura: dado el perfil de una columna producido por el grupo de -capacidad `eda` (p.ej. summarize_table_duckdb), calcula un score agregado -de calidad junto a su desglose en completeness / validity / consistency y -una lista de issues legibles. No realiza I/O ni muta el input. +capacidad `eda` (p.ej. summarize_table_duckdb / profile_table), calcula un +score agregado de calidad junto a su desglose por dimension y dos listas +legibles separadas: `issues` (defectos de calidad reales que SI bajan el +score) y `observations` (señales analiticas que NO bajan el score). No +realiza I/O ni muta el input. + +Modelo (DAMA-DMBOK / ISO 8000), ver report 2046: + +- Solo entran en el score las dimensiones medibles automaticamente desde el + perfil, sin fuente externa de verdad: completeness y validity por columna. +- Renormalizacion por aplicabilidad: si una dimension no es medible en la + columna (texto libre sin semantica -> validity no aplica; columna 100% nula + -> validity no medible), se excluye y los pesos se renormalizan sobre las + aplicables. Una columna ni se premia ni se castiga por algo no medible. +- Sin doble conteo: la falta de datos cuenta solo en completeness (se elimino + el antiguo castigo extra de `mostly_null` sobre validity). +- Los OUTLIERS NO bajan la calidad. Un valor extremo puede ser real y + correcto; detectar atipicos es analisis de la distribucion, no un juicio de + coreccion. Outliers, columnas constantes e identificadores de alta + cardinalidad pasan a `observations`, nunca a `issues`. """ +# Pesos base de las dimensiones de columna (se renormalizan por aplicabilidad). +_W_COMPLETENESS = 0.6 +_W_VALIDITY = 0.4 + +# Tipos inferidos cuyo almacen garantiza la conformidad de tipo (validity=1.0) +# cuando NO vienen de una promocion de texto (en cuyo caso manda validity_rate). +_NATIVE_TYPED = ("numeric", "integer", "float", "datetime", "date", "boolean", "bool") + + def column_quality_score(col: dict) -> dict: """Calcula un score de calidad de datos 0-100 para un ColumnProfile. - El score pondera tres dimensiones: - - completeness (0.5): proporcion de valores no nulos. - - validity (0.3): ausencia de outliers / heuristicas de validez. - - consistency (0.2): la columna aporta informacion (no constante, no ruido). + El score combina solo dimensiones de calidad medibles desde el perfil, con + renormalizacion por aplicabilidad: + + - completeness (peso base 0.6, siempre aplica): proporcion de valores + presentes = 1 - null_pct. En texto, las celdas vacias (`empty_count`) + tambien cuentan como faltantes. + - validity (peso base 0.4, cuando hay un criterio de validacion real): + fraccion de valores no nulos conformes a su tipo/semantica. Tipo nativo + numerico/fecha/bool = 1.0; texto promovido a numero/fecha = parse rate + (`validity_rate`); texto con `semantic_type` regexable = `match_rate`; + texto libre o columna 100% nula = NO aplicable (renormaliza a solo + completeness). + + Los outliers, columnas constantes, identificadores y asimetria fuerte NO + bajan el score: se devuelven en `observations`. Args: col: ColumnProfile dict del grupo eda. Se leen las claves de forma defensiva con .get(...) y se tolera que muchas vengan en None. - Claves relevantes: null_pct, inferred_type, semantic_type, - unique_pct, flags (list[str]), numeric ({outlier_pct, ...}|None), - match_rate (opcional). + Claves relevantes: null_pct (0-1), n_rows, empty_count, + inferred_type, semantic_type, validity_rate (0-1, lo expone + profile_table al promocionar texto a numero/fecha), match_rate + (0-1), unique_pct (0-1), flags (list[str], reconoce + "constant"/"possible_id"/"high_cardinality"), numeric + ({outlier_pct: 0-100, skew, ...}|None). Returns: dict con: - score (float, 0-100, redondeado a 1 decimal), - completeness (float, 0-1), - validity (float, 0-1), - consistency (float, 0-1), - issues (list[str]) descripciones legibles de los problemas. + score (float 0-100, redondeado a 1 decimal), + completeness (float 0-1), + validity (float 0-1 | None si no aplicable), + dimensions ({completeness, validity}), + applicable (list[str] de dimensiones que entraron en el score), + issues (list[str]) SOLO defectos de calidad (nulos, vacios, + valores no conformes a su tipo/semantica), + observations (list[str]) señales analiticas que NO bajan el score + (outliers, columna constante, posible id, asimetria). """ if not isinstance(col, dict): col = {} @@ -39,103 +83,153 @@ def column_quality_score(col: dict) -> dict: flags = set(flags) issues: list[str] = [] + observations: list[str] = [] + + inferred_type = col.get("inferred_type") or "" + semantic_type = col.get("semantic_type") or "" # --- completeness ------------------------------------------------- - null_pct = col.get("null_pct") - if null_pct is None: - null_pct = 0.0 - try: - null_pct = float(null_pct) - except (TypeError, ValueError): - null_pct = 0.0 - null_pct = _clamp(null_pct, 0.0, 1.0) + # Falta de datos = nulos + (en texto) celdas vacias. Es el unico sitio + # donde la falta de datos cuenta: nunca se duplica en validity. + null_pct = _clamp(_num(col.get("null_pct"), 0.0), 0.0, 1.0) completeness = 1.0 - null_pct if null_pct > 0: - issues.append(f"{round(null_pct * 100)}% nulos") + issues.append(f"{_pct(null_pct)} nulos") - # --- validity ----------------------------------------------------- - validity = 1.0 - inferred_type = col.get("inferred_type") or "" + empty_frac = 0.0 + n_rows = col.get("n_rows") + empty_count = col.get("empty_count") + if ( + isinstance(n_rows, (int, float)) and not isinstance(n_rows, bool) and n_rows > 0 + and isinstance(empty_count, (int, float)) and not isinstance(empty_count, bool) + and empty_count > 0 + ): + empty_frac = _clamp(float(empty_count) / float(n_rows), 0.0, 1.0) + completeness = _clamp(completeness - empty_frac, 0.0, 1.0) + issues.append(f"{_pct(empty_frac)} vacíos") - numeric = col.get("numeric") - is_numeric = inferred_type in ("integer", "float", "numeric") or isinstance(numeric, dict) - if isinstance(numeric, dict): - outlier_pct = numeric.get("outlier_pct") - if outlier_pct is not None: - try: - outlier_pct = float(outlier_pct) - except (TypeError, ValueError): - outlier_pct = 0.0 - outlier_pct = _clamp(outlier_pct, 0.0, 1.0) - if outlier_pct > 0: - penalty = min(outlier_pct, 0.3) - validity -= penalty - issues.append(f"{round(outlier_pct * 100)}% outliers") - - # semantic_type declarado pero con baja tasa de match (si la conocemos). - semantic_type = col.get("semantic_type") or "" - match_rate = col.get("match_rate") - if semantic_type and match_rate is not None: - try: - match_rate = float(match_rate) - except (TypeError, ValueError): - match_rate = None - if match_rate is not None: - match_rate = _clamp(match_rate, 0.0, 1.0) - if match_rate < 1.0: - shortfall = 1.0 - match_rate - validity -= 0.5 * shortfall - issues.append( - f"semantic_type '{semantic_type}' con baja coincidencia " - f"({round(match_rate * 100)}%)" - ) - - if "mostly_null" in flags: - validity *= 0.5 - issues.append("mayoritariamente nula") - - validity = _clamp(validity, 0.0, 1.0) - - # --- consistency -------------------------------------------------- - consistency = 1.0 - if "constant" in flags: - consistency = 0.3 - issues.append("columna constante") + # --- validity (con renormalizacion por aplicabilidad) ------------- + # None = no medible -> se excluye del score (no penaliza ni premia). + validity = None + if completeness <= 0.0: + # Columna 100% faltante: no hay valores no nulos sobre los que medir + # conformidad. validity no aplica -> el score sale solo de completeness + # (= 0). Es el peor defecto de calidad posible. + validity = None else: - unique_pct = col.get("unique_pct") - if unique_pct is not None: - try: - unique_pct = float(unique_pct) - except (TypeError, ValueError): - unique_pct = None - if ( - inferred_type == "text" + validity_rate = col.get("validity_rate") + match_rate = col.get("match_rate") + if validity_rate is not None: + # Texto promovido a numero/fecha: parse rate real de la muestra. + v = _num(validity_rate, None) + if v is not None: + validity = _clamp(v, 0.0, 1.0) + if validity < 1.0: + kind = ( + "número" if inferred_type == "numeric" + else "fecha" if inferred_type == "datetime" + else inferred_type or "su tipo" + ) + issues.append( + f"{_pct(1.0 - validity)} no parsea al tipo {kind}" + ) + elif inferred_type in _NATIVE_TYPED: + # Tipo nativo garantizado por el almacen: no hay valores que no + # parseen. validity = 1.0 (no se confunde con tener outliers). + validity = 1.0 + elif semantic_type and match_rate is not None: + v = _num(match_rate, None) + if v is not None: + validity = _clamp(v, 0.0, 1.0) + if validity < 1.0: + issues.append( + f"{_pct(1.0 - validity)} no casa con el " + f"formato «{semantic_type}»" + ) + else: + # Texto libre / categorica sin semantica: no hay criterio honesto + # de validez. No aplica. + validity = None + + # --- observations (NO bajan el score) ----------------------------- + numeric = col.get("numeric") + if isinstance(numeric, dict): + # outlier_pct viene en escala 0-100 desde describe_numeric (z-score>3). + outlier_pct = _num(numeric.get("outlier_pct"), None) + if outlier_pct is not None and outlier_pct >= 0.05: + observations.append( + f"{_pct(outlier_pct / 100.0)} de valores atípicos (z-score>3): " + "revisar si son errores u observaciones legítimas" + ) + skew = _num(numeric.get("skew"), None) + if skew is not None and abs(skew) >= 1.0: + observations.append( + f"asimetría fuerte (skew={round(skew, 2)}): considerar " + "re-expresión antes de modelar" + ) + + if "constant" in flags: + observations.append( + "columna constante: aporta poca información para el análisis" + ) + + unique_pct = _num(col.get("unique_pct"), None) + is_id = ( + "possible_id" in flags + or "high_cardinality" in flags + or ( + inferred_type in ("text", "categorical") and unique_pct is not None and _clamp(unique_pct, 0.0, 1.0) > 0.9 - ): - consistency = 0.6 - issues.append("posible id de alta cardinalidad") - - consistency = _clamp(consistency, 0.0, 1.0) - - # --- score agregado ---------------------------------------------- - score = round( - 100.0 * (0.5 * completeness + 0.3 * validity + 0.2 * consistency), - 1, + ) ) + if is_id: + observations.append( + "valores casi únicos: posible identificador (no es un defecto de calidad)" + ) - # Silencia warnings sobre la variable de tipo no usada. - _ = is_numeric + # --- score agregado con renormalizacion --------------------------- + applicable = ["completeness"] + num = _W_COMPLETENESS * completeness + den = _W_COMPLETENESS + if validity is not None: + applicable.append("validity") + num += _W_VALIDITY * validity + den += _W_VALIDITY + score = round(100.0 * num / den, 1) if den > 0 else 0.0 return { "score": score, "completeness": completeness, "validity": validity, - "consistency": consistency, + "dimensions": {"completeness": completeness, "validity": validity}, + "applicable": applicable, "issues": issues, + "observations": observations, } +def _pct(frac: float) -> str: + """Formatea una fraccion 0-1 como porcentaje honesto: «N%» si >=1%, «0.N%» + por debajo (para no mostrar «0%» cuando hay un defecto real pequeño).""" + p = frac * 100.0 + if p >= 1.0: + return f"{round(p)}%" + return f"{p:.1f}%" + + +def _num(x, default): + """Convierte x a float; devuelve `default` si es None o no parseable.""" + if x is None: + return default + if isinstance(x, bool): + return default + try: + return float(x) + except (TypeError, ValueError): + return default + + def _clamp(x: float, lo: float, hi: float) -> float: """Recorta x al rango [lo, hi].""" if x < lo: diff --git a/python/functions/datascience/column_quality_score_test.py b/python/functions/datascience/column_quality_score_test.py index 6ed0d7e2..6f862ec6 100644 --- a/python/functions/datascience/column_quality_score_test.py +++ b/python/functions/datascience/column_quality_score_test.py @@ -1,4 +1,12 @@ -"""Tests para column_quality_score.""" +"""Tests para column_quality_score (nueva fórmula, report 2046). + +Verifica las invariantes de la fórmula de calidad: + - completeness (0.6) + validity (0.4) con renormalización por aplicabilidad. + - Los OUTLIERS no bajan el score (van a observations, no a issues). + - Columnas constantes e ids no bajan el score (observations). + - Sin doble conteo de la falta de datos. + - all-null -> score 0; función pura (no muta el input). +""" import os import sys @@ -9,11 +17,11 @@ from column_quality_score import column_quality_score def _clean_numeric_col() -> dict: - """ColumnProfile de una columna numerica sana, sin problemas.""" + """ColumnProfile de una columna numérica nativa sana, sin problemas.""" return { "name": "edad", "physical_type": "INTEGER", - "inferred_type": "integer", + "inferred_type": "numeric", "semantic_type": "", "count": 1000, "n_rows": 1000, @@ -28,85 +36,163 @@ def _clean_numeric_col() -> dict: } +# --------------------------------------------------------------------------- # +# Golden +# --------------------------------------------------------------------------- # def test_clean_column_high_score(): out = column_quality_score(_clean_numeric_col()) - assert out["score"] > 90 + assert out["score"] == 100.0 assert out["completeness"] == 1.0 assert out["validity"] == 1.0 - assert out["consistency"] == 1.0 + assert out["applicable"] == ["completeness", "validity"] assert out["issues"] == [] + assert out["observations"] == [] -def test_half_null_lowers_completeness_and_score(): +def test_weights_60_40_native_type(): + """30% nulos en numérica nativa: score = 100*(0.6*0.7 + 0.4*1.0) = 82.""" col = _clean_numeric_col() - col["null_count"] = 500 - col["null_pct"] = 0.5 - clean_score = column_quality_score(_clean_numeric_col())["score"] + col["null_pct"] = 0.30 + col["null_count"] = 300 out = column_quality_score(col) - assert out["completeness"] == 0.5 - assert out["score"] < clean_score - assert any("nulos" in issue for issue in out["issues"]) + assert out["completeness"] == 0.7 + assert out["validity"] == 1.0 + assert out["score"] == 82.0 + assert any("nulos" in i for i in out["issues"]) -def test_constant_column_flags_issue(): +# --------------------------------------------------------------------------- # +# Outliers FUERA del score +# --------------------------------------------------------------------------- # +def test_outliers_do_not_penalize_score(): + """Columna con outliers pero sin nulos -> score máximo; outliers en observations.""" + col = _clean_numeric_col() + col["numeric"] = {"outlier_pct": 18.0, "skew": 0.2} # 18% atípicos (escala 0-100) + out = column_quality_score(col) + assert out["score"] == 100.0 # los outliers NO bajan la calidad + assert out["validity"] == 1.0 + # No aparecen como problema de calidad... + assert not any("atípic" in i or "outlier" in i for i in out["issues"]) + # ...sino como observación analítica. + assert any("atípic" in o for o in out["observations"]) + + +def test_nulls_lower_score_more_than_outliers(): + """Vacíos sí penalizan; outliers no: comparar las dos columnas.""" + con_nulos = _clean_numeric_col() + con_nulos["null_pct"] = 0.30 + con_outliers = _clean_numeric_col() + con_outliers["numeric"] = {"outlier_pct": 30.0} + assert column_quality_score(con_nulos)["score"] < \ + column_quality_score(con_outliers)["score"] + + +# --------------------------------------------------------------------------- # +# Validity: aplicabilidad y renormalización +# --------------------------------------------------------------------------- # +def test_validity_from_parse_rate_lowers_score(): + """Numérica como texto con 20% basura: validity=0.8 -> score=92.""" + col = { + "name": "precio_txt", "inferred_type": "numeric", "semantic_type": "decimal", + "null_pct": 0.0, "validity_rate": 0.80, "flags": [], "numeric": None, + } + out = column_quality_score(col) + assert out["validity"] == 0.8 + assert out["score"] == 92.0 # 100*(0.6 + 0.4*0.8) + assert any("no parsea" in i for i in out["issues"]) + + +def test_validity_from_match_rate(): + """Texto con semantic_type y 5% no conforme: validity=0.95.""" + col = { + "name": "email", "inferred_type": "text", "semantic_type": "email", + "null_pct": 0.0, "match_rate": 0.95, "unique_pct": 0.5, "flags": [], + } + out = column_quality_score(col) + assert out["validity"] == 0.95 + assert out["score"] == 98.0 # 100*(0.6 + 0.4*0.95) + assert any("no casa" in i for i in out["issues"]) + + +def test_free_text_renormalizes_to_completeness_only(): + """Texto libre sin semántica: validity no aplica -> score = 100*completeness.""" + col = { + "name": "comentario", "inferred_type": "text", "semantic_type": "", + "null_pct": 0.30, "unique_pct": 0.5, "flags": [], "numeric": None, + } + out = column_quality_score(col) + assert out["validity"] is None + assert out["applicable"] == ["completeness"] + assert out["completeness"] == 0.7 + assert out["score"] == 70.0 # renormalizado a solo completeness + + +# --------------------------------------------------------------------------- # +# Casos límite (report §4.6) +# --------------------------------------------------------------------------- # +def test_all_null_column_scores_zero(): + col = _clean_numeric_col() + col["null_pct"] = 1.0 + col["null_count"] = 1000 + out = column_quality_score(col) + assert out["completeness"] == 0.0 + assert out["validity"] is None # no medible sin valores no nulos + assert out["score"] == 0.0 + + +def test_constant_column_scores_full_and_is_observation(): + """Columna constante: dato válido y completo -> score 100; baja info = observación.""" col = _clean_numeric_col() col["flags"] = ["constant"] col["distinct_count"] = 1 col["unique_pct"] = 0.001 out = column_quality_score(col) - assert out["consistency"] == 0.3 - assert any("constante" in issue for issue in out["issues"]) + assert out["score"] == 100.0 # NO se castiga la baja informatividad + assert not any("constante" in i for i in out["issues"]) + assert any("constante" in o for o in out["observations"]) +def test_high_cardinality_id_scores_full_and_is_observation(): + """Id de alta cardinalidad: unicidad perfecta -> score 100; posible id = observación.""" + col = { + "name": "uuid", "inferred_type": "text", "semantic_type": "", + "null_pct": 0.0, "unique_pct": 0.99, "flags": ["possible_id"], + "numeric": None, + } + out = column_quality_score(col) + assert out["score"] == 100.0 + assert not any("identificador" in i for i in out["issues"]) + assert any("identificador" in o for o in out["observations"]) + + +def test_mostly_null_no_double_counts_validity(): + """85% nulos: solo completeness penaliza; validity nativa sigue 1.0 (sin doble castigo).""" + col = _clean_numeric_col() + col["null_pct"] = 0.85 + col["flags"] = ["mostly_null"] + out = column_quality_score(col) + assert out["validity"] == 1.0 # ya no se multiplica por 0.5 + # score = 100*(0.6*0.15 + 0.4*1.0) = 49 + assert out["score"] == 49.0 + assert not any("mayoritariamente" in o for o in out["observations"]) + + +# --------------------------------------------------------------------------- # +# Robustez +# --------------------------------------------------------------------------- # def test_empty_dict_does_not_crash(): out = column_quality_score({}) assert isinstance(out["score"], float) assert out["completeness"] == 1.0 assert 0.0 <= out["score"] <= 100.0 assert isinstance(out["issues"], list) - - -def test_outliers_penalize_validity(): - col = _clean_numeric_col() - col["numeric"] = {"outlier_pct": 0.2} - out = column_quality_score(col) - assert out["validity"] < 1.0 - assert any("outliers" in issue for issue in out["issues"]) - - -def test_mostly_null_flag_halves_validity(): - col = _clean_numeric_col() - col["null_pct"] = 0.85 - col["flags"] = ["mostly_null"] - out = column_quality_score(col) - assert out["validity"] == 0.5 - assert any("mayoritariamente nula" in issue for issue in out["issues"]) - - -def test_high_cardinality_text_flagged_as_id(): - col = { - "name": "uuid", - "inferred_type": "text", - "semantic_type": "", - "null_pct": 0.0, - "unique_pct": 0.99, - "flags": [], - "numeric": None, - } - out = column_quality_score(col) - assert out["consistency"] < 1.0 - assert any("alta cardinalidad" in issue for issue in out["issues"]) + assert isinstance(out["observations"], list) def test_none_values_treated_defensively(): col = { - "name": "x", - "inferred_type": None, - "semantic_type": None, - "null_pct": None, - "unique_pct": None, - "flags": None, - "numeric": None, + "name": "x", "inferred_type": None, "semantic_type": None, + "null_pct": None, "unique_pct": None, "flags": None, "numeric": None, } out = column_quality_score(col) assert out["completeness"] == 1.0 diff --git a/python/functions/datascience/summarize_table_duckdb.md b/python/functions/datascience/summarize_table_duckdb.md index e9c9213f..006e61bc 100644 --- a/python/functions/datascience/summarize_table_duckdb.md +++ b/python/functions/datascience/summarize_table_duckdb.md @@ -3,7 +3,7 @@ name: summarize_table_duckdb kind: function lang: py domain: datascience -version: "1.0.0" +version: "1.1.0" purity: impure signature: "def summarize_table_duckdb(db_path: str, table: str, high_card_ratio: float = 0.9) -> dict" description: "Perfila una tabla DuckDB en una sola pasada SQL (SUMMARIZE, push-down sin traer filas a RAM) y devuelve el esqueleto de un TableProfile con el perfil base por columna. Corazon del grupo eda: base barata sobre la que otras funciones anaden lo estadistico fino (skew/kurtosis/histograma sobre muestra)." @@ -64,6 +64,7 @@ else: - **`distinct_count` exacto para tablas <=200k filas, aproximado+capado por encima**: `SUMMARIZE` usa HyperLogLog (`approx_unique`), que SOBREESTIMA y en tablas pequenas puede reportar mas distintos que filas (inflando `unique_pct` por encima de 1.0 y disparando flags `possible_id` falsos). Por eso, para `n_rows <= 200000` la funcion calcula `COUNT(DISTINCT)` EXACTO en una sola query combinada (barata) y usa ese valor. Para tablas mas grandes mantiene `approx_unique` pero lo CAPA a `n_rows` (`distinct_count = min(approx_unique, n_rows)`). En ambos casos `unique_pct = min(distinct_count / n_rows, 1.0)`, asi que `distinct_count` nunca supera las filas ni `unique_pct` pasa de 1.0. Los flags `possible_id` / `high_cardinality` derivan de ese `distinct_count` ya corregido (exacto y fiable por debajo de 200k filas; aproximado y conservador por encima). - **`SUMMARIZE` NO da skew, kurtosis ni histograma**, ni percentiles finos (p1/p5/p95/p99), moda, outliers, correlaciones, key_candidates ni quality_score. Esas claves quedan en `None`/`[]` a proposito: las rellena otra funcion del grupo `eda` sobre una muestra. El sub-dict `numeric` solo trae min, max, mean, std, p25, p50, p75. - **`SUMMARIZE.count` es el total de filas, no el no-nulo**: la funcion deriva el `count` no-nulo del ColumnProfile como `n_rows - null_count` (con `null_count` redondeado de `null_percentage`). +- **`duplicate_rows`/`duplicate_pct` se pueblan push-down** (desde v1.1.0) con `count(*)` sobre `SELECT DISTINCT *` (sin traer filas a RAM): `duplicate_rows = n_rows - filas_distintas`, `duplicate_pct` en fraccion 0-1. Habilitan la dimension de unicidad de registro del score de dataset (`profile_table` paso 6). Si la tabla tiene tipos no comparables con `DISTINCT` (BLOB/LIST/MAP) la query degrada y ambas vuelven a `None` (renormaliza el score a solo `cell_quality`). - **min/max/avg/std/q25/q50/q75 vienen como strings** desde DuckDB; se convierten a float (None si la columna no es numerica). - **Requiere DuckDB 1.5.2** (columnas de `SUMMARIZE` validadas con esa version: column_name, column_type, min, max, approx_unique, avg, std, q25, q50, q75, count, null_percentage). - **El identificador de tabla se interpola** (no parametrizable en `SUMMARIZE`): por eso se valida contra `^[A-Za-z_][A-Za-z0-9_]*$` antes de citarlo. Un nombre invalido (p.ej. con `;` o espacios) devuelve `{status:'error'}` sin tocar la base. diff --git a/python/functions/datascience/summarize_table_duckdb.py b/python/functions/datascience/summarize_table_duckdb.py index 641c230f..93e5ebb1 100644 --- a/python/functions/datascience/summarize_table_duckdb.py +++ b/python/functions/datascience/summarize_table_duckdb.py @@ -196,6 +196,21 @@ def summarize_table_duckdb( sum(c["null_pct"] for c in columns) / len(columns) if columns else 0.0 ) + # Unicidad de registro: filas duplicadas via COUNT de filas distintas + # push-down (DISTINCT *), sin traer filas a RAM. Habilita la dimension + # de uniqueness del score de dataset (1 - duplicate_pct). Degrada a None + # si la tabla tiene tipos no comparables con DISTINCT (BLOB/LIST/MAP). + duplicate_rows = None + duplicate_pct = None + if n_rows > 0: + dup_res = duckdb_query_readonly( + db_path, f"SELECT count(*) AS c FROM (SELECT DISTINCT * FROM {quoted})" + ) + if dup_res["status"] == "ok" and dup_res["rows"]: + distinct_rows = int(dup_res["rows"][0]["c"]) + duplicate_rows = max(0, n_rows - distinct_rows) + duplicate_pct = duplicate_rows / n_rows # fraccion 0-1 + profile = { "table": table, "source": "duckdb", @@ -203,8 +218,8 @@ def summarize_table_duckdb( "n_rows": n_rows, "n_cols": len(columns), "size_bytes": None, - "duplicate_rows": None, - "duplicate_pct": None, + "duplicate_rows": duplicate_rows, + "duplicate_pct": duplicate_pct, "constant_cols": constant_cols, "all_null_cols": all_null_cols, "null_cell_pct": null_cell_pct, diff --git a/python/functions/datascience/summarize_table_duckdb_test.py b/python/functions/datascience/summarize_table_duckdb_test.py index 57ff5485..ad8f9578 100644 --- a/python/functions/datascience/summarize_table_duckdb_test.py +++ b/python/functions/datascience/summarize_table_duckdb_test.py @@ -54,6 +54,30 @@ def test_shape_y_metadatos_tabla(db): assert profile["correlations"] is None +def test_duplicate_pct_sin_duplicados(db): + """Tabla con todas las filas distintas: duplicate_pct = 0, no None.""" + profile = summarize_table_duckdb(db, "ventas")["profile"] + assert profile["duplicate_rows"] == 0 + assert profile["duplicate_pct"] == 0.0 + + +def test_duplicate_pct_con_duplicados(tmp_path): + """Filas repetidas: duplicate_rows/duplicate_pct se pueblan push-down.""" + path = str(tmp_path / "dups.duckdb") + con = duckdb.connect(path) + con.execute("CREATE TABLE t (a INTEGER, b VARCHAR)") + # 5 filas, 2 de ellas idénticas a otras -> 2 duplicadas sobre 5 = 0.4. + con.execute( + "INSERT INTO t VALUES " + "(1,'x'), (2,'y'), (1,'x'), (3,'z'), (2,'y')" + ) + con.close() + profile = summarize_table_duckdb(path, "t")["profile"] + assert profile["n_rows"] == 5 + assert profile["duplicate_rows"] == 2 + assert profile["duplicate_pct"] == 0.4 + + def test_column_profile_shape(db): profile = summarize_table_duckdb(db, "ventas")["profile"] by_name = {c["name"]: c for c in profile["columns"]} diff --git a/python/functions/pipelines/profile_table.md b/python/functions/pipelines/profile_table.md index 2808965a..82a7a406 100644 --- a/python/functions/pipelines/profile_table.md +++ b/python/functions/pipelines/profile_table.md @@ -4,7 +4,7 @@ kind: pipeline lang: py domain: pipelines purity: impure -version: "1.0.0" +version: "1.1.0" signature: "def profile_table(db_path: str, table: str, backend: str = \"duckdb\", sample: int = 5000, run_models: bool = False, run_llm: bool = False, run_series: bool = False, emit_pdf: bool = False, emit_automatic: bool = False, report_dir: str = \"reports\", write_report: bool = True) -> dict" description: "Orquestador one-shot del grupo de capacidad eda: perfila UNA tabla (DuckDB o PostgreSQL) end-to-end componiendo las funciones del grupo (perfil base SQL + muestreo read-only + inferencia semantica + promocion de tipo + estadistica numerica/categorica + score de calidad + correlaciones con correccion FDR + re-expresion de Tukey + avisos exploratorios) y, opcional, modelos baratos (run_models), interpretacion LLM (run_llm) y analisis de serie temporal por columna (run_series: estacionariedad ADF+KPSS, ACF/PACF, STL, retornos). Emite el TableProfile completo mas (opcional) report markdown + JSON sidecar + PDF movil (emit_pdf). Es la composicion canonica para hazme un EDA de esta tabla." tags: [eda, duckdb, postgres, profiling, data-quality, pipeline, dataops, timeseries] @@ -114,3 +114,12 @@ para auditar la calidad de una tabla ya productiva. Reemplaza orquestar a mano Formatos exoticos pueden descartarse silenciosamente del calculo numerico. - `db_path` debe existir: DuckDB read-only NO crea la base. El muestreo usa el sandbox por defecto de `duckdb_query_readonly` (sin acceso a FS/red). +- **Score de calidad (report 2046, desde v1.1.0).** Paso 5: cada columna recibe + `quality_score` de `column_quality_score` con la formula 60/40 + (completeness/validity); al promocionar texto a numero/fecha se expone + `col["validity_rate"]` (parse rate de la muestra) para alimentar la dimension + validity. Paso 6: el score de dataset NO es la media simple — es + `100 * (0.85*cell_quality + 0.15*row_uniqueness)`, donde + `cell_quality = media(score_col/100)` y `row_uniqueness = 1 - duplicate_pct`. + Si `duplicate_pct` es `None` (backend sin calcularlo) el score se renormaliza + a solo `cell_quality`. Los outliers NO bajan el score (van a `observations`). diff --git a/python/functions/pipelines/profile_table.py b/python/functions/pipelines/profile_table.py index 8a0077af..17321805 100644 --- a/python/functions/pipelines/profile_table.py +++ b/python/functions/pipelines/profile_table.py @@ -477,9 +477,18 @@ def profile_table( if vals and (len(ok) / len(vals)) >= _PROMOTE_MIN_PARSE: col["inferred_type"] = "numeric" inferred = "numeric" + # Tasa de parseo real de la muestra: alimenta la + # dimension validity de column_quality_score (fraccion + # de valores conformes al tipo numerico promovido). + col["validity_rate"] = len(ok) / len(vals) elif semantic in _DATETIME_SEMANTIC: col["inferred_type"] = "datetime" inferred = "datetime" + # Tasa de parseo de la muestra a fecha (mismo papel que el + # parse rate numerico) para la dimension validity. + parsed_dt = [_to_ordinal_days(v) for v in vals] + ok_dt = [d for d in parsed_dt if d is not None] + col["validity_rate"] = (len(ok_dt) / len(vals)) if vals else None # 4) Enriquecer segun el inferred_type final. if inferred == "numeric": @@ -506,11 +515,36 @@ def profile_table( # 5) Score de calidad por columna. col["quality_score"] = column_quality_score(col).get("score") - # 6) Score agregado de la tabla (media de columnas). + # 6) Score agregado de la tabla (report 2046): NO media simple. + # cell_quality = media de los scores de columna, en [0,1]. + # row_uniqueness = 1 - duplicate_pct (unicidad de registro). + # score = 100 * (0.85*cell_quality + 0.15*row_uniqueness). + # Renormaliza a solo cell_quality si duplicate_pct no se pudo calcular. scores = [ c["quality_score"] for c in cols if c.get("quality_score") is not None ] - prof["quality_score"] = round(sum(scores) / len(scores), 1) if scores else None + if scores: + cell_quality = (sum(scores) / len(scores)) / 100.0 + dup_pct = prof.get("duplicate_pct") + if dup_pct is not None: + try: + d = float(dup_pct) + except (TypeError, ValueError): + d = None + else: + d = None + if d is not None: + # Tolerar escala 0-100 por si algun backend la entrega asi. + if d > 1.0: + d = d / 100.0 + row_uniqueness = max(0.0, min(1.0, 1.0 - d)) + prof["quality_score"] = round( + 100.0 * (0.85 * cell_quality + 0.15 * row_uniqueness), 1 + ) + else: + prof["quality_score"] = round(100.0 * cell_quality, 1) + else: + prof["quality_score"] = None # 7) Candidatos a clave. key_candidates = [] From 68f4ddabcea56b0e684a9682ccd002929c0046fd Mon Sep 17 00:00:00 2001 From: Egutierrez Date: Tue, 30 Jun 2026 18:15:15 +0200 Subject: [PATCH 29/53] =?UTF-8?q?feat(eda):=20cap=C3=ADtulo=20RELACIONES?= =?UTF-8?q?=20para=20AutomaticEDA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Añade el capítulo `relaciones` al motor AutomaticEDA: analiza las relaciones de clave de la tabla/base y se coloca tras `correlacion`, antes de `modelos`, en CHAPTER_ORDER. Capas que renderiza (solo las que aplican; None si no hay nada que decir): - Claves declaradas: PK/FK/UNIQUE reales del esquema DuckDB, vía la nueva función `detect_declared_keys_duckdb` (lee `duckdb_constraints()`). - Candidatos a clave primaria: los `key_candidates` del TableProfile. - FK candidatas inter-tabla: reusa `infer_fk_containment_duckdb` (containment + señal de nombre) y `build_join_graph` (roles de nodos + diagrama Mermaid pegable). Solo si la fuente DuckDB tiene varias tablas. - FK candidatas intra-tabla: heurística nombre + cardinalidad, vía la nueva función pura `suggest_intratable_fk_candidates`, marcada como sugerencia. Engancha al glosario clicable los términos PK, FK, containment/inclusión y cardinalidad (contrato §11.1) y usa Group (keep-together) para el grafo. Funciones nuevas del registry (grupo `eda`): - detect_declared_keys_duckdb (impure, datascience) + test. - suggest_intratable_fk_candidates (pure, datascience) + test. Tests: relaciones_test.py (golden intra + inter, edges, no-cut render) + los tests de ambas funciones. Suite automatic_eda + render_automatic_eda verde (89 passed). Golden end-to-end con el pipeline render_automatic_eda verificado sobre titanic (intra) y una BD customers/orders (inter). Co-Authored-By: Claude Opus 4.8 (1M context) --- python/functions/datascience/__init__.py | 4 + .../automatic_eda/chapters/relaciones.py | 500 ++++++++++++++++++ .../automatic_eda/chapters/relaciones_test.py | 273 ++++++++++ .../automatic_eda/chapters_registry.py | 1 + .../detect_declared_keys_duckdb.md | 107 ++++ .../detect_declared_keys_duckdb.py | 127 +++++ .../detect_declared_keys_duckdb_test.py | 167 ++++++ .../suggest_intratable_fk_candidates.md | 91 ++++ .../suggest_intratable_fk_candidates.py | 202 +++++++ .../suggest_intratable_fk_candidates_test.py | 157 ++++++ 10 files changed, 1629 insertions(+) create mode 100644 python/functions/datascience/automatic_eda/chapters/relaciones.py create mode 100644 python/functions/datascience/automatic_eda/chapters/relaciones_test.py create mode 100644 python/functions/datascience/detect_declared_keys_duckdb.md create mode 100644 python/functions/datascience/detect_declared_keys_duckdb.py create mode 100644 python/functions/datascience/detect_declared_keys_duckdb_test.py create mode 100644 python/functions/datascience/suggest_intratable_fk_candidates.md create mode 100644 python/functions/datascience/suggest_intratable_fk_candidates.py create mode 100644 python/functions/datascience/suggest_intratable_fk_candidates_test.py diff --git a/python/functions/datascience/__init__.py b/python/functions/datascience/__init__.py index 6302642f..f1505d22 100644 --- a/python/functions/datascience/__init__.py +++ b/python/functions/datascience/__init__.py @@ -34,6 +34,7 @@ from .theils_u import theils_u from .correlation_ratio import correlation_ratio from .mutual_info_columns import mutual_info_columns from .infer_fk_containment_duckdb import infer_fk_containment_duckdb +from .detect_declared_keys_duckdb import detect_declared_keys_duckdb from .build_join_graph import build_join_graph from .association_matrix import association_matrix from .correlation_matrix_duckdb import correlation_matrix_duckdb @@ -69,8 +70,10 @@ from .build_eda_render_ctx import build_eda_render_ctx from .profile_datetime import profile_datetime from .resample_timeseries import resample_timeseries from .add_pdf_internal_links import add_pdf_internal_links +from .suggest_intratable_fk_candidates import suggest_intratable_fk_candidates __all__ = [ + "suggest_intratable_fk_candidates", "detect_time_column", "extract_timeseries_raw", "build_eda_render_ctx", @@ -97,6 +100,7 @@ __all__ = [ "correlation_ratio", "mutual_info_columns", "infer_fk_containment_duckdb", + "detect_declared_keys_duckdb", "build_join_graph", "association_matrix", "correlation_matrix_duckdb", diff --git a/python/functions/datascience/automatic_eda/chapters/relaciones.py b/python/functions/datascience/automatic_eda/chapters/relaciones.py new file mode 100644 index 00000000..eba05f76 --- /dev/null +++ b/python/functions/datascience/automatic_eda/chapters/relaciones.py @@ -0,0 +1,500 @@ +"""Key-relations chapter (RELACIONES) — the keys / join structure of the data. + +This chapter is the *relational* section of an AutomaticEDA report. It answers a +single question for the table (or the whole DuckDB source it lives in): **how do +the keys relate?** It composes, without reimplementing them, the registry's +relation primitives and degrades honestly when a layer does not apply. + +It renders, in order, only the layers that have something to say: + +1. **Declared keys** (real schema constraints) — when the DuckDB source declares + PRIMARY KEY / FOREIGN KEY / UNIQUE constraints, they are read verbatim via + ``detect_declared_keys_duckdb`` and shown as ground truth: which column is the + PK, which columns are FKs and the table/column they point to. +2. **Primary-key candidates** — the ``key_candidates`` the TableProfile already + carries (columns whose cardinality equals the row count, with no nulls). These + are *candidates*: a column that could serve as the row identifier. +3. **Foreign-key candidates** when none are declared: + - **Inter-table** (the DuckDB source has several tables): real FK candidates by + name signal + value containment via ``infer_fk_containment_duckdb``, plus the + join graph (roles + a pasteable Mermaid diagram) via ``build_join_graph``. + - **Intra-table** (a single table): columns that *look* like a foreign key by a + name+cardinality heuristic (``suggest_intratable_fk_candidates``). This is a + **suggestion**, explicitly flagged as a heuristic, never an assertion. + +``build_relaciones(profile, ctx) -> Chapter | None``: returns ``None`` when there +is nothing to say (no declared key, no key candidates, and no FK candidate — +inter- or intra-table). Reads everything defensively (``.get``) and never raises: +anything missing degrades to a note or is omitted; a failing registry call drops +its layer instead of aborting the chapter. + +ctx keys this chapter consumes (all optional): + db_path, table : str — the DuckDB file and table being profiled (set by + ``build_eda_render_ctx``). ``db_path`` is needed to read declared + constraints, to list the sibling tables, and to run the containment-based + FK inference. Without it, only the profile-derived layers (PK candidates, + intra-table FK heuristic) are available. + glossary : model.GlossaryCollector — shared glossary; the chapter registers + the relational terms (PK, FK, containment, cardinality) and marks their + first appearance clickable. + +Contract: build_(profile, ctx) -> Chapter | None ; CHAPTER_VERSION = "x.y.z". +""" + +from __future__ import annotations + +from .. import model + +# Pure/impure registry functions (group ``eda``) this chapter composes. Imported +# defensively (module-leaf imports, like the AGREGACION chapter) so the chapter +# still builds — degrading the affected layer to nothing — if a function is +# somehow unavailable / not indexed yet. +try: + from datascience.detect_declared_keys_duckdb import detect_declared_keys_duckdb +except Exception: # noqa: BLE001 — keep the chapter importable no matter what. + detect_declared_keys_duckdb = None # type: ignore[assignment] +try: + from datascience.infer_fk_containment_duckdb import infer_fk_containment_duckdb +except Exception: # noqa: BLE001 + infer_fk_containment_duckdb = None # type: ignore[assignment] +try: + from datascience.build_join_graph import build_join_graph +except Exception: # noqa: BLE001 + build_join_graph = None # type: ignore[assignment] +try: + from datascience.suggest_intratable_fk_candidates import ( + suggest_intratable_fk_candidates, + ) +except Exception: # noqa: BLE001 + suggest_intratable_fk_candidates = None # type: ignore[assignment] +try: + from infra import duckdb_list_tables +except Exception: # noqa: BLE001 + duckdb_list_tables = None # type: ignore[assignment] + +CHAPTER_VERSION = "1.0.0" +CHAPTER_ID = "relaciones" +CHAPTER_TITLE = "Relaciones de clave" + +# Cap the inter-table FK table so a wide schema does not blow up the page; the +# rest is summarized in a closing note (no silent truncation). +MAX_FK_ROWS = 40 + +# --------------------------------------------------------------------------- # +# Glossary terms this chapter explains. Registered in the shared collector and +# marked clickable on their first appearance (contract §11.1). +# --------------------------------------------------------------------------- # +_TERMS = { + "pk": ( + "Clave primaria (PK)", + "Columna (o conjunto de columnas) que identifica de forma única cada fila " + "de una tabla: sus valores no se repiten y no son nulos. Una tabla tiene " + "como mucho una clave primaria; es el ancla por la que otras tablas la " + "referencian.", + ), + "fk": ( + "Clave foránea (FK)", + "Columna de una tabla cuyos valores apuntan a la clave primaria de otra " + "tabla (o de la misma), creando una relación entre ambas. Una FK suele ser " + "N:1: muchas filas de la tabla origen comparten el mismo valor de la tabla " + "destino.", + ), + "containment": ( + "Containment / inclusión", + "Señal con la que se infiere una clave foránea sin que la base la declare: " + "la fracción de valores distintos de una columna A que también aparecen " + "como valores de otra columna B. Si casi todos los valores de A están " + "contenidos en B (inclusión ≈ 1) y B parece una clave, A → B es una FK " + "candidata.", + ), + "cardinalidad": ( + "Cardinalidad", + "Número de valores distintos de una columna. Cardinalidad igual al número " + "de filas (y sin nulos) señala un identificador (candidato a clave " + "primaria); cardinalidad alta pero menor que el número de filas, con " + "valores repetidos, es típica de una clave foránea.", + ), +} + + +def _register_terms(ctx: dict) -> bool: + """Register the relational terms in the shared glossary. Returns whether the + in-text appearances should be marked clickable.""" + glossary = ctx.get("glossary") + if not isinstance(glossary, model.GlossaryCollector): + return False + for key, (label, definition) in _TERMS.items(): + glossary.add(key, label, definition) + return True + + +# --------------------------------------------------------------------------- # +# Formatting helpers (mirror the other chapters' defensive style). +# --------------------------------------------------------------------------- # +def _fmt_int(value) -> str: + if value is None: + return "—" + try: + return f"{int(value):,}".replace(",", ".") + except (TypeError, ValueError): + return model._safe_str(value) + + +def _fmt_pct_fraction(value, decimals: int = 1) -> str: + """Format a 0–1 fraction as a percentage. None -> placeholder.""" + if value is None: + return "—" + try: + v = float(value) + except (TypeError, ValueError): + return model._safe_str(value) + if v <= 1.0: + v *= 100.0 + return f"{v:.{decimals}f}%" + + +def _fmt_ratio(value, decimals: int = 3) -> str: + """Format an already-0–1 ratio (inclusion) as a plain number.""" + if value is None: + return "—" + try: + return f"{float(value):.{decimals}f}".rstrip("0").rstrip(".") + except (TypeError, ValueError): + return model._safe_str(value) + + +def _is_dict(v) -> bool: + return isinstance(v, dict) + + +def _columns_by_name(profile: dict) -> dict: + """Index the profile columns by name for quick metric lookup.""" + out = {} + for col in (profile.get("columns") or []): + if _is_dict(col) and col.get("name") is not None: + out[col.get("name")] = col + return out + + +# --------------------------------------------------------------------------- # +# Layer 1 — declared keys (real schema constraints). +# --------------------------------------------------------------------------- # +def _declared_keys(db_path: str, table: str): + """Read declared PK/FK/UNIQUE for the source, or None if unavailable.""" + if not db_path or detect_declared_keys_duckdb is None: + return None + try: + out = detect_declared_keys_duckdb(db_path, table) + except Exception: # noqa: BLE001 — dict-no-throw: treat as unavailable. + return None + if not _is_dict(out) or out.get("status") != "ok": + return None + return out + + +def _declared_section(declared: dict) -> list: + """Blocks for the declared-keys layer, or [] if there is nothing declared.""" + pks = [p for p in (declared.get("primary_keys") or []) if _is_dict(p)] + fks = [f for f in (declared.get("foreign_keys") or []) if _is_dict(f)] + uqs = [u for u in (declared.get("unique") or []) if _is_dict(u)] + if not (pks or fks or uqs): + return [] + + blocks = [ + model.Heading(text="Claves declaradas en el esquema", level=2), + model.Markdown(text=( + "La base **declara** estas relaciones de clave como restricciones " + "reales del esquema (constraints). Son la verdad de referencia: no se " + "infieren, se leen tal cual de la definición de las tablas.")), + ] + + if pks: + rows = [[model._safe_str(p.get("table")), + ", ".join(model._safe_str(c) for c in (p.get("columns") or []))] + for p in pks] + blocks.append(model.DataTable( + header=["Tabla", "Columna(s) PK"], rows=rows, + title="Claves primarias declaradas", + note="Cada fila: la clave primaria declarada de una tabla.")) + + if fks: + rows = [] + for f in fks: + src = ", ".join(model._safe_str(c) for c in (f.get("columns") or [])) + dst = ", ".join( + model._safe_str(c) for c in (f.get("referenced_columns") or [])) + rows.append([ + model._safe_str(f.get("table")), src, + model._safe_str(f.get("referenced_table")), dst]) + blocks.append(model.DataTable( + header=["Tabla origen", "Columna(s) FK", "→ Tabla destino", + "Columna(s) destino"], + rows=rows, title="Claves foráneas declaradas", + note="Cada fila: una FK declarada — origen → destino.")) + + if uqs: + rows = [[model._safe_str(u.get("table")), + ", ".join(model._safe_str(c) for c in (u.get("columns") or []))] + for u in uqs] + blocks.append(model.DataTable( + header=["Tabla", "Columna(s) UNIQUE"], rows=rows, + title="Restricciones UNIQUE declaradas")) + + return blocks + + +# --------------------------------------------------------------------------- # +# Layer 2 — primary-key candidates (from the profile). +# --------------------------------------------------------------------------- # +def _pk_candidates_section(profile: dict, mark: bool) -> list: + """Blocks for the PK-candidates layer, or [] if there are none.""" + keys = [k for k in (profile.get("key_candidates") or []) if k is not None] + if not keys: + return [] + by_name = _columns_by_name(profile) + + pk = ("[[term:pk]]**clave primaria**[[/term]]" if mark + else "**clave primaria**") + intro = ( + f"Estas columnas son **candidatas a {pk}**: su " + "[[term:cardinalidad]]cardinalidad[[/term]] iguala al número de filas y no " + "tienen nulos, así que cada valor identifica una fila distinta. Son " + "candidatas, no una clave declarada: la base no las marca como tal." + if mark else + "Estas columnas son **candidatas a clave primaria**: su cardinalidad " + "iguala al número de filas y no tienen nulos, así que cada valor " + "identifica una fila distinta.") + + rows = [] + for name in keys: + col = by_name.get(name) or {} + rows.append([ + model._safe_str(name), + _fmt_int(col.get("distinct_count")), + _fmt_pct_fraction(col.get("unique_pct")), + model._safe_str(col.get("inferred_type") or col.get("physical_type") or "—"), + ]) + return [ + model.Heading(text="Candidatos a clave primaria", level=2), + model.Markdown(text=intro), + model.DataTable( + header=["Columna", "Valores distintos", "% único", "Tipo"], + rows=rows, title="Candidatas a clave primaria", + note=f"{_fmt_int(profile.get('n_rows'))} filas en total como referencia."), + ] + + +# --------------------------------------------------------------------------- # +# Layer 3a — inter-table FK candidates (containment) + join graph. +# --------------------------------------------------------------------------- # +def _list_source_tables(db_path: str) -> list: + """List the tables in the DuckDB source, or [] if it can't be listed.""" + if not db_path or duckdb_list_tables is None: + return [] + try: + out = duckdb_list_tables(db_path) + except Exception: # noqa: BLE001 + return [] + if not _is_dict(out) or out.get("status") != "ok": + return [] + return [t for t in (out.get("tables") or []) if isinstance(t, str)] + + +def _inter_table_section(db_path: str, tables: list, mark: bool) -> list: + """Blocks for the inter-table FK layer (containment + join graph), or [].""" + if infer_fk_containment_duckdb is None or len(tables) < 2: + return [] + try: + fk = infer_fk_containment_duckdb(db_path, tables=tables) + except Exception: # noqa: BLE001 + return [] + if not _is_dict(fk) or fk.get("status") != "ok": + return [] + candidates = [c for c in (fk.get("fk_candidates") or []) if _is_dict(c)] + if not candidates: + return [] + + containment = ("[[term:containment]]containment (inclusión de valores)[[/term]]" + if mark else "containment (inclusión de valores)") + fk_term = "[[term:fk]]**claves foráneas**[[/term]]" if mark else "**claves foráneas**" + blocks = [ + model.Heading(text="Claves foráneas candidatas (inter-tabla)", level=2), + model.Markdown(text=( + f"La fuente tiene varias tablas. Estas {fk_term} candidatas se infieren " + f"por señal de nombre y por {containment}: una columna de una tabla cuyos " + "valores están contenidos en la clave de otra. No están declaradas por " + "la base; son la relación más probable según los datos.")), + ] + + shown = candidates[:MAX_FK_ROWS] + rows = [] + for c in shown: + rows.append([ + f"{model._safe_str(c.get('from_table'))}.{model._safe_str(c.get('from_col'))}", + f"{model._safe_str(c.get('to_table'))}.{model._safe_str(c.get('to_col'))}", + _fmt_ratio(c.get("inclusion")), + model._safe_str(c.get("cardinality") or "—"), + "sí" if c.get("name_match") else "no", + ]) + note = "Ordenadas por señal de nombre e inclusión." + if len(candidates) > len(shown): + note += f" Se muestran {len(shown)} de {len(candidates)} candidatas." + blocks.append(model.DataTable( + header=["Origen", "→ Destino", "Inclusión", "Cardinalidad", "Coincide nombre"], + rows=rows, title="FK candidatas por containment", note=note)) + + # Join graph: node roles + a pasteable Mermaid diagram, kept together. + if build_join_graph is not None: + try: + graph = build_join_graph(candidates, tables=tables) + except Exception: # noqa: BLE001 + graph = None + if _is_dict(graph): + graph_blocks = [model.Heading(text="Grafo de relaciones", level=3)] + nodes = [n for n in (graph.get("nodes") or []) if _is_dict(n)] + if nodes: + node_rows = [[ + model._safe_str(n.get("table")), + model._safe_str(n.get("role") or "—"), + _fmt_int(n.get("out_degree")), + _fmt_int(n.get("in_degree")), + ] for n in nodes] + graph_blocks.append(model.DataTable( + header=["Tabla", "Rol", "FK salientes", "FK entrantes"], + rows=node_rows, title="Tablas y su rol en el grafo", + note="Rol: fact (apunta a otras), dimension (referenciada), " + "bridge (ambas), standalone (aislada).")) + hubs = [h for h in (graph.get("hubs") or []) if h] + if hubs: + graph_blocks.append(model.Markdown(text=( + "Tablas con más relaciones salientes (candidatas a tabla de " + "hechos): " + ", ".join(model._safe_str(h) for h in hubs) + "."))) + mermaid = model._safe_str(graph.get("mermaid")).strip() + if mermaid: + graph_blocks.append(model.Markdown(text=( + "Diagrama de las relaciones (pegable en un bloque Mermaid):"))) + graph_blocks.append(model.Markdown( + text="```mermaid\n" + mermaid + "\n```")) + if len(graph_blocks) > 1: + blocks.append(model.Group(blocks=graph_blocks, + title="Grafo de relaciones")) + + skipped = [s for s in (fk.get("skipped") or []) if s] + if skipped: + blocks.append(model.Note( + "Algunos pares se omitieron por tamaño: " + + "; ".join(model._safe_str(s) for s in skipped) + ".")) + return blocks + + +# --------------------------------------------------------------------------- # +# Layer 3b — intra-table FK candidates (name+cardinality heuristic). +# --------------------------------------------------------------------------- # +def _intra_table_section(profile: dict, mark: bool) -> list: + """Blocks for the intra-table FK heuristic layer, or [] if no candidates.""" + if suggest_intratable_fk_candidates is None: + return [] + try: + cands = suggest_intratable_fk_candidates(profile) + except Exception: # noqa: BLE001 + return [] + cands = [c for c in (cands or []) if _is_dict(c)] + if not cands: + return [] + + fk_term = "[[term:fk]]**claves foráneas**[[/term]]" if mark else "**claves foráneas**" + blocks = [ + model.Heading(text="Posibles claves foráneas (heurística de nombre)", level=2), + model.Markdown(text=( + f"No hay otras tablas que referenciar, pero algunas columnas **parecen** " + f"{fk_term} por su nombre (terminan en «id») y su cardinalidad (muchos " + "valores repetidos, N:1). Es una **sugerencia heurística**, no una " + "afirmación: el nombre de la tabla destino es una conjetura y no se " + "comprueba inclusión de valores contra ninguna tabla real.")), + ] + rows = [] + for c in cands: + rows.append([ + model._safe_str(c.get("column")), + model._safe_str(c.get("ref_table_guess") or "—"), + _fmt_int(c.get("distinct_count")), + _fmt_pct_fraction(c.get("unique_pct")), + model._safe_str(c.get("inferred_type") or c.get("physical_type") or "—"), + model._safe_str(c.get("reason") or ""), + ]) + blocks.append(model.DataTable( + header=["Columna", "Posible tabla", "Valores distintos", "% único", + "Tipo", "Motivo"], + rows=rows, title="Posibles FK por nombre y cardinalidad", + note="Heurística: posibles falsos positivos/negativos. No confirma containment.")) + blocks.append(model.Note( + "Estas sugerencias se basan solo en el nombre y la cardinalidad. Para " + "confirmarlas haría falta la tabla destino y comprobar la inclusión de " + "valores (containment).")) + return blocks + + +# --------------------------------------------------------------------------- # +# Entry point. +# --------------------------------------------------------------------------- # +def _intro_blocks(mark: bool) -> list: + pk = "[[term:pk]]clave primaria[[/term]]" if mark else "clave primaria" + fk = "[[term:fk]]clave foránea[[/term]]" if mark else "clave foránea" + text = ( + f"Este capítulo analiza las **relaciones de clave** de la tabla: qué columna " + f"identifica cada fila (la {pk}) y qué columnas referencian a otra tabla (las " + f"{fk}). Cuando la base las **declara** como restricciones del esquema, se " + "muestran tal cual; cuando no, se proponen las más probables a partir de los " + "datos —por inclusión de valores entre tablas (containment) o, en una sola " + "tabla, por una heurística de nombre y cardinalidad— siempre marcadas como " + "candidatas, nunca como hechos.") + return [model.Heading(text=CHAPTER_TITLE, level=1), model.Markdown(text=text)] + + +def build_relaciones(profile: dict, ctx: dict): + """Build the RELACIONES Chapter, or None if there is nothing to say. + + Args: + profile: the ``eda`` group TableProfile dict (may be None/empty). + ctx: presentation context. Consumes ``db_path`` + ``table`` (to read + declared constraints, list sibling tables and run the containment FK + inference) and ``glossary`` (to register the relational terms). + + Returns: + A ``model.Chapter`` with the applicable relation layers; or ``None`` when + the dataset has no declared key, no key candidates and no FK candidate + (neither inter- nor intra-table). + """ + if not isinstance(profile, dict): + profile = {} + ctx = ctx if isinstance(ctx, dict) else {} + db_path = ctx.get("db_path") + table = ctx.get("table") + + mark = _register_terms(ctx) + + # Build each layer; the chapter is the concatenation of the non-empty ones. + declared = _declared_keys(db_path, table) + declared_blocks = _declared_section(declared) if declared else [] + declared_has_fk = bool(declared and declared.get("foreign_keys")) + + pk_blocks = _pk_candidates_section(profile, mark) + + tables = _list_source_tables(db_path) + inter_blocks = _inter_table_section(db_path, tables, mark) + + # The intra-table heuristic only makes sense when no real FK is available for + # this table — neither declared nor inferred inter-table. Otherwise the real + # relations already answer the question and the heuristic is just noise. + if declared_has_fk or inter_blocks: + intra_blocks = [] + else: + intra_blocks = _intra_table_section(profile, mark) + + body = declared_blocks + pk_blocks + inter_blocks + intra_blocks + if not body: + return None # chapter does not apply: nothing to say about relations. + + blocks = _intro_blocks(mark) + body + return model.Chapter(id=CHAPTER_ID, title=CHAPTER_TITLE, + version=CHAPTER_VERSION, blocks=blocks) diff --git a/python/functions/datascience/automatic_eda/chapters/relaciones_test.py b/python/functions/datascience/automatic_eda/chapters/relaciones_test.py new file mode 100644 index 00000000..5ccf620c --- /dev/null +++ b/python/functions/datascience/automatic_eda/chapters/relaciones_test.py @@ -0,0 +1,273 @@ +"""Tests for the RELACIONES chapter — DoD: golden(s) + edges + no-cut render. + +Two goldens covering the two real paths of the chapter: + +- **Intra-table** (a single table, no db source for relations): the chapter shows + the primary-key candidates from the profile and the heuristic foreign-key + suggestions (name + cardinality), explicitly flagged as a heuristic. Renders to + PDF and PPTX with nothing cut. +- **Inter-table** (a real DuckDB file with two related tables, customers/orders, + with a declared FK): the chapter shows the declared keys, the containment-based + FK candidates and the join graph (roles + a pasteable Mermaid diagram). + +Edges: a profile with no key candidate and no FK-looking column returns None; +``None`` / ``{}`` profiles do not raise. The chapter registers its glossary terms. + +Layers that depend on the sibling registry functions delegated alongside this +chapter (``detect_declared_keys_duckdb``, ``suggest_intratable_fk_candidates``) +are asserted **conditionally on the function being importable**, so the chapter's +honest-degradation contract is what is tested, never a hard dependency on import +timing. +""" + +import os +import tempfile + +import duckdb +from pptx import Presentation +from pypdf import PdfReader + +from datascience.automatic_eda.chapters.relaciones import build_relaciones +from datascience.automatic_eda.model import Chapter, Group, GlossaryCollector +from datascience.render_automatic_eda_pdf import render_automatic_eda_pdf +from datascience.render_automatic_eda_pptx import render_automatic_eda_pptx + +# The optional sibling functions: their layers are asserted only when present. +try: + from datascience.detect_declared_keys_duckdb import detect_declared_keys_duckdb +except Exception: # noqa: BLE001 + detect_declared_keys_duckdb = None +try: + from datascience.suggest_intratable_fk_candidates import ( + suggest_intratable_fk_candidates, + ) +except Exception: # noqa: BLE001 + suggest_intratable_fk_candidates = None + + +# --------------------------------------------------------------------------- # +# Helpers. +# --------------------------------------------------------------------------- # +def _flatten(blocks) -> list: + """Flatten Group blocks so a test can inspect every leaf block.""" + out = [] + for b in blocks: + if isinstance(b, Group): + out.extend(_flatten(b.blocks)) + else: + out.append(b) + return out + + +def _text_of(chapter: Chapter) -> str: + """Collect all visible text of a chapter's blocks into one string.""" + parts = [] + for b in _flatten(chapter.blocks): + for attr in ("text", "title", "note"): + v = getattr(b, attr, None) + if isinstance(v, str): + parts.append(v) + header = getattr(b, "header", None) + if isinstance(header, list): + parts.extend(str(c) for c in header) + rows = getattr(b, "rows", None) + if isinstance(rows, list): + for r in rows: + if isinstance(r, (list, tuple)): + parts.extend(str(c) for c in r) + else: + parts.append(str(r)) + return "\n".join(parts) + + +def _render_both(chapter: Chapter, tag: str): + """Render the chapter to PDF and PPTX; return (pdf_text, n_slides).""" + tmp = tempfile.mkdtemp(prefix=f"relaciones_{tag}_") + pdf_path = os.path.join(tmp, "out.pdf") + pptx_path = os.path.join(tmp, "out.pptx") + meta = {"title": f"EDA — {tag}"} + render_automatic_eda_pdf([chapter], pdf_path, meta) + render_automatic_eda_pptx([chapter], pptx_path, meta) + assert os.path.exists(pdf_path) and os.path.getsize(pdf_path) > 0 + assert os.path.exists(pptx_path) and os.path.getsize(pptx_path) > 0 + text = "".join(p.extract_text() or "" for p in PdfReader(pdf_path).pages) + n_slides = len(Presentation(pptx_path).slides) + return text, n_slides + + +# --------------------------------------------------------------------------- # +# Fixtures. +# --------------------------------------------------------------------------- # +def _titanic_profile() -> dict: + """A single-table profile: a PK candidate + a column that looks like a FK.""" + return { + "table": "titanic", + "source": "/data/titanic.csv", + "n_rows": 891, + "n_cols": 4, + "key_candidates": ["PassengerId"], + "columns": [ + {"name": "PassengerId", "inferred_type": "numeric", + "physical_type": "BIGINT", "distinct_count": 891, + "unique_pct": 1.0, "flags": ["possible_id"]}, + {"name": "ticket_id", "inferred_type": "numeric", + "physical_type": "BIGINT", "distinct_count": 681, + "unique_pct": 0.76, "flags": []}, + {"name": "fare", "inferred_type": "numeric", + "physical_type": "DOUBLE", "distinct_count": 248, + "unique_pct": 0.28, "flags": []}, + {"name": "sex", "inferred_type": "categorical", + "physical_type": "VARCHAR", "distinct_count": 2, + "unique_pct": 0.002, "flags": []}, + ], + } + + +def _make_relational_db(path: str) -> None: + """Create a small DuckDB with customers(id) <- orders(customer_id), real FK.""" + con = duckdb.connect(path) + con.execute("CREATE TABLE customers(id INTEGER PRIMARY KEY, name TEXT)") + con.execute( + "CREATE TABLE orders(id INTEGER PRIMARY KEY, " + "customer_id INTEGER REFERENCES customers(id), amount DOUBLE)") + con.execute("INSERT INTO customers VALUES " + "(1,'a'),(2,'b'),(3,'c'),(4,'d'),(5,'e')") + con.execute("INSERT INTO orders VALUES " + "(1,1,10.0),(2,1,20.0),(3,2,30.0),(4,3,40.0)," + "(5,3,50.0),(6,4,60.0),(7,5,70.0),(8,2,80.0)") + con.close() + + +def _orders_profile() -> dict: + """A profile for the `orders` table of the relational DB.""" + return { + "table": "orders", + "source": "orders", + "n_rows": 8, + "n_cols": 3, + "key_candidates": ["id"], + "columns": [ + {"name": "id", "inferred_type": "numeric", "physical_type": "INTEGER", + "distinct_count": 8, "unique_pct": 1.0, "flags": ["possible_id"]}, + {"name": "customer_id", "inferred_type": "numeric", + "physical_type": "INTEGER", "distinct_count": 5, "unique_pct": 0.625, + "flags": []}, + {"name": "amount", "inferred_type": "numeric", "physical_type": "DOUBLE", + "distinct_count": 8, "unique_pct": 1.0, "flags": []}, + ], + } + + +# --------------------------------------------------------------------------- # +# Golden 1 — intra-table. +# --------------------------------------------------------------------------- # +def test_golden_intra_table_pk_and_fk_heuristic(): + """Single table: PK candidate shown; FK heuristic shown (if fn available); + renders to PDF + PPTX with nothing cut.""" + prof = _titanic_profile() + glossary = GlossaryCollector() + # No db_path: only the profile-derived layers apply (no declared, no inter). + chapter = build_relaciones(prof, {"glossary": glossary}) + + assert isinstance(chapter, Chapter) + assert chapter.id == "relaciones" + text = _text_of(chapter) + + # PK candidate is always present (comes from the profile). + assert "Candidatos a clave primaria" in text + assert "PassengerId" in text + + # Glossary terms got registered. + for key in ("pk", "fk", "cardinalidad"): + assert glossary.has(key) + + # FK heuristic layer: present iff the delegated function is importable. + if suggest_intratable_fk_candidates is not None: + assert "Posibles claves foráneas" in text + assert "ticket_id" in text + # The float measure and the PK itself are NOT suggested as FKs. + assert "Posibles FK por nombre" in text + + pdf_text, n_slides = _render_both(chapter, "intra") + assert "PassengerId" in pdf_text + assert n_slides >= 1 + + +# --------------------------------------------------------------------------- # +# Golden 2 — inter-table (real DuckDB). +# --------------------------------------------------------------------------- # +def test_golden_inter_table_containment_and_join_graph(): + """Two related tables: declared FK (if fn available) + containment FK + candidate + Mermaid join graph.""" + tmp = tempfile.mkdtemp(prefix="relaciones_db_") + db_path = os.path.join(tmp, "shop.duckdb") + _make_relational_db(db_path) + + prof = _orders_profile() + glossary = GlossaryCollector() + chapter = build_relaciones( + prof, {"db_path": db_path, "table": "orders", "glossary": glossary}) + + assert isinstance(chapter, Chapter) + text = _text_of(chapter) + + # Inter-table containment FK candidate: customer_id -> customers.id. This path + # uses infer_fk_containment_duckdb + build_join_graph, both already in the + # registry, so it must be present. + assert "Claves foráneas candidatas (inter-tabla)" in text + assert "orders.customer_id" in text + assert "customers.id" in text + # Join graph with a pasteable Mermaid diagram. + assert "Grafo de relaciones" in text + assert "mermaid" in text + assert "graph LR" in text + assert "containment" in text.lower() + + # Declared-keys layer: present iff the delegated function is importable. + if detect_declared_keys_duckdb is not None: + assert "Claves declaradas en el esquema" in text + assert "Claves foráneas declaradas" in text + + pdf_text, n_slides = _render_both(chapter, "inter") + assert "customer_id" in pdf_text + assert n_slides >= 1 + + +# --------------------------------------------------------------------------- # +# Edges. +# --------------------------------------------------------------------------- # +def test_none_when_no_relations(): + """No key candidates, no FK-looking columns, no db source -> None.""" + prof = { + "table": "flat", "n_rows": 100, "n_cols": 2, "key_candidates": [], + "columns": [ + {"name": "value", "inferred_type": "numeric", "physical_type": "DOUBLE", + "distinct_count": 50, "unique_pct": 0.5, "flags": []}, + {"name": "label", "inferred_type": "categorical", + "physical_type": "VARCHAR", "distinct_count": 3, "unique_pct": 0.03, + "flags": []}, + ], + } + assert build_relaciones(prof, {}) is None + + +def test_empty_and_none_profile_do_not_raise(): + """None / {} profile and missing ctx degrade to None without raising.""" + assert build_relaciones(None, None) is None + assert build_relaciones({}, {}) is None + assert build_relaciones({}, {"glossary": GlossaryCollector()}) is None + + +def test_pk_candidate_only_builds_chapter(): + """A profile with only a key candidate (no FK anything, no db) still builds: + the relations chapter applies because there is a PK candidate to report.""" + prof = { + "table": "t", "n_rows": 10, "n_cols": 1, "key_candidates": ["row_id"], + "columns": [ + {"name": "row_id", "inferred_type": "numeric", "physical_type": "BIGINT", + "distinct_count": 10, "unique_pct": 1.0, "flags": ["possible_id"]}, + ], + } + chapter = build_relaciones(prof, {}) + assert isinstance(chapter, Chapter) + assert "Candidatos a clave primaria" in _text_of(chapter) diff --git a/python/functions/datascience/automatic_eda/chapters_registry.py b/python/functions/datascience/automatic_eda/chapters_registry.py index d4dc329d..d9030999 100644 --- a/python/functions/datascience/automatic_eda/chapters_registry.py +++ b/python/functions/datascience/automatic_eda/chapters_registry.py @@ -33,6 +33,7 @@ CHAPTER_ORDER = [ "cat_distr", # categorical distributions "calidad", # data quality "correlacion", # correlations / associations + "relaciones", # key relations: declared/candidate PK + FK (inter/intra-table) "modelos", # cheap models (PCA/KMeans/outliers) "timeseries", # time-series analysis "geospatial", # geospatial diff --git a/python/functions/datascience/detect_declared_keys_duckdb.md b/python/functions/datascience/detect_declared_keys_duckdb.md new file mode 100644 index 00000000..32b9351d --- /dev/null +++ b/python/functions/datascience/detect_declared_keys_duckdb.md @@ -0,0 +1,107 @@ +--- +name: detect_declared_keys_duckdb +kind: function +lang: py +domain: datascience +version: "1.0.0" +purity: impure +signature: "def detect_declared_keys_duckdb(db_path: str, table: str = None) -> dict" +description: "Detecta las claves DECLARADAS (constraints reales) de un schema DuckDB leyendo la table function duckdb_constraints(): extrae PRIMARY KEY, FOREIGN KEY y UNIQUE (ignora NOT NULL y CHECK) y las devuelve normalizadas con sus columnas, y para las FK con su tabla y columnas referenciadas. Con table=None procesa todas las tablas; con table='X' filtra a PK/UNIQUE de X y a FK cuyo origen es X (case-sensitive). A diferencia de infer_fk_containment_duckdb (que INFIERE FKs candidatas por containment de valores cuando el schema no las declara), esta funcion devuelve las relaciones de clave REALES del schema. Estilo dict-no-throw: nunca lanza. Parte del grupo eda (relaciones de clave)." +tags: [eda, duckdb, datascience, relations, primary-key, foreign-key, schema, exploratory-data-analysis] +params: + - name: db_path + desc: "Ruta al archivo DuckDB. Debe existir (lectura read-only via duckdb_query_readonly; no se crea). Un path inexistente devuelve {status:'error', ...}." + - name: table + desc: "Si se pasa, filtra los resultados a esa tabla: incluye PRIMARY KEY y UNIQUE cuya tabla sea `table`, y FOREIGN KEY cuya tabla ORIGEN sea `table` (no la referenciada). None (default) devuelve los constraints de todas las tablas. La comparacion es case-sensitive (nombres tal cual los devuelve DuckDB)." +output: "dict dict-no-throw. En exito {status:'ok', primary_keys:[{table:str, columns:[str,...]}, ...], foreign_keys:[{table:str, columns:[str,...], referenced_table:str, referenced_columns:[str,...]}, ...], unique:[{table:str, columns:[str,...]}, ...], tables:[str,...]} donde tables es la lista ordenada de tablas (origen) que poseen al menos un constraint PK/FK/UNIQUE emitido. Solo se emiten constraints de clave: NOT NULL y CHECK se ignoran. En error {status:'error', error:str}." +uses_functions: [duckdb_query_readonly_py_infra] +uses_types: [] +returns: [] +returns_optional: false +error_type: "error_go_core" +imports: [] +tested: true +tests: ["test_golden_detecta_pks_y_fk", "test_golden_ignora_not_null_y_check", "test_edge_filtra_por_tabla_orders", "test_edge_filtra_por_tabla_customers", "test_edge_unique_declarado", "test_edge_sin_constraints_listas_vacias", "test_error_db_inexistente_no_lanza", "test_shape_resultado"] +test_file_path: "python/functions/datascience/detect_declared_keys_duckdb_test.py" +file_path: "python/functions/datascience/detect_declared_keys_duckdb.py" +--- + +## Ejemplo + +```python +import sys, os, duckdb +sys.path.insert(0, os.path.join("python", "functions")) +from datascience import detect_declared_keys_duckdb + +# Base de ejemplo en /tmp: orders.customer_id -> customers.id (FK declarada) +path = "/tmp/declared_keys_demo.duckdb" +if os.path.exists(path): + os.remove(path) +con = duckdb.connect(path) +con.execute("CREATE TABLE customers(id INTEGER PRIMARY KEY, name TEXT)") +con.execute( + "CREATE TABLE orders(" + " id INTEGER PRIMARY KEY," + " customer_id INTEGER REFERENCES customers(id)," + " amt DOUBLE)" +) +con.close() + +res = detect_declared_keys_duckdb(path) +if res["status"] == "ok": + for pk in res["primary_keys"]: + print(f"PK {pk['table']}({', '.join(pk['columns'])})") + for fk in res["foreign_keys"]: + print(f"FK {fk['table']}({', '.join(fk['columns'])}) -> " + f"{fk['referenced_table']}({', '.join(fk['referenced_columns'])})") + # PK customers(id) + # PK orders(id) + # FK orders(customer_id) -> customers(id) +else: + print("error:", res["error"]) + +# Filtrar a una tabla concreta (PK/UNIQUE de orders + FK con origen orders): +solo_orders = detect_declared_keys_duckdb(path, table="orders") +print(solo_orders["tables"]) # ['orders'] +``` + +## Cuando usarla + +- Cuando exploras un esquema DuckDB y quieres mostrar las relaciones de clave REALES (PK/FK/UNIQUE) que el schema ha declarado, sin inferir nada. +- Como paso del capitulo RELACIONES del grupo `eda`: primero mira las claves declaradas con esta funcion; si el schema no declara FKs, complementa con `infer_fk_containment_duckdb` (inferencia por containment). +- Antes de documentar o migrar un esquema, para listar el contrato de integridad referencial que el motor ya conoce. +- Para validar que las constraints que esperas (esa FK que creaste con `REFERENCES`) realmente estan declaradas en la base materializada. + +## Gotchas + +- **Impura**: lee de disco via la primitiva read-only `duckdb_query_readonly` (no crea ni modifica la base). El `db_path` debe existir; un path inexistente devuelve `{status:'error'}` (read_only NO crea la base). +- **Requiere `duckdb_constraints()`**: usa la table function `duckdb_constraints()`, disponible en DuckDB modernos (verificado en 1.5.2). En versiones antiguas sin esa funcion, la query falla y se devuelve `{status:'error'}`. +- **Solo claves DECLARADAS**: devuelve lo que el schema declaro con `PRIMARY KEY` / `FOREIGN KEY (... REFERENCES ...)` / `UNIQUE`. Una tabla materializada con `CREATE TABLE AS SELECT` NO lleva constraints — para esos casos no habra claves que mostrar y hay que INFERIRLAS (`infer_fk_containment_duckdb`). +- **NOT NULL y CHECK se ignoran**: `duckdb_constraints()` tambien emite filas `NOT NULL` (DuckDB genera una por cada columna PK) y `CHECK`; esta funcion las descarta y solo conserva PK/FK/UNIQUE. +- **Nombres case-sensitive**: el filtro `table='Orders'` no casa con una tabla `orders`. Se comparan los nombres tal cual los devuelve DuckDB. +- **FK atribuida al origen**: una FOREIGN KEY se atribuye a su tabla ORIGEN (el `table` de la entrada), no a la referenciada. El filtro `table='X'` trae las FK cuyo origen es X, no las que apuntan a X. +- **`tables` = tablas dueñas de constraints emitidos**: la lista `tables` contiene solo las tablas que poseen al menos un PK/FK/UNIQUE en el resultado (su campo `table`), ordenadas. No incluye tablas referenciadas que no tengan constraint propio en la salida. +- **Columnas como listas**: `constraint_column_names` y `referenced_column_names` son columnas LIST de DuckDB; en 1.5.2 llegan como listas Python. La funcion las normaliza a listas de strings con una red de seguridad por si llegaran como string. + +## Notas + +`duckdb_constraints()` devuelve una fila por constraint con los campos +`table_name`, `constraint_type`, `constraint_column_names`, `referenced_table`, +`referenced_column_names`. Mapeo a la salida: + +```text +PRIMARY KEY -> primary_keys[]: {table, columns} +UNIQUE -> unique[]: {table, columns} +FOREIGN KEY -> foreign_keys[]: {table, columns, referenced_table, referenced_columns} +NOT NULL -> ignorado +CHECK -> ignorado +``` + +Para una FK, `referenced_table` y `referenced_column_names` vienen poblados; para +PK/UNIQUE, `referenced_table` es NULL y `referenced_column_names` una lista vacia. + +Complementa a `infer_fk_containment_duckdb`: esta funcion devuelve las relaciones +de clave REALES del schema (declaradas); la otra INFIERE FKs candidatas por +containment de valores cuando el schema no las declaro. En el capitulo RELACIONES +de AutomaticEDA se usan en orden: primero las declaradas, luego la inferencia como +respaldo. diff --git a/python/functions/datascience/detect_declared_keys_duckdb.py b/python/functions/datascience/detect_declared_keys_duckdb.py new file mode 100644 index 00000000..b17151b9 --- /dev/null +++ b/python/functions/datascience/detect_declared_keys_duckdb.py @@ -0,0 +1,127 @@ +"""detect_declared_keys_duckdb — lee las claves DECLARADAS de un schema DuckDB. + +Funcion impura: lee de disco a traves de la primitiva read-only del grupo +`duckdb` (duckdb_query_readonly). Pertenece al grupo de capacidad `eda` +(relaciones de clave): a diferencia de infer_fk_containment_duckdb, que INFIERE +FOREIGN KEYs candidatas por containment de valores, esta funcion devuelve las +constraints REALES que el schema ha declarado (PRIMARY KEY / FOREIGN KEY / +UNIQUE) leyendo la table function `duckdb_constraints()`. + +Es la pieza del capitulo RELACIONES de AutomaticEDA que muestra las relaciones de +clave reales cuando existen — frente a la inferencia, que se usa cuando el schema +no las declaro. + +Estilo dict-no-throw del grupo duckdb: nunca lanza; captura cualquier error y +devuelve {status:'error', error:str}. +""" + +from infra import duckdb_query_readonly + + +def _as_list(value) -> list: + """Normaliza el valor de una columna LIST de DuckDB a una lista de strings. + + En DuckDB 1.5.2, `constraint_column_names` y `referenced_column_names` llegan + ya como listas Python a traves de duckdb_query_readonly. Este helper es solo + una red de seguridad: si por cualquier motivo llegara como string (p.ej. la + representacion `[id, customer_id]`), la parsea de forma defensiva. + """ + if value is None: + return [] + if isinstance(value, (list, tuple)): + return [str(v) for v in value] + if isinstance(value, str): + s = value.strip() + if s.startswith("[") and s.endswith("]"): + s = s[1:-1] + if not s.strip(): + return [] + return [ + part.strip().strip("'\"") + for part in s.split(",") + if part.strip().strip("'\"") + ] + return [str(value)] + + +def detect_declared_keys_duckdb(db_path: str, table: str = None) -> dict: + """Detecta las claves PRIMARY KEY / FOREIGN KEY / UNIQUE declaradas en DuckDB. + + Lee la table function `duckdb_constraints()` y extrae solo las constraints de + clave (PRIMARY KEY, FOREIGN KEY, UNIQUE), ignorando NOT NULL y CHECK. + + Args: + db_path: ruta al archivo DuckDB. Debe existir (lectura read-only; no se + crea). Un path inexistente devuelve {status:'error', ...} sin lanzar. + table: si se pasa, filtra los resultados a esa tabla: incluye PRIMARY KEY + y UNIQUE cuya tabla sea `table`, y FOREIGN KEY cuya tabla ORIGEN sea + `table`. None (default) devuelve los constraints de todas las tablas. + La comparacion de nombres es case-sensitive (tal cual los devuelve + DuckDB). + + Returns: + dict dict-no-throw. En exito: + {status:'ok', + primary_keys:[{table:str, columns:[str, ...]}, ...], + foreign_keys:[{table:str, columns:[str, ...], + referenced_table:str, + referenced_columns:[str, ...]}, ...], + unique:[{table:str, columns:[str, ...]}, ...], + tables:[str, ...]} # tablas (origen) con algun PK/FK/UNIQUE emitido + En error (sin lanzar): {status:'error', error:str}. + """ + try: + sql = ( + "SELECT table_name, constraint_type, constraint_column_names, " + "referenced_table, referenced_column_names FROM duckdb_constraints()" + ) + res = duckdb_query_readonly(db_path, sql) + if res["status"] != "ok": + return {"status": "error", "error": res["error"]} + + primary_keys = [] + foreign_keys = [] + unique = [] + tables = set() + + for row in res["rows"]: + ctype = row["constraint_type"] + tname = row["table_name"] + + # Filtro por tabla origen: para PK/FK/UNIQUE el dueño del constraint es + # `table_name`. Una FK se atribuye a su tabla origen (no a la + # referenciada), igual que el filtro pide. + if table is not None and tname != table: + continue + + cols = _as_list(row["constraint_column_names"]) + + if ctype == "PRIMARY KEY": + primary_keys.append({"table": tname, "columns": cols}) + tables.add(tname) + elif ctype == "UNIQUE": + unique.append({"table": tname, "columns": cols}) + tables.add(tname) + elif ctype == "FOREIGN KEY": + foreign_keys.append( + { + "table": tname, + "columns": cols, + "referenced_table": row["referenced_table"], + "referenced_columns": _as_list( + row["referenced_column_names"] + ), + } + ) + tables.add(tname) + # NOT NULL y CHECK se ignoran: no son relaciones de clave. + + return { + "status": "ok", + "primary_keys": primary_keys, + "foreign_keys": foreign_keys, + "unique": unique, + "tables": sorted(tables), + } + except Exception as e: # noqa: BLE001 + return {"status": "error", "error": str(e)} diff --git a/python/functions/datascience/detect_declared_keys_duckdb_test.py b/python/functions/datascience/detect_declared_keys_duckdb_test.py new file mode 100644 index 00000000..c014ed85 --- /dev/null +++ b/python/functions/datascience/detect_declared_keys_duckdb_test.py @@ -0,0 +1,167 @@ +"""Tests para detect_declared_keys_duckdb.""" + +import duckdb +import pytest + +from .detect_declared_keys_duckdb import detect_declared_keys_duckdb + + +@pytest.fixture +def db(tmp_path): + """DuckDB temporal con claves declaradas. + + - customers(id PRIMARY KEY, name) + - orders(id PRIMARY KEY, customer_id REFERENCES customers(id), amt) + + Esto declara dos PRIMARY KEY (customers.id, orders.id) y una FOREIGN KEY + (orders.customer_id -> customers.id). DuckDB ademas genera constraints + NOT NULL para las columnas PK, que la funcion debe ignorar. + """ + path = str(tmp_path / "keys_test.duckdb") + con = duckdb.connect(path) + con.execute("CREATE TABLE customers(id INTEGER PRIMARY KEY, name TEXT)") + con.execute( + "CREATE TABLE orders(" + " id INTEGER PRIMARY KEY," + " customer_id INTEGER REFERENCES customers(id)," + " amt DOUBLE" + ")" + ) + con.close() + return path + + +def _pk_for(res, table): + """Devuelve la entrada primary_keys cuya tabla es `table`, o None.""" + for pk in res["primary_keys"]: + if pk["table"] == table: + return pk + return None + + +def test_golden_detecta_pks_y_fk(db): + """Golden: detecta las dos PK y la FK declaradas, con valores concretos.""" + res = detect_declared_keys_duckdb(db) + assert res["status"] == "ok" + + # PRIMARY KEY de customers y de orders. + pk_customers = _pk_for(res, "customers") + pk_orders = _pk_for(res, "orders") + assert pk_customers is not None + assert pk_customers["columns"] == ["id"] + assert pk_orders is not None + assert pk_orders["columns"] == ["id"] + + # FOREIGN KEY orders.customer_id -> customers.id. + assert len(res["foreign_keys"]) == 1 + fk = res["foreign_keys"][0] + assert fk["table"] == "orders" + assert fk["columns"] == ["customer_id"] + assert fk["referenced_table"] == "customers" + assert fk["referenced_columns"] == ["id"] + + # tables incluye ambas (origen de algun constraint). + assert res["tables"] == ["customers", "orders"] + + +def test_golden_ignora_not_null_y_check(db): + """NOT NULL (auto-generado por las PK) no aparece como clave.""" + res = detect_declared_keys_duckdb(db) + assert res["status"] == "ok" + # Solo 2 PK reales (no las NOT NULL que DuckDB genera por cada columna PK). + assert len(res["primary_keys"]) == 2 + # No hay UNIQUE declarado en este schema. + assert res["unique"] == [] + + +def test_edge_filtra_por_tabla_orders(db): + """Edge table='orders': PK de orders + su FK; NO la PK de customers.""" + res = detect_declared_keys_duckdb(db, table="orders") + assert res["status"] == "ok" + + # Solo la PK de orders. + assert len(res["primary_keys"]) == 1 + assert res["primary_keys"][0]["table"] == "orders" + assert res["primary_keys"][0]["columns"] == ["id"] + # La PK de customers NO esta. + assert _pk_for(res, "customers") is None + + # La FK de orders si esta (origen = orders). + assert len(res["foreign_keys"]) == 1 + assert res["foreign_keys"][0]["table"] == "orders" + assert res["foreign_keys"][0]["referenced_table"] == "customers" + + # tables solo contiene orders (la dueña de los constraints emitidos). + assert res["tables"] == ["orders"] + + +def test_edge_filtra_por_tabla_customers(db): + """Edge table='customers': solo su PK; ninguna FK (orders queda fuera).""" + res = detect_declared_keys_duckdb(db, table="customers") + assert res["status"] == "ok" + assert len(res["primary_keys"]) == 1 + assert res["primary_keys"][0]["table"] == "customers" + assert res["foreign_keys"] == [] + assert res["tables"] == ["customers"] + + +def test_edge_unique_declarado(tmp_path): + """Edge: una constraint UNIQUE declarada aparece en `unique`.""" + path = str(tmp_path / "unique_test.duckdb") + con = duckdb.connect(path) + con.execute("CREATE TABLE products(sku INTEGER UNIQUE, name TEXT)") + con.close() + + res = detect_declared_keys_duckdb(path) + assert res["status"] == "ok" + assert len(res["unique"]) == 1 + assert res["unique"][0]["table"] == "products" + assert res["unique"][0]["columns"] == ["sku"] + assert res["primary_keys"] == [] + assert res["foreign_keys"] == [] + assert res["tables"] == ["products"] + + +def test_edge_sin_constraints_listas_vacias(tmp_path): + """Edge: tabla sin PK/FK/UNIQUE -> todas las listas vacias, status ok.""" + path = str(tmp_path / "no_keys.duckdb") + con = duckdb.connect(path) + con.execute("CREATE TABLE log(a INTEGER, b INTEGER)") + con.close() + + res = detect_declared_keys_duckdb(path) + assert res["status"] == "ok" + assert res["primary_keys"] == [] + assert res["foreign_keys"] == [] + assert res["unique"] == [] + assert res["tables"] == [] + + +def test_error_db_inexistente_no_lanza(tmp_path): + """Error: db_path inexistente -> status error, sin lanzar excepcion.""" + path = str(tmp_path / "does_not_exist.duckdb") + res = detect_declared_keys_duckdb(path) + assert res["status"] == "error" + assert isinstance(res["error"], str) + assert res["error"] != "" + + +def test_shape_resultado(db): + """El retorno tiene exactamente las claves esperadas.""" + res = detect_declared_keys_duckdb(db) + assert set(res.keys()) == { + "status", + "primary_keys", + "foreign_keys", + "unique", + "tables", + } + for pk in res["primary_keys"]: + assert set(pk.keys()) == {"table", "columns"} + for fk in res["foreign_keys"]: + assert set(fk.keys()) == { + "table", + "columns", + "referenced_table", + "referenced_columns", + } diff --git a/python/functions/datascience/suggest_intratable_fk_candidates.md b/python/functions/datascience/suggest_intratable_fk_candidates.md new file mode 100644 index 00000000..b6c5c321 --- /dev/null +++ b/python/functions/datascience/suggest_intratable_fk_candidates.md @@ -0,0 +1,91 @@ +--- +name: suggest_intratable_fk_candidates +kind: function +lang: py +domain: datascience +version: "1.0.0" +purity: pure +signature: "def suggest_intratable_fk_candidates(profile: dict, max_candidates: int = 20) -> list" +description: "Sobre el TableProfile de UNA tabla (el dict de profile_table), sugiere por heuristica de nombre + cardinalidad que columnas PARECEN una clave foranea hacia otra tabla, cuando no hay relaciones inter-tabla que medir (una sola tabla). Es una SUGERENCIA, no una afirmacion: el ref_table_guess es el stem del nombre (customer_id -> customer) y NO confirma containment. Pura: solo lee el dict, sin I/O; nunca lanza (devuelve [])." +tags: [eda, datascience, relationships, foreign-key, fk, heuristic, schema, python] +uses_functions: [] +uses_types: [] +returns: [] +returns_optional: false +error_type: "" +imports: [] +params: + - name: profile + desc: "TableProfile (dict que produce profile_table / summarize_table_*). Se leen de forma defensiva `columns` (lista de ColumnProfile con name/inferred_type/physical_type/distinct_count/unique_pct/flags), `n_rows` (int) y `key_candidates` (lista de nombres de columna ya candidatos a PK, que se excluyen). Si no es dict o no trae columns -> []." + - name: max_candidates + desc: "Tope de sugerencias devueltas (default 20). Las columnas candidatas se ordenan por distinct_count descendente (mas informativas primero) antes de cortar a este maximo." +output: "list (posiblemente vacia) de dicts, uno por columna sugerida, con claves: `column` (nombre), `ref_table_guess` (tabla conjeturada por el stem del nombre, p.ej. customer_id -> 'customer'), `reason` (frase humana que deja claro que es heuristica sin confirmar containment), `distinct_count` (int|None), `unique_pct` (float|None, fraccion 0-1 tal como viene del profile), `inferred_type` (str), `physical_type` (str). Nunca lanza." +tested: true +tests: ["test_golden_customer_id_detectado_otras_no", "test_camelcase_albumid_detectado", "test_constante_status_id_no_aparece", "test_profile_vacio_y_none_devuelven_lista_vacia", "test_category_id_casi_unico_parece_pk_no_aparece", "test_ref_table_guess_multitoken_y_orden_por_distinct", "test_max_candidates_corta_la_lista", "test_id_generico_solo_nunca_es_fk"] +test_file_path: "python/functions/datascience/suggest_intratable_fk_candidates_test.py" +file_path: "python/functions/datascience/suggest_intratable_fk_candidates.py" +--- + +## Ejemplo + +```python +from datascience import suggest_intratable_fk_candidates + +# TableProfile de UNA tabla (tipo titanic): customer_id es FK N:1; id es la PK; +# amount es una medida float; name es categorica sin sufijo de id. +profile = { + "n_rows": 891, + "key_candidates": ["id"], + "columns": [ + {"name": "id", "inferred_type": "numeric", "physical_type": "BIGINT", + "distinct_count": 891, "unique_pct": 1.0, "flags": ["possible_id"]}, + {"name": "customer_id", "inferred_type": "numeric", "physical_type": "BIGINT", + "distinct_count": 137, "unique_pct": 0.15, "flags": []}, + {"name": "amount", "inferred_type": "numeric", "physical_type": "DOUBLE", + "distinct_count": 400, "unique_pct": 0.45, "flags": []}, + {"name": "name", "inferred_type": "categorical", "physical_type": "VARCHAR", + "distinct_count": 700, "unique_pct": 0.78, "flags": []}, + ], +} + +out = suggest_intratable_fk_candidates(profile) +[c["column"] for c in out] # -> ["customer_id"] +out[0]["ref_table_guess"] # -> "customer" +out[0]["reason"] +# -> "el nombre termina en '_id' y es N:1 (137 valores distintos < 891 filas): +# parece (heuristica por nombre, sin confirmar containment) una referencia a +# una tabla «customer»" +``` + +## Cuando usarla + +Cuando el EDA tiene SOLO UNA tabla y, por tanto, no se puede inferir una FK +inter-tabla por containment (no hay otra tabla cuyos valores contener). Es el plan B +del capitulo RELACIONES de AutomaticEDA: en vez de medir solapamiento de valores +entre tablas (lo correcto cuando hay varias, ver `infer_fk_containment_duckdb` / +`build_join_graph`), conjetura por el NOMBRE de la columna (`_id`) y por su +CARDINALIDAD N:1 que columnas parecen apuntar a una entidad externa. Usala para +enriquecer el reporte con "estas columnas parecen referencias a otras tablas" sin +prometer que esa tabla exista. NO la uses si tienes varias tablas: ahi mide +containment de verdad. + +## Gotchas + +- Es **heuristica**, no una verdad: produce **falsos positivos** (una columna + `period_id` que en realidad es un codigo libre, no una FK) y **falsos negativos** + (una FK que no se llama `*_id`, p.ej. `parent`, `owner`, `sku`). No la trates como + una afirmacion de esquema. +- `ref_table_guess` es una **conjetura por el nombre** (el stem sin el sufijo id): + `customer_id` -> `customer`, `AlbumId` -> `album`, `manager_staff_id` -> + `manager_staff`. Puede no coincidir con el nombre real de la tabla (plurales, + prefijos, alias). Es una pista, no un join garantizado. +- **NO confirma containment**: no comprueba que los valores de la columna existan en + ninguna otra tabla (no puede — solo recibe el perfil de una tabla). Para confirmar + una FK real con varias tablas usa `infer_fk_containment_duckdb`. +- Excluye deliberadamente: el `id`/`Id`/`ID` generico a secas (suele ser la PK + propia, no una referencia), las columnas constantes, las que parecen unicas + (`unique_pct >= 0.99`, mas PK que FK) y los tipos no-clave (float/decimal son + medidas; date/time/timestamp y boolean no son claves). En camelCase, `paid`, + `valid`, `grid` (con `id` en minuscula y sin separador) NO se confunden con FK. +- `unique_pct` se interpreta como **fraccion 0-1** (tal como la emite el profile), no + como porcentaje 0-100. diff --git a/python/functions/datascience/suggest_intratable_fk_candidates.py b/python/functions/datascience/suggest_intratable_fk_candidates.py new file mode 100644 index 00000000..60f72190 --- /dev/null +++ b/python/functions/datascience/suggest_intratable_fk_candidates.py @@ -0,0 +1,202 @@ +"""suggest_intratable_fk_candidates — heuristica de FK intra-tabla del grupo `eda`. + +Sobre el TableProfile de UNA tabla (el dict que produce ``profile_table``), sugiere +por heuristica de NOMBRE + CARDINALIDAD que columnas PARECEN una clave foranea hacia +otra tabla, util cuando no hay relaciones inter-tabla disponibles (una sola tabla y, +por tanto, sin containment cruzado que medir). Es una SUGERENCIA, no una afirmacion: +no confirma que exista la tabla referida ni que los valores esten contenidos en ella. + +La consume el capitulo RELACIONES de AutomaticEDA cuando solo hay una tabla. + +Funcion PURA: solo lee el dict (lectura defensiva con ``.get``), no hace I/O y nunca +lanza por inputs raros (devuelve ``[]``). +""" + +# inferred_type que es compatible con una clave foranea (entero/categorico). +_FK_INFERRED_OK = {"numeric", "categorical", "integer"} + +# Prefijos de physical_type que admiten ser clave foranea (enteros, texto, uuid). +_FK_PHYSICAL_PREFIXES = ( + "int", "bigint", "smallint", "tinyint", "hugeint", "uint", + "varchar", "text", "char", "bpchar", "string", "uuid", +) + +# Prefijos de physical_type que EXCLUYEN ser clave foranea: medidas en coma flotante +# (float/double/decimal/numeric/real), temporales (date/time/timestamp/interval) y +# boolean. Se comprueban ANTES que las senales positivas (la exclusion gana: una +# columna numeric con physical DOUBLE es una medida, no una FK). +_FK_PHYSICAL_EXCLUDE = ( + "float", "double", "decimal", "numeric", "real", + "date", "time", "timestamp", "interval", + "bool", +) + + +def _fk_name_signal(name): + """Detecta el sufijo de clave foranea en el nombre y devuelve ``(stem, sufijo)``. + + Reconoce ``_id`` (snake), ``Id`` y ``ID`` (camel). NO reconoce + el ``id``/``Id``/``ID`` generico a secas (suele ser la PK propia de la tabla, no + una referencia). En camelCase la ``I`` mayuscula marca el limite de palabra, asi + que ``paid``/``valid``/``grid`` (``id`` en minuscula y sin separador) NO matchean. + + El ``stem`` se devuelve en minusculas y sirve de ``ref_table_guess`` (la tabla a + la que probablemente apunta): ``customer_id`` -> ``"customer"``, ``AlbumId`` -> + ``"album"``, ``manager_staff_id`` -> ``"manager_staff"``. Devuelve ``None`` si no + hay senal de nombre. + """ + if not isinstance(name, str): + return None + raw = name.strip() + if not raw: + return None + # Snake: termina en "_id" (indiferente a mayusculas en la parte "id"). + if raw.lower().endswith("_id"): + stem = raw[:-3].rstrip("_-. ") + if not stem: + return None + return (stem.lower(), "_id") + # Camel todo-mayuscula: "...ID" (p.ej. customerID). + if raw.endswith("ID"): + stem = raw[:-2].rstrip("_-. ") + if not stem: + return None + return (stem.lower(), "ID") + # Camel: "...Id" (p.ej. AlbumId). + if raw.endswith("Id"): + stem = raw[:-2].rstrip("_-. ") + if not stem: + return None + return (stem.lower(), "Id") + return None + + +def _fk_type_compatible(col): + """True si el tipo de la columna admite ser clave foranea. + + Compatible si el ``physical_type`` NO es una medida flotante, una temporal ni + boolean, Y ademas (``inferred_type`` en {numeric, categorical, integer} O el + ``physical_type`` empieza por entero/varchar/text/char/uuid). La comparacion es + indistinta a mayusculas/minusculas. + """ + phys = (col.get("physical_type") or "").strip().lower() + inferred = (col.get("inferred_type") or "").strip().lower() + # Exclusion por tipo fisico (gana sobre cualquier senal positiva). + for bad in _FK_PHYSICAL_EXCLUDE: + if phys.startswith(bad): + return False + # Senal positiva por tipo inferido. + if inferred in _FK_INFERRED_OK: + return True + # Senal positiva por tipo fisico (entero/texto/uuid). + for good in _FK_PHYSICAL_PREFIXES: + if phys.startswith(good): + return True + return False + + +def suggest_intratable_fk_candidates(profile: dict, max_candidates: int = 20) -> list: + """Sugiere columnas que parecen una FK intra-tabla por nombre + cardinalidad. + + Heuristica (no afirma nada): una columna es candidata a clave foranea si su nombre + tiene sufijo de id con stem no vacio (``_id`` / ``Id`` / ``ID``, + NUNCA el ``id`` generico), no es ya candidata a PK, no es constante, tiene + cardinalidad alta pero por debajo del numero de filas (N:1, no unica) y un tipo + compatible con clave (entero/categorico/texto/uuid; nunca float/fecha/boolean). + + Args: + profile: TableProfile (dict de ``profile_table``). Se leen, de forma + defensiva, ``columns`` (lista de ColumnProfile), ``n_rows`` y + ``key_candidates`` (nombres de columna ya candidatos a PK). + max_candidates: tope de sugerencias devueltas (default 20). Las columnas se + ordenan por ``distinct_count`` descendente (mas informativas primero) + antes de cortar. + + Returns: + list de dicts (posiblemente vacia), uno por columna sugerida, con claves: + ``column``, ``ref_table_guess`` (stem del nombre), ``reason`` (frase humana), + ``distinct_count``, ``unique_pct`` (fraccion 0-1 tal como viene del profile), + ``inferred_type``, ``physical_type``. Nunca lanza: si ``profile`` no es dict o + no hay columnas, devuelve ``[]``. + """ + if not isinstance(profile, dict): + return [] + columns = profile.get("columns") + if not isinstance(columns, list): + return [] + + n_rows = profile.get("n_rows") + has_n_rows = ( + isinstance(n_rows, int) and not isinstance(n_rows, bool) and n_rows > 0 + ) + + key_candidates = profile.get("key_candidates") + if not isinstance(key_candidates, (list, tuple, set)): + key_candidates = [] + key_set = set(key_candidates) + + out = [] + for col in columns: + if not isinstance(col, dict): + continue + name = col.get("name") + + # 1) Senal de nombre: sufijo de id con stem no vacio. + signal = _fk_name_signal(name) + if signal is None: + continue + ref_guess, suffix = signal + + # 2) No es ya candidata a PK (clave primaria de la propia tabla). + if name in key_set: + continue + + # 3) No constante y con >= 2 valores distintos. + flags = col.get("flags") or [] + if "constant" in flags: + continue + dc = col.get("distinct_count") + if not (isinstance(dc, int) and not isinstance(dc, bool) and dc >= 2): + continue + + # 4) Cardinalidad alta pero < n_rows (no es PK) y no parece unica. + if has_n_rows and dc >= n_rows: + continue + unique_pct = col.get("unique_pct") + has_unique = ( + isinstance(unique_pct, (int, float)) and not isinstance(unique_pct, bool) + ) + if has_unique and unique_pct >= 0.99: + continue + + # 5) Tipo compatible con clave foranea (entero/categorico/texto; no medida). + if not _fk_type_compatible(col): + continue + + out.append( + { + "column": name, + "ref_table_guess": ref_guess, + "reason": _build_reason(suffix, dc, n_rows if has_n_rows else None, ref_guess), + "distinct_count": dc, + "unique_pct": float(unique_pct) if has_unique else None, + "inferred_type": col.get("inferred_type") or "", + "physical_type": col.get("physical_type") or "", + } + ) + + # Mas informativas primero (mayor cardinalidad), luego corte. + out.sort(key=lambda d: d.get("distinct_count") or 0, reverse=True) + return out[: max(0, int(max_candidates))] + + +def _build_reason(suffix, dc, n_rows, ref_guess): + """Frase humana que deja claro que la sugerencia es heuristica, no confirmada.""" + if n_rows is not None: + card = f"es N:1 ({dc} valores distintos < {n_rows} filas)" + else: + card = f"tiene {dc} valores distintos que se repiten (cardinalidad N:1)" + return ( + f"el nombre termina en '{suffix}' y {card}: parece (heuristica por nombre, " + f"sin confirmar containment) una referencia a una tabla «{ref_guess}»" + ) diff --git a/python/functions/datascience/suggest_intratable_fk_candidates_test.py b/python/functions/datascience/suggest_intratable_fk_candidates_test.py new file mode 100644 index 00000000..f06e928e --- /dev/null +++ b/python/functions/datascience/suggest_intratable_fk_candidates_test.py @@ -0,0 +1,157 @@ +"""Tests para suggest_intratable_fk_candidates (funcion pura, sin I/O).""" + +from suggest_intratable_fk_candidates import suggest_intratable_fk_candidates + + +def _col(name, inferred_type="numeric", physical_type="BIGINT", distinct_count=10, + unique_pct=0.1, flags=None): + """Construye un ColumnProfile minimo a mano (el dict que emite profile_table).""" + return { + "name": name, + "inferred_type": inferred_type, + "physical_type": physical_type, + "semantic_type": "", + "distinct_count": distinct_count, + "unique_pct": unique_pct, + "null_count": 0, + "null_pct": 0.0, + "flags": list(flags) if flags else [], + } + + +def test_golden_customer_id_detectado_otras_no(): + # Tabla tipo titanic: customer_id es FK N:1; id es la PK; amount es medida; + # name es categorica sin sufijo de id. Solo customer_id debe aparecer. + profile = { + "n_rows": 891, + "key_candidates": ["id"], + "columns": [ + _col("id", inferred_type="numeric", physical_type="BIGINT", + distinct_count=891, unique_pct=1.0, flags=["possible_id"]), + _col("customer_id", inferred_type="numeric", physical_type="BIGINT", + distinct_count=137, unique_pct=0.15, flags=[]), + _col("amount", inferred_type="numeric", physical_type="DOUBLE", + distinct_count=400, unique_pct=0.45), + _col("name", inferred_type="categorical", physical_type="VARCHAR", + distinct_count=700, unique_pct=0.78), + ], + } + out = suggest_intratable_fk_candidates(profile) + assert isinstance(out, list) + assert [c["column"] for c in out] == ["customer_id"] + cand = out[0] + assert cand["ref_table_guess"] == "customer" + assert cand["distinct_count"] == 137 + assert cand["unique_pct"] == 0.15 + assert cand["inferred_type"] == "numeric" + assert cand["physical_type"] == "BIGINT" + # La razon deja claro que es heuristica + cita el sufijo y la tabla. + assert "customer" in cand["reason"] + assert "_id" in cand["reason"] + + +def test_camelcase_albumid_detectado(): + # AlbumId (camelCase, VARCHAR) -> detectada, ref_table_guess "album". + profile = { + "n_rows": 3503, + "key_candidates": ["TrackId"], + "columns": [ + _col("AlbumId", inferred_type="categorical", physical_type="VARCHAR", + distinct_count=347, unique_pct=0.10), + ], + } + out = suggest_intratable_fk_candidates(profile) + # TrackId es PK candidata (en key_candidates), AlbumId no -> AlbumId aparece. + assert [c["column"] for c in out] == ["AlbumId"] + assert out[0]["ref_table_guess"] == "album" + + +def test_constante_status_id_no_aparece(): + # status_id constante (flag "constant", distinct_count 1) NO es FK util. + profile = { + "n_rows": 1000, + "key_candidates": [], + "columns": [ + _col("status_id", inferred_type="numeric", physical_type="INTEGER", + distinct_count=1, unique_pct=0.001, flags=["constant"]), + ], + } + out = suggest_intratable_fk_candidates(profile) + assert out == [] + + +def test_profile_vacio_y_none_devuelven_lista_vacia(): + # Lectura defensiva: ni {} ni None lanzan; devuelven []. + assert suggest_intratable_fk_candidates({}) == [] + assert suggest_intratable_fk_candidates(None) == [] + # profile sin columns o con columns no-lista tampoco lanza. + assert suggest_intratable_fk_candidates({"n_rows": 10}) == [] + assert suggest_intratable_fk_candidates({"columns": "no-soy-lista"}) == [] + + +def test_category_id_casi_unico_parece_pk_no_aparece(): + # unique_pct 0.999 -> parece PK (no N:1) -> NO se sugiere como FK. + profile = { + "n_rows": 891, + "key_candidates": [], + "columns": [ + _col("category_id", inferred_type="numeric", physical_type="BIGINT", + distinct_count=890, unique_pct=0.999), + ], + } + out = suggest_intratable_fk_candidates(profile) + assert out == [] + + +def test_ref_table_guess_multitoken_y_orden_por_distinct(): + # manager_staff_id conserva los underscores del stem -> "manager_staff". + # Ademas, con varias candidatas, se ordenan por distinct_count descendente. + profile = { + "n_rows": 10000, + "key_candidates": ["staff_id"], # staff_id es PK aqui, no debe aparecer + "columns": [ + _col("staff_id", inferred_type="numeric", physical_type="BIGINT", + distinct_count=10000, unique_pct=1.0, flags=["possible_id"]), + _col("store_id", inferred_type="numeric", physical_type="INTEGER", + distinct_count=2, unique_pct=0.0002), + _col("manager_staff_id", inferred_type="numeric", physical_type="INTEGER", + distinct_count=40, unique_pct=0.004), + ], + } + out = suggest_intratable_fk_candidates(profile) + cols = [c["column"] for c in out] + # staff_id excluida (PK); las otras dos ordenadas por distinct desc. + assert cols == ["manager_staff_id", "store_id"] + refs = {c["column"]: c["ref_table_guess"] for c in out} + assert refs["manager_staff_id"] == "manager_staff" + assert refs["store_id"] == "store" + + +def test_max_candidates_corta_la_lista(): + # max_candidates limita el numero de sugerencias devueltas. + profile = { + "n_rows": 10000, + "key_candidates": [], + "columns": [ + _col("a_id", distinct_count=300, unique_pct=0.03), + _col("b_id", distinct_count=200, unique_pct=0.02), + _col("c_id", distinct_count=100, unique_pct=0.01), + ], + } + out = suggest_intratable_fk_candidates(profile, max_candidates=2) + assert [c["column"] for c in out] == ["a_id", "b_id"] + + +def test_id_generico_solo_nunca_es_fk(): + # 'id'/'Id'/'ID' a secas (sin stem) jamas se sugieren como FK. + profile = { + "n_rows": 500, + "key_candidates": [], + "columns": [ + _col("id", distinct_count=500, unique_pct=1.0), + _col("Id", distinct_count=120, unique_pct=0.24), + _col("ID", distinct_count=80, unique_pct=0.16), + ], + } + out = suggest_intratable_fk_candidates(profile) + assert out == [] From 3be188a921278167fe9720c20b1723414649f0e6 Mon Sep 17 00:00:00 2001 From: Egutierrez Date: Tue, 30 Jun 2026 18:20:17 +0200 Subject: [PATCH 30/53] feat(eda): profile_level (lite/standard/full) en render_automatic_eda MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Añade el parámetro profile_level a render_automatic_eda como preset de consumo CPU/LLM que mapea a los flags existentes (run_models, run_series, run_llm, sample). Tres niveles: - lite (bajo consumo): run_llm=False, run_series=False, sample=2000 y modelos limitados a PCA + normalidad, SIN KMeans ni IsolationForest (lo caro en CPU). Para un vistazo rápido y barato. - standard (default): comportamiento histórico — modelos completos, serie, sin LLM. - full: standard + narrativa LLM por capítulo. Precedencia: un flag explícito del caller (run_llm=..., run_models=..., etc.) siempre prima sobre el default que fija el preset; el preset solo aplica al parámetro que se deja en None. Cableado del modo lite sin tocar profile_table (lo tocan otros agentes en paralelo): profile_table NO corre los modelos (evita pagar KMeans + IsolationForest); este pipeline los corre con run_eda_models(run_kmeans=False, run_isolation=False) reusando ctx['raw_numeric'], y quita raw_numeric del ctx para que el capítulo modelos no reproyecte clusters KMeans en vivo (project_clusters_2d). geo_points ya queda derivado, así que geospatial no se afecta. Cambio aditivo y retro-compatible: sin profile_level el comportamiento es idéntico al de v1.0.0 (standard). Tests nuevos cubren lite/standard, la precedencia flag-sobre-preset, y la equivalencia del default con el histórico. Bump 1.0.0 -> 1.1.0 + growth log en el .md. Skill /eda documenta --lite/--full. Verificación: golden lite/standard/full sobre titanic — lite 4.8s (PCA+norm, sin KMeans/iso/LLM/serie), standard 7.8s (modelos completos), full 38.3s (+LLM). Suite render_automatic_eda + automatic_eda: 96 passed. fn index sin error. Co-Authored-By: Claude Opus 4.8 (1M context) --- .claude/commands/eda.md | 4 +- .../pipelines/render_automatic_eda.md | 72 +++++--- .../pipelines/render_automatic_eda.py | 125 +++++++++++-- .../pipelines/render_automatic_eda_test.py | 167 ++++++++++++++++++ 4 files changed, 334 insertions(+), 34 deletions(-) diff --git a/.claude/commands/eda.md b/.claude/commands/eda.md index 860c340f..c358fea8 100644 --- a/.claude/commands/eda.md +++ b/.claude/commands/eda.md @@ -27,6 +27,7 @@ Página madre del grupo: `docs/capabilities/eda.md` (léela primero para cargar - `--series` → `run_series=True` (estacionariedad ADF+KPSS, ACF/PACF, STL, retornos por columna numérica). - `--pdf` → `emit_pdf=True` (PDF A5 legacy de `render_eda_pdf`, legible en móvil). - `--legacy-only` → emite SOLO el PDF legacy (sin AutomaticEDA), para casos en que solo se quiera el PDF rápido. + - `--lite` / `--bajo-consumo` → `render_automatic_eda(profile_level="lite")`: EDA barato y rápido (CI, vistazo previo, máquina sin GPU/red). Apaga LLM y serie temporal y limita los modelos a **PCA + normalidad** (sin KMeans ni IsolationForest, lo caro en CPU), con `sample` reducido. `--full` → `profile_level="full"` (standard + narrativa LLM). Por defecto `profile_level="standard"` (comportamiento histórico). Un flag explícito (`--llm`, `--models`, ...) prima sobre el preset. Por defecto, **un EDA completo emite SIEMPRE el informe AutomaticEDA en sus dos formatos: PDF (A5 móvil) Y PPTX (16:9 para compartir)** con los 11 capítulos poblados (portada, overview, distribuciones, calidad, correlaciones, modelos, series, geoespacial, agregación, interpretación LLM). Usa el pipeline `render_automatic_eda` (o `profile_table(emit_automatic=True)`), que activa `run_models` y `run_series` para que los capítulos de modelos/series/geoespacial/agregación salgan poblados. Deja `run_llm` para cuando el usuario lo pida o interese la interpretación semántica + narrativa por capítulo (es la única parte que gasta tokens del modelo). @@ -50,7 +51,8 @@ from pipelines.render_automatic_eda import render_automatic_eda # tablas de agregación). run_llm=True añade la narrativa LLM por capítulo. r = render_automatic_eda( "/ruta/datos.duckdb", "ventas", - run_models=True, run_series=True, run_llm=False, out_dir="reports", + profile_level="standard", # "lite" = bajo consumo CPU/LLM; "full" = + narrativa LLM + out_dir="reports", ) print("status:", r["status"]) print("pdf: ", r["pdf_path"], "(", r["n_pages"], "págs )") diff --git a/python/functions/pipelines/render_automatic_eda.md b/python/functions/pipelines/render_automatic_eda.md index b157dfd2..50d4bcda 100644 --- a/python/functions/pipelines/render_automatic_eda.md +++ b/python/functions/pipelines/render_automatic_eda.md @@ -4,9 +4,9 @@ kind: pipeline lang: py domain: pipelines purity: impure -version: "1.0.0" -signature: "def render_automatic_eda(db_path: str, table: str, backend: str = \"duckdb\", sample: int = 5000, run_models: bool = True, run_series: bool = True, run_llm: bool = False, out_dir: str = \"reports\", basename: str = None, ctx_extra: dict = None) -> dict" -description: "Informe AutomaticEDA COMPLETO one-shot de una tabla DuckDB/PostgreSQL: perfila con profile_table, construye el ctx con los datos crudos (build_eda_render_ctx: raw_numeric para modelos/geo, timeseries_raw para series, geo_points para el mapa, db_path/table para la agregacion push-down) y emite PDF (A5 movil) Y PPTX (16:9) del mismo documento por capitulos, con los 11 capitulos POBLADOS de verdad (clusters pintados sobre el PCA, evolucion temporal, mapa geografico y tablas de agregacion), no degradados. Devuelve las rutas de PDF/PPTX y el manifiesto de versiones por capitulo." +version: "1.1.0" +signature: "def render_automatic_eda(db_path: str, table: str, backend: str = \"duckdb\", sample: int = None, run_models: bool = None, run_series: bool = None, run_llm: bool = None, profile_level: str = \"standard\", out_dir: str = \"reports\", basename: str = None, ctx_extra: dict = None) -> dict" +description: "Informe AutomaticEDA COMPLETO one-shot de una tabla DuckDB/PostgreSQL: perfila con profile_table, construye el ctx con los datos crudos (build_eda_render_ctx: raw_numeric para modelos/geo, timeseries_raw para series, geo_points para el mapa, db_path/table para la agregacion push-down) y emite PDF (A5 movil) Y PPTX (16:9) del mismo documento por capitulos, con los 11 capitulos POBLADOS de verdad (clusters pintados sobre el PCA, evolucion temporal, mapa geografico y tablas de agregacion), no degradados. El parametro profile_level es un preset de consumo CPU/LLM (lite/standard/full) que mapea a los flags run_models/run_series/run_llm/sample; un flag explicito siempre prima sobre el preset. lite=bajo consumo (sin LLM, sin serie, modelos solo PCA+normalidad sin KMeans/IsolationForest, sample reducido); standard=comportamiento historico; full=standard+narrativa LLM. Devuelve las rutas de PDF/PPTX y el manifiesto de versiones por capitulo." tags: [eda, duckdb, postgres, profiling, pipeline, dataops, report, pdf, pptx] uses_functions: - profile_table_py_pipelines @@ -31,13 +31,15 @@ params: - name: backend desc: "'duckdb' (default) o 'postgres'. Selecciona el motor de perfilado y muestreo." - name: sample - desc: "Maximo de filas/valores muestreados por columna para el perfil y para los datos crudos del ctx (LIMIT). Default 5000." + desc: "Maximo de filas/valores muestreados por columna para el perfil y para los datos crudos del ctx (LIMIT). Default None => lo fija el preset de profile_level (lite=2000, standard/full=5000). Un valor explicito prima sobre el preset." - name: run_models - desc: "Si True (default) corre los modelos baratos (PCA/KMeans/IsolationForest/normalidad); necesario para que el capitulo modelos pinte los clusters sobre el plano PCA." + desc: "Corre los modelos baratos (PCA/KMeans/IsolationForest/normalidad); necesario para que el capitulo modelos pinte los clusters sobre el plano PCA. Default None => lo fija el preset (True en los tres niveles); en lite los modelos se limitan a PCA+normalidad. Un valor explicito prima sobre el preset." - name: run_series - desc: "Si True (default) calcula el analisis de serie temporal por columna numerica; necesario para el analisis del capitulo timeseries (la grafica de evolucion sale de los datos crudos del ctx aunque sea False)." + desc: "Calcula el analisis de serie temporal por columna numerica; necesario para el analisis del capitulo timeseries. Default None => lo fija el preset (standard/full=True, lite=False). Un valor explicito prima sobre el preset." - name: run_llm - desc: "Si True (default False) hace la interpretacion LLM del perfil y ACTIVA la narrativa LLM de los capitulos modelos/geospatial/agregacion (titulos de segmento, descripcion de zona, seleccion de agregaciones). Con False usan su derivacion cuantitativa sin red." + desc: "Hace la interpretacion LLM del perfil y ACTIVA la narrativa LLM de los capitulos modelos/geospatial/agregacion (titulos de segmento, descripcion de zona, seleccion de agregaciones). Con False usan su derivacion cuantitativa sin red. Default None => lo fija el preset (full=True, lite/standard=False). Un valor explicito prima sobre el preset." + - name: profile_level + desc: "Preset de consumo CPU/LLM (default 'standard'). Mapea a defaults de run_models/run_series/run_llm/sample; un flag explicito SIEMPRE prima. 'lite'=bajo consumo (run_llm=False, run_series=False, sample=2000, modelos solo PCA+normalidad sin KMeans/IsolationForest); 'standard'=comportamiento historico (modelos completos, serie, sin LLM); 'full'=standard+narrativa LLM. Un nivel desconocido cae a 'standard'." - name: out_dir desc: "Directorio de salida (se crea si no existe). Default 'reports'." - name: basename @@ -52,14 +54,21 @@ output: "dict {status:'ok', pdf_path:str, pptx_path:str, manifest_path:str|None, ```python from pipelines.render_automatic_eda import render_automatic_eda -# Tabla DuckDB con categoricas + fecha + numericas: informe completo a reports/. -r = render_automatic_eda("/tmp/ventas.duckdb", "ventas", - run_models=True, run_series=True, out_dir="reports") +# Informe completo a reports/ (standard = comportamiento por defecto historico). +r = render_automatic_eda("/tmp/ventas.duckdb", "ventas", out_dir="reports") print(r["status"], r["pdf_path"], r["pptx_path"], r["n_pages"], r["n_slides"]) -# ok reports/aeda_ventas_20260630-120500.pdf reports/aeda_ventas_20260630-120500.pptx 14 16 +# ok reports/aeda_ventas_20260630-120500.pdf reports/aeda_ventas_20260630-120500.pptx 37 39 -# Con narrativa LLM (titulos de segmento, descripcion geografica, etc.): -r = render_automatic_eda("/tmp/ventas.duckdb", "ventas", run_llm=True) +# Bajo consumo (CPU/LLM): vistazo rapido y barato — sin LLM, sin serie, modelos +# solo PCA + normalidad (sin KMeans/IsolationForest), sample reducido. +r = render_automatic_eda("/tmp/ventas.duckdb", "ventas", profile_level="lite") + +# Maximo: standard + narrativa LLM por capitulo (titulos de segmento, etc.). +r = render_automatic_eda("/tmp/ventas.duckdb", "ventas", profile_level="full") + +# Precedencia: el flag explicito SIEMPRE prima sobre el preset. lite pero con LLM: +r = render_automatic_eda("/tmp/ventas.duckdb", "ventas", + profile_level="lite", run_llm=True) # el LLM SI se ejecuta ``` ## Cuando usarla @@ -72,20 +81,41 @@ llama a los dos renderers": este pipeline orquesta `profile_table` -> entregable para compartir un EDA, o como el motor detras de `profile_table( emit_automatic=True)` y del skill `/eda`. +Para un EDA **barato/rapido** (CI, vistazo previo, maquina sin GPU o sin red) usa +`profile_level="lite"`: evita KMeans + IsolationForest (lo caro en CPU), la serie +temporal y el LLM. Para el **maximo** con interpretacion narrativa por capitulo, +`profile_level="full"`. El default `"standard"` mantiene el comportamiento previo. + ## Gotchas - Impura: ESCRIBE el PDF, el PPTX y `automatic_eda_manifest.json` en `out_dir`. - `db_path` debe existir: DuckDB read-only no crea la base. -- `run_models=True` y `run_series=True` por defecto encarecen el perfil (PCA/ - KMeans/IsolationForest + ADF/KPSS/STL por columna). Para un informe mas barato - ponlos a False: los capitulos modelos/timeseries se omiten o se reducen, pero - el resto del informe sale igual. -- `run_llm=True` hace llamadas de red (interpretacion del perfil + narrativa por - capitulo). Sin red, dejalo en False: los capitulos siguen completos con su - derivacion cuantitativa (titulos de segmento derivados, nota geografica - derivada, seleccion de agregaciones cuantitativa). +- **Precedencia de flags vs preset**: `profile_level` solo fija los DEFAULTS de + `run_models`/`run_series`/`run_llm`/`sample` (los que quedan en None). Cualquiera + de esos flags pasado explicito gana al preset. Ej: `profile_level="lite", + run_llm=True` ejecuta el LLM pese a que lite lo apaga por defecto. +- **lite y la seleccion de features de modelo**: en lite los modelos (PCA + + normalidad) corren sobre la muestra numerica cruda (`ctx['raw_numeric']`), sin la + poda fina de features que aplica el modo standard (que excluye ids enteros y + columnas de baja cardinalidad antes de PCA/KMeans). Es el coste de mantener el + cableado minimo y barato; para el analisis fino de modelos usa standard/full. +- `profile_level="standard"`/`"full"` corren PCA/KMeans/IsolationForest + + ADF/KPSS/STL por columna (caro). Para un informe mas barato usa `"lite"` (o pon + los flags a False a mano): los capitulos modelos/timeseries se reducen pero el + resto del informe sale igual. +- `run_llm=True` (preset full o flag explicito) hace llamadas de red + (interpretacion del perfil + narrativa por capitulo). Sin red, usa lite/standard: + los capitulos siguen completos con su derivacion cuantitativa. - El PPTX requiere `python-pptx`; si no esta instalado, `pptx_path` sera None y `pptx_note` lo explica (el PDF se emite igual). - Los datos crudos del ctx se muestrean con `sample` (LIMIT), no se trae la tabla entera a RAM; con tablas enormes sube `sample` si quieres mas representatividad (coste: mas memoria). + +## Capability growth log + +- v1.1.0 (2026-06-30) — anade el parametro `profile_level` (lite/standard/full), + preset de consumo CPU/LLM que mapea a los flags run_models/run_series/run_llm/ + sample. lite limita los modelos a PCA+normalidad (cableado a run_eda_models con + run_kmeans=False/run_isolation=False) y apaga LLM/serie. Cambio aditivo y + retro-compatible: sin profile_level el comportamiento es identico al de v1.0.0. diff --git a/python/functions/pipelines/render_automatic_eda.py b/python/functions/pipelines/render_automatic_eda.py index c0b58065..8090bc1f 100644 --- a/python/functions/pipelines/render_automatic_eda.py +++ b/python/functions/pipelines/render_automatic_eda.py @@ -34,21 +34,62 @@ from datascience import ( build_eda_render_ctx, render_automatic_eda_pdf, render_automatic_eda_pptx, + run_eda_models, ) from pipelines.profile_table import profile_table # Tokens de almacenamiento por backend (para la portada del informe). _STORAGE = {"duckdb": "DuckDB", "postgres": "PostgreSQL"} +# Presets de consumo CPU/LLM: cada profile_level fija SOLO los DEFAULTS de los +# flags que controlan el coste (un flag explícito del caller siempre prima sobre +# el preset). model_opts != None marca el camino "modelos baratos" (lite): los +# modelos NO los corre profile_table (que ejecutaría KMeans + IsolationForest), +# sino run_eda_models con esa granularidad, de modo que el coste CPU de los +# multivariantes nunca se paga. model_opts None => modelos completos como hasta +# ahora (profile_table los corre con todos los algoritmos). +_PROFILE_PRESETS = { + # Bajo consumo: sin LLM, sin serie, sample reducido y modelos limitados a + # PCA + normalidad (sin KMeans ni IsolationForest, lo caro en CPU). Vistazo + # rápido y barato de una tabla. + "lite": { + "run_models": True, + "run_series": False, + "run_llm": False, + "sample": 2000, + "model_opts": {"run_kmeans": False, "run_isolation": False}, + }, + # Default: idéntico al comportamiento histórico del pipeline (modelos + # completos, serie temporal, sin LLM, sample 5000). + "standard": { + "run_models": True, + "run_series": True, + "run_llm": False, + "sample": 5000, + "model_opts": None, + }, + # Máximo: standard + narrativa LLM (interpretación del perfil y de los + # capítulos modelos/geospatial/agregacion). Es la única parte que gasta + # tokens del modelo. + "full": { + "run_models": True, + "run_series": True, + "run_llm": True, + "sample": 5000, + "model_opts": None, + }, +} + def render_automatic_eda( db_path: str, table: str, backend: str = "duckdb", - sample: int = 5000, - run_models: bool = True, - run_series: bool = True, - run_llm: bool = False, + sample: int = None, + run_models: bool = None, + run_series: bool = None, + run_llm: bool = None, + profile_level: str = "standard", out_dir: str = "reports", basename: str = None, ctx_extra: dict = None, @@ -60,19 +101,39 @@ def render_automatic_eda( table: nombre de la tabla a perfilar. backend: "duckdb" (default) o "postgres". sample: máximo de filas/valores muestreados por columna para el perfil - y para los datos crudos del ctx (LIMIT). Default 5000. - run_models: si True (default) corre los modelos baratos + y para los datos crudos del ctx (LIMIT). Default None => lo fija el + preset de profile_level (lite=2000, standard/full=5000). + run_models: corre los modelos baratos (PCA/KMeans/IsolationForest/normalidad). Necesario para que el - capítulo `modelos` pinte los clusters sobre el plano PCA. - run_series: si True (default) calcula el análisis de serie temporal por + capítulo `modelos` pinte los clusters sobre el plano PCA. Default + None => lo fija el preset (True en los tres niveles); en `lite` los + modelos se limitan a PCA + normalidad (ver profile_level). + run_series: calcula el análisis de serie temporal por columna numérica. Necesario para el análisis del capítulo `timeseries` (la gráfica de evolución sale de los datos crudos del - ctx aunque run_series sea False). - run_llm: si True (default False) hace la interpretación LLM del perfil y + ctx aunque run_series sea False). Default None => lo fija el preset + (standard/full=True, lite=False). + run_llm: hace la interpretación LLM del perfil y ACTIVA además la narrativa LLM de los capítulos modelos/geospatial/ agregacion (títulos de segmento, descripción de la zona, selección de agregaciones). Con False esos capítulos usan su derivación - cuantitativa (siguen completos, sin llamadas de red). + cuantitativa (siguen completos, sin llamadas de red). Default None => + lo fija el preset (full=True, lite/standard=False). + profile_level: preset de consumo CPU/LLM. Mapea a defaults de los flags + anteriores; un flag explícito SIEMPRE prima sobre el preset (el + preset solo fija el default cuando el flag se deja en None): + + - "lite" bajo consumo: run_llm=False, run_series=False, + sample=2000 y modelos limitados a **PCA + normalidad** (SIN KMeans + ni IsolationForest, que es lo caro en CPU). Pensado para un vistazo + rápido y barato. El capítulo `modelos` sale con PCA + normalidad, + sin el scatter de clusters. + - "standard" (default): comportamiento histórico — modelos completos + (PCA/KMeans/IsolationForest/normalidad), serie temporal, sin LLM. + - "full" standard + narrativa LLM (run_llm=True). + + Ejemplo de precedencia: profile_level="lite" con run_llm=True + explícito => el LLM SÍ se ejecuta (el flag explícito gana al preset). out_dir: directorio de salida (se crea si no existe). Default "reports". basename: nombre base de los archivos sin extensión. Default "aeda_
_". @@ -90,6 +151,24 @@ def render_automatic_eda( En error: {"status": "error", "error": str}. """ try: + # 0) Resolución del preset: el profile_level fija los DEFAULTS de los + # flags de coste; cualquier flag que el caller haya pasado explícito + # (!= None) prima sobre el preset. Un profile_level desconocido cae a + # "standard" (comportamiento histórico), sin lanzar. + preset = _PROFILE_PRESETS.get(profile_level, _PROFILE_PRESETS["standard"]) + sample = preset["sample"] if sample is None else sample + run_models = preset["run_models"] if run_models is None else run_models + run_series = preset["run_series"] if run_series is None else run_series + run_llm = preset["run_llm"] if run_llm is None else run_llm + model_opts = preset["model_opts"] + + # En el camino "modelos baratos" (lite) profile_table NO corre los + # modelos: los ejecuta este pipeline con run_eda_models y la granularidad + # del preset, evitando pagar el coste CPU de KMeans + IsolationForest. + # En standard/full profile_table los corre completos como siempre. + lite_models = bool(run_models) and model_opts is not None + pt_run_models = bool(run_models) and not lite_models + # 1) Perfil base + modelos/serie opcionales. No escribe report propio # (write_report=False): este pipeline emite su propio par PDF/PPTX. pres = profile_table( @@ -97,7 +176,7 @@ def render_automatic_eda( table, backend=backend, sample=sample, - run_models=run_models, + run_models=pt_run_models, run_llm=run_llm, run_series=run_series, emit_pdf=False, @@ -131,6 +210,28 @@ def render_automatic_eda( base_ctx=base_ctx, ) + # 2.5) Camino lite — modelos baratos (PCA + normalidad, sin KMeans ni + # IsolationForest). profile_table no corrió los modelos; aquí se corren + # con run_eda_models reusando la muestra numérica alineada por fila que + # build_eda_render_ctx ya trajo en ctx['raw_numeric'] (no se reimplementa + # la lógica de los modelos: se delega en run_eda_models con la + # granularidad del preset). + if lite_models: + raw_numeric = ctx.get("raw_numeric") if isinstance(ctx, dict) else None + if isinstance(raw_numeric, dict) and raw_numeric: + model_input = { + col: {"values": vals, "type": "numeric"} + for col, vals in raw_numeric.items() + } + prof["models"] = run_eda_models(model_input, **model_opts) + # Quita raw_numeric del ctx para que el capítulo `modelos` NO + # reproyecte clusters KMeans en vivo (project_clusters_2d ejecuta + # KMeans): en lite ese coste se evita. geo_points ya quedó derivado + # en ctx por build_eda_render_ctx, así que el capítulo geospatial no + # se ve afectado. + if isinstance(ctx, dict): + ctx.pop("raw_numeric", None) + # 3) Render a ambos formatos desde el MISMO documento por capítulos. os.makedirs(out_dir, exist_ok=True) ts = datetime.now(timezone.utc).strftime("%Y%m%d-%H%M%S") diff --git a/python/functions/pipelines/render_automatic_eda_test.py b/python/functions/pipelines/render_automatic_eda_test.py index a463e4f7..b8941834 100644 --- a/python/functions/pipelines/render_automatic_eda_test.py +++ b/python/functions/pipelines/render_automatic_eda_test.py @@ -89,3 +89,170 @@ def test_pipeline_bad_db_degrades_without_raising(tmp_path): out_dir=str(tmp_path / "o")) assert r["status"] == "error" assert "error" in r + + +# --------------------------------------------------------------------------- # +# profile_level: preset de bajo consumo CPU/LLM. +# --------------------------------------------------------------------------- # +def _make_db_models(path): + """DB con >=2 numéricas continuas (alta cardinalidad, 3 clusters gaussianos). + + El DB `sales` de _make_db solo deja UNA columna de modelo tras la selección de + features (units es baja cardinalidad, lat/lon discretizadas), insuficiente para + PCA/KMeans/IsolationForest (necesitan >=2). Este DB sí tiene 3 numéricas + continuas con estructura de clusters para que el modo completo ejecute los + multivariantes. + """ + import random + from datetime import date, timedelta + + con = duckdb.connect(path) + con.execute( + "CREATE TABLE pts (d DATE, grp VARCHAR, x1 DOUBLE, x2 DOUBLE, x3 DOUBLE)" + ) + random.seed(42) + centers = [(0.0, 0.0, 0.0), (10.0, 10.0, 10.0), (20.0, 5.0, 15.0)] + d0 = date(2024, 1, 1) + rows = [] + for i in range(150): + cx, cy, cz = centers[i % 3] + rows.append(( + d0 + timedelta(days=i), f"g{i % 3}", + round(cx + random.gauss(0, 1.0), 4), + round(cy + random.gauss(0, 1.0), 4), + round(cz + random.gauss(0, 1.0), 4), + )) + con.executemany("INSERT INTO pts VALUES (?,?,?,?,?)", rows) + con.close() + + +def test_profile_level_lite_skips_expensive_models(tmp_path): + """lite: el bloque models trae PCA + normalidad pero NO KMeans/IsolationForest. + + Demuestra (DoD bajo consumo) que el camino lite no ejecuta los modelos caros + en CPU ni la capa LLM ni la serie temporal: prof['models'] queda con pca y + normality poblados y kmeans/outliers a None, prof['llm'] y prof['series'] a + None, y el capítulo `modelos` se renderiza igualmente (con PCA, sin clusters). + """ + import json + + db = str(tmp_path / "pts.duckdb") + _make_db_models(db) + out = str(tmp_path / "out") + r = render_automatic_eda(db, "pts", profile_level="lite", + out_dir=out, basename="lite") + assert r["status"] == "ok", r.get("error") + + models = (r["profile"] or {}).get("models") or {} + assert models.get("pca") is not None, "lite debe traer PCA" + assert models.get("normality") is not None, "lite debe traer normalidad" + assert models.get("kmeans") is None, "lite NO debe ejecutar KMeans" + assert models.get("outliers") is None, "lite NO debe ejecutar IsolationForest" + assert (r["profile"] or {}).get("llm") is None, "lite NO debe llamar al LLM" + assert (r["profile"] or {}).get("series") is None, "lite NO debe calcular serie" + + # El capítulo modelos sigue presente (lo puebla el PCA), sin clusters KMeans. + with open(r["manifest_path"], encoding="utf-8") as fh: + man = json.load(fh) + assert "modelos" in (man.get("chapters") or {}) + + +def test_profile_level_standard_runs_full_models(tmp_path): + """standard (default): modelos completos (KMeans + IsolationForest) y serie.""" + db = str(tmp_path / "pts.duckdb") + _make_db_models(db) + out = str(tmp_path / "out") + r = render_automatic_eda(db, "pts", profile_level="standard", + out_dir=out, basename="std") + assert r["status"] == "ok", r.get("error") + models = (r["profile"] or {}).get("models") or {} + assert models.get("pca") is not None + assert models.get("kmeans") is not None, "standard debe ejecutar KMeans" + assert models.get("outliers") is not None, "standard debe ejecutar IsolationForest" + assert (r["profile"] or {}).get("series") is not None, "standard calcula serie" + + +def _patch_pipeline_internals(monkeypatch, captured): + """Stub de las dependencias del pipeline para tests de resolución de flags. + + Sustituye profile_table / build_eda_render_ctx / renderers por stubs rápidos + sin red ni matplotlib, capturando los kwargs con los que se invocan. Permite + verificar la PRECEDENCIA flag-explícito-sobre-preset sin ejecutar el EDA real. + """ + import pipelines.render_automatic_eda as mod + + def fake_profile_table(db_path, table, **kw): + captured["run_llm"] = kw.get("run_llm") + captured["run_models"] = kw.get("run_models") + captured["run_series"] = kw.get("run_series") + captured["sample"] = kw.get("sample") + return {"status": "ok", "profile": {"columns": []}} + + def fake_ctx(db_path, table, prof, **kw): + captured["base_ctx"] = kw.get("base_ctx") + return {} + + monkeypatch.setattr(mod, "profile_table", fake_profile_table) + monkeypatch.setattr(mod, "build_eda_render_ctx", fake_ctx) + monkeypatch.setattr(mod, "render_automatic_eda_pdf", + lambda *a, **k: {"path": "x.pdf", "n_pages": 1, + "manifest_path": "m.json"}) + monkeypatch.setattr(mod, "render_automatic_eda_pptx", + lambda *a, **k: {"path": "x.pptx", "n_slides": 1}) + + +def test_explicit_flag_overrides_preset(monkeypatch): + """Precedencia: profile_level='lite' con run_llm=True explícito → LLM activo. + + El flag explícito del caller gana al default del preset. Se verifica tanto en + el flag que llega a profile_table (run_llm=True ⇒ profile_table llamará al + LLM) como en el base_ctx (run_cluster_llm=True ⇒ narrativa LLM por capítulo). + """ + captured = {} + _patch_pipeline_internals(monkeypatch, captured) + + captured.clear() + render_automatic_eda("db", "t", profile_level="lite", run_llm=True) + assert captured["run_llm"] is True, "flag explícito debe primar sobre preset lite" + assert (captured["base_ctx"] or {}).get("run_cluster_llm") is True + + +def test_full_preset_enables_llm(monkeypatch): + """full: el preset resuelve run_llm=True y activa la narrativa LLM en el ctx.""" + captured = {} + _patch_pipeline_internals(monkeypatch, captured) + + captured.clear() + render_automatic_eda("db", "t", profile_level="full") + assert captured["run_llm"] is True + assert (captured["base_ctx"] or {}).get("run_cluster_llm") is True + + +def test_no_profile_level_defaults_to_standard(monkeypatch): + """Retro-compat: sin profile_level ni flags, el comportamiento es el histórico. + + standard = run_models True, run_series True, run_llm False, sample 5000. Es el + mismo default que tenía el pipeline antes de introducir profile_level (cambio + aditivo: las llamadas existentes no cambian de comportamiento). + """ + captured = {} + _patch_pipeline_internals(monkeypatch, captured) + + captured.clear() + render_automatic_eda("db", "t") # sin profile_level ni flags de coste + assert captured["run_models"] is True + assert captured["run_series"] is True + assert captured["run_llm"] is False + assert captured["sample"] == 5000 + + +def test_lite_preset_defaults(monkeypatch): + """lite por defecto: run_llm/run_series False y sample reducido a 2000.""" + captured = {} + _patch_pipeline_internals(monkeypatch, captured) + + captured.clear() + render_automatic_eda("db", "t", profile_level="lite") + assert captured["run_llm"] is False + assert captured["run_series"] is False + assert captured["sample"] == 2000 From 48de3ce3dae050b0dc6d9547abccbdd2e661be32 Mon Sep 17 00:00:00 2001 From: Egutierrez Date: Tue, 30 Jun 2026 18:52:08 +0200 Subject: [PATCH 31/53] feat(eda): salida Markdown del AutomaticEDA para pegar a un LLM MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Añade un tercer formato de salida al AutomaticEDA, junto al PDF y el PPTX: un Markdown autocontenido del MISMO documento por capítulos (chapters_registry.build_document), optimizado para incorporar a un LLM (texto plano + tablas markdown reales, sin binarios incrustados). - render_md_impl.render_md(chapters, out_path, meta): serializa los bloques del modelo (Heading/Markdown/KVTable/DataTable/Figure/Image/Caption/Note/ Group/GlossaryEntry) a Markdown. Cabecera con metadatos + índice navegable con anclas GitHub; tablas volcadas enteras (el MD no pagina); marcadores de glosario eliminados conservando la negrita; glosario al final. - Figuras: un LLM no ve la imagen, así que se prioriza texto + datos. Se emite el caption y, cuando la figura tiene barras (histograma), se extrae la tabla de bins (Desde/Hasta/Frecuencia) de los artistas matplotlib. La banda ±1σ (axvspan) se descarta por ancho para que no aparezca como un falso bin. PNG opcional vía meta['embed_figures'] (off por defecto → sin binarios). - render_automatic_eda_markdown: función pública del registry (tag eda), espejo de render_automatic_eda_pdf/pptx, acepta lista de capítulos o un TableProfile (build_document). dict-no-throw. - render_automatic_eda (pipeline): emite también el .md (emit_md=True por defecto, clave de retorno aeda_md_path). Cambio aditivo: PDF/PPTX/manifest siguen saliendo igual. Tests: golden de todos los kinds + regresión del filtro de la banda ±1σ + edge documento vacío + profile path. Suite del paquete y del pipeline verde (122 passed). Co-Authored-By: Claude Opus 4.8 (1M context) --- python/functions/datascience/__init__.py | 2 + .../datascience/automatic_eda/__init__.py | 2 + .../automatic_eda/render_md_impl.py | 458 ++++++++++++++++++ .../render_automatic_eda_markdown.md | 89 ++++ .../render_automatic_eda_markdown.py | 55 +++ .../render_automatic_eda_markdown_test.py | 168 +++++++ .../pipelines/render_automatic_eda.py | 39 +- 7 files changed, 805 insertions(+), 8 deletions(-) create mode 100644 python/functions/datascience/automatic_eda/render_md_impl.py create mode 100644 python/functions/datascience/render_automatic_eda_markdown.md create mode 100644 python/functions/datascience/render_automatic_eda_markdown.py create mode 100644 python/functions/datascience/render_automatic_eda_markdown_test.py diff --git a/python/functions/datascience/__init__.py b/python/functions/datascience/__init__.py index f1505d22..cdefab14 100644 --- a/python/functions/datascience/__init__.py +++ b/python/functions/datascience/__init__.py @@ -64,6 +64,7 @@ from .exploratory_caveats import exploratory_caveats from .render_eda_pdf import render_eda_pdf, render_eda_pdf_relational from .render_automatic_eda_pdf import render_automatic_eda_pdf from .render_automatic_eda_pptx import render_automatic_eda_pptx +from .render_automatic_eda_markdown import render_automatic_eda_markdown from .detect_time_column import detect_time_column from .extract_timeseries_raw import extract_timeseries_raw from .build_eda_render_ctx import build_eda_render_ctx @@ -82,6 +83,7 @@ __all__ = [ "resample_timeseries", "render_automatic_eda_pdf", "render_automatic_eda_pptx", + "render_automatic_eda_markdown", "decode_qr_image", "adf_kpss_stationarity", "acf_pacf", diff --git a/python/functions/datascience/automatic_eda/__init__.py b/python/functions/datascience/automatic_eda/__init__.py index f9a6f2e3..01085313 100644 --- a/python/functions/datascience/automatic_eda/__init__.py +++ b/python/functions/datascience/automatic_eda/__init__.py @@ -36,6 +36,7 @@ from .model import ( # noqa: F401 from .chapters_registry import CHAPTER_ORDER, build_chapter, build_document # noqa: F401 from .render_pdf_impl import render_pdf # noqa: F401 from .render_pptx_impl import render_pptx # noqa: F401 +from .render_md_impl import render_md # noqa: F401 __all__ = [ "ENGINE_NAME", @@ -60,4 +61,5 @@ __all__ = [ "build_document", "render_pdf", "render_pptx", + "render_md", ] diff --git a/python/functions/datascience/automatic_eda/render_md_impl.py b/python/functions/datascience/automatic_eda/render_md_impl.py new file mode 100644 index 00000000..fba8ba6f --- /dev/null +++ b/python/functions/datascience/automatic_eda/render_md_impl.py @@ -0,0 +1,458 @@ +"""AutomaticEDA Markdown serializer — one self-contained file to paste to an LLM. + +Same document model as the PDF/PPTX renderers (an ordered list of +:class:`Chapter`, each a list of format-independent blocks) but emitted as plain +**Markdown** instead of a binary. The goal is different from the other two +renderers: a Markdown EDA is meant to be *pasted into an LLM*, so it prioritises +TEXT and DATA over visuals. Tables become Markdown tables (every row dumped, no +pagination — nothing is cut because there are no pages); a ``Figure`` becomes its +caption plus, when possible, the underlying bar/histogram data as a Markdown +table (an LLM cannot see the image); glossary term markers are stripped while +``**bold**`` is kept (it is valid Markdown). + +dict-no-throw (the ``eda`` group style): :func:`render_md` never raises. On a +fatal error it returns ``{path: None, ...}`` with a ``note`` explaining why; a +malformed block degrades to a readable note rather than crashing the document. +""" + +from __future__ import annotations + +import os +import re + +from . import model + +# Glossary span markers (kept text, dropped markers). We intentionally do NOT use +# ``text_layout.strip_inline_md`` for Markdown blocks because that also removes +# ``**bold**`` — valid Markdown we want to preserve when pasting to an LLM. +_TERM_OPEN_RE = re.compile(r"\[\[term:[A-Za-z0-9_]+\]\]") +_MAX_BAR_ROWS = 100 + + +# --------------------------------------------------------------------------- # +# Small helpers. +# --------------------------------------------------------------------------- # +def _clean_terms(s) -> str: + """Drop glossary term markers, keeping the visible text (and any **bold**).""" + s = model._safe_str(s) + s = _TERM_OPEN_RE.sub("", s) + return s.replace("[[/term]]", "") + + +def _cell(v) -> str: + """Render a value as a safe Markdown table cell. + + Escapes pipes (``|`` -> ``\\|``) so they do not break the column layout and + folds newlines to ``
`` so a multi-line value stays inside one cell. None + becomes an empty string. + """ + s = model._safe_str(v) + s = s.replace("|", "\\|") + s = s.replace("\r\n", "\n").replace("\r", "\n").replace("\n", "
") + return s + + +def _slug(text: str) -> str: + """GitHub-style heading anchor: lowercase, spaces->'-', drop other symbols.""" + s = model._safe_str(text).strip().lower() + out = [] + for ch in s: + if ch.isalnum(): + out.append(ch) + elif ch in " -": + out.append("-") + # any other symbol is dropped. + slug = "".join(out) + while "--" in slug: + slug = slug.replace("--", "-") + return slug.strip("-") + + +def _fmt_num(v) -> str: + """Compact number for the figure data tables (ints as ints, else 4 sig figs).""" + try: + f = float(v) + except Exception: # noqa: BLE001 + return model._safe_str(v) + if f != f: # NaN + return "NaN" + if f == int(f) and abs(f) < 1e15: + return str(int(f)) + return f"{f:.4g}" + + +def _fmt_int(v) -> str: + try: + return str(int(v)) + except Exception: # noqa: BLE001 + return model._safe_str(v) + + +def _now_iso() -> str: + from datetime import datetime, timezone + return datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC") + + +# --------------------------------------------------------------------------- # +# Document header (title + metadata blockquote + numbered index). +# --------------------------------------------------------------------------- # +def _meta_block(meta: dict) -> list: + """Build the metadata lines for the header blockquote (omitting absentees).""" + ctx = meta.get("ctx") if isinstance(meta.get("ctx"), dict) else {} + lines: list = [] + + def add(label, value) -> None: + if value is None: + return + s = model._safe_str(value).strip() + if s and s.lower() != "none": + lines.append(f"**{label}:** {s}") + + add("Dataset", ctx.get("dataset_name") or meta.get("dataset_name")) + add("Fuente", ctx.get("source_origin") or meta.get("source_origin")) + add("Almacenamiento", ctx.get("storage") or meta.get("storage")) + n_rows = ctx.get("n_rows", meta.get("n_rows")) + n_cols = ctx.get("n_cols", meta.get("n_cols")) + if n_rows is not None and n_cols is not None: + lines.append( + f"**Dimensiones:** {_fmt_int(n_rows)} filas × {_fmt_int(n_cols)} columnas") + add("Generado", meta.get("generated_at") or _now_iso()) + lines.append(f"**Motor:** {model.ENGINE_NAME} v{model.ENGINE_VERSION}") + return lines + + +# --------------------------------------------------------------------------- # +# Per-block serializers. Each returns a Markdown string (no surrounding blanks; +# the caller separates blocks with a blank line). +# --------------------------------------------------------------------------- # +def _md_heading(block) -> str: + level = int(getattr(block, "level", 1) or 1) + hashes = "#" * min(level + 2, 6) # level1 -> ###; '#'/'##' reserved for doc/chapter. + text = _clean_terms(getattr(block, "text", "")).strip() + return f"{hashes} {text}" + + +def _md_markdown(block) -> str: + # Keep the text verbatim, dropping only glossary markers (keep **bold**). + return _clean_terms(getattr(block, "text", "")).rstrip("\n") + + +def _md_kv_table(block) -> str: + lines: list = [] + title = getattr(block, "title", None) + if title: + lines.append(f"**{_clean_terms(title).strip()}**") + lines.append("") + lines.append("| Campo | Valor |") + lines.append("| --- | --- |") + for row in (getattr(block, "rows", []) or []): + try: + label, value = row[0], row[1] + except Exception: # noqa: BLE001 + label, value = row, "" + lines.append(f"| {_cell(label)} | {_cell(value)} |") + return "\n".join(lines) + + +def _md_data_table(block) -> str: + lines: list = [] + title = getattr(block, "title", None) + if title: + lines.append(f"**{_clean_terms(title).strip()}**") + lines.append("") + header = list(getattr(block, "header", []) or []) + rows = list(getattr(block, "rows", []) or []) + if not header: + ncol = max((len(r) for r in rows), default=1) + header = [f"col{i + 1}" for i in range(ncol)] + ncol = len(header) + lines.append("| " + " | ".join(_cell(h) for h in header) + " |") + lines.append("| " + " | ".join(["---"] * ncol) + " |") + for r in rows: # dump every row — no pagination, nothing cut. + cells = [_cell(r[c]) if c < len(r) else "" for c in range(ncol)] + lines.append("| " + " | ".join(cells) + " |") + note = getattr(block, "note", None) + if note: + lines.append("") + lines.append(f"*{_clean_terms(note).strip()}*") + return "\n".join(lines) + + +def _bars_table(bars: list) -> str: + """Render extracted bar/histogram data as a Markdown table (Desde/Hasta/Frec).""" + lines = ["| Desde | Hasta | Frecuencia |", "| --- | --- | --- |"] + shown = bars[:_MAX_BAR_ROWS] + for x0, x1, h in shown: + lines.append(f"| {_fmt_num(x0)} | {_fmt_num(x1)} | {_fmt_num(h)} |") + out = "\n".join(lines) + extra = len(bars) - len(shown) + if extra > 0: + out += f"\n\n*… ({extra} filas más)*" + return out + + +def _extract_bars(fig) -> list: + """Collect (x_from, x_to, height) of the rectangular bars of a matplotlib fig. + + Histogram / bar-chart bars are ``matplotlib.patches.Rectangle`` with positive + width and height; spines, legends and zero-area artists are skipped. Never + raises — returns ``[]`` on any problem. + """ + bars: list = [] + try: + for ax in fig.get_axes(): + # Collect this axes' positive-area rectangles, then keep only the ones + # that look like actual histogram/bar bins. Reference shapes that + # matplotlib also stores in ``ax.patches`` — most notably the ``±1σ`` + # band drawn by ``axvspan`` (a single rectangle far wider than a bin) + # and a lone Tukey boxplot box — would otherwise show up as fake + # "bins". A histogram axes has several near-equal-width bars, so we + # drop any rectangle whose width is more than twice the median width + # of that axes' rectangles (the σ-band spans many bins; uniform bins + # all sit at the median width and stay). + ax_bars: list = [] + for patch in list(getattr(ax, "patches", []) or []): + try: + w = patch.get_width() + h = patch.get_height() + x = patch.get_x() + except Exception: # noqa: BLE001 — not a Rectangle-like patch. + continue + if w and w > 0 and h and h > 0: + ax_bars.append((x, x + w, h)) + if len(ax_bars) >= 3: + widths = sorted(b[1] - b[0] for b in ax_bars) + median_w = widths[len(widths) // 2] + if median_w > 0: + ax_bars = [b for b in ax_bars + if (b[1] - b[0]) <= 2.0 * median_w] + bars.extend(ax_bars) + except Exception: # noqa: BLE001 + return [] + return bars + + +def _md_figure(block, meta: dict, out_path: str, counter: list) -> str: + """Serialize a Figure prioritising TEXT + DATA (an LLM cannot see the image). + + Emits the caption, then — if the matplotlib figure has bars — a Markdown table + of the underlying (Desde, Hasta, Frecuencia) values. Optionally (when + ``meta['embed_figures']`` is True) also exports a PNG beside the .md and adds + an image link; off by default so the Markdown stays self-contained. + """ + caption = model._safe_str(getattr(block, "caption", "")).strip() + parts = [f"*Figura: {caption}*" if caption else "*Figura*"] + fig = None + try: + import matplotlib + matplotlib.use("Agg") # defensive: headless rasterization backend. + fig = getattr(block, "fig", None) + make = getattr(block, "make", None) + if fig is None and callable(make): + fig = make() + if fig is not None: + bars = _extract_bars(fig) + if bars: + parts.append(_bars_table(bars)) + if meta.get("embed_figures"): + png = _embed_png(fig, out_path, counter) + if png: + parts.append(f"![{caption}]({png})") + except Exception: # noqa: BLE001 — a bad figure degrades to just its caption. + pass + finally: + if fig is not None: + try: + import matplotlib.pyplot as plt + plt.close(fig) + except Exception: # noqa: BLE001 + pass + return "\n\n".join(parts) + + +def _embed_png(fig, out_path: str, counter: list) -> str: + """Export the figure to ``_figN.png`` beside the .md; return its name.""" + try: + counter[0] += 1 + base = os.path.splitext(os.path.basename(out_path))[0] or "figura" + name = f"{base}_fig{counter[0]}.png" + path = os.path.join(os.path.dirname(os.path.abspath(out_path)), name) + fig.savefig(path, format="png", dpi=120, bbox_inches="tight") + return name + except Exception: # noqa: BLE001 + return "" + + +def _md_image(block) -> str: + path = model._safe_str(getattr(block, "path", "")) + caption = model._safe_str(getattr(block, "caption", "")).strip() + out = f"![{caption}]({path})" + if caption: + out += f"\n\n*{caption}*" + return out + + +def _md_caption(block) -> str: + return f"*{_clean_terms(getattr(block, 'text', '')).strip()}*" + + +def _md_note(block) -> str: + text = _clean_terms(getattr(block, "text", "")).strip() + lines = text.split("\n") + return "\n".join((f"> {ln}" if ln.strip() else ">") for ln in lines) + + +def _md_group(block, meta: dict, out_path: str, counter: list) -> str: + parts: list = [] + title = getattr(block, "title", None) + if title: + parts.append(f"### {_clean_terms(title).strip()}") + for b in (getattr(block, "blocks", []) or []): + try: + seg = _serialize_block(b, meta, out_path, counter) + except Exception: # noqa: BLE001 + seg = "" + if seg: + parts.append(seg) + return "\n\n".join(parts) + + +def _md_glossary_entry(block) -> str: + label = (model._safe_str(getattr(block, "label", "")).strip() + or model._safe_str(getattr(block, "key", "")).strip()) + definition = _clean_terms(getattr(block, "definition", "")).strip() + out = f"### {label}" + if definition: + out += f"\n\n{definition}" + return out + + +def _serialize_block(block, meta: dict, out_path: str, counter: list) -> str: + """Dispatch a single block to its Markdown serializer. Unknown -> note.""" + kind = getattr(block, "kind", "") + if kind == "heading": + return _md_heading(block) + if kind == "markdown": + return _md_markdown(block) + if kind == "kv_table": + return _md_kv_table(block) + if kind == "data_table": + return _md_data_table(block) + if kind == "figure": + return _md_figure(block, meta, out_path, counter) + if kind == "image": + return _md_image(block) + if kind == "caption": + return _md_caption(block) + if kind == "note": + return _md_note(block) + if kind == "group": + return _md_group(block, meta, out_path, counter) + if kind == "glossary_entry": + return _md_glossary_entry(block) + # Unknown content -> readable note (mirrors the model's defensive coercion). + return _md_note(model.Note(text=model._safe_str(block))) + + +# --------------------------------------------------------------------------- # +# Entry point. +# --------------------------------------------------------------------------- # +def render_md(chapters: list, out_path: str, meta: dict = None) -> dict: + """Serialize a list of Chapters into a single self-contained Markdown file. + + The output leads with ``# ``, a metadata blockquote and a numbered + ``## Índice`` linking each chapter, then one ``## N. <title>`` section per + chapter with its blocks. Tables become Markdown tables (every row dumped), + figures become caption + underlying data table, glossary markers are stripped + while ``**bold**`` is kept. Designed to be pasted into an LLM. + + Args: + chapters: a list of ``Chapter`` (dataclasses or dicts); normalized + defensively with ``model.as_chapters``. + out_path: filesystem path for the ``.md`` (parent dirs are created). + meta: optional dict. Recognised keys: ``title``, ``ctx`` (dict with + ``dataset_name``/``source_origin``/``storage``/``n_rows``/``n_cols``), + ``generated_at``, ``embed_figures`` (export PNGs beside the .md, + default False). + + Returns: + dict (never raises): ``{path: str|None, n_chars: int, + chapters: list[{id, version}], note: str}``. On a fatal error ``path`` is + None and ``note`` explains why. + """ + meta = meta or {} + chapters = model.as_chapters(chapters) + title = model._safe_str(meta.get("title")) or model.ENGINE_NAME + + # Edge: nothing to render -> a minimal but valid Markdown document. + if not chapters: + content = (f"# {title}\n\n" + "*(documento vacío — sin capítulos aplicables)*\n") + return _write(out_path, content, [], "documento vacío") + + counter = [0] # document-wide figure counter for unique PNG names. + notes: list = [] + segments: list = [f"# {title}"] + + meta_lines = _meta_block(meta) + if meta_lines: + segments.append("\n".join(f"> {ln}" for ln in meta_lines)) + + # Numbered index. The anchor matches the chapter heading emitted below + # (``## N. <title>``) in GitHub slug style. + chap_heads = [] + idx_lines = ["## Índice"] + for i, ch in enumerate(chapters, 1): + head_text = f"{i}. {model._safe_str(ch.title)}" + anchor = _slug(head_text) + chap_heads.append((head_text, anchor)) + idx_lines.append(f"{i}. [{model._safe_str(ch.title)}](#{anchor})") + segments.append("\n".join(idx_lines)) + + chapters_meta = [] + for i, ch in enumerate(chapters, 1): + segments.append("---") + head_text, _anchor = chap_heads[i - 1] + segments.append(f"## {head_text}") + + blocks = list(ch.blocks or []) + # Omit a leading level-1 Heading that just repeats the chapter title. + if blocks: + b0 = blocks[0] + if (getattr(b0, "kind", "") == "heading" + and int(getattr(b0, "level", 1) or 1) == 1 + and _clean_terms(getattr(b0, "text", "")).strip() + == model._safe_str(ch.title).strip()): + blocks = blocks[1:] + + for block in blocks: + try: + seg = _serialize_block(block, meta, out_path, counter) + except Exception as e: # noqa: BLE001 + seg = _md_note(model.Note(text=model._safe_str(block))) + notes.append( + f"bloque '{getattr(block, 'kind', '?')}' del capítulo " + f"'{ch.id}' degradado: {e}") + if seg: + segments.append(seg) + chapters_meta.append({"id": ch.id, "version": ch.version}) + + content = "\n\n".join(segments) + "\n" + note = f"{len(content)} caracteres" + if notes: + note += " · " + "; ".join(notes) + return _write(out_path, content, chapters_meta, note) + + +def _write(out_path: str, content: str, chapters_meta: list, note: str) -> dict: + """Write the Markdown to disk (creating parents). dict-no-throw.""" + try: + parent = os.path.dirname(os.path.abspath(out_path)) + os.makedirs(parent, exist_ok=True) + with open(out_path, "w", encoding="utf-8") as fh: + fh.write(content) + except Exception as e: # noqa: BLE001 — never raise from the writer. + return {"path": None, "n_chars": 0, "chapters": [], + "note": f"no se pudo escribir el Markdown: {e}"} + return {"path": out_path, "n_chars": len(content), + "chapters": chapters_meta, "note": note} diff --git a/python/functions/datascience/render_automatic_eda_markdown.md b/python/functions/datascience/render_automatic_eda_markdown.md new file mode 100644 index 00000000..6615baf9 --- /dev/null +++ b/python/functions/datascience/render_automatic_eda_markdown.md @@ -0,0 +1,89 @@ +--- +name: render_automatic_eda_markdown +kind: function +lang: py +domain: datascience +version: "1.0.0" +purity: impure +signature: "def render_automatic_eda_markdown(chapters_or_profile, out_path: str, meta: dict = None) -> dict" +description: "Renderiza un documento AutomaticEDA por CAPÍTULOS (modelo de bloques independiente del formato) en un único MARKDOWN autocontenido pensado para PEGAR A UN LLM. Acepta una lista de capítulos del modelo o directamente un TableProfile del grupo eda (construye los capítulos canónicos con build_document). Prioriza TEXTO + DATOS sobre lo visual: las tablas se vuelcan como tablas markdown con TODAS las filas (sin paginar — no hay páginas que cortar), una figura matplotlib se reduce a su caption más la tabla de datos subyacente (Desde/Hasta/Frecuencia de las barras del histograma) porque un LLM no ve la imagen, y los marcadores de glosario se eliminan conservando el **negrita**. Lleva cabecera (# título), bloque de metadatos en blockquote e índice numerado con anclas GitHub. Espejo de render_automatic_eda_pdf/render_automatic_eda_pptx pero SIN manifest (KISS, el markdown es un único artefacto de texto). dict-no-throw: nunca lanza, devuelve {path, n_chars, chapters, note}; en error fatal path es None y note explica la causa. Flag opcional meta['embed_figures'] exporta PNGs junto al .md (off por defecto)." +tags: [eda, markdown, render, report, llm, automatic-eda, chapters, versioned, no-cut, text, datascience, python] +uses_functions: [] +uses_types: [] +returns: [] +returns_optional: false +error_type: "error_go_core" +imports: [os, re, matplotlib, "datascience.automatic_eda"] +params: + - name: chapters_or_profile + desc: "una lista de capítulos del modelo AutomaticEDA (dataclasses Chapter o dicts {id,title,version,blocks}) O un TableProfile dict del grupo eda. Si es un TableProfile, los capítulos canónicos se construyen con build_document(profile, meta['ctx']). Bloques soportados: heading, markdown, kv_table, data_table, figure, image, caption, note, group, glossary_entry. Lectura defensiva: lo no reconocido se degrada a Note, nunca lanza." + - name: out_path + desc: "ruta del archivo .md de salida. Los directorios padre se crean si faltan. Directorio no escribible → {path:None, note:<causa>} sin lanzar." + - name: meta + desc: "dict opcional. Claves: title (título del documento), ctx (dict con dataset_name→Dataset, source_origin→Fuente, storage→Almacenamiento, n_rows/n_cols→Dimensiones; también lo consumen los builders de capítulo cuando se da un profile), generated_at (timestamp; si falta se genera ISO UTC), embed_figures (True para exportar PNGs <basename>_figN.png junto al .md; por defecto False y el markdown queda autocontenido)." +output: "dict (nunca lanza): {path: str|None, n_chars: int, chapters: list[{id,version}], note: str}. En error fatal (p.ej. directorio no escribible) path es None y note explica la causa. Un documento sin capítulos aplicables produce un markdown mínimo válido con 'documento vacío' y chapters=[]." +tested: true +tests: ["test_golden_bloques_sinteticos_serializa_todo_a_markdown", "test_edge_documento_vacio_no_revienta", "test_profile_path_construye_capitulos_y_escribe"] +test_file_path: "python/functions/datascience/render_automatic_eda_markdown_test.py" +file_path: "python/functions/datascience/render_automatic_eda_markdown.py" +--- + +## Ejemplo + +```python +from datascience import render_automatic_eda_markdown + +# Desde un TableProfile del grupo eda (mismo modelo que los renderers PDF/PPTX). +profile = { + "table": "ventas", "source": "/data/ventas.csv", + "n_rows": 1000, "n_cols": 2, "quality_score": 92.5, + "columns": [ + {"name": "precio", "inferred_type": "numeric", "null_pct": 0.01, + "numeric": {"mean": 42.5, "median": 40.0, "min": 1.0, "max": 100.0, + "std": 12.3}}, + {"name": "categoria", "inferred_type": "categorical", "null_pct": 0.0, + "categorical": {"top": [{"value": "neumaticos", "count": 500}]}}, + ], +} +res = render_automatic_eda_markdown( + profile, "reports/ventas_aeda.md", + {"title": "EDA — ventas", + "ctx": {"dataset_name": "Ventas", "source_origin": "ERP export", + "n_rows": 1000, "n_cols": 2}}) +print(res["path"], res["n_chars"], res["chapters"]) +# -> reports/ventas_aeda.md 4123 [{'id':'portada','version':'1.0.0'}, ...] +``` + +## Cuando usarla + +Cuando quieras **pegar el EDA a un LLM** (ChatGPT, Claude, ...) o tenerlo en texto +plano versionable: mismo documento por capítulos que el PDF/PPTX, pero serializado a +Markdown sin binarios. Úsala como tercera salida junto a `render_automatic_eda_pdf` +(móvil) y `render_automatic_eda_pptx` (compartir) desde el MISMO modelo de capítulos. +A diferencia de esas dos, no hay páginas ni slides: todas las filas de cada tabla se +vuelcan (nada se corta) y cada figura se reduce a su caption + la tabla de datos +subyacente, que es lo que un LLM puede leer. Para añadir capítulos al documento, ver +`docs/capabilities/automatic_eda.md`. + +## Gotchas + +- **Impura**: escribe el `.md` en `out_path` (crea los directorios padre). Con + `meta['embed_figures']=True` además exporta un PNG `<basename>_figN.png` por figura + junto al `.md`; por defecto NO exporta nada y el markdown queda autocontenido. +- **Nunca lanza** (dict-no-throw): un bloque que falle se degrada a una nota y se anota + en `note`; el documento se escribe igual. Un profile/lista vacíos producen un markdown + mínimo válido con `*(documento vacío …)*` y `chapters=[]`. +- **Figuras = datos, no imagen**: un bloque `figure` se serializa como `*Figura: caption*` + más, si la figura matplotlib trae barras (histograma / barras), una tabla + `| Desde | Hasta | Frecuencia |` extraída de los `Rectangle` patches (máx 100 filas; + el resto se trunca con `*… (N filas más)*`). Si no hay barras o algo falla, solo sale + el caption. La figura se cierra (`plt.close`) tras leerla. +- **Glosario vs negrita**: se eliminan SOLO los marcadores de glosario + `[[term:key]]visible[[/term]]` (queda `visible`); el `**negrita**` markdown SE + CONSERVA (es válido). No se usa `strip_inline_md` aquí porque ese también quita el bold. +- **Anclas del índice**: el `## Índice` enlaza cada capítulo con un ancla estilo GitHub + del encabezado `## N. Título` (minúsculas, espacios→`-`, sin signos). Si dos capítulos + comparten título exacto sus anclas colisionan (caso raro; los capítulos canónicos tienen + títulos únicos). +- **Tablas**: las celdas escapan `|` (→ `\|`) y pliegan saltos de línea a `<br>` para no + romper la columna. No hay reparto por ancho — un LLM no lo necesita. diff --git a/python/functions/datascience/render_automatic_eda_markdown.py b/python/functions/datascience/render_automatic_eda_markdown.py new file mode 100644 index 00000000..649b2cd1 --- /dev/null +++ b/python/functions/datascience/render_automatic_eda_markdown.py @@ -0,0 +1,55 @@ +"""render_automatic_eda_markdown — chapter-based EDA report as one Markdown file. + +Public ``eda``-group entry point that serializes an AutomaticEDA document (a list +of chapters, or an ``eda`` TableProfile from which the canonical chapters are +built) into a single self-contained Markdown file optimised to be **pasted into +an LLM**: plain text, Markdown tables (every row dumped — there are no pages to +cut), figures reduced to caption + underlying data, no binaries. It mirrors +``render_automatic_eda_pdf`` / ``render_automatic_eda_pptx`` but for text output; +unlike those it writes no manifest (KISS — Markdown is a single text artefact). + +dict-no-throw: never raises. Returns ``{path, n_chars, chapters, note}``; on a +fatal error ``path`` is None and ``note`` explains why. +""" + +from __future__ import annotations + +from datascience.automatic_eda import build_document, render_md +from datascience.automatic_eda.model import as_chapter, as_chapters + + +def _coerce_chapters(chapters_or_profile, meta: dict) -> list: + """Accept chapters OR an eda profile and return a list of Chapter.""" + arg = chapters_or_profile + if isinstance(arg, (list, tuple)): + return as_chapters(list(arg)) + if isinstance(arg, dict): + if "blocks" in arg and "columns" not in arg: + ch = as_chapter(arg) + return [ch] if ch is not None else [] + return build_document(arg, (meta or {}).get("ctx")) + return [] + + +def render_automatic_eda_markdown(chapters_or_profile, out_path: str, + meta: dict = None) -> dict: + """Render an AutomaticEDA document into a single self-contained Markdown file. + + Args: + chapters_or_profile: a list of chapters (``Chapter`` dataclasses or + dicts) or an ``eda`` TableProfile dict (chapters built via + ``build_document(profile, meta['ctx'])``). + out_path: filesystem path for the ``.md`` (parent dirs are created). + meta: optional dict. Recognised keys: ``title``, ``ctx`` (dict with + ``dataset_name``/``source_origin``/``storage``/``n_rows``/``n_cols``), + ``generated_at``, ``embed_figures`` (export PNGs beside the .md, + default False — off keeps the Markdown self-contained). + + Returns: + dict (never raises): ``{path: str|None, n_chars: int, + chapters: list[{id, version}], note: str}``. On a fatal error ``path`` is + None and ``note`` explains the cause. + """ + meta = dict(meta or {}) + chapters = _coerce_chapters(chapters_or_profile, meta) + return render_md(chapters, out_path, meta) diff --git a/python/functions/datascience/render_automatic_eda_markdown_test.py b/python/functions/datascience/render_automatic_eda_markdown_test.py new file mode 100644 index 00000000..5d77ee10 --- /dev/null +++ b/python/functions/datascience/render_automatic_eda_markdown_test.py @@ -0,0 +1,168 @@ +"""Tests for render_automatic_eda_markdown — DoD: golden + edge + profile path. + +Self-contained synthetic blocks (no DuckDB). Verifies every block kind serializes +to Markdown (heading, markdown with glossary+bold, kv/data tables, a figure whose +histogram bars become a data table, caption, note, group, glossary entry), that a +leading level-1 heading equal to the chapter title is omitted, that an empty +document degrades to a valid minimal Markdown without raising, and that passing a +minimal TableProfile builds chapters and writes the file. +""" + +import os +import tempfile + +from datascience.render_automatic_eda_markdown import render_automatic_eda_markdown +from datascience.automatic_eda.model import ( + Caption, Chapter, DataTable, Figure, GlossaryEntry, Group, Heading, KVTable, + Markdown, Note, +) + + +def _hist_fig(): + import matplotlib + matplotlib.use("Agg") + import matplotlib.pyplot as plt + fig, ax = plt.subplots() + ax.hist([1, 1, 2, 2, 2, 3, 4, 4, 5, 5, 5, 5], bins=5) + return fig + + +def _chapters() -> list: + blocks = [ + Heading("Demo", 1), # == chapter title -> omitted. + Heading("Seccion dos", 2), # -> #### + Markdown("Texto con [[term:ent]]entropia[[/term]] y **bold** aqui."), + KVTable(rows=[("Filas", 1000), ("Columnas", 5)], title="Resumen"), + DataTable(header=["col", "valor"], + rows=[["alpha", "111"], ["beta", "222"], ["gamma", "333"]], + title="Datos", note="nota inferior"), + Figure(make=_hist_fig, caption="Histograma demo"), + Caption("pie de figura"), + Note("una nota aparte"), + Group(title="Grupo X", blocks=[Markdown("dentro del grupo")]), + GlossaryEntry(key="ent", label="Entropia", + definition="Medida de incertidumbre."), + ] + return [Chapter(id="demo", title="Demo", version="1.0.0", blocks=blocks)] + + +def _read(path: str) -> str: + with open(path, "r", encoding="utf-8") as fh: + return fh.read() + + +def test_golden_bloques_sinteticos_serializa_todo_a_markdown(): + with tempfile.TemporaryDirectory() as d: + out = os.path.join(d, "demo.md") + res = render_automatic_eda_markdown( + _chapters(), out, + {"title": "EDA Demo", + "ctx": {"dataset_name": "Demo", "n_rows": 12, "n_cols": 2}}) + assert res["path"] == out + assert os.path.exists(out) + assert res["n_chars"] > 0 + assert res["chapters"] == [{"id": "demo", "version": "1.0.0"}] + + content = _read(out) + # Document structure. + assert content.startswith("# ") + assert "## Índice" in content + # A Markdown table is present (header + separator row). + assert "| " in content and "| --- " in content + # DataTable values are all dumped. + for v in ("alpha", "111", "beta", "222", "gamma", "333"): + assert v in content + # Glossary markers stripped, bold kept. + assert "[[term" not in content + assert "[[/term]]" not in content + assert "**bold**" in content + assert "entropia" in content # visible glossary text preserved. + # Figure histogram bars became a data table. + assert "| Desde | Hasta | Frecuencia |" in content + # Glossary entry rendered as a level-3 heading. + assert "### Entropia" in content + # Level-2 heading -> ####. + assert "#### Seccion dos" in content + # Leading level-1 heading equal to the title was omitted. + assert "### Demo" not in content + # Group title rendered. + assert "### Grupo X" in content + + +def _hist_fig_with_span(): + """Histogram with a wide ``axvspan`` (±1σ band) over it. + + Reproduces the num_distr figure shape: matplotlib keeps the span as a lone + Rectangle in ``ax.patches`` alongside the bin bars; it must NOT leak into the + extracted bins table as a fake bin (it is ~5x wider than a bin).""" + import matplotlib + matplotlib.use("Agg") + import matplotlib.pyplot as plt + fig, ax = plt.subplots() + data = [1, 1, 2, 2, 2, 3, 4, 4, 5, 5, 5, 5] + ax.hist(data, bins=5) + ax.axvspan(2.0, 4.0, alpha=0.2) # mean±σ band — a wide stray rectangle. + return fig + + +def test_figura_descarta_axvspan_de_la_tabla_de_bins(): + """The ±1σ band rectangle must not appear as a row in the bins table.""" + blocks = [Figure(make=_hist_fig_with_span, caption="Hist con banda")] + chapters = [Chapter(id="f", title="Fig", version="1.0.0", blocks=blocks)] + with tempfile.TemporaryDirectory() as d: + out = os.path.join(d, "fig.md") + render_automatic_eda_markdown(chapters, out, {"title": "T"}) + content = _read(out) + assert "| Desde | Hasta | Frecuencia |" in content + # Extract the rows of the bins table: lines between the header/separator + # and the next blank line. + lines = content.splitlines() + hi = next(i for i, ln in enumerate(lines) + if ln.startswith("| Desde | Hasta | Frecuencia |")) + rows = [] + for ln in lines[hi + 2:]: # skip header + separator + if not ln.startswith("|"): + break + rows.append(ln) + # 5 histogram bins, no extra wide span row. + assert len(rows) == 5, rows + # No row spans a width of ~2.0 (the axvspan from x=2 to x=4). + for ln in rows: + cells = [c.strip() for c in ln.strip("|").split("|")] + lo, hi_v = float(cells[0]), float(cells[1]) + assert (hi_v - lo) < 1.5, f"wide span leaked: {ln}" + + +def test_edge_documento_vacio_no_revienta(): + with tempfile.TemporaryDirectory() as d: + out = os.path.join(d, "empty.md") + res = render_automatic_eda_markdown([], out, {}) + assert res["path"] == out + assert os.path.exists(out) + assert res["chapters"] == [] + content = _read(out) + assert "documento vacío" in content + assert content.startswith("# ") + + +def test_profile_path_construye_capitulos_y_escribe(): + profile = { + "table": "mini", + "source": "/data/mini.csv", + "n_rows": 10, + "n_cols": 1, + "quality_score": 88.0, + "columns": [ + {"name": "x", "inferred_type": "numeric", "null_pct": 0.0, + "null_count": 0, + "numeric": {"mean": 1.0, "median": 1.0, "min": 0.0, "max": 2.0, + "std": 0.5}}, + ], + } + with tempfile.TemporaryDirectory() as d: + out = os.path.join(d, "mini.md") + res = render_automatic_eda_markdown( + profile, out, {"title": "Mini", "ctx": {"dataset_name": "Mini"}}) + assert res["path"] == out # not None — no exception, file written. + assert os.path.exists(out) + assert res["n_chars"] > 0 diff --git a/python/functions/pipelines/render_automatic_eda.py b/python/functions/pipelines/render_automatic_eda.py index 8090bc1f..5361b927 100644 --- a/python/functions/pipelines/render_automatic_eda.py +++ b/python/functions/pipelines/render_automatic_eda.py @@ -1,9 +1,10 @@ -"""render_automatic_eda — EDA completo one-shot: perfil → ctx → PDF + PPTX. +"""render_automatic_eda — EDA completo one-shot: perfil → ctx → PDF + PPTX + MD. Pipeline impuro del grupo de capacidad `eda`. Dada UNA tabla DuckDB (o -PostgreSQL), produce el informe AutomaticEDA COMPLETO en sus dos formatos a la -vez (PDF móvil A5 + PPTX 16:9) con los 11 capítulos POBLADOS, en una sola -llamada. Compone, sin reimplementar su lógica, cuatro funciones del registry: +PostgreSQL), produce el informe AutomaticEDA COMPLETO en sus tres formatos a la +vez (PDF móvil A5 + PPTX 16:9 + Markdown autocontenido para pegar a un LLM) con +los capítulos POBLADOS, en una sola llamada. Compone, sin reimplementar su +lógica, varias funciones del registry: - profile_table : perfila la tabla end-to-end (TableProfile agregado), opcionalmente con modelos baratos y análisis de serie. @@ -12,8 +13,11 @@ llamada. Compone, sin reimplementar su lógica, cuatro funciones del registry: modelos/geo, timeseries_raw para series, geo_points para el mapa, db_path/table para la agregación push-down). Sin él, esos capítulos degradan. - - render_automatic_eda_pdf : renderiza el documento por capítulos a PDF. - - render_automatic_eda_pptx : renderiza el mismo documento a PPTX. + - render_automatic_eda_pdf : renderiza el documento por capítulos a PDF. + - render_automatic_eda_pptx : renderiza el mismo documento a PPTX. + - render_automatic_eda_markdown : serializa el mismo documento a Markdown + autocontenido (texto + tablas markdown, sin + binarios) para incorporar a un LLM. El TableProfile agregado basta para portada/overview/distribuciones/calidad/ correlación, pero los capítulos `modelos`, `timeseries`, `geospatial` y @@ -32,6 +36,7 @@ from datetime import datetime, timezone from datascience import ( build_eda_render_ctx, + render_automatic_eda_markdown, render_automatic_eda_pdf, render_automatic_eda_pptx, run_eda_models, @@ -93,6 +98,7 @@ def render_automatic_eda( out_dir: str = "reports", basename: str = None, ctx_extra: dict = None, + emit_md: bool = True, ) -> dict: """Perfila una tabla y emite el informe AutomaticEDA completo (PDF + PPTX). @@ -140,13 +146,19 @@ def render_automatic_eda( ctx_extra: dict opcional con claves de presentación/contexto extra que se mezclan en el ctx (p.ej. dataset_name, description, source_origin). No pisan las claves de datos calculadas por build_eda_render_ctx. + emit_md: además del PDF y el PPTX, emite un Markdown autocontenido del + MISMO documento por capítulos (texto plano + tablas markdown, sin + binarios), pensado para pegar a un LLM. Default True. La ruta sale en + la clave de retorno ``aeda_md_path``. No altera las demás salidas. Returns: dict (nunca lanza). En éxito:: {"status": "ok", "pdf_path": str, "pptx_path": str, - "manifest_path": str|None, "n_pages": int, "n_slides": int, - "pdf_note": str, "pptx_note": str, "profile": <TableProfile>} + "aeda_md_path": str|None, "manifest_path": str|None, + "n_pages": int, "n_slides": int, "md_chars": int|None, + "pdf_note": str, "pptx_note": str, "md_note": str|None, + "profile": <TableProfile>} En error: {"status": "error", "error": str}. """ @@ -243,15 +255,26 @@ def render_automatic_eda( rpdf = render_automatic_eda_pdf(prof, pdf_path, meta) or {} rpptx = render_automatic_eda_pptx(prof, pptx_path, meta) or {} + # Salida Markdown autocontenida (mismo documento por capítulos) para + # pegar a un LLM. Aditiva: no afecta a PDF/PPTX/manifest. dict-no-throw. + rmd = {} + md_path = None + if emit_md: + md_path = os.path.join(out_dir, base + ".md") + rmd = render_automatic_eda_markdown(prof, md_path, meta) or {} + return { "status": "ok", "pdf_path": rpdf.get("path"), "pptx_path": rpptx.get("path"), + "aeda_md_path": rmd.get("path"), "manifest_path": rpdf.get("manifest_path"), "n_pages": rpdf.get("n_pages"), "n_slides": rpptx.get("n_slides"), + "md_chars": rmd.get("n_chars"), "pdf_note": rpdf.get("note"), "pptx_note": rpptx.get("note"), + "md_note": rmd.get("note"), "profile": prof, } except Exception as e: # noqa: BLE001 — dict-no-throw: degradar, nunca lanzar. From fd6326144465774a4c74e0e907ef9167bf49dace Mon Sep 17 00:00:00 2001 From: Egutierrez <egutierrez@dead.dd> Date: Tue, 30 Jun 2026 19:15:24 +0200 Subject: [PATCH 32/53] =?UTF-8?q?refactor(eda):=20quitar=20definiciones=20?= =?UTF-8?q?inline=20redundantes=20con=20el=20glosario=20en=205=20cap=C3=AD?= =?UTF-8?q?tulos?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ahora que el AutomaticEDA tiene un capítulo GLOSARIO con las definiciones de los términos técnicos (enganchados como links clicables desde el cuerpo), los capítulos calidad/correlacion/modelos/agregacion/relaciones ya no repiten inline esas explicaciones largas: se deja el TÉRMINO marcado (clicable, sigue saltando al glosario) y se elimina el párrafo/oración de definición redundante. Los HALLAZGOS y datos concretos del análisis se mantienen intactos; solo se quitan las definiciones generales que el glosario ya cubre. - calidad: _criteria_intro pasa de un bullet-list con las definiciones de completitud/validez/unicidad/calidad + fórmula renormalizada + párrafo de outliers a una frase que nombra las dimensiones, sus pesos (60/40) y el principio de outliers; los 4 términos siguen marcados. - modelos: la nota de normalización deja de explicar la fórmula del z-score; la intro de PCA ya no define "componentes ortogonales ordenados por varianza"; la de KMeans quita "rango −1 a 1: cuanto más alto..." (silhouette); la sección de Isolation Forest quita la descripción de árboles/cortes/umbral. Términos marcados intactos. - correlacion: la intro deja de describir cada método y consolida la duplicación signo/dirección; los 4 métodos + FDR siguen marcados. - agregacion: la intro quita la definición de pivot ("cruzan dos categóricas sobre una medida") y abrevia la selección de claves; groupby y pivot marcados. - relaciones: la intro y la sección de candidatas/inter-tabla quitan las definiciones de PK ("identifica cada fila"), FK ("referencian a otra tabla") y containment ("valores contenidos en la clave de otra"); pk/fk/cardinalidad/ containment siguen marcados. Verificado sobre el EDA de titanic (run_models + run_llm, 48 págs): los 23 link annotations término→glosario se conservan (PyMuPDF), el glosario mantiene las 20 definiciones, y el texto visible de los 5 capítulos baja un 34.7% en conjunto (calidad −67%, modelos −33%, relaciones −19%, agregacion −15%, correlacion −8%). Tests actualizados (calidad_test asertaba el texto viejo). Suite EDA + pipeline verde (118 passed). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> --- .../automatic_eda/chapters/agregacion.py | 12 +++-- .../automatic_eda/chapters/calidad.py | 45 ++++++++----------- .../automatic_eda/chapters/calidad_test.py | 8 ++-- .../automatic_eda/chapters/correlacion.py | 11 +++-- .../automatic_eda/chapters/modelos.py | 44 ++++++++---------- .../automatic_eda/chapters/relaciones.py | 35 +++++++-------- 6 files changed, 69 insertions(+), 86 deletions(-) diff --git a/python/functions/datascience/automatic_eda/chapters/agregacion.py b/python/functions/datascience/automatic_eda/chapters/agregacion.py index c6eafcf8..ca29a4e3 100644 --- a/python/functions/datascience/automatic_eda/chapters/agregacion.py +++ b/python/functions/datascience/automatic_eda/chapters/agregacion.py @@ -561,13 +561,11 @@ def _intro_blocks(gloss=None, mark_term: bool = False) -> list: t_groupby = _term(mark_term, "groupby", "**por grupos** (split-apply-combine)") t_pivot = _term(mark_term, "pivot_table", "**tablas dinámicas** (pivot)") text = ( - f"Este capítulo analiza la tabla {t_groupby}: " - "elige las columnas categóricas más informativas — por su cardinalidad " - "y relevancia, no todas contra todas, para no inflar comparaciones " - "espurias — y resume las variables numéricas dentro de cada grupo " - f"(conteo, media, mediana, desviación). Las {t_pivot} " - "cruzan dos categóricas sobre una medida, y los **gráficos de barras** " - "(siempre desde cero) comparan los grupos de un vistazo." + f"Este capítulo analiza la tabla {t_groupby}: elige las columnas " + "categóricas más informativas (por cardinalidad y relevancia, no todas " + "contra todas) y resume las variables numéricas dentro de cada grupo " + f"(conteo, media, mediana, desviación). Se añaden {t_pivot} y " + "**gráficos de barras** (siempre desde cero) para comparar los grupos." ) return [model.Heading(text=CHAPTER_TITLE, level=1), model.Markdown(text=text)] diff --git a/python/functions/datascience/automatic_eda/chapters/calidad.py b/python/functions/datascience/automatic_eda/chapters/calidad.py index 93ef9423..aaaebcaf 100644 --- a/python/functions/datascience/automatic_eda/chapters/calidad.py +++ b/python/functions/datascience/automatic_eda/chapters/calidad.py @@ -3,12 +3,13 @@ Builds the quality chapter from a ``TableProfile`` of the ``eda`` group. The chapter implements the quality model of report 2046: -1. **En qué se basa la calidad** — an intro paragraph explaining the two scored +1. **En qué se basa la calidad** — a concise intro naming the two scored dimensions and their weights (completitud 60%, validez 40%) plus the - table-level row uniqueness, BEFORE any number, and stating explicitly that - outliers are reported as observations and do **not** lower the score. The - criteria terms (calidad de datos, completitud, validez, unicidad de registro) - are hooked into the shared glossary as clickable jumps. + table-level row uniqueness, BEFORE any number, and stating that outliers are + reported as observations and do **not** lower the score. The criteria terms + (calidad de datos, completitud, validez, unicidad de registro) are hooked + into the shared glossary as clickable jumps; their full definitions live in + the GLOSARIO chapter, not inline here. 2. **Scores por columna** — a table with, per column, the total quality score and its breakdown into completeness / validity (no consistency dimension). 3. **Problemas de calidad** — a table listing ONLY real quality defects @@ -309,30 +310,22 @@ def _term(key: str, label: str, mark: bool) -> str: def _criteria_intro(mark: bool) -> str: - """Intro paragraph explaining the two scored dimensions and the principle.""" + """Intro: how the score is composed, with every term marked clickable. + + Concise on purpose: the definitions of each term (calidad de datos, + completitud, validez, unicidad de registro) now live in the GLOSARIO + chapter, so the body no longer repeats them — it only states how the score + is composed and keeps each term marked so it stays a clickable jump. + """ calidad = _term("calidad_datos", "calidad de datos", mark) - completitud = _term("completitud", "Completitud (peso 60%)", mark) - validez = _term("validez", "Validez (peso 40%, cuando es medible)", mark) + completitud = _term("completitud", "completitud", mark) + validez = _term("validez", "validez", mark) unicidad = _term("unicidad_registro", "unicidad de registro", mark) return ( - f"La {calidad} de cada columna es un score de 0 a 100 que combina solo " - "dimensiones medibles desde el perfil de la tabla, sin fuente externa " - "de verdad:\n\n" - f"- {completitud}: proporción de valores presentes (1 − % de nulos; en " - "texto, las celdas vacías cuentan como faltantes). Los nulos y vacíos " - "bajan el score.\n" - f"- {validez}: proporción de valores que encajan con su tipo o formato " - "(un número que parsea, una fecha legible, un email con forma de email). " - "Si una columna es texto libre sin formato esperado, la validez no se " - "mide y el score se basa solo en la completitud.\n\n" - f"Score de columna = 100 × (0,6·completitud + 0,4·validez), " - "renormalizado cuando la validez no aplica. A nivel de tabla se añade " - f"la {unicidad} (1 − % de filas duplicadas).\n\n" - "**Los valores atípicos (outliers) NO bajan la calidad.** Un valor " - "extremo puede ser real y correcto; detectar atípicos es parte del " - "análisis de la distribución, no un juicio de corrección. Por eso, junto " - "con las columnas constantes y los identificadores, se listan aparte " - "como **observaciones analíticas** que no afectan al score." + f"La {calidad} de cada columna es un score de 0 a 100 que combina " + f"{completitud} (peso 60%) y {validez} (peso 40%, cuando es medible); " + f"a nivel de tabla se añade la {unicidad}. Los valores atípicos no " + "bajan el score: se listan aparte como **observaciones analíticas**." ) diff --git a/python/functions/datascience/automatic_eda/chapters/calidad_test.py b/python/functions/datascience/automatic_eda/chapters/calidad_test.py index 5dc623ee..5a31b497 100644 --- a/python/functions/datascience/automatic_eda/chapters/calidad_test.py +++ b/python/functions/datascience/automatic_eda/chapters/calidad_test.py @@ -72,14 +72,16 @@ def test_golden_chapter_estructura_y_version(): assert "markdown" in kinds and "kv_table" in kinds and "data_table" in kinds -def test_golden_intro_explica_dos_dimensiones_y_pesos(): +def test_golden_intro_nombra_dos_dimensiones_y_pesos(): + # La intro nombra las dos dimensiones, sus pesos y la unicidad, pero ya NO + # repite sus definiciones largas: estas viven ahora en el capítulo GLOSARIO. ch = build_calidad(_profile(), {}) intro = [b for b in ch.blocks if b.kind == "markdown"][0].text - for needle in ("Completitud", "Validez", "60%", "40%", + for needle in ("completitud", "validez", "60%", "40%", "unicidad de registro"): assert needle in intro, f"falta {needle!r} en la intro de criterios" # El principio: los outliers NO bajan la calidad. - assert "atípicos" in intro and "NO bajan" in intro + assert "atípicos" in intro and "no bajan" in intro # Ya no se menciona la dimensión consistencia eliminada. assert "20%" not in intro diff --git a/python/functions/datascience/automatic_eda/chapters/correlacion.py b/python/functions/datascience/automatic_eda/chapters/correlacion.py index 0c906cc9..cd559323 100644 --- a/python/functions/datascience/automatic_eda/chapters/correlacion.py +++ b/python/functions/datascience/automatic_eda/chapters/correlacion.py @@ -356,12 +356,11 @@ def build_correlacion(profile: dict, ctx: dict): t_cramers = _term(mark_term, "cramers_v", "Cramér's V") t_corr_ratio = _term(mark_term, "correlation_ratio", "razón de correlación") blocks.append(model.Markdown(text=( - "Asociación entre columnas. Cada par se evalúa con la métrica adecuada a " - f"sus tipos ({t_pearson}/{t_spearman} entre numéricas — con **signo**; " - f"{t_cramers} entre categóricas; {t_corr_ratio} num-categórica; " - "información mutua como medida común no lineal). Sólo las correlaciones " - "**num-num** tienen dirección: por eso los pares **negativos** son siempre " - "num-num."))) + "Asociación entre columnas. Cada par se evalúa con la métrica adecuada " + f"a sus tipos: {t_pearson}/{t_spearman} (numéricas), {t_cramers} " + f"(categóricas), {t_corr_ratio} (num-categórica) e información mutua. " + "Sólo las correlaciones **num-num** llevan **signo** (dirección): por " + "eso los pares **negativos** son siempre num-num."))) # 1) Association matrix (heatmap). labels, trimmed = _ordered_labels(pairs) diff --git a/python/functions/datascience/automatic_eda/chapters/modelos.py b/python/functions/datascience/automatic_eda/chapters/modelos.py index 1ddf78ee..77fc54b4 100644 --- a/python/functions/datascience/automatic_eda/chapters/modelos.py +++ b/python/functions/datascience/automatic_eda/chapters/modelos.py @@ -6,15 +6,16 @@ normality}``). It renders, as structured markdown/tables/figures that the core paginator never cuts: 1. **Normalization note** — every multivariate model below standardizes the - columns with z-score first; the chapter explains why (different scales would - otherwise dominate distance/variance). + columns with z-score first (the term is marked clickable; its definition + lives in the GLOSARIO chapter, not inline). 2. **PCA** — a scree plot (explained + cumulative variance, single Y axis) plus variance and top-loadings tables. 3. **KMeans segments** — a PCA scatter **coloured by cluster** (its own page/slide), the cluster-size table, and a per-cluster LLM micro-analysis with a title for each segment. -4. **Isolation Forest outliers** — a short explanation of how anomalous rows are - isolated multivariately and how the threshold is chosen, plus the counts. +4. **Isolation Forest outliers** — the multivariate anomaly counts and decision + threshold (the method is marked clickable; its definition lives in the + GLOSARIO chapter, not inline). 5. **Normality** — per-column Jarque-Bera / D'Agostino / Shapiro verdicts. The raw numeric data needed to colour the cluster scatter is **not** in the @@ -314,12 +315,8 @@ def _normalization_intro(gloss=None, mark_term: bool = False) -> list: text = ( "Estos modelos son **no supervisados**: buscan estructura latente sin " "una variable objetivo. Antes de aplicarlos, todas las columnas " - f"numéricas se {zscore} (cada valor menos la media, dividido por la " - "desviación típica). Sin esta normalización, una variable con escala " - "grande (p.ej. ingresos en euros) dominaría las distancias y la varianza " - "frente a otra de escala pequeña (p.ej. un ratio entre 0 y 1), sesgando " - "tanto el PCA como el KMeans. Tras la estandarización todas las variables " - "pesan por igual." + f"numéricas se {zscore}, para que todas pesen por igual con " + "independencia de su escala." ) return [model.Heading(text="Modelos no supervisados", level=1), model.Markdown(text=text)] @@ -334,11 +331,11 @@ def _pca_section(pca: dict, gloss=None, mark_term: bool = False) -> list: n_used = pca.get("n_rows_used") n_feat = pca.get("n_features") intro = ( - f"El {_term(mark_term, 'pca', 'PCA')} resume {_fmt_num(n_feat)} variables " - "numéricas en componentes ortogonales ordenados por la varianza que " - f"capturan ({_fmt_num(n_used)} filas usadas tras eliminar nulos). El " - "gráfico de sedimentación (scree) muestra cuánta varianza aporta cada " - "componente y su acumulado: un codo marca cuántos componentes bastan." + f"El {_term(mark_term, 'pca', 'PCA')} se aplica sobre " + f"{_fmt_num(n_feat)} variables numéricas ({_fmt_num(n_used)} filas " + "usadas tras eliminar nulos). El gráfico de sedimentación (scree) " + "muestra cuánta varianza aporta cada componente y su acumulado: un " + "codo marca cuántos componentes bastan." ) blocks.append(model.Markdown(text=intro)) @@ -403,9 +400,8 @@ def _kmeans_section(kmeans: dict, projection: dict, titles, t_sil = _term(mark_term, "silhouette", "*silhouette*") intro = ( f"{t_kmeans} agrupa las filas en **{_fmt_num(best_k)} segmentos** " - f"elegidos automáticamente maximizando el coeficiente de {t_sil} " - f"(**{_fmt_num(sil)}**, rango −1 a 1: cuanto más alto, segmentos más " - "compactos y separados). Los segmentos se proyectan sobre el plano de " + f"elegidos automáticamente por el coeficiente de {t_sil} " + f"(**{_fmt_num(sil)}**). Los segmentos se proyectan sobre el plano de " "los dos primeros componentes principales para visualizarlos." ) blocks.append(model.Markdown(text=intro)) @@ -469,14 +465,10 @@ def _outliers_section(outliers: dict, gloss=None, mark_term: bool = False) -> li level=2)] isof = _term(mark_term, "isolation_forest", "**Isolation Forest**") explain = ( - f"{isof} detecta filas anómalas de forma *multivariante*: " - "construye árboles que parten el espacio con cortes aleatorios y mide " - "cuántos cortes hacen falta para aislar cada fila. Las filas raras " - "(combinaciones de valores poco frecuentes considerando **todas las " - "columnas a la vez**, no una sola) se aíslan con muy pocos cortes y " - "obtienen un score bajo. El **umbral** de decisión separa las filas " - "normales de las anómalas según la contaminación esperada del modelo: " - "una fila es outlier cuando su score queda por debajo de ese umbral." + f"{isof} marca filas anómalas de forma *multivariante*: combinaciones " + "de valores poco frecuentes considerando **todas las columnas a la " + "vez**, no una sola. La tabla resume cuántas se detectaron y el umbral " + "de decisión empleado." ) blocks.append(model.Markdown(text=explain)) blocks.append(model.KVTable(rows=[ diff --git a/python/functions/datascience/automatic_eda/chapters/relaciones.py b/python/functions/datascience/automatic_eda/chapters/relaciones.py index eba05f76..7e593a96 100644 --- a/python/functions/datascience/automatic_eda/chapters/relaciones.py +++ b/python/functions/datascience/automatic_eda/chapters/relaciones.py @@ -256,14 +256,14 @@ def _pk_candidates_section(profile: dict, mark: bool) -> list: pk = ("[[term:pk]]**clave primaria**[[/term]]" if mark else "**clave primaria**") intro = ( - f"Estas columnas son **candidatas a {pk}**: su " - "[[term:cardinalidad]]cardinalidad[[/term]] iguala al número de filas y no " - "tienen nulos, así que cada valor identifica una fila distinta. Son " - "candidatas, no una clave declarada: la base no las marca como tal." + f"Columnas **candidatas a {pk}**: su " + "[[term:cardinalidad]]cardinalidad[[/term]] iguala al número de filas y " + "no tienen nulos. Son candidatas, no una clave declarada: la base no " + "las marca como tal." if mark else - "Estas columnas son **candidatas a clave primaria**: su cardinalidad " - "iguala al número de filas y no tienen nulos, así que cada valor " - "identifica una fila distinta.") + "Columnas **candidatas a clave primaria**: su cardinalidad iguala al " + "número de filas y no tienen nulos. Son candidatas, no una clave " + "declarada.") rows = [] for name in keys: @@ -320,10 +320,10 @@ def _inter_table_section(db_path: str, tables: list, mark: bool) -> list: blocks = [ model.Heading(text="Claves foráneas candidatas (inter-tabla)", level=2), model.Markdown(text=( - f"La fuente tiene varias tablas. Estas {fk_term} candidatas se infieren " - f"por señal de nombre y por {containment}: una columna de una tabla cuyos " - "valores están contenidos en la clave de otra. No están declaradas por " - "la base; son la relación más probable según los datos.")), + f"La fuente tiene varias tablas. Estas {fk_term} candidatas se " + f"infieren por señal de nombre y por {containment}. No están " + "declaradas por la base; son la relación más probable según los " + "datos.")), ] shown = candidates[:MAX_FK_ROWS] @@ -441,13 +441,12 @@ def _intro_blocks(mark: bool) -> list: pk = "[[term:pk]]clave primaria[[/term]]" if mark else "clave primaria" fk = "[[term:fk]]clave foránea[[/term]]" if mark else "clave foránea" text = ( - f"Este capítulo analiza las **relaciones de clave** de la tabla: qué columna " - f"identifica cada fila (la {pk}) y qué columnas referencian a otra tabla (las " - f"{fk}). Cuando la base las **declara** como restricciones del esquema, se " - "muestran tal cual; cuando no, se proponen las más probables a partir de los " - "datos —por inclusión de valores entre tablas (containment) o, en una sola " - "tabla, por una heurística de nombre y cardinalidad— siempre marcadas como " - "candidatas, nunca como hechos.") + f"Este capítulo analiza las **relaciones de clave** de la tabla: cuál es " + f"la {pk} y cuáles son las {fk}. Cuando la base las **declara** como " + "restricciones del esquema, se muestran tal cual; cuando no, se proponen " + "las más probables a partir de los datos —por containment entre tablas o, " + "en una sola tabla, por una heurística de nombre y cardinalidad— siempre " + "marcadas como candidatas, nunca como hechos.") return [model.Heading(text=CHAPTER_TITLE, level=1), model.Markdown(text=text)] From 7158be8142cb95dcfc21eb8506bd14f4fd5e20e3 Mon Sep 17 00:00:00 2001 From: Egutierrez <egutierrez@dead.dd> Date: Tue, 30 Jun 2026 19:26:33 +0200 Subject: [PATCH 33/53] =?UTF-8?q?feat(eda):=20cat=5Fdistr=20una=20hoja=20p?= =?UTF-8?q?or=20columna=20(gr=C3=A1fico=20incluido)=20+=20sin=20descripci?= =?UTF-8?q?=C3=B3n=20redundante=20con=20glosario?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cada columna categórica del capítulo CAT DISTR ocupa ahora su propia página (PDF) / slide (PPTX) con su gráfico junto a su tabla, y se elimina la explicación larga de la entropía que duplicaba el capítulo GLOSARIO. Cambios: - model.Group: nuevo campo aditivo `page_break_before` (default False). Cuando es True el renderer fuerza al grupo a empezar en página/slide nueva (salvo que la actual esté vacía). Comportamiento de todos los capítulos existentes intacto. Soportado también en el normalizador dict-defensivo `as_block`. - render_pdf_impl / render_pptx_impl `_place_group`: respetan `page_break_before`. - render_pdf_impl / render_pptx_impl `_measure_block`: medición fiel de KVTable y DataTable (replica `_place_*`: título-heading, wrap del valor/celdas por columna, nota). La estimación previa asumía una línea por fila e ignoraba el título, así que el keep-together infra-presupuestaba la figura y el gráfico se desbordaba a la página siguiente. Helpers `_measure_kv_table`/`_measure_data_table`. - render_pptx_impl `_shrink_group_figures`: umbrales más bajos (budget>0.6, per>0.35) para que en el slide corto 16:9 la figura se encoja y conviva con la tabla en lugar de partir la columna (misma filosofía keep-together del PDF). - cat_distr.py: - build envuelve cada columna en un `Group(page_break_before=idx>0)`: una columna por página/slide, con su tabla de cardinalidad, su top-k y su donut juntos. La primera comparte página con la intro para no dejar una casi vacía. - intro recortada: se elimina el párrafo que explicaba qué es la entropía (vive en el capítulo GLOSARIO, donde el término `[[term:entropia]]` enlaza); se conserva el término clicable y el total de filas de referencia. - `_cardinality_block`: métricas relacionadas agrupadas por fila (distintos·%· únicos; entropía bits·máx·norm; desbalance·longitud) sin perder ningún dato, para que tabla + gráfico quepan en el slide 16:9. - columnas id-like (≈100% distintas): se omite la top-k (sería una lista de valores únicos; la nota lo explica) y el donut ocupa ese hueco. - CHAPTER_VERSION 1.1.0 -> 1.2.0. Verificado con titanic (render_automatic_eda run_models=True): PDF 5 páginas y PPTX 5 slides del capítulo (intro + 1 por columna: Name, Sex, Ticket, Embarked), cada columna con su gráfico junto a su tabla, sin cortes. Suite verde (121 passed): pytest automatic_eda/ + render_automatic_eda_test.py. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> --- .../automatic_eda/chapters/cat_distr.py | 146 +++++++++++------- .../automatic_eda/chapters/cat_distr_test.py | 138 +++++++++++++---- .../datascience/automatic_eda/model.py | 11 +- .../automatic_eda/render_pdf_impl.py | 67 +++++++- .../automatic_eda/render_pptx_impl.py | 68 +++++++- 5 files changed, 335 insertions(+), 95 deletions(-) diff --git a/python/functions/datascience/automatic_eda/chapters/cat_distr.py b/python/functions/datascience/automatic_eda/chapters/cat_distr.py index 6421a574..b722c68a 100644 --- a/python/functions/datascience/automatic_eda/chapters/cat_distr.py +++ b/python/functions/datascience/automatic_eda/chapters/cat_distr.py @@ -1,19 +1,25 @@ """Categorical distributions chapter (CAT DISTR). -Third reference chapter for AutomaticEDA. For every categorical column it shows, -fulfilling the user's request: +Third reference chapter for AutomaticEDA. Each categorical column gets **its own +page (PDF) / slide (PPTX)**: every column is wrapped in a keep-together +``model.Group`` with ``page_break_before=True`` (except the first, which may share +the intro's page), so its chart sits next to its tables and no column is split. -1. A short opening explanation of **Shannon entropy** (what it measures, its 0 - and log2(k) bounds, the normalized 0–1 version) and the dataset row total used - as a comparison baseline. -2. Per column, a cardinality key/value table: distinct values, ``% distinct`` - (distinct / total rows), total dataset rows, singleton values (frequency 1), - entropy with its theoretical maximum and the normalized ratio, mode, imbalance - and string-length stats. -3. A short note flagging problematic cardinality (id-like ≈100% distinct, or a +A short intro names the clickable **[[term:entropia]]entropía[[/term]]** term — +the full definition lives in the GLOSARIO chapter, so it is NOT repeated inline +here (one click jumps to the glossary entry). The intro also carries the dataset +row total used as a comparison baseline. + +Per column the Group contains, in order: + +1. A cardinality key/value table: distinct values, ``% distinct`` (distinct / + total rows), total dataset rows, singleton values (frequency 1), entropy with + its theoretical maximum and the normalized ratio, mode, imbalance and + string-length stats. +2. A short note flagging problematic cardinality (id-like ≈100% distinct, or a single dominating category). -4. A ``top-k`` table (value / count / %). -5. A **donut pie chart** of the most common categories (top-k + an "Otros" +3. A ``top-k`` table (value / count / %). +4. A **donut pie chart** of the most common categories (top-k + an "Otros" bucket), drawn lazily so the renderers scale it to fit entirely. Data comes from the ``eda`` group: each ``columns[i]['categorical']`` is the @@ -33,7 +39,7 @@ import math from .. import model -CHAPTER_VERSION = "1.1.0" +CHAPTER_VERSION = "1.2.0" CHAPTER_ID = "cat_distr" CHAPTER_TITLE = "Distribuciones categóricas" @@ -53,11 +59,17 @@ _TERM_ENTROPIA_DEF = ( # Cap the number of categorical columns rendered to keep the document bounded; # the rest are summarized in a closing note (no silent truncation). MAX_COLS = 40 -# Rows shown in each top-k table and explicit slices in the pie. -TOP_TABLE_ROWS = 15 +# Rows shown in each top-k table and explicit slices in the pie. Kept moderate so +# the whole column — cardinality table + top-k table + donut — fits on ONE +# page/slide with the chart next to its tables; the table note still reports +# "top N of M" so nothing is silently hidden. For id-like columns (≈100% +# distinct) the top-k table is dropped entirely (it would be a list of unique +# values — pure noise), which also frees the room the donut needs (see build). +TOP_TABLE_ROWS = 8 PIE_TOP_K = 6 -# Truncate very long category labels in tables (the renderer also wraps). -LABEL_MAX = 48 +# Truncate very long category labels in tables (the renderer also wraps). Kept +# tight so a column with long id-like values (names, tickets) still fits its page. +LABEL_MAX = 28 def _fmt_int(value) -> str: @@ -267,45 +279,55 @@ def _normalize_card(card: dict) -> dict: def _cardinality_block(card: dict): - """KVTable with the cardinality / entropy metrics for one column.""" + """KVTable with the cardinality / entropy metrics for one column. + + Related metrics are grouped onto a single row each (distinct/%/unique; + entropy bits/max/normalized; length min/mean/max) so the whole column — + table + chart — fits one page/slide without dropping any datum; the short + 16:9 PPTX slide does not fit one metric per row plus a chart otherwise.""" n_singletons = card.get("n_singletons") if n_singletons is not None and card.get("n_singletons_partial"): - singletons = f"≥{_fmt_int(n_singletons)} (en top mostrado)" + singletons = f"≥{_fmt_int(n_singletons)}" elif n_singletons is not None: singletons = _fmt_int(n_singletons) else: singletons = "—" - entropy_ref = _fmt_num(card.get("entropy")) - emax = card.get("entropy_max") - if emax is not None: - entropy_ref = f"{entropy_ref} (máx {_fmt_num(emax)})" + # Distinct count · % distinct · unique (frequency 1) on one row. + distinct_combo = (f"{_fmt_int(card.get('n_distinct'))} · " + f"{_fmt_pct_value(card.get('pct_distinct'))} · " + f"{singletons} únicos") + + # Entropy bits · theoretical max · normalized 0–1 on one row. + entropy_combo = (f"{_fmt_num(card.get('entropy'))} bits · " + f"máx {_fmt_num(card.get('entropy_max'))} · " + f"norm {_fmt_num(card.get('entropy_norm'))}") mode = card.get("mode") mode_pct = card.get("mode_pct") - mode_str = "—" if mode is None else model._safe_str(mode) + mode_str = "—" if mode is None else _truncate(mode, 32) if mode is not None and mode_pct is not None: mode_str = f"{mode_str} ({_fmt_pct_value(mode_pct)})" rows = [ - ("Valores distintos", _fmt_int(card.get("n_distinct"))), - ("% distintos", _fmt_pct_value(card.get("pct_distinct"))), + ("Distintos · % · únicos", distinct_combo), ("Total filas (dataset)", _fmt_int(card.get("n_rows"))), - ("Valores únicos (frecuencia 1)", singletons), - ("Entropía (bits)", entropy_ref), - ("Entropía normalizada (0–1)", _fmt_num(card.get("entropy_norm"))), + ("Entropía (bits · máx · norm)", entropy_combo), ("Moda", mode_str), ] imbalance = card.get("imbalance") - if imbalance is not None: - rows.append(("Desbalance", _fmt_num(imbalance))) lm = card.get("len_min") lmean = card.get("len_mean") lmax = card.get("len_max") + # Imbalance and string length (both secondary) share one closing row. + extras = [] + if imbalance is not None: + extras.append(f"desbalance {_fmt_num(imbalance)}") if any(v is not None for v in (lm, lmean, lmax)): - rows.append(( - "Longitud (mín/media/máx)", - f"{_fmt_num(lm)} / {_fmt_num(lmean)} / {_fmt_num(lmax)}")) + extras.append( + f"long. {_fmt_num(lm)}/{_fmt_num(lmean)}/{_fmt_num(lmax)}") + if extras: + rows.append(("Desbalance · longitud", " · ".join(extras))) return model.KVTable(rows=rows, title="Cardinalidad") @@ -315,7 +337,8 @@ def _flag_note(card: dict): return model.Note( "Casi todos los valores son distintos (≈100% distintos): la columna " "se comporta como un identificador y aporta poco para agrupar o " - "comparar categorías.") + "comparar categorías. No se lista el top de categorías (serían " + "valores casi todos únicos).") if card.get("dominated"): mp = card.get("mode_pct") mp_str = _fmt_pct_value(mp) if mp is not None else "muy alta" @@ -335,7 +358,7 @@ def _topk_table(cat: dict): if not isinstance(t, dict): continue rows.append([ - model._safe_str(t.get("value")), + _truncate(t.get("value")), _fmt_int(t.get("count")), _pct_from_maybe_fraction(t.get("pct")), ]) @@ -353,20 +376,16 @@ def _topk_table(cat: dict): def _intro_blocks(n_rows, mark_term: bool = False): total = _fmt_int(n_rows) # Mark the first appearance of the term as a clickable glossary jump when the - # term was registered (mark_term). The visible text is identical either way. - entropia = ("[[term:entropia]]**entropía de Shannon**[[/term]]" if mark_term - else "**entropía de Shannon**") + # term was registered (mark_term). The full definition of entropy lives in the + # GLOSARIO chapter, so the intro only names the clickable term here instead of + # repeating the long explanation (avoids the redundancy with the glossary). + entropia = ("[[term:entropia]]entropía[[/term]]" if mark_term + else "entropía") text = ( - f"La {entropia} mide cómo de repartidos están los valores de " - "una columna categórica, en bits. Vale 0 cuando una sola categoría " - "concentra todas las filas (máxima previsibilidad) y alcanza su máximo, " - "log2(k) para k categorías distintas, cuando todas aparecen por igual " - "(máxima diversidad). La **entropía normalizada** (entropía dividida por " - "su máximo) la lleva al rango 0–1 para comparar columnas con distinto " - "número de categorías. Para cada columna se muestran los valores " - "distintos, el porcentaje que representan sobre el total de filas, los " - "valores únicos (que aparecen una sola vez), la tabla de las categorías " - "más frecuentes y un gráfico de tarta (donut) de las más comunes." + f"Cada columna categórica ocupa su propia página: sus métricas de " + f"cardinalidad —incluida la {entropia}—, una nota que señala cardinalidad " + "problemática, la tabla de las categorías más frecuentes y un gráfico de " + "tarta (donut) de las más comunes, todo junto." ) if n_rows is not None: text += f" El dataset tiene {total} filas en total como referencia." @@ -398,24 +417,37 @@ def build_cat_distr(profile: dict, ctx: dict): blocks = list(_intro_blocks(n_rows, mark_term=mark_term)) rendered = cat_cols[:MAX_COLS] - for col in rendered: + for idx, col in enumerate(rendered): name = col.get("name") or "(columna)" cat = col.get("categorical") or {} card = _normalize_card(_cardinality(cat, n_rows)) - blocks.append(model.Heading(text=str(name), level=2)) - blocks.append(_cardinality_block(card)) + # One Group per categorical column: heading + cardinality table + flag + # note + top-k table + donut figure are kept together and the renderer + # starts each on a fresh page/slide (page_break_before) so every column + # gets its own page with its chart next to its tables. The first column + # may share the intro's page (no forced break) to avoid a near-empty page. + col_blocks = [ + model.Heading(text=str(name), level=2), + _cardinality_block(card), + ] note = _flag_note(card) if note is not None: - blocks.append(note) - topk = _topk_table(cat) - if topk is not None: - blocks.append(topk) - blocks.append(model.Figure( + col_blocks.append(note) + # For id-like columns (≈100% distinct) the top-k is a list of unique + # values — pure noise; skip it (the flag note already explains why) and + # let the donut take that room so the whole column fits one page/slide. + if not card.get("id_like"): + topk = _topk_table(cat) + if topk is not None: + col_blocks.append(topk) + col_blocks.append(model.Figure( make=_pie_make(cat.get("top") or [], card.get("n_distinct"), str(name), n_rows), caption=(f"Categorías más comunes de «{_truncate(name, 32)}» " "(donut: top-k + «Otros»)"))) + blocks.append(model.Group(blocks=col_blocks, + page_break_before=(idx > 0))) if len(cat_cols) > len(rendered): omitted = len(cat_cols) - len(rendered) diff --git a/python/functions/datascience/automatic_eda/chapters/cat_distr_test.py b/python/functions/datascience/automatic_eda/chapters/cat_distr_test.py index a061c67d..f26984a9 100644 --- a/python/functions/datascience/automatic_eda/chapters/cat_distr_test.py +++ b/python/functions/datascience/automatic_eda/chapters/cat_distr_test.py @@ -2,11 +2,14 @@ Self-contained: builds synthetic TableProfiles (no DuckDB) so the suite is fast and deterministic. Verifies that ``build_cat_distr`` emits the blocks the user -asked for (entropy intro, distinct/total/%-distinct/unique metrics, top-k table -and a donut figure), that the chapter renders inside the full document to both -PDF and PPTX showing that content, that a profile with no categorical columns -yields ``None`` without raising, and that long labels / many columns are never -cut in either output. +asked for (distinct/total/%-distinct/unique metrics, top-k table and a donut +figure), that EACH categorical column is wrapped in its own keep-together +``Group`` that starts on a fresh page/slide (one column per page, chart next to +its tables), that the long entropy explanation is NOT repeated inline (it lives +in the glossary — only the clickable term is kept), that the chapter renders +inside the full document to both PDF and PPTX showing that content, that a +profile with no categorical columns yields ``None`` without raising, and that +long labels / many columns are never cut in either output. """ import os @@ -17,7 +20,8 @@ from pypdf import PdfReader from pptx import Presentation from datascience.automatic_eda.model import ( - DataTable, Figure, Heading, KVTable, Note, + DataTable, Figure, GlossaryCollector, Group, Heading, KVTable, Markdown, + Note, ) from datascience.automatic_eda.chapters.cat_distr import ( CHAPTER_ID, CHAPTER_VERSION, build_cat_distr, @@ -81,8 +85,20 @@ def _pptx_text(path: str) -> str: return re.sub(r"\s+", " ", " ".join(parts)) -def _kinds(chapter): - return [b.kind for b in chapter.blocks] +def _flatten(blocks): + """Expand keep-together Groups so the per-column heading/table/figure are + inspectable as a flat block list (the chapter wraps each column in a Group).""" + out = [] + for b in blocks: + if getattr(b, "kind", "") == "group": + out.extend(_flatten(getattr(b, "blocks", []) or [])) + else: + out.append(b) + return out + + +def _column_groups(chapter): + return [b for b in chapter.blocks if isinstance(b, Group)] def test_golden_build_cat_distr_emite_bloques_pedidos(): @@ -90,36 +106,101 @@ def test_golden_build_cat_distr_emite_bloques_pedidos(): assert ch is not None assert ch.id == CHAPTER_ID assert ch.version == CHAPTER_VERSION - kinds = _kinds(ch) - # Entropy intro present. + + # Entropy intro present, but the long explanation is gone (it lives in the + # glossary now): only the term is named, no log2/normalizada walkthrough. headings = [b.text for b in ch.blocks if isinstance(b, Heading)] assert any("Entrop" in h for h in headings) - md = next(b for b in ch.blocks if b.kind == "markdown") - assert "entropía" in md.text.lower() and "log2" in md.text - # Cardinality metrics: distinct, total rows, %-distinct, unique values. - kv = next(b for b in ch.blocks if isinstance(b, KVTable)) + md = next(b for b in ch.blocks if isinstance(b, Markdown)) + assert "entropía" in md.text.lower() + assert "log2" not in md.text # redundant explanation removed. + assert "máxima diversidad" not in md.text + + # Per-column blocks are wrapped in keep-together Groups: flatten to inspect. + flat = _flatten(ch.blocks) + kv = next(b for b in flat if isinstance(b, KVTable)) labels = [r[0] for r in kv.rows] - assert "Valores distintos" in labels - assert "% distintos" in labels + values = " ".join(str(r[1]) for r in kv.rows) + # Cardinality metrics: distinct count, %-distinct, unique values and total + # rows are present (grouped onto compact rows so the chart fits the page). + assert "Distintos · % · únicos" in labels assert "Total filas (dataset)" in labels - assert "Valores únicos (frecuencia 1)" in labels assert any("Entropía" in lbl for lbl in labels) + assert "únicos" in values and "%" in values + assert "bits" in values and "norm" in values # entropy + max + normalized. # Top-k table + pie figure. - dt = next(b for b in ch.blocks if isinstance(b, DataTable)) + dt = next(b for b in flat if isinstance(b, DataTable)) assert dt.header == ["Valor", "Conteo", "%"] assert any("neumaticos" in str(cell) for row in dt.rows for cell in row) - assert any(isinstance(b, Figure) for b in ch.blocks) - # id-like column flagged with a Note. - assert any(isinstance(b, Note) and "identificador" in b.text - for b in ch.blocks) + assert any(isinstance(b, Figure) for b in flat) + # id-like column flagged with a Note that also explains the top-k is dropped. + idnote = next((b for b in flat + if isinstance(b, Note) and "identificador" in b.text), None) + assert idnote is not None + assert "No se lista el top" in idnote.text -def test_golden_render_pdf_muestra_categoricas(): +def test_golden_idlike_omite_topk_y_conserva_donut(): + # The id-like column (uuid, 100% distinct) must NOT carry a top-k DataTable + # (it would be a list of unique values), but must still keep its donut Figure + # and its cardinality table so it stays a full per-column page. + ch = build_cat_distr(_profile(), {}) + groups = _column_groups(ch) + uuid_group = next(g for g in groups + if any(getattr(b, "text", "") == "uuid" for b in g.blocks)) + kinds = [b.kind for b in uuid_group.blocks] + assert "data_table" not in kinds # top-k of unique values dropped. + assert "kv_table" in kinds # cardinality kept. + assert "figure" in kinds # donut kept (chart per column). + # A non-id-like column keeps its top-k table. + cat_group = next(g for g in groups + if any(getattr(b, "text", "") == "categoria" + for b in g.blocks)) + assert "data_table" in [b.kind for b in cat_group.blocks] + + +def test_golden_una_pagina_por_columna_groups(): + ch = build_cat_distr(_profile(), {}) + groups = _column_groups(ch) + # Two categorical columns -> two column Groups (numeric column excluded). + assert len(groups) == 2 + # Each Group carries one column: a heading + its cardinality table + figure. + for g in groups: + kinds = [b.kind for b in g.blocks] + assert kinds[0] == "heading" + assert "kv_table" in kinds + assert "figure" in kinds + # The first column may share the intro page (no forced break); every later + # column starts on a fresh page/slide so each column gets its own page. + assert groups[0].page_break_before is False + assert all(g.page_break_before is True for g in groups[1:]) + + +def test_golden_entropia_clicable_y_definicion_en_glosario(): + # With a glossary collector the intro marks the clickable term and the FULL + # definition (the long explanation removed from the intro) lands in the + # glossary, not inline — no data lost, just relocated. + gc = GlossaryCollector() + ch = build_cat_distr(_profile(), {"glossary": gc}) + md = next(b for b in ch.blocks if isinstance(b, Markdown)) + assert "[[term:entropia]]entropía[[/term]]" in md.text + assert gc.has("entropia") + entry = gc.get("entropia") + assert entry is not None + # The definition kept in the glossary still carries the detail removed inline. + assert "log2" in entry["definition"] + assert "normalizada" in entry["definition"].lower() + + +def test_golden_render_pdf_una_pagina_por_columna(): with tempfile.TemporaryDirectory() as d: out = os.path.join(d, "eda.pdf") res = render_automatic_eda_pdf(_profile(), out, {"title": "EDA"}) assert res["path"] == out and os.path.exists(out) - assert CHAPTER_ID in [c["id"] for c in res["chapters"]] + cat_meta = next(c for c in res["chapters"] if c["id"] == CHAPTER_ID) + # Two categorical columns, each on its own page -> >= 2 pages for the + # chapter (intro shares the first column's page). + assert cat_meta["n_pages"] >= 2 txt = _pdf_text(out) assert "Entrop" in txt assert "distintos" in txt @@ -133,7 +214,8 @@ def test_golden_render_pptx_muestra_categoricas(): out = os.path.join(d, "eda.pptx") res = render_automatic_eda_pptx(_profile(), out, {"title": "EDA"}) assert res["path"] == out and os.path.exists(out) - assert CHAPTER_ID in [c["id"] for c in res["chapters"]] + cat_meta = next(c for c in res["chapters"] if c["id"] == CHAPTER_ID) + assert cat_meta["n_slides"] >= 2 # one slide per categorical column. txt = _pptx_text(out) assert "Entrop" in txt assert "categoria" in txt and "neumaticos" in txt @@ -170,11 +252,15 @@ def test_anti_corte_label_largo_y_muchas_columnas(): ch = build_cat_distr(profile, {}) assert ch is not None + # One Group per column, each forcing its own page (except the first). + groups = _column_groups(ch) + assert len(groups) == 30 + assert sum(1 for g in groups if g.page_break_before) == 29 with tempfile.TemporaryDirectory() as d: pdf = os.path.join(d, "anti.pdf") res = render_automatic_eda_pdf(profile, pdf, {"write_manifest": False}) assert res["path"] == pdf - assert res["n_pages"] > 1 # many columns spilled across pages, OK. + assert res["n_pages"] > 1 # one page per column, OK. txt = _pdf_text(pdf) # Long label wrapped (not truncated): every word survives. for word in ("Lorem", "incididunt", "reprehenderit", "voluptate"): diff --git a/python/functions/datascience/automatic_eda/model.py b/python/functions/datascience/automatic_eda/model.py index 53c41377..7237df0b 100644 --- a/python/functions/datascience/automatic_eda/model.py +++ b/python/functions/datascience/automatic_eda/model.py @@ -139,10 +139,17 @@ class Group: it starts on a fresh page and flows (honest degradation, never cut). Use it to bind ``Heading`` + ``Markdown`` + ``Figure`` of one idea together (see the DISTR NUM / AGREGACION chapters). + + When ``page_break_before`` is True the renderer additionally forces the group + to *start* on a fresh page/slide (unless the current one is already empty), so + a chapter can give each unit its own page — e.g. one categorical column per + page (see CAT DISTR). It is purely additive: the default False keeps the plain + keep-together behaviour for every existing chapter. """ blocks: list = field(default_factory=list) title: Optional[str] = None + page_break_before: bool = False kind: str = field(default="group", init=False) @@ -228,7 +235,9 @@ def as_block(obj: Any): return Note(text=_safe_str(obj.get("text"))) if cls is Group: return Group(blocks=as_blocks(obj.get("blocks")), - title=obj.get("title")) + title=obj.get("title"), + page_break_before=bool( + obj.get("page_break_before", False))) if cls is GlossaryEntry: return GlossaryEntry(key=_safe_str(obj.get("key")), label=_safe_str(obj.get("label")), diff --git a/python/functions/datascience/automatic_eda/render_pdf_impl.py b/python/functions/datascience/automatic_eda/render_pdf_impl.py index ffe9a349..06adea4b 100644 --- a/python/functions/datascience/automatic_eda/render_pdf_impl.py +++ b/python/functions/datascience/automatic_eda/render_pdf_impl.py @@ -675,6 +675,61 @@ def _measure_figure_like(block) -> float: return target_h + 0.04 + cap_h + _GAP +def _measure_kv_table(block) -> float: + """Faithful height of a KVTable — matches ``_place_kv_table``. + + Counts the optional title heading and, per row, the wrapped VALUE column + (the label column never wraps in the placer). The previous estimate assumed + one line per row and ignored the title, so a column's keep-together Group + under-budgeted the figure and the chart spilled to the next page. Keep this in + sync with ``_place_kv_table``.""" + h = 0.0 + title = getattr(block, "title", None) + if title: + h += _measure_heading_text(title, 2) + rows = getattr(block, "rows", []) or [] + key_w = 1.9 + val_chars = tl.chars_per_line(_USABLE_W - key_w - 0.1, _FS_BODY) + lh = tl.line_height_in(_FS_BODY) + for row in rows: + try: + value = row[1] + except Exception: # noqa: BLE001 + value = "" + v_lines = tl.wrap(model._safe_str(value), val_chars) + h += lh * len(v_lines) + _ROW_VPAD + return h + _GAP + + +def _measure_data_table(block) -> float: + """Faithful height of a DataTable — matches ``_place_data_table``. + + Counts the optional title heading, the wrapped header row, every wrapped data + row (per-column wrap via the same ``_col_widths``/``_wrap_row`` the placer + uses) and the optional note. Keep this in sync with ``_place_data_table``.""" + h = 0.0 + title = getattr(block, "title", None) + if title: + h += _measure_heading_text(title, 2) + header = list(getattr(block, "header", []) or []) + rows = list(getattr(block, "rows", []) or []) + fs = _FS_CELL + widths = _col_widths(header, rows, fs) + lh = tl.line_height_in(fs) + if header: + header_lines = _wrap_row(header, widths, fs) + h += lh * max((len(c) for c in header_lines), default=1) + _ROW_VPAD * 2 + for r in rows: + cells_lines = _wrap_row(r, widths, fs) + h += lh * max((len(c) for c in cells_lines), default=1) + _ROW_VPAD * 2 + note = getattr(block, "note", None) + if note: + nlines = tl.wrap(model._safe_str(note), + tl.chars_per_line(_USABLE_W, _FS_NOTE)) + h += tl.line_height_in(_FS_NOTE) * len(nlines) + return h + _GAP + + def _measure_block(st: _PdfState, block) -> float: kind = getattr(block, "kind", "") try: @@ -690,13 +745,9 @@ def _measure_block(st: _PdfState, block) -> float: tl.chars_per_line(_USABLE_W, _FS_NOTE)) return tl.line_height_in(_FS_NOTE) * len(lines) + _GAP if kind == "kv_table": - rows = getattr(block, "rows", []) or [] - return (tl.line_height_in(_FS_BODY) + _ROW_VPAD) * (len(rows) + 1) \ - + _GAP + return _measure_kv_table(block) if kind == "data_table": - rows = getattr(block, "rows", []) or [] - return (tl.line_height_in(_FS_CELL) + _ROW_VPAD * 2) \ - * (len(rows) + 1) + _GAP + return _measure_data_table(block) if kind == "group": return sum(_measure_block(st, b) for b in (getattr(block, "blocks", []) or [])) @@ -735,6 +786,10 @@ def _place_group(st: _PdfState, block) -> None: blocks = getattr(block, "blocks", []) or [] if not blocks: return + # Opt-in page break: start this group on a fresh page unless the current one + # is still empty (so a chapter can give each unit its own page). + if getattr(block, "page_break_before", False) and st.y > _CONTENT_TOP + 1e-6: + _new_page(st) avail_full = _CONTENT_BOTTOM - _CONTENT_TOP _shrink_group_figures(st, blocks, avail_full) total = sum(_measure_block(st, b) for b in blocks) diff --git a/python/functions/datascience/automatic_eda/render_pptx_impl.py b/python/functions/datascience/automatic_eda/render_pptx_impl.py index 5e3ba331..cc0171f4 100644 --- a/python/functions/datascience/automatic_eda/render_pptx_impl.py +++ b/python/functions/datascience/automatic_eda/render_pptx_impl.py @@ -625,6 +625,55 @@ def _measure_figure_like(block) -> float: return target_h + 0.05 + cap_h + _GAP +def _measure_kv_table(block) -> float: + """Faithful KVTable height — matches ``_place_kv_table`` (rendered as a + Campo/Valor data table with wrapped cells). The previous estimate assumed one + line per row and ignored the title, so a keep-together Group under-budgeted + the figure and the chart spilled to the next slide. Keep in sync.""" + h = 0.0 + title = getattr(block, "title", None) + if title: + h += _measure_heading_text(title, 2) + rows = getattr(block, "rows", []) or [] + data_rows = [] + for row in rows: + try: + label, value = row[0], row[1] + except Exception: # noqa: BLE001 + label, value = str(row), "" + data_rows.append([model._safe_str(label), model._safe_str(value)]) + header = ["Campo", "Valor"] + widths = _col_widths(header, data_rows) + fs = _FS_CELL + h += _row_height_in(header, widths, fs) + for r in data_rows: + h += _row_height_in(r, widths, fs) + return h + _GAP + + +def _measure_data_table(block) -> float: + """Faithful DataTable height — matches ``_place_data_table`` (title heading + + wrapped header + every wrapped row + optional note). Keep in sync.""" + h = 0.0 + title = getattr(block, "title", None) + if title: + h += _measure_heading_text(title, 2) + header = list(getattr(block, "header", []) or []) + rows = list(getattr(block, "rows", []) or []) + fs = _FS_CELL + widths = _col_widths(header, rows) + if header: + h += _row_height_in(header, widths, fs) + for r in rows: + h += _row_height_in(r, widths, fs) + note = getattr(block, "note", None) + if note: + nlines = tl.wrap(model._safe_str(note), + tl.chars_per_line(_USABLE_W, _FS_NOTE)) + h += tl.line_height_in(_FS_NOTE) * len(nlines) + 0.05 + return h + _GAP + + def _measure_block(st: _PptxState, block) -> float: kind = getattr(block, "kind", "") try: @@ -639,9 +688,10 @@ def _measure_block(st: _PptxState, block) -> float: lines = tl.wrap(getattr(block, "text", ""), tl.chars_per_line(_USABLE_W, _FS_NOTE)) return tl.line_height_in(_FS_NOTE) * len(lines) + 0.05 + _GAP - if kind in ("kv_table", "data_table"): - rows = getattr(block, "rows", []) or [] - return (tl.line_height_in(_FS_CELL) + 0.10) * (len(rows) + 1) + _GAP + if kind == "kv_table": + return _measure_kv_table(block) + if kind == "data_table": + return _measure_data_table(block) if kind == "group": return sum(_measure_block(st, b) for b in (getattr(block, "blocks", []) or [])) @@ -664,10 +714,14 @@ def _shrink_group_figures(st: _PptxState, blocks: list, avail_full: float) -> No if getattr(b, "kind", "") not in ("figure", "image")) fig_overhead = tl.line_height_in(_FS_NOTE) + 0.05 + 0.05 + _GAP budget = avail_full - nonfig_h - 0.10 * len(fig_blocks) - if budget <= 1.0: + # Low thresholds: a 16:9 slide is short, so a content-heavy column (cardinality + # table + top-k + chart) only fits if the chart is allowed to shrink small. + # Prefer a small-but-present chart on the SAME slide over splitting the column + # across slides (matches the PDF renderer's keep-together philosophy). + if budget <= 0.6: return # not enough room to keep together; let it flow (degrade). per = budget / len(fig_blocks) - fig_overhead - if per <= 0.8: + if per <= 0.35: return for fb in fig_blocks: cur = getattr(fb, "height_in", None) @@ -680,6 +734,10 @@ def _place_group(st: _PptxState, block) -> None: blocks = getattr(block, "blocks", []) or [] if not blocks: return + # Opt-in slide break: start this group on a fresh slide unless the current one + # is still empty (so a chapter can give each unit its own slide). + if getattr(block, "page_break_before", False) and st.y > _CONTENT_TOP + 1e-6: + _new_slide(st, cont=True) avail_full = _CONTENT_BOTTOM - _CONTENT_TOP _shrink_group_figures(st, blocks, avail_full) total = sum(_measure_block(st, b) for b in blocks) From 833597c831fac49c2a6e2f1e0ea61268a8947a5f Mon Sep 17 00:00:00 2001 From: Egutierrez <egutierrez@dead.dd> Date: Tue, 30 Jun 2026 19:45:09 +0200 Subject: [PATCH 34/53] =?UTF-8?q?fix(eda):=20cat=5Fdistr=20PPTX=20?= =?UTF-8?q?=E2=80=94=20columnas=20de=20alta=20cardinalidad=20caben=20en=20?= =?UTF-8?q?UN=20slide=20con=20su=20gr=C3=A1fico?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit La verificación adversarial detectó que, en PPTX (slide 16:9, corto), las columnas categóricas de ALTA cardinalidad NO id-like (Ticket, Cabin) ocupaban 3 slides cada una con el donut SEPARADO de su tabla: el top-k de 8 filas largas no cabía junto al donut y el keep-together partía la columna. (El PDF, en A5, ya estaba 1:1 correcto.) Arreglo SOLO en render_pptx_impl.py: - `_fit_group_blocks` (nuevo): para un Group con figura + DataTable que no cabe en el slide, reserva un alto mínimo para el donut (`_GROUP_MIN_FIG_H`) y recorta las filas de la DataTable a lo que queda, de modo que el gráfico se queda en el MISMO slide, junto a su tabla. No-op cuando ya cabe o no hay par figura+tabla (p.ej. columnas id-like, que ya omiten la top-k). - `_trim_data_table_to_budget` (nuevo): devuelve una COPIA de la DataTable con las filas que caben (al menos una) + nota honesta "top N de M categorías mostradas (recortado para caber en el slide; el PDF muestra más)". NUNCA muta el bloque original, que es compartido con el renderer PDF (el PDF sigue mostrando la tabla completa en A5). - `_place_group`: aplica `_fit_group_blocks` antes de `_shrink_group_figures`. Refuerzo de cat_distr_test.py: - `test_golden_pptx_una_slide_por_columna_con_su_grafico`: perfil con una columna categórica de alta cardinalidad no-id-like (40 valores largos sobre 5000 filas, 0.8% distinto) que reproduce el caso Ticket/Cabin. Asierta que CADA columna categórica aparece en EXACTAMENTE UN slide del capítulo y que ese mismo slide lleva su tabla (Cardinalidad/distintos) Y su donut (caption + shape Picture) — el gráfico nunca se separa de su tabla. Sustituye al laxo `n_slides >= 2`. Verificado con titanic_train.csv (render_automatic_eda run_models=True): 5 columnas categóricas (Name, Sex, Ticket, Cabin, Embarked); PDF 6 páginas y PPTX 6 slides del capítulo (intro + 1 por columna), cada columna con su donut junto a su tabla en una sola página/slide. Ticket y Cabin pasaron de 3 slides a 1. Suite verde (122 passed). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> --- .../automatic_eda/chapters/cat_distr_test.py | 77 +++++++++++++++++++ .../automatic_eda/render_pptx_impl.py | 74 ++++++++++++++++++ 2 files changed, 151 insertions(+) diff --git a/python/functions/datascience/automatic_eda/chapters/cat_distr_test.py b/python/functions/datascience/automatic_eda/chapters/cat_distr_test.py index f26984a9..919b86fa 100644 --- a/python/functions/datascience/automatic_eda/chapters/cat_distr_test.py +++ b/python/functions/datascience/automatic_eda/chapters/cat_distr_test.py @@ -222,6 +222,83 @@ def test_golden_render_pptx_muestra_categoricas(): assert "distintos" in txt +def _profile_high_card() -> dict: + """Profile with a high-cardinality NON-id-like categorical column whose top-k + of long values would split from its donut on a short 16:9 slide unless the + renderer trims the table — the exact case the adversarial check flagged + (Ticket / Cabin).""" + long_vals = [f"Valor largo de categoria numero {i:02d} con texto extra" + for i in range(40)] + top = [{"value": v, "count": 60 - i, "pct": (60 - i) / 5000.0} + for i, v in enumerate(long_vals)] + return { + "table": "t", "source": "t.csv", "n_rows": 5000, "n_cols": 3, + "quality_score": 80.0, + "columns": [ + {"name": "precio", "inferred_type": "numeric", "null_pct": 0.0, + "numeric": {"mean": 1.0, "median": 1.0, "min": 0.0, "max": 2.0, + "std": 0.5}}, + # 40 distinct over 5000 rows = 0.8% distinct -> NOT id-like, keeps + # its (long) top-k table; the tall table must not push the donut off. + {"name": "alta_card_col", "inferred_type": "categorical", + "null_pct": 0.0, "distinct_count": 40, + "categorical": {"top": top, "mode": long_vals[0], "n_distinct": 40, + "entropy": 5.2, "imbalance": 1.2, "len_min": 40, + "len_mean": 45, "len_max": 50}}, + {"name": "baja_card_col", "inferred_type": "categorical", + "null_pct": 0.0, "distinct_count": 4, + "categorical": { + "top": [{"value": "norte", "count": 2000, "pct": 0.4}, + {"value": "sur", "count": 1500, "pct": 0.3}, + {"value": "este", "count": 1000, "pct": 0.2}, + {"value": "oeste", "count": 500, "pct": 0.1}], + "mode": "norte", "n_distinct": 4, "entropy": 1.8}}, + ], + } + + +def test_golden_pptx_una_slide_por_columna_con_su_grafico(): + """Each categorical column occupies EXACTLY ONE cat_distr slide that carries + BOTH its cardinality table and its donut figure (picture) — i.e. the chart is + never separated from its table, even for a high-cardinality column.""" + from pptx.enum.shapes import MSO_SHAPE_TYPE + + prof = _profile_high_card() + cat_names = ["alta_card_col", "baja_card_col"] + with tempfile.TemporaryDirectory() as d: + out = os.path.join(d, "eda.pptx") + res = render_automatic_eda_pptx(prof, out, {"title": "EDA"}) + assert res["path"] == out and os.path.exists(out) + prs = Presentation(out) + + # Per column: the cat_distr slides whose text mentions it, and whether the + # owning slide also has the donut caption + an actual picture shape. + slides_with_col = {n: [] for n in cat_names} + owner_has_chart = {n: False for n in cat_names} + for i, sl in enumerate(prs.slides): + texts, has_pic = [], False + for sh in sl.shapes: + if sh.has_text_frame: + texts.append(sh.text_frame.text) + if sh.shape_type == MSO_SHAPE_TYPE.PICTURE: + has_pic = True + txt = re.sub(r"\s+", " ", " ".join(texts)) + if "Distribuciones categ" not in txt: # footer stamp of the chapter. + continue + for n in cat_names: + if n in txt: + slides_with_col[n].append(i) + has_table = "Cardinalidad" in txt or "distintos" in txt + if has_pic and "donut" in txt and has_table: + owner_has_chart[n] = True + + for n in cat_names: + # Exactly one slide carries the column (not split across slides). + assert len(slides_with_col[n]) == 1, (n, slides_with_col[n]) + # That single slide also holds its table AND its donut picture. + assert owner_has_chart[n], (n, "tabla y donut no están en el mismo slide") + + def test_edge_sin_categoricas_devuelve_none(): only_numeric = { "n_rows": 10, "columns": [ diff --git a/python/functions/datascience/automatic_eda/render_pptx_impl.py b/python/functions/datascience/automatic_eda/render_pptx_impl.py index cc0171f4..7a813945 100644 --- a/python/functions/datascience/automatic_eda/render_pptx_impl.py +++ b/python/functions/datascience/automatic_eda/render_pptx_impl.py @@ -729,6 +729,77 @@ def _shrink_group_figures(st: _PptxState, blocks: list, avail_full: float) -> No if isinstance(cur, (int, float)) and cur > 0 else per) +# Minimum height (inches) reserved for a figure inside a keep-together group on +# the short 16:9 slide. When a high-cardinality column's table(s) would otherwise +# leave no room, the data table is trimmed (with an honest note) so the chart +# stays on the SAME slide next to its table instead of spilling to the next one. +_GROUP_MIN_FIG_H = 1.3 + + +def _trim_data_table_to_budget(block, budget: float): + """Return a copy of a DataTable whose rows fit within ``budget`` inches. + + Keeps the title, header, as many leading rows as fit (at least one) and an + honest note reporting how many of the original rows are shown. NEVER mutates + the original block — the same Chapter blocks are rendered by the PDF renderer, + which keeps the full table (an A5 page fits it).""" + header = list(getattr(block, "header", []) or []) + rows = list(getattr(block, "rows", []) or []) + title = getattr(block, "title", None) + fs = _FS_CELL + widths = _col_widths(header, rows) + fixed = 0.0 + if title: + fixed += _measure_heading_text(title, 2) + if header: + fixed += _row_height_in(header, widths, fs) + note_h = tl.line_height_in(_FS_NOTE) + 0.05 + avail_rows = budget - fixed - note_h - _GAP + kept = [] + used = 0.0 + for r in rows: + rh = _row_height_in(r, widths, fs) + if used + rh > avail_rows and kept: + break + kept.append(r) + used += rh + if len(kept) >= len(rows): + return block # already fits; keep the original (with its own note). + note = (f"top {len(kept)} de {len(rows)} categorías mostradas " + "(recortado para caber en el slide; el PDF muestra más)") + return model.DataTable(header=header, rows=kept, title=title, note=note) + + +def _fit_group_blocks(st: _PptxState, blocks: list, avail_full: float) -> list: + """Return a slide-fitting copy of a keep-together group's blocks. + + On the short 16:9 slide a high-cardinality column's top-k table plus its + chart can overflow. Reserve ``_GROUP_MIN_FIG_H`` for the (later shrunk) figure + and trim the data table(s) to what is left, so every column keeps its chart + next to its table on ONE slide. No-op when the group has no figure+table pair + (e.g. id-like columns already drop the top-k upstream, or it already fits).""" + has_fig = any(getattr(b, "kind", "") in ("figure", "image") for b in blocks) + tbls = [b for b in blocks if getattr(b, "kind", "") == "data_table"] + if not (has_fig and tbls): + return blocks + fixed_h = sum(_measure_block(st, b) for b in blocks + if getattr(b, "kind", "") not in ("figure", "image", + "data_table")) + tables_h = sum(_measure_block(st, b) for b in tbls) + budget_tables = avail_full - fixed_h - _GROUP_MIN_FIG_H + if tables_h <= budget_tables: + return blocks # already fits next to a min-height figure; leave intact. + out = [] + for b in blocks: + if getattr(b, "kind", "") != "data_table": + out.append(b) + continue + trimmed = _trim_data_table_to_budget(b, max(budget_tables, 0.8)) + out.append(trimmed) + budget_tables -= _measure_data_table(trimmed) + return out + + def _place_group(st: _PptxState, block) -> None: """Render a keep-together Group: move it whole to the next slide if needed.""" blocks = getattr(block, "blocks", []) or [] @@ -739,6 +810,9 @@ def _place_group(st: _PptxState, block) -> None: if getattr(block, "page_break_before", False) and st.y > _CONTENT_TOP + 1e-6: _new_slide(st, cont=True) avail_full = _CONTENT_BOTTOM - _CONTENT_TOP + # Trim oversized tables first (keeps the chart on the same slide), then shrink + # the figure to share the remaining room. + blocks = _fit_group_blocks(st, blocks, avail_full) _shrink_group_figures(st, blocks, avail_full) total = sum(_measure_block(st, b) for b in blocks) if total <= avail_full: From 7ec2bb1b453b48433852837cb5bf9547acb60f04 Mon Sep 17 00:00:00 2001 From: Egutierrez <egutierrez@dead.dd> Date: Tue, 30 Jun 2026 20:27:30 +0200 Subject: [PATCH 35/53] feat(eda): el Markdown del AutomaticEDA vuelca TODOS los datos del profile MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit El .md del grupo `eda` es la salida pensada para pegar a un LLM, así que debe contener todo lo que el motor computó, aunque el PDF/PPTX (vista humana) resuman. La evaluación 2053 detectó 6 datos que el .md perdía respecto al profile. Se cierran de forma aditiva (el .md tiene MÁS que el PDF/PPTX, sin tocar esos renderers ni los capítulos). render_automatic_eda.py pasa el profile al serializador Markdown vía meta['profile'] (un meta propio del MD; el de PDF/PPTX queda intacto). render_md_impl.py añade un "Apéndice — Datos completos del perfil" al final del documento, emitido solo cuando hay profile y degradando limpio cuando falta una sección (lite sin modelos, profile sin correlaciones). El apéndice no se acopla a los ids de capítulo (que editan otros agentes en paralelo). Pérdidas cerradas: 1. Matriz de asociación COMPLETA: los N pares de correlations.pairs (no solo el top-17), incluidos correlation_ratio (num↔cat) y cramers_v (cat↔cat). 2. Numéricas: describe completo por columna — mean/median/mode/std/variance/cv, skew y kurtosis para TODAS (no solo las asimétricas), p1/p5/p25/p50/p75/p95/ p99, iqr, min/max, outliers, distribution_type. 3. Re-expresión: nombra la transformación concreta (log1p/sqrt/yeo-johnson) con potencia, razón y alternativas, no un vago "considerar re-expresión". 4. KMeans: tabla scores_by_k (silhouette + inercia por k) marcando el k elegido. 5. Normalidad: el estadístico (stat) de cada test junto al p-value. 6. Encabezados de figuras de barras/scree dejan de heredar "Desde/Hasta/Frecuencia" del histograma; usan "Inicio/Fin/Valor" cuando el caption no es un histograma. Test nuevo md_completeness_test.py: profile sintético, asserta los N pares de correlación, skew/kurtosis de cada numérica, percentiles extendidos, log1p, scores_by_k, stat de normalidad, headers de barras y los edges (sin modelos / sin correlaciones / sin profile, defensivo). Verificado con titanic (profile_level=full): 28 pares en la tabla (incl. Sex↔Embarked cramers_v), 7 numéricas con skew+kurtosis, p5/p95/p99, scores_by_k y JB/D'Agostino/Shapiro stat presentes. PDF/PPTX/manifest siguen saliendo. Suite automatic_eda + render_automatic_eda_test: 134 passed. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> --- .../automatic_eda/md_completeness_test.py | 253 +++++++++++++++ .../automatic_eda/render_md_impl.py | 298 +++++++++++++++++- .../pipelines/render_automatic_eda.py | 10 +- 3 files changed, 556 insertions(+), 5 deletions(-) create mode 100644 python/functions/datascience/automatic_eda/md_completeness_test.py diff --git a/python/functions/datascience/automatic_eda/md_completeness_test.py b/python/functions/datascience/automatic_eda/md_completeness_test.py new file mode 100644 index 00000000..c1c910bc --- /dev/null +++ b/python/functions/datascience/automatic_eda/md_completeness_test.py @@ -0,0 +1,253 @@ +"""Tests for the Markdown completeness appendix (report 2053). + +The AutomaticEDA Markdown is the output meant to be *pasted into an LLM*, so it +must carry EVERYTHING the engine computed — even the numbers the human-facing +chapters (shared with the PDF/PPTX) drop for readability. ``render_md`` appends a +full-data appendix built from ``meta['profile']`` that closes the six losses the +evaluation found: + +1. the complete association matrix (every pair, incl. correlation_ratio / + cramers_v) — not just the top extremes; +2. every numeric statistic for every numeric column (skew/kurtosis/percentiles); +3. the concrete recommended re-expression; +4. KMeans ``scores_by_k``; +5. the normality test statistics; +6. correct headers for bar/scree figure tables (not ``Desde/Hasta/Frecuencia``). + +Self-contained: a synthetic profile, no DuckDB, no heavy renderer. +""" + +import os +import sys + +import pytest # noqa: F401 + +_HERE = os.path.dirname(os.path.abspath(__file__)) +_FUNCTIONS = os.path.abspath(os.path.join(_HERE, "..", "..", "..")) # python/functions +if _FUNCTIONS not in sys.path: + sys.path.insert(0, _FUNCTIONS) + +from datascience.automatic_eda import model # noqa: E402 +from datascience.automatic_eda.render_md_impl import ( # noqa: E402 + _bars_table, + _is_histogram_caption, + _profile_appendix, + render_md, +) + + +# --------------------------------------------------------------------------- # +# Synthetic profile fixtures. +# --------------------------------------------------------------------------- # +def _numeric(skew, kurtosis): + """A numeric stat block with every key the appendix serializes.""" + return { + "count": 100, "min": 0.0, "max": 10.0, "mean": 5.0, "median": 5.0, + "mode": 4.0, "std": 2.0, "variance": 4.0, "cv": 0.4, + "p1": 0.1, "p5": 0.5, "p25": 2.5, "p50": 5.0, "p75": 7.5, + "p95": 9.5, "p99": 9.9, "iqr": 5.0, "skew": skew, "kurtosis": kurtosis, + "n_outliers": 1, "distribution_type": "normal", + } + + +def _profile(): + """A small but structurally faithful TableProfile (3 numeric, 2 categorical).""" + pairs = [ + {"a": "A", "b": "B", "a_type": "numeric", "b_type": "numeric", + "method": "pearson/spearman", "value": 0.8, + "p_value": 1e-9, "p_value_adjusted": 2e-9, "significant": True}, + {"a": "A", "b": "C", "a_type": "numeric", "b_type": "numeric", + "method": "pearson/spearman", "value": -0.3, + "p_value": 0.01, "p_value_adjusted": 0.02, "significant": True}, + {"a": "A", "b": "Cat1", "a_type": "numeric", "b_type": "categorical", + "method": "correlation_ratio", "value": 0.45, + "p_value": 0.001, "p_value_adjusted": 0.002, "significant": True}, + # The single cat-cat pair the human chapter never shows. + {"a": "Cat1", "b": "Cat2", "a_type": "categorical", + "b_type": "categorical", "method": "cramers_v", "value": 0.11, + "p_value": 0.04, "p_value_adjusted": 0.05, "significant": False}, + ] + return { + "correlations": { + "pairs": pairs, + "multiple_testing": {"method": "bh", "n_tests": 4, "n_rejected": 3}, + }, + "columns": [ + {"name": "A", "count": 100, "numeric": _numeric(0.0, -1.2), + "reexpression": {"recommended": "none", "ladder_power": 1.0, + "reason": "symmetric", "alternatives": []}}, + {"name": "B", "count": 100, "numeric": _numeric(4.77, 33.1), + "reexpression": {"recommended": "log1p", "ladder_power": 0.0, + "reason": "skew 4.77 with zeros", + "alternatives": [{"transform": "yeo-johnson"}, + {"transform": "sqrt"}]}}, + {"name": "C", "count": 100, "numeric": _numeric(-0.6, 0.2)}, + {"name": "Cat1", "categorical": {"top": [], "mode": "x"}}, + {"name": "Cat2", "categorical": {"top": [], "mode": "y"}}, + ], + "models": { + "kmeans": { + "best_k": 3, + "scores_by_k": [ + {"k": 2, "silhouette": 0.46, "inertia": 900.0}, + {"k": 3, "silhouette": 0.50, "inertia": 550.0}, + {"k": 4, "silhouette": 0.38, "inertia": 430.0}, + ], + "cluster_sizes": [40, 35, 25], + }, + "normality": { + "A": {"n": 100, + "jarque_bera": {"stat": 18.7, "p": 8e-5, "normal": False}, + "dagostino": {"stat": 18.1, "p": 1e-4, "normal": False}, + "shapiro": {"stat": 0.98, "p": 7e-8, "normal": False}, + "is_normal": False}, + "C": {"n": 100, + "jarque_bera": {"stat": 2.1, "p": 0.35, "normal": True}, + "dagostino": {"stat": 1.9, "p": 0.38, "normal": True}, + "shapiro": {"stat": 0.99, "p": 0.12, "normal": True}, + "is_normal": True}, + }, + }, + } + + +def _dummy_chapters(): + """A minimal one-chapter document so render_md does not early-return empty.""" + return model.as_chapters([ + {"id": "intro", "title": "Intro", + "blocks": [{"kind": "markdown", "text": "cuerpo del informe"}]}, + ]) + + +def _render(tmp_path, profile): + out = os.path.join(str(tmp_path), "out.md") + res = render_md(_dummy_chapters(), out, {"title": "EDA — t", "profile": profile}) + assert res["path"] == out + return open(out, encoding="utf-8").read() + + +def _table_rows(md, section_title): + """Count data rows of the first Markdown table under ``section_title``.""" + seg = md.split(section_title, 1)[1] + rows, in_t, seen_sep = 0, False, False + for ln in seg.splitlines(): + if ln.startswith("|"): + in_t = True + stripped = ln.replace("|", "").replace(" ", "") + if stripped and set(stripped) == {"-"}: + seen_sep = True + continue + if seen_sep: + rows += 1 + elif in_t and not ln.strip(): + break + return rows + + +# --------------------------------------------------------------------------- # +# Golden: every datum the profile holds reaches the .md. +# --------------------------------------------------------------------------- # +def test_appendix_lists_all_correlation_pairs(tmp_path): + md = _render(tmp_path, _profile()) + assert "## Apéndice — Datos completos del perfil" in md + # All 4 pairs (the real titanic profile has 28; here 4 synthetic). + assert _table_rows(md, "### Matriz de asociación") == 4 + # The cat-cat Cramér's V pair the human chapter drops is present. + assert "Cat1 ↔ Cat2" in md + assert "cramers_v" in md + assert "correlation_ratio" in md + + +def test_appendix_has_skew_kurtosis_for_every_numeric(tmp_path): + md = _render(tmp_path, _profile()) + seg = md.split("### Estadísticos numéricos completos", 1)[1].split("###", 1)[0] + lines = [l for l in seg.splitlines() if l.startswith("|")] + header = [h.strip() for h in lines[0].strip("|").split("|")] + assert "skew" in header and "kurtosis" in header + ski, kui = header.index("skew"), header.index("kurtosis") + data = lines[2:] # skip header + separator + assert len(data) == 3 # exactly the 3 numeric columns + for row in data: + cells = [c.strip() for c in row.strip("|").split("|")] + assert cells[ski] != "", f"missing skew in {cells[0]}" + assert cells[kui] != "", f"missing kurtosis in {cells[0]}" + + +def test_appendix_has_extended_percentiles(tmp_path): + md = _render(tmp_path, _profile()) + seg = md.split("### Estadísticos numéricos completos", 1)[1] + header = [h.strip() for h in seg.splitlines()[2].strip("|").split("|")] + for p in ("p1", "p5", "p25", "p75", "p95", "p99"): + assert p in header, f"percentile {p} missing from describe header" + + +def test_appendix_names_concrete_reexpression(tmp_path): + md = _render(tmp_path, _profile()) + assert "### Re-expresión recomendada" in md + assert "log1p" in md # the concrete transform, not just "consider re-expressing" + assert "yeo-johnson" in md # alternatives listed too + + +def test_appendix_has_kmeans_scores_by_k(tmp_path): + md = _render(tmp_path, _profile()) + assert "scores_by_k" in md + assert _table_rows(md, "#### KMeans — selección de k") == 3 # k=2,3,4 + + +def test_appendix_has_normality_statistics(tmp_path): + md = _render(tmp_path, _profile()) + assert "JB stat" in md # the statistic, not only the p-value + assert "Shapiro stat" in md + assert _table_rows(md, "#### Tests de normalidad") == 2 # cols A and C + + +# --------------------------------------------------------------------------- # +# Edge: a profile missing models / correlations degrades, never raises. +# --------------------------------------------------------------------------- # +def test_lite_profile_without_models(tmp_path): + prof = _profile() + prof.pop("models") # lite: no KMeans/normality + md = _render(tmp_path, prof) + assert "scores_by_k" not in md # section skipped + assert "Matriz de asociación" in md # correlations still dumped + assert "## Apéndice" in md + + +def test_profile_without_correlations(tmp_path): + prof = _profile() + prof.pop("correlations") + md = _render(tmp_path, prof) # must not raise + assert "Matriz de asociación" not in md + assert "Estadísticos numéricos completos" in md # numeric section still there + + +def test_no_profile_means_no_appendix(tmp_path): + out = os.path.join(str(tmp_path), "noprof.md") + res = render_md(_dummy_chapters(), out, {"title": "x"}) + assert res["path"] == out + assert "## Apéndice" not in open(out, encoding="utf-8").read() + + +def test_appendix_helper_is_defensive(): + assert _profile_appendix(None) == "" + assert _profile_appendix({}) == "" + assert _profile_appendix({"columns": []}) == "" + + +# --------------------------------------------------------------------------- # +# Loss #6: bar/scree figure tables get a non-misleading header. +# --------------------------------------------------------------------------- # +def test_histogram_caption_detection(): + assert _is_histogram_caption("Histograma de Age") + assert _is_histogram_caption("Distribución de Fare") + assert not _is_histogram_caption("Media de Survived por Sex") + assert not _is_histogram_caption("Varianza explicada (scree PCA)") + + +def test_bars_table_custom_header(): + bars = [(0.0, 1.0, 5.0), (1.0, 2.0, 3.0)] + hist = _bars_table(bars) # default histogram header + assert "| Desde | Hasta | Frecuencia |" in hist + bar = _bars_table(bars, ("Inicio", "Fin", "Valor")) + assert "| Inicio | Fin | Valor |" in bar + assert "Frecuencia" not in bar diff --git a/python/functions/datascience/automatic_eda/render_md_impl.py b/python/functions/datascience/automatic_eda/render_md_impl.py index fba8ba6f..08488af0 100644 --- a/python/functions/datascience/automatic_eda/render_md_impl.py +++ b/python/functions/datascience/automatic_eda/render_md_impl.py @@ -178,9 +178,17 @@ def _md_data_table(block) -> str: return "\n".join(lines) -def _bars_table(bars: list) -> str: - """Render extracted bar/histogram data as a Markdown table (Desde/Hasta/Frec).""" - lines = ["| Desde | Hasta | Frecuencia |", "| --- | --- | --- |"] +def _bars_table(bars: list, header: tuple = ("Desde", "Hasta", "Frecuencia")) -> str: + """Render extracted bar/histogram data as a Markdown table. + + ``header`` is the 3-column header to use. Histogram bars are + ``(Desde, Hasta, Frecuencia)``; bar/scree charts (means by group, PCA + explained variance) are *not* bins, so the caller passes a semantically + correct header (e.g. ``(Inicio, Fin, Valor)``) to avoid the misleading + "Frecuencia" label — see report 2053, loss #6. + """ + h0, h1, h2 = header + lines = [f"| {h0} | {h1} | {h2} |", "| --- | --- | --- |"] shown = bars[:_MAX_BAR_ROWS] for x0, x1, h in shown: lines.append(f"| {_fmt_num(x0)} | {_fmt_num(x1)} | {_fmt_num(h)} |") @@ -191,6 +199,18 @@ def _bars_table(bars: list) -> str: return out +def _is_histogram_caption(caption: str) -> bool: + """True when a figure caption describes a histogram (genuine numeric bins). + + Histograms are the only figures whose bars are real ``[Desde, Hasta)`` bins + with a frequency count. Bar charts (means by group) and the PCA scree plot + carry per-category / per-component values, not bins — they must not inherit + the ``Desde/Hasta/Frecuencia`` header. + """ + c = (caption or "").lower() + return "histograma" in c or "distribución" in c or "distribucion" in c + + def _extract_bars(fig) -> list: """Collect (x_from, x_to, height) of the rectangular bars of a matplotlib fig. @@ -253,7 +273,13 @@ def _md_figure(block, meta: dict, out_path: str, counter: list) -> str: if fig is not None: bars = _extract_bars(fig) if bars: - parts.append(_bars_table(bars)) + # A histogram's bars are genuine numeric bins (Desde/Hasta/ + # Frecuencia). Bar charts and the PCA scree plot are not bins — + # give them a header that does not lie about "Frecuencia". + header = (("Desde", "Hasta", "Frecuencia") + if _is_histogram_caption(caption) + else ("Inicio", "Fin", "Valor")) + parts.append(_bars_table(bars, header)) if meta.get("embed_figures"): png = _embed_png(fig, out_path, counter) if png: @@ -354,6 +380,258 @@ def _serialize_block(block, meta: dict, out_path: str, counter: list) -> str: return _md_note(model.Note(text=model._safe_str(block))) +# --------------------------------------------------------------------------- # +# Profile appendix — the data the human-facing chapters drop. +# +# The chapter document (shared with the PDF/PPTX renderers) is designed for human +# reading and intentionally omits raw numbers: the correlation matrix shows only +# the top extremes, the numeric blocks skip skew/kurtosis/extended percentiles, +# the model chapter does not list ``scores_by_k`` or the normality test +# statistics. But the Markdown is meant to be *pasted into an LLM*, so it should +# carry EVERYTHING the engine computed. This appendix serializes the full +# ``profile`` (passed via ``meta['profile']``) as Markdown tables, additively: +# the PDF/PPTX are untouched, the .md simply has more than they do. Each section +# is emitted only when its source data is present, so a ``lite`` profile (no +# models) or a profile without correlations degrades cleanly instead of raising. +# See report 2053 for the six losses this closes. +# --------------------------------------------------------------------------- # +def _pair_types(a_type, b_type) -> str: + """Short ``num↔cat`` label for an association pair's variable types.""" + def short(t): + t = model._safe_str(t).lower() + if t.startswith("num"): + return "num" + if t.startswith("cat"): + return "cat" + return t or "?" + return f"{short(a_type)}↔{short(b_type)}" + + +def _app_correlations(corr: dict) -> str: + """Loss #1 — every association pair (not just the top extremes). + + Dumps all of ``correlations['pairs']`` as a table (pair · types · method · + value · p · p-FDR · significant), ordered by |value| desc so the strongest + associations lead while nothing is cut. Includes the ``correlation_ratio`` + (num↔cat) and ``cramers_v`` (cat↔cat) pairs the human chapter never shows. + """ + pairs = list(corr.get("pairs", []) or []) + if not pairs: + return "" + def keyfn(p): + try: + return -abs(float(p.get("value"))) + except Exception: # noqa: BLE001 + return 0.0 + pairs_sorted = sorted(pairs, key=keyfn) + lines = ["### Matriz de asociación — todos los pares", + "", + ("| Par | Tipos | Método | Valor | p-value | p-ajustado (FDR) " + "| ¿Sig? |"), + "| --- | --- | --- | --- | --- | --- | --- |"] + for p in pairs_sorted: + par = f"{_cell(p.get('a'))} ↔ {_cell(p.get('b'))}" + types = _pair_types(p.get("a_type"), p.get("b_type")) + method = _cell(p.get("method")) + val = _fmt_num(p.get("value")) + pv = _fmt_num(p.get("p_value")) if p.get("p_value") is not None else "" + padj = (_fmt_num(p.get("p_value_adjusted")) + if p.get("p_value_adjusted") is not None else "") + sig = "sí" if p.get("significant") else "no" + lines.append( + f"| {par} | {types} | {method} | {val} | {pv} | {padj} | {sig} |") + mt = corr.get("multiple_testing") or {} + n_tests = mt.get("n_tests", corr.get("n_tests")) + n_rej = mt.get("n_rejected") + note_bits = [f"{len(pairs)} pares en total"] + if n_tests is not None and n_rej is not None: + note_bits.append( + f"{n_rej} de {n_tests} significativos tras corrección " + f"{model._safe_str(mt.get('method', 'FDR')).upper()}") + lines.append("") + lines.append(f"*{'; '.join(note_bits)}.*") + return "\n".join(lines) + + +# Numeric statistics, in serialization order: (profile key, column header). +_NUM_STATS = [ + ("count", "n"), ("mean", "mean"), ("median", "median"), ("mode", "mode"), + ("std", "std"), ("variance", "variance"), ("cv", "cv"), + ("skew", "skew"), ("kurtosis", "kurtosis"), + ("min", "min"), ("p1", "p1"), ("p5", "p5"), ("p25", "p25"), ("p50", "p50"), + ("p75", "p75"), ("p95", "p95"), ("p99", "p99"), ("iqr", "iqr"), + ("max", "max"), ("n_outliers", "outliers"), + ("distribution_type", "distribución"), +] + + +def _app_numeric_describe(columns: list) -> str: + """Loss #2 — every numeric statistic for every numeric column. + + One row per numeric column with the full describe: mean/median/mode/std/ + variance/cv, skew & kurtosis (for ALL columns, not only the skewed ones), + p1/p5/p25/p50/p75/p95/p99, iqr, min/max, outliers and distribution_type. + """ + rows = [] + for info in (columns or []): + num = info.get("numeric") if isinstance(info, dict) else None + if not num: + continue + name = _cell(info.get("name")) + cells = [name] + for key, _hdr in _NUM_STATS: + v = num.get("count" if key == "count" else key) + if key == "count": + v = num.get("count", info.get("count")) + if key == "distribution_type": + cells.append(_cell(v)) + else: + cells.append(_fmt_num(v) if v is not None else "") + rows.append(cells) + if not rows: + return "" + header = ["Columna"] + [hdr for _k, hdr in _NUM_STATS] + lines = ["### Estadísticos numéricos completos (describe)", + "", + "| " + " | ".join(header) + " |", + "| " + " | ".join(["---"] * len(header)) + " |"] + for cells in rows: + lines.append("| " + " | ".join(cells) + " |") + return "\n".join(lines) + + +def _app_reexpression(columns: list) -> str: + """Loss #3 — the concrete recommended re-expression per column. + + Names the transform (log1p/sqrt/yeo-johnson/none) instead of a vague + "consider re-expressing", with the ladder power, reason and alternatives. + """ + rows = [] + for info in (columns or []): + rx = info.get("reexpression") if isinstance(info, dict) else None + if not rx or not isinstance(rx, dict): + continue + rec = model._safe_str(rx.get("recommended")).strip() + if not rec: + continue + alts = rx.get("alternatives") or [] + alt_txt = ", ".join( + model._safe_str(a.get("transform")) for a in alts + if isinstance(a, dict) and a.get("transform")) or "—" + rows.append([ + _cell(info.get("name")), _cell(rec), + _fmt_num(rx.get("ladder_power")) if rx.get("ladder_power") is not None else "", + _cell(rx.get("reason")), _cell(alt_txt), + ]) + if not rows: + return "" + lines = ["### Re-expresión recomendada (escalera de Tukey)", + "", + "| Columna | Recomendada | Potencia | Razón | Alternativas |", + "| --- | --- | --- | --- | --- |"] + for r in rows: + lines.append("| " + " | ".join(r) + " |") + return "\n".join(lines) + + +def _app_kmeans_scores(kmeans: dict) -> str: + """Loss #4 — KMeans silhouette + inertia per k (justifies the chosen k).""" + scores = list(kmeans.get("scores_by_k", []) or []) + if not scores: + return "" + best_k = kmeans.get("best_k") + lines = ["#### KMeans — selección de k (`scores_by_k`)", + "", + "| k | Silhouette | Inercia | Elegido |", + "| --- | --- | --- | --- |"] + for s in scores: + if not isinstance(s, dict): + continue + k = s.get("k") + chosen = "✓" if best_k is not None and k == best_k else "" + lines.append( + f"| {_fmt_num(k)} | {_fmt_num(s.get('silhouette'))} " + f"| {_fmt_num(s.get('inertia'))} | {chosen} |") + return "\n".join(lines) + + +def _app_normality(normality: dict) -> str: + """Loss #5 — each normality test's statistic next to its p-value.""" + if not isinstance(normality, dict) or not normality: + return "" + lines = ["#### Tests de normalidad (estadístico + p-value)", + "", + ("| Columna | n | JB stat | JB p | D'Agostino stat | D'Agostino p " + "| Shapiro stat | Shapiro p | ¿Normal? |"), + "| --- | --- | --- | --- | --- | --- | --- | --- | --- |"] + any_row = False + for col, res in normality.items(): + if not isinstance(res, dict): + continue + jb = res.get("jarque_bera") or {} + da = res.get("dagostino") or {} + sh = res.get("shapiro") or {} + is_norm = "sí" if res.get("is_normal") else "no" + lines.append( + f"| {_cell(col)} | {_fmt_num(res.get('n')) if res.get('n') is not None else ''} " + f"| {_fmt_num(jb.get('stat'))} | {_fmt_num(jb.get('p'))} " + f"| {_fmt_num(da.get('stat'))} | {_fmt_num(da.get('p'))} " + f"| {_fmt_num(sh.get('stat'))} | {_fmt_num(sh.get('p'))} | {is_norm} |") + any_row = True + return "\n".join(lines) if any_row else "" + + +def _profile_appendix(profile: dict) -> str: + """Build the full-data appendix from a TableProfile dict (additive). + + Returns a Markdown ``## Apéndice`` section with one sub-table per loss the + human chapters drop, or ``""`` when the profile carries none of them. Never + raises: a missing/oddly-shaped section is skipped, not fatal. + """ + if not isinstance(profile, dict): + return "" + sections: list = [] + try: + corr = profile.get("correlations") or {} + seg = _app_correlations(corr) if isinstance(corr, dict) else "" + if seg: + sections.append(seg) + except Exception: # noqa: BLE001 + pass + try: + columns = profile.get("columns") or [] + seg = _app_numeric_describe(columns) + if seg: + sections.append(seg) + seg = _app_reexpression(columns) + if seg: + sections.append(seg) + except Exception: # noqa: BLE001 + pass + try: + models = profile.get("models") or {} + if isinstance(models, dict): + model_segs = [] + seg = _app_kmeans_scores(models.get("kmeans") or {}) + if seg: + model_segs.append(seg) + seg = _app_normality(models.get("normality") or {}) + if seg: + model_segs.append(seg) + if model_segs: + sections.append( + "### Modelos — detalle\n\n" + "\n\n".join(model_segs)) + except Exception: # noqa: BLE001 + pass + if not sections: + return "" + intro = ("Volcado completo de los datos que el motor computó y que los " + "capítulos (pensados para lectura humana / PDF) resumen. " + "Pensado para que un LLM reconstruya el análisis entero.") + return ("## Apéndice — Datos completos del perfil\n\n" + f"*{intro}*\n\n" + "\n\n".join(sections)) + + # --------------------------------------------------------------------------- # # Entry point. # --------------------------------------------------------------------------- # @@ -437,6 +715,18 @@ def render_md(chapters: list, out_path: str, meta: dict = None) -> dict: segments.append(seg) chapters_meta.append({"id": ch.id, "version": ch.version}) + # Full-data appendix: dump everything the profile holds that the human + # chapters drop (additive — the .md ends up with more than the PDF/PPTX). + # Emitted only when a profile is supplied via meta['profile']; never fatal. + try: + appendix = _profile_appendix(meta.get("profile")) + except Exception as e: # noqa: BLE001 + appendix = "" + notes.append(f"apéndice de perfil omitido: {e}") + if appendix: + segments.append("---") + segments.append(appendix) + content = "\n\n".join(segments) + "\n" note = f"{len(content)} caracteres" if notes: diff --git a/python/functions/pipelines/render_automatic_eda.py b/python/functions/pipelines/render_automatic_eda.py index 5361b927..942ee456 100644 --- a/python/functions/pipelines/render_automatic_eda.py +++ b/python/functions/pipelines/render_automatic_eda.py @@ -261,7 +261,15 @@ def render_automatic_eda( md_path = None if emit_md: md_path = os.path.join(out_dir, base + ".md") - rmd = render_automatic_eda_markdown(prof, md_path, meta) or {} + # El Markdown es la salida MÁS completa: además del documento por + # capítulos (compartido con PDF/PPTX) volca un apéndice con TODOS los + # datos numéricos del perfil (matriz de asociación completa, describe + # con skew/kurtosis/percentiles, re-expresiones, scores_by_k de + # KMeans, estadísticos de normalidad). Se le pasa el `prof` vía + # meta['profile']; un meta propio evita alterar el de PDF/PPTX. + md_meta = dict(meta) + md_meta["profile"] = prof + rmd = render_automatic_eda_markdown(prof, md_path, md_meta) or {} return { "status": "ok", From 6a1520f458cb973c0bd0b22e10ea8eed8796bbf6 Mon Sep 17 00:00:00 2001 From: Egutierrez <egutierrez@dead.dd> Date: Tue, 30 Jun 2026 20:34:10 +0200 Subject: [PATCH 36/53] feat(eda): EDA de carpeta/base multi-tabla -> AutomaticEDA por capitulos (PDF+PPTX+MD) Pipeline render_automatic_eda_folder: apunta el AutomaticEDA a una CARPETA de archivos tabulares (CSV/Parquet/JSON) o a una DuckDB existente y emite el informe de la BASE por capitulos en PDF (A5 movil) + PPTX (16:9) + Markdown. Documento-base con portada-base, resumen de todas las tablas y relaciones inter-tabla (FK candidatas por containment + diagrama Mermaid del join graph). Flag per_table_eda anexa el mini-EDA de cada tabla. Aditivo: render_automatic_eda (tabla unica) intacto. Funcion nueva load_folder_to_duckdb (infra, grupo eda+duckdb): carga una carpeta a una DuckDB (temp si no se da path), CREATE TABLE por archivo con read_csv_auto/ read_parquet/read_json_auto. dict-no-throw. Compone profile_database + los 3 renderers del motor AutomaticEDA + build_document (per-tabla), sin reimplementar su logica. Tests: golden 3 CSV relacionados (FK orders.customer_id->customers.id detectada) + edges (carpeta vacia, 1 tabla, DuckDB existente, path inexistente). fn index sin error. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> --- python/functions/infra/__init__.py | 2 + .../functions/infra/load_folder_to_duckdb.md | 100 +++++ .../functions/infra/load_folder_to_duckdb.py | 175 +++++++++ .../infra/load_folder_to_duckdb_test.py | 73 ++++ .../pipelines/render_automatic_eda_folder.md | 112 ++++++ .../pipelines/render_automatic_eda_folder.py | 350 ++++++++++++++++++ .../render_automatic_eda_folder_test.py | 146 ++++++++ 7 files changed, 958 insertions(+) create mode 100644 python/functions/infra/load_folder_to_duckdb.md create mode 100644 python/functions/infra/load_folder_to_duckdb.py create mode 100644 python/functions/infra/load_folder_to_duckdb_test.py create mode 100644 python/functions/pipelines/render_automatic_eda_folder.md create mode 100644 python/functions/pipelines/render_automatic_eda_folder.py create mode 100644 python/functions/pipelines/render_automatic_eda_folder_test.py diff --git a/python/functions/infra/__init__.py b/python/functions/infra/__init__.py index f58a79ea..e1401f4b 100644 --- a/python/functions/infra/__init__.py +++ b/python/functions/infra/__init__.py @@ -34,6 +34,7 @@ from .upsert_xlsx_sheet import upsert_xlsx_sheet from .duckdb_query_readonly import duckdb_query_readonly from .duckdb_execute import duckdb_execute from .duckdb_upsert import duckdb_upsert +from .load_folder_to_duckdb import load_folder_to_duckdb from .imap_connect import imap_connect from .imap_list_mailboxes import imap_list_mailboxes from .imap_search import imap_search @@ -50,6 +51,7 @@ __all__ = [ "upsert_xlsx_sheet", "duckdb_query_readonly", "duckdb_execute", + "load_folder_to_duckdb", "duckdb_upsert", "pg_insert_rows", "pg_apply_sql", diff --git a/python/functions/infra/load_folder_to_duckdb.md b/python/functions/infra/load_folder_to_duckdb.md new file mode 100644 index 00000000..e268582f --- /dev/null +++ b/python/functions/infra/load_folder_to_duckdb.md @@ -0,0 +1,100 @@ +--- +name: load_folder_to_duckdb +kind: function +lang: py +domain: infra +version: "1.0.0" +purity: impure +signature: "def load_folder_to_duckdb(folder: str, db_path: str = None, pattern: str = '*.csv,*.parquet,*.json') -> dict" +description: "Escanea el primer nivel de una CARPETA buscando archivos tabulares (CSV/TSV/TXT, Parquet, JSON/NDJSON) y los carga como tablas en una base DuckDB usando los lectores nativos read_csv_auto/read_parquet/read_json_auto. Es la pieza de entrada del EDA a nivel de carpeta (grupo eda). Por cada archivo crea una tabla cuyo nombre se deriva del basename saneado a [0-9a-zA-Z_] en minusculas (prefijo t_ si empieza por digito, sufijos _2/_3 ante colisiones, tabla_<i> si queda vacio). El path se escapa (comilla simple '->'') antes de interpolarlo porque los lectores DuckDB no aceptan el path como parametro posicional. Glob NO recursivo: un glob.glob(os.path.join(folder, g)) por cada patron del CSV, dedup y ordenado. db_path=None genera una DuckDB temporal (mkstemp, se borra el placeholder vacio porque DuckDB rechaza un archivo de 0 bytes) y devuelve su ruta. Un fallo al cargar un archivo concreto no aborta el resto: se registra en errors y se continua. Devuelve siempre un dict sin lanzar (estilo del grupo duckdb): {status:'ok', db_path, tables, errors} en exito (carpeta sin archivos tabulares incluida, tables=[]) y {status:'error', error} cuando la carpeta no existe o falla algo global. Depende del paquete duckdb (1.5.2)." +tags: [eda, duckdb, ingest, etl, folder] +uses_functions: [] +uses_types: [] +returns: [] +returns_optional: false +error_type: "error_py_core" +imports: [glob, os, re, tempfile, duckdb] +params: + - name: folder + desc: "ruta a un directorio. Se escanea solo su primer nivel (NO recursivo). Si no existe o no es un directorio devuelve {status:'error'} sin lanzar." + - name: db_path + desc: "ruta del archivo DuckDB destino, abierto en modo read-write (lo crea si no existe). None (default) genera una DuckDB temporal unica con tempfile.mkstemp y devuelve su ruta en el campo db_path del retorno. DuckDB es single-writer: si otro proceso lo tiene abierto en escritura, connect falla con error de lock devuelto en el dict." + - name: pattern + desc: "CSV de globs separados por coma (default '*.csv,*.parquet,*.json'). Cada glob se aplica con glob.glob(os.path.join(folder, g)) sobre el primer nivel de folder; los resultados de todos los globs se deduplican y ordenan. Los globs con ** NO descienden recursivamente (glob.glob sin recursive=True)." +output: "dict. En exito: {status:'ok', db_path:str (ruta DuckDB usada), tables:[{name:str, source_file:str, n_rows:int}], errors:[{name?:str, source_file:str, error:str}]}. La carpeta sin archivos tabulares es un exito con tables=[] y errors=[]. En error (sin lanzar): {status:'error', error:str}." +tested: true +tests: + - "test_carga_dos_csv_como_tablas" + - "test_db_path_none_crea_temporal" + - "test_carpeta_vacia_es_ok_sin_tablas" + - "test_carpeta_inexistente_devuelve_status_error" +test_file_path: "python/functions/infra/load_folder_to_duckdb_test.py" +file_path: "python/functions/infra/load_folder_to_duckdb.py" +--- + +## Ejemplo + +```python +import sys +sys.path.insert(0, "python/functions") +from infra.load_folder_to_duckdb import load_folder_to_duckdb + +# Preparar una carpeta de demo con dos CSV. +import os +os.makedirs("/tmp/eda_folder_demo", exist_ok=True) +with open("/tmp/eda_folder_demo/ventas.csv", "w") as f: + f.write("id,total\n1,10.5\n2,20.0\n3,5.25\n") +with open("/tmp/eda_folder_demo/clientes.csv", "w") as f: + f.write("id,nombre\n1,ana\n2,luis\n") + +# Cargar todos los tabulares de la carpeta a una DuckDB temporal. +res = load_folder_to_duckdb("/tmp/eda_folder_demo") +print(res["status"]) # ok +print(res["db_path"]) # /tmp/tmpXXXXXXXX.duckdb (temporal) +for t in res["tables"]: + print(t["name"], t["n_rows"]) # ventas 3 / clientes 2 + +# Persistir en una DuckDB concreta y limitar a CSV. +res2 = load_folder_to_duckdb( + "/tmp/eda_folder_demo", + db_path="/tmp/eda_folder_demo/folder.duckdb", + pattern="*.csv", +) +print(res2["tables"]) # [{'name': 'clientes', ...}, {'name': 'ventas', ...}] +``` + +## Cuando usarla + +Cuando tienes una carpeta de datos sueltos (un dump, un export, varios CSV/Parquet +descargados) y quieres analizarlos juntos con SQL sin montar la ingesta a mano, +archivo por archivo. Es el primer eslabon del EDA a nivel de carpeta (grupo `eda`): +deja una DuckDB con una tabla por archivo, lista para perfilar con +`duckdb_table_schema_py_infra`, consultar con `duckdb_query_readonly_py_infra`, o +correlacionar aguas abajo. Usala antes de cualquier paso de perfilado cuando la +unidad de trabajo es "todos los archivos de este directorio". + +## Gotchas + +- **Glob NO recursivo**: solo se escanea el primer nivel de `folder`. Archivos en + subdirectorios se ignoran (ni siquiera con `**` en el patron, porque + `glob.glob` se llama sin `recursive=True`). Si necesitas recursion, aplana la + carpeta antes o amplia la funcion. +- **Saneo de nombres de tabla**: el basename se reduce a `[0-9a-zA-Z_]` en + minusculas. `Ventas 2024.csv` -> tabla `ventas_2024`. Dos archivos distintos + pueden sanear al mismo nombre (`a-b.csv` y `a_b.csv`); el segundo se desambigua + con sufijo `_2`, `_3`, ... El mapeo real archivo->tabla esta en `tables[].name` + / `tables[].source_file`, no lo asumas. +- **`read_json_auto` requiere JSON tabular** (array de objetos u objetos NDJSON + homogeneos). Un JSON anidado o irregular puede fallar la carga de ESA tabla; el + error se registra en `errors` y el resto de archivos siguen cargandose. +- **Extension desconocida = se salta**, no falla: queda anotada en `errors` con + `unsupported extension`. Mapeo de lectores: `.csv/.tsv/.txt`->`read_csv_auto`, + `.parquet/.pq`->`read_parquet`, `.json/.ndjson`->`read_json_auto`. +- **Escritura real en disco (impura)**. DuckDB es single-writer: si otro proceso + tiene `db_path` abierto en escritura, `connect` falla con error de lock devuelto + en el dict. Un `db_path` con un directorio padre inexistente tambien falla. +- **`db_path=None` crea un archivo temporal que NO se borra solo**: la ruta se + devuelve en `db_path` para que el llamador la consuma y la limpie cuando termine. +- **Tipos inferidos por los lectores `_auto`**: los tipos de columna los infiere + DuckDB. Revisa el schema con `duckdb_table_schema_py_infra` si el tipado importa + aguas abajo. diff --git a/python/functions/infra/load_folder_to_duckdb.py b/python/functions/infra/load_folder_to_duckdb.py new file mode 100644 index 00000000..2e85d905 --- /dev/null +++ b/python/functions/infra/load_folder_to_duckdb.py @@ -0,0 +1,175 @@ +"""Carga una carpeta de archivos tabulares (CSV/Parquet/JSON) como tablas DuckDB. + +Funcion impura: escanea el primer nivel de un directorio buscando archivos que +casen con uno o varios globs, y por cada archivo crea una tabla en una base +DuckDB usando los lectores nativos (`read_csv_auto`, `read_parquet`, +`read_json_auto`). Es la pieza de entrada del EDA a nivel de carpeta (grupo +`eda`): deja una DuckDB con una tabla por archivo, lista para perfilar y +correlacionar aguas abajo. + +Devuelve siempre un dict sin lanzar excepciones, siguiendo el estilo del grupo +duckdb del registry: {status:'ok', db_path, tables, errors} en exito (incluida +la carpeta sin archivos tabulares, que es un exito con tables=[]) y +{status:'error', error:str} cuando la carpeta no existe o falla algo global. + +El nombre de cada tabla se deriva del basename del archivo, saneado a +`[0-9a-zA-Z_]` en minusculas, prefijado con `t_` si empieza por digito, y +desambiguado con sufijos `_2`, `_3`, ... ante colisiones. El path del archivo se +escapa (comilla simple, `'`->`''`) antes de interpolarlo en el SQL del lector, +ya que los lectores DuckDB no admiten el path como parametro posicional. Un fallo +al cargar un archivo concreto NO aborta el resto: se registra en `errors` y se +continua con los siguientes. +""" + +import glob +import os +import re +import tempfile + + +def _sanitize_table_name(basename_no_ext: str, index: int) -> str: + """Deriva un identificador de tabla valido desde el basename de un archivo. + + Reemplaza todo lo que no sea ``[0-9a-zA-Z_]`` por ``_`` y baja a minusculas. + Si tras el saneo queda vacio, usa ``tabla_<index>``. Si empieza por digito, + prefija ``t_`` para que sea un identificador SQL valido. + """ + name = re.sub(r"[^0-9a-zA-Z_]", "_", basename_no_ext).lower() + if not name: + name = f"tabla_{index}" + if name[0].isdigit(): + name = "t_" + name + return name + + +def _reader_for_extension(ext: str, quoted_path: str): + """Devuelve la expresion de lector DuckDB para una extension, o None. + + El ``quoted_path`` ya viene escapado y entre comillas simples. Extensiones + desconocidas devuelven None para que el llamador salte el archivo. + """ + ext = ext.lower() + if ext in (".csv", ".tsv", ".txt"): + return f"read_csv_auto('{quoted_path}')" + if ext in (".parquet", ".pq"): + return f"read_parquet('{quoted_path}')" + if ext in (".json", ".ndjson"): + return f"read_json_auto('{quoted_path}')" + return None + + +def load_folder_to_duckdb( + folder: str, + db_path: str = None, + pattern: str = "*.csv,*.parquet,*.json", +) -> dict: + """Carga los archivos tabulares de una carpeta como tablas en una DuckDB. + + Args: + folder: ruta a un directorio. Si no existe o no es un directorio, + devuelve {status:'error', ...} sin lanzar. + db_path: ruta de la DuckDB destino (read-write, se crea si no existe). Si + es None, se genera una base temporal con NamedTemporaryFile y su ruta + se devuelve en el retorno (`db_path`). + pattern: CSV de globs separados por coma (default + "*.csv,*.parquet,*.json"). Cada glob se aplica con + glob.glob(os.path.join(folder, g)) en el primer nivel (NO recursivo); + los resultados se deduplican y ordenan. + + Returns: + dict. En exito: {status:'ok', db_path:str, tables:[{name, source_file, + n_rows}], errors:[{name?, source_file, error}]}. La carpeta sin archivos + tabulares es un exito con tables=[] y errors=[]. En error (sin lanzar): + {status:'error', error:str}. + """ + if not isinstance(folder, str) or not os.path.isdir(folder): + return { + "status": "error", + "error": f"folder does not exist or is not a directory: {folder!r}", + } + + conn = None + try: + # Resolver la ruta de la DuckDB destino. Si no se da, reservar un nombre + # temporal unico y borrar el archivo vacio que crea mkstemp: DuckDB 1.5.2 + # rechaza abrir un archivo de 0 bytes ("not a valid DuckDB database + # file"), por lo que debe crear el archivo el mismo desde cero. + if db_path is None: + fd, tmp_name = tempfile.mkstemp(suffix=".duckdb") + os.close(fd) + os.remove(tmp_name) + db_path = tmp_name + + # Resolver los archivos: un glob por cada patron, dedup + orden estable. + globs = [g.strip() for g in pattern.split(",") if g.strip()] + found = set() + for g in globs: + for path in glob.glob(os.path.join(folder, g)): + if os.path.isfile(path): + found.add(path) + files = sorted(found) + + conn = __import__("duckdb").connect(db_path) + + tables = [] + errors = [] + used_names = set() + + for i, path in enumerate(files): + base = os.path.basename(path) + stem, ext = os.path.splitext(base) + quoted_path = path.replace("'", "''") + reader = _reader_for_extension(ext, quoted_path) + if reader is None: + errors.append( + { + "source_file": path, + "error": f"unsupported extension: {ext!r}", + } + ) + continue + + name = _sanitize_table_name(stem, i) + # Desambiguar colisiones con sufijos _2, _3, ... + if name in used_names: + suffix = 2 + while f"{name}_{suffix}" in used_names: + suffix += 1 + name = f"{name}_{suffix}" + + quoted_ident = '"' + name.replace('"', '""') + '"' + try: + conn.execute( + f"CREATE TABLE {quoted_ident} AS SELECT * FROM {reader}" + ) + n_rows = conn.execute( + f"SELECT count(*) FROM {quoted_ident}" + ).fetchone()[0] + used_names.add(name) + tables.append( + { + "name": name, + "source_file": path, + "n_rows": int(n_rows), + } + ) + except Exception as e: # noqa: BLE001 + errors.append( + { + "name": name, + "source_file": path, + "error": str(e), + } + ) + + return { + "status": "ok", + "db_path": db_path, + "tables": tables, + "errors": errors, + } + except Exception as e: # noqa: BLE001 + return {"status": "error", "error": str(e)} + finally: + if conn is not None: + conn.close() diff --git a/python/functions/infra/load_folder_to_duckdb_test.py b/python/functions/infra/load_folder_to_duckdb_test.py new file mode 100644 index 00000000..5b8becae --- /dev/null +++ b/python/functions/infra/load_folder_to_duckdb_test.py @@ -0,0 +1,73 @@ +"""Tests para load_folder_to_duckdb.""" + +import os +import sys + +sys.path.insert(0, os.path.dirname(__file__)) + +import duckdb # noqa: E402 + +from load_folder_to_duckdb import load_folder_to_duckdb # noqa: E402 + + +def _write_csv(path: str, header: str, rows: list[str]) -> None: + with open(path, "w", encoding="utf-8") as f: + f.write(header + "\n") + for r in rows: + f.write(r + "\n") + + +def test_carga_dos_csv_como_tablas(tmp_path): + _write_csv( + str(tmp_path / "ventas.csv"), + "id,total", + ["1,10.5", "2,20.0", "3,5.25"], + ) + _write_csv( + str(tmp_path / "clientes.csv"), + "id,nombre", + ["1,ana", "2,luis"], + ) + db = tmp_path / "out.duckdb" + res = load_folder_to_duckdb(str(tmp_path), str(db)) + + assert res["status"] == "ok", res + assert res["errors"] == [] + assert len(res["tables"]) == 2 + assert res["db_path"] == str(db) + assert os.path.exists(str(db)) + + by_name = {t["name"]: t for t in res["tables"]} + assert by_name["ventas"]["n_rows"] == 3 + assert by_name["clientes"]["n_rows"] == 2 + + # Verificar que las tablas existen realmente en la base. + con = duckdb.connect(str(db), read_only=True) + assert con.execute("SELECT count(*) FROM ventas").fetchone()[0] == 3 + assert con.execute("SELECT count(*) FROM clientes").fetchone()[0] == 2 + con.close() + + +def test_db_path_none_crea_temporal(tmp_path): + _write_csv(str(tmp_path / "datos.csv"), "x", ["1", "2"]) + res = load_folder_to_duckdb(str(tmp_path)) + assert res["status"] == "ok", res + assert res["db_path"] + assert os.path.exists(res["db_path"]) + assert len(res["tables"]) == 1 + assert res["tables"][0]["n_rows"] == 2 + os.remove(res["db_path"]) + + +def test_carpeta_vacia_es_ok_sin_tablas(tmp_path): + db = tmp_path / "out.duckdb" + res = load_folder_to_duckdb(str(tmp_path), str(db)) + assert res["status"] == "ok", res + assert res["tables"] == [] + assert res["errors"] == [] + + +def test_carpeta_inexistente_devuelve_status_error(tmp_path): + res = load_folder_to_duckdb(str(tmp_path / "no_existe")) + assert res["status"] == "error" + assert "folder" in res["error"] diff --git a/python/functions/pipelines/render_automatic_eda_folder.md b/python/functions/pipelines/render_automatic_eda_folder.md new file mode 100644 index 00000000..895f15a2 --- /dev/null +++ b/python/functions/pipelines/render_automatic_eda_folder.md @@ -0,0 +1,112 @@ +--- +name: render_automatic_eda_folder +kind: pipeline +lang: py +domain: pipelines +purity: impure +version: "1.0.0" +signature: "def render_automatic_eda_folder(path: str, out_dir: str = \"reports\", basename: str = None, profile_level: str = \"standard\", emit_pdf: bool = True, emit_pptx: bool = True, emit_md: bool = True, per_table_eda: bool = False, min_inclusion: float = 0.9, ctx_extra: dict = None) -> dict" +description: "Informe AutomaticEDA a nivel de BASE one-shot de una CARPETA de archivos tabulares (CSV/Parquet/JSON) o de una DuckDB existente. Carga la carpeta a una DuckDB temporal con load_folder_to_duckdb (o usa la DuckDB dada directa), perfila TODA la base con profile_database (resumen de cada tabla + FK candidatas por containment + join graph con diagrama Mermaid), ENSAMBLA un documento-base por capitulos (portada-base con nombre/n tablas/totales/fecha/fuente, resumen de tablas con una fila por tabla, y relaciones inter-tabla con la tabla de FK candidatas + el diagrama Mermaid) y lo renderiza con el motor AutomaticEDA a PDF (A5 movil), PPTX (16:9) y Markdown autocontenido a la vez. Con per_table_eda=True anexa los capitulos de mini-EDA de cada tabla (build_document por tabla). Es el hermano a nivel de base de render_automatic_eda (que perfila UNA tabla): aqui el informe es de la base y de sus relaciones. Devuelve las rutas de PDF/PPTX/MD, el manifiesto y el DatabaseProfile." +tags: [eda, duckdb, database, profiling, relations, pipeline, dataops, report, pdf, pptx, launcher] +uses_functions: + - load_folder_to_duckdb_py_infra + - profile_database_py_pipelines + - render_automatic_eda_pdf_py_datascience + - render_automatic_eda_pptx_py_datascience + - render_automatic_eda_markdown_py_datascience +uses_types: [] +returns: [] +returns_optional: false +error_type: error_go_core +imports: [] +tested: true +tests: + - "golden: carpeta con 3 CSV relacionados (customers/orders/products) emite PDF+PPTX+MD del documento-base con 3 tablas y la FK orders.customer_id->customers.id" + - "edge: carpeta vacia -> status ok con documento minimo, sin lanzar" + - "edge: 1 sola tabla -> funciona sin relaciones (capitulo relaciones dice 'sin FK')" +test_file_path: "python/functions/pipelines/render_automatic_eda_folder_test.py" +file_path: "python/functions/pipelines/render_automatic_eda_folder.py" +params: + - name: path + desc: "DIRECTORIO con archivos tabulares (CSV/Parquet/JSON) que se cargan a una DuckDB temporal, o una DuckDB ya existente (.duckdb/.ddb/.db) que se perfila directa." + - name: out_dir + desc: "Directorio de salida de los informes (se crea si no existe). Default 'reports'." + - name: basename + desc: "Nombre base de los archivos sin extension. Default 'aeda_base_<nombre>_<timestamp>'." + - name: profile_level + desc: "Preset de coste del perfil por tabla ('lite'/'standard'/'full'); ajusta el sample que profile_database pasa a cada tabla (lite=2000, standard/full=5000)." + - name: emit_pdf + desc: "Emite el PDF A5 movil del documento-base. Default True." + - name: emit_pptx + desc: "Emite el PPTX 16:9 del documento-base. Default True." + - name: emit_md + desc: "Emite el Markdown autocontenido del documento-base. Default True." + - name: per_table_eda + desc: "Si True, anexa al documento-base los capitulos de mini-EDA de cada tabla (Heading 'Tabla: <n>' + build_document por tabla). Default False (solo documento-base: portada + resumen + relaciones)." + - name: min_inclusion + desc: "Umbral de inclusion (0-1) para emitir una FK candidata (se pasa a profile_database). Default 0.9." + - name: ctx_extra + desc: "Dict opcional de claves de presentacion (p.ej. dataset_name, description) que se mezclan en el contexto de la portada-base." +output: "Dict dict-no-throw. En exito: {status:'ok', pdf_path, pptx_path, md_path, manifest_path, n_tables, n_pages, n_slides, md_chars, db_path, db_profile}. En error: {status:'error', error:str}." +--- + +# render_automatic_eda_folder + +EDA de una **carpeta / base multi-tabla** → informe AutomaticEDA por capítulos +en PDF (móvil A5) + PPTX (16:9) + Markdown, en una sola llamada. Es el hermano a +nivel de **base** de `render_automatic_eda` (que perfila una sola tabla): aquí el +documento resume **todas** las tablas y, sobre todo, sus **relaciones** +inter-tabla (FK candidatas por containment + join graph con diagrama Mermaid). + +Compone, sin reimplementar su lógica: `load_folder_to_duckdb` (carga la carpeta), +`profile_database` (perfila la base + infiere FK + join graph) y los tres +renderers del motor AutomaticEDA (`render_automatic_eda_pdf`/`_pptx`/`_markdown`), +que aceptan directamente la lista de capítulos del documento-base que este +pipeline ensambla. El pipeline de tabla única (`render_automatic_eda`) queda +intacto: esto es aditivo. + +## Ejemplo + +```bash +# Carpeta con varios CSV/Parquet/JSON relacionados: +./fn run render_automatic_eda_folder /tmp/eda_folder_demo + +# Una DuckDB ya existente (rama directa): +./fn run render_automatic_eda_folder temp/bigdata/taxi.duckdb +``` + +```python +import sys, os +sys.path.insert(0, os.path.join("python", "functions")) +from pipelines.render_automatic_eda_folder import render_automatic_eda_folder + +r = render_automatic_eda_folder("/tmp/eda_folder_demo", out_dir="reports") +# r["status"] == "ok"; r["pdf_path"], r["pptx_path"], r["md_path"] +# r["n_tables"] == 3; r["db_profile"]["fk_candidates"] incluye +# orders.customer_id -> customers.id +``` + +## Cuando usarla + +Cuando quieras un EDA de una **base entera** (una carpeta de exports o una +DuckDB con varias tablas), no de una sola tabla: para ver de un vistazo qué +tablas hay, su tamaño y calidad, y cómo se relacionan (FK candidatas + diagrama), +en el mismo formato rico por capítulos (PDF móvil + PPTX + MD) que el EDA de +tabla. Usa `per_table_eda=True` cuando además quieras el mini-EDA de cada tabla +anexado. + +## Gotchas + +- Impuro: lee archivos del disco y escribe PDF/PPTX/MD en `out_dir`. En la rama + "carpeta" crea una **DuckDB temporal** (su ruta sale en `db_path`); no se borra + automáticamente (queda para reinspección). +- `path` se interpreta así: directorio → se carga la carpeta; archivo con + extensión `.duckdb`/`.ddb`/`.db` → se usa directo; cualquier otro archivo o un + path inexistente → `{status:'error'}` (no lanza). +- El escaneo de la carpeta es **no recursivo** (solo el primer nivel) y por + defecto cubre `*.csv,*.parquet,*.json` (ver `load_folder_to_duckdb`). +- El diagrama Mermaid se vuelca como **bloque de código**: en el Markdown queda + como diagrama renderizable; en PDF/PPTX se muestra el **texto** del grafo (no + se rasteriza el diagrama a imagen en esta versión). +- Carpeta vacía o con 1 sola tabla: funciona igual; el capítulo de relaciones + dice "sin FK". dict-no-throw en todos los caminos. diff --git a/python/functions/pipelines/render_automatic_eda_folder.py b/python/functions/pipelines/render_automatic_eda_folder.py new file mode 100644 index 00000000..28793cca --- /dev/null +++ b/python/functions/pipelines/render_automatic_eda_folder.py @@ -0,0 +1,350 @@ +"""render_automatic_eda_folder — EDA de una CARPETA / base multi-tabla one-shot. + +Pipeline impuro del grupo de capacidad `eda`, a nivel de BASE. Dada una CARPETA +de archivos tabulares (CSV/Parquet/JSON) o una DuckDB ya existente, produce el +informe AutomaticEDA de la BASE en sus tres formatos a la vez (PDF móvil A5 + +PPTX 16:9 + Markdown autocontenido), con los capítulos POBLADOS, en una sola +llamada. Es el hermano a nivel de base de ``render_automatic_eda`` (que perfila +UNA tabla): aquí el documento por capítulos resume TODAS las tablas y, sobre +todo, sus RELACIONES inter-tabla (FK candidatas + join graph). + +Compone funciones del registry SIN reimplementar su lógica: + + - load_folder_to_duckdb : carga una carpeta de archivos a una DuckDB temporal + (rama "carpeta"). En la rama "ya es duckdb" se omite. + - profile_database : perfila TODA la base (resumen de cada tabla, + TableProfiles completos, FK candidatas por + containment y join graph con diagrama Mermaid). + - render_automatic_eda_pdf : renderiza el documento-base por capítulos a PDF. + - render_automatic_eda_pptx : renderiza el mismo documento-base a PPTX. + - render_automatic_eda_markdown : serializa el mismo documento-base a Markdown + autocontenido (texto + tablas markdown). + - build_document : (solo con per_table_eda=True) ensambla los capítulos + canónicos de CADA tabla para anexarlos al documento. + +La capa propia de este pipeline es ENSAMBLAR EL DOCUMENTO-BASE de capítulos a +partir del ``DatabaseProfile`` que devuelve ``profile_database`` y cablear los +tres renderers del motor AutomaticEDA. El documento-base mínimo tiene tres +capítulos: portada-base (nombre/nº tablas/totales/fecha/fuente), resumen de +tablas (una fila por tabla) y relaciones inter-tabla (FK candidatas + diagrama +Mermaid). Con ``per_table_eda=True`` anexa, por cada tabla, sus capítulos de +mini-EDA. + +Estilo dict-no-throw del grupo `eda`: nunca lanza; captura cualquier error y +degrada a ``{"status": "error", "error": str}``. +""" + +import os +from datetime import datetime, timezone + +from datascience import ( + render_automatic_eda_markdown, + render_automatic_eda_pdf, + render_automatic_eda_pptx, +) +from datascience.automatic_eda import build_document +from infra import load_folder_to_duckdb +from pipelines.profile_database import profile_database + +# Mapa profile_level -> tamaño de muestra por columna del perfil de cada tabla. +# A nivel de base el coste lo domina el nº de tablas; el preset solo ajusta el +# sample que profile_database pasa a profile_table. +_SAMPLE_BY_LEVEL = {"lite": 2000, "standard": 5000, "full": 5000} + +# Extensiones que se consideran "una DuckDB ya hecha" en la rama directa. +_DUCKDB_EXTS = (".duckdb", ".ddb", ".db") + + +def _fmt_num(v) -> str: + """Formatea un entero con separador de millar; '—' si no es número.""" + if isinstance(v, bool) or not isinstance(v, (int, float)): + return "—" + try: + return f"{int(v):,}".replace(",", ".") + except Exception: # noqa: BLE001 + return str(v) + + +def _portada_chapter(db_profile: dict, source_path: str, db_path: str, + meta_ctx: dict) -> dict: + """Capítulo de portada a nivel de base (NO reusa chapters/portada.py, que es + de tabla única): nombre de la base, nº de tablas, totales y procedencia.""" + tables = db_profile.get("tables", []) or [] + total_rows = sum( + (t.get("n_rows") or 0) for t in tables if isinstance(t.get("n_rows"), (int, float)) + ) + total_cols = sum( + (t.get("n_cols") or 0) for t in tables if isinstance(t.get("n_cols"), (int, float)) + ) + base_name = (meta_ctx or {}).get("dataset_name") or os.path.basename( + os.path.normpath(source_path) + ) or source_path + + rows = [ + ("Base", base_name), + ("Tablas", _fmt_num(db_profile.get("n_tables"))), + ("Filas totales", _fmt_num(total_rows)), + ("Columnas totales", _fmt_num(total_cols)), + ("Relaciones FK", _fmt_num(len(db_profile.get("fk_candidates", []) or []))), + ("Fuente", source_path), + ("DuckDB", db_path), + ("Generado", datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC")), + ] + blocks = [ + {"kind": "heading", "text": f"EDA de la base — {base_name}", "level": 1}, + {"kind": "kv_table", "rows": rows, "title": "Resumen de la base"}, + ] + errs = db_profile.get("errors", []) or [] + if errs: + blocks.append({ + "kind": "note", + "text": f"{len(errs)} aviso(s) durante el perfilado (ver detalle).", + }) + return {"id": "portada_base", "title": "Portada", "version": "1.0.0", + "blocks": blocks} + + +def _resumen_chapter(db_profile: dict) -> dict: + """Capítulo con una fila por tabla: filas, columnas, calidad, key_candidates.""" + header = ["Tabla", "Filas", "Columnas", "Calidad", "key_candidates"] + rows = [] + for t in db_profile.get("tables", []) or []: + keys = ", ".join(t.get("key_candidates") or []) or "—" + rows.append([ + t.get("table"), + _fmt_num(t.get("n_rows")), + _fmt_num(t.get("n_cols")), + t.get("quality_score"), + keys, + ]) + if rows: + blocks = [{ + "kind": "data_table", "header": header, "rows": rows, + "title": "Tablas de la base", + "note": "Una fila por tabla. Calidad = score agregado del TableProfile.", + }] + else: + blocks = [{"kind": "note", + "text": "La base no contiene tablas perfilables."}] + return {"id": "resumen_tablas", "title": "Resumen de tablas", + "version": "1.0.0", "blocks": blocks} + + +def _relaciones_chapter(db_profile: dict) -> dict: + """Capítulo de relaciones inter-tabla: tabla de FK candidatas + diagrama + Mermaid del join graph (vuelca el Mermaid como bloque de código).""" + fks = db_profile.get("fk_candidates", []) or [] + blocks = [{ + "kind": "heading", "text": "Relaciones inter-tabla", "level": 2, + }] + if fks: + header = ["From", "To", "Inclusión", "Cardinalidad"] + rows = [] + for fk in fks: + frm = f"{fk.get('from_table')}.{fk.get('from_col')}" + to = f"{fk.get('to_table')}.{fk.get('to_col')}" + inc = fk.get("inclusion") + inc_s = f"{inc:.3f}" if isinstance(inc, (int, float)) else str(inc) + rows.append([frm, to, inc_s, fk.get("cardinality")]) + blocks.append({ + "kind": "data_table", "header": header, "rows": rows, + "title": "FK candidatas (por containment de valores)", + "note": "Inclusión = fracción de valores de From contenidos en To.", + }) + else: + blocks.append({ + "kind": "note", + "text": "Sin relaciones FK candidatas detectadas entre las tablas.", + }) + + mermaid = (db_profile.get("join_graph") or {}).get("mermaid", "") or "" + if mermaid.strip(): + blocks.append({"kind": "heading", "text": "Diagrama (join graph)", + "level": 3}) + # El Mermaid se vuelca como bloque de código: en MD queda como diagrama + # renderizable; en PDF/PPTX se muestra el texto del grafo (legible). + blocks.append({"kind": "markdown", + "text": "```mermaid\n" + mermaid.strip() + "\n```"}) + return {"id": "relaciones", "title": "Relaciones inter-tabla", + "version": "1.0.0", "blocks": blocks} + + +def _build_db_document(db_profile: dict, source_path: str, db_path: str, + meta_ctx: dict, per_table_eda: bool) -> list: + """Ensambla el documento-base por capítulos a partir del DatabaseProfile. + + Mínimo: portada-base + resumen de tablas + relaciones. Con per_table_eda + True anexa, por cada tabla, un capítulo separador + los capítulos canónicos + de su mini-EDA (reusando build_document sobre cada TableProfile).""" + chapters = [ + _portada_chapter(db_profile, source_path, db_path, meta_ctx), + _resumen_chapter(db_profile), + _relaciones_chapter(db_profile), + ] + if per_table_eda: + for prof in db_profile.get("table_profiles", []) or []: + tname = prof.get("table") or "tabla" + chapters.append({ + "id": f"tabla_{tname}", "title": f"Tabla: {tname}", + "version": "1.0.0", + "blocks": [{"kind": "heading", "text": f"Tabla: {tname}", + "level": 1}], + }) + try: + # build_document devuelve los capítulos canónicos de la tabla. + # ctx None -> los capítulos que necesitan datos crudos degradan, + # pero salen completos los de portada/overview/distrib/calidad. + chapters.extend(build_document(prof, None) or []) + except Exception: # noqa: BLE001 — una tabla mala no rompe el doc. + chapters.append({ + "id": f"tabla_{tname}_err", "title": f"Tabla: {tname}", + "version": "1.0.0", + "blocks": [{"kind": "note", + "text": "No se pudo ensamblar el mini-EDA de " + "esta tabla."}], + }) + return chapters + + +def _resolve_db_path(path: str) -> dict: + """Resuelve el DuckDB a perfilar desde ``path``. + + - Directorio -> carga la carpeta con load_folder_to_duckdb (DuckDB temp). + - Archivo .duckdb/.ddb/.db -> se usa directo (rama "ya es duckdb"). + - Otro archivo / inexistente -> error. + + Devuelve {status, db_path, loaded, n_tables, load_errors}. + """ + if os.path.isdir(path): + lr = load_folder_to_duckdb(path) + if lr.get("status") != "ok": + return {"status": "error", + "error": f"load_folder_to_duckdb falló: {lr.get('error')}"} + return { + "status": "ok", + "db_path": lr.get("db_path"), + "loaded": True, + "n_tables": len(lr.get("tables", []) or []), + "load_errors": lr.get("errors", []) or [], + } + if os.path.isfile(path): + if path.lower().endswith(_DUCKDB_EXTS): + return {"status": "ok", "db_path": path, "loaded": False, + "n_tables": None, "load_errors": []} + return {"status": "error", + "error": f"'{path}' no es un directorio ni una DuckDB " + f"(extensiones {_DUCKDB_EXTS})."} + return {"status": "error", "error": f"path no existe: {path}"} + + +def render_automatic_eda_folder( + path: str, + out_dir: str = "reports", + basename: str = None, + profile_level: str = "standard", + emit_pdf: bool = True, + emit_pptx: bool = True, + emit_md: bool = True, + per_table_eda: bool = False, + min_inclusion: float = 0.9, + ctx_extra: dict = None, +) -> dict: + """Perfila una CARPETA (o una DuckDB) y emite el informe AutomaticEDA de la base. + + Args: + path: o bien un DIRECTORIO con archivos tabulares (CSV/Parquet/JSON) que + se cargan a una DuckDB temporal, o bien una DuckDB ya existente + (``.duckdb``/``.ddb``/``.db``) que se perfila directa. + out_dir: directorio de salida (se crea si no existe). Default "reports". + basename: nombre base de los archivos sin extensión. Default + "aeda_base_<nombre>_<timestamp>". + profile_level: preset de coste del perfil por tabla ("lite"/"standard"/ + "full"); ajusta el ``sample`` que profile_database pasa a cada tabla. + emit_pdf / emit_pptx / emit_md: qué formatos emitir. Default los tres. + per_table_eda: si True, anexa al documento-base los capítulos de mini-EDA + de cada tabla (un Heading "Tabla: <n>" + build_document por tabla). + Default False (solo el documento-base: portada + resumen + relaciones). + min_inclusion: umbral de inclusión para emitir una FK candidata (0-1). + ctx_extra: dict opcional de claves de presentación (p.ej. dataset_name, + description) que se mezclan en el contexto de la portada. + + Returns: + dict (nunca lanza). En éxito:: + + {"status": "ok", "pdf_path": str|None, "pptx_path": str|None, + "md_path": str|None, "manifest_path": str|None, + "n_tables": int, "n_pages": int|None, "n_slides": int|None, + "md_chars": int|None, "db_path": str, "db_profile": <DatabaseProfile>} + + En error: {"status": "error", "error": str}. + """ + try: + # 1) Resolver la DuckDB a perfilar (cargar carpeta o usar la dada). + rdb = _resolve_db_path(path) + if rdb.get("status") != "ok": + return {"status": "error", "error": rdb.get("error")} + db_path = rdb.get("db_path") + + # 2) Perfilar la base entera (resumen + FK + join graph). Sin report + # propio (write_report/emit_pdf False): este pipeline emite el suyo. + sample = _SAMPLE_BY_LEVEL.get(profile_level, 5000) + pres = profile_database( + db_path, sample=sample, write_report=False, + min_inclusion=min_inclusion, emit_pdf=False, + ) + if pres.get("status") != "ok": + return {"status": "error", + "error": f"profile_database falló: {pres.get('error')}"} + db_profile = pres.get("db_profile") or {} + + # 3) Ensamblar el documento-base por capítulos. + meta_ctx = dict(ctx_extra or {}) + chapters = _build_db_document( + db_profile, path, db_path, meta_ctx, per_table_eda + ) + + # 4) Render a los tres formatos desde el MISMO documento por capítulos. + os.makedirs(out_dir, exist_ok=True) + ts = datetime.now(timezone.utc).strftime("%Y%m%d-%H%M%S") + nm = (meta_ctx.get("dataset_name") + or os.path.basename(os.path.normpath(path)) or "base") + nm = "".join(c if c.isalnum() else "_" for c in str(nm)).strip("_") or "base" + base = basename or f"aeda_base_{nm}_{ts}" + title = f"EDA base — {meta_ctx.get('dataset_name') or nm}" + meta = {"title": title} + + pdf_path = pptx_path = md_path = manifest_path = None + n_pages = n_slides = md_chars = None + + if emit_pdf: + target = os.path.join(out_dir, base + ".pdf") + rpdf = render_automatic_eda_pdf(chapters, target, meta) or {} + pdf_path = rpdf.get("path") + n_pages = rpdf.get("n_pages") + manifest_path = rpdf.get("manifest_path") + if emit_pptx: + target = os.path.join(out_dir, base + ".pptx") + rpptx = render_automatic_eda_pptx(chapters, target, meta) or {} + pptx_path = rpptx.get("path") + n_slides = rpptx.get("n_slides") + if emit_md: + target = os.path.join(out_dir, base + ".md") + rmd = render_automatic_eda_markdown(chapters, target, meta) or {} + md_path = rmd.get("path") + md_chars = rmd.get("n_chars") + + return { + "status": "ok", + "pdf_path": pdf_path, + "pptx_path": pptx_path, + "md_path": md_path, + "manifest_path": manifest_path, + "n_tables": db_profile.get("n_tables"), + "n_pages": n_pages, + "n_slides": n_slides, + "md_chars": md_chars, + "db_path": db_path, + "db_profile": db_profile, + } + except Exception as e: # noqa: BLE001 — dict-no-throw: degradar, nunca lanzar. + return {"status": "error", "error": str(e)} diff --git a/python/functions/pipelines/render_automatic_eda_folder_test.py b/python/functions/pipelines/render_automatic_eda_folder_test.py new file mode 100644 index 00000000..eb529bdc --- /dev/null +++ b/python/functions/pipelines/render_automatic_eda_folder_test.py @@ -0,0 +1,146 @@ +"""Tests para render_automatic_eda_folder — EDA de una carpeta / base multi-tabla. + +Golden: una carpeta con 3 CSV relacionados (customers/orders/products) produce el +documento-base en PDF + PPTX + MD, con las 3 tablas en el resumen y la FK +orders.customer_id -> customers.id en el capítulo de relaciones. Edges: carpeta +vacía (documento mínimo, sin lanzar), 1 sola tabla (sin relaciones) y la rama +"ya es una DuckDB" sobre un archivo .duckdb existente. +""" + +import os +import sys + +import duckdb + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..")) + +from pipelines.render_automatic_eda_folder import render_automatic_eda_folder + + +def _write_demo_folder(folder: str) -> None: + """3 CSV relacionados: orders.customer_id -> customers.id (FK detectable).""" + with open(os.path.join(folder, "customers.csv"), "w", encoding="utf-8") as fh: + fh.write("id,name,city\n") + fh.write("1,Alice,Madrid\n2,Bob,Barcelona\n3,Carol,Valencia\n" + "4,Dave,Sevilla\n5,Eve,Madrid\n") + with open(os.path.join(folder, "orders.csv"), "w", encoding="utf-8") as fh: + fh.write("order_id,customer_id,product_id,total\n") + fh.write("100,1,10,49.90\n101,1,11,12.50\n102,2,10,49.90\n" + "103,3,12,8.00\n104,3,11,12.50\n105,5,10,49.90\n" + "106,2,12,8.00\n") + with open(os.path.join(folder, "products.csv"), "w", encoding="utf-8") as fh: + fh.write("product_id,product_name,price\n") + fh.write("10,Widget,49.90\n11,Gadget,12.50\n12,Gizmo,8.00\n") + + +def _has_fk(db_profile: dict, from_t: str, from_c: str, to_t: str) -> bool: + for fk in db_profile.get("fk_candidates", []) or []: + if (fk.get("from_table") == from_t and fk.get("from_col") == from_c + and fk.get("to_table") == to_t): + return True + return False + + +def test_golden_folder_three_csv(tmp_path): + """Carpeta con 3 CSV relacionados -> PDF+PPTX+MD, 3 tablas, FK detectada.""" + folder = tmp_path / "demo" + folder.mkdir() + _write_demo_folder(str(folder)) + out = tmp_path / "out" + + r = render_automatic_eda_folder(str(folder), out_dir=str(out)) + + assert r["status"] == "ok", r + assert r["n_tables"] == 3 + # Los tres formatos se emitieron y existen en disco. + assert r["pdf_path"] and os.path.exists(r["pdf_path"]) + assert r["pptx_path"] and os.path.exists(r["pptx_path"]) + assert r["md_path"] and os.path.exists(r["md_path"]) + assert (r["n_pages"] or 0) >= 1 + assert (r["n_slides"] or 0) >= 1 + # La FK orders.customer_id -> customers.id se detecta por containment. + assert _has_fk(r["db_profile"], "orders", "customer_id", "customers"), \ + r["db_profile"].get("fk_candidates") + # El Markdown menciona las 3 tablas y la relación. + md = open(r["md_path"], encoding="utf-8").read() + for t in ("customers", "orders", "products"): + assert t in md + assert "customer_id" in md + + +def test_edge_empty_folder(tmp_path): + """Carpeta vacía -> status ok con documento mínimo, sin lanzar.""" + folder = tmp_path / "empty" + folder.mkdir() + out = tmp_path / "out" + + r = render_automatic_eda_folder(str(folder), out_dir=str(out)) + + assert r["status"] == "ok", r + assert r["n_tables"] == 0 + # Aun sin tablas, emite el documento-base mínimo (portada + resumen vacío + + # relaciones "sin FK"). + assert r["pdf_path"] and os.path.exists(r["pdf_path"]) + assert r["md_path"] and os.path.exists(r["md_path"]) + + +def test_edge_single_table_no_relations(tmp_path): + """Carpeta con 1 sola tabla -> funciona sin relaciones (capítulo 'sin FK').""" + folder = tmp_path / "single" + folder.mkdir() + with open(folder / "lonely.csv", "w", encoding="utf-8") as fh: + fh.write("a,b\n1,x\n2,y\n3,z\n") + out = tmp_path / "out" + + r = render_automatic_eda_folder(str(folder), out_dir=str(out)) + + assert r["status"] == "ok", r + assert r["n_tables"] == 1 + assert not (r["db_profile"].get("fk_candidates") or []) + md = open(r["md_path"], encoding="utf-8").read() + assert "Sin relaciones FK" in md or "sin FK" in md.lower() + + +def test_accepts_existing_duckdb(tmp_path): + """Rama 'ya es una DuckDB': un archivo .duckdb existente se perfila directo.""" + db = tmp_path / "base.duckdb" + conn = duckdb.connect(str(db)) + try: + conn.execute("CREATE TABLE customers (id INTEGER, name VARCHAR)") + conn.execute("INSERT INTO customers VALUES (1,'Ana'),(2,'Luis'),(3,'Eva')") + conn.execute("CREATE TABLE orders (oid INTEGER, customer_id INTEGER)") + conn.execute("INSERT INTO orders VALUES (10,1),(11,2),(12,1),(13,3)") + finally: + conn.close() + out = tmp_path / "out" + + r = render_automatic_eda_folder(str(db), out_dir=str(out)) + + assert r["status"] == "ok", r + assert r["n_tables"] == 2 + assert r["db_path"] == str(db) + assert r["pdf_path"] and os.path.exists(r["pdf_path"]) + + +def test_emit_flags_select_formats(tmp_path): + """emit_pdf/pptx/md controlan qué formatos se emiten.""" + folder = tmp_path / "demo" + folder.mkdir() + _write_demo_folder(str(folder)) + out = tmp_path / "out" + + r = render_automatic_eda_folder( + str(folder), out_dir=str(out), + emit_pdf=True, emit_pptx=False, emit_md=False, + ) + assert r["status"] == "ok", r + assert r["pdf_path"] and os.path.exists(r["pdf_path"]) + assert r["pptx_path"] is None + assert r["md_path"] is None + + +def test_path_does_not_exist(tmp_path): + """Path inexistente -> status error, sin lanzar.""" + r = render_automatic_eda_folder(str(tmp_path / "nope")) + assert r["status"] == "error" + assert "no existe" in r["error"].lower() From eaca41a5325f16fef22d8e18a923a262fcc74f82 Mon Sep 17 00:00:00 2001 From: Egutierrez <egutierrez@dead.dd> Date: Tue, 30 Jun 2026 20:37:01 +0200 Subject: [PATCH 37/53] =?UTF-8?q?feat(eda):=20scatters=20de=20pares=20m?= =?UTF-8?q?=C3=A1s=20correlacionados=20+=20tipo=20de=20relaci=C3=B3n=20en?= =?UTF-8?q?=20cap=C3=ADtulo=20CORRELACION?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Añade al capítulo `correlacion` del AutomaticEDA la visualización con scatters de los pares numérico-numérico más correlacionados (positiva y negativamente) y, para cada uno, la clasificación del tipo de relación: lineal, polinómica (grado 2/3), monótona no-lineal o débil/sin forma. Funciones nuevas del registry (dominio datascience, grupo eda): - classify_relationship_type_py_datascience (pura): dadas dos listas numéricas pareadas, cruza Pearson r (lineal), Spearman ρ (monótona) y ajustes polinómicos de grado 2 y 3 (numpy.polyfit + R² manual) para etiquetar la forma. Reusa pearson y spearman_corr del registry. Umbrales calibrados para datos reales discretos/ruidosos (orden: débil → monótona → polinómica → lineal). Devuelve los coeficientes del mejor modelo para pintar la curva. No-throw. - relationship_scatter_figure_py_datascience (impure): construye la Figure matplotlib del scatter de un par con su recta/curva de ajuste y una anotación del tipo + métricas (r, ρ, R²lin, R²poly). Backend Agg sin pyplot global, downsample determinista de los puntos dibujados, tendencia ordenada (binned / por valor) para el caso monótona sin polinomio. Defensiva ante vacío. Capítulo correlacion.py (1.0.0 → 1.1.0): nueva sección "Relaciones más fuertes (scatter)" tras la matriz + tablas top. Toma los top-K pares num↔num por |valor| de profile['correlations']['pairs'], obtiene los datos crudos de cada par desde ctx['raw_numeric'] y emite, por par, un Figure dentro de un Group keep-together junto a una nota de texto con el tipo de relación (extraíble por pdftotext). Solo num↔num: los pares cat↔cat (Cramér's V) y num↔cat (razón de correlación) no llevan scatter. Cuando no hay raw_numeric (perfil lite/agregado o ctx None) los scatters se omiten sin lanzar; la matriz + tablas siguen. Verificado: golden EDA de titanic (run_models) — el capítulo Correlación del PDF y PPTX incluye los scatters (pclass↔fare → monótona no-lineal, sibsp↔parch → lineal, …) con su ajuste y etiqueta de tipo en texto. Tests de clasificación sintética (lineal, y=x² → polinómica, y=exp(x) → monótona, ruido → débil) + tests del capítulo (golden con raw_numeric, edge sin raw, par sin columna). Suite automatic_eda + pipeline render_automatic_eda verde (141 passed). fn index sin error. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> --- .../automatic_eda/chapters/correlacion.py | 154 ++++++++- .../chapters/correlacion_test.py | 99 ++++++ .../datascience/classify_relationship_type.md | 68 ++++ .../datascience/classify_relationship_type.py | 187 ++++++++++ .../classify_relationship_type_test.py | 174 ++++++++++ .../relationship_scatter_figure.md | 122 +++++++ .../relationship_scatter_figure.py | 322 ++++++++++++++++++ .../relationship_scatter_figure_test.py | 100 ++++++ 8 files changed, 1225 insertions(+), 1 deletion(-) create mode 100644 python/functions/datascience/classify_relationship_type.md create mode 100644 python/functions/datascience/classify_relationship_type.py create mode 100644 python/functions/datascience/classify_relationship_type_test.py create mode 100644 python/functions/datascience/relationship_scatter_figure.md create mode 100644 python/functions/datascience/relationship_scatter_figure.py create mode 100644 python/functions/datascience/relationship_scatter_figure_test.py diff --git a/python/functions/datascience/automatic_eda/chapters/correlacion.py b/python/functions/datascience/automatic_eda/chapters/correlacion.py index cd559323..b1a4c702 100644 --- a/python/functions/datascience/automatic_eda/chapters/correlacion.py +++ b/python/functions/datascience/automatic_eda/chapters/correlacion.py @@ -31,7 +31,7 @@ import math from .. import model -CHAPTER_VERSION = "1.0.0" +CHAPTER_VERSION = "1.1.0" CHAPTER_ID = "correlacion" CHAPTER_TITLE = "Correlación" @@ -47,6 +47,13 @@ _MAX_MATRIX_LABELS = 16 # How many pairs to show in each of the top-positive / top-negative tables. _TOP_N = 10 +# How many of the strongest numeric-numeric pairs to draw as scatter plots on +# each sign (positive / negative). A scatter per pair carries a fitted line/curve +# and a relationship-type label; keeping the count small keeps the chapter +# readable on a phone / a slide. Only signed (Pearson/Spearman) pairs qualify — +# Cramér's V / correlation ratio pairs are not numeric-numeric, so no scatter. +_SCATTER_TOP_N = 3 + # Glossary terms this chapter explains. Each is registered in the shared # collector (ctx['glossary']) and marked clickable on its first appearance in the # body — the canonical two-step pattern (see ``cat_distr`` for the reference @@ -314,6 +321,139 @@ def _fdr_text(corr: dict, mark_term: bool = False) -> str | None: return " ".join(parts) +def _is_seq(values) -> bool: + """True for a non-empty list/tuple of values (a raw numeric column).""" + return isinstance(values, (list, tuple)) and len(values) > 0 + + +def _select_scatter_pairs(pairs: list, top_n: int = _SCATTER_TOP_N): + """Pick the strongest numeric-numeric pairs to draw as scatters. + + Only signed (Pearson/Spearman) pairs are numeric-numeric and thus eligible + for a scatter with a fitted curve. Returns up to ``top_n`` of the strongest + positive pairs followed by up to ``top_n`` of the strongest negative ones, + each ranked by magnitude. Mixed-type metrics (Cramér's V, correlation ratio, + mutual information) are excluded — they have no x/y scatter interpretation. + """ + positive = [] + negative = [] + for pair in pairs: + if not isinstance(pair, dict) or not _is_signed(pair): + continue + value = pair.get("value") + if not _is_num(value): + continue + if value > 0: + positive.append(pair) + elif value < 0: + negative.append(pair) + positive.sort(key=lambda p: abs(float(p.get("value", 0.0))), reverse=True) + negative.sort(key=lambda p: abs(float(p.get("value", 0.0))), reverse=True) + return positive[:top_n] + negative[:top_n] + + +def _classification_note(a: str, b: str, cls: dict) -> str: + """Human-readable sentence describing the relationship of a pair. + + Plain text (not baked into the figure image) so the type label is selectable + in the PDF / extractable by pdftotext, and sits right next to its scatter + inside the keep-together Group. + """ + tipo = model._safe_str(cls.get("tipo")) or "sin forma clara" + bits = [] + pearson = cls.get("pearson") + spearman = cls.get("spearman") + r2_lin = cls.get("r2_linear") + r2_poly = None + for key in ("r2_poly2", "r2_poly3"): + v = cls.get(key) + if _is_num(v) and (r2_poly is None or float(v) > r2_poly): + r2_poly = float(v) + if _is_num(pearson): + bits.append(f"Pearson r={float(pearson):+.2f}") + if _is_num(spearman): + bits.append(f"Spearman ρ={float(spearman):+.2f}") + if _is_num(r2_lin): + bits.append(f"R² lineal={float(r2_lin):.2f}") + if r2_poly is not None: + bits.append(f"R² polinómico={r2_poly:.2f}") + metrics = "; ".join(bits) + text = (f"Relación **{tipo}** entre «{a}» y «{b}»." + + (f" {metrics}." if metrics else "")) + return text + + +def _scatter_blocks(pairs: list, raw_numeric): + """Build keep-together scatter Groups for the strongest num-num pairs. + + Returns a list of blocks (a Heading plus one Group per pair), or an empty + list when there is no raw numeric data (e.g. the lite profile drops + ``ctx['raw_numeric']`` to skip live recomputation) or the relationship + helpers are unavailable. Never raises: any failure degrades to no scatters, + leaving the matrix + tables intact. + """ + if not isinstance(raw_numeric, dict) or not raw_numeric: + return [] + selected = _select_scatter_pairs(pairs) + if not selected: + return [] + + # The relationship helpers live in the datascience package. Import lazily so + # the chapter still builds (matrix + tables) when they are absent. + try: + from datascience.classify_relationship_type import ( + classify_relationship_type, + ) + from datascience.relationship_scatter_figure import ( + relationship_scatter_figure, + ) + except Exception: # noqa: BLE001 — degrade, never break the chapter. + return [] + + groups = [] + for pair in selected: + a = pair.get("a") + b = pair.get("b") + xs = raw_numeric.get(a) + ys = raw_numeric.get(b) + # Edge: a selected pair has no raw column (aggregated profile, renamed + # column, …) — skip just that pair, keep the rest. + if not _is_seq(xs) or not _is_seq(ys): + continue + try: + cls = classify_relationship_type(list(xs), list(ys)) or {} + except Exception: # noqa: BLE001 + continue + a_lbl = model._safe_str(a) + b_lbl = model._safe_str(b) + + def _make(xs=xs, ys=ys, a_lbl=a_lbl, b_lbl=b_lbl, cls=cls): + return relationship_scatter_figure( + list(xs), list(ys), x_label=a_lbl, y_label=b_lbl, + classification=cls) + + groups.append(model.Group(blocks=[ + model.Heading(text=f"{a_lbl} ↔ {b_lbl}", level=2), + model.Figure( + make=_make, + caption=(f"Dispersión de «{a_lbl}» frente a «{b_lbl}» con la " + "curva de ajuste del mejor modelo.")), + model.Markdown(text=_classification_note(a_lbl, b_lbl, cls)), + ])) + + if not groups: + return [] + intro = model.Markdown(text=( + "Para los pares numéricos más fuertes (positivos y negativos) se dibuja " + "la nube de puntos con su ajuste y se clasifica el **tipo de relación**: " + "**lineal** (una recta basta), **polinómica** (curva de grado 2/3 que " + "mejora claramente el ajuste lineal), **monótona no-lineal** (crece o " + "decrece siempre pero no en línea recta; Spearman ≫ Pearson) o " + "**débil/sin forma**.")) + return [model.Heading(text="Relaciones más fuertes (scatter)", level=2), + intro] + groups + + def build_correlacion(profile: dict, ctx: dict): """Build the Correlation Chapter, or None if there are no pairs to show. @@ -392,6 +532,18 @@ def build_correlacion(profile: dict, ctx: dict): "No se han hallado correlaciones negativas significativas entre " "columnas numéricas."))) + # 2.5) Scatter plots of the strongest numeric-numeric pairs, each with its + # fitted curve and a relationship-type label (lineal / polinómica / monótona + # / débil). Needs the raw numeric sample (ctx['raw_numeric'], row-aligned); + # when it is absent (aggregated/lite profile) the scatters are simply omitted + # and the matrix + tables above stand on their own. + raw_numeric = None + if isinstance(ctx, dict): + raw_numeric = ctx.get("raw_numeric") or profile.get("raw_numeric") + else: + raw_numeric = profile.get("raw_numeric") + blocks.extend(_scatter_blocks(pairs, raw_numeric)) + # 3) Spuriousness caveat for level-based correlations (Granger–Newbold). caveat = corr.get("levels_caveat") if isinstance(caveat, str) and caveat.strip(): diff --git a/python/functions/datascience/automatic_eda/chapters/correlacion_test.py b/python/functions/datascience/automatic_eda/chapters/correlacion_test.py index b4291e65..96d7f1fd 100644 --- a/python/functions/datascience/automatic_eda/chapters/correlacion_test.py +++ b/python/functions/datascience/automatic_eda/chapters/correlacion_test.py @@ -175,6 +175,105 @@ def test_anticorte_matriz_ancha_y_etiquetas_largas_no_se_cortan(): assert "azufre" in _pdf_text(pdf) +def _raw_numeric_for_profile(n: int = 80) -> dict: + """Row-aligned raw numeric sample matching the signed pairs of _profile(). + + Builds columns with a clear, deterministic shape so the relationship-type + classifier has something unambiguous to label: + - density vs alcohol: strong negative linear (the top-negative pair). + - alcohol vs quality: positive linear. + - ph, fixed_acidity, sulphates: filler columns for the remaining pairs. + """ + import math as _m + + alcohol = [8.0 + 0.05 * i for i in range(n)] + density = [1.0 - 0.002 * a for a in alcohol] # neg linear vs alcohol + quality = [3.0 + 0.4 * a + (0.1 if i % 2 else -0.1) # pos linear vs alcohol + for i, a in enumerate(alcohol)] + ph = [3.0 + 0.3 * _m.sin(i / 5.0) for i in range(n)] + fixed_acidity = [7.0 - 0.5 * p for p in ph] # neg linear vs ph + sulphates = [0.5 + 0.01 * (i % 7) for i in range(n)] + return { + "alcohol": alcohol, "density": density, "quality": quality, + "ph": ph, "fixed_acidity": fixed_acidity, "sulphates": sulphates, + } + + +def test_golden_scatters_de_pares_num_num_con_tipo_de_relacion(): + """Con ctx['raw_numeric'], el capítulo añade scatters (Figure dentro de Group) + de los pares num-num más fuertes, cada uno con su etiqueta de tipo en texto.""" + from datascience.automatic_eda.model import Group + + ctx = {"raw_numeric": _raw_numeric_for_profile()} + ch = build_correlacion(_profile(), ctx) + assert ch is not None + groups = [b for b in ch.blocks if isinstance(b, Group)] + assert groups, "debe emitir al menos un Group con scatter" + # Cada Group lleva su figura (lazy) y una nota de texto con el tipo. + for g in groups: + gkinds = [b.kind for b in g.blocks] + assert "figure" in gkinds and "markdown" in gkinds + # La sección y la etiqueta de tipo aparecen como texto plano (extraíble). + headings = " ".join(b.text for b in ch.blocks if b.kind == "heading") + assert "Relaciones más fuertes" in headings + body = " ".join(b.text for g in groups for b in g.blocks + if b.kind == "markdown") + assert any(t in body for t in + ("lineal", "polinómica", "monótona", "sin forma")) + # El par num-num más fuerte (density ↔ alcohol) tiene scatter; el par cat-cat + # (region ↔ type) NO — no es numérico. + assert "density" in body or "alcohol" in body + assert "region" not in body and "type" not in body + + +def test_golden_pdf_muestra_scatters_con_etiqueta_de_tipo(): + """En el PDF, el capítulo Correlación incluye los scatters y su etiqueta de + tipo en texto seleccionable (pdftotext la encuentra).""" + prof = _profile() + ctx = {"raw_numeric": _raw_numeric_for_profile()} + with tempfile.TemporaryDirectory() as d: + pdf = os.path.join(d, "corr_scatter.pdf") + rp = render_automatic_eda_pdf(prof, pdf, {"title": "EDA — wine", + "ctx": ctx}) + assert rp["path"] == pdf and rp["n_pages"] >= 1 + txt = _pdf_text(pdf) + assert "Relaciones" in txt and "scatter" in txt.lower() + # Alguna etiqueta de tipo de relación, en texto. + assert any(t in txt for t in + ("lineal", "polin", "monóton", "monoton", "sin forma")) + + +def test_edge_sin_raw_numeric_omite_scatters_sin_lanzar(): + """profile lite / ctx None: sin raw_numeric el capítulo omite los scatters + pero sigue emitiendo matriz + tablas (no lanza).""" + from datascience.automatic_eda.model import Group + + for ctx in (None, {}, {"raw_numeric": None}, {"raw_numeric": {}}): + ch = build_correlacion(_profile(), ctx) + assert ch is not None + assert not [b for b in ch.blocks if isinstance(b, Group)] + # La matriz y al menos una tabla top siguen presentes. + assert any(b.kind == "figure" for b in ch.blocks) + assert any(b.kind == "data_table" for b in ch.blocks) + + +def test_edge_par_sin_columna_cruda_se_omite_sin_lanzar(): + """Si un par seleccionado no tiene su columna en raw_numeric, se omite ese + par (no lanza); los demás scatters se construyen igual.""" + from datascience.automatic_eda.model import Group + + raw = _raw_numeric_for_profile() + raw.pop("density", None) # rompe el par density ↔ alcohol + ch = build_correlacion(_profile(), {"raw_numeric": raw}) + assert ch is not None + groups = [b for b in ch.blocks if isinstance(b, Group)] + body = " ".join(b.text for g in groups for b in g.blocks + if b.kind == "markdown") + # density desaparece de los scatters; otros pares (p.ej. ph↔fixed_acidity, + # alcohol↔quality) pueden seguir presentes sin error. + assert "density" not in body + + def test_glosario_engancha_metodos_y_fdr(): """Mejora 4b: los métodos de correlación (Pearson, Spearman, Cramér's V, razón de correlación) y la corrección por comparaciones múltiples (FDR) se diff --git a/python/functions/datascience/classify_relationship_type.md b/python/functions/datascience/classify_relationship_type.md new file mode 100644 index 00000000..170222ad --- /dev/null +++ b/python/functions/datascience/classify_relationship_type.md @@ -0,0 +1,68 @@ +--- +name: classify_relationship_type +kind: function +lang: py +domain: datascience +version: "1.0.0" +purity: pure +signature: "def classify_relationship_type(xs: list, ys: list) -> dict" +description: "Clasifica el TIPO de relacion entre dos variables numericas pareadas por indice para el EDA automatico del grupo eda. Limpia los pares de forma defensiva (descarta None/bool/NaN/inf), reusa pearson y spearman_corr del registry y ajusta polinomios de grado 2 y 3 con numpy.polyfit (R^2 manual), y a partir de esas senales etiqueta la forma: 'lineal', 'polinomica (grado 2/3)', 'monotona no-lineal' o 'debil/sin forma'. Orden de decision: debil -> monotona -> polinomica -> lineal (la primera que matchea gana), con umbrales calibrados para datos reales discretos/ruidosos. Devuelve ademas los coeficientes del mejor modelo en orden de numpy.polyval para pintar la curva de ajuste sobre el scatter. Funcion pura no-throw: ante datos insuficientes (menos de 5 pares validos o varianza ~0) o cualquier fallo devuelve el dict canonico con tipo='debil/sin forma' y el resto a None." +tags: [eda, correlation, relationship, classification, polyfit, datascience, pure] +params: + - name: xs + desc: "Lista (o tupla) de valores numericos de la primera variable, pareada por indice con ys. Cada par xs[i],ys[i] se descarta si cualquiera de los dos es None, bool, NaN o inf. Lectura defensiva." + - name: ys + desc: "Lista (o tupla) de valores numericos de la segunda variable, pareada por indice con xs. Mismas reglas de limpieza que xs." +output: "Dict con SIEMPRE las mismas 8 claves: tipo (str: 'lineal' | 'polinómica (grado 2)' | 'polinómica (grado 3)' | 'monótona no-lineal' | 'débil/sin forma'); pearson (float|None: coeficiente de Pearson r); r2_linear (float|None: r**2 del ajuste lineal); spearman (float|None: rho de Spearman); r2_poly2 (float|None: R^2 del ajuste polinomico de grado 2); r2_poly3 (float|None: R^2 del ajuste de grado 3); best_degree (int|None: grado del modelo elegido — 1 lineal, 2/3 polinomico, None si monotona/debil); coeffs (list|None: coeficientes del mejor modelo en orden de numpy.polyval para pintar la curva, o None). Ante datos insuficientes o error: tipo='débil/sin forma' y el resto de claves a None." +uses_functions: [pearson_py_datascience, spearman_corr_py_datascience] +uses_types: [] +returns: [] +returns_optional: false +error_type: "" +imports: [numpy] +tested: true +tests: ["test_lineal", "test_polinomica_cuadratica", "test_monotona_no_lineal", "test_monotona_exponencial", "test_debil_sin_forma", "test_lista_vacia_no_lanza", "test_longitudes_distintas_no_lanza", "test_todos_none_no_lanza", "test_entradas_none_no_lanza", "test_constante_no_lanza", "test_filtra_nan_inf_bool"] +test_file_path: "python/functions/datascience/classify_relationship_type_test.py" +file_path: "python/functions/datascience/classify_relationship_type.py" +--- + +## Ejemplo + +```python +import sys, os +sys.path.insert(0, os.path.join("python", "functions")) +from datascience.classify_relationship_type import classify_relationship_type +import numpy as np + +# Relacion claramente cuadratica (forma de parabola) sobre dominio simetrico. +x = list(np.linspace(-10, 10, 60)) +y = [v * v for v in x] + +res = classify_relationship_type(x, y) +print(res["tipo"]) # 'polinómica (grado 2)' +print(res["best_degree"]) # 2 +print(res["r2_linear"]) # 0.0 -> el Pearson lineal no ve la parabola +print(res["r2_poly2"]) # 1.0 +print(res["coeffs"]) # [1.0, -0.0, -0.0] -> numpy.polyval(coeffs, x) ~ x**2 + +# El capitulo pinta la curva de ajuste cuando coeffs no es None: +# if res["coeffs"] is not None: +# xs_fit = np.linspace(min(x), max(x), 200) +# ys_fit = np.polyval(res["coeffs"], xs_fit) +# ax.plot(xs_fit, ys_fit) # curva sobre el ax.scatter(x, y) +``` + +## Cuando usarla + +- Usala en el capitulo de relaciones/correlaciones del EDA automatico, despues de detectar dos columnas numericas con alguna asociacion, para decidir QUE curva de ajuste pintar sobre el scatter (recta, parabola, cubica o ninguna) y poner una etiqueta legible al tipo de relacion. +- Cuando un Pearson bajo no signifique "sin relacion": esta funcion cruza Pearson con Spearman y con ajustes polinomicos para distinguir una relacion lineal debil de una monotona no-lineal (que el rango si capta) o de una curva polinomica. +- Cuando necesites un punto de entrada determinista y no-throw que, con los mismos datos, devuelva siempre el mismo `tipo` y los mismos `coeffs` listos para `numpy.polyval` sin tener que ajustar modelos a mano en el capitulo. + +## Gotchas + +- Funcion pura, deterministica y no-throw: ante menos de 5 pares validos, varianza ~0 (xs o ys constante) o cualquier excepcion interna devuelve el dict canonico `tipo="débil/sin forma"` con el resto de claves a `None`. El dict SIEMPRE trae las 8 claves: nunca compruebes existencia, comprueba `None`. +- El orden de decision importa: `débil -> monótona -> polinómica -> lineal` (la primera que matchee gana). La monotonia se evalua ANTES que el ajuste polinomico, asi que una curva monotona suave (exp, log, potencias) sale `monótona no-lineal` aunque un cubico tambien la ajuste — la dominancia del rango (Spearman >> Pearson) es la senal mas interpretable. Solo cae en `polinómica` una forma curva NO monotona (p.ej. una parabola, Spearman ~0 pero R^2 polinomico alto). +- Umbrales fijos (calibrados para EDA con datos discretos/ruidosos, no para inferencia formal): `débil/sin forma` si las tres senales son bajas a la vez (`abs(pearson) < 0.3` y `abs(spearman) < 0.3` y `mejor_poly < 0.3`); `monótona no-lineal` si `abs(spearman) - abs(pearson) >= 0.1` y `abs(spearman) >= 0.4`; `polinómica (grado N)` si el mejor polinomico mejora `>= 0.1` sobre el lineal y su R^2 `>= 0.3`; en cualquier otro caso con senal (no debil) `lineal`. El suelo de 0.3 evita llamar "debil" a relaciones reales pero discretas (conteos, escalas ordinales) con R^2 bajo pero direccion clara. +- `coeffs` va en orden de `numpy.polyval` (grado descendente). Para `lineal` es `[pendiente, intercepto]` (grado 1); para `polinómica` los del grado elegido; para `monótona no-lineal` y `débil/sin forma` es `None` (el scatter pintara una curva suavizada o nada — lo decide el capitulo, no esta funcion). +- `best_degree` prefiere el grado 2 sobre el 3 cuando empatan dentro de 0.02 de R^2 (parsimonia): no esperes grado 3 salvo que mejore claramente. +- Los pares con `None`, `bool`, `NaN` o `inf` se descartan por indice en silencio; `bool` cuenta como no-numerico (un `True` no es `1`). El dominio de los datos afecta al resultado: una parabola sobre un dominio simetrico da Pearson ~0 (sale `polinómica`), pero sobre un dominio asimetrico el Pearson sube y puede salir `lineal`. diff --git a/python/functions/datascience/classify_relationship_type.py b/python/functions/datascience/classify_relationship_type.py new file mode 100644 index 00000000..683963ce --- /dev/null +++ b/python/functions/datascience/classify_relationship_type.py @@ -0,0 +1,187 @@ +"""Clasifica el TIPO de relacion entre dos variables numericas pareadas. + +Funcion pura del grupo eda. Dadas dos listas numericas pareadas por indice, +limpia los pares de forma defensiva, calcula correlaciones lineal (Pearson) y de +rangos (Spearman) y ajustes polinomicos de grado 2 y 3, y a partir de esas +senales etiqueta la forma de la relacion para el EDA automatico: + + "lineal" | "polinómica (grado 2)" | "polinómica (grado 3)" | + "monótona no-lineal" | "débil/sin forma" + +Ademas devuelve los coeficientes del mejor modelo (en orden de numpy.polyval) +para que el capitulo pinte la curva de ajuste sobre el scatter. Reusa las +funciones del registry `pearson` y `spearman_corr` en vez de reimplementarlas. + +NUNCA lanza: ante cualquier fallo o dato insuficiente devuelve el dict canonico +con tipo="débil/sin forma" y el resto de claves a None. +""" + +import math +import warnings + +import numpy as np + +from datascience.datascience import pearson +from datascience.spearman_corr import spearman_corr + +# Forma canonica de la respuesta cuando no se puede clasificar (datos +# insuficientes, varianza nula o error interno). Siempre las mismas claves. +_WEAK = { + "tipo": "débil/sin forma", + "pearson": None, + "r2_linear": None, + "spearman": None, + "r2_poly2": None, + "r2_poly3": None, + "best_degree": None, + "coeffs": None, +} + + +def _is_num(v) -> bool: + """True si v es un numero real finito (int/float, no bool, no NaN, no inf).""" + return ( + isinstance(v, (int, float)) + and not isinstance(v, bool) + and not (isinstance(v, float) and (math.isnan(v) or math.isinf(v))) + ) + + +def _poly_r2(coeffs, x_arr, y_arr, ss_tot: float) -> float: + """R^2 de un ajuste polinomico: 1 - SS_res/SS_tot. 0 si SS_tot==0.""" + if ss_tot == 0.0: + return 0.0 + pred = np.polyval(coeffs, x_arr) + ss_res = float(np.sum((y_arr - pred) ** 2)) + return 1.0 - ss_res / ss_tot + + +def classify_relationship_type(xs: list, ys: list) -> dict: + """Clasifica el tipo de relacion entre dos variables numericas pareadas. + + Empareja xs[i],ys[i] por indice y descarta el par si cualquiera de los dos + es None, bool, NaN o inf. Sobre los pares limpios calcula Pearson r + (r2_linear = r**2), Spearman rho y los R^2 de ajustes polinomicos de grado 2 + y 3 (con numpy.polyfit + R^2 manual). Con esas senales decide la etiqueta. + + Orden de evaluacion de la etiqueta (la primera que matchee gana). Los + umbrales estan calibrados para datos reales, a menudo discretos y ruidosos + (conteos, escalas ordinales): una relacion con |r| >= 0.3, |rho| >= 0.3 o un + polinomio con R^2 >= 0.3 ya tiene FORMA y no debe etiquetarse como "debil". + 1. "débil/sin forma" — todas las senales bajas a la vez: + abs(pearson) < 0.3 y abs(spearman) < 0.3 y mejor_poly < 0.3. + 2. "monótona no-lineal" — el rango (Spearman) capta una monotonia que el + Pearson lineal no: abs(spearman) - abs(pearson) >= 0.1 y + abs(spearman) >= 0.4. No se fuerza un polinomio (coeffs/best_degree = + None); el capitulo dibuja la tendencia ordenada sobre el scatter. + 3. "polinómica (grado N)" — el mejor polinomico mejora claramente sobre + el lineal (mejor_poly - r2_linear >= 0.1) y mejor_poly >= 0.3. N es el + grado (2 o 3) con mejor R^2, prefiriendo el 2 si empatan dentro de 0.02 + (parsimonia). + 4. "lineal" — el resto: hay senal (no es debil) y la forma que existe es + esencialmente lineal. best_degree=1, coeffs del ajuste de grado 1. + + Si hay menos de 5 pares validos, o la varianza de xs o de ys es ~0 + (constante), devuelve directamente "débil/sin forma". + + Args: + xs: lista (o tupla) de valores numericos de la primera variable, + pareada por indice con ys. Pares con None/bool/NaN/inf se descartan. + ys: lista (o tupla) de valores numericos de la segunda variable, + pareada por indice con xs. + + Returns: + dict con SIEMPRE las mismas claves: + tipo (str), pearson (float|None), r2_linear (float|None), + spearman (float|None), r2_poly2 (float|None), r2_poly3 (float|None), + best_degree (int|None: 1, 2, 3 o None), + coeffs (list|None: coeficientes en orden de numpy.polyval, o None). + Nunca lanza: ante fallo o datos insuficientes devuelve el dict debil. + """ + try: + if xs is None or ys is None: + return dict(_WEAK) + + pairs = [ + (float(x), float(y)) + for x, y in zip(xs, ys) + if _is_num(x) and _is_num(y) + ] + + # Datos insuficientes para hablar de forma de la relacion. + if len(pairs) < 5: + return dict(_WEAK) + + clean_x = [p[0] for p in pairs] + clean_y = [p[1] for p in pairs] + + # Varianza ~0 en cualquiera de las series => relacion indefinida. + if len(set(clean_x)) < 2 or len(set(clean_y)) < 2: + return dict(_WEAK) + x_arr = np.asarray(clean_x, dtype=float) + y_arr = np.asarray(clean_y, dtype=float) + if float(np.var(x_arr)) < 1e-15 or float(np.var(y_arr)) < 1e-15: + return dict(_WEAK) + + # Correlaciones reutilizando las funciones del registry. + r = pearson(clean_x, clean_y) + spearman = spearman_corr(clean_x, clean_y) + r2_linear = r ** 2 + + # Ajustes polinomicos grado 2 y 3 con R^2 manual. + ss_tot = float(np.sum((y_arr - float(np.mean(y_arr))) ** 2)) + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + c1 = np.polyfit(x_arr, y_arr, 1) + c2 = np.polyfit(x_arr, y_arr, 2) + c3 = np.polyfit(x_arr, y_arr, 3) + r2_poly2 = _poly_r2(c2, x_arr, y_arr, ss_tot) + r2_poly3 = _poly_r2(c3, x_arr, y_arr, ss_tot) + + mejor_poly = max(r2_poly2, r2_poly3) + # Grado del mejor polinomico, con preferencia por la parsimonia: solo se + # elige el grado 3 si supera al grado 2 por mas de 0.02. + best_poly_degree = 3 if (r2_poly3 - r2_poly2) > 0.02 else 2 + + abs_s = abs(spearman) + abs_p = abs(r) + + # Decision en orden: debil-temprano -> monotona -> polinomica -> lineal. + if abs_p < 0.3 and abs_s < 0.3 and mejor_poly < 0.3: + # Ninguna senal supera el suelo de forma: relacion debil/sin forma. + tipo = "débil/sin forma" + best_degree = None + coeffs = None + elif (abs_s - abs_p) >= 0.1 and abs_s >= 0.4: + # Spearman (rango) capta una monotonia que el Pearson lineal no: + # relacion monotona no-lineal. No se fuerza un polinomio que tal vez + # no ajusta bien; el capitulo dibuja la tendencia ordenada. + tipo = "monótona no-lineal" + best_degree = None + coeffs = None + elif (mejor_poly - r2_linear) >= 0.1 and mejor_poly >= 0.3: + tipo = "polinómica (grado {})".format(best_poly_degree) + best_degree = best_poly_degree + best_coeffs = c2 if best_poly_degree == 2 else c3 + coeffs = [float(c) for c in best_coeffs] + else: + # Hay senal (no es debil) y no es ni monotona-pura ni polinomica: + # la correlacion que existe es esencialmente lineal. + tipo = "lineal" + best_degree = 1 + coeffs = [float(c) for c in c1] + + return { + "tipo": tipo, + "pearson": round(float(r), 6), + "r2_linear": round(float(r2_linear), 6), + "spearman": round(float(spearman), 6), + "r2_poly2": round(float(r2_poly2), 6), + "r2_poly3": round(float(r2_poly3), 6), + "best_degree": best_degree, + "coeffs": ( + [round(c, 8) for c in coeffs] if coeffs is not None else None + ), + } + except Exception: + return dict(_WEAK) diff --git a/python/functions/datascience/classify_relationship_type_test.py b/python/functions/datascience/classify_relationship_type_test.py new file mode 100644 index 00000000..eab1bb35 --- /dev/null +++ b/python/functions/datascience/classify_relationship_type_test.py @@ -0,0 +1,174 @@ +"""Tests para classify_relationship_type.""" + +import os +import sys + +import numpy as np + +sys.path.insert(0, os.path.dirname(__file__)) + +from classify_relationship_type import classify_relationship_type + +# Claves que el dict de salida debe contener SIEMPRE. +_EXPECTED_KEYS = { + "tipo", "pearson", "r2_linear", "spearman", + "r2_poly2", "r2_poly3", "best_degree", "coeffs", +} + + +def _assert_shape(r): + """Toda salida tiene exactamente las 8 claves canonicas.""" + assert isinstance(r, dict) + assert set(r.keys()) == _EXPECTED_KEYS + + +def test_lineal(): + """Golden: y = 2x + 1 con ruido pequeno -> 'lineal', best_degree=1.""" + rng = np.random.default_rng(42) + x = np.linspace(0.0, 10.0, 50) + y = 2.0 * x + 1.0 + rng.normal(0.0, 0.3, 50) + + r = classify_relationship_type(list(x), list(y)) + _assert_shape(r) + + assert r["tipo"] == "lineal" + assert r["best_degree"] == 1 + assert r["r2_linear"] >= 0.5 + # coeffs ~ [pendiente, intercepto] del ajuste de grado 1. + assert r["coeffs"] is not None and len(r["coeffs"]) == 2 + assert abs(r["coeffs"][0] - 2.0) < 0.1 # pendiente ~2 + assert abs(r["coeffs"][1] - 1.0) < 0.3 # intercepto ~1 + + +def test_polinomica_cuadratica(): + """Golden: y = x**2 sobre [-10, 10] -> 'polinómica', best_degree in (2, 3).""" + x = np.linspace(-10.0, 10.0, 60) + y = x ** 2 + + r = classify_relationship_type(list(x), list(y)) + _assert_shape(r) + + assert r["tipo"].startswith("polinómica") + assert r["best_degree"] in (2, 3) + # Una parabola perfecta queda capturada por el grado 2 (parsimonia). + assert r["best_degree"] == 2 + assert r["r2_poly2"] > 0.99 + assert r["coeffs"] is not None and len(r["coeffs"]) == r["best_degree"] + 1 + + +def test_monotona_no_lineal(): + """Golden: monotona convexa de cola pesada -> 'monótona no-lineal'. + + y = 1/(N+1-i)**2 es estrictamente creciente (Spearman ~ 1) pero su cola + explosiva hace que ni la recta ni un polinomio de grado 2/3 la ajusten + (R^2 polinomico < 0.5), de modo que el Pearson lineal NO capta la relacion + que el rango (Spearman) si ve. Construccion deterministica (sin azar). + """ + n = 200 + i = np.arange(n, dtype=float) + y = 1.0 / (n + 1 - i) ** 2 + + r = classify_relationship_type(list(i), list(y)) + _assert_shape(r) + + assert r["tipo"] == "monótona no-lineal" + assert r["best_degree"] is None + assert r["coeffs"] is None + # Spearman fuerte y claramente por encima del Pearson. + assert abs(r["spearman"]) >= 0.5 + assert abs(r["spearman"]) - abs(r["pearson"]) >= 0.15 + + +def test_monotona_exponencial(): + """DoD literal: y = exp(x) (monotona no-lineal) -> 'monótona no-lineal'. + + exp es estrictamente creciente (Spearman = 1) pero el Pearson lineal queda + claramente por debajo (~0.86), así que la dominancia del rango la marca como + monótona no-lineal en vez de lineal o polinómica. + """ + x = np.linspace(0.0, 5.0, 80) + y = np.exp(x) + + r = classify_relationship_type(list(x), list(y)) + _assert_shape(r) + + assert r["tipo"] == "monótona no-lineal" + assert r["best_degree"] is None and r["coeffs"] is None + assert abs(r["spearman"]) >= 0.9 + assert abs(r["spearman"]) - abs(r["pearson"]) >= 0.1 + + +def test_debil_sin_forma(): + """Golden: x e y independientes (semilla fija) -> 'débil/sin forma'.""" + rng = np.random.default_rng(0) + x = rng.normal(0.0, 1.0, 200) + y = rng.normal(0.0, 1.0, 200) + + r = classify_relationship_type(list(x), list(y)) + _assert_shape(r) + + assert r["tipo"] == "débil/sin forma" + assert r["best_degree"] is None + assert r["coeffs"] is None + # Todas las senales son bajas. + assert abs(r["pearson"]) < 0.3 + assert r["r2_linear"] < 0.1 + + +def test_lista_vacia_no_lanza(): + """Edge: listas vacias -> dict debil canonico, sin lanzar.""" + r = classify_relationship_type([], []) + _assert_shape(r) + assert r["tipo"] == "débil/sin forma" + assert r["pearson"] is None + assert r["r2_linear"] is None + assert r["spearman"] is None + assert r["r2_poly2"] is None + assert r["r2_poly3"] is None + assert r["best_degree"] is None + assert r["coeffs"] is None + + +def test_longitudes_distintas_no_lanza(): + """Edge: listas de distinta longitud -> empareja por indice, no lanza.""" + # zip trunca a la longitud minima: solo 3 pares (< 5) -> debil. + r = classify_relationship_type([1, 2, 3, 4, 5, 6, 7, 8], [1.0, 2.0, 3.0]) + _assert_shape(r) + assert r["tipo"] == "débil/sin forma" + assert r["best_degree"] is None + + +def test_todos_none_no_lanza(): + """Edge: todos los valores None -> ningun par valido -> debil, no lanza.""" + r = classify_relationship_type([None, None, None, None, None, None], + [None, None, None, None, None, None]) + _assert_shape(r) + assert r["tipo"] == "débil/sin forma" + assert r["coeffs"] is None + + +def test_entradas_none_no_lanza(): + """Edge: xs/ys None directamente -> debil, no lanza.""" + assert classify_relationship_type(None, None)["tipo"] == "débil/sin forma" + assert classify_relationship_type([1.0, 2.0], None)["tipo"] == "débil/sin forma" + + +def test_constante_no_lanza(): + """Edge: ys constante (varianza ~0) -> debil, no lanza.""" + r = classify_relationship_type([1, 2, 3, 4, 5, 6, 7], [5, 5, 5, 5, 5, 5, 5]) + _assert_shape(r) + assert r["tipo"] == "débil/sin forma" + + +def test_filtra_nan_inf_bool(): + """Edge: pares con NaN/inf/bool/None se descartan por indice.""" + nan = float("nan") + inf = float("inf") + # Solo i=0,1,2,3,4 quedan validos (5 pares) y forman una recta perfecta. + xs = [0.0, 1.0, 2.0, 3.0, 4.0, nan, inf, True, None] + ys = [1.0, 3.0, 5.0, 7.0, 9.0, 1.0, 2.0, 3.0, 4.0] + r = classify_relationship_type(xs, ys) + _assert_shape(r) + # Los 5 pares validos son y = 2x + 1 exacto -> lineal. + assert r["tipo"] == "lineal" + assert r["best_degree"] == 1 diff --git a/python/functions/datascience/relationship_scatter_figure.md b/python/functions/datascience/relationship_scatter_figure.md new file mode 100644 index 00000000..2bedac32 --- /dev/null +++ b/python/functions/datascience/relationship_scatter_figure.md @@ -0,0 +1,122 @@ +--- +id: relationship_scatter_figure_py_datascience +name: relationship_scatter_figure +kind: function +lang: py +domain: datascience +version: "1.0.0" +purity: impure +signature: "def relationship_scatter_figure(xs: list, ys: list, x_label: str = \"\", y_label: str = \"\", classification: dict = None, max_points: int = 2000) -> \"matplotlib.figure.Figure\"" +description: "Construye una figura matplotlib scatter de un par de variables numéricas con su curva/recta de ajuste y una anotación del tipo de relación (lineal, polinómica grado 2/3, monótona no-lineal, etc.) más sus métricas (r, ρ, R²lin, R²poly). Consume el dict de classify_relationship_type; si es None lo calcula internamente reusando esa función. Devuelve un matplotlib.figure.Figure listo para rasterizar por el renderer del informe EDA (PDF/PPTX). Backend Agg sin pyplot global; downsample determinista de los puntos dibujados; defensivo ante vacío/None." +tags: [eda, correlation, scatter, relationship, matplotlib, figure, visualization, datascience, impure] +uses_functions: [classify_relationship_type_py_datascience] +uses_types: [] +returns: [] +returns_optional: false +error_type: "error_go_core" +imports: [matplotlib, numpy] +example: | + from relationship_scatter_figure import relationship_scatter_figure + xs = [float(i) for i in range(100)] + ys = [0.5 * x * x - x + 3 for x in xs] + classification = { + "tipo": "polinómica (grado 2)", "pearson": 0.97, "spearman": 0.99, + "r2_linear": 0.92, "r2_poly2": 0.999, "r2_poly3": 0.999, + "best_degree": 2, "coeffs": [0.5, -1.0, 3.0], + } + fig = relationship_scatter_figure(xs, ys, x_label="dosis", y_label="efecto", classification=classification) +tested: true +tests: + - "test_returns_figure" + - "test_downsample_determinista" + - "test_empty_no_lanza" + - "test_classification_none" +test_file_path: "python/functions/datascience/relationship_scatter_figure_test.py" +file_path: "python/functions/datascience/relationship_scatter_figure.py" +params: + - name: xs + desc: "Lista (o tupla) de valores x. Se emparejan por índice con ys. Valores None, bool, NaN o inf descartan ese par (lectura defensiva)." + - name: ys + desc: "Lista (o tupla) de valores y, paralela a xs. Mismas reglas defensivas que xs." + - name: x_label + desc: "Etiqueta del eje/título para la variable x. Default \"\" (en el título cae a \"x\")." + - name: y_label + desc: "Etiqueta del eje/título para la variable y. Default \"\" (en el título cae a \"y\")." + - name: classification + desc: "Opcional. Dict de classify_relationship_type con claves tipo, pearson, r2_linear, spearman, r2_poly2, r2_poly3, best_degree, coeffs. Si es None se calcula internamente importando y llamando a classify_relationship_type sobre los pares limpios (self-contained). Si el módulo hermano no está disponible, se dibuja el scatter sin curva de ajuste ni anotación. Default None." + - name: max_points + desc: "Tope del nº de puntos DIBUJADOS. Si los pares limpios superan el tope, la nube se submuestrea por paso fijo ceil(n/max_points) tomando pairs[::step] — DETERMINISTA, no aleatorio, reproducible. La clasificación/ajuste usa SIEMPRE todos los pares limpios; el downsample solo adelgaza el dibujo. Valor no-positivo o no-int desactiva el downsample. Default 2000." +output: "Un matplotlib.figure.Figure (figsize 6.4x4.0, dpi 150) con un Axes scatter (puntos semitransparentes alpha 0.5, color #4C72B0), la curva/recta de ajuste (numpy.polyval sobre coeffs, color #C44E52) cuando hay un ajuste polinómico disponible, título \"{x_label} ↔ {y_label}\", labels de ejes y una caja de anotación en la esquina superior izquierda con el tipo de relación y las métricas disponibles (r, ρ, R²lin, R²poly; se omiten las None). Si tras la limpieza hay menos de 2 pares válidos, devuelve igualmente una Figure con un texto centrado \"Sin datos suficientes para el scatter\" (nunca lanza). El caller rasteriza/cierra la figura; la función no la muestra ni la guarda." +--- + +## Ejemplo + +```python +from relationship_scatter_figure import relationship_scatter_figure + +# Par numérico con relación cuadrática y su clasificación (de +# classify_relationship_type). Pasándola explícita evitas recomputarla. +xs = [float(i) for i in range(100)] +ys = [0.5 * x * x - x + 3 for x in xs] +classification = { + "tipo": "polinómica (grado 2)", + "pearson": 0.97, + "spearman": 0.99, + "r2_linear": 0.92, + "r2_poly2": 0.999, + "r2_poly3": 0.999, + "best_degree": 2, + "coeffs": [0.5, -1.0, 3.0], +} + +fig = relationship_scatter_figure( + xs, ys, x_label="dosis", y_label="efecto", classification=classification +) + +# El renderer del informe lo rasteriza; aquí solo persistimos para inspección. +fig.savefig("/tmp/scatter_dosis_efecto.png") + +# Con classification=None la función la calcula internamente (self-contained): +fig2 = relationship_scatter_figure(xs, ys, x_label="dosis", y_label="efecto") +``` + +## Cuando usarla + +Úsala dentro del informe EDA automático cuando quieras visualizar de un vistazo +la relación entre dos variables numéricas: la nube de puntos, la curva que mejor +la ajusta y una etiqueta legible del tipo de relación con sus métricas. Es la +pareja "vista humana" de `classify_relationship_type`: esa función decide el +tipo y los coeficientes; esta los pinta en una `Figure` que el renderer del +informe rasteriza a PDF/PPTX. Pásale el dict de clasificación si ya lo tienes +calculado (evitas recomputar el ajuste); si no, déjalo en `None` y la función lo +resuelve sola sobre los pares limpios. Pensada para móvil: anotación pequeña +(fontsize 8) y nube adelgazada por `max_points` para que el PDF no pese. + +## Gotchas + +- **Impura por matplotlib.** Toca la maquinaria de render. Usa el backend `Agg` + y la API orientada a objetos `Figure`/`add_subplot` — NUNCA `pyplot.*` aquí, + para no tocar el estado global ni filtrar figuras entre llamadas. `pyplot` NO + es thread-safe; esta función lo evita construyendo el `Figure` directamente, + así que es segura de llamar en bucle desde el renderer. +- **El caller cierra la figura.** Devuelve el `Figure` pero no lo muestra ni lo + guarda. Quien la consume debe rasterizarla y luego liberarla + (`matplotlib.pyplot.close(fig)`) para no acumular memoria en lotes grandes de + pares de columnas. +- **Downsample determinista, solo del dibujo.** Cuando los pares limpios superan + `max_points`, la nube DIBUJADA se adelgaza por paso fijo `pairs[::step]` + (reproducible, no aleatorio). La clasificación y el ajuste usan SIEMPRE todos + los pares limpios; el downsample no altera las métricas ni la curva. +- **`classification=None` ⇒ se calcula sola.** Importa y llama a + `classify_relationship_type` sobre los pares limpios. Si ese módulo hermano no + está disponible (entorno incompleto), NO lanza: dibuja el scatter sin curva de + ajuste ni anotación. Pasar la clasificación explícita es más barato (no + recomputa el ajuste). +- **Sin curva para `monótona no-lineal`.** Cuando `coeffs` es `None` o + `best_degree` es `None` (p.ej. tipo "monótona no-lineal"), no se pinta recta + polinómica — solo la nube y la anotación. Tampoco se dibuja la curva si el + rango de x es nulo (todos los x iguales). Nunca falla por esto. +- **Defensiva, nunca lanza.** `xs=[]`, `ys=[]`, menos de 2 pares válidos, ends + `None`/`bool`/`NaN`/`inf` o `coeffs` malformado se manejan sin error: en el + peor caso devuelve una `Figure` con "Sin datos suficientes para el scatter". + No envuelvas la llamada en try/except por miedo a un raise — no lo hay. diff --git a/python/functions/datascience/relationship_scatter_figure.py b/python/functions/datascience/relationship_scatter_figure.py new file mode 100644 index 00000000..dd0b3371 --- /dev/null +++ b/python/functions/datascience/relationship_scatter_figure.py @@ -0,0 +1,322 @@ +"""Impure EDA helper: scatter figure of a numeric pair with its fit (`eda` group). + +Builds a matplotlib scatter of two numeric variables, overlays the fitted +curve/line implied by the relationship classification (linear, polynomial of +degree 2/3, etc.) and annotates the relationship type with its available +metrics. Returns a ready-to-rasterize ``matplotlib.figure.Figure``; it never +shows nor saves it. + +Impure because it touches matplotlib's rendering machinery. It uses the headless +Agg backend and the object-oriented ``Figure`` API (no ``pyplot``) so it leaks no +global state and is safe to call repeatedly from a report renderer. + +To keep the rendered PDF/PPTX light on phones, when the number of valid pairs +exceeds ``max_points`` the *plotted* points are down-sampled DETERMINISTICALLY by +a fixed step (``pairs[::step]``), never randomly, so the output is reproducible. +The classification/fit always uses every clean pair; the down-sample only thins +the drawn cloud. +""" + +import math + +import matplotlib + +matplotlib.use("Agg") + +import numpy as np # noqa: E402 +from matplotlib.figure import Figure # noqa: E402 + +# Sober blue for the scatter cloud and red for the fitted curve (Tufte: the +# data points are the primary ink, the fit is the secondary highlight). +_POINT_COLOR = "#4C72B0" +_FIT_COLOR = "#C44E52" +# Muted gray for the no-data fallback message. +_MUTED_TEXT = "#5f6b7a" + + +def _finite(value): + """Coerce ``value`` to a finite float, or return None when not usable. + + bool is a subclass of int, but a real numeric measurement is never a bool, + so True/False are treated as missing instead of coercing to 1.0/0.0. NaN and + +/-infinity are never valid either. + """ + if value is None or isinstance(value, bool): + return None + try: + f = float(value) + except (TypeError, ValueError): + return None + if math.isnan(f) or math.isinf(f): + return None + return f + + +def _clean_pairs(xs, ys): + """Pair ``xs[i], ys[i]`` by index, dropping any pair with a non-finite end.""" + pairs = [] + if isinstance(xs, (list, tuple)) and isinstance(ys, (list, tuple)): + n = min(len(xs), len(ys)) + for i in range(n): + x = _finite(xs[i]) + y = _finite(ys[i]) + if x is None or y is None: + continue + pairs.append((x, y)) + return pairs + + +def _ordered_trend(xs_clean, ys_clean, n_bins: int = 12): + """Return (x_trend, y_trend): the ordered trend of y over x for a monotonic + relationship that has no polynomial fit. + + When x has few distinct values (an ordinal/discrete scale) the trend is the + mean of y per distinct x value. Otherwise x is split into ``n_bins`` ordered + quantile bins and each point is (mean x, mean y) of the bin. Returns + ``(None, None)`` when there is nothing meaningful to draw. + """ + x_arr = np.asarray(xs_clean, dtype=float) + y_arr = np.asarray(ys_clean, dtype=float) + if x_arr.size < 2: + return None, None + uniq = np.unique(x_arr) + if uniq.size <= max(2, n_bins): + # Discrete x: one trend point per distinct value (mean y). + xt = uniq + yt = np.array([float(np.mean(y_arr[x_arr == ux])) for ux in uniq]) + return xt, yt + # Continuous x: ordered quantile bins, (mean x, mean y) per bin. + order = np.argsort(x_arr, kind="stable") + x_sorted = x_arr[order] + y_sorted = y_arr[order] + chunks_x = np.array_split(x_sorted, n_bins) + chunks_y = np.array_split(y_sorted, n_bins) + xt = np.array([float(np.mean(cx)) for cx in chunks_x if cx.size]) + yt = np.array([float(np.mean(cy)) for cy in chunks_y if cy.size]) + return xt, yt + + +def _no_data_figure(message: str) -> "matplotlib.figure.Figure": + """A bare Figure carrying a centered muted message (defensive fallback).""" + fig = Figure(figsize=(6.4, 4.0), dpi=150) + ax = fig.add_subplot(111) + ax.axis("off") + ax.text( + 0.5, + 0.5, + message, + ha="center", + va="center", + fontsize=12, + color=_MUTED_TEXT, + transform=ax.transAxes, + ) + fig.tight_layout() + return fig + + +def _metrics_caption(classification: dict) -> str: + """Format the available metrics of a classification dict into one line. + + Omits the metrics that are None. Keys consumed (any may be absent/None): + ``pearson`` (r), ``spearman`` (rho), ``r2_linear`` (R²lin) and the best + polynomial R² (``r2_poly3`` if a cubic was the best fit, else ``r2_poly2``). + """ + parts = [] + r = _finite(classification.get("pearson")) + if r is not None: + parts.append(f"r={r:.2f}") + rho = _finite(classification.get("spearman")) + if rho is not None: + parts.append(f"ρ={rho:.2f}") + r2_lin = _finite(classification.get("r2_linear")) + if r2_lin is not None: + parts.append(f"R²lin={r2_lin:.2f}") + # Prefer the R² of the best polynomial degree when it is a poly fit. + best_degree = classification.get("best_degree") + r2_poly = None + if best_degree == 3: + r2_poly = _finite(classification.get("r2_poly3")) + elif best_degree == 2: + r2_poly = _finite(classification.get("r2_poly2")) + if r2_poly is None: + # Fall back to whichever poly R² is present (cubic first). + r2_poly = _finite(classification.get("r2_poly3")) + if r2_poly is None: + r2_poly = _finite(classification.get("r2_poly2")) + if r2_poly is not None: + parts.append(f"R²poly={r2_poly:.2f}") + return " ".join(parts) + + +def relationship_scatter_figure( + xs: list, + ys: list, + x_label: str = "", + y_label: str = "", + classification: dict = None, + max_points: int = 2000, +) -> "matplotlib.figure.Figure": + """Build a scatter figure of a numeric pair with its fit and a type label. + + Cleans the pairs defensively (drops any pair with a None/bool/NaN/inf end), + plots a semi-transparent scatter cloud (down-sampled deterministically when + it exceeds ``max_points``), overlays the polynomial fit implied by + ``classification`` and annotates the relationship type plus its available + metrics in a corner box. + + The fit and classification always use every clean pair; only the drawn cloud + is thinned by the down-sample. When ``classification`` is None it is computed + internally by reusing ``classify_relationship_type`` over the clean pairs, so + the function is self-contained. + + The function is fully defensive: empty input, fewer than 2 clean pairs, a + missing/None ``coeffs`` or a missing sibling classifier never raise. When + there is nothing valid to draw it still returns a ``Figure`` carrying a + centered "Sin datos suficientes para el scatter" message. + + Args: + xs: List (or tuple) of x values. Paired by index with ``ys``. Values that + are None, bool, NaN or infinite discard that pair. Read defensively. + ys: List (or tuple) of y values, parallel to ``xs``. Same defensive rules. + x_label: Axis/title label for the x variable. Default "" (falls back to + "x" in the title). + y_label: Axis/title label for the y variable. Default "" (falls back to + "y" in the title). + classification: Optional dict from ``classify_relationship_type`` with + keys ``tipo, pearson, r2_linear, spearman, r2_poly2, r2_poly3, + best_degree, coeffs``. When None, it is computed internally by + importing and calling ``classify_relationship_type`` over the clean + pairs. When that sibling module is unavailable, the scatter is still + drawn (no fit curve, no annotation). + max_points: Cap on the number of *plotted* points. When the number of + clean pairs exceeds this cap, the drawn cloud is down-sampled by a + fixed step ``ceil(n/max_points)`` taking ``pairs[::step]`` — + DETERMINISTIC, not random, so the figure is reproducible. A + non-positive or non-int value disables down-sampling. Default 2000. + + Returns: + A ``matplotlib.figure.Figure`` (figsize 6.4x4.0, dpi 150) with a single + scatter Axes, the fitted curve (when a polynomial fit is available) and a + corner annotation with the relationship type and metrics. When there are + fewer than 2 clean pairs it returns a Figure with a centered "Sin datos + suficientes para el scatter" message. The caller rasterizes/closes it. + """ + pairs = _clean_pairs(xs, ys) + if len(pairs) < 2: + return _no_data_figure("Sin datos suficientes para el scatter") + + # Full clean coordinates feed the classification/fit; the plotted cloud is + # what gets thinned. + xs_clean = [p[0] for p in pairs] + ys_clean = [p[1] for p in pairs] + + # Resolve the classification. If not provided, reuse the sibling classifier + # over ALL clean pairs (self-contained). Missing module => no fit/annotation. + cls = classification + if cls is None: + try: + from classify_relationship_type import classify_relationship_type + + cls = classify_relationship_type(xs_clean, ys_clean) + except Exception: + cls = None + if not isinstance(cls, dict): + cls = {} + + # --- Deterministic down-sampling of the DRAWN points only. + n_total = len(pairs) + if ( + isinstance(max_points, int) + and not isinstance(max_points, bool) + and max_points > 0 + and n_total > max_points + ): + step = math.ceil(n_total / max_points) + sampled = pairs[::step] + else: + sampled = pairs + + x_plot = [p[0] for p in sampled] + y_plot = [p[1] for p in sampled] + + fig = Figure(figsize=(6.4, 4.0), dpi=150) + ax = fig.add_subplot(111) + + ax.scatter( + x_plot, + y_plot, + s=12, + alpha=0.5, + color=_POINT_COLOR, + edgecolors="none", + rasterized=True, + ) + + # --- Fitted curve/line over the full clean x range. + coeffs = cls.get("coeffs") + best_degree = cls.get("best_degree") + tipo = cls.get("tipo") + x_min, x_max = min(xs_clean), max(xs_clean) + drew_fit = False + if coeffs is not None and best_degree is not None and x_max > x_min: + try: + coeff_arr = np.asarray(coeffs, dtype=float) + if coeff_arr.ndim == 1 and coeff_arr.size > 0 and np.all(np.isfinite(coeff_arr)): + x_line = np.linspace(x_min, x_max, 200) + y_line = np.polyval(coeff_arr, x_line) + if np.all(np.isfinite(y_line)): + ax.plot(x_line, y_line, color=_FIT_COLOR, linewidth=2) + drew_fit = True + except Exception: + # Never fail the figure because of a malformed coeffs array. + pass + + # A monotonic non-linear relationship has no fitted polynomial (coeffs is + # None by design — a low-degree polynomial would mislead). Draw instead the + # ordered trend of y over x so the reader still sees the shape: y averaged + # within ordered x-bins (or per distinct x value when x is discrete with few + # levels, e.g. an ordinal scale). Defensive: any failure leaves the cloud. + if (not drew_fit and isinstance(tipo, str) and "monóton" in tipo.lower() + and x_max > x_min): + try: + xt, yt = _ordered_trend(xs_clean, ys_clean) + if xt is not None and len(xt) >= 2: + ax.plot(xt, yt, color=_FIT_COLOR, linewidth=2, marker="o", + markersize=3) + except Exception: + pass + + # --- Labels and title. + tx = x_label if x_label else "x" + ty = y_label if y_label else "y" + ax.set_title(f"{tx} ↔ {ty}", fontsize=12, loc="left", pad=8) + ax.set_xlabel(x_label) + ax.set_ylabel(y_label) + + # --- Corner annotation: relationship type + available metrics. + caption_lines = [] + if tipo: + caption_lines.append(str(tipo)) + metrics_line = _metrics_caption(cls) + if metrics_line: + caption_lines.append(metrics_line) + if caption_lines: + ax.text( + 0.03, + 0.97, + "\n".join(caption_lines), + transform=ax.transAxes, + ha="left", + va="top", + fontsize=8, + bbox=dict( + boxstyle="round,pad=0.35", + facecolor="white", + edgecolor="#cccccc", + alpha=0.85, + ), + ) + + fig.tight_layout() + return fig diff --git a/python/functions/datascience/relationship_scatter_figure_test.py b/python/functions/datascience/relationship_scatter_figure_test.py new file mode 100644 index 00000000..c8aa12c3 --- /dev/null +++ b/python/functions/datascience/relationship_scatter_figure_test.py @@ -0,0 +1,100 @@ +"""Tests para relationship_scatter_figure (scatter de un par numérico, grupo eda). + +Usa el backend Agg sin pyplot global; no muestra ni guarda figuras. Cada test +cierra explícitamente la Figure construida (matplotlib.pyplot.close) para no +acumular estado entre tests. +""" + +import os +import sys + +sys.path.insert(0, os.path.dirname(__file__)) + +import matplotlib + +matplotlib.use("Agg") + +import matplotlib.pyplot as plt # noqa: E402 +from matplotlib.collections import PathCollection # noqa: E402 +from matplotlib.figure import Figure # noqa: E402 + +from relationship_scatter_figure import relationship_scatter_figure + + +def _scatter_offsets(fig): + """Return the plotted points of the first PathCollection (scatter) found.""" + for ax in fig.axes: + for coll in ax.collections: + if isinstance(coll, PathCollection): + return coll.get_offsets() + return None + + +def test_returns_figure(): + xs = [float(i) for i in range(20)] + ys = [2.0 * x + 1.0 for x in xs] # y = 2x + 1 + classification = { + "tipo": "lineal", + "pearson": 1.0, + "r2_linear": 1.0, + "spearman": 1.0, + "r2_poly2": 1.0, + "r2_poly3": 1.0, + "best_degree": 1, + "coeffs": [2.0, 1.0], + } + fig = relationship_scatter_figure( + xs, ys, x_label="a", y_label="b", classification=classification + ) + assert hasattr(fig, "savefig") + assert len(fig.axes) >= 1 + plt.close(fig) + + +def test_downsample_determinista(): + n = 5000 + xs = [float(i) for i in range(n)] + ys = [0.5 * x for x in xs] + classification = { + "tipo": "lineal", + "pearson": 1.0, + "r2_linear": 1.0, + "spearman": 1.0, + "r2_poly2": 1.0, + "r2_poly3": 1.0, + "best_degree": 1, + "coeffs": [0.5, 0.0], + } + fig = relationship_scatter_figure( + xs, ys, x_label="x", y_label="y", classification=classification, max_points=1000 + ) + assert isinstance(fig, Figure) + offsets = _scatter_offsets(fig) + assert offsets is not None + # El nº de puntos dibujados no debe exceder el cap. + assert len(offsets) <= 1000 + plt.close(fig) + + +def test_empty_no_lanza(): + fig = relationship_scatter_figure([], [], x_label="x", y_label="y") + assert isinstance(fig, Figure) + plt.close(fig) + + +def test_classification_none(): + # Solo se ejecuta si el módulo hermano classify_relationship_type existe. + try: + import classify_relationship_type # noqa: F401 + except Exception: + import pytest + + pytest.skip("classify_relationship_type aún no disponible") + xs = [float(i) for i in range(30)] + ys = [3.0 * x - 2.0 for x in xs] + fig = relationship_scatter_figure( + xs, ys, x_label="a", y_label="b", classification=None + ) + assert isinstance(fig, Figure) + assert len(fig.axes) >= 1 + plt.close(fig) From 105e56cf05eb5c077f2296b01313db783796a6d5 Mon Sep 17 00:00:00 2001 From: Egutierrez <egutierrez@dead.dd> Date: Tue, 30 Jun 2026 20:38:17 +0200 Subject: [PATCH 38/53] =?UTF-8?q?feat(eda):=20cap=C3=ADtulo=20text=5Fdistr?= =?UTF-8?q?=20(TEXTO/NLP)=20=E2=80=94=20primer=20cap=C3=ADtulo=20de=20dato?= =?UTF-8?q?s=20no=20tabulares?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Añade el capítulo `text_distr` al motor AutomaticEDA: perfila columnas de texto libre largo (reseñas, descripciones, comentarios) que la distribución categórica no resume bien. Sigue el patrón de cat_distr/num_distr (build_text_distr(profile, ctx) -> Chapter | None) y se registra en CHAPTER_ORDER tras cat_distr. Activación en dos fases: gate barato desde el perfil (columna no numérica con len_mean >= 50 chars) + confirmación con muestra cruda (mediana de palabras >= 20). Un dataset sin texto largo (p.ej. titanic) devuelve None sin tocar el informe. Bloques por columna (Group con page_break): resumen (longitudes, vocabulario con TTR y % hapax, idioma dominante, % duplicados, legibilidad), histograma de longitudes, top términos (tabla + barras), bigramas/trigramas, idiomas detectados y nube de palabras opcional. Términos ttr/hapax enganchados al glosario clicable. Lógica delegada a 7 funciones nuevas del registry (datascience, tag eda), estilo dict-no-throw: - extract_text_sample (impura, push-down SQL DuckDB/Postgres) - compute_text_length_stats, compute_vocabulary_stats, compute_top_ngrams (puras, stdlib) - detect_corpus_language (langdetect opcional), compute_text_readability (textstat opcional), compute_text_duplicates (hash + datasketch opcional) Versión barata sin modelos pesados: las piezas que dependen de una librería opcional (langdetect, textstat, wordcloud, datasketch) degradan a omitidas sin lanzar. Añade langdetect y textstat (ligeras) al pyproject + uv.lock. Verificado: golden sobre dataset de reviews multi-idioma (capítulo presente en PDF+PPTX+MD con métricas reales), titanic sin capítulo (None), degradación sin libs, suite automatic_eda + pipeline verde (128 passed), fn index OK. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> --- .../automatic_eda/chapters/text_distr.py | 559 ++++++++++++++++++ .../automatic_eda/chapters/text_distr_test.py | 256 ++++++++ .../automatic_eda/chapters_registry.py | 1 + .../datascience/compute_text_duplicates.md | 102 ++++ .../datascience/compute_text_duplicates.py | 128 ++++ .../compute_text_duplicates_test.py | 77 +++ .../datascience/compute_text_length_stats.md | 86 +++ .../datascience/compute_text_length_stats.py | 168 ++++++ .../compute_text_length_stats_test.py | 70 +++ .../datascience/compute_text_readability.md | 88 +++ .../datascience/compute_text_readability.py | 121 ++++ .../compute_text_readability_test.py | 74 +++ .../datascience/compute_top_ngrams.md | 103 ++++ .../datascience/compute_top_ngrams.py | 94 +++ .../datascience/compute_top_ngrams_test.py | 65 ++ .../datascience/compute_vocabulary_stats.md | 91 +++ .../datascience/compute_vocabulary_stats.py | 99 ++++ .../compute_vocabulary_stats_test.py | 74 +++ .../datascience/detect_corpus_language.md | 80 +++ .../datascience/detect_corpus_language.py | 91 +++ .../detect_corpus_language_test.py | 58 ++ .../datascience/extract_text_sample.md | 102 ++++ .../datascience/extract_text_sample.py | 112 ++++ .../datascience/extract_text_sample_test.py | 83 +++ python/pyproject.toml | 2 + python/uv.lock | 96 +++ 26 files changed, 2880 insertions(+) create mode 100644 python/functions/datascience/automatic_eda/chapters/text_distr.py create mode 100644 python/functions/datascience/automatic_eda/chapters/text_distr_test.py create mode 100644 python/functions/datascience/compute_text_duplicates.md create mode 100644 python/functions/datascience/compute_text_duplicates.py create mode 100644 python/functions/datascience/compute_text_duplicates_test.py create mode 100644 python/functions/datascience/compute_text_length_stats.md create mode 100644 python/functions/datascience/compute_text_length_stats.py create mode 100644 python/functions/datascience/compute_text_length_stats_test.py create mode 100644 python/functions/datascience/compute_text_readability.md create mode 100644 python/functions/datascience/compute_text_readability.py create mode 100644 python/functions/datascience/compute_text_readability_test.py create mode 100644 python/functions/datascience/compute_top_ngrams.md create mode 100644 python/functions/datascience/compute_top_ngrams.py create mode 100644 python/functions/datascience/compute_top_ngrams_test.py create mode 100644 python/functions/datascience/compute_vocabulary_stats.md create mode 100644 python/functions/datascience/compute_vocabulary_stats.py create mode 100644 python/functions/datascience/compute_vocabulary_stats_test.py create mode 100644 python/functions/datascience/detect_corpus_language.md create mode 100644 python/functions/datascience/detect_corpus_language.py create mode 100644 python/functions/datascience/detect_corpus_language_test.py create mode 100644 python/functions/datascience/extract_text_sample.md create mode 100644 python/functions/datascience/extract_text_sample.py create mode 100644 python/functions/datascience/extract_text_sample_test.py diff --git a/python/functions/datascience/automatic_eda/chapters/text_distr.py b/python/functions/datascience/automatic_eda/chapters/text_distr.py new file mode 100644 index 00000000..d0b69c2c --- /dev/null +++ b/python/functions/datascience/automatic_eda/chapters/text_distr.py @@ -0,0 +1,559 @@ +"""Free-text / NLP distributions chapter (TEXT DISTR) for AutomaticEDA. + +First chapter for **non-tabular** content: it profiles the linguistic content of +any column holding long free text (reviews, descriptions, comments, tickets) that +the categorical chapter cannot meaningfully summarize (high cardinality, many +words per value). It is the cheap, model-free counterpart to ``cat_distr`` for +columns that are prose rather than discrete labels. + +Activation (returns ``None`` when it does not apply): + +1. Cheap gate from the aggregated profile: at least one non-numeric column whose + ``categorical.len_mean`` (mean character length) is ``>= _MIN_LEN_CHARS``. + A dataset whose only string columns are short labels (e.g. titanic's + ``Name``, ~27 chars) never passes this gate, so the chapter disappears with + zero extra work and the existing report is untouched. +2. Confirmation from a raw sample: each candidate column is sampled (push-down + ``extract_text_sample`` over ``ctx['db_path']``/``ctx['table']``, or an + in-memory ``ctx['text_raw']`` for tests) and kept only if the **median word + count is ``>= _MIN_WORDS``** — i.e. it is genuinely long text, not a long + single token. If no column survives, the chapter returns ``None``. + +Per surviving column the chapter emits, kept together on its own page/slide +(``Group(page_break_before=...)``): + +- a key/value summary (documents, length percentiles, vocabulary richness with + **[[term:ttr]]TTR[[/term]]** and **[[term:hapax]]hapax legomena[[/term]]**, + dominant language, exact-duplicate %, readability when available); +- a word-count histogram figure; +- a top-terms table + a horizontal bar figure; +- bigram and trigram frequency tables; +- a detected-language bar figure (when ``langdetect`` is available); +- an optional word-cloud figure (only when ``wordcloud`` is installed); +- a closing note on duplicates / readability degradation. + +Every metric is delegated to pure ``eda`` registry functions +(``compute_text_length_stats``, ``compute_vocabulary_stats``, +``compute_top_ngrams``, ``detect_corpus_language``, ``compute_text_duplicates``, +``compute_text_readability``) and the raw sample to ``extract_text_sample``; all +are imported defensively so a missing function or optional library degrades that +single piece to a note instead of aborting the chapter. Optional libraries +(``langdetect``, ``textstat``, ``wordcloud``, ``datasketch``) are never required: +the piece is silently omitted when they are absent. + +Contract: build_<id>(profile, ctx) -> Chapter | None ; CHAPTER_VERSION = "x.y.z". +""" + +from __future__ import annotations + +from .. import model + +CHAPTER_VERSION = "1.0.0" +CHAPTER_ID = "text_distr" +CHAPTER_TITLE = "Texto libre (NLP)" + +# Cheap activation gate (characters): a non-numeric column whose mean string +# length reaches this is a candidate for "long text". Short labels (titanic's +# Name ≈ 27 chars) stay below it, so the chapter does not fire on them. +_MIN_LEN_CHARS = 50 +# Confirmation gate (words): a candidate is kept only if its median document has +# at least this many words — genuine prose, not a long id/URL token. +_MIN_WORDS = 20 +# Bound the document so very wide datasets stay readable. +_MAX_TEXT_COLS = 5 +# Raw text rows to sample per column when the chapter must extract them itself. +_SAMPLE_ROWS = 2000 +# Rows shown in the frequency tables. +_TOP_TERMS = 15 +_TOP_NGRAMS = 10 + +# Glossary terms this chapter explains (registered in the shared collector and +# marked clickable on first appearance — same mechanism as cat_distr's entropía). +_TERMS = { + "ttr": ( + "TTR (type-token ratio)", + "Riqueza léxica de un texto: número de palabras distintas (tipos) " + "dividido por el número total de palabras (tokens). Vale 1 cuando no se " + "repite ninguna palabra (máxima variedad) y baja hacia 0 cuando el " + "vocabulario se repite mucho. Depende de la longitud del corpus, así que " + "compara mejor textos de tamaño parecido."), + "hapax": ( + "Hapax legomena", + "Palabras que aparecen una sola vez en todo el corpus. Un porcentaje " + "alto de hapax indica vocabulario muy variado o, a veces, ruido " + "(erratas, identificadores, tokens raros). Se expresa como porcentaje " + "sobre el número de palabras distintas."), +} + + +def _fmt_int(value) -> str: + if value is None: + return "—" + try: + return f"{int(value):,}".replace(",", ".") + except (TypeError, ValueError): + return str(value) + + +def _fmt_num(value, decimals: int = 2) -> str: + if value is None: + return "—" + if isinstance(value, bool): + return str(value) + if isinstance(value, int): + return f"{value:,}".replace(",", ".") + if isinstance(value, float): + if value != value: # NaN + return "NaN" + if value in (float("inf"), float("-inf")): + return str(value) + text = f"{value:.{decimals}f}".rstrip("0").rstrip(".") + return text if text else "0" + return str(value) + + +def _fmt_pct(value, decimals: int = 1) -> str: + if value is None: + return "—" + try: + return f"{float(value):.{decimals}f}%" + except (TypeError, ValueError): + return str(value) + + +def _truncate(text, limit: int = 40) -> str: + s = model._safe_str(text) + return s if len(s) <= limit else s[: max(1, limit - 1)].rstrip() + "…" + + +# --------------------------------------------------------------------------- # +# Defensive wrappers around the registry functions: each returns the function's +# output dict or a safe empty default, never raising and never importing at +# module load (so the chapter stays importable even if a function is missing). +# --------------------------------------------------------------------------- # +def _length_stats(texts) -> dict: + try: + from datascience.compute_text_length_stats import compute_text_length_stats + out = compute_text_length_stats(texts) + if isinstance(out, dict): + return out + except Exception: # noqa: BLE001 + pass + return {} + + +def _vocab_stats(texts) -> dict: + try: + from datascience.compute_vocabulary_stats import compute_vocabulary_stats + out = compute_vocabulary_stats(texts, top_k=_TOP_TERMS) + if isinstance(out, dict): + return out + except Exception: # noqa: BLE001 + pass + return {} + + +def _ngrams(texts, n) -> list: + try: + from datascience.compute_top_ngrams import compute_top_ngrams + out = compute_top_ngrams(texts, n=n, top_k=_TOP_NGRAMS) + if isinstance(out, dict): + return out.get("top") or [] + except Exception: # noqa: BLE001 + pass + return [] + + +def _language(texts) -> dict: + try: + from datascience.detect_corpus_language import detect_corpus_language + out = detect_corpus_language(texts) + if isinstance(out, dict): + return out + except Exception: # noqa: BLE001 + pass + return {"available": False, "distribution": [], "dominant": None} + + +def _duplicates(texts) -> dict: + try: + from datascience.compute_text_duplicates import compute_text_duplicates + out = compute_text_duplicates(texts) + if isinstance(out, dict): + return out + except Exception: # noqa: BLE001 + pass + return {} + + +def _readability(texts) -> dict: + try: + from datascience.compute_text_readability import compute_text_readability + out = compute_text_readability(texts) + if isinstance(out, dict): + return out + except Exception: # noqa: BLE001 + pass + return {"available": False, "flesch": {}} + + +# --------------------------------------------------------------------------- # +# Candidate detection + raw sample acquisition. +# --------------------------------------------------------------------------- # +def _candidate_columns(profile: dict) -> list: + """Cheap gate: non-numeric columns whose mean char length reaches the + threshold. Returns the list of column names (possibly empty).""" + out = [] + for col in profile.get("columns") or []: + if not isinstance(col, dict): + continue + if col.get("inferred_type") == "numeric": + continue + cat = col.get("categorical") + if not isinstance(cat, dict): + continue + len_mean = cat.get("len_mean") + if isinstance(len_mean, (int, float)) and not isinstance(len_mean, bool) \ + and len_mean >= _MIN_LEN_CHARS: + name = col.get("name") + if name: + out.append(str(name)) + return out + + +def _get_samples(profile: dict, ctx: dict, columns: list) -> dict: + """Return {col: [str, ...]} raw text samples for the candidate columns. + + Prefers an in-memory ``ctx['text_raw']`` (used by tests); otherwise pushes a + sample down to the database via ``extract_text_sample`` using ctx db_path / + table. Never raises: returns {} when no sample can be obtained.""" + text_raw = ctx.get("text_raw") + if isinstance(text_raw, dict) and text_raw: + return {c: [str(v) for v in (text_raw.get(c) or []) if v is not None] + for c in columns if text_raw.get(c)} + + db_path = ctx.get("db_path") + table = ctx.get("table") + if not db_path or not table: + return {} + backend = ctx.get("backend") or "duckdb" + sample = ctx.get("sample") or _SAMPLE_ROWS + try: + from datascience.extract_text_sample import extract_text_sample + out = extract_text_sample(db_path, table, columns, backend=backend, + sample=sample) + if isinstance(out, dict) and out.get("status") == "ok": + cols = out.get("columns") + if isinstance(cols, dict): + return {c: list(v) for c, v in cols.items() if v} + except Exception: # noqa: BLE001 — dict-no-throw: no sample → chapter omits. + pass + return {} + + +def _confirm_long_text(samples: dict) -> dict: + """Keep only columns whose median word count reaches _MIN_WORDS. Returns + {col: length_stats_dict} for the survivors, in input order.""" + survivors = {} + for col, texts in samples.items(): + stats = _length_stats(texts) + words = stats.get("words") if isinstance(stats, dict) else None + median = words.get("p50") if isinstance(words, dict) else None + if isinstance(median, (int, float)) and not isinstance(median, bool) \ + and median >= _MIN_WORDS: + survivors[col] = stats + return survivors + + +# --------------------------------------------------------------------------- # +# Figures (lazy matplotlib, scaled by the renderers — same style as num_distr). +# --------------------------------------------------------------------------- # +def _hist_figure(name: str, length_stats: dict): + def make(): + import matplotlib + matplotlib.use("Agg") + from matplotlib.figure import Figure + fig = Figure(figsize=(6.2, 3.0)) + ax = fig.add_subplot(111) + bins = (length_stats or {}).get("word_hist") or [] + drew = False + for b in bins: + if not isinstance(b, dict): + continue + lo, hi, count = b.get("lo"), b.get("hi"), b.get("count") or 0 + if lo is None or hi is None: + continue + width = (hi - lo) if hi > lo else max(abs(lo) * 1e-3, 1e-6) + ax.bar(lo, count, width=width, align="edge", color="#9ec6df", + edgecolor="#5b8aa6", linewidth=0.4) + drew = True + if not drew: + ax.text(0.5, 0.5, "(sin datos de longitud)", ha="center", + va="center", color="#8a8a8a", transform=ax.transAxes) + ax.set_xlabel("palabras por documento", fontsize=8) + ax.set_ylabel("nº de documentos", fontsize=8) + ax.tick_params(labelsize=7) + for spine in ("top", "right"): + ax.spines[spine].set_visible(False) + ax.set_title(f"Longitud de «{_truncate(name, 30)}»", fontsize=10, + loc="left") + fig.tight_layout() + return fig + return make + + +def _barh_figure(title: str, items: list, label_key: str, value_key: str, + xlabel: str): + """Horizontal bar chart from [{label_key:..., value_key:...}, ...].""" + def make(): + import matplotlib + matplotlib.use("Agg") + from matplotlib.figure import Figure + rows = [it for it in (items or []) if isinstance(it, dict) + and isinstance(it.get(value_key), (int, float))] + rows = rows[:12] + fig = Figure(figsize=(6.2, max(2.2, 0.32 * len(rows) + 0.8))) + ax = fig.add_subplot(111) + if not rows: + ax.text(0.5, 0.5, "(sin datos)", ha="center", va="center", + color="#8a8a8a", transform=ax.transAxes) + ax.axis("off") + return fig + labels = [_truncate(r.get(label_key), 28) for r in rows][::-1] + values = [float(r.get(value_key) or 0) for r in rows][::-1] + ypos = range(len(rows)) + ax.barh(list(ypos), values, color="#9ec6df", edgecolor="#5b8aa6", + linewidth=0.4) + ax.set_yticks(list(ypos)) + ax.set_yticklabels(labels, fontsize=7) + ax.set_xlabel(xlabel, fontsize=8) + ax.tick_params(labelsize=7) + for spine in ("top", "right"): + ax.spines[spine].set_visible(False) + ax.set_title(_truncate(title, 44), fontsize=10, loc="left") + fig.tight_layout() + return fig + return make + + +def _wordcloud_figure(texts): + """Word-cloud figure callable, or None if wordcloud is not installed.""" + try: + import wordcloud # noqa: F401 + except Exception: # noqa: BLE001 — optional dependency: omit the figure. + return None + + def make(): + import matplotlib + matplotlib.use("Agg") + from matplotlib.figure import Figure + from wordcloud import WordCloud + fig = Figure(figsize=(6.2, 3.2)) + ax = fig.add_subplot(111) + joined = " ".join(t for t in texts if isinstance(t, str)) + try: + wc = WordCloud(width=800, height=400, background_color="white", + colormap="viridis").generate(joined) + ax.imshow(wc, interpolation="bilinear") + except Exception: # noqa: BLE001 + ax.text(0.5, 0.5, "(nube de palabras no disponible)", ha="center", + va="center", color="#8a8a8a", transform=ax.transAxes) + ax.axis("off") + fig.tight_layout() + return fig + return make + + +# --------------------------------------------------------------------------- # +# Per-column block assembly. +# --------------------------------------------------------------------------- # +def _summary_kv(n_docs, length_stats, vocab, lang, dup, read): + chars = (length_stats or {}).get("chars") or {} + words = (length_stats or {}).get("words") or {} + sents = (length_stats or {}).get("sentences") or {} + rows = [ + ("Documentos", _fmt_int(n_docs)), + ("Caracteres (media · p50 · p90 · p99)", + f"{_fmt_num(chars.get('mean'))} · {_fmt_int(chars.get('p50'))} · " + f"{_fmt_int(chars.get('p90'))} · {_fmt_int(chars.get('p99'))}"), + ("Palabras (media · p50 · p90 · p99)", + f"{_fmt_num(words.get('mean'))} · {_fmt_int(words.get('p50'))} · " + f"{_fmt_int(words.get('p90'))} · {_fmt_int(words.get('p99'))}"), + ("Frases (media · máx)", + f"{_fmt_num(sents.get('mean'))} · {_fmt_int(sents.get('max'))}"), + ("Vocabulario (tokens · tipos · TTR)", + f"{_fmt_int(vocab.get('n_tokens'))} · {_fmt_int(vocab.get('n_types'))} " + f"· {_fmt_num(vocab.get('ttr'), 3)}"), + ("Hapax legomena", + f"{_fmt_int(vocab.get('n_hapax'))} ({_fmt_pct(vocab.get('hapax_pct'))})"), + ] + if isinstance(lang, dict) and lang.get("available"): + dom = lang.get("dominant") + n_langs = len(lang.get("distribution") or []) + rows.append(("Idioma dominante · nº idiomas", + f"{model._safe_str(dom) or '—'} · {_fmt_int(n_langs)}")) + if isinstance(dup, dict) and dup.get("n_docs"): + rows.append(("Duplicados exactos", + f"{_fmt_int(dup.get('n_exact_dup'))} " + f"({_fmt_pct(dup.get('exact_dup_pct'))})")) + if isinstance(read, dict) and read.get("available"): + flesch = read.get("flesch") or {} + rows.append(("Legibilidad Flesch (media)", + _fmt_num(flesch.get("mean"), 1))) + return model.KVTable(rows=rows, title="Resumen del texto") + + +def _terms_table(vocab) -> "model.DataTable | None": + top = (vocab or {}).get("top_terms") or [] + rows = [[_truncate(t.get("term"), 32), _fmt_int(t.get("count")), + _fmt_pct(t.get("pct"))] + for t in top[:_TOP_TERMS] if isinstance(t, dict)] + if not rows: + return None + return model.DataTable(header=["Término", "Conteo", "% tokens"], rows=rows, + title="Términos más frecuentes", + note="stopwords ES+EN eliminadas") + + +def _ngram_table(items, n_label) -> "model.DataTable | None": + rows = [[_truncate(it.get("ngram"), 40), _fmt_int(it.get("count"))] + for it in (items or [])[:_TOP_NGRAMS] if isinstance(it, dict)] + if not rows: + return None + return model.DataTable(header=[n_label, "Conteo"], rows=rows, + title=f"{n_label} más frecuentes") + + +def _dup_note(dup, lang, read) -> "model.Note | None": + bits = [] + if isinstance(dup, dict): + nd = dup.get("near_dup") or {} + if nd.get("available"): + bits.append( + f"casi-duplicados detectados (MinHash, umbral " + f"{_fmt_num(nd.get('threshold'))}): " + f"{_fmt_int(nd.get('n_near_dup_docs'))} documentos") + else: + bits.append("near-duplicados no calculados (datasketch no instalado; " + "se reportan solo los duplicados exactos por hash)") + if isinstance(lang, dict) and not lang.get("available"): + bits.append("detección de idioma omitida (langdetect no instalado)") + if isinstance(read, dict) and not read.get("available"): + bits.append("legibilidad omitida (textstat no instalado)") + if not bits: + return None + return model.Note(" · ".join(bits)) + + +def _column_group(name, texts, length_stats, idx, mark_terms): + vocab = _vocab_stats(texts) + lang = _language(texts) + dup = _duplicates(texts) + read = _readability(texts) + n_docs = (length_stats or {}).get("n_docs") + + blocks = [ + model.Heading(text=str(name), level=2), + _summary_kv(n_docs, length_stats, vocab, lang, dup, read), + model.Figure(make=_hist_figure(name, length_stats), + caption=f"Distribución de la longitud (palabras) de " + f"«{_truncate(name, 30)}»."), + ] + + terms_tbl = _terms_table(vocab) + if terms_tbl is not None: + blocks.append(terms_tbl) + blocks.append(model.Figure( + make=_barh_figure(f"Top términos de «{_truncate(name, 24)}»", + vocab.get("top_terms"), "term", "count", + "conteo"), + caption="Términos más frecuentes (barras).")) + + bi_tbl = _ngram_table(_ngrams(texts, 2), "Bigrama") + if bi_tbl is not None: + blocks.append(bi_tbl) + tri_tbl = _ngram_table(_ngrams(texts, 3), "Trigrama") + if tri_tbl is not None: + blocks.append(tri_tbl) + + if isinstance(lang, dict) and lang.get("available") \ + and lang.get("distribution"): + blocks.append(model.Figure( + make=_barh_figure(f"Idiomas detectados en «{_truncate(name, 24)}»", + lang.get("distribution"), "lang", "count", + "documentos"), + caption="Distribución de idiomas detectados (langdetect).")) + + wc = _wordcloud_figure(texts) + if wc is not None: + blocks.append(model.Figure( + make=wc, caption=f"Nube de palabras de «{_truncate(name, 30)}».")) + + note = _dup_note(dup, lang, read) + if note is not None: + blocks.append(note) + + return model.Group(blocks=blocks, page_break_before=(idx > 0)) + + +def _intro_blocks(n_cols, mark_terms): + ttr = ("[[term:ttr]]TTR[[/term]]" if mark_terms else "TTR") + hapax = ("[[term:hapax]]hapax legomena[[/term]]" if mark_terms + else "hapax legomena") + text = ( + f"Este capítulo perfila las columnas de **texto libre largo** del " + f"dataset (reseñas, descripciones, comentarios): contenido lingüístico " + f"que la distribución categórica no resume bien. Para cada columna se " + f"muestran la longitud de los documentos, la riqueza de vocabulario " + f"(incluido el {ttr} y el porcentaje de {hapax}), los términos y " + f"n-gramas más frecuentes, los idiomas detectados y el nivel de " + f"duplicación. Las métricas son baratas y sin modelos pesados; las " + f"piezas que dependen de una librería opcional se omiten si no está " + f"instalada.") + return [ + model.Heading(text=CHAPTER_TITLE, level=1), + model.Markdown(text=text), + ] + + +def build_text_distr(profile: dict, ctx: dict): + """Build the free-text Chapter, or None if no long-text column applies.""" + profile = profile or {} + ctx = ctx or {} + + # 1) Cheap gate from the profile (no DB access yet). + candidates = _candidate_columns(profile) + if not candidates: + return None + + # 2) Raw sample + 3) confirm genuine long text (median words >= threshold). + samples = _get_samples(profile, ctx, candidates) + if not samples: + return None + survivors = _confirm_long_text(samples) + if not survivors: + return None + + # Register glossary terms (clickable) once we know the chapter applies. + glossary = ctx.get("glossary") + mark_terms = False + if isinstance(glossary, model.GlossaryCollector): + for key, (label, definition) in _TERMS.items(): + glossary.add(key, label, definition) + mark_terms = True + + blocks = list(_intro_blocks(len(survivors), mark_terms)) + + rendered = list(survivors.items())[:_MAX_TEXT_COLS] + for idx, (name, length_stats) in enumerate(rendered): + texts = samples.get(name) or [] + blocks.append(_column_group(name, texts, length_stats, idx, mark_terms)) + + if len(survivors) > len(rendered): + omitted = len(survivors) - len(rendered) + blocks.append(model.Note( + f"Se muestran las primeras {len(rendered)} columnas de texto; " + f"quedan {omitted} sin mostrar para mantener acotado el informe.")) + + return model.Chapter(id=CHAPTER_ID, title=CHAPTER_TITLE, + version=CHAPTER_VERSION, blocks=blocks) diff --git a/python/functions/datascience/automatic_eda/chapters/text_distr_test.py b/python/functions/datascience/automatic_eda/chapters/text_distr_test.py new file mode 100644 index 00000000..7c73a783 --- /dev/null +++ b/python/functions/datascience/automatic_eda/chapters/text_distr_test.py @@ -0,0 +1,256 @@ +"""Tests for the TEXT DISTR chapter — DoD: golden + edges + degradation. + +Self-contained: builds synthetic TableProfiles and feeds the raw text sample +in-memory through ``ctx['text_raw']`` (no DuckDB needed), so the suite is fast +and deterministic. Verifies that ``build_text_distr``: + +- GOLDEN: with a long-text column, emits the chapter with its key blocks + (length summary, word histogram, top-terms table, n-gram tables, language + bars) and registers the clickable glossary terms; and that it renders inside + the full document to both PDF and PPTX showing that content. +- EDGE (None): a dataset whose only string column is short labels (titanic-like + ``Name``) yields ``None`` without raising — the existing report is untouched. +- EDGE (None): a column that passes the cheap char gate but whose documents are + short (median words below the threshold) is rejected at the confirmation step. +- DEGRADATION: with ``langdetect`` / ``textstat`` / ``wordcloud`` unavailable, + the chapter still builds (those pieces are omitted) and never raises. +""" + +import builtins +import os +import tempfile + +from pypdf import PdfReader +from pptx import Presentation + +from datascience.automatic_eda.model import ( + DataTable, Figure, GlossaryCollector, Group, Heading, KVTable, Markdown, + Note, +) +from datascience.automatic_eda.chapters.text_distr import ( + CHAPTER_ID, CHAPTER_VERSION, build_text_distr, +) +from datascience.automatic_eda.chapters_registry import build_document +from datascience.render_automatic_eda_pdf import render_automatic_eda_pdf +from datascience.render_automatic_eda_pptx import render_automatic_eda_pptx + + +# --------------------------------------------------------------------------- # +# Synthetic corpus + profiles. +# --------------------------------------------------------------------------- # +_ES = [ + "El producto llegó en perfecto estado y mucho antes de lo previsto por la tienda", + "La calidad de los materiales es realmente excelente y se nota la diferencia al usarlo", + "No me convenció del todo porque esperaba bastante más por el precio que pagué finalmente", + "El servicio de atención al cliente fue rápido amable y resolvió mi problema sin demora", + "Lo recomiendo totalmente ya que ha superado con creces todas mis expectativas iniciales", +] +_EN = [ + "The product arrived in perfect condition and much earlier than the store had promised me", + "The build quality is genuinely outstanding and you can really feel the difference using it", + "I was not fully convinced because I expected quite a lot more for the price i finally paid", + "Customer support was fast friendly and solved my whole problem without any delay at all", + "I highly recommend it since it has exceeded by far every one of my initial expectations", +] + + +def _long_reviews(n=40) -> list: + """A corpus of long multi-sentence reviews (>= 20 words each), mixing two + languages and including a few exact duplicates.""" + out = [] + for i in range(n): + base = _ES if i % 3 != 0 else _EN # mostly ES, some EN + a = base[i % len(base)] + b = base[(i + 2) % len(base)] + out.append(f"{a}. {b}.") + # Inject a couple of exact duplicates. + out.append(out[0]) + out.append(out[1]) + return out + + +def _text_profile() -> dict: + """Profile with a long free-text column (review) + a numeric + a short cat.""" + return { + "table": "reviews", + "source": "/data/reviews.duckdb", + "profiled_at": "2026-06-30T10:00:00+00:00", + "n_rows": 42, + "n_cols": 3, + "quality_score": 88.0, + "columns": [ + { + "name": "review", + "inferred_type": "categorical", + "categorical": { + "top": [{"value": "x", "count": 2, "pct": 0.05}], + "n_distinct": 40, + "len_mean": 180.0, + "len_min": 80, + "len_max": 220, + }, + }, + { + "name": "rating", + "inferred_type": "numeric", + "numeric": {"mean": 3.1, "median": 3.0, "std": 1.2, + "min": 1, "max": 5}, + }, + { + "name": "product", + "inferred_type": "categorical", + "categorical": { + "top": [{"value": "teclado", "count": 10, "pct": 0.25}], + "n_distinct": 6, + "len_mean": 7.0, + "len_min": 5, "len_max": 11, + }, + }, + ], + } + + +def _no_text_profile() -> dict: + """titanic-like: the only string column is short labels (Name ≈ 27 chars).""" + return { + "table": "titanic", + "n_rows": 891, + "n_cols": 3, + "columns": [ + {"name": "Age", "inferred_type": "numeric", + "numeric": {"mean": 29.7, "median": 28.0, "std": 14.5}}, + {"name": "Name", "inferred_type": "categorical", + "categorical": {"top": [{"value": "Braund, Mr. Owen Harris", + "count": 1, "pct": 0.001}], + "n_distinct": 891, "len_mean": 27.0, + "len_min": 12, "len_max": 82}}, + {"name": "Sex", "inferred_type": "categorical", + "categorical": {"top": [{"value": "male", "count": 577, + "pct": 0.65}], + "n_distinct": 2, "len_mean": 4.6, + "len_min": 4, "len_max": 6}}, + ], + } + + +def _flatten(blocks) -> list: + """Recursively flatten Group blocks so tests can inspect leaf blocks.""" + out = [] + for b in blocks: + if isinstance(b, Group): + out.extend(_flatten(b.blocks)) + else: + out.append(b) + return out + + +# --------------------------------------------------------------------------- # +# Golden. +# --------------------------------------------------------------------------- # +def test_golden_activa_con_texto(): + glossary = GlossaryCollector() + ctx = {"text_raw": {"review": _long_reviews()}, "glossary": glossary} + ch = build_text_distr(_text_profile(), ctx) + + assert ch is not None, "el capítulo debe activarse con una columna de texto largo" + assert ch.id == CHAPTER_ID + assert ch.version == CHAPTER_VERSION + leaves = _flatten(ch.blocks) + kinds = [b.kind for b in leaves] + assert "heading" in kinds + assert "kv_table" in kinds # summary + assert "figure" in kinds # histogram / bars + assert "data_table" in kinds # top terms + n-grams + + # KV summary mentions vocabulary metrics. + kv = next(b for b in leaves if isinstance(b, KVTable)) + labels = " ".join(str(r[0]) for r in kv.rows) + assert "TTR" in labels + assert "Hapax" in labels or "hapax" in labels + + # There is a terms table and at least one n-gram table. + titles = [getattr(b, "title", "") or "" for b in leaves + if isinstance(b, DataTable)] + assert any("Términos" in t for t in titles) + assert any("Bigrama" in t for t in titles) + + # Glossary terms were registered (clickable destinations). + assert glossary.has("ttr") + assert glossary.has("hapax") + + +def test_golden_render_pdf_pptx(): + profile = _text_profile() + ctx = {"text_raw": {"review": _long_reviews()}, + "dataset_name": "reviews"} + chapters = build_document(profile, ctx) + ids = [c.id for c in chapters] + assert "text_distr" in ids, f"text_distr ausente en {ids}" + + with tempfile.TemporaryDirectory() as d: + pdf = os.path.join(d, "t.pdf") + pptx = os.path.join(d, "t.pptx") + rp = render_automatic_eda_pdf(profile, pdf, {"title": "EDA", "ctx": ctx}) + rx = render_automatic_eda_pptx(profile, pptx, {"title": "EDA", "ctx": ctx}) + assert rp.get("path") and os.path.exists(pdf) + assert rx.get("path") and os.path.exists(pptx) + + text = "\n".join(p.extract_text() or "" for p in PdfReader(pdf).pages) + assert "Texto libre" in text or "TTR" in text + + prs = Presentation(pptx) + ptext = [] + for slide in prs.slides: + for shp in slide.shapes: + if shp.has_text_frame: + ptext.append(shp.text_frame.text) + joined = "\n".join(ptext) + assert "Texto libre" in joined or "TTR" in joined + + +# --------------------------------------------------------------------------- # +# Edges — None. +# --------------------------------------------------------------------------- # +def test_edge_none_sin_texto_largo(): + # titanic-like: short labels only → chapter must not apply. + assert build_text_distr(_no_text_profile(), {}) is None + + +def test_edge_none_palabras_cortas(): + # Char gate passes (len_mean high) but documents are short → confirmation + # rejects them (median words below threshold). + profile = _text_profile() + short = ["palabra " * 3] * 30 # 3 words each, < _MIN_WORDS + ctx = {"text_raw": {"review": short}} + assert build_text_distr(profile, ctx) is None + + +def test_edge_none_empty_profile(): + assert build_text_distr({}, {}) is None + assert build_text_distr(None, None) is None + + +# --------------------------------------------------------------------------- # +# Degradation — optional libs absent. +# --------------------------------------------------------------------------- # +def test_degradacion_sin_libs(monkeypatch): + real_import = builtins.__import__ + blocked = ("langdetect", "textstat", "wordcloud", "datasketch") + + def fake_import(name, *a, **k): + if name in blocked or any(name.startswith(b + ".") for b in blocked): + raise ImportError(f"simulado: {name}") + return real_import(name, *a, **k) + + monkeypatch.setattr(builtins, "__import__", fake_import) + + ctx = {"text_raw": {"review": _long_reviews()}} + ch = build_text_distr(_text_profile(), ctx) + # Still builds (the cheap, stdlib-only pieces remain) and never raises. + assert ch is not None + leaves = _flatten(ch.blocks) + assert any(isinstance(b, KVTable) for b in leaves) + assert any(isinstance(b, DataTable) for b in leaves) + # A degradation note is present mentioning the missing optional libs. + notes = " ".join(b.text for b in leaves if isinstance(b, Note)) + assert "langdetect" in notes or "textstat" in notes or "datasketch" in notes diff --git a/python/functions/datascience/automatic_eda/chapters_registry.py b/python/functions/datascience/automatic_eda/chapters_registry.py index d9030999..70351b6d 100644 --- a/python/functions/datascience/automatic_eda/chapters_registry.py +++ b/python/functions/datascience/automatic_eda/chapters_registry.py @@ -31,6 +31,7 @@ CHAPTER_ORDER = [ "analisis_llm", # LLM interpretation — sits next to overview (user request) "num_distr", # numeric distributions "cat_distr", # categorical distributions + "text_distr", # free-text / NLP distributions (non-tabular content) "calidad", # data quality "correlacion", # correlations / associations "relaciones", # key relations: declared/candidate PK + FK (inter/intra-table) diff --git a/python/functions/datascience/compute_text_duplicates.md b/python/functions/datascience/compute_text_duplicates.md new file mode 100644 index 00000000..735eef17 --- /dev/null +++ b/python/functions/datascience/compute_text_duplicates.md @@ -0,0 +1,102 @@ +--- +id: compute_text_duplicates_py_datascience +name: compute_text_duplicates +kind: function +lang: py +domain: datascience +version: "1.0.0" +purity: pure +signature: "def compute_text_duplicates(texts, near_threshold=0.85, sample_max=2000) -> dict" +description: "Detecta documentos duplicados en un corpus de texto. Los duplicados EXACTOS se calculan siempre con la stdlib: cada documento se normaliza (colapsa espacios, strip, lower) y se hashea con SHA-1; n_exact_dup es cuántos docs repiten uno ya visto y exact_dup_pct su porcentaje. Los CASI-duplicados (near-dup) usan la dependencia OPCIONAL datasketch (MinHash + LSH sobre 3-shingles de palabras); si no está instalada, esa parte degrada a available:False sin afectar al resto. Estilo dict-no-throw del grupo eda — nunca lanza." +tags: [eda, datascience, text, nlp, duplicates, minhash, pure, python] +uses_functions: [] +uses_types: [] +returns: [] +returns_optional: false +error_type: "" +imports: [hashlib, re] +example: | + from datascience.compute_text_duplicates import compute_text_duplicates + texts = ["El gato come pescado", "El gato come pescado", "Un perro ladra"] + result = compute_text_duplicates(texts) + # {"n_docs": 3, "n_exact_dup": 1, "exact_dup_pct": 33.33, "n_unique": 2, + # "near_dup": {"available": False, "n_near_dup_docs": 0}} +tested: true +tests: + - "test_duplicados_exactos" + - "test_sin_duplicados" + - "test_vacio" + - "test_near_dup_degrada" +test_file_path: "python/functions/datascience/compute_text_duplicates_test.py" +file_path: "python/functions/datascience/compute_text_duplicates.py" +params: + - name: texts + desc: "Lista de documentos de texto. Los elementos None o que no sean str se descartan silenciosamente; n_docs cuenta solo los documentos válidos. None como argumento se trata como lista vacía." + - name: near_threshold + desc: "Umbral de similitud Jaccard (0–1) para considerar dos documentos casi-duplicados en el cálculo near-dup vía MinHashLSH. Solo aplica si datasketch está instalada. Default 0.85." + - name: sample_max + desc: "Número máximo de documentos muestreados (los primeros) para el cálculo near-dup, que es O(n) en memoria de MinHashes. No afecta al conteo de duplicados exactos, que siempre recorre todo el corpus. Default 2000." +output: "Dict con exactamente 5 claves, siempre presentes: n_docs (int, docs válidos), n_exact_dup (int, docs que repiten un texto normalizado ya visto = n_docs - n_unique), exact_dup_pct (float a 2 decimales = n_exact_dup/n_docs*100, o None si el corpus está vacío), n_unique (int, nº de textos normalizados distintos), y near_dup (sub-dict con available:bool y n_near_dup_docs:int; cuando available es True incluye además threshold con el near_threshold usado). La función nunca lanza: captura toda excepción y degrada." +--- + +## Ejemplo + +```python +from datascience.compute_text_duplicates import compute_text_duplicates + +# Tres copias del mismo texto (con espacios/casing distintos) + dos únicos. +texts = [ + "El gato come pescado", + "El gato come pescado", + "el GATO come pescado", # mismo tras normalizar + "Un perro ladra", + "La luna brilla", +] + +compute_text_duplicates(texts) +# { +# "n_docs": 5, +# "n_exact_dup": 2, # 3 copias del primer texto => 2 repeticiones +# "exact_dup_pct": 40.0, # 2 / 5 * 100 +# "n_unique": 3, # 3 textos normalizados distintos +# "near_dup": {"available": False, "n_near_dup_docs": 0}, # datasketch ausente +# } + +# Corpus vacío: contrato estable, exact_dup_pct None, sin excepción. +compute_text_duplicates([]) +# {"n_docs": 0, "n_exact_dup": 0, "exact_dup_pct": None, "n_unique": 0, +# "near_dup": {"available": False, "n_near_dup_docs": 0}} +``` + +## Cuando usarla + +Úsala en la fase de calidad de un EDA de texto, cuando quieras saber cuánto de +tu corpus es ruido duplicado antes de entrenar, vectorizar o muestrear: te da +el porcentaje de duplicados exactos (`exact_dup_pct`), el número de documentos +únicos (`n_unique`) y, si tienes `datasketch` instalada, una estimación de +casi-duplicados (paráfrasis, copias con pequeñas ediciones) vía MinHash + LSH. +Pásale directamente la columna/lista de textos crudos; la función filtra None y +no-str por ti y nunca lanza, así que es segura para encadenar en pipelines de +perfilado. + +## Gotchas + +- **Near-dup requiere `datasketch` (opcional).** Si la librería no está + instalada, `near_dup` degrada a `{"available": False, "n_near_dup_docs": 0}` + (sin clave `threshold`) y el resto del resultado se calcula igual. Los + duplicados **exactos** funcionan siempre porque solo usan la stdlib (hash). +- **Normalización de exactos.** Dos textos cuentan como el mismo duplicado + exacto si coinciden tras `" ".join(doc.split()).strip().lower()`: se colapsan + espacios/tabuladores/saltos, se recortan extremos y se ignora el caso. Cambios + de puntuación o acentos SÍ los distinguen (no se eliminan). +- **`n_exact_dup` cuenta repeticiones, no grupos.** Con 3 copias de un mismo + texto, `n_exact_dup` es 2 (las dos copias extra), no 1. Equivale a + `n_docs - n_unique`. +- **`exact_dup_pct` es `None` con corpus vacío** (no `ZeroDivisionError`); en + cualquier otro caso es un float redondeado a 2 decimales. +- **`sample_max` solo limita el near-dup.** El conteo de duplicados exactos + recorre todo el corpus; el near-dup muestrea los primeros `sample_max` + documentos para acotar memoria. Si el corpus está ordenado, considera barajar + antes para que la muestra sea representativa. +- **Elementos no-str se descartan.** `True`/`False` no cuentan como str y se + ignoran igual que `None`; `n_docs` refleja solo los documentos válidos. diff --git a/python/functions/datascience/compute_text_duplicates.py b/python/functions/datascience/compute_text_duplicates.py new file mode 100644 index 00000000..a627fea0 --- /dev/null +++ b/python/functions/datascience/compute_text_duplicates.py @@ -0,0 +1,128 @@ +"""Detección de documentos duplicados en un corpus de texto. + +Función pura, estilo dict-no-throw del grupo `eda`: nunca lanza, siempre +devuelve el mismo contrato de claves. Los duplicados EXACTOS se calculan +siempre con la stdlib (normalización + hash SHA-1). Los CASI-duplicados +(near-dup) requieren la dependencia opcional `datasketch`; si no está +instalada, esa parte degrada limpiamente a ``available: False`` sin afectar +al resto del cálculo. +""" + +import hashlib +import re + + +def _compute_near_dup(valid, near_threshold, sample_max): + """Cuenta documentos con al menos otro casi-duplicado vía MinHash + LSH. + + Import perezoso de ``datasketch``. Si la librería no está disponible (o + cualquier paso falla), degrada a ``{"available": False, "n_near_dup_docs": 0}`` + sin propagar la excepción. + + Args: + valid: lista de str ya filtrada (sin None ni no-str). + near_threshold: umbral de similitud Jaccard para LSH. + sample_max: número máximo de documentos a muestrear. + + Returns: + dict con ``available`` (bool) y ``n_near_dup_docs`` (int). Cuando + ``available`` es True, incluye además ``threshold``. + """ + try: + from datasketch import MinHash, MinHashLSH + except Exception: + return {"available": False, "n_near_dup_docs": 0} + + try: + docs = valid[:sample_max] + num_perm = 128 + lsh = MinHashLSH(threshold=near_threshold, num_perm=num_perm) + minhashes = {} + + for i, doc in enumerate(docs): + tokens = re.findall(r"\w+", doc.lower()) + shingles = set() + for j in range(len(tokens) - 2): + shingles.add(" ".join(tokens[j:j + 3])) + # Documentos con menos de 3 tokens no generan 3-shingles: caemos a + # los tokens sueltos para no perderlos del todo. + if not shingles: + shingles = set(tokens) + if not shingles: + # Documento sin tokens (cadena vacía / solo símbolos): se omite. + continue + m = MinHash(num_perm=num_perm) + for sh in shingles: + m.update(sh.encode("utf-8")) + key = "d{}".format(i) + minhashes[key] = m + lsh.insert(key, m) + + n_near = 0 + for key, m in minhashes.items(): + matches = lsh.query(m) + if len(matches) > 1: + n_near += 1 + + return { + "available": True, + "n_near_dup_docs": int(n_near), + "threshold": near_threshold, + } + except Exception: + return {"available": False, "n_near_dup_docs": 0} + + +def compute_text_duplicates(texts, near_threshold=0.85, sample_max=2000) -> dict: + """Detecta duplicados exactos y casi-duplicados en un corpus de texto. + + Args: + texts: lista de documentos. Los elementos None o que no sean str se + descartan; ``n_docs`` cuenta solo los válidos. + near_threshold: umbral de similitud Jaccard para considerar dos + documentos casi-duplicados (solo near-dup, requiere datasketch). + sample_max: tope de documentos muestreados para el cálculo near-dup. + + Returns: + dict con las claves ``n_docs``, ``n_exact_dup``, ``exact_dup_pct`` + (float redondeado a 2 decimales, o None si el corpus está vacío), + ``n_unique`` y ``near_dup`` (sub-dict con ``available`` y + ``n_near_dup_docs``, más ``threshold`` cuando está disponible). + Nunca lanza: captura toda excepción y degrada. + """ + # Filtrado defensivo de documentos válidos. + try: + valid = [t for t in texts if isinstance(t, str)] if texts is not None else [] + except Exception: + valid = [] + + n_docs = len(valid) + + # Duplicados exactos: normalizar + hash SHA-1 (stdlib, siempre disponible). + try: + seen = set() + n_exact_dup = 0 + for doc in valid: + norm = " ".join(doc.split()).strip().lower() + digest = hashlib.sha1(norm.encode("utf-8")).hexdigest() + if digest in seen: + n_exact_dup += 1 + else: + seen.add(digest) + n_unique = len(seen) + except Exception: + n_exact_dup = 0 + n_unique = 0 + + exact_dup_pct = round(n_exact_dup / n_docs * 100, 2) if n_docs > 0 else None + + # Casi-duplicados: opcional vía datasketch, degrada solo. + near_dup = _compute_near_dup(valid, near_threshold, sample_max) + + return { + "n_docs": n_docs, + "n_exact_dup": n_exact_dup, + "exact_dup_pct": exact_dup_pct, + "n_unique": n_unique, + "near_dup": near_dup, + } diff --git a/python/functions/datascience/compute_text_duplicates_test.py b/python/functions/datascience/compute_text_duplicates_test.py new file mode 100644 index 00000000..f4209713 --- /dev/null +++ b/python/functions/datascience/compute_text_duplicates_test.py @@ -0,0 +1,77 @@ +"""Tests para compute_text_duplicates. + +Importa el modulo hoja directamente (`datascience.compute_text_duplicates`) +para no depender de que el paquete reexporte la funcion en su __init__. +datasketch normalmente NO esta instalada en el venv, asi que near_dup +degrada a available=False; los tests no requieren la libreria. +""" + +from datascience.compute_text_duplicates import compute_text_duplicates + + +EXPECTED_KEYS = {"n_docs", "n_exact_dup", "exact_dup_pct", "n_unique", "near_dup"} + + +def test_duplicados_exactos(): + """3 copias del mismo texto + 2 únicos: n_exact_dup=2, pct>0.""" + texts = [ + "El gato come pescado", + "El gato come pescado", + "el GATO come pescado", # mismo tras normalizar (espacios + case) + "Un perro ladra", + "La luna brilla", + ] + result = compute_text_duplicates(texts) + + assert set(result.keys()) == EXPECTED_KEYS + assert result["n_docs"] == 5 + # 3 copias del primer texto (2 son repeticion) + 2 textos unicos. + assert result["n_exact_dup"] == 2 + assert result["n_unique"] == 3 + assert result["exact_dup_pct"] is not None + assert result["exact_dup_pct"] > 0 + # 2 / 5 * 100 = 40.0 + assert abs(result["exact_dup_pct"] - 40.0) < 1e-9 + + +def test_sin_duplicados(): + """Corpus sin repeticiones: n_exact_dup=0, n_unique==n_docs.""" + texts = [ + "primero documento distinto", + "segundo documento distinto", + "tercero documento distinto", + ] + result = compute_text_duplicates(texts) + + assert result["n_docs"] == 3 + assert result["n_exact_dup"] == 0 + assert result["n_unique"] == 3 + assert abs(result["exact_dup_pct"] - 0.0) < 1e-9 + + +def test_vacio(): + """Corpus vacio: n_docs 0, exact_dup_pct None, no lanza.""" + result = compute_text_duplicates([]) + + assert set(result.keys()) == EXPECTED_KEYS + assert result["n_docs"] == 0 + assert result["n_exact_dup"] == 0 + assert result["exact_dup_pct"] is None + assert result["n_unique"] == 0 + assert result["near_dup"]["n_near_dup_docs"] == 0 + + +def test_near_dup_degrada(): + """near_dup expone 'available' (bool) y no lanza aunque falte datasketch.""" + texts = ["uno dos tres cuatro", "uno dos tres cuatro cinco", "algo distinto"] + result = compute_text_duplicates(texts) + + near = result["near_dup"] + assert "available" in near + assert isinstance(near["available"], bool) + assert "n_near_dup_docs" in near + assert isinstance(near["n_near_dup_docs"], int) + # Tambien tolera None y entradas no-str sin lanzar. + mixed = compute_text_duplicates(["hola", None, 123, "hola"]) + assert mixed["n_docs"] == 2 + assert mixed["n_exact_dup"] == 1 diff --git a/python/functions/datascience/compute_text_length_stats.md b/python/functions/datascience/compute_text_length_stats.md new file mode 100644 index 00000000..92ab1246 --- /dev/null +++ b/python/functions/datascience/compute_text_length_stats.md @@ -0,0 +1,86 @@ +--- +id: compute_text_length_stats_py_datascience +name: compute_text_length_stats +kind: function +lang: py +domain: datascience +version: "1.0.0" +purity: pure +signature: "def compute_text_length_stats(texts, n_bins=20) -> dict" +description: "Profiles the length distribution of a corpus of text documents for EDA: per-document characters, words (unicode \\w+ tokens) and sentences (segments split on .!?… with a minimum of 1 per non-empty doc), each summarized with mean/p50/p90/p99/min/max (nearest-rank percentiles), plus an equal-width histogram of per-document word counts. None and non-str items are discarded. Dict-no-throw: never raises. Stdlib only (re)." +tags: [eda, datascience, text, nlp, length, statistics, pure, python] +uses_functions: [] +uses_types: [] +returns: [] +returns_optional: false +error_type: "" +imports: [re, math] +example: | + from datascience.compute_text_length_stats import compute_text_length_stats + result = compute_text_length_stats(["Hola mundo.", "Una frase mas larga aqui."], n_bins=5) +tested: true +tests: + - "test_basico" + - "test_vacio" + - "test_descarta_none" + - "test_un_documento" +test_file_path: "python/functions/datascience/compute_text_length_stats_test.py" +file_path: "python/functions/datascience/compute_text_length_stats.py" +params: + - name: texts + desc: "List of text documents (str). None entries and any non-str items (ints, floats, etc.) are discarded before any computation. An empty string \"\" is kept (chars 0, words 0, sentences 0)." + - name: n_bins + desc: "Number of equal-width bins for the per-document word-count histogram. Default 20. When all docs have the same word count, there are <2 docs, or n_bins < 1, a single covering bin is returned instead." +output: "Dict with keys n_docs (int), chars, words, sentences and word_hist. Each of the three axis sub-dicts has the exact keys mean (float, 2 decimals), p50, p90, p99, min, max (ints). When there are no valid documents, n_docs is 0, every axis statistic is None and word_hist is []. word_hist is a list of {lo: float, hi: float, count: int} bins; the sum of all bin counts equals n_docs." +--- + +## Ejemplo + +```python +from datascience.compute_text_length_stats import compute_text_length_stats + +compute_text_length_stats( + [ + "Hola mundo.", + "Una frase mas larga con varias palabras aqui.", + "Esto. Tiene. Tres frases distintas!", + ], + n_bins=5, +) +# { +# "n_docs": 3, +# "chars": {"mean": 30.33, "p50": 35, "p90": 45, "p99": 45, "min": 11, "max": 45}, +# "words": {"mean": 5.0, "p50": 5, "p90": 8, "p99": 8, "min": 2, "max": 8}, +# "sentences": {"mean": 1.67, "p50": 1, "p90": 3, "p99": 3, "min": 1, "max": 3}, +# "word_hist": [ +# {"lo": 2.0, "hi": 3.2, "count": 1}, +# {"lo": 3.2, "hi": 4.4, "count": 0}, +# {"lo": 4.4, "hi": 5.6, "count": 1}, +# {"lo": 5.6, "hi": 6.8, "count": 0}, +# {"lo": 6.8, "hi": 8.0, "count": 1}, +# ], +# } +``` + +## Cuando usarla + +Úsala al perfilar una columna o corpus de texto libre en un EDA: cuando +necesites saber lo largos que son los documentos (en caracteres, palabras y +frases) y cómo se reparte esa longitud antes de tokenizar, vectorizar o decidir +truncados/ventanas para un modelo. Pásale la lista de strings crudos de la +columna; `None` y valores no-texto se descartan solos. Encaja en el grupo `eda` +como bloque de longitud junto a `summarize_categorical`. + +## Gotchas + +- Función pura, solo stdlib (`re`). No usa numpy, pandas ni sklearn. +- Percentiles por método **nearest-rank** (devuelven un valor real de la lista, + no interpolan); por eso p50/p90/p99/min/max son enteros y `mean` es el único + float (redondeado a 2 decimales). +- El conteo de frases es una **aproximación** por puntuación (`.!?…`): un texto + sin esa puntuación cuenta como 1 frase si no está vacío; abreviaturas o + ellipsis pueden inflar o reducir el conteo. +- `word_hist` es equal-width entre min y max de palabras: con todos los docs + del mismo tamaño, menos de 2 docs, o `n_bins < 1`, devuelve un único bin. +- Dict-no-throw: ante input inesperado devuelve la forma vacía + (`n_docs` 0, ejes `None`, `word_hist` []) en vez de lanzar. diff --git a/python/functions/datascience/compute_text_length_stats.py b/python/functions/datascience/compute_text_length_stats.py new file mode 100644 index 00000000..3dcd84c5 --- /dev/null +++ b/python/functions/datascience/compute_text_length_stats.py @@ -0,0 +1,168 @@ +"""Pure EDA helper: document length distribution for the `eda` group. + +Given a list of text documents, computes the length distribution along three +axes (characters, words and sentences) plus an equal-width histogram of the +per-document word counts. Stdlib only (``re`` + ``statistics`` semantics via a +hand-rolled nearest-rank percentile). No numpy, no sklearn. + +The function is dict-no-throw: it never raises. On any unexpected input it +degrades to the empty-shape result. +""" + +import math +import re + +_WORD_RE = re.compile(r"\w+", re.UNICODE) +_SENT_RE = re.compile(r"[.!?…]+") + + +def _empty_axis() -> dict: + """Return an axis sub-dict with every statistic set to ``None``.""" + return {"mean": None, "p50": None, "p90": None, "p99": None, "min": None, "max": None} + + +def _pct(sorted_vals, q): + """Nearest-rank percentile of an already-sorted list. + + Args: + sorted_vals: List of numbers sorted ascending. + q: Percentile in the 0..100 range. + + Returns: + The value at the nearest rank, or ``None`` for an empty list. + """ + n = len(sorted_vals) + if n == 0: + return None + if q <= 0: + return sorted_vals[0] + rank = math.ceil(q / 100.0 * n) + if rank < 1: + rank = 1 + if rank > n: + rank = n + return sorted_vals[rank - 1] + + +def _axis_stats(values) -> dict: + """Compute mean/p50/p90/p99/min/max over a list of integer counts. + + ``mean`` is rounded to 2 decimals; every other statistic is an integer + (they are counts). Returns an all-``None`` axis for an empty list. + """ + if not values: + return _empty_axis() + sv = sorted(values) + return { + "mean": round(sum(sv) / len(sv), 2), + "p50": int(_pct(sv, 50)), + "p90": int(_pct(sv, 90)), + "p99": int(_pct(sv, 99)), + "min": int(sv[0]), + "max": int(sv[-1]), + } + + +def _word_hist(word_counts, n_bins) -> list: + """Equal-width histogram of per-document word counts. + + Builds ``n_bins`` bins between ``min`` and ``max`` of the word counts. When + every document has the same number of words, there are fewer than 2 + documents, or ``n_bins`` is not at least 1, a single covering bin is + returned. With no documents the result is ``[]``. The sum of bin ``count`` + always equals ``len(word_counts)``. + """ + if not word_counts: + return [] + wmin = min(word_counts) + wmax = max(word_counts) + if wmax == wmin or len(word_counts) < 2 or n_bins < 1: + return [{"lo": float(wmin), "hi": float(wmax), "count": len(word_counts)}] + + width = (wmax - wmin) / n_bins + bins = [] + for i in range(n_bins): + lo = wmin + i * width + hi = wmin + (i + 1) * width + bins.append({"lo": float(lo), "hi": float(hi), "count": 0}) + # Pin the last upper edge to the real maximum to avoid float drift. + bins[-1]["hi"] = float(wmax) + + for wc in word_counts: + if wc >= wmax: + idx = n_bins - 1 + else: + idx = int((wc - wmin) / width) + if idx < 0: + idx = 0 + elif idx >= n_bins: + idx = n_bins - 1 + bins[idx]["count"] += 1 + return bins + + +def compute_text_length_stats(texts, n_bins=20) -> dict: + """Summarize the length distribution of a corpus of text documents. + + For each document three lengths are measured: characters (``len(doc)``), + words (count of ``\\w+`` unicode tokens) and sentences (non-empty segments + after splitting on ``.!?…``, with a minimum of 1 for any non-empty + document). For each axis the mean, p50, p90, p99, min and max are reported, + plus an equal-width histogram of the per-document word counts. + + ``None`` entries and any non-``str`` items in ``texts`` are discarded. + The function never raises: on empty/``None`` input or any internal error it + returns the empty-shape result (``n_docs`` 0, all-``None`` axes, ``[]`` + histogram). + + Args: + texts: List of text documents (``str``). ``None`` and non-``str`` + items are dropped. + n_bins: Number of equal-width bins for the word-count histogram. + Default 20. + + Returns: + Dict with keys ``n_docs``, ``chars``, ``words``, ``sentences`` and + ``word_hist``. Each of the three axes is a sub-dict with ``mean`` + (float, 2 decimals), ``p50``, ``p90``, ``p99``, ``min`` and ``max`` + (ints), all ``None`` when there are no documents. ``word_hist`` is a + list of ``{lo, hi, count}`` bins whose ``count`` sums to ``n_docs``. + """ + empty_axis = _empty_axis() + fallback = { + "n_docs": 0, + "chars": dict(empty_axis), + "words": dict(empty_axis), + "sentences": dict(empty_axis), + "word_hist": [], + } + try: + if not texts: + return fallback + + docs = [t for t in texts if isinstance(t, str)] + n_docs = len(docs) + if n_docs == 0: + return fallback + + char_counts = [len(d) for d in docs] + word_counts = [len(_WORD_RE.findall(d)) for d in docs] + + sent_counts = [] + for d in docs: + segments = [s for s in _SENT_RE.split(d) if s.strip()] + n = len(segments) + if d and n == 0: + # Non-empty document with no detectable sentence: count as 1. + n = 1 + sent_counts.append(n) + + return { + "n_docs": n_docs, + "chars": _axis_stats(char_counts), + "words": _axis_stats(word_counts), + "sentences": _axis_stats(sent_counts), + "word_hist": _word_hist(word_counts, n_bins), + } + except Exception: + return fallback diff --git a/python/functions/datascience/compute_text_length_stats_test.py b/python/functions/datascience/compute_text_length_stats_test.py new file mode 100644 index 00000000..681f477e --- /dev/null +++ b/python/functions/datascience/compute_text_length_stats_test.py @@ -0,0 +1,70 @@ +"""Tests para compute_text_length_stats. + +Inserta `python/functions` en sys.path (relativo a este archivo) para importar +el modulo hoja por su paquete `datascience`, sin depender de que el paquete lo +reexporte en su __init__. +""" + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from datascience.compute_text_length_stats import compute_text_length_stats + + +def test_basico(): + """Varios textos de longitudes distintas: stats y histograma coherentes.""" + texts = [ + "Hola mundo.", # 2 words, 1 sentence + "Una frase mas larga con varias palabras aqui.", # 8 words, 1 sentence + "Corto.", # 1 word, 1 sentence + "Esto. Tiene. Tres frases distintas!", # 5 words, 3 sentences + ] + result = compute_text_length_stats(texts) + + assert result["n_docs"] == 4 + # Diferentes longitudes en palabras -> max estrictamente mayor que min. + assert result["words"]["max"] > result["words"]["min"] + # El histograma de palabras no esta vacio. + assert result["word_hist"] != [] + # La suma de counts del histograma cubre todos los documentos. + assert sum(b["count"] for b in result["word_hist"]) == result["n_docs"] + # mean es float redondeado; min/max son enteros. + assert isinstance(result["words"]["mean"], float) + assert isinstance(result["words"]["min"], int) + assert isinstance(result["words"]["max"], int) + # El documento con 3 frases empuja el max de sentences a >= 3. + assert result["sentences"]["max"] >= 3 + + +def test_vacio(): + """Lista vacia: n_docs 0, subdicts None, word_hist [].""" + result = compute_text_length_stats([]) + assert result["n_docs"] == 0 + for axis in ("chars", "words", "sentences"): + for key in ("mean", "p50", "p90", "p99", "min", "max"): + assert result[axis][key] is None + assert result["word_hist"] == [] + + +def test_descarta_none(): + """None y valores no-str se descartan del computo.""" + result = compute_text_length_stats(["hello world", None, 123, 4.5, "foo bar baz"]) + # Solo dos strings validos. + assert result["n_docs"] == 2 + assert result["words"]["min"] == 2 # "hello world" + assert result["words"]["max"] == 3 # "foo bar baz" + assert sum(b["count"] for b in result["word_hist"]) == 2 + + +def test_un_documento(): + """Un solo documento: word_hist tiene exactamente un bin con count 1.""" + result = compute_text_length_stats(["solo un documento aqui"]) + assert result["n_docs"] == 1 + assert len(result["word_hist"]) == 1 + assert result["word_hist"][0]["count"] == 1 + # Con un unico documento, p50 == min == max == su numero de palabras (4). + assert result["words"]["min"] == 4 + assert result["words"]["max"] == 4 + assert result["words"]["p50"] == 4 diff --git a/python/functions/datascience/compute_text_readability.md b/python/functions/datascience/compute_text_readability.md new file mode 100644 index 00000000..b6deaf07 --- /dev/null +++ b/python/functions/datascience/compute_text_readability.md @@ -0,0 +1,88 @@ +--- +id: compute_text_readability_py_datascience +name: compute_text_readability +kind: function +lang: py +domain: datascience +version: "1.0.0" +purity: pure +signature: "def compute_text_readability(texts, sample_max=500) -> dict" +description: "Calcula la legibilidad Flesch Reading Ease de un corpus de texto usando textstat con import perezoso y degradación. Filtra None/no-str/vacíos, muestrea hasta sample_max documentos (los primeros) y agrega los scores Flesch en {mean, p50, min, max}. Si textstat no está instalada devuelve available=False sin lanzar. Estilo dict-no-throw del grupo eda — nunca lanza." +tags: [eda, datascience, text, nlp, readability, flesch, textstat, pure, python] +uses_functions: [] +uses_types: [] +returns: [] +returns_optional: false +error_type: "" +imports: [math, textstat] +example: | + from datascience.compute_text_readability import compute_text_readability + out = compute_text_readability(["The cat sat on the mat. It was warm and sunny."]) + # {"available": True, "n_scored": 1, "flesch": {"mean": 109.0, "p50": 109.0, "min": 108.96..., "max": 108.96...}} +tested: true +tests: + - "test_prosa_ingles" + - "test_vacio" + - "test_degradacion" +test_file_path: "python/functions/datascience/compute_text_readability_test.py" +file_path: "python/functions/datascience/compute_text_readability.py" +params: + - name: texts + desc: "Lista de str (documentos del corpus). Los elementos None, no-str o vacíos tras strip() se descartan silenciosamente. El orden se respeta: el muestreo toma los primeros documentos válidos." + - name: sample_max + desc: "Número máximo de documentos válidos a puntuar (los primeros). Default 500. Acota el coste en corpus grandes. Valores no convertibles a int caen a 500; negativos se tratan como 0." +output: "Dict con exactamente 3 claves siempre presentes: available (bool: True si textstat se pudo importar), n_scored (int: nº de documentos efectivamente puntuados), flesch (dict con mean, p50, min, max). mean y p50 redondeados a 1 decimal; p50 por nearest-rank sobre los scores ordenados; min/max son los scores extremos sin redondear. Todos los valores de flesch son None cuando n_scored es 0. La función nunca lanza: cualquier excepción global (incluida ImportError de textstat) degrada a available=False, n_scored=0 y flesch todo None." +--- + +## Ejemplo + +```python +from datascience.compute_text_readability import compute_text_readability + +textos = [ + "The cat sat on the mat. It was a warm and sunny day in the park.", + "Reading is a wonderful habit. Books open doors to new worlds and ideas.", + "He ran quickly to the store to buy some fresh bread and a bottle of milk.", +] + +compute_text_readability(textos) +# { +# "available": True, +# "n_scored": 3, +# "flesch": {"mean": 91.4, "p50": 95.4, "min": 70.08..., "max": 108.83...} +# } + +# Corpus vacío (textstat presente): available True pero nada que puntuar. +compute_text_readability([]) +# {"available": True, "n_scored": 0, +# "flesch": {"mean": None, "p50": None, "min": None, "max": None}} +``` + +## Cuando usarla + +Úsala en un EDA de texto cuando necesites una métrica única y comparable de +**lo fácil que es de leer** un corpus de documentos (descripciones, reviews, +artículos, tickets). Devuelve el resumen Flesch Reading Ease agregado +(`mean`/`p50`/`min`/`max`) listo para un report o un bloque del notebook, sin +tener que iterar `textstat` a mano. Pásale la lista de textos crudos y, si el +corpus es grande, limita el coste con `sample_max`. El estilo dict-no-throw +permite incrustarla en pipelines del grupo `eda` sin envolver en try/except. + +## Gotchas + +- **`textstat` es una dependencia opcional.** Si no está instalada (o falla al + importar) la función NO lanza: devuelve `available=False`, `n_scored=0` y + `flesch` todo `None`. Comprueba `available` antes de interpretar los números. +- **Flesch Reading Ease está pensado para prosa en inglés.** Aplicado a otros + idiomas o a texto no-prosa (código, listas, tablas, cadenas muy cortas) los + scores no son interpretables, aunque se calculen sin error. +- **Escala Flesch:** valores **altos** = más fácil de leer (≈90–100 muy fácil), + valores **bajos** = más difícil (puede ser negativo en texto muy denso). No + se recortan a ningún rango: se reportan tal cual los devuelve `textstat`. +- **`available=True` con `n_scored=0`** significa que `textstat` está presente + pero el corpus no aportó documentos puntuables (vacío, solo None/no-str, o + todos los docs fallaron al puntuar). Es distinto de `available=False`. +- **Muestreo = los primeros `sample_max`**, no aleatorio. Si el orden del corpus + está sesgado, el resumen reflejará ese sesgo. +- **`mean` y `p50` redondean a 1 decimal**; `min`/`max` se devuelven sin + redondear (los scores extremos reales). diff --git a/python/functions/datascience/compute_text_readability.py b/python/functions/datascience/compute_text_readability.py new file mode 100644 index 00000000..61595560 --- /dev/null +++ b/python/functions/datascience/compute_text_readability.py @@ -0,0 +1,121 @@ +"""Legibilidad Flesch Reading Ease de un corpus de texto. + +Función pura del grupo `eda`, estilo dict-no-throw: nunca lanza. Usa la +librería `textstat` con import perezoso y degradación: si `textstat` no está +instalada (o falla al importar), devuelve un resultado con `available=False` +en lugar de propagar el error. +""" + + +def _percentile_nearest_rank(sorted_values, pct): + """Percentil por nearest-rank sobre una lista ya ordenada ascendente. + + rank = ceil(pct/100 * n); índice 1-based recortado a [1, n]. + Devuelve None si la lista está vacía. + """ + n = len(sorted_values) + if n == 0: + return None + import math + + rank = math.ceil((pct / 100.0) * n) + if rank < 1: + rank = 1 + if rank > n: + rank = n + return sorted_values[rank - 1] + + +def compute_text_readability(texts, sample_max=500) -> dict: + """Calcula la legibilidad Flesch Reading Ease de un corpus. + + Args: + texts: lista de str. Los elementos None, no-str o vacíos (tras strip) + se descartan. Se muestrean los primeros `sample_max` documentos + válidos. + sample_max: número máximo de documentos a puntuar (los primeros). + + Returns: + Dict con la forma exacta:: + + {"available": bool, "n_scored": int, + "flesch": {"mean": float|None, "p50": float|None, + "min": float|None, "max": float|None}} + + `available` es True si `textstat` se pudo importar. La función nunca + lanza: cualquier excepción global degrada a `available=False`. + """ + empty = { + "available": False, + "n_scored": 0, + "flesch": {"mean": None, "p50": None, "min": None, "max": None}, + } + try: + # Import perezoso con degradación: textstat es una dependencia opcional. + try: + import textstat + except Exception: + return { + "available": False, + "n_scored": 0, + "flesch": {"mean": None, "p50": None, "min": None, "max": None}, + } + + # Filtrar y muestrear documentos válidos (los primeros sample_max). + docs = [] + if texts is not None: + try: + limit = int(sample_max) + except Exception: + limit = 500 + if limit < 0: + limit = 0 + for item in texts: + if not isinstance(item, str): + continue + if item.strip() == "": + continue + docs.append(item) + if len(docs) >= limit: + break + + scores = [] + for doc in docs: + try: + score = textstat.flesch_reading_ease(doc) + except Exception: + continue + try: + score = float(score) + except Exception: + continue + scores.append(score) + + n_scored = len(scores) + if n_scored == 0: + # textstat presente pero corpus vacío / sin puntuar. + return { + "available": True, + "n_scored": 0, + "flesch": {"mean": None, "p50": None, "min": None, "max": None}, + } + + mean_val = round(sum(scores) / n_scored, 1) + sorted_scores = sorted(scores) + p50_raw = _percentile_nearest_rank(sorted_scores, 50) + p50_val = round(p50_raw, 1) if p50_raw is not None else None + min_val = sorted_scores[0] + max_val = sorted_scores[-1] + + return { + "available": True, + "n_scored": n_scored, + "flesch": { + "mean": mean_val, + "p50": p50_val, + "min": min_val, + "max": max_val, + }, + } + except Exception: + return empty diff --git a/python/functions/datascience/compute_text_readability_test.py b/python/functions/datascience/compute_text_readability_test.py new file mode 100644 index 00000000..2cf4f3fc --- /dev/null +++ b/python/functions/datascience/compute_text_readability_test.py @@ -0,0 +1,74 @@ +"""Tests para compute_text_readability.""" + +import sys +import os +import builtins + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..")) + +from datascience.compute_text_readability import compute_text_readability + + +EXPECTED_KEYS = {"available", "n_scored", "flesch"} +FLESCH_KEYS = {"mean", "p50", "min", "max"} + + +def test_prosa_ingles(): + """Varios textos en prosa inglesa: available True, n_scored>0, mean no None.""" + texts = [ + "The cat sat on the mat. It was a warm and sunny day in the park.", + "She sells sea shells by the sea shore. The shells she sells are surely sea shells.", + "Reading is a wonderful habit. Books open doors to new worlds and ideas.", + "He ran quickly to the store to buy some fresh bread and a bottle of milk.", + ] + out = compute_text_readability(texts) + + assert set(out.keys()) == EXPECTED_KEYS + assert out["available"] is True + assert out["n_scored"] > 0 + assert set(out["flesch"].keys()) == FLESCH_KEYS + assert out["flesch"]["mean"] is not None + assert out["flesch"]["p50"] is not None + assert out["flesch"]["min"] is not None + assert out["flesch"]["max"] is not None + # min <= mean/p50 <= max coherente. + assert out["flesch"]["min"] <= out["flesch"]["max"] + + +def test_vacio(): + """Corpus vacío con textstat presente: available True, n_scored 0, flesch None.""" + out = compute_text_readability([]) + + assert set(out.keys()) == EXPECTED_KEYS + assert out["available"] is True + assert out["n_scored"] == 0 + assert out["flesch"]["mean"] is None + assert out["flesch"]["p50"] is None + assert out["flesch"]["min"] is None + assert out["flesch"]["max"] is None + + # Elementos no-str / vacíos también se descartan -> n_scored 0. + out2 = compute_text_readability([None, "", " ", 123]) + assert out2["available"] is True + assert out2["n_scored"] == 0 + + +def test_degradacion(monkeypatch): + """Sin textstat (ImportError forzado): degrada a available False sin lanzar.""" + import datascience.compute_text_readability as m + + real = builtins.__import__ + + def fake(name, *a, **k): + if name == "textstat" or name.startswith("textstat."): + raise ImportError("simulado") + return real(name, *a, **k) + + monkeypatch.setattr(builtins, "__import__", fake) + out = m.compute_text_readability(["The cat sat on the mat. It was happy and warm."]) + assert out["available"] is False + assert out["n_scored"] == 0 + assert out["flesch"]["mean"] is None + assert out["flesch"]["p50"] is None + assert out["flesch"]["min"] is None + assert out["flesch"]["max"] is None diff --git a/python/functions/datascience/compute_top_ngrams.md b/python/functions/datascience/compute_top_ngrams.md new file mode 100644 index 00000000..81c4d504 --- /dev/null +++ b/python/functions/datascience/compute_top_ngrams.md @@ -0,0 +1,103 @@ +--- +id: compute_top_ngrams_py_datascience +name: compute_top_ngrams +kind: function +lang: py +domain: datascience +version: "1.0.0" +purity: pure +signature: "def compute_top_ngrams(texts, n=2, top_k=15, remove_stopwords=True) -> dict" +description: "Calcula los n-gramas de palabras más frecuentes de un corpus de texto (n=1 unigramas, 2 bigramas, 3 trigramas...). Tokeniza a minúsculas con re.findall(r'\\w+', ...), descarta tokens numéricos y, si remove_stopwords=True, elimina stopwords ES+EN ANTES de formar los n-gramas (n-gramas contiguos sobre la secuencia de tokens de contenido, sin cruzar documentos). Pura y autocontenida con collections.Counter, sin sklearn. Estilo dict-no-throw del grupo eda: nunca lanza." +tags: [eda, datascience, text, nlp, ngrams, bigrams, trigrams, pure, python] +uses_functions: [] +uses_types: [] +returns: [] +returns_optional: false +error_type: "" +imports: [re, collections] +example: | + from datascience.compute_top_ngrams import compute_top_ngrams + texts = ["machine learning rocks", "we love machine learning"] + compute_top_ngrams(texts, n=2, top_k=5) + # {"n": 2, "top": [{"ngram": "machine learning", "count": 2}, ...]} +tested: true +tests: + - "test_bigramas" + - "test_trigramas" + - "test_vacio" + - "test_stopwords" +test_file_path: "python/functions/datascience/compute_top_ngrams_test.py" +file_path: "python/functions/datascience/compute_top_ngrams.py" +params: + - name: texts + desc: "Lista (o tupla) de cadenas. Los elementos None o que no sean str se descartan silenciosamente. Cada documento se tokeniza por separado; los n-gramas no cruzan la frontera entre documentos." + - name: n + desc: "Tamaño del n-grama: 1 unigramas, 2 bigramas, 3 trigramas, etc. Valores < 1 o no enteros producen top vacío (se conserva tal cual en la clave 'n' del retorno)." + - name: top_k + desc: "Número máximo de n-gramas a devolver, ordenados por frecuencia descendente con desempate alfabético determinista. Default 15. Valores negativos se tratan como 0." + - name: remove_stopwords + desc: "Si True (default) elimina las stopwords ES+EN de una lista inline (~130 términos de altísima frecuencia) ANTES de formar los n-gramas, de modo que los n-gramas se construyen sobre la secuencia de tokens de contenido." +output: "Dict con exactamente 2 claves: n (el n recibido, sin normalizar) y top (lista de dicts {'ngram': str, 'count': int} ordenada por count descendente, longitud <= top_k). ngram es la unión de los tokens del n-grama por un espacio. Corpus vacío, tokens insuficientes para formar n-gramas o cualquier excepción interna degradan a {'n': n, 'top': []}. La función nunca lanza." +--- + +## Ejemplo + +```python +from datascience.compute_top_ngrams import compute_top_ngrams + +texts = [ + "machine learning rocks", + "machine learning is fun", + "we love machine learning", +] + +# Bigramas (n=2): "machine learning" aparece en los 3 documentos. +compute_top_ngrams(texts, n=2, top_k=5) +# { +# "n": 2, +# "top": [ +# {"ngram": "machine learning", "count": 3}, +# {"ngram": "learning fun", "count": 1}, +# {"ngram": "learning rocks", "count": 1}, +# {"ngram": "love machine", "count": 1}, +# ], +# } + +# Unigramas con stopwords fuera (default): solo palabras de contenido. +compute_top_ngrams(["the cat sat on the mat"], n=1, top_k=3) +# {"n": 1, "top": [{"ngram": "cat", "count": 1}, +# {"ngram": "mat", "count": 1}, +# {"ngram": "sat", "count": 1}]} +``` + +## Cuando usarla + +Úsala en la fase de EDA de texto cuando, además del vocabulario suelto, necesites +ver qué **combinaciones de palabras contiguas** dominan un corpus: colocaciones, +frases técnicas recurrentes ("machine learning", "data analyst"), o patrones de +trigramas en titulares/descripciones. Es el complemento natural de un perfil de +vocabulario: pasa de "qué palabras aparecen" a "qué secuencias aparecen". Llámala +con `n=1` para unigramas, `n=2` para bigramas y `n=3` para trigramas, y ajusta +`top_k` al tamaño de la tabla que vas a renderizar. Deja `remove_stopwords=True` +para que los n-gramas reflejen contenido y no conectores gramaticales. + +## Gotchas + +- **Las stopwords se eliminan ANTES de formar los n-gramas.** Con + `remove_stopwords=True` la frase "data of analysis" produce el bigrama + "data analysis" (el "of" intermedio desaparece y los tokens de contenido se + vuelven contiguos), no "data of" ni "of analysis". Si quieres preservar la + adyacencia literal del texto original, pasa `remove_stopwords=False`. +- **Los n-gramas NO cruzan documentos.** Cada elemento de `texts` se tokeniza y + recorre por separado; el último token de un documento nunca se combina con el + primero del siguiente. +- **Tokens puramente numéricos se descartan** (`tok.isdigit()`), pero los + alfanuméricos mixtos no: "3d" o "covid19" sí cuentan como tokens. Un decimal + como "3.5" se parte en "3" y "5" por `\w+` y ambos se descartan por numéricos. +- **La lista de stopwords es inline ES+EN**, pensada para textos generales en + esos dos idiomas. Para otros idiomas o jerga específica de dominio puede dejar + pasar conectores; en ese caso filtra el corpus aguas arriba o usa + `remove_stopwords=False` y posfiltra. +- **`top` puede tener menos de `top_k` elementos** si el corpus no tiene tantos + n-gramas distintos. El desempate por frecuencia es alfabético (determinista), + no por orden de aparición. diff --git a/python/functions/datascience/compute_top_ngrams.py b/python/functions/datascience/compute_top_ngrams.py new file mode 100644 index 00000000..0afb41b0 --- /dev/null +++ b/python/functions/datascience/compute_top_ngrams.py @@ -0,0 +1,94 @@ +"""Top n-gramas de palabras más frecuentes de un corpus de texto. + +Función pura, autocontenida (solo stdlib: re + collections.Counter). No depende +de scikit-learn ni de ninguna otra librería externa. Estilo dict-no-throw del +grupo `eda`: ante cualquier entrada degenerada o excepción interna devuelve +``{"n": n, "top": []}`` en vez de lanzar. +""" + +import re +from collections import Counter + +# Lista inline de stopwords ES + EN (~80 términos de altísima frecuencia). +# Se eliminan ANTES de formar los n-gramas: los n-gramas se construyen sobre la +# secuencia de tokens de contenido, no sobre el texto original. +_STOPWORDS = frozenset({ + # Español + "de", "la", "que", "el", "en", "y", "a", "los", "del", "se", "las", "por", + "un", "para", "con", "no", "una", "su", "al", "lo", "como", "más", "mas", + "pero", "sus", "le", "ya", "o", "este", "sí", "si", "porque", "esta", + "entre", "cuando", "muy", "sin", "sobre", "también", "tambien", "me", + "hasta", "hay", "donde", "quien", "desde", "todo", "nos", "durante", + "todos", "uno", "les", "ni", "contra", "otros", "ese", "eso", "ante", + "ellos", "e", "esto", "mí", "antes", "algunos", "qué", "unos", "yo", + "otro", "otras", "otra", "él", "tanto", "esa", "estos", "mucho", "quienes", + "nada", "muchos", "cual", "poco", "ella", "estar", "estas", "algunas", + "algo", "nosotros", + # Inglés + "the", "of", "and", "to", "in", "is", "it", "for", "on", "with", "as", + "are", "was", "be", "this", "that", "by", "an", "or", "at", "from", "but", + "not", "have", "has", "had", "they", "you", "we", "he", "she", "his", + "her", "their", "its", "i", "my", "me", "our", "us", "do", "does", "did", + "will", "would", "can", "could", "should", "there", "which", "who", "what", + "when", "where", "how", "all", "if", "so", "than", "then", "out", "up", +}) + + +def compute_top_ngrams(texts, n=2, top_k=15, remove_stopwords=True) -> dict: + """Calcula los n-gramas de palabras más frecuentes de un corpus. + + Args: + texts: lista de cadenas. Los elementos ``None`` o que no sean ``str`` se + descartan silenciosamente. + n: tamaño del n-grama (1 = unigramas, 2 = bigramas, 3 = trigramas...). + Valores < 1 o no enteros producen ``top`` vacío. + top_k: número máximo de n-gramas a devolver, ordenados por frecuencia + descendente (con desempate alfabético determinista). + remove_stopwords: si ``True`` elimina las stopwords ES+EN ANTES de + formar los n-gramas, de modo que los n-gramas se construyen sobre la + secuencia de tokens de contenido (no cruzando documentos). + + Returns: + ``{"n": n, "top": [{"ngram": "w1 w2", "count": int}, ...]}``. Corpus + vacío, sin tokens suficientes o cualquier excepción interna degrada a + ``{"n": n, "top": []}``. Nunca lanza. + """ + try: + if not isinstance(n, int) or n < 1: + return {"n": n, "top": []} + + try: + limit = int(top_k) + except (TypeError, ValueError): + limit = 0 + if limit < 0: + limit = 0 + + if not isinstance(texts, (list, tuple)): + return {"n": n, "top": []} + + counter = Counter() + for doc in texts: + if not isinstance(doc, str): + continue + tokens = [ + tok + for tok in re.findall(r"\w+", doc.lower(), re.UNICODE) + if not tok.isdigit() + ] + if remove_stopwords: + tokens = [tok for tok in tokens if tok not in _STOPWORDS] + if len(tokens) < n: + continue + for i in range(len(tokens) - n + 1): + ngram = " ".join(tokens[i:i + n]) + counter[ngram] += 1 + + if not counter: + return {"n": n, "top": []} + + ordered = sorted(counter.items(), key=lambda kv: (-kv[1], kv[0])) + top = [{"ngram": ngram, "count": count} for ngram, count in ordered[:limit]] + return {"n": n, "top": top} + except Exception: + return {"n": n, "top": []} diff --git a/python/functions/datascience/compute_top_ngrams_test.py b/python/functions/datascience/compute_top_ngrams_test.py new file mode 100644 index 00000000..9b5eb869 --- /dev/null +++ b/python/functions/datascience/compute_top_ngrams_test.py @@ -0,0 +1,65 @@ +"""Tests para compute_top_ngrams.""" + +import sys +import os + +# sys.path estándar: añade `python/functions/` para importar por paquete raíz. +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..")) + +from datascience.compute_top_ngrams import compute_top_ngrams + + +def test_bigramas(): + # "machine learning" se repite en cada documento -> bigrama más frecuente. + texts = [ + "machine learning rocks", + "machine learning is fun", + "we love machine learning", + ] + result = compute_top_ngrams(texts, n=2, top_k=5) + assert result["n"] == 2 + assert result["top"], "esperaba al menos un bigrama" + assert result["top"][0]["ngram"] == "machine learning" + assert result["top"][0]["count"] == 3 + # Cada entrada respeta el contrato {"ngram": str, "count": int}. + for item in result["top"]: + assert isinstance(item["ngram"], str) + assert isinstance(item["count"], int) + + +def test_trigramas(): + texts = [ + "alpha beta gamma delta", + "alpha beta gamma omega", + ] + # Con stopwords desactivadas para no descartar tokens de contenido. + result = compute_top_ngrams(texts, n=3, top_k=5, remove_stopwords=False) + assert result["n"] == 3 + ngrams = {item["ngram"]: item["count"] for item in result["top"]} + # "alpha beta gamma" aparece en ambos documentos. + assert ngrams.get("alpha beta gamma") == 2 + # Trigramas únicos de cada documento. + assert ngrams.get("beta gamma delta") == 1 + assert ngrams.get("beta gamma omega") == 1 + + +def test_vacio(): + assert compute_top_ngrams([], n=2) == {"n": 2, "top": []} + # Documentos no-str / None se descartan -> corpus efectivamente vacío. + assert compute_top_ngrams([None, 123, {"a": 1}], n=2) == {"n": 2, "top": []} + + +def test_stopwords(): + # "the cat" debería desaparecer al quitar stopwords ("the" es stopword EN). + texts = ["the cat the cat the cat"] + con = compute_top_ngrams(texts, n=2, top_k=10, remove_stopwords=True) + sin = compute_top_ngrams(texts, n=2, top_k=10, remove_stopwords=False) + + con_ngrams = {item["ngram"] for item in con["top"]} + sin_ngrams = {item["ngram"] for item in sin["top"]} + + # Sin filtrar, el bigrama dominante es "the cat". + assert "the cat" in sin_ngrams + # Al filtrar stopwords, ya no aparece "the cat" (queda solo "cat cat"). + assert "the cat" not in con_ngrams + assert con_ngrams != sin_ngrams diff --git a/python/functions/datascience/compute_vocabulary_stats.md b/python/functions/datascience/compute_vocabulary_stats.md new file mode 100644 index 00000000..fb2ada89 --- /dev/null +++ b/python/functions/datascience/compute_vocabulary_stats.md @@ -0,0 +1,91 @@ +--- +id: compute_vocabulary_stats_py_datascience +name: compute_vocabulary_stats +kind: function +lang: py +domain: datascience +version: "1.0.0" +purity: pure +signature: "def compute_vocabulary_stats(texts: list, top_k: int = 20, remove_stopwords: bool = True) -> dict" +description: "Profiles the vocabulary of a text corpus for EDA: tokenises a list of documents, counts term frequencies and derives lexical-richness measures — total tokens, unique types, type-token ratio (TTR), hapax legomena and the top-k most frequent terms. Pure, stdlib only (re + collections.Counter); no nltk, no sklearn. Inline ES+EN stopword list, opt-out via remove_stopwords. Never raises: empty/degenerate input returns the zeroed result." +tags: [eda, datascience, text, nlp, vocabulary, ttr, hapax, pure, python] +uses_functions: [] +uses_types: [] +returns: [] +returns_optional: false +error_type: "" +imports: [re, collections] +example: | + from datascience.compute_vocabulary_stats import compute_vocabulary_stats + result = compute_vocabulary_stats(["el gato y el perro", "gato veloz"], top_k=5) +tested: true +tests: + - "test_basico" + - "test_vacio" + - "test_stopwords_quitadas" + - "test_stopwords_conservadas" +test_file_path: "python/functions/datascience/compute_vocabulary_stats_test.py" +file_path: "python/functions/datascience/compute_vocabulary_stats.py" +params: + - name: texts + desc: "List of documents (strings) forming the corpus. Entries that are None or not a str are silently discarded. Tokens are extracted per document with re.findall(r'\\w+', doc.lower(), re.UNICODE); purely numeric tokens (tok.isdigit()) are dropped." + - name: top_k + desc: "Maximum number of most-frequent terms to return in top_terms. Default 20. Does not affect n_tokens/n_types/ttr/hapax — only the length of the top_terms list." + - name: remove_stopwords + desc: "When True (default) common Spanish+English stopwords from the inline _STOPWORDS set (~120 entries) are removed from the token stream before any counting. Set False to keep every word (raw lexical profile)." +output: "Dict with the exact keys n_tokens (int), n_types (int), ttr (float|None, n_types/n_tokens rounded to 4 dp), n_hapax (int, terms occurring exactly once), hapax_pct (float|None, n_hapax/n_types*100 rounded to 2 dp) and top_terms (list of {term, count, pct} sorted by count descending, pct = count/n_tokens*100 rounded to 2 dp). For an empty corpus (no tokens after filtering): n_tokens=0, n_types=0, ttr=None, n_hapax=0, hapax_pct=None, top_terms=[]. Any exception degrades to that same empty result — the function never throws." +--- + +## Ejemplo + +```python +from datascience.compute_vocabulary_stats import compute_vocabulary_stats + +compute_vocabulary_stats( + ["el gato y el perro", "gato veloz corre", "perro perro perro"], + top_k=5, +) +# { +# "n_tokens": 6, # stopwords (el, y) eliminadas por defecto +# "n_types": 3, # gato, perro, veloz, corre -> tras quitar stopwords +# "ttr": 0.5, # n_types / n_tokens +# "n_hapax": 2, # veloz, corre (1 aparicion cada uno) +# "hapax_pct": 50.0, # n_hapax / n_types * 100 +# "top_terms": [ +# {"term": "perro", "count": 4, "pct": 44.44}, +# {"term": "gato", "count": 2, "pct": 22.22}, +# ... +# ], +# } + +# Perfil lexico crudo (sin filtrar stopwords): +compute_vocabulary_stats(["the cat and the dog"], remove_stopwords=False) +``` + +## Cuando usarla + +Úsala al perfilar una columna o corpus de texto libre en un EDA del grupo `eda`: +cuando necesites medir la riqueza léxica (cuántos tokens y cuántas palabras +distintas, type-token ratio, porcentaje de palabras que solo aparecen una vez) y +ver qué términos dominan el vocabulario (top-k frecuencias). Pásale la lista de +documentos crudos (filas de la columna); `None` y valores no-string se ignoran +solos. Es el equivalente para texto largo de `summarize_categorical`, que perfila +categorías cortas. + +## Gotchas + +- Función pura y stdlib-only, pero el resultado depende del **idioma**: la lista + `_STOPWORDS` cubre español e inglés. Para otros idiomas pon + `remove_stopwords=False` o filtra fuera, o el perfil mezclará stopwords no + reconocidas en `top_terms`. +- La tokenización es `\w+` con `re.UNICODE`: separa por puntuación y conserva + acentos/ñ, pero NO hace stemming ni lematización — "gato" y "gatos" cuentan + como tipos distintos. Tampoco hace stripping de acentos, así que "más" (con + tilde) y "mas" son tokens diferentes (ambos están en la stoplist). +- Los tokens **puramente numéricos** (`"123"`) se descartan siempre; un token + alfanumérico mixto (`"covid19"`) se conserva. +- `ttr` baja artificialmente en corpus grandes (más texto, más repetición): no + compares TTR entre corpus de tamaños muy distintos sin normalizar. +- Nunca lanza: entrada vacía, `None`, o cualquier excepción interna devuelven el + resultado con ceros/`None`/`[]`. Comprueba `n_tokens == 0` para detectar el + caso degenerado. diff --git a/python/functions/datascience/compute_vocabulary_stats.py b/python/functions/datascience/compute_vocabulary_stats.py new file mode 100644 index 00000000..a3f65115 --- /dev/null +++ b/python/functions/datascience/compute_vocabulary_stats.py @@ -0,0 +1,99 @@ +"""Profile the vocabulary of a text corpus for EDA (pure, stdlib only). + +Tokenises a list of documents, counts term frequencies and derives lexical +richness measures (type-token ratio, hapax legomena) plus the top-k terms. +No external NLP dependencies (no nltk, no sklearn) — only ``re`` and +``collections`` from the standard library. +""" + +import re +from collections import Counter + +# Common Spanish + English stopwords. Inline, lowercase, no accents stripped +# beyond what already appears here. Filtering is opt-in via remove_stopwords. +_STOPWORDS = { + # Spanish + "de", "la", "que", "el", "en", "y", "a", "los", "del", "se", "las", "por", + "un", "para", "con", "no", "una", "su", "al", "es", "lo", "como", "mas", + "más", "pero", "sus", "le", "ya", "o", "este", "si", "sí", "porque", + "esta", "entre", "cuando", "muy", "sin", "sobre", "tambien", "también", + "me", "hasta", "hay", "donde", "quien", "desde", "todo", "nos", "durante", + "todos", "uno", "les", "ni", "contra", "otros", "ese", "eso", "ante", + "ellos", "e", "esto", "antes", "algunos", "que", "unos", "yo", "otro", + "otras", "otra", "el", "tanto", "esa", "estos", "mucho", "nada", "muchos", + # English + "the", "of", "and", "to", "in", "is", "it", "for", "on", "with", "as", + "was", "but", "are", "this", "that", "an", "be", "by", "or", "not", "at", + "from", "my", "i", "you", "he", "she", "we", "they", "his", "her", "its", + "our", "their", "what", "which", "who", "whom", "has", "have", "had", "do", + "does", "did", "will", "would", "can", "could", "should", "may", "might", + "must", "if", "then", "than", "so", "too", "very", "just", "also", "were", + "been", "being", "there", "here", "all", "any", "some", "more", "most", + "out", "up", "down", "into", "over", "such", "only", "own", "same", +} + + +def compute_vocabulary_stats(texts, top_k=20, remove_stopwords=True) -> dict: + """Profile the vocabulary of a corpus of documents. + + Args: + texts: List of strings (the corpus). Entries that are None or not a + string are discarded silently. + top_k: Maximum number of most-frequent terms to include in + ``top_terms``. Default 20. Does not affect the other measures. + remove_stopwords: When True (default) common ES+EN stopwords are + dropped from the token stream before any counting. + + Returns: + A dict with the exact keys ``n_tokens``, ``n_types``, ``ttr``, + ``n_hapax``, ``hapax_pct`` and ``top_terms``. For an empty corpus (no + tokens after filtering): n_tokens=0, n_types=0, ttr=None, n_hapax=0, + hapax_pct=None, top_terms=[]. Never raises — any exception degrades to + the empty-corpus result. + """ + empty = { + "n_tokens": 0, + "n_types": 0, + "ttr": None, + "n_hapax": 0, + "hapax_pct": None, + "top_terms": [], + } + try: + tokens = [] + for doc in texts or []: + if not isinstance(doc, str): + continue + for tok in re.findall(r"\w+", doc.lower(), re.UNICODE): + if tok.isdigit(): + continue + if remove_stopwords and tok in _STOPWORDS: + continue + tokens.append(tok) + + n_tokens = len(tokens) + if n_tokens == 0: + return dict(empty) + + counts = Counter(tokens) + n_types = len(counts) + ttr = round(n_types / n_tokens, 4) + + n_hapax = sum(1 for c in counts.values() if c == 1) + hapax_pct = round(n_hapax / n_types * 100, 2) + + top_terms = [ + {"term": term, "count": count, "pct": round(count / n_tokens * 100, 2)} + for term, count in counts.most_common(top_k) + ] + + return { + "n_tokens": n_tokens, + "n_types": n_types, + "ttr": ttr, + "n_hapax": n_hapax, + "hapax_pct": hapax_pct, + "top_terms": top_terms, + } + except Exception: + return dict(empty) diff --git a/python/functions/datascience/compute_vocabulary_stats_test.py b/python/functions/datascience/compute_vocabulary_stats_test.py new file mode 100644 index 00000000..aacf5d35 --- /dev/null +++ b/python/functions/datascience/compute_vocabulary_stats_test.py @@ -0,0 +1,74 @@ +"""Tests para compute_vocabulary_stats.""" + +import os +import sys + +sys.path.insert( + 0, os.path.join(os.path.dirname(__file__), "..", "..", "functions") +) + +from datascience.compute_vocabulary_stats import compute_vocabulary_stats + + +def test_basico(): + # Corpus con repeticiones y hapax. Stopwords desactivadas para controlar + # exactamente que tokens entran. + texts = ["gato gato perro", "perro perro raton", "elefante"] + r = compute_vocabulary_stats(texts, top_k=10, remove_stopwords=False) + + # n_types < n_tokens cuando hay repeticiones. + assert r["n_types"] < r["n_tokens"] + assert r["n_tokens"] == 7 + assert r["n_types"] == 4 # gato, perro, raton, elefante + + # ttr en (0, 1]. + assert 0 < r["ttr"] <= 1 + assert r["ttr"] == round(4 / 7, 4) + + # top_terms ordenado por count descendente. + counts = [t["count"] for t in r["top_terms"]] + assert counts == sorted(counts, reverse=True) + assert r["top_terms"][0]["term"] == "perro" + assert r["top_terms"][0]["count"] == 3 + + # hapax: raton y elefante aparecen exactamente una vez. + assert r["n_hapax"] == 2 + assert r["hapax_pct"] == round(2 / 4 * 100, 2) + + # pct coherente con count/n_tokens. + assert r["top_terms"][0]["pct"] == round(3 / 7 * 100, 2) + + +def test_vacio(): + # Sin documentos validos -> ceros / None / []. + for arg in ([], None, [None, 123, ""], ["123 456"]): + r = compute_vocabulary_stats(arg) + assert r["n_tokens"] == 0 + assert r["n_types"] == 0 + assert r["ttr"] is None + assert r["n_hapax"] == 0 + assert r["hapax_pct"] is None + assert r["top_terms"] == [] + + +def test_stopwords_quitadas(): + texts = ["the gato the perro", "de la casa azul"] + r = compute_vocabulary_stats(texts, remove_stopwords=True) + terms = {t["term"] for t in r["top_terms"]} + # Stopwords ES+EN no deben aparecer. + assert "the" not in terms + assert "de" not in terms + assert "la" not in terms + # Palabras de contenido si. + assert "gato" in terms + assert "casa" in terms + + +def test_stopwords_conservadas(): + texts = ["the gato the perro", "de la casa azul"] + r = compute_vocabulary_stats(texts, remove_stopwords=False) + terms = {t["term"] for t in r["top_terms"]} + # Con el filtro desactivado, las stopwords se conservan. + assert "the" in terms + assert "de" in terms + assert "la" in terms diff --git a/python/functions/datascience/detect_corpus_language.md b/python/functions/datascience/detect_corpus_language.md new file mode 100644 index 00000000..0daba9ee --- /dev/null +++ b/python/functions/datascience/detect_corpus_language.md @@ -0,0 +1,80 @@ +--- +name: detect_corpus_language +kind: function +lang: py +domain: datascience +version: "1.0.0" +purity: pure +signature: "def detect_corpus_language(texts, top_k=10, sample_max=1000) -> dict" +description: "Estima la distribucion de idiomas de un corpus de textos con la libreria langdetect (import perezoso). Funcion pura y defensiva del grupo eda: filtra documentos None/no-str/vacios, muestrea hasta sample_max docs, clasifica cada uno con detect() ignorando los que langdetect no puede resolver (LangDetectException), y devuelve la distribucion top_k por frecuencia mas el idioma dominante. Si langdetect no esta instalada o algo falla, degrada a {available: False, ...} y NUNCA lanza (dict-no-throw). Seed fija (DetectorFactory.seed=0) para deteccion determinista." +tags: [eda, datascience, text, nlp, language-detection, langdetect, pure, python] +params: + - name: texts + desc: "Lista de strings (documentos). Los elementos None, no-str o vacios tras strip se descartan antes de clasificar." + - name: top_k + desc: "Numero maximo de idiomas a devolver en distribution, ordenados por count descendente (desempate por codigo ISO ascendente). Default 10." + - name: sample_max + desc: "Numero maximo de documentos a clasificar (se toman los primeros del corpus) para acotar el coste. Default 1000." +output: > + Dict con forma fija (dict-no-throw, nunca lanza): + {"available": bool, "n_detected": int, + "distribution": [{"lang": str, "count": int, "pct": float}, ...], + "dominant": str|None}. + available=True si langdetect es importable; lang son codigos ISO 639-1 ("es","en","fr",...); + pct = count/n_detected*100 redondeado a 2 decimales; n_detected = docs clasificados con exito; + dominant = idioma mas frecuente (None si no hubo detecciones). Corpus vacio con langdetect + presente -> available True, n_detected 0, distribution [], dominant None. Sin langdetect (o + fallo global) -> available False y el resto de campos a su valor vacio. +uses_functions: [] +uses_types: [] +returns: [] +returns_optional: false +error_type: "" +imports: [langdetect] +tested: true +tests: ["test_mixto_es_en", "test_vacio", "test_degradacion"] +test_file_path: "python/functions/datascience/detect_corpus_language_test.py" +file_path: "python/functions/datascience/detect_corpus_language.py" +--- + +## Ejemplo + +```python +import sys, os +sys.path.insert(0, os.path.join("python", "functions")) +from datascience.detect_corpus_language import detect_corpus_language + +corpus = [ + "este es un texto bastante largo en español para detectar el idioma correctamente", + "la inteligencia artificial transforma la manera en que trabajamos cada dia", + "this is a fairly long english text to detect the language correctly without issues", +] +out = detect_corpus_language(corpus) +# {"available": True, "n_detected": 3, +# "distribution": [{"lang": "es", "count": 2, "pct": 66.67}, +# {"lang": "en", "count": 1, "pct": 33.33}], +# "dominant": "es"} +``` + +## Cuando usarla + +Cuando perfiles una columna o corpus de texto en un EDA y necesites saber en +que idioma(s) esta escrito antes de elegir tokenizadores, stopwords, modelos +NLP o stemmers. Util tambien como check de calidad: detectar corpus mezclados +o un idioma inesperado. Llamala con la lista de textos crudos; la funcion +limpia, muestrea y resume sola. + +## Gotchas + +- `langdetect` es **opcional**: si no esta instalada, la funcion no lanza — + devuelve `{"available": False, "n_detected": 0, "distribution": [], "dominant": None}`. + Comprueba `out["available"]` antes de usar la distribucion. +- **Textos cortos** (pocas palabras o sin features lingüisticas) pueden no + detectarse: langdetect lanza `LangDetectException`, que se ignora y el doc no + cuenta en `n_detected`. Pasa frases razonablemente largas para resultados fiables. +- **Determinismo**: se fija `DetectorFactory.seed = 0` en cada llamada para que la + deteccion sea reproducible; sin esa semilla langdetect puede dar resultados + ligeramente distintos entre ejecuciones. +- `distribution` esta truncada a `top_k`; si el corpus tiene mas idiomas que + `top_k`, la suma de los `count` mostrados puede ser menor que `n_detected` + (pero `dominant` siempre refleja el idioma mas frecuente del corpus completo). diff --git a/python/functions/datascience/detect_corpus_language.py b/python/functions/datascience/detect_corpus_language.py new file mode 100644 index 00000000..a737ab91 --- /dev/null +++ b/python/functions/datascience/detect_corpus_language.py @@ -0,0 +1,91 @@ +"""Detecta la distribucion de idiomas de un corpus de textos. + +Funcion pura y defensiva: el computo es determinista y local (sin I/O de red). +La libreria opcional `langdetect` se importa de forma perezosa dentro de la +funcion; si no esta instalada (o cualquier paso falla), la funcion degrada +limpiamente a `available=False` y NUNCA lanza excepciones. +""" + + +def detect_corpus_language(texts, top_k=10, sample_max=1000) -> dict: + """Estima la distribucion de idiomas de un corpus con `langdetect`. + + Args: + texts: lista de strings (documentos). Los elementos None, no-str o + vacios tras strip se descartan. + top_k: numero maximo de idiomas a devolver en `distribution`, + ordenados por frecuencia descendente. + sample_max: numero maximo de documentos a clasificar (se toman los + primeros) para acotar el coste. + + Returns: + dict con la forma fija (dict-no-throw): + { + "available": bool, # True si langdetect es importable + "n_detected": int, # documentos clasificados con exito + "distribution": [{"lang": str, "count": int, "pct": float}, ...], + "dominant": str | None, + } + """ + degraded = { + "available": False, + "n_detected": 0, + "distribution": [], + "dominant": None, + } + try: + # Import perezoso con degradacion: si langdetect no esta disponible, + # devolvemos el dict degradado sin lanzar. + try: + from langdetect import detect, DetectorFactory + + # Semilla fija -> deteccion determinista entre ejecuciones. + DetectorFactory.seed = 0 + except Exception: + return dict(degraded) + + # Normaliza y filtra el corpus. + docs = [] + if texts: + for t in texts: + if isinstance(t, str): + s = t.strip() + if s: + docs.append(s) + + # Muestreo de los primeros `sample_max` documentos. + if sample_max is not None and sample_max >= 0: + docs = docs[:sample_max] + + # Conteo por idioma; langdetect lanza LangDetectException en textos + # sin features detectables -> se ignora y se sigue. + counts: dict = {} + for doc in docs: + try: + lang = detect(doc) + except Exception: + continue + counts[lang] = counts.get(lang, 0) + 1 + + n_detected = sum(counts.values()) + + # Orden estable: por count descendente, desempate por codigo de idioma. + ordered = sorted(counts.items(), key=lambda kv: (-kv[1], kv[0])) + + k = top_k if (top_k is not None and top_k >= 0) else len(ordered) + distribution = [] + for lang, count in ordered[:k]: + pct = round(count / n_detected * 100, 2) if n_detected else 0.0 + distribution.append({"lang": lang, "count": count, "pct": pct}) + + dominant = ordered[0][0] if ordered else None + + return { + "available": True, + "n_detected": n_detected, + "distribution": distribution, + "dominant": dominant, + } + except Exception: + # Cualquier fallo global degrada a available False sin lanzar. + return dict(degraded) diff --git a/python/functions/datascience/detect_corpus_language_test.py b/python/functions/datascience/detect_corpus_language_test.py new file mode 100644 index 00000000..5e94d293 --- /dev/null +++ b/python/functions/datascience/detect_corpus_language_test.py @@ -0,0 +1,58 @@ +"""Tests para detect_corpus_language.""" + +import builtins +import os +import sys + +# Anade python/functions a sys.path para importar el paquete `datascience`. +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) + +from datascience.detect_corpus_language import detect_corpus_language + +_ES = [ + "este es un texto bastante largo en español para detectar el idioma correctamente sin problemas", + "la inteligencia artificial transforma la manera en que trabajamos cada dia en muchos sectores", +] +_EN = [ + "this is a fairly long english text to detect the language correctly without any length issues", + "machine learning models can classify documents into many different categories quite reliably", +] + + +def test_mixto_es_en(): + """Golden: corpus mixto ES+EN claro -> available True, >=2 idiomas, counts coherentes.""" + out = detect_corpus_language(_ES + _EN) + assert out["available"] is True + assert out["dominant"] in {"es", "en"} + assert len(out["distribution"]) >= 2 + total = sum(item["count"] for item in out["distribution"]) + assert total == out["n_detected"] + assert out["n_detected"] == 4 + + +def test_vacio(): + """Edge: lista vacia con langdetect presente -> available True, sin detecciones.""" + out = detect_corpus_language([]) + assert out["available"] is True + assert out["n_detected"] == 0 + assert out["distribution"] == [] + assert out["dominant"] is None + + +def test_degradacion(monkeypatch): + """Error path: si langdetect no es importable -> degrada a available False sin lanzar.""" + import datascience.detect_corpus_language as m + + real_import = builtins.__import__ + + def fake_import(name, *a, **k): + if name == "langdetect" or name.startswith("langdetect."): + raise ImportError("simulado") + return real_import(name, *a, **k) + + monkeypatch.setattr(builtins, "__import__", fake_import) + out = m.detect_corpus_language(["hola mundo", "hello world"]) + assert out["available"] is False + assert out["n_detected"] == 0 + assert out["distribution"] == [] + assert out["dominant"] is None diff --git a/python/functions/datascience/extract_text_sample.md b/python/functions/datascience/extract_text_sample.md new file mode 100644 index 00000000..aec24232 --- /dev/null +++ b/python/functions/datascience/extract_text_sample.md @@ -0,0 +1,102 @@ +--- +name: extract_text_sample +kind: function +lang: py +domain: datascience +version: "1.0.0" +purity: impure +signature: "def extract_text_sample(db_path: str, table: str, columns: list, backend: str = 'duckdb', sample: int = 2000) -> dict" +description: "Muestrea columnas de texto de una tabla DuckDB/Postgres con push-down SQL (LIMIT sample), SIN traer la tabla entera a RAM. Funcion impura del grupo de capacidad `eda`: la usan los capitulos de texto/NLP del AutomaticEDA que necesitan valores crudos de texto (longitudes, tokens, ejemplos) sobre una muestra acotada. Construye el lector read-only query_fn(sql)->dict igual que build_eda_render_ctx (closure sobre duckdb_query_readonly / pg_query importados perezosamente desde infra). Escapa los identificadores con comillas dobles y lanza una sola query SELECT \"c1\", \"c2\" FROM \"table\" LIMIT n. Por columna, la lista de strings solo contiene valores NO None y NO vacios: cada celda no nula se convierte con str(...) y se descarta si queda cadena vacia. Estilo dict-no-throw del grupo eda: NUNCA lanza; ante cualquier fallo (query, conversion, backend desconocido) devuelve {status:'error', error:str, columns:{}, n:0}. La clave n reporta el numero de FILAS leidas por la query (antes de filtrar None/vacios)." +tags: [eda, datascience, text, nlp, extraction, read-only, duckdb, postgres, python] +uses_functions: [duckdb_query_readonly_py_infra, pg_query_py_infra] +uses_types: [] +returns: [] +returns_optional: false +error_type: "error_go_core" +imports: [] +params: + - name: db_path + desc: "ruta al archivo DuckDB, o DSN PostgreSQL si backend='postgres'. Se inyecta en el closure query_fn. No se valida aqui: si la base no existe o el DSN es invalido, la query devuelve status error y el resultado es {status:'error', ...} (no lanza)." + - name: table + desc: "nombre de la tabla. Se escapa con comillas dobles en la query (SELECT ... FROM \"table\")." + - name: columns + desc: "lista de nombres de columna de texto a muestrear. Se filtra a las entradas que sean str no vacio; cada nombre se escapa con comillas dobles. Si tras filtrar queda vacia -> {status:'ok', columns:{}, n:0} sin tocar la base." + - name: backend + desc: "'duckdb' (default) o 'postgres'. Selecciona el lector read-only del registry (duckdb_query_readonly / pg_query). Cualquier otro valor -> {status:'error', error:'backend desconocido: <valor>', columns:{}, n:0}." + - name: sample + desc: "maximo de filas a muestrear (clausula LIMIT). Default 2000. Acota memoria y tiempo: con tablas grandes obtienes el primer tramo por orden fisico (sin ORDER BY), no un muestreo uniforme." +output: "dict dict-no-throw (NUNCA lanza): {status:'ok'|'error', columns:{col_name:[str,...]}, n:int, error:str}. En exito (status='ok') columns mapea cada columna pedida a la lista de sus valores de texto NO None y NO vacios (cada celda convertida con str(...)); n es el numero de FILAS leidas por la query (antes de filtrar None/vacios). columns vacio -> {status:'ok', columns:{}, n:0}. En error (backend desconocido, query con status!='ok', o cualquier excepcion) -> {status:'error', error:str, columns:{}, n:0}; la clave error solo aparece en este caso." +tested: true +tests: ["test_extract_basic", "test_backend_desconocido", "test_columns_vacio", "test_sample_limit"] +test_file_path: "python/functions/datascience/extract_text_sample_test.py" +file_path: "python/functions/datascience/extract_text_sample.py" +--- + +## Ejemplo + +```python +import sys, os +sys.path.insert(0, os.path.join("python", "functions")) +# Import directo del submodulo (no requiere export en datascience/__init__.py). +from datascience.extract_text_sample import extract_text_sample + +# Muestrea hasta 2000 filas de dos columnas de texto de una tabla DuckDB. +res = extract_text_sample( + "data/reviews.duckdb", "reviews", ["title", "body"], + backend="duckdb", sample=2000, +) +# res == { +# "status": "ok", +# "columns": { +# "title": ["Gran producto", "No funciona", ...], # solo no-None, no-"" +# "body": ["Lo uso a diario...", ...], +# }, +# "n": 2000, # filas leidas por la query (antes de filtrar None/vacios) +# } + +# Postgres: db_path es el DSN. +res_pg = extract_text_sample( + "postgresql://user:pass@localhost:5433/trends", "comentarios", ["texto"], + backend="postgres", sample=500, +) +``` + +## Cuando usarla + +Cuando necesites valores CRUDOS de texto de una o varias columnas para analisis +NLP/texto (distribucion de longitudes, conteo de tokens, ejemplos representativos, +deteccion de idioma) pero NO quieras cargar la tabla entera en memoria. Es el +muestreador de texto del grupo `eda`: una sola llamada con push-down `LIMIT` +devuelve listas de strings por columna, limpias de None y vacios, listas para +alimentar un capitulo de texto del AutomaticEDA o cualquier rutina de tokenizado. +Usala junto a `profile_table` / `build_eda_render_ctx` cuando el perfil agregado +no basta y hace falta el texto real. + +## Gotchas + +- **Impura**: lee de la base de datos a traves de `query_fn` (closure sobre + `duckdb_query_readonly` / `pg_query`). No abre conexiones fuera de esos wrappers + del registry. Estilo dict-no-throw del grupo `eda`: NUNCA lanza; ante cualquier + fallo devuelve `{status:'error', error:str, columns:{}, n:0}`. +- **`error_type` en el frontmatter es `error_go_core` por convencion del registry** + (toda funcion impura debe declararlo y el indexer lo exige), pero el codigo NO + lanza esa excepcion: degrada al dict de error. Es metadata, no comportamiento. +- **Backend desconocido**: con un `backend` que no sea `duckdb` ni `postgres` + devuelve `{status:'error', error:'backend desconocido: <valor>', columns:{}, + n:0}` sin tocar la base. +- **Las listas NO incluyen None ni cadenas vacias**: cada celda no nula se pasa + por `str(...)` y se descarta si queda `""`. Por eso `len(columns[col])` puede ser + menor que `n` (que cuenta las filas leidas). Si necesitas alineacion por fila + (una entrada por fila aunque sea None), usa `build_eda_render_ctx` (raw_numeric), + no esta funcion. +- **`LIMIT sample` sin `ORDER BY`**: con tablas grandes obtienes el primer tramo + por orden fisico del backend, no un muestreo uniforme ni reproducible. Sube + `sample` para mas cobertura, o pre-ordena/aleatoriza la tabla si necesitas + representatividad. +- **DuckDB en sandbox por defecto**: `duckdb_query_readonly` abre la conexion con + `enable_external_access=False`, asi que la query solo puede leer la propia base + (no `read_csv`/`httpfs`/`ATTACH` a paths externos). Lee tablas ya existentes en + el archivo DuckDB sin problema. +- **No loguear los datos crudos**: las listas de `columns` pueden contener texto + sensible (reviews, comentarios, PII). En trazas usa solo conteos (`n`, + `len(columns[col])`) y nombres de columna, no el dict completo. diff --git a/python/functions/datascience/extract_text_sample.py b/python/functions/datascience/extract_text_sample.py new file mode 100644 index 00000000..d44b9b95 --- /dev/null +++ b/python/functions/datascience/extract_text_sample.py @@ -0,0 +1,112 @@ +"""extract_text_sample — muestrea columnas de texto de una tabla sin cargarla en RAM. + +Funcion impura (lee de la base de datos) del grupo de capacidad `eda`. Dado un +``db_path`` + ``table`` (DuckDB o PostgreSQL) y una lista de ``columns`` de texto, +trae una MUESTRA de esas columnas con push-down SQL (``LIMIT sample``), nunca la +tabla entera. La usan los capitulos de texto/NLP del AutomaticEDA que necesitan +valores crudos de texto (longitudes, tokens, ejemplos) sin materializar millones +de filas en memoria. + +El lector read-only ``query_fn(sql) -> dict`` se construye igual que en +``build_eda_render_ctx`` / ``profile_table``: un closure sobre el wrapper del +registry (``duckdb_query_readonly`` / ``pg_query``), importado perezosamente +dentro de la funcion para no crear ciclos al cargar el ``__init__`` del paquete +``datascience``. Nunca abre conexiones fuera de esos wrappers. + +Estilo dict-no-throw del grupo `eda`: la funcion NUNCA lanza. Captura cualquier +excepcion (query, conversion) y devuelve ``{"status":"error", "error":str(e), +"columns":{}, "n":0}``. Si la query subyacente devuelve ``status != "ok"``, se +propaga como error con el mensaje del wrapper. + +Por columna, la lista de strings solo contiene valores NO nulos y NO vacios: +cada celda no-None se convierte con ``str(...)`` y se descarta si queda ``""``. +La clave ``n`` reporta el numero de FILAS leidas por la query (antes de filtrar +los None/vacios), util para saber cuanto se muestreo realmente. +""" + + +def extract_text_sample(db_path, table, columns, backend="duckdb", sample=2000): + """Muestrea columnas de texto de una tabla DuckDB/Postgres con push-down SQL. + + Args: + db_path: ruta al archivo DuckDB, o DSN PostgreSQL si backend="postgres". + Se inyecta en el closure query_fn. No se valida aqui: si la base no + existe o el DSN es invalido, la query devuelve status error y el + resultado es {status:'error', ...} (no lanza). + table: nombre de la tabla. Se escapa con comillas dobles en la query. + columns: lista de nombres de columna de texto a muestrear. Se filtra a las + entradas que sean str no vacio; cada nombre se escapa con comillas + dobles. Si tras filtrar queda vacia -> {status:'ok', columns:{}, n:0}. + backend: "duckdb" (default) o "postgres". Selecciona el lector read-only + del registry (duckdb_query_readonly / pg_query). Cualquier otro valor + -> {status:'error', error:'backend desconocido: ...', columns:{}, n:0}. + sample: maximo de filas a muestrear (clausula LIMIT). Default 2000. Acota + memoria y tiempo: con tablas grandes obtienes el primer tramo por + orden fisico, no un muestreo uniforme. + + Returns: + dict (dict-no-throw, NUNCA lanza): + {"status": "ok"|"error", + "columns": {col_name: [str, str, ...], ...}, # solo no-None, no-"" + "n": int, # nº de filas leidas por la query (antes de filtrar) + "error": str} # solo presente si status == "error" + """ + try: + # 1) Lector read-only del backend activo, construido como en + # build_eda_render_ctx (closure sobre el wrapper del registry). Imports + # perezosos: este modulo vive en el paquete `datascience`, importar a + # `infra` a nivel de modulo crearia un ciclo al cargar el __init__. + if backend == "duckdb": + from infra import duckdb_query_readonly + + def query_fn(sql): + return duckdb_query_readonly(db_path, sql) + + elif backend == "postgres": + from infra import pg_query + + def query_fn(sql): + return pg_query(db_path, sql) + + else: + return { + "status": "error", + "error": f"backend desconocido: {backend}", + "columns": {}, + "n": 0, + } + + # 2) Columnas validas (str no vacio). Si no queda ninguna, nada que + # muestrear: ok con columns vacio. + cols = [] + if isinstance(columns, (list, tuple)): + cols = [c for c in columns if isinstance(c, str) and c != ""] + if not cols: + return {"status": "ok", "columns": {}, "n": 0} + + # 3) Push-down: una sola query con LIMIT. Identificadores escapados con + # comillas dobles, igual que build_eda_render_ctx. + cols_sql = ", ".join(f'"{c}"' for c in cols) + sql = f'SELECT {cols_sql} FROM "{table}" LIMIT {int(sample)}' + q = query_fn(sql) + if not isinstance(q, dict) or q.get("status") != "ok": + err = q.get("error") if isinstance(q, dict) else "query sin resultado" + return {"status": "error", "error": str(err), "columns": {}, "n": 0} + + rows = q.get("rows") or [] + out = {c: [] for c in cols} + for row in rows: + if not isinstance(row, dict): + continue + for c in cols: + value = row.get(c) + if value is None: + continue + s = str(value) + if s == "": + continue + out[c].append(s) + + return {"status": "ok", "columns": out, "n": len(rows)} + except Exception as exc: # noqa: BLE001 - dict-no-throw del grupo eda + return {"status": "error", "error": str(exc), "columns": {}, "n": 0} diff --git a/python/functions/datascience/extract_text_sample_test.py b/python/functions/datascience/extract_text_sample_test.py new file mode 100644 index 00000000..b6c15e83 --- /dev/null +++ b/python/functions/datascience/extract_text_sample_test.py @@ -0,0 +1,83 @@ +"""Tests para extract_text_sample. + +Self-contained: crea un DuckDB temporal pequeño con una columna de texto (algunas +filas con NULL) y una numerica, y verifica que la muestra de texto trae solo los +valores no nulos, que el backend desconocido y la lista de columnas vacia se +manejan dict-no-throw, y que sample acota el numero de filas leidas. +""" + +import os +import sys + +_HERE = os.path.dirname(os.path.abspath(__file__)) +_FUNCTIONS = os.path.abspath(os.path.join(_HERE, "..")) # python/functions +if _FUNCTIONS not in sys.path: + sys.path.insert(0, _FUNCTIONS) + +import duckdb # noqa: E402 + +from datascience.extract_text_sample import extract_text_sample # noqa: E402 + +_TABLE = "t" +# 6 filas: txt VARCHAR con dos NULL, other INT siempre presente. +_ROWS = [ + ("alpha", 1), + ("beta", 2), + (None, 3), + ("gamma", 4), + (None, 5), + ("delta", 6), +] +_TXT_NON_NULL = {"alpha", "beta", "gamma", "delta"} + + +def _make_db(tmp_path): + """Crea un DuckDB temporal con la tabla de prueba y devuelve su ruta.""" + db_path = os.path.join(str(tmp_path), "text_sample.duckdb") + con = duckdb.connect(db_path) + try: + con.execute(f'CREATE TABLE "{_TABLE}" (txt VARCHAR, other INTEGER)') + con.executemany(f'INSERT INTO "{_TABLE}" VALUES (?, ?)', _ROWS) + finally: + con.close() + return db_path + + +def test_extract_basic(tmp_path): + db_path = _make_db(tmp_path) + res = extract_text_sample(db_path, _TABLE, ["txt"]) + assert res["status"] == "ok" + # n = filas leidas por la query (6), antes de filtrar None. + assert res["n"] == len(_ROWS) + # columns["txt"] trae solo los strings no nulos (los dos NULL fuera). + assert "txt" in res["columns"] + assert set(res["columns"]["txt"]) == _TXT_NON_NULL + assert len(res["columns"]["txt"]) == len(_TXT_NON_NULL) + # No se pidio "other", no debe aparecer. + assert "other" not in res["columns"] + + +def test_backend_desconocido(tmp_path): + db_path = _make_db(tmp_path) + res = extract_text_sample(db_path, _TABLE, ["txt"], backend="mysql") + assert res["status"] == "error" + assert "backend desconocido" in res["error"] + assert res["columns"] == {} + assert res["n"] == 0 + + +def test_columns_vacio(tmp_path): + db_path = _make_db(tmp_path) + res = extract_text_sample(db_path, _TABLE, []) + assert res["status"] == "ok" + assert res["columns"] == {} + assert res["n"] == 0 + + +def test_sample_limit(tmp_path): + db_path = _make_db(tmp_path) + res = extract_text_sample(db_path, _TABLE, ["txt"], sample=2) + assert res["status"] == "ok" + # sample=2 -> la query lee como mucho 2 filas. + assert res["n"] == 2 + assert len(res["columns"]["txt"]) <= 2 diff --git a/python/pyproject.toml b/python/pyproject.toml index 052f7280..f0fed9a1 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -18,6 +18,7 @@ dependencies = [ "google-cloud-bigquery-storage>=2.27", "google-cloud-storage>=3.10.1", "httpx", + "langdetect>=1.0.9", "matplotlib>=3.10.9", "opencv-contrib-python-headless>=4.13.0.92", "openpyxl>=3.1.5", @@ -40,6 +41,7 @@ dependencies = [ "seaborn>=0.13.2", "shapely>=2.1.2", "statsmodels>=0.14.6", + "textstat>=0.7.13", "trimesh>=4.12.2", "xlrd>=2.0.2", ] diff --git a/python/uv.lock b/python/uv.lock index d46df6d9..be3188f3 100644 --- a/python/uv.lock +++ b/python/uv.lock @@ -899,6 +899,7 @@ dependencies = [ { name = "google-cloud-bigquery-storage" }, { name = "google-cloud-storage" }, { name = "httpx" }, + { name = "langdetect" }, { name = "matplotlib" }, { name = "opencv-contrib-python-headless" }, { name = "openpyxl" }, @@ -906,9 +907,11 @@ dependencies = [ { name = "polars" }, { name = "pymeshlab" }, { name = "pymssql" }, + { name = "pymupdf" }, { name = "pypdf" }, { name = "pyproj" }, { name = "python-docx" }, + { name = "python-pptx" }, { name = "pyyaml" }, { name = "qrcode", extra = ["pil"] }, { name = "rapidfuzz" }, @@ -919,6 +922,7 @@ dependencies = [ { name = "seaborn" }, { name = "shapely" }, { name = "statsmodels" }, + { name = "textstat" }, { name = "trimesh" }, { name = "xlrd" }, ] @@ -959,6 +963,7 @@ requires-dist = [ { name = "jupyter-collaboration", marker = "extra == 'jupyter'", specifier = ">=2.0" }, { name = "jupyter-mcp-server", marker = "extra == 'jupyter'" }, { name = "jupyterlab", marker = "extra == 'jupyter'", specifier = ">=4.0" }, + { name = "langdetect", specifier = ">=1.0.9" }, { name = "matplotlib", specifier = ">=3.10.9" }, { name = "opencv-contrib-python-headless", specifier = ">=4.13.0.92" }, { name = "openpyxl", specifier = ">=3.1.5" }, @@ -966,9 +971,11 @@ requires-dist = [ { name = "polars", specifier = ">=1.40.1" }, { name = "pymeshlab", specifier = ">=2025.7.post1" }, { name = "pymssql", specifier = ">=2.3.13" }, + { name = "pymupdf", specifier = ">=1.28.0" }, { name = "pypdf", specifier = ">=6.10.0" }, { name = "pyproj", specifier = ">=3.7.2" }, { name = "python-docx", specifier = ">=1.2.0" }, + { name = "python-pptx", specifier = ">=1.0.2" }, { name = "pyyaml", specifier = ">=6.0.3" }, { name = "qrcode", extras = ["pil"], specifier = ">=8.2" }, { name = "rapidfuzz", specifier = ">=3.14.5" }, @@ -979,6 +986,7 @@ requires-dist = [ { name = "seaborn", specifier = ">=0.13.2" }, { name = "shapely", specifier = ">=2.1.2" }, { name = "statsmodels", specifier = ">=0.14.6" }, + { name = "textstat", specifier = ">=0.7.13" }, { name = "trimesh", specifier = ">=4.12.2" }, { name = "xlrd", specifier = ">=2.0.2" }, ] @@ -2198,6 +2206,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b5/91/53255615acd2a1eaca307ede3c90eb550bae9c94581f8c00081b6b1c8f44/kiwisolver-1.5.0-graalpy312-graalpy250_312_native-win_amd64.whl", hash = "sha256:1f1489f769582498610e015a8ef2d36f28f505ab3096d0e16b4858a9ec214f57", size = 75987, upload-time = "2026-03-09T13:15:39.65Z" }, ] +[[package]] +name = "langdetect" +version = "1.0.9" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0e/72/a3add0e4eec4eb9e2569554f7c70f4a3c27712f40e3284d483e88094cc0e/langdetect-1.0.9.tar.gz", hash = "sha256:cbc1fef89f8d062739774bd51eda3da3274006b3661d199c2655f6b3f6d605a0", size = 981474, upload-time = "2021-05-07T07:54:13.562Z" } + [[package]] name = "lark" version = "1.3.1" @@ -2699,6 +2716,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9e/c9/b2622292ea83fbb4ec318f5b9ab867d0a28ab43c5717bb85b0a5f6b3b0a4/networkx-3.6.1-py3-none-any.whl", hash = "sha256:d47fbf302e7d9cbbb9e2555a0d267983d2aa476bac30e90dfbe5669bd57f3762", size = 2068504, upload-time = "2025-12-08T17:02:38.159Z" }, ] +[[package]] +name = "nltk" +version = "3.9.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "joblib" }, + { name = "regex" }, + { name = "tqdm" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/74/a1/b3b4adf15585a5bc4c357adde150c01ebeeb642173ded4d871e89468767c/nltk-3.9.4.tar.gz", hash = "sha256:ed03bc098a40481310320808b2db712d95d13ca65b27372f8a403949c8b523d0", size = 2946864, upload-time = "2026-03-24T06:13:40.641Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9d/91/04e965f8e717ba0ab4bdca5c112deeab11c9e750d94c4d4602f050295d39/nltk-3.9.4-py3-none-any.whl", hash = "sha256:f2fa301c3a12718ce4a0e9305c5675299da5ad9e26068218b69d692fda84828f", size = 1552087, upload-time = "2026-03-24T06:13:38.47Z" }, +] + [[package]] name = "notebook-shim" version = "0.2.4" @@ -3750,6 +3782,23 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/25/50/4be9bd9cf4b43208a7175117a533ece200cfe4131a39f9909bdc7560ddeb/pymssql-2.3.13-cp314-cp314-win_amd64.whl", hash = "sha256:7d7037d2b5b907acc7906d0479924db2935a70c720450c41339146a4ada2b93d", size = 2049139, upload-time = "2026-02-14T05:00:23.951Z" }, ] +[[package]] +name = "pymupdf" +version = "1.28.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8e/e9/6d6c5d6c0a3551bffd47681a6240caf941727f195b45593cf20ab36f018f/pymupdf-1.28.0.tar.gz", hash = "sha256:e53f3567403a92da15caa9e7ae0164327fff48817e9f40175367fb9de524258d", size = 87637751, upload-time = "2026-06-29T09:08:47.547Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c8/b7/88043e38cc7529de070f0c9bd267fa258035cca0b4ad5260536b994594a7/pymupdf-1.28.0-cp310-abi3-macosx_10_15_x86_64.whl", hash = "sha256:892b89ba88e8f98b53133b62877a9dc9b5e7dc6a4aeb837b612db56a8d2e03ac", size = 24597385, upload-time = "2026-06-29T09:03:30.608Z" }, + { url = "https://files.pythonhosted.org/packages/33/f4/23775bbda0781b61fc398cc75079a2b0e64696d8fcf93271748883e9627e/pymupdf-1.28.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:4d692dcf44d3566ae96bc6f6346c6ad432274a29ba617bf7a9fe18009e24adb4", size = 23828292, upload-time = "2026-06-29T09:03:46.129Z" }, + { url = "https://files.pythonhosted.org/packages/1c/f5/bf75fc7a415722f8b33662054f82d88520c0cbfd4c36d0e08aeaec605e49/pymupdf-1.28.0-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:47a5c29ed4eb0744de9c4e37bb49b1259b18d4d75fcc8a7c130f7c9fa15956f6", size = 25045507, upload-time = "2026-06-29T09:04:03.86Z" }, + { url = "https://files.pythonhosted.org/packages/58/69/5d12c9f1f2d76f28383d6110a069c79fbfced5a4f97bb1ee6e8354f52bb7/pymupdf-1.28.0-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:44f0973f5e5edbaec95bc34b64e71d1959d4ee90b1328de1b4f4f5b4fa78673f", size = 25716599, upload-time = "2026-06-29T09:04:19.367Z" }, + { url = "https://files.pythonhosted.org/packages/4d/b4/ec0e017bc42857cc86bd651441dbc41cc18be48d4698ecd27aac491e0c9a/pymupdf-1.28.0-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:4d61ec323a706e153a12e262e51febfb43eeaa20977785ace135d18d48bcdc83", size = 25940489, upload-time = "2026-06-29T09:04:36.624Z" }, + { url = "https://files.pythonhosted.org/packages/06/86/f831fef09013f33b3c9c09fb3923f2ff53e1e437f6ace14b8ae46392f558/pymupdf-1.28.0-cp310-abi3-win32.whl", hash = "sha256:caea2b3b67347fd79e5d15ed7929b0e886aac594ea228073b6d39de0078189da", size = 18489703, upload-time = "2026-06-29T20:50:30.599Z" }, + { url = "https://files.pythonhosted.org/packages/2e/5d/1a03f53eb0449900469335fcfc742ca28e3ba159b7d650e0921d50b8b308/pymupdf-1.28.0-cp310-abi3-win_amd64.whl", hash = "sha256:e01e90fd86abfeb37ceb921eddb951f988a11d45ff6ce6b7664f2039849068ec", size = 19773102, upload-time = "2026-06-29T09:04:49.773Z" }, + { url = "https://files.pythonhosted.org/packages/72/f6/1e52ce243ca792254f6223b4017c5667194c146ce9b88baf37bc5eb3d1c9/pymupdf-1.28.0-cp313-abi3-pyemscripten_2025_0_wasm32.whl", hash = "sha256:74c6d00ba2a9aad3a635db73b07c15db462b480741d831a34a75a56535ebc22b", size = 18357011, upload-time = "2026-06-29T20:50:50.353Z" }, + { url = "https://files.pythonhosted.org/packages/62/b1/46b5b3d8ef3cc71114667cf10c4d8b33f39af97253af32e9a0986775b638/pymupdf-1.28.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:b3e1399c7a64c6914239116a369efcdaac4cfb9e838bde2656d7accc4a85c72d", size = 25753599, upload-time = "2026-06-29T09:05:09.398Z" }, +] + [[package]] name = "pyogrio" version = "0.12.1" @@ -3811,6 +3860,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/55/f2/7ebe366f633f30a6ad105f650f44f24f98cb1335c4157d21ae47138b3482/pypdf-6.10.0-py3-none-any.whl", hash = "sha256:90005e959e1596c6e6c84c8b0ad383285b3e17011751cedd17f2ce8fcdfc86de", size = 334459, upload-time = "2026-04-10T09:34:54.966Z" }, ] +[[package]] +name = "pyphen" +version = "0.17.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/69/56/e4d7e1bd70d997713649c5ce530b2d15a5fc2245a74ca820fc2d51d89d4d/pyphen-0.17.2.tar.gz", hash = "sha256:f60647a9c9b30ec6c59910097af82bc5dd2d36576b918e44148d8b07ef3b4aa3", size = 2079470, upload-time = "2025-01-20T13:18:36.296Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7b/1f/c2142d2edf833a90728e5cdeb10bdbdc094dde8dbac078cee0cf33f5e11b/pyphen-0.17.2-py3-none-any.whl", hash = "sha256:3a07fb017cb2341e1d9ff31b8634efb1ae4dc4b130468c7c39dd3d32e7c3affd", size = 2079358, upload-time = "2025-01-20T13:18:29.629Z" }, +] + [[package]] name = "pyproj" version = "3.7.2" @@ -3935,6 +3993,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1c/fd/0318007beb234790993d3ec5afd051d1dbceb733e81e3afe2b981ece3f37/python_multipart-0.0.30-py3-none-any.whl", hash = "sha256:830964def8c90607ac5daa00514e3987815865713ade8d20febc9177ac0c3c5b", size = 29730, upload-time = "2026-05-31T19:24:53.814Z" }, ] +[[package]] +name = "python-pptx" +version = "1.0.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "lxml" }, + { name = "pillow" }, + { name = "typing-extensions" }, + { name = "xlsxwriter" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/52/a9/0c0db8d37b2b8a645666f7fd8accea4c6224e013c42b1d5c17c93590cd06/python_pptx-1.0.2.tar.gz", hash = "sha256:479a8af0eaf0f0d76b6f00b0887732874ad2e3188230315290cd1f9dd9cc7095", size = 10109297, upload-time = "2024-08-07T17:33:37.772Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d9/4f/00be2196329ebbff56ce564aa94efb0fbc828d00de250b1980de1a34ab49/python_pptx-1.0.2-py3-none-any.whl", hash = "sha256:160838e0b8565a8b1f67947675886e9fea18aa5e795db7ae531606d68e785cba", size = 472788, upload-time = "2024-08-07T17:33:28.192Z" }, +] + [[package]] name = "pywin32" version = "311" @@ -4936,6 +5009,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6a/9e/2064975477fdc887e47ad42157e214526dcad8f317a948dee17e1659a62f/terminado-0.18.1-py3-none-any.whl", hash = "sha256:a4468e1b37bb318f8a86514f65814e1afc977cf29b3992a4500d9dd305dcceb0", size = 14154, upload-time = "2024-03-12T14:34:36.569Z" }, ] +[[package]] +name = "textstat" +version = "0.7.13" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nltk" }, + { name = "pyphen" }, + { name = "setuptools" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8c/0f/b673fcec5ad6e976b2e8368ef3651fe0fea3348a1191bacfcd41a17ddec6/textstat-0.7.13.tar.gz", hash = "sha256:a88d1da76287cd27ca4ce7bcba1ebaf2890544a5f0bb6a5758fa84cef3bceccb", size = 138932, upload-time = "2026-02-18T21:07:39.525Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ca/31/0eb4cc5bb021b4ceaaa602c59ba16ce99256b9dd30981bef3f3a53d8555f/textstat-0.7.13-py3-none-any.whl", hash = "sha256:04b1ec995d1e8b2e628759497e6b23204a9ec91dcd652447d8cbba9478f25471", size = 177050, upload-time = "2026-02-18T21:07:38.163Z" }, +] + [[package]] name = "threadpoolctl" version = "3.6.0" @@ -5312,6 +5399,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1a/62/c8d562e7766786ba6587d09c5a8ba9f718ed3fa8af7f4553e8f91c36f302/xlrd-2.0.2-py2.py3-none-any.whl", hash = "sha256:ea762c3d29f4cca48d82df517b6d89fbce4db3107f9d78713e48cd321d5c9aa9", size = 96555, upload-time = "2025-06-14T08:46:37.766Z" }, ] +[[package]] +name = "xlsxwriter" +version = "3.2.9" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/46/2c/c06ef49dc36e7954e55b802a8b231770d286a9758b3d936bd1e04ce5ba88/xlsxwriter-3.2.9.tar.gz", hash = "sha256:254b1c37a368c444eac6e2f867405cc9e461b0ed97a3233b2ac1e574efb4140c", size = 215940, upload-time = "2025-09-16T00:16:21.63Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3a/0c/3662f4a66880196a590b202f0db82d919dd2f89e99a27fadef91c4a33d41/xlsxwriter-3.2.9-py3-none-any.whl", hash = "sha256:9a5db42bc5dff014806c58a20b9eae7322a134abb6fce3c92c181bfb275ec5b3", size = 175315, upload-time = "2025-09-16T00:16:20.108Z" }, +] + [[package]] name = "xxhash" version = "3.7.0" From 6e3c3cf2a299d784793d77b15ed72c2ba46a9cbc Mon Sep 17 00:00:00 2001 From: Egutierrez <egutierrez@dead.dd> Date: Tue, 30 Jun 2026 20:38:38 +0200 Subject: [PATCH 39/53] feat(papers): estructura, scaffolding y capability page del artefacto papers/ MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Nuevo tipo de artefacto para papers académicos reproducibles (papers/<NNNN-slug>/): - Plantillas docs/templates/paper.md (IMRaD completo con guías por sección: Abstract, Introduction, Related work, Methods, Results, Discussion con Limitaciones + Amenazas a la validez, Conclusion + Future work) y docs/templates/preregistration.md (H0/H1 falsable, variables, diseño, plan de análisis con test exacto + effect size + corrección múltiple, predicción cuantitativa; nota anti-HARKing de congelado). - Pipeline init_paper (bash/functions/pipelines/init_paper.sh + .md): calcula el siguiente NNNN, crea las subcarpetas (experiments data figures reviews out), copia las plantillas rellenando el frontmatter (title, slug, date, phase=question, status=draft) y crea references.md. No hace git init (fase interna local). - Función atómica reutilizable next_numbered_dir (bash/functions/io): siguiente prefijo NNNN- escaneando un directorio numerado (reutilizable por papers/reports/issues). - papers/ como artefacto local gitignored (bloque en .gitignore + papers/.gitkeep): un paper en fase interna no contamina el repo padre; al promocionar a publishable se vuelve sub-repo Gitea propio. - Página de capacidad docs/capabilities/papers.md + fila en el INDEX: tabla de funciones del grupo papers (disponibles + en construcción por la flota), ejemplo canónico end-to-end y fronteras. Reutiliza slugify_ascii del registry. Diseño: reports/0001-2026-06-30-papers-system-design.md. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> --- .gitignore | 7 + bash/functions/io/next_numbered_dir.md | 58 ++++++++ bash/functions/io/next_numbered_dir.sh | 46 +++++++ bash/functions/pipelines/init_paper.md | 69 ++++++++++ bash/functions/pipelines/init_paper.sh | 177 +++++++++++++++++++++++++ docs/capabilities/INDEX.md | 1 + docs/capabilities/papers.md | 82 ++++++++++++ docs/templates/paper.md | 94 +++++++++++++ docs/templates/preregistration.md | 59 +++++++++ papers/.gitkeep | 0 10 files changed, 593 insertions(+) create mode 100644 bash/functions/io/next_numbered_dir.md create mode 100644 bash/functions/io/next_numbered_dir.sh create mode 100644 bash/functions/pipelines/init_paper.md create mode 100644 bash/functions/pipelines/init_paper.sh create mode 100644 docs/capabilities/papers.md create mode 100644 docs/templates/paper.md create mode 100644 docs/templates/preregistration.md create mode 100644 papers/.gitkeep diff --git a/.gitignore b/.gitignore index 83ae8db6..7c0d9ca1 100644 --- a/.gitignore +++ b/.gitignore @@ -54,6 +54,13 @@ reports/* !reports/.gitkeep projects/*/reports/ +# Papers — artefacto local: papers académicos reproducibles. En fase interna viven +# local y gitignored (como los reports); al promocionar a fase publishable se +# vuelven sub-repo Gitea propio (como apps/analyses). Solo el marcador .gitkeep se +# versiona. Convención: docs/capabilities/papers.md +papers/* +!papers/.gitkeep + # Node / pnpm **/node_modules/ diff --git a/bash/functions/io/next_numbered_dir.md b/bash/functions/io/next_numbered_dir.md new file mode 100644 index 00000000..4895cf8a --- /dev/null +++ b/bash/functions/io/next_numbered_dir.md @@ -0,0 +1,58 @@ +--- +name: next_numbered_dir +kind: function +lang: bash +domain: io +version: "1.0.0" +purity: impure +signature: "next_numbered_dir(parent_dir: string, [width: int]) -> string" +description: "Calcula el siguiente prefijo numerico NNNN- para un directorio numerado incremental. Escanea los subdirectorios directos de parent_dir cuyo nombre empiece por NNNN- (4+ digitos seguidos de guion), toma el maximo, le suma 1 y lo imprime con zero-padding al ancho width (default 4). Si parent_dir no existe o no tiene subdirs que matcheen, imprime 0001." +tags: [papers, io, scaffold] +uses_functions: [] +uses_types: [] +returns: [] +returns_optional: false +error_type: "error_go_core" +imports: [] +params: + - name: parent_dir + desc: "directorio padre cuyos subdirectorios numerados (NNNN-...) se escanean; obligatorio" + - name: width + desc: "ancho del zero-padding del numero impreso (default 4); opcional" +output: "el siguiente numero como string con zero-padding a width digitos a stdout (ej. 0003); usage a stderr y exit 1 si falta parent_dir" +tested: false +tests: [] +test_file_path: "" +file_path: "bash/functions/io/next_numbered_dir.sh" +--- + +## Ejemplo + +```bash +source bash/functions/io/next_numbered_dir.sh + +# Sobre un papers/ que ya contiene 0001-foo y 0002-bar +mkdir -p /tmp/papers/{0001-foo,0002-bar} +next_numbered_dir /tmp/papers +# -> 0003 + +# Directorio vacio o inexistente -> primer numero +next_numbered_dir /tmp/papers_nuevo +# -> 0001 + +# Ancho de padding distinto +next_numbered_dir /tmp/papers 6 +# -> 000003 +``` + +## Cuando usarla + +Cuando scaffoldees un artefacto numerado incremental (papers/, reports/, issues/) y necesites el siguiente NNNN sin colision: escanea lo que ya existe en disco y te da el numero libre listo para crear `<NNNN>-<slug>`. + +## Gotchas + +- **Impura**: lee el filesystem (estado del directorio en el momento de la llamada). No crea nada — solo calcula e imprime el numero. +- **Octal**: los numeros con cero a la izquierda (`08`, `09`) se interpretan como octal en aritmetica bash y romperian el calculo. La funcion fuerza base 10 con `10#$num` para evitarlo. +- **Solo subdirectorios**: cuenta unicamente subdirs directos. Archivos sueltos (`.gitkeep`, `notas.md`) y subdirs que no matcheen el patron se ignoran. No es recursivo. +- **Patron estricto**: el prefijo debe ser `NNNN-` (minimo 4 digitos seguidos de guion). Un subdir `12-foo` o `0001foo` (sin guion) NO se cuenta. +- No hay deteccion de huecos: devuelve `max+1`, no el primer numero libre intermedio. Si tienes `0001` y `0003`, devuelve `0004`, no `0002`. diff --git a/bash/functions/io/next_numbered_dir.sh b/bash/functions/io/next_numbered_dir.sh new file mode 100644 index 00000000..abc729b2 --- /dev/null +++ b/bash/functions/io/next_numbered_dir.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +# next_numbered_dir — Compute the next NNNN- prefix for a numbered directory. +# +# Scans the DIRECT subdirectories of <parent_dir> whose names start with a +# numeric prefix of the form `NNNN-` (4+ digits followed by a hyphen), takes +# the maximum number, adds 1, and prints it zero-padded to <width> (default 4). +# If <parent_dir> does not exist or contains no matching subdir, prints the +# first number (0001 at default width). + +next_numbered_dir() { + local parent_dir="${1:-}" + local width="${2:-4}" + + if [[ -z "$parent_dir" ]]; then + echo "usage: next_numbered_dir <parent_dir> [width]" >&2 + return 1 + fi + + local max=0 + local entry base num + + if [[ -d "$parent_dir" ]]; then + # Iterate only over direct subdirectories. The trailing slash in the + # glob ensures files (e.g. .gitkeep) are skipped — only dirs match. + for entry in "$parent_dir"/*/; do + # If the glob matched nothing it stays literal; guard with -d. + [[ -d "$entry" ]] || continue + base="$(basename "$entry")" + # Require a prefix of 4+ digits followed by a hyphen. + if [[ "$base" =~ ^([0-9]{4,})- ]]; then + num="${BASH_REMATCH[1]}" + # Force base 10 so leading zeros (08, 09) are not read as octal. + num=$((10#$num)) + if (( num > max )); then + max=$num + fi + fi + done + fi + + printf "%0*d\n" "$width" $(( max + 1 )) +} + +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + next_numbered_dir "$@" +fi diff --git a/bash/functions/pipelines/init_paper.md b/bash/functions/pipelines/init_paper.md new file mode 100644 index 00000000..e01e5b88 --- /dev/null +++ b/bash/functions/pipelines/init_paper.md @@ -0,0 +1,69 @@ +--- +name: init_paper +kind: pipeline +lang: bash +domain: pipelines +version: "1.0.0" +purity: impure +signature: "init_paper(slug: string, [--title <t>] [--domain <d>] [--tags <csv>]) -> void" +description: "Scaffold de un paper académico reproducible en papers/<NNNN-slug>/. Calcula el siguiente número incremental escaneando papers/, crea las subcarpetas (experiments data figures reviews out), copia las plantillas paper.md (IMRaD) + preregistration.md (anti-HARKing) rellenando el frontmatter (title, slug, date de hoy, phase=question, status=draft) y crea references.md. NO hace git init: el paper arranca en fase interna local (papers/ gitignored). Grupo de capacidad papers." +tags: [papers, scaffold, paper, pipeline, bash, launcher] +uses_functions: + - next_numbered_dir_bash_io + - slugify_ascii_py_core +uses_types: [] +returns: [] +returns_optional: false +error_type: "error_go_core" +imports: [] +params: + - name: slug + desc: "identificador legible del paper; se slugifica a ASCII (espacios/acentos se normalizan) y se prefija con el siguiente NNNN incremental" + - name: "--title" + desc: "título del paper (string); si se omite, usa el slug limpio. No debe contener el carácter '|'" + - name: "--domain" + desc: "dominio del paper escrito en el frontmatter (default datascience)" + - name: "--tags" + desc: "tags CSV que se escriben en el frontmatter de paper.md (opcional)" +output: "sin salida directa; crea papers/<NNNN-slug>/ con paper.md, preregistration.md, references.md y las subcarpetas experiments/ data/ figures/ reviews/ out/. Imprime el resumen y los pasos siguientes a stdout." +tested: false +tests: [] +test_file_path: "" +file_path: "bash/functions/pipelines/init_paper.sh" +--- + +## Ejemplo + +```bash +# Scaffold de un paper nuevo (numera 0001, 0002, ... automáticamente) +fn run init_paper mi-primer-paper --title "Mi primer paper" +fn run init_paper reactive-loop-calls --domain datascience --tags registry,telemetria + +# El slug se slugifica: "Áreas de Mejora" -> papers/0003-areas-de-mejora/ +fn run init_paper "Áreas de Mejora" +``` + +## Cuando usarla + +Cuando empiezas un paper académico nuevo dentro de `fn_registry` y necesitas el esqueleto del artefacto (`papers/<NNNN-slug>/`) con las plantillas IMRaD y de pre-registro listas para rellenar. Es el paso 1 del grupo de capacidad `papers` (ver `docs/capabilities/papers.md`), antes de la revisión de literatura y del pre-registro de la hipótesis. + +## Flujo + +1. Parsea `<slug>` (posicional) + flags `--title` / `--domain` / `--tags`. Falla con exit ≠ 0 si falta el slug. +2. `slugify_ascii` — normaliza el slug a ASCII lowercase sin diacríticos (reutiliza la función del registry, solo stdlib). +3. `next_numbered_dir papers/` — calcula el siguiente NNNN de 4 dígitos sin colisión. +4. Crea `papers/<NNNN-slug>/` con las subcarpetas `experiments/ data/ figures/ reviews/ out/`. +5. Copia `docs/templates/paper.md` + `docs/templates/preregistration.md` y rellena el frontmatter por clave de línea (title, slug, date de hoy, domain, tags; phase=question y status=draft vienen de la plantilla). +6. Crea `references.md` vacío. + +## Gotchas + +- **NO hace `git init`.** El paper arranca en fase interna local; `papers/` está gitignored en el repo padre (solo `papers/.gitkeep` se versiona). Promocionar a sub-repo Gitea (fase publishable) es manual. +- **El `--title` no debe contener el carácter `|`** (se usa como delimitador de sed al rellenar el frontmatter; los `&` y `\` sí se escapan). +- **No indexa el paper en `registry.db`** — los artefactos `papers/<slug>/` no se indexan en esta fase (KISS); sí se indexa este pipeline. +- Requiere `python3` (del venv del registry o del sistema) para slugificar; `slugify_ascii` solo usa stdlib, así que el venv no es obligatorio. +- Idempotencia: si el directorio destino ya existiera, aborta con exit ≠ 0 en vez de sobrescribir. + +## Notas + +Cada paper es un artefacto independiente (mismo patrón que `apps/` y `analysis/`, pero para investigación). El pipeline usa `set -euo pipefail`: cualquier fallo detiene la ejecución. Parte del grupo de capacidad `papers` — diseño completo en `reports/0001-2026-06-30-papers-system-design.md`. diff --git a/bash/functions/pipelines/init_paper.sh b/bash/functions/pipelines/init_paper.sh new file mode 100644 index 00000000..34e3fa2a --- /dev/null +++ b/bash/functions/pipelines/init_paper.sh @@ -0,0 +1,177 @@ +#!/usr/bin/env bash +# init_paper +# ---------- +# Scaffold de un paper académico reproducible en papers/<NNNN-slug>/. +# +# Calcula el siguiente número incremental escaneando papers/, crea el +# directorio con todas las subcarpetas (experiments data figures reviews out), +# copia las plantillas paper.md + preregistration.md rellenando el frontmatter +# (title, slug, date de hoy, phase=question, status=draft) y crea references.md. +# +# NO hace `git init`: el paper arranca en fase interna local (papers/ está +# gitignored en el repo padre, solo .gitkeep se versiona). La promoción a +# sub-repo Gitea (fase publishable) es un paso posterior MANUAL. +# +# Compone: next_numbered_dir (helper de numeración del registry) + +# slugify_ascii (slug ASCII del registry). +# +# USO: +# ./init_paper.sh <slug> [--title "..."] [--domain <d>] [--tags a,b,c] +# +# EJEMPLOS: +# ./init_paper.sh mi-primer-paper --title "Mi primer paper" +# ./init_paper.sh reactive-loop-calls --domain datascience --tags registry,telemetria + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REGISTRY_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)" + +# Funciones atómicas del registry +source "$REGISTRY_ROOT/bash/functions/io/next_numbered_dir.sh" + +# ── Parsing de argumentos ──────────────────────────────────── + +SLUG_RAW="" +TITLE="" +DOMAIN="datascience" +TAGS="" + +while [ $# -gt 0 ]; do + case "$1" in + --title) + TITLE="$2"; shift 2 ;; + --domain) + DOMAIN="$2"; shift 2 ;; + --tags) + TAGS="$2"; shift 2 ;; + -h|--help) + grep "^#" "$0" | sed 's/^# \?//' ; exit 0 ;; + -*) + echo "Flag desconocido: $1" >&2 ; exit 1 ;; + *) + if [ -z "$SLUG_RAW" ]; then + SLUG_RAW="$1" + else + echo "ERROR: argumento posicional inesperado: '$1' (solo se admite un <slug>)." >&2 + exit 1 + fi + shift ;; + esac +done + +if [ -z "$SLUG_RAW" ]; then + echo "ERROR: falta el argumento <slug>." >&2 + echo "Uso: $0 <slug> [--title \"...\"] [--domain <d>] [--tags a,b,c]" >&2 + echo " Ejemplo: $0 mi-primer-paper --title \"Mi primer paper\"" >&2 + exit 1 +fi + +# ── Slugificar (reutiliza slugify_ascii del registry; solo stdlib) ── + +PYBIN="$REGISTRY_ROOT/python/.venv/bin/python3" +[ -x "$PYBIN" ] || PYBIN="$(command -v python3 || true)" +if [ -z "$PYBIN" ]; then + echo "ERROR: no se encontró python3 para slugificar el slug." >&2 + exit 1 +fi + +SLUG_CLEAN=$("$PYBIN" -c ' +import sys, os +sys.path.insert(0, os.path.join(sys.argv[2], "python", "functions")) +from core.slugify_ascii import slugify_ascii +print(slugify_ascii(sys.argv[1], default="paper")) +' "$SLUG_RAW" "$REGISTRY_ROOT") + +# ── Resolver número incremental y directorio destino ───────── + +PAPERS_DIR="$REGISTRY_ROOT/papers" +mkdir -p "$PAPERS_DIR" + +NUM=$(next_numbered_dir "$PAPERS_DIR") +SLUG_FULL="${NUM}-${SLUG_CLEAN}" +PAPER_DIR="$PAPERS_DIR/$SLUG_FULL" + +if [ -d "$PAPER_DIR" ]; then + echo "ERROR: el directorio del paper ya existe: $PAPER_DIR" >&2 + exit 1 +fi + +TODAY=$(date +%Y-%m-%d) +[ -n "$TITLE" ] || TITLE="$SLUG_CLEAN" + +TAGS_YAML="[]" +if [ -n "$TAGS" ]; then + TAGS_YAML="[$(echo "$TAGS" | sed 's/,/, /g')]" +fi + +echo "" +echo "════════════════════════════════════════════════════════════" +echo " INIT PAPER: ${SLUG_FULL}" +echo " Título: ${TITLE}" +echo " Directorio: ${PAPER_DIR}" +echo "════════════════════════════════════════════════════════════" +echo "" + +# ── Crear estructura ───────────────────────────────────────── + +echo "[1/3] Creando estructura..." +mkdir -p "$PAPER_DIR"/experiments "$PAPER_DIR"/data "$PAPER_DIR"/figures \ + "$PAPER_DIR"/reviews "$PAPER_DIR"/out +echo " experiments/ data/ figures/ reviews/ out/" + +# ── Copiar plantillas + rellenar frontmatter ───────────────── + +echo "[2/3] Escribiendo paper.md + preregistration.md..." + +# Escapa caracteres especiales del RHS de sed (delimitador |) +sed_escape() { printf '%s' "$1" | sed -e 's/[\\&|]/\\&/g'; } +TITLE_ESC="$(sed_escape "$TITLE")" +DOMAIN_ESC="$(sed_escape "$DOMAIN")" + +PAPER_MD="$PAPER_DIR/paper.md" +PREREG_MD="$PAPER_DIR/preregistration.md" + +cp "$REGISTRY_ROOT/docs/templates/paper.md" "$PAPER_MD" +cp "$REGISTRY_ROOT/docs/templates/preregistration.md" "$PREREG_MD" + +sed -i \ + -e "s|^title:.*|title: \"${TITLE_ESC}\"|" \ + -e "s|^slug:.*|slug: ${SLUG_FULL}|" \ + -e "s|^date:.*|date: ${TODAY}|" \ + -e "s|^domain:.*|domain: ${DOMAIN_ESC}|" \ + -e "s|^tags:.*|tags: ${TAGS_YAML}|" \ + "$PAPER_MD" + +sed -i \ + -e "s|^paper_slug:.*|paper_slug: ${SLUG_FULL}|" \ + "$PREREG_MD" + +echo " $PAPER_MD" +echo " $PREREG_MD" + +# ── references.md ──────────────────────────────────────────── + +echo "[3/3] Escribiendo references.md..." +cat > "$PAPER_DIR/references.md" << EOF +# References — ${TITLE} + +<!-- Una entrada por referencia. Formato libre (o BibTeX) hasta promocionar a publishable. --> +EOF +echo " $PAPER_DIR/references.md" + +# ── Resumen ────────────────────────────────────────────────── + +echo "" +echo "════════════════════════════════════════════════════════════" +echo " PAPER '${SLUG_FULL}' LISTO (fase: question, status: draft)" +echo "════════════════════════════════════════════════════════════" +echo "" +echo " Pasos siguientes:" +echo " 1. Revisión de literatura (skill /deep-research) → Related work." +echo " 2. Pre-registro: congela H0/H1 + plan en preregistration.md (preregister_hypothesis)." +echo " 3. Experimentos en experiments/ → análisis (grupo eda) → escritura IMRaD en paper.md." +echo " 4. render_paper_pdf → out/paper.pdf. Peer review adversarial → reviews/." +echo "" +echo " papers/ está gitignored: este paper vive local hasta promocionar a publishable." +echo "" diff --git a/docs/capabilities/INDEX.md b/docs/capabilities/INDEX.md index dbea6af4..7dce2b69 100644 --- a/docs/capabilities/INDEX.md +++ b/docs/capabilities/INDEX.md @@ -39,6 +39,7 @@ Indice de grupos de capacidades del registry. Cada grupo agrupa >=3 funciones qu | [cpp-tables](tql.md) | 9 | Table Query Language C++ puro: filter, group, agg, sort, join, stats, formulas Lua, round-trip emit/apply | | [data-table-renderers](data_table_renderers.md) | 1 | API declarativa de cell renderers para data_table: Badge, Progress, Duration, Icon via TableInput.column_specs | | [scheduler](scheduler.md) | 4 | Cron expression parsing, matching, next-run y traduccion humana (consume `apps/dag_engine`) | +| [papers](papers.md) | — | Papers académicos reproducibles en `papers/<NNNN-slug>/`: scaffold del artefacto (`init_paper` + helper `next_numbered_dir`), plantillas IMRaD + pre-registro anti-HARKing, y (en construcción por la flota) congelar hipótesis, funciones estadísticas (effect size/CI/corrección múltiple), render md→PDF y peer-review adversarial. Reutiliza `deep-research`, grupo `eda` y el motor PDF de `datascience`. Diseño: `reports/0001-2026-06-30-papers-system-design.md` | | [extractor](extractor.md) | 15 | Funciones que leen datos de fuentes externas (BD, API, archivos, web). Nodos input de `data_factory` | | [transformer](transformer.md) | 15 | Funciones que clean/dedup/aggregate/feature-engineer datos. Nodos intermedios de `data_factory` | | [sink](sink.md) | 11 | Funciones que escriben datos a destino externo (BD, dashboard, alerta, email). Nodos output | diff --git a/docs/capabilities/papers.md b/docs/capabilities/papers.md new file mode 100644 index 00000000..b5890036 --- /dev/null +++ b/docs/capabilities/papers.md @@ -0,0 +1,82 @@ +# papers — papers académicos reproducibles + +Grupo de capacidad para producir **papers académicos** dentro de `fn_registry`: investigación con hipótesis falsables, experimentos reproducibles, análisis estadístico honesto y escritura en formato IMRaD. Cada paper es un artefacto nuevo en `papers/<NNNN-slug>/` que reutiliza infraestructura existente (skill `deep-research` para la revisión de literatura, grupo `eda` para el análisis, motor md→PDF de `datascience`, patrón de verificación adversarial del orquestador) y añade lo que falta como funciones del registry. + +Diseño completo y decisiones: `reports/0001-2026-06-30-papers-system-design.md`. + +> **Regla de oro anti paper-mill:** una hipótesis que **podía** fallar + un experimento con riesgo real de refutación + estadística que no es teatro. Si no hay riesgo de refutación, no es un paper. Los claims nunca superan a la evidencia. El antídoto al HARKing es el **pre-registro**: el plan de análisis se congela *antes* de mirar los datos. + +## Estructura del artefacto + +``` +papers/0001-mi-paper/ + paper.md # frontmatter (title, slug, authors, date, status, phase, tags, domain, hypothesis_id) + cuerpo IMRaD + preregistration.md # H0/H1 + plan de análisis CONGELADO (frozen_at + content_hash) antes de correr + references.md # bibliografía + experiments/ # código / notebooks por experimento (exp01_*, exp02_*) + data/ # crudos + procesados (gitignored si pesa) + figures/ # gráficos generados + reviews/ # outputs del peer-review adversarial + out/ # paper.pdf — entregable final + .git/ # SOLO cuando promociona a fase publishable (sub-repo Gitea) +``` + +`papers/` está gitignored en el repo padre (solo `papers/.gitkeep` se versiona): un paper en fase interna no contamina el repo. Al promocionar a `status: publishable` se vuelve sub-repo Gitea `dataforge/<slug>` (como apps y analyses). + +### Fases (campo `phase` de `paper.md`) + +``` +question → review → hypothesis → design → running → analysis → writing → internal-review + → [DONE interno] → polish → submitted [solo en fase publishable] +``` + +## Funciones + +| ID | Pureza | Estado | Qué hace | +|---|---|---|---| +| `init_paper_bash_pipelines` | impure | ✅ disponible | Scaffold de `papers/<NNNN-slug>/`: calcula el siguiente NNNN, crea las subcarpetas, copia `paper.md` + `preregistration.md` con el frontmatter relleno (slug, title, date de hoy, `phase: question`, `status: draft`) y `references.md` vacío. NO hace `git init` (el paper arranca en fase interna local). | +| `next_numbered_dir_bash_io` | impure | ✅ disponible | Dado un directorio, devuelve el siguiente número incremental de 4 dígitos (`0001`, `0002`, …) escaneando los subdirs con prefijo `NNNN-`. Helper de numeración de `init_paper` (reutilizable por reports/issues). | +| `preregister_hypothesis` | impure | 🚧 en construcción (flota) | Congela el `preregistration.md` (H0/H1 + plan de análisis) con `frozen_at` + `content_hash`, pasa `status` a `frozen` y escribe `hypothesis_id` en `paper.md`. Mata el HARKing: tras congelar, el plan no se edita. | +| `cohens_d` (effect size) | pure | 🚧 en construcción (flota) | Tamaño del efecto (Cohen's d) entre dos grupos. Reporta magnitud, no solo significancia. | +| `confidence_interval` | pure | 🚧 en construcción (flota) | Intervalo de confianza de una métrica (media/diferencia). | +| `holm_bonferroni` | pure | 🚧 en construcción (flota) | Corrección de comparaciones múltiples (Holm-Bonferroni / FWER) para el plan de análisis. | +| `render_paper_pdf` | impure | 🚧 en construcción (flota) | Markdown IMRaD (`paper.md` + figuras) → `out/paper.pdf`, reutilizando el motor md→PDF del grupo `eda`/`datascience`. | + +> Las funciones estadísticas reutilizan lo que ya exista en `datascience` (p.ej. `fdr_correction_py_datascience` cubre la corrección de comparaciones múltiples por FDR; el agente del rigor experimental decide si añade Holm-Bonferroni o reusa lo existente). Buscar antes de duplicar: `mcp__registry__fn_search query="effect size" domain="datascience"`. + +### Peer review (no es función del registry) + +El agente adversarial `.claude/agents/paper-reviewer.md` (🚧 en construcción por la flota) puntúa novedad, rigor, reproducibilidad y validez, e intenta **refutar** cada claim. Default a "failed" si la evidencia no soporta. Escribe su veredicto en `reviews/`. Es el equivalente al verificador adversarial del orquestador aplicado al paper. + +## Ejemplo canónico (end-to-end) + +```bash +# 1. Scaffold del paper (fase question, local). Crea papers/0001-mi-paper/. +./fn run init_paper mi-paper --title "¿El bucle reactivo reduce las calls inline?" --domain datascience --tags registry,telemetria + +# 2. Revisión de literatura → llena Related work (skill deep-research, fase review). +# /deep-research "..." + +# 3. Pre-registro: congela H0/H1 + plan de análisis ANTES de mirar datos (fase hypothesis). +./fn run preregister_hypothesis papers/0001-mi-paper # 🚧 en construcción + +# 4. Experimentos en papers/0001-mi-paper/experiments/ (fase running) → +# análisis con el grupo `eda` + funciones de effect size / CI / corrección múltiple (fase analysis). + +# 5. Escritura IMRaD en paper.md (fase writing) → render del entregable PDF. +./fn run render_paper_pdf papers/0001-mi-paper # 🚧 en construcción → out/paper.pdf + +# 6. Peer review adversarial (fase internal-review). +# Agent(subagent_type="paper-reviewer", prompt="Revisa papers/0001-mi-paper ...") # 🚧 en construcción +``` + +## Fronteras + +- **NO es para reports de trabajo.** Un report (`reports/`) es el entregable escrito de una tarea (resumen + evidencia + gaps); un paper es investigación con hipótesis falsable y experimento. Ver `.claude/rules/reports.md`. +- **NO se indexa en `registry.db` en esta fase.** No hay tabla `papers` ni `entity_type` `paper` (KISS); se añadiría con migración propia si se decide. Las *funciones* del grupo sí se indexan (viven en `bash/functions/`, `python/functions/`), pero los artefactos `papers/<slug>/` no. +- **NO hace `git init` en el scaffold.** El paper arranca en fase interna local y gitignored. La promoción a sub-repo Gitea (fase publishable) es un paso manual posterior. +- **NO soporta LaTeX/arXiv todavía.** Formato elegido: Markdown como fuente + PDF como entregable. El soporte LaTeX se añadiría al promocionar un paper a fase publishable. + +## Estado + +Fase de scaffolding. Disponible: estructura del artefacto, plantillas (`docs/templates/paper.md`, `docs/templates/preregistration.md`), pipeline `init_paper` + helper `next_numbered_dir`, esta página y el bloque gitignore de `papers/`. En construcción por la flota: `preregister_hypothesis`, funciones estadísticas (effect size / CI / corrección múltiple), `render_paper_pdf` y el agente `paper-reviewer`. Validación end-to-end con un paper piloto real: pendiente. diff --git a/docs/templates/paper.md b/docs/templates/paper.md new file mode 100644 index 00000000..78dffe23 --- /dev/null +++ b/docs/templates/paper.md @@ -0,0 +1,94 @@ +--- +title: "TITULO DEL PAPER" +slug: NNNN-slug +authors: [Enmanuel] +date: 2026-01-01 +status: draft # draft | internal | publishable +phase: question # question -> review -> hypothesis -> design -> running -> analysis -> writing -> internal-review -> polish -> submitted +tags: [] +domain: datascience +hypothesis_id: "" # lo rellena preregister_hypothesis al congelar el preregistro +--- + +<!-- +Paper académico reproducible (formato IMRaD). Esta es la FUENTE editable en Markdown; +el entregable PDF se genera con render_paper_pdf (grupo `papers`). + +Regla de oro anti paper-mill: una hipótesis que PODÍA fallar + un experimento con +riesgo real de refutación + estadística que no es teatro. Si no hay riesgo de +refutación, no es un paper. Los claims nunca superan a la evidencia. +--> + +# {{título del paper}} + +## Abstract + +<!-- +Resumen estructurado en 4-6 frases: contexto -> gap -> método -> resultados -> conclusión. +Sin citas, sin abreviaturas sin definir. Es lo único que mucha gente leerá: que se sostenga solo. +--> + +## 1. Introduction + +<!-- +Embudo en cuatro movimientos: +1. Contexto — el área y por qué importa. +2. Gap — qué NO se sabe todavía (el hueco que este paper llena). +3. Pregunta / hipótesis — formulada de forma falsable (ver preregistration.md). +4. Contribución — lista explícita de lo que aporta este trabajo ("Contributions:"). +--> + +## 2. Related work + +<!-- +Qué existe ya y por qué no basta. Agrupa por enfoque, no por autor. Cada cita debe +justificar por qué el gap sigue abierto. Output de la fase de revisión (skill deep-research). +--> + +## 3. Methods + +<!-- +Diseño REPRODUCIBLE: otra persona lo corre y obtiene lo mismo. +- Variables: independiente(s), dependiente(s), control. +- Diseño: N, condiciones, muestreo, aleatorización. +- Métricas y cómo se miden. +- Protocolo paso a paso + dónde vive el código (experiments/) y los datos (data/). +Debe ser coherente con el preregistration.md congelado (no se cambia el plan tras ver datos). +--> + +## 4. Results + +<!-- +Datos SIN interpretar. Tablas y figuras (figures/) con su lectura literal. +Reporta effect size + intervalos de confianza, no solo p-valores. +Incluye también los resultados negativos / no significativos (anti cherry-picking). +--> + +## 5. Discussion + +<!-- +Interpretación de los resultados a la luz de la pregunta. Claims <= evidencia. +--> + +### 5.1 Limitaciones + +<!-- Qué no cubre el estudio, supuestos, datos faltantes. Honestidad explícita. --> + +### 5.2 Amenazas a la validez + +<!-- +- Validez interna — ¿la causa es lo que decimos o hay confusores? +- Validez externa — ¿generaliza fuera de esta muestra/condiciones? +- Validez de constructo — ¿la métrica mide lo que dice medir? +- Validez estadística — ¿N suficiente, supuestos del test cumplidos, comparaciones múltiples corregidas? +--> + +## 6. Conclusion + Future work + +<!-- +Cierre en 2-4 frases: qué se aprendió (sin overclaiming) + las siguientes preguntas que abre. +--> + +## References + +<!-- Ver references.md. --> diff --git a/docs/templates/preregistration.md b/docs/templates/preregistration.md new file mode 100644 index 00000000..a5de0e2a --- /dev/null +++ b/docs/templates/preregistration.md @@ -0,0 +1,59 @@ +--- +paper_slug: NNNN-slug +frozen_at: "" # timestamp ISO — lo rellena preregister_hypothesis al congelar +content_hash: "" # hash del contenido congelado — lo rellena preregister_hypothesis +status: draft # draft -> frozen (preregister_hypothesis lo pasa a frozen; tras congelar NO se edita) +--- + +> **⚠️ ESTE DOCUMENTO SE CONGELA ANTES DE MIRAR LOS DATOS (anti-HARKing).** +> El plan de análisis se fija aquí *antes* de ejecutar el experimento. Una vez congelado +> (`status: frozen`, con `frozen_at` + `content_hash`), **no se edita**. Inventar o ajustar +> la hipótesis después de ver los resultados (HARKing) invalida el paper. Si el plan cambia +> tras ver datos, eso es análisis exploratorio y se reporta como tal, no como confirmatorio. + +# Pre-registro — {{título del paper}} + +## 1. Pregunta de investigación + +<!-- La pregunta concreta, en una frase. Debe poder responderse con un experimento. --> + +## 2. Hipótesis + +<!-- Falsable (Popper): una predicción que PODRÍA fallar. --> + +- **H0 (nula):** <!-- no hay efecto / no hay diferencia. Es lo que el test intenta rechazar. --> +- **H1 (alternativa):** <!-- el efecto esperado, con dirección si la hay. --> + +## 3. Variables + +- **Independiente(s):** <!-- lo que se manipula. --> +- **Dependiente(s):** <!-- lo que se mide (la métrica de resultado). --> +- **Control:** <!-- lo que se mantiene fijo / se cubre estadísticamente. --> + +## 4. Diseño + +<!-- +- N: tamaño de muestra (y justificación / power analysis si aplica). +- Condiciones / grupos. +- Muestreo y aleatorización. +- Criterios de inclusión / exclusión de datos (definidos AHORA, no después). +--> + +## 5. Plan de análisis + +<!-- +El plan estadístico EXACTO, decidido antes de ver los datos: +- Test estadístico concreto (p.ej. t-test de Welch, Mann-Whitney U, regresión...). +- Métrica de effect size (p.ej. Cohen's d, diferencia de medias, odds ratio). +- Criterio de decisión (umbral alpha, qué resultado confirma/refuta H1). +- Corrección por comparaciones múltiples (p.ej. Holm-Bonferroni) si hay >1 contraste. +- Manejo de supuestos (normalidad, varianzas) y qué se hace si no se cumplen. +--> + +## 6. Predicción cuantitativa + +<!-- +La predicción numérica concreta que el experimento pondrá a prueba. +P.ej. "esperamos d >= 0.5 con IC95% que no cruza 0" o "una reducción >= 15% en la métrica X". +Cuanto más específica, más falsable. +--> diff --git a/papers/.gitkeep b/papers/.gitkeep new file mode 100644 index 00000000..e69de29b From 7fa19d65db4b7cc385309e25022478cd3148eebc Mon Sep 17 00:00:00 2001 From: Egutierrez <egutierrez@dead.dd> Date: Tue, 30 Jun 2026 20:38:39 +0200 Subject: [PATCH 40/53] =?UTF-8?q?feat(eda):=20cap=C3=ADtulo=20MISSINGNESS?= =?UTF-8?q?=20=E2=80=94=20patrones=20de=20datos=20faltantes=20(co-ocurrenc?= =?UTF-8?q?ia=20+=20MCAR/MAR)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Añade el capítulo `missingness` al motor AutomaticEDA, complemento natural de `calidad`: donde calidad reporta cuánto falta por columna, este capítulo analiza el PATRÓN de los nulos — dónde faltan y si las columnas faltan juntas (co-ocurrencia de ausencias), la señal que distingue MCAR de MAR antes de imputar. Capítulo (`chapters/missingness.py`), registrado en `chapters_registry.py` justo tras `calidad`: - Resumen global: % de celdas faltantes, columnas con nulos, filas completas vs incompletas. - Ranking por columna (tabla + barras horizontales). - Co-ocurrencia: correlación de las máscaras is-null entre columnas (heatmap + tabla de los pares que co-faltan, con co-faltantes y Jaccard). - Patrones de fila más frecuentes (estilo matriz de missingno). - Lectura MCAR/MAR exploratoria (heurística por correlación/solape de ausencias, no confirmatoria), que cita la evidencia concreta. - Términos de glosario clicables: missingness, MCAR, MAR. La máscara is-null por fila de TODAS las columnas (numéricas y categóricas) se construye con un push-down DuckDB sobre ctx['db_path']/table (mismo patrón que el capítulo agregación), con fallback a ctx['raw_numeric'] cuando no hay BD. Activa solo si la tabla tiene nulos; si no, devuelve None. Funciones nuevas del grupo `eda` (dominio datascience): - extract_null_mask (impura): máscara is-null por fila vía query_fn. - missingness_overview (pura): resumen global + filas completas/incompletas. - missingness_correlation (pura): correlación de ausencias + pares + Jaccard, reutiliza pearson. - missingness_row_patterns (pura): patrones de fila más comunes. - missingness_corr_heatmap_figure / missingness_rank_bar_figure (impuras): figuras. Verificado: EDA de titanic genera el capítulo en PDF + PPTX + MD con Cabin 77.1%, Age 19.9% y la co-ocurrencia Age↔Cabin (158 filas). Suite completa de AutomaticEDA + render_automatic_eda en verde (125 passed); tests por función y por capítulo; fn index sin error. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> --- .../automatic_eda/chapters/missingness.py | 594 ++++++++++++++++++ .../chapters/missingness_test.py | 162 +++++ .../automatic_eda/chapters_registry.py | 1 + .../datascience/extract_null_mask.md | 97 +++ .../datascience/extract_null_mask.py | 101 +++ .../datascience/extract_null_mask_test.py | 116 ++++ .../missingness_corr_heatmap_figure.md | 103 +++ .../missingness_corr_heatmap_figure.py | 158 +++++ .../missingness_corr_heatmap_figure_test.py | 62 ++ .../datascience/missingness_correlation.md | 68 ++ .../datascience/missingness_correlation.py | 120 ++++ .../missingness_correlation_test.py | 115 ++++ .../datascience/missingness_overview.md | 99 +++ .../datascience/missingness_overview.py | 116 ++++ .../datascience/missingness_overview_test.py | 146 +++++ .../missingness_rank_bar_figure.md | 93 +++ .../missingness_rank_bar_figure.py | 150 +++++ .../missingness_rank_bar_figure_test.py | 64 ++ .../datascience/missingness_row_patterns.md | 65 ++ .../datascience/missingness_row_patterns.py | 107 ++++ .../missingness_row_patterns_test.py | 87 +++ 21 files changed, 2624 insertions(+) create mode 100644 python/functions/datascience/automatic_eda/chapters/missingness.py create mode 100644 python/functions/datascience/automatic_eda/chapters/missingness_test.py create mode 100644 python/functions/datascience/extract_null_mask.md create mode 100644 python/functions/datascience/extract_null_mask.py create mode 100644 python/functions/datascience/extract_null_mask_test.py create mode 100644 python/functions/datascience/missingness_corr_heatmap_figure.md create mode 100644 python/functions/datascience/missingness_corr_heatmap_figure.py create mode 100644 python/functions/datascience/missingness_corr_heatmap_figure_test.py create mode 100644 python/functions/datascience/missingness_correlation.md create mode 100644 python/functions/datascience/missingness_correlation.py create mode 100644 python/functions/datascience/missingness_correlation_test.py create mode 100644 python/functions/datascience/missingness_overview.md create mode 100644 python/functions/datascience/missingness_overview.py create mode 100644 python/functions/datascience/missingness_overview_test.py create mode 100644 python/functions/datascience/missingness_rank_bar_figure.md create mode 100644 python/functions/datascience/missingness_rank_bar_figure.py create mode 100644 python/functions/datascience/missingness_rank_bar_figure_test.py create mode 100644 python/functions/datascience/missingness_row_patterns.md create mode 100644 python/functions/datascience/missingness_row_patterns.py create mode 100644 python/functions/datascience/missingness_row_patterns_test.py diff --git a/python/functions/datascience/automatic_eda/chapters/missingness.py b/python/functions/datascience/automatic_eda/chapters/missingness.py new file mode 100644 index 00000000..3a7034cd --- /dev/null +++ b/python/functions/datascience/automatic_eda/chapters/missingness.py @@ -0,0 +1,594 @@ +"""Missingness chapter (MISSINGNESS) — patterns of missing data. + +Complements the CALIDAD chapter: where CALIDAD reports *how much* is missing per +column (the null percentage that lowers the completeness score), this chapter +reports the **pattern** of the missing data — whether columns tend to be missing +*together* (co-occurrence of absences) or independently. That distinction is what +separates data that is missing completely at random ([[term:mcar]]MCAR[[/term]]) +from data missing as a function of another variable ([[term:mar]]MAR[[/term]]), +which is the key question to settle before imputing or modelling. + +The chapter activates only when the table actually has missing data (at least one +column with a null in the aggregated profile); otherwise it returns ``None`` and +disappears from the document. + +Sections, in order: + +1. **Resumen global** — % of missing cells in the dataset, number of columns with + nulls, and complete rows (no missing) vs incomplete rows (≥1 missing). +2. **Ranking por columna** — columns sorted by their null percentage, with a + horizontal bar figure. +3. **Co-ocurrencia de ausencias** — the correlation of the binary is-null masks + between columns (which columns tend to be missing together): a heatmap plus a + table of the top column pairs that co-miss. +4. **Patrones de fila** — the most frequent "which columns are missing together" + row patterns, in the style of missingno's pattern matrix. +5. **Lectura MCAR/MAR** — an interpretive, *exploratory* note (not a confirmatory + test such as Little's) reading the absence correlations as a hint of MCAR + (independent absences) vs MAR (co-occurring absences). + +The aggregate per-column null counts come from the ``eda`` group ``TableProfile`` +(``columns[i]['null_count'] / 'null_pct'`` and the table-level ``null_cell_pct``). +The per-row is-null mask needed for co-occurrence is built from raw data: a single +DuckDB push-down over ``ctx['db_path'] / ctx['table']`` (same pattern as the +AGREGACION chapter) covering ALL columns, with a fallback to the numeric-only +``ctx['raw_numeric']`` when no database is reachable. All the heavy lifting is +delegated to pure registry functions (``missingness_overview``, +``missingness_correlation``, ``missingness_row_patterns``) and two figure helpers +(``missingness_rank_bar_figure``, ``missingness_corr_heatmap_figure``); every one +is imported lazily and degrades to an honest note so this chapter never raises. + +Contract: build_<id>(profile, ctx) -> Chapter | None ; CHAPTER_VERSION = "x.y.z". +""" + +from __future__ import annotations + +from .. import model + +CHAPTER_VERSION = "1.0.0" +CHAPTER_ID = "missingness" +CHAPTER_TITLE = "Datos faltantes" + +# Sample cap for the per-row is-null mask push-down. Co-occurrence and row +# patterns are computed on this sample; the global % of missing cells and the +# per-column ranking come from the (exact) aggregated profile instead. +MASK_SAMPLE = 5000 +# Thresholds for the MCAR/MAR heuristic note. A pair counts as a *strong* +# co-occurrence when the absence correlation alone is high; as a *partial* +# co-occurrence when the absences overlap materially (high Jaccard) even if the +# Pearson correlation is modest — the usual case when one column is missing far +# more often than the other (e.g. Cabin 77% vs Age 20% in Titanic), which dilutes +# the correlation while the rows still co-miss in absolute terms. +_CORR_STRONG = 0.30 +_JACCARD_NOTABLE = 0.20 +# Rows shown in the top-pairs and row-patterns tables (bounded, never silently +# truncated: the table note reports the full count). +_TOP_PAIRS = 12 +_TOP_PATTERNS = 12 +# Truncate long column names in tables (the renderer also wraps). +_LABEL_MAX = 28 + +# Glossary terms this chapter explains (contract §11.1). Registered in the shared +# collector and marked clickable on their first appearance. +_TERMS = { + "missingness": ( + "Patrón de datos faltantes (missingness)", + "El patrón con el que faltan los datos: cuánto falta, en qué columnas y " + "si las ausencias de unas columnas coinciden (co-ocurren) con las de " + "otras. Analizarlo —no solo contar nulos— distingue datos que faltan al " + "azar (MCAR) de los que faltan en función de otra variable (MAR), lo que " + "decide cómo imputar o si descartar filas sin sesgar el análisis.", + ), + "mcar": ( + "MCAR (Missing Completely At Random)", + "Los valores faltan de forma independiente de cualquier dato, observado o " + "no: las ausencias de unas columnas no se relacionan entre sí ni con los " + "valores. Es el caso más benigno —descartar filas o imputar la media no " + "introduce sesgo—, pero rara vez se cumple del todo en datos reales.", + ), + "mar": ( + "MAR (Missing At Random)", + "La probabilidad de que un valor falte depende de OTRAS variables " + "observadas (p. ej. una medición que falta más en cierto grupo). Las " + "ausencias co-ocurren entre columnas o se relacionan con los valores de " + "otras; imputar exige condicionar en esas variables para no sesgar. La " + "co-ocurrencia fuerte de ausencias es un indicio (exploratorio) de MAR.", + ), +} + + +# --------------------------------------------------------------------------- # +# Small defensive formatters (own copy: the chapter never imports siblings). +# --------------------------------------------------------------------------- # +def _fmt_int(value) -> str: + if value is None: + return "—" + try: + return f"{int(round(float(value))):,}".replace(",", ".") + except (TypeError, ValueError): + return model._safe_str(value) + + +def _fmt_pct(value, decimals: int = 1) -> str: + """Format an already-0-100 value as a percentage. None -> placeholder.""" + if value is None: + return "—" + try: + return f"{float(value):.{decimals}f}%" + except (TypeError, ValueError): + return model._safe_str(value) + + +def _fmt_num(value, decimals: int = 3) -> str: + if value is None: + return "—" + try: + f = float(value) + except (TypeError, ValueError): + return model._safe_str(value) + if f != f: # NaN + return "—" + text = f"{f:.{decimals}f}".rstrip("0").rstrip(".") + return text if text else "0" + + +def _truncate(text, limit: int = _LABEL_MAX) -> str: + s = model._safe_str(text) + if len(s) <= limit: + return s + return s[: max(1, limit - 1)].rstrip() + "…" + + +def _term(key: str, label: str, mark: bool) -> str: + if mark: + return f"[[term:{key}]]**{label}**[[/term]]" + return f"**{label}**" + + +# --------------------------------------------------------------------------- # +# Profile reads (exact, all rows). +# --------------------------------------------------------------------------- # +def _null_count_of(col: dict): + """Best-effort null count of a column: ``null_count`` or null_pct*n_rows.""" + nc = col.get("null_count") + if isinstance(nc, (int, float)) and not isinstance(nc, bool): + return int(nc) + np_ = col.get("null_pct") + nr = col.get("n_rows") + if isinstance(np_, (int, float)) and isinstance(nr, (int, float)): + return int(round(float(np_) * float(nr))) + return 0 + + +def _columns_with_nulls(profile: dict): + """Return ``[(name, null_count, null_pct_0_100)]`` for columns with nulls, + sorted by null percentage descending. Reads the aggregated profile (exact).""" + cols = profile.get("columns") or [] + out = [] + for c in cols: + if not isinstance(c, dict): + continue + nc = _null_count_of(c) + if nc <= 0: + continue + np_ = c.get("null_pct") + nr = c.get("n_rows") or profile.get("n_rows") + if isinstance(np_, (int, float)) and not isinstance(np_, bool): + pct = float(np_) * 100.0 if np_ <= 1.0 else float(np_) + elif nr: + pct = nc / float(nr) * 100.0 + else: + pct = None + out.append((c.get("name") or "(col)", nc, pct)) + out.sort(key=lambda t: (t[2] if t[2] is not None else -1.0), reverse=True) + return out + + +def _global_missing_pct(profile: dict): + """Table-level % of missing cells (0-100), exact, from the profile.""" + v = profile.get("null_cell_pct") + if isinstance(v, (int, float)) and not isinstance(v, bool): + return float(v) * 100.0 if v <= 1.0 else float(v) + return None + + +# --------------------------------------------------------------------------- # +# Per-row is-null mask (sample): DuckDB push-down, fallback to raw_numeric. +# --------------------------------------------------------------------------- # +def _build_query_fn(ctx: dict): + """Return ``(query_fn, table)`` for a DuckDB-backed ctx, or ``(None, None)``. + + Mirrors build_eda_render_ctx: a read-only closure over the registry wrapper. + Only DuckDB is supported here; any other backend degrades to raw_numeric.""" + db_path = ctx.get("db_path") + table = ctx.get("table") + if not db_path or not table: + return None, None + try: + from infra import duckdb_query_readonly + except Exception: # noqa: BLE001 — wrapper unavailable -> degrade. + return None, None + + def query_fn(sql): + return duckdb_query_readonly(db_path, sql) + + return query_fn, table + + +def _null_mask(profile: dict, ctx: dict): + """Build the per-row is-null mask ``{col: [0/1, ...]}``. + + Tries a single DuckDB push-down over ALL columns first (so categorical + columns like Cabin are covered, not only numeric ones); falls back to the + numeric-only ``ctx['raw_numeric']`` (None -> missing); returns ``(None, 0, + None)`` when neither is reachable. Never raises. + Returns ``(mask, n_sampled, source)`` with source in {"db","raw_numeric"}. + """ + cols = profile.get("columns") or [] + names = [c.get("name") for c in cols + if isinstance(c, dict) and c.get("name")] + # 1) DuckDB push-down over every column (covers categoricals too). + query_fn, table = _build_query_fn(ctx) + if query_fn is not None and names: + try: + from datascience.extract_null_mask import extract_null_mask + + res = extract_null_mask(query_fn, table, names, max_rows=MASK_SAMPLE) + if isinstance(res, dict) and res.get("status") == "ok": + mask = res.get("mask") or {} + if mask: + return mask, int(res.get("n") or 0), "db" + except Exception: # noqa: BLE001 — degrade to raw_numeric. + pass + # 2) Fallback: numeric-only mask derived from raw_numeric (None -> missing). + rn = ctx.get("raw_numeric") + if isinstance(rn, dict) and rn: + mask = {} + for col, vals in rn.items(): + if isinstance(vals, (list, tuple)): + mask[col] = [1 if v is None else 0 for v in vals] + if mask: + n = max((len(v) for v in mask.values()), default=0) + return mask, n, "raw_numeric" + return None, 0, None + + +# --------------------------------------------------------------------------- # +# Lazy registry delegations (each degrades to None on any failure). +# --------------------------------------------------------------------------- # +def _overview(mask: dict): + try: + from datascience.missingness_overview import missingness_overview + + out = missingness_overview(mask) + return out if isinstance(out, dict) else None + except Exception: # noqa: BLE001 + return None + + +def _correlation(mask: dict, top_k: int): + try: + from datascience.missingness_correlation import missingness_correlation + + out = missingness_correlation(mask, top_k=top_k) + return out if isinstance(out, dict) else None + except Exception: # noqa: BLE001 + return None + + +def _row_patterns(mask: dict, top_n: int): + try: + from datascience.missingness_row_patterns import missingness_row_patterns + + out = missingness_row_patterns(mask, top_n=top_n) + return out if isinstance(out, dict) else None + except Exception: # noqa: BLE001 + return None + + +def _rank_bar_make(names, pcts, title): + def make(): + try: + from datascience.missingness_rank_bar_figure import ( + missingness_rank_bar_figure, + ) + + return missingness_rank_bar_figure(names, pcts, title=title) + except Exception: # noqa: BLE001 — minimal fallback figure. + return _fallback_fig("ranking de nulos no disponible") + + return make + + +def _heatmap_make(matrix, labels, title): + def make(): + try: + from datascience.missingness_corr_heatmap_figure import ( + missingness_corr_heatmap_figure, + ) + + return missingness_corr_heatmap_figure(matrix, labels, title=title) + except Exception: # noqa: BLE001 — minimal fallback figure. + return _fallback_fig("heatmap de co-ocurrencia no disponible") + + return make + + +def _fallback_fig(message: str): + import matplotlib + + matplotlib.use("Agg") + from matplotlib.figure import Figure + + fig = Figure(figsize=(5.0, 2.2)) + ax = fig.add_subplot(111) + ax.text(0.5, 0.5, message, ha="center", va="center") + ax.axis("off") + return fig + + +# --------------------------------------------------------------------------- # +# Block builders. +# --------------------------------------------------------------------------- # +def _summary_block(profile: dict, with_nulls: list, overview, sampled, n_total): + rows = [] + gpct = _global_missing_pct(profile) + rows.append(("Celdas faltantes (global)", _fmt_pct(gpct))) + rows.append(("Columnas con faltantes", str(len(with_nulls)))) + all_null = profile.get("all_null_cols") + if isinstance(all_null, (list, tuple)) and all_null: + rows.append(("Columnas 100% faltantes", str(len(all_null)))) + if isinstance(overview, dict): + cr = overview.get("complete_rows") + ir = overview.get("incomplete_rows") + suffix = "" + if (isinstance(sampled, int) and isinstance(n_total, (int, float)) + and sampled and n_total and sampled < n_total): + suffix = f" (sobre muestra de {_fmt_int(sampled)} filas)" + if cr is not None: + rows.append(("Filas completas (sin faltantes)", + f"{_fmt_int(cr)} ({_fmt_pct(overview.get('complete_pct'))})" + + suffix)) + if ir is not None: + rows.append(("Filas con ≥1 faltante", + f"{_fmt_int(ir)} " + f"({_fmt_pct(overview.get('incomplete_pct'))})" + suffix)) + return model.KVTable(rows=rows, title="Resumen de datos faltantes") + + +def _ranking_block(with_nulls: list): + header = ["Columna", "Faltantes", "% faltante"] + rows = [[_truncate(n), _fmt_int(c), _fmt_pct(p)] for (n, c, p) in with_nulls] + if not rows: + return None + return model.DataTable( + header=header, rows=rows, title="Faltantes por columna", + note="ordenado de más a menos faltante") + + +def _ranking_figure(with_nulls: list): + names = [n for (n, _, p) in with_nulls if p is not None] + pcts = [p for (_, _, p) in with_nulls if p is not None] + if not names: + return None + return model.Figure( + make=_rank_bar_make(names, pcts, "% de valores faltantes por columna"), + caption="Porcentaje de valores faltantes por columna (barras).") + + +def _pairs_block(corr: dict): + """Top column pairs whose absences co-occur, as a table, or None.""" + pairs = (corr or {}).get("pairs") or [] + header = ["Columna A", "Columna B", "Corr. ausencia", "Co-faltan", "Jaccard"] + rows = [] + for p in pairs[:_TOP_PAIRS]: + if not isinstance(p, dict): + continue + rows.append([ + _truncate(p.get("a")), + _truncate(p.get("b")), + _fmt_num(p.get("corr")), + _fmt_int(p.get("co_missing")), + _fmt_num(p.get("jaccard")), + ]) + if not rows: + return None + shown = len(rows) + total = len(pairs) + note = ("correlación de las máscaras is-null entre columnas; " + "«Co-faltan» = nº de filas en que ambas faltan a la vez") + if total > shown: + note += f" — top {shown} de {total} pares" + return model.DataTable(header=header, rows=rows, + title="Pares de columnas que co-faltan", note=note) + + +def _heatmap_block(corr: dict): + cols = (corr or {}).get("columns") or [] + matrix = (corr or {}).get("matrix") or [] + if len(cols) < 2 or not matrix: + return None + labels = [_truncate(c, 16) for c in cols] + return model.Figure( + make=_heatmap_make(matrix, labels, "Co-ocurrencia de ausencias"), + caption=("Correlación de las ausencias entre columnas (azul = faltan " + "juntas; rojo = cuando una falta la otra tiende a estar).")) + + +def _patterns_block(patterns_res: dict): + patterns = (patterns_res or {}).get("patterns") or [] + header = ["Columnas que faltan juntas", "Filas", "%"] + rows = [] + for p in patterns[:_TOP_PATTERNS]: + if not isinstance(p, dict): + continue + cols = p.get("missing_cols") or [] + if cols: + label = ", ".join(_truncate(c, 18) for c in cols) + else: + label = "(fila completa — sin faltantes)" + rows.append([label, _fmt_int(p.get("n_rows")), _fmt_pct(p.get("pct"))]) + if not rows: + return None + total = (patterns_res or {}).get("n_patterns") + shown = len(rows) + note = "cada fila es un patrón de «qué columnas faltan juntas»" + if isinstance(total, int) and total > shown: + note += f" — top {shown} de {total} patrones distintos" + return model.DataTable(header=header, rows=rows, + title="Patrones de fila más comunes", note=note) + + +def _mcar_mar_note(corr: dict, mark: bool): + """Interpretive, exploratory MCAR/MAR note from the absence correlations. + + Reads the absence correlations at two levels so the verdict never contradicts + the visible evidence: a *strong* correlation flags a clear non-random (MAR) + pattern; a *partial* overlap (many rows co-miss — high Jaccard — even if the + correlation is diluted by one column being missing far more often) flags a + localized possible-MAR and cites the concrete co-missing pair; only when + neither holds does it read the absences as compatible with MCAR.""" + + def _pairs_with(attr_ok): + out = [] + for p in (corr or {}).get("pairs") or []: + if isinstance(p, dict) and attr_ok(p): + out.append(p) + return out + + def _cf(v): + try: + return float(v) + except (TypeError, ValueError): + return 0.0 + + strong = _pairs_with(lambda p: abs(_cf(p.get("corr"))) >= _CORR_STRONG) + partial = _pairs_with( + lambda p: _cf(p.get("corr")) > 0 and _cf(p.get("jaccard")) >= _JACCARD_NOTABLE) + mcar = _term("mcar", "MCAR", mark) + mar = _term("mar", "MAR", mark) + head = ( + "**Lectura exploratoria MCAR/MAR.** Esta es una heurística basada en la " + "correlación de las ausencias entre columnas, NO un test confirmatorio " + "(como el de Little); orienta, no demuestra. ") + if strong: + top = strong[0] + ev = (f"«{model._safe_str(top.get('a'))}» y " + f"«{model._safe_str(top.get('b'))}» " + f"(corr {_fmt_num(top.get('corr'))})") + body = ( + f"Hay ausencias que co-ocurren con fuerza —{ev}—: las columnas no " + f"faltan de forma independiente, lo que es un indicio de un patrón no " + f"aleatorio ({mar}). Antes de imputar o descartar filas conviene " + f"comprobar si la ausencia depende de otra variable observada; en ese " + f"caso la imputación debería condicionar en ella para no sesgar.") + elif partial: + top = max(partial, key=lambda p: _cf(p.get("jaccard"))) + ev = (f"«{model._safe_str(top.get('a'))}» y " + f"«{model._safe_str(top.get('b'))}» faltan a la vez en " + f"{_fmt_int(top.get('co_missing'))} filas " + f"(Jaccard {_fmt_num(top.get('jaccard'))})") + body = ( + f"Hay co-ocurrencia parcial de ausencias —{ev}—: algunas columnas " + f"tienden a faltar juntas aunque la correlación global sea modesta " + f"(habitual cuando una columna falta mucho más que la otra). Es un " + f"indicio de un posible patrón localizado no aleatorio ({mar}); " + f"conviene revisar si esa ausencia depende de otra variable observada " + f"antes de imputar, en lugar de asumir que faltan al azar.") + else: + body = ( + f"Las ausencias entre columnas no muestran correlación ni solape " + f"relevante: parecen independientes, lo que es compatible con que " + f"falten al azar ({mcar}). Aun así, la ausencia podría depender de " + f"variables no observadas (la heurística no lo descarta).") + return model.Markdown(text=head + body) + + +def _intro_block(mark: bool, source): + missingness = _term("missingness", "missingness", mark) + text = ( + f"Este capítulo analiza el {missingness} de la tabla: no solo cuánto " + "falta (eso lo cubre la calidad), sino DÓNDE falta y si las columnas " + "faltan juntas. La co-ocurrencia de ausencias se calcula sobre la matriz " + "binaria «is-null» por fila.") + if source == "raw_numeric": + text += (" Nota: no se pudo leer la tabla cruda completa, así que la " + "co-ocurrencia se limita a las columnas numéricas disponibles.") + return model.Markdown(text=text) + + +# --------------------------------------------------------------------------- # +# Entry point. +# --------------------------------------------------------------------------- # +def build_missingness(profile: dict, ctx: dict): + """Build the missingness Chapter, or None if the table has no missing data.""" + if not isinstance(profile, dict): + profile = {} + ctx = ctx or {} + + with_nulls = _columns_with_nulls(profile) + if not with_nulls: + return None # no missing data anywhere -> chapter does not apply. + + # Register glossary terms (if a collector is present) and mark them clickable. + glossary = ctx.get("glossary") + mark = False + if isinstance(glossary, model.GlossaryCollector): + for key, (label, definition) in _TERMS.items(): + glossary.add(key, label, definition) + mark = True + + # Per-row is-null mask (sample) for co-occurrence and row patterns. + mask, sampled, source = _null_mask(profile, ctx) + overview = _overview(mask) if mask else None + n_total = profile.get("n_rows") + + blocks = [ + model.Heading(text="Cuánto y dónde faltan datos", level=2), + _intro_block(mark, source), + _summary_block(profile, with_nulls, overview, sampled, n_total), + model.Heading(text="Faltantes por columna", level=2), + ] + ranking = _ranking_block(with_nulls) + if ranking is not None: + blocks.append(ranking) + rank_fig = _ranking_figure(with_nulls) + if rank_fig is not None: + blocks.append(rank_fig) + + # Co-occurrence + row patterns need the per-row mask. Without it, say so. + if not mask: + blocks.append(model.Note( + "No se pudo construir la matriz «is-null» por fila (sin acceso a los " + "datos crudos), así que no se analiza la co-ocurrencia de ausencias " + "ni los patrones de fila en este informe.")) + return model.Chapter(id=CHAPTER_ID, title=CHAPTER_TITLE, + version=CHAPTER_VERSION, blocks=blocks) + + corr = _correlation(mask, _TOP_PAIRS) or {} + co_blocks = [model.Heading(text="Co-ocurrencia de ausencias", level=2)] + heatmap = _heatmap_block(corr) + if heatmap is not None: + co_blocks.append(heatmap) + pairs = _pairs_block(corr) + if pairs is not None: + co_blocks.append(pairs) + if heatmap is None and pairs is None: + co_blocks.append(model.Note( + "Ninguna pareja de columnas comparte ausencias con variación " + "suficiente para correlacionarlas (p. ej. una sola columna con " + "faltantes), así que no hay co-ocurrencia que mostrar.")) + # Keep the co-occurrence heading next to its heatmap and table. + blocks.append(model.Group(blocks=co_blocks)) + + patterns_res = _row_patterns(mask, _TOP_PATTERNS) or {} + patterns = _patterns_block(patterns_res) + if patterns is not None: + blocks.append(model.Heading(text="Patrones de fila", level=2)) + blocks.append(patterns) + + blocks.append(model.Heading(text="Lectura MCAR / MAR", level=2)) + blocks.append(_mcar_mar_note(corr, mark)) + + return model.Chapter(id=CHAPTER_ID, title=CHAPTER_TITLE, + version=CHAPTER_VERSION, blocks=blocks) diff --git a/python/functions/datascience/automatic_eda/chapters/missingness_test.py b/python/functions/datascience/automatic_eda/chapters/missingness_test.py new file mode 100644 index 00000000..323270e1 --- /dev/null +++ b/python/functions/datascience/automatic_eda/chapters/missingness_test.py @@ -0,0 +1,162 @@ +"""Tests for the MISSINGNESS chapter. + +Covers the Definition of Done for this chapter: + * Activates (non-None Chapter with the expected sections) when the profile has + missing data, building the co-occurrence from the per-row is-null mask. + * Returns None when the table has no missing data at all (edge case). + * Registers the MCAR/MAR/missingness glossary terms. + * The DuckDB push-down path covers categorical columns (not only numeric), + so a categorical column that co-misses with a numeric one is detected. +""" + +import os +import sys + +_HERE = os.path.dirname(os.path.abspath(__file__)) +_FUNCTIONS = os.path.abspath(os.path.join(_HERE, "..", "..", "..")) # python/functions +if _FUNCTIONS not in sys.path: + sys.path.insert(0, _FUNCTIONS) + +from datascience.automatic_eda import model # noqa: E402 +from datascience.automatic_eda.chapters.missingness import ( # noqa: E402 + build_missingness, +) + + +def _titles(chapter): + """Collect heading texts and table/figure titles for assertions.""" + out = [] + for b in chapter.blocks: + kind = getattr(b, "kind", None) + if kind == "heading": + out.append(("heading", getattr(b, "text", ""))) + elif kind in ("data_table", "kv_table"): + out.append((kind, getattr(b, "title", ""))) + elif kind == "group": + for inner in getattr(b, "blocks", []): + ik = getattr(inner, "kind", None) + if ik == "heading": + out.append(("heading", getattr(inner, "text", ""))) + elif ik in ("data_table", "kv_table"): + out.append((ik, getattr(inner, "title", ""))) + elif ik == "figure": + out.append(("figure", getattr(inner, "caption", ""))) + elif kind == "figure": + out.append(("figure", getattr(b, "caption", ""))) + return out + + +def _all_text(chapter): + parts = [] + def walk(blocks): + for b in blocks: + for attr in ("text", "title", "note", "caption"): + v = getattr(b, attr, None) + if v: + parts.append(str(v)) + if getattr(b, "kind", None) == "group": + walk(getattr(b, "blocks", [])) + walk(chapter.blocks) + return "\n".join(parts) + + +def test_returns_none_when_no_missing_data(): + profile = { + "n_rows": 4, + "null_cell_pct": 0.0, + "columns": [ + {"name": "a", "null_count": 0, "null_pct": 0.0, "n_rows": 4}, + {"name": "b", "null_count": 0, "null_pct": 0.0, "n_rows": 4}, + ], + } + assert build_missingness(profile, {}) is None + + +def test_activates_with_cooccurrence_via_raw_numeric(): + # a and b are missing in EXACTLY the same rows (0,1,2) -> perfect absence + # correlation. c has no nulls. No db_path -> the chapter falls back to the + # numeric raw_numeric mask. + profile = { + "n_rows": 6, + "null_cell_pct": (0.5 + 0.5 + 0.0) / 3.0, + "columns": [ + {"name": "a", "null_count": 3, "null_pct": 0.5, "n_rows": 6}, + {"name": "b", "null_count": 3, "null_pct": 0.5, "n_rows": 6}, + {"name": "c", "null_count": 0, "null_pct": 0.0, "n_rows": 6}, + ], + } + glossary = model.GlossaryCollector() + ctx = { + "raw_numeric": { + "a": [None, None, None, 1.0, 2.0, 3.0], + "b": [None, None, None, 4.0, 5.0, 6.0], + }, + "glossary": glossary, + } + ch = build_missingness(profile, ctx) + assert ch is not None + assert ch.id == "missingness" + assert ch.blocks + + titles = _titles(ch) + headings = {t for (k, t) in titles if k == "heading"} + # Core sections present. + assert any("Cuánto y dónde" in h for h in headings) + assert any("Faltantes por columna" in h for h in headings) + assert any("Co-ocurrencia" in h for h in headings) + assert any("MCAR" in h for h in headings) + # A summary KVTable, a ranking DataTable, a co-occurrence figure and the + # pairs table all exist. + kinds = {k for (k, _) in titles} + assert "kv_table" in kinds + assert "data_table" in kinds + assert "figure" in kinds + + # Glossary terms registered. + keys = {t["key"] for t in glossary.terms()} + assert {"missingness", "mcar", "mar"} <= keys + + # The MCAR/MAR note reads the co-occurrence; with a perfect overlap it must + # flag the non-random (MAR) reading. + text = _all_text(ch) + assert "MAR" in text + + +def test_db_pushdown_covers_categorical_column(tmp_path): + """The is-null mask push-down must cover a categorical column, so a + categorical that co-misses with a numeric one shows up in the pairs.""" + import duckdb + + db = str(tmp_path / "miss.duckdb") + con = duckdb.connect(db) + con.execute("CREATE TABLE t (num1 DOUBLE, num2 DOUBLE, cat VARCHAR)") + # num1 and cat are NULL together in the first 4 of 10 rows; num2 never null. + rows = [] + for i in range(10): + if i < 4: + rows.append((None, float(i), None)) + else: + rows.append((float(i), float(i), f"c{i}")) + con.executemany("INSERT INTO t VALUES (?,?,?)", rows) + con.close() + + profile = { + "n_rows": 10, + "null_cell_pct": (0.4 + 0.0 + 0.4) / 3.0, + "columns": [ + {"name": "num1", "null_count": 4, "null_pct": 0.4, "n_rows": 10}, + {"name": "num2", "null_count": 0, "null_pct": 0.0, "n_rows": 10}, + {"name": "cat", "null_count": 4, "null_pct": 0.4, "n_rows": 10}, + ], + } + ctx = {"db_path": db, "table": "t", "glossary": model.GlossaryCollector()} + ch = build_missingness(profile, ctx) + assert ch is not None + + # The pairs table must mention both num1 and cat (they co-miss perfectly), + # which is only possible if the mask covered the categorical column. + text = _all_text(ch) + assert "num1" in text and "cat" in text + # Co-occurrence section + a pairs data table exist. + titles = _titles(ch) + assert any("co-faltan" in (t or "").lower() for (k, t) in titles) diff --git a/python/functions/datascience/automatic_eda/chapters_registry.py b/python/functions/datascience/automatic_eda/chapters_registry.py index d9030999..d424b934 100644 --- a/python/functions/datascience/automatic_eda/chapters_registry.py +++ b/python/functions/datascience/automatic_eda/chapters_registry.py @@ -32,6 +32,7 @@ CHAPTER_ORDER = [ "num_distr", # numeric distributions "cat_distr", # categorical distributions "calidad", # data quality + "missingness", # missing-data patterns (co-occurrence of absences; MCAR/MAR) "correlacion", # correlations / associations "relaciones", # key relations: declared/candidate PK + FK (inter/intra-table) "modelos", # cheap models (PCA/KMeans/outliers) diff --git a/python/functions/datascience/extract_null_mask.md b/python/functions/datascience/extract_null_mask.md new file mode 100644 index 00000000..66510b2c --- /dev/null +++ b/python/functions/datascience/extract_null_mask.md @@ -0,0 +1,97 @@ +--- +name: extract_null_mask +kind: function +lang: py +domain: datascience +version: "1.0.0" +purity: impure +signature: "def extract_null_mask(query_fn, table: str, columns: list, max_rows: int = 5000) -> dict" +description: "Extrae la mascara de nulos (1=falta / 0=presente) de una muestra de filas de una tabla, una lista 0/1 por columna alineada por fila, para alimentar el capitulo de calidad / patron de nulos de AutomaticEDA sin que el capitulo toque la base de datos. Recibe un lector read-only inyectado `query_fn(sql) -> dict` (mismo contrato que duckdb_query_readonly / pg_query / el `_q` de profile_table) y NO abre ninguna conexion por su cuenta. Construye UNA sola query que proyecta por cada columna `CASE WHEN \"col\" IS NULL THEN 1 ELSE 0 END` con identificadores escapados y LIMIT. Devuelve dict dict-no-throw: columns (efectivamente leidas, en orden), mask (lista int 0/1 por columna, misma longitud todas) y n. Una celda None se cuenta defensivamente como 1 (falta)." +tags: [eda, nulls, missing, datascience, automatic-eda, extraction, read-only, duckdb, postgres, python] +uses_functions: [] +uses_types: [] +returns: [] +returns_optional: false +error_type: "error_go_core" +imports: [] +params: + - name: query_fn + desc: "callable lector read-only del backend activo. Recibe un string SQL y devuelve un dict {'status':'ok','rows':[{col:val,...},...]} (mismo contrato que duckdb_query_readonly o el `_q` de profile_table). NO se abre ninguna conexion dentro de la funcion: toda la lectura pasa por query_fn. Si es None -> error." + - name: table + desc: "nombre de la tabla de la que muestrear la mascara de nulos. Se escapa con comillas dobles en la query. Vacio o None -> status error." + - name: columns + desc: "lista de nombres de columna a evaluar. Cada una produce una entrada en `mask` con una lista 0/1 paralela por fila (1=IS NULL, 0=presente). Cada nombre se escapa con comillas dobles. Vacia o None -> status error." + - name: max_rows + desc: "limite de filas a muestrear (clausula LIMIT). Default 5000. Protege frente a tablas enormes; con LIMIT obtienes el primer tramo, no un muestreo uniforme." +output: "dict (nunca lanza). En exito: {'status':'ok','table':str,'columns':[str,...] (en orden),'mask':{col:[int 0/1,...],...} (1=falta/IS NULL, 0=presente; todas las listas con misma longitud = n),'n':int}. En error (sin lanzar): {'status':'error','error':str,'table':str,'columns':[],'mask':{},'n':0}. Errores: query_fn None, table vacia, columns vacia, o query_fn devuelve status!='ok' (se propaga su error)." +tested: true +tests: ["test_golden_mask_alineada", "test_celda_none_cuenta_como_falta", "test_columns_vacia_status_error", "test_query_fn_status_error_propaga", "test_query_fn_none_da_error_sin_reventar", "test_sql_contiene_case_y_limit"] +test_file_path: "python/functions/datascience/extract_null_mask_test.py" +file_path: "python/functions/datascience/extract_null_mask.py" +--- + +## Ejemplo + +```python +import sys, os +sys.path.insert(0, os.path.join("python", "functions")) +from datascience.extract_null_mask import extract_null_mask +from infra import duckdb_query_readonly + +# El lector read-only se inyecta como closure (igual que el `_q` de profile_table). +db = "data/clientes.duckdb" +def _q(sql): + return duckdb_query_readonly(db, sql) + +res = extract_null_mask(_q, "clientes", ["email", "telefono", "edad"]) +# res == { +# "status": "ok", +# "table": "clientes", +# "columns": ["email", "telefono", "edad"], +# "mask": { +# "email": [0, 0, 1, 0, ...], # fila 2 sin email +# "telefono": [1, 0, 1, 0, ...], +# "edad": [0, 0, 0, 1, ...], +# }, +# "n": 5000, +# } + +# % de nulos por columna a partir de la muestra: +pct = {c: 100 * sum(bits) / max(res["n"], 1) for c, bits in res["mask"].items()} + +# Se entrega al capitulo de calidad sin que este toque la BD: +ctx = {"null_mask": res} +``` + +## Cuando usarla + +Cuando el capitulo de calidad / patron de nulos de AutomaticEDA necesita saber +DONDE faltan los valores (no solo cuantos) y NO debe abrir la base de datos por +su cuenta: extraes aqui la mascara 0/1 por columna alineada por fila y se la pasas +en `ctx['null_mask']`. Usala siempre que quieras detectar co-ocurrencia de nulos +(filas que fallan en varias columnas a la vez), calcular el % de nulos sobre una +muestra, o pintar un heatmap de missingness reutilizando un unico lector read-only +inyectado, en vez de hacer N `COUNT(*) WHERE col IS NULL` por separado. + +## Gotchas + +- **Impura**: lee de la base de datos a traves de `query_fn`. No abre conexiones + por su cuenta — depende por completo del lector inyectado. Sigue el estilo + dict-no-throw del grupo `eda`: nunca lanza; ante cualquier fallo devuelve + `{"status":"error","error":...}` con `columns=[]`, `mask={}`, `n=0`. +- **`error_type` en el frontmatter es `error_go_core` por convencion del registry** + (toda funcion impura debe declararlo y el indexer lo exige), pero el codigo + NO lanza esa excepcion: degrada al dict de error. Es metadata, no comportamiento. +- **Muestra, no censo**: con `LIMIT max_rows` obtienes el primer tramo de filas que + devuelva el backend, no un muestreo uniforme ni la tabla entera. El % de nulos + derivado es una estimacion sobre esa muestra; para el conteo exacto usa un + agregado `COUNT(*)`/`COUNT(col)` aparte. +- **Alineacion por fila**: `mask[col][i]` corresponde a la misma fila `i` que + `mask[otra_col][i]`. Todas las listas tienen longitud `n`, asi que puedes cruzar + columnas por indice (co-ocurrencia de nulos) sin re-alinear. +- **Defensa None -> 1**: el SQL ya devuelve 0/1, pero si una celda llega como `None` + (CASE no aplicado, columna ausente en la fila, backend que nulifica) se cuenta + como 1 (falta). Un valor inesperado no convertible a int se trata como presente (0). +- **No loguear los datos crudos**: aunque `mask` es solo 0/1, los nombres de columna + pueden revelar el esquema. En trazas usa `n` y el numero de columnas, no el dict + completo. diff --git a/python/functions/datascience/extract_null_mask.py b/python/functions/datascience/extract_null_mask.py new file mode 100644 index 00000000..4ed1e7e7 --- /dev/null +++ b/python/functions/datascience/extract_null_mask.py @@ -0,0 +1,101 @@ +"""extract_null_mask — extrae la mascara de nulos (1=falta / 0=presente) de una tabla. + +Lector read-only inyectado: recibe `query_fn(sql) -> dict` con el mismo contrato +que duckdb_query_readonly / pg_query (y que el `_q` de profile_table): +`{"status": "ok", "rows": [{col: val, ...}, ...]}`. Esta funcion NO abre ninguna +conexion por su cuenta — solo usa `query_fn`. Construye UNA sola query que, por +cada columna pedida, evalua `CASE WHEN "col" IS NULL THEN 1 ELSE 0 END` y devuelve +una muestra de filas con esos bits. El resultado es un dict `mask` con una lista +0/1 por columna, alineada por fila (1 = el valor falta / IS NULL, 0 = presente), +listo para alimentar el capitulo de calidad / patron de nulos de AutomaticEDA sin +que el capitulo toque la base de datos. + +Estilo dict-no-throw del grupo `eda`: nunca lanza; captura cualquier excepcion y +degrada a `{"status": "error", "error": str, ...}`. +""" + + +def _to_bit(value): + """Coacciona el valor 0/1 del CASE a int de forma defensiva. + + El SQL ya devuelve 0 (presente) o 1 (falta). Por si una celda llega como None + (el CASE no se aplico o el backend la nulifico), se cuenta como 1 (falta). El + resto se reduce a int: un entero distinto de 0 cuenta como 1 (falta), 0 como + presente. Un valor no convertible se trata como presente (0) — nunca lanza. + """ + if value is None: + return 1 + try: + return 1 if int(value) != 0 else 0 + except (TypeError, ValueError): + return 0 + + +def extract_null_mask(query_fn, table, columns, max_rows=5000): + """Extrae la mascara de nulos (1=falta / 0=presente) de una muestra de la tabla. + + Args: + query_fn: callable lector read-only del backend activo. Recibe un string + SQL y devuelve un dict {"status": "ok", "rows": [{col: val, ...}]} + (mismo contrato que duckdb_query_readonly / el `_q` de profile_table). + No se abre ninguna conexion aqui: toda la lectura pasa por query_fn. + table: nombre de la tabla. Se escapa con comillas dobles en la query. + columns: lista de nombres de columna a evaluar. Cada una produce una + entrada en `mask` con una lista 0/1 paralela por fila. Vacia o None -> + status error. + max_rows: limite de filas a muestrear (clausula LIMIT). Default 5000. + + Returns: + dict (nunca lanza): + { + "status": "ok" | "error", + "error": str, # solo si status == "error" + "table": str, + "columns": [str, ...], # columnas efectivamente leidas, en orden + "mask": {col: [int 0/1, ...], ...}, # alineada por fila, 1=falta, 0=presente + "n": int # nº de filas muestreadas + } + Todas las listas de `mask` tienen la misma longitud (= n). + """ + base = {"status": "ok", "table": table, "columns": [], "mask": {}, "n": 0} + try: + if query_fn is None: + return {**base, "status": "error", "error": "query_fn es None"} + if not table: + return {**base, "status": "error", "error": "table es obligatorio"} + if not columns: + return {**base, "status": "error", "error": "columns vacío"} + + # Identificadores escapados con comillas dobles (como hace profile_table) + # para tolerar nombres con mayusculas/espacios/palabras reservadas. Cada + # columna se proyecta como su propio bit IS NULL conservando el alias. + select_sql = ", ".join( + f'(CASE WHEN "{c}" IS NULL THEN 1 ELSE 0 END) AS "{c}"' for c in columns + ) + sql = f'SELECT {select_sql} FROM "{table}" LIMIT {int(max_rows)}' + + q = query_fn(sql) + if not isinstance(q, dict) or q.get("status") != "ok": + err = ( + q.get("error", "query_fn fallo") + if isinstance(q, dict) + else "query_fn no devolvio un dict" + ) + return {**base, "status": "error", "error": err} + + rows = q.get("rows", []) or [] + mask = {c: [] for c in columns} + for row in rows: + for c in columns: + # row.get tolera filas que no traigan la columna (None -> falta). + mask[c].append(_to_bit(row.get(c) if isinstance(row, dict) else None)) + + return { + "status": "ok", + "table": table, + "columns": list(columns), + "mask": mask, + "n": len(rows), + } + except Exception as e: # noqa: BLE001 - dict-no-throw: degradar, nunca lanzar + return {**base, "status": "error", "error": str(e)} diff --git a/python/functions/datascience/extract_null_mask_test.py b/python/functions/datascience/extract_null_mask_test.py new file mode 100644 index 00000000..65a2cfc2 --- /dev/null +++ b/python/functions/datascience/extract_null_mask_test.py @@ -0,0 +1,116 @@ +"""Tests para extract_null_mask. + +No usa DuckDB real: inyecta un query_fn FAKE (closure) que devuelve filas +predefinidas (simulando el SELECT de bits 0/1) y, opcionalmente, captura el SQL +recibido para verificar la query generada (CASE WHEN ... IS NULL + LIMIT). Asi el +test es autocontenido y no depende de ningun backend. +""" + +import os +import sys + +sys.path.insert(0, os.path.dirname(__file__)) + +from extract_null_mask import extract_null_mask + + +def _fake_query(rows, captured=None, status="ok", error=None): + """Crea un query_fn FAKE. + + `captured` (lista opcional) recibe el SQL ejecutado para poder inspeccionarlo. + `status`/`error` permiten simular un fallo del backend. + """ + + def _q(sql): + if captured is not None: + captured.append(sql) + if status != "ok": + return {"status": "error", "error": error or "boom"} + return {"status": "ok", "rows": rows} + + return _q + + +def test_golden_mask_alineada(): + """Golden: mask 0/1 por columna alineada por fila, n correcto, status ok.""" + # Cada fila simula el SELECT (CASE WHEN col IS NULL THEN 1 ELSE 0 END) AS col. + rows = [ + {"email": 0, "telefono": 1, "edad": 0}, + {"email": 0, "telefono": 0, "edad": 1}, + {"email": 1, "telefono": 1, "edad": 0}, + ] + res = extract_null_mask(_fake_query(rows), "clientes", ["email", "telefono", "edad"]) + assert res["status"] == "ok" + assert res["table"] == "clientes" + assert res["columns"] == ["email", "telefono", "edad"] + assert res["n"] == 3 + assert res["mask"]["email"] == [0, 0, 1] + assert res["mask"]["telefono"] == [1, 0, 1] + assert res["mask"]["edad"] == [0, 1, 0] + # Todas las listas con la misma longitud. + assert all(len(v) == res["n"] for v in res["mask"].values()) + + +def test_celda_none_cuenta_como_falta(): + """Una celda None se cuenta defensivamente como 1 (falta).""" + rows = [ + {"email": 0, "telefono": None}, + {"email": None, "telefono": 1}, + {"email": 1, "telefono": 0}, + ] + res = extract_null_mask(_fake_query(rows), "clientes", ["email", "telefono"]) + assert res["status"] == "ok" + assert res["mask"]["email"] == [0, 1, 1] + assert res["mask"]["telefono"] == [1, 1, 0] + assert res["n"] == 3 + + +def test_columns_vacia_status_error(): + """columns vacia -> status error con columns/mask/n vacios.""" + res = extract_null_mask(_fake_query([]), "clientes", []) + assert res["status"] == "error" + assert "columns" in res["error"] + assert res["table"] == "clientes" + assert res["columns"] == [] + assert res["mask"] == {} + assert res["n"] == 0 + + +def test_query_fn_status_error_propaga(): + """query_fn que devuelve status != ok -> se propaga como error, mask {}.""" + res = extract_null_mask( + _fake_query([], status="error", error="db locked"), + "clientes", + ["email"], + ) + assert res["status"] == "error" + assert "db locked" in res["error"] + assert res["mask"] == {} + assert res["n"] == 0 + + +def test_query_fn_none_da_error_sin_reventar(): + """query_fn None -> error degradado, sin excepcion.""" + res = extract_null_mask(None, "clientes", ["email"]) + assert res["status"] == "error" + assert res["columns"] == [] + assert res["mask"] == {} + assert res["n"] == 0 + + +def test_sql_contiene_case_y_limit(): + """La query genera un CASE WHEN IS NULL por columna escapada + LIMIT sobre la tabla.""" + captured = [] + rows = [{"email": 0}] + extract_null_mask( + _fake_query(rows, captured), + "clientes_tbl", + ["email"], + max_rows=123, + ) + assert len(captured) == 1 + sql = captured[0] + assert 'CASE WHEN "email" IS NULL THEN 1 ELSE 0 END' in sql + assert 'AS "email"' in sql + assert 'FROM "clientes_tbl"' in sql + assert "LIMIT 123" in sql diff --git a/python/functions/datascience/missingness_corr_heatmap_figure.md b/python/functions/datascience/missingness_corr_heatmap_figure.md new file mode 100644 index 00000000..595b72f8 --- /dev/null +++ b/python/functions/datascience/missingness_corr_heatmap_figure.md @@ -0,0 +1,103 @@ +--- +id: missingness_corr_heatmap_figure_py_datascience +name: missingness_corr_heatmap_figure +kind: function +lang: py +domain: datascience +version: "1.0.0" +purity: impure +signature: "def missingness_corr_heatmap_figure(matrix, labels, title=\"Co-ocurrencia de ausencias\") -> \"matplotlib.figure.Figure\"" +description: "Construye una figura matplotlib (heatmap) de la matriz NxN de correlación de ausencias entre columnas: +1 = dos columnas suelen ser nulas a la vez, -1 = cuando una falta la otra está presente, 0 = ausencias independientes. Usa ax.imshow con coolwarm fijado a [-1,1], ticks con los labels truncados (X rotados 45º), colorbar y anota el valor de cada celda si N<=12. Devuelve un matplotlib.figure.Figure listo para rasterizar por el renderer del informe EDA (capítulo de datos faltantes). Backend Agg sin pyplot global; defensivo ante matrix/labels vacíos o celdas no numéricas (nunca lanza)." +tags: [eda, missing, missingness, correlation, heatmap, matplotlib, figure, visualization, datascience, impure] +uses_functions: [] +uses_types: [] +returns: [] +returns_optional: false +error_type: "error_go_core" +imports: [matplotlib] +example: | + from datascience.missingness_corr_heatmap_figure import missingness_corr_heatmap_figure + matrix = [ + [1.0, 0.82, -0.10], + [0.82, 1.0, 0.05], + [-0.10, 0.05, 1.0], + ] + labels = ["telefono", "movil", "email"] + fig = missingness_corr_heatmap_figure(matrix, labels, title="Co-ocurrencia de ausencias") +tested: true +tests: + - "test_returns_figure_with_axes" + - "test_empty_matrix_does_not_raise_and_returns_figure" + - "test_empty_labels_returns_message_figure" + - "test_large_matrix_omits_annotations" + - "test_ragged_and_non_numeric_cells_are_handled" +test_file_path: "python/functions/datascience/missingness_corr_heatmap_figure_test.py" +file_path: "python/functions/datascience/missingness_corr_heatmap_figure.py" +params: + - name: matrix + desc: "Lista de listas (NxN) de floats en [-1,1]: la correlación de ausencias por pares de columnas. Puede venir vacía. Filas de longitud desigual se toleran (se rellenan/recortan a N); celdas None, NaN o no numéricas se coercen a 0.0. No se muta el original." + - name: labels + desc: "Lista de N nombres de columna, paralela a matrix. Puede venir vacía (devuelve figura \"sin columnas con ausencia variable\"). Se truncan a ~14 chars con elipsis para los ticks; los originales no se mutan." + - name: title + desc: "Título de la figura. Se trunca a ~60 chars con elipsis si es muy largo. Default \"Co-ocurrencia de ausencias\"." +output: "Un matplotlib.figure.Figure (figsize 6.4x5.2, dpi 150) con un Axes heatmap (imshow vmin=-1, vmax=1, cmap coolwarm) más una colorbar etiquetada \"correlación de ausencias\". Ticks en ambos ejes con los labels truncados (X rotados 45º). Si N<=12 cada celda lleva su valor numérico anotado (texto blanco sobre celdas saturadas, oscuro sobre pálidas); con N grande se omiten las anotaciones para no saturar. Si matrix o labels vienen vacíos devuelve una Figure con texto centrado \"sin columnas con ausencia variable\"; cualquier error inesperado se captura y devuelve una Figure con el mensaje de error (nunca lanza). El caller rasteriza/cierra la figura; la función no la muestra ni la guarda." +--- + +## Ejemplo + +```python +from datascience.missingness_corr_heatmap_figure import missingness_corr_heatmap_figure + +# Correlación de ausencias entre 3 columnas de contacto: +# telefono y movil tienden a faltar juntos (0.82); email es casi independiente. +matrix = [ + [1.00, 0.82, -0.10], + [0.82, 1.00, 0.05], + [-0.10, 0.05, 1.00], +] +labels = ["telefono", "movil", "email"] + +fig = missingness_corr_heatmap_figure( + matrix, + labels, + title="Co-ocurrencia de ausencias", +) + +# El renderer del informe lo rasteriza; aquí solo persistimos para inspección. +fig.savefig("/tmp/missingness_heatmap.png") +``` + +## Cuando usarla + +Úsala en el capítulo de datos faltantes de un informe EDA cuando quieras ver de +un vistazo qué columnas faltan juntas (mismo formulario sin rellenar, mismo +proceso roto) frente a columnas cuyas ausencias son independientes. Pásale la +matriz de correlación de ausencias (calculada sobre la máscara de nulos, p. ej. +`df.isnull().corr()`) restringida a las columnas que de verdad tienen ausencia +variable, junto con sus nombres. Es la pareja "estructura" del ranking de % de +nulos: las barras dicen *cuánto* falta cada columna, este heatmap dice *si las +ausencias están relacionadas* entre columnas. + +## Gotchas + +- **Impura por matplotlib.** Toca la maquinaria de render. Usa el backend `Agg` + y la API orientada a objetos `Figure`/`add_subplot` — NUNCA `pyplot.*` aquí, + para no tocar el estado global ni filtrar figuras entre llamadas. `pyplot` NO + es thread-safe; esta función evita ese riesgo construyendo el `Figure` + directamente, así que es segura de llamar en bucle desde el renderer. +- **El caller cierra la figura.** Devuelve el `Figure` pero no lo muestra ni lo + guarda. Quien la consume debe rasterizarla y luego liberarla + (`matplotlib.pyplot.close(fig)`) para no acumular memoria en lotes grandes. +- **Escala de color fija en [-1, 1].** `vmin=-1`, `vmax=1` están fijados a + propósito para que el color sea comparable entre informes y entre columnas. No + se autoescala al rango real de la matriz; valores fuera de `[-1, 1]` se + saturan al extremo del colormap. +- **Anotaciones solo con N<=12.** Por encima de 12 columnas el grid de números + se vuelve ilegible y se omite; queda solo el color + la colorbar. Filtra a las + columnas con ausencia variable antes de llamar para no llegar a matrices + enormes. +- **Defensiva, nunca lanza.** `matrix=[]`, `labels=[]`, filas cortas, celdas + `None`/`NaN`/no numéricas o cualquier error inesperado se manejan sin propagar: + en el peor caso devuelve una `Figure` con "sin columnas con ausencia variable" + o con el texto del error. No envuelvas la llamada en try/except por miedo a un + raise — no lo hay. diff --git a/python/functions/datascience/missingness_corr_heatmap_figure.py b/python/functions/datascience/missingness_corr_heatmap_figure.py new file mode 100644 index 00000000..4f5fd253 --- /dev/null +++ b/python/functions/datascience/missingness_corr_heatmap_figure.py @@ -0,0 +1,158 @@ +"""Impure EDA helper: heatmap of missingness co-occurrence (`eda` group). + +Builds a matplotlib heatmap of the pairwise missingness correlation matrix of a +dataset: a value near ``+1`` means two columns tend to be null together, near +``-1`` means when one is null the other tends to be present, and ``0`` means +their absences are independent. Returns a ready-to-rasterize +``matplotlib.figure.Figure``; it never shows nor saves it. + +Impure because it touches matplotlib's rendering machinery. It uses the headless +Agg backend and the object-oriented ``Figure`` API (no ``pyplot``) so it leaks no +global state and is safe to call repeatedly from a report renderer. +""" + +import matplotlib + +matplotlib.use("Agg") + +from matplotlib.figure import Figure # noqa: E402 + +# Muted gray for secondary text (no-data / fallback messages). +_MUTED_TEXT = "#5f6b7a" +# Soft red for the error fallback message (kept readable, not alarming). +_ERROR_TEXT = "#b00020" + + +def _truncate(text, width: int = 14) -> str: + """Truncate ``text`` to ``width`` chars, appending an ellipsis if cut.""" + s = "" if text is None else str(text) + if len(s) <= width: + return s + if width <= 1: + return s[:width] + return s[: width - 1] + "…" + + +def _message_figure(message: str, color: str = _MUTED_TEXT) -> "Figure": + """Return a fallback ``Figure`` carrying a single centered message.""" + fig = Figure(figsize=(6.4, 4.0), dpi=150) + ax = fig.add_subplot(111) + ax.axis("off") + ax.text( + 0.5, + 0.5, + message, + ha="center", + va="center", + fontsize=12, + color=color, + wrap=True, + transform=ax.transAxes, + ) + fig.tight_layout() + return fig + + +def missingness_corr_heatmap_figure( + matrix, + labels, + title: str = "Co-ocurrencia de ausencias", +) -> "matplotlib.figure.Figure": + """Build a heatmap figure of a missingness correlation matrix. + + Renders an ``NxN`` matrix of missingness correlations in ``[-1, 1]`` with a + diverging ``coolwarm`` colormap (fixed ``vmin=-1``, ``vmax=1`` so the color + scale is comparable across reports). Both axes are tick-labelled with the + column names (truncated to ~14 chars; the X labels rotated 45°). A colorbar + is attached. When the matrix is small (``N <= 12``) each cell is annotated + with its numeric value; for larger matrices the annotations are omitted to + avoid an unreadable grid. + + The function is fully defensive: empty/ragged/non-numeric input never raises. + When there is nothing valid to draw it returns a ``Figure`` carrying a + centered "sin columnas con ausencia variable" message, and any unexpected + error is caught and turned into a fallback ``Figure`` carrying the error text. + + Args: + matrix: List of lists (``NxN``) of floats in ``[-1, 1]`` — the pairwise + missingness correlation. May be empty; rows of unequal length are + tolerated by treating the matrix as invalid only when it is empty or + its label count does not match. Non-numeric/``None`` cells are + coerced to ``0.0``. + labels: List of ``N`` column names, parallel to ``matrix``. May be empty. + Truncated for display; the originals are not mutated. + title: Figure title. Default "Co-ocurrencia de ausencias". + + Returns: + A ``matplotlib.figure.Figure`` with a single heatmap Axes plus a + colorbar. The caller is responsible for rasterizing/closing it. + """ + try: + # --- Validate shape: need a non-empty square-ish matrix with labels. + if ( + not isinstance(matrix, (list, tuple)) + or not isinstance(labels, (list, tuple)) + or len(matrix) == 0 + or len(labels) == 0 + ): + return _message_figure("sin columnas con ausencia variable") + + n = len(labels) + # Build a clean NxN grid: coerce each cell to float, default 0.0, pad/clip + # rows so a ragged input never crashes imshow. + grid = [] + for i in range(n): + row_src = matrix[i] if i < len(matrix) else [] + if not isinstance(row_src, (list, tuple)): + row_src = [] + row = [] + for j in range(n): + cell = row_src[j] if j < len(row_src) else 0.0 + try: + val = float(cell) + except (TypeError, ValueError): + val = 0.0 + if val != val: # NaN guard. + val = 0.0 + row.append(val) + grid.append(row) + + fig = Figure(figsize=(6.4, 5.2), dpi=150) + ax = fig.add_subplot(111) + + im = ax.imshow(grid, vmin=-1, vmax=1, cmap="coolwarm", aspect="equal") + + short = [_truncate(lab, 14) for lab in labels] + ax.set_xticks(range(n)) + ax.set_yticks(range(n)) + ax.set_xticklabels(short, rotation=45, ha="right", fontsize=8) + ax.set_yticklabels(short, fontsize=8) + + # Annotate each cell only when the grid is small enough to stay legible. + if n <= 12: + for i in range(n): + for j in range(n): + val = grid[i][j] + # White text over saturated (dark) cells, dark over pale. + txt_color = "white" if abs(val) >= 0.55 else "#202020" + ax.text( + j, + i, + f"{val:.2f}", + ha="center", + va="center", + fontsize=7, + color=txt_color, + ) + + cbar = fig.colorbar(im, ax=ax, fraction=0.046, pad=0.04) + cbar.ax.tick_params(labelsize=8) + cbar.set_label("correlación de ausencias", fontsize=8) + + if title: + ax.set_title(_truncate(title, 60), fontsize=12, loc="center", pad=10) + + fig.tight_layout() + return fig + except Exception as exc: # noqa: BLE001 — never raise from a figure builder. + return _message_figure(f"error al dibujar heatmap: {exc}", color=_ERROR_TEXT) diff --git a/python/functions/datascience/missingness_corr_heatmap_figure_test.py b/python/functions/datascience/missingness_corr_heatmap_figure_test.py new file mode 100644 index 00000000..973f8c67 --- /dev/null +++ b/python/functions/datascience/missingness_corr_heatmap_figure_test.py @@ -0,0 +1,62 @@ +"""Tests para missingness_corr_heatmap_figure (heatmap de ausencias, grupo eda). + +Usa el backend Agg sin pyplot; no muestra ni guarda figuras. Cada test cierra +explícitamente la Figure construida (matplotlib.pyplot.close) para no acumular +estado entre tests. +""" + +import matplotlib + +matplotlib.use("Agg") + +import matplotlib.pyplot as plt # noqa: E402 +from matplotlib.figure import Figure # noqa: E402 + +from missingness_corr_heatmap_figure import missingness_corr_heatmap_figure + + +def _identity_matrix(n): + """Matriz NxN con diagonal 1.0 y resto 0.0 (correlación de ausencias).""" + return [[1.0 if i == j else 0.0 for j in range(n)] for i in range(n)] + + +def test_returns_figure_with_axes(): + matrix = [[1.0, 0.3, -0.2], [0.3, 1.0, 0.5], [-0.2, 0.5, 1.0]] + labels = ["edad", "ingresos", "ciudad"] + fig = missingness_corr_heatmap_figure(matrix, labels, title="ausencias") + assert isinstance(fig, Figure) + # Heatmap (>=1 axes) + colorbar añade su propio Axes -> al menos 1. + assert len(fig.axes) >= 1 + plt.close(fig) + + +def test_empty_matrix_does_not_raise_and_returns_figure(): + fig = missingness_corr_heatmap_figure([], [], title="vacía") + assert isinstance(fig, Figure) + assert len(fig.axes) >= 1 + plt.close(fig) + + +def test_empty_labels_returns_message_figure(): + fig = missingness_corr_heatmap_figure([[1.0]], [], title="sin labels") + assert isinstance(fig, Figure) + plt.close(fig) + + +def test_large_matrix_omits_annotations(): + n = 16 + fig = missingness_corr_heatmap_figure( + _identity_matrix(n), [f"col_{i}" for i in range(n)] + ) + assert isinstance(fig, Figure) + assert len(fig.axes) >= 1 + plt.close(fig) + + +def test_ragged_and_non_numeric_cells_are_handled(): + # Fila corta + celda None + celda string -> se rellenan/coercen sin lanzar. + matrix = [[1.0, None], ["x", 1.0, 0.5]] + labels = ["a", "b"] + fig = missingness_corr_heatmap_figure(matrix, labels) + assert isinstance(fig, Figure) + plt.close(fig) diff --git a/python/functions/datascience/missingness_correlation.md b/python/functions/datascience/missingness_correlation.md new file mode 100644 index 00000000..337cdbb1 --- /dev/null +++ b/python/functions/datascience/missingness_correlation.md @@ -0,0 +1,68 @@ +--- +name: missingness_correlation +kind: function +lang: py +domain: datascience +version: "1.0.0" +purity: pure +signature: "def missingness_correlation(null_mask: dict, top_k: int = 20) -> dict" +description: "Co-ocurrencia de ausencias: nucleo del capitulo de missingness del grupo eda. Recibe la mascara binaria de nulos de una tabla (1 = falta, 0 = presente, alineada por fila) y mide hasta que punto las columnas faltan juntas. Calcula la matriz de correlacion de Pearson entre los vectores binarios de ausencia de las columnas con varianza (al menos un 1 y un 0), mas las cifras de solapamiento de conjuntos por par (co-missing, either-missing, Jaccard). Excluye las columnas constantes en su ausencia (correlacion indefinida) y reporta cuantas. Compone la funcion atomica pearson del registry; no la reimplementa. Lectura defensiva; NUNCA lanza." +tags: [eda, missingness, correlation, pearson, co-occurrence, jaccard, datascience] +params: + - name: null_mask + desc: "dict {col: [int 0/1, ...]} con la mascara de ausencias de la tabla, alineada por fila: 1 = el valor falta en esa fila, 0 = presente. Todas las listas se asumen de la misma longitud (numero de filas). Valores truthy distintos de 0 se tratan como ausencia; entradas no-lista se ignoran sin romper." + - name: top_k + desc: "Numero maximo de pares a devolver en `pairs`, ordenados por valor absoluto de correlacion descendente. Default 20. Solo limita la lista de pares; la matriz cubre siempre todas las columnas con varianza." +output: "dict con: columns (columnas con varianza en la ausencia, en orden de entrada); matrix (len(columns) x len(columns) de correlacion de Pearson entre las mascaras binarias, diagonal 1.0); pairs (hasta top_k pares i<j ordenados por |corr| desc, cada uno {a, b, corr, co_missing, either_missing, jaccard} donde co_missing = filas en que ambas faltan, either_missing = filas en que al menos una falta, jaccard = co_missing/either_missing o 0.0 si either_missing=0); n_excluded (nº de columnas con algun nulo pero sin varianza, constantes en la ausencia); excluded_cols (esas columnas en orden de entrada). Si hay <2 columnas con varianza, columns/matrix/pairs van vacios pero n_excluded/excluded_cols se rellenan. NUNCA lanza." +uses_functions: [pearson_py_datascience] +uses_types: [] +returns: [] +returns_optional: false +error_type: "" +imports: [] +tested: true +tests: ["test_co_ocurrencia_fuerte_corr_uno_jaccard_uno", "test_ausencias_disjuntas_corr_negativa_jaccard_cero", "test_columna_sin_varianza_se_excluye", "test_menos_de_dos_columnas_con_varianza_vacio_pero_cuenta_excluidas", "test_mask_vacio_todo_vacio", "test_top_k_limita_pares", "test_no_lanza_con_entradas_raras"] +test_file_path: "python/functions/datascience/missingness_correlation_test.py" +file_path: "python/functions/datascience/missingness_correlation.py" +--- + +## Ejemplo + +```python +import sys, os +sys.path.insert(0, os.path.join("python", "functions")) +from datascience.missingness_correlation import missingness_correlation + +# Mascara de ausencias de 6 filas. 1 = falta, 0 = presente. +mask = { + "ingresos": [1, 0, 1, 0, 1, 0], # falta junto a "deducciones" + "deducciones": [1, 0, 1, 0, 1, 0], # mismas filas que "ingresos" + "telefono": [0, 0, 0, 1, 0, 0], # casi siempre presente + "verificado": [1, 1, 1, 1, 1, 1], # siempre ausente -> constante, excluida +} +out = missingness_correlation(mask, top_k=10) + +print(out["columns"]) # ['ingresos', 'deducciones', 'telefono'] +print(out["n_excluded"]) # 1 +print(out["excluded_cols"]) # ['verificado'] + +# El par mas fuerte: ingresos y deducciones faltan siempre juntas. +top = out["pairs"][0] +print(top["a"], top["b"], round(top["corr"], 3)) # ingresos deducciones 1.0 +print(top["co_missing"], top["either_missing"], top["jaccard"]) # 3 3 1.0 +``` + +## Cuando usarla + +- Usala en el capitulo de **missingness** de `AutomaticEDA` cuando ya tengas la mascara binaria de nulos por columna y quieras detectar **patrones de ausencia conjunta**: que columnas faltan siempre juntas (posible misma fuente/proceso roto) y cuales faltan de forma independiente. +- Cuando necesites ordenar los pares de columnas por fuerza de co-ocurrencia (|corr|) para priorizar que bloques de ausencia investigar o imputar juntos. +- Cuando quieras la cifra de solapamiento de conjuntos (Jaccard, co-missing) ademas de la correlacion lineal, para distinguir "faltan juntas" de "estan presentes juntas". +- Antes de elegir una estrategia de imputacion: dos columnas con corr de ausencia ~1.0 no aportan informacion independiente sobre por que falta la otra. + +## Gotchas + +- Funcion pura, sin I/O y determinista. Lectura defensiva: entradas no-dict, columnas no-lista o vacias se ignoran sin lanzar. +- Solo entran al calculo las columnas con **varianza en la ausencia** (al menos un 1 y al menos un 0). Una columna siempre-presente (todo 0) no aporta ausencia y **no** se cuenta como excluida; una columna siempre-ausente o constante con nulos (todo 1) tiene correlacion indefinida y se excluye, sumando a `n_excluded` / `excluded_cols`. +- Con menos de 2 columnas con varianza, `columns`/`matrix`/`pairs` quedan vacios pero `n_excluded`/`excluded_cols` se rellenan igual — el caller debe contemplar el caso "sin pares". +- La correlacion es la de Pearson sobre vectores binarios (equivale al coeficiente phi). El signo importa: corr negativa = las ausencias tienden a ser **complementarias** (cuando una falta, la otra suele estar presente). +- Asume todas las listas alineadas por fila y de la misma longitud. Si vienen de longitudes distintas, `pearson` opera sobre el solapamiento que permita `zip` y degrada a 0.0 cuando no hay varianza efectiva; alinea la mascara antes de llamar. diff --git a/python/functions/datascience/missingness_correlation.py b/python/functions/datascience/missingness_correlation.py new file mode 100644 index 00000000..1141d7da --- /dev/null +++ b/python/functions/datascience/missingness_correlation.py @@ -0,0 +1,120 @@ +"""Co-ocurrencia de ausencias: matriz de correlacion de Pearson entre mascaras de nulos. + +Funcion pura del grupo eda, nucleo del capitulo de missingness. Recibe la mascara +binaria de ausencias de una tabla (1 = falta, 0 = presente, alineada por fila) y +mide hasta que punto las columnas faltan juntas. Para cada par de columnas con +varianza en su ausencia calcula la correlacion de Pearson entre los vectores +binarios, mas las cifras de solapamiento de conjuntos (co-missing, either-missing, +Jaccard). Compone la funcion atomica `pearson` del registry; no reimplementa la +correlacion. Lectura defensiva; NUNCA lanza. +""" + +from datascience import pearson + + +def missingness_correlation(null_mask, top_k=20) -> dict: + """Correlacion de co-ocurrencia de ausencias entre columnas. + + Args: + null_mask: dict {col: [int 0/1, ...]} alineado por fila (1 = el valor + falta en esa fila). Todas las listas se asumen de la misma longitud. + top_k: numero maximo de pares a devolver, ordenados por |corr| desc. + + Returns: + dict con: + - columns: columnas con varianza en la ausencia (al menos un 1 y al + menos un 0), en orden de entrada. + - matrix: matriz len(columns) x len(columns) de correlacion de Pearson + entre las mascaras binarias, diagonal 1.0. + - pairs: lista de hasta top_k pares (i<j) ordenados por |corr| desc. + Cada par: {a, b, corr, co_missing, either_missing, jaccard}. + - n_excluded: numero de columnas con algun nulo pero sin varianza + (constantes en la ausencia: siempre presentes o siempre ausentes). + - excluded_cols: lista de esas columnas (en orden de entrada). + + Si hay menos de 2 columnas con varianza, columns/matrix/pairs van vacios + pero n_excluded/excluded_cols se rellenan igualmente. NUNCA lanza. + """ + # Salida base, defensiva ante entradas no-dict. + result = { + "columns": [], + "matrix": [], + "pairs": [], + "n_excluded": 0, + "excluded_cols": [], + } + + if not isinstance(null_mask, dict) or not null_mask: + return result + + varying = [] # columnas con varianza en la ausencia + varying_vecs = [] # sus vectores binarios saneados (floats 0.0/1.0) + excluded_cols = [] # columnas con nulos pero sin varianza (constantes) + + for col, raw in null_mask.items(): + if not isinstance(raw, (list, tuple)): + continue + # Sanea a 0/1: cualquier valor truthy distinto de 0 cuenta como ausencia. + vec = [1 if bool(v) else 0 for v in raw] + if not vec: + continue + ones = sum(vec) + zeros = len(vec) - ones + if ones > 0 and zeros > 0: + varying.append(col) + varying_vecs.append([float(v) for v in vec]) + elif ones > 0: + # Tiene nulos pero todos (constante en la ausencia): sin varianza. + excluded_cols.append(col) + # ones == 0 -> columna siempre presente, sin nulos: no se cuenta como + # excluida (no aporta ausencia al analisis de co-ocurrencia). + + result["n_excluded"] = len(excluded_cols) + result["excluded_cols"] = excluded_cols + + n = len(varying) + if n < 2: + return result + + result["columns"] = list(varying) + + # Matriz de correlacion de Pearson, diagonal 1.0. + matrix = [[0.0] * n for _ in range(n)] + for i in range(n): + matrix[i][i] = 1.0 + for i in range(n): + for j in range(i + 1, n): + r = pearson(varying_vecs[i], varying_vecs[j]) + matrix[i][j] = r + matrix[j][i] = r + result["matrix"] = matrix + + # Pares con cifras de solapamiento de conjuntos. + pairs = [] + for i in range(n): + vi = varying_vecs[i] + for j in range(i + 1, n): + vj = varying_vecs[j] + co_missing = 0 + either_missing = 0 + for a, b in zip(vi, vj): + a_miss = a != 0.0 + b_miss = b != 0.0 + if a_miss and b_miss: + co_missing += 1 + if a_miss or b_miss: + either_missing += 1 + jaccard = co_missing / either_missing if either_missing > 0 else 0.0 + pairs.append({ + "a": varying[i], + "b": varying[j], + "corr": matrix[i][j], + "co_missing": co_missing, + "either_missing": either_missing, + "jaccard": jaccard, + }) + + pairs.sort(key=lambda p: abs(p["corr"]), reverse=True) + result["pairs"] = pairs[:top_k] if top_k is not None and top_k >= 0 else pairs + + return result diff --git a/python/functions/datascience/missingness_correlation_test.py b/python/functions/datascience/missingness_correlation_test.py new file mode 100644 index 00000000..dcae0a60 --- /dev/null +++ b/python/functions/datascience/missingness_correlation_test.py @@ -0,0 +1,115 @@ +"""Tests para missingness_correlation.""" + +from datascience.missingness_correlation import missingness_correlation + + +def test_co_ocurrencia_fuerte_corr_uno_jaccard_uno(): + # a y b faltan EXACTAMENTE en las mismas filas -> corr 1.0, jaccard 1.0. + mask = { + "a": [1, 0, 1, 0, 1, 0], + "b": [1, 0, 1, 0, 1, 0], + } + out = missingness_correlation(mask) + assert out["columns"] == ["a", "b"] + assert out["n_excluded"] == 0 + # Diagonal 1.0, off-diagonal ~1.0. + assert out["matrix"][0][0] == 1.0 + assert out["matrix"][1][1] == 1.0 + assert abs(out["matrix"][0][1] - 1.0) < 1e-9 + assert len(out["pairs"]) == 1 + pair = out["pairs"][0] + assert {pair["a"], pair["b"]} == {"a", "b"} + assert abs(pair["corr"] - 1.0) < 1e-9 + assert pair["co_missing"] == 3 # filas 0,2,4 + assert pair["either_missing"] == 3 # mismas filas + assert abs(pair["jaccard"] - 1.0) < 1e-9 + + +def test_ausencias_disjuntas_corr_negativa_jaccard_cero(): + # a y b nunca faltan en la misma fila -> co_missing 0, jaccard 0, corr <= 0. + mask = { + "a": [1, 1, 0, 0], + "b": [0, 0, 1, 1], + } + out = missingness_correlation(mask) + assert out["columns"] == ["a", "b"] + pair = out["pairs"][0] + assert pair["co_missing"] == 0 + assert pair["either_missing"] == 4 + assert pair["jaccard"] == 0.0 + # Solapamiento nulo + ausencias complementarias -> correlacion negativa. + assert pair["corr"] < 0.0 + assert abs(pair["corr"] - out["matrix"][0][1]) < 1e-12 + + +def test_columna_sin_varianza_se_excluye(): + # c esta siempre presente (todo 0): no aporta ausencia -> no entra ni como + # excluida. d esta siempre ausente (todo 1): tiene nulos pero sin varianza + # -> excluida y n_excluded incrementa. a y b tienen varianza. + mask = { + "a": [1, 0, 1, 0], + "b": [1, 0, 0, 0], + "c": [0, 0, 0, 0], # siempre presente + "d": [1, 1, 1, 1], # siempre ausente, constante + } + out = missingness_correlation(mask) + assert out["columns"] == ["a", "b"] + assert "d" in out["excluded_cols"] + assert "c" not in out["excluded_cols"] + assert out["n_excluded"] == 1 + # Matriz solo de las columnas con varianza. + assert len(out["matrix"]) == 2 + assert len(out["matrix"][0]) == 2 + + +def test_menos_de_dos_columnas_con_varianza_vacio_pero_cuenta_excluidas(): + # Solo una columna con varianza (a) + una constante-ausente (d). + mask = { + "a": [1, 0, 1, 0], + "d": [1, 1, 1, 1], + } + out = missingness_correlation(mask) + assert out["columns"] == [] + assert out["matrix"] == [] + assert out["pairs"] == [] + assert out["n_excluded"] == 1 + assert out["excluded_cols"] == ["d"] + + +def test_mask_vacio_todo_vacio(): + out = missingness_correlation({}) + assert out == { + "columns": [], + "matrix": [], + "pairs": [], + "n_excluded": 0, + "excluded_cols": [], + } + + +def test_top_k_limita_pares(): + # 4 columnas con varianza -> 6 pares; top_k=2 deja 2. + mask = { + "a": [1, 0, 1, 0, 0], + "b": [1, 0, 0, 1, 0], + "c": [0, 1, 1, 0, 1], + "d": [1, 1, 0, 0, 1], + } + out = missingness_correlation(mask, top_k=2) + assert len(out["columns"]) == 4 + assert len(out["pairs"]) == 2 + # Ordenados por |corr| desc. + assert abs(out["pairs"][0]["corr"]) >= abs(out["pairs"][1]["corr"]) + + +def test_no_lanza_con_entradas_raras(): + # Valores no-lista y no-dict no deben romper. + assert missingness_correlation(None)["columns"] == [] + mask = { + "a": [1, 0, 1, 0], + "b": [1, 0, 1, 0], + "bad": "not a list", + "empty": [], + } + out = missingness_correlation(mask) + assert out["columns"] == ["a", "b"] diff --git a/python/functions/datascience/missingness_overview.md b/python/functions/datascience/missingness_overview.md new file mode 100644 index 00000000..d33bb8ab --- /dev/null +++ b/python/functions/datascience/missingness_overview.md @@ -0,0 +1,99 @@ +--- +id: missingness_overview_py_datascience +name: missingness_overview +kind: function +lang: py +domain: datascience +version: "1.0.0" +purity: pure +signature: "def missingness_overview(null_mask) -> dict" +description: "Resumen de ausencias a nivel de dataset a partir de una máscara de nulos 0/1 por columna ({col: [1=falta, 0=presente]} alineada por fila). Calcula celdas y porcentaje de datos faltantes, cuántas columnas tienen algún nulo y cuántas filas son completas vs. incompletas. Estilo dict-no-throw del grupo eda: nunca lanza. Lectura defensiva — no-dict o dict vacío devuelve todo a 0; columnas no-lista se tratan como vacías; listas de longitud distinta se alinean a la longitud máxima rellenando la cola corta como presente (0); valores None/no-int cuentan como presente; sin ZeroDivisionError." +tags: [eda, missing, missingness, nulls, profiling, datascience, pure] +uses_functions: [] +uses_types: [] +returns: [] +returns_optional: false +error_type: "" +imports: [] +example: | + from datascience.missingness_overview import missingness_overview + mask = { + "a": [1, 0, 0, 0, 1], + "b": [1, 0, 1, 0, 0], + "c": [0, 0, 0, 0, 1], + } + missingness_overview(mask) + # n_missing_cells=5, missing_cell_pct≈33.33, complete_rows=2, incomplete_rows=3 +tested: true +tests: + - "test_cooccurrence_three_cols_exact" + - "test_empty_dict_all_zero" + - "test_output_keys_contract" + - "test_not_a_dict_returns_zero" + - "test_no_nulls_all_complete" + - "test_none_values_treated_as_present" + - "test_unequal_lengths_pad_with_max" + - "test_columns_present_but_no_rows" + - "test_never_raises_on_garbage" +test_file_path: "python/functions/datascience/missingness_overview_test.py" +file_path: "python/functions/datascience/missingness_overview.py" +params: + - name: null_mask + desc: "Dict {col_name: [int 0/1, ...]} con la máscara de nulos por columna, alineada por fila (1 = el valor falta, 0 = el valor está presente). Normalmente todas las listas tienen la misma longitud = nº de filas. Lectura defensiva: si no es dict o está vacío se devuelve todo a 0; columnas cuyo valor no es lista/tupla se tratan como vacías; listas de longitud distinta se alinean a la longitud máxima (las posiciones inexistentes de las columnas más cortas cuentan como presentes, 0); valores None o no enteros cuentan como presentes." +output: "Dict con exactamente 9 claves, todas siempre presentes (la función nunca lanza): n_rows (longitud de fila = longitud máxima entre columnas, 0 si vacío), n_cols (nº de columnas), n_cols_with_null (columnas con >=1 falta), n_missing_cells (suma total de 1s), missing_cell_pct (0-100 = n_missing_cells / (n_rows*n_cols) * 100), complete_rows (filas sin ninguna falta), incomplete_rows (filas con >=1 falta), complete_pct (0-100), incomplete_pct (0-100). Los porcentajes son 0.0 cuando el denominador es 0 (sin ZeroDivisionError)." +--- + +## Ejemplo + +```python +from datascience.missingness_overview import missingness_overview + +# Máscara de nulos por columna: 1 = falta, 0 = presente, alineada por fila. +mask = { + "a": [1, 0, 0, 0, 1], + "b": [1, 0, 1, 0, 0], + "c": [0, 0, 0, 0, 1], +} + +missingness_overview(mask) +# { +# "n_rows": 5, +# "n_cols": 3, +# "n_cols_with_null": 3, # a, b y c tienen al menos una falta +# "n_missing_cells": 5, # 2 (a) + 2 (b) + 1 (c) +# "missing_cell_pct": 33.33, # 5 / (5*3) * 100 +# "complete_rows": 2, # filas 1 y 3 sin ninguna falta +# "incomplete_rows": 3, # filas 0 (a&b), 2 (b), 4 (a&c) +# "complete_pct": 40.0, # 2 / 5 * 100 +# "incomplete_pct": 60.0, # 3 / 5 * 100 +# } + +missingness_overview({}) +# Todo a 0: {"n_rows": 0, "n_cols": 0, "n_cols_with_null": 0, +# "n_missing_cells": 0, "missing_cell_pct": 0.0, +# "complete_rows": 0, "incomplete_rows": 0, +# "complete_pct": 0.0, "incomplete_pct": 0.0} +``` + +## Cuando usarla + +Úsala al perfilar un dataset cuando ya tienes una máscara de nulos 0/1 por +columna (p. ej. derivada del paso de carga/perfilado del EDA) y quieres la foto +global de ausencias en una llamada: cuánta proporción de celdas falta, cuántas +columnas están afectadas y, sobre todo, cuántas filas quedan completas vs. +incompletas. Es el bloque resumen del capítulo de calidad/missingness de un EDA, +y la base para decidir estrategias de imputación o de borrado de filas. Como es +pura y dict-no-throw, puedes alimentarla con la máscara tal cual sin validarla +antes: entradas malformadas degradan a ceros en vez de romper el pipeline. + +## Gotchas + +- **`n_rows` es la longitud máxima entre columnas.** Con listas de longitud + desigual, las posiciones que faltan en las columnas más cortas se cuentan como + presentes (`0`); no se descartan filas. En el caso normal (todas las listas de + igual longitud) `n_rows` es simplemente esa longitud. +- **Solo el valor exacto `1` cuenta como falta.** `None`, `0`, cadenas y + cualquier otro valor se tratan como presentes. `True` (== 1) también cuenta + como falta por la igualdad. +- **Porcentajes en escala 0-100**, no fracciones. División por cero protegida: + con `n_rows*n_cols == 0` los porcentajes salen `0.0`. diff --git a/python/functions/datascience/missingness_overview.py b/python/functions/datascience/missingness_overview.py new file mode 100644 index 00000000..04fdfec9 --- /dev/null +++ b/python/functions/datascience/missingness_overview.py @@ -0,0 +1,116 @@ +"""Pure EDA helper: dataset-level missingness overview from a 0/1 null mask. + +Part of the `eda` capability group. Consumes a per-column null mask +(``{col_name: [int 0/1, ...]}`` aligned by row, ``1`` = value is missing, +``0`` = value is present) and derives dataset-wide missingness metrics: cell +count and percentage of missing data, how many columns carry any null, and how +many rows are complete vs. incomplete. + +Dict-no-throw style of the `eda` group: it NEVER raises. A non-dict, an empty +dict, malformed columns, ragged lists or non-int cell values all degrade +gracefully to the zero/contract output. Stdlib only. + +Ragged-length policy: columns are allowed to have different lengths. ``n_rows`` +is the **maximum** column length; positions that don't exist in a shorter +column are treated as present (``0``). This keeps the ``n_rows * n_cols`` cell +grid well defined without dropping rows. +""" + + +def _is_missing(value) -> int: + """Return ``1`` iff ``value`` denotes a missing cell, else ``0``. + + Only an exact equality to ``1`` (covers ``int`` ``1`` and ``float`` ``1.0``) + counts as missing. ``None``, ``0``, strings and any other value are treated + as present. The comparison cannot raise for standard inputs. + """ + try: + return 1 if value == 1 else 0 + except Exception: + return 0 + + +def missingness_overview(null_mask) -> dict: + """Summarize dataset-level missingness from a 0/1 null mask. + + Args: + null_mask: Dict ``{col_name: [int 0/1, ...]}`` where each list is aligned + by row (``1`` = missing, ``0`` = present). Lists are normally all the + same length (= number of rows). Defensive: a non-dict or empty dict + returns the all-zero contract; non-list columns are treated as empty; + ragged lists are aligned to the maximum length, padding the missing + tail of shorter columns as present (``0``); ``None`` / non-int cells + count as present. + + Returns: + Dict with exactly these keys, all always present (the function never + raises): ``n_rows``, ``n_cols``, ``n_cols_with_null``, + ``n_missing_cells``, ``missing_cell_pct`` (0-100), ``complete_rows``, + ``incomplete_rows``, ``complete_pct`` (0-100), ``incomplete_pct`` + (0-100). Percentages are ``0.0`` when the denominator is zero (no + ``ZeroDivisionError``). + """ + zero = { + "n_rows": 0, + "n_cols": 0, + "n_cols_with_null": 0, + "n_missing_cells": 0, + "missing_cell_pct": 0.0, + "complete_rows": 0, + "incomplete_rows": 0, + "complete_pct": 0.0, + "incomplete_pct": 0.0, + } + + if not isinstance(null_mask, dict) or not null_mask: + return dict(zero) + + # Normalize every column to a list; non-list columns become empty. + cols = {} + for name, seq in null_mask.items(): + cols[name] = seq if isinstance(seq, (list, tuple)) else [] + + n_cols = len(cols) + lengths = [len(seq) for seq in cols.values()] + n_rows = max(lengths) if lengths else 0 + + if n_rows == 0: + # Columns exist but carry no rows: everything zero except n_cols. + out = dict(zero) + out["n_cols"] = n_cols + return out + + n_missing_cells = 0 + n_cols_with_null = 0 + row_has_missing = [False] * n_rows + + for seq in cols.values(): + col_len = len(seq) + col_has_null = False + for r in range(n_rows): + if r < col_len and _is_missing(seq[r]): + n_missing_cells += 1 + row_has_missing[r] = True + col_has_null = True + if col_has_null: + n_cols_with_null += 1 + + incomplete_rows = sum(1 for flag in row_has_missing if flag) + complete_rows = n_rows - incomplete_rows + + total_cells = n_rows * n_cols + missing_cell_pct = (n_missing_cells / total_cells * 100.0) if total_cells else 0.0 + complete_pct = complete_rows / n_rows * 100.0 + incomplete_pct = incomplete_rows / n_rows * 100.0 + + return { + "n_rows": n_rows, + "n_cols": n_cols, + "n_cols_with_null": n_cols_with_null, + "n_missing_cells": n_missing_cells, + "missing_cell_pct": missing_cell_pct, + "complete_rows": complete_rows, + "incomplete_rows": incomplete_rows, + "complete_pct": complete_pct, + "incomplete_pct": incomplete_pct, + } diff --git a/python/functions/datascience/missingness_overview_test.py b/python/functions/datascience/missingness_overview_test.py new file mode 100644 index 00000000..4cee5452 --- /dev/null +++ b/python/functions/datascience/missingness_overview_test.py @@ -0,0 +1,146 @@ +"""Tests para missingness_overview.""" + +import sys +import os + +import pytest + +sys.path.insert(0, os.path.dirname(__file__)) + +from missingness_overview import missingness_overview + + +# Output contract: every call returns exactly these 9 keys. +EXPECTED_KEYS = { + "n_rows", + "n_cols", + "n_cols_with_null", + "n_missing_cells", + "missing_cell_pct", + "complete_rows", + "incomplete_rows", + "complete_pct", + "incomplete_pct", +} + + +def test_cooccurrence_three_cols_exact(): + # 3 columns, 5 rows. Hand-computed expectations: + # col a missing at rows 0, 4 -> 2 + # col b missing at rows 0, 2 -> 2 + # col c missing at row 4 -> 1 + # n_missing_cells = 5, total_cells = 5*3 = 15 -> 33.333...% + # row 0 (a&b co-occur) -> incomplete + # row 1 (all present) -> complete + # row 2 (b only) -> incomplete + # row 3 (all present) -> complete + # row 4 (a&c co-occur) -> incomplete + mask = { + "a": [1, 0, 0, 0, 1], + "b": [1, 0, 1, 0, 0], + "c": [0, 0, 0, 0, 1], + } + out = missingness_overview(mask) + assert out["n_rows"] == 5 + assert out["n_cols"] == 3 + assert out["n_cols_with_null"] == 3 + assert out["n_missing_cells"] == 5 + assert out["missing_cell_pct"] == pytest.approx(33.33333333, abs=1e-6) + assert out["complete_rows"] == 2 + assert out["incomplete_rows"] == 3 + assert out["complete_pct"] == pytest.approx(40.0) + assert out["incomplete_pct"] == pytest.approx(60.0) + + +def test_empty_dict_all_zero(): + out = missingness_overview({}) + assert out == { + "n_rows": 0, + "n_cols": 0, + "n_cols_with_null": 0, + "n_missing_cells": 0, + "missing_cell_pct": 0.0, + "complete_rows": 0, + "incomplete_rows": 0, + "complete_pct": 0.0, + "incomplete_pct": 0.0, + } + + +def test_output_keys_contract(): + # The 9-key contract holds even for the garbage/zero path. + assert set(missingness_overview({}).keys()) == EXPECTED_KEYS + assert set(missingness_overview({"a": [1, 0]}).keys()) == EXPECTED_KEYS + + +def test_not_a_dict_returns_zero(): + for bad in (None, [1, 0, 1], 42, "nope", 3.14): + out = missingness_overview(bad) + assert out["n_rows"] == 0 + assert out["n_cols"] == 0 + assert out["n_missing_cells"] == 0 + assert out["missing_cell_pct"] == 0.0 + + +def test_no_nulls_all_complete(): + mask = {"a": [0, 0, 0], "b": [0, 0, 0]} + out = missingness_overview(mask) + assert out["n_rows"] == 3 + assert out["n_cols"] == 2 + assert out["n_cols_with_null"] == 0 + assert out["n_missing_cells"] == 0 + assert out["missing_cell_pct"] == 0.0 + assert out["complete_rows"] == 3 + assert out["incomplete_rows"] == 0 + assert out["complete_pct"] == pytest.approx(100.0) + assert out["incomplete_pct"] == pytest.approx(0.0) + + +def test_none_values_treated_as_present(): + # None and other non-1 values count as present (0). + mask = {"a": [None, 1, None, "x", 0]} + out = missingness_overview(mask) + assert out["n_rows"] == 5 + assert out["n_cols"] == 1 + assert out["n_missing_cells"] == 1 # only the explicit 1 at row 1 + assert out["n_cols_with_null"] == 1 + assert out["complete_rows"] == 4 + assert out["incomplete_rows"] == 1 + + +def test_unequal_lengths_pad_with_max(): + # Ragged lists: n_rows = max length; shorter column padded as present. + # a = [1, 1] -> missing at rows 0, 1 + # b = [0] -> row 1 padded to present + # n_rows = 2, n_cols = 2, total_cells = 4, n_missing_cells = 2 -> 50% + mask = {"a": [1, 1], "b": [0]} + out = missingness_overview(mask) + assert out["n_rows"] == 2 + assert out["n_cols"] == 2 + assert out["n_cols_with_null"] == 1 + assert out["n_missing_cells"] == 2 + assert out["missing_cell_pct"] == pytest.approx(50.0) + assert out["complete_rows"] == 0 + assert out["incomplete_rows"] == 2 + assert out["incomplete_pct"] == pytest.approx(100.0) + + +def test_columns_present_but_no_rows(): + # Columns exist but all empty -> zero metrics, n_cols preserved. + out = missingness_overview({"a": [], "b": []}) + assert out["n_rows"] == 0 + assert out["n_cols"] == 2 + assert out["n_missing_cells"] == 0 + assert out["missing_cell_pct"] == 0.0 + assert out["complete_pct"] == 0.0 + + +def test_never_raises_on_garbage(): + # Non-list column values, mixed junk -> must not raise. + mask = {"a": "not a list", "b": 123, "c": [1, 0, 1]} + out = missingness_overview(mask) + assert set(out.keys()) == EXPECTED_KEYS + assert out["n_rows"] == 3 + assert out["n_cols"] == 3 + assert out["n_missing_cells"] == 2 # only col c contributes + assert out["n_cols_with_null"] == 1 diff --git a/python/functions/datascience/missingness_rank_bar_figure.md b/python/functions/datascience/missingness_rank_bar_figure.md new file mode 100644 index 00000000..c75f4613 --- /dev/null +++ b/python/functions/datascience/missingness_rank_bar_figure.md @@ -0,0 +1,93 @@ +--- +id: missingness_rank_bar_figure_py_datascience +name: missingness_rank_bar_figure +kind: function +lang: py +domain: datascience +version: "1.0.0" +purity: impure +signature: "def missingness_rank_bar_figure(names, pcts, title=\"% de valores faltantes por columna\") -> \"matplotlib.figure.Figure\"" +description: "Construye una figura matplotlib de barras horizontales que ordena las columnas de un dataset por su porcentaje de valores faltantes (0-100), la mayor arriba, etiquetando cada barra con su NN.N% al final. Usa ax.barh, eje X fijo 0-100 y labels truncados a ~22 chars. Devuelve un matplotlib.figure.Figure listo para rasterizar por el renderer del informe EDA (capítulo de datos faltantes). Backend Agg sin pyplot global; defensivo ante listas vacías, longitudes desiguales o valores no numéricos (nunca lanza)." +tags: [eda, missing, missingness, ranking, bar, barh, matplotlib, figure, visualization, datascience, impure] +uses_functions: [] +uses_types: [] +returns: [] +returns_optional: false +error_type: "error_go_core" +imports: [matplotlib] +example: | + from datascience.missingness_rank_bar_figure import missingness_rank_bar_figure + names = ["edad", "ingresos", "ciudad", "email"] + pcts = [12.5, 40.0, 3.2, 0.0] + fig = missingness_rank_bar_figure(names, pcts, title="% de valores faltantes por columna") +tested: true +tests: + - "test_returns_figure_with_axes" + - "test_sorted_descending_largest_on_top" + - "test_empty_lists_do_not_raise_and_returns_figure" + - "test_xlim_is_zero_to_hundred" + - "test_length_mismatch_and_non_numeric_are_handled" +test_file_path: "python/functions/datascience/missingness_rank_bar_figure_test.py" +file_path: "python/functions/datascience/missingness_rank_bar_figure.py" +params: + - name: names + desc: "Lista de nombres de columna. Puede venir vacía (devuelve figura \"sin datos faltantes\"). Los items se convierten a str y se truncan a ~22 chars con elipsis para las etiquetas del eje Y; los originales no se mutan." + - name: pcts + desc: "Lista paralela a names con el % de nulos en [0,100]. Valores None, NaN o no numéricos se coercen a 0.0 y los negativos se recortan a 0. Si len(names) != len(pcts) se recorta al menor de ambos para no romper." + - name: title + desc: "Título de la figura. Se trunca a ~60 chars con elipsis si es muy largo. Default \"% de valores faltantes por columna\"." +output: "Un matplotlib.figure.Figure (figsize 6.4 x alto adaptativo según nº de barras, dpi 150) con un Axes de barras horizontales (ax.barh) ordenadas por % descendente, la mayor arriba. Eje X fijado a [0,100] con label \"% faltante\", etiquetas del eje Y truncadas a ~22 chars, y cada barra anotada con su NN.N% al final. Si names o pcts vienen vacíos devuelve una Figure con texto centrado \"sin datos faltantes\"; cualquier error inesperado se captura y devuelve una Figure con el mensaje de error (nunca lanza). El caller rasteriza/cierra la figura; la función no la muestra ni la guarda." +--- + +## Ejemplo + +```python +from datascience.missingness_rank_bar_figure import missingness_rank_bar_figure + +# % de nulos por columna (p. ej. (df.isnull().mean() * 100). +names = ["edad", "ingresos", "ciudad", "email"] +pcts = [12.5, 40.0, 3.2, 0.0] + +fig = missingness_rank_bar_figure( + names, + pcts, + title="% de valores faltantes por columna", +) + +# ingresos (40.0%) queda arriba; email (0.0%) abajo. +# El renderer del informe lo rasteriza; aquí solo persistimos para inspección. +fig.savefig("/tmp/missingness_rank.png") +``` + +## Cuando usarla + +Úsala al abrir el capítulo de datos faltantes de un informe EDA para responder +"¿qué columnas están más incompletas?" de un vistazo. Pásale los nombres de +columna y el % de nulos de cada una (`(df.isnull().mean() * 100).round(1)`); la +función se encarga de ordenar de mayor a menor y poner la peor arriba. Es la +pareja "magnitud" del heatmap de co-ocurrencia: las barras dicen *cuánto* falta +en cada columna, el heatmap dice *si esas ausencias están relacionadas* entre +columnas. + +## Gotchas + +- **Impura por matplotlib.** Toca la maquinaria de render. Usa el backend `Agg` + y la API orientada a objetos `Figure`/`add_subplot` — NUNCA `pyplot.*` aquí, + para no tocar el estado global ni filtrar figuras entre llamadas. `pyplot` NO + es thread-safe; esta función evita ese riesgo construyendo el `Figure` + directamente, así que es segura de llamar en bucle desde el renderer. +- **El caller cierra la figura.** Devuelve el `Figure` pero no lo muestra ni lo + guarda. Quien la consume debe rasterizarla y luego liberarla + (`matplotlib.pyplot.close(fig)`) para no acumular memoria en lotes grandes. +- **Espera porcentajes 0-100, no fracciones 0-1.** El eje X está fijado a + `[0, 100]`. Si pasas fracciones (`0.4` en vez de `40.0`) las barras saldrán + pegadas al origen. Multiplica por 100 antes de llamar. +- **Alto adaptativo.** La altura de la figura crece con el número de barras + (hasta un tope) para que reports con muchas columnas sigan legibles; aun así, + conviene filtrar a las columnas con algún nulo antes de llamar para no listar + decenas de barras a 0%. +- **Defensiva, nunca lanza.** Listas vacías, longitudes desiguales, valores + `None`/`NaN`/no numéricos o cualquier error inesperado se manejan sin propagar: + en el peor caso devuelve una `Figure` con "sin datos faltantes" o con el texto + del error. No envuelvas la llamada en try/except por miedo a un raise — no lo + hay. diff --git a/python/functions/datascience/missingness_rank_bar_figure.py b/python/functions/datascience/missingness_rank_bar_figure.py new file mode 100644 index 00000000..58643ff3 --- /dev/null +++ b/python/functions/datascience/missingness_rank_bar_figure.py @@ -0,0 +1,150 @@ +"""Impure EDA helper: ranked bar figure of missing-value share (`eda` group). + +Builds a horizontal bar chart ranking the columns of a dataset by their +percentage of missing values (0-100), largest at the top, each bar labelled with +its ``NN.N%`` at the end. Returns a ready-to-rasterize +``matplotlib.figure.Figure``; it never shows nor saves it. + +Impure because it touches matplotlib's rendering machinery. It uses the headless +Agg backend and the object-oriented ``Figure`` API (no ``pyplot``) so it leaks no +global state and is safe to call repeatedly from a report renderer. +""" + +import matplotlib + +matplotlib.use("Agg") + +from matplotlib.figure import Figure # noqa: E402 + +# Muted gray for secondary text (no-data / fallback messages). +_MUTED_TEXT = "#5f6b7a" +# Soft red for the error fallback message. +_ERROR_TEXT = "#b00020" +# Bar fill — a calm blue that reads well on white at report size. +_BAR_COLOR = "#4C72B0" + + +def _truncate(text, width: int = 22) -> str: + """Truncate ``text`` to ``width`` chars, appending an ellipsis if cut.""" + s = "" if text is None else str(text) + if len(s) <= width: + return s + if width <= 1: + return s[:width] + return s[: width - 1] + "…" + + +def _message_figure(message: str, color: str = _MUTED_TEXT) -> "Figure": + """Return a fallback ``Figure`` carrying a single centered message.""" + fig = Figure(figsize=(6.4, 4.0), dpi=150) + ax = fig.add_subplot(111) + ax.axis("off") + ax.text( + 0.5, + 0.5, + message, + ha="center", + va="center", + fontsize=12, + color=color, + wrap=True, + transform=ax.transAxes, + ) + fig.tight_layout() + return fig + + +def missingness_rank_bar_figure( + names, + pcts, + title: str = "% de valores faltantes por columna", +) -> "matplotlib.figure.Figure": + """Build a horizontal ranked bar figure of missing-value share per column. + + Pairs each column name with its missing percentage, sorts by percentage + descending and draws horizontal bars with the largest at the top. The X axis + is pinned to ``[0, 100]`` so bars are comparable across reports, each bar is + annotated with its ``NN.N%`` at the end, and the Y tick labels are truncated + to ~22 chars. + + The function is fully defensive: empty/mismatched/non-numeric input never + raises. When there is nothing valid to draw it returns a ``Figure`` carrying + a centered "sin datos faltantes" message, and any unexpected error is caught + and turned into a fallback ``Figure`` carrying the error text. + + Args: + names: List of column names. May be empty. Items are stringified and + truncated for display; the originals are not mutated. + pcts: List parallel to ``names`` of missing-value percentages in + ``[0, 100]``. Non-numeric/``None`` values are coerced to ``0.0`` and + negatives are clamped to ``0``. The list is truncated to + ``min(len(names), len(pcts))`` so a length mismatch never crashes. + title: Figure title. Default "% de valores faltantes por columna". + + Returns: + A ``matplotlib.figure.Figure`` with a single horizontal-bar Axes. The + caller is responsible for rasterizing/closing it. + """ + try: + if ( + not isinstance(names, (list, tuple)) + or not isinstance(pcts, (list, tuple)) + or len(names) == 0 + or len(pcts) == 0 + ): + return _message_figure("sin datos faltantes") + + # --- Pair names with coerced percentages, tolerating length mismatch. + pairs = [] + for name, pct in zip(names, pcts): + try: + val = float(pct) + except (TypeError, ValueError): + val = 0.0 + if val != val: # NaN guard. + val = 0.0 + val = max(0.0, val) + pairs.append((name, val)) + + if not pairs: + return _message_figure("sin datos faltantes") + + # Sort by percentage descending; barh draws bottom-up, so the largest + # ends at the top when we reverse the order before plotting. + pairs.sort(key=lambda p: p[1], reverse=True) + ordered = list(reversed(pairs)) # smallest first -> largest on top. + + labels = [_truncate(name, 22) for name, _ in ordered] + values = [val for _, val in ordered] + y_pos = range(len(ordered)) + + # Height scales with the number of bars so dense reports stay readable. + height = max(2.4, min(0.4 * len(ordered) + 1.2, 14.0)) + fig = Figure(figsize=(6.4, height), dpi=150) + ax = fig.add_subplot(111) + + ax.barh(list(y_pos), values, color=_BAR_COLOR, edgecolor="white") + ax.set_yticks(list(y_pos)) + ax.set_yticklabels(labels, fontsize=8) + ax.set_xlim(0, 100) + ax.set_xlabel("% faltante", fontsize=9) + + # Annotate each bar with its percentage at the end of the bar. + for y, val in zip(y_pos, values): + ax.text( + min(val + 1.5, 99.0), + y, + f"{val:.1f}%", + va="center", + ha="left" if val < 90 else "right", + fontsize=7, + color="#202020", + ) + + if title: + ax.set_title(_truncate(title, 60), fontsize=12, loc="left", pad=10) + + fig.tight_layout() + return fig + except Exception as exc: # noqa: BLE001 — never raise from a figure builder. + return _message_figure(f"error al dibujar barras: {exc}", color=_ERROR_TEXT) diff --git a/python/functions/datascience/missingness_rank_bar_figure_test.py b/python/functions/datascience/missingness_rank_bar_figure_test.py new file mode 100644 index 00000000..42b41b19 --- /dev/null +++ b/python/functions/datascience/missingness_rank_bar_figure_test.py @@ -0,0 +1,64 @@ +"""Tests para missingness_rank_bar_figure (barras de % faltante, grupo eda). + +Usa el backend Agg sin pyplot; no muestra ni guarda figuras. Cada test cierra +explícitamente la Figure construida (matplotlib.pyplot.close) para no acumular +estado entre tests. +""" + +import matplotlib + +matplotlib.use("Agg") + +import matplotlib.pyplot as plt # noqa: E402 +from matplotlib.figure import Figure # noqa: E402 + +from missingness_rank_bar_figure import missingness_rank_bar_figure + + +def test_returns_figure_with_axes(): + names = ["edad", "ingresos", "ciudad"] + pcts = [12.5, 40.0, 3.2] + fig = missingness_rank_bar_figure(names, pcts, title="faltantes") + assert isinstance(fig, Figure) + assert len(fig.axes) >= 1 + plt.close(fig) + + +def test_sorted_descending_largest_on_top(): + names = ["a", "b", "c"] + pcts = [10.0, 50.0, 25.0] + fig = missingness_rank_bar_figure(names, pcts) + ax = fig.axes[0] + # barh dibuja de abajo arriba; la mayor (50, "b") debe quedar arriba (mayor y). + bars = ax.patches + # El último parche (mayor índice y) corresponde a la barra superior. + widths = [b.get_width() for b in bars] + assert max(widths) == 50.0 + # La barra con la mayor anchura es la de mayor coordenada y (arriba). + top_bar = max(bars, key=lambda b: b.get_y()) + assert top_bar.get_width() == 50.0 + plt.close(fig) + + +def test_empty_lists_do_not_raise_and_returns_figure(): + fig = missingness_rank_bar_figure([], [], title="vacía") + assert isinstance(fig, Figure) + assert len(fig.axes) >= 1 + plt.close(fig) + + +def test_xlim_is_zero_to_hundred(): + fig = missingness_rank_bar_figure(["a"], [42.0]) + ax = fig.axes[0] + assert ax.get_xlim() == (0.0, 100.0) + plt.close(fig) + + +def test_length_mismatch_and_non_numeric_are_handled(): + # Más names que pcts + un pct None -> zip recorta y None se coacciona a 0. + names = ["a", "b", "c"] + pcts = [None, 30.0] + fig = missingness_rank_bar_figure(names, pcts) + assert isinstance(fig, Figure) + assert len(fig.axes) >= 1 + plt.close(fig) diff --git a/python/functions/datascience/missingness_row_patterns.md b/python/functions/datascience/missingness_row_patterns.md new file mode 100644 index 00000000..ad1437b5 --- /dev/null +++ b/python/functions/datascience/missingness_row_patterns.md @@ -0,0 +1,65 @@ +--- +name: missingness_row_patterns +kind: function +lang: py +domain: datascience +version: "1.0.0" +purity: pure +signature: "def missingness_row_patterns(null_mask, top_n=10) -> dict" +description: "Agrupa las filas de un dataset por su patron de ausencias (estilo matriz de missingno): para cada fila, el patron es la tupla ORDENADA de columnas que faltan en esa fila (las que tienen 1 en el null_mask). Cuenta la frecuencia de cada patron distinto, incluido el patron vacio (fila completa). Devuelve el top_n por frecuencia con su pct sobre el total. Pura, lectura defensiva, NUNCA lanza; {} -> n_rows 0." +tags: [eda, missingness, missingno, patterns, profiling, datascience, data-quality] +params: + - name: null_mask + desc: "Dict {col: [0/1, ...]} alineado por fila, donde 1 = la celda falta en esa fila y 0 = presente. Todas las columnas deberian tener la misma longitud (una entrada por fila); si difieren, n_rows es la lista mas larga y las celdas fuera de rango cuentan como presentes. Las claves se ordenan por str(col) para canonizar el patron. {} (o no-dict) -> n_rows 0." + - name: top_n + desc: "Maximo de patrones devueltos en `patterns`, rankeados por n_rows desc (desempate: menos columnas primero, luego nombres de columna). El recuento total de patrones distintos siempre se reporta en `n_patterns`, no se trunca. Default 10. Valores negativos -> 0; no-int -> 10." +output: "Dict {n_rows: int (filas totales), n_patterns: int (patrones distintos, incluye el patron vacio = fila completa), complete_rows: int (filas con patron vacio, nada falta), patterns: lista del top_n ordenada por n_rows desc con [{missing_cols: [col,...] (vacio = fila completa), n_rows: int, pct: float 0-100 sobre n_rows total, redondeado a 2 decimales}]}. Para {} devuelve n_rows 0 y patterns []. NUNCA lanza." +uses_functions: [] +uses_types: [] +returns: [] +returns_optional: false +error_type: "" +imports: [] +tested: true +tests: ["test_patron_dominante_completas_singleton", "test_mask_vacio", "test_top_n_trunca_pero_cuenta_todos"] +test_file_path: "python/functions/datascience/missingness_row_patterns_test.py" +file_path: "python/functions/datascience/missingness_row_patterns.py" +--- + +## Ejemplo + +```python +import sys, os +sys.path.insert(0, os.path.join("python", "functions")) +from datascience.missingness_row_patterns import missingness_row_patterns + +# null_mask alineado por fila: 1 = la celda falta en esa fila. +null_mask = { + "A": [1, 1, 1, 1, 0, 0, 0, 0, 0, 0], + "B": [1, 1, 1, 1, 0, 0, 0, 0, 0, 0], + "C": [0, 0, 0, 0, 0, 0, 0, 0, 0, 1], +} +out = missingness_row_patterns(null_mask, top_n=10) +print(out["n_rows"], out["n_patterns"], out["complete_rows"]) # 10 3 5 +for p in out["patterns"]: + label = p["missing_cols"] or "(fila completa)" + print(label, p["n_rows"], p["pct"]) +# (fila completa) 5 50.0 +# ['A', 'B'] 4 40.0 +# ['C'] 1 10.0 +``` + +## Cuando usarla + +- Usala en el capitulo de calidad/ausencias de `AutomaticEDA` para mostrar la "matriz de patrones de missingno": en vez de pintar celda a celda, resume que combinaciones de columnas se quedan en blanco juntas y con que frecuencia. +- Cuando ya tengas el null_mask por columna (1=falta) y quieras detectar co-ausencia estructural ("A y B siempre faltan juntas") antes de decidir una imputacion o un drop conjunto de columnas. +- Cuando necesites una tabla compacta "patron -> nº filas -> pct" para un report o un grafico de barras de los patrones de ausencia mas comunes, separando ademas cuantas filas estan completas (`complete_rows`). + +## Gotchas + +- Funcion pura, sin I/O y determinista. Lectura defensiva: `{}` o un no-dict devuelven `n_rows` 0 con `patterns` []. NUNCA lanza. +- El patron vacio (fila completa, `missing_cols=[]`) SI cuenta como patron: aparece en `n_patterns` y puede aparecer en `patterns`. El consumidor lo etiqueta como "(fila completa)". +- `pct` es sobre `n_rows` total (0-100), redondeado a 2 decimales. La suma de los `pct` de TODOS los patrones es 100; si `top_n` trunca, los `pct` mostrados sumaran menos. +- Las columnas se ordenan por `str(col)` para canonizar cada patron, asi `{A,B}` y `{B,A}` colapsan al mismo patron `["A", "B"]`. +- Una celda cuenta como ausente solo si vale 1 (`int(cell) == 1`); 0, None y valores no numericos se tratan como presentes. +- Si las listas de columnas tienen longitudes distintas, `n_rows` es la mas larga y las posiciones fuera de rango de una columna corta cuentan como presentes (0). diff --git a/python/functions/datascience/missingness_row_patterns.py b/python/functions/datascience/missingness_row_patterns.py new file mode 100644 index 00000000..7bf034f8 --- /dev/null +++ b/python/functions/datascience/missingness_row_patterns.py @@ -0,0 +1,107 @@ +"""missingness_row_patterns — distinct per-row missingness patterns (missingno matrix style). + +Pure function: no I/O, deterministic, NEVER raises. Given a per-column null mask +aligned by row ({col: [0/1, ...]}, 1 = missing), it groups rows by their missing +"pattern" — the sorted tuple of column names that are missing in that row — and +counts how often each distinct pattern occurs. + +This mirrors the missingno matrix idea: instead of plotting per-cell nullity, it +collapses each row to the SET of columns it lacks, surfacing co-missing structure +(e.g. "A and B always go missing together"). The empty pattern (a fully complete +row) is a first-class pattern and may appear in the result with missing_cols=[]; +the caller labels it "(fila completa)". +""" + + +def _is_missing(cell) -> bool: + """A cell counts as missing when it equals 1 (truthy 0/1 mask). + + None / 0 / non-numeric are treated as present. Defensive: never raises. + """ + try: + return int(cell) == 1 + except (TypeError, ValueError): + return bool(cell) + + +def missingness_row_patterns(null_mask, top_n=10) -> dict: + """Count distinct per-row missingness patterns from a column null mask. + + For each row, its pattern is the sorted tuple of column names missing in that + row (the columns whose value is 1). The frequency of each distinct pattern is + counted, including the empty pattern (a complete row with nothing missing). + + Args: + null_mask: Dict {col: [0/1, ...]} aligned by row, where 1 means the cell + is missing in that row. Read defensively; columns with differing + lengths are tolerated (n_rows is the longest list; out-of-range cells + count as present). Empty dict -> n_rows 0. + top_n: Maximum number of patterns returned in `patterns`, ranked by + n_rows desc (tiebreak: fewer columns first, then column names). The + full count of distinct patterns is always reported in `n_patterns`. + + Returns: + Dict: + { + "n_rows": int, # total rows + "n_patterns": int, # distinct patterns (incl. the empty pattern) + "complete_rows": int, # rows with the empty pattern (nothing missing) + "patterns": [ # top_n patterns, n_rows desc + {"missing_cols": [col, ...], "n_rows": int, "pct": float} # [] = complete row + ], + } + For {} (or a non-dict) returns n_rows 0 and patterns []. NEVER raises. + """ + empty = {"n_rows": 0, "n_patterns": 0, "complete_rows": 0, "patterns": []} + if not isinstance(null_mask, dict) or not null_mask: + return empty + + # Stable, canonical column order so each row's pattern tuple is sorted. + items = sorted(null_mask.items(), key=lambda kv: str(kv[0])) + names = [str(k) for k, _ in items] + lists = [v if isinstance(v, (list, tuple)) else [] for _, v in items] + + n_rows = max((len(lst) for lst in lists), default=0) + if n_rows == 0: + return empty + + # Defensive parsing of top_n. + try: + limit = int(top_n) + except (TypeError, ValueError): + limit = 10 + if limit < 0: + limit = 0 + + counts: dict = {} + n_cols = len(names) + for r in range(n_rows): + # names is sorted, so iterating in order yields an already-sorted tuple. + pattern = tuple( + names[c] + for c in range(n_cols) + if r < len(lists[c]) and _is_missing(lists[c][r]) + ) + counts[pattern] = counts.get(pattern, 0) + 1 + + complete_rows = counts.get((), 0) + n_patterns = len(counts) + + # Rank: n_rows desc, then fewer columns first, then column names (deterministic). + ordered = sorted(counts.items(), key=lambda kv: (-kv[1], len(kv[0]), kv[0])) + + patterns = [ + { + "missing_cols": list(pat), + "n_rows": cnt, + "pct": round(100.0 * cnt / n_rows, 2), + } + for pat, cnt in ordered[:limit] + ] + + return { + "n_rows": n_rows, + "n_patterns": n_patterns, + "complete_rows": complete_rows, + "patterns": patterns, + } diff --git a/python/functions/datascience/missingness_row_patterns_test.py b/python/functions/datascience/missingness_row_patterns_test.py new file mode 100644 index 00000000..28e97068 --- /dev/null +++ b/python/functions/datascience/missingness_row_patterns_test.py @@ -0,0 +1,87 @@ +"""Tests para missingness_row_patterns.""" + +import os +import sys + +sys.path.insert(0, os.path.dirname(__file__)) + +from missingness_row_patterns import missingness_row_patterns + +_EXPECTED_KEYS = {"n_rows", "n_patterns", "complete_rows", "patterns"} + + +def test_patron_dominante_completas_singleton(): + """Golden: {A,B} co-faltan en 4 filas + 5 filas completas + 1 singleton {C}.""" + # 10 filas. A y B faltan juntas en las filas 0-3; filas 4-8 completas; + # la fila 9 solo le falta C. + null_mask = { + "A": [1, 1, 1, 1, 0, 0, 0, 0, 0, 0], + "B": [1, 1, 1, 1, 0, 0, 0, 0, 0, 0], + "C": [0, 0, 0, 0, 0, 0, 0, 0, 0, 1], + } + out = missingness_row_patterns(null_mask) + + assert set(out.keys()) == _EXPECTED_KEYS + assert out["n_rows"] == 10 + # 3 patrones distintos: (A,B), () y (C,). + assert out["n_patterns"] == 3 + # 5 filas completas (filas 4-8). + assert out["complete_rows"] == 5 + + # Orden: n_rows desc; desempate menos columnas primero. + # () tiene 5 filas, (A,B) 4, (C,) 1. + pats = out["patterns"] + assert len(pats) == 3 + + assert pats[0]["missing_cols"] == [] + assert pats[0]["n_rows"] == 5 + assert pats[0]["pct"] == 50.0 + + assert pats[1]["missing_cols"] == ["A", "B"] + assert pats[1]["n_rows"] == 4 + assert pats[1]["pct"] == 40.0 + + assert pats[2]["missing_cols"] == ["C"] + assert pats[2]["n_rows"] == 1 + assert pats[2]["pct"] == 10.0 + + # Tipos de salida. + assert isinstance(out["n_rows"], int) + assert isinstance(pats[0]["pct"], float) + + +def test_mask_vacio(): + """{} -> n_rows 0, sin patrones, nunca lanza.""" + out = missingness_row_patterns({}) + assert out == { + "n_rows": 0, + "n_patterns": 0, + "complete_rows": 0, + "patterns": [], + } + # No dict / None tambien degradan a vacio sin lanzar. + assert missingness_row_patterns(None)["n_rows"] == 0 + # Columnas presentes pero listas vacias -> n_rows 0. + assert missingness_row_patterns({"A": [], "B": []})["patterns"] == [] + + +def test_top_n_trunca_pero_cuenta_todos(): + """top_n limita `patterns`, pero n_patterns reporta TODOS los distintos.""" + null_mask = { + "A": [0, 1, 1, 0, 1], + "B": [0, 0, 0, 1, 1], + "C": [0, 0, 0, 0, 1], + } + # Filas: () (A,) (A,) (B,) (A,B,C) + out = missingness_row_patterns(null_mask, top_n=2) + + assert out["n_rows"] == 5 + assert out["n_patterns"] == 4 # (), (A,), (B,), (A,B,C) + assert out["complete_rows"] == 1 + # Solo 2 patrones devueltos pese a haber 4. + assert len(out["patterns"]) == 2 + # (A,) domina con 2 filas; desempate del 2o entre los de 1 fila -> () (0 cols). + assert out["patterns"][0]["missing_cols"] == ["A"] + assert out["patterns"][0]["n_rows"] == 2 + assert out["patterns"][1]["missing_cols"] == [] + assert out["patterns"][1]["n_rows"] == 1 From 9c1b7dd0f3a088f44d241febbb4714bb2d8428ed Mon Sep 17 00:00:00 2001 From: Egutierrez <egutierrez@dead.dd> Date: Tue, 30 Jun 2026 20:39:59 +0200 Subject: [PATCH 41/53] =?UTF-8?q?feat(papers):=20render=5Fpaper=5Fpdf=20(M?= =?UTF-8?q?arkdown=20IMRaD=20=E2=86=92=20PDF)=20+=20agente=20paper-reviewe?= =?UTF-8?q?r?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Subsistema papers/: pieza de entrega + control de calidad. - render_paper_pdf_py_datascience (Python, impure, dominio datascience, grupo `papers`): convierte papers/<slug>/paper.md (frontmatter YAML + cuerpo IMRaD) en papers/<slug>/out/paper.pdf. Reutiliza el motor de paginación de flujo del paquete automatic_eda (matplotlib PdfPages, el mismo PDF móvil A5 de los informes EDA) — no reimplementa paginación ni toca matplotlib, y no añade dependencias. Cada sección IMRaD (# H1) → un Chapter en página nueva; portada desde el frontmatter (title/authors/date europea/abstract); detecta las imágenes Markdown ![alt](src) que el motor no entiende y las parte en bloques Image resueltos contra base_dir y base_dir/figures/. dict-no-throw estricto. 5 tests verdes (golden + edges: sin frontmatter, path inexistente, figura inexistente, ruta directa al .md). - .claude/agents/paper-reviewer: revisor académico adversarial read-only (gate anti paper-mill). Puntúa novedad/rigor/reproducibilidad/validez (0-5), intenta refutar cada claim contra la evidencia citada, detecta HARKing contra el preregistration.md, exige limitaciones declaradas y claims ≤ evidencia, y emite veredicto estructurado JSON (accept|major_revision|reject) con default conservador. Tools: Read, Grep, Glob, Bash (sin Edit/Write: solo juzga). Diseño completo: reports/0001-2026-06-30-papers-system-design.md (agente C). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> --- .claude/agents/paper-reviewer/SKILL.md | 141 +++++++++ python/functions/datascience/__init__.py | 2 + .../functions/datascience/render_paper_pdf.md | 96 ++++++ .../functions/datascience/render_paper_pdf.py | 297 ++++++++++++++++++ .../datascience/render_paper_pdf_test.py | 118 +++++++ 5 files changed, 654 insertions(+) create mode 100644 .claude/agents/paper-reviewer/SKILL.md create mode 100644 python/functions/datascience/render_paper_pdf.md create mode 100644 python/functions/datascience/render_paper_pdf.py create mode 100644 python/functions/datascience/render_paper_pdf_test.py diff --git a/.claude/agents/paper-reviewer/SKILL.md b/.claude/agents/paper-reviewer/SKILL.md new file mode 100644 index 00000000..f0652e34 --- /dev/null +++ b/.claude/agents/paper-reviewer/SKILL.md @@ -0,0 +1,141 @@ +--- +name: paper-reviewer +description: "Revisor académico adversarial (read-only) para los papers del subsistema `papers/`. Recibe el directorio de un paper (`papers/<slug>/`) y su `preregistration.md`, y lo juzga sin piedad: puntúa novedad, rigor, reproducibilidad y validez (0-5 cada uno), intenta REFUTAR cada claim contra la evidencia citada, detecta HARKing contra el pre-registro, y emite un veredicto estructurado (accept|major_revision|reject) con default conservador. Es el gate anti paper-mill: NO modifica el paper, solo lo evalúa." +model: opus +tools: Read, Grep, Glob, Bash +--- + +# Agente Paper-Reviewer — peer review adversarial + +Eres un revisor académico **hostil pero justo**. Tu trabajo NO es ayudar al autor a sentirse bien: es proteger la integridad del registro científico. Asumes la posición de un revisor de conferencia top que ha visto cientos de papers inflados y sabe oler el humo. Por defecto **desconfías** de cada afirmación hasta que la evidencia citada la sostenga. Eres específico, citas líneas y archivos, y no rellenas con elogios. + +Este agente es el **gate anti paper-mill** del subsistema `papers/`. El riesgo que combates: papers que *parecen* rigurosos (estructura IMRaD impecable, lenguaje académico, tablas bonitas) pero sin sustancia — hipótesis que no podían fallar, estadística de teatro, claims que exceden la evidencia, análisis inventados después de ver los datos. Si no hubo riesgo real de refutación, no es un paper. + +--- + +## REGLA FUNDAMENTAL: read-only, solo juzgas + +- **Lectura:** `paper.md`, `preregistration.md`, `references.md`/`.bib`, y todo lo que haya en `experiments/`, `data/`, `figures/`, `reviews/` del paper. +- **Escritura:** NINGUNA. No tienes Edit ni Write. No modificas el paper, no arreglas su prosa, no corriges sus tablas. Solo emites un veredicto. +- **Bash es read-only:** úsalo para inspeccionar evidencia (`ls`, `cat`, `head`, `wc`, `grep`, re-correr un script de análisis que YA exista en `experiments/` para verificar un número reportado, contar filas de un dataset, comprobar que una figura referenciada existe). NUNCA escribas archivos, NUNCA borres, NUNCA mutes estado externo (sin red con efectos, sin deploys). + +--- + +## Input + +Recibes el path de un directorio de paper: + +- `paper_dir` (ej. `papers/0001-bucle-reactivo-calls`). Dentro esperas al menos `paper.md`; idealmente también `preregistration.md`, `experiments/`, `data/`, `figures/`. + +Si falta `paper.md`, reporta que no hay paper que revisar y sal. Si falta `preregistration.md`, NO es excusa para aprobar: la ausencia de pre-registro es en sí misma una **amenaza grave a la validez** (no puedes distinguir análisis confirmatorios de exploratorios) y debe bajar el eje de rigor y reproducibilidad. + +--- + +## Algoritmo de revisión + +### 1. Lee todo el material primero + +- `paper.md` completo (frontmatter + cuerpo IMRaD). +- `preregistration.md` (H0/H1, plan de análisis congelado, timestamp/hash si lo tiene). +- Inventaria la evidencia: `ls -R experiments/ data/ figures/`. Anota qué tablas, figuras, scripts y datasets existen REALMENTE en disco. +- Si hay `reviews/` previos, léelos para no repetir y para ver si el autor respondió a críticas anteriores. + +No puntúes nada hasta haber leído el material. Una revisión sin abrir la evidencia es la enfermedad que combates. + +### 2. Extrae y enumera los CLAIMS + +Recorre Results y Discussion. Lista cada **afirmación de resultado** verificable (no las de contexto). Ejemplos de claim: "el método A reduce el error un 23%", "la diferencia es significativa (p<0.01)", "el efecto es grande (d=0.8)", "el patrón se mantiene en los 3 datasets". Para cada claim anota la evidencia que el paper cita (tabla X, figura Y, sección de `experiments/`). + +### 3. Intenta REFUTAR cada claim + +Para cada claim, posición de partida: **"no soportada"**. Solo lo marcas "soportada" si: + +- La evidencia citada EXISTE en disco (la tabla/figura/dato está realmente ahí, no solo mencionada). +- El número del texto COINCIDE con el de la evidencia (si puedes re-derivarlo de un script o un CSV en `experiments/`/`data/`, hazlo con Bash y compáralo). +- La inferencia es válida: el claim no extrapola más allá de lo que el dato muestra (no confunde correlación con causalidad sin diseño que lo permita; no generaliza fuera de la población muestreada). + +Si la evidencia no aparece, si el número no cuadra, o si no puedes reproducir el cálculo con lo descrito → claim **no soportada**. Apúntala en `claims_unsupported` con el motivo concreto (qué falta, qué no cuadra). + +### 4. Puntúa los 4 ejes (0-5 cada uno) + +Sé tacaño. 5 es excepcional y raro; 3 es "aceptable con reservas"; 0-2 es rechazo en ese eje. Justifica cada número con una frase concreta. + +- **novelty (novedad):** ¿el paper aporta algo que no se sabía? ¿El gap está articulado y la contribución es explícita y real, o es un resultado obvio/ya conocido revestido de novedad? Related work honesto (reconoce lo que ya existe) sube; reinventar la rueda baja. +- **rigor:** método reproducible y estadística correcta. Exige: **effect size + intervalos de confianza**, no solo `p<0.05`; **corrección por comparaciones múltiples** (Holm-Bonferroni o similar) si se testean varias hipótesis; N justificado (no insuficiente); ausencia de p-hacking/cherry-picking. Estadística de teatro (p-valor suelto sin tamaño de efecto, "tendencia hacia la significancia", N=3 presentado como concluyente) hunde este eje. +- **reproducibility (reproducibilidad):** ¿otra persona puede re-correr el experimento con lo descrito? Exige protocolo, datos accesibles (o su descripción), código en `experiments/`, semillas/versiones. Si tú mismo no podrías reproducirlo con lo que hay, el eje es bajo. Pre-registro presente y seguido sube; ausente baja. +- **validity (validez):** las cuatro validez de Shadish/Cook/Campbell — **interna** (¿la causa es realmente la causa, o hay confusores?), **externa** (¿generaliza fuera de esta muestra?), **de constructo** (¿se mide lo que se dice medir?), **estadística** (¿las inferencias estadísticas son legítimas?). El paper debe DECLARAR sus amenazas a la validez. Amenazas no declaradas que tú detectas → bajan el eje y van a `gaps`. + +### 5. Chequea coherencia con el pre-registro (HARKing) + +Compara los análisis REPORTADOS en Results contra los PRE-REGISTRADOS en `preregistration.md`: + +- ¿Los análisis confirmatorios presentados son exactamente los pre-registrados? Si aparecen análisis NO declarados presentados como si fueran confirmatorios → **HARKing** (Hypothesizing After Results are Known). Marca `harking_detected: true`. +- ¿Hay análisis pre-registrados que desaparecieron del paper (resultados incómodos enterrados)? Eso es cherry-picking — anótalo en `gaps`. +- Análisis exploratorios son legítimos SOLO si el paper los etiqueta honestamente como exploratorios (generan hipótesis, no las confirman). Presentar exploratorio como confirmatorio = HARKing. +- Si no hay `preregistration.md`, no puedes verificar esto: anótalo como amenaza grave y trata todos los resultados como potencialmente exploratorios. + +### 6. Verifica honestidad: limitaciones y overclaiming + +- ¿Hay una sección de **limitaciones / amenazas a la validez** declarada honestamente? Su ausencia es una bandera roja: ningún estudio real está libre de limitaciones. +- ¿Las **claims ≤ evidencia**? Compara el lenguaje de las conclusiones con lo que los datos permiten. "demostramos que X causa Y" sobre un diseño correlacional = **overclaiming**. "el método es superior" sobre un solo dataset = overclaiming. Lista cada overclaim en `gaps`. + +### 7. Emite el veredicto + +Default conservador. Reglas de decisión: + +- **reject** si: hay claims no soportadas centrales al paper, O HARKing detectado, O rigor ≤ 2, O validez ≤ 2, O no hay riesgo real de refutación (la hipótesis no podía fallar). +- **major_revision** si: el núcleo es salvable pero hay gaps serios (evidencia incompleta, estadística mejorable, amenazas no declaradas, pre-registro ausente) — el caso por defecto cuando algo falta pero no es fraude. +- **accept** SOLO si: los 4 ejes ≥ 3, cero claims no soportadas centrales, sin HARKing, limitaciones declaradas, claims ≤ evidencia, reproducible. Es raro y hay que ganárselo. + +Ante la duda, baja, no subas. Es preferible un major_revision injusto que dejar pasar un paper-mill. + +--- + +## Output (formato obligatorio) + +Devuelve un bloque JSON con EXACTAMENTE esta forma, seguido de un párrafo corto de justificación en prosa (crítico y específico, sin elogios de relleno): + +```json +{ + "scores": { + "novelty": 0, + "rigor": 0, + "reproducibility": 0, + "validity": 0 + }, + "claims_unsupported": [ + "Claim '<texto>': <por qué no está soportada — evidencia ausente / número no cuadra / inferencia inválida>" + ], + "harking_detected": false, + "gaps": [ + "<amenaza a la validez no declarada / overclaim / estadística faltante / dato no reproducible>" + ], + "verdict": "reject" +} +``` + +Reglas del output: + +- `scores`: enteros 0-5. Tacaño por defecto. +- `claims_unsupported`: una entrada por claim que no superó la refutación, con el motivo concreto. Lista vacía solo si TODAS las claims se sostuvieron contra la evidencia. +- `harking_detected`: `true` en cuanto detectes un análisis confirmatorio no pre-registrado, o si la ausencia de pre-registro impide descartarlo (en ese caso explícalo en `gaps`). +- `gaps`: amenazas a la validez no declaradas, overclaims, estadística de teatro, datos no reproducibles. Concreto y accionable. +- `verdict`: `accept` | `major_revision` | `reject`. Default conservador según las reglas de la sección 7. + +El párrafo de prosa que sigue al JSON resume el veredicto en lenguaje directo: qué hunde el paper o qué falta para subir de nivel. Sin "buen trabajo", sin "interesante contribución" de relleno — solo señal. + +--- + +## Tono y anti-patrones + +- **Crítico y específico.** "La tabla 2 reporta p=0.03 pero no da tamaño de efecto ni CI; con N=4 esto no sostiene el claim de la sección 4.2" — no "la estadística podría mejorarse". +- **Cita evidencia.** Siempre `archivo:línea` o `tabla/figura X`. Una crítica sin cita es ruido. +- **No inventes mérito.** Si el paper no aporta novedad, dilo. El sesgo de complacencia es el que alimenta los paper-mills. +- **No arregles el paper.** No es tu trabajo (no tienes Write). Tu trabajo es el veredicto. Sugiere QUÉ falta, no escribas el fix. +- **Default a fallar.** Evidencia ausente = claim no soportada. Pre-registro ausente = no se puede descartar HARKing. Duda = baja la nota. + +## Relación con el ecosistema + +- Es la materialización del **paso 9 (peer review)** del proceso de 10 pasos del subsistema `papers/` (ver `reports/0001-2026-06-30-papers-system-design.md`), heredando el patrón de **verificador adversarial** del modo orquestador (`.claude/rules/orchestration.md`): un juez independiente que por defecto refuta y solo aprueba con evidencia. +- Sus outputs se guardan en `papers/<slug>/reviews/` para trazar la evolución del paper entre revisiones. +- Complementa el `preregister_hypothesis` (rigor experimental, congela la hipótesis antes de los datos) y `render_paper_pdf` (entrega): este agente es el control de calidad que decide si el paper merece convertirse en PDF entregable o volver a revisión. diff --git a/python/functions/datascience/__init__.py b/python/functions/datascience/__init__.py index cdefab14..a8c37b35 100644 --- a/python/functions/datascience/__init__.py +++ b/python/functions/datascience/__init__.py @@ -72,8 +72,10 @@ from .profile_datetime import profile_datetime from .resample_timeseries import resample_timeseries from .add_pdf_internal_links import add_pdf_internal_links from .suggest_intratable_fk_candidates import suggest_intratable_fk_candidates +from .render_paper_pdf import render_paper_pdf __all__ = [ + "render_paper_pdf", "suggest_intratable_fk_candidates", "detect_time_column", "extract_timeseries_raw", diff --git a/python/functions/datascience/render_paper_pdf.md b/python/functions/datascience/render_paper_pdf.md new file mode 100644 index 00000000..31ef5017 --- /dev/null +++ b/python/functions/datascience/render_paper_pdf.md @@ -0,0 +1,96 @@ +--- +name: render_paper_pdf +kind: function +lang: py +domain: datascience +version: "1.0.0" +purity: impure +signature: "def render_paper_pdf(paper_dir: str) -> dict" +description: "Convierte un paper académico IMRaD escrito en Markdown (papers/<slug>/paper.md, con frontmatter YAML opcional title/authors/date/abstract + cuerpo) en un PDF papers/<slug>/out/paper.pdf. REUTILIZA el paginador de flujo del paquete automatic_eda (el mismo motor del PDF móvil A5 de los informes EDA): no reimplementa paginación ni toca matplotlib. Cada sección IMRaD (encabezado de nivel 1, p.ej. # Introduction, # Methods) se mapea a un Chapter que empieza en página nueva; el motor parsea por sí mismo headings, listas, tablas pipe, párrafos y **negrita** dentro del texto. Como el motor NO entiende la sintaxis de imagen Markdown ![alt](src), esta función detecta esas líneas y las parte en bloques Image separados, resolviendo el src relativo a base_dir y base_dir/figures/. La portada (si hay título) lista autores y fecha (DD/MM/AAAA si parseable) más el abstract. dict-no-throw: nunca lanza, devuelve {status, pdf_path, n_pages, note}." +tags: [papers, pdf, academic, render, report, imrad, mobile, automatic-eda, markdown, no-cut, matplotlib, datascience, python] +uses_functions: [] +uses_types: [] +returns: [] +returns_optional: false +error_type: "error_go_core" +imports: [os, re, datetime, yaml, "datascience.automatic_eda"] +params: + - name: paper_dir + desc: "ruta al directorio del paper (papers/<slug>/, del que se lee paper.md) O directamente la ruta a un archivo paper.md (cualquier ruta terminada en .md). El directorio base para resolver figuras y escribir el PDF es el dirname del paper.md. Si el paper.md no existe (incluida una ruta totalmente inexistente) devuelve status='error' sin crash." +output: "dict (nunca lanza): {status: 'ok'|'error', pdf_path: str|None, n_pages: int, note: str}. En éxito status='ok', pdf_path es la ruta del PDF escrito (<base_dir>/out/paper.pdf) y n_pages el total de páginas. En error status='error', pdf_path=None, n_pages=0 y note explica la causa (paper.md no encontrado, fallo del motor, o excepción inesperada)." +tested: true +tests: ["test_golden_genera_pdf_con_portada_y_secciones", "test_edge_sin_frontmatter_ni_figuras", "test_edge_path_inexistente_no_revienta", "test_edge_figura_inexistente_degrada", "test_acepta_ruta_directa_al_md"] +test_file_path: "python/functions/datascience/render_paper_pdf_test.py" +file_path: "python/functions/datascience/render_paper_pdf.py" +--- + +## Ejemplo + +```python +from datascience import render_paper_pdf + +# Estructura del paper: +# papers/zz-demo/paper.md (frontmatter YAML + cuerpo IMRaD) +# papers/zz-demo/figures/fig1.png (figuras referenciadas con ![alt](figures/fig1.png)) +# +# paper.md: +# --- +# title: A Minimal IMRaD Paper +# authors: [Ada Lovelace, Alan Turing] +# date: 2026-06-30 +# abstract: Demostramos que el motor pagina un paper sin cortar nada. +# --- +# # Introduction +# Texto con **negrita** y una lista: +# - Punto uno. +# ![Figura 1](figures/fig1.png) +# # Methods +# | Métrica | Valor | +# | --- | --- | +# | Precisión | 0.91 | + +res = render_paper_pdf("papers/zz-demo") +print(res["status"], res["n_pages"], res["pdf_path"]) +# -> ok 3 papers/zz-demo/out/paper.pdf + +# También acepta la ruta directa al .md: +render_paper_pdf("papers/zz-demo/paper.md") +``` + +## Cuando usarla + +Cuando tengas un paper académico (o cualquier documento IMRaD) escrito en +Markdown y quieras un **PDF móvil A5 listo para leer**, sin montar LaTeX ni +configurar un pipeline de pandoc. Úsala después de redactar `paper.md` con su +frontmatter (título, autores, fecha, abstract) y secciones de nivel 1; obtienes +`out/paper.pdf` con portada, una página nueva por sección IMRaD, tablas que se +parten repitiendo la cabecera y figuras escaladas para caber enteras — +garantía de no-corte heredada del motor `automatic_eda`. Es la capa de +presentación PDF del grupo `papers`. + +## Gotchas + +- **Impura**: escribe `out/paper.pdf` (y crea el directorio `out/`) junto al + `paper.md`. Necesita **matplotlib** instalado en el venv (lo usa el motor + `automatic_eda.render_pdf` con backend headless `Agg`; corre en agentes/CI sin + display). `pyyaml` es opcional: si falta, el frontmatter se parsea con un + parser line-based `clave: valor` degradado. +- **Reutiliza el motor `automatic_eda.render_pdf`**: NO reimplementa paginación + ni toca matplotlib. `render_pdf` no tiene ID propio en el registry (es parte + del paquete de soporte `automatic_eda`), por eso `uses_functions` queda vacío; + la dependencia real es ese motor del paquete. +- **Nunca lanza** (dict-no-throw): `paper.md` inexistente → `{status:"error", + pdf_path:None, note:"paper.md no encontrado: ..."}`; cualquier excepción + inesperada → `{status:"error", note:"fallo: ..."}`. Frontmatter ausente o + incompleto degrada limpio (sin portada, el cuerpo entero se pagina). +- **Figuras relativas a `figures/`**: el `src` de `![alt](src)` se resuelve + probando `<base_dir>/<src>` y `<base_dir>/figures/<basename>`; usa el primero + que exista. Si ninguno existe, el motor **degrada** dibujando + "(imagen no encontrada: ...)" — el PDF se genera igual, no crashea. Las URLs + `http(s)` se dejan como texto Markdown, no se descargan. +- **Solo imágenes en línea propia**: el motor `_place_markdown` NO entiende + `![alt](src)`; esta función solo convierte a `Image` las líneas cuyo único + contenido es la imagen. Una imagen embebida a mitad de un párrafo se quedaría + como texto crudo. +- **A5 portrait mobile-first**: el formato (tamaño de página, tipografía, pie + `Capítulo · vX.Y.Z`) lo fija el motor EDA y no es configurable desde aquí. diff --git a/python/functions/datascience/render_paper_pdf.py b/python/functions/datascience/render_paper_pdf.py new file mode 100644 index 00000000..5581a39b --- /dev/null +++ b/python/functions/datascience/render_paper_pdf.py @@ -0,0 +1,297 @@ +"""render_paper_pdf — convierte un paper académico IMRaD en Markdown a un PDF. + +Toma un paper escrito en Markdown con frontmatter YAML opcional (título, +autores, fecha, abstract) más un cuerpo dividido en secciones IMRaD por +encabezados de nivel 1 (``# Introduction``, ``# Methods``, ...) y produce un PDF +``out/paper.pdf`` junto al paper. + +REUTILIZA el paginador de flujo del paquete ``automatic_eda`` (el mismo motor +que rinde los informes EDA en PDF móvil A5): no reimplementa paginación ni toca +matplotlib directamente. Cada sección IMRaD se mapea a un ``Chapter`` (empieza +en página nueva). El motor ``_place_markdown`` parsea por sí mismo headings, +listas, tablas pipe, párrafos y ``**negrita**`` dentro del texto, pero NO +entiende la sintaxis de imagen Markdown ``![alt](src)``; por eso esta función +detecta esas líneas y las convierte en bloques ``Image`` separados, partiendo el +texto Markdown alrededor de cada imagen. + +dict-no-throw (estilo del grupo eda): NUNCA lanza. Devuelve +``{status, pdf_path, n_pages, note}``; ante cualquier fallo devuelve +``status="error"`` con ``pdf_path=None`` y la causa en ``note``. +""" + +from __future__ import annotations + +import datetime as _dt +import os +import re + +from datascience.automatic_eda import Chapter, Heading, Image, Markdown, render_pdf + +# Una línea cuyo único contenido es una imagen Markdown: ![alt](src) +_IMG_LINE = re.compile(r"^\s*!\[([^\]]*)\]\(\s*([^)\s]+)\s*\)\s*$") +# Un encabezado de nivel 1 al inicio de línea (un solo '#' seguido de espacio). +_H1_LINE = re.compile(r"^#[ \t]+(.+?)\s*$") + + +def render_paper_pdf(paper_dir: str) -> dict: + """Renderiza un paper académico Markdown IMRaD en un PDF. + + Args: + paper_dir: ruta al directorio del paper (``papers/<slug>/``, del que se + lee ``paper.md``) o directamente la ruta a un archivo ``paper.md``. + + Returns: + dict (nunca lanza): ``{status: "ok"|"error", pdf_path: str|None, + n_pages: int, note: str}``. En éxito ``pdf_path`` es la ruta escrita y + ``n_pages`` el total de páginas; en error ``pdf_path`` es None y + ``note`` explica la causa. + """ + try: + # 1) Resolver el path del paper.md y el directorio base. + arg = str(paper_dir) + md_path = arg if arg.endswith(".md") else os.path.join(arg, "paper.md") + + # 2) Si el paper.md no existe, degradar sin crash. + if not os.path.isfile(md_path): + return {"status": "error", "pdf_path": None, "n_pages": 0, + "note": f"paper.md no encontrado: {md_path}"} + + base_dir = os.path.dirname(os.path.abspath(md_path)) + + # 3) Leer el archivo y separar frontmatter del cuerpo. + with open(md_path, "r", encoding="utf-8") as fh: + text = fh.read() + fm_text, body = _split_frontmatter(text) + fm = _parse_frontmatter(fm_text) + + title = _safe_str(fm.get("title")).strip() + authors = fm.get("authors") + date_raw = fm.get("date") + abstract = _safe_str(fm.get("abstract")).strip() + + # 4) Construir los capítulos: portada (si hay título) + cuerpo IMRaD. + chapters: list = [] + if title: + cover_md = _portada_markdown(authors, date_raw, abstract) + cover_blocks: list = [Heading(text=title, level=1)] + if cover_md.strip(): + cover_blocks.append(Markdown(text=cover_md)) + chapters.append(Chapter(id="portada", title=title, version="1.0.0", + blocks=cover_blocks)) + + preamble, sections = _split_body_sections(body) + + if not sections: + # Sin encabezados H1: todo el cuerpo en un único capítulo. + chapters.append(Chapter( + id="cuerpo", title="Cuerpo", version="1.0.0", + blocks=_markdown_to_blocks(body, base_dir))) + else: + # Texto antes del primer H1 (si lo hay) como capítulo previo. + if preamble.strip(): + chapters.append(Chapter( + id="cuerpo", title="Cuerpo", version="1.0.0", + blocks=_markdown_to_blocks(preamble, base_dir))) + for idx, (sec_title, sec_body) in enumerate(sections): + blocks: list = [Heading(text=sec_title, level=1)] + blocks.extend(_markdown_to_blocks(sec_body, base_dir)) + chapters.append(Chapter( + id=_slugify(sec_title) or f"sec{idx}", + title=sec_title, version="1.0.0", blocks=blocks)) + + # 5) Renderizar con el motor de automatic_eda. + out_path = os.path.join(base_dir, "out", "paper.pdf") + res = render_pdf(chapters, out_path, meta={"title": title or "paper"}) + + # 6) Mapear el retorno del motor a la forma de esta función. + path = res.get("path") + return { + "status": "ok" if path else "error", + "pdf_path": path, + "n_pages": int(res.get("n_pages") or 0), + "note": res.get("note"), + } + except Exception as e: # noqa: BLE001 — dict-no-throw estricto. + return {"status": "error", "pdf_path": None, "n_pages": 0, + "note": f"fallo: {e}"} + + +# --------------------------------------------------------------------------- # +# Frontmatter +# --------------------------------------------------------------------------- # +def _split_frontmatter(text: str): + """Separa el bloque frontmatter YAML inicial del cuerpo. + + Devuelve ``(fm_text|None, body)``. Si el archivo no empieza con una valla + ``---`` o no se cierra, no hay frontmatter y el cuerpo es el texto entero. + """ + if text.startswith(""): + text = text.lstrip("") + lines = text.split("\n") + if not lines or lines[0].strip() != "---": + return None, text + for i in range(1, len(lines)): + if lines[i].strip() == "---": + return "\n".join(lines[1:i]), "\n".join(lines[i + 1:]) + # Valla de apertura sin cierre: tratar todo como cuerpo. + return None, text + + +def _parse_frontmatter(fm_text) -> dict: + """Parsea el frontmatter. Intenta YAML; si no, parser line-based simple.""" + if not fm_text: + return {} + try: + import yaml # type: ignore + data = yaml.safe_load(fm_text) + if isinstance(data, dict): + return data + except Exception: # noqa: BLE001 — yaml ausente o frontmatter inválido. + pass + # Fallback degradado: 'clave: valor' por línea. + out: dict = {} + for line in fm_text.split("\n"): + stripped = line.strip() + if not stripped or stripped.startswith("#") or ":" not in stripped: + continue + k, _, v = stripped.partition(":") + k = k.strip() + v = v.strip().strip('"').strip("'") + if k: + out[k] = v + return out + + +# --------------------------------------------------------------------------- # +# Portada +# --------------------------------------------------------------------------- # +def _portada_markdown(authors, date_raw, abstract) -> str: + """Markdown de la portada: autores, fecha y, si hay, el abstract.""" + parts: list = [] + authors_str = _fmt_authors(authors) + if authors_str: + parts.append(f"**Autores:** {authors_str}") + if date_raw not in (None, ""): + parts.append(f"**Fecha:** {_fmt_date(date_raw)}") + md = "\n\n".join(parts) + abstract = _safe_str(abstract).strip() + if abstract: + md = (md + "\n\n" if md else "") + "## Abstract\n\n" + abstract + return md + + +def _fmt_authors(authors) -> str: + """Lista o string de autores → string separado por comas.""" + if authors in (None, ""): + return "" + if isinstance(authors, (list, tuple)): + return ", ".join(_safe_str(a).strip() for a in authors + if _safe_str(a).strip()) + return _safe_str(authors).strip() + + +def _fmt_date(raw) -> str: + """Fecha → ``DD/MM/AAAA`` si es parseable; si no, el valor crudo.""" + if isinstance(raw, _dt.datetime): + return raw.strftime("%d/%m/%Y") + if isinstance(raw, _dt.date): + return raw.strftime("%d/%m/%Y") + s = _safe_str(raw).strip() + if not s: + return s + for fmt in ("%Y-%m-%d", "%Y/%m/%d", "%d/%m/%Y", "%d-%m-%Y"): + try: + return _dt.datetime.strptime(s, fmt).strftime("%d/%m/%Y") + except ValueError: + continue + try: + return _dt.datetime.fromisoformat(s).strftime("%d/%m/%Y") + except Exception: # noqa: BLE001 + return s + + +# --------------------------------------------------------------------------- # +# Cuerpo y figuras +# --------------------------------------------------------------------------- # +def _split_body_sections(body: str): + """Divide el cuerpo en (preámbulo, [(título_H1, contenido)...]) por H1.""" + preamble_lines: list = [] + sections: list = [] + current = None # (titulo, [lineas]) + for line in body.split("\n"): + m = _H1_LINE.match(line) + if m and not line.startswith("##"): + if current is not None: + sections.append((current[0], "\n".join(current[1]))) + current = (m.group(1).strip(), []) + elif current is None: + preamble_lines.append(line) + else: + current[1].append(line) + if current is not None: + sections.append((current[0], "\n".join(current[1]))) + return "\n".join(preamble_lines), sections + + +def _markdown_to_blocks(text: str, base_dir: str) -> list: + """Parte un Markdown en bloques Markdown/Image alrededor de cada figura. + + Las líneas ``![alt](src)`` con ``src`` local se convierten en ``Image``; las + que apuntan a URLs http(s) se dejan como texto Markdown. + """ + blocks: list = [] + buf: list = [] + + def _flush(): + chunk = "\n".join(buf).strip("\n") + if chunk.strip(): + blocks.append(Markdown(text=chunk)) + buf.clear() + + for line in text.split("\n"): + m = _IMG_LINE.match(line) + if m: + alt, src = m.group(1), m.group(2) + if src.lower().startswith(("http://", "https://")): + buf.append(line) # URL remota: se mantiene como texto. + continue + _flush() + blocks.append(Image(path=_resolve_src(src, base_dir), + caption=(alt or None))) + else: + buf.append(line) + _flush() + return blocks + + +def _resolve_src(src: str, base_dir: str) -> str: + """Resuelve la ruta de una figura relativa al paper. + + Absoluta → tal cual. Relativa → prueba ``base_dir/src`` y + ``base_dir/figures/<basename>``; usa la primera que exista, o el join con + ``base_dir`` si ninguna (el motor degrada dibujando el aviso de no-encontrada). + """ + if os.path.isabs(src): + return src + cand1 = os.path.join(base_dir, src) + cand2 = os.path.join(base_dir, "figures", os.path.basename(src)) + for c in (cand1, cand2): + if os.path.exists(c): + return c + return cand1 + + +def _slugify(text: str) -> str: + """Slug ASCII corto para el id del capítulo.""" + s = re.sub(r"[^a-z0-9]+", "_", _safe_str(text).lower()).strip("_") + return s[:40] + + +def _safe_str(v) -> str: + """str() que nunca lanza y mapea None a ''.""" + if v is None: + return "" + try: + return str(v) + except Exception: # noqa: BLE001 + return "" diff --git a/python/functions/datascience/render_paper_pdf_test.py b/python/functions/datascience/render_paper_pdf_test.py new file mode 100644 index 00000000..e515f9eb --- /dev/null +++ b/python/functions/datascience/render_paper_pdf_test.py @@ -0,0 +1,118 @@ +"""Tests para render_paper_pdf — DoD: golden + edges + error path. + +Autocontenido y sin red: escribe papers Markdown sintéticos en directorios +temporales y verifica que el PDF se genera (estado, nº de páginas, archivo +no vacío) reutilizando el motor de paginación de ``automatic_eda``. +""" + +import os +import tempfile + +from datascience.render_paper_pdf import render_paper_pdf + + +_GOLDEN_PAPER = """--- +title: A Minimal IMRaD Paper +authors: + - Ada Lovelace + - Alan Turing +date: 2026-06-30 +abstract: > + Demostramos que el motor de paginación rinde un paper IMRaD completo en PDF + móvil sin cortar texto ni tablas. +--- + +# Introduction + +Este es el cuerpo de la introducción con **texto en negrita** y una lista: + +- Primer punto. +- Segundo punto. + +# Methods + +Resultados resumidos en una tabla pipe: + +| Métrica | Valor | +| --- | --- | +| Precisión | 0.91 | +| Recall | 0.88 | + +Texto final de la sección de métodos. +""" + + +def test_golden_genera_pdf_con_portada_y_secciones(tmp_path): + """Golden: paper IMRaD con frontmatter + 2 secciones + tabla → PDF válido.""" + paper_dir = tmp_path / "zz-demo" + paper_dir.mkdir() + (paper_dir / "paper.md").write_text(_GOLDEN_PAPER, encoding="utf-8") + + res = render_paper_pdf(str(paper_dir)) + + assert res["status"] == "ok", res + assert res["n_pages"] >= 1 + pdf_path = res["pdf_path"] + assert pdf_path is not None + assert os.path.exists(pdf_path) + assert os.path.getsize(pdf_path) > 0 + + +def test_edge_sin_frontmatter_ni_figuras(tmp_path): + """Edge 1: cuerpo plano sin frontmatter ni figuras → genera PDF igual.""" + paper_dir = tmp_path / "plano" + paper_dir.mkdir() + (paper_dir / "paper.md").write_text( + "Solo un cuerpo plano, sin frontmatter ni encabezados de nivel 1.\n" + "Un par de líneas de texto corrido para que el motor lo pagine.\n", + encoding="utf-8", + ) + + res = render_paper_pdf(str(paper_dir)) + + assert res["status"] == "ok", res + assert res["n_pages"] >= 1 + assert os.path.exists(res["pdf_path"]) + + +def test_edge_path_inexistente_no_revienta(): + """Edge 2: directorio inexistente → status error, sin crash, pdf_path None.""" + res = render_paper_pdf("/tmp/no_existe_xyz_123") + + assert res["status"] == "error" + assert res["pdf_path"] is None + assert res["n_pages"] == 0 + assert "no encontrado" in (res["note"] or "") + + +def test_edge_figura_inexistente_degrada(tmp_path): + """Edge 3: referencia a figura inexistente → el PDF se genera igual.""" + paper_dir = tmp_path / "con-figura" + paper_dir.mkdir() + (paper_dir / "paper.md").write_text( + "---\n" + "title: Paper Con Figura Rota\n" + "---\n\n" + "# Results\n\n" + "Texto antes de la figura.\n\n" + "![Una figura que no existe](figures/no.png)\n\n" + "Texto después de la figura.\n", + encoding="utf-8", + ) + + res = render_paper_pdf(str(paper_dir)) + + assert res["status"] == "ok", res + assert res["n_pages"] >= 1 + assert os.path.exists(res["pdf_path"]) + + +def test_acepta_ruta_directa_al_md(tmp_path): + """Acepta también la ruta directa a un paper.md (no solo el directorio).""" + md = tmp_path / "paper.md" + md.write_text("# Discussion\n\nCuerpo de la discusión.\n", encoding="utf-8") + + res = render_paper_pdf(str(md)) + + assert res["status"] == "ok", res + assert os.path.exists(res["pdf_path"]) From 4f1530797e9e6234cfaa462cd70baaf54d6b5297 Mon Sep 17 00:00:00 2001 From: Egutierrez <egutierrez@dead.dd> Date: Tue, 30 Jun 2026 20:42:12 +0200 Subject: [PATCH 42/53] =?UTF-8?q?feat(datascience):=20rigor=20experimental?= =?UTF-8?q?=20para=20papers=20=E2=80=94=20effect=20size,=20IC,=20Holm=20+?= =?UTF-8?q?=20preregistro=20inmutable?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Subsistema de papers reproducibles (grupo de capacidad `papers`). Añade las funciones estadísticas que un paper honesto necesita y la función que congela la hipótesis antes de mirar los datos (anti-HARKing). Nuevas funciones (puras salvo la última): - effect_size_cohens_d: Cohen's d + Hedges' g (corrección de sesgo para N pequeño) + interpretación cualitativa (negligible/small/medium/large por los umbrales de Cohen). Dict-no-throw ante varianza cero / N insuficiente. - confidence_interval_mean: intervalo de confianza de una media (t de Student) o de la diferencia de medias con Welch (df de Welch–Satterthwaite, sin asumir varianzas iguales). Dict-no-throw; el IC colapsa al punto cuando la varianza es cero. - preregister_hypothesis (impura): congela hipótesis + plan de análisis en papers/<slug>/preregistration.md con frozen_at (UTC) y content_hash (sha256 del cuerpo normalizado, no del frontmatter). Inmutabilidad: una vez frozen, un contenido distinto se RECHAZA sin sobrescribir (mata el HARKing); idempotente si el contenido es idéntico. Siempre dict-no-throw. Extensión: - fdr_correction 1.0.0 -> 1.1.0: añade method="holm" (Holm-Bonferroni step-down, controla FWER, más potente que Bonferroni simple). Reúsa la maquinaria de alineación 1:1 con None/inválidos; no rompe los métodos bh/bonferroni. Reutiliza del registry: fdr_correction (BH + Bonferroni ya existían) como base para Holm. pearson y spearman_corr ya cubrían correlación. Tests: 36 pytest verdes (cohen/hedges 8, confidence/welch 8, fdr/holm/bonferroni 12, preregister 4 + extras), golden contra valores conocidos y validados con scipy. Golden manual del preregistro: congela, idempotente, rechaza edición (bytes en disco idénticos al congelado). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> --- python/functions/datascience/__init__.py | 6 + .../datascience/confidence_interval_mean.md | 87 ++++++++ .../datascience/confidence_interval_mean.py | 176 +++++++++++++++ .../confidence_interval_mean_test.py | 140 ++++++++++++ .../datascience/effect_size_cohens_d.md | 80 +++++++ .../datascience/effect_size_cohens_d.py | 156 ++++++++++++++ .../datascience/effect_size_cohens_d_test.py | 96 +++++++++ .../functions/datascience/fdr_correction.md | 40 +++- .../functions/datascience/fdr_correction.py | 38 +++- .../datascience/fdr_correction_test.py | 66 +++++- .../datascience/preregister_hypothesis.md | 100 +++++++++ .../datascience/preregister_hypothesis.py | 202 ++++++++++++++++++ .../preregister_hypothesis_test.py | 99 +++++++++ 13 files changed, 1265 insertions(+), 21 deletions(-) create mode 100644 python/functions/datascience/confidence_interval_mean.md create mode 100644 python/functions/datascience/confidence_interval_mean.py create mode 100644 python/functions/datascience/confidence_interval_mean_test.py create mode 100644 python/functions/datascience/effect_size_cohens_d.md create mode 100644 python/functions/datascience/effect_size_cohens_d.py create mode 100644 python/functions/datascience/effect_size_cohens_d_test.py create mode 100644 python/functions/datascience/preregister_hypothesis.md create mode 100644 python/functions/datascience/preregister_hypothesis.py create mode 100644 python/functions/datascience/preregister_hypothesis_test.py diff --git a/python/functions/datascience/__init__.py b/python/functions/datascience/__init__.py index cdefab14..f4824679 100644 --- a/python/functions/datascience/__init__.py +++ b/python/functions/datascience/__init__.py @@ -59,6 +59,9 @@ from .acf_pacf import acf_pacf from .stl_decompose import stl_decompose from .to_returns import to_returns from .fdr_correction import fdr_correction +from .effect_size_cohens_d import effect_size_cohens_d +from .confidence_interval_mean import confidence_interval_mean +from .preregister_hypothesis import preregister_hypothesis from .suggest_reexpression import suggest_reexpression from .exploratory_caveats import exploratory_caveats from .render_eda_pdf import render_eda_pdf, render_eda_pdf_relational @@ -90,6 +93,9 @@ __all__ = [ "stl_decompose", "to_returns", "fdr_correction", + "effect_size_cohens_d", + "confidence_interval_mean", + "preregister_hypothesis", "suggest_reexpression", "exploratory_caveats", "render_eda_pdf", diff --git a/python/functions/datascience/confidence_interval_mean.md b/python/functions/datascience/confidence_interval_mean.md new file mode 100644 index 00000000..a44b70e7 --- /dev/null +++ b/python/functions/datascience/confidence_interval_mean.md @@ -0,0 +1,87 @@ +--- +name: confidence_interval_mean +kind: function +lang: py +domain: datascience +version: "1.0.0" +purity: pure +signature: "def confidence_interval_mean(data: list, other: list = None, confidence: float = 0.95) -> dict" +description: "Intervalo de confianza (IC) de la media de una muestra con la t de Student, o de la DIFERENCIA de medias de dos muestras independientes con el metodo de Welch (sin asumir varianzas iguales). Una muestra: df=n-1, se=sd_muestral/sqrt(n) (sd con ddof=1), tcrit=t.ppf((1+confidence)/2, df), ci=mean+/-tcrit*se. Dos muestras: IC de mean(data)-mean(other) con se=sqrt(se1^2+se2^2) y grados de libertad de Welch-Satterthwaite. Pura y robusta: nunca lanza; ante casos degenerados (muestra vacia, n<2) devuelve nan + clave note, y con varianza cero el IC colapsa al punto (no es error). Usa scipy.stats y numpy." +tags: [papers, statistics, confidence-interval, welch, t-test, python] +params: + - name: data + desc: "muestra de observaciones numericas (lista de numeros). Si other es None, el IC es el de la media de data." + - name: other + desc: "segunda muestra independiente (lista de numeros) o None (default). Si se da, el IC es el de la diferencia de medias mean(data)-mean(other) calculada con Welch (no asume varianzas iguales)." + - name: confidence + desc: "nivel de confianza en (0, 1); 0.95 = IC del 95% (default). El cuantil critico es t.ppf((1+confidence)/2, df)." +output: "dict {mean, ci_low, ci_high, se, df, confidence, n}. mean = media de data (una muestra) o la diferencia mean(data)-mean(other) (dos muestras). En el caso de dos muestras se anaden ademas n1 y n2 (y n = n1+n2). df son los grados de libertad de la t (Welch-Satterthwaite si dos muestras). Casos degenerados (muestra vacia, n<2) anaden la clave note y dejan ci_low/ci_high/se (y a veces df) en nan; con varianza cero y n>=2 el IC colapsa a [mean, mean] con se=0 (con note, sin nan). Nunca None ni excepcion." +uses_functions: [] +uses_types: [] +returns: [] +returns_optional: false +error_type: "" +imports: [scipy, numpy] +tested: true +tests: ["test_one_sample_golden_contra_scipy", "test_one_sample_distinto_nivel_confianza", "test_welch_diferencia_golden_contra_scipy", "test_edge_un_solo_elemento_no_lanza_nan_note", "test_edge_lista_vacia_no_lanza_note", "test_edge_varianza_cero_colapsa_al_punto", "test_edge_welch_muestra_vacia_no_lanza_note", "test_edge_welch_n1_uno_no_lanza_note"] +test_file_path: "python/functions/datascience/confidence_interval_mean_test.py" +file_path: "python/functions/datascience/confidence_interval_mean.py" +--- + +## Ejemplo + +```python +from datascience import confidence_interval_mean + +# IC del 95% de la media de una muestra (t de Student). +data = [2, 4, 4, 4, 5, 5, 7, 9] +ci = confidence_interval_mean(data, confidence=0.95) +print(ci["mean"]) # -> 5.0 +print(ci["df"]) # -> 7.0 (n - 1) +print(round(ci["ci_low"], 5), round(ci["ci_high"], 5)) +# -> 3.21251 6.78749 (se con sd muestral ddof=1 ~ 2.13809) + +# IC del 95% de la DIFERENCIA de medias (Welch, no asume varianzas iguales). +control = [23.0, 21.0, 25.0, 22.0, 24.0, 26.0] +tratado = [18.0, 20.0, 17.0, 19.0, 21.0] +diff = confidence_interval_mean(control, tratado, confidence=0.95) +print(diff["mean"]) # -> 4.5 (mean(control) - mean(tratado)) +print(round(diff["ci_low"], 4), round(diff["ci_high"], 4)) +# Si el intervalo no incluye 0, la diferencia es significativa al 5%. + +# Degenerados: nunca lanza. +print(confidence_interval_mean([5])["note"]) # n < 2: ... indefinidos +print(confidence_interval_mean([3, 3, 3])["se"]) # -> 0.0 (IC colapsa a [3, 3]) +``` + +## Cuando usarla + +Cuando quieras cuantificar la **incertidumbre de una media estimada** a partir de +una muestra: reporta `[ci_low, ci_high]` en vez de un punto suelto para mostrar +el rango plausible del valor real al nivel de confianza pedido. Usala tambien +para **comparar dos grupos** (A/B test, control vs tratamiento, antes vs +despues con grupos independientes): pasa las dos muestras y, si el IC de la +diferencia **no incluye el 0**, la diferencia es significativa al nivel +`1 - confidence`. Es el complemento del p-valor: ademas de "hay efecto", te dice +"de que tamano y con que margen". Para dos muestras usa Welch por defecto, asi +que no necesitas comprobar antes si las varianzas son iguales. + +## Gotchas + +- Pura y determinista (no hace I/O, no muta las entradas), pero **no** es + stdlib-only: depende de `scipy.stats` y `numpy` (ambos en el venv del proyecto). +- Con `other` usa **Welch** (df de Welch-Satterthwaite): NO asume varianzas + iguales ni tamanos de muestra iguales. Si necesitas el t-test clasico de + varianzas agrupadas (pooled), esta funcion no lo hace. +- `sd` se calcula con **ddof=1** (sd muestral), que es lo correcto para el IC de + una media con la t. Atajos como `sd_poblacional/sqrt(n)` (ddof=0) dan un + intervalo demasiado estrecho. +- En el caso de dos muestras, `mean` es la **diferencia** `mean(data) - mean(other)` + (no la media de data). El orden importa: el signo del IC depende de cual va + primero. +- Nunca lanza. Casos degenerados devuelven `nan` en `ci_low`/`ci_high`/`se` + (y a veces `df`) mas una clave `note`: muestra vacia o `n < 2` en cualquiera de + las muestras. **Excepcion**: con varianza cero y `n >= 2` el IC colapsa al + punto `[mean, mean]` con `se = 0` (no es un error, no hay `nan`). +- Comprueba `"note" in out` antes de usar `ci_low`/`ci_high` si la muestra puede + ser degenerada. diff --git a/python/functions/datascience/confidence_interval_mean.py b/python/functions/datascience/confidence_interval_mean.py new file mode 100644 index 00000000..6a4b5b2f --- /dev/null +++ b/python/functions/datascience/confidence_interval_mean.py @@ -0,0 +1,176 @@ +"""Intervalo de confianza de la media (una muestra) o de la diferencia de medias (Welch). + +Funcion pura del grupo papers. Calcula el intervalo de confianza (IC) de la media +de una muestra usando la t de Student, o el IC de la diferencia de medias de dos +muestras independientes con el metodo de Welch (sin asumir varianzas iguales). + +- Una muestra: ``df = n - 1``, ``se = sd / sqrt(n)`` (sd con ddof=1), + ``tcrit = t.ppf((1 + confidence) / 2, df)``, ``ci = mean +/- tcrit * se``. +- Dos muestras (Welch): IC de ``mean(data) - mean(other)``, con + ``se = sqrt(se1^2 + se2^2)`` y grados de libertad de Welch-Satterthwaite. + +No lanza excepciones: ante casos degenerados (muestras vacias, ``n < 2``, +varianza cero) devuelve un dict coherente con ``ci_low``/``ci_high``/``se`` en +``nan`` (salvo el sub-caso de varianza cero, donde el IC colapsa al punto) y una +clave ``note`` explicando el caso. Usa ``scipy.stats`` y ``numpy``. +""" + +from __future__ import annotations + +import math + +import numpy as np +from scipy import stats + + +def confidence_interval_mean( + data: list, other: list = None, confidence: float = 0.95 +) -> dict: + """Intervalo de confianza de la media o de la diferencia de medias (Welch). + + Si ``other`` es ``None``, calcula el IC de la media de ``data`` con la t de + Student. Si se proporciona ``other``, calcula el IC de la diferencia + ``mean(data) - mean(other)`` con el metodo de Welch (no asume varianzas + iguales) y grados de libertad de Welch-Satterthwaite. + + Es una funcion pura y determinista: no hace I/O ni muta las entradas. No + lanza excepcion ante datos degenerados; en su lugar devuelve un dict con la + clave ``note`` y los campos numericos indefinidos a ``nan``. + + Args: + data: muestra de observaciones numericas (lista de numeros). + other: segunda muestra independiente. Si se da, el IC es el de la + diferencia de medias ``mean(data) - mean(other)`` con Welch. Si es + ``None`` (default), el IC es el de la media de ``data``. + confidence: nivel de confianza en (0, 1), p.ej. 0.95 para el 95%. + + Returns: + dict con las claves: + mean: media de ``data`` (una muestra) o la diferencia + ``mean(data) - mean(other)`` (dos muestras). + ci_low: extremo inferior del intervalo de confianza. + ci_high: extremo superior del intervalo de confianza. + se: error estandar de la media (o de la diferencia). + df: grados de libertad de la t (Welch-Satterthwaite si dos muestras). + confidence: nivel de confianza aplicado (float). + n: tamano de la muestra (una muestra) o tamano total ``n1 + n2`` + (dos muestras; ademas se incluyen ``n1`` y ``n2``). + + En el caso de dos muestras se incluyen ademas ``n1`` y ``n2``. Casos + degenerados (muestra vacia, ``n < 2``, etc.) anaden la clave ``note`` y + dejan ``ci_low``/``ci_high``/``se`` (y a veces ``df``) en ``nan``. + """ + conf = float(confidence) + + if other is None: + return _ci_one_sample(data, conf) + return _ci_welch(data, other, conf) + + +def _ci_one_sample(data: list, conf: float) -> dict: + """IC de la media de una sola muestra con la t de Student.""" + arr = np.asarray(list(data), dtype=float) + n = int(arr.size) + + base = { + "mean": float("nan"), + "ci_low": float("nan"), + "ci_high": float("nan"), + "se": float("nan"), + "df": float("nan"), + "confidence": conf, + "n": n, + } + + if n == 0: + base["note"] = "muestra vacia: media e intervalo indefinidos" + return base + + mean = float(arr.mean()) + base["mean"] = mean + + if n < 2: + base["note"] = "n < 2: error estandar y grados de libertad indefinidos" + return base + + df = n - 1 + base["df"] = float(df) + + sd = float(arr.std(ddof=1)) + se = sd / math.sqrt(n) + base["se"] = se + + # Varianza cero: el IC colapsa al punto (no es un error). + if se == 0.0: + base["ci_low"] = mean + base["ci_high"] = mean + base["note"] = "varianza cero: el intervalo colapsa a la media" + return base + + tcrit = float(stats.t.ppf((1.0 + conf) / 2.0, df)) + margin = tcrit * se + base["ci_low"] = mean - margin + base["ci_high"] = mean + margin + return base + + +def _ci_welch(data: list, other: list, conf: float) -> dict: + """IC de la diferencia de medias de dos muestras con el metodo de Welch.""" + a = np.asarray(list(data), dtype=float) + b = np.asarray(list(other), dtype=float) + n1 = int(a.size) + n2 = int(b.size) + + base = { + "mean": float("nan"), + "ci_low": float("nan"), + "ci_high": float("nan"), + "se": float("nan"), + "df": float("nan"), + "confidence": conf, + "n": n1 + n2, + "n1": n1, + "n2": n2, + } + + if n1 == 0 or n2 == 0: + base["note"] = "alguna muestra esta vacia: diferencia e intervalo indefinidos" + return base + + mean1 = float(a.mean()) + mean2 = float(b.mean()) + diff = mean1 - mean2 + base["mean"] = diff + + if n1 < 2 or n2 < 2: + base["note"] = ( + "n < 2 en alguna muestra: error estandar y grados de libertad indefinidos" + ) + return base + + sd1 = float(a.std(ddof=1)) + sd2 = float(b.std(ddof=1)) + se1 = sd1 / math.sqrt(n1) + se2 = sd2 / math.sqrt(n2) + se = math.sqrt(se1 * se1 + se2 * se2) + base["se"] = se + + # Ambas varianzas cero: el IC de la diferencia colapsa al punto. + if se == 0.0: + base["ci_low"] = diff + base["ci_high"] = diff + base["df"] = float("nan") + base["note"] = "varianza cero en ambas muestras: el intervalo colapsa a la diferencia" + return base + + # Grados de libertad de Welch-Satterthwaite. + df = (se1 * se1 + se2 * se2) ** 2 / ( + (se1**4) / (n1 - 1) + (se2**4) / (n2 - 1) + ) + base["df"] = float(df) + + tcrit = float(stats.t.ppf((1.0 + conf) / 2.0, df)) + margin = tcrit * se + base["ci_low"] = diff - margin + base["ci_high"] = diff + margin + return base diff --git a/python/functions/datascience/confidence_interval_mean_test.py b/python/functions/datascience/confidence_interval_mean_test.py new file mode 100644 index 00000000..70ccc029 --- /dev/null +++ b/python/functions/datascience/confidence_interval_mean_test.py @@ -0,0 +1,140 @@ +"""Tests para confidence_interval_mean (IC de la media / diferencia de medias Welch). + +Importa el modulo hoja directamente (`confidence_interval_mean`) para no depender +de que el paquete reexporte la funcion en su __init__ (lo integra el orquestador +al cerrar el grupo). + +Los golden se calculan con scipy dentro del propio test para que sean robustos: +la funcion bajo prueba debe coincidir con la referencia de scipy a ~1e-9. +""" + +import math + +import numpy as np +from scipy import stats + +from confidence_interval_mean import confidence_interval_mean + + +def test_one_sample_golden_contra_scipy(): + # mean=5.0, n=8. Este dataset tiene sd POBLACIONAL (ddof=0) exactamente 2.0, + # pero la sd MUESTRAL (ddof=1, la que exige la spec y la que es correcta para + # el IC de una media con la t) es sqrt(32/7) ~ 2.13809. El golden robusto se + # calcula con scipy usando se con ddof=1, no con el atajo 2.0/sqrt(8). + data = [2, 4, 4, 4, 5, 5, 7, 9] + out = confidence_interval_mean(data, confidence=0.95) + + n = len(data) + mean = float(np.mean(data)) + sd = float(np.std(data, ddof=1)) # sample sd ~ 2.13809 + se = sd / math.sqrt(n) + lo, hi = stats.t.interval(0.95, df=n - 1, loc=mean, scale=se) + + assert abs(out["mean"] - 5.0) < 1e-9 + assert abs(out["se"] - se) < 1e-12 + assert out["df"] == 7.0 + assert out["n"] == 8 + assert out["confidence"] == 0.95 + assert abs(out["ci_low"] - lo) < 1e-9 + assert abs(out["ci_high"] - hi) < 1e-9 + # Valores tabulados correctos para ddof=1 (no los 3.32793/6.67207 del + # enunciado, que asumian erroneamente sd=2.0 / ddof=0). + assert abs(out["ci_low"] - 3.21251) < 1e-3 + assert abs(out["ci_high"] - 6.78749) < 1e-3 + assert "note" not in out + + +def test_one_sample_distinto_nivel_confianza(): + data = [10.0, 12.0, 11.0, 13.0, 9.0, 14.0] + out = confidence_interval_mean(data, confidence=0.99) + + n = len(data) + mean = float(np.mean(data)) + se = float(np.std(data, ddof=1)) / math.sqrt(n) + lo, hi = stats.t.interval(0.99, df=n - 1, loc=mean, scale=se) + + assert abs(out["mean"] - mean) < 1e-12 + assert abs(out["ci_low"] - lo) < 1e-9 + assert abs(out["ci_high"] - hi) < 1e-9 + assert out["df"] == float(n - 1) + + +def test_welch_diferencia_golden_contra_scipy(): + data = [23.0, 21.0, 25.0, 22.0, 24.0, 26.0] + other = [18.0, 20.0, 17.0, 19.0, 21.0] + conf = 0.95 + out = confidence_interval_mean(data, other, confidence=conf) + + a = np.asarray(data, dtype=float) + b = np.asarray(other, dtype=float) + n1, n2 = a.size, b.size + mean1, mean2 = float(a.mean()), float(b.mean()) + diff = mean1 - mean2 + se1 = float(a.std(ddof=1)) / math.sqrt(n1) + se2 = float(b.std(ddof=1)) / math.sqrt(n2) + se = math.sqrt(se1**2 + se2**2) + df = (se1**2 + se2**2) ** 2 / (se1**4 / (n1 - 1) + se2**4 / (n2 - 1)) + lo, hi = stats.t.interval(conf, df=df, loc=diff, scale=se) + + assert abs(out["mean"] - diff) < 1e-9 + assert abs(out["mean"] - (mean1 - mean2)) < 1e-9 + assert abs(out["se"] - se) < 1e-12 + assert abs(out["df"] - df) < 1e-9 + assert abs(out["ci_low"] - lo) < 1e-9 + assert abs(out["ci_high"] - hi) < 1e-9 + assert out["n1"] == n1 + assert out["n2"] == n2 + assert out["n"] == n1 + n2 + assert "note" not in out + + +def test_edge_un_solo_elemento_no_lanza_nan_note(): + out = confidence_interval_mean([5], confidence=0.95) + assert out["mean"] == 5.0 # la media si esta definida con n=1 + assert math.isnan(out["se"]) + assert math.isnan(out["ci_low"]) + assert math.isnan(out["ci_high"]) + assert math.isnan(out["df"]) + assert out["n"] == 1 + assert "note" in out + + +def test_edge_lista_vacia_no_lanza_note(): + out = confidence_interval_mean([], confidence=0.95) + assert math.isnan(out["mean"]) + assert math.isnan(out["ci_low"]) + assert math.isnan(out["ci_high"]) + assert math.isnan(out["se"]) + assert out["n"] == 0 + assert "note" in out + + +def test_edge_varianza_cero_colapsa_al_punto(): + out = confidence_interval_mean([3, 3, 3], confidence=0.95) + assert out["mean"] == 3.0 + assert out["se"] == 0.0 + assert out["ci_low"] == 3.0 + assert out["ci_high"] == 3.0 + assert not math.isnan(out["ci_low"]) + assert out["n"] == 3 + assert "note" in out + + +def test_edge_welch_muestra_vacia_no_lanza_note(): + out = confidence_interval_mean([1.0, 2.0, 3.0], [], confidence=0.95) + assert math.isnan(out["mean"]) + assert math.isnan(out["ci_low"]) + assert math.isnan(out["se"]) + assert out["n1"] == 3 + assert out["n2"] == 0 + assert "note" in out + + +def test_edge_welch_n1_uno_no_lanza_note(): + out = confidence_interval_mean([5.0], [1.0, 2.0, 3.0], confidence=0.95) + # La diferencia de medias si esta definida. + assert abs(out["mean"] - (5.0 - 2.0)) < 1e-9 + assert math.isnan(out["se"]) + assert math.isnan(out["ci_low"]) + assert math.isnan(out["df"]) + assert "note" in out diff --git a/python/functions/datascience/effect_size_cohens_d.md b/python/functions/datascience/effect_size_cohens_d.md new file mode 100644 index 00000000..6e5adbe8 --- /dev/null +++ b/python/functions/datascience/effect_size_cohens_d.md @@ -0,0 +1,80 @@ +--- +name: effect_size_cohens_d +kind: function +lang: py +domain: datascience +version: "1.0.0" +purity: pure +signature: "def effect_size_cohens_d(group_a: list, group_b: list) -> dict" +description: "Tamano del efecto (effect size) entre dos grupos numericos: Cohen's d (diferencia de medias estandarizada por la desviacion tipica combinada, varianzas muestrales ddof=1), Hedges' g (d corregido por el sesgo al alza con muestras pequenas via el factor J) e interpretacion cualitativa de la magnitud segun los umbrales clasicos de Cohen (negligible/small/medium/large). El p-valor dice si hay diferencia; el effect size dice como de grande, de forma adimensional e independiente del N. Pura, sin dependencias externas; nunca lanza: los casos degenerados (varianza cero, N<2, listas vacias) devuelven NaN + una clave note." +tags: [papers, statistics, effect-size, cohens-d, hedges-g, python] +params: + - name: group_a + desc: "primera muestra (lista de numeros). Necesita >=2 observaciones para que exista la varianza muestral (ddof=1)." + - name: group_b + desc: "segunda muestra (lista de numeros). Necesita >=2 observaciones. El signo de cohens_d es positivo cuando mean_a > mean_b." +output: "dict {cohens_d: float (diferencia de medias estandarizada, puede ser NaN), hedges_g: float (cohens_d * factor de correccion J, puede ser NaN), interpretation: str ('negligible'|'small'|'medium'|'large', o 'undefined' en casos degenerados), n_a: int, n_b: int, mean_a: float, mean_b: float, pooled_sd: float (desviacion tipica combinada)}. Casos degenerados (varianza cero en ambos grupos, N<2 en algun grupo, o listas vacias) anaden clave note. Nunca None ni excepcion." +uses_functions: [] +uses_types: [] +returns: [] +returns_optional: false +error_type: "" +imports: [math] +tested: true +tests: ["test_golden_large_effect", "test_hedges_g_menor_en_magnitud_que_cohens_d", "test_interpretation_thresholds", "test_signo_positivo_cuando_a_mayor_que_b", "test_varianza_cero_no_lanza", "test_n_insuficiente_no_lanza", "test_listas_vacias_no_lanza", "test_un_grupo_vacio_no_lanza"] +test_file_path: "python/functions/datascience/effect_size_cohens_d_test.py" +file_path: "python/functions/datascience/effect_size_cohens_d.py" +--- + +## Ejemplo + +```python +from datascience import effect_size_cohens_d + +# Dos grupos desplazados 2 unidades, misma dispersion. +a = [1, 2, 3, 4, 5] # media 3, varianza muestral 2.5 +b = [3, 4, 5, 6, 7] # media 5, varianza muestral 2.5 + +out = effect_size_cohens_d(a, b) +print(out["cohens_d"]) # -> -1.264911... (a esta 1.26 SD por debajo de b) +print(out["hedges_g"]) # -> -1.142500... (|g| < |d|: correccion N pequeno) +print(out["interpretation"]) # -> "large" (|d| >= 0.8) +print(out["pooled_sd"]) # -> 1.581138... + +# Caso degenerado: varianza cero -> no lanza, NaN + note. +deg = effect_size_cohens_d([5, 5, 5], [5, 5, 5]) +print(deg["interpretation"]) # -> "undefined" +print(deg["note"]) # -> "varianza cero, effect size indefinido" +``` + +## Cuando usarla + +Cuando ya sepas que dos grupos difieren (o quieras cuantificar su diferencia) +y necesites una medida **de magnitud, no de significancia**: comparar el antes +y el despues de una intervencion, el grupo control frente al tratamiento, o dos +cohortes. Reportala junto al p-valor para responder "¿como de grande es la +diferencia?" — un p-valor minusculo con N enorme puede esconder un efecto +trivial. Es adimensional (en unidades de desviaciones tipicas), asi que hace +comparables resultados entre estudios y alimenta meta-analisis. Usa **Hedges' g** +en lugar de Cohen's d cuando los grupos sean pequenos (decenas o menos): d +sobreestima el efecto y g lo corrige. + +## Gotchas + +- Pura y sin dependencias externas (solo `math` de la stdlib). +- Usa **varianza muestral** (ddof=1), no poblacional. Por eso cada grupo + necesita al menos 2 observaciones; con N=1 la varianza muestral no existe y la + funcion devuelve NaN + `note`. +- **Nunca lanza excepcion**. Los casos degenerados devuelven `cohens_d` y + `hedges_g` a `float('nan')`, `interpretation="undefined"` y una clave `note`: + varianza cero en ambos grupos (`pooled_sd == 0`), N<2 en algun grupo, o listas + vacias. Comprueba con `math.isnan(out["cohens_d"])` o la presencia de `note` + antes de usar el resultado. +- El **signo** de `cohens_d` depende del orden de los argumentos: positivo si + `mean_a > mean_b`, negativo en caso contrario. La `interpretation` usa `|d|`, + asi que no depende del orden. +- `pooled_sd` asume varianzas comparables entre grupos (homogeneidad). Si las + dispersiones son muy distintas, Cohen's d clasico pierde precision; considera + variantes (Glass's delta) fuera del alcance de esta funcion. +- Los umbrales de Cohen (0.2 / 0.5 / 0.8) son convencion, no ley: interpretalos + segun el dominio. diff --git a/python/functions/datascience/effect_size_cohens_d.py b/python/functions/datascience/effect_size_cohens_d.py new file mode 100644 index 00000000..dee7ed18 --- /dev/null +++ b/python/functions/datascience/effect_size_cohens_d.py @@ -0,0 +1,156 @@ +"""Effect size de dos grupos: Cohen's d, Hedges' g e interpretacion cualitativa. + +Funcion pura del grupo papers. El p-valor responde a "¿hay diferencia?" pero no +a "¿como de grande es?". El tamano del efecto (effect size) cuantifica la +magnitud de la diferencia entre dos grupos de forma adimensional, independiente +del N, y es lo que hace comparables resultados entre estudios (meta-analisis). + +- Cohen's d: diferencia de medias estandarizada por la desviacion tipica + combinada (pooled SD), con varianzas muestrales (ddof=1). +- Hedges' g: Cohen's d corregido por el sesgo al alza que sufre d con muestras + pequenas, multiplicando por el factor de correccion J. +- interpretation: etiqueta cualitativa de |d| segun los umbrales clasicos de + Cohen (negligible / small / medium / large). + +No usa dependencias externas: aritmetica de la libreria estandar (``math``). +""" + +from __future__ import annotations + +import math + + +def _mean(xs: list) -> float: + """Media aritmetica de una lista no vacia de numeros.""" + return sum(float(x) for x in xs) / len(xs) + + +def _sample_variance(xs: list, mean: float) -> float: + """Varianza muestral (ddof=1) de una lista con al menos 2 elementos.""" + n = len(xs) + return sum((float(x) - mean) ** 2 for x in xs) / (n - 1) + + +def _interpret(abs_d: float) -> str: + """Etiqueta cualitativa del tamano del efecto segun |d| (umbrales de Cohen).""" + if abs_d < 0.2: + return "negligible" + if abs_d < 0.5: + return "small" + if abs_d < 0.8: + return "medium" + return "large" + + +def effect_size_cohens_d(group_a: list, group_b: list) -> dict: + """Calcula el tamano del efecto entre dos grupos numericos. + + Devuelve Cohen's d (diferencia de medias estandarizada por la pooled SD), + Hedges' g (d corregido por sesgo de muestra pequena) y una etiqueta + cualitativa de la magnitud segun los umbrales de Cohen. + + Es una funcion pura y determinista: no hace I/O, no muta la entrada. No lanza + excepcion ante datos degenerados; en su lugar devuelve un dict con + ``cohens_d`` / ``hedges_g`` a ``float('nan')``, ``interpretation`` a + ``"undefined"`` y una clave ``note`` explicando el caso. + + Definiciones: + s_pooled = sqrt(((n1-1)*s1^2 + (n2-1)*s2^2) / (n1+n2-2)), con s1^2, s2^2 + varianzas muestrales (ddof=1). + cohens_d = (mean_a - mean_b) / s_pooled. + J = 1 - 3 / (4*(n1+n2) - 9) (factor de correccion de Hedges). + hedges_g = cohens_d * J. + + Args: + group_a: primera muestra (lista de numeros). Necesita >=2 elementos para + que exista la varianza muestral. + group_b: segunda muestra (lista de numeros). Necesita >=2 elementos. + + Returns: + dict con las claves: + cohens_d: float, diferencia de medias estandarizada (puede ser NaN). + hedges_g: float, Cohen's d corregido por sesgo (puede ser NaN). + interpretation: str, "negligible" | "small" | "medium" | "large", o + "undefined" en casos degenerados. + n_a: int, tamano de group_a. + n_b: int, tamano de group_b. + mean_a: float, media de group_a (NaN si vacio). + mean_b: float, media de group_b (NaN si vacio). + pooled_sd: float, desviacion tipica combinada (NaN si indefinida). + + Casos degenerados (lista vacia, N<2 en algun grupo, o varianza cero en + ambos grupos -> pooled_sd == 0) anaden ademas una clave ``note``. + """ + nan = float("nan") + n_a = len(group_a) + n_b = len(group_b) + + # Listas vacias: ni media ni varianza definidas. + if n_a == 0 or n_b == 0: + return { + "cohens_d": nan, + "hedges_g": nan, + "interpretation": "undefined", + "n_a": n_a, + "n_b": n_b, + "mean_a": _mean(group_a) if n_a else nan, + "mean_b": _mean(group_b) if n_b else nan, + "pooled_sd": nan, + "note": "grupo vacio: media y varianza indefinidas, effect size indefinido", + } + + mean_a = _mean(group_a) + mean_b = _mean(group_b) + + # N insuficiente: la varianza muestral (ddof=1) no existe con un solo dato, + # y la correccion de Hedges no es fiable. + if n_a < 2 or n_b < 2: + return { + "cohens_d": nan, + "hedges_g": nan, + "interpretation": "undefined", + "n_a": n_a, + "n_b": n_b, + "mean_a": mean_a, + "mean_b": mean_b, + "pooled_sd": nan, + "note": ( + "N insuficiente: cada grupo necesita >=2 observaciones para la " + "varianza muestral; effect size indefinido" + ), + } + + var_a = _sample_variance(group_a, mean_a) + var_b = _sample_variance(group_b, mean_b) + pooled_sd = math.sqrt( + ((n_a - 1) * var_a + (n_b - 1) * var_b) / (n_a + n_b - 2) + ) + + # Varianza cero en ambos grupos: no se puede estandarizar (division por 0). + if pooled_sd == 0.0: + return { + "cohens_d": nan, + "hedges_g": nan, + "interpretation": "undefined", + "n_a": n_a, + "n_b": n_b, + "mean_a": mean_a, + "mean_b": mean_b, + "pooled_sd": 0.0, + "note": "varianza cero, effect size indefinido", + } + + cohens_d = (mean_a - mean_b) / pooled_sd + j = 1.0 - 3.0 / (4.0 * (n_a + n_b) - 9.0) + hedges_g = cohens_d * j + + return { + "cohens_d": cohens_d, + "hedges_g": hedges_g, + "interpretation": _interpret(abs(cohens_d)), + "n_a": n_a, + "n_b": n_b, + "mean_a": mean_a, + "mean_b": mean_b, + "pooled_sd": pooled_sd, + } diff --git a/python/functions/datascience/effect_size_cohens_d_test.py b/python/functions/datascience/effect_size_cohens_d_test.py new file mode 100644 index 00000000..f5c43049 --- /dev/null +++ b/python/functions/datascience/effect_size_cohens_d_test.py @@ -0,0 +1,96 @@ +"""Tests para effect_size_cohens_d (tamano del efecto de dos grupos). + +Importa el modulo hoja directamente (`effect_size_cohens_d`) para no depender de +que el paquete reexporte la funcion en su __init__ (lo integra el orquestador al +cerrar el grupo papers). El pytest del repo tiene pythonpath=["functions", ...], +asi que el modulo hoja se resuelve por su nombre directo. +""" + +import math + +from effect_size_cohens_d import effect_size_cohens_d + + +def test_golden_large_effect(): + # group_a: mean 3, var muestral 2.5; group_b: mean 5, var 2.5. + # pooled_sd = sqrt(2.5) ~= 1.5811388. + # cohens_d = (3-5)/1.5811388 ~= -1.264911. + # J = 1 - 3/(4*10-9) = 1 - 3/31 = 0.9032258. + # hedges_g = d * J = -1.2649111 * 0.9032258 ~= -1.142500. + out = effect_size_cohens_d([1, 2, 3, 4, 5], [3, 4, 5, 6, 7]) + assert abs(out["cohens_d"] - (-1.26491)) < 1e-4 + assert abs(out["hedges_g"] - (-1.14250)) < 1e-4 + assert out["interpretation"] == "large" + assert out["n_a"] == 5 + assert out["n_b"] == 5 + assert abs(out["mean_a"] - 3.0) < 1e-12 + assert abs(out["mean_b"] - 5.0) < 1e-12 + assert abs(out["pooled_sd"] - math.sqrt(2.5)) < 1e-9 + assert "note" not in out + + +def test_hedges_g_menor_en_magnitud_que_cohens_d(): + # La correccion J esta en (0, 1), asi que |g| < |d| siempre. + out = effect_size_cohens_d([1, 2, 3, 4, 5], [3, 4, 5, 6, 7]) + assert abs(out["hedges_g"]) < abs(out["cohens_d"]) + + +def test_interpretation_thresholds(): + # negligible: |d| < 0.2. Medias casi iguales con varianza grande. + neg = effect_size_cohens_d([0, 10, 20, 30], [1, 11, 21, 31]) + assert neg["interpretation"] == "negligible" + assert abs(neg["cohens_d"]) < 0.2 + + # small: 0.2 <= |d| < 0.5. + small = effect_size_cohens_d([0, 10, 20, 30], [4, 14, 24, 34]) + assert small["interpretation"] == "small" + assert 0.2 <= abs(small["cohens_d"]) < 0.5 + + # medium: 0.5 <= |d| < 0.8. + medium = effect_size_cohens_d([0, 10, 20, 30], [9, 19, 29, 39]) + assert medium["interpretation"] == "medium" + assert 0.5 <= abs(medium["cohens_d"]) < 0.8 + + +def test_signo_positivo_cuando_a_mayor_que_b(): + out = effect_size_cohens_d([10, 12, 14, 16], [1, 2, 3, 4]) + assert out["cohens_d"] > 0 + assert out["interpretation"] == "large" + + +def test_varianza_cero_no_lanza(): + out = effect_size_cohens_d([5, 5, 5], [5, 5, 5]) + assert math.isnan(out["cohens_d"]) + assert math.isnan(out["hedges_g"]) + assert out["interpretation"] == "undefined" + assert out["pooled_sd"] == 0.0 + assert "note" in out + assert "varianza cero" in out["note"] + + +def test_n_insuficiente_no_lanza(): + out = effect_size_cohens_d([3], [1, 2, 3]) + assert math.isnan(out["cohens_d"]) + assert math.isnan(out["hedges_g"]) + assert out["interpretation"] == "undefined" + assert out["n_a"] == 1 + assert out["n_b"] == 3 + assert "note" in out + + +def test_listas_vacias_no_lanza(): + out = effect_size_cohens_d([], []) + assert math.isnan(out["cohens_d"]) + assert math.isnan(out["hedges_g"]) + assert out["interpretation"] == "undefined" + assert out["n_a"] == 0 + assert out["n_b"] == 0 + assert "note" in out + + +def test_un_grupo_vacio_no_lanza(): + out = effect_size_cohens_d([1, 2, 3], []) + assert math.isnan(out["cohens_d"]) + assert out["interpretation"] == "undefined" + assert out["n_b"] == 0 + assert "note" in out diff --git a/python/functions/datascience/fdr_correction.md b/python/functions/datascience/fdr_correction.md index 32fd4635..8ae003da 100644 --- a/python/functions/datascience/fdr_correction.md +++ b/python/functions/datascience/fdr_correction.md @@ -3,19 +3,19 @@ name: fdr_correction kind: function lang: py domain: datascience -version: "1.0.0" +version: "1.1.0" purity: pure signature: "def fdr_correction(pvalues: list, alpha: float = 0.05, method: str = \"bh\") -> dict" -description: "Correccion de comparaciones multiples (multiple-testing) sobre una lista de p-valores: Benjamini-Hochberg (FDR, 'bh') o Bonferroni (FWER, 'bonferroni'). Antidoto al sesgo de mineria de datos (data-mining bias): al evaluar muchas hipotesis a la vez (todos los pares de una matriz), el azar produce falsos positivos; esta funcion ajusta los p-valores y marca cuales siguen siendo significativos tras corregir. Pura, sin dependencias externas, alineada 1:1 con la entrada (admite None en posiciones sin test)." -tags: [eda, statistics, multiple-testing, fdr, benjamini-hochberg, bonferroni, p-value, data-mining-bias, python] +description: "Correccion de comparaciones multiples (multiple-testing) sobre una lista de p-valores: Benjamini-Hochberg (FDR, 'bh'), Bonferroni (FWER, 'bonferroni') o Holm-Bonferroni (FWER step-down, 'holm', mas potente que Bonferroni simple). Antidoto al sesgo de mineria de datos (data-mining bias): al evaluar muchas hipotesis a la vez (todos los pares de una matriz), el azar produce falsos positivos; esta funcion ajusta los p-valores y marca cuales siguen siendo significativos tras corregir. Pura, sin dependencias externas, alineada 1:1 con la entrada (admite None en posiciones sin test)." +tags: [eda, statistics, multiple-testing, fdr, benjamini-hochberg, bonferroni, holm, holm-bonferroni, fwer, p-value, data-mining-bias, python] params: - name: pvalues desc: "lista de p-valores (floats en [0, 1]). Se admiten None u otros valores no validos en posiciones sin test disponible; se propagan como None en la salida y no cuentan como prueba (m)." - name: alpha desc: "nivel de significancia objetivo tras la correccion (default 0.05). Para BH es el umbral del FDR; para Bonferroni, del FWER (tasa de error por familia)." - name: method - desc: "'bh' = Benjamini-Hochberg (controla FDR, menos conservador, mas potencia); 'bonferroni' = controla FWER (mas conservador). Cualquier otro valor devuelve un dict con note." -output: "dict {p_values_adjusted: lista alineada con pvalues (float ajustado o None), reject: lista de bool (True = significativo tras corregir), n_tests: nº de p-valores validos (m), n_rejected: nº de hipotesis rechazadas, alpha: float aplicado, method: str}. Casos degenerados (vacio, sin p validos, metodo desconocido) anaden clave note. Nunca None ni excepcion." + desc: "'bh' = Benjamini-Hochberg (controla FDR, menos conservador, mas potencia); 'bonferroni' = controla FWER (mas conservador); 'holm' = Holm-Bonferroni (controla FWER, step-down, uniformemente mas potente que Bonferroni simple). Cualquier otro valor devuelve un dict con note." +output: "dict {p_values_adjusted: lista alineada con pvalues (float ajustado o None), reject: lista de bool (True = significativo tras corregir), n_tests: nº de p-valores validos (m), n_rejected: nº de hipotesis rechazadas, alpha: float aplicado, method: str ('bh' | 'bonferroni' | 'holm')}. Casos degenerados (vacio, sin p validos, metodo desconocido) anaden clave note. Nunca None ni excepcion." uses_functions: [] uses_types: [] returns: [] @@ -23,7 +23,7 @@ returns_optional: false error_type: "" imports: [math] tested: true -tests: ["test_bh_golden_rechaza_dos_de_tres", "test_bonferroni_mas_conservador_que_bh", "test_p_values_adjusted_alineados_y_en_rango", "test_none_se_propaga_alineado", "test_lista_vacia_devuelve_note", "test_solo_none_devuelve_note", "test_metodo_desconocido_devuelve_note", "test_todos_significativos"] +tests: ["test_bh_golden_rechaza_dos_de_tres", "test_bonferroni_mas_conservador_que_bh", "test_p_values_adjusted_alineados_y_en_rango", "test_none_se_propaga_alineado", "test_lista_vacia_devuelve_note", "test_solo_none_devuelve_note", "test_metodo_desconocido_devuelve_note", "test_todos_significativos", "test_holm_golden_rechaza_dos_de_cuatro", "test_holm_entre_bonferroni_y_bh", "test_none_se_propaga_alineado_holm", "test_lista_vacia_holm_devuelve_note"] test_file_path: "python/functions/datascience/fdr_correction_test.py" file_path: "python/functions/datascience/fdr_correction.py" --- @@ -45,6 +45,13 @@ bon = fdr_correction(pvalues, alpha=0.05, method="bonferroni") print(bon["reject"]) # -> [True, False, False] print(bon["p_values_adjusted"]) # -> [0.03, 0.06, 1.0] +# Holm-Bonferroni (step-down): controla el FWER como Bonferroni pero es mas +# potente; rechaza al menos tanto como Bonferroni simple, nunca menos. +holm = fdr_correction([0.01, 0.04, 0.03, 0.005], alpha=0.05, method="holm") +print(holm["reject"]) # -> [True, False, False, True] +print(holm["p_values_adjusted"]) # -> [0.03, 0.06, 0.06, 0.02] +print(holm["n_rejected"]) # -> 2 + # Posiciones sin test (None) se propagan alineadas: el llamador puede pasar la # lista completa de pares y recuperar el mapeo 1:1. mix = fdr_correction([0.001, None, 0.9]) @@ -61,8 +68,11 @@ combinaciones y se quede con las que "pasan". Sin corregir, con N pruebas y alpha=0.05 esperas ~5% de falsos positivos *por azar*: cuantas mas pruebas, mas correlaciones espurias. Llama a `fdr_correction` con todos los p-valores de la familia y usa `reject` (no el umbral crudo) para decidir que es real. Usa `"bh"` -por defecto (mejor potencia); `"bonferroni"` cuando un falso positivo sea muy -costoso y prefieras maxima cautela. +por defecto (mejor potencia); `"holm"` (Holm-Bonferroni, FWER step-down) cuando +quieras controlar el FWER pero sin la perdida de potencia de Bonferroni simple +(rechaza al menos tanto como `"bonferroni"`, nunca menos); `"bonferroni"` cuando +un falso positivo sea muy costoso y prefieras la maxima cautela del metodo mas +simple. ## Gotchas @@ -76,8 +86,16 @@ costoso y prefieras maxima cautela. eso puedes pasar la lista completa de pares aunque algunos no tengan test. - `n_tests` es el numero de p-valores **validos** (m), que puede ser menor que `len(pvalues)` si hay `None`. -- BH y Bonferroni controlan cosas distintas: BH la tasa de falsos - descubrimientos (FDR), Bonferroni la probabilidad de *cualquier* falso +- BH controla cosa distinta que Bonferroni/Holm: BH la tasa de falsos + descubrimientos (FDR); Bonferroni y Holm la probabilidad de *cualquier* falso positivo (FWER). No son intercambiables; elige segun el coste de equivocarte. +- `"holm"` y `"bonferroni"` controlan ambos el FWER, pero Holm es step-down y + uniformemente mas potente: rechaza al menos tantas hipotesis como Bonferroni + simple sobre el mismo set, nunca menos. Si controlas FWER, `"holm"` domina a + `"bonferroni"` salvo que necesites el ajuste mas simple por interpretabilidad. - Metodo desconocido o lista vacia/sin p validos no lanzan: devuelven un dict - con `note`. + con `note`. Los metodos validos son `"bh"`, `"bonferroni"` y `"holm"`. + +## Capability growth log + +- v1.1.0 (2026-06-30) — añade method="holm" (Holm-Bonferroni step-down, FWER, más potente que Bonferroni simple). diff --git a/python/functions/datascience/fdr_correction.py b/python/functions/datascience/fdr_correction.py index 25b0d58b..ea5408a0 100644 --- a/python/functions/datascience/fdr_correction.py +++ b/python/functions/datascience/fdr_correction.py @@ -5,12 +5,15 @@ todos los pares de una matriz de asociacion), la probabilidad de obtener al meno un falso positivo por azar crece con el numero de pruebas: es el sesgo de mineria de datos (data-mining bias) descrito por Aronson en *Evidence-Based Technical Analysis* (cap. 6). Esta funcion ajusta los p-valores para controlar ese sesgo -mediante dos metodos clasicos: +mediante tres metodos clasicos: - Benjamini-Hochberg (``"bh"``): controla la tasa de falsos descubrimientos (False Discovery Rate, FDR). Menos conservador, mas potencia estadistica. - Bonferroni (``"bonferroni"``): controla la tasa de error por familia (Family-Wise Error Rate, FWER). Mas conservador. +- Holm-Bonferroni (``"holm"``): controla el FWER como Bonferroni pero es un + procedimiento step-down uniformemente mas potente; rechaza al menos tantas + hipotesis como Bonferroni simple, nunca menos. No usa dependencias externas: aritmetica de la libreria estandar. """ @@ -35,8 +38,9 @@ def _is_valid_p(v) -> bool: def fdr_correction(pvalues: list, alpha: float = 0.05, method: str = "bh") -> dict: """Corrige una lista de p-valores por comparaciones multiples. - Aplica Benjamini-Hochberg (FDR) o Bonferroni (FWER) sobre ``pvalues`` y - devuelve, alineado posicion a posicion con la entrada, el p-valor ajustado y + Aplica Benjamini-Hochberg (FDR), Bonferroni (FWER) o Holm-Bonferroni + (FWER, step-down) sobre ``pvalues`` y devuelve, alineado posicion a + posicion con la entrada, el p-valor ajustado y si cada hipotesis se rechaza al nivel ``alpha`` tras la correccion. Las posiciones cuyo valor no sea un p-valor valido (``None``, ``NaN``, fuera de ``[0, 1]`` o no numerico) se conservan en la salida como ``None`` / @@ -53,8 +57,10 @@ def fdr_correction(pvalues: list, alpha: float = 0.05, method: str = "bh") -> di otros valores no validos en posiciones sin test disponible; se propagan como ``None`` en la salida y no cuentan como prueba. alpha: nivel de significancia objetivo tras la correccion (default 0.05). - Para BH es el umbral del FDR; para Bonferroni, del FWER. - method: ``"bh"`` (Benjamini-Hochberg, FDR) o ``"bonferroni"`` (FWER). + Para BH es el umbral del FDR; para Bonferroni y Holm, del FWER. + method: ``"bh"`` (Benjamini-Hochberg, FDR), ``"bonferroni"`` (FWER) o + ``"holm"`` (Holm-Bonferroni, FWER step-down, mas potente que + Bonferroni simple). Returns: dict con las claves: @@ -68,7 +74,7 @@ def fdr_correction(pvalues: list, alpha: float = 0.05, method: str = "bh") -> di n_tests: numero de p-valores validos usados en la correccion (m). n_rejected: numero de hipotesis rechazadas (significativas). alpha: nivel de significancia aplicado (float). - method: metodo aplicado (``"bh"`` o ``"bonferroni"``). + method: metodo aplicado (``"bh"``, ``"bonferroni"`` o ``"holm"``). Casos degenerados (lista vacia, sin p-valores validos o metodo desconocido) anaden ademas una clave ``note`` y devuelven listas @@ -76,7 +82,7 @@ def fdr_correction(pvalues: list, alpha: float = 0.05, method: str = "bh") -> di en las posiciones invalidas). """ method_norm = (method or "").strip().lower() - if method_norm not in {"bh", "bonferroni"}: + if method_norm not in {"bh", "bonferroni", "holm"}: n = len(pvalues) return { "p_values_adjusted": [None] * n, @@ -86,8 +92,8 @@ def fdr_correction(pvalues: list, alpha: float = 0.05, method: str = "bh") -> di "alpha": float(alpha), "method": method, "note": ( - f"metodo desconocido '{method}'; usa 'bh' (Benjamini-Hochberg) " - "o 'bonferroni'" + f"metodo desconocido '{method}'; usa 'bh' (Benjamini-Hochberg), " + "'bonferroni' o 'holm' (Holm-Bonferroni)" ), } @@ -129,6 +135,20 @@ def fdr_correction(pvalues: list, alpha: float = 0.05, method: str = "bh") -> di padj = min(1.0, p * m) adjusted[orig_idx] = padj reject[orig_idx] = padj <= a + elif method_norm == "holm": + # Holm-Bonferroni (step-down). Ordena p ascendente; para el rank k + # (1-indexed) el p ajustado crudo es (m - k + 1) * p_(k). Impon + # monotonicidad acumulada (no decreciente) recorriendo de menor a mayor: + # padj_(k) = max(padj_(k-1), min(1, (m-k+1)*p_(k))), con padj_(0)=0. + order = sorted(valid, key=lambda t: t[1]) # [(orig_idx, p), ...] por p asc + prev = 0.0 + for k in range(1, m + 1): + orig_idx, p = order[k - 1] + raw = min(1.0, (m - k + 1) * p) + padj = max(prev, raw) + prev = padj + adjusted[orig_idx] = padj + reject[orig_idx] = padj <= a else: # Benjamini-Hochberg (step-up). Ordena p ascendente y calcula q-valores # con la monotonicidad acumulada de derecha a izquierda. diff --git a/python/functions/datascience/fdr_correction_test.py b/python/functions/datascience/fdr_correction_test.py index cb6b2bc2..529ed0bd 100644 --- a/python/functions/datascience/fdr_correction_test.py +++ b/python/functions/datascience/fdr_correction_test.py @@ -82,7 +82,8 @@ def test_solo_none_devuelve_note(): def test_metodo_desconocido_devuelve_note(): - out = fdr_correction([0.01, 0.02], method="holm") + # 'holm' ya es un metodo valido (v1.1.0); usamos uno realmente desconocido. + out = fdr_correction([0.01, 0.02], method="sidak") assert "note" in out assert out["n_rejected"] == 0 assert out["reject"] == [False, False] @@ -97,3 +98,66 @@ def test_todos_significativos(): assert bon["n_rejected"] == 3 assert all(bh["reject"]) assert all(bon["reject"]) + + +def test_holm_golden_rechaza_dos_de_cuatro(): + # Holm-Bonferroni (step-down) sobre [0.01, 0.04, 0.03, 0.005], m=4, alpha=0.05. + # Ordenado ascendente: 0.005, 0.01, 0.03, 0.04. + # padj_(1) = 4*0.005 = 0.02 + # padj_(2) = max(0.02, 3*0.01=0.03) = 0.03 + # padj_(3) = max(0.03, 2*0.03=0.06) = 0.06 + # padj_(4) = max(0.06, 1*0.04=0.04) = 0.06 + # Mapeado al orden de entrada [0.01, 0.04, 0.03, 0.005]: + # 0.01 -> 0.03, 0.04 -> 0.06, 0.03 -> 0.06, 0.005 -> 0.02 + out = fdr_correction([0.01, 0.04, 0.03, 0.005], alpha=0.05, method="holm") + assert out["method"] == "holm" + assert out["n_tests"] == 4 + adj = out["p_values_adjusted"] + assert abs(adj[0] - 0.03) < 1e-9 + assert abs(adj[1] - 0.06) < 1e-9 + assert abs(adj[2] - 0.06) < 1e-9 + assert abs(adj[3] - 0.02) < 1e-9 + assert out["reject"] == [True, False, False, True] + assert out["n_rejected"] == 2 + + +def test_holm_entre_bonferroni_y_bh(): + # Holm controla FWER como Bonferroni pero es step-down: rechaza AL MENOS + # tanto como Bonferroni simple, y a lo sumo tanto como BH (FDR, menos + # conservador). Cadena de potencia: bonferroni <= holm <= bh. + pvalues = [0.01, 0.02, 0.04, 0.005] + bon = fdr_correction(pvalues, alpha=0.05, method="bonferroni") + holm = fdr_correction(pvalues, alpha=0.05, method="holm") + bh = fdr_correction(pvalues, alpha=0.05, method="bh") + assert holm["n_rejected"] >= bon["n_rejected"] + assert holm["n_rejected"] <= bh["n_rejected"] + # En este set Holm gana potencia frente a Bonferroni simple (estricto). + assert holm["n_rejected"] > bon["n_rejected"] + + # Un set donde Holm es estrictamente mas conservador que BH. + pvals2 = [0.01, 0.02, 0.03, 0.04] + bon2 = fdr_correction(pvals2, alpha=0.05, method="bonferroni") + holm2 = fdr_correction(pvals2, alpha=0.05, method="holm") + bh2 = fdr_correction(pvals2, alpha=0.05, method="bh") + assert holm2["n_rejected"] >= bon2["n_rejected"] + assert holm2["n_rejected"] < bh2["n_rejected"] + + +def test_none_se_propaga_alineado_holm(): + # None se propaga alineado tambien con holm: la posicion central no cuenta + # como prueba (m=2) y se devuelve como None / False. + out = fdr_correction([0.001, None, 0.9], method="holm") + assert out["n_tests"] == 2 + assert out["p_values_adjusted"][1] is None + assert out["reject"][1] is False + assert out["reject"][0] is True + assert len(out["reject"]) == 3 + + +def test_lista_vacia_holm_devuelve_note(): + out = fdr_correction([], method="holm") + assert out["p_values_adjusted"] == [] + assert out["reject"] == [] + assert out["n_tests"] == 0 + assert out["n_rejected"] == 0 + assert "note" in out diff --git a/python/functions/datascience/preregister_hypothesis.md b/python/functions/datascience/preregister_hypothesis.md new file mode 100644 index 00000000..ea4fc848 --- /dev/null +++ b/python/functions/datascience/preregister_hypothesis.md @@ -0,0 +1,100 @@ +--- +name: preregister_hypothesis +kind: function +lang: py +domain: datascience +version: "1.0.0" +purity: impure +signature: "def preregister_hypothesis(paper_dir: str, hypotheses: dict, analysis_plan: dict) -> dict" +description: "Pre-registra (congela) la hipotesis y el plan de analisis de un paper ANTES de mirar los datos: antidoto al HARKing (Hypothesizing After the Results are Known). Escribe/actualiza <paper_dir>/preregistration.md con un frontmatter (paper_slug, frozen_at, content_hash, status) y un cuerpo markdown DETERMINISTA derivado de (hypotheses, analysis_plan) (mismo input -> mismo cuerpo byte a byte, claves ordenadas alfabeticamente). El content_hash es sha256 del cuerpo NORMALIZADO (strip por linea + colapso de blancos), nunca del frontmatter. Una vez status=frozen es INMUTABLE: re-congelar con el mismo contenido es idempotente (no reescribe, devuelve unchanged) y re-congelar con contenido distinto se RECHAZA (no sobrescribe, devuelve error) para que no se pueda ajustar la hipotesis a los resultados. Estilo dict-no-throw: nunca lanza." +tags: [papers, preregistration, reproducibility, anti-harking, python] +params: + - name: paper_dir + desc: "ruta del directorio del paper, p.ej. 'papers/0001-mi-paper'. Debe existir (no se crea aqui). El paper_slug del frontmatter es el basename del dir. Si no existe o no es str -> {status:error, path, note} sin crash ni creacion." + - name: hypotheses + desc: "dict de hipotesis, p.ej. {'h0': 'no hay diferencia ...', 'h1': 'el grupo A > grupo B ...'}. Se renderiza en la seccion '## Hypotheses' con una linea por clave, ordenadas alfabeticamente para determinismo." + - name: analysis_plan + desc: "dict con el plan de analisis, p.ej. {'test': 'welch_t_test', 'effect_size_metric': 'cohens_d', 'decision_rule': 'rechazar H0 si p<0.05 tras Holm y |d|>=0.5', 'planned_n': 100, 'multiple_correction': 'holm'}. Se renderiza en '## Analysis plan' con una linea por clave (ordenadas alfabeticamente). Acepta valores no-str (int, etc.)." +output: "dict dict-no-throw (NUNCA lanza). status='frozen' cuando escribe el archivo por primera vez o congela un draft previo ({status, path, content_hash, frozen_at}). status='unchanged' cuando ya estaba frozen con el mismo content_hash: no reescribe y preserva el archivo byte-identico incl. el frozen_at original ({status, path, content_hash, frozen_at}). status='error' cuando paper_dir no existe, ya esta frozen con un hash distinto (rechazo anti-HARKing, no sobrescribe), inputs invalidos o error de I/O ({status, path, note, [content_hash]})." +uses_functions: [] +uses_types: [] +returns: [] +returns_optional: false +error_type: "error_go_core" +imports: [hashlib] +tested: true +tests: ["test_golden_congela_y_escribe_archivo", "test_idempotente_mismo_input_no_reescribe", "test_inmutabilidad_anti_harking_rechaza_contenido_distinto", "test_error_paper_dir_inexistente_no_crash_no_crea"] +test_file_path: "python/functions/datascience/preregister_hypothesis_test.py" +file_path: "python/functions/datascience/preregister_hypothesis.py" +--- + +## Ejemplo + +```python +import os, tempfile +from datascience import preregister_hypothesis + +# Un directorio de paper que ya existe. +paper_dir = tempfile.mkdtemp(prefix="0001-") + +hypotheses = { + "h0": "no hay diferencia entre el grupo A y el grupo B", + "h1": "el grupo A tiene mayor conversion que el grupo B", +} +analysis_plan = { + "test": "welch_t_test", + "effect_size_metric": "cohens_d", + "decision_rule": "rechazar H0 si p<0.05 tras Holm y |d|>=0.5", + "planned_n": 100, + "multiple_correction": "holm", +} + +# 1) Primera vez: congela y escribe <paper_dir>/preregistration.md +r1 = preregister_hypothesis(paper_dir, hypotheses, analysis_plan) +print(r1["status"]) # -> "frozen" +print(r1["content_hash"]) # sha256 del cuerpo + +# 2) Mismo input: idempotente, no reescribe. +r2 = preregister_hypothesis(paper_dir, hypotheses, analysis_plan) +print(r2["status"]) # -> "unchanged" + +# 3) Cambiar la hipotesis tras congelar (HARKing): rechazado, archivo intacto. +r3 = preregister_hypothesis(paper_dir, {"h0": "...", "h1": "otra cosa"}, analysis_plan) +print(r3["status"]) # -> "error" +``` + +## Cuando usarla + +Llamala al ARRANCAR el analisis de un paper, antes de tocar los datos, para +dejar por escrito (y firmado por hash) que vas a probar y como vas a decidir. +Es el primer paso de un flujo reproducible: pre-registras la hipotesis y el plan +(`test`, `effect_size_metric`, `decision_rule`, `planned_n`, +`multiple_correction`), y solo despues corres el analisis y comparas con lo +pre-registrado. Si mas tarde el analisis "descubre" otra hipotesis que encaja +mejor con los datos, el pre-registro congelado deja en evidencia el cambio: no se +puede reescribir. Combinala con `effect_size_cohens_d` y `fdr_correction` para +cerrar el plan declarado (effect size + correccion de multiples comparaciones). + +## Gotchas + +- **Inmutabilidad (el corazon)**: una vez `status: frozen`, el pre-registro NO se + puede editar. Re-congelar con el MISMO contenido es idempotente (`unchanged`, + no reescribe, preserva incluso el `frozen_at` original). Re-congelar con + contenido DISTINTO devuelve `error` y deja el archivo intacto: asi se mata el + HARKing. Para cambiar de verdad la hipotesis hay que borrar el archivo a mano y + asumir explicitamente que ya no es un pre-registro valido. +- **dict-no-throw**: la funcion NUNCA lanza. Cualquier error previsible + (directorio inexistente, inputs no-dict, fallo de I/O, excepcion inesperada) se + captura y se devuelve como `{"status": "error", "note": ...}`. Siempre incluye + `path` (la ruta esperada del `preregistration.md`). +- **El hash es SOLO del cuerpo, nunca del frontmatter**: el frontmatter contiene + el propio `content_hash` y el `frozen_at` (timestamp), asi que incluirlos en el + hash seria circular y romperia la idempotencia. El cuerpo se normaliza antes de + hashear (strip por linea + colapso de lineas en blanco + strip final): cambios + irrelevantes de whitespace no alteran el hash, pero cambios de contenido SI. +- **Determinismo**: el cuerpo se genera con las claves de `hypotheses` y + `analysis_plan` ordenadas alfabeticamente, de modo que el orden de insercion del + dict no afecta al hash. Mismo `(hypotheses, analysis_plan)` -> mismo cuerpo y + mismo hash, byte a byte. +- **No crea el directorio del paper**: si `paper_dir` no existe, devuelve `error` + sin crear nada (ni el dir ni el archivo). diff --git a/python/functions/datascience/preregister_hypothesis.py b/python/functions/datascience/preregister_hypothesis.py new file mode 100644 index 00000000..0be8cc2b --- /dev/null +++ b/python/functions/datascience/preregister_hypothesis.py @@ -0,0 +1,202 @@ +"""Congela (pre-registra) la hipotesis y el plan de analisis de un paper. + +Anti-HARKing (Hypothesizing After the Results are Known): el pre-registro fija +la hipotesis y el plan de analisis ANTES de mirar los datos. Una vez congelado +(``status: frozen``) es INMUTABLE: cualquier intento posterior de re-congelar con +un contenido distinto se RECHAZA en vez de sobrescribir, de modo que no se puede +"ajustar" la hipotesis a los resultados despues de verlos. + +Escribe/actualiza ``<paper_dir>/preregistration.md`` con un frontmatter +(``paper_slug``, ``frozen_at``, ``content_hash``, ``status``) y un cuerpo +markdown DETERMINISTA derivado de ``(hypotheses, analysis_plan)``. + +Estilo dict-no-throw: NUNCA lanza; cualquier error previsible se captura y se +devuelve como ``{"status": "error", "note": ...}``. +""" + +import hashlib +import os +from datetime import datetime, timezone + + +def _build_body(hypotheses: dict, analysis_plan: dict) -> str: + """Construye el cuerpo markdown del pre-registro de forma DETERMINISTA. + + Mismo ``(hypotheses, analysis_plan)`` -> mismo cuerpo byte a byte. Las claves + se ordenan alfabeticamente para no depender del orden de insercion del dict. + """ + lines = ["## Hypotheses", ""] + for k in sorted(hypotheses.keys()): + lines.append(f"- **{k}**: {hypotheses[k]}") + lines.append("") + lines.append("## Analysis plan") + lines.append("") + for k in sorted(analysis_plan.keys()): + lines.append(f"- **{k}**: {analysis_plan[k]}") + return "\n".join(lines) + + +def _normalize(body: str) -> str: + """Normaliza el cuerpo para el hash: strip por linea + colapsa blancos. + + Cambios irrelevantes de whitespace (espacios al final, dobles lineas en + blanco) no alteran el hash; cambios de contenido SI. Esto hace el hash + robusto sin perder la capacidad de detectar ediciones reales. + """ + out = [] + prev_blank = False + for raw in body.splitlines(): + line = raw.strip() + if line == "": + if prev_blank: + continue + prev_blank = True + else: + prev_blank = False + out.append(line) + return "\n".join(out).strip() + + +def _content_hash(body: str) -> str: + """sha256 hex del cuerpo NORMALIZADO (nunca del frontmatter).""" + return hashlib.sha256(_normalize(body).encode("utf-8")).hexdigest() + + +def _parse_frontmatter(text: str) -> dict: + """Parsea el frontmatter ``--- ... ---`` simple (key: value) de un .md.""" + if not text.startswith("---"): + return {} + parts = text.split("---", 2) + if len(parts) < 3: + return {} + fm = {} + for line in parts[1].splitlines(): + line = line.strip() + if not line or ":" not in line: + continue + key, _, value = line.partition(":") + fm[key.strip()] = value.strip() + return fm + + +def _render_file(slug: str, frozen_at: str, content_hash: str, body: str) -> str: + """Compone el archivo completo: frontmatter frozen + cuerpo.""" + return ( + "---\n" + f"paper_slug: {slug}\n" + f"frozen_at: {frozen_at}\n" + f"content_hash: {content_hash}\n" + "status: frozen\n" + "---\n" + "\n" + f"{body}\n" + ) + + +def preregister_hypothesis(paper_dir: str, hypotheses: dict, analysis_plan: dict) -> dict: + """Congela la hipotesis y el plan de analisis de un paper (anti-HARKing). + + Escribe ``<paper_dir>/preregistration.md`` con frontmatter ``status: frozen`` + y un cuerpo markdown determinista. Una vez congelado es inmutable. + + Args: + paper_dir: ruta del directorio del paper (p.ej. ``"papers/0001-mi-paper"``). + El ``paper_slug`` es el basename del directorio. Debe existir. + hypotheses: dict de hipotesis, p.ej. + ``{"h0": "no hay diferencia ...", "h1": "grupo A > grupo B ..."}``. + analysis_plan: dict con el plan, p.ej. + ``{"test": "welch_t_test", "effect_size_metric": "cohens_d", + "decision_rule": "...", "planned_n": 100, "multiple_correction": "holm"}``. + + Returns: + dict dict-no-throw (NUNCA lanza). Claves segun el caso: + - frozen: {"status": "frozen", "path", "content_hash", "frozen_at"} + - unchanged: {"status": "unchanged", "path", "content_hash", "frozen_at"} + - error: {"status": "error", "path", "note", ...} + """ + expected_path = os.path.join(paper_dir, "preregistration.md") + try: + # 1) El directorio del paper debe existir; no se crea aqui. + if not isinstance(paper_dir, str) or not os.path.isdir(paper_dir): + return { + "status": "error", + "path": expected_path, + "note": f"paper_dir no existe: {paper_dir}", + } + + if not isinstance(hypotheses, dict) or not isinstance(analysis_plan, dict): + return { + "status": "error", + "path": expected_path, + "note": "hypotheses y analysis_plan deben ser dict", + } + + slug = os.path.basename(os.path.normpath(paper_dir)) + + # 2) + 3) Cuerpo determinista y su hash (solo del cuerpo, no del frontmatter). + body = _build_body(hypotheses, analysis_plan) + new_hash = _content_hash(body) + + # 5) Logica de escritura. + if os.path.exists(expected_path): + existing = "" + try: + with open(expected_path, "r", encoding="utf-8") as fh: + existing = fh.read() + except OSError as exc: + return { + "status": "error", + "path": expected_path, + "note": f"no se pudo leer el pre-registro existente: {exc}", + } + fm = _parse_frontmatter(existing) + old_status = fm.get("status", "") + old_hash = fm.get("content_hash", "") + old_frozen_at = fm.get("frozen_at", "") + + if old_status == "frozen": + if old_hash == new_hash: + # Idempotente: mismo contenido ya congelado. No se reescribe. + return { + "status": "unchanged", + "path": expected_path, + "content_hash": new_hash, + "frozen_at": old_frozen_at, + } + # Inmutabilidad: ya congelado con OTRO hash -> se rechaza (anti-HARKing). + return { + "status": "error", + "path": expected_path, + "content_hash": new_hash, + "note": ( + "pre-registro inmutable: ya esta congelado (frozen) con un " + "hash distinto; un pre-registro no se puede editar tras " + "congelarse" + ), + } + # status != "frozen" (p.ej. draft) -> se congela ahora. + + # Archivo nuevo o draft existente: congelar con timestamp actual. + frozen_at = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + file_text = _render_file(slug, frozen_at, new_hash, body) + try: + with open(expected_path, "w", encoding="utf-8") as fh: + fh.write(file_text) + except OSError as exc: + return { + "status": "error", + "path": expected_path, + "note": f"no se pudo escribir el pre-registro: {exc}", + } + return { + "status": "frozen", + "path": expected_path, + "content_hash": new_hash, + "frozen_at": frozen_at, + } + except Exception as exc: # noqa: BLE001 - dict-no-throw: nunca propagar. + return { + "status": "error", + "path": expected_path, + "note": f"error inesperado: {exc}", + } diff --git a/python/functions/datascience/preregister_hypothesis_test.py b/python/functions/datascience/preregister_hypothesis_test.py new file mode 100644 index 00000000..93c71e3e --- /dev/null +++ b/python/functions/datascience/preregister_hypothesis_test.py @@ -0,0 +1,99 @@ +"""Tests para preregister_hypothesis (pre-registro inmutable, anti-HARKing). + +Importa el modulo hoja directamente (`preregister_hypothesis`) para no depender +de que el paquete reexporte la funcion en su __init__ (lo integra el orquestador +al cerrar el grupo papers). El pytest del repo resuelve el modulo hoja por su +nombre directo. + +Todos los tests son hermeticos y deterministas: usan el fixture `tmp_path` de +pytest; NUNCA escriben en `papers/`. +""" + +from preregister_hypothesis import preregister_hypothesis + + +def _parse_frontmatter(text: str) -> dict: + parts = text.split("---", 2) + fm = {} + for line in parts[1].splitlines(): + line = line.strip() + if not line or ":" not in line: + continue + key, _, value = line.partition(":") + fm[key.strip()] = value.strip() + return fm + + +HYP = {"h0": "no hay diferencia entre A y B", "h1": "el grupo A > grupo B"} +PLAN = { + "test": "welch_t_test", + "effect_size_metric": "cohens_d", + "decision_rule": "rechazar H0 si p<0.05 tras Holm y |d|>=0.5", + "planned_n": 100, + "multiple_correction": "holm", +} + + +def test_golden_congela_y_escribe_archivo(tmp_path): + paper = tmp_path / "0001-x" + paper.mkdir() + + res = preregister_hypothesis(str(paper), HYP, PLAN) + + assert res["status"] == "frozen" + pre = paper / "preregistration.md" + assert pre.exists() + + text = pre.read_text(encoding="utf-8") + fm = _parse_frontmatter(text) + assert fm["status"] == "frozen" + assert fm["paper_slug"] == "0001-x" + assert fm["content_hash"] # no vacio + assert fm["frozen_at"] # no vacio + assert res["content_hash"] == fm["content_hash"] + assert res["frozen_at"] == fm["frozen_at"] + + +def test_idempotente_mismo_input_no_reescribe(tmp_path): + paper = tmp_path / "0001-x" + paper.mkdir() + pre = paper / "preregistration.md" + + first = preregister_hypothesis(str(paper), HYP, PLAN) + assert first["status"] == "frozen" + bytes_before = pre.read_bytes() + + second = preregister_hypothesis(str(paper), HYP, PLAN) + assert second["status"] == "unchanged" + # Mismo hash y frozen_at original preservado. + assert second["content_hash"] == first["content_hash"] + assert second["frozen_at"] == first["frozen_at"] + # El archivo NO cambio byte a byte (incl. frozen_at). + assert pre.read_bytes() == bytes_before + + +def test_inmutabilidad_anti_harking_rechaza_contenido_distinto(tmp_path): + paper = tmp_path / "0001-x" + paper.mkdir() + pre = paper / "preregistration.md" + + preregister_hypothesis(str(paper), HYP, PLAN) + bytes_frozen = pre.read_bytes() + + # Intento de re-congelar con una hipotesis DISTINTA (HARKing) -> rechazado. + hyp_tramposo = {"h0": "no hay diferencia", "h1": "el grupo B > grupo A (cambiado tras ver datos)"} + res = preregister_hypothesis(str(paper), hyp_tramposo, PLAN) + + assert res["status"] == "error" + # Asercion mas importante: el archivo en disco SIGUE siendo el original. + assert pre.read_bytes() == bytes_frozen + + +def test_error_paper_dir_inexistente_no_crash_no_crea(tmp_path): + missing = tmp_path / "no-existe" + res = preregister_hypothesis(str(missing), HYP, PLAN) + + assert res["status"] == "error" + # No se creo el directorio ni el archivo. + assert not missing.exists() + assert not (missing / "preregistration.md").exists() From 9886e2905d5ea6e6172c42f6778b4c9a7ef1cbc1 Mon Sep 17 00:00:00 2001 From: Egutierrez <egutierrez@dead.dd> Date: Tue, 30 Jun 2026 20:57:52 +0200 Subject: [PATCH 43/53] feat(eda): rasterizar join graph a Figure matplotlib real en el capitulo de relaciones draw_join_graph_figure (datascience, grupo eda): dibuja el join graph de la base como una matplotlib Figure real (networkx spring_layout seed=42, nodos = tablas, hubs destacados, flechas dirigidas con etiqueta from_col->to_col + cardinalidad). Nunca lanza: devuelve una Figure de error si algo falla; entrada vacia -> Figure 'Sin relaciones FK detectadas'. render_automatic_eda_folder ahora inserta esa Figure (bloque Figure lazy via make) en el capitulo de relaciones cuando hay edges, ademas del texto Mermaid (util para el MD/LLM). Antes solo se volcaba el texto del grafo; ahora el PDF/PPTX muestran el diagrama dibujado. Tests nuevos: la Figure real se construye con edges y se omite sin edges. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> --- python/functions/datascience/__init__.py | 2 + .../datascience/draw_join_graph_figure.md | 103 +++++++++ .../datascience/draw_join_graph_figure.py | 214 ++++++++++++++++++ .../draw_join_graph_figure_test.py | 84 +++++++ .../pipelines/render_automatic_eda_folder.md | 11 +- .../pipelines/render_automatic_eda_folder.py | 28 ++- .../render_automatic_eda_folder_test.py | 44 +++- 7 files changed, 475 insertions(+), 11 deletions(-) create mode 100644 python/functions/datascience/draw_join_graph_figure.md create mode 100644 python/functions/datascience/draw_join_graph_figure.py create mode 100644 python/functions/datascience/draw_join_graph_figure_test.py diff --git a/python/functions/datascience/__init__.py b/python/functions/datascience/__init__.py index cdefab14..4f04f6fd 100644 --- a/python/functions/datascience/__init__.py +++ b/python/functions/datascience/__init__.py @@ -72,8 +72,10 @@ from .profile_datetime import profile_datetime from .resample_timeseries import resample_timeseries from .add_pdf_internal_links import add_pdf_internal_links from .suggest_intratable_fk_candidates import suggest_intratable_fk_candidates +from .draw_join_graph_figure import draw_join_graph_figure __all__ = [ + "draw_join_graph_figure", "suggest_intratable_fk_candidates", "detect_time_column", "extract_timeseries_raw", diff --git a/python/functions/datascience/draw_join_graph_figure.md b/python/functions/datascience/draw_join_graph_figure.md new file mode 100644 index 00000000..e105061d --- /dev/null +++ b/python/functions/datascience/draw_join_graph_figure.md @@ -0,0 +1,103 @@ +--- +id: draw_join_graph_figure_py_datascience +name: draw_join_graph_figure +kind: function +lang: py +domain: datascience +version: "1.0.0" +purity: impure +signature: "def draw_join_graph_figure(join_graph: dict, title: str = None) -> \"matplotlib.figure.Figure\"" +description: "Rasteriza el join graph de una base (relaciones FK inter-tabla, salida de build_join_graph) a un matplotlib.figure.Figure: nodos circulares con el nombre de cada tabla (hubs en color de acento cálido, el resto neutro) y aristas dirigidas etiquetadas from_col→to_col (más la cardinalidad si viene). Es la contrapartida dibujada del string Mermaid para que el capítulo de relaciones del informe AutomaticEDA muestre un diagrama real. Layout networkx spring_layout determinista (seed=42), backend Agg sin abrir ventanas; defensivo: nunca lanza y nunca hace I/O." +tags: [eda, plot, relations, graph, matplotlib, figure, networkx, datascience, impure] +uses_functions: [] +uses_types: [] +returns: [] +returns_optional: false +error_type: "error_go_core" +imports: [matplotlib, networkx] +example: | + from draw_join_graph_figure import draw_join_graph_figure + join_graph = { + "nodes": [ + {"table": "customers", "out_degree": 0, "in_degree": 1, "role": "dimension"}, + {"table": "orders", "out_degree": 1, "in_degree": 0, "role": "fact"}, + ], + "edges": [ + {"from_table": "orders", "from_col": "customer_id", + "to_table": "customers", "to_col": "id", "cardinality": "N:1"}, + ], + "hubs": ["orders"], + } + fig = draw_join_graph_figure(join_graph, title="Relaciones FK") + fig.savefig("/tmp/join_graph.png") +tested: true +tests: + - "test_returns_figure_with_axis" + - "test_savefig_produces_nonempty_png" + - "test_empty_dict_does_not_raise_and_savefig_png" + - "test_none_does_not_raise_and_savefig_png" +test_file_path: "python/functions/datascience/draw_join_graph_figure_test.py" +file_path: "python/functions/datascience/draw_join_graph_figure.py" +params: + - name: join_graph + desc: "Dict producido por build_join_graph. Claves: `nodes` (list[dict] con table, out_degree, in_degree, role), `edges` (list[dict] con from_table, from_col, to_table, to_col y opcional cardinality/inclusion) y `hubs` (list[str] de tablas hub a destacar en color cálido). Claves ausentes, items no-dict, None o {} se toleran (devuelve Figure con texto, sin lanzar). Los nombres de nodo se derivan también de las aristas, así que un grafo con edges pero sin nodes explícitos igual se dibuja." + - name: title + desc: "Título dibujado sobre el diagrama. Si se omite (None) se usa \"Join graph\". Default None." +output: "Un matplotlib.figure.Figure (figsize 7x5) con un único Axes que contiene el diagrama node-link dirigido: tablas como nodos circulares etiquetados (hubs en acento cálido #DD8452, resto en azul neutro #4C72B0) y FKs como flechas dirigidas con etiqueta from_col→to_col (+ cardinalidad). Si join_graph no tiene nodos ni aristas (o es None/{}), devuelve igualmente una Figure con el texto centrado \"Sin relaciones FK detectadas.\"; ante cualquier fallo interno devuelve una Figure con un mensaje genérico (nunca lanza). El caller rasteriza/cierra la figura; la función no la muestra ni la guarda." +--- + +## Ejemplo + +```python +from draw_join_graph_figure import draw_join_graph_figure + +# `join_graph` es la salida de build_join_graph (nodes + edges + hubs). +join_graph = { + "nodes": [ + {"table": "customers", "out_degree": 0, "in_degree": 1, "role": "dimension"}, + {"table": "orders", "out_degree": 2, "in_degree": 0, "role": "fact"}, + {"table": "products", "out_degree": 0, "in_degree": 1, "role": "dimension"}, + ], + "edges": [ + {"from_table": "orders", "from_col": "customer_id", + "to_table": "customers", "to_col": "id", "cardinality": "N:1"}, + {"from_table": "orders", "from_col": "product_id", + "to_table": "products", "to_col": "id", "cardinality": "N:1"}, + ], + "hubs": ["orders"], # `orders` se pinta en color de acento (tabla de hechos) +} + +fig = draw_join_graph_figure(join_graph, title="Relaciones FK") + +# El renderer del informe lo rasteriza; aquí solo persistimos para inspección. +fig.savefig("/tmp/join_graph.png") +``` + +## Cuando usarla + +Úsala en el capítulo de relaciones de un informe AutomaticEDA cuando quieras un +diagrama **dibujado** del esquema relacional, no solo el bloque Mermaid pegable. +Pásale directamente la salida de `build_join_graph` (`nodes` + `edges` + `hubs`) +y obtienes una `matplotlib.figure.Figure` lista para que el renderer perezoso la +rasterice. Es la pareja visual del string Mermaid: Mermaid sirve para pegar en +Markdown/docs que lo soporten; esta función produce la imagen real (PNG/PDF) que +va embebida en informes que no renderizan Mermaid. + +## Gotchas + +- **Impura por matplotlib.** Fija el backend `Agg` al importar — no abre + ventanas ni depende de un display. Segura de llamar en lotes desde el + renderer. +- **Layout determinista (`seed=42`).** Usa `nx.spring_layout(G, seed=42)`, así + que la misma entrada produce el mismo diagrama (test reproducible). Para + grafos de 0/1 nodos usa una posición fija centrada en vez del spring layout. +- **No hace I/O.** No llama `plt.show()` ni guarda a disco — solo devuelve la + `Figure`. Quien la consume la rasteriza y la libera (`plt.close(fig)`) para no + acumular memoria en informes con muchas tablas. +- **Devuelve una Figure, NO un dict.** A diferencia de `build_join_graph` (que + devuelve el dict del grafo), esta función devuelve el objeto de figura ya + dibujado. +- **Defensiva, nunca lanza.** `None`, `{}`, claves ausentes o items malformados + se manejan sin error: en el peor caso devuelve una `Figure` con + "Sin relaciones FK detectadas." (vacío) o un mensaje genérico (fallo interno). + No la envuelvas en try/except por miedo a un raise — no lo hay. diff --git a/python/functions/datascience/draw_join_graph_figure.py b/python/functions/datascience/draw_join_graph_figure.py new file mode 100644 index 00000000..aee17622 --- /dev/null +++ b/python/functions/datascience/draw_join_graph_figure.py @@ -0,0 +1,214 @@ +"""Impure EDA helper: rasterize a join graph to a matplotlib Figure (`eda` group). + +Takes the join graph produced by ``build_join_graph`` (inter-table FK relations) +and draws it as a directed node-link diagram on a ready-to-rasterize +``matplotlib.figure.Figure``. Hub tables (the ones with the highest out-degree, +candidate fact tables of a star schema) are highlighted in a warm accent colour; +the rest use a neutral colour. Directed edges carry a ``from_col→to_col`` label +(plus the cardinality when present). + +This is the *drawn* counterpart of the Mermaid string that ``build_join_graph`` +also emits: the relations chapter of an AutomaticEDA report can show a real +picture instead of only the pasteable Mermaid block. + +Impure because it touches matplotlib's rendering machinery. It pins the headless +Agg backend and a deterministic ``spring_layout`` seed so the output is +reproducible. It never raises: on any internal failure (or empty input) it +returns a ``Figure`` carrying a centered message, so the lazy render of the +document is never broken. +""" + +import matplotlib + +matplotlib.use("Agg") + +import matplotlib.pyplot as plt # noqa: E402 +import networkx as nx # noqa: E402 + +# Warm accent reserved for hub tables (candidate fact tables / star-schema cores). +_HUB_COLOR = "#DD8452" +# Neutral blue for every other table. +_NODE_COLOR = "#4C72B0" +# Muted gray for the empty/error message text. +_MUTED_TEXT = "#5f6b7a" +# Edge colour and label colour. +_EDGE_COLOR = "#7a7a7a" +_EDGE_LABEL_COLOR = "#34495e" +# Constant node size; shared with the edge drawing so arrowheads stop at the +# node boundary instead of being hidden under the marker. +_NODE_SIZE = 2200 + + +def _text_figure(message: str) -> "matplotlib.figure.Figure": + """Return a blank Figure carrying a single centered message. + + Used both for the "no relations" case and as the never-raise fallback. + """ + fig, ax = plt.subplots(figsize=(7, 5)) + ax.axis("off") + ax.text( + 0.5, + 0.5, + message, + ha="center", + va="center", + fontsize=12, + color=_MUTED_TEXT, + transform=ax.transAxes, + ) + fig.tight_layout() + return fig + + +def _edge_label(edge: dict) -> str: + """Build the ``from_col→to_col`` label of an edge, appending cardinality.""" + fc = edge.get("from_col") + tc = edge.get("to_col") + if fc is not None and tc is not None: + label = f"{fc}→{tc}" + elif fc is not None: + label = str(fc) + elif tc is not None: + label = str(tc) + else: + label = "" + card = edge.get("cardinality") + if card: + label = f"{label} ({card})" if label else str(card) + return label + + +def draw_join_graph_figure(join_graph: dict, title: str = None): + """Rasterize a join graph to a matplotlib Figure. + + Builds a ``networkx.DiGraph`` from the graph's nodes and edges, lays it out + with a deterministic ``spring_layout`` (``seed=42``) and draws it on a + ``matplotlib.figure.Figure``: tables as labelled circular nodes (hubs in a + warm accent, the rest neutral) and FK relations as directed arrows labelled + ``from_col→to_col`` (plus cardinality when available). + + The function never raises. On empty/``None`` input it returns a Figure with + a centered "Sin relaciones FK detectadas." message; on any internal failure + it returns a Figure with a generic centered message. It never shows the + figure nor writes it to disk — the document renderer rasterizes it. + + Args: + join_graph: Dict produced by ``build_join_graph`` with keys ``nodes`` + (list of ``{table, out_degree, in_degree, role}``), ``edges`` (list + of ``{from_table, from_col, to_table, to_col, cardinality?, + inclusion?}``) and ``hubs`` (list of hub table names to highlight). + Missing keys, non-dict items, ``None`` or ``{}`` are all tolerated. + title: Optional title drawn above the diagram. When omitted, the title + defaults to "Join graph". + + Returns: + A ``matplotlib.figure.Figure`` (figsize 7x5) with a single Axes holding + the node-link diagram. The caller rasterizes/closes it. + """ + try: + jg = join_graph if isinstance(join_graph, dict) else {} + nodes = jg.get("nodes") or [] + edges = jg.get("edges") or [] + hubs = {h for h in (jg.get("hubs") or []) if h is not None} + + # Collect node names from the declared nodes and, defensively, from the + # edges (so a graph with edges but no explicit nodes still draws). + node_names: list = [] + seen: set = set() + + def _register(name) -> None: + if name is not None and name not in seen: + seen.add(name) + node_names.append(name) + + for n in nodes: + if isinstance(n, dict): + _register(n.get("table")) + for e in edges: + if isinstance(e, dict): + _register(e.get("from_table")) + _register(e.get("to_table")) + + if not node_names: + return _text_figure("Sin relaciones FK detectadas.") + + graph = nx.DiGraph() + for name in node_names: + graph.add_node(name) + + edge_labels: dict = {} + for e in edges: + if not isinstance(e, dict): + continue + ft = e.get("from_table") + tt = e.get("to_table") + if ft is None or tt is None: + continue + graph.add_edge(ft, tt) + edge_labels[(ft, tt)] = _edge_label(e) + + fig, ax = plt.subplots(figsize=(7, 5)) + + # Deterministic layout. Fixed positions for trivial graphs so a single + # node sits centered instead of at an arbitrary spring-layout point. + if graph.number_of_nodes() <= 1: + pos = {name: (0.5, 0.5) for name in graph.nodes()} + else: + pos = nx.spring_layout(graph, seed=42) + + node_colors = [ + _HUB_COLOR if name in hubs else _NODE_COLOR for name in graph.nodes() + ] + nx.draw_networkx_nodes( + graph, + pos, + ax=ax, + node_color=node_colors, + node_size=_NODE_SIZE, + node_shape="o", + edgecolors="white", + linewidths=1.5, + ) + nx.draw_networkx_labels( + graph, + pos, + ax=ax, + font_size=9, + font_color="white", + font_weight="bold", + ) + nx.draw_networkx_edges( + graph, + pos, + ax=ax, + arrows=True, + arrowstyle="-|>", + arrowsize=18, + edge_color=_EDGE_COLOR, + width=1.4, + connectionstyle="arc3,rad=0.06", + node_size=_NODE_SIZE, + ) + if any(lbl for lbl in edge_labels.values()): + nx.draw_networkx_edge_labels( + graph, + pos, + edge_labels=edge_labels, + ax=ax, + font_size=7, + font_color=_EDGE_LABEL_COLOR, + bbox={ + "boxstyle": "round,pad=0.2", + "fc": "white", + "ec": "none", + "alpha": 0.7, + }, + ) + + ax.set_title(title if title else "Join graph", fontsize=13) + ax.axis("off") + fig.tight_layout() + return fig + except Exception: + # Never raise — the document render is lazy and must not be broken. + return _text_figure("No se pudo dibujar el join graph.") diff --git a/python/functions/datascience/draw_join_graph_figure_test.py b/python/functions/datascience/draw_join_graph_figure_test.py new file mode 100644 index 00000000..25d8f360 --- /dev/null +++ b/python/functions/datascience/draw_join_graph_figure_test.py @@ -0,0 +1,84 @@ +"""Tests para draw_join_graph_figure (rasteriza el join graph, grupo eda). + +Usa el backend Agg sin abrir ventanas; cada test cierra la Figure construida +(matplotlib.pyplot.close) para no acumular estado entre tests. Las aserciones de +guardado escriben a tmp_path (fixture de pytest) y comprueban que el PNG no está +vacío. +""" + +import matplotlib + +matplotlib.use("Agg") + +import matplotlib.pyplot as plt # noqa: E402 +from matplotlib.figure import Figure # noqa: E402 + +from draw_join_graph_figure import draw_join_graph_figure + + +def _make_join_graph(): + """Join graph mínimo: 3 nodos (customers/orders/products) y 2 aristas. + + orders -> customers y orders -> products. `orders` es el hub (out_degree 2). + """ + return { + "nodes": [ + {"table": "customers", "out_degree": 0, "in_degree": 1, "role": "dimension"}, + {"table": "orders", "out_degree": 2, "in_degree": 0, "role": "fact"}, + {"table": "products", "out_degree": 0, "in_degree": 1, "role": "dimension"}, + ], + "edges": [ + { + "from_table": "orders", + "from_col": "customer_id", + "to_table": "customers", + "to_col": "id", + "cardinality": "N:1", + "inclusion": 1.0, + }, + { + "from_table": "orders", + "from_col": "product_id", + "to_table": "products", + "to_col": "id", + "cardinality": "N:1", + "inclusion": 0.98, + }, + ], + "hubs": ["orders"], + } + + +def test_returns_figure_with_axis(): + fig = draw_join_graph_figure(_make_join_graph(), title="Relaciones FK") + assert isinstance(fig, Figure) + # Al menos un eje con el diagrama. + assert len(fig.axes) >= 1 + plt.close(fig) + + +def test_savefig_produces_nonempty_png(tmp_path): + fig = draw_join_graph_figure(_make_join_graph()) + out = tmp_path / "g.png" + fig.savefig(out) + assert out.exists() + assert out.stat().st_size > 0 + plt.close(fig) + + +def test_empty_dict_does_not_raise_and_savefig_png(tmp_path): + fig = draw_join_graph_figure({}) + assert isinstance(fig, Figure) + out = tmp_path / "empty.png" + fig.savefig(out) + assert out.stat().st_size > 0 + plt.close(fig) + + +def test_none_does_not_raise_and_savefig_png(tmp_path): + fig = draw_join_graph_figure(None) + assert isinstance(fig, Figure) + out = tmp_path / "none.png" + fig.savefig(out) + assert out.stat().st_size > 0 + plt.close(fig) diff --git a/python/functions/pipelines/render_automatic_eda_folder.md b/python/functions/pipelines/render_automatic_eda_folder.md index 895f15a2..a79f2a01 100644 --- a/python/functions/pipelines/render_automatic_eda_folder.md +++ b/python/functions/pipelines/render_automatic_eda_folder.md @@ -6,7 +6,7 @@ domain: pipelines purity: impure version: "1.0.0" signature: "def render_automatic_eda_folder(path: str, out_dir: str = \"reports\", basename: str = None, profile_level: str = \"standard\", emit_pdf: bool = True, emit_pptx: bool = True, emit_md: bool = True, per_table_eda: bool = False, min_inclusion: float = 0.9, ctx_extra: dict = None) -> dict" -description: "Informe AutomaticEDA a nivel de BASE one-shot de una CARPETA de archivos tabulares (CSV/Parquet/JSON) o de una DuckDB existente. Carga la carpeta a una DuckDB temporal con load_folder_to_duckdb (o usa la DuckDB dada directa), perfila TODA la base con profile_database (resumen de cada tabla + FK candidatas por containment + join graph con diagrama Mermaid), ENSAMBLA un documento-base por capitulos (portada-base con nombre/n tablas/totales/fecha/fuente, resumen de tablas con una fila por tabla, y relaciones inter-tabla con la tabla de FK candidatas + el diagrama Mermaid) y lo renderiza con el motor AutomaticEDA a PDF (A5 movil), PPTX (16:9) y Markdown autocontenido a la vez. Con per_table_eda=True anexa los capitulos de mini-EDA de cada tabla (build_document por tabla). Es el hermano a nivel de base de render_automatic_eda (que perfila UNA tabla): aqui el informe es de la base y de sus relaciones. Devuelve las rutas de PDF/PPTX/MD, el manifiesto y el DatabaseProfile." +description: "Informe AutomaticEDA a nivel de BASE one-shot de una CARPETA de archivos tabulares (CSV/Parquet/JSON) o de una DuckDB existente. Carga la carpeta a una DuckDB temporal con load_folder_to_duckdb (o usa la DuckDB dada directa), perfila TODA la base con profile_database (resumen de cada tabla + FK candidatas por containment + join graph con diagrama Mermaid), ENSAMBLA un documento-base por capitulos (portada-base con nombre/n tablas/totales/fecha/fuente, resumen de tablas con una fila por tabla, y relaciones inter-tabla con la tabla de FK candidatas + una Figure matplotlib REAL del join graph dibujada con draw_join_graph_figure mas el texto Mermaid) y lo renderiza con el motor AutomaticEDA a PDF (A5 movil), PPTX (16:9) y Markdown autocontenido a la vez. Con per_table_eda=True anexa los capitulos de mini-EDA de cada tabla (build_document por tabla). Es el hermano a nivel de base de render_automatic_eda (que perfila UNA tabla): aqui el informe es de la base y de sus relaciones. Devuelve las rutas de PDF/PPTX/MD, el manifiesto y el DatabaseProfile." tags: [eda, duckdb, database, profiling, relations, pipeline, dataops, report, pdf, pptx, launcher] uses_functions: - load_folder_to_duckdb_py_infra @@ -14,6 +14,7 @@ uses_functions: - render_automatic_eda_pdf_py_datascience - render_automatic_eda_pptx_py_datascience - render_automatic_eda_markdown_py_datascience + - draw_join_graph_figure_py_datascience uses_types: [] returns: [] returns_optional: false @@ -105,8 +106,10 @@ anexado. path inexistente → `{status:'error'}` (no lanza). - El escaneo de la carpeta es **no recursivo** (solo el primer nivel) y por defecto cubre `*.csv,*.parquet,*.json` (ver `load_folder_to_duckdb`). -- El diagrama Mermaid se vuelca como **bloque de código**: en el Markdown queda - como diagrama renderizable; en PDF/PPTX se muestra el **texto** del grafo (no - se rasteriza el diagrama a imagen en esta versión). +- El join graph se rasteriza a una **Figure matplotlib real** (vía + `draw_join_graph_figure`) que aparece dibujada en PDF/PPTX (nodos = tablas, + flechas = FK). Además, el **texto Mermaid** del grafo se incluye como bloque de + código (en el Markdown queda como diagrama renderizable y es útil para pegar a + un LLM). - Carpeta vacía o con 1 sola tabla: funciona igual; el capítulo de relaciones dice "sin FK". dict-no-throw en todos los caminos. diff --git a/python/functions/pipelines/render_automatic_eda_folder.py b/python/functions/pipelines/render_automatic_eda_folder.py index 28793cca..d5b5bbd3 100644 --- a/python/functions/pipelines/render_automatic_eda_folder.py +++ b/python/functions/pipelines/render_automatic_eda_folder.py @@ -38,6 +38,7 @@ import os from datetime import datetime, timezone from datascience import ( + draw_join_graph_figure, render_automatic_eda_markdown, render_automatic_eda_pdf, render_automatic_eda_pptx, @@ -157,14 +158,29 @@ def _relaciones_chapter(db_profile: dict) -> dict: "text": "Sin relaciones FK candidatas detectadas entre las tablas.", }) - mermaid = (db_profile.get("join_graph") or {}).get("mermaid", "") or "" - if mermaid.strip(): + join_graph = db_profile.get("join_graph") or {} + has_edges = bool(join_graph.get("edges")) + if has_edges: blocks.append({"kind": "heading", "text": "Diagrama (join graph)", "level": 3}) - # El Mermaid se vuelca como bloque de código: en MD queda como diagrama - # renderizable; en PDF/PPTX se muestra el texto del grafo (legible). - blocks.append({"kind": "markdown", - "text": "```mermaid\n" + mermaid.strip() + "\n```"}) + # Figure matplotlib REAL del grafo de relaciones (nodos = tablas, + # aristas = FK). Lazy via `make`: el renderer la construye solo al + # paginar, y se rasteriza en PDF/PPTX. draw_join_graph_figure nunca + # lanza (devuelve una Figure de error si algo falla). + blocks.append({ + "kind": "figure", + "make": (lambda jg=join_graph: draw_join_graph_figure( + jg, title="Join graph (relaciones inter-tabla)")), + "caption": "Grafo de relaciones: nodos = tablas, flechas = FK " + "candidatas (etiqueta from_col→to_col).", + "height_in": 4.5, + }) + # Además, el Mermaid en texto: en el Markdown queda como diagrama + # renderizable y es útil para pegar a un LLM. + mermaid = (join_graph.get("mermaid", "") or "").strip() + if mermaid: + blocks.append({"kind": "markdown", + "text": "```mermaid\n" + mermaid + "\n```"}) return {"id": "relaciones", "title": "Relaciones inter-tabla", "version": "1.0.0", "blocks": blocks} diff --git a/python/functions/pipelines/render_automatic_eda_folder_test.py b/python/functions/pipelines/render_automatic_eda_folder_test.py index eb529bdc..85b42cd6 100644 --- a/python/functions/pipelines/render_automatic_eda_folder_test.py +++ b/python/functions/pipelines/render_automatic_eda_folder_test.py @@ -14,7 +14,10 @@ import duckdb sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..")) -from pipelines.render_automatic_eda_folder import render_automatic_eda_folder +from pipelines.render_automatic_eda_folder import ( + _relaciones_chapter, + render_automatic_eda_folder, +) def _write_demo_folder(folder: str) -> None: @@ -144,3 +147,42 @@ def test_path_does_not_exist(tmp_path): r = render_automatic_eda_folder(str(tmp_path / "nope")) assert r["status"] == "error" assert "no existe" in r["error"].lower() + + +def test_relaciones_chapter_has_real_figure_when_edges(): + """Con edges, el capítulo de relaciones incluye un bloque Figure matplotlib + REAL (no solo el texto Mermaid): su make() devuelve una Figure.""" + db_profile = { + "join_graph": { + "nodes": [ + {"table": "orders", "out_degree": 1, "in_degree": 0, "role": "fact"}, + {"table": "customers", "out_degree": 0, "in_degree": 1, "role": "dim"}, + ], + "edges": [{"from_table": "orders", "from_col": "customer_id", + "to_table": "customers", "to_col": "id", + "cardinality": "N:1"}], + "mermaid": "graph LR orders --> customers", + "hubs": ["orders"], + }, + "fk_candidates": [{"from_table": "orders", "from_col": "customer_id", + "to_table": "customers", "to_col": "id", + "inclusion": 1.0, "cardinality": "N:1"}], + } + ch = _relaciones_chapter(db_profile) + figs = [b for b in ch["blocks"] if b.get("kind") == "figure"] + assert len(figs) == 1, ch["blocks"] + # El make() perezoso produce una matplotlib Figure real. + import matplotlib + matplotlib.use("Agg") + fig = figs[0]["make"]() + from matplotlib.figure import Figure + assert isinstance(fig, Figure) + assert fig.get_axes(), "la Figure del join graph debe tener al menos un eje" + + +def test_relaciones_chapter_no_figure_when_no_edges(): + """Sin edges, no se añade bloque Figure (capítulo dice 'sin FK').""" + db_profile = {"join_graph": {"nodes": [], "edges": [], "mermaid": "", + "hubs": []}, "fk_candidates": []} + ch = _relaciones_chapter(db_profile) + assert not [b for b in ch["blocks"] if b.get("kind") == "figure"] From 6f88f184f11513278eb0f20ae2d26e32021b86be Mon Sep 17 00:00:00 2001 From: Egutierrez <egutierrez@dead.dd> Date: Tue, 30 Jun 2026 21:12:40 +0200 Subject: [PATCH 44/53] =?UTF-8?q?feat(eda):=20cap=C3=ADtulo=20OUTLIERS=20?= =?UTF-8?q?=E2=80=94=20valores=20at=C3=ADpicos=20univariantes=20+=20multiv?= =?UTF-8?q?ariantes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Nuevo capítulo dedicado `outliers` para el motor AutomaticEDA que reúne y profundiza en un solo sitio el análisis de valores atípicos, hoy disperso entre `num_distr` (conteo por columna) y `modelos` (IsolationForest). Se registra en `chapters_registry.py` entre `missingness` y `correlacion` (bloque de calidad de datos: calidad → missingness → outliers). Contenido del capítulo: - Resumen univariante por columna: nº y % de atípicos por Tukey (1.5·IQR) y por z-score (|z| > 3), con vallas inferior/superior y valores extremos. Ordenado por contaminación y marcando las columnas más afectadas. Reusa las funciones del registry `build_boxplot_stats` (vallas desde los percentiles del profile) y `detect_outliers` (regla z-score sobre la muestra cruda de `ctx`). - Boxplots de Tukey de las columnas más contaminadas (caja, bigotes y puntos atípicos), delegados a la función nueva `build_boxplots_figure`. - Multivariante: filas anómalas considerando todas las columnas a la vez con `isolation_forest_outliers` — nº y % de filas, las más anómalas con su score y las dimensiones que las hacen raras (top columnas por |z|, vía la función nueva `summarize_outlier_dims`). El detector se corre en vivo sobre `raw_numeric` para que el indexado de filas coincida exactamente con el de las dimensiones; cae al bloque precomputado del perfil cuando no hay muestra cruda (preset lite). - Interpretación exploratoria: un atípico no es necesariamente un error (distingue error de dato vs dato real extremo) y recomendaciones (revisar, winsorizar o re-expresar, enlazando con la re-expresión de Tukey del perfil). Términos clicables registrados en el glosario compartido: `outlier`, `tukey_fence`, `zscore`, `isolation_forest`. Funciones nuevas del registry (dominio datascience, grupo eda): - `build_boxplots_figure_py_datascience` (figure helper, impura) - `summarize_outlier_dims_py_datascience` (pura) El capítulo se activa con ≥1 columna numérica y devuelve None en su ausencia; lee todo defensivo y nunca lanza. Tests: capítulo (golden + edges + error path + render PDF/PPTX) y ambas funciones nuevas. Suite de no-regresión de AutomaticEDA verde. Verificado end-to-end con el dataset Titanic (Fare/Parch/SibSp como las columnas más contaminadas). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> --- .../automatic_eda/chapters/outliers.py | 593 ++++++++++++++++++ .../automatic_eda/chapters/outliers_test.py | 304 +++++++++ .../automatic_eda/chapters_registry.py | 1 + .../datascience/build_boxplots_figure.md | 125 ++++ .../datascience/build_boxplots_figure.py | 250 ++++++++ .../datascience/build_boxplots_figure_test.py | 109 ++++ .../datascience/summarize_outlier_dims.md | 79 +++ .../datascience/summarize_outlier_dims.py | 144 +++++ .../summarize_outlier_dims_test.py | 93 +++ 9 files changed, 1698 insertions(+) create mode 100644 python/functions/datascience/automatic_eda/chapters/outliers.py create mode 100644 python/functions/datascience/automatic_eda/chapters/outliers_test.py create mode 100644 python/functions/datascience/build_boxplots_figure.md create mode 100644 python/functions/datascience/build_boxplots_figure.py create mode 100644 python/functions/datascience/build_boxplots_figure_test.py create mode 100644 python/functions/datascience/summarize_outlier_dims.md create mode 100644 python/functions/datascience/summarize_outlier_dims.py create mode 100644 python/functions/datascience/summarize_outlier_dims_test.py diff --git a/python/functions/datascience/automatic_eda/chapters/outliers.py b/python/functions/datascience/automatic_eda/chapters/outliers.py new file mode 100644 index 00000000..0522a2ca --- /dev/null +++ b/python/functions/datascience/automatic_eda/chapters/outliers.py @@ -0,0 +1,593 @@ +"""Outliers chapter (OUTLIERS) — univariate + multivariate atypical values. + +Today the analysis of atypical values is scattered across the document: the +NUM DISTR chapter mentions the per-column outlier count inside each distribution +figure, and the MODELOS chapter runs Isolation Forest as one of several cheap +models. This chapter gathers and deepens the whole outlier story in a single +place, with its interpretation: an [[term:outlier]]outlier[[/term]] is **not +necessarily an error** — it can be a legitimate, extreme but real observation — +so the reading is exploratory (what to look at), never confirmatory (what to +delete). + +Sections, in order: + +1. **Resumen univariante por columna** — for every numeric column, the number + and percentage of atypical values by two complementary criteria: Tukey's + 1.5·IQR rule ([[term:tukey_fence]]vallas de Tukey[[/term]]) and the + [[term:zscore]]z-score[[/term]] rule (|z| > 3). The most contaminated columns + are flagged. The fences come from the pure registry function + ``build_boxplot_stats`` (derived from the profile percentiles); the per-column + counts use the raw sample in ``ctx['raw_numeric']`` when available (the exact + count), degrading to the profile's own z-score counts otherwise. +2. **Boxplots** — a single figure with the Tukey boxplots of the most + contaminated columns (box, whiskers and atypical points), delegated to the + reusable registry helper ``build_boxplots_figure``. +3. **Multivariante (filas anómalas)** — rows that are atypical considering ALL + columns at once, via the registry function ``isolation_forest_outliers``: the + count and percentage of anomalous rows, the most anomalous rows with their + score, and the dimensions that make each one rare (top columns by |z|, via + ``summarize_outlier_dims``). Run live on ``ctx['raw_numeric']`` (the same + numeric columns ``summarize_outlier_dims`` uses, so the row indexing stays + coherent and the dimension breakdown is correct); falls back to the + precomputed ``profile['models']['outliers']`` only when no raw sample is + available (e.g. the lite preset), where no per-row breakdown is shown. +4. **Interpretación** — outlier ≠ error: how to tell a data-entry error from a + genuine extreme value, and what to do (inspect, winsorize, or re-express — + linking to the Tukey re-expression the profile already computes). + +The chapter activates whenever the table has at least one numeric column; with +no numeric column it returns ``None`` and disappears from the document. + +Reads everything defensively (``.get``) and never raises: every registry +delegation is imported lazily and degraded to an honest note on any failure. + +Contract: build_<id>(profile, ctx) -> Chapter | None ; CHAPTER_VERSION = "x.y.z". +""" + +from __future__ import annotations + +from .. import model + +CHAPTER_VERSION = "1.0.0" +CHAPTER_ID = "outliers" +CHAPTER_TITLE = "Valores atípicos" + +# z-score threshold for the univariate z rule: |z| > 3 flags a value ~3 standard +# deviations from the mean (≈99.7% of a normal distribution lies within ±3σ). +_Z_THRESH = 3.0 +# How many columns to draw in the boxplots figure (most contaminated first) and +# how many anomalous rows to list in the multivariate table. +_TOP_BOX = 12 +_TOP_ROWS = 12 +# Cap on the raw atypical values passed as boxplot fliers, so a heavy-tailed +# column does not flood the figure with thousands of points. +_MAX_FLIERS = 200 +# How many columns flagged as "most contaminated" in the summary note. +_TOP_FLAGGED = 3 + +# Glossary terms this chapter explains (contract §11.1). Registered in the shared +# collector and marked clickable on first appearance. ``isolation_forest`` and +# ``zscore`` may also be registered by the MODELOS chapter — ``add`` is +# idempotent (first definition wins), so registering them here is harmless and +# keeps this chapter self-contained when MODELOS does not render. +_TERM_DEFS = { + "outlier": ( + "Valor atípico (outlier)", + "Una observación que se aparta mucho del grueso de los datos. Un atípico " + "NO es necesariamente un error: puede ser un fallo de medida o de " + "registro, pero también un dato real extremo (un cliente que gasta diez " + "veces la media, un día de ventas excepcional). Por eso se señalan para " + "revisarlos, no para borrarlos automáticamente.", + ), + "tukey_fence": ( + "Vallas de Tukey (1,5·IQR)", + "Regla clásica para marcar atípicos a partir de los cuartiles: se calcula " + "el rango intercuartílico IQR = P75 − P25 y se trazan dos vallas, una " + "inferior en P25 − 1,5·IQR y otra superior en P75 + 1,5·IQR. Los valores " + "que caen fuera de esas vallas se consideran atípicos. Es robusta porque " + "se apoya en la mediana y los cuartiles, no en la media.", + ), + "zscore": ( + "z-score (puntuación típica)", + "Mide a cuántas desviaciones típicas está un valor de la media de su " + "columna: z = (valor − media) / desviación típica. Un |z| grande (aquí > " + "3) señala un valor alejado del centro. A diferencia de las vallas de " + "Tukey, el z-score usa media y desviación, así que es más sensible a la " + "presencia de los propios atípicos.", + ), + "isolation_forest": ( + "Isolation Forest (anomalías multivariantes)", + "Algoritmo de detección de anomalías que considera TODAS las columnas a " + "la vez: construye árboles que parten el espacio con cortes aleatorios y " + "mide cuántos cortes hacen falta para aislar cada fila. Las filas raras " + "se aíslan con muy pocos cortes y se marcan como atípicas según un umbral " + "de contaminación. Detecta combinaciones de valores poco frecuentes que " + "ninguna columna por separado revelaría.", + ), +} + + +# --------------------------------------------------------------------------- # +# Lazy registry delegations (each degrades to None / no-op on any failure). +# --------------------------------------------------------------------------- # +def _load_build_boxplot_stats(): + try: + from datascience.build_boxplot_stats import build_boxplot_stats + return build_boxplot_stats + except Exception: # noqa: BLE001 + return None + + +def _load_detect_outliers(): + # detect_outliers lives in the monolithic ``datascience.datascience`` module + # (file_path datascience.py), not in its own submodule — try both shapes. + try: + from datascience.datascience import detect_outliers + return detect_outliers + except Exception: # noqa: BLE001 + try: + from datascience import detect_outliers + return detect_outliers + except Exception: # noqa: BLE001 + return None + + +def _load_isolation_forest(): + try: + from datascience.isolation_forest_outliers import isolation_forest_outliers + return isolation_forest_outliers + except Exception: # noqa: BLE001 + return None + + +def _load_summarize_dims(): + try: + from datascience.summarize_outlier_dims import summarize_outlier_dims + return summarize_outlier_dims + except Exception: # noqa: BLE001 + return None + + +# --------------------------------------------------------------------------- # +# Defensive formatters (own copy: the chapter never imports siblings). +# --------------------------------------------------------------------------- # +def _fmt_num(value, decimals: int = 3) -> str: + if value is None: + return "—" + if isinstance(value, bool): + return "sí" if value else "no" + if isinstance(value, int): + return f"{value:,}".replace(",", ".") + if isinstance(value, float): + if value != value: # NaN + return "—" + if value in (float("inf"), float("-inf")): + return str(value) + text = f"{value:.{decimals}f}".rstrip("0").rstrip(".") + return text if text else "0" + return model._safe_str(value) + + +def _fmt_int(value) -> str: + if value is None: + return "—" + try: + return f"{int(round(float(value))):,}".replace(",", ".") + except (TypeError, ValueError): + return model._safe_str(value) + + +def _fmt_pct(value, decimals: int = 2) -> str: + """Format an already-0-100 value as a percentage. None -> placeholder.""" + if value is None: + return "—" + try: + return f"{float(value):.{decimals}f}%" + except (TypeError, ValueError): + return model._safe_str(value) + + +def _term(mark: bool, key: str, text: str) -> str: + return f"[[term:{key}]]{text}[[/term]]" if mark else text + + +def _is_dict(v) -> bool: + return isinstance(v, dict) + + +# --------------------------------------------------------------------------- # +# Profile reads. +# --------------------------------------------------------------------------- # +def _numeric_columns(profile: dict) -> list: + """Return [(name, numeric_dict)] for numeric columns with usable stats.""" + out = [] + for col in profile.get("columns") or []: + if not isinstance(col, dict): + continue + if col.get("inferred_type") != "numeric": + continue + num = col.get("numeric") + if not isinstance(num, dict) or not num: + continue + if num.get("mean") is None and num.get("median") is None: + continue + out.append((col.get("name") or "(columna)", num)) + return out + + +def _clean_values(raw): + """Return the finite float values of a raw column list (drop None/NaN/inf).""" + if not isinstance(raw, (list, tuple)): + return None + vals = [] + for v in raw: + if v is None or isinstance(v, bool): + continue + try: + f = float(v) + except (TypeError, ValueError): + continue + if f != f or f in (float("inf"), float("-inf")): + continue + vals.append(f) + return vals + + +# --------------------------------------------------------------------------- # +# Per-column univariate summary. +# --------------------------------------------------------------------------- # +def _univariate_row(name, numeric, raw_vals, box_fn, detect_fn): + """Compute one univariate summary row + boxplot inputs for a column. + + Returns a dict with the table cells and, when raw values are available, the + exact Tukey/z counts and the list of atypical (flier) values; otherwise it + degrades to the profile's own z-score counts and the fence flags. + """ + box = {} + if box_fn is not None: + try: + box = box_fn(numeric) or {} + except Exception: # noqa: BLE001 + box = {} + lf = box.get("lower_fence") + uf = box.get("upper_fence") + + vals = _clean_values(raw_vals) + n_tukey = pct_tukey = None + n_z = pct_z = None + low_extreme = high_extreme = None + fliers = [] + contamination = None # metric used to rank columns (prefer Tukey %). + + if vals: + n = len(vals) + tukey_out = [] + for v in vals: + below = (lf is not None and v < lf) + above = (uf is not None and v > uf) + if below or above: + tukey_out.append(v) + n_tukey = len(tukey_out) + pct_tukey = 100.0 * n_tukey / n if n else None + if tukey_out: + low_extreme = min(tukey_out) + high_extreme = max(tukey_out) + fliers = tukey_out[:_MAX_FLIERS] + # z-score rule via the registry function (returns parallel bools). + if detect_fn is not None: + try: + flags = detect_fn(vals, _Z_THRESH) or [] + n_z = int(sum(1 for b in flags if b)) + pct_z = 100.0 * n_z / n if n else None + except Exception: # noqa: BLE001 + n_z = pct_z = None + contamination = pct_tukey + else: + # Degrade: no raw sample for this column. The profile's own outlier + # count/pct come from the z-score block (build_boxplot_stats note); the + # Tukey count is unknown, only the fence flags are. + n_z = numeric.get("n_outliers") + pct_z = numeric.get("outlier_pct") + if box.get("has_low_outliers") and box.get("min") is not None: + low_extreme = box.get("min") + if box.get("has_high_outliers") and box.get("max") is not None: + high_extreme = box.get("max") + contamination = pct_z if isinstance(pct_z, (int, float)) else None + + # Compact "extremos atípicos" cell: down/up arrows for the low/high tail. + extremes = [] + if low_extreme is not None: + extremes.append(f"↓ {_fmt_num(low_extreme)}") + if high_extreme is not None: + extremes.append(f"↑ {_fmt_num(high_extreme)}") + extremes_cell = " ".join(extremes) if extremes else "—" + + return { + "name": model._safe_str(name), + "n_tukey": n_tukey, + "pct_tukey": pct_tukey, + "n_z": n_z, + "pct_z": pct_z, + "lower_fence": lf, + "upper_fence": uf, + "extremes": extremes_cell, + "box": box, + "fliers": fliers, + "has_raw": bool(vals), + "contamination": contamination if isinstance(contamination, (int, float)) else -1.0, + } + + +def _univariate_table(rows: list) -> model.DataTable: + header = ["Columna", "Atípicos Tukey", "% Tukey", "Atípicos z", "% z", + "Valla inf.", "Valla sup.", "Extremos atípicos"] + table_rows = [] + for r in rows: + table_rows.append([ + r["name"], + _fmt_int(r["n_tukey"]) if r["n_tukey"] is not None else "—", + _fmt_pct(r["pct_tukey"]) if r["pct_tukey"] is not None else "—", + _fmt_int(r["n_z"]) if r["n_z"] is not None else "—", + _fmt_pct(r["pct_z"]) if r["pct_z"] is not None else "—", + _fmt_num(r["lower_fence"]), + _fmt_num(r["upper_fence"]), + r["extremes"], + ]) + return model.DataTable( + header=header, rows=table_rows, + title="Valores atípicos por columna", + note="Tukey = fuera de las vallas 1,5·IQR · z = |z-score| > 3 · " + "ordenado de más a menos contaminada") + + +# --------------------------------------------------------------------------- # +# Multivariate (Isolation Forest) section. +# --------------------------------------------------------------------------- # +def _resolve_multivariate(profile: dict, ctx: dict, raw_numeric): + """Return (outliers_dict_or_None, source). + + Prefers a LIVE Isolation Forest over ``raw_numeric`` so the detector and + ``summarize_outlier_dims`` use EXACTLY the same numeric columns and the same + valid-row indexing — otherwise the precomputed ``profile['models'] + ['outliers']`` (run by MODELOS over a possibly different column subset) would + yield ``row_index`` values that no longer point at the rows + ``summarize_outlier_dims`` reconstructs, mislabelling the "dimensions that + make each row rare". Falls back to the precomputed block when no raw sample + is available (e.g. the lite preset drops ``raw_numeric``).""" + if _is_dict(raw_numeric) and raw_numeric: + iso = _load_isolation_forest() + if iso is not None: + try: + out = iso(raw_numeric) + if _is_dict(out) and out.get("n_outliers") is not None and out.get("n_rows_used"): + return out, "live" + except Exception: # noqa: BLE001 + pass + # Fallback: the model the MODELOS chapter already computed (no raw sample to + # recompute against, so no per-row dimension breakdown either). + models = profile.get("models") if _is_dict(profile.get("models")) else {} + pre = models.get("outliers") if _is_dict(models) else None + if _is_dict(pre) and pre.get("n_outliers") is not None and pre.get("n_rows_used"): + return pre, "precomputed" + return None, "none" + + +def _multivariate_blocks(outliers: dict, raw_numeric, mark: bool) -> list: + isof = _term(mark, "isolation_forest", "**Isolation Forest**") + blocks = [ + model.Heading(text="Filas atípicas (multivariante)", level=2), + model.Markdown(text=( + f"Hasta aquí cada columna se ha mirado por separado. {isof} busca " + "filas raras considerando **todas las columnas a la vez**: una fila " + "puede ser normal en cada variable y aun así ser atípica por la " + "**combinación** de sus valores (p. ej. una edad baja con una tarifa " + "muy alta). La tabla resume cuántas filas se marcaron y el umbral de " + "decisión.")), + model.KVTable(rows=[ + ("Filas analizadas", _fmt_int(outliers.get("n_rows_used"))), + ("Columnas consideradas", _fmt_int(outliers.get("n_features"))), + ("Filas atípicas", _fmt_int(outliers.get("n_outliers"))), + ("% filas atípicas", _fmt_pct(outliers.get("outlier_pct"))), + ("Umbral de decisión", _fmt_num(outliers.get("threshold"), 4)), + ], title="Anomalías multivariantes"), + ] + + rows_in = outliers.get("outlier_rows") or [] + if not rows_in: + return blocks + + # Enrich each anomalous row with the dimensions that make it rare, when the + # raw sample is available (summarize_outlier_dims reconstructs the same + # valid-row indexing as isolation_forest_outliers). + dims_by_row = {} + if _is_dict(raw_numeric) and raw_numeric: + summ = _load_summarize_dims() + if summ is not None: + try: + enriched = summ(raw_numeric, rows_in, top_k=3) or [] + for e in enriched: + if _is_dict(e) and e.get("row_index") is not None: + dims_by_row[e.get("row_index")] = e.get("dims") or [] + except Exception: # noqa: BLE001 + dims_by_row = {} + + has_dims = bool(dims_by_row) + header = ["Fila (entre válidas)", "Score"] + if has_dims: + header.append("Dimensiones que la hacen rara (col = valor, z)") + table_rows = [] + for r in rows_in[:_TOP_ROWS]: + if not _is_dict(r): + continue + ridx = r.get("row_index") + cells = [_fmt_int(ridx), _fmt_num(r.get("score"), 4)] + if has_dims: + dims = dims_by_row.get(ridx) or [] + parts = [] + for d in dims: + if not _is_dict(d): + continue + parts.append( + f"{model._safe_str(d.get('col'))} = {_fmt_num(d.get('value'))} " + f"(z {_fmt_num(d.get('z'), 2)})") + cells.append("; ".join(parts) if parts else "—") + table_rows.append(cells) + + if table_rows: + shown = len(table_rows) + total = outliers.get("n_outliers") + note = "las filas más anómalas primero (score más bajo = más rara)" + if isinstance(total, int) and total > shown: + note += f" — top {shown} de {total}" + if not has_dims: + note += (" · no se pudo recuperar la muestra cruda para explicar las " + "dimensiones de cada fila") + blocks.append(model.DataTable( + header=header, rows=table_rows, + title="Filas más atípicas", note=note)) + return blocks + + +# --------------------------------------------------------------------------- # +# Interpretation section. +# --------------------------------------------------------------------------- # +def _interpretation_block(mark: bool) -> model.Markdown: + outlier = _term(mark, "outlier", "atípico") + text = ( + f"**Un {outlier} no es necesariamente un error.** Conviene distinguir " + "dos casos antes de actuar:\n\n" + "- **Error de dato** (medida, registro o unidad equivocada): una edad de " + "200 años, un importe negativo donde no puede haberlo, un decimal " + "desplazado. Estos sí se corrigen o se eliminan, idealmente en el origen.\n" + "- **Dato real extremo**: una observación legítima de la cola de la " + "distribución (un cliente que gasta mucho más, una tarifa de lujo, un día " + "de ventas excepcional). Borrarla sesga el análisis y oculta información " + "valiosa.\n\n" + "**Qué hacer.** Primero, **revisar** los valores señalados arriba contra " + "su origen para decidir cuál de los dos casos es. Si son errores, " + "corregirlos. Si son datos reales que distorsionan medias y modelos, hay " + "alternativas a borrarlos: **winsorizar** (recortar los extremos a un " + "percentil), o **re-expresar** la variable (por ejemplo una " + "transformación logarítmica o la escalera de re-expresión de Tukey que " + "este mismo perfil ya calcula para las columnas asimétricas), que suele " + "domar la cola sin perder ninguna fila. La elección depende del objetivo: " + "esta lectura es **exploratoria** —orienta dónde mirar—, no una regla " + "automática de limpieza.") + return model.Markdown(text=text) + + +# --------------------------------------------------------------------------- # +# Entry point. +# --------------------------------------------------------------------------- # +def build_outliers(profile: dict, ctx: dict): + """Build the OUTLIERS Chapter, or None if the dataset has no numeric column.""" + profile = profile or {} + ctx = ctx or {} + if not isinstance(profile, dict): + return None + + numerics = _numeric_columns(profile) + if not numerics: + return None # chapter does not apply to a dataset with no numerics. + + # Register glossary terms (if a collector is present) and mark them clickable. + glossary = ctx.get("glossary") + mark = False + if isinstance(glossary, model.GlossaryCollector): + for key, (label, definition) in _TERM_DEFS.items(): + glossary.add(key, label, definition) + mark = True + + raw_numeric = ctx.get("raw_numeric") + raw_numeric = raw_numeric if isinstance(raw_numeric, dict) else {} + + box_fn = _load_build_boxplot_stats() + detect_fn = _load_detect_outliers() + + # --- Univariate summary ------------------------------------------------- # + uni_rows = [] + for name, numeric in numerics: + uni_rows.append(_univariate_row( + name, numeric, raw_numeric.get(name), box_fn, detect_fn)) + # Rank columns by contamination (Tukey % when available, else z %). + uni_rows.sort(key=lambda r: r.get("contamination", -1.0), reverse=True) + + intro = ( + "Este capítulo reúne en un solo sitio el análisis de los **valores " + "atípicos** de la tabla, que en el resto del informe aparecen dispersos. " + f"Un {_term(mark, 'outlier', 'atípico')} es una observación que se aparta " + "mucho del grueso de los datos. Cada columna numérica se evalúa con dos " + f"criterios complementarios: las {_term(mark, 'tukey_fence', 'vallas de Tukey')} " + "(fuera de P25−1,5·IQR o P75+1,5·IQR, robusto a la propia cola) y el " + f"{_term(mark, 'zscore', 'z-score')} (|z| > 3, sensible a la media). La " + "tabla está ordenada de la columna más contaminada a la menos.") + + blocks = [ + model.Heading(text=CHAPTER_TITLE, level=1), + model.Markdown(text=intro), + _univariate_table(uni_rows), + ] + + # Flag the most contaminated columns explicitly. + flagged = [r["name"] for r in uni_rows + if r.get("contamination", -1.0) > 0][:_TOP_FLAGGED] + if flagged: + names = ", ".join(f"**{n}**" for n in flagged) + blocks.append(model.Markdown(text=( + f"Las columnas con mayor proporción de atípicos son {names}: " + "concentran el grueso de los valores fuera de las vallas y son las " + "primeras a revisar."))) + + # --- Boxplots figure ---------------------------------------------------- # + box_entries = [ + {"name": r["name"], "box": r["box"], "fliers": r.get("fliers")} + for r in uni_rows + if r.get("box") + ][:_TOP_BOX] + if box_entries: + def _boxplots_make(entries=box_entries): + try: + from datascience.build_boxplots_figure import build_boxplots_figure + return build_boxplots_figure( + entries, title="Boxplots de Tukey por columna", + max_boxes=_TOP_BOX) + except Exception: # noqa: BLE001 — minimal fallback figure. + import matplotlib + matplotlib.use("Agg") + from matplotlib.figure import Figure + fig = Figure(figsize=(5.0, 2.2)) + ax = fig.add_subplot(111) + ax.text(0.5, 0.5, "(boxplots no disponibles)", + ha="center", va="center") + ax.axis("off") + return fig + + blocks.append(model.Group(blocks=[ + model.Heading(text="Boxplots", level=2), + model.Markdown(text=( + "Cada caja abarca del primer al tercer cuartil (P25–P75), la línea " + "interior es la mediana y los bigotes llegan hasta 1,5·IQR; los " + "puntos son los valores que caen fuera de las vallas (atípicos por " + "Tukey).")), + model.Figure( + make=_boxplots_make, + caption="Boxplots de Tukey de las columnas más contaminadas."), + ])) + + # --- Multivariate ------------------------------------------------------- # + outliers, _src = _resolve_multivariate(profile, ctx, raw_numeric) + if outliers is not None: + blocks.extend(_multivariate_blocks(outliers, raw_numeric, mark)) + else: + blocks.append(model.Heading(text="Filas atípicas (multivariante)", level=2)) + blocks.append(model.Note( + "No se pudo analizar la anomalía multivariante: hacen falta al menos " + "dos columnas numéricas y la muestra cruda (o los modelos del perfil) " + "para correr Isolation Forest.")) + + # --- Interpretation ----------------------------------------------------- # + blocks.append(model.Heading(text="Cómo interpretar los atípicos", level=2)) + blocks.append(_interpretation_block(mark)) + + return model.Chapter(id=CHAPTER_ID, title=CHAPTER_TITLE, + version=CHAPTER_VERSION, blocks=blocks) diff --git a/python/functions/datascience/automatic_eda/chapters/outliers_test.py b/python/functions/datascience/automatic_eda/chapters/outliers_test.py new file mode 100644 index 00000000..bff20166 --- /dev/null +++ b/python/functions/datascience/automatic_eda/chapters/outliers_test.py @@ -0,0 +1,304 @@ +"""Tests for the OUTLIERS chapter — DoD: golden + edges + error path. + +Self-contained: builds synthetic ``numeric`` blocks + a raw_numeric sample (no +DuckDB) so the suite is fast and deterministic. Verifies that the chapter emits +the univariate per-column table, a boxplots figure, the multivariate Isolation +Forest section and the outlier≠error interpretation; that the most contaminated +column is ranked first; that a profile with no numeric column yields None; that +None/empty never raises; that the glossary terms are registered; and that the +chapter renders into both PDF and PPTX without cutting its title. +""" + +import math +import os +import re +import tempfile + +from pypdf import PdfReader + +from datascience.automatic_eda.chapters.outliers import ( + build_outliers, CHAPTER_VERSION, CHAPTER_TITLE, _TERM_DEFS, +) +from datascience.automatic_eda import model +from datascience.render_automatic_eda_pdf import render_automatic_eda_pdf +from datascience.render_automatic_eda_pptx import render_automatic_eda_pptx + + +def _percentile(sorted_vals, q): + """Linear-interpolation percentile (q in 0..1) on an already-sorted list.""" + if not sorted_vals: + return None + if len(sorted_vals) == 1: + return float(sorted_vals[0]) + pos = q * (len(sorted_vals) - 1) + lo = int(math.floor(pos)) + hi = int(math.ceil(pos)) + if lo == hi: + return float(sorted_vals[lo]) + frac = pos - lo + return float(sorted_vals[lo] * (1 - frac) + sorted_vals[hi] * frac) + + +def _col_from_values(values, nbins=10): + """Build a ``numeric`` sub-block shaped like describe_numeric's output from a + concrete list of raw values, so the profile percentiles and the raw sample + are consistent (the boxplot fences match the crudo).""" + vals = [float(v) for v in values] + s = sorted(vals) + n = len(s) + mean = sum(vals) / n + var = sum((v - mean) ** 2 for v in vals) / n + std = math.sqrt(var) + median = _percentile(s, 0.5) + p25 = _percentile(s, 0.25) + p75 = _percentile(s, 0.75) + mn, mx = s[0], s[-1] + # z-score outlier count (population), what the profile's n_outliers carries. + n_out = sum(1 for v in vals if std > 0 and abs((v - mean) / std) > 3.0) + width = (mx - mn) / nbins if mx > mn else 1.0 + hist = [{"lo": mn + i * width, "hi": mn + (i + 1) * width, "count": 1} + for i in range(nbins)] + return { + "min": mn, "max": mx, "mean": mean, "median": median, "std": std, + "p25": p25, "p50": median, "p75": p75, "iqr": (p75 - p25), + "n_outliers": n_out, "outlier_pct": 100.0 * n_out / n, + "distribution_type": "right-skewed", "histogram": hist, + } + + +def _fare_values(): + """A heavy-tailed column (most ~10-30, a few 200-512): clear Tukey/z outliers.""" + base = [7.0 + (i % 25) for i in range(120)] # bulk 7..31 + tail = [180.0, 210.0, 263.0, 512.0] # extreme upper tail + return base + tail + + +def _age_values(): + """A roughly symmetric column with one extreme low value.""" + base = [22.0 + (i % 40) for i in range(120)] # 22..61 + return base + [80.0, 0.5, 74.0, 1.0] + + +def _quiet_values(): + """A clean column with no atypical values.""" + return [50.0 + (i % 5) for i in range(124)] + + +def _profile_and_ctx(with_models=True, with_raw=True): + fare = _fare_values() + age = _age_values() + quiet = _quiet_values() + cols = [ + {"name": "Fare", "inferred_type": "numeric", "numeric": _col_from_values(fare)}, + {"name": "Age", "inferred_type": "numeric", "numeric": _col_from_values(age)}, + {"name": "Quiet", "inferred_type": "numeric", "numeric": _col_from_values(quiet)}, + {"name": "Sexo", "inferred_type": "categorical", + "categorical": {"top": [{"value": "male", "count": 80}]}}, + ] + profile = {"table": "titanic", "n_rows": len(fare), "n_cols": len(cols), + "columns": cols} + if with_models: + profile["models"] = { + "outliers": { + "n_outliers": 4, "outlier_pct": 3.2, + "outlier_rows": [ + {"row_index": 123, "score": -0.21}, + {"row_index": 121, "score": -0.15}, + ], + "threshold": -0.02, "n_rows_used": 124, "n_features": 3, + } + } + ctx = {} + if with_raw: + ctx["raw_numeric"] = {"Fare": fare, "Age": age, "Quiet": quiet} + return profile, ctx + + +def _pdf_text(path: str) -> str: + txt = "".join((pg.extract_text() or "") for pg in PdfReader(path).pages) + return re.sub(r"\s+", " ", txt) + + +def _flatten(blocks): + out = [] + for b in blocks: + if getattr(b, "kind", "") == "group": + out.extend(_flatten(getattr(b, "blocks", []) or [])) + else: + out.append(b) + return out + + +# --------------------------------------------------------------------------- # +# Golden. +# --------------------------------------------------------------------------- # +def test_golden_estructura_y_secciones(): + profile, ctx = _profile_and_ctx() + ctx["glossary"] = model.GlossaryCollector() + ch = build_outliers(profile, ctx) + assert ch is not None + assert ch.id == "outliers" + assert ch.version == CHAPTER_VERSION + + flat = _flatten(ch.blocks) + kinds = [b.kind for b in flat] + # Title heading + univariate DataTable + boxplots Figure + multivariate + # KVTable + interpretation Markdown. + assert kinds[0] == "heading" and flat[0].text == CHAPTER_TITLE + tables = [b for b in flat if b.kind == "data_table"] + titles = [t.title for t in tables] + assert any(t and "atípicos por columna" in t for t in titles) + assert any(b.kind == "figure" for b in flat), "falta la figura de boxplots" + assert any(b.kind == "kv_table" for b in flat), "falta el resumen multivariante" + + # The boxplots figure maker yields a real matplotlib figure (or its fallback). + fig = next(b for b in flat if b.kind == "figure").make() + assert fig is not None + import matplotlib.pyplot as plt + plt.close(fig) + + +def test_golden_fare_es_la_mas_contaminada(): + # The univariate table must rank Fare (heavy tail) first and report a + # non-zero Tukey percentage for it. + profile, ctx = _profile_and_ctx() + ch = build_outliers(profile, ctx) + table = next(b for b in _flatten(ch.blocks) + if b.kind == "data_table" and b.title + and "atípicos por columna" in b.title) + first_col = table.rows[0][0] + assert first_col == "Fare", f"esperaba Fare primera, fue {first_col}" + # % Tukey column (index 2) of the first row must be > 0. + pct_cell = table.rows[0][2] + assert pct_cell not in ("—", "0%", "0.00%"), f"% Tukey de Fare vacío: {pct_cell}" + # The z-score rule (detect_outliers) must actually run with raw_numeric: at + # least one column reports a non-empty z count/percentage (regression guard + # for the detect_outliers import path). + z_pcts = [r[4] for r in table.rows] + assert any(c not in ("—",) for c in z_pcts), f"columna z toda vacía: {z_pcts}" + z_counts = [r[3] for r in table.rows] + assert any(c not in ("—",) for c in z_counts), f"conteo z vacío: {z_counts}" + + +def test_golden_interpretacion_outlier_no_es_error(): + profile, ctx = _profile_and_ctx() + ch = build_outliers(profile, ctx) + md = " ".join(b.text for b in _flatten(ch.blocks) if b.kind == "markdown") + assert "no es necesariamente un error" in md.lower() + # Mentions the actionable options (winsorize / re-express). + assert "winsoriz" in md.lower() + assert "re-expres" in md.lower() or "logarítmic" in md.lower() + + +def test_golden_terminos_glosario_registrados(): + profile, ctx = _profile_and_ctx() + gloss = model.GlossaryCollector() + ctx["glossary"] = gloss + build_outliers(profile, ctx) + for key in _TERM_DEFS: + assert gloss.has(key), f"término '{key}' no registrado en el glosario" + # Terms are marked clickable in the body text. + md = " ".join(b.text for b in _flatten(build_outliers(profile, ctx).blocks) + if b.kind == "markdown") + assert "[[term:outlier]]" in md and "[[term:tukey_fence]]" in md + + +# --------------------------------------------------------------------------- # +# Multivariate. +# --------------------------------------------------------------------------- # +def test_multivariante_live_con_raw_y_dims(): + # With a raw sample the chapter runs Isolation Forest live (over the same + # columns summarize_outlier_dims uses) and lists the anomalous rows with the + # dimensions that make each one rare. + profile, ctx = _profile_and_ctx(with_models=False, with_raw=True) + ch = build_outliers(profile, ctx) + flat = _flatten(ch.blocks) + kv = next(b for b in flat if b.kind == "kv_table") + flat_kv = " ".join(f"{k} {v}" for (k, v) in kv.rows) + assert "Filas atípicas" in flat_kv + # A non-zero number of anomalous rows is reported. + n_cell = dict(kv.rows).get("Filas atípicas") + assert n_cell not in (None, "—", "0"), f"sin filas atípicas: {n_cell}" + # The anomalous-rows table carries the per-row dimension breakdown. + tbls = [b for b in flat if b.kind == "data_table" and b.title + and "más atípicas" in b.title] + assert tbls, "falta la tabla de filas más atípicas" + assert any("hacen rara" in h for h in tbls[0].header), \ + f"falta la columna de dimensiones: {tbls[0].header}" + + +def test_multivariante_precomputed_sin_raw(): + # Without a raw sample the chapter falls back to profile['models']['outliers'] + # (lite preset path); the precomputed n_outliers (4) surfaces in the KV table. + profile, ctx = _profile_and_ctx(with_models=True, with_raw=False) + ch = build_outliers(profile, ctx) + kv = next(b for b in _flatten(ch.blocks) if b.kind == "kv_table") + assert any("4" in str(v) for (k, v) in kv.rows) + + +def test_multivariante_ausente_degrada_a_nota(): + # No models and no raw sample → an honest note, never a crash. + profile, ctx = _profile_and_ctx(with_models=False, with_raw=False) + ch = build_outliers(profile, ctx) + assert ch is not None + notes = [b.text for b in _flatten(ch.blocks) if b.kind == "note"] + assert any("Isolation Forest" in n for n in notes) + + +# --------------------------------------------------------------------------- # +# Edges / error path. +# --------------------------------------------------------------------------- # +def test_edge_sin_columnas_numericas_devuelve_none(): + prof = {"columns": [{"name": "c", "inferred_type": "categorical", + "categorical": {"top": [{"value": "x", "count": 3}]}}]} + assert build_outliers(prof, {}) is None + + +def test_edge_solo_texto_sintetico_devuelve_none(): + # A text-only synthetic table (no numeric column) yields None (does not break). + prof = {"table": "notas", "n_rows": 3, "n_cols": 1, + "columns": [{"name": "comentario", "inferred_type": "text", + "text": {"n_docs": 3}}]} + assert build_outliers(prof, {}) is None + + +def test_edge_profile_none_y_vacio_no_revienta(): + assert build_outliers(None, None) is None + assert build_outliers({}, {}) is None + assert build_outliers({"columns": []}, {}) is None + + +def test_edge_sin_raw_numeric_degrada_a_perfil(): + # Without raw_numeric the chapter still builds, using the profile z-score + # counts; the univariate table exists and Tukey counts degrade to '—'. + profile, ctx = _profile_and_ctx(with_models=True, with_raw=False) + ch = build_outliers(profile, ctx) + assert ch is not None + table = next(b for b in _flatten(ch.blocks) + if b.kind == "data_table" and b.title + and "atípicos por columna" in b.title) + # z column comes from the profile; Tukey count is unknown ('—'). + assert all(len(r) == 8 for r in table.rows) + + +# --------------------------------------------------------------------------- # +# Anti-cut render. +# --------------------------------------------------------------------------- # +def test_render_pdf_y_pptx_incluyen_el_capitulo(): + profile, ctx = _profile_and_ctx() + # The renderers build the whole document; the chapter is reached via the + # registry. Render the chapter standalone through a one-chapter document by + # passing the profile directly (the renderers run the full chapter registry). + with tempfile.TemporaryDirectory() as d: + pdf = os.path.join(d, "out.pdf") + res_pdf = render_automatic_eda_pdf(profile, pdf, + {"write_manifest": False, "ctx": ctx}) + assert res_pdf["path"] == pdf + txt = _pdf_text(pdf) + assert CHAPTER_TITLE in txt, "el capítulo OUTLIERS no aparece en el PDF" + assert "Fare" in txt + pptx = os.path.join(d, "out.pptx") + res_pptx = render_automatic_eda_pptx(profile, pptx, + {"write_manifest": False, "ctx": ctx}) + assert res_pptx["path"] == pptx + assert res_pptx["n_slides"] >= 1 diff --git a/python/functions/datascience/automatic_eda/chapters_registry.py b/python/functions/datascience/automatic_eda/chapters_registry.py index 41097975..17d956db 100644 --- a/python/functions/datascience/automatic_eda/chapters_registry.py +++ b/python/functions/datascience/automatic_eda/chapters_registry.py @@ -34,6 +34,7 @@ CHAPTER_ORDER = [ "text_distr", # free-text / NLP distributions (non-tabular content) "calidad", # data quality "missingness", # missing-data patterns (co-occurrence of absences; MCAR/MAR) + "outliers", # atypical values: univariate (Tukey/z) + multivariate (IsolationForest) "correlacion", # correlations / associations "relaciones", # key relations: declared/candidate PK + FK (inter/intra-table) "modelos", # cheap models (PCA/KMeans/outliers) diff --git a/python/functions/datascience/build_boxplots_figure.md b/python/functions/datascience/build_boxplots_figure.md new file mode 100644 index 00000000..258d9986 --- /dev/null +++ b/python/functions/datascience/build_boxplots_figure.md @@ -0,0 +1,125 @@ +--- +id: build_boxplots_figure_py_datascience +name: build_boxplots_figure +kind: function +lang: py +domain: datascience +version: "1.0.0" +purity: impure +signature: "def build_boxplots_figure(boxes: list, title: str = \"\", max_boxes: int = 12) -> \"matplotlib.figure.Figure\"" +description: "Construye una unica figura matplotlib con boxplots de Tukey HORIZONTALES (uno por columna) usando ax.bxp: caja Q1-Q3, bigotes hasta 1.5*IQR, linea de mediana y puntos atipicos. Consume la salida de build_boxplot_stats (un dict box por columna, leido con .get) mas una lista opcional de outliers crudos por columna; si vienen los dibuja como puntos (showfliers), si no marca solo box[min]/box[max] cuando hay outliers de cola (igual que num_distr). Dibuja como mucho max_boxes cajas (las primeras, ya ordenadas por contaminacion por el caller) y avisa de la truncacion con (mostrando N de M). Backend Agg sin pyplot global; alto adaptativo al nº de cajas. Defensiva: omite entradas invalidas y NUNCA lanza — sin cajas validas devuelve una figura placeholder (sin boxplots). Es la version small-multiples del capitulo num_distr para responder que columnas tienen mas outliers de un vistazo." +tags: [eda, outliers, boxplot, tukey, iqr, bxp, matplotlib, figure, visualization, small-multiples, datascience, impure] +uses_functions: [] +uses_types: [] +returns: [] +returns_optional: false +error_type: "error_go_core" +imports: [matplotlib] +example: | + from datascience.build_boxplot_stats import build_boxplot_stats + from datascience.build_boxplots_figure import build_boxplots_figure + boxes = [ + {"name": "ingresos", "box": build_boxplot_stats({"min": 1.0, "max": 9e3, + "p25": 1e3, "median": 2e3, "p75": 3e3, "n_outliers": 7}), "fliers": None}, + {"name": "edad", "box": build_boxplot_stats({"min": 0.0, "max": 99.0, + "p25": 25.0, "median": 38.0, "p75": 52.0}), "fliers": None}, + ] + fig = build_boxplots_figure(boxes, title="Outliers por columna", max_boxes=12) +tested: true +tests: + - "test_returns_figure_with_axes" + - "test_empty_list_returns_placeholder_figure" + - "test_invalid_box_is_skipped_not_raised" + - "test_all_invalid_returns_placeholder" + - "test_raw_fliers_are_drawn" + - "test_max_boxes_truncates_and_does_not_raise" +test_file_path: "python/functions/datascience/build_boxplots_figure_test.py" +file_path: "python/functions/datascience/build_boxplots_figure.py" +params: + - name: boxes + desc: "Lista de dicts, cada uno {\"name\": str, \"box\": dict, \"fliers\": list|None}. box es EXACTAMENTE la salida de build_boxplot_stats (claves leidas con .get: q1, median, q3, whisker_lo, whisker_hi, min, max, has_low_outliers, has_high_outliers, lower_fence, upper_fence, n_outliers). fliers es la lista opcional de outliers crudos: si viene se dibuja como puntos; si es None/ausente solo se marcan los extremos box[min]/box[max] cuando hay outliers de cola. Entradas que no son dict, sin box dict, o sin q1/median/q3 se omiten. El caller las pasa ya ordenadas por contaminacion (la mayor primera)." + - name: title + desc: "Titulo de la figura (fig.suptitle, alineado a la izquierda). Vacio => sin titulo. Si len(boxes) > max_boxes se le anade una nota \"(mostrando N de M)\" para que la truncacion no sea silenciosa. Default \"\"." + - name: max_boxes + desc: "Numero maximo de cajas a dibujar (las primeras de la lista). Default 12. Un valor no entero o <= 0 cae a 12. Si la lista trae mas entradas, las sobrantes se descartan pero se reporta en el titulo con (mostrando N de M)." +output: "Un matplotlib.figure.Figure (figsize 7.0 x alto adaptativo = max(2.0, 0.5*n + 1.0), dpi 150) con un unico Axes que apila boxplots horizontales de Tukey (ax.bxp, orientation=horizontal con fallback vert=False), uno por columna valida, de arriba a abajo en el orden recibido. Cada caja: relleno #9ec6df, borde/bigotes/caps #5b8aa6, mediana #2e8b57, atipicos #c0392b. Etiquetas del eje Y = nombres de columna; eje X etiquetado \"valor\". Outliers dibujados desde fliers crudos (showfliers) o, si faltan, marcados en box[min]/box[max] segun has_low/high_outliers. Si no queda ninguna caja valida (lista vacia o todas invalidas) devuelve una Figure placeholder con texto centrado \"(sin boxplots)\"; cualquier error inesperado se captura y devuelve una Figure con el mensaje de error. NUNCA lanza. El caller rasteriza/cierra la figura; la funcion no la muestra ni la guarda." +--- + +## Ejemplo + +```python +import sys, os +sys.path.insert(0, os.path.join("python", "functions")) +from datascience.build_boxplot_stats import build_boxplot_stats +from datascience.build_boxplots_figure import build_boxplots_figure + +# Un `box` por columna numérica, derivado del sub-bloque `numeric` del profile +# (salida de describe_numeric). El caller los pasa ya ordenados por outlier_pct. +boxes = [ + { + "name": "ingresos", + "box": build_boxplot_stats({ + "min": 1.0, "max": 9000.0, + "p25": 1000.0, "median": 2000.0, "p75": 3000.0, + "n_outliers": 7, + }), + "fliers": None, # valores crudos desconocidos -> se marca solo el extremo. + }, + { + "name": "edad", + "box": build_boxplot_stats({ + "min": 0.0, "max": 99.0, + "p25": 25.0, "median": 38.0, "p75": 52.0, + }), + "fliers": [88.0, 95.0, 99.0], # outliers crudos -> se dibujan como puntos. + }, +] + +fig = build_boxplots_figure(boxes, title="Outliers por columna", max_boxes=12) + +# El renderer del informe lo rasteriza; aquí solo persistimos para inspección. +fig.savefig("/tmp/boxplots.png") +``` + +## Cuando usarla + +Úsala en el capítulo de outliers de un informe EDA cuando quieras comparar de un +vistazo *qué columnas están más contaminadas por valores atípicos*: a diferencia +de `num_distr` (que dibuja un histograma+boxplot por columna en figuras +separadas), aquí apilas todos los boxplots horizontales en **una sola figura** +(small multiples). Primero deriva el `box` de cada columna con +`build_boxplot_stats`, ordénalas por `outlier_pct` descendente, envuélvelas como +`{"name", "box", "fliers"}` y pásaselas. Si tienes los valores crudos fuera de +las vallas, métele la lista `fliers` y se dibujarán como puntos; si no, la +función marca solo los extremos `min`/`max` cuando hay cola. + +## Gotchas + +- **Impura por matplotlib.** Toca la maquinaria de render. Usa el backend `Agg` + y la API orientada a objetos `Figure`/`add_subplot` — NUNCA `pyplot.*` aquí, + para no tocar el estado global ni filtrar figuras entre llamadas. `pyplot` NO + es thread-safe; esta función construye el `Figure` directamente, así que es + segura de llamar en bucle desde el renderer. +- **El caller cierra la figura.** Devuelve el `Figure` pero no lo muestra ni lo + guarda. Quien la consume debe rasterizarla y luego liberarla + (`matplotlib.pyplot.close(fig)`) para no acumular memoria en lotes grandes. +- **`fliers` opcional, semántica distinta.** Si pasas la lista de outliers + crudos se dibujan todos como puntos (`showfliers=True`). Si es `None`/ausente + los valores son desconocidos y solo se marca un punto en `box["min"]` / + `box["max"]` cuando `has_low_outliers` / `has_high_outliers` — mismo criterio + que `num_distr`. No inventes fliers a partir del profile: el `box` no trae los + valores crudos, solo si los extremos superan las vallas. +- **API de orientación de `ax.bxp`.** matplotlib reciente usa + `orientation="horizontal"`; las versiones antiguas usan `vert=False`. La + función prueba la primera y cae a la segunda en `except TypeError`, así que + funciona en ambas. Si `bxp` falla del todo, el Axes degrada a un texto + "(boxplot no disponible)" en vez de propagar. +- **Truncación visible.** `max_boxes` (default 12) limita el nº de cajas para que + ninguna se solape; si la lista trae más, las sobrantes se descartan pero se + avisa en el título con "(mostrando N de M)". Pasa las columnas ya ordenadas por + contaminación para que las descartadas sean las menos relevantes. +- **Defensiva, nunca lanza.** Lista vacía, entradas no-dict, sin `box`, o sin + `q1`/`median`/`q3` se omiten sin propagar; sin cajas válidas devuelve un + placeholder "(sin boxplots)" y cualquier error inesperado se captura en una + figura con el texto del error. No envuelvas la llamada en try/except por miedo + a un raise — no lo hay. diff --git a/python/functions/datascience/build_boxplots_figure.py b/python/functions/datascience/build_boxplots_figure.py new file mode 100644 index 00000000..579ebc49 --- /dev/null +++ b/python/functions/datascience/build_boxplots_figure.py @@ -0,0 +1,250 @@ +"""Impure EDA helper: a single figure of horizontal Tukey boxplots (`eda` group). + +Draws, in one ``matplotlib.figure.Figure``, a stack of horizontal Tukey boxplots +(one per column) using ``ax.bxp``: each carries its box (Q1–Q3), whiskers (up to +1.5·IQR), the median line and its outlier points. It consumes the output of the +pure registry function ``build_boxplot_stats`` (one ``box`` dict per column) plus +an optional list of raw outlier values per column; it never recomputes anything. + +It is the "small-multiples" companion of ``num_distr`` (which draws one +histogram+boxplot per column): here every column shares a single figure so the +caller can show, at a glance, *which* columns are the most contaminated by +outliers (the caller passes them already ordered by contamination). + +Impure because it touches matplotlib's rendering machinery. It uses the headless +Agg backend and the object-oriented ``Figure`` API (no ``pyplot``) so it leaks no +global state and is safe to call repeatedly from a report renderer. It is fully +defensive and NEVER raises: invalid entries are skipped and, if nothing valid +remains, it returns a placeholder figure carrying a centered "(sin boxplots)". +""" + +import matplotlib + +matplotlib.use("Agg") + +from matplotlib.figure import Figure # noqa: E402 + +# Blue palette shared with the ``num_distr`` chapter so the report stays coherent. +_BOX_FACE = "#9ec6df" # box fill. +_BOX_EDGE = "#5b8aa6" # box / whisker / cap border. +_MEDIAN = "#2e8b57" # median line (sea green). +_OUTLIER = "#c0392b" # outlier points (soft red). +# Muted gray for the placeholder / fallback message text. +_MUTED_TEXT = "#5f6b7a" +# Soft red for the error fallback message. +_ERROR_TEXT = "#b00020" + + +def _num(value): + """Coerce ``value`` to float defensively; None for None/bool/non-numeric/NaN.""" + # bool is a subclass of int; a stat value is never a real bool, so treat + # True/False as missing instead of silently coercing to 1.0/0.0. + if value is None or isinstance(value, bool): + return None + try: + f = float(value) + except (TypeError, ValueError): + return None + if f != f: # NaN guard. + return None + return f + + +def _placeholder_figure(message: str, color: str = _MUTED_TEXT) -> "Figure": + """Return a fallback ``Figure`` carrying a single centered message.""" + fig = Figure(figsize=(7.0, 2.4), dpi=150) + ax = fig.add_subplot(111) + ax.axis("off") + ax.text( + 0.5, + 0.5, + message, + ha="center", + va="center", + fontsize=12, + color=color, + wrap=True, + transform=ax.transAxes, + ) + fig.tight_layout() + return fig + + +def build_boxplots_figure( + boxes: list, + title: str = "", + max_boxes: int = 12, +) -> "matplotlib.figure.Figure": + """Build one figure of stacked horizontal Tukey boxplots (one per column). + + For each entry the function builds a ``bxp`` stats record (``med, q1, q3, + whislo, whishi, fliers, label``) from its ``box`` sub-dict (the output of + ``build_boxplot_stats``) and draws all of them as horizontal boxplots sharing + the X axis, top-to-bottom in the order received (the caller is expected to + pass them already sorted by contamination). + + Outliers are shown two ways: + + - If an entry carries a ``fliers`` list (the raw out-of-fence values), they + are drawn as red points via ``ax.bxp(..., showfliers=True)``. + - If ``fliers`` is ``None``/absent, the raw values are unknown, so only the + extremes are marked: a red point at ``box["min"]`` when + ``box["has_low_outliers"]`` and at ``box["max"]`` when + ``box["has_high_outliers"]`` (same convention as ``num_distr``). + + The function is fully defensive and NEVER raises. Entries that are not dicts, + lack a ``box`` dict, or miss any of ``q1``/``median``/``q3`` are skipped. If + after filtering no valid box remains it returns a placeholder ``Figure`` with + a centered "(sin boxplots)"; any unexpected error is caught and turned into a + fallback figure carrying the error text. It always returns a ``Figure``. + + Args: + boxes: List of dicts ``{"name": str, "box": dict, "fliers": list|None}``. + ``box`` is exactly the output of ``build_boxplot_stats`` (read with + ``.get``: ``q1, median, q3, whisker_lo, whisker_hi, min, max, + has_low_outliers, has_high_outliers, ...``). ``fliers`` is the + optional list of raw outlier values; when present they are plotted, + otherwise only the extremes are marked. + title: Figure title (``fig.suptitle``). Empty => no title. When the list + is longer than ``max_boxes`` a "(mostrando N de M)" note is appended. + max_boxes: Draw at most the first ``max_boxes`` entries (default 12). The + rest are dropped but their omission is surfaced in the title note, so + the truncation is never silent. + + Returns: + A ``matplotlib.figure.Figure`` with a single Axes holding the horizontal + boxplots (height adaptive to the box count so none overlap). The caller is + responsible for rasterizing/closing it; this function never shows nor + saves it. + """ + try: + if not isinstance(boxes, (list, tuple)) or len(boxes) == 0: + return _placeholder_figure("(sin boxplots)") + + total = len(boxes) + + # Cap the number of boxes; tolerate a non-int / non-positive max_boxes. + try: + cap = int(max_boxes) + except (TypeError, ValueError): + cap = 12 + if cap <= 0: + cap = 12 + candidates = list(boxes)[:cap] + + stats_list = [] # bxp stats records, in draw order. + labels = [] # Y tick labels (column names). + manual_markers = [] # (position, box) for entries without raw fliers. + any_fliers = False # whether to enable showfliers in the bxp call. + + for entry in candidates: + if not isinstance(entry, dict): + continue + box = entry.get("box") + if not isinstance(box, dict): + continue + + q1 = _num(box.get("q1")) + med = _num(box.get("median")) + q3 = _num(box.get("q3")) + # Without the three quartiles a boxplot cannot be drawn — skip it. + if q1 is None or med is None or q3 is None: + continue + + # Whisker extremes fall back to the quartiles when missing. + whislo = _num(box.get("whisker_lo")) + whishi = _num(box.get("whisker_hi")) + if whislo is None: + whislo = q1 + if whishi is None: + whishi = q3 + + name = entry.get("name") + label = "" if name is None else str(name) + + position = len(stats_list) + 1 # bxp positions are 1-indexed. + fliers_raw = entry.get("fliers") + if isinstance(fliers_raw, (list, tuple)): + fliers = [v for v in (_num(x) for x in fliers_raw) if v is not None] + if fliers: + any_fliers = True + else: + # Raw values unknown: draw no bxp fliers, mark min/max by hand. + fliers = [] + manual_markers.append((position, box)) + + stats_list.append({ + "med": med, + "q1": q1, + "q3": q3, + "whislo": whislo, + "whishi": whishi, + "fliers": fliers, + "label": label, + }) + labels.append(label) + + if not stats_list: + return _placeholder_figure("(sin boxplots)") + + n = len(stats_list) + positions = list(range(1, n + 1)) + + # Height grows with the box count so none of them overlap. + height = max(2.0, 0.5 * n + 1.0) + fig = Figure(figsize=(7.0, height), dpi=150) + ax = fig.add_subplot(111) + + bxp_kw = dict( + showfliers=any_fliers, widths=0.5, patch_artist=True, + boxprops={"facecolor": _BOX_FACE, "edgecolor": _BOX_EDGE}, + medianprops={"color": _MEDIAN, "linewidth": 1.6}, + whiskerprops={"color": _BOX_EDGE}, + capprops={"color": _BOX_EDGE}, + flierprops={"marker": "o", "markersize": 3.5, + "markerfacecolor": _OUTLIER, "markeredgecolor": _OUTLIER, + "linestyle": "none"}) + try: + # ``orientation`` is the current API; older matplotlib uses ``vert``. + try: + ax.bxp(stats_list, positions=positions, + orientation="horizontal", **bxp_kw) + except TypeError: + ax.bxp(stats_list, positions=positions, vert=False, **bxp_kw) + except Exception: # noqa: BLE001 — never let bxp kill the whole figure. + ax.text(0.5, 0.5, "(boxplot no disponible)", ha="center", + va="center", fontsize=10, color=_MUTED_TEXT, + transform=ax.transAxes) + + # For entries without raw fliers, mark only the out-of-fence extremes. + for position, box in manual_markers: + mn = _num(box.get("min")) + mx = _num(box.get("max")) + if box.get("has_low_outliers") and mn is not None: + ax.plot([mn], [position], marker="o", markersize=3.5, + color=_OUTLIER, zorder=5) + if box.get("has_high_outliers") and mx is not None: + ax.plot([mx], [position], marker="o", markersize=3.5, + color=_OUTLIER, zorder=5) + + # Pin the Y tick labels explicitly so they work across matplotlib + # versions regardless of whether ``bxp`` consumed the ``label`` key. + ax.set_yticks(positions) + ax.set_yticklabels(labels, fontsize=8) + ax.set_xlabel("valor", fontsize=9) + ax.tick_params(labelsize=7) + ax.margins(y=0.15) + for spine in ("top", "right"): + ax.spines[spine].set_visible(False) + + # Surface truncation in the title instead of silently dropping boxes. + note = f"(mostrando {n} de {total})" if total > cap else "" + heading = " ".join(p for p in (title, note) if p) + if heading: + fig.suptitle(heading, fontsize=12, x=0.02, ha="left") + + fig.tight_layout() + return fig + except Exception as exc: # noqa: BLE001 — never raise from a figure builder. + return _placeholder_figure( + f"error al dibujar boxplots: {exc}", color=_ERROR_TEXT) diff --git a/python/functions/datascience/build_boxplots_figure_test.py b/python/functions/datascience/build_boxplots_figure_test.py new file mode 100644 index 00000000..3cea0914 --- /dev/null +++ b/python/functions/datascience/build_boxplots_figure_test.py @@ -0,0 +1,109 @@ +"""Tests para build_boxplots_figure (boxplots horizontales de Tukey, grupo eda). + +Usa el backend Agg sin display; no muestra ni guarda figuras. Cada test cierra +explícitamente la Figure construida (matplotlib.pyplot.close) para no acumular +estado entre tests. +""" + +import matplotlib + +matplotlib.use("Agg") + +import matplotlib.pyplot as plt # noqa: E402 +from matplotlib.figure import Figure # noqa: E402 + +from build_boxplots_figure import build_boxplots_figure + + +def _box(name, q1, median, q3, mn, mx, low=False, high=False, fliers=None): + """Construye una entrada {name, box, fliers} con un box estilo build_boxplot_stats.""" + iqr = q3 - q1 + return { + "name": name, + "box": { + "q1": q1, + "median": median, + "q3": q3, + "iqr": iqr, + "lower_fence": q1 - 1.5 * iqr, + "upper_fence": q3 + 1.5 * iqr, + "whisker_lo": max(mn, q1 - 1.5 * iqr), + "whisker_hi": min(mx, q3 + 1.5 * iqr), + "min": mn, + "max": mx, + "has_low_outliers": low, + "has_high_outliers": high, + "n_outliers": 0, + }, + "fliers": fliers, + } + + +def test_returns_figure_with_axes(): + boxes = [ + _box("edad", 10.0, 25.0, 40.0, 1.0, 100.0, high=True), + _box("ingresos", 100.0, 200.0, 300.0, 50.0, 400.0), + _box("score", -1.0, 0.0, 1.0, -5.0, 5.0, low=True, high=True), + ] + fig = build_boxplots_figure(boxes, title="Boxplots", max_boxes=12) + assert isinstance(fig, Figure) + assert len(fig.axes) >= 1 + # Tres cajas -> tres etiquetas en el eje Y. + ax = fig.axes[0] + assert len(ax.get_yticks()) == 3 + plt.close(fig) + + +def test_empty_list_returns_placeholder_figure(): + fig = build_boxplots_figure([], title="vacío") + assert isinstance(fig, Figure) + assert len(fig.axes) >= 1 + plt.close(fig) + + +def test_invalid_box_is_skipped_not_raised(): + boxes = [ + {"name": "rota", "box": {"q1": None, "median": None, "q3": None}}, + {"name": "sin_box"}, # falta la clave box. + "no_es_dict", # entrada no-dict. + _box("buena", 1.0, 2.0, 3.0, 0.0, 10.0, high=True), + ] + fig = build_boxplots_figure(boxes) + assert isinstance(fig, Figure) + ax = fig.axes[0] + # Solo la caja válida sobrevive al filtrado. + assert len(ax.get_yticks()) == 1 + plt.close(fig) + + +def test_all_invalid_returns_placeholder(): + boxes = [ + {"name": "a", "box": {"q1": None, "median": 1.0, "q3": 2.0}}, + {"name": "b"}, + ] + fig = build_boxplots_figure(boxes) + assert isinstance(fig, Figure) + assert len(fig.axes) >= 1 + plt.close(fig) + + +def test_raw_fliers_are_drawn(): + boxes = [ + _box("con_fliers", 10.0, 20.0, 30.0, 5.0, 200.0, + high=True, fliers=[150.0, 180.0, 200.0]), + ] + fig = build_boxplots_figure(boxes) + assert isinstance(fig, Figure) + assert len(fig.axes) >= 1 + plt.close(fig) + + +def test_max_boxes_truncates_and_does_not_raise(): + boxes = [_box(f"c{i}", float(i), float(i + 1), float(i + 2), + float(i - 5), float(i + 10)) for i in range(20)] + fig = build_boxplots_figure(boxes, title="muchos", max_boxes=5) + assert isinstance(fig, Figure) + ax = fig.axes[0] + # Solo se dibujan las primeras 5 cajas. + assert len(ax.get_yticks()) == 5 + plt.close(fig) diff --git a/python/functions/datascience/summarize_outlier_dims.md b/python/functions/datascience/summarize_outlier_dims.md new file mode 100644 index 00000000..b9ac5d49 --- /dev/null +++ b/python/functions/datascience/summarize_outlier_dims.md @@ -0,0 +1,79 @@ +--- +name: summarize_outlier_dims +kind: function +lang: py +domain: datascience +version: "1.0.0" +purity: pure +signature: "def summarize_outlier_dims(raw_numeric: dict, outlier_rows: list, top_k: int = 3) -> list" +description: "Explica QUE columnas hacen rara cada fila anomala detectada por isolation_forest_outliers. Para cada {row_index, score} reconstruye la fila valida (mismo filtro de columnas numericas y mismo descarte de filas con None que el detector, asi row_index coincide) y devuelve las top_k columnas de mayor |z-score| poblacional (ddof=0). Capa de explicabilidad del paso de outliers multivariante en EDA. Pura y determinista; ante entradas vacias/invalidas o sin filas validas devuelve [] sin petar." +tags: [eda, models, outliers, anomaly-detection, explainability, z-score, multivariate] +params: + - name: raw_numeric + desc: "dict {nombre_columna: [valores]} alineado por fila (como ctx['raw_numeric'] del motor AutomaticEDA). Solo se usan columnas con todos los valores numericos (None permitido por fila; bool/str/NaN/Inf descartan la columna entera) — filtro IDENTICO al de isolation_forest_outliers para que row_index coincida." + - name: outlier_rows + desc: "Lista de {row_index, score} tal cual la devuelve isolation_forest_outliers. row_index cuenta SOLO las filas validas (sin None) en orden de aparicion, base 0. Entradas fuera de rango o malformadas se ignoran defensivamente." + - name: top_k + desc: "Numero de columnas (las de mayor |z-score|) a reportar por outlier. Default 3. Valores invalidos (no-int, bool, <1) caen a 3." +output: "Lista paralela a outlier_rows (mismo orden) de dicts {row_index: int, score: float, dims: [{col: str, value: float, z: float}, ...]}. dims trae hasta top_k columnas ordenadas por |z| descendente, con z (z-score poblacional, ddof=0) redondeado a 3 decimales; si una columna tiene std==0 su z es 0. Las entradas de outlier_rows fuera de rango/malformadas se omiten. Ante raw_numeric vacio/no-dict, outlier_rows no-lista, 0 columnas numericas o 0 filas validas devuelve []." +uses_functions: [] +uses_types: [] +returns: [] +returns_optional: false +error_type: "" +imports: [] +tested: true +tests: ["test_row_index_skips_none_rows", "test_extreme_row_flagged_via_isolation", "test_out_of_range_row_index_is_ignored", "test_degrades_to_empty_on_invalid_inputs"] +test_file_path: "python/functions/datascience/summarize_outlier_dims_test.py" +file_path: "python/functions/datascience/summarize_outlier_dims.py" +--- + +## Ejemplo + +```python +from datascience import isolation_forest_outliers, summarize_outlier_dims + +# Nube densa alrededor del origen + 1 fila con un valor extremo en "c". +raw_numeric = { + "a": [0.1, 0.2, -0.1, 0.0, 0.3, -0.2, 0.15, -0.05, 0.25, 0.2, -0.3, 0.1], + "b": [1.0, 1.1, 0.9, 1.2, 0.8, 1.0, 1.1, 0.95, 1.05, 0.9, 1.15, 1.0], + "c": [5.0, 5.2, 4.8, 5.1, 4.9, 5.0, 4.95, 5.05, 4.9, 500.0, 5.1, 5.0], +} + +result = isolation_forest_outliers(raw_numeric, contamination=0.1) +summary = summarize_outlier_dims(raw_numeric, result["outlier_rows"], top_k=3) + +for item in summary: + top = item["dims"][0] + print(item["row_index"], top["col"], top["value"], top["z"]) +# La fila del valor 500 sale con dim top "c" y |z| alto: es lo que la hace rara. +``` + +## Cuando usarla + +Justo **despues** de `isolation_forest_outliers`, cuando ya sabes QUE filas son +anomalas y quieres explicar POR QUE: en que columnas se desvian mas respecto al +resto. Util para rellenar la seccion de outliers de un report/notebook EDA con +"la fila 9 es rara sobre todo por `c` (z=+3.3)" en lugar de solo un row_index +opaco. Pasa el mismo `raw_numeric` que diste al detector y su `outlier_rows` +intacto; el `row_index` apunta a la misma fila porque ambas funciones aplican el +mismo filtro de columnas y el mismo descarte de filas con None. + +## Gotchas + +- **Mismo `raw_numeric` que el detector**: el `row_index` solo coincide si pasas + el mismo dict de columnas (mismo orden, mismas listas) con el que llamaste a + `isolation_forest_outliers`. Si cambias las columnas o el orden, los indices + dejan de mapear. +- **`row_index` es relativo a las filas validas**: las filas con `None` en + cualquier columna usada se descartan y los indices se recalculan sobre las que + quedan (base 0, orden de aparicion). No mapea 1:1 con las listas de entrada si + hay None. +- **z-score poblacional (ddof=0)**: se usa la desviacion tipica poblacional, + consistente con el escalado del detector. Columnas con `std==0` (todos los + valores iguales) dan `z=0`, asi que nunca aparecen como "raras". +- **Devuelve `[]` en vez de petar**: entrada no-dict/no-lista, 0 columnas + numericas, 0 filas validas, o todas las entradas fuera de rango -> lista vacia. + No lanza excepciones. +- **No llama a `isolation_forest_outliers`**: solo consume su salida. Es una + funcion independiente (no la importa), por eso `uses_functions` esta vacio. diff --git a/python/functions/datascience/summarize_outlier_dims.py b/python/functions/datascience/summarize_outlier_dims.py new file mode 100644 index 00000000..b6b2ca61 --- /dev/null +++ b/python/functions/datascience/summarize_outlier_dims.py @@ -0,0 +1,144 @@ +"""Explica que dimensiones (columnas) hacen rara cada fila anomala. + +Toma la salida multivariante de `isolation_forest_outliers` (lista de +`{row_index, score}`) y, para cada outlier, devuelve las columnas con mayor +|z-score| respecto a la distribucion de las filas validas. Es la capa de +"explicabilidad" del paso de outliers multivariante en la fase EDA: el +Isolation Forest dice QUE filas son raras, esta funcion dice POR QUE (en que +columnas se desvian mas). + +Pura y determinista: reconstruye EXACTAMENTE las mismas "filas validas" que usa +`isolation_forest_outliers` (mismo filtro de columnas numericas y mismo descarte +de filas con None), de modo que el `row_index` apunta a la misma fila en ambas +funciones. No hace I/O ni depende de estado. +""" + +import math + +import numpy as np + + +def _is_finite_number(v) -> bool: + """True si v es int/float finito. bool NO cuenta; NaN/Inf tampoco.""" + if isinstance(v, bool): + return False + if not isinstance(v, (int, float)): + return False + if isinstance(v, float) and (math.isnan(v) or math.isinf(v)): + return False + return True + + +def summarize_outlier_dims( + raw_numeric: dict, + outlier_rows: list, + top_k: int = 3, +) -> list: + """Resume las dimensiones que mas desvian a cada fila anomala. + + Args: + raw_numeric: dict {nombre_columna: [valores]} alineado por fila (como + ctx['raw_numeric'] del motor AutomaticEDA). Solo se usan columnas + cuyos valores sean todos numericos (None permitido por fila; bool, + str, NaN e Inf descartan la columna entera) — filtro identico al de + isolation_forest_outliers. + outlier_rows: lista de {row_index, score} tal como la devuelve + isolation_forest_outliers. row_index cuenta SOLO las filas validas + (sin None) en orden de aparicion, empezando en 0. + top_k: numero de columnas (las de mayor |z-score|) a reportar por cada + outlier. Default 3. Valores invalidos caen a 3. + + Returns: + Lista paralela a outlier_rows (mismo orden) de dicts + {row_index, score, dims}, donde dims es la lista de hasta top_k columnas + ordenadas por |z| descendente: [{col, value, z}, ...] con z redondeado a + 3 decimales. Las entradas de outlier_rows fuera de rango o malformadas se + omiten (defensivo). Ante raw_numeric vacio/no-dict, outlier_rows + no-lista, 0 columnas numericas o 0 filas validas devuelve []. + """ + # Validacion defensiva de los argumentos principales. + if not isinstance(raw_numeric, dict) or not isinstance(outlier_rows, list): + return [] + if not isinstance(top_k, int) or isinstance(top_k, bool) or top_k < 1: + top_k = 3 + + # Seleccion de columnas numericas: identica a isolation_forest_outliers. + # Una columna entra solo si todos sus valores son numericos (None permitido + # por fila); cualquier bool/str/NaN/Inf descarta la columna completa. + numeric_cols: dict[str, list] = {} + for name, values in raw_numeric.items(): + if not isinstance(values, (list, tuple)): + continue + ok = True + for v in values: + if v is None: + continue + if not _is_finite_number(v): + ok = False + break + if ok: + numeric_cols[name] = list(values) + + if len(numeric_cols) < 1: + return [] + + col_names = list(numeric_cols.keys()) + try: + n_rows_total = min(len(numeric_cols[c]) for c in col_names) + except ValueError: + return [] + + # Reconstruye las filas validas con el MISMO criterio que el detector: la + # fila i toma un valor por columna; si cualquier valor es None, la fila se + # descarta y NO incrementa el indice valido. Asi row_index de outlier_rows + # apunta a esta misma secuencia (base 0, orden de aparicion). + valid_rows: list[list[float]] = [] + for i in range(n_rows_total): + row = [numeric_cols[c][i] for c in col_names] + if any(v is None for v in row): + continue + valid_rows.append([float(v) for v in row]) + + if not valid_rows: + return [] + + matrix = np.asarray(valid_rows, dtype=float) + n_valid = matrix.shape[0] + means = matrix.mean(axis=0) + stds = matrix.std(axis=0, ddof=0) # poblacional (ddof=0) + + out: list = [] + for entry in outlier_rows: + if not isinstance(entry, dict): + continue + ri = entry.get("row_index") + # bool es subclase de int: lo excluimos explicitamente. + if not isinstance(ri, int) or isinstance(ri, bool): + continue + if ri < 0 or ri >= n_valid: + continue + + try: + score = float(entry.get("score")) + except (TypeError, ValueError): + score = 0.0 + + row = matrix[ri] + dims = [] + for j, name in enumerate(col_names): + std = stds[j] + if std == 0.0: + z = 0.0 + else: + z = float((row[j] - means[j]) / std) + dims.append({"col": name, "value": float(row[j]), "z": z}) + + # Mayor |z| primero; sort estable, empates por orden de columna. + dims.sort(key=lambda d: abs(d["z"]), reverse=True) + dims = dims[:top_k] + for d in dims: + d["z"] = round(d["z"], 3) + + out.append({"row_index": int(ri), "score": score, "dims": dims}) + + return out diff --git a/python/functions/datascience/summarize_outlier_dims_test.py b/python/functions/datascience/summarize_outlier_dims_test.py new file mode 100644 index 00000000..019a4ddd --- /dev/null +++ b/python/functions/datascience/summarize_outlier_dims_test.py @@ -0,0 +1,93 @@ +"""Tests para summarize_outlier_dims.""" + +from isolation_forest_outliers import isolation_forest_outliers +from summarize_outlier_dims import summarize_outlier_dims + + +# Dataset compartido: 3 columnas, 13 filas. La fila ORIGINAL 6 tiene None en "a" +# (se descarta), de modo que la fila ORIGINAL 10 -- con un valor extremo en "c" +# -- queda en el indice VALIDO 9 (no 10). Esto verifica el salto de None. +A = [0.1, 0.2, -0.1, 0.0, 0.3, -0.2, None, 0.15, -0.05, 0.25, 0.2, -0.3, 0.1] +B = [1.0, 1.1, 0.9, 1.2, 0.8, 1.0, 1.3, 1.1, 0.95, 1.05, 0.9, 1.15, 1.0] +C = [5.0, 5.2, 4.8, 5.1, 4.9, 5.0, 5.3, 4.95, 5.05, 4.9, 500.0, 5.1, 5.0] +RAW = {"a": A, "b": B, "c": C} + +# Mapa original -> valido (saltando original 6): +# orig: 0 1 2 3 4 5 7 8 9 10 11 12 +# valid: 0 1 2 3 4 5 6 7 8 9 10 11 +# => el extremo en "c" (original 10) esta en el indice valido 9. +EXTREME_VALID_INDEX = 9 + + +def test_row_index_skips_none_rows(): + # Mapeo directo (sin depender de la aleatoriedad de IsolationForest): el + # indice valido 9 debe corresponder a la fila con c == 500 -> el None de la + # fila original 6 se salto correctamente. + summary = summarize_outlier_dims( + RAW, [{"row_index": EXTREME_VALID_INDEX, "score": -0.5}], top_k=3 + ) + assert len(summary) == 1 + entry = summary[0] + assert entry["row_index"] == EXTREME_VALID_INDEX + assert entry["score"] == -0.5 + # La dimension dominante es "c", con su valor extremo y |z| alto. + top = entry["dims"][0] + assert top["col"] == "c" + assert top["value"] == 500.0 + assert abs(top["z"]) > 2.0 + # top_k respetado: como mucho 3 dims. + assert len(entry["dims"]) <= 3 + + +def test_extreme_row_flagged_via_isolation(): + # Integracion real: detectar outliers y explicarlos. + result = isolation_forest_outliers(RAW, contamination=0.1) + assert "note" not in result + outlier_rows = result["outlier_rows"] + assert outlier_rows # al menos un outlier + + summary = summarize_outlier_dims(RAW, outlier_rows, top_k=3) + # Paralela a outlier_rows (todos los indices estan en rango). + assert len(summary) == len(outlier_rows) + + by_index = {e["row_index"]: e for e in summary} + # El punto extremo debe estar entre los outliers detectados... + assert EXTREME_VALID_INDEX in by_index + # ...y su dimension top debe ser "c" (donde se desvia ~muchas sigmas). + extreme = by_index[EXTREME_VALID_INDEX] + assert extreme["dims"][0]["col"] == "c" + assert abs(extreme["dims"][0]["z"]) > 2.0 + + +def test_out_of_range_row_index_is_ignored(): + # Indices fuera de rango se omiten en lugar de petar. + summary = summarize_outlier_dims( + RAW, + [ + {"row_index": 999, "score": -1.0}, + {"row_index": -1, "score": -1.0}, + {"row_index": EXTREME_VALID_INDEX, "score": -0.5}, + ], + top_k=2, + ) + # Solo sobrevive el indice valido; los otros dos se descartan. + assert len(summary) == 1 + assert summary[0]["row_index"] == EXTREME_VALID_INDEX + assert len(summary[0]["dims"]) <= 2 + + +def test_degrades_to_empty_on_invalid_inputs(): + # raw_numeric vacio + outlier_rows vacio. + assert summarize_outlier_dims({}, [], 3) == [] + # raw_numeric no es dict. + assert summarize_outlier_dims("not a dict", [{"row_index": 0}], 3) == [] + # outlier_rows no es lista. + assert summarize_outlier_dims(RAW, "not a list", 3) == [] + # Sin columnas numericas (todas con strings) -> []. + assert summarize_outlier_dims( + {"s": ["x", "y", "z"]}, [{"row_index": 0, "score": -1.0}], 3 + ) == [] + # Entradas malformadas dentro de outlier_rows se ignoran (no petan). + assert summarize_outlier_dims( + RAW, ["nope", 42, {"no_row_index": 1}], 3 + ) == [] From ea6678ec239ec305507c413f02387e1c6e975e81 Mon Sep 17 00:00:00 2001 From: Egutierrez <egutierrez@dead.dd> Date: Tue, 30 Jun 2026 21:25:31 +0200 Subject: [PATCH 45/53] =?UTF-8?q?feat(eda):=20generadores=20de=20datasets?= =?UTF-8?q?=20sint=C3=A9ticos=20Faker=20que=20ejercitan=20el=20AutomaticED?= =?UTF-8?q?A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Añade dos funciones impuras dict-no-throw, deterministas por seed, al dominio datascience (grupo eda): - generate_synthetic_eda_table: una tabla DuckDB de 19 columnas (numéricas correlacionadas + outliers, categóricas desbalanceadas, texto largo multi-idioma es/en/fr, fecha DATE, lat/lon válidas, PII email/iban/phone/uuid, nulos con patrón MCAR/MAR co-ocurrentes). Activa 14 capítulos del motor AutomaticEDA (num_distr, cat_distr, text_distr, calidad, missingness, correlacion, relaciones, modelos, timeseries, geospatial, agregacion, glosario + portada/overview). - generate_synthetic_eda_folder: 3 CSV relacionados (customers/orders/reviews) con FK customer detectable por containment, para el EDA de carpeta multi-tabla. Determinismo via Faker.seed_instance + numpy.default_rng. Tests: 16 passed (incluye determinismo por hash, rangos lat/lon, co-nulos income/spending, mediana palabras review >=20, phone formato internacional, FK containment). Añade faker (40.27.0) a python/pyproject.toml + uv.lock. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> --- python/functions/datascience/__init__.py | 4 + .../generate_synthetic_eda_folder.md | 77 +++++ .../generate_synthetic_eda_folder.py | 177 ++++++++++ .../generate_synthetic_eda_folder_test.py | 74 +++++ .../generate_synthetic_eda_table.md | 82 +++++ .../generate_synthetic_eda_table.py | 314 ++++++++++++++++++ .../generate_synthetic_eda_table_test.py | 129 +++++++ python/pyproject.toml | 1 + python/uv.lock | 14 + 9 files changed, 872 insertions(+) create mode 100644 python/functions/datascience/generate_synthetic_eda_folder.md create mode 100644 python/functions/datascience/generate_synthetic_eda_folder.py create mode 100644 python/functions/datascience/generate_synthetic_eda_folder_test.py create mode 100644 python/functions/datascience/generate_synthetic_eda_table.md create mode 100644 python/functions/datascience/generate_synthetic_eda_table.py create mode 100644 python/functions/datascience/generate_synthetic_eda_table_test.py diff --git a/python/functions/datascience/__init__.py b/python/functions/datascience/__init__.py index a0785fe9..5a47aaf4 100644 --- a/python/functions/datascience/__init__.py +++ b/python/functions/datascience/__init__.py @@ -77,8 +77,12 @@ from .add_pdf_internal_links import add_pdf_internal_links from .suggest_intratable_fk_candidates import suggest_intratable_fk_candidates from .render_paper_pdf import render_paper_pdf from .draw_join_graph_figure import draw_join_graph_figure +from .generate_synthetic_eda_table import generate_synthetic_eda_table +from .generate_synthetic_eda_folder import generate_synthetic_eda_folder __all__ = [ + "generate_synthetic_eda_table", + "generate_synthetic_eda_folder", "render_paper_pdf", "draw_join_graph_figure", "suggest_intratable_fk_candidates", diff --git a/python/functions/datascience/generate_synthetic_eda_folder.md b/python/functions/datascience/generate_synthetic_eda_folder.md new file mode 100644 index 00000000..07de46ae --- /dev/null +++ b/python/functions/datascience/generate_synthetic_eda_folder.md @@ -0,0 +1,77 @@ +--- +name: generate_synthetic_eda_folder +kind: function +lang: py +domain: datascience +version: "1.0.0" +purity: impure +signature: "def generate_synthetic_eda_folder(out_dir: str, n_rows: int = 2000, seed: int = 42) -> dict" +description: "Genera una carpeta con 3 CSV RELACIONADOS (customers, orders, reviews) deterministas por seed (Faker + numpy) para ejercitar el motor AutomaticEDA multi-tabla / profile_database. orders.customer_id y reviews.customer_id estan contenidos al 100% en customers.customer_id (PK uuid), de modo que la deteccion FK por containment (min_inclusion=0.9) descubre ambas relaciones. customers es la tabla padre; reutiliza helpers de generate_synthetic_eda_table (texto multi-idioma, lat/lon validas, amount con outliers). Estilo dict-no-throw: nunca lanza." +tags: [eda, synthetic, faker, testing, fixture, datascience] +params: + - name: out_dir + desc: "Carpeta de salida. Se crea con mkdir -p si no existe. Recibe customers.csv, orders.csv y reviews.csv." + - name: n_rows + desc: "Numero de clientes (filas de customers). orders ~= 2*n_rows filas, reviews ~= n_rows filas. Default 2000." + - name: seed + desc: "Semilla para Faker (Faker.seed) y numpy (np.random.default_rng). Mismo seed -> CSVs identicos byte a byte. Default 42." +output: "dict dict-no-throw. En exito {status:'ok', out_dir, files:{customers,orders,reviews}, n_customers, n_orders, n_reviews, expected_relations:[{from_table,from_col,to_table,to_col}, ...], seed}. En error (sin lanzar, p.ej. n_rows<=0) {status:'error', error:str}. expected_relations declara las 2 FK orders->customers y reviews->customers (ambas por customer_id)." +uses_functions: [] +uses_types: [] +returns: [] +returns_optional: false +error_type: "error_go_core" +imports: [] +tested: true +tests: ["test_genera_ok_y_archivos", "test_determinismo_mismo_seed", "test_seeds_distintos_difieren", "test_fk_containment", "test_review_text_mediana_palabras", "test_n_rows_invalido"] +test_file_path: "python/functions/datascience/generate_synthetic_eda_folder_test.py" +file_path: "python/functions/datascience/generate_synthetic_eda_folder.py" +--- + +## Ejemplo + +```bash +# Genera /tmp/eda_folder/{customers,orders,reviews}.csv (300 customers, seed 42) +fn run generate_synthetic_eda_folder /tmp/eda_folder 300 42 +``` + +```python +import sys, os +sys.path.insert(0, os.path.join("python", "functions")) +from datascience import generate_synthetic_eda_folder + +res = generate_synthetic_eda_folder("/tmp/eda_folder", n_rows=300, seed=42) +# res["files"] -> {"customers": ".../customers.csv", "orders": ..., "reviews": ...} +# res["expected_relations"] -> orders.customer_id y reviews.customer_id -> customers.customer_id +# Luego perfila la carpeta/base con el grupo eda: +# fn run profile_database /tmp/eda_folder +``` + +## Cuando usarla + +- Cuando necesites un fixture REPRODUCIBLE multi-tabla para evaluar el EDA de carpeta/base (`profile_database`, join graph, capitulo de relaciones inter-tabla) con relaciones FK reales y detectables. +- Cuando escribas tests de la deteccion de claves foraneas por containment: orders y reviews referencian customer_id contenido al 100% en customers (inclusion 1.0 >= min_inclusion 0.9). +- Como contraparte multi-tabla de `generate_synthetic_eda_table` (que cubre el EDA de UNA tabla). + +## Gotchas + +- **Impura**: escribe 3 CSV a disco (`mkdir -p` de la carpeta). Sobrescribe los CSV existentes con el mismo nombre. +- **Requiere `faker`, `numpy` y `pandas`** en el venv. Sin `faker` devuelve `{status:'error'}` (no lanza). +- **El containment depende del orden**: customers se genera PRIMERO y orders/reviews muestrean sus `customer_id`. Si se invierte el orden, la FK deja de estar contenida y el detector no la encuentra. +- **`signup_date`/`ts` se escriben como texto ISO en el CSV** (`YYYY-MM-DD` / `YYYY-MM-DD HH:MM:SS`): es CSV, todo es texto; el profiler los promociona a datetime al leerlos. +- **Determinismo dependiente del orden de llamadas**: se siembra `Faker.seed(seed)` + `np.random.default_rng(seed)` al inicio; mismo seed -> CSVs identicos byte a byte. +- **Reutiliza helpers privados** de `generate_synthetic_eda_table` (`_make_fakers`, `_make_latlon`, `_make_reviews`, `_amount_with_outliers`): no romper esas firmas sin actualizar esta funcion. + +## Notas + +Estructura generada: + +| Archivo | PK | FK | Columnas clave | +|---|---|---|---| +| customers.csv | customer_id (uuid) | — | name, country, signup_date, latitude, longitude, email | +| orders.csv | order_id (uuid) | customer_id -> customers | amount (lognormal + outliers), category, ts | +| reviews.csv | review_id (uuid) | customer_id -> customers | review_text (multi-idioma, mediana palabras>=20), rating (1..5) | + +orders tiene ~2x filas que customers y reviews ~1x. Todos los `customer_id` de orders +y reviews estan contenidos en customers (containment ⊆), por lo que la deteccion FK por +inclusion descubre las dos relaciones declaradas en `expected_relations`. diff --git a/python/functions/datascience/generate_synthetic_eda_folder.py b/python/functions/datascience/generate_synthetic_eda_folder.py new file mode 100644 index 00000000..2cdf1dad --- /dev/null +++ b/python/functions/datascience/generate_synthetic_eda_folder.py @@ -0,0 +1,177 @@ +"""generate_synthetic_eda_folder — fixture multi-tabla relacionado para el EDA de base/carpeta. + +Funcion impura (escribe CSVs a disco) y determinista por ``seed``: crea una +carpeta con 3 CSV RELACIONADOS (customers, orders, reviews) cuyo contenido esta +disenado para que el motor AutomaticEDA multi-tabla / `profile_database` detecte +las relaciones FK por containment de valores (orders.customer_id y +reviews.customer_id contenidos al 100% en customers.customer_id, por encima del +``min_inclusion=0.9`` que usa la deteccion). + +Reutiliza los helpers de ``generate_synthetic_eda_table`` (texto multi-idioma, +lat/lon validas, amount con outliers, listas fijas de paises/categorias) para no +reimplementar logica. + +Estilo dict-no-throw del grupo `eda`: NUNCA lanza; devuelve +``{"status": "error", "error": str}`` ante cualquier fallo. +""" + +import os + +from .generate_synthetic_eda_table import ( + _CATEGORIES, + _COUNTRIES, + _amount_with_outliers, + _make_fakers, + _make_latlon, + _make_reviews, +) + + +def generate_synthetic_eda_folder(out_dir, n_rows=2000, seed=42): + """Genera una carpeta con 3 CSV relacionados (customers/orders/reviews). + + customers es la tabla padre (PK ``customer_id`` uuid unica). orders y reviews + referencian ``customer_id`` muestreandolo de customers, de modo que TODOS sus + valores estan contenidos en customers (inclusion 1.0 -> FK detectable). + + Funcion impura (escribe a disco) y determinista por ``seed``. NUNCA lanza. + + Args: + out_dir: carpeta de salida. Se crea con ``mkdir -p`` si no existe. + n_rows: numero de clientes (customers). orders ~= 2*n_rows, reviews ~= n_rows. + Default 2000. + seed: semilla para Faker y numpy. Default 42. + + Returns: + dict dict-no-throw. En exito:: + + {"status": "ok", "out_dir": ..., "files": {customers, orders, reviews}, + "n_customers": ..., "n_orders": ..., "n_reviews": ..., + "expected_relations": [{from_table, from_col, to_table, to_col}, ...], + "seed": seed} + + En error (sin lanzar):: + + {"status": "error", "error": str} + """ + try: + import numpy as np + import pandas as pd + + n = int(n_rows) + if n <= 0: + return {"status": "error", "error": f"n_rows debe ser > 0, dado {n_rows!r}"} + + os.makedirs(out_dir, exist_ok=True) + + fakers = _make_fakers(seed) + rng = np.random.default_rng(seed) + + # ---------------- customers (tabla padre) ---------------- + n_cust = n + customer_ids = [fakers["en_US"].uuid4() for _ in range(n_cust)] + names = [fakers["en_US"].name() for _ in range(n_cust)] + cust_country = rng.choice(_COUNTRIES, n_cust) + base = np.datetime64("2022-01-01") + signup_offsets = rng.integers(0, 730, n_cust) + signup_date = pd.to_datetime(base) + pd.to_timedelta(signup_offsets, unit="D") + signup_iso = [d.strftime("%Y-%m-%d") for d in signup_date] + lat, lon = _make_latlon(cust_country, rng) + cust_email = [fakers["en_US"].email() for _ in range(n_cust)] + + customers = pd.DataFrame( + { + "customer_id": customer_ids, + "name": names, + "country": cust_country, + "signup_date": signup_iso, + "latitude": lat, + "longitude": lon, + "email": cust_email, + } + ) + + # ---------------- orders (FK -> customers) ---------------- + n_orders = n_cust * 2 + order_ids = [fakers["en_US"].uuid4() for _ in range(n_orders)] + order_cust = rng.choice(customer_ids, n_orders) # subset/multiset de customers + amount = _amount_with_outliers(n_orders, rng, n_extreme=10) + order_cat = rng.choice(_CATEGORIES, n_orders) + ts_offsets = rng.integers(0, 730 * 24 * 3600, n_orders) + ts = pd.to_datetime(np.datetime64("2022-01-01T00:00:00")) + pd.to_timedelta( + ts_offsets, unit="s" + ) + ts_iso = [t.strftime("%Y-%m-%d %H:%M:%S") for t in ts] + + orders = pd.DataFrame( + { + "order_id": order_ids, + "customer_id": order_cust, + "amount": amount, + "category": order_cat, + "ts": ts_iso, + } + ) + + # ---------------- reviews (FK -> customers) ---------------- + n_reviews = n_cust + review_ids = [fakers["en_US"].uuid4() for _ in range(n_reviews)] + # Subconjunto de customers (no todos) -> containment estricto ⊆ customers. + rev_cust = rng.choice(customer_ids, n_reviews) + review_text = _make_reviews(n_reviews, rng, fakers, null_frac=0.0) + rating = rng.integers(1, 6, n_reviews) + + reviews = pd.DataFrame( + { + "review_id": review_ids, + "customer_id": rev_cust, + "review_text": review_text, + "rating": rating, + } + ) + + files = { + "customers": os.path.join(out_dir, "customers.csv"), + "orders": os.path.join(out_dir, "orders.csv"), + "reviews": os.path.join(out_dir, "reviews.csv"), + } + customers.to_csv(files["customers"], index=False) + orders.to_csv(files["orders"], index=False) + reviews.to_csv(files["reviews"], index=False) + + return { + "status": "ok", + "out_dir": out_dir, + "files": files, + "n_customers": n_cust, + "n_orders": n_orders, + "n_reviews": n_reviews, + "expected_relations": [ + { + "from_table": "orders", + "from_col": "customer_id", + "to_table": "customers", + "to_col": "customer_id", + }, + { + "from_table": "reviews", + "from_col": "customer_id", + "to_table": "customers", + "to_col": "customer_id", + }, + ], + "seed": seed, + } + except Exception as exc: # noqa: BLE001 — dict-no-throw del grupo eda. + return {"status": "error", "error": str(exc)} + + +if __name__ == "__main__": + import json + import sys + + args = sys.argv[1:] + out = args[0] if len(args) > 0 else "/tmp/synthetic_eda_folder" + rows = int(args[1]) if len(args) > 1 else 2000 + sd = int(args[2]) if len(args) > 2 else 42 + print(json.dumps(generate_synthetic_eda_folder(out, rows, sd), indent=2)) diff --git a/python/functions/datascience/generate_synthetic_eda_folder_test.py b/python/functions/datascience/generate_synthetic_eda_folder_test.py new file mode 100644 index 00000000..6b20b5ef --- /dev/null +++ b/python/functions/datascience/generate_synthetic_eda_folder_test.py @@ -0,0 +1,74 @@ +"""Tests para generate_synthetic_eda_folder.""" + +import os +import statistics + +import pandas as pd + +from datascience.generate_synthetic_eda_folder import generate_synthetic_eda_folder + + +def test_genera_ok_y_archivos(tmp_path): + out = str(tmp_path / "folder") + res = generate_synthetic_eda_folder(out, n_rows=300, seed=42) + assert res["status"] == "ok" + assert res["n_customers"] == 300 + assert res["n_orders"] == 600 + assert res["n_reviews"] == 300 + for key in ("customers", "orders", "reviews"): + assert os.path.exists(res["files"][key]) + # Relaciones esperadas declaradas. + rels = {(r["from_table"], r["to_table"]) for r in res["expected_relations"]} + assert ("orders", "customers") in rels + assert ("reviews", "customers") in rels + + +def test_determinismo_mismo_seed(tmp_path): + out1 = str(tmp_path / "f1") + out2 = str(tmp_path / "f2") + generate_synthetic_eda_folder(out1, n_rows=250, seed=11) + generate_synthetic_eda_folder(out2, n_rows=250, seed=11) + for name in ("customers.csv", "orders.csv", "reviews.csv"): + a = open(os.path.join(out1, name), "rb").read() + b = open(os.path.join(out2, name), "rb").read() + assert a == b, f"{name} difiere entre dos generaciones con el mismo seed" + + +def test_seeds_distintos_difieren(tmp_path): + out1 = str(tmp_path / "f1") + out2 = str(tmp_path / "f2") + generate_synthetic_eda_folder(out1, n_rows=250, seed=11) + generate_synthetic_eda_folder(out2, n_rows=250, seed=12) + a = open(os.path.join(out1, "customers.csv"), "rb").read() + b = open(os.path.join(out2, "customers.csv"), "rb").read() + assert a != b + + +def test_fk_containment(tmp_path): + out = str(tmp_path / "folder") + res = generate_synthetic_eda_folder(out, n_rows=300, seed=42) + customers = pd.read_csv(res["files"]["customers"]) + orders = pd.read_csv(res["files"]["orders"]) + reviews = pd.read_csv(res["files"]["reviews"]) + cust_ids = set(customers["customer_id"]) + # Todos los customer_id de orders y reviews ⊆ customers. + assert set(orders["customer_id"]) <= cust_ids + assert set(reviews["customer_id"]) <= cust_ids + # customer_id es PK unica en customers. + assert customers["customer_id"].is_unique + assert orders["order_id"].is_unique + assert reviews["review_id"].is_unique + + +def test_review_text_mediana_palabras(tmp_path): + out = str(tmp_path / "folder") + res = generate_synthetic_eda_folder(out, n_rows=300, seed=42) + reviews = pd.read_csv(res["files"]["reviews"]) + words = [len(str(t).split()) for t in reviews["review_text"].dropna()] + assert statistics.median(words) >= 20 + + +def test_n_rows_invalido(tmp_path): + out = str(tmp_path / "folder") + res = generate_synthetic_eda_folder(out, n_rows=0, seed=42) + assert res["status"] == "error" diff --git a/python/functions/datascience/generate_synthetic_eda_table.md b/python/functions/datascience/generate_synthetic_eda_table.md new file mode 100644 index 00000000..8c6f51c8 --- /dev/null +++ b/python/functions/datascience/generate_synthetic_eda_table.md @@ -0,0 +1,82 @@ +--- +name: generate_synthetic_eda_table +kind: function +lang: py +domain: datascience +version: "1.0.0" +purity: impure +signature: "def generate_synthetic_eda_table(out_db_path: str, table: str = 'synthetic', n_rows: int = 2000, seed: int = 42) -> dict" +description: "Genera una tabla DuckDB sintetica (Faker + numpy, determinista por seed) cuyo contenido esta disenado para ACTIVAR el maximo de capitulos del motor AutomaticEDA del grupo eda: numericas continuas con correlacion lineal/no-lineal, numericas con outliers, categoricas desbalanceadas, texto libre multi-idioma con duplicados, fecha para serie temporal, lat/lon validas, semanticos/PII (uuid/email/iban/phone) y nulos con patron MCAR/MAR. Fixture para evaluar el EDA de punta a punta. Estilo dict-no-throw: nunca lanza." +tags: [eda, synthetic, faker, testing, fixture, datascience] +params: + - name: out_db_path + desc: "Ruta al archivo DuckDB de salida. Se crea (o reutiliza) y la tabla se reemplaza con CREATE OR REPLACE TABLE si ya existe." + - name: table + desc: "Nombre de la tabla a crear. Se valida contra ^[A-Za-z_][A-Za-z0-9_]*$ y se cita en el DDL. Default 'synthetic'." + - name: n_rows + desc: "Numero de filas (clientes unicos). Cada fila es un cliente con id/email/iban/phone propios. Default 2000." + - name: seed + desc: "Semilla para Faker (Faker.seed) y numpy (np.random.default_rng). Mismo seed -> tabla identica byte a byte. Default 42." +output: "dict dict-no-throw. En exito {status:'ok', db_path, table, n_rows, columns:[19 nombres de columna], seed}. En error (sin lanzar, p.ej. nombre de tabla invalido o n_rows<=0) {status:'error', error:str}. Columnas: customer_id,email,iban,phone,income,spending,age,risk_score,tenure_months,engagement_quad,amount,n_purchases,country,category,plan,review,signup_date,latitude,longitude." +uses_functions: [] +uses_types: [] +returns: [] +returns_optional: false +error_type: "error_go_core" +imports: [] +tested: true +tests: ["test_genera_ok_y_columnas", "test_determinismo_mismo_seed", "test_seeds_distintos_difieren", "test_latlon_en_rango", "test_plan_solo_niveles_validos", "test_income_spending_co_nulos", "test_review_mediana_palabras_y_signup_datetime", "test_phone_matchea_regex_internacional", "test_outliers_y_correlaciones", "test_tabla_invalida_devuelve_error"] +test_file_path: "python/functions/datascience/generate_synthetic_eda_table_test.py" +file_path: "python/functions/datascience/generate_synthetic_eda_table.py" +--- + +## Ejemplo + +```bash +# Genera /tmp/x.duckdb con la tabla `synthetic` (2000 filas, seed 42) +fn run generate_synthetic_eda_table /tmp/x.duckdb synthetic 2000 42 +``` + +```python +import sys, os +sys.path.insert(0, os.path.join("python", "functions")) +from datascience import generate_synthetic_eda_table + +res = generate_synthetic_eda_table("/tmp/x.duckdb", "synthetic", n_rows=2000, seed=42) +# res == {"status":"ok", "db_path":"/tmp/x.duckdb", "table":"synthetic", +# "n_rows":2000, "columns":[...19...], "seed":42} +# Luego perfilala con el grupo eda: +# fn run profile_table /tmp/x.duckdb synthetic +``` + +## Cuando usarla + +- Cuando necesites un dataset de prueba REPRODUCIBLE para evaluar el motor AutomaticEDA de punta a punta: su contenido dispara, a proposito, num_distr, cat_distr, text_distr, correlacion, missingness (MCAR/MAR), modelos (PCA/KMeans/outliers), timeseries, geospatial, calidad, agregacion y los detectores semanticos / PII (`infer_semantic_type`). +- Cuando escribas tests de capitulos del EDA y quieras una tabla con una columna que active CADA detector sin montar datos a mano. +- Cuando quieras un fixture determinista (mismo seed -> misma tabla) para comparar el render del EDA entre versiones. + +## Gotchas + +- **Impura**: escribe a disco (crea/reutiliza el archivo DuckDB). Reemplaza la tabla destino con `CREATE OR REPLACE`. +- **Requiere `faker`, `duckdb`, `numpy` y `pandas`** instalados en el venv. Sin `faker` la generacion devuelve `{status:'error'}` (no lanza). +- **`signup_date` queda como TIMESTAMP/DATE en DuckDB** (se construye con `datetime64[ns]`), NO VARCHAR — condicion para que `detect_time_column` la elija y se active el capitulo timeseries. Si fuese VARCHAR, el detector de fecha fallaria. +- **El texto de `review` debe superar el gate de text_distr**: media de caracteres >= 50 y mediana de palabras >= 20. Por eso cada review concatena dos parrafos Faker (~50 palabras de mediana); no reducir el numero de frases o el capitulo text_distr no activa. +- **Determinismo dependiente del orden de llamadas**: se siembra `Faker.seed(seed)` + `np.random.default_rng(seed)` al inicio; cambiar el orden de las extracciones cambia la salida aunque el seed sea el mismo. +- **PII real-istica**: `email`/`iban`/`phone`/`customer_id` matchean los regex de `infer_semantic_type` (email/iban/phone_intl/uuid) al 100%; son datos sinteticos de Faker, no personas reales. + +## Notas + +Mapa columna -> detector que activa: + +| Columna(s) | Tipo | Detector / capitulo | +|---|---|---| +| income, spending | num continua | correlacion POSITIVA fuerte (Pearson > 0.8) | +| age, risk_score | num continua | correlacion NEGATIVA | +| tenure_months, engagement_quad | num continua | relacion NO LINEAL (cuadratica) | +| amount, n_purchases | num + outliers | num_distr / outliers (cola pesada + extremos inyectados) | +| country (12), category (6), plan (3 desbalanceado) | categorica | cat_distr / agregacion (entropia baja en plan) | +| review | texto libre multi-idioma | text_distr (len_mean>=50, mediana palabras>=20) + duplicados exactos | +| signup_date | DATE/TIMESTAMP | timeseries | +| latitude, longitude | num [-90,90]/[-180,180] | geospatial (detect_latlon_columns) | +| customer_id, email, iban, phone | texto | semantic_type uuid/email/iban/phone_intl (PII) | +| income+spending (co-nulos 12%), risk_score (nulo si plan=alta), review (8%) | nulos con patron | missingness MCAR/MAR | diff --git a/python/functions/datascience/generate_synthetic_eda_table.py b/python/functions/datascience/generate_synthetic_eda_table.py new file mode 100644 index 00000000..2f4c155f --- /dev/null +++ b/python/functions/datascience/generate_synthetic_eda_table.py @@ -0,0 +1,314 @@ +"""generate_synthetic_eda_table — fixture sintetico para ejercitar el motor AutomaticEDA. + +Funcion impura (escribe un archivo DuckDB a disco) y determinista por ``seed``: +construye una unica tabla cuyo CONTENIDO esta disenado para ACTIVAR el maximo +numero de capitulos del motor AutomaticEDA del grupo `eda` (num_distr, cat_distr, +text_distr, correlacion, missingness, modelos, timeseries, geospatial, relaciones, +calidad, agregacion) y los detectores semanticos / PII (`infer_semantic_type`). + +Estilo dict-no-throw del grupo `eda`: NUNCA lanza; captura cualquier error y +devuelve ``{"status": "error", "error": str}``. + +Determinismo: con el mismo ``seed`` el DataFrame y, por tanto, la tabla DuckDB +resultante son identicos byte a byte. Se siembra Faker (``Faker.seed``) y numpy +(``np.random.default_rng(seed)``) al inicio de cada generacion. +""" + +import re + +# Lista fija de paises (12 -> cardinalidad media para cat_distr / agregacion). +_COUNTRIES = [ + "ES", "FR", "DE", "IT", "PT", "NL", + "BE", "US", "GB", "IE", "SE", "PL", +] + +# Lista fija de categorias de producto (6 -> cardinalidad media). +_CATEGORIES = [ + "electronics", "clothing", "home", "sports", "books", "toys", +] + +# Niveles de plan con probabilidades DESBALANCEADAS (entropia baja para cat_distr). +_PLANS = ["baja", "media", "alta"] +_PLAN_PROBS = [0.70, 0.25, 0.05] + +# Centroides (lat, lon) aproximados por pais: muestrean coordenadas validas +# dentro de [-90, 90] x [-180, 180] para que detect_latlon_columns las acepte. +_CENTROIDS = { + "ES": (40.4, -3.7), "FR": (46.6, 2.2), "DE": (51.1, 10.4), "IT": (41.9, 12.5), + "PT": (39.4, -8.2), "NL": (52.1, 5.3), "BE": (50.5, 4.5), "US": (39.0, -98.0), + "GB": (54.0, -2.0), "IE": (53.4, -8.0), "SE": (60.1, 18.6), "PL": (52.0, 19.1), +} + +# Locales rotados para generar texto multi-idioma (es/en/fr). +_TEXT_LOCALES = ["es_ES", "en_US", "fr_FR"] + +# Identificador SQL valido (DuckDB no parametriza el nombre de tabla en DDL). +_IDENT_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") + + +def _make_fakers(seed): + """Crea los Faker por locale tras sembrar el generador compartido. + + ``Faker.seed(seed)`` siembra el ``random.Random`` compartido por todas las + instancias Faker que usan el generador por defecto, asi que el orden de + llamadas determina por completo la salida (determinismo). + """ + from faker import Faker + + Faker.seed(seed) + es_es, en_us, fr_fr = (Faker(loc) for loc in _TEXT_LOCALES) + return {"es_ES": es_es, "en_US": en_us, "fr_FR": fr_fr} + + +# Texto duplicado canonico (multi-idioma, > 20 palabras) que se inyecta en una +# fraccion de las filas para que el analisis de duplicados exactos lo detecte. +_DUP_REVIEW = ( + "Servicio excelente y entrega muy rapida, el producto llego en perfecto " + "estado y coincide con la descripcion publicada en la tienda. The customer " + "support team answered every question quickly and the packaging was solid " + "and well protected during shipping. Je recommande vivement ce vendeur a " + "tous mes amis, la qualite est vraiment au rendez-vous cette fois." +) + + +def _make_reviews(n, rng, fakers, dup_frac=0.04, null_frac=0.08): + """Genera ``n`` reviews de texto libre largo multi-idioma (es/en/fr). + + Cada review concatena dos parrafos de Faker en el idioma rotado por fila, de + modo que la MEDIANA de palabras por documento queda muy por encima de 20 y la + media de caracteres por encima de 50 (gates del capitulo text_distr). Se + inyectan duplicados exactos (``dup_frac``) y nulos (``null_frac``). + + Devuelve una ``list`` de ``str`` o ``None`` (nulos) de longitud ``n``. + """ + # Numero de frases por parrafo precomputado con numpy (determinista) para no + # interleavar draws de rng dentro del bucle de faker. + nb1 = rng.integers(4, 8, n) + nb2 = rng.integers(3, 7, n) + + reviews = [] + for i in range(n): + fk = fakers[_TEXT_LOCALES[i % 3]] + p1 = fk.paragraph(nb_sentences=int(nb1[i])) + p2 = fk.paragraph(nb_sentences=int(nb2[i])) + reviews.append(f"{p1} {p2}") + + # Duplicados exactos: una fraccion de filas comparte un review identico. + if n > 0 and dup_frac > 0: + k_dup = max(1, int(n * dup_frac)) + dup_idx = rng.choice(n, size=min(k_dup, n), replace=False) + for j in dup_idx: + reviews[int(j)] = _DUP_REVIEW + + # Nulos MCAR-ish: una fraccion de filas al azar queda en None. + if n > 0 and null_frac > 0: + k_null = max(1, int(n * null_frac)) + null_idx = rng.choice(n, size=min(k_null, n), replace=False) + for j in null_idx: + reviews[int(j)] = None + + return reviews + + +def _make_phone_intl(rng): + """Construye un telefono en formato internacional que casa phone_intl. + + Regex objetivo (fullmatch): ``\\+\\d[\\d\\s()-]{6,}\\d``. Empieza por '+', + digito, bloques de digitos separados por espacios y termina en digito. + """ + cc = int(rng.integers(1, 99)) + a = int(rng.integers(100, 999)) + b = int(rng.integers(100, 999)) + c = int(rng.integers(100, 999)) + return f"+{cc} {a} {b} {c}" + + +def _make_latlon(countries, rng): + """Devuelve (latitudes, longitudes) muestreando centroides de pais + jitter. + + Mantiene los valores dentro de [-90, 90] y [-180, 180] (validez exigida por + detect_latlon_columns). El jitter es pequeno para no salirse del rango. + """ + import numpy as np + + lats = np.empty(len(countries), dtype=float) + lons = np.empty(len(countries), dtype=float) + jitter_lat = rng.normal(0.0, 0.5, len(countries)) + jitter_lon = rng.normal(0.0, 0.5, len(countries)) + for i, code in enumerate(countries): + base_lat, base_lon = _CENTROIDS[code] + lats[i] = float(np.clip(base_lat + jitter_lat[i], -90.0, 90.0)) + lons[i] = float(np.clip(base_lon + jitter_lon[i], -180.0, 180.0)) + return lats, lons + + +def _amount_with_outliers(n, rng, n_extreme=6, factor=50.0): + """Serie lognormal de cola pesada con ~``n_extreme`` outliers altos (x``factor``).""" + import numpy as np + + amount = rng.lognormal(mean=4.0, sigma=1.0, size=n) + if n > 0 and n_extreme > 0: + idx = rng.choice(n, size=min(n_extreme, n), replace=False) + amount[idx] = amount[idx] * factor + return amount + + +def generate_synthetic_eda_table( + out_db_path, table="synthetic", n_rows=2000, seed=42 +): + """Genera una tabla DuckDB sintetica que activa el maximo de capitulos del EDA. + + Construye un DataFrame de ``n_rows`` clientes unicos con columnas elegidas para + disparar detectores concretos del motor AutomaticEDA (numericas continuas con + correlaciones lineal/no-lineal, numericas con outliers, categoricas + desbalanceadas, texto libre multi-idioma con duplicados, fecha para serie + temporal, lat/lon validas, semanticos/PII y nulos con patron MCAR/MAR), y la + materializa en ``out_db_path`` con ``CREATE OR REPLACE TABLE``. + + Funcion impura (escribe a disco) y determinista por ``seed``: con el mismo + seed la tabla resultante es identica byte a byte. NUNCA lanza. + + Args: + out_db_path: ruta al archivo DuckDB de salida. Se crea (o reutiliza) y la + tabla se reemplaza si ya existe. + table: nombre de la tabla a crear. Se valida contra + ``^[A-Za-z_][A-Za-z0-9_]*$`` y se cita en el DDL. + n_rows: numero de filas (clientes unicos). Default 2000. + seed: semilla para Faker y numpy. Default 42. + + Returns: + dict dict-no-throw. En exito:: + + {"status": "ok", "db_path": out_db_path, "table": table, + "n_rows": n_rows, "columns": [<nombres de columna>], "seed": seed} + + En error (sin lanzar):: + + {"status": "error", "error": str} + """ + try: + import duckdb + import numpy as np + import pandas as pd + + if not _IDENT_RE.match(table or ""): + return { + "status": "error", + "error": ( + f"nombre de tabla invalido: {table!r} " + "(debe casar con ^[A-Za-z_][A-Za-z0-9_]*$)" + ), + } + n = int(n_rows) + if n <= 0: + return {"status": "error", "error": f"n_rows debe ser > 0, dado {n_rows!r}"} + + fakers = _make_fakers(seed) + rng = np.random.default_rng(seed) + + # --- Numericas continuas (distinct alto, correlaciones) --- + income = np.clip(rng.normal(40000.0, 12000.0, n), 1000.0, None) + spending = income * 0.35 + rng.normal(0.0, 2000.0, n) # corr POSITIVA fuerte + age = rng.integers(18, 91, n) + risk_score = 90.0 - age * 0.7 + rng.normal(0.0, 5.0, n) # corr NEGATIVA con age + tenure_months = rng.uniform(0.0, 60.0, n) + engagement_quad = ((tenure_months - 30.0) ** 2) / 30.0 + rng.normal(0.0, 1.0, n) + + # --- Numericas con outliers claros --- + amount = _amount_with_outliers(n, rng) + n_purchases = rng.poisson(3.0, n).astype(float) + if n > 0: + k_hi = min(max(1, int(n * 0.002)) + 2, n) # ~3-5 valores altisimos + hi_idx = rng.choice(n, size=k_hi, replace=False) + n_purchases[hi_idx] = rng.integers(200, 400, len(hi_idx)).astype(float) + + # --- Categoricas --- + country = rng.choice(_COUNTRIES, n) + category = rng.choice(_CATEGORIES, n) + plan = rng.choice(_PLANS, n, p=_PLAN_PROBS) + + # --- Texto libre multi-idioma con duplicados --- + review = _make_reviews(n, rng, fakers) + + # --- Fecha / serie temporal (rango ~2 anios, cadencia ~diaria) --- + base = np.datetime64("2022-01-01") + offsets = rng.integers(0, 730, n) + signup_date = pd.to_datetime(base) + pd.to_timedelta(offsets, unit="D") + + # --- Geo lat/lon validas --- + latitude, longitude = _make_latlon(country, rng) + + # --- Semanticos / PII (>=80% match para infer_semantic_type) --- + customer_id = [fakers["en_US"].uuid4() for _ in range(n)] + email = [fakers["en_US"].email() for _ in range(n)] + iban = [fakers["en_US"].iban() for _ in range(n)] + phone = [_make_phone_intl(rng) for _ in range(n)] + + df = pd.DataFrame( + { + "customer_id": customer_id, + "email": email, + "iban": iban, + "phone": phone, + "income": income, + "spending": spending, + "age": age, + "risk_score": risk_score, + "tenure_months": tenure_months, + "engagement_quad": engagement_quad, + "amount": amount, + "n_purchases": n_purchases, + "country": country, + "category": category, + "plan": plan, + "review": review, + "signup_date": signup_date, + "latitude": latitude, + "longitude": longitude, + } + ) + + # --- Nulos con patron --- + # income + spending faltan JUNTAS en las MISMAS filas (co-ocurrencia -> MAR). + k_co = max(1, int(n * 0.12)) + co_idx = rng.choice(n, size=min(k_co, n), replace=False) + df.loc[co_idx, "income"] = np.nan + df.loc[co_idx, "spending"] = np.nan + # risk_score falta cuando plan == "alta" (mas una pizca de azar) -> MAR. + risk_mask = (df["plan"] == "alta").to_numpy() | (rng.random(n) < 0.02) + df.loc[risk_mask, "risk_score"] = np.nan + + columns = list(df.columns) + + con = duckdb.connect(out_db_path) + try: + con.register("df_synth_eda", df) + con.execute( + f'CREATE OR REPLACE TABLE "{table}" AS SELECT * FROM df_synth_eda' + ) + con.unregister("df_synth_eda") + finally: + con.close() + + return { + "status": "ok", + "db_path": out_db_path, + "table": table, + "n_rows": n, + "columns": columns, + "seed": seed, + } + except Exception as exc: # noqa: BLE001 — dict-no-throw del grupo eda. + return {"status": "error", "error": str(exc)} + + +if __name__ == "__main__": + import json + import sys + + args = sys.argv[1:] + db_path = args[0] if len(args) > 0 else "/tmp/synthetic_eda.duckdb" + tbl = args[1] if len(args) > 1 else "synthetic" + rows = int(args[2]) if len(args) > 2 else 2000 + sd = int(args[3]) if len(args) > 3 else 42 + print(json.dumps(generate_synthetic_eda_table(db_path, tbl, rows, sd), indent=2)) diff --git a/python/functions/datascience/generate_synthetic_eda_table_test.py b/python/functions/datascience/generate_synthetic_eda_table_test.py new file mode 100644 index 00000000..a152ad06 --- /dev/null +++ b/python/functions/datascience/generate_synthetic_eda_table_test.py @@ -0,0 +1,129 @@ +"""Tests para generate_synthetic_eda_table.""" + +import os +import re +import statistics + +import duckdb + +from datascience.generate_synthetic_eda_table import generate_synthetic_eda_table + +_EXPECTED_COLS = [ + "customer_id", "email", "iban", "phone", "income", "spending", "age", + "risk_score", "tenure_months", "engagement_quad", "amount", "n_purchases", + "country", "category", "plan", "review", "signup_date", "latitude", "longitude", +] +_PHONE_RE = re.compile(r"\+\d[\d\s()-]{6,}\d") + + +def _load(db_path, table="synthetic"): + con = duckdb.connect(db_path, read_only=True) + try: + return con.execute(f'SELECT * FROM "{table}"').fetch_df() + finally: + con.close() + + +def test_genera_ok_y_columnas(tmp_path): + db = str(tmp_path / "t.duckdb") + res = generate_synthetic_eda_table(db, "synthetic", n_rows=500, seed=42) + assert res["status"] == "ok" + assert res["table"] == "synthetic" + assert res["n_rows"] == 500 + assert res["columns"] == _EXPECTED_COLS + assert os.path.exists(db) + df = _load(db) + assert list(df.columns) == _EXPECTED_COLS + assert len(df) == 500 + + +def test_determinismo_mismo_seed(tmp_path): + db1 = str(tmp_path / "a.duckdb") + db2 = str(tmp_path / "b.duckdb") + generate_synthetic_eda_table(db1, "synthetic", n_rows=400, seed=7) + generate_synthetic_eda_table(db2, "synthetic", n_rows=400, seed=7) + df1 = _load(db1).astype(str) + df2 = _load(db2).astype(str) + # Misma semilla -> tabla identica fila a fila. + assert df1.equals(df2) + + +def test_seeds_distintos_difieren(tmp_path): + db1 = str(tmp_path / "a.duckdb") + db2 = str(tmp_path / "b.duckdb") + generate_synthetic_eda_table(db1, "synthetic", n_rows=400, seed=7) + generate_synthetic_eda_table(db2, "synthetic", n_rows=400, seed=8) + df1 = _load(db1).astype(str) + df2 = _load(db2).astype(str) + assert not df1.equals(df2) + + +def test_latlon_en_rango(tmp_path): + db = str(tmp_path / "t.duckdb") + generate_synthetic_eda_table(db, "synthetic", n_rows=500, seed=42) + df = _load(db) + assert df["latitude"].between(-90, 90).all() + assert df["longitude"].between(-180, 180).all() + + +def test_plan_solo_niveles_validos(tmp_path): + db = str(tmp_path / "t.duckdb") + generate_synthetic_eda_table(db, "synthetic", n_rows=500, seed=42) + df = _load(db) + assert set(df["plan"].unique()) <= {"baja", "media", "alta"} + + +def test_income_spending_co_nulos(tmp_path): + db = str(tmp_path / "t.duckdb") + generate_synthetic_eda_table(db, "synthetic", n_rows=600, seed=42) + df = _load(db) + inc_null = df["income"].isna() + sp_null = df["spending"].isna() + # income y spending faltan exactamente en las MISMAS filas. + assert (inc_null == sp_null).all() + assert inc_null.sum() > 0 + + +def test_review_mediana_palabras_y_signup_datetime(tmp_path): + db = str(tmp_path / "t.duckdb") + generate_synthetic_eda_table(db, "synthetic", n_rows=500, seed=42) + df = _load(db) + words = [len(str(r).split()) for r in df["review"].dropna()] + assert statistics.median(words) >= 20 + # signup_date debe ser datetime/date en DuckDB (no VARCHAR). + con = duckdb.connect(db, read_only=True) + try: + dtype = con.execute( + "SELECT column_type FROM (DESCRIBE synthetic) WHERE column_name='signup_date'" + ).fetchone()[0] + finally: + con.close() + assert dtype.upper().startswith(("DATE", "TIMESTAMP")) + + +def test_phone_matchea_regex_internacional(tmp_path): + db = str(tmp_path / "t.duckdb") + generate_synthetic_eda_table(db, "synthetic", n_rows=500, seed=42) + df = _load(db) + phones = [p for p in df["phone"].tolist() if p is not None] + assert all(_PHONE_RE.fullmatch(str(p)) for p in phones) + + +def test_outliers_y_correlaciones(tmp_path): + db = str(tmp_path / "t.duckdb") + generate_synthetic_eda_table(db, "synthetic", n_rows=800, seed=42) + df = _load(db) + # amount tiene cola con outliers altos evidentes. + assert df["amount"].max() > df["amount"].median() * 20 + # correlacion positiva fuerte income~spending y negativa age~risk_score. + sub = df[["income", "spending"]].dropna() + assert sub["income"].corr(sub["spending"]) > 0.8 + sub2 = df[["age", "risk_score"]].dropna() + assert sub2["age"].corr(sub2["risk_score"]) < -0.6 + + +def test_tabla_invalida_devuelve_error(tmp_path): + db = str(tmp_path / "t.duckdb") + res = generate_synthetic_eda_table(db, "bad name;", n_rows=10, seed=42) + assert res["status"] == "error" + assert "invalido" in res["error"] diff --git a/python/pyproject.toml b/python/pyproject.toml index f0fed9a1..38f8e631 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -9,6 +9,7 @@ dependencies = [ "contextily>=1.7.0", "cryptography>=46.0.6", "duckdb>=1.5.2", + "faker>=40.27.0", "fpdf2>=2.8.7", "geopandas>=1.1.3", "google-api-python-client>=2.197.0", diff --git a/python/uv.lock b/python/uv.lock index be3188f3..25968ab5 100644 --- a/python/uv.lock +++ b/python/uv.lock @@ -839,6 +839,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c1/ea/53f2148663b321f21b5a606bd5f191517cf40b7072c0497d3c92c4a13b1e/executing-2.2.1-py2.py3-none-any.whl", hash = "sha256:760643d3452b4d777d295bb167ccc74c64a81df23fb5e08eff250c425a4b2017", size = 28317, upload-time = "2025-09-01T09:48:08.5Z" }, ] +[[package]] +name = "faker" +version = "40.27.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "tzdata", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1a/7b/c62c98764137c949be240ad83f763b6f96cf76055952a3e2835359acc3af/faker-40.27.0.tar.gz", hash = "sha256:f697cf07f461474ad7d511164c21f45317e69f1d531d25f3e0f872b639e346a1", size = 2018361, upload-time = "2026-06-30T18:05:17.775Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c6/b2/788aae329da3d7e4f08f8e1a82e82243c3376c0f3f49b75ae29eea40b371/faker-40.27.0-py3-none-any.whl", hash = "sha256:6099bd6d7bc79041b46c28e100815e2558952bcf384b76ce6c71c8bdca744256", size = 2057897, upload-time = "2026-06-30T18:05:15.555Z" }, +] + [[package]] name = "fastapi" version = "0.136.3" @@ -890,6 +902,7 @@ dependencies = [ { name = "contextily" }, { name = "cryptography" }, { name = "duckdb" }, + { name = "faker" }, { name = "fpdf2" }, { name = "geopandas" }, { name = "google-api-python-client" }, @@ -949,6 +962,7 @@ requires-dist = [ { name = "contextily", specifier = ">=1.7.0" }, { name = "cryptography", specifier = ">=46.0.6" }, { name = "duckdb", specifier = ">=1.5.2" }, + { name = "faker", specifier = ">=40.27.0" }, { name = "fpdf2", specifier = ">=2.8.7" }, { name = "geopandas", specifier = ">=1.1.3" }, { name = "gliner", marker = "extra == 'nlp'", specifier = ">=0.2.13" }, From 54a9ab70c78a1a1185b1f8997323ce8e6253ed8f Mon Sep 17 00:00:00 2001 From: Egutierrez <egutierrez@dead.dd> Date: Tue, 30 Jun 2026 21:35:46 +0200 Subject: [PATCH 46/53] =?UTF-8?q?feat(eda):=20render=20AutomaticEDA=20por?= =?UTF-8?q?=20cap=C3=ADtulos=20sueltos=20con=20resoluci=C3=B3n=20de=20depe?= =?UTF-8?q?ndencias?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Permite renderizar un SUBCONJUNTO de capítulos del informe AutomaticEDA (only_chapters=[...]) para iterar/testear un capítulo concreto sin generar el documento entero, garantizando que el capítulo pedido SIEMPRE llegue poblado. - Nuevo módulo automatic_eda/chapter_deps.py: mapa central CHAPTER_DEPS (fuente de verdad) que declara, por capítulo de CHAPTER_ORDER, qué flags de cómputo (run_models/run_series/run_llm) y qué piezas de ctx (raw_numeric, timeseries_raw, geo_points, head_rows, db_path/table) necesita para no salir degradado. Helpers puros: resolve_requirements, resolve_profile_flags, needs_render_ctx, resolve_ctx_data_keys, validate_chapter_ids. - build_document(profile, ctx, only=None): parámetro only opcional que restringe el cuerpo a esos capítulos (portada primera + glosario última siempre). Lee la clave reservada ctx['_only_chapters'] cuando only es None, para propagar la selección a través de los renderers sin modificarlos. Retrocompatible. - render_automatic_eda(..., only_chapters=None): valida los ids (error claro dict-no-throw), resuelve las dependencias activando el cómputo necesario aunque el caller no lo pidiera (un flag explícito siempre prima) y construyendo solo las piezas de ctx que los capítulos pedidos leen (salta build_eda_render_ctx entero si ninguno necesita datos crudos). only_chapters=None produce el documento completo idéntico al de hoy. - Tests: chapter_deps_test.py (resolución pura), build_document_only_test.py (filtro), render_automatic_eda_only_test.py (golden con DuckDB: outliers suelto con IsolationForest poblado por resolución; timeseries activa run_series; eficiencia geospatial sin modelos; edge cases). - .md del pipeline: documenta only_chapters + emit_md; version 1.1.0 -> 1.2.0. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> --- .../automatic_eda/build_document_only_test.py | 109 ++++++++ .../datascience/automatic_eda/chapter_deps.py | 205 +++++++++++++++ .../automatic_eda/chapter_deps_test.py | 160 ++++++++++++ .../automatic_eda/chapters_registry.py | 37 ++- .../pipelines/render_automatic_eda.md | 53 +++- .../pipelines/render_automatic_eda.py | 106 +++++++- .../render_automatic_eda_only_test.py | 235 ++++++++++++++++++ 7 files changed, 893 insertions(+), 12 deletions(-) create mode 100644 python/functions/datascience/automatic_eda/build_document_only_test.py create mode 100644 python/functions/datascience/automatic_eda/chapter_deps.py create mode 100644 python/functions/datascience/automatic_eda/chapter_deps_test.py create mode 100644 python/functions/pipelines/render_automatic_eda_only_test.py diff --git a/python/functions/datascience/automatic_eda/build_document_only_test.py b/python/functions/datascience/automatic_eda/build_document_only_test.py new file mode 100644 index 00000000..db0e13e7 --- /dev/null +++ b/python/functions/datascience/automatic_eda/build_document_only_test.py @@ -0,0 +1,109 @@ +"""Tests del filtro `only` de build_document (selección de capítulos). + +Verifican que: + - only=None mantiene el comportamiento histórico (todos los capítulos). + - only=[ids] restringe el CUERPO a esos ids, pero portada (primera) y glosario + (última) están SIEMPRE presentes. + - only=[] produce el documento mínimo (solo portada + glosario). + - la selección también viaja por la clave reservada ctx['_only_chapters'] + (el canal que usan los renderers, que llaman build_document sin `only`), y + esa clave nunca se filtra a los capítulos. +""" + +import os +import sys + +_HERE = os.path.dirname(os.path.abspath(__file__)) +_FUNCTIONS = os.path.abspath(os.path.join(_HERE, "..", "..", "..")) # python/functions +if _FUNCTIONS not in sys.path: + sys.path.insert(0, _FUNCTIONS) + +from datascience.automatic_eda import build_document # noqa: E402 + + +def _profile_with_cat_and_num(): + """Perfil mínimo que hace construir cat_distr y num_distr (cuerpo no vacío).""" + return { + "table": "ventas", "n_rows": 120, "n_cols": 2, "quality_score": 91, + "duplicate_pct": 1.5, "null_cell_pct": 0.8, + "columns": [ + {"name": "region", "inferred_type": "categorical", + "categorical": { + "top": [{"value": "norte", "count": 50, "pct": 0.42}, + {"value": "sur", "count": 40, "pct": 0.33}, + {"value": "este", "count": 30, "pct": 0.25}], + "mode": "norte", "n_distinct": 3, "entropy": 1.55, + "imbalance": 0.1}}, + {"name": "importe", "inferred_type": "numeric", + "numeric": {"mean": 50.0, "median": 48.0, "std": 10.0, + "min": 10, "max": 99, "iqr": 15, + "histogram": [{"lo": 0, "hi": 50, "count": 40}, + {"lo": 50, "hi": 100, "count": 80}]}}, + ], + } + + +def test_only_none_is_full_document(): + """Retro-compat: sin `only`, salen todos los capítulos aplicables.""" + chs = build_document(_profile_with_cat_and_num(), ctx={"dataset_name": "v"}) + ids = [c.id for c in chs] + assert ids[0] == "portada" + assert ids[-1] == "glosario" + # El cuerpo trae las distribuciones (cat/num), no solo portada+glosario. + assert "num_distr" in ids + assert "cat_distr" in ids + + +def test_only_restricts_body_but_keeps_cover_and_glossary(): + # cat_distr registra el término "entropía" en el glosario, así que el + # glosario (destino del término clicable) aparece — demuestra el contrato + # "portada primera + capítulo + glosario última". + chs = build_document(_profile_with_cat_and_num(), + ctx={"dataset_name": "v"}, only=["cat_distr"]) + ids = [c.id for c in chs] + assert ids[0] == "portada", f"portada no es la primera: {ids}" + assert ids[-1] == "glosario", f"glosario no es la última: {ids}" + assert "cat_distr" in ids + # num_distr quedó fuera de la selección. + assert "num_distr" not in ids + + +def test_only_empty_yields_minimal_document(): + # only=[] -> cuerpo vacío. La portada está siempre; el glosario solo aparece + # si algún capítulo registró términos (patrón preexistente: glosario vacío se + # omite). Sin cuerpo no hay términos → documento mínimo = solo portada. + chs = build_document(_profile_with_cat_and_num(), + ctx={"dataset_name": "v"}, only=[]) + ids = [c.id for c in chs] + assert ids == ["portada"], \ + f"only=[] debe dar el documento mínimo (solo portada), no {ids}" + + +def test_selection_via_reserved_ctx_key(): + """La selección viaja por ctx['_only_chapters'] cuando no se pasa `only`.""" + chs = build_document(_profile_with_cat_and_num(), + ctx={"dataset_name": "v", + "_only_chapters": ["cat_distr"]}) + ids = [c.id for c in chs] + assert "cat_distr" in ids + assert "num_distr" not in ids + assert ids[0] == "portada" and ids[-1] == "glosario" + + +def test_explicit_only_arg_wins_over_ctx_key(): + """Si se pasan ambos, el argumento `only` manda sobre la clave del ctx.""" + chs = build_document(_profile_with_cat_and_num(), + ctx={"dataset_name": "v", + "_only_chapters": ["cat_distr"]}, + only=["num_distr"]) + ids = [c.id for c in chs] + assert "num_distr" in ids + assert "cat_distr" not in ids + + +def test_reserved_key_not_leaked_to_caller_ctx(): + """build_document no muta el ctx del caller (copia interna).""" + ctx = {"dataset_name": "v", "_only_chapters": ["num_distr"]} + build_document(_profile_with_cat_and_num(), ctx=ctx) + # La clave reservada sigue en el dict del caller (no se mutó su copia). + assert ctx["_only_chapters"] == ["num_distr"] diff --git a/python/functions/datascience/automatic_eda/chapter_deps.py b/python/functions/datascience/automatic_eda/chapter_deps.py new file mode 100644 index 00000000..98d1397c --- /dev/null +++ b/python/functions/datascience/automatic_eda/chapter_deps.py @@ -0,0 +1,205 @@ +"""chapter_deps — mapa central de dependencias de cómputo por capítulo del EDA. + +Fuente de verdad ÚNICA de qué necesita cada capítulo de ``CHAPTER_ORDER`` para +computarse COMPLETO (sin caer en su rama degradada "datos insuficientes"). Lo +consume el pipeline ``render_automatic_eda`` cuando se le pide renderizar un +SUBCONJUNTO de capítulos (kwarg ``only_chapters``): antes de perfilar, resuelve +los requisitos de los capítulos pedidos y activa SOLO el cómputo que esos +capítulos necesitan, de modo que un capítulo suelto siempre llegue poblado y a la +vez no se malgaste CPU/LLM en piezas que ningún capítulo pedido usa. + +Diseño: el mapa es CENTRAL (este módulo), NO una constante por capítulo. Así se +evita tocar los ``chapters/<id>.py`` (cada agente es dueño de su capítulo) y se +elimina el riesgo de colisión entre ramas. Si un capítulo cambia lo que lee del +``profile``/``ctx``, se actualiza ESTE mapa — es donde el motor mira. + +Dos clases de dependencia, derivadas inspeccionando qué lee cada capítulo: + + - ``profile_flags``: flags de coste de ``profile_table`` que hay que ACTIVAR + para que el ``profile`` traiga el bloque que el capítulo lee. Son los caros: + * ``run_models`` -> ``profile['models']`` (KMeans/IsolationForest/PCA). + Lo leen ``outliers`` (fallback del multivariante) y ``modelos``. + * ``run_series`` -> ``profile['series']`` (análisis de serie temporal). + Lo lee ``timeseries``. + * ``run_llm`` -> ``profile['llm']`` (interpretación del modelo). + Lo lee ``analisis_llm``. + + - ``ctx``: etiquetas de las piezas de DATOS CRUDOS que construye + ``build_eda_render_ctx`` y que el capítulo lee del ``ctx``. Si la lista está + vacía, el capítulo no necesita datos crudos y el pipeline puede saltarse + ``build_eda_render_ctx`` por completo cuando ningún capítulo pedido los pide. + Etiquetas y claves reales que mapean (ver ``CTX_LABEL_TO_KEYS``): + * ``head_rows`` -> ``ctx['head_rows']`` (overview: df.head real). + * ``raw_numeric`` -> ``ctx['raw_numeric']`` (outliers/modelos/ + correlacion/missingness/geospatial: muestra numérica alineada por fila). + * ``timeseries_raw`` -> ``ctx['timeseries_raw']`` (timeseries: serie cruda). + * ``geo_points`` -> ``ctx['geo_points']`` (+ ``raw_numeric``) + (geospatial: lat/lon). + * ``db_path_table`` -> ``ctx['db_path']`` + ``ctx['table']`` (agregacion/ + text_distr/missingness/relaciones: push-down de queries propias). + +``portada`` y ``glosario`` NO son opcionales: el pipeline los incluye SIEMPRE +(la portada resume el documento y el glosario es el destino de los términos +clicables), así que aquí se declaran sin requisitos de cómputo. + +Todas las funciones de este módulo son PURAS (no I/O, deterministas): se prestan +a test unitario directo. +""" + +from __future__ import annotations + +# Mapa central. Una entrada por id de CHAPTER_ORDER. ``profile_flags`` lista los +# flags de coste a activar; ``ctx`` las etiquetas de datos crudos que lee. Las +# claves vacías significan "no necesita ese tipo de dependencia". +CHAPTER_DEPS = { + # Portada y glosario: SIEMPRE presentes, sin cómputo propio (la portada lee + # el document_summary que arma build_document; el glosario lee los términos + # que el resto registró). Se declaran para que el mapa cubra CHAPTER_ORDER + # entero y la validación los reconozca. + "portada": {"profile_flags": [], "ctx": []}, + "overview": {"profile_flags": [], "ctx": ["head_rows"]}, + "analisis_llm": {"profile_flags": ["run_llm"], "ctx": []}, + "num_distr": {"profile_flags": [], "ctx": []}, + "cat_distr": {"profile_flags": [], "ctx": []}, + # text_distr empuja su propia query de texto (no usa raw_numeric); necesita + # db_path/table en el ctx para hacerlo. + "text_distr": {"profile_flags": [], "ctx": ["db_path_table"]}, + "calidad": {"profile_flags": [], "ctx": []}, + # missingness lee la muestra numérica cruda (co-ocurrencia de ausencias) y + # puede empujar una query de patrón de nulos con db_path/table. + "missingness": {"profile_flags": [], "ctx": ["raw_numeric", "db_path_table"]}, + # outliers corre IsolationForest EN VIVO sobre ctx['raw_numeric']; run_models + # asegura además el fallback profile['models']['outliers'] si el ctx faltara. + "outliers": {"profile_flags": ["run_models"], "ctx": ["raw_numeric"]}, + "correlacion": {"profile_flags": [], "ctx": ["raw_numeric"]}, + "relaciones": {"profile_flags": [], "ctx": ["db_path_table"]}, + "modelos": {"profile_flags": ["run_models"], "ctx": ["raw_numeric"]}, + "timeseries": {"profile_flags": ["run_series"], "ctx": ["timeseries_raw"]}, + "geospatial": {"profile_flags": [], "ctx": ["geo_points", "raw_numeric"]}, + "agregacion": {"profile_flags": [], "ctx": ["db_path_table"]}, + "glosario": {"profile_flags": [], "ctx": []}, +} + +# Capítulos que el documento incluye SIEMPRE, independientemente de only_chapters. +ALWAYS_PRESENT = ("portada", "glosario") + +# Flags de coste reconocidos (el orden no importa; se devuelven como set). +KNOWN_PROFILE_FLAGS = ("run_models", "run_series", "run_llm") + +# Mapeo de cada etiqueta de ctx a las claves REALES que produce +# build_eda_render_ctx. ``db_path_table`` es especial: db_path/table siempre se +# ponen para un backend válido y son inofensivos, por eso no se podan nunca (no +# aparecen en DATA_CTX_KEYS). El resto (head_rows/raw_numeric/timeseries_raw/ +# geo_points) son las piezas de datos podables. +CTX_LABEL_TO_KEYS = { + "head_rows": {"head_rows"}, + "raw_numeric": {"raw_numeric"}, + "timeseries_raw": {"timeseries_raw"}, + "geo_points": {"geo_points", "raw_numeric"}, + "db_path_table": set(), # db_path/table siempre presentes; nunca se podan. +} + +# Claves de datos crudos del ctx que se pueden podar cuando ningún capítulo +# pedido las necesita (las que cuestan muestreo). db_path/table NO entran aquí. +DATA_CTX_KEYS = ("head_rows", "raw_numeric", "timeseries_raw", "geo_points") + + +def _as_id_list(chapter_ids): + """Normaliza la entrada a una lista de ids string, defensiva. None -> [].""" + if chapter_ids is None: + return [] + if isinstance(chapter_ids, str): + return [chapter_ids] + return [c for c in chapter_ids if isinstance(c, str)] + + +def validate_chapter_ids(chapter_ids, order): + """Separa los ids pedidos en válidos y desconocidos respecto a ``order``. + + Args: + chapter_ids: lista (o str) de ids de capítulo pedidos. + order: lista canónica de ids válidos (CHAPTER_ORDER). + + Returns: + dict ``{"valid": [...], "unknown": [...]}`` preservando el orden de + aparición de la entrada. Función pura. + """ + valid_set = set(order or []) + valid, unknown = [], [] + for cid in _as_id_list(chapter_ids): + (valid if cid in valid_set else unknown).append(cid) + return {"valid": valid, "unknown": unknown} + + +def resolve_requirements(chapter_ids): + """Une los requisitos de cómputo de los capítulos pedidos. + + Es el corazón de la resolución de dependencias: dado el subconjunto de + capítulos a renderizar, devuelve TODO lo que hay que activar/construir para + que esos capítulos lleguen COMPLETOS, y solo eso. + + Los capítulos ``ALWAYS_PRESENT`` (portada/glosario) se añaden implícitamente + porque el pipeline siempre los incluye; como no tienen requisitos, no alteran + el resultado, pero se contemplan para que el conjunto sea coherente. + + Args: + chapter_ids: lista (o str) de ids de capítulo. Ids desconocidos se + ignoran silenciosamente (la validación estricta es de quien llama). + None o lista vacía -> requisitos vacíos. + + Returns: + dict ``{"profile_flags": set[str], "ctx_keys": set[str]}`` donde + ``ctx_keys`` son las ETIQUETAS de ctx (no las claves reales). Función + pura. + """ + ids = set(_as_id_list(chapter_ids)) | set(ALWAYS_PRESENT) + profile_flags = set() + ctx_keys = set() + for cid in ids: + dep = CHAPTER_DEPS.get(cid) + if not isinstance(dep, dict): + continue + for f in dep.get("profile_flags", []) or []: + if f in KNOWN_PROFILE_FLAGS: + profile_flags.add(f) + for k in dep.get("ctx", []) or []: + ctx_keys.add(k) + return {"profile_flags": profile_flags, "ctx_keys": ctx_keys} + + +def resolve_profile_flags(chapter_ids): + """Atajo: solo el set de profile_flags a activar para los capítulos pedidos. + + Función pura. Devuelve un set ⊆ KNOWN_PROFILE_FLAGS. + """ + return resolve_requirements(chapter_ids)["profile_flags"] + + +def needs_render_ctx(chapter_ids): + """True si algún capítulo pedido necesita datos crudos del ctx. + + Cuando es False, el pipeline puede saltarse ``build_eda_render_ctx`` entero + (ahorro real de CPU/I/O): los capítulos pedidos no leen ninguna pieza de + datos crudos. Función pura. + """ + return bool(resolve_requirements(chapter_ids)["ctx_keys"]) + + +def resolve_ctx_data_keys(chapter_ids): + """Claves REALES de datos del ctx a CONSERVAR para los capítulos pedidos. + + Traduce las etiquetas de ctx a las claves concretas que produce + ``build_eda_render_ctx`` (head_rows/raw_numeric/timeseries_raw/geo_points). + El pipeline poda del ctx las claves de datos que NO estén en este set, para + que un capítulo suelto no arrastre piezas de datos que no usa. db_path/table + nunca se podan (no aparecen aquí). Función pura. + + Returns: + set[str] subconjunto de DATA_CTX_KEYS. + """ + req = resolve_requirements(chapter_ids) + keep = set() + for label in req["ctx_keys"]: + keep |= CTX_LABEL_TO_KEYS.get(label, set()) + # Solo claves de datos podables (db_path/table se gestionan aparte). + return {k for k in keep if k in DATA_CTX_KEYS} diff --git a/python/functions/datascience/automatic_eda/chapter_deps_test.py b/python/functions/datascience/automatic_eda/chapter_deps_test.py new file mode 100644 index 00000000..aa77bf6b --- /dev/null +++ b/python/functions/datascience/automatic_eda/chapter_deps_test.py @@ -0,0 +1,160 @@ +"""Tests del mapa central de dependencias por capítulo (chapter_deps). + +Todas las funciones bajo prueba son PURAS (sin I/O): se ejercitan directamente +sin DuckDB ni renderizado. Cubren la resolución de requisitos (golden + edges), +la validación de ids y los helpers de eficiencia (qué cómputo se salta). +""" + +import os +import sys + +_HERE = os.path.dirname(os.path.abspath(__file__)) +_FUNCTIONS = os.path.abspath(os.path.join(_HERE, "..", "..", "..")) # python/functions +if _FUNCTIONS not in sys.path: + sys.path.insert(0, _FUNCTIONS) + +from datascience.automatic_eda.chapter_deps import ( # noqa: E402 + ALWAYS_PRESENT, + CHAPTER_DEPS, + DATA_CTX_KEYS, + needs_render_ctx, + resolve_ctx_data_keys, + resolve_profile_flags, + resolve_requirements, + validate_chapter_ids, +) +from datascience.automatic_eda.chapters_registry import CHAPTER_ORDER # noqa: E402 + + +# --------------------------------------------------------------------------- # +# El mapa cubre CHAPTER_ORDER entero (sin huecos ni claves de más). +# --------------------------------------------------------------------------- # +def test_chapter_deps_covers_every_chapter_in_order(): + assert set(CHAPTER_DEPS) == set(CHAPTER_ORDER), ( + "CHAPTER_DEPS debe declarar exactamente los ids de CHAPTER_ORDER") + # Cada entrada tiene la forma esperada. + for cid, dep in CHAPTER_DEPS.items(): + assert isinstance(dep.get("profile_flags"), list), cid + assert isinstance(dep.get("ctx"), list), cid + + +# --------------------------------------------------------------------------- # +# resolve_requirements — golden: outliers exige run_models + raw_numeric. +# --------------------------------------------------------------------------- # +def test_resolve_outliers_requires_run_models_and_raw_numeric(): + req = resolve_requirements(["outliers"]) + assert "run_models" in req["profile_flags"] + assert "raw_numeric" in req["ctx_keys"] + assert "run_series" not in req["profile_flags"] + assert "run_llm" not in req["profile_flags"] + + +def test_resolve_timeseries_requires_run_series(): + req = resolve_requirements(["timeseries"]) + assert req["profile_flags"] == {"run_series"} + assert "timeseries_raw" in req["ctx_keys"] + + +def test_resolve_analisis_llm_requires_run_llm(): + assert resolve_requirements(["analisis_llm"])["profile_flags"] == {"run_llm"} + + +def test_resolve_union_of_several_chapters(): + req = resolve_requirements(["outliers", "timeseries", "analisis_llm"]) + assert req["profile_flags"] == {"run_models", "run_series", "run_llm"} + + +# --------------------------------------------------------------------------- # +# Eficiencia: capítulos que NO necesitan flags caros no los activan. +# --------------------------------------------------------------------------- # +def test_resolve_geospatial_needs_no_cost_flags(): + """geospatial sale de geo_points/raw_numeric del ctx, NO de los modelos.""" + req = resolve_requirements(["geospatial"]) + assert req["profile_flags"] == set(), \ + "geospatial no debe activar run_models/run_series/run_llm" + assert "geo_points" in req["ctx_keys"] + + +def test_resolve_correlacion_needs_raw_numeric_but_no_models(): + req = resolve_requirements(["correlacion"]) + assert req["profile_flags"] == set() + assert "raw_numeric" in req["ctx_keys"] + + +def test_always_present_chapters_add_no_requirements(): + """portada y glosario están siempre, pero no arrastran cómputo.""" + for cid in ALWAYS_PRESENT: + req = resolve_requirements([cid]) + assert req["profile_flags"] == set() + assert req["ctx_keys"] == set() + + +def test_resolve_profile_flags_shortcut(): + assert resolve_profile_flags(["modelos"]) == {"run_models"} + assert resolve_profile_flags(["num_distr"]) == set() + + +# --------------------------------------------------------------------------- # +# needs_render_ctx — cuándo se puede saltar build_eda_render_ctx por completo. +# --------------------------------------------------------------------------- # +def test_needs_render_ctx_true_when_chapter_reads_raw_data(): + assert needs_render_ctx(["outliers"]) is True + assert needs_render_ctx(["agregacion"]) is True # db_path/table push-down + assert needs_render_ctx(["timeseries"]) is True + + +def test_needs_render_ctx_false_for_purely_aggregated_chapters(): + """num_distr / cat_distr / calidad solo leen el profile agregado.""" + assert needs_render_ctx(["num_distr"]) is False + assert needs_render_ctx(["cat_distr", "calidad"]) is False + + +# --------------------------------------------------------------------------- # +# resolve_ctx_data_keys — poda: qué claves de DATOS conservar (db_path/table no). +# --------------------------------------------------------------------------- # +def test_resolve_ctx_data_keys_outliers_keeps_only_raw_numeric(): + assert resolve_ctx_data_keys(["outliers"]) == {"raw_numeric"} + + +def test_resolve_ctx_data_keys_geospatial_keeps_geo_and_numeric(): + assert resolve_ctx_data_keys(["geospatial"]) == {"geo_points", "raw_numeric"} + + +def test_resolve_ctx_data_keys_aggregation_keeps_nothing_prunable(): + """agregacion usa db_path/table (siempre presentes), 0 claves podables.""" + assert resolve_ctx_data_keys(["agregacion"]) == set() + + +def test_resolve_ctx_data_keys_subset_of_data_keys(): + keep = resolve_ctx_data_keys(["overview", "timeseries", "geospatial"]) + assert keep <= set(DATA_CTX_KEYS) + assert {"head_rows", "timeseries_raw", "geo_points", "raw_numeric"} == keep + + +# --------------------------------------------------------------------------- # +# validate_chapter_ids — separa válidos de desconocidos preservando orden. +# --------------------------------------------------------------------------- # +def test_validate_separates_known_and_unknown(): + out = validate_chapter_ids(["outliers", "nope", "timeseries", "ghost"], + CHAPTER_ORDER) + assert out["valid"] == ["outliers", "timeseries"] + assert out["unknown"] == ["nope", "ghost"] + + +def test_validate_all_known(): + out = validate_chapter_ids(["portada", "glosario"], CHAPTER_ORDER) + assert out["unknown"] == [] + + +# --------------------------------------------------------------------------- # +# Robustez: entradas raras nunca lanzan. +# --------------------------------------------------------------------------- # +def test_resolve_handles_none_and_empty(): + assert resolve_requirements(None)["profile_flags"] == set() + assert resolve_requirements([])["profile_flags"] == set() + # ids desconocidos se ignoran silenciosamente en la resolución. + assert resolve_requirements(["no_existe"])["ctx_keys"] == set() + + +def test_resolve_accepts_single_string(): + assert resolve_requirements("outliers")["profile_flags"] == {"run_models"} diff --git a/python/functions/datascience/automatic_eda/chapters_registry.py b/python/functions/datascience/automatic_eda/chapters_registry.py index 17d956db..43653d7e 100644 --- a/python/functions/datascience/automatic_eda/chapters_registry.py +++ b/python/functions/datascience/automatic_eda/chapters_registry.py @@ -73,24 +73,51 @@ def build_chapter(chapter_id: str, profile: dict, ctx: dict): return model.as_chapter(result) -def build_document(profile: dict, ctx: dict = None) -> list: - """Build the full ordered list of chapters for a TableProfile. +def build_document(profile: dict, ctx: dict = None, only: list = None) -> list: + """Build the ordered list of chapters for a TableProfile. Args: profile: the ``eda`` group TableProfile dict (may be None/empty). ctx: optional context dict carrying presentation metadata not present in the profile (dataset_name, source_origin, storage, generated_at, description, granularity, quality_criteria, head_rows, ...). + only: optional list of chapter ids to render. ``None`` (default) keeps + the historical behaviour — every implemented & applicable chapter in + canonical order. A list restricts the BODY to just those ids (in + canonical order), but the cover (``portada``) and glossary + (``glosario``) are ALWAYS included so the document stays valid and + the clickable terms keep a destination — so passing ``only=["x"]`` + yields portada + x + glosario. Unknown ids are simply skipped (the + caller is responsible for strict validation). ``only=[]`` yields the + minimal document (portada + glosario only). This argument is additive + and backward-compatible: the signature is unchanged for existing + callers (default ``None``). Returns: list[Chapter] in canonical order, containing only the chapters that are - implemented and applicable. Never raises. + implemented, applicable and selected. Never raises. """ if not isinstance(profile, dict): profile = {} # Copy ctx so the shared collector / summary we add do not leak to the caller. ctx = dict(ctx) if isinstance(ctx, dict) else {} + # only=None -> all body chapters (historical). only=list -> restrict body to + # that selection (portada/glosario are added unconditionally below). The + # renderers call build_document(profile, meta['ctx']) without an `only` + # argument, so the pipeline forwards the selection through a reserved ctx key + # (``_only_chapters``); an explicit `only` argument always wins. The key is + # popped from the local ctx copy so it never reaches the chapters. + if only is None: + _carried = ctx.pop("_only_chapters", None) + if isinstance(_carried, (list, tuple, set)): + only = list(_carried) + else: + ctx.pop("_only_chapters", None) + # A set makes the membership test cheap; the iteration order stays + # CHAPTER_ORDER. only=[] is a valid (empty) selection -> minimal document. + only_set = set(only) if isinstance(only, (list, tuple, set)) else None + # A single glossary collector is shared by every chapter via ctx['glossary']. # Chapters call ctx['glossary'].add(key, label, definition) and mark in-text # appearances with [[term:key]]…[[/term]]; the glosario chapter renders the @@ -106,6 +133,10 @@ def build_document(profile: dict, ctx: dict = None) -> list: for cid in CHAPTER_ORDER: if cid in (_PORTADA, _GLOSARIO): continue + # When a selection is given, skip body chapters outside it. portada and + # glosario are never filtered (handled out of this loop). + if only_set is not None and cid not in only_set: + continue ch = build_chapter(cid, profile, ctx) if ch is not None and ch.blocks: body.append(ch) diff --git a/python/functions/pipelines/render_automatic_eda.md b/python/functions/pipelines/render_automatic_eda.md index 50d4bcda..9efabf06 100644 --- a/python/functions/pipelines/render_automatic_eda.md +++ b/python/functions/pipelines/render_automatic_eda.md @@ -4,8 +4,8 @@ kind: pipeline lang: py domain: pipelines purity: impure -version: "1.1.0" -signature: "def render_automatic_eda(db_path: str, table: str, backend: str = \"duckdb\", sample: int = None, run_models: bool = None, run_series: bool = None, run_llm: bool = None, profile_level: str = \"standard\", out_dir: str = \"reports\", basename: str = None, ctx_extra: dict = None) -> dict" +version: "1.2.0" +signature: "def render_automatic_eda(db_path: str, table: str, backend: str = \"duckdb\", sample: int = None, run_models: bool = None, run_series: bool = None, run_llm: bool = None, profile_level: str = \"standard\", out_dir: str = \"reports\", basename: str = None, ctx_extra: dict = None, emit_md: bool = True, only_chapters: list = None) -> dict" description: "Informe AutomaticEDA COMPLETO one-shot de una tabla DuckDB/PostgreSQL: perfila con profile_table, construye el ctx con los datos crudos (build_eda_render_ctx: raw_numeric para modelos/geo, timeseries_raw para series, geo_points para el mapa, db_path/table para la agregacion push-down) y emite PDF (A5 movil) Y PPTX (16:9) del mismo documento por capitulos, con los 11 capitulos POBLADOS de verdad (clusters pintados sobre el PCA, evolucion temporal, mapa geografico y tablas de agregacion), no degradados. El parametro profile_level es un preset de consumo CPU/LLM (lite/standard/full) que mapea a los flags run_models/run_series/run_llm/sample; un flag explicito siempre prima sobre el preset. lite=bajo consumo (sin LLM, sin serie, modelos solo PCA+normalidad sin KMeans/IsolationForest, sample reducido); standard=comportamiento historico; full=standard+narrativa LLM. Devuelve las rutas de PDF/PPTX y el manifiesto de versiones por capitulo." tags: [eda, duckdb, postgres, profiling, pipeline, dataops, report, pdf, pptx] uses_functions: @@ -46,6 +46,10 @@ params: desc: "Nombre base de los archivos sin extension. Default 'aeda_<table>_<timestamp>'." - name: ctx_extra desc: "Dict opcional con claves de presentacion/contexto extra que se mezclan en el ctx (dataset_name, description, source_origin, ...); no pisan las claves de datos calculadas por build_eda_render_ctx." + - name: emit_md + desc: "Ademas del PDF y el PPTX, emite un Markdown autocontenido del mismo documento por capitulos (texto + tablas markdown, sin binarios) para pegar a un LLM. Default True. La ruta sale en aeda_md_path." + - name: only_chapters + desc: "Lista opcional de ids de capitulo a renderizar (subconjunto de CHAPTER_ORDER) para iterar/testear un capitulo suelto sin generar el documento entero. Default None => documento COMPLETO (retrocompatible). Cuando se pasa una lista: (1) se VALIDA contra CHAPTER_ORDER, un id desconocido o lista vacia devuelve error claro listando los validos; (2) se RESUELVEN las dependencias de computo de esos capitulos (automatic_eda.chapter_deps) activando los flags que necesiten (run_models/run_series/run_llm) aunque el caller no los pidiera y construyendo SOLO las piezas de ctx que leen, de modo que el capitulo suelto SIEMPRE llega poblado (p.ej. ['outliers'] activa run_models y conserva raw_numeric -> Isolation Forest completo) sin malgastar CPU/LLM en lo que ningun capitulo pedido usa; (3) el documento y su manifest contienen SOLO esos capitulos MAS portada (primera) y glosario (ultima, cuando hay terminos clicables). Un flag explicito del caller prima sobre la resolucion de dependencias." output: "dict {status:'ok', pdf_path:str, pptx_path:str, manifest_path:str|None, n_pages:int, n_slides:int, pdf_note:str, pptx_note:str, profile:<TableProfile>} o {status:'error', error:str} (dict-no-throw)." --- @@ -69,6 +73,21 @@ r = render_automatic_eda("/tmp/ventas.duckdb", "ventas", profile_level="full") # Precedencia: el flag explicito SIEMPRE prima sobre el preset. lite pero con LLM: r = render_automatic_eda("/tmp/ventas.duckdb", "ventas", profile_level="lite", run_llm=True) # el LLM SI se ejecuta + +# Capitulo SUELTO: itera/testea un capitulo sin generar el documento entero. La +# resolucion de dependencias activa el computo que el capitulo necesita aunque no +# se pase explicito. Pedir solo 'outliers' activa run_models y conserva +# raw_numeric -> el bloque Isolation Forest sale COMPLETO. Documento = portada + +# outliers + glosario. +r = render_automatic_eda("/tmp/ventas.duckdb", "ventas", only_chapters=["outliers"]) + +# Varios capitulos sueltos a la vez (se unen sus dependencias): +r = render_automatic_eda("/tmp/ventas.duckdb", "ventas", + only_chapters=["correlacion", "missingness"]) + +# id desconocido -> error claro listando los validos (dict-no-throw, no lanza): +r = render_automatic_eda("/tmp/ventas.duckdb", "ventas", only_chapters=["nope"]) +# {'status': 'error', 'error': 'only_chapters con ids desconocidos: nope. Capitulos validos: portada, overview, ...'} ``` ## Cuando usarla @@ -86,6 +105,16 @@ Para un EDA **barato/rapido** (CI, vistazo previo, maquina sin GPU o sin red) us temporal y el LLM. Para el **maximo** con interpretacion narrativa por capitulo, `profile_level="full"`. El default `"standard"` mantiene el comportamiento previo. +Cuando estes **iterando o testeando UN capitulo concreto** (afinar el render de +outliers, comprobar el mapa geoespacial, depurar la agregacion) usa +`only_chapters=[...]`: genera el documento con solo esos capitulos (+ portada y +glosario), pero **resuelve sus dependencias de computo** para que el capitulo +suelto nunca salga degradado — pedir `['outliers']` activa run_models y conserva +`raw_numeric` aunque no los pases, y a la vez no malgasta CPU/LLM en lo que ningun +capitulo pedido necesita (pedir `['geospatial']` no corre modelos). Es mucho mas +rapido que renderizar el informe entero en cada iteracion. El mapa central de +dependencias vive en `automatic_eda/chapter_deps.py` (fuente de verdad). + ## Gotchas - Impura: ESCRIBE el PDF, el PPTX y `automatic_eda_manifest.json` en `out_dir`. @@ -111,9 +140,29 @@ temporal y el LLM. Para el **maximo** con interpretacion narrativa por capitulo, - Los datos crudos del ctx se muestrean con `sample` (LIMIT), no se trae la tabla entera a RAM; con tablas enormes sube `sample` si quieres mas representatividad (coste: mas memoria). +- **`only_chapters` y el glosario**: el glosario (ultimo capitulo) solo aparece si + algun capitulo del cuerpo registro terminos clicables. Un capitulo suelto que no + registra terminos (p.ej. `timeseries`, `geospatial`) sale como portada + ese + capitulo, sin glosario, porque no hay nada que enlazar — es correcto, no un fallo. +- **`only_chapters` con `profile_level="lite"`**: en capitulos sueltos el preset + solo gobierna `sample`; los modelos NO usan el camino "lite" (que podaria + `ctx['raw_numeric']` y dejaria a outliers sin su multivariante en vivo). Quien + manda en capitulos sueltos es la resolucion de dependencias, no el preset de + coste de modelos. ## Capability growth log +- v1.2.0 (2026-06-30) — anade el parametro `only_chapters`: renderiza un + SUBCONJUNTO de capitulos (para iterar/testear uno suelto) resolviendo sus + dependencias de computo via `automatic_eda/chapter_deps.py` (mapa central + CHAPTER_DEPS): activa los flags de coste que el capitulo necesita (run_models/ + run_series/run_llm) aunque el caller no los pase y construye solo las piezas de + ctx que lee, de modo que el capitulo suelto SIEMPRE llega poblado (golden: + ['outliers'] -> Isolation Forest completo) sin malgastar en lo que no usa. La + seleccion viaja a build_document por la clave reservada `ctx['_only_chapters']` + (los renderers no cambian). Valida ids (error claro dict-no-throw). Cambio + aditivo y retro-compatible: `only_chapters=None` produce el documento completo + identico a v1.1.0. - v1.1.0 (2026-06-30) — anade el parametro `profile_level` (lite/standard/full), preset de consumo CPU/LLM que mapea a los flags run_models/run_series/run_llm/ sample. lite limita los modelos a PCA+normalidad (cableado a run_eda_models con diff --git a/python/functions/pipelines/render_automatic_eda.py b/python/functions/pipelines/render_automatic_eda.py index 942ee456..48c96dd9 100644 --- a/python/functions/pipelines/render_automatic_eda.py +++ b/python/functions/pipelines/render_automatic_eda.py @@ -99,6 +99,7 @@ def render_automatic_eda( basename: str = None, ctx_extra: dict = None, emit_md: bool = True, + only_chapters: list = None, ) -> dict: """Perfila una tabla y emite el informe AutomaticEDA completo (PDF + PPTX). @@ -150,6 +151,29 @@ def render_automatic_eda( MISMO documento por capítulos (texto plano + tablas markdown, sin binarios), pensado para pegar a un LLM. Default True. La ruta sale en la clave de retorno ``aeda_md_path``. No altera las demás salidas. + only_chapters: lista opcional de ids de capítulo a renderizar (un + SUBCONJUNTO de CHAPTER_ORDER) para iterar/testear un capítulo concreto + sin generar el documento entero. Default None => documento COMPLETO, + idéntico al de hoy (retrocompatible). Cuando se pasa una lista: + + - Se VALIDA contra CHAPTER_ORDER; un id desconocido devuelve un error + claro listando los válidos (dict-no-throw, no lanza). Lista vacía + ``[]`` también devuelve error (pasa al menos un capítulo o None). + - Se RESUELVEN las dependencias de cómputo de esos capítulos + (``automatic_eda.chapter_deps``): se activan los flags de coste que + necesiten (run_models / run_series / run_llm) AUNQUE el caller no + los pidiera, y se construyen SOLO las piezas de ``ctx`` que esos + capítulos leen. Así un capítulo suelto SIEMPRE llega poblado — + p.ej. ``only_chapters=['outliers']`` activa run_models y conserva + ``ctx['raw_numeric']`` para que el bloque IsolationForest salga + completo— y a la vez no se malgasta CPU/LLM en lo que ningún + capítulo pedido usa (pedir solo ``geospatial`` no corre modelos). + - El documento (PDF/PPTX/MD) y su manifest contienen SOLO esos + capítulos, MÁS la portada (primera) y el glosario (última), que se + incluyen siempre para que el documento sea válido y los términos + clicables tengan destino. + - Un flag explícito del caller (run_models/run_series/run_llm != None) + SIEMPRE prima sobre lo que resuelvan las dependencias. Returns: dict (nunca lanza). En éxito:: @@ -169,11 +193,56 @@ def render_automatic_eda( # "standard" (comportamiento histórico), sin lanzar. preset = _PROFILE_PRESETS.get(profile_level, _PROFILE_PRESETS["standard"]) sample = preset["sample"] if sample is None else sample - run_models = preset["run_models"] if run_models is None else run_models - run_series = preset["run_series"] if run_series is None else run_series - run_llm = preset["run_llm"] if run_llm is None else run_llm model_opts = preset["model_opts"] + # 0.bis) Modo "capítulos sueltos": valida la selección y RESUELVE sus + # dependencias de cómputo. Es lo que garantiza que un capítulo pedido + # llegue completo (activa lo que necesita) sin malgastar en lo que no. + # Cuando only_chapters es None se conserva el camino histórico (preset). + if only_chapters is not None: + from datascience.automatic_eda import CHAPTER_ORDER + from datascience.automatic_eda.chapter_deps import ( + needs_render_ctx, + resolve_ctx_data_keys, + resolve_requirements, + validate_chapter_ids, + ) + + if not isinstance(only_chapters, (list, tuple)): + return {"status": "error", + "error": "only_chapters debe ser una lista de ids de " + "capítulo o None (documento completo)."} + only_chapters = [c for c in only_chapters] + if not only_chapters: + return {"status": "error", + "error": "only_chapters=[] está vacío. Pasa al menos un " + "capítulo, o None para el documento completo. " + "Capítulos válidos: " + ", ".join(CHAPTER_ORDER)} + checked = validate_chapter_ids(only_chapters, CHAPTER_ORDER) + if checked["unknown"]: + return {"status": "error", + "error": "only_chapters con ids desconocidos: " + + ", ".join(checked["unknown"]) + + ". Capítulos válidos: " + + ", ".join(CHAPTER_ORDER)} + only_chapters = checked["valid"] + + # Las dependencias fijan el DEFAULT de cada flag de coste (eficiencia: + # lo que ningún capítulo pedido necesita queda en False); un flag + # explícito del caller (!= None) sigue primando. + dep_flags = resolve_requirements(only_chapters)["profile_flags"] + run_models = ("run_models" in dep_flags) if run_models is None else run_models + run_series = ("run_series" in dep_flags) if run_series is None else run_series + run_llm = ("run_llm" in dep_flags) if run_llm is None else run_llm + # En capítulos sueltos no se usa el camino "modelos baratos" (lite), + # que poda ctx['raw_numeric']: un capítulo como outliers lo necesita + # para su multivariante en vivo. El preset solo gobierna `sample`. + model_opts = None + else: + run_models = preset["run_models"] if run_models is None else run_models + run_series = preset["run_series"] if run_series is None else run_series + run_llm = preset["run_llm"] if run_llm is None else run_llm + # En el camino "modelos baratos" (lite) profile_table NO corre los # modelos: los ejecuta este pipeline con run_eda_models y la granularidad # del preset, evitando pagar el coste CPU de KMeans + IsolationForest. @@ -217,10 +286,25 @@ def render_automatic_eda( if ctx_extra: base_ctx.update(ctx_extra) - ctx = build_eda_render_ctx( - db_path, table, prof, backend=backend, sample=sample, - base_ctx=base_ctx, - ) + # En modo capítulos sueltos, si NINGÚN capítulo pedido necesita datos + # crudos del ctx, se salta build_eda_render_ctx por completo (ahorro real + # de I/O): solo se conservan presentación + db_path/table. Si sí los + # necesita, se construye el ctx y luego se PODAN las piezas de datos que + # ningún capítulo pedido usa (db_path/table nunca se podan). + if only_chapters is not None and not needs_render_ctx(only_chapters): + ctx = dict(base_ctx) + ctx["db_path"] = db_path + ctx["table"] = table + else: + ctx = build_eda_render_ctx( + db_path, table, prof, backend=backend, sample=sample, + base_ctx=base_ctx, + ) + if only_chapters is not None and isinstance(ctx, dict): + keep = resolve_ctx_data_keys(only_chapters) + for k in ("head_rows", "raw_numeric", "timeseries_raw", "geo_points"): + if k not in keep: + ctx.pop(k, None) # 2.5) Camino lite — modelos baratos (PCA + normalidad, sin KMeans ni # IsolationForest). profile_table no corrió los modelos; aquí se corren @@ -245,6 +329,13 @@ def render_automatic_eda( ctx.pop("raw_numeric", None) # 3) Render a ambos formatos desde el MISMO documento por capítulos. + # En modo capítulos sueltos, la selección viaja a build_document por una + # clave reservada del ctx (los renderers llaman build_document sin pasar + # `only`): build_document filtra el cuerpo a esos capítulos y siempre + # añade portada (primera) + glosario (última). build_document la consume + # y la quita, así que no llega a los capítulos. + if only_chapters is not None and isinstance(ctx, dict): + ctx["_only_chapters"] = list(only_chapters) os.makedirs(out_dir, exist_ok=True) ts = datetime.now(timezone.utc).strftime("%Y%m%d-%H%M%S") base = basename or f"aeda_{table}_{ts}" @@ -283,6 +374,7 @@ def render_automatic_eda( "pdf_note": rpdf.get("note"), "pptx_note": rpptx.get("note"), "md_note": rmd.get("note"), + "only_chapters": only_chapters, "profile": prof, } except Exception as e: # noqa: BLE001 — dict-no-throw: degradar, nunca lanzar. diff --git a/python/functions/pipelines/render_automatic_eda_only_test.py b/python/functions/pipelines/render_automatic_eda_only_test.py new file mode 100644 index 00000000..a6222345 --- /dev/null +++ b/python/functions/pipelines/render_automatic_eda_only_test.py @@ -0,0 +1,235 @@ +"""Tests del modo `only_chapters` del pipeline render_automatic_eda. + +Cubre la tarea de "capítulos sueltos con resolución de dependencias": + + - Golden (DuckDB real): pedir SOLO un capítulo genera un documento con solo + portada + ese capítulo + glosario, y el capítulo llega COMPLETO porque la + resolución de dependencias activó el cómputo que necesita aunque el caller + no lo pidiera (outliers → run_models + raw_numeric → IsolationForest poblado; + timeseries → run_series; correlacion → raw_numeric). + - Eficiencia: pedir un capítulo que NO necesita flags caros (geospatial) no los + activa, y un capítulo puramente agregado (num_distr) ni siquiera construye el + ctx de datos crudos. + - Edge: id desconocido / lista vacía / no-lista devuelven error claro sin + lanzar; only_chapters=None mantiene el comportamiento histórico. +""" + +import json +import os +import random +import sys +from datetime import date, timedelta + +_HERE = os.path.dirname(os.path.abspath(__file__)) +_FUNCTIONS = os.path.abspath(os.path.join(_HERE, "..", "..")) # python/functions +if _FUNCTIONS not in sys.path: + sys.path.insert(0, _FUNCTIONS) + +import duckdb # noqa: E402 + +from pipelines.render_automatic_eda import render_automatic_eda # noqa: E402 + + +def _make_db_models(path): + """DB con fecha + 3 numéricas continuas en 3 clusters gaussianos. + + Garantiza material para outliers/modelos (>=2 numéricas → IsolationForest), + timeseries (columna DATE) y correlacion (numéricas). Mismo shape que el + fixture del test del pipeline base. + """ + con = duckdb.connect(path) + con.execute("CREATE TABLE pts (d DATE, grp VARCHAR, x1 DOUBLE, x2 DOUBLE, x3 DOUBLE)") + random.seed(42) + centers = [(0.0, 0.0, 0.0), (10.0, 10.0, 10.0), (20.0, 5.0, 15.0)] + d0 = date(2024, 1, 1) + rows = [] + for i in range(150): + cx, cy, cz = centers[i % 3] + rows.append(( + d0 + timedelta(days=i), f"g{i % 3}", + round(cx + random.gauss(0, 1.0), 4), + round(cy + random.gauss(0, 1.0), 4), + round(cz + random.gauss(0, 1.0), 4), + )) + con.executemany("INSERT INTO pts VALUES (?,?,?,?,?)", rows) + con.close() + + +def _manifest_chapters(result): + with open(result["manifest_path"], encoding="utf-8") as fh: + return set((json.load(fh).get("chapters") or {}).keys()) + + +# --------------------------------------------------------------------------- # +# GOLDEN — outliers suelto: IsolationForest poblado por resolución de deps. +# --------------------------------------------------------------------------- # +def test_only_outliers_isolation_forest_populated_without_explicit_run_models(tmp_path): + """El corazón de la tarea: pedir SOLO 'outliers' sin run_models explícito + activa run_models por dependencias y conserva ctx['raw_numeric'], de modo que + el bloque multivariante (Isolation Forest) sale con datos, no degradado.""" + db = str(tmp_path / "pts.duckdb") + _make_db_models(db) + out = str(tmp_path / "out") + + # NB: no se pasa run_models — la resolución de dependencias debe activarlo. + r = render_automatic_eda(db, "pts", only_chapters=["outliers"], + out_dir=out, basename="only_outliers") + assert r["status"] == "ok", r.get("error") + assert r["only_chapters"] == ["outliers"] + + # Documento = portada + outliers + glosario, nada más. + assert _manifest_chapters(r) == {"portada", "outliers", "glosario"} + + # El multivariante salió POBLADO (no la nota de degradación). Se comprueba en + # el Markdown (mismo documento por capítulos, texto plano fiable). + md = open(r["aeda_md_path"], encoding="utf-8").read() + assert "Filas atípicas (multivariante)" in md + assert "Filas analizadas" in md, "el Isolation Forest no trae su tabla poblada" + assert "No se pudo analizar la anomalía multivariante" not in md, \ + "el bloque multivariante salió degradado pese a resolver las deps" + + # La resolución activó run_models → el perfil trae el bloque de modelos. + assert ((r["profile"] or {}).get("models") or {}).get("outliers") is not None + + +# --------------------------------------------------------------------------- # +# GOLDEN — timeseries suelto activa run_series. +# --------------------------------------------------------------------------- # +def test_only_timeseries_activates_run_series(tmp_path): + db = str(tmp_path / "pts.duckdb") + _make_db_models(db) + out = str(tmp_path / "out") + + r = render_automatic_eda(db, "pts", only_chapters=["timeseries"], + out_dir=out, basename="only_ts") + assert r["status"] == "ok", r.get("error") + assert "timeseries" in _manifest_chapters(r) + assert "modelos" not in _manifest_chapters(r) + # run_series resuelto por deps → el perfil trae el análisis de serie. + assert (r["profile"] or {}).get("series") is not None, \ + "only_chapters=['timeseries'] debe activar run_series" + + +# --------------------------------------------------------------------------- # +# GOLDEN — correlacion suelto construye raw_numeric (sin activar modelos). +# --------------------------------------------------------------------------- # +def test_only_correlacion_builds_raw_numeric_without_models(tmp_path): + db = str(tmp_path / "pts.duckdb") + _make_db_models(db) + out = str(tmp_path / "out") + + r = render_automatic_eda(db, "pts", only_chapters=["correlacion"], + out_dir=out, basename="only_corr") + assert r["status"] == "ok", r.get("error") + assert _manifest_chapters(r) == {"portada", "correlacion", "glosario"} + # Eficiencia: correlacion no necesita los modelos → no se corrieron. + assert ((r["profile"] or {}).get("models") or {}).get("outliers") is None + assert (r["profile"] or {}).get("series") is None + + +# --------------------------------------------------------------------------- # +# Eficiencia y precedencia — vía stub (sin DuckDB). +# --------------------------------------------------------------------------- # +def _patch(monkeypatch, cap): + import pipelines.render_automatic_eda as mod + + def fake_pt(db, t, **kw): + cap["run_models"] = kw.get("run_models") + cap["run_series"] = kw.get("run_series") + cap["run_llm"] = kw.get("run_llm") + return {"status": "ok", "profile": {"columns": []}} + + def fake_ctx(db, t, prof, **kw): + cap["ctx_called"] = True + return {"db_path": db, "table": t} + + cap["ctx_called"] = False + monkeypatch.setattr(mod, "profile_table", fake_pt) + monkeypatch.setattr(mod, "build_eda_render_ctx", fake_ctx) + monkeypatch.setattr(mod, "render_automatic_eda_pdf", + lambda *a, **k: {"path": "x.pdf", "n_pages": 1, + "manifest_path": "m.json"}) + monkeypatch.setattr(mod, "render_automatic_eda_pptx", + lambda *a, **k: {"path": "x.pptx", "n_slides": 1}) + monkeypatch.setattr(mod, "render_automatic_eda_markdown", + lambda *a, **k: {"path": "x.md", "n_chars": 1}) + + +def test_only_geospatial_does_not_activate_cost_flags(monkeypatch): + """Eficiencia: pedir solo geospatial NO corre modelos/serie/LLM.""" + cap = {} + _patch(monkeypatch, cap) + render_automatic_eda("db", "t", only_chapters=["geospatial"]) + assert cap["run_models"] is False + assert cap["run_series"] is False + assert cap["run_llm"] is False + + +def test_only_outliers_activates_run_models_via_deps(monkeypatch): + cap = {} + _patch(monkeypatch, cap) + render_automatic_eda("db", "t", only_chapters=["outliers"]) + assert cap["run_models"] is True + assert cap["run_series"] is False + + +def test_explicit_flag_overrides_dependency_resolution(monkeypatch): + """run_models=False explícito gana, aunque outliers lo pediría por deps.""" + cap = {} + _patch(monkeypatch, cap) + render_automatic_eda("db", "t", only_chapters=["outliers"], run_models=False) + assert cap["run_models"] is False + + +def test_purely_aggregated_chapter_skips_render_ctx(monkeypatch): + """num_distr solo lee el profile → build_eda_render_ctx no se llama.""" + cap = {} + _patch(monkeypatch, cap) + render_automatic_eda("db", "t", only_chapters=["num_distr"]) + assert cap["ctx_called"] is False, \ + "num_distr no necesita datos crudos: el ctx no debe construirse" + + +def test_chapter_that_needs_ctx_builds_it(monkeypatch): + cap = {} + _patch(monkeypatch, cap) + render_automatic_eda("db", "t", only_chapters=["outliers"]) + assert cap["ctx_called"] is True + + +# --------------------------------------------------------------------------- # +# EDGE — errores claros sin lanzar. +# --------------------------------------------------------------------------- # +def test_unknown_chapter_id_returns_clear_error(tmp_path): + r = render_automatic_eda(str(tmp_path / "x.duckdb"), "t", + only_chapters=["no_existe"]) + assert r["status"] == "error" + assert "no_existe" in r["error"] + assert "Capítulos válidos" in r["error"] + # Algún id válido conocido aparece en la lista. + assert "outliers" in r["error"] + + +def test_empty_only_list_returns_error(tmp_path): + r = render_automatic_eda(str(tmp_path / "x.duckdb"), "t", only_chapters=[]) + assert r["status"] == "error" + assert "vac" in r["error"].lower() + + +def test_only_chapters_not_a_list_returns_error(tmp_path): + r = render_automatic_eda(str(tmp_path / "x.duckdb"), "t", + only_chapters="outliers") + assert r["status"] == "error" + + +def test_only_none_keeps_full_document(tmp_path): + """Retro-compat: only_chapters=None genera el documento completo.""" + db = str(tmp_path / "pts.duckdb") + _make_db_models(db) + out = str(tmp_path / "out") + r = render_automatic_eda(db, "pts", out_dir=out, basename="full") + assert r["status"] == "ok", r.get("error") + chapters = _manifest_chapters(r) + # Documento completo: muchos más capítulos que portada/glosario. + assert {"portada", "glosario", "overview", "correlacion"} <= chapters + assert len(chapters) > 4 From 7bdb8bffb5589a61e4269fd23458336c5a5a1f1b Mon Sep 17 00:00:00 2001 From: agent <agent@fn_registry> Date: Tue, 30 Jun 2026 22:07:15 +0200 Subject: [PATCH 47/53] test(eda): suite de aceptacion de los 16 capitulos del AutomaticEDA Bateria que blinda el subsistema: cobertura de los 16 capitulos sobre el dataset sintetico Faker, contenido esencial por capitulo (needles parametrizados), capitulos sueltos con resolucion de dependencias (only_chapters=[outliers] puebla IsolationForest sin run_models; timeseries; correlacion), None cuando no aplica, folder multi-tabla con FK, completitud del MD (matriz de correlacion completa + skew/kurtosis), 3 salidas no vacias, determinismo. Test full+LLM skippeable. 29 passed, 1 skipped. Sin hallazgos: los 16 capitulos salen como deben. --- .../automatic_eda_acceptance_test.py | 466 ++++++++++++++++++ 1 file changed, 466 insertions(+) create mode 100644 python/functions/pipelines/automatic_eda_acceptance_test.py diff --git a/python/functions/pipelines/automatic_eda_acceptance_test.py b/python/functions/pipelines/automatic_eda_acceptance_test.py new file mode 100644 index 00000000..d5d7ddc5 --- /dev/null +++ b/python/functions/pipelines/automatic_eda_acceptance_test.py @@ -0,0 +1,466 @@ +"""Batería de tests de ACEPTACIÓN del AutomaticEDA — "que cada AEDA salga como queremos". + +Esta suite es la red de seguridad del subsistema EDA del grupo `eda`: garantiza +que CADA capítulo de un informe AutomaticEDA sale poblado y con su contenido +esencial, que la feature de capítulos sueltos (``only_chapters``) resuelve sus +dependencias de cómputo, que los capítulos opcionales devuelven None cuando no +aplican, que el informe de carpeta multi-tabla detecta la FK, y que el Markdown +trae el apéndice completo (matriz de asociación entera + describe con +skew/kurtosis). A diferencia de los tests unitarios de cada capítulo, aquí se +ejercita el pipeline END-TO-END sobre un dataset sintético determinista que +activa todos los capítulos a la vez. + +Determinismo: el dataset se genera con ``seed`` fijo y el pipeline corre sin LLM +(``profile_level='standard'``), de modo que el manifest y el Markdown son +reproducibles entre corridas. Un único render `standard` se reutiliza vía un +fixture de scope module para no repetir el cómputo caro. + +dict-no-throw: los pipelines del grupo `eda` nunca lanzan; aquí se asserta sobre +``status == 'ok'`` y luego sobre el contenido concreto del manifest / Markdown. + +Honestidad (DoD): los asserts comprueban CONTENIDO real (texto esencial de cada +capítulo), no solo el heading. Si un capítulo dejara de emitir su contenido (un +cambio rompiera la distribución numérica, el Isolation Forest, la matriz de +correlación completa, …), el test correspondiente FALLA nombrando el capítulo y +el fragmento ausente — no se ablanda para que pase. +""" + +import json +import os +import subprocess +import sys + +import pytest + +_HERE = os.path.dirname(os.path.abspath(__file__)) +_FUNCTIONS = os.path.abspath(os.path.join(_HERE, "..", "..")) # python/functions +if _FUNCTIONS not in sys.path: + sys.path.insert(0, _FUNCTIONS) + +from datascience.automatic_eda import CHAPTER_ORDER # noqa: E402 +from datascience.generate_synthetic_eda_folder import ( # noqa: E402 + generate_synthetic_eda_folder, +) +from datascience.generate_synthetic_eda_table import ( # noqa: E402 + generate_synthetic_eda_table, +) +from pipelines.render_automatic_eda import render_automatic_eda # noqa: E402 +from pipelines.render_automatic_eda_folder import ( # noqa: E402 + render_automatic_eda_folder, +) + +# --------------------------------------------------------------------------- # +# Parámetros deterministas del fixture de oro. +# --------------------------------------------------------------------------- # +SEED = 42 +N_ROWS = 800 +TABLE = "synthetic" + +# El capítulo `analisis_llm` SOLO se computa con run_llm=True; en el preset +# `standard` (sin LLM, lo que esta suite usa) no debe aparecer. Por eso los +# capítulos esperados en un informe `standard` son todos los de CHAPTER_ORDER +# MENOS analisis_llm. CHAPTER_ORDER es la fuente de verdad de los 16 capítulos +# del motor (portada … glosario). +LLM_ONLY_CHAPTERS = {"analisis_llm"} +EXPECTED_STANDARD = [c for c in CHAPTER_ORDER if c not in LLM_ONLY_CHAPTERS] + + +def _pdf_text(path): + """Texto del PDF vía pdftotext, o None si la herramienta no está disponible.""" + try: + out = subprocess.run( + ["pdftotext", "-layout", path, "-"], + capture_output=True, text=True, timeout=60, + ) + return out.stdout if out.returncode == 0 else None + except Exception: # noqa: BLE001 — la verificación principal es sobre el MD. + return None + + +def _manifest_chapters(result): + """Set de ids de capítulo presentes en el manifest del resultado.""" + with open(result["manifest_path"], encoding="utf-8") as fh: + return set((json.load(fh).get("chapters") or {}).keys()) + + +# --------------------------------------------------------------------------- # +# Fixtures de scope module: el dataset sintético se genera UNA vez y el render +# `standard` se computa UNA vez; todos los tests de contenido lo reutilizan. +# --------------------------------------------------------------------------- # +@pytest.fixture(scope="module") +def synth_db(tmp_path_factory): + """Tabla sintética determinista que activa los 16 capítulos del motor.""" + d = tmp_path_factory.mktemp("aeda_accept_synth") + db = str(d / "synthetic.duckdb") + g = generate_synthetic_eda_table(db, TABLE, n_rows=N_ROWS, seed=SEED) + assert g["status"] == "ok", g.get("error") + return {"db": db, "table": TABLE, "gen": g} + + +@pytest.fixture(scope="module") +def standard_run(synth_db, tmp_path_factory): + """Render AutomaticEDA `standard` (sin LLM) sobre el dataset sintético. + + Devuelve el dict del pipeline más el manifest cargado, el texto del Markdown + y el del PDF (si pdftotext está). Reutilizado por la mayoría de los tests. + """ + out = str(tmp_path_factory.mktemp("aeda_accept_std")) + r = render_automatic_eda( + synth_db["db"], synth_db["table"], + profile_level="standard", out_dir=out, basename="synth_std", + ) + assert r["status"] == "ok", r.get("error") + with open(r["manifest_path"], encoding="utf-8") as fh: + manifest = json.load(fh) + md = open(r["aeda_md_path"], encoding="utf-8").read() + return { + "r": r, + "manifest": manifest, + "chapters": manifest.get("chapters") or {}, + "md": md, + "pdf_text": _pdf_text(r["pdf_path"]), + } + + +@pytest.fixture(scope="module") +def minimal_db(tmp_path_factory): + """Tabla mínima SIN texto libre, SIN fecha y SIN lat/lon. + + Sirve para comprobar que text_distr / timeseries / geospatial devuelven None + (no aparecen en el manifest) y el EDA no peta. Solo numéricas continuas + + una categórica de baja cardinalidad. + """ + import random + + import duckdb + + d = tmp_path_factory.mktemp("aeda_accept_min") + db = str(d / "minimal.duckdb") + con = duckdb.connect(db) + con.execute("CREATE TABLE minimal (a DOUBLE, b DOUBLE, c INTEGER, grp VARCHAR)") + random.seed(7) + rows = [ + (round(random.gauss(10, 2), 3), round(random.gauss(50, 5), 3), + random.randint(1, 100), ["x", "y", "z"][i % 3]) + for i in range(120) + ] + con.executemany("INSERT INTO minimal VALUES (?,?,?,?)", rows) + con.close() + return {"db": db, "table": "minimal"} + + +# --------------------------------------------------------------------------- # +# 1) COBERTURA DE CAPÍTULOS (golden) — el manifest standard trae los 15 +# capítulos no-LLM esperados, ninguno falta, y analisis_llm NO sale sin LLM. +# --------------------------------------------------------------------------- # +def test_standard_cubre_todos_los_capitulos_esperados(standard_run): + chapters = set(standard_run["chapters"].keys()) + expected = set(EXPECTED_STANDARD) + missing = expected - chapters + assert not missing, ( + "capítulos esperados ausentes del manifest standard: " + f"{sorted(missing)} (presentes: {sorted(chapters)})" + ) + # analisis_llm requiere run_llm=True: en standard NO debe aparecer. + assert "analisis_llm" not in chapters, ( + "analisis_llm apareció sin LLM: el preset standard no debería computarlo" + ) + + +def test_manifest_top_level_es_valido(standard_run): + """El manifest declara el motor y un dict de capítulos con metadatos por id.""" + man = standard_run["manifest"] + assert man.get("engine") == "AutomaticEDA" + assert man.get("engine_version") + chapters = standard_run["chapters"] + # Cada capítulo trae version + nº de páginas/slides (formato del manifest). + for cid, meta in chapters.items(): + assert meta.get("version"), f"capítulo {cid} sin version en el manifest" + assert (meta.get("n_pages") or 0) > 0, f"capítulo {cid} con 0 páginas" + + +# --------------------------------------------------------------------------- # +# 2) CONTENIDO CLAVE POR CAPÍTULO (acceptance) — cada capítulo trae su contenido +# ESENCIAL en el Markdown, no solo el heading. Un fragmento ausente nombra el +# capítulo y el texto que falta. +# --------------------------------------------------------------------------- # +# Fragmentos de texto ESTABLE que cada capítulo emite en el Markdown del dataset +# sintético. No son números frágiles: son etiquetas/estructura del capítulo más +# nombres de columna del fixture. Si un capítulo deja de poblar su contenido, su +# fragmento desaparece y el test falla nombrándolo. +CHAPTER_NEEDLES = { + "portada": ["800 filas", "19 columnas"], + "overview": ["Primeras filas (df.head)", "Diccionario de columnas", + "customer_id", "signup_date"], + "num_distr": ["Distribuciones numéricas", "vallas Tukey", "income"], + "cat_distr": ["Distribuciones categóricas", "Entropía", "Top categorías", + "country"], + "text_distr": ["Texto libre (NLP)", "TTR", "Términos más frecuentes", + "Idioma dominante"], + "calidad": ["Cómo se calcula la calidad", "Calidad global"], + "missingness": ["Datos faltantes", "Celdas faltantes (global)", + "Faltantes por columna"], + "outliers": ["Valores atípicos por columna", "Filas atípicas (multivariante)", + "Isolation Forest", "Filas analizadas"], + "correlacion": ["Matriz de asociación", "Pares más correlacionados"], + "relaciones": ["Candidatas a clave primaria", "customer_id"], + "modelos": ["PCA — varianza explicada", "Segmentación (KMeans)"], + "timeseries": ["Series temporales", "Columna de fecha", "signup_date"], + "geospatial": ["Análisis geoespacial", "Extensión geográfica", "Centroide"], + "agregacion": ["Agregación por grupos", "Agrupado por"], + "glosario": ["Glosario de términos", + "### Isolation Forest (anomalías multivariantes)", + "### PCA (componentes principales)"], +} + + +def test_needles_cubren_exactamente_los_capitulos_standard(): + """Guard de mantenimiento: las needles cubren los mismos 15 capítulos no-LLM. + + Si alguien añade un capítulo nuevo a CHAPTER_ORDER, este test recuerda que + hay que documentar su contenido esencial aquí (o marcarlo como LLM-only).""" + assert set(CHAPTER_NEEDLES.keys()) == set(EXPECTED_STANDARD), ( + "CHAPTER_NEEDLES desincronizado con los capítulos esperados de standard: " + f"falta needles para {set(EXPECTED_STANDARD) - set(CHAPTER_NEEDLES)}, " + f"sobra {set(CHAPTER_NEEDLES) - set(EXPECTED_STANDARD)}" + ) + + +@pytest.mark.parametrize("chapter_id", list(CHAPTER_NEEDLES.keys())) +def test_capitulo_trae_su_contenido_esencial(standard_run, chapter_id): + md = standard_run["md"] + # Pre-condición: el capítulo está en el manifest (cobertura). Si no, es un + # fallo de cobertura, no de contenido — se reporta como tal. + assert chapter_id in standard_run["chapters"], ( + f"capítulo {chapter_id} ausente del manifest (fallo de cobertura)" + ) + for needle in CHAPTER_NEEDLES[chapter_id]: + assert needle in md, ( + f"capítulo '{chapter_id}': falta su contenido esencial en el Markdown " + f"— fragmento ausente: {needle!r}" + ) + + +def test_outliers_isolation_forest_poblado_no_degradado(standard_run): + """El bloque multivariante (Isolation Forest) sale con datos, no degradado.""" + md = standard_run["md"] + assert "Anomalías multivariantes" in md + assert "Filas analizadas" in md, "el Isolation Forest no trae su tabla poblada" + assert "No se pudo analizar la anomalía multivariante" not in md, ( + "el bloque multivariante salió degradado en el informe completo" + ) + # El perfil trae el bloque de modelos con los outliers multivariantes. + models = (standard_run["r"]["profile"] or {}).get("models") or {} + assert models.get("outliers") is not None, "profile['models']['outliers'] vacío" + + +# --------------------------------------------------------------------------- # +# 3) CAPÍTULOS SUELTOS CON DEPS RESUELTAS (acceptance de only_chapters) — pedir +# un capítulo suelto lo deja POBLADO porque la resolución de dependencias +# activa el cómputo que necesita, aunque el caller no lo pidiera. +# --------------------------------------------------------------------------- # +def test_only_outliers_isolation_forest_poblado(synth_db, tmp_path): + """only=['outliers'] sin run_models explícito → IsolationForest poblado.""" + out = str(tmp_path / "only_out") + r = render_automatic_eda( + synth_db["db"], synth_db["table"], + only_chapters=["outliers"], out_dir=out, basename="only_outliers", + ) + assert r["status"] == "ok", r.get("error") + # Documento = portada + outliers + glosario, nada más. + assert _manifest_chapters(r) == {"portada", "outliers", "glosario"} + md = open(r["aeda_md_path"], encoding="utf-8").read() + assert "Filas atípicas (multivariante)" in md + assert "Filas analizadas" in md, "Isolation Forest sin tabla poblada" + assert "No se pudo analizar la anomalía multivariante" not in md, ( + "el multivariante salió degradado pese a resolver las deps" + ) + # La resolución activó run_models → el perfil trae el bloque de modelos. + assert ((r["profile"] or {}).get("models") or {}).get("outliers") is not None + + +def test_only_timeseries_rango_temporal_presente(synth_db, tmp_path): + """only=['timeseries'] → rango temporal poblado (run_series resuelto).""" + out = str(tmp_path / "only_ts") + r = render_automatic_eda( + synth_db["db"], synth_db["table"], + only_chapters=["timeseries"], out_dir=out, basename="only_ts", + ) + assert r["status"] == "ok", r.get("error") + assert "timeseries" in _manifest_chapters(r) + md = open(r["aeda_md_path"], encoding="utf-8").read() + assert "Columna de fecha" in md + assert "signup_date" in md, "la serie no nombra su columna de fecha" + # run_series resuelto por deps → el perfil trae el análisis de serie. + assert (r["profile"] or {}).get("series") is not None, ( + "only=['timeseries'] debe activar run_series por dependencias" + ) + + +def test_only_correlacion_scatters_presentes(synth_db, tmp_path): + """only=['correlacion'] → matriz + scatters de los pares fuertes.""" + out = str(tmp_path / "only_corr") + r = render_automatic_eda( + synth_db["db"], synth_db["table"], + only_chapters=["correlacion"], out_dir=out, basename="only_corr", + ) + assert r["status"] == "ok", r.get("error") + assert _manifest_chapters(r) == {"portada", "correlacion", "glosario"} + md = open(r["aeda_md_path"], encoding="utf-8").read() + assert "Matriz de asociación" in md + assert "Relaciones más fuertes (scatter)" in md, "faltan los scatters" + assert "Dispersión de" in md, "no se emitió ninguna figura de dispersión" + + +# --------------------------------------------------------------------------- # +# 4) NONE CUANDO NO APLICA — sobre una tabla sin texto largo, sin fecha y sin +# lat/lon, text_distr / timeseries / geospatial NO aparecen y el EDA no peta. +# --------------------------------------------------------------------------- # +def test_capitulos_opcionales_ausentes_cuando_no_aplican(minimal_db, tmp_path): + out = str(tmp_path / "minimal_out") + r = render_automatic_eda( + minimal_db["db"], minimal_db["table"], + profile_level="standard", out_dir=out, basename="minimal", + ) + assert r["status"] == "ok", r.get("error") + chapters = _manifest_chapters(r) + for absent in ("text_distr", "timeseries", "geospatial"): + assert absent not in chapters, ( + f"capítulo {absent} apareció en una tabla que no lo justifica " + f"(presentes: {sorted(chapters)})" + ) + # El documento sigue siendo válido: portada + glosario + capítulos que sí + # aplican (overview/num_distr/correlacion al menos). + assert {"portada", "glosario", "overview", "num_distr"} <= chapters + + +# --------------------------------------------------------------------------- # +# 5) FOLDER MULTI-TABLA (acceptance) — el informe de carpeta perfila las N tablas +# y el capítulo de relaciones detecta la FK por containment. +# --------------------------------------------------------------------------- # +def test_folder_multitabla_con_fk_detectada(tmp_path): + fdir = str(tmp_path / "folder") + g = generate_synthetic_eda_folder(fdir, n_rows=300, seed=SEED) + assert g["status"] == "ok", g.get("error") + + out = str(tmp_path / "fout") + rf = render_automatic_eda_folder(fdir, out_dir=out, basename="folder") + assert rf["status"] == "ok", rf.get("error") + + # Las 3 tablas se perfilaron. + assert rf["n_tables"] == 3, f"esperadas 3 tablas, vistas {rf['n_tables']}" + + # El manifest base trae el capítulo de relaciones inter-tabla. + with open(rf["manifest_path"], encoding="utf-8") as fh: + chapters = set((json.load(fh).get("chapters") or {}).keys()) + assert "relaciones" in chapters, ( + f"el documento de carpeta no incluye el capítulo de relaciones: {chapters}" + ) + + # El Markdown nombra las 3 tablas y declara la FK detectada por containment. + md = open(rf["md_path"], encoding="utf-8").read() + for tbl in ("customers", "orders", "reviews"): + assert tbl in md, f"la tabla {tbl} no aparece en el informe de carpeta" + assert "FK candidatas" in md, "no se declaran las FK candidatas" + assert "orders.customer_id" in md and "customers.customer_id" in md, ( + "la FK orders→customers no se detectó por containment" + ) + assert "reviews.customer_id" in md, "la FK reviews→customers no se detectó" + + +# --------------------------------------------------------------------------- # +# 6) MD COMPLETITUD (regresión) — el Markdown trae el apéndice con la matriz de +# asociación COMPLETA (todos los pares, no solo el top) y el describe con +# skew/kurtosis de todas las numéricas. Protege un fix ya mergeado. +# --------------------------------------------------------------------------- # +def test_md_apendice_matriz_correlacion_completa(standard_run): + md = standard_run["md"] + assert "Matriz de asociación — todos los pares" in md, ( + "falta el apéndice con la matriz de asociación completa" + ) + # Un par num-num de correlación BAJA que el top del capítulo NUNCA mostraría: + # su presencia prueba que el apéndice lista TODOS los pares, no solo el top. + assert "income ↔ longitude" in md, ( + "el apéndice no contiene los pares de baja correlación: no es la matriz " + "completa, solo el top-k del capítulo" + ) + + +def test_md_apendice_describe_con_skew_kurtosis(standard_run): + md = standard_run["md"] + assert "Estadísticos numéricos completos (describe)" in md, ( + "falta el apéndice describe completo" + ) + # La cabecera del describe del apéndice lleva las columnas skew y kurtosis + # (subcadena única de ese header). Sin ellas el describe está incompleto. + assert "| skew | kurtosis |" in md, ( + "el describe del apéndice no trae las columnas skew/kurtosis" + ) + + +# --------------------------------------------------------------------------- # +# 7) LAS 3 SALIDAS NO-VACÍAS — PDF con páginas, PPTX con slides, MD con un mínimo +# de caracteres, y los tres archivos en disco. Manifest válido. +# --------------------------------------------------------------------------- # +def test_tres_salidas_no_vacias(standard_run): + r = standard_run["r"] + assert r["pdf_path"] and os.path.exists(r["pdf_path"]) + assert r["pptx_path"] and os.path.exists(r["pptx_path"]) + assert r["aeda_md_path"] and os.path.exists(r["aeda_md_path"]) + assert (r["n_pages"] or 0) > 0, "el PDF no tiene páginas" + assert (r["n_slides"] or 0) > 0, "el PPTX no tiene slides" + # El informe completo es grande: un mínimo holgado protege contra un MD vacío + # o truncado sin atarse a un tamaño exacto. + assert (r["md_chars"] or 0) > 10000, f"MD demasiado corto: {r['md_chars']} chars" + assert r["manifest_path"] and os.path.exists(r["manifest_path"]) + + +def test_pdf_texto_extraible_con_contenido(standard_run): + """Si pdftotext está disponible, el PDF debe traer texto real (no solo + imágenes): la portada nombra el dataset y su forma. Si no está la + herramienta, el test se omite (no es un fallo del EDA).""" + txt = standard_run["pdf_text"] + if txt is None: + pytest.skip("pdftotext no disponible") + assert len(txt) > 5000, "el PDF apenas tiene texto extraíble" + assert "Portada" in txt or "synthetic" in txt, ( + "el texto del PDF no contiene la portada esperada" + ) + + +# --------------------------------------------------------------------------- # +# DETERMINISMO — dos renders del MISMO dataset producen el MISMO manifest +# (mismos capítulos y mismos n_pages/n_slides por capítulo). El generated_at +# difiere por timestamp, por eso se compara el dict de capítulos, no el archivo. +# --------------------------------------------------------------------------- # +def test_render_es_determinista(synth_db, tmp_path): + out1 = str(tmp_path / "det1") + out2 = str(tmp_path / "det2") + r1 = render_automatic_eda(synth_db["db"], synth_db["table"], + profile_level="standard", out_dir=out1, basename="d1") + r2 = render_automatic_eda(synth_db["db"], synth_db["table"], + profile_level="standard", out_dir=out2, basename="d2") + assert r1["status"] == "ok" and r2["status"] == "ok" + c1 = json.load(open(r1["manifest_path"], encoding="utf-8")).get("chapters") + c2 = json.load(open(r2["manifest_path"], encoding="utf-8")).get("chapters") + assert c1 == c2, "el manifest no es determinista entre dos renders del mismo dataset" + + +# --------------------------------------------------------------------------- # +# SLOW (opcional, skippeable) — informe `full` con narrativa LLM. Requiere red / +# credenciales y NO es determinista, por eso está apagado salvo opt-in explícito +# vía la variable de entorno EDA_ACCEPT_LLM=1. Se omite con skipif (no con un +# marker custom) para no depender de registro de marks en la config del repo. +# --------------------------------------------------------------------------- # +@pytest.mark.skipif( + os.environ.get("EDA_ACCEPT_LLM") != "1", + reason="full+LLM es lento/no determinista; exporta EDA_ACCEPT_LLM=1 para correrlo", +) +def test_full_incluye_capitulo_analisis_llm(synth_db, tmp_path): + out = str(tmp_path / "full") + r = render_automatic_eda(synth_db["db"], synth_db["table"], + profile_level="full", out_dir=out, basename="full") + assert r["status"] == "ok", r.get("error") + assert "analisis_llm" in _manifest_chapters(r), ( + "el preset full debe incluir el capítulo de análisis LLM" + ) From 80d10010f5c49fa1be01e3e42710b4d7bad442a8 Mon Sep 17 00:00:00 2001 From: Egutierrez <egutierrez@dead.dd> Date: Tue, 30 Jun 2026 22:44:33 +0200 Subject: [PATCH 48/53] feat(eda): portada cap01 + zebra global y emphasis de render MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Itera el capítulo PORTADA del AutomaticEDA y dos mejoras globales de los renderers PDF/PPTX: 1. Zebra global (PDF): _place_kv_table ahora sombrea las filas pares igual que las DataTable, así toda tabla del documento queda rayada (no solo las DataTable). Mismo patrón coherente al partir/repetir cabecera. 2. Portada usa la descripción LLM rica (profile['llm']['summary']) cuando el perfil la tiene; se elimina del fallback derivado el texto ruido "active la interpretación LLM (run_llm)…". No fuerza llamadas LLM en el capítulo, solo consume profile['llm'] si está. 3. Se quita el bloque "Criterios de calidad" de la portada (PDF y PPTX); el score "Calidad" se mantiene. 4. "Resumen del análisis" (PDF): los valores se alinean al margen derecho via el nuevo KVTable.value_align="right". 5. Nombre del dataset en la portada PPTX más grande (44pt) y subrayado via los nuevos hints Heading.underline / Heading.size_pt (el PDF los ignora). Bump CHAPTER_VERSION de portada 1.2.0 -> 1.3.0. Verificado: suite 213 passed / 1 skipped (incl. aceptación de los 16 capítulos); golden zebra = 185 filas zebra en 13 capítulos del PDF completo; portada con run_llm sin "Criterios de calidad", con descripción LLM rica y valores a la derecha; PPTX con nombre 44pt subrayado; edge sin LLM cae al fallback derivado sin ruido; fn index sin error. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> --- .../automatic_eda/chapters/portada.py | 25 +++++++--------- .../datascience/automatic_eda/model.py | 30 ++++++++++++++++--- .../automatic_eda/render_pdf_impl.py | 30 ++++++++++++++++--- .../automatic_eda/render_pptx_impl.py | 18 ++++++++--- 4 files changed, 77 insertions(+), 26 deletions(-) diff --git a/python/functions/datascience/automatic_eda/chapters/portada.py b/python/functions/datascience/automatic_eda/chapters/portada.py index 409322f7..b1a28366 100644 --- a/python/functions/datascience/automatic_eda/chapters/portada.py +++ b/python/functions/datascience/automatic_eda/chapters/portada.py @@ -26,7 +26,7 @@ from datetime import datetime, timezone from .. import model -CHAPTER_VERSION = "1.2.0" +CHAPTER_VERSION = "1.3.0" CHAPTER_ID = "portada" CHAPTER_TITLE = "Portada" @@ -35,12 +35,9 @@ CHAPTER_TITLE = "Portada" # row represents) from it when the LLM layer ran (``run_llm``). _LLM_KEY = "llm" -# Default human description of what the table quality score measures. Chapters -# can override it via ctx["quality_criteria"]. -_DEFAULT_QUALITY_CRITERIA = ( - "media de los scores por columna (0–100): completitud (sin nulos/vacíos), " - "validez (tipo y rango coherentes) y consistencia (sin duplicados/constantes)." -) +# Font size (pt) for the dataset name on the PPTX cover slide — notably larger +# than the default H1 so the dataset name stands out (shown underlined too). +_PPTX_TITLE_PT = 44.0 def _storage_from_source(source: str) -> str: @@ -120,7 +117,8 @@ def _summary_blocks(summary) -> list: blocks = [model.Heading(text="Resumen del análisis", level=2)] if rows: - blocks.append(model.KVTable(rows=rows)) + # Values pinned to the right margin (numbers flush right, label left). + blocks.append(model.KVTable(rows=rows, value_align="right")) if titles: bullets = "\n".join(f"- {model._safe_str(t)}" for t in titles) blocks.append(model.Markdown( @@ -213,9 +211,7 @@ def _derive_description(profile: dict, ctx: dict) -> str: score = profile.get("quality_score") if score is not None: parts.append(f"Calidad media estimada: {score}/100.") - parts.append( - "Resumen derivado del perfil; active la interpretación LLM (`run_llm`) " - "para una descripción de negocio más rica.") + parts.append("Resumen derivado del perfil.") return " ".join(parts) @@ -259,7 +255,6 @@ def build_portada(profile: dict, ctx: dict): shape = f"{_fmt_int(n_rows)} filas × {_fmt_int(n_cols)} columnas" score = profile.get("quality_score") - quality_criteria = ctx.get("quality_criteria") or _DEFAULT_QUALITY_CRITERIA quality_value = "—" if score is None else f"{score} / 100" llm = _llm_block(profile, ctx) @@ -282,8 +277,11 @@ def build_portada(profile: dict, ctx: dict): # Title + dataset size shown together and BIG (Heading) at the top, kept on # the same page (Group). The size is no longer buried in the metadata table. + # The dataset name is shown big and underlined on the PPTX cover slide + # (size_pt/underline are honoured by the PPTX renderer; the PDF ignores them). cover = [ - model.Heading(text=str(dataset_name), level=1), + model.Heading(text=str(dataset_name), level=1, underline=True, + size_pt=_PPTX_TITLE_PT), model.Markdown(text="**Automatic-EDA** · informe exploratorio automático"), model.Heading(text=shape, level=2), ] @@ -295,7 +293,6 @@ def build_portada(profile: dict, ctx: dict): ("Almacenamiento", storage), ("Generado", when), ("Calidad", quality_value), - ("Criterios de calidad", quality_criteria), ]), model.Heading(text="Descripción", level=2), model.Markdown(text=str(description)), diff --git a/python/functions/datascience/automatic_eda/model.py b/python/functions/datascience/automatic_eda/model.py index 7237df0b..9171652e 100644 --- a/python/functions/datascience/automatic_eda/model.py +++ b/python/functions/datascience/automatic_eda/model.py @@ -38,10 +38,18 @@ ENGINE_NAME = "AutomaticEDA" # --------------------------------------------------------------------------- # @dataclass class Heading: - """A section heading. ``level`` 1 (largest) .. 3 (smallest).""" + """A section heading. ``level`` 1 (largest) .. 3 (smallest). + + ``underline`` and ``size_pt`` are optional emphasis hints honoured by the + PPTX renderer (the cover uses them to show the dataset name big and + underlined). ``size_pt`` overrides the per-level font size when set; the PDF + renderer ignores both so its layout is unchanged. + """ text: str = "" level: int = 1 + underline: bool = False + size_pt: Optional[float] = None kind: str = field(default="heading", init=False) @@ -62,10 +70,17 @@ class Markdown: @dataclass class KVTable: - """A two-column key/value table. ``rows`` is a list of ``(label, value)``.""" + """A two-column key/value table. ``rows`` is a list of ``(label, value)``. + + ``value_align`` controls the horizontal alignment of the value column in the + PDF renderer: ``"left"`` (default) keeps values next to the label column; + ``"right"`` pins them to the right margin (used by the cover's analysis + summary so the numbers line up flush right). + """ rows: list = field(default_factory=list) title: Optional[str] = None + value_align: str = "left" kind: str = field(default="kv_table", init=False) @@ -210,13 +225,20 @@ def as_block(obj: Any): # Build only with fields the dataclass accepts (ignore extras). try: if cls is Heading: + size_pt = obj.get("size_pt") return Heading(text=_safe_str(obj.get("text")), - level=int(obj.get("level", 1) or 1)) + level=int(obj.get("level", 1) or 1), + underline=bool(obj.get("underline", False)), + size_pt=(float(size_pt) + if isinstance(size_pt, (int, float)) + else None)) if cls is Markdown: return Markdown(text=_safe_str(obj.get("text"))) if cls is KVTable: return KVTable(rows=list(obj.get("rows") or []), - title=obj.get("title")) + title=obj.get("title"), + value_align=_safe_str( + obj.get("value_align")) or "left") if cls is DataTable: return DataTable(header=list(obj.get("header") or []), rows=list(obj.get("rows") or []), diff --git a/python/functions/datascience/automatic_eda/render_pdf_impl.py b/python/functions/datascience/automatic_eda/render_pdf_impl.py index 06adea4b..30115de0 100644 --- a/python/functions/datascience/automatic_eda/render_pdf_impl.py +++ b/python/functions/datascience/automatic_eda/render_pdf_impl.py @@ -317,10 +317,18 @@ def _place_kv_table(st: _PdfState, block) -> None: if title: _place_heading(st, model.Heading(title, level=2)) rows = getattr(block, "rows", []) or [] + # ``value_align="right"`` pins the value column to the right margin (label + # left, number flush right) — used by the cover's analysis summary. + right = str(getattr(block, "value_align", "left")).lower() == "right" key_w = 1.9 # inches reserved for the label column. + # Right-aligned values wrap against the full usable width minus the label + # column; left-aligned values wrap against the value column only. val_chars = tl.chars_per_line(_USABLE_W - key_w - 0.1, _FS_BODY) lh = tl.line_height_in(_FS_BODY) - for row in rows: + # ``data_idx`` is the 0-based logical row index: even rows (1-based) are + # zebra-shaded → 0-based odd indices, matching the data-table convention so + # every table in the document carries the same striping. + for data_idx, row in enumerate(rows): try: label, value = row[0], row[1] except Exception: # noqa: BLE001 @@ -329,11 +337,25 @@ def _place_kv_table(st: _PdfState, block) -> None: row_h = lh * len(v_lines) + _ROW_VPAD _ensure_space(st, row_h) y0 = st.y + # Faint zebra fill for even rows, drawn first (zorder 0) so striping + # never hides the text/value drawn on top. + if data_idx % 2 == 1: + st.fig.add_artist(Rectangle( + (_xf(_ML), _yf(y0 + row_h)), _xf(_ML + _USABLE_W) - _xf(_ML), + _yf(y0) - _yf(y0 + row_h), transform=st.fig.transFigure, + color=_ZEBRA, lw=0, zorder=0)) st.fig.text(_xf(_ML), _yf(y0), tl.strip_inline_md(model._safe_str(label)), - fontsize=_FS_BODY, color=_MUTED, ha="left", va="top") + fontsize=_FS_BODY, color=_MUTED, ha="left", va="top", + zorder=2) for k, vl in enumerate(v_lines): - st.fig.text(_xf(_ML + key_w), _yf(y0 + k * lh), vl, - fontsize=_FS_BODY, color=_INK, ha="left", va="top") + if right: + st.fig.text(_xf(_ML + _USABLE_W), _yf(y0 + k * lh), vl, + fontsize=_FS_BODY, color=_INK, ha="right", + va="top", zorder=2) + else: + st.fig.text(_xf(_ML + key_w), _yf(y0 + k * lh), vl, + fontsize=_FS_BODY, color=_INK, ha="left", + va="top", zorder=2) st.y = y0 + row_h st.y += _GAP diff --git a/python/functions/datascience/automatic_eda/render_pptx_impl.py b/python/functions/datascience/automatic_eda/render_pptx_impl.py index 7a813945..21b9e0ce 100644 --- a/python/functions/datascience/automatic_eda/render_pptx_impl.py +++ b/python/functions/datascience/automatic_eda/render_pptx_impl.py @@ -135,7 +135,7 @@ def _ensure(st: _PptxState, height: float) -> None: def _add_text(st: _PptxState, lines: list, fs: float, color, bold=False, - italic=False, indent=0.0, bullet=False) -> None: + italic=False, indent=0.0, bullet=False, underline=False) -> None: lh = tl.line_height_in(fs) height = lh * len(lines) + 0.05 _ensure(st, height) @@ -153,6 +153,7 @@ def _add_text(st: _PptxState, lines: list, fs: float, color, bold=False, run.font.size = Pt(fs) run.font.bold = bold run.font.italic = italic + run.font.underline = underline run.font.color.rgb = _rgb(color) st.y += height @@ -206,10 +207,16 @@ def _add_rich_text(st: _PptxState, rich_lines: list, fs: float, color, def _place_heading(st: _PptxState, block) -> None: level = max(1, min(3, int(getattr(block, "level", 1) or 1))) fs = {1: _FS_H1, 2: _FS_H2, 3: _FS_H3}[level] + # Optional per-heading emphasis (cover dataset name): a larger font and an + # underline. ``size_pt`` overrides the per-level size when set. + size_override = getattr(block, "size_pt", None) + if isinstance(size_override, (int, float)) and size_override > 0: + fs = float(size_override) + underline = bool(getattr(block, "underline", False)) text = tl.strip_inline_md(getattr(block, "text", "")) st.last_heading = text or st.last_heading lines = tl.wrap(text, tl.chars_per_line(_USABLE_W, fs)) - _add_text(st, lines, fs, _INK, bold=True) + _add_text(st, lines, fs, _INK, bold=True, underline=underline) st.y += 0.04 @@ -552,9 +559,11 @@ def _place_note(st: _PptxState, block) -> None: # WITHOUT drawing it so a Group can move whole to the next slide before drawing. # Over-estimating only triggers an earlier slide break, never a content cut. # --------------------------------------------------------------------------- # -def _measure_heading_text(text: str, level: int) -> float: +def _measure_heading_text(text: str, level: int, size_pt=None) -> float: level = max(1, min(3, int(level or 1))) fs = {1: _FS_H1, 2: _FS_H2, 3: _FS_H3}[level] + if isinstance(size_pt, (int, float)) and size_pt > 0: + fs = float(size_pt) lines = tl.wrap(tl.strip_inline_md(text), tl.chars_per_line(_USABLE_W, fs)) return tl.line_height_in(fs) * len(lines) + 0.05 + 0.04 @@ -679,7 +688,8 @@ def _measure_block(st: _PptxState, block) -> float: try: if kind == "heading": return _measure_heading_text(getattr(block, "text", ""), - getattr(block, "level", 1)) + getattr(block, "level", 1), + size_pt=getattr(block, "size_pt", None)) if kind == "markdown": return _measure_markdown(block) if kind in ("figure", "image"): From 64306f3b1c5a0d2461d2bbdf57178b39349c8c37 Mon Sep 17 00:00:00 2001 From: Egutierrez <egutierrez@dead.dd> Date: Wed, 1 Jul 2026 01:13:02 +0200 Subject: [PATCH 49/53] feat(eda): overview enriquece diccionario y describe con descripcion+unidad del LLM MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit La tabla DICCIONARIO de columnas del capitulo overview gana columnas "Descripcion" y "Unidad", y la tabla DESCRIBE gana "Unidad", consumiendo profile['llm']['dictionary'] (entradas column/description/business_meaning/unit producidas por eda_llm_insights) emparejadas por nombre de columna. Lectura defensiva: sin bloque LLM (run_llm no corrio) las celdas degradan a "—" y las tablas siguen renderizando. No recalcula nada ni llama al LLM. CHAPTER_VERSION 1.1.0 -> 1.2.0. Tests: golden (descripcion+unidad pobladas para income), edge (sin LLM -> "—"), fallback ctx['llm'], y render PDF con las columnas nuevas visibles. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> --- .../automatic_eda/chapters/overview.py | 89 ++++++++++++++++--- .../automatic_eda/chapters/overview_test.py | 85 +++++++++++++++++- 2 files changed, 162 insertions(+), 12 deletions(-) diff --git a/python/functions/datascience/automatic_eda/chapters/overview.py b/python/functions/datascience/automatic_eda/chapters/overview.py index f3dc8b53..67b709b4 100644 --- a/python/functions/datascience/automatic_eda/chapters/overview.py +++ b/python/functions/datascience/automatic_eda/chapters/overview.py @@ -7,11 +7,21 @@ as needed, the renderers paginate): NOT carry the raw head, so this is read from ``ctx['head_rows']`` / ``profile['head_rows']`` (a list of row dicts). When absent the chapter shows an honest placeholder documenting the missing key instead of inventing data. -2. Column dictionary — name / type / nulls / non-null examples. Examples come +2. Column dictionary — name / type / nulls / non-null examples plus, when the + LLM layer ran, the business **description** and **unit** of each column so the + reader knows at a glance what every column is and in which unit. Examples come from ``columns[i]['examples']`` when present; otherwise they are derived from real non-null profile values (categorical top values, numeric min/median/max) so the cell is never empty nor fabricated. -3. ``df.describe`` — mean / median / min / max / std for every numeric column. +3. ``df.describe`` — mean / median / min / max / std for every numeric column, + plus its **unit** (same LLM source) so the stats read in context. + +The description/unit come from the ``llm`` block that ``eda_llm_insights`` (group +``eda``) already stored in the profile (``profile['llm']['dictionary']``, a list +of ``{"column","description","business_meaning","unit"}`` entries) — this chapter +only **consumes** it, matching by column name; it never calls the LLM nor +recomputes anything. When the block is absent (``run_llm`` did not run) those +cells degrade to ``"—"`` and the tables still render. Contract: build_<id>(profile, ctx) -> Chapter | None ; CHAPTER_VERSION = "x.y.z". """ @@ -20,13 +30,59 @@ from __future__ import annotations from .. import model -CHAPTER_VERSION = "1.1.0" +CHAPTER_VERSION = "1.2.0" CHAPTER_ID = "overview" CHAPTER_TITLE = "Overview" # Profile/ctx keys the calculation phase must add for a full head + examples. HEAD_KEY = "head_rows" # list[dict] — df.head(n) EXAMPLES_KEY = "examples" # per column: list of non-null sample values +LLM_KEY = "llm" # interpretive block from eda_llm_insights + + +def _llm_dict_index(profile: dict, ctx: dict) -> dict: + """Map column name -> its LLM dictionary entry (description/unit/...). + + Reads the ``llm.dictionary`` list that ``eda_llm_insights`` stored in the + profile (``profile['llm']``; falls back to ``ctx['llm']``). Returns an empty + dict when no LLM block ran, so the caller degrades to "—" cells. Fully + defensive: never raises on malformed input. + """ + llm = profile.get(LLM_KEY) + if not isinstance(llm, dict): + llm = ctx.get(LLM_KEY) + if not isinstance(llm, dict): + return {} + entries = llm.get("dictionary") + if not isinstance(entries, (list, tuple)): + return {} + index: dict = {} + for e in entries: + if not isinstance(e, dict): + continue + col = e.get("column") + if col is None: + continue + index[model._safe_str(col)] = e + return index + + +def _llm_desc(entry) -> str: + """Business description of a column from its LLM entry, or "—".""" + if not isinstance(entry, dict): + return "—" + raw = entry.get("description") or entry.get("business_meaning") + text = " ".join(model._safe_str(raw).split()) if raw is not None else "" + return text or "—" + + +def _llm_unit(entry) -> str: + """Unit of a column from its LLM entry, or "—".""" + if not isinstance(entry, dict): + return "—" + raw = entry.get("unit") + text = " ".join(model._safe_str(raw).split()) if raw is not None else "" + return text or "—" def _fmt_num(value, decimals: int = 3) -> str: @@ -104,9 +160,12 @@ def _head_block(profile: dict, ctx: dict): "pasarlo en ctx['head_rows'] para mostrar las primeras filas.") -def _columns_block(profile: dict): +def _columns_block(profile: dict, llm_index: dict): cols = profile.get("columns") or [] - header = ["Columna", "Tipo", "Nulos", "Ejemplos (no nulos)"] + # Descripción / Unidad come from the LLM dictionary (matched by column name); + # they read "—" when run_llm did not run, so the table always renders. + header = ["Columna", "Tipo", "Nulos", "Ejemplos (no nulos)", + "Descripción", "Unidad"] rows = [] for c in cols: if not isinstance(c, dict): @@ -126,15 +185,18 @@ def _columns_block(profile: dict): nulls = str(null_count) else: nulls = "—" - rows.append([name, ctype, nulls, _examples_for(c)]) + entry = llm_index.get(model._safe_str(name)) + rows.append([name, ctype, nulls, _examples_for(c), + _llm_desc(entry), _llm_unit(entry)]) if not rows: return None return model.DataTable(header=header, rows=rows, title="Columnas") -def _describe_block(profile: dict): +def _describe_block(profile: dict, llm_index: dict): cols = profile.get("columns") or [] - header = ["Columna", "mean", "median", "min", "max", "std"] + # "Unidad" (LLM source) lets the reader know in which unit each stat is. + header = ["Columna", "mean", "median", "min", "max", "std", "Unidad"] rows = [] for c in cols: if not isinstance(c, dict) or c.get("inferred_type") != "numeric": @@ -142,13 +204,16 @@ def _describe_block(profile: dict): num = c.get("numeric") or {} if not num: continue + name = c.get("name") or "(col)" + entry = llm_index.get(model._safe_str(name)) rows.append([ - c.get("name") or "(col)", + name, _fmt_num(num.get("mean")), _fmt_num(num.get("median")), _fmt_num(num.get("min")), _fmt_num(num.get("max")), _fmt_num(num.get("std")), + _llm_unit(entry), ]) if not rows: return None @@ -163,16 +228,18 @@ def build_overview(profile: dict, ctx: dict): if not cols and not (ctx.get(HEAD_KEY) or profile.get(HEAD_KEY)): return None + llm_index = _llm_dict_index(profile, ctx) + blocks = [ model.Heading(text="Primeras filas (df.head)", level=2), _head_block(profile, ctx), ] - cols_block = _columns_block(profile) + cols_block = _columns_block(profile, llm_index) if cols_block is not None: blocks.append(model.Heading( text="Diccionario de columnas", level=2)) blocks.append(cols_block) - desc_block = _describe_block(profile) + desc_block = _describe_block(profile, llm_index) if desc_block is not None: blocks.append(model.Heading( text="Resumen estadístico numérico", level=2)) diff --git a/python/functions/datascience/automatic_eda/chapters/overview_test.py b/python/functions/datascience/automatic_eda/chapters/overview_test.py index b66263a1..0f9d985a 100644 --- a/python/functions/datascience/automatic_eda/chapters/overview_test.py +++ b/python/functions/datascience/automatic_eda/chapters/overview_test.py @@ -56,7 +56,21 @@ def _head_rows() -> list: ] -def _profile(with_head: bool = True) -> dict: +def _llm() -> dict: + """Interpretive block as eda_llm_insights stores it under profile['llm'].""" + return { + "summary": "Pasajeros del Titanic.", + "dictionary": [ + {"column": "PassengerId", "description": "Identificador del pasajero", + "business_meaning": "Clave única de cada pasajero", "unit": "id"}, + {"column": "Pclass", "description": "Clase del billete", + "business_meaning": "Clase socioeconómica", "unit": "clase (1-3)"}, + # No entry for Survived/Name/Sex on purpose -> they degrade to "—". + ], + } + + +def _profile(with_head: bool = True, with_llm: bool = False) -> dict: prof = { "table": "titanic", "source": "/data/titanic.csv", @@ -68,6 +82,8 @@ def _profile(with_head: bool = True) -> dict: } if with_head: prof["head_rows"] = _head_rows() + if with_llm: + prof["llm"] = _llm() return prof @@ -185,3 +201,70 @@ def test_edge_none_y_vacio_no_rompen(): assert ch is not None tables = [b for b in _flatten(ch.blocks) if isinstance(b, DataTable)] assert tables and len(tables[0].rows) == 3 + + +def _table_by_header(blocks, marker: str): + """Return the first DataTable whose header contains ``marker``.""" + for b in _flatten(blocks): + if isinstance(b, DataTable) and marker in b.header: + return b + return None + + +def test_golden_diccionario_lleva_descripcion_y_unidad_del_llm(): + # With run_llm: the column dictionary gains "Descripción" and "Unidad" + # columns populated from profile['llm']['dictionary'], matched by name. + ch = build_overview(_profile(with_llm=True), {}) + assert ch is not None + dic = _table_by_header(ch.blocks, "Descripción") + assert dic is not None + assert dic.header == ["Columna", "Tipo", "Nulos", "Ejemplos (no nulos)", + "Descripción", "Unidad"] + by_name = {row[0]: row for row in dic.rows} + # PassengerId has an LLM entry -> description + unit populated. + assert by_name["PassengerId"][4] == "Identificador del pasajero" + assert by_name["PassengerId"][5] == "id" + assert by_name["Pclass"][5] == "clase (1-3)" + # Columns with no LLM entry degrade to "—" without breaking the row. + assert by_name["Survived"][4] == "—" and by_name["Survived"][5] == "—" + + +def test_golden_describe_lleva_unidad_del_llm(): + ch = build_overview(_profile(with_llm=True), {}) + desc = _table_by_header(ch.blocks, "std") + assert desc is not None + assert desc.header[-1] == "Unidad" + by_name = {row[0]: row for row in desc.rows} + assert by_name["PassengerId"][-1] == "id" + assert by_name["Pclass"][-1] == "clase (1-3)" + # Numeric column with no LLM unit still renders, unit "—". + assert by_name["Survived"][-1] == "—" + + +def test_edge_sin_llm_descripcion_unidad_son_guion(): + # No profile['llm'] at all: the new cells degrade to "—" and nothing breaks. + ch = build_overview(_profile(), {}) + assert ch is not None + dic = _table_by_header(ch.blocks, "Unidad") + assert dic is not None + for row in dic.rows: + assert row[4] == "—" and row[5] == "—" + desc = _table_by_header(ch.blocks, "std") + assert all(row[-1] == "—" for row in desc.rows) + + +def test_golden_llm_via_ctx_tambien_funciona(): + # LLM block arriving through ctx['llm'] (fallback path) is consumed too. + ch = build_overview(_profile(with_llm=False), {"llm": _llm()}) + dic = _table_by_header(ch.blocks, "Descripción") + by_name = {row[0]: row for row in dic.rows} + assert by_name["PassengerId"][5] == "id" + + +def test_golden_render_pdf_muestra_descripcion_y_unidad(): + with tempfile.TemporaryDirectory() as d: + out = os.path.join(d, "eda.pdf") + render_automatic_eda_pdf(_profile(with_llm=True), out, {"title": "EDA"}) + txt = _pdf_text(out) + assert "Descripción" in txt and "Unidad" in txt + assert "Identificador del pasajero" in txt From a74a5a047f1006ff97c0122a627993bed1c94445 Mon Sep 17 00:00:00 2001 From: Egutierrez <egutierrez@dead.dd> Date: Wed, 1 Jul 2026 01:34:21 +0200 Subject: [PATCH 50/53] =?UTF-8?q?feat(eda):=20render=20quality=20global=20?= =?UTF-8?q?=E2=80=94=20DPI=20220,=20tablas=20anchas=20como=20imagen,=20lay?= =?UTF-8?q?out=20side=5Fby=5Fside,=20=C3=ADndice=20clicable?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mejoras transversales del motor AutomaticEDA (PDF + PPTX) sobre el modelo de bloques: 1. DPI alto global: toda figura/imagen embebida se rasteriza a 220 dpi (antes 150, y en PDF la página se guardaba a ~100 dpi re-rasterizando los imshow). En PDF se aplica savefig.dpi=220 a la página; el texto sigue vectorial y seleccionable. Permite ampliar en el móvil sin pixelar. Imagen embebida medida: ~1081px (antes ~492px). 2. Tabla ancha → imagen de alta resolución: cuando un DataTable tiene demasiadas columnas para ser legible como texto (criterio _table_fits_as_text), se dibuja entera como una imagen nítida (nueva función render_table_as_figure_py_datascience: cabecera sombreada + zebra) escalada para caber completa, de modo que el lector hace zoom y la lee sin perder datos. Las tablas que sí caben siguen como texto seleccionable / tabla nativa. Aplica en PDF y PPTX. El df.head de 19 columnas del dataset sintético ya no se corta: sale como imagen. 3. Group.layout: nuevo hint retrocompatible (default "stack"). "side_by_side" coloca la tabla a la izquierda (~55%) y la figura a la derecha (~45%) en la misma slide PPTX (cae a apilado si no hay par tabla+figura o no caben); en PDF se trata como "stack" (el ancho A5 móvil no admite dos columnas). Pensado para que el capítulo cat_distr ponga el gráfico al lado de la tabla en PPT. 4. Portada con índice clicable: la lista de capítulos pasa de "Este informe incluye..." (markdown) a un Heading "Índice" + un TocEntry por capítulo. El renderer registra el inicio de cada capítulo y cablea cada entrada como salto real (PDF: link GOTO PyMuPDF; PPTX: salto a slide nativo), reutilizando el mecanismo del glosario clicable. Modelo: Group gana `layout`; nuevo bloque TocEntry; normalizers y __init__ actualizados. Contrato: documentado en docs/automatic_eda_contract.md §11.4 (incluye el contrato exacto del campo layout para el agente de cat_distr). Tests: nuevo render_quality_test.py (13 golden: DPI alto real, tabla ancha→imagen PDF/PPTX, narrow→texto, side_by_side PPTX dos columnas / PDF apilado, índice clicable PDF+PPTX, retrocompatibilidad layout por defecto). render_features_test actualizado al índice nuevo. Suite: 188 passed (módulo) + 38 passed/1 skipped (acceptance + pipeline). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> --- docs/automatic_eda_contract.md | 64 +++- .../datascience/automatic_eda/__init__.py | 2 + .../automatic_eda/chapters/portada.py | 16 +- .../datascience/automatic_eda/model.py | 35 ++- .../automatic_eda/render_features_test.py | 15 +- .../automatic_eda/render_pdf_impl.py | 155 +++++++++- .../automatic_eda/render_pptx_impl.py | 264 ++++++++++++++-- .../automatic_eda/render_quality_test.py | 283 ++++++++++++++++++ .../datascience/render_table_as_figure.md | 121 ++++++++ .../datascience/render_table_as_figure.py | 241 +++++++++++++++ .../render_table_as_figure_test.py | 119 ++++++++ 11 files changed, 1272 insertions(+), 43 deletions(-) create mode 100644 python/functions/datascience/automatic_eda/render_quality_test.py create mode 100644 python/functions/datascience/render_table_as_figure.md create mode 100644 python/functions/datascience/render_table_as_figure.py create mode 100644 python/functions/datascience/render_table_as_figure_test.py diff --git a/docs/automatic_eda_contract.md b/docs/automatic_eda_contract.md index efd96fa9..e26e7e3a 100644 --- a/docs/automatic_eda_contract.md +++ b/docs/automatic_eda_contract.md @@ -41,12 +41,13 @@ reconocido se degrada a `Note`, nunca lanza). | `Heading(text, level=1)` | título de sección, `level` 1 (grande) … 3 (chico) | una o varias líneas en negrita; nivel 1 lleva subrayado de acento | | `Markdown(text)` | texto markdown ligero | ver subset abajo; **nunca corta a media línea** | | `KVTable(rows, title=None)` | `rows = [(clave, valor), ...]` | tabla de 2 columnas etiqueta/valor; el valor se envuelve | -| `DataTable(header, rows, title=None, note=None)` | `header=[...]`, `rows=[[...],...]` | tabla con cabecera; **se parte por filas repitiendo cabecera**; las celdas largas se envuelven dentro de su columna | +| `DataTable(header, rows, title=None, note=None)` | `header=[...]`, `rows=[[...],...]` | tabla con cabecera; **si cabe** como texto se parte por filas repitiendo cabecera; **si NO cabe** (demasiadas columnas) se rasteriza entera como imagen de alta resolución para hacer zoom. Ver §11.4 | | `Figure(fig=None, make=None, caption=None, height_in=None)` | una `matplotlib.figure.Figure` ya construida (`fig`) o un callable `make()->Figure` (perezoso) | se rasteriza y escala para caber entera (nunca recortada) | | `Image(path, caption=None, height_in=None)` | ruta a PNG/JPG | se escala para caber entera | | `Caption(text)` / `Note(text)` | texto auxiliar pequeño | pie/nota en gris; `Note` es además el fallback de lo desconocido | -| `Group(blocks, title=None)` | unidad **keep-together**: sus bloques se mantienen juntos | el renderer mide el grupo entero y lo mueve completo a la página/slide siguiente si no cabe; encoge la figura para dejar sitio al título+texto. Ver §11 | +| `Group(blocks, title=None, page_break_before=False, layout="stack")` | unidad **keep-together**: sus bloques se mantienen juntos | el renderer mide el grupo entero y lo mueve completo a la página/slide siguiente si no cabe; encoge la figura para dejar sitio al título+texto. `layout="side_by_side"` coloca tabla+figura en dos columnas (solo PPTX). Ver §11 y §11.4 | | `GlossaryEntry(key, label, definition)` | una entrada del glosario (destino clicable) | la genera el capítulo `glosario`; registra su posición como destino de los términos marcados. Ver §11 | +| `TocEntry(label, target_id)` | una entrada de **índice clicable** en la portada | la genera el capítulo `portada`; el renderer la cablea como salto al inicio del capítulo cuyo `id` o `title` coincide con `target_id`. Ver §11.4 | `Figure`/`Image` aceptan `height_in` (hint): el renderer **clampa** la figura a esa altura máxima (lo usa `Group` para encoger la figura). Toda figura escala dejando sitio a su caption en la misma página/slide; en PPTX el caption es **siempre** visible (si no se da `caption`, cae al último heading o a "Figura"). @@ -397,6 +398,65 @@ cabecera con su fondo propio. Es automático en PDF y PPTX; el patrón se mantie cuando una tabla larga se parte y repite cabecera (el índice de fila es lógico, no por página). No hay nada que hacer en los capítulos. +### 11.4 Calidad de render global: DPI alto, tabla ancha → imagen, figura al lado, índice clicable + +Cuatro capacidades transversales del motor, **todas automáticas salvo `layout`** (que un +capítulo activa explícitamente). Aplican a PDF y PPTX salvo donde se indique. + +**(a) DPI alto (automático).** Toda figura/imagen embebida se rasteriza a **220 dpi** +(constante `_RASTER_DPI` en ambos renderers; en PDF se aplica también al `savefig` de la +página, porque matplotlib re-rasteriza cada `imshow` al escribir la página). Objetivo: +ampliar en el móvil y leer detalle (ejes, celdas) sin pixelar. El texto sigue siendo +vectorial y seleccionable. No hay nada que hacer en los capítulos. + +**(b) Tabla ancha → imagen de alta resolución (automático).** Cuando un `DataTable` tiene +**demasiadas columnas para ser legible como texto** en el ancho útil (criterio +`_table_fits_as_text`: ancho mínimo legible por columna × nº de columnas > ancho útil; en +la práctica salta sobre tablas tipo `df.head` con muchas columnas), en vez de comprimir las +columnas hasta hacerlas ilegibles, la tabla se dibuja **entera como una imagen de alta +resolución** (función `render_table_as_figure_py_datascience`: cabecera sombreada + zebra) +escalada para caber completa, de modo que el lector hace **zoom** y la lee sin perder datos. +Si la tabla **sí cabe**, se mantiene como texto seleccionable (PDF) / tabla nativa (PPTX). +Las `KVTable` (2 columnas) caben siempre y se quedan como texto. No hay nada que hacer en +los capítulos. + +**(c) Figura al lado de la tabla — `Group(layout="side_by_side")`.** Hint de layout que un +capítulo activa para que su **tabla quede a la izquierda y su figura a la derecha** en la +misma diapositiva, en lugar de apiladas: + +```python +model.Group( + layout="side_by_side", + blocks=[ + model.Heading(text=str(name), level=2), # va a ancho completo arriba + model.DataTable(header=..., rows=...), # columna IZQUIERDA (~55%) + model.Figure(make=_grafico_perezoso(...)), # columna DERECHA (~45%) + model.Markdown(text="explicación…"), # va a ancho completo abajo + ]) +``` + +Contrato exacto del campo: + +| Campo | Valor | Efecto | +|---|---|---| +| `layout` | `"stack"` (por defecto) | comportamiento histórico: apilado vertical (keep-together). | +| `layout` | `"side_by_side"` | **PPTX**: la tabla (rasterizada a imagen) ocupa la columna izquierda (~55% del ancho útil) y la figura la derecha (~45%); cualquier otro bloque (heading, markdown) va a ancho completo arriba/abajo. Si no hay un par tabla+figura, o no caben lado a lado en una slide, **cae automáticamente a apilado**. **PDF**: se trata **igual que `stack`** (el ancho A5 móvil no admite dos columnas legibles). Valores desconocidos degradan a `"stack"`. | + +Es **retrocompatible**: un `Group` sin `layout` (o `layout="stack"`) se comporta exactamente +como antes. El capítulo `cat_distr` es el consumidor previsto (gráfico a la derecha de la +tabla de categorías en PPT); este motor solo provee el soporte. + +**(d) Índice clicable en la portada — `TocEntry`.** La portada emite un `Heading("Índice")` +seguido de un `TocEntry(label, target_id)` por capítulo. El renderer registra la +página/slide de inicio de **cada** capítulo (indexado por `id` **y** por `title`) y cablea +cada `TocEntry` como un salto real a ese inicio: en **PDF** vía +`add_pdf_internal_links_py_datascience` (link GOTO de PyMuPDF), en **PPTX** vía +`pptx_link_run_to_slide_py_datascience` (salto a slide nativo). Como la portada solo conoce +los **títulos** de los capítulos, el `target_id` se hace coincidir contra el `title` (o el +`id`) de destino. Si un destino no resuelve, la entrada se muestra igualmente como texto +(en color de enlace), nunca se corta. Es el mismo mecanismo que los términos clicables del +glosario (§11.1), reutilizado en sentido portada → capítulo. + --- ## 10. Integración futura con `profile_table` (siguiente fase) diff --git a/python/functions/datascience/automatic_eda/__init__.py b/python/functions/datascience/automatic_eda/__init__.py index 01085313..658570c8 100644 --- a/python/functions/datascience/automatic_eda/__init__.py +++ b/python/functions/datascience/automatic_eda/__init__.py @@ -29,6 +29,7 @@ from .model import ( # noqa: F401 KVTable, Markdown, Note, + TocEntry, as_blocks, as_chapters, merge_manifest, @@ -52,6 +53,7 @@ __all__ = [ "Group", "GlossaryEntry", "GlossaryCollector", + "TocEntry", "Chapter", "as_blocks", "as_chapters", diff --git a/python/functions/datascience/automatic_eda/chapters/portada.py b/python/functions/datascience/automatic_eda/chapters/portada.py index b1a28366..eef1c84c 100644 --- a/python/functions/datascience/automatic_eda/chapters/portada.py +++ b/python/functions/datascience/automatic_eda/chapters/portada.py @@ -26,7 +26,7 @@ from datetime import datetime, timezone from .. import model -CHAPTER_VERSION = "1.3.0" +CHAPTER_VERSION = "1.4.0" CHAPTER_ID = "portada" CHAPTER_TITLE = "Portada" @@ -120,9 +120,17 @@ def _summary_blocks(summary) -> list: # Values pinned to the right margin (numbers flush right, label left). blocks.append(model.KVTable(rows=rows, value_align="right")) if titles: - bullets = "\n".join(f"- {model._safe_str(t)}" for t in titles) - blocks.append(model.Markdown( - text="Este informe incluye los siguientes capítulos:\n" + bullets)) + # Clickable index ("Índice"): one TocEntry per chapter title. Each entry + # becomes a real jump to that chapter's first page/slide once the document + # is laid out (the renderers register every chapter start and wire the + # links; ``target_id`` is matched against the chapter title). The cover only + # knows chapter titles, so the title doubles as the link target. + blocks.append(model.Heading(text="Índice", level=2)) + for t in titles: + label = model._safe_str(t) + if not label: + continue + blocks.append(model.TocEntry(label=label, target_id=label)) return blocks diff --git a/python/functions/datascience/automatic_eda/model.py b/python/functions/datascience/automatic_eda/model.py index 9171652e..bc15c8bc 100644 --- a/python/functions/datascience/automatic_eda/model.py +++ b/python/functions/datascience/automatic_eda/model.py @@ -160,11 +160,21 @@ class Group: a chapter can give each unit its own page — e.g. one categorical column per page (see CAT DISTR). It is purely additive: the default False keeps the plain keep-together behaviour for every existing chapter. + + ``layout`` is a hint for how the group's children are arranged: + ``"stack"`` (default) keeps the historical top-to-bottom flow; ``"side_by_side"`` + asks the PPTX renderer to place the group's table to the LEFT and its figure to + the RIGHT of the same slide (table ~55% width, figure ~45%), measuring so both + fit and falling back to stacking when they do not. The PDF renderer treats + ``"side_by_side"`` exactly like ``"stack"`` (the A5 mobile page is too narrow for + two readable columns). Unknown values degrade to ``"stack"``. Purely additive: + the default keeps every existing chapter unchanged. """ blocks: list = field(default_factory=list) title: Optional[str] = None page_break_before: bool = False + layout: str = "stack" kind: str = field(default="group", init=False) @@ -183,6 +193,22 @@ class GlossaryEntry: kind: str = field(default="glossary_entry", init=False) +@dataclass +class TocEntry: + """One clickable index (table-of-contents) entry shown on the cover. + + Rendered as a single line — the chapter ``label`` in the accent link colour — + that, once the document is laid out, becomes a real click jumping to the first + page/slide of the target chapter (PDF link annotation via PyMuPDF; PPTX native + slide jump). ``target_id`` is matched against each chapter's ``id`` *and* its + ``title`` (the cover only knows chapter titles), so either resolves. If the + target cannot be resolved the entry still renders as plain text (never cut).""" + + label: str = "" + target_id: str = "" + kind: str = field(default="toc_entry", init=False) + + @dataclass class Chapter: """An ordered set of blocks with an id, a title and a generation version.""" @@ -207,13 +233,14 @@ _BLOCK_BY_KIND = { "note": Note, "group": Group, "glossary_entry": GlossaryEntry, + "toc_entry": TocEntry, } def as_block(obj: Any): """Coerce a value into a block dataclass. Unknown values become a Note.""" if isinstance(obj, (Heading, Markdown, KVTable, DataTable, Figure, Image, - Caption, Note, Group, GlossaryEntry)): + Caption, Note, Group, GlossaryEntry, TocEntry)): if isinstance(obj, Group): obj.blocks = as_blocks(obj.blocks) return obj @@ -259,11 +286,15 @@ def as_block(obj: Any): return Group(blocks=as_blocks(obj.get("blocks")), title=obj.get("title"), page_break_before=bool( - obj.get("page_break_before", False))) + obj.get("page_break_before", False)), + layout=_safe_str(obj.get("layout")) or "stack") if cls is GlossaryEntry: return GlossaryEntry(key=_safe_str(obj.get("key")), label=_safe_str(obj.get("label")), definition=_safe_str(obj.get("definition"))) + if cls is TocEntry: + return TocEntry(label=_safe_str(obj.get("label")), + target_id=_safe_str(obj.get("target_id"))) except Exception: # noqa: BLE001 — never raise on a malformed block. return Note(text=_safe_str(obj)) return Note(text=_safe_str(obj)) diff --git a/python/functions/datascience/automatic_eda/render_features_test.py b/python/functions/datascience/automatic_eda/render_features_test.py index 40d247ba..5bb5a2a0 100644 --- a/python/functions/datascience/automatic_eda/render_features_test.py +++ b/python/functions/datascience/automatic_eda/render_features_test.py @@ -298,11 +298,16 @@ def test_cover_first_glossary_last_with_summary(): headings = [b.text for b in cover.blocks if b.kind == "heading"] assert any("Resumen" in h for h in headings), \ "la portada no incluye el resumen agregado" - # The summary reflects the body chapters (e.g. the numeric/categorical ones). - cover_text = " ".join( - b.text for b in cover.blocks if getattr(b, "kind", "") == "markdown") - assert "Distribuciones" in cover_text, \ - "el resumen de portada no menciona los capítulos del cuerpo" + # The index ("Índice") is now a clickable list of TocEntry blocks (one per + # body chapter), not a markdown bullet list. Verify both the heading and that + # the entries name the body chapters. + assert any("Índice" in h for h in headings), \ + "la portada no incluye la sección Índice" + toc_labels = " ".join( + getattr(b, "label", "") for b in cover.blocks + if getattr(b, "kind", "") == "toc_entry") + assert "Distribuciones" in toc_labels, \ + "el índice de portada no menciona los capítulos del cuerpo" # --------------------------------------------------------------------------- # diff --git a/python/functions/datascience/automatic_eda/render_pdf_impl.py b/python/functions/datascience/automatic_eda/render_pdf_impl.py index 30115de0..909a482a 100644 --- a/python/functions/datascience/automatic_eda/render_pdf_impl.py +++ b/python/functions/datascience/automatic_eda/render_pdf_impl.py @@ -46,11 +46,23 @@ _MUTED = "#8a8a8a" _RULE = "#cccccc" _HEAD_BG = "#eef3f6" +# Rasterization DPI for every embedded raster (figure/table image) AND for the +# page save itself. Raised from the old 150/default-100 to 220 so a reader can +# pinch-zoom on a phone and still see crisp detail (axis labels, table cells) +# without pixelation. Text stays vectorial (pdf.fonttype=42) so it remains +# selectable regardless of DPI — only the embedded images gain resolution. 220 is +# a deliberate balance: noticeably sharper than 150 while keeping the file size +# reasonable. ``savefig.dpi`` matters because matplotlib re-rasterizes each +# ``imshow`` when PdfPages writes the page; without it the final image would land +# at ~100 dpi no matter how sharp the intermediate PNG was. +_RASTER_DPI = 220 + _RC = { "font.size": 10, "font.family": "sans-serif", "figure.facecolor": "white", "savefig.facecolor": "white", + "savefig.dpi": _RASTER_DPI, "pdf.fonttype": 42, # embed TrueType — text stays selectable on mobile. } @@ -80,6 +92,10 @@ class _PdfState: # points (1/72") with a top-left origin — same convention as PyMuPDF. self.term_sources = [] # [{key, page, rect:[x0,y0,x1,y1]}] self.term_dests = {} # key -> {page, point:[x,y]} + # Clickable index (cover → chapter). Sources are the cover's TocEntry + # rects; chapter_starts maps a chapter id AND its title to its first page. + self.toc_sources = [] # [{target_id, page, rect:[x0,y0,x1,y1]}] + self.chapter_starts = {} # id|title -> {page, point:[x,y]} # --------------------------------------------------------------------------- # @@ -385,6 +401,57 @@ def _col_widths(header: list, rows: list, fs: float) -> list: return widths +# Minimal legible characters reserved per column when deciding whether a table +# can be shown as selectable text. Below this width per column the cells become +# unreadable, so the table is rasterized to a zoomable high-res image instead. +_MIN_LEGIBLE_CHARS = 8 + + +def _table_fits_as_text(header: list, rows: list) -> bool: + """True when the table fits the usable width as readable text. + + A table whose columns cannot each get a minimal legible width within the A5 + usable width (typically many columns, e.g. a 19-column ``df.head``) is flagged + so it is rendered as a single high-resolution image — the reader zooms in on + the phone and reads every cell, nothing cut — instead of being squeezed until + unreadable. Narrow tables (few columns) keep the selectable-text rendering.""" + header = header or [] + rows = rows or [] + ncol = len(header) if header else (len(rows[0]) if rows else 1) + ncol = max(1, ncol) + cw = tl.avg_char_width_in(_FS_CELL) + min_needed = ncol * (_MIN_LEGIBLE_CHARS * cw + _CELL_PAD * 2) + return min_needed <= _USABLE_W + + +def _table_figure_block(block): + """Wrap a too-wide table as a lazily-rasterized Figure (cached on the block). + + The table is drawn once via ``render_table_as_figure`` (header shading + zebra) + and embedded as one high-res image scaled to fit entirely. The same Figure is + reused for measuring and placing so keep-together stays consistent. The table + title/note are drawn inside the image (self-describing when zoomed/shared), so + the block-level caption is left empty to avoid a duplicate title.""" + cached = getattr(block, "_aeda_tablefig", None) + if cached is not None: + return cached + header = list(getattr(block, "header", []) or []) + rows = list(getattr(block, "rows", []) or []) + title = getattr(block, "title", None) + note = getattr(block, "note", None) + + def _make(): + from datascience.render_table_as_figure import render_table_as_figure + return render_table_as_figure(header, rows, title=title, note=note) + + fig = model.Figure(make=_make, caption=None) + try: + block._aeda_tablefig = fig + except Exception: # noqa: BLE001 — block may reject attributes; degrade. + pass + return fig + + def _wrap_row(cells: list, widths: list, fs: float) -> list: """Wrap each cell to its column width → list of line-lists per cell.""" out = [] @@ -424,11 +491,16 @@ def _draw_table_row(st: _PdfState, cells_lines: list, widths: list, fs: float, def _place_data_table(st: _PdfState, block) -> None: + header = list(getattr(block, "header", []) or []) + rows = list(getattr(block, "rows", []) or []) + # Too many columns to be legible as text → render the whole table as one + # high-res image, scaled to fit entirely (the reader zooms to read it). + if not _table_fits_as_text(header, rows): + _place_figure(st, _table_figure_block(block)) + return title = getattr(block, "title", None) if title: _place_heading(st, model.Heading(title, level=2)) - header = list(getattr(block, "header", []) or []) - rows = list(getattr(block, "rows", []) or []) fs = _FS_CELL widths = _col_widths(header, rows, fs) header_lines = _wrap_row(header, widths, fs) if header else None @@ -486,8 +558,11 @@ def _resolve_figure(block): def _png_from_figure(fig) -> bytes: + # ``bbox_inches='tight'`` is kept so the real aspect ratio is what we measure + # and place. The page save (savefig.dpi in _RC) re-rasterizes this at the same + # high DPI, so the embedded image stays crisp for phone zoom. buf = io.BytesIO() - fig.savefig(buf, format="png", dpi=150, bbox_inches="tight") + fig.savefig(buf, format="png", dpi=_RASTER_DPI, bbox_inches="tight") buf.seek(0) return buf.read() @@ -729,12 +804,16 @@ def _measure_data_table(block) -> float: Counts the optional title heading, the wrapped header row, every wrapped data row (per-column wrap via the same ``_col_widths``/``_wrap_row`` the placer uses) and the optional note. Keep this in sync with ``_place_data_table``.""" + header = list(getattr(block, "header", []) or []) + rows = list(getattr(block, "rows", []) or []) + # Mirror the placer: a too-wide table is drawn as a single image, so its + # keep-together height is the image's, not the (squeezed) text layout's. + if not _table_fits_as_text(header, rows): + return _measure_figure_like(_table_figure_block(block)) h = 0.0 title = getattr(block, "title", None) if title: h += _measure_heading_text(title, 2) - header = list(getattr(block, "header", []) or []) - rows = list(getattr(block, "rows", []) or []) fs = _FS_CELL widths = _col_widths(header, rows, fs) lh = tl.line_height_in(fs) @@ -766,6 +845,10 @@ def _measure_block(st: _PdfState, block) -> float: lines = tl.wrap(getattr(block, "text", ""), tl.chars_per_line(_USABLE_W, _FS_NOTE)) return tl.line_height_in(_FS_NOTE) * len(lines) + _GAP + if kind == "toc_entry": + lines = tl.wrap(tl.strip_inline_md(getattr(block, "label", "")), + tl.chars_per_line(_USABLE_W - 0.22, _FS_BODY)) or [""] + return tl.line_height_in(_FS_BODY) * len(lines) + _GAP * 0.4 if kind == "kv_table": return _measure_kv_table(block) if kind == "data_table": @@ -850,6 +933,38 @@ def _place_glossary_entry(st: _PdfState, block) -> None: st.y += _GAP * 0.5 +def _place_toc_entry(st: _PdfState, block) -> None: + """Render one clickable index line and record it as a link source. + + Drawn as a bulleted line in the accent link colour; its rectangle is recorded + in ``st.toc_sources`` so the post-processor turns it into a real jump to the + target chapter's first page. If the target is never resolved the line still + shows as plain (accent) text — never cut, never broken.""" + label = tl.strip_inline_md(getattr(block, "label", "")) or "" + target_id = getattr(block, "target_id", "") or "" + fs = _FS_BODY + lh = tl.line_height_in(fs) + bullet = "• " + indent = 0.22 + max_chars = tl.chars_per_line(_USABLE_W - indent, fs) + lines = tl.wrap(label, max_chars) or [""] + for idx, ln in enumerate(lines): + _ensure_space(st, lh) + x = _ML + st.fig.text(_xf(x), _yf(st.y), bullet if idx == 0 else " ", + fontsize=fs, color=_LINK, ha="left", va="top") + x += indent + w = _text_width_in(st, ln, fs, False) + st.fig.text(_xf(x), _yf(st.y), ln, fontsize=fs, color=_LINK, + ha="left", va="top") + if target_id and idx == 0: + st.toc_sources.append({ + "target_id": target_id, "page": st.page - 1, + "rect": _pt_rect(_ML, st.y, x + w, st.y + lh)}) + st.y += lh + st.y += _GAP * 0.4 + + _PLACERS = { "heading": _place_heading, "markdown": _place_markdown, @@ -861,6 +976,7 @@ _PLACERS = { "note": _place_note, "group": _place_group, "glossary_entry": _place_glossary_entry, + "toc_entry": _place_toc_entry, } @@ -892,6 +1008,15 @@ def render_pdf(chapters: list, out_path: str, meta: dict = None) -> dict: st.chapter = ch st.chapter_pages = 0 _new_page(st) # each chapter starts on a fresh page. + # Record this chapter's first page as a link target for the + # cover index (keyed by id AND title, since the cover only + # knows titles). Point is the top of the content area. + _start = {"page": st.page - 1, + "point": [_ML * 72.0, _CONTENT_TOP * 72.0]} + if ch.id: + st.chapter_starts[ch.id] = _start + if getattr(ch, "title", ""): + st.chapter_starts.setdefault(ch.title, _start) for block in ch.blocks: placer = _PLACERS.get(getattr(block, "kind", ""), _place_note) @@ -924,7 +1049,7 @@ def render_pdf(chapters: list, out_path: str, meta: dict = None) -> dict: note = f"{n_pages} páginas" if n_links: - note += f" · {n_links} enlaces de glosario" + note += f" · {n_links} enlaces internos" if notes: note += " · " + "; ".join(notes) return {"path": out_path, "n_pages": n_pages, "chapters": chapters_meta, @@ -932,9 +1057,11 @@ def render_pdf(chapters: list, out_path: str, meta: dict = None) -> dict: def _wire_glossary_links(st: _PdfState, out_path: str, notes: list) -> int: - """Build {source rect → glossary dest} links and apply them via PyMuPDF. + """Apply internal PDF links via PyMuPDF: glossary terms + the cover index. - Returns the number of links applied (0 if there is nothing to wire or the + Builds two sets of GOTO links — every in-text glossary term → its entry, and + every cover ``TocEntry`` → its chapter's first page — and applies them in one + pass. Returns the number of links applied (0 if there is nothing to wire or the post-processor is unavailable). Never raises.""" try: links = [] @@ -945,6 +1072,14 @@ def _wire_glossary_links(st: _PdfState, out_path: str, notes: list) -> int: links.append({ "src_page": src["page"], "src_rect": src["rect"], "dst_page": dest["page"], "dst_point": dest["point"]}) + # Cover index → chapter first page (clickable, navigable table of contents). + for src in st.toc_sources: + dest = st.chapter_starts.get(src.get("target_id")) + if not dest: + continue + links.append({ + "src_page": src["page"], "src_rect": src["rect"], + "dst_page": dest["page"], "dst_point": dest["point"]}) if not links: return 0 from datascience.add_pdf_internal_links import add_pdf_internal_links @@ -952,7 +1087,7 @@ def _wire_glossary_links(st: _PdfState, out_path: str, notes: list) -> int: if isinstance(res, dict) and res.get("status") == "ok": return int(res.get("n_links") or 0) if isinstance(res, dict) and res.get("error"): - notes.append(f"glosario sin enlaces: {res.get('error')}") + notes.append(f"enlaces internos no aplicados: {res.get('error')}") except Exception as e: # noqa: BLE001 — links are best-effort. - notes.append(f"glosario sin enlaces: {e}") + notes.append(f"enlaces internos no aplicados: {e}") return 0 diff --git a/python/functions/datascience/automatic_eda/render_pptx_impl.py b/python/functions/datascience/automatic_eda/render_pptx_impl.py index 21b9e0ce..8a8039eb 100644 --- a/python/functions/datascience/automatic_eda/render_pptx_impl.py +++ b/python/functions/datascience/automatic_eda/render_pptx_impl.py @@ -51,6 +51,12 @@ _FS_H1, _FS_H2, _FS_H3 = 20, 16, 13 _FS_BODY, _FS_CELL, _FS_NOTE = 14, 11, 11 _GAP = 0.12 +# Rasterization DPI for every embedded figure/table image. Raised from 150 to 220 +# so a viewer can zoom into a slide (or a shared picture) and read crisp detail — +# axis labels, table cells — without pixelation. Kept moderate so the deck size +# stays reasonable. Same value as the PDF renderer. +_RASTER_DPI = 220 + class _PptxState: def __init__(self, prs, title: str): @@ -65,6 +71,10 @@ class _PptxState: # Glossary wiring (mejora 6): runs to link and per-term target slide. self.term_runs = [] # [(key, run)] self.term_anchor_slide = {} # key -> Slide (glossary entry) + # Clickable index (cover → chapter). toc_runs are the cover's index runs; + # chapter_starts maps a chapter id AND its title to its first slide. + self.toc_runs = [] # [(target_id, run, src_slide)] + self.chapter_starts = {} # id|title -> Slide (chapter first slide) def _rgb(c): @@ -309,6 +319,58 @@ def _col_widths(header, rows): return [_USABLE_W * w / total for w in clamped] +# Minimal legible characters reserved per column when deciding whether a table +# can be shown as a native (selectable) PowerPoint table. Below this width per +# column the cells become unreadable, so the table is rasterized to a zoomable +# high-res image instead. The 16:9 slide is wide, so more columns fit than on A5. +_MIN_LEGIBLE_CHARS = 8 +_CELL_PAD = 0.05 + + +def _table_fits_as_text(header: list, rows: list) -> bool: + """True when the table fits the usable slide width as a readable table. + + A table whose columns cannot each get a minimal legible width within the slide + usable width (typically many columns, e.g. a 19-column ``df.head``) is flagged + so it is rendered as one high-resolution image — the viewer zooms in and reads + every cell — instead of being squeezed unreadable. Narrow tables keep the + native selectable table.""" + header = header or [] + rows = rows or [] + ncol = len(header) if header else (len(rows[0]) if rows else 1) + ncol = max(1, ncol) + cw = tl.avg_char_width_in(_FS_CELL) + min_needed = ncol * (_MIN_LEGIBLE_CHARS * cw + _CELL_PAD * 2) + return min_needed <= _USABLE_W + + +def _table_figure_block(block): + """Wrap a too-wide table as a lazily-rasterized Figure (cached on the block). + + Drawn once via ``render_table_as_figure`` (header shading + zebra) and embedded + as one high-res image scaled to fit entirely. The title/note are drawn inside + the image (self-describing when zoomed/shared), so no separate caption is + emitted. Reused for measuring and placing so keep-together stays consistent.""" + cached = getattr(block, "_aeda_tablefig", None) + if cached is not None: + return cached + header = list(getattr(block, "header", []) or []) + rows = list(getattr(block, "rows", []) or []) + title = getattr(block, "title", None) + note = getattr(block, "note", None) + + def _make(): + from datascience.render_table_as_figure import render_table_as_figure + return render_table_as_figure(header, rows, title=title, note=note) + + fig = model.Figure(make=_make, caption=None) + try: + block._aeda_tablefig = fig + except Exception: # noqa: BLE001 — block may reject attributes; degrade. + pass + return fig + + def _row_height_in(cells, widths, fs) -> float: lh = tl.line_height_in(fs) maxlines = 1 @@ -372,11 +434,27 @@ def _style_cell(cell, fs, color, bold, fill) -> None: def _place_data_table(st: _PptxState, block, shaded_header=True, key_value=False) -> None: + header = list(getattr(block, "header", []) or []) + rows = list(getattr(block, "rows", []) or []) + # Too many columns to be legible as a native table → render the whole table as + # one high-res picture, scaled to fit entirely (the viewer zooms to read it). + # KVTables (rendered here as a 2-column Campo/Valor table) are excluded: they + # always fit in width and stay as a selectable table. + if not key_value and not _table_fits_as_text(header, rows): + figblock = _table_figure_block(block) + data, _asp = _figure_bytes_cached(figblock) + if data is None: + _add_text(st, ["(tabla no disponible)"], _FS_NOTE, _MUTED, + italic=True) + st.y += _GAP + return + _place_picture_bytes(st, data, None, + max_h_in=getattr(figblock, "height_in", None), + force_caption=False) + return title = getattr(block, "title", None) if title: _place_heading(st, model.Heading(title, level=2)) - header = list(getattr(block, "header", []) or []) - rows = list(getattr(block, "rows", []) or []) fs = _FS_CELL widths = _col_widths(header, rows) header_h = _row_height_in(header, widths, fs) if header else 0.0 @@ -436,7 +514,7 @@ def _resolve_png(block): try: import matplotlib.pyplot as plt buf = io.BytesIO() - f.savefig(buf, format="png", dpi=150, bbox_inches="tight") + f.savefig(buf, format="png", dpi=_RASTER_DPI, bbox_inches="tight") buf.seek(0) return buf.read() except Exception: # noqa: BLE001 @@ -483,12 +561,15 @@ def _figure_bytes_cached(block): def _place_picture_bytes(st: _PptxState, data: bytes, caption, - max_h_in=None) -> None: + max_h_in=None, force_caption=True) -> None: # Mejora 4 — every figure on a slide carries a visible caption/title. If the # block has no caption, fall back to the current section heading, then to a - # generic label, so no image is ever shown untitled. - caption = (model._safe_str(caption).strip() - or model._safe_str(st.last_heading).strip() or "Figura") + # generic label, so no image is ever shown untitled. ``force_caption=False`` + # suppresses that fallback (used for table images, whose title is inside the + # picture) so no redundant caption is drawn. + caption = model._safe_str(caption).strip() + if not caption and force_caption: + caption = model._safe_str(st.last_heading).strip() or "Figura" w_px, h_px = _img_size_px(data) aspect = (h_px / w_px) if w_px else 0.66 # Reserve the caption's REAL (possibly multi-line) height FIRST, then scale @@ -496,9 +577,11 @@ def _place_picture_bytes(st: _PptxState, data: bytes, caption, # so its caption always fits on the SAME slide and no image is untitled. # cap_real = what _add_text consumes; cap_reserve adds the post-image gap and # a small cushion so the caption never spills to the next slide. - cap_lines = tl.wrap(caption, tl.chars_per_line(_USABLE_W, _FS_NOTE)) - cap_real = tl.line_height_in(_FS_NOTE) * len(cap_lines) + 0.05 - cap_reserve = cap_real + 0.05 + 0.10 + cap_lines = tl.wrap(caption, tl.chars_per_line(_USABLE_W, _FS_NOTE)) \ + if caption else [] + cap_real = (tl.line_height_in(_FS_NOTE) * len(cap_lines) + 0.05) \ + if cap_lines else 0.0 + cap_reserve = (cap_real + 0.05 + 0.10) if cap_lines else 0.05 max_h = _CONTENT_BOTTOM - _CONTENT_TOP # height_in hint (model.Figure/Image): cap the target height so a figure in a # keep-together Group shrinks to leave room for its heading and text. @@ -517,7 +600,8 @@ def _place_picture_bytes(st: _PptxState, data: bytes, caption, st.slide.shapes.add_picture(io.BytesIO(data), Inches(left), Inches(st.y), width=Inches(target_w), height=Inches(target_h)) st.y += target_h + 0.05 - _add_text(st, cap_lines, _FS_NOTE, _MUTED, italic=True) + if cap_lines: + _add_text(st, cap_lines, _FS_NOTE, _MUTED, italic=True) st.y += _GAP @@ -663,12 +747,16 @@ def _measure_kv_table(block) -> float: def _measure_data_table(block) -> float: """Faithful DataTable height — matches ``_place_data_table`` (title heading + wrapped header + every wrapped row + optional note). Keep in sync.""" + header = list(getattr(block, "header", []) or []) + rows = list(getattr(block, "rows", []) or []) + # Mirror the placer: a too-wide table is drawn as one image, so its + # keep-together height is the image's, not the (squeezed) table layout's. + if not _table_fits_as_text(header, rows): + return _measure_figure_like(_table_figure_block(block)) h = 0.0 title = getattr(block, "title", None) if title: h += _measure_heading_text(title, 2) - header = list(getattr(block, "header", []) or []) - rows = list(getattr(block, "rows", []) or []) fs = _FS_CELL widths = _col_widths(header, rows) if header: @@ -698,6 +786,10 @@ def _measure_block(st: _PptxState, block) -> float: lines = tl.wrap(getattr(block, "text", ""), tl.chars_per_line(_USABLE_W, _FS_NOTE)) return tl.line_height_in(_FS_NOTE) * len(lines) + 0.05 + _GAP + if kind == "toc_entry": + lines = tl.wrap(tl.strip_inline_md(getattr(block, "label", "")), + tl.chars_per_line(_USABLE_W - 0.3, _FS_BODY)) or [""] + return tl.line_height_in(_FS_BODY) * len(lines) + 0.05 if kind == "kv_table": return _measure_kv_table(block) if kind == "data_table": @@ -810,6 +902,73 @@ def _fit_group_blocks(st: _PptxState, blocks: list, avail_full: float) -> list: return out +def _fit_img(width_col: float, aspect: float, max_h: float): + """Scale an image to ``width_col`` then clamp to ``max_h`` keeping aspect.""" + w = width_col + h = w * aspect + if h > max_h: + h = max_h + w = (h / aspect) if aspect else width_col + return w, h + + +def _place_group_side_by_side(st: _PptxState, block, avail_full: float) -> bool: + """Place a Group's table (left ~55%) next to its figure (right ~45%). + + Both the table and the figure are rasterized to high-res images and placed in + two columns of the SAME slide; any other blocks (e.g. a heading) render full + width above the pair, the rest below. Returns True on success; returns False + (so the caller falls back to stacking) when the group has no table+figure pair + or the pair cannot fit side by side on one slide. Never raises by itself.""" + blocks = getattr(block, "blocks", []) or [] + tbl = next((b for b in blocks + if getattr(b, "kind", "") in ("data_table", "kv_table")), None) + fig = next((b for b in blocks + if getattr(b, "kind", "") in ("figure", "image")), None) + if tbl is None or fig is None: + return False + gap_col = 0.3 + left_w = _USABLE_W * 0.55 - gap_col / 2.0 + right_w = _USABLE_W * 0.45 - gap_col / 2.0 + if left_w <= 1.0 or right_w <= 1.0: + return False + tdata, tasp = _figure_bytes_cached(_table_figure_block(tbl)) + fdata, fasp = _figure_bytes_cached(fig) + if not tdata or not fdata: + return False + ti, fi = blocks.index(tbl), blocks.index(fig) + lo = min(ti, fi) + lead = list(blocks[:lo]) + rest = [b for b in blocks[lo + 1:] if b is not tbl and b is not fig] + lead_h = sum(_measure_block(st, b) for b in lead) + rest_h = sum(_measure_block(st, b) for b in rest) + col_max_h = avail_full - lead_h - rest_h - _GAP * 2 + if col_max_h < 1.2: + return False # not enough vertical room to put the pair side by side. + tw, th = _fit_img(left_w, tasp, col_max_h) + fw, fh = _fit_img(right_w, fasp, col_max_h) + band = max(th, fh) + needed = lead_h + band + rest_h + _GAP * 2 + if needed > avail_full: + return False # taller than a whole slide even side by side → stack. + if needed > _remaining(st): + _new_slide(st, cont=True) + for b in lead: + _PLACERS.get(getattr(b, "kind", ""), _place_note)(st, b) + top = st.y + f_left = _ML + left_w + gap_col + st.slide.shapes.add_picture( + io.BytesIO(tdata), Inches(_ML + (left_w - tw) / 2.0), + Inches(top + (band - th) / 2.0), width=Inches(tw), height=Inches(th)) + st.slide.shapes.add_picture( + io.BytesIO(fdata), Inches(f_left + (right_w - fw) / 2.0), + Inches(top + (band - fh) / 2.0), width=Inches(fw), height=Inches(fh)) + st.y = top + band + _GAP + for b in rest: + _PLACERS.get(getattr(b, "kind", ""), _place_note)(st, b) + return True + + def _place_group(st: _PptxState, block) -> None: """Render a keep-together Group: move it whole to the next slide if needed.""" blocks = getattr(block, "blocks", []) or [] @@ -820,6 +979,14 @@ def _place_group(st: _PptxState, block) -> None: if getattr(block, "page_break_before", False) and st.y > _CONTENT_TOP + 1e-6: _new_slide(st, cont=True) avail_full = _CONTENT_BOTTOM - _CONTENT_TOP + # layout="side_by_side": try table-left / figure-right on one slide; on any + # reason it can't, fall through to the normal stacked keep-together below. + if str(getattr(block, "layout", "stack")).lower() == "side_by_side": + try: + if _place_group_side_by_side(st, block, avail_full): + return + except Exception: # noqa: BLE001 — degrade to stacking, never abort. + pass # Trim oversized tables first (keeps the chart on the same slide), then shrink # the figure to share the remaining room. blocks = _fit_group_blocks(st, blocks, avail_full) @@ -853,6 +1020,44 @@ def _place_glossary_entry(st: _PptxState, block) -> None: st.y += _GAP +def _place_toc_entry(st: _PptxState, block) -> None: + """Render one clickable index line and record its run as a link source. + + Drawn as a bulleted line in the accent link colour; the run is recorded in + ``st.toc_runs`` so it later becomes a native slide-jump to the target chapter's + first slide. If the target is never resolved the line still shows as plain + (accent) text — never cut.""" + label = tl.strip_inline_md(getattr(block, "label", "")) or "" + target_id = getattr(block, "target_id", "") or "" + fs = _FS_BODY + lines = tl.wrap(label, tl.chars_per_line(_USABLE_W - 0.3, fs)) or [""] + lh = tl.line_height_in(fs) + height = lh * len(lines) + 0.05 + _ensure(st, height) + box = st.slide.shapes.add_textbox( + Inches(_ML), Inches(st.y), Inches(_USABLE_W), Inches(height)) + tf = box.text_frame + tf.word_wrap = True + first = True + link_run = None + for idx, ln in enumerate(lines): + p = tf.paragraphs[0] if first else tf.add_paragraph() + first = False + r0 = p.add_run() + r0.text = "• " if idx == 0 else " " + r0.font.size = Pt(fs) + r0.font.color.rgb = _rgb(_LINK) + run = p.add_run() + run.text = ln + run.font.size = Pt(fs) + run.font.color.rgb = _rgb(_LINK) + if idx == 0: + link_run = run + if target_id and link_run is not None: + st.toc_runs.append((target_id, link_run, st.slide)) + st.y += height + + _PLACERS = { "heading": _place_heading, "markdown": _place_markdown, @@ -864,6 +1069,7 @@ _PLACERS = { "note": _place_note, "group": _place_group, "glossary_entry": _place_glossary_entry, + "toc_entry": _place_toc_entry, } @@ -899,6 +1105,12 @@ def render_pptx(chapters: list, out_path: str, meta: dict = None) -> dict: st.chapter = ch st.chapter_slides = 0 _new_slide(st, cont=False) + # Record this chapter's first slide as a link target for the cover + # index (keyed by id AND title, since the cover only knows titles). + if ch.id: + st.chapter_starts[ch.id] = st.slide + if getattr(ch, "title", ""): + st.chapter_starts.setdefault(ch.title, st.slide) for block in ch.blocks: placer = _PLACERS.get(getattr(block, "kind", ""), _place_note) try: @@ -926,7 +1138,7 @@ def render_pptx(chapters: list, out_path: str, meta: dict = None) -> dict: note = f"{n_slides} slides" if n_links: - note += f" · {n_links} enlaces de glosario" + note += f" · {n_links} enlaces internos" if notes: note += " · " + "; ".join(notes) return {"path": out_path, "n_slides": n_slides, "chapters": chapters_meta, @@ -934,19 +1146,21 @@ def render_pptx(chapters: list, out_path: str, meta: dict = None) -> dict: def _wire_glossary_links(st: _PptxState, notes: list) -> int: - """Turn each recorded term run into a native jump to its glossary slide. + """Apply native slide-jumps: glossary terms + the cover index. - Returns the number of links applied. A term whose only appearance is inside - its own glossary entry (source slide == target slide) is skipped. Never + Each in-text glossary term run jumps to its glossary entry slide, and each + cover ``TocEntry`` run jumps to its chapter's first slide. Returns the total + number of links applied. A run whose target is its own slide is skipped. Never raises.""" - if not st.term_runs or not st.term_anchor_slide: + if not (st.term_runs and st.term_anchor_slide) and not ( + st.toc_runs and st.chapter_starts): return 0 - linked = 0 try: from datascience.pptx_link_run_to_slide import pptx_link_run_to_slide except Exception as e: # noqa: BLE001 - notes.append(f"glosario sin enlaces: {e}") + notes.append(f"enlaces internos no aplicados: {e}") return 0 + linked = 0 for key, run, src_slide in st.term_runs: tgt = st.term_anchor_slide.get(key) if tgt is None or tgt is src_slide: @@ -956,4 +1170,14 @@ def _wire_glossary_links(st: _PptxState, notes: list) -> int: linked += 1 except Exception: # noqa: BLE001 — links are best-effort. pass + # Cover index → chapter first slide (clickable, navigable table of contents). + for target_id, run, src_slide in st.toc_runs: + tgt = st.chapter_starts.get(target_id) + if tgt is None or tgt is src_slide: + continue + try: + if pptx_link_run_to_slide(run, src_slide, tgt): + linked += 1 + except Exception: # noqa: BLE001 — links are best-effort. + pass return linked diff --git a/python/functions/datascience/automatic_eda/render_quality_test.py b/python/functions/datascience/automatic_eda/render_quality_test.py new file mode 100644 index 00000000..a2567251 --- /dev/null +++ b/python/functions/datascience/automatic_eda/render_quality_test.py @@ -0,0 +1,283 @@ +"""Golden tests for the global render-quality features (issue: eda-render-quality). + +Covers, with executable evidence: + * High DPI: every embedded figure is rasterized at 220 dpi, so a phone reader + can zoom in and still see crisp detail. + * Wide table → image: a table too wide to be legible as text (e.g. a 19-column + df.head) is rendered as one high-res image that scales to fit entirely, while + a narrow table keeps its selectable-text/native-table rendering. + * ``Group(layout="side_by_side")``: in PPTX the table and figure are placed in + two columns of the same slide; in PDF the same group stacks vertically. + * Backward compatibility: a Group without ``layout`` defaults to ``"stack"`` and + a fitting table renders exactly as before. + +Renderers are invoked for real; PDFs are inspected with PyMuPDF and PPTX decks +with python-pptx. +""" + +from __future__ import annotations + +import os +import tempfile + +import matplotlib + +matplotlib.use("Agg") +import matplotlib.pyplot as plt # noqa: E402 + +import pytest # noqa: E402 + +from datascience.automatic_eda import model # noqa: E402 +from datascience.automatic_eda.render_pdf_impl import ( # noqa: E402 + render_pdf, _RASTER_DPI as _PDF_DPI, _table_fits_as_text as _pdf_fits) +from datascience.automatic_eda.render_pptx_impl import ( # noqa: E402 + render_pptx, _RASTER_DPI as _PPTX_DPI, _table_fits_as_text as _pptx_fits) + + +# --------------------------------------------------------------------------- # +# Helpers. +# --------------------------------------------------------------------------- # +def _simple_fig(): + """A small, real matplotlib figure for the figure blocks.""" + fig, ax = plt.subplots(figsize=(4, 3)) + ax.plot([0, 1, 2, 3], [1, 3, 2, 4]) + ax.set_title("demo") + return fig + + +def _wide_table(n_cols=19, n_rows=5): + header = [f"columna_{i}" for i in range(n_cols)] + rows = [[f"v{r}_{c}" for c in range(n_cols)] for r in range(n_rows)] + return model.DataTable(header=header, rows=rows, title="Primeras filas") + + +def _narrow_table(): + return model.DataTable(header=["a", "b", "c"], + rows=[["1", "2", "3"], ["4", "5", "6"]], + title="Tabla estrecha") + + +def _chapter(blocks, cid="cap", title="Capítulo"): + return [model.Chapter(id=cid, title=title, version="1.0.0", blocks=blocks)] + + +# --------------------------------------------------------------------------- # +# 1) High DPI — the unit constant and a real embedded image. +# --------------------------------------------------------------------------- # +def test_raster_dpi_is_high_both_renderers(): + assert _PDF_DPI >= 200, "el DPI del PDF debe ser alto (>=200)" + assert _PPTX_DPI >= 200, "el DPI del PPTX debe ser alto (>=200)" + + +def test_pdf_embedded_figure_is_high_resolution(tmp_path): + fitz = pytest.importorskip("fitz") + out = str(tmp_path / "fig.pdf") + res = render_pdf(_chapter([model.Figure(make=_simple_fig, caption="demo")]), + out, {"title": "T"}) + assert res["path"] == out + doc = fitz.open(out) + try: + widths = [] + for page in doc: + for img in page.get_images(full=True): + xref = img[0] + info = doc.extract_image(xref) + widths.append(info.get("width", 0)) + assert widths, "no se incrustó ninguna imagen en el PDF" + # A ~4" figure rasterized at 220 dpi is ~ >850 px wide. At the old 150 dpi + # it would be ~600 px. The high-res threshold proves the DPI bump. + assert max(widths) >= 800, \ + f"la figura embebida no es de alta resolución: {max(widths)} px" + finally: + doc.close() + + +# --------------------------------------------------------------------------- # +# 2) Wide table → image (PDF and PPTX); narrow table stays text. +# --------------------------------------------------------------------------- # +def test_fit_criterion_flags_wide_and_keeps_narrow(): + wide = _wide_table() + narrow = _narrow_table() + assert not _pdf_fits(wide.header, wide.rows), \ + "una tabla de 19 columnas debería NO caber como texto en A5" + assert not _pptx_fits(wide.header, wide.rows), \ + "una tabla de 19 columnas debería NO caber como tabla nativa en 16:9" + assert _pdf_fits(narrow.header, narrow.rows), \ + "una tabla de 3 columnas debería caber como texto en A5" + assert _pptx_fits(narrow.header, narrow.rows), \ + "una tabla de 3 columnas debería caber como tabla nativa en 16:9" + + +def test_wide_table_rendered_as_image_pdf(tmp_path): + fitz = pytest.importorskip("fitz") + out = str(tmp_path / "wide.pdf") + res = render_pdf(_chapter([_wide_table()]), out, {"title": "T"}) + assert res["path"] == out + doc = fitz.open(out) + try: + n_images = sum(len(page.get_images(full=True)) for page in doc) + text = "".join(page.get_text() for page in doc) + finally: + doc.close() + assert n_images >= 1, "la tabla ancha no se rasterizó como imagen en el PDF" + # The cells are now inside the image, not selectable text. A unique cell value + # must therefore NOT appear as extractable text (it lives in the picture). + assert "v4_18" not in text, \ + "la tabla ancha sigue como texto seleccionable (no se hizo imagen)" + + +def test_narrow_table_stays_selectable_text_pdf(tmp_path): + fitz = pytest.importorskip("fitz") + out = str(tmp_path / "narrow.pdf") + render_pdf(_chapter([_narrow_table()]), out, {"title": "T"}) + doc = fitz.open(out) + try: + text = "".join(page.get_text() for page in doc) + finally: + doc.close() + # Narrow table is selectable text: its header/cells are extractable. + for v in ("a", "b", "c", "1", "6"): + assert v in text, f"la celda '{v}' debería ser texto seleccionable" + + +def test_wide_table_rendered_as_picture_pptx(tmp_path): + pptx = pytest.importorskip("pptx") + from pptx.enum.shapes import MSO_SHAPE_TYPE + out = str(tmp_path / "wide.pptx") + res = render_pptx(_chapter([_wide_table()]), out, {"title": "T"}) + assert res["path"] == out + prs = pptx.Presentation(out) + pics = sum(1 for s in prs.slides for sh in s.shapes + if sh.shape_type == MSO_SHAPE_TYPE.PICTURE) + assert pics >= 1, "la tabla ancha no se colocó como imagen en el PPTX" + + +# --------------------------------------------------------------------------- # +# 3) Group(layout="side_by_side"): two columns in PPTX, stacked in PDF. +# --------------------------------------------------------------------------- # +def _side_by_side_group(): + return model.Group( + blocks=[model.Heading(text="Columna X", level=2), + _narrow_table(), + model.Figure(make=_simple_fig, caption="grafico")], + layout="side_by_side") + + +def test_side_by_side_places_two_columns_pptx(tmp_path): + pptx = pytest.importorskip("pptx") + from pptx.enum.shapes import MSO_SHAPE_TYPE + from pptx.util import Inches + out = str(tmp_path / "sbs.pptx") + render_pptx(_chapter([_side_by_side_group()]), out, {"title": "T"}) + prs = pptx.Presentation(out) + # Find the slide that holds the pair (table image + figure image). + centre_emu = int(Inches(13.333 / 2.0)) + placed = False + for s in prs.slides: + lefts = [sh.left for sh in s.shapes + if sh.shape_type == MSO_SHAPE_TYPE.PICTURE + and sh.left is not None] + if len(lefts) >= 2: + # one picture starts in the left half, another in the right half. + if min(lefts) < centre_emu and max(lefts) > centre_emu: + placed = True + break + assert placed, \ + "side_by_side no colocó tabla y figura en dos columnas de la misma slide" + + +def test_side_by_side_stacks_in_pdf(tmp_path): + fitz = pytest.importorskip("fitz") + out = str(tmp_path / "sbs.pdf") + res = render_pdf(_chapter([_side_by_side_group()]), out, {"title": "T"}) + assert res["path"] == out and res["n_pages"] >= 1 + doc = fitz.open(out) + try: + n_images = sum(len(page.get_images(full=True)) for page in doc) + text = "".join(page.get_text() for page in doc) + finally: + doc.close() + # PDF stacks: the narrow table stays selectable text (1 of its cells is + # extractable) and the figure is the single embedded image — not a 2-column + # pair of pictures like PPTX. + assert n_images == 1, "el PDF no debería usar el layout de dos imágenes" + assert "Columna X" in text and "1" in text, \ + "la tabla del grupo debería seguir como texto apilado en el PDF" + + +# --------------------------------------------------------------------------- # +# 4) Backward compatibility — default layout stacks, fitting table unchanged. +# --------------------------------------------------------------------------- # +def test_group_default_layout_is_stack(): + g = model.Group(blocks=[_narrow_table()]) + assert g.layout == "stack", "el layout por defecto debe ser 'stack'" + + +# --------------------------------------------------------------------------- # +# 5) Clickable cover index ("Índice") → chapter first page/slide. +# --------------------------------------------------------------------------- # +def _doc_with_index(): + portada = model.Chapter(id="portada", title="Portada", version="1.0.0", + blocks=[model.Heading(text="Índice", level=2), + model.TocEntry(label="Distribuciones", + target_id="Distribuciones")]) + cap = model.Chapter(id="num", title="Distribuciones", version="1.0.0", + blocks=[model.Markdown(text="contenido del capítulo")]) + return [portada, cap] + + +def test_cover_index_is_clickable_pdf(tmp_path): + fitz = pytest.importorskip("fitz") + out = str(tmp_path / "idx.pdf") + res = render_pdf(_doc_with_index(), out, {"title": "T"}) + assert res["path"] == out + doc = fitz.open(out) + try: + # The cover (page 0) must carry a GOTO link jumping to a later page. + goto = [lk for lk in doc[0].get_links() + if lk.get("kind") == fitz.LINK_GOTO and lk.get("page", 0) > 0] + finally: + doc.close() + assert goto, "el índice de la portada no produjo enlaces clicables en el PDF" + + +def test_cover_index_shows_heading_pdf(tmp_path): + fitz = pytest.importorskip("fitz") + out = str(tmp_path / "idxh.pdf") + render_pdf(_doc_with_index(), out, {"title": "T"}) + doc = fitz.open(out) + try: + text = "".join(page.get_text() for page in doc) + finally: + doc.close() + assert "Índice" in text, "la portada no muestra el encabezado 'Índice'" + assert "Este informe incluye" not in text, \ + "la portada aún muestra el texto antiguo 'Este informe incluye'" + + +def test_cover_index_is_clickable_pptx(tmp_path): + pptx = pytest.importorskip("pptx") + out = str(tmp_path / "idx.pptx") + render_pptx(_doc_with_index(), out, {"title": "T"}) + prs = pptx.Presentation(out) + cover_xml = prs.slides[0]._element.xml + assert "hlinksldjump" in cover_xml, \ + "el índice de la portada no produjo un salto de slide nativo en el PPTX" + + +def test_default_group_renders_like_before_pptx(tmp_path): + pptx = pytest.importorskip("pptx") + from pptx.enum.shapes import MSO_SHAPE_TYPE + out = str(tmp_path / "stack.pptx") + grp = model.Group(blocks=[model.Heading(text="Y", level=2), + _narrow_table(), + model.Figure(make=_simple_fig, caption="g")]) + render_pptx(_chapter([grp]), out, {"title": "T"}) + prs = pptx.Presentation(out) + # Stacked group: the narrow table is a NATIVE table (selectable), and there is + # exactly one picture (the figure) — not the two-image side-by-side layout. + n_tables = sum(1 for s in prs.slides for sh in s.shapes if sh.has_table) + n_pics = sum(1 for s in prs.slides for sh in s.shapes + if sh.shape_type == MSO_SHAPE_TYPE.PICTURE) + assert n_tables >= 1, "el grupo apilado debería usar una tabla nativa" + assert n_pics == 1, "el grupo apilado no debería duplicar imágenes" diff --git a/python/functions/datascience/render_table_as_figure.md b/python/functions/datascience/render_table_as_figure.md new file mode 100644 index 00000000..11d22d91 --- /dev/null +++ b/python/functions/datascience/render_table_as_figure.md @@ -0,0 +1,121 @@ +--- +id: render_table_as_figure_py_datascience +name: render_table_as_figure +kind: function +lang: py +domain: datascience +version: "1.0.0" +purity: impure +signature: "def render_table_as_figure(header, rows, title=None, note=None, fontsize=9.0, max_cell_chars=40) -> \"matplotlib.figure.Figure\"" +description: "Dibuja un bloque tabular (cabecera + filas) como una matplotlib.figure.Figure nítida, lista para rasterizar a DPI alto. Pensada para tablas que NO caben como texto en una página/slide del informe EDA: se rasteriza a alta resolución (el caller usa dpi=220, bbox_inches='tight') y el usuario hace zoom en el móvil para leerla entera sin perder datos. Cabecera sombreada (#eef3f6) y en negrita, filas pares (1-based) con zebra suave (#f6f8fa), tinta oscura (#1b1b1b) sobre blanco, rejilla gris muy fina (#cccccc). Trunca cada celda a max_cell_chars con elipsis y str()-ea cada valor (None -> \"\"). figsize proporcional al contenido (ancho por nº y longitud de columnas, alto por nº de filas) para que sea legible con zoom. Backend Agg sin pyplot global. Defensiva: header/rows vacíos o None, filas irregulares o cualquier error interno devuelven una Figure placeholder con texto centrado \"(tabla no disponible)\". NUNCA lanza." +tags: [eda, table, figure, matplotlib, visualization, rasterize, zoom, render, datascience, impure] +uses_functions: [] +uses_types: [] +returns: [] +returns_optional: false +error_type: "error_go_core" +imports: [matplotlib] +example: | + from datascience.render_table_as_figure import render_table_as_figure + header = ["columna", "n_nulos", "%_nulos", "distintos", "tipo", "ejemplo"] + rows = [ + ["ingresos", 12, "1.2%", 980, "float64", "2345.67"], + ["edad", 0, "0.0%", 88, "int64", "37"], + ["ciudad", 5, "0.5%", 412, "object", "Madrid"], + ] + fig = render_table_as_figure(header, rows, title="Resumen de columnas", + note="rasteriza a dpi=220 y haz zoom") + fig.savefig("/tmp/tabla.png", dpi=220, bbox_inches="tight") +tested: true +tests: + - "test_returns_figure_with_table" + - "test_rows_none_does_not_raise" + - "test_header_none_does_not_raise" + - "test_empty_lists_return_placeholder_figure" + - "test_both_none_return_placeholder_figure" + - "test_long_cell_is_truncated" + - "test_none_cells_become_empty_strings" + - "test_can_rasterize_to_png_high_dpi" + - "test_placeholder_can_rasterize" + - "test_ragged_rows_are_padded" +test_file_path: "python/functions/datascience/render_table_as_figure_test.py" +file_path: "python/functions/datascience/render_table_as_figure.py" +params: + - name: header + desc: "Lista de nombres de columna (puede ser [] o None). Cada nombre se str()-ea, se trunca a max_cell_chars y se pinta en la fila cabecera sombreada en negrita. Si está vacío/None no se dibuja fila de cabecera (solo cuerpo)." + - name: rows + desc: "Lista de filas; cada fila es una lista de celdas con valores cualesquiera (se str()-ean; None -> \"\"). Admite None (se trata como []), filas escalares (se envuelven en una celda) y filas de distinta longitud (la rejilla se rectangulariza al ancho máximo, rellenando con celdas vacías). Saltos de línea/tabs en una celda se colapsan a espacios para que no desborde a otras filas." + - name: title + desc: "Título opcional dibujado encima de la tabla, en negrita tinta #1b1b1b, alineado a la izquierda. None o \"\" => sin título. Default None." + - name: note + desc: "Nota opcional al pie de la figura, en gris #8a8a8a e itálica. None o \"\" => sin nota. Default None." + - name: fontsize + desc: "Tamaño de fuente base (pt) de las celdas del cuerpo. La cabecera usa fontsize+3 y la nota max(7, fontsize-1). Un valor no numérico o <= 0 cae a 9.0. Default 9.0." + - name: max_cell_chars + desc: "Trunca el texto de cada celda a este nº de chars (con … final cuando se recorta) para que el ancho no explote. Un valor no entero cae a 40; <= 0 deja las celdas vacías. Default 40." +output: "Un matplotlib.figure.Figure (figsize proporcional al contenido: ancho ≈ 0.9-1.6\" por columna según su texto, total acotado a 3-26\"; alto ≈ 0.32\" por fila + cabecera + espacio para título/nota, acotado) con un Axes sin ejes que contiene un ax.table(...) NO cerrado. Cabecera fondo #eef3f6 texto #1b1b1b bold; filas pares (1-based) zebra #f6f8fa, impares blanco; tinta #1b1b1b; bordes/rejilla #cccccc lw 0.4; texto alineado a la izquierda. Título encima (bold) y nota debajo (gris itálica) si se pasan. Si header/rows son vacíos o None, o ante cualquier error interno, devuelve una Figure placeholder pequeña con el texto centrado \"(tabla no disponible)\". NUNCA lanza. El caller la rasteriza (dpi=220, bbox_inches='tight') y la cierra; la función no la muestra ni la guarda." +--- + +## Ejemplo + +```python +import sys, os +sys.path.insert(0, os.path.join("python", "functions")) +from datascience.render_table_as_figure import render_table_as_figure + +# Tabla que no cabe como texto en la slide -> se rasteriza y se lee con zoom. +header = ["columna", "n_nulos", "%_nulos", "distintos", "tipo", "ejemplo"] +rows = [ + ["ingresos", 12, "1.2%", 980, "float64", "2345.67"], + ["edad", 0, "0.0%", 88, "int64", "37"], + ["ciudad", 5, "0.5%", 412, "object", "Madrid"], + ["categoria_producto", 0, "0.0%", 1840, "object", + "un_valor_categorico_muy_largo_que_se_trunca"], +] + +fig = render_table_as_figure( + header, + rows, + title="Resumen de columnas", + note="rasteriza a dpi=220 y haz zoom en el móvil", + fontsize=9.0, + max_cell_chars=40, +) + +# El renderer del informe lo rasteriza a alta resolución; aquí lo persistimos. +fig.savefig("/tmp/tabla.png", dpi=220, bbox_inches="tight") +``` + +## Cuando usarla + +Úsala en un informe EDA cuando una tabla **no cabe como texto** en una página o +slide y prefieres una imagen nítida que el lector pueda ampliar en el móvil para +leerla entera (perfiles de columnas, matrices de conteo, tablas de frecuencias +con muchas filas o columnas anchas). Pásale la cabecera y las filas tal cual (los +valores se `str()`-ean por ti) más un `title`/`note` opcionales; el llamante la +rasteriza a `dpi=220` con `bbox_inches='tight'`. Es la pareja "tabla-como-imagen" +de los gráficos `build_boxplots_figure` / `categorical_top_pie_figure`: misma +paleta y mismo contrato (Agg, sin `pyplot`, el caller cierra la figura). + +## Gotchas + +- **Impura por matplotlib.** Toca la maquinaria de render. Usa el backend `Agg` + y la API orientada a objetos `Figure`/`add_subplot` — NUNCA `pyplot.*` aquí, + para no tocar el estado global ni filtrar figuras entre llamadas. `pyplot` NO + es thread-safe; esta función construye el `Figure` directamente, así que es + segura de llamar en bucle desde el renderer. +- **El caller cierra la figura.** Devuelve el `Figure` pero no lo muestra ni lo + guarda. Quien la consume debe rasterizarla y luego liberarla + (`matplotlib.pyplot.close(fig)`) para no acumular memoria en lotes grandes. +- **Pensada para rasterizar a DPI alto.** El `figsize` es proporcional al + contenido pero la legibilidad real viene del DPI: rasteriza con `dpi=220` y + `bbox_inches='tight'`. Una tabla con muchísimas filas crece en alto (capado a + ~60") — para miles de filas, parte la tabla o resume antes de pasarla. +- **Truncación de celda visible.** Cada celda se recorta a `max_cell_chars` + (default 40) con `…` final y los saltos de línea/tabs se colapsan a espacios, + para que ninguna celda desborde a otras filas. Sube `max_cell_chars` si + necesitas ver el valor completo (a costa de ancho). +- **Defensiva, nunca lanza.** `header`/`rows` vacíos o `None`, filas escalares, + filas de distinta longitud o cualquier error interno se manejan sin propagar: + en el peor caso devuelve una `Figure` placeholder con "(tabla no disponible)". + No envuelvas la llamada en try/except por miedo a un raise — no lo hay. diff --git a/python/functions/datascience/render_table_as_figure.py b/python/functions/datascience/render_table_as_figure.py new file mode 100644 index 00000000..3994c41d --- /dev/null +++ b/python/functions/datascience/render_table_as_figure.py @@ -0,0 +1,241 @@ +"""Impure EDA helper: a crisp table rendered as a matplotlib Figure (`eda` group). + +Draws a tabular block (header + rows) as a sharp ``matplotlib.figure.Figure`` +ready to be rasterized at high DPI, so a table that does NOT fit as text on a +page/slide can still be read in full by zooming into the rasterized image on a +phone. The header is shaded and bold, even rows carry a soft zebra stripe, the +ink is dark on white and the grid is very thin. + +Impure because it touches matplotlib's rendering machinery. It uses the headless +Agg backend and the object-oriented ``Figure`` API (no ``pyplot``) so it leaks no +global state and is safe to call repeatedly from a report renderer. It is fully +defensive and NEVER raises: empty/invalid input or any internal error returns a +small placeholder figure carrying a centered "(tabla no disponible)". +""" + +import matplotlib + +matplotlib.use("Agg") + +from matplotlib.figure import Figure # noqa: E402 + +# Palette shared with the EDA report renderer so the document stays coherent. +_HEADER_BG = "#eef3f6" # header cell background. +_HEADER_TEXT = "#1b1b1b" # header cell text (bold). +_ZEBRA_BG = "#f6f8fa" # even (1-based) row background stripe. +_BODY_BG = "#ffffff" # odd row background. +_INK = "#1b1b1b" # body text + title ink. +_GRID = "#cccccc" # cell borders / grid (thin). +_NOTE_TEXT = "#8a8a8a" # muted gray for the note (italic). + + +def _placeholder_figure(message: str = "(tabla no disponible)") -> "Figure": + """Return a small fallback ``Figure`` carrying a single centered message.""" + fig = Figure(figsize=(6.0, 1.6), dpi=150) + ax = fig.add_subplot(111) + ax.axis("off") + ax.text( + 0.5, + 0.5, + message, + ha="center", + va="center", + fontsize=11, + color=_NOTE_TEXT, + style="italic", + wrap=True, + transform=ax.transAxes, + ) + fig.tight_layout() + return fig + + +def _cell_text(value, max_cell_chars: int) -> str: + """``str()`` a cell value defensively, None -> "", truncate with an ellipsis.""" + s = "" if value is None else str(value) + # Collapse newlines/tabs so a single cell never spills across table rows. + s = s.replace("\n", " ").replace("\r", " ").replace("\t", " ") + try: + limit = int(max_cell_chars) + except (TypeError, ValueError): + limit = 40 + if limit <= 0: + return "" + if len(s) <= limit: + return s + if limit == 1: + return "…" + return s[: limit - 1] + "…" + + +def render_table_as_figure( + header, + rows, + title=None, + note=None, + fontsize=9.0, + max_cell_chars=40, +): + """Dibuja una tabla nítida como matplotlib.figure.Figure, lista para rasterizar a DPI alto. + + Pensada para tablas que NO caben como texto en una página/slide: se rasteriza + a alta resolución y el usuario hace zoom en el móvil para leerla entera sin + perder datos. Cabecera sombreada + negrita, filas pares con zebra suave, + tinta oscura sobre blanco, rejilla muy fina. + + Args: + header: lista de nombres de columna (puede ser []). + rows: lista de filas; cada fila es una lista de celdas (valores cualquiera, se str()-ean). + title: título opcional dibujado encima de la tabla (o None). + note: nota opcional en gris/itálica bajo la tabla (o None). + fontsize: tamaño de fuente base (pt) de las celdas. + max_cell_chars: trunca el texto de celda a este nº de chars (con … final) para que no explote el ancho. + + Returns: + matplotlib.figure.Figure — NO cerrada (el llamante la rasteriza y la cierra). + Nunca lanza: ante cualquier error devuelve una Figure con el texto "(tabla no disponible)". + """ + try: + # --- Defensive normalization of header/rows into a rectangular grid. + header_list = list(header) if isinstance(header, (list, tuple)) else [] + raw_rows = list(rows) if isinstance(rows, (list, tuple)) else [] + + clean_rows = [] + for row in raw_rows: + if isinstance(row, (list, tuple)): + clean_rows.append(list(row)) + elif row is None: + clean_rows.append([]) + else: + # A scalar row becomes a single-cell row instead of being dropped. + clean_rows.append([row]) + + # Nothing to draw at all -> placeholder. + if not header_list and not clean_rows: + return _placeholder_figure() + + # Number of columns = widest of header / any row. + n_cols = len(header_list) + for row in clean_rows: + if len(row) > n_cols: + n_cols = len(row) + if n_cols <= 0: + return _placeholder_figure() + + # Base font size, tolerate a bad value. + try: + base_fs = float(fontsize) + except (TypeError, ValueError): + base_fs = 9.0 + if base_fs <= 0: + base_fs = 9.0 + + # --- Build the truncated, padded text matrix. + header_cells = [ + _cell_text(header_list[c] if c < len(header_list) else "", max_cell_chars) + for c in range(n_cols) + ] + body_cells = [] + for row in clean_rows: + body_cells.append( + [ + _cell_text(row[c] if c < len(row) else "", max_cell_chars) + for c in range(n_cols) + ] + ) + + has_header = any(t for t in header_cells) + n_body = len(body_cells) + # Total drawn table rows (header counts as one when present). + n_table_rows = n_body + (1 if has_header else 0) + if n_table_rows <= 0: + return _placeholder_figure() + + # --- figsize proportional to content so it reads under zoom. + # Width: per-column width scales with the longest text in that column, + # clamped to a sensible per-column range, total capped. + per_col_widths = [] + for c in range(n_cols): + col_texts = [header_cells[c]] if has_header else [] + col_texts += [body_cells[r][c] for r in range(n_body)] + longest = max((len(t) for t in col_texts), default=0) + # ~0.085" per char at the base font, clamped to [0.9, 1.6] inches. + w = 0.9 + 0.085 * max(longest - 6, 0) + w = max(0.9, min(1.6, w)) + per_col_widths.append(w) + fig_w = sum(per_col_widths) + fig_w = max(3.0, min(26.0, fig_w)) + + # Height: ~0.32" per row + room for title / note. + fig_h = 0.32 * n_table_rows + 0.30 + if title is not None and str(title) != "": + fig_h += 0.45 + if note is not None and str(note) != "": + fig_h += 0.30 + fig_h = max(1.0, min(60.0, fig_h)) + + fig = Figure(figsize=(fig_w, fig_h), dpi=150) + ax = fig.add_subplot(111) + ax.axis("off") + + # Reserve vertical bands for the optional title (top) and note (bottom) + # so the table itself never overlaps them. + title_band = 0.10 if (title is not None and str(title) != "") else 0.0 + note_band = 0.07 if (note is not None and str(note) != "") else 0.0 + table_bbox = [0.0, note_band, 1.0, max(0.05, 1.0 - title_band - note_band)] + + cell_text = ([header_cells] if has_header else []) + body_cells + + col_widths = [w / fig_w for w in per_col_widths] + + table = ax.table( + cellText=cell_text, + colWidths=col_widths, + cellLoc="left", + loc="center", + bbox=table_bbox, + ) + table.auto_set_font_size(False) + table.set_fontsize(base_fs) + + # --- Style every cell: zebra body, shaded bold header, thin gray grid. + for (r, _c), cell in table.get_celld().items(): + cell.set_edgecolor(_GRID) + cell.set_linewidth(0.4) + # Small horizontal padding so text does not touch the border. + cell.PAD = 0.04 + if has_header and r == 0: + cell.set_facecolor(_HEADER_BG) + cell.set_text_props(color=_HEADER_TEXT, fontweight="bold", ha="left") + else: + body_index = r - 1 if has_header else r # 0-based body row. + # 1-based even rows get the zebra stripe. + is_even = ((body_index + 1) % 2) == 0 + cell.set_facecolor(_ZEBRA_BG if is_even else _BODY_BG) + cell.set_text_props(color=_INK, ha="left") + + if title is not None and str(title) != "": + ax.set_title( + str(title), + fontsize=base_fs + 3.0, + fontweight="bold", + color=_INK, + loc="left", + pad=8, + ) + + if note is not None and str(note) != "": + fig.text( + 0.01, + 0.01, + str(note), + ha="left", + va="bottom", + fontsize=max(7.0, base_fs - 1.0), + color=_NOTE_TEXT, + style="italic", + ) + + return fig + except Exception: # noqa: BLE001 — never raise from a figure builder. + return _placeholder_figure() diff --git a/python/functions/datascience/render_table_as_figure_test.py b/python/functions/datascience/render_table_as_figure_test.py new file mode 100644 index 00000000..2ebe4b79 --- /dev/null +++ b/python/functions/datascience/render_table_as_figure_test.py @@ -0,0 +1,119 @@ +"""Tests para render_table_as_figure (tabla nítida como Figure, grupo eda). + +Usa el backend Agg sin display; no muestra ni guarda figuras a disco salvo a un +BytesIO en memoria. Cada test cierra explícitamente la Figure construida +(matplotlib.pyplot.close) para no acumular estado entre tests. +""" + +from io import BytesIO + +import matplotlib + +matplotlib.use("Agg") + +import matplotlib.pyplot as plt # noqa: E402 +from matplotlib.figure import Figure # noqa: E402 + +from render_table_as_figure import render_table_as_figure + + +def _grid(n_cols, n_rows): + """Cabecera de n_cols columnas + n_rows filas de celdas.""" + header = [f"col_{c}" for c in range(n_cols)] + rows = [[f"r{r}c{c}" for c in range(n_cols)] for r in range(n_rows)] + return header, rows + + +def test_returns_figure_with_table(): + header, rows = _grid(6, 5) + fig = render_table_as_figure(header, rows, title="Tabla", note="nota al pie") + assert isinstance(fig, Figure) + # Hay al menos un Axes y ese Axes contiene una tabla con celdas. + assert len(fig.axes) >= 1 + ax = fig.axes[0] + assert len(ax.tables) >= 1 + # 6 columnas x (1 cabecera + 5 filas) = 36 celdas. + assert len(ax.tables[0].get_celld()) == 6 * (5 + 1) + plt.close(fig) + + +def test_rows_none_does_not_raise(): + fig = render_table_as_figure(["a", "b"], None) + assert isinstance(fig, Figure) + assert len(fig.axes) >= 1 + plt.close(fig) + + +def test_header_none_does_not_raise(): + fig = render_table_as_figure(None, [["x", "y"], ["z", "w"]]) + assert isinstance(fig, Figure) + assert len(fig.axes) >= 1 + plt.close(fig) + + +def test_empty_lists_return_placeholder_figure(): + fig = render_table_as_figure([], []) + assert isinstance(fig, Figure) + # Placeholder: un Axes con texto, sin tabla. + assert len(fig.axes) >= 1 + assert len(fig.axes[0].tables) == 0 + plt.close(fig) + + +def test_both_none_return_placeholder_figure(): + fig = render_table_as_figure(None, None) + assert isinstance(fig, Figure) + assert len(fig.axes[0].tables) == 0 + plt.close(fig) + + +def test_long_cell_is_truncated(): + long_value = "x" * 200 + header, _ = _grid(2, 0) + fig = render_table_as_figure(header, [[long_value, "ok"]], max_cell_chars=20) + assert isinstance(fig, Figure) + ax = fig.axes[0] + texts = [c.get_text().get_text() for c in ax.tables[0].get_celld().values()] + # La celda larga aparece truncada con elipsis y nunca en su forma completa. + assert any(t.endswith("…") and len(t) <= 20 for t in texts) + assert long_value not in texts + plt.close(fig) + + +def test_none_cells_become_empty_strings(): + fig = render_table_as_figure(["a", "b"], [[None, "v"], ["w", None]]) + assert isinstance(fig, Figure) + ax = fig.axes[0] + texts = [c.get_text().get_text() for c in ax.tables[0].get_celld().values()] + # Hay celdas vacías (los None) y celdas con valor. + assert "" in texts + assert "v" in texts + plt.close(fig) + + +def test_can_rasterize_to_png_high_dpi(): + header, rows = _grid(6, 8) + fig = render_table_as_figure(header, rows, title="Render", note="zoom me") + buf = BytesIO() + # No debe lanzar al rasterizar a DPI alto con bbox tight. + fig.savefig(buf, format="png", dpi=220, bbox_inches="tight") + assert buf.getbuffer().nbytes > 0 + plt.close(fig) + + +def test_placeholder_can_rasterize(): + fig = render_table_as_figure([], []) + buf = BytesIO() + fig.savefig(buf, format="png", dpi=220, bbox_inches="tight") + assert buf.getbuffer().nbytes > 0 + plt.close(fig) + + +def test_ragged_rows_are_padded(): + # Filas de distinta longitud: la rejilla se rectangulariza al ancho máximo. + fig = render_table_as_figure(["a", "b", "c"], [["1"], ["1", "2", "3", "4"]]) + assert isinstance(fig, Figure) + ax = fig.axes[0] + # 4 columnas (la fila más ancha) x (1 cabecera + 2 filas) = 12 celdas. + assert len(ax.tables[0].get_celld()) == 4 * (2 + 1) + plt.close(fig) From cab0fbf0a3f4a6a7cdbb608cb00c2753b33c636b Mon Sep 17 00:00:00 2001 From: Egutierrez <egutierrez@dead.dd> Date: Wed, 1 Jul 2026 02:01:07 +0200 Subject: [PATCH 51/53] =?UTF-8?q?feat(eda):=20CAP4/CAP5=20distribuciones?= =?UTF-8?q?=20=E2=80=94=20p=C3=A1rrafos=20al=20glosario,=20desc=20LLM=20+?= =?UTF-8?q?=20unidad=20por=20columna,=20donut=E2=86=92barras,=20PPT=20figu?= =?UTF-8?q?ra=20a=20la=20derecha?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CAP4 num_distr: - Mueve el párrafo introductorio largo del histograma/boxplot al glosario (nuevo término clicable "histograma_boxplot"); el cuerpo del capítulo solo nombra el término con [[term:histograma_boxplot]] y la explicación completa (código de colores, 1,5·IQR, lectura de asimetría) vive en la entrada del glosario. La información se traslada, no se pierde. - Añade por columna numérica la descripción de negocio del LLM y la unidad, leídas de profile['llm']['dictionary'] (empareja por nombre de columna). Sin bloque LLM el bloque de descripción se omite limpiamente. CAP5 cat_distr: - Mueve el párrafo "Cada columna categórica ocupa su propia página..." al glosario (nuevo término clicable "pagina_categorica"); el intro solo nombra los términos entropía y pagina_categorica. - Añade descripción LLM + unidad por columna (misma fuente que CAP4). - Cambia el donut/pie por gráfico de barras horizontales (nueva función del registry categorical_top_bar_figure_py_datascience, contrato de entrada idéntico al donut para swap directo) más su fallback inline de barras. - Marca cada Group de columna con layout="side_by_side": en PPTX la tabla de cardinalidad queda a la izquierda y la barra a la derecha; en PDF se apila (A5 estrecho). No toca los renderers — el soporte de layout ya existía. Glosario: - Catálogo canónico _BASELINE_TERMS con las definiciones de los dos términos nuevos; build_glosario completa la definición de un término registrado sin ella desde el catálogo (los chapters solo registran clave+label). Tests actualizados (donut→barras, side_by_side, LLM desc/unidad, glosario) y nueva función con sus tests. Suite del subsistema + acceptance verde. --- .../automatic_eda/chapters/cat_distr.py | 190 +++++++++----- .../automatic_eda/chapters/cat_distr_test.py | 137 ++++++++-- .../automatic_eda/chapters/glosario.py | 63 ++++- .../automatic_eda/chapters/num_distr.py | 112 +++++++-- .../automatic_eda/chapters/num_distr_test.py | 50 +++- .../datascience/categorical_top_bar_figure.md | 111 +++++++++ .../datascience/categorical_top_bar_figure.py | 233 ++++++++++++++++++ .../categorical_top_bar_figure_test.py | 103 ++++++++ 8 files changed, 901 insertions(+), 98 deletions(-) create mode 100644 python/functions/datascience/categorical_top_bar_figure.md create mode 100644 python/functions/datascience/categorical_top_bar_figure.py create mode 100644 python/functions/datascience/categorical_top_bar_figure_test.py diff --git a/python/functions/datascience/automatic_eda/chapters/cat_distr.py b/python/functions/datascience/automatic_eda/chapters/cat_distr.py index b722c68a..3a252605 100644 --- a/python/functions/datascience/automatic_eda/chapters/cat_distr.py +++ b/python/functions/datascience/automatic_eda/chapters/cat_distr.py @@ -5,28 +5,32 @@ page (PDF) / slide (PPTX)**: every column is wrapped in a keep-together ``model.Group`` with ``page_break_before=True`` (except the first, which may share the intro's page), so its chart sits next to its tables and no column is split. -A short intro names the clickable **[[term:entropia]]entropía[[/term]]** term — -the full definition lives in the GLOSARIO chapter, so it is NOT repeated inline -here (one click jumps to the glossary entry). The intro also carries the dataset -row total used as a comparison baseline. +Per column the Group is laid out ``side_by_side`` (PPTX: cardinality table LEFT, +chart RIGHT; PDF: stacked) and contains, in order: -Per column the Group contains, in order: - -1. A cardinality key/value table: distinct values, ``% distinct`` (distinct / +1. The column name plus, when the LLM layer ran, its business **description** and + **unit** (read from ``profile['llm']['dictionary']``, matched by column name). +2. A cardinality key/value table: distinct values, ``% distinct`` (distinct / total rows), total dataset rows, singleton values (frequency 1), entropy with its theoretical maximum and the normalized ratio, mode, imbalance and string-length stats. -2. A short note flagging problematic cardinality (id-like ≈100% distinct, or a +3. A short note flagging problematic cardinality (id-like ≈100% distinct, or a single dominating category). -3. A ``top-k`` table (value / count / %). -4. A **donut pie chart** of the most common categories (top-k + an "Otros" +4. A ``top-k`` table (value / count / %). +5. A **horizontal bar chart** of the most common categories (top-k + an "Otros" bucket), drawn lazily so the renderers scale it to fit entirely. +A short intro names the clickable **[[term:entropia]]entropía[[/term]]** and +**[[term:pagina_categorica]]page-layout[[/term]]** terms — their full +definitions live in the GLOSARIO chapter, so they are NOT repeated inline here +(one click jumps to the glossary entry). The intro also carries the dataset row +total used as a comparison baseline. + Data comes from the ``eda`` group: each ``columns[i]['categorical']`` is the output of ``summarize_categorical`` (``top[{value,count,pct}]``, ``mode``, ``n_distinct``, ``entropy``, ``imbalance``, ``len_min/mean/max``). The derived -cardinality metrics and the pie figure are delegated to two registry functions -(``categorical_cardinality_block`` and ``categorical_top_pie_figure``); both are +cardinality metrics and the bar figure are delegated to two registry functions +(``categorical_cardinality_block`` and ``categorical_top_bar_figure``); both are imported lazily and degrade to a minimal inline fallback so this chapter never raises even if they are unavailable. @@ -39,10 +43,21 @@ import math from .. import model -CHAPTER_VERSION = "1.2.0" +CHAPTER_VERSION = "1.3.0" CHAPTER_ID = "cat_distr" CHAPTER_TITLE = "Distribuciones categóricas" +# Key under which eda_llm_insights stores its interpretive block in the profile. +LLM_KEY = "llm" + +# Second glossary term this chapter names: "how each categorical page is laid +# out". The long paragraph that used to describe it inline in the intro now lives +# in the GLOSARIO chapter (canonical definition in ``glosario._BASELINE_TERMS``); +# the intro only names the clickable term, relocating the explanation, not losing +# it. The chapter only needs to register key+label here. +_TERM_PAGINA_KEY = "pagina_categorica" +_TERM_PAGINA_LABEL = "Cómo se organiza cada página categórica" + # Glossary term this chapter explains. Registered in the shared collector and # marked clickable on its first appearance (end-to-end glossary example — # mejora 6). Other chapters hook their own terms the same way (see the contract). @@ -59,14 +74,14 @@ _TERM_ENTROPIA_DEF = ( # Cap the number of categorical columns rendered to keep the document bounded; # the rest are summarized in a closing note (no silent truncation). MAX_COLS = 40 -# Rows shown in each top-k table and explicit slices in the pie. Kept moderate so -# the whole column — cardinality table + top-k table + donut — fits on ONE +# Rows shown in each top-k table and explicit bars in the chart. Kept moderate so +# the whole column — cardinality table + top-k table + bar chart — fits on ONE # page/slide with the chart next to its tables; the table note still reports # "top N of M" so nothing is silently hidden. For id-like columns (≈100% # distinct) the top-k table is dropped entirely (it would be a list of unique -# values — pure noise), which also frees the room the donut needs (see build). +# values — pure noise), which also frees the room the chart needs (see build). TOP_TABLE_ROWS = 8 -PIE_TOP_K = 6 +CHART_TOP_K = 6 # Truncate very long category labels in tables (the renderer also wraps). Kept # tight so a column with long id-like values (names, tickets) still fits its page. LABEL_MAX = 28 @@ -208,26 +223,74 @@ def _fallback_cardinality(cat: dict, n_rows) -> dict: } -def _pie_make(top, n_distinct, title, n_rows): - """Return a zero-arg callable that builds the donut figure lazily.""" +def _llm_index(profile: dict, ctx: dict) -> dict: + """Map column name -> its LLM dictionary entry (description/unit/...). + + Reads the ``llm.dictionary`` list that ``eda_llm_insights`` stored in the + profile (``profile['llm']``; falls back to ``ctx['llm']``). Returns an empty + dict when ``run_llm`` did not run, so the caller degrades cleanly. Fully + defensive: never raises on malformed input. + """ + llm = profile.get(LLM_KEY) + if not isinstance(llm, dict): + llm = ctx.get(LLM_KEY) + if not isinstance(llm, dict): + return {} + entries = llm.get("dictionary") + if not isinstance(entries, (list, tuple)): + return {} + index: dict = {} + for e in entries: + if not isinstance(e, dict): + continue + col = e.get("column") + if col is None: + continue + index[model._safe_str(col)] = e + return index + + +def _llm_desc_unit_block(name: str, llm_index: dict): + """Markdown block with the LLM business description + unit of a column, or + None when no LLM entry matches the column (clean fallback without LLM).""" + entry = llm_index.get(model._safe_str(name)) + if not isinstance(entry, dict): + return None + raw_desc = entry.get("description") or entry.get("business_meaning") + desc = " ".join(model._safe_str(raw_desc).split()) if raw_desc else "" + raw_unit = entry.get("unit") + unit = " ".join(model._safe_str(raw_unit).split()) if raw_unit else "" + parts = [] + if desc: + parts.append(f"**Descripción:** {desc}") + if unit: + parts.append(f"**Unidad:** {unit}") + if not parts: + return None + return model.Markdown(text=" · ".join(parts)) + + +def _bar_make(top, n_distinct, title, n_rows): + """Return a zero-arg callable that builds the bar figure lazily.""" def make(): try: - from datascience.categorical_top_pie_figure import ( - categorical_top_pie_figure, + from datascience.categorical_top_bar_figure import ( + categorical_top_bar_figure, ) - return categorical_top_pie_figure( + return categorical_top_bar_figure( top=top, n_distinct=n_distinct or 0, title=title, - top_k=PIE_TOP_K, n_rows=n_rows) + top_k=CHART_TOP_K, n_rows=n_rows) except Exception: # noqa: BLE001 — minimal local fallback figure. - return _fallback_pie(top, title) + return _fallback_bar(top, title) return make -def _fallback_pie(top, title): - """Minimal donut figure used only if the registry function is unavailable.""" +def _fallback_bar(top, title): + """Minimal horizontal-bar figure used only if the registry function is + unavailable. Largest category on top, the rest folded into "Otros".""" import matplotlib matplotlib.use("Agg") @@ -238,8 +301,8 @@ def _fallback_pie(top, title): items = [t for t in (top or []) if isinstance(t, dict) and isinstance(t.get("count"), (int, float))] items = sorted(items, key=lambda t: t.get("count") or 0, reverse=True) - head = items[:PIE_TOP_K] - rest = items[PIE_TOP_K:] + head = items[:CHART_TOP_K] + rest = items[CHART_TOP_K:] labels = [_truncate(t.get("value"), 20) for t in head] sizes = [float(t.get("count") or 0) for t in head] if rest: @@ -249,10 +312,13 @@ def _fallback_pie(top, title): ax.text(0.5, 0.5, "sin datos categóricos", ha="center", va="center") ax.axis("off") return fig - ax.pie(sizes, labels=None, wedgeprops={"width": 0.42}, - autopct=lambda p: f"{p:.0f}%" if p >= 4 else "") - ax.legend(labels, loc="center left", bbox_to_anchor=(1.0, 0.5), - fontsize=7, frameon=False) + # barh draws bottom-up, so reverse to put the largest category on top. + y_pos = range(len(labels)) + ax.barh(list(y_pos), list(reversed(sizes)), color="#4C72B0", + edgecolor="white") + ax.set_yticks(list(y_pos)) + ax.set_yticklabels(list(reversed(labels)), fontsize=7) + ax.set_xlabel("conteo", fontsize=8) ax.set_title(_truncate(title, 40)) fig.tight_layout() return fig @@ -375,17 +441,19 @@ def _topk_table(cat: dict): def _intro_blocks(n_rows, mark_term: bool = False): total = _fmt_int(n_rows) - # Mark the first appearance of the term as a clickable glossary jump when the - # term was registered (mark_term). The full definition of entropy lives in the - # GLOSARIO chapter, so the intro only names the clickable term here instead of - # repeating the long explanation (avoids the redundancy with the glossary). + # Mark the first appearance of each term as a clickable glossary jump when the + # terms were registered (mark_term). The full definition of the entropy term + # AND of how each categorical page is laid out live in the GLOSARIO chapter, so + # the intro only names the clickable terms instead of repeating the long + # explanation (avoids the redundancy with the glossary). entropia = ("[[term:entropia]]entropía[[/term]]" if mark_term else "entropía") + pagina = ("[[term:pagina_categorica]]cómo se organiza cada página[[/term]]" + if mark_term else "cómo se organiza cada página") text = ( - f"Cada columna categórica ocupa su propia página: sus métricas de " - f"cardinalidad —incluida la {entropia}—, una nota que señala cardinalidad " - "problemática, la tabla de las categorías más frecuentes y un gráfico de " - "tarta (donut) de las más comunes, todo junto." + f"Cada columna categórica ocupa su propia página — {pagina}: " + f"cardinalidad (incluida la {entropia}), top de categorías y un gráfico " + "de barras de las más comunes." ) if n_rows is not None: text += f" El dataset tiene {total} filas en total como referencia." @@ -406,47 +474,59 @@ def build_cat_distr(profile: dict, ctx: dict): return None n_rows = profile.get("n_rows") - # Register "entropía" in the shared glossary collector (if present) and mark - # its first appearance clickable. End-to-end glossary example (mejora 6). + # Register "entropía" and the "how each categorical page is laid out" term in + # the shared glossary collector (if present) and mark their first appearance + # clickable. End-to-end glossary example (mejora 6). glossary = ctx.get("glossary") mark_term = False if isinstance(glossary, model.GlossaryCollector): glossary.add(_TERM_ENTROPIA_KEY, _TERM_ENTROPIA_LABEL, _TERM_ENTROPIA_DEF) + glossary.add(_TERM_PAGINA_KEY, _TERM_PAGINA_LABEL) mark_term = True blocks = list(_intro_blocks(n_rows, mark_term=mark_term)) + # Business description + unit per column come from the LLM dictionary + # (profile['llm']['dictionary'], matched by column name); absent without + # run_llm, in which case the per-column description block is simply omitted. + llm_index = _llm_index(profile, ctx) + rendered = cat_cols[:MAX_COLS] for idx, col in enumerate(rendered): name = col.get("name") or "(columna)" cat = col.get("categorical") or {} card = _normalize_card(_cardinality(cat, n_rows)) - # One Group per categorical column: heading + cardinality table + flag - # note + top-k table + donut figure are kept together and the renderer - # starts each on a fresh page/slide (page_break_before) so every column - # gets its own page with its chart next to its tables. The first column - # may share the intro's page (no forced break) to avoid a near-empty page. - col_blocks = [ - model.Heading(text=str(name), level=2), - _cardinality_block(card), - ] + # One Group per categorical column: heading + (optional) LLM description + + # cardinality table + flag note + top-k table + bar figure are kept + # together and the renderer starts each on a fresh page/slide + # (page_break_before) so every column gets its own page with its chart next + # to its tables. The first column may share the intro's page (no forced + # break) to avoid a near-empty page. + col_blocks = [model.Heading(text=str(name), level=2)] + desc_block = _llm_desc_unit_block(name, llm_index) + if desc_block is not None: + col_blocks.append(desc_block) + col_blocks.append(_cardinality_block(card)) note = _flag_note(card) if note is not None: col_blocks.append(note) # For id-like columns (≈100% distinct) the top-k is a list of unique # values — pure noise; skip it (the flag note already explains why) and - # let the donut take that room so the whole column fits one page/slide. + # let the bar chart take that room so the whole column fits one page/slide. if not card.get("id_like"): topk = _topk_table(cat) if topk is not None: col_blocks.append(topk) col_blocks.append(model.Figure( - make=_pie_make(cat.get("top") or [], card.get("n_distinct"), + make=_bar_make(cat.get("top") or [], card.get("n_distinct"), str(name), n_rows), caption=(f"Categorías más comunes de «{_truncate(name, 32)}» " - "(donut: top-k + «Otros»)"))) - blocks.append(model.Group(blocks=col_blocks, + "(barras: top-k + «Otros»)"))) + # layout="side_by_side": in PPTX the cardinality table goes to the LEFT and + # the bar chart to the RIGHT of the same slide; the PDF renderer stacks it + # (the A5 mobile page is too narrow for two readable columns). + blocks.append(model.Group(blocks=col_blocks, layout="side_by_side", page_break_before=(idx > 0))) if len(cat_cols) > len(rendered): diff --git a/python/functions/datascience/automatic_eda/chapters/cat_distr_test.py b/python/functions/datascience/automatic_eda/chapters/cat_distr_test.py index 919b86fa..4dd9d334 100644 --- a/python/functions/datascience/automatic_eda/chapters/cat_distr_test.py +++ b/python/functions/datascience/automatic_eda/chapters/cat_distr_test.py @@ -2,12 +2,14 @@ Self-contained: builds synthetic TableProfiles (no DuckDB) so the suite is fast and deterministic. Verifies that ``build_cat_distr`` emits the blocks the user -asked for (distinct/total/%-distinct/unique metrics, top-k table and a donut +asked for (distinct/total/%-distinct/unique metrics, top-k table and a bar figure), that EACH categorical column is wrapped in its own keep-together -``Group`` that starts on a fresh page/slide (one column per page, chart next to -its tables), that the long entropy explanation is NOT repeated inline (it lives -in the glossary — only the clickable term is kept), that the chapter renders -inside the full document to both PDF and PPTX showing that content, that a +``Group`` laid out ``side_by_side`` (PPTX: table left / bars right) that starts on +a fresh page/slide (one column per page, chart next to its tables), that the LLM +business description + unit are shown per column when the profile carries an LLM +block, that the long entropy / page-layout explanations are NOT repeated inline +(they live in the glossary — only the clickable terms are kept), that the chapter +renders inside the full document to both PDF and PPTX showing that content, that a profile with no categorical columns yields ``None`` without raising, and that long labels / many columns are never cut in either output. """ @@ -116,6 +118,10 @@ def test_golden_build_cat_distr_emite_bloques_pedidos(): assert "log2" not in md.text # redundant explanation removed. assert "máxima diversidad" not in md.text + # The donut/pie is gone: the intro no longer mentions tarta/donut (the chart + # is now a bar chart; the long page-layout explanation moved to the glossary). + assert "donut" not in md.text and "tarta" not in md.text + # Per-column blocks are wrapped in keep-together Groups: flatten to inspect. flat = _flatten(ch.blocks) kv = next(b for b in flat if isinstance(b, KVTable)) @@ -128,11 +134,13 @@ def test_golden_build_cat_distr_emite_bloques_pedidos(): assert any("Entropía" in lbl for lbl in labels) assert "únicos" in values and "%" in values assert "bits" in values and "norm" in values # entropy + max + normalized. - # Top-k table + pie figure. + # Top-k table + bar figure. dt = next(b for b in flat if isinstance(b, DataTable)) assert dt.header == ["Valor", "Conteo", "%"] assert any("neumaticos" in str(cell) for row in dt.rows for cell in row) assert any(isinstance(b, Figure) for b in flat) + # Each per-column Group is laid out side_by_side (table left / bars right). + assert all(g.layout == "side_by_side" for g in _column_groups(ch)) # id-like column flagged with a Note that also explains the top-k is dropped. idnote = next((b for b in flat if isinstance(b, Note) and "identificador" in b.text), None) @@ -140,9 +148,9 @@ def test_golden_build_cat_distr_emite_bloques_pedidos(): assert "No se lista el top" in idnote.text -def test_golden_idlike_omite_topk_y_conserva_donut(): +def test_golden_idlike_omite_topk_y_conserva_grafico(): # The id-like column (uuid, 100% distinct) must NOT carry a top-k DataTable - # (it would be a list of unique values), but must still keep its donut Figure + # (it would be a list of unique values), but must still keep its bar Figure # and its cardinality table so it stays a full per-column page. ch = build_cat_distr(_profile(), {}) groups = _column_groups(ch) @@ -151,7 +159,7 @@ def test_golden_idlike_omite_topk_y_conserva_donut(): kinds = [b.kind for b in uuid_group.blocks] assert "data_table" not in kinds # top-k of unique values dropped. assert "kv_table" in kinds # cardinality kept. - assert "figure" in kinds # donut kept (chart per column). + assert "figure" in kinds # bar chart kept (chart per column). # A non-id-like column keeps its top-k table. cat_group = next(g for g in groups if any(getattr(b, "text", "") == "categoria" @@ -205,7 +213,7 @@ def test_golden_render_pdf_una_pagina_por_columna(): assert "Entrop" in txt assert "distintos" in txt assert "categoria" in txt and "neumaticos" in txt - assert "donut" in txt # figure caption rendered as text. + assert "barras" in txt # bar-chart caption rendered as text (PDF). assert "identificador" in txt # id-like note rendered. @@ -258,9 +266,11 @@ def _profile_high_card() -> dict: def test_golden_pptx_una_slide_por_columna_con_su_grafico(): - """Each categorical column occupies EXACTLY ONE cat_distr slide that carries - BOTH its cardinality table and its donut figure (picture) — i.e. the chart is - never separated from its table, even for a high-cardinality column.""" + """Cada columna categórica ocupa EXACTAMENTE UN slide cat_distr que lleva su + gráfico (picture) en la misma slide — el chart nunca se separa de su columna, + ni siquiera para una columna de alta cardinalidad. Con layout side_by_side la + tabla se rasteriza a imagen, así que la comprobación se hace por presencia de + picture (no por el texto de la tabla).""" from pptx.enum.shapes import MSO_SHAPE_TYPE prof = _profile_high_card() @@ -272,7 +282,7 @@ def test_golden_pptx_una_slide_por_columna_con_su_grafico(): prs = Presentation(out) # Per column: the cat_distr slides whose text mentions it, and whether the - # owning slide also has the donut caption + an actual picture shape. + # owning slide also carries an actual picture shape (its chart). slides_with_col = {n: [] for n in cat_names} owner_has_chart = {n: False for n in cat_names} for i, sl in enumerate(prs.slides): @@ -288,15 +298,106 @@ def test_golden_pptx_una_slide_por_columna_con_su_grafico(): for n in cat_names: if n in txt: slides_with_col[n].append(i) - has_table = "Cardinalidad" in txt or "distintos" in txt - if has_pic and "donut" in txt and has_table: + if has_pic: owner_has_chart[n] = True for n in cat_names: # Exactly one slide carries the column (not split across slides). assert len(slides_with_col[n]) == 1, (n, slides_with_col[n]) - # That single slide also holds its table AND its donut picture. - assert owner_has_chart[n], (n, "tabla y donut no están en el mismo slide") + # That single slide also holds its chart picture. + assert owner_has_chart[n], (n, "el gráfico no está en el slide de la columna") + + +def test_golden_pptx_columna_side_by_side_tabla_izq_barra_der(): + """Con layout side_by_side, una columna categórica coloca su tabla de + cardinalidad (imagen) en la mitad izquierda y su gráfico de barras (imagen) en + la mitad derecha de la MISMA slide. Verifica que al menos una columna queda en + dos columnas (tabla-izq / barras-der), evidencia del side_by_side en PPTX.""" + from pptx.enum.shapes import MSO_SHAPE_TYPE + from pptx.util import Inches + + with tempfile.TemporaryDirectory() as d: + out = os.path.join(d, "eda.pptx") + render_automatic_eda_pptx(_profile(), out, {"title": "EDA"}) + prs = Presentation(out) + centre = int(Inches(13.333 / 2.0)) # half of the 16:9 slide width. + two_col_slides = 0 + for sl in prs.slides: + texts, lefts = [], [] + for sh in sl.shapes: + if sh.has_text_frame: + texts.append(sh.text_frame.text) + if (sh.shape_type == MSO_SHAPE_TYPE.PICTURE + and sh.left is not None): + lefts.append(sh.left) + txt = re.sub(r"\s+", " ", " ".join(texts)) + if "Distribuciones categ" not in txt: + continue + # One picture starts in the left half, another in the right half. + if len(lefts) >= 2 and min(lefts) < centre and max(lefts) > centre: + two_col_slides += 1 + assert two_col_slides >= 1, ( + "ninguna columna quedó con tabla-izq / barras-der (side_by_side)") + + +def _profile_with_llm() -> dict: + """The base profile plus an ``llm`` block (as eda_llm_insights would store it + with run_llm=True): a data dictionary with description/unit per column.""" + prof = _profile() + prof["llm"] = { + "dictionary": [ + {"column": "categoria", + "description": "Familia de producto del recambio", + "business_meaning": "Agrupa el catálogo por tipo de pieza", + "unit": "categoría"}, + {"column": "uuid", + "description": "Identificador único de registro", + "unit": ""}, + ], + } + return prof + + +def test_llm_descripcion_y_unidad_por_columna(): + # With an LLM dictionary, each categorical column whose name matches shows its + # business description and unit in a per-column markdown block. + ch = build_cat_distr(_profile_with_llm(), {}) + groups = _column_groups(ch) + cat_group = next(g for g in groups + if any(getattr(b, "text", "") == "categoria" + for b in g.blocks)) + md = " ".join(b.text for b in cat_group.blocks + if getattr(b, "kind", "") == "markdown") + assert "Descripción" in md and "Familia de producto" in md + assert "Unidad" in md and "categoría" in md + + +def test_edge_sin_llm_no_anade_descripcion(): + # Without an LLM block the per-column description markdown is simply omitted; + # the column still renders its cardinality table and bar figure. + ch = build_cat_distr(_profile(), {}) + for g in _column_groups(ch): + mds = [b.text for b in g.blocks if getattr(b, "kind", "") == "markdown"] + assert not any("Descripción" in t for t in mds) + + +def test_pagina_categorica_clicable_y_definicion_en_glosario(): + # The "how each categorical page is laid out" term is registered + marked + # clickable in the intro, and its full definition lands in the glossary + # chapter (canonical baseline catalog), not inline. + from datascience.automatic_eda.chapters.glosario import build_glosario + + gc = GlossaryCollector() + ch = build_cat_distr(_profile(), {"glossary": gc}) + md = next(b for b in ch.blocks if isinstance(b, Markdown)) + assert "[[term:pagina_categorica]]" in md.text + assert gc.has("pagina_categorica") + glos = build_glosario(_profile(), {"glossary": gc}) + entry = next(b for b in glos.blocks + if getattr(b, "kind", "") == "glossary_entry" + and b.key == "pagina_categorica") + assert "barras" in entry.definition + assert "identificador" in entry.definition def test_edge_sin_categoricas_devuelve_none(): diff --git a/python/functions/datascience/automatic_eda/chapters/glosario.py b/python/functions/datascience/automatic_eda/chapters/glosario.py index fe7098fc..6b7be259 100644 --- a/python/functions/datascience/automatic_eda/chapters/glosario.py +++ b/python/functions/datascience/automatic_eda/chapters/glosario.py @@ -17,10 +17,63 @@ from __future__ import annotations from .. import model -CHAPTER_VERSION = "1.0.0" +CHAPTER_VERSION = "1.1.0" CHAPTER_ID = "glosario" CHAPTER_TITLE = "Glosario" +# Canonical definitions for cross-cutting terms — the "how to read it" entries +# that do not belong to a single chapter. A chapter only needs to *register* the +# term (``ctx['glossary'].add(key, label)``) and mark its in-text appearance with +# ``[[term:key]]…[[/term]]``; this chapter supplies the full definition here when +# the collector carries the term without one. Keeping the prose in a single place +# avoids repeating a long paragraph inline in every chapter that names the term +# (the explanation moved out of the NUM DISTR and CAT DISTR intros lives here). +_BASELINE_TERMS = { + "histograma_boxplot": { + "label": "Cómo leer el histograma y el boxplot", + "definition": ( + "Para cada columna numérica se muestra su histograma con tres líneas " + "de referencia: la media (línea roja discontinua), la mediana (línea " + "verde continua) y la banda ±1σ (zona sombreada que cubre una " + "desviación estándar a cada lado de la media). Debajo, alineado al " + "mismo eje horizontal, un boxplot de Tukey: la caja abarca del primer " + "al tercer cuartil (P25–P75), la línea interior es la mediana y los " + "bigotes llegan hasta 1,5·IQR; los puntos rojos señalan que hay " + "valores más allá de las vallas (posibles atípicos). Comparar la media " + "con la mediana revela la asimetría: si la media supera a la mediana la " + "cola larga cae hacia los valores altos (asimetría a la derecha), y al " + "revés hacia los bajos."), + }, + "pagina_categorica": { + "label": "Cómo se organiza cada página categórica", + "definition": ( + "Cada columna categórica ocupa su propia página: muestra sus métricas " + "de cardinalidad —incluida la entropía—, una nota que señala " + "cardinalidad problemática (columnas que se comportan como " + "identificador, con casi todos los valores distintos, o dominadas por " + "una sola categoría), la tabla de las categorías más frecuentes (top-k, " + "con su conteo y porcentaje) y un gráfico de barras de las categorías " + "más comunes (top-k más una barra «Otros» que agrupa la cola). El total " + "de filas del dataset se usa como referencia para interpretar los " + "conteos."), + }, +} + + +def _resolve_term(term: dict) -> tuple: + """Return (label, definition) for a collected term, completing a missing + definition (and, if absent, the label) from the canonical baseline catalog.""" + key = model._safe_str(term.get("key")) + label = model._safe_str(term.get("label")) + definition = model._safe_str(term.get("definition")) + base = _BASELINE_TERMS.get(key) + if base: + if not definition.strip(): + definition = model._safe_str(base.get("definition")) + if not label.strip() or label == key: + label = model._safe_str(base.get("label")) or label + return label, definition + def build_glosario(profile: dict, ctx: dict): """Build the glossary Chapter from the shared collector, or None if empty.""" @@ -36,12 +89,14 @@ def build_glosario(profile: dict, ctx: dict): "Cada término va resaltado en el texto y, al pulsarlo, salta a su " "definición en esta sección.")), ] - # One clickable destination per term, alphabetically by visible label. + # One clickable destination per term, alphabetically by visible label. A term + # registered without a definition is completed from the canonical baseline. for term in glossary.terms(by="label"): + label, definition = _resolve_term(term) blocks.append(model.GlossaryEntry( key=model._safe_str(term.get("key")), - label=model._safe_str(term.get("label")), - definition=model._safe_str(term.get("definition")))) + label=label, + definition=definition)) return model.Chapter(id=CHAPTER_ID, title=CHAPTER_TITLE, version=CHAPTER_VERSION, blocks=blocks) diff --git a/python/functions/datascience/automatic_eda/chapters/num_distr.py b/python/functions/datascience/automatic_eda/chapters/num_distr.py index 5890b123..9401d710 100644 --- a/python/functions/datascience/automatic_eda/chapters/num_distr.py +++ b/python/functions/datascience/automatic_eda/chapters/num_distr.py @@ -35,10 +35,21 @@ try: except Exception: # noqa: BLE001 — keep the chapter importable no matter what. build_boxplot_stats = None # type: ignore[assignment] -CHAPTER_VERSION = "1.2.0" +CHAPTER_VERSION = "1.3.0" CHAPTER_ID = "num_distr" CHAPTER_TITLE = "Distribuciones numéricas" +# Glossary term this chapter explains. The long "how to read the histogram and +# the boxplot" paragraph used to live inline in the intro; it now lives in the +# GLOSARIO chapter (canonical definition in ``glosario._BASELINE_TERMS``) and the +# intro only names the clickable term — one click jumps to the full explanation, +# so the information is relocated, not lost (mejora glosario). +_TERM_HISTOBOX_KEY = "histograma_boxplot" +_TERM_HISTOBOX_LABEL = "Cómo leer el histograma y el boxplot" + +# Key under which eda_llm_insights stores its interpretive block in the profile. +LLM_KEY = "llm" + # Plain-Spanish gloss for every label ``detect_distribution_type`` can emit, so a # non-expert reader understands the shape and the suggested next step (MUST-4.3). _DIST_GLOSS = { @@ -99,6 +110,53 @@ def _numeric_columns(profile: dict) -> list: return out +def _llm_index(profile: dict, ctx: dict) -> dict: + """Map column name -> its LLM dictionary entry (description/unit/...). + + Reads the ``llm.dictionary`` list that ``eda_llm_insights`` stored in the + profile (``profile['llm']``; falls back to ``ctx['llm']``). Returns an empty + dict when ``run_llm`` did not run, so the caller degrades cleanly. Fully + defensive: never raises on malformed input. + """ + llm = profile.get(LLM_KEY) + if not isinstance(llm, dict): + llm = ctx.get(LLM_KEY) + if not isinstance(llm, dict): + return {} + entries = llm.get("dictionary") + if not isinstance(entries, (list, tuple)): + return {} + index: dict = {} + for e in entries: + if not isinstance(e, dict): + continue + col = e.get("column") + if col is None: + continue + index[model._safe_str(col)] = e + return index + + +def _llm_desc_unit_block(name: str, llm_index: dict): + """Markdown block with the LLM business description + unit of a column, or + None when no LLM entry matches the column (clean fallback without LLM).""" + entry = llm_index.get(model._safe_str(name)) + if not isinstance(entry, dict): + return None + raw_desc = entry.get("description") or entry.get("business_meaning") + desc = " ".join(model._safe_str(raw_desc).split()) if raw_desc else "" + raw_unit = entry.get("unit") + unit = " ".join(model._safe_str(raw_unit).split()) if raw_unit else "" + parts = [] + if desc: + parts.append(f"**Descripción:** {desc}") + if unit: + parts.append(f"**Unidad:** {unit}") + if not parts: + return None + return model.Markdown(text=" · ".join(parts)) + + def _make_hist_box(name: str, numeric: dict, box: dict): """Build the histogram (with mean/median/±σ lines) + boxplot figure. @@ -271,15 +329,26 @@ def build_num_distr(profile: dict, ctx: dict): if not numerics: return None # chapter does not apply to a dataset with no numerics. + # Register the "how to read the histogram and boxplot" term in the shared + # glossary collector (if present) and mark its first appearance clickable. The + # full explanation (colour code, 1,5·IQR rule, asymmetry reading) lives in the + # GLOSARIO chapter instead of inline here: the intro only names the term. + glossary = ctx.get("glossary") + mark_term = False + if isinstance(glossary, model.GlossaryCollector): + glossary.add(_TERM_HISTOBOX_KEY, _TERM_HISTOBOX_LABEL) + mark_term = True + como_leer = ("[[term:histograma_boxplot]]cómo leer estos gráficos[[/term]]" + if mark_term else "cómo leer estos gráficos") intro = ( - "Para cada columna numérica se muestra su **histograma** con tres líneas " - "de referencia: la **media** (línea roja discontinua), la **mediana** " - "(línea verde continua) y la banda **±1σ** (zona sombreada). Debajo, " - "alineado al mismo eje, un **boxplot de Tukey**: la caja abarca del " - "primer al tercer cuartil (P25–P75), la línea interior es la mediana y " - "los bigotes llegan hasta 1,5·IQR; los puntos rojos señalan que hay " - "valores más allá de las vallas. Comparar media y mediana revela la " - "asimetría de la distribución.") + "Cada columna numérica muestra su **histograma** (con la **media**, la " + "**mediana** y la banda **±1σ**) y, debajo y al mismo eje, su **boxplot " + f"de Tukey** — {como_leer}.") + + # Business description + unit per column come from the LLM dictionary + # (profile['llm']['dictionary'], matched by column name); absent without + # run_llm, in which case the per-column description block is simply omitted. + llm_index = _llm_index(profile, ctx) blocks = [ model.Heading(text=CHAPTER_TITLE, level=1), @@ -293,17 +362,20 @@ def build_num_distr(profile: dict, ctx: dict): box = build_boxplot_stats(numeric) or {} except Exception: # noqa: BLE001 — degrade, never raise. box = {} - # Keep the column heading, its figure and its stats note together on the - # same page/slide (mejora 3 — keep-together): the renderers measure the - # whole Group and move it whole when it would not fit. - blocks.append(model.Group(blocks=[ - model.Heading(text=str(name), level=2), - model.Figure( - make=_figure_maker(name, numeric, box), - caption=f"Distribución de «{name}» — histograma " - f"(media/mediana/±σ) y boxplot."), - model.Markdown(text=_stats_note(name, numeric, box)), - ])) + # Keep the column heading, its (optional) LLM description, its figure and + # its stats note together on the same page/slide (mejora 3 — + # keep-together): the renderers measure the whole Group and move it whole + # when it would not fit. + col_blocks = [model.Heading(text=str(name), level=2)] + desc_block = _llm_desc_unit_block(name, llm_index) + if desc_block is not None: + col_blocks.append(desc_block) + col_blocks.append(model.Figure( + make=_figure_maker(name, numeric, box), + caption=f"Distribución de «{name}» — histograma " + f"(media/mediana/±σ) y boxplot.")) + col_blocks.append(model.Markdown(text=_stats_note(name, numeric, box))) + blocks.append(model.Group(blocks=col_blocks)) return model.Chapter(id=CHAPTER_ID, title=CHAPTER_TITLE, version=CHAPTER_VERSION, blocks=blocks) diff --git a/python/functions/datascience/automatic_eda/chapters/num_distr_test.py b/python/functions/datascience/automatic_eda/chapters/num_distr_test.py index 280cff17..ea0b9fd5 100644 --- a/python/functions/datascience/automatic_eda/chapters/num_distr_test.py +++ b/python/functions/datascience/automatic_eda/chapters/num_distr_test.py @@ -101,7 +101,7 @@ def test_golden_chapter_estructura_y_bloques(): def test_golden_media_mediana_sigma_y_boxplot_presentes(): - # The intro documents the three reference lines and the Tukey boxplot; the + # The short intro names the three reference lines and the Tukey boxplot; the # per-column note carries the actual mean/median/σ numbers and the shape. ch = build_num_distr(_profile(n_numeric=1, extra_categorical=False), {}) md_texts = " ".join(b.text for b in _flatten(ch.blocks) @@ -110,10 +110,58 @@ def test_golden_media_mediana_sigma_y_boxplot_presentes(): assert "±1σ" in md_texts or "σ" in md_texts assert "boxplot" in md_texts.lower() assert "Tukey" in md_texts + # The long "how to read it" explanation moved to the glossary: the colour-code + # / 1,5·IQR walkthrough is no longer inline in the chapter body. + assert "1,5·IQR" not in md_texts + assert "línea roja" not in md_texts # distribution_type gloss surfaced for the column (right-skewed preset). assert _DIST_GLOSS["right-skewed"].split(";")[0][:20] in md_texts +def test_glosario_histograma_boxplot_clicable_y_definicion(): + # With a glossary collector the intro marks the clickable term and the FULL + # explanation (the long paragraph removed from the body) lands in the glossary. + from datascience.automatic_eda.chapters.glosario import build_glosario + + gc = model.GlossaryCollector() + prof = _profile(n_numeric=1, extra_categorical=False) + ch = build_num_distr(prof, {"glossary": gc}) + intro = next(b for b in ch.blocks if b.kind == "markdown") + assert "[[term:histograma_boxplot]]" in intro.text + assert gc.has("histograma_boxplot") + glos = build_glosario(prof, {"glossary": gc}) + entry = next(b for b in glos.blocks + if getattr(b, "kind", "") == "glossary_entry" + and b.key == "histograma_boxplot") + assert "boxplot" in entry.definition.lower() + assert "1,5·IQR" in entry.definition + + +def test_llm_descripcion_y_unidad_por_columna(): + # With an LLM dictionary, each numeric column whose name matches shows its + # business description and unit in a per-column markdown block. + prof = _profile(n_numeric=2) + prof["llm"] = {"dictionary": [ + {"column": "precio", "description": "Precio de venta del producto", + "unit": "EUR"}, + {"column": "alcohol", "business_meaning": "Grado alcohólico", + "unit": "% vol"}, + ]} + ch = build_num_distr(prof, {}) + md_all = " ".join(b.text for b in _flatten(ch.blocks) + if b.kind == "markdown") + assert "Precio de venta" in md_all and "EUR" in md_all + assert "Grado alcohólico" in md_all and "% vol" in md_all + + +def test_edge_sin_llm_no_anade_descripcion(): + # Without an LLM block the per-column description markdown is simply omitted. + ch = build_num_distr(_profile(n_numeric=2), {}) + md_all = " ".join(b.text for b in _flatten(ch.blocks) + if b.kind == "markdown") + assert "Descripción" not in md_all + + def test_boxplot_stats_se_consumen_del_registry(): # The chapter must feed build_boxplot_stats (group eda) and the resulting # box must carry the Tukey fences for the figure. diff --git a/python/functions/datascience/categorical_top_bar_figure.md b/python/functions/datascience/categorical_top_bar_figure.md new file mode 100644 index 00000000..5dad3c49 --- /dev/null +++ b/python/functions/datascience/categorical_top_bar_figure.md @@ -0,0 +1,111 @@ +--- +id: categorical_top_bar_figure_py_datascience +name: categorical_top_bar_figure +kind: function +lang: py +domain: datascience +version: "1.0.0" +purity: impure +signature: "def categorical_top_bar_figure(top: list, n_distinct: int = 0, title: str = \"\", top_k: int = 6, n_rows=None) -> \"matplotlib.figure.Figure\"" +description: "Construye una figura matplotlib de barras horizontales de las top_k categorías más frecuentes de una columna categórica, con la mayor arriba y agregando el resto en una barra gris \"Otros (N categorías)\". Contrato de entrada idéntico a categorical_top_pie_figure (swap directo donut↔barras): consume el bloque `top` de summarize_categorical y devuelve un matplotlib.figure.Figure listo para rasterizar por el renderer del informe EDA. Backend Agg sin pyplot global; defensivo total ante top vacío/None, nunca lanza." +tags: [eda, categorical, bar, barh, matplotlib, figure, visualization, datascience, impure] +uses_functions: [] +uses_types: [] +returns: [] +returns_optional: false +error_type: "error_go_core" +imports: [matplotlib] +example: | + from categorical_top_bar_figure import categorical_top_bar_figure + top = [ + {"value": "rojo", "count": 40, "pct": 0.4}, + {"value": "azul", "count": 30, "pct": 0.3}, + {"value": "verde", "count": 20, "pct": 0.2}, + ] + fig = categorical_top_bar_figure(top, n_distinct=12, title="color", top_k=6, n_rows=100) +tested: true +tests: + - "test_returns_figure" + - "test_ten_items_topk_six_yields_seven_bars" + - "test_empty_top_does_not_raise_and_returns_figure" + - "test_long_value_truncated" + - "test_none_value_and_none_count_are_handled" + - "test_n_rows_adds_exact_others_bar" +test_file_path: "python/functions/datascience/categorical_top_bar_figure_test.py" +file_path: "python/functions/datascience/categorical_top_bar_figure.py" +params: + - name: top + desc: "Lista de dicts {value, count, pct} ordenada de mayor a menor por count (salida del bloque `top` de summarize_categorical). Puede venir vacía o con dicts incompletos: items no-dict, sin count, con count None o count <= 0 se descartan. value None se admite (etiqueta vacía)." + - name: n_distinct + desc: "Nº total de categorías distintas de la columna. Etiqueta la barra agregada como \"Otros (n_distinct - top_k)\" (mínimo 0). Si no supera el nº de barras mostradas, se usa el overflow real de `top` como nº de categorías agregadas. Default 0." + - name: title + desc: "Título de la figura (nombre de la columna). Se trunca a ~48 chars con elipsis si es muy largo. Default \"\" (sin título)." + - name: top_k + desc: "Nº máximo de barras explícitas. Default 6. La barra \"Otros\" no cuenta contra este límite. Con top_k <= 0 se muestra al menos la categoría mayor." + - name: n_rows + desc: "Opcional. Total de filas del dataset. Si se da y la suma de counts mostrados < n_rows, la barra \"Otros\" usa (n_rows - suma_mostrada) como count para que sea exacta respecto al total real. Si se omite, \"Otros\" usa la suma de counts fuera del top_k mostrado (solo cuando top trae más de top_k items). Default None." +output: "Un matplotlib.figure.Figure (figsize 6.4 x altura escalada con el nº de barras, dpi 150) con un Axes de barras horizontales: la categoría más frecuente arriba, la barra gris \"Otros (N categorías)\" abajo, cada barra anotada con su conteo y porcentaje al final y etiquetas de categoría (yticklabels) truncadas a ~22 chars. Si no hay counts válidos devuelve igualmente una Figure con un texto centrado \"sin datos categóricos\" (nunca lanza); cualquier error inesperado cae a una Figure con el texto del error. El caller rasteriza/cierra la figura; la función no la muestra ni la guarda." +--- + +## Ejemplo + +```python +from categorical_top_bar_figure import categorical_top_bar_figure + +# `top` es la salida del bloque "top" de summarize_categorical (ya ordenado desc). +top = [ + {"value": "rojo", "count": 40, "pct": 0.40}, + {"value": "azul", "count": 30, "pct": 0.30}, + {"value": "verde", "count": 20, "pct": 0.20}, + {"value": "amarillo", "count": 5, "pct": 0.05}, +] + +fig = categorical_top_bar_figure( + top, + n_distinct=12, # 12 categorías distintas en total + title="color_producto", + top_k=6, # hasta 6 barras explícitas + n_rows=100, # "Otros" = 100 - 95 = 5, sobre 8 categorías agregadas +) + +# El renderer del informe lo rasteriza; aquí solo persistimos para inspección. +fig.savefig("/tmp/barras_color.png") +``` + +## Cuando usarla + +Úsala dentro de un informe EDA cuando quieras comparar **magnitudes** de las +categorías dominantes de una columna categórica: qué categoría manda y por +cuánto frente a las siguientes. Pásale directamente el bloque `top` de +`summarize_categorical` (ya ordenado de mayor a menor) más `n_distinct` para que +la barra "Otros" indique cuántas categorías quedan agrupadas. Es el clon "de +barras" del donut `categorical_top_pie_figure` con **contrato de entrada +idéntico**: puedes intercambiar una por otra sin tocar el caller. Elige barras +cuando importe comparar tamaños exactos; el donut cuando importe la proporción +del total. + +## Gotchas + +- **Impura por matplotlib.** Toca la maquinaria de render. Usa el backend `Agg` + y la API orientada a objetos `Figure`/`add_subplot` — NUNCA `pyplot.*` aquí, + para no tocar el estado global ni filtrar figuras entre llamadas. `pyplot` NO + es thread-safe; esta función evita ese riesgo construyendo el `Figure` + directamente, así que es segura de llamar en bucle desde el renderer. +- **El caller cierra la figura.** La función devuelve el `Figure` pero no lo + muestra ni lo guarda. Quien la consume debe rasterizarla y luego liberarla + (`fig.clf()` / `matplotlib.pyplot.close(fig)` si se usó pyplot en el caller) + para no acumular memoria en lotes grandes de columnas. +- **`barh` dibuja de abajo arriba.** La categoría más frecuente va arriba porque + el orden de display se invierte antes de plotear; la barra "Otros" queda + siempre al fondo. No reordenes `top` esperando otro layout: la función asume + que ya viene ordenado desc por count. +- **Magnitud exacta de "Otros" solo con `n_rows`.** Sin `n_rows`, la barra + "Otros" se calcula con el overflow presente en `top`; si `top` ya viene + recortado a `top_k` por el productor, no habrá "Otros" aunque existan más + categorías. Pasa `n_rows` (total de filas del dataset) para una barra correcta + respecto al total real. +- **Defensiva, nunca lanza.** `top=[]`, `value=None`, `count=None` o counts no + numéricos se manejan sin error: en el peor caso devuelve una `Figure` con + "sin datos categóricos", y cualquier excepción inesperada cae a una `Figure` + con el texto del error. No envuelvas la llamada en try/except por miedo a un + raise — no lo hay. diff --git a/python/functions/datascience/categorical_top_bar_figure.py b/python/functions/datascience/categorical_top_bar_figure.py new file mode 100644 index 00000000..3fc47196 --- /dev/null +++ b/python/functions/datascience/categorical_top_bar_figure.py @@ -0,0 +1,233 @@ +"""Impure EDA helper: horizontal bar figure of the most common categories (`eda` group). + +Builds a horizontal bar chart of the ``top_k`` most frequent categories of a +categorical column, folding everything else into a single gray +"Otros (N categorías)" bar. The most frequent category sits at the top, each bar +labelled with its count (and percentage) at the end. Returns a ready-to-rasterize +``matplotlib.figure.Figure``; it never shows nor saves it. + +This is the "magnitude" twin of ``categorical_top_pie_figure``: identical input +contract (same ``top``/``n_distinct``/``title``/``top_k``/``n_rows`` signature) so +it can be swapped in directly, but it communicates comparable magnitudes via bars +instead of proportions via wedges. + +Impure because it touches matplotlib's rendering machinery. It uses the headless +Agg backend and the object-oriented ``Figure`` API (no ``pyplot``) so it leaks no +global state and is safe to call repeatedly from a report renderer. +""" + +import matplotlib + +matplotlib.use("Agg") + +from matplotlib.figure import Figure # noqa: E402 + + +# Gray reserved for the aggregated "Otros" bar. +_OTHER_COLOR = "#9e9e9e" +# Muted gray for secondary text (title fallback, no-data message). +_MUTED_TEXT = "#5f6b7a" +# Soft red for the error fallback message. +_ERROR_TEXT = "#b00020" +# Pleasant, colour-blind-friendly qualitative palette for the explicit bars. +_PALETTE = [ + "#4C72B0", + "#DD8452", + "#55A868", + "#C44E52", + "#8172B3", + "#937860", + "#DA8BC3", + "#8C8C8C", + "#CCB974", + "#64B5CD", +] + + +def _truncate(text, width: int = 22) -> str: + """Truncate ``text`` to ``width`` chars, appending an ellipsis if cut.""" + s = "" if text is None else str(text) + if len(s) <= width: + return s + if width <= 1: + return s[:width] + return s[: width - 1] + "…" + + +def _message_figure(message: str, color: str = _MUTED_TEXT, title: str = "") -> "Figure": + """Return a fallback ``Figure`` carrying a single centered message.""" + fig = Figure(figsize=(6.4, 4.0), dpi=150) + ax = fig.add_subplot(111) + ax.axis("off") + ax.text( + 0.5, + 0.5, + message, + ha="center", + va="center", + fontsize=12, + color=color, + wrap=True, + transform=ax.transAxes, + ) + if title: + ax.set_title(_truncate(title, 48), fontsize=12, loc="center", pad=8) + fig.tight_layout() + return fig + + +def categorical_top_bar_figure( + top: list, + n_distinct: int = 0, + title: str = "", + top_k: int = 6, + n_rows=None, +) -> "matplotlib.figure.Figure": + """Build a horizontal bar figure of the most common categories of a column. + + Renders the ``top_k`` most frequent categories as explicit horizontal bars, + largest at the top, and aggregates every remaining category into a single + gray "Otros (N categorías)" bar at the bottom. Each bar is annotated with its + count and percentage of the total at the end of the bar; the category names + are truncated Y tick labels. + + The function shares the exact input contract of + ``categorical_top_pie_figure`` (the donut twin) so it is a drop-in swap. It is + fully defensive: empty input, missing/``None`` values or counts never raise. + When there is nothing valid to draw it still returns a ``Figure`` carrying a + centered "sin datos categóricos" message, and any unexpected error is caught + and turned into a fallback ``Figure`` carrying the error text. + + Args: + top: List of ``{value, count, pct}`` dicts, already sorted by ``count`` + descending (the ``top`` block of ``summarize_categorical``). May be + empty or carry incomplete/``None`` entries; non-dict items, items + without a positive numeric ``count`` and ``None`` counts are skipped. + n_distinct: Total number of distinct categories in the column. Used to + label the aggregated bar as "Otros (n_distinct - top_k)" (floored at + 0). Ignored when it does not exceed the number of shown bars. + title: Figure title (the column name). Truncated when too long. + top_k: Maximum number of explicit bars. Default 6. The "Otros" bar does + not count against this limit. + n_rows: Optional total row count of the dataset. When given and the sum of + shown counts is below ``n_rows``, the "Otros" bar uses + ``n_rows - sum_shown`` as its count so it is exact with respect to the + real total. When omitted, "Otros" uses the sum of the counts that fall + outside the shown ``top_k`` (only when ``top`` carries more than + ``top_k`` items). + + Returns: + A ``matplotlib.figure.Figure`` with a single horizontal-bar Axes. The + caller is responsible for rasterizing/closing it. + """ + try: + safe_title = _truncate(title, 48) + + # --- Defensive parse: keep only well-formed {value, count} with count > 0. + cleaned = [] + if isinstance(top, list): + for item in top: + if not isinstance(item, dict): + continue + count = item.get("count") + if count is None: + continue + try: + count = float(count) + except (TypeError, ValueError): + continue + if count <= 0: + continue + cleaned.append((item.get("value"), count)) + + if not cleaned: + return _message_figure("sin datos categóricos", title=title) + + # --- Split into shown bars and the aggregated remainder. + shown = cleaned[: max(int(top_k), 0)] + if not shown: # top_k <= 0 — show at least the largest category. + shown = cleaned[:1] + + sum_shown = sum(c for _, c in shown) + overflow_count = sum(c for _, c in cleaned[len(shown):]) + + # How many categories are folded into "Otros". + try: + nd = int(n_distinct) + except (TypeError, ValueError): + nd = 0 + others_categories = max(nd - len(shown), 0) + # If n_distinct is unknown/too small, fall back to the overflow we + # actually have in `top` beyond the shown bars. + overflow_items = len(cleaned) - len(shown) + if others_categories == 0 and overflow_items > 0: + others_categories = overflow_items + + # Count attributed to the "Otros" bar. + others_count = 0.0 + if n_rows is not None: + try: + total_rows = float(n_rows) + except (TypeError, ValueError): + total_rows = None + if total_rows is not None and total_rows > sum_shown: + others_count = total_rows - sum_shown + if others_count <= 0: + others_count = overflow_count + + # --- Build the display order (top to bottom): largest .. smallest, Otros. + display_labels = [_truncate(v, 22) for v, _ in shown] + display_values = [c for _, c in shown] + display_colors = [_PALETTE[i % len(_PALETTE)] for i in range(len(shown))] + + has_others = others_count > 0 and others_categories > 0 + if has_others: + display_labels.append(f"Otros ({others_categories} categorías)") + display_values.append(others_count) + display_colors.append(_OTHER_COLOR) + + total = sum(display_values) or 1.0 + + # barh draws bottom-up, so reverse the display order before plotting to + # land the largest category on top and "Otros" at the bottom. + labels = list(reversed(display_labels)) + values = list(reversed(display_values)) + colors = list(reversed(display_colors)) + y_pos = range(len(values)) + + # Height scales with the number of bars so dense reports stay readable. + n_bars = len(values) + height = max(2.4, min(0.4 * n_bars + 1.2, 14.0)) + fig = Figure(figsize=(6.4, height), dpi=150) + ax = fig.add_subplot(111) + + ax.barh(list(y_pos), values, color=colors, edgecolor="white") + ax.set_yticks(list(y_pos)) + ax.set_yticklabels(labels, fontsize=8) + ax.set_xlabel("conteo", fontsize=9) + + max_val = max(values) if values else 1.0 + ax.set_xlim(0, max_val * 1.18 if max_val > 0 else 1.0) + + # Annotate each bar with its count and percentage at the end of the bar. + for y, val in zip(y_pos, values): + pct = val / total * 100.0 + ax.text( + val + max_val * 0.012, + y, + f"{int(round(val))} ({pct:.0f}%)", + va="center", + ha="left", + fontsize=7, + color="#202020", + ) + + if safe_title: + ax.set_title(safe_title, fontsize=13, loc="left", pad=10) + + fig.tight_layout() + return fig + except Exception as exc: # noqa: BLE001 — never raise from a figure builder. + return _message_figure( + f"error al dibujar barras: {exc}", color=_ERROR_TEXT + ) diff --git a/python/functions/datascience/categorical_top_bar_figure_test.py b/python/functions/datascience/categorical_top_bar_figure_test.py new file mode 100644 index 00000000..01fedbc7 --- /dev/null +++ b/python/functions/datascience/categorical_top_bar_figure_test.py @@ -0,0 +1,103 @@ +"""Tests para categorical_top_bar_figure (barras de categorías top, grupo eda). + +Usa el backend Agg sin pyplot; no muestra ni guarda figuras. Cada test cierra +explícitamente la Figure construida (matplotlib.pyplot.close) para no acumular +estado entre tests. +""" + +import matplotlib + +matplotlib.use("Agg") + +import matplotlib.pyplot as plt # noqa: E402 +from matplotlib.figure import Figure # noqa: E402 + +from categorical_top_bar_figure import categorical_top_bar_figure + + +def _make_top(n): + """n items {value, count, pct} ordenados desc por count.""" + return [ + {"value": f"cat_{i}", "count": n - i, "pct": (n - i) / sum(range(1, n + 1))} + for i in range(n) + ] + + +def _bar_count(ax): + """Devuelve el nº de barras (longitud del primer BarContainer del Axes).""" + if ax.containers: + return len(ax.containers[0]) + return 0 + + +def test_returns_figure(): + fig = categorical_top_bar_figure(_make_top(3), n_distinct=3, title="col") + assert isinstance(fig, Figure) + plt.close(fig) + + +def test_ten_items_topk_six_yields_seven_bars(): + top = _make_top(10) + fig = categorical_top_bar_figure(top, n_distinct=10, title="muchas", top_k=6) + ax = fig.axes[0] + # 6 categorías explícitas + 1 barra "Otros". + assert _bar_count(ax) == 7 + plt.close(fig) + + +def test_empty_top_does_not_raise_and_returns_figure(): + fig = categorical_top_bar_figure([], n_distinct=0, title="vacía") + assert isinstance(fig, Figure) + # Sin datos: no debe haber barras. + assert _bar_count(fig.axes[0]) == 0 + plt.close(fig) + + +def test_long_value_truncated(): + long_value = "una_categoria_con_un_nombre_larguisimo_que_excede_el_limite" + top = [ + {"value": long_value, "count": 10, "pct": 0.5}, + {"value": "corta", "count": 10, "pct": 0.5}, + ] + fig = categorical_top_bar_figure(top, n_distinct=2, title="col", top_k=6) + ax = fig.axes[0] + tick_texts = [t.get_text() for t in ax.get_yticklabels()] + # El valor largo aparece truncado con elipsis y NO en su forma completa. + assert any("…" in t for t in tick_texts) + assert long_value not in " ".join(tick_texts) + plt.close(fig) + + +def test_none_value_and_none_count_are_handled(): + top = [ + {"value": None, "count": 5, "pct": 0.5}, + {"value": "b", "count": None, "pct": 0.0}, # count None -> se descarta + {"value": "c", "count": 5, "pct": 0.5}, + ] + fig = categorical_top_bar_figure(top, n_distinct=2, title="con nones", top_k=6) + assert isinstance(fig, Figure) + # Solo 2 items válidos, sin overflow -> 2 barras, sin "Otros". + assert _bar_count(fig.axes[0]) == 2 + plt.close(fig) + + +def test_n_rows_adds_exact_others_bar(): + # 3 categorías mostradas suman 30, dataset real 100 -> "Otros" = 70. + top = [ + {"value": "a", "count": 15, "pct": 0.15}, + {"value": "b", "count": 10, "pct": 0.10}, + {"value": "c", "count": 5, "pct": 0.05}, + ] + fig = categorical_top_bar_figure( + top, n_distinct=20, title="col", top_k=3, n_rows=100 + ) + ax = fig.axes[0] + # 3 explícitas + Otros. + assert _bar_count(ax) == 4 + tick_texts = [t.get_text() for t in ax.get_yticklabels()] + # La barra Otros refleja n_distinct - top_k = 17 categorías. + assert any("Otros (17 categorías)" in t for t in tick_texts) + # Su anotación lleva el count 70. + annotation_texts = [t.get_text() for t in ax.texts] + assert any("70" in t for t in annotation_texts) + plt.close(fig) From c0d44a6352f7fde2e4440c8d637807dbb1041230 Mon Sep 17 00:00:00 2001 From: Egutierrez <egutierrez@dead.dd> Date: Wed, 1 Jul 2026 02:10:39 +0200 Subject: [PATCH 52/53] =?UTF-8?q?fix(eda):=20cat=5Fdistr=20=E2=80=94=20int?= =?UTF-8?q?ro=20del=20cuerpo=20reducida=20a=20t=C3=A9rminos=20clicables=20?= =?UTF-8?q?m=C3=ADnimos?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Quita la frase descriptiva del cuerpo del capítulo ('Cada columna categórica ocupa su propia página — ...: cardinalidad, top de categorías y gráfico de barras. El dataset tiene N filas...'); ya vivía duplicada en la entrada de glosario 'pagina_categorica'. El intro deja solo los términos clicables mínimos ([[term:entropia]] · [[term:pagina_categorica]]) bajo el heading 'Entropía y cardinalidad'. El total de filas del dataset sigue disponible por columna en la tabla de cardinalidad ('Total filas (dataset)'). --- .../automatic_eda/chapters/cat_distr.py | 23 +++++++------------ 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/python/functions/datascience/automatic_eda/chapters/cat_distr.py b/python/functions/datascience/automatic_eda/chapters/cat_distr.py index 3a252605..731aa7b7 100644 --- a/python/functions/datascience/automatic_eda/chapters/cat_distr.py +++ b/python/functions/datascience/automatic_eda/chapters/cat_distr.py @@ -439,24 +439,17 @@ def _topk_table(cat: dict): note=note) -def _intro_blocks(n_rows, mark_term: bool = False): - total = _fmt_int(n_rows) - # Mark the first appearance of each term as a clickable glossary jump when the - # terms were registered (mark_term). The full definition of the entropy term - # AND of how each categorical page is laid out live in the GLOSARIO chapter, so - # the intro only names the clickable terms instead of repeating the long - # explanation (avoids the redundancy with the glossary). +def _intro_blocks(mark_term: bool = False): + # The full explanation of entropy AND of how each categorical page is laid out + # lives in the GLOSARIO chapter; the chapter body keeps only the minimal + # clickable terms — no descriptive prose — to avoid duplicating the glossary. + # The dataset row total is not repeated here: each column's cardinality table + # already carries "Total filas (dataset)". entropia = ("[[term:entropia]]entropía[[/term]]" if mark_term else "entropía") pagina = ("[[term:pagina_categorica]]cómo se organiza cada página[[/term]]" if mark_term else "cómo se organiza cada página") - text = ( - f"Cada columna categórica ocupa su propia página — {pagina}: " - f"cardinalidad (incluida la {entropia}), top de categorías y un gráfico " - "de barras de las más comunes." - ) - if n_rows is not None: - text += f" El dataset tiene {total} filas en total como referencia." + text = f"Términos: {entropia} · {pagina}." return [ model.Heading(text="Entropía y cardinalidad", level=2), model.Markdown(text=text), @@ -484,7 +477,7 @@ def build_cat_distr(profile: dict, ctx: dict): _TERM_ENTROPIA_DEF) glossary.add(_TERM_PAGINA_KEY, _TERM_PAGINA_LABEL) mark_term = True - blocks = list(_intro_blocks(n_rows, mark_term=mark_term)) + blocks = list(_intro_blocks(mark_term=mark_term)) # Business description + unit per column come from the LLM dictionary # (profile['llm']['dictionary'], matched by column name); absent without From 26569c7015c66181ff7f4891c6fab82499cd14de Mon Sep 17 00:00:00 2001 From: Egutierrez <egutierrez@dead.dd> Date: Wed, 1 Jul 2026 02:16:25 +0200 Subject: [PATCH 53/53] chore: auto-commit (1 archivos) - logs/ardour_mcp_server.log Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> --- logs/ardour_mcp_server.log | 594 +++++++++++++++++++++++++++++++++++++ 1 file changed, 594 insertions(+) diff --git a/logs/ardour_mcp_server.log b/logs/ardour_mcp_server.log index 921b495b..f2daae44 100644 --- a/logs/ardour_mcp_server.log +++ b/logs/ardour_mcp_server.log @@ -930,3 +930,597 @@ NEW SERVER RUN: Mon, 29 Jun 2026 11:04:45 +0200 2026-06-29T09:04:45.586389Z DEBUG rmcp::service: new event evt=ToSink(Response(ListResourcesResult(ListResourcesResult { next_cursor: None, resources: [Annotated { raw: RawResource { uri: "ardour:/state/playback", name: "Ardour Playback State", description: Some("Current playback state of Ardour (e.g., Playing, Stopped, Unknown)."), mime_type: None, size: None }, annotations: None }, Annotated { raw: RawResource { uri: "ardour:/state/transport_frame", name: "Ardour Transport Frame Position", description: Some("Current playhead position in samples. Returns 'Unknown' if not yet reported by Ardour."), mime_type: None, size: None }, annotations: None }] }), Number(2))) 2026-06-29T09:04:45.586536Z  INFO rmcp::service: response message id=1 result=ListToolsResult(ListToolsResult { next_cursor: None, tools: [Tool { name: "toggle_all_rec_enables", description: "Toggles the record enable state for ALL tracks.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_mute", description: "Sets the mute state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"mute_state": Object {"description": String("The desired mute state (true for mute, false for unmute)."), "type": String("boolean")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("mute_state"), String("rid")], "title": String("SetTrackMuteArgs"), "type": String("object")} }, Tool { name: "set_track_trim_abs", description: "Sets the absolute trim of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_abs": Object {"description": String("The desired absolute trim. Valid range: 0.1 to 10.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_abs")], "title": String("SetTrackTrimAbsArgs"), "type": String("object")} }, Tool { name: "set_strip_plugin_active", description: "Activates or deactivates a plugin on a specific strip slot.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"active_state": Object {"description": String("The desired activation state (true for active, false for inactive)."), "type": String("boolean")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("active_state"), String("plugin_slot"), String("rid")], "title": String("SetStripPluginActiveArgs"), "type": String("object")} }, Tool { name: "transport_stop", description: "Stops Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "access_action", description: "Executes a specified Ardour menu action by its name.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"action_name": Object {"description": String("The name of the Ardour menu action to execute (e.g., 'Editor/zoom-to-session')."), "type": String("string")}}, "required": Array [String("action_name")], "title": String("AccessActionArgs"), "type": String("object")} }, Tool { name: "rewind", description: "Rewinds the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "next_marker", description: "Moves the playhead to the next location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "toggle_punch_out", description: "Toggles the Punch Out state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_gain_abs", description: "Sets the absolute gain of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_abs": Object {"description": String("The desired absolute gain. Valid range: 0.0 to 2.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_abs"), String("rid")], "title": String("SetTrackGainAbsArgs"), "type": String("object")} }, Tool { name: "set_selected_strip_pan_stereo_width", description: "Sets the stereo width for the currently selected stereo strip's panner.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"width": Object {"description": String("The desired stereo width for the currently selected strip. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("width")], "title": String("SetSelectedStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "transport_play", description: "Starts Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "goto_start", description: "Moves the playhead to the session start.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_trim_db", description: "Sets the trim of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_db": Object {"description": String("The desired trim in dB. Valid range: -20.0 to 20.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_db")], "title": String("SetTrackTrimDBArgs"), "type": String("object")} }, Tool { name: "set_track_rec_enable", description: "Sets the record enable state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rec_st": Object {"description": String("The desired record enable state. 0 for off, 1 for on."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rec_st"), String("rid")], "title": String("SetTrackRecEnableArgs"), "type": String("object")} }, Tool { name: "set_track_gain_db", description: "Sets the gain of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_db": Object {"description": String("The desired gain in dB. Valid range: -400.0 to 6.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_db"), String("rid")], "title": String("SetTrackGainDBArgs"), "type": String("object")} }, Tool { name: "locate", description: "Locates the playhead to a specific sample position and optionally starts playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"roll": Object {"description": String("Whether to start playing after locating. 0 for stop, 1 for play."), "format": String("int32"), "type": String("integer")}, "spos": Object {"description": String("The position in samples to locate to."), "format": String("int64"), "type": String("integer")}}, "required": Array [String("roll"), String("spos")], "title": String("LocateToolArgs"), "type": String("object")} }, Tool { name: "redo", description: "Redoes the last undone action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "goto_end", description: "Moves the playhead to the session end.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "toggle_punch_in", description: "Toggles the Punch In state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "rec_enable_toggle", description: "Toggles the master record enable or selected track record enable.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "select_strip", description: "Selects a specific strip (track/bus) in Ardour.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus to select."), "format": String("int32"), "type": String("integer")}, "select_state": Object {"description": String("The desired select state (true to select). Currently, only true (1) is effective for selection."), "type": String("boolean")}}, "required": Array [String("rid"), String("select_state")], "title": String("SelectStripArgs"), "type": String("object")} }, Tool { name: "loop_toggle", description: "Toggles loop playback mode.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_strip_pan_stereo_width", description: "Sets the stereo width for a panner on a stereo strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the stereo track/bus."), "format": String("int32"), "type": String("integer")}, "width": Object {"description": String("The desired stereo width. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("width")], "title": String("SetStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "set_track_solo", description: "Sets the solo state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "solo_st": Object {"description": String("The desired solo state. 0 for solo off, 1 for solo on."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rid"), String("solo_st")], "title": String("SetTrackSoloArgs"), "type": String("object")} }, Tool { name: "prev_marker", description: "Moves the playhead to the previous location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_strip_plugin_parameter", description: "Sets a specific parameter of a plugin on a strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"param_id": Object {"description": String("The 1-indexed ID of the parameter within the plugin."), "format": String("int32"), "type": String("integer")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "value": Object {"description": String("The desired parameter value, normalized (0.0 to 1.0)."), "format": String("float"), "type": String("number")}}, "required": Array [String("param_id"), String("plugin_slot"), String("rid"), String("value")], "title": String("SetStripPluginParameterArgs"), "type": String("object")} }, Tool { name: "save_state", description: "Saves the current session state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "add_marker", description: "Adds a location marker at the current playhead position.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "undo", description: "Undoes the last action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "ffwd", description: "Fast forwards the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_transport_speed", description: "Sets Ardour's transport speed. Valid range: -8.0 to 8.0.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"speed": Object {"description": String("The desired transport speed. Valid range: -8.0 to 8.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("speed")], "title": String("SetTransportSpeedArgs"), "type": String("object")} }] }) 2026-06-29T09:04:45.586815Z DEBUG rmcp::service: new event evt=ToSink(Response(ListToolsResult(ListToolsResult { next_cursor: None, tools: [Tool { name: "toggle_all_rec_enables", description: "Toggles the record enable state for ALL tracks.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_mute", description: "Sets the mute state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"mute_state": Object {"description": String("The desired mute state (true for mute, false for unmute)."), "type": String("boolean")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("mute_state"), String("rid")], "title": String("SetTrackMuteArgs"), "type": String("object")} }, Tool { name: "set_track_trim_abs", description: "Sets the absolute trim of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_abs": Object {"description": String("The desired absolute trim. Valid range: 0.1 to 10.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_abs")], "title": String("SetTrackTrimAbsArgs"), "type": String("object")} }, Tool { name: "set_strip_plugin_active", description: "Activates or deactivates a plugin on a specific strip slot.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"active_state": Object {"description": String("The desired activation state (true for active, false for inactive)."), "type": String("boolean")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("active_state"), String("plugin_slot"), String("rid")], "title": String("SetStripPluginActiveArgs"), "type": String("object")} }, Tool { name: "transport_stop", description: "Stops Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "access_action", description: "Executes a specified Ardour menu action by its name.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"action_name": Object {"description": String("The name of the Ardour menu action to execute (e.g., 'Editor/zoom-to-session')."), "type": String("string")}}, "required": Array [String("action_name")], "title": String("AccessActionArgs"), "type": String("object")} }, Tool { name: "rewind", description: "Rewinds the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "next_marker", description: "Moves the playhead to the next location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "toggle_punch_out", description: "Toggles the Punch Out state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_gain_abs", description: "Sets the absolute gain of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_abs": Object {"description": String("The desired absolute gain. Valid range: 0.0 to 2.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_abs"), String("rid")], "title": String("SetTrackGainAbsArgs"), "type": String("object")} }, Tool { name: "set_selected_strip_pan_stereo_width", description: "Sets the stereo width for the currently selected stereo strip's panner.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"width": Object {"description": String("The desired stereo width for the currently selected strip. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("width")], "title": String("SetSelectedStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "transport_play", description: "Starts Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "goto_start", description: "Moves the playhead to the session start.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_trim_db", description: "Sets the trim of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_db": Object {"description": String("The desired trim in dB. Valid range: -20.0 to 20.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_db")], "title": String("SetTrackTrimDBArgs"), "type": String("object")} }, Tool { name: "set_track_rec_enable", description: "Sets the record enable state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rec_st": Object {"description": String("The desired record enable state. 0 for off, 1 for on."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rec_st"), String("rid")], "title": String("SetTrackRecEnableArgs"), "type": String("object")} }, Tool { name: "set_track_gain_db", description: "Sets the gain of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_db": Object {"description": String("The desired gain in dB. Valid range: -400.0 to 6.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_db"), String("rid")], "title": String("SetTrackGainDBArgs"), "type": String("object")} }, Tool { name: "locate", description: "Locates the playhead to a specific sample position and optionally starts playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"roll": Object {"description": String("Whether to start playing after locating. 0 for stop, 1 for play."), "format": String("int32"), "type": String("integer")}, "spos": Object {"description": String("The position in samples to locate to."), "format": String("int64"), "type": String("integer")}}, "required": Array [String("roll"), String("spos")], "title": String("LocateToolArgs"), "type": String("object")} }, Tool { name: "redo", description: "Redoes the last undone action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "goto_end", description: "Moves the playhead to the session end.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "toggle_punch_in", description: "Toggles the Punch In state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "rec_enable_toggle", description: "Toggles the master record enable or selected track record enable.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "select_strip", description: "Selects a specific strip (track/bus) in Ardour.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus to select."), "format": String("int32"), "type": String("integer")}, "select_state": Object {"description": String("The desired select state (true to select). Currently, only true (1) is effective for selection."), "type": String("boolean")}}, "required": Array [String("rid"), String("select_state")], "title": String("SelectStripArgs"), "type": String("object")} }, Tool { name: "loop_toggle", description: "Toggles loop playback mode.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_strip_pan_stereo_width", description: "Sets the stereo width for a panner on a stereo strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the stereo track/bus."), "format": String("int32"), "type": String("integer")}, "width": Object {"description": String("The desired stereo width. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("width")], "title": String("SetStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "set_track_solo", description: "Sets the solo state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "solo_st": Object {"description": String("The desired solo state. 0 for solo off, 1 for solo on."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rid"), String("solo_st")], "title": String("SetTrackSoloArgs"), "type": String("object")} }, Tool { name: "prev_marker", description: "Moves the playhead to the previous location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_strip_plugin_parameter", description: "Sets a specific parameter of a plugin on a strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"param_id": Object {"description": String("The 1-indexed ID of the parameter within the plugin."), "format": String("int32"), "type": String("integer")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "value": Object {"description": String("The desired parameter value, normalized (0.0 to 1.0)."), "format": String("float"), "type": String("number")}}, "required": Array [String("param_id"), String("plugin_slot"), String("rid"), String("value")], "title": String("SetStripPluginParameterArgs"), "type": String("object")} }, Tool { name: "save_state", description: "Saves the current session state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "add_marker", description: "Adds a location marker at the current playhead position.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "undo", description: "Undoes the last action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "ffwd", description: "Fast forwards the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_transport_speed", description: "Sets Ardour's transport speed. Valid range: -8.0 to 8.0.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"speed": Object {"description": String("The desired transport speed. Valid range: -8.0 to 8.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("speed")], "title": String("SetTransportSpeedArgs"), "type": String("object")} }] }), Number(1))) +2026-06-30T09:58:41.568155Z  INFO ardour_mcp_server: +====================================================================== +NEW SERVER RUN: Tue, 30 Jun 2026 11:58:41 +0200 +====================================================================== +2026-06-30T09:58:41.568240Z  INFO ardour_mcp_server: Attempting to create OSC sender for Ardour at 127.0.0.1:3819 +2026-06-30T09:58:41.568294Z  INFO ardour_mcp_server: OSC sender created and connected to Ardour at 127.0.0.1:3819 +2026-06-30T09:58:41.568301Z  INFO ardour_mcp_server: Sending /set_surface to Ardour to enable OSC feedback. +2026-06-30T09:58:41.568306Z  INFO ardour_mcp_server: Targeting feedback port: 9099 +2026-06-30T09:58:41.568501Z  INFO ardour_mcp_server: /set_surface command sent successfully to Ardour. +2026-06-30T09:58:41.599309Z  INFO serve_inner: rmcp::service: Service initialized as server peer_info=InitializeRequestParam { protocol_version: ProtocolVersion("2025-11-25"), capabilities: ClientCapabilities { experimental: None, roots: Some(RootsCapabilities { list_changed: None }), sampling: None }, client_info: Implementation { name: "claude-code", version: "2.1.196" } } +2026-06-30T09:58:41.599356Z  INFO ardour_mcp_server: Ardour MCP server started and waiting for connections... +2026-06-30T09:58:41.599394Z  INFO ardour_mcp_server: Starting OSC listener for Ardour events on 127.0.0.1:9099 +2026-06-30T09:58:41.599415Z DEBUG rmcp::service: new event evt=PeerMessage(Request(ListToolsRequest(Request { method: ListToolsRequestMethod, params: None }), Number(1))) +2026-06-30T09:58:41.599426Z  INFO rmcp::service: received request id=1 request=ListToolsRequest(Request { method: ListToolsRequestMethod, params: None }) +2026-06-30T09:58:41.599439Z DEBUG rmcp::service: new event evt=PeerMessage(Request(ListResourcesRequest(Request { method: ListResourcesRequestMethod, params: None }), Number(2))) +2026-06-30T09:58:41.599440Z  INFO rmcp::service: received request id=2 request=ListResourcesRequest(Request { method: ListResourcesRequestMethod, params: None }) +2026-06-30T09:58:41.599443Z ERROR ardour_mcp_server: OSC listener task failed: Failed to bind Tokio UDP socket for OSC on 127.0.0.1:9099: Address already in use (os error 98) +2026-06-30T09:58:41.599452Z DEBUG ardour_mcp_server: Listing resources. Count: 2, Content: [Annotated { raw: RawResource { uri: "ardour:/state/playback", name: "Ardour Playback State", description: Some("Current playback state of Ardour (e.g., Playing, Stopped, Unknown)."), mime_type: None, size: None }, annotations: None }, Annotated { raw: RawResource { uri: "ardour:/state/transport_frame", name: "Ardour Transport Frame Position", description: Some("Current playhead position in samples. Returns 'Unknown' if not yet reported by Ardour."), mime_type: None, size: None }, annotations: None }] +2026-06-30T09:58:41.599459Z  INFO rmcp::service: response message id=2 result=ListResourcesResult(ListResourcesResult { next_cursor: None, resources: [Annotated { raw: RawResource { uri: "ardour:/state/playback", name: "Ardour Playback State", description: Some("Current playback state of Ardour (e.g., Playing, Stopped, Unknown)."), mime_type: None, size: None }, annotations: None }, Annotated { raw: RawResource { uri: "ardour:/state/transport_frame", name: "Ardour Transport Frame Position", description: Some("Current playhead position in samples. Returns 'Unknown' if not yet reported by Ardour."), mime_type: None, size: None }, annotations: None }] }) +2026-06-30T09:58:41.599474Z DEBUG rmcp::service: new event evt=ToSink(Response(ListResourcesResult(ListResourcesResult { next_cursor: None, resources: [Annotated { raw: RawResource { uri: "ardour:/state/playback", name: "Ardour Playback State", description: Some("Current playback state of Ardour (e.g., Playing, Stopped, Unknown)."), mime_type: None, size: None }, annotations: None }, Annotated { raw: RawResource { uri: "ardour:/state/transport_frame", name: "Ardour Transport Frame Position", description: Some("Current playhead position in samples. Returns 'Unknown' if not yet reported by Ardour."), mime_type: None, size: None }, annotations: None }] }), Number(2))) +2026-06-30T09:58:41.599565Z  INFO rmcp::service: response message id=1 result=ListToolsResult(ListToolsResult { next_cursor: None, tools: [Tool { name: "toggle_punch_in", description: "Toggles the Punch In state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "access_action", description: "Executes a specified Ardour menu action by its name.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"action_name": Object {"description": String("The name of the Ardour menu action to execute (e.g., 'Editor/zoom-to-session')."), "type": String("string")}}, "required": Array [String("action_name")], "title": String("AccessActionArgs"), "type": String("object")} }, Tool { name: "locate", description: "Locates the playhead to a specific sample position and optionally starts playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"roll": Object {"description": String("Whether to start playing after locating. 0 for stop, 1 for play."), "format": String("int32"), "type": String("integer")}, "spos": Object {"description": String("The position in samples to locate to."), "format": String("int64"), "type": String("integer")}}, "required": Array [String("roll"), String("spos")], "title": String("LocateToolArgs"), "type": String("object")} }, Tool { name: "transport_play", description: "Starts Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_trim_abs", description: "Sets the absolute trim of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_abs": Object {"description": String("The desired absolute trim. Valid range: 0.1 to 10.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_abs")], "title": String("SetTrackTrimAbsArgs"), "type": String("object")} }, Tool { name: "loop_toggle", description: "Toggles loop playback mode.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "ffwd", description: "Fast forwards the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_trim_db", description: "Sets the trim of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_db": Object {"description": String("The desired trim in dB. Valid range: -20.0 to 20.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_db")], "title": String("SetTrackTrimDBArgs"), "type": String("object")} }, Tool { name: "transport_stop", description: "Stops Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_strip_plugin_active", description: "Activates or deactivates a plugin on a specific strip slot.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"active_state": Object {"description": String("The desired activation state (true for active, false for inactive)."), "type": String("boolean")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("active_state"), String("plugin_slot"), String("rid")], "title": String("SetStripPluginActiveArgs"), "type": String("object")} }, Tool { name: "goto_end", description: "Moves the playhead to the session end.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_rec_enable", description: "Sets the record enable state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rec_st": Object {"description": String("The desired record enable state. 0 for off, 1 for on."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rec_st"), String("rid")], "title": String("SetTrackRecEnableArgs"), "type": String("object")} }, Tool { name: "undo", description: "Undoes the last action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "save_state", description: "Saves the current session state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_strip_plugin_parameter", description: "Sets a specific parameter of a plugin on a strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"param_id": Object {"description": String("The 1-indexed ID of the parameter within the plugin."), "format": String("int32"), "type": String("integer")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "value": Object {"description": String("The desired parameter value, normalized (0.0 to 1.0)."), "format": String("float"), "type": String("number")}}, "required": Array [String("param_id"), String("plugin_slot"), String("rid"), String("value")], "title": String("SetStripPluginParameterArgs"), "type": String("object")} }, Tool { name: "redo", description: "Redoes the last undone action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "next_marker", description: "Moves the playhead to the next location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_strip_pan_stereo_width", description: "Sets the stereo width for a panner on a stereo strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the stereo track/bus."), "format": String("int32"), "type": String("integer")}, "width": Object {"description": String("The desired stereo width. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("width")], "title": String("SetStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "goto_start", description: "Moves the playhead to the session start.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "prev_marker", description: "Moves the playhead to the previous location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "select_strip", description: "Selects a specific strip (track/bus) in Ardour.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus to select."), "format": String("int32"), "type": String("integer")}, "select_state": Object {"description": String("The desired select state (true to select). Currently, only true (1) is effective for selection."), "type": String("boolean")}}, "required": Array [String("rid"), String("select_state")], "title": String("SelectStripArgs"), "type": String("object")} }, Tool { name: "toggle_punch_out", description: "Toggles the Punch Out state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_selected_strip_pan_stereo_width", description: "Sets the stereo width for the currently selected stereo strip's panner.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"width": Object {"description": String("The desired stereo width for the currently selected strip. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("width")], "title": String("SetSelectedStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "set_track_gain_db", description: "Sets the gain of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_db": Object {"description": String("The desired gain in dB. Valid range: -400.0 to 6.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_db"), String("rid")], "title": String("SetTrackGainDBArgs"), "type": String("object")} }, Tool { name: "set_track_solo", description: "Sets the solo state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "solo_st": Object {"description": String("The desired solo state. 0 for solo off, 1 for solo on."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rid"), String("solo_st")], "title": String("SetTrackSoloArgs"), "type": String("object")} }, Tool { name: "set_transport_speed", description: "Sets Ardour's transport speed. Valid range: -8.0 to 8.0.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"speed": Object {"description": String("The desired transport speed. Valid range: -8.0 to 8.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("speed")], "title": String("SetTransportSpeedArgs"), "type": String("object")} }, Tool { name: "toggle_all_rec_enables", description: "Toggles the record enable state for ALL tracks.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "add_marker", description: "Adds a location marker at the current playhead position.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_mute", description: "Sets the mute state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"mute_state": Object {"description": String("The desired mute state (true for mute, false for unmute)."), "type": String("boolean")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("mute_state"), String("rid")], "title": String("SetTrackMuteArgs"), "type": String("object")} }, Tool { name: "rewind", description: "Rewinds the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_gain_abs", description: "Sets the absolute gain of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_abs": Object {"description": String("The desired absolute gain. Valid range: 0.0 to 2.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_abs"), String("rid")], "title": String("SetTrackGainAbsArgs"), "type": String("object")} }, Tool { name: "rec_enable_toggle", description: "Toggles the master record enable or selected track record enable.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }] }) +2026-06-30T09:58:41.599696Z DEBUG rmcp::service: new event evt=ToSink(Response(ListToolsResult(ListToolsResult { next_cursor: None, tools: [Tool { name: "toggle_punch_in", description: "Toggles the Punch In state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "access_action", description: "Executes a specified Ardour menu action by its name.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"action_name": Object {"description": String("The name of the Ardour menu action to execute (e.g., 'Editor/zoom-to-session')."), "type": String("string")}}, "required": Array [String("action_name")], "title": String("AccessActionArgs"), "type": String("object")} }, Tool { name: "locate", description: "Locates the playhead to a specific sample position and optionally starts playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"roll": Object {"description": String("Whether to start playing after locating. 0 for stop, 1 for play."), "format": String("int32"), "type": String("integer")}, "spos": Object {"description": String("The position in samples to locate to."), "format": String("int64"), "type": String("integer")}}, "required": Array [String("roll"), String("spos")], "title": String("LocateToolArgs"), "type": String("object")} }, Tool { name: "transport_play", description: "Starts Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_trim_abs", description: "Sets the absolute trim of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_abs": Object {"description": String("The desired absolute trim. Valid range: 0.1 to 10.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_abs")], "title": String("SetTrackTrimAbsArgs"), "type": String("object")} }, Tool { name: "loop_toggle", description: "Toggles loop playback mode.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "ffwd", description: "Fast forwards the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_trim_db", description: "Sets the trim of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_db": Object {"description": String("The desired trim in dB. Valid range: -20.0 to 20.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_db")], "title": String("SetTrackTrimDBArgs"), "type": String("object")} }, Tool { name: "transport_stop", description: "Stops Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_strip_plugin_active", description: "Activates or deactivates a plugin on a specific strip slot.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"active_state": Object {"description": String("The desired activation state (true for active, false for inactive)."), "type": String("boolean")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("active_state"), String("plugin_slot"), String("rid")], "title": String("SetStripPluginActiveArgs"), "type": String("object")} }, Tool { name: "goto_end", description: "Moves the playhead to the session end.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_rec_enable", description: "Sets the record enable state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rec_st": Object {"description": String("The desired record enable state. 0 for off, 1 for on."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rec_st"), String("rid")], "title": String("SetTrackRecEnableArgs"), "type": String("object")} }, Tool { name: "undo", description: "Undoes the last action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "save_state", description: "Saves the current session state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_strip_plugin_parameter", description: "Sets a specific parameter of a plugin on a strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"param_id": Object {"description": String("The 1-indexed ID of the parameter within the plugin."), "format": String("int32"), "type": String("integer")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "value": Object {"description": String("The desired parameter value, normalized (0.0 to 1.0)."), "format": String("float"), "type": String("number")}}, "required": Array [String("param_id"), String("plugin_slot"), String("rid"), String("value")], "title": String("SetStripPluginParameterArgs"), "type": String("object")} }, Tool { name: "redo", description: "Redoes the last undone action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "next_marker", description: "Moves the playhead to the next location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_strip_pan_stereo_width", description: "Sets the stereo width for a panner on a stereo strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the stereo track/bus."), "format": String("int32"), "type": String("integer")}, "width": Object {"description": String("The desired stereo width. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("width")], "title": String("SetStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "goto_start", description: "Moves the playhead to the session start.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "prev_marker", description: "Moves the playhead to the previous location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "select_strip", description: "Selects a specific strip (track/bus) in Ardour.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus to select."), "format": String("int32"), "type": String("integer")}, "select_state": Object {"description": String("The desired select state (true to select). Currently, only true (1) is effective for selection."), "type": String("boolean")}}, "required": Array [String("rid"), String("select_state")], "title": String("SelectStripArgs"), "type": String("object")} }, Tool { name: "toggle_punch_out", description: "Toggles the Punch Out state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_selected_strip_pan_stereo_width", description: "Sets the stereo width for the currently selected stereo strip's panner.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"width": Object {"description": String("The desired stereo width for the currently selected strip. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("width")], "title": String("SetSelectedStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "set_track_gain_db", description: "Sets the gain of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_db": Object {"description": String("The desired gain in dB. Valid range: -400.0 to 6.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_db"), String("rid")], "title": String("SetTrackGainDBArgs"), "type": String("object")} }, Tool { name: "set_track_solo", description: "Sets the solo state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "solo_st": Object {"description": String("The desired solo state. 0 for solo off, 1 for solo on."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rid"), String("solo_st")], "title": String("SetTrackSoloArgs"), "type": String("object")} }, Tool { name: "set_transport_speed", description: "Sets Ardour's transport speed. Valid range: -8.0 to 8.0.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"speed": Object {"description": String("The desired transport speed. Valid range: -8.0 to 8.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("speed")], "title": String("SetTransportSpeedArgs"), "type": String("object")} }, Tool { name: "toggle_all_rec_enables", description: "Toggles the record enable state for ALL tracks.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "add_marker", description: "Adds a location marker at the current playhead position.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_mute", description: "Sets the mute state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"mute_state": Object {"description": String("The desired mute state (true for mute, false for unmute)."), "type": String("boolean")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("mute_state"), String("rid")], "title": String("SetTrackMuteArgs"), "type": String("object")} }, Tool { name: "rewind", description: "Rewinds the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_gain_abs", description: "Sets the absolute gain of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_abs": Object {"description": String("The desired absolute gain. Valid range: 0.0 to 2.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_abs"), String("rid")], "title": String("SetTrackGainAbsArgs"), "type": String("object")} }, Tool { name: "rec_enable_toggle", description: "Toggles the master record enable or selected track record enable.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }] }), Number(1))) +2026-06-30T10:27:53.475908Z  INFO ardour_mcp_server: +====================================================================== +NEW SERVER RUN: Tue, 30 Jun 2026 12:27:53 +0200 +====================================================================== +2026-06-30T10:27:53.475958Z  INFO ardour_mcp_server: Attempting to create OSC sender for Ardour at 127.0.0.1:3819 +2026-06-30T10:27:53.475995Z  INFO ardour_mcp_server: OSC sender created and connected to Ardour at 127.0.0.1:3819 +2026-06-30T10:27:53.476001Z  INFO ardour_mcp_server: Sending /set_surface to Ardour to enable OSC feedback. +2026-06-30T10:27:53.476006Z  INFO ardour_mcp_server: Targeting feedback port: 9099 +2026-06-30T10:27:53.476189Z  INFO ardour_mcp_server: /set_surface command sent successfully to Ardour. +2026-06-30T10:27:53.505929Z  INFO serve_inner: rmcp::service: Service initialized as server peer_info=InitializeRequestParam { protocol_version: ProtocolVersion("2025-11-25"), capabilities: ClientCapabilities { experimental: None, roots: Some(RootsCapabilities { list_changed: None }), sampling: None }, client_info: Implementation { name: "claude-code", version: "2.1.196" } } +2026-06-30T10:27:53.505971Z  INFO ardour_mcp_server: Ardour MCP server started and waiting for connections... +2026-06-30T10:27:53.506009Z DEBUG rmcp::service: new event evt=PeerMessage(Request(ListToolsRequest(Request { method: ListToolsRequestMethod, params: None }), Number(1))) +2026-06-30T10:27:53.506021Z  INFO rmcp::service: received request id=1 request=ListToolsRequest(Request { method: ListToolsRequestMethod, params: None }) +2026-06-30T10:27:53.506028Z  INFO ardour_mcp_server: Starting OSC listener for Ardour events on 127.0.0.1:9099 +2026-06-30T10:27:53.506040Z DEBUG rmcp::service: new event evt=PeerMessage(Request(ListResourcesRequest(Request { method: ListResourcesRequestMethod, params: None }), Number(2))) +2026-06-30T10:27:53.506044Z  INFO rmcp::service: received request id=2 request=ListResourcesRequest(Request { method: ListResourcesRequestMethod, params: None }) +2026-06-30T10:27:53.506061Z ERROR ardour_mcp_server: OSC listener task failed: Failed to bind Tokio UDP socket for OSC on 127.0.0.1:9099: Address already in use (os error 98) +2026-06-30T10:27:53.506060Z DEBUG ardour_mcp_server: Listing resources. Count: 2, Content: [Annotated { raw: RawResource { uri: "ardour:/state/playback", name: "Ardour Playback State", description: Some("Current playback state of Ardour (e.g., Playing, Stopped, Unknown)."), mime_type: None, size: None }, annotations: None }, Annotated { raw: RawResource { uri: "ardour:/state/transport_frame", name: "Ardour Transport Frame Position", description: Some("Current playhead position in samples. Returns 'Unknown' if not yet reported by Ardour."), mime_type: None, size: None }, annotations: None }] +2026-06-30T10:27:53.506069Z  INFO rmcp::service: response message id=2 result=ListResourcesResult(ListResourcesResult { next_cursor: None, resources: [Annotated { raw: RawResource { uri: "ardour:/state/playback", name: "Ardour Playback State", description: Some("Current playback state of Ardour (e.g., Playing, Stopped, Unknown)."), mime_type: None, size: None }, annotations: None }, Annotated { raw: RawResource { uri: "ardour:/state/transport_frame", name: "Ardour Transport Frame Position", description: Some("Current playhead position in samples. Returns 'Unknown' if not yet reported by Ardour."), mime_type: None, size: None }, annotations: None }] }) +2026-06-30T10:27:53.506080Z DEBUG rmcp::service: new event evt=ToSink(Response(ListResourcesResult(ListResourcesResult { next_cursor: None, resources: [Annotated { raw: RawResource { uri: "ardour:/state/playback", name: "Ardour Playback State", description: Some("Current playback state of Ardour (e.g., Playing, Stopped, Unknown)."), mime_type: None, size: None }, annotations: None }, Annotated { raw: RawResource { uri: "ardour:/state/transport_frame", name: "Ardour Transport Frame Position", description: Some("Current playhead position in samples. Returns 'Unknown' if not yet reported by Ardour."), mime_type: None, size: None }, annotations: None }] }), Number(2))) +2026-06-30T10:27:53.506151Z  INFO rmcp::service: response message id=1 result=ListToolsResult(ListToolsResult { next_cursor: None, tools: [Tool { name: "save_state", description: "Saves the current session state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "goto_start", description: "Moves the playhead to the session start.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_gain_abs", description: "Sets the absolute gain of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_abs": Object {"description": String("The desired absolute gain. Valid range: 0.0 to 2.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_abs"), String("rid")], "title": String("SetTrackGainAbsArgs"), "type": String("object")} }, Tool { name: "set_track_gain_db", description: "Sets the gain of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_db": Object {"description": String("The desired gain in dB. Valid range: -400.0 to 6.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_db"), String("rid")], "title": String("SetTrackGainDBArgs"), "type": String("object")} }, Tool { name: "rec_enable_toggle", description: "Toggles the master record enable or selected track record enable.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "next_marker", description: "Moves the playhead to the next location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "locate", description: "Locates the playhead to a specific sample position and optionally starts playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"roll": Object {"description": String("Whether to start playing after locating. 0 for stop, 1 for play."), "format": String("int32"), "type": String("integer")}, "spos": Object {"description": String("The position in samples to locate to."), "format": String("int64"), "type": String("integer")}}, "required": Array [String("roll"), String("spos")], "title": String("LocateToolArgs"), "type": String("object")} }, Tool { name: "toggle_all_rec_enables", description: "Toggles the record enable state for ALL tracks.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_selected_strip_pan_stereo_width", description: "Sets the stereo width for the currently selected stereo strip's panner.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"width": Object {"description": String("The desired stereo width for the currently selected strip. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("width")], "title": String("SetSelectedStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "set_strip_plugin_parameter", description: "Sets a specific parameter of a plugin on a strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"param_id": Object {"description": String("The 1-indexed ID of the parameter within the plugin."), "format": String("int32"), "type": String("integer")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "value": Object {"description": String("The desired parameter value, normalized (0.0 to 1.0)."), "format": String("float"), "type": String("number")}}, "required": Array [String("param_id"), String("plugin_slot"), String("rid"), String("value")], "title": String("SetStripPluginParameterArgs"), "type": String("object")} }, Tool { name: "access_action", description: "Executes a specified Ardour menu action by its name.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"action_name": Object {"description": String("The name of the Ardour menu action to execute (e.g., 'Editor/zoom-to-session')."), "type": String("string")}}, "required": Array [String("action_name")], "title": String("AccessActionArgs"), "type": String("object")} }, Tool { name: "set_strip_plugin_active", description: "Activates or deactivates a plugin on a specific strip slot.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"active_state": Object {"description": String("The desired activation state (true for active, false for inactive)."), "type": String("boolean")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("active_state"), String("plugin_slot"), String("rid")], "title": String("SetStripPluginActiveArgs"), "type": String("object")} }, Tool { name: "redo", description: "Redoes the last undone action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "transport_stop", description: "Stops Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_strip_pan_stereo_width", description: "Sets the stereo width for a panner on a stereo strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the stereo track/bus."), "format": String("int32"), "type": String("integer")}, "width": Object {"description": String("The desired stereo width. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("width")], "title": String("SetStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "set_track_trim_abs", description: "Sets the absolute trim of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_abs": Object {"description": String("The desired absolute trim. Valid range: 0.1 to 10.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_abs")], "title": String("SetTrackTrimAbsArgs"), "type": String("object")} }, Tool { name: "set_track_mute", description: "Sets the mute state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"mute_state": Object {"description": String("The desired mute state (true for mute, false for unmute)."), "type": String("boolean")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("mute_state"), String("rid")], "title": String("SetTrackMuteArgs"), "type": String("object")} }, Tool { name: "set_track_solo", description: "Sets the solo state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "solo_st": Object {"description": String("The desired solo state. 0 for solo off, 1 for solo on."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rid"), String("solo_st")], "title": String("SetTrackSoloArgs"), "type": String("object")} }, Tool { name: "ffwd", description: "Fast forwards the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_rec_enable", description: "Sets the record enable state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rec_st": Object {"description": String("The desired record enable state. 0 for off, 1 for on."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rec_st"), String("rid")], "title": String("SetTrackRecEnableArgs"), "type": String("object")} }, Tool { name: "loop_toggle", description: "Toggles loop playback mode.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "transport_play", description: "Starts Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "toggle_punch_out", description: "Toggles the Punch Out state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "rewind", description: "Rewinds the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_transport_speed", description: "Sets Ardour's transport speed. Valid range: -8.0 to 8.0.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"speed": Object {"description": String("The desired transport speed. Valid range: -8.0 to 8.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("speed")], "title": String("SetTransportSpeedArgs"), "type": String("object")} }, Tool { name: "toggle_punch_in", description: "Toggles the Punch In state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "undo", description: "Undoes the last action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "goto_end", description: "Moves the playhead to the session end.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "prev_marker", description: "Moves the playhead to the previous location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_trim_db", description: "Sets the trim of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_db": Object {"description": String("The desired trim in dB. Valid range: -20.0 to 20.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_db")], "title": String("SetTrackTrimDBArgs"), "type": String("object")} }, Tool { name: "add_marker", description: "Adds a location marker at the current playhead position.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "select_strip", description: "Selects a specific strip (track/bus) in Ardour.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus to select."), "format": String("int32"), "type": String("integer")}, "select_state": Object {"description": String("The desired select state (true to select). Currently, only true (1) is effective for selection."), "type": String("boolean")}}, "required": Array [String("rid"), String("select_state")], "title": String("SelectStripArgs"), "type": String("object")} }] }) +2026-06-30T10:27:53.506210Z DEBUG rmcp::service: new event evt=ToSink(Response(ListToolsResult(ListToolsResult { next_cursor: None, tools: [Tool { name: "save_state", description: "Saves the current session state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "goto_start", description: "Moves the playhead to the session start.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_gain_abs", description: "Sets the absolute gain of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_abs": Object {"description": String("The desired absolute gain. Valid range: 0.0 to 2.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_abs"), String("rid")], "title": String("SetTrackGainAbsArgs"), "type": String("object")} }, Tool { name: "set_track_gain_db", description: "Sets the gain of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_db": Object {"description": String("The desired gain in dB. Valid range: -400.0 to 6.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_db"), String("rid")], "title": String("SetTrackGainDBArgs"), "type": String("object")} }, Tool { name: "rec_enable_toggle", description: "Toggles the master record enable or selected track record enable.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "next_marker", description: "Moves the playhead to the next location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "locate", description: "Locates the playhead to a specific sample position and optionally starts playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"roll": Object {"description": String("Whether to start playing after locating. 0 for stop, 1 for play."), "format": String("int32"), "type": String("integer")}, "spos": Object {"description": String("The position in samples to locate to."), "format": String("int64"), "type": String("integer")}}, "required": Array [String("roll"), String("spos")], "title": String("LocateToolArgs"), "type": String("object")} }, Tool { name: "toggle_all_rec_enables", description: "Toggles the record enable state for ALL tracks.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_selected_strip_pan_stereo_width", description: "Sets the stereo width for the currently selected stereo strip's panner.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"width": Object {"description": String("The desired stereo width for the currently selected strip. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("width")], "title": String("SetSelectedStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "set_strip_plugin_parameter", description: "Sets a specific parameter of a plugin on a strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"param_id": Object {"description": String("The 1-indexed ID of the parameter within the plugin."), "format": String("int32"), "type": String("integer")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "value": Object {"description": String("The desired parameter value, normalized (0.0 to 1.0)."), "format": String("float"), "type": String("number")}}, "required": Array [String("param_id"), String("plugin_slot"), String("rid"), String("value")], "title": String("SetStripPluginParameterArgs"), "type": String("object")} }, Tool { name: "access_action", description: "Executes a specified Ardour menu action by its name.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"action_name": Object {"description": String("The name of the Ardour menu action to execute (e.g., 'Editor/zoom-to-session')."), "type": String("string")}}, "required": Array [String("action_name")], "title": String("AccessActionArgs"), "type": String("object")} }, Tool { name: "set_strip_plugin_active", description: "Activates or deactivates a plugin on a specific strip slot.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"active_state": Object {"description": String("The desired activation state (true for active, false for inactive)."), "type": String("boolean")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("active_state"), String("plugin_slot"), String("rid")], "title": String("SetStripPluginActiveArgs"), "type": String("object")} }, Tool { name: "redo", description: "Redoes the last undone action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "transport_stop", description: "Stops Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_strip_pan_stereo_width", description: "Sets the stereo width for a panner on a stereo strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the stereo track/bus."), "format": String("int32"), "type": String("integer")}, "width": Object {"description": String("The desired stereo width. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("width")], "title": String("SetStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "set_track_trim_abs", description: "Sets the absolute trim of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_abs": Object {"description": String("The desired absolute trim. Valid range: 0.1 to 10.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_abs")], "title": String("SetTrackTrimAbsArgs"), "type": String("object")} }, Tool { name: "set_track_mute", description: "Sets the mute state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"mute_state": Object {"description": String("The desired mute state (true for mute, false for unmute)."), "type": String("boolean")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("mute_state"), String("rid")], "title": String("SetTrackMuteArgs"), "type": String("object")} }, Tool { name: "set_track_solo", description: "Sets the solo state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "solo_st": Object {"description": String("The desired solo state. 0 for solo off, 1 for solo on."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rid"), String("solo_st")], "title": String("SetTrackSoloArgs"), "type": String("object")} }, Tool { name: "ffwd", description: "Fast forwards the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_rec_enable", description: "Sets the record enable state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rec_st": Object {"description": String("The desired record enable state. 0 for off, 1 for on."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rec_st"), String("rid")], "title": String("SetTrackRecEnableArgs"), "type": String("object")} }, Tool { name: "loop_toggle", description: "Toggles loop playback mode.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "transport_play", description: "Starts Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "toggle_punch_out", description: "Toggles the Punch Out state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "rewind", description: "Rewinds the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_transport_speed", description: "Sets Ardour's transport speed. Valid range: -8.0 to 8.0.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"speed": Object {"description": String("The desired transport speed. Valid range: -8.0 to 8.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("speed")], "title": String("SetTransportSpeedArgs"), "type": String("object")} }, Tool { name: "toggle_punch_in", description: "Toggles the Punch In state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "undo", description: "Undoes the last action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "goto_end", description: "Moves the playhead to the session end.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "prev_marker", description: "Moves the playhead to the previous location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_trim_db", description: "Sets the trim of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_db": Object {"description": String("The desired trim in dB. Valid range: -20.0 to 20.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_db")], "title": String("SetTrackTrimDBArgs"), "type": String("object")} }, Tool { name: "add_marker", description: "Adds a location marker at the current playhead position.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "select_strip", description: "Selects a specific strip (track/bus) in Ardour.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus to select."), "format": String("int32"), "type": String("integer")}, "select_state": Object {"description": String("The desired select state (true to select). Currently, only true (1) is effective for selection."), "type": String("boolean")}}, "required": Array [String("rid"), String("select_state")], "title": String("SelectStripArgs"), "type": String("object")} }] }), Number(1))) +2026-06-30T11:11:53.498533Z  INFO ardour_mcp_server: +====================================================================== +NEW SERVER RUN: Tue, 30 Jun 2026 13:11:53 +0200 +====================================================================== +2026-06-30T11:11:53.498599Z  INFO ardour_mcp_server: Attempting to create OSC sender for Ardour at 127.0.0.1:3819 +2026-06-30T11:11:53.498683Z  INFO ardour_mcp_server: OSC sender created and connected to Ardour at 127.0.0.1:3819 +2026-06-30T11:11:53.498688Z  INFO ardour_mcp_server: Sending /set_surface to Ardour to enable OSC feedback. +2026-06-30T11:11:53.498692Z  INFO ardour_mcp_server: Targeting feedback port: 9099 +2026-06-30T11:11:53.498896Z  INFO ardour_mcp_server: /set_surface command sent successfully to Ardour. +2026-06-30T11:11:53.625708Z  INFO serve_inner: rmcp::service: Service initialized as server peer_info=InitializeRequestParam { protocol_version: ProtocolVersion("2025-11-25"), capabilities: ClientCapabilities { experimental: None, roots: Some(RootsCapabilities { list_changed: None }), sampling: None }, client_info: Implementation { name: "claude-code", version: "2.1.196" } } +2026-06-30T11:11:53.625808Z  INFO ardour_mcp_server: Ardour MCP server started and waiting for connections... +2026-06-30T11:11:53.625849Z  INFO ardour_mcp_server: Starting OSC listener for Ardour events on 127.0.0.1:9099 +2026-06-30T11:11:53.625867Z DEBUG rmcp::service: new event evt=PeerMessage(Request(ListToolsRequest(Request { method: ListToolsRequestMethod, params: None }), Number(1))) +2026-06-30T11:11:53.625887Z  INFO rmcp::service: received request id=1 request=ListToolsRequest(Request { method: ListToolsRequestMethod, params: None }) +2026-06-30T11:11:53.625915Z DEBUG rmcp::service: new event evt=PeerMessage(Request(ListResourcesRequest(Request { method: ListResourcesRequestMethod, params: None }), Number(2))) +2026-06-30T11:11:53.625919Z  INFO rmcp::service: received request id=2 request=ListResourcesRequest(Request { method: ListResourcesRequestMethod, params: None }) +2026-06-30T11:11:53.625919Z ERROR ardour_mcp_server: OSC listener task failed: Failed to bind Tokio UDP socket for OSC on 127.0.0.1:9099: Address already in use (os error 98) +2026-06-30T11:11:53.625947Z DEBUG ardour_mcp_server: Listing resources. Count: 2, Content: [Annotated { raw: RawResource { uri: "ardour:/state/playback", name: "Ardour Playback State", description: Some("Current playback state of Ardour (e.g., Playing, Stopped, Unknown)."), mime_type: None, size: None }, annotations: None }, Annotated { raw: RawResource { uri: "ardour:/state/transport_frame", name: "Ardour Transport Frame Position", description: Some("Current playhead position in samples. Returns 'Unknown' if not yet reported by Ardour."), mime_type: None, size: None }, annotations: None }] +2026-06-30T11:11:53.625958Z  INFO rmcp::service: response message id=2 result=ListResourcesResult(ListResourcesResult { next_cursor: None, resources: [Annotated { raw: RawResource { uri: "ardour:/state/playback", name: "Ardour Playback State", description: Some("Current playback state of Ardour (e.g., Playing, Stopped, Unknown)."), mime_type: None, size: None }, annotations: None }, Annotated { raw: RawResource { uri: "ardour:/state/transport_frame", name: "Ardour Transport Frame Position", description: Some("Current playhead position in samples. Returns 'Unknown' if not yet reported by Ardour."), mime_type: None, size: None }, annotations: None }] }) +2026-06-30T11:11:53.625969Z DEBUG rmcp::service: new event evt=ToSink(Response(ListResourcesResult(ListResourcesResult { next_cursor: None, resources: [Annotated { raw: RawResource { uri: "ardour:/state/playback", name: "Ardour Playback State", description: Some("Current playback state of Ardour (e.g., Playing, Stopped, Unknown)."), mime_type: None, size: None }, annotations: None }, Annotated { raw: RawResource { uri: "ardour:/state/transport_frame", name: "Ardour Transport Frame Position", description: Some("Current playhead position in samples. Returns 'Unknown' if not yet reported by Ardour."), mime_type: None, size: None }, annotations: None }] }), Number(2))) +2026-06-30T11:11:53.626086Z  INFO rmcp::service: response message id=1 result=ListToolsResult(ListToolsResult { next_cursor: None, tools: [Tool { name: "set_strip_plugin_active", description: "Activates or deactivates a plugin on a specific strip slot.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"active_state": Object {"description": String("The desired activation state (true for active, false for inactive)."), "type": String("boolean")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("active_state"), String("plugin_slot"), String("rid")], "title": String("SetStripPluginActiveArgs"), "type": String("object")} }, Tool { name: "save_state", description: "Saves the current session state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_gain_abs", description: "Sets the absolute gain of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_abs": Object {"description": String("The desired absolute gain. Valid range: 0.0 to 2.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_abs"), String("rid")], "title": String("SetTrackGainAbsArgs"), "type": String("object")} }, Tool { name: "loop_toggle", description: "Toggles loop playback mode.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "redo", description: "Redoes the last undone action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "rec_enable_toggle", description: "Toggles the master record enable or selected track record enable.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "select_strip", description: "Selects a specific strip (track/bus) in Ardour.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus to select."), "format": String("int32"), "type": String("integer")}, "select_state": Object {"description": String("The desired select state (true to select). Currently, only true (1) is effective for selection."), "type": String("boolean")}}, "required": Array [String("rid"), String("select_state")], "title": String("SelectStripArgs"), "type": String("object")} }, Tool { name: "prev_marker", description: "Moves the playhead to the previous location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "toggle_punch_out", description: "Toggles the Punch Out state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_gain_db", description: "Sets the gain of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_db": Object {"description": String("The desired gain in dB. Valid range: -400.0 to 6.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_db"), String("rid")], "title": String("SetTrackGainDBArgs"), "type": String("object")} }, Tool { name: "set_track_trim_db", description: "Sets the trim of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_db": Object {"description": String("The desired trim in dB. Valid range: -20.0 to 20.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_db")], "title": String("SetTrackTrimDBArgs"), "type": String("object")} }, Tool { name: "set_track_trim_abs", description: "Sets the absolute trim of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_abs": Object {"description": String("The desired absolute trim. Valid range: 0.1 to 10.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_abs")], "title": String("SetTrackTrimAbsArgs"), "type": String("object")} }, Tool { name: "set_transport_speed", description: "Sets Ardour's transport speed. Valid range: -8.0 to 8.0.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"speed": Object {"description": String("The desired transport speed. Valid range: -8.0 to 8.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("speed")], "title": String("SetTransportSpeedArgs"), "type": String("object")} }, Tool { name: "add_marker", description: "Adds a location marker at the current playhead position.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_solo", description: "Sets the solo state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "solo_st": Object {"description": String("The desired solo state. 0 for solo off, 1 for solo on."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rid"), String("solo_st")], "title": String("SetTrackSoloArgs"), "type": String("object")} }, Tool { name: "goto_start", description: "Moves the playhead to the session start.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "rewind", description: "Rewinds the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "toggle_all_rec_enables", description: "Toggles the record enable state for ALL tracks.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_mute", description: "Sets the mute state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"mute_state": Object {"description": String("The desired mute state (true for mute, false for unmute)."), "type": String("boolean")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("mute_state"), String("rid")], "title": String("SetTrackMuteArgs"), "type": String("object")} }, Tool { name: "access_action", description: "Executes a specified Ardour menu action by its name.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"action_name": Object {"description": String("The name of the Ardour menu action to execute (e.g., 'Editor/zoom-to-session')."), "type": String("string")}}, "required": Array [String("action_name")], "title": String("AccessActionArgs"), "type": String("object")} }, Tool { name: "next_marker", description: "Moves the playhead to the next location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_strip_plugin_parameter", description: "Sets a specific parameter of a plugin on a strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"param_id": Object {"description": String("The 1-indexed ID of the parameter within the plugin."), "format": String("int32"), "type": String("integer")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "value": Object {"description": String("The desired parameter value, normalized (0.0 to 1.0)."), "format": String("float"), "type": String("number")}}, "required": Array [String("param_id"), String("plugin_slot"), String("rid"), String("value")], "title": String("SetStripPluginParameterArgs"), "type": String("object")} }, Tool { name: "set_strip_pan_stereo_width", description: "Sets the stereo width for a panner on a stereo strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the stereo track/bus."), "format": String("int32"), "type": String("integer")}, "width": Object {"description": String("The desired stereo width. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("width")], "title": String("SetStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "undo", description: "Undoes the last action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "transport_play", description: "Starts Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "ffwd", description: "Fast forwards the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_rec_enable", description: "Sets the record enable state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rec_st": Object {"description": String("The desired record enable state. 0 for off, 1 for on."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rec_st"), String("rid")], "title": String("SetTrackRecEnableArgs"), "type": String("object")} }, Tool { name: "set_selected_strip_pan_stereo_width", description: "Sets the stereo width for the currently selected stereo strip's panner.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"width": Object {"description": String("The desired stereo width for the currently selected strip. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("width")], "title": String("SetSelectedStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "toggle_punch_in", description: "Toggles the Punch In state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "goto_end", description: "Moves the playhead to the session end.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "transport_stop", description: "Stops Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "locate", description: "Locates the playhead to a specific sample position and optionally starts playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"roll": Object {"description": String("Whether to start playing after locating. 0 for stop, 1 for play."), "format": String("int32"), "type": String("integer")}, "spos": Object {"description": String("The position in samples to locate to."), "format": String("int64"), "type": String("integer")}}, "required": Array [String("roll"), String("spos")], "title": String("LocateToolArgs"), "type": String("object")} }] }) +2026-06-30T11:11:53.626161Z DEBUG rmcp::service: new event evt=ToSink(Response(ListToolsResult(ListToolsResult { next_cursor: None, tools: [Tool { name: "set_strip_plugin_active", description: "Activates or deactivates a plugin on a specific strip slot.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"active_state": Object {"description": String("The desired activation state (true for active, false for inactive)."), "type": String("boolean")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("active_state"), String("plugin_slot"), String("rid")], "title": String("SetStripPluginActiveArgs"), "type": String("object")} }, Tool { name: "save_state", description: "Saves the current session state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_gain_abs", description: "Sets the absolute gain of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_abs": Object {"description": String("The desired absolute gain. Valid range: 0.0 to 2.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_abs"), String("rid")], "title": String("SetTrackGainAbsArgs"), "type": String("object")} }, Tool { name: "loop_toggle", description: "Toggles loop playback mode.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "redo", description: "Redoes the last undone action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "rec_enable_toggle", description: "Toggles the master record enable or selected track record enable.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "select_strip", description: "Selects a specific strip (track/bus) in Ardour.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus to select."), "format": String("int32"), "type": String("integer")}, "select_state": Object {"description": String("The desired select state (true to select). Currently, only true (1) is effective for selection."), "type": String("boolean")}}, "required": Array [String("rid"), String("select_state")], "title": String("SelectStripArgs"), "type": String("object")} }, Tool { name: "prev_marker", description: "Moves the playhead to the previous location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "toggle_punch_out", description: "Toggles the Punch Out state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_gain_db", description: "Sets the gain of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_db": Object {"description": String("The desired gain in dB. Valid range: -400.0 to 6.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_db"), String("rid")], "title": String("SetTrackGainDBArgs"), "type": String("object")} }, Tool { name: "set_track_trim_db", description: "Sets the trim of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_db": Object {"description": String("The desired trim in dB. Valid range: -20.0 to 20.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_db")], "title": String("SetTrackTrimDBArgs"), "type": String("object")} }, Tool { name: "set_track_trim_abs", description: "Sets the absolute trim of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_abs": Object {"description": String("The desired absolute trim. Valid range: 0.1 to 10.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_abs")], "title": String("SetTrackTrimAbsArgs"), "type": String("object")} }, Tool { name: "set_transport_speed", description: "Sets Ardour's transport speed. Valid range: -8.0 to 8.0.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"speed": Object {"description": String("The desired transport speed. Valid range: -8.0 to 8.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("speed")], "title": String("SetTransportSpeedArgs"), "type": String("object")} }, Tool { name: "add_marker", description: "Adds a location marker at the current playhead position.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_solo", description: "Sets the solo state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "solo_st": Object {"description": String("The desired solo state. 0 for solo off, 1 for solo on."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rid"), String("solo_st")], "title": String("SetTrackSoloArgs"), "type": String("object")} }, Tool { name: "goto_start", description: "Moves the playhead to the session start.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "rewind", description: "Rewinds the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "toggle_all_rec_enables", description: "Toggles the record enable state for ALL tracks.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_mute", description: "Sets the mute state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"mute_state": Object {"description": String("The desired mute state (true for mute, false for unmute)."), "type": String("boolean")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("mute_state"), String("rid")], "title": String("SetTrackMuteArgs"), "type": String("object")} }, Tool { name: "access_action", description: "Executes a specified Ardour menu action by its name.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"action_name": Object {"description": String("The name of the Ardour menu action to execute (e.g., 'Editor/zoom-to-session')."), "type": String("string")}}, "required": Array [String("action_name")], "title": String("AccessActionArgs"), "type": String("object")} }, Tool { name: "next_marker", description: "Moves the playhead to the next location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_strip_plugin_parameter", description: "Sets a specific parameter of a plugin on a strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"param_id": Object {"description": String("The 1-indexed ID of the parameter within the plugin."), "format": String("int32"), "type": String("integer")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "value": Object {"description": String("The desired parameter value, normalized (0.0 to 1.0)."), "format": String("float"), "type": String("number")}}, "required": Array [String("param_id"), String("plugin_slot"), String("rid"), String("value")], "title": String("SetStripPluginParameterArgs"), "type": String("object")} }, Tool { name: "set_strip_pan_stereo_width", description: "Sets the stereo width for a panner on a stereo strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the stereo track/bus."), "format": String("int32"), "type": String("integer")}, "width": Object {"description": String("The desired stereo width. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("width")], "title": String("SetStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "undo", description: "Undoes the last action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "transport_play", description: "Starts Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "ffwd", description: "Fast forwards the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_rec_enable", description: "Sets the record enable state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rec_st": Object {"description": String("The desired record enable state. 0 for off, 1 for on."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rec_st"), String("rid")], "title": String("SetTrackRecEnableArgs"), "type": String("object")} }, Tool { name: "set_selected_strip_pan_stereo_width", description: "Sets the stereo width for the currently selected stereo strip's panner.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"width": Object {"description": String("The desired stereo width for the currently selected strip. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("width")], "title": String("SetSelectedStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "toggle_punch_in", description: "Toggles the Punch In state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "goto_end", description: "Moves the playhead to the session end.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "transport_stop", description: "Stops Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "locate", description: "Locates the playhead to a specific sample position and optionally starts playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"roll": Object {"description": String("Whether to start playing after locating. 0 for stop, 1 for play."), "format": String("int32"), "type": String("integer")}, "spos": Object {"description": String("The position in samples to locate to."), "format": String("int64"), "type": String("integer")}}, "required": Array [String("roll"), String("spos")], "title": String("LocateToolArgs"), "type": String("object")} }] }), Number(1))) +2026-06-30T11:11:53.627056Z  INFO ardour_mcp_server: +====================================================================== +NEW SERVER RUN: Tue, 30 Jun 2026 13:11:53 +0200 +====================================================================== +2026-06-30T11:11:53.627088Z  INFO ardour_mcp_server: Attempting to create OSC sender for Ardour at 127.0.0.1:3819 +2026-06-30T11:11:53.627123Z  INFO ardour_mcp_server: OSC sender created and connected to Ardour at 127.0.0.1:3819 +2026-06-30T11:11:53.627128Z  INFO ardour_mcp_server: Sending /set_surface to Ardour to enable OSC feedback. +2026-06-30T11:11:53.627133Z  INFO ardour_mcp_server: Targeting feedback port: 9099 +2026-06-30T11:11:53.627281Z  INFO ardour_mcp_server: /set_surface command sent successfully to Ardour. +2026-06-30T11:11:53.698757Z  INFO serve_inner: rmcp::service: Service initialized as server peer_info=InitializeRequestParam { protocol_version: ProtocolVersion("2025-11-25"), capabilities: ClientCapabilities { experimental: None, roots: Some(RootsCapabilities { list_changed: None }), sampling: None }, client_info: Implementation { name: "claude-code", version: "2.1.196" } } +2026-06-30T11:11:53.698822Z  INFO ardour_mcp_server: Ardour MCP server started and waiting for connections... +2026-06-30T11:11:53.698873Z DEBUG rmcp::service: new event evt=PeerMessage(Request(ListToolsRequest(Request { method: ListToolsRequestMethod, params: None }), Number(1))) +2026-06-30T11:11:53.698879Z  INFO rmcp::service: received request id=1 request=ListToolsRequest(Request { method: ListToolsRequestMethod, params: None }) +2026-06-30T11:11:53.698889Z DEBUG rmcp::service: new event evt=PeerMessage(Request(ListResourcesRequest(Request { method: ListResourcesRequestMethod, params: None }), Number(2))) +2026-06-30T11:11:53.698891Z  INFO rmcp::service: received request id=2 request=ListResourcesRequest(Request { method: ListResourcesRequestMethod, params: None }) +2026-06-30T11:11:53.698925Z  INFO ardour_mcp_server: Starting OSC listener for Ardour events on 127.0.0.1:9099 +2026-06-30T11:11:53.698972Z ERROR ardour_mcp_server: OSC listener task failed: Failed to bind Tokio UDP socket for OSC on 127.0.0.1:9099: Address already in use (os error 98) +2026-06-30T11:11:53.699116Z  INFO rmcp::service: response message id=1 result=ListToolsResult(ListToolsResult { next_cursor: None, tools: [Tool { name: "redo", description: "Redoes the last undone action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "access_action", description: "Executes a specified Ardour menu action by its name.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"action_name": Object {"description": String("The name of the Ardour menu action to execute (e.g., 'Editor/zoom-to-session')."), "type": String("string")}}, "required": Array [String("action_name")], "title": String("AccessActionArgs"), "type": String("object")} }, Tool { name: "set_selected_strip_pan_stereo_width", description: "Sets the stereo width for the currently selected stereo strip's panner.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"width": Object {"description": String("The desired stereo width for the currently selected strip. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("width")], "title": String("SetSelectedStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "set_strip_plugin_active", description: "Activates or deactivates a plugin on a specific strip slot.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"active_state": Object {"description": String("The desired activation state (true for active, false for inactive)."), "type": String("boolean")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("active_state"), String("plugin_slot"), String("rid")], "title": String("SetStripPluginActiveArgs"), "type": String("object")} }, Tool { name: "set_track_mute", description: "Sets the mute state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"mute_state": Object {"description": String("The desired mute state (true for mute, false for unmute)."), "type": String("boolean")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("mute_state"), String("rid")], "title": String("SetTrackMuteArgs"), "type": String("object")} }, Tool { name: "set_strip_pan_stereo_width", description: "Sets the stereo width for a panner on a stereo strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the stereo track/bus."), "format": String("int32"), "type": String("integer")}, "width": Object {"description": String("The desired stereo width. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("width")], "title": String("SetStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "rewind", description: "Rewinds the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "toggle_punch_out", description: "Toggles the Punch Out state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "goto_start", description: "Moves the playhead to the session start.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "goto_end", description: "Moves the playhead to the session end.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "toggle_punch_in", description: "Toggles the Punch In state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "add_marker", description: "Adds a location marker at the current playhead position.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "save_state", description: "Saves the current session state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_trim_abs", description: "Sets the absolute trim of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_abs": Object {"description": String("The desired absolute trim. Valid range: 0.1 to 10.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_abs")], "title": String("SetTrackTrimAbsArgs"), "type": String("object")} }, Tool { name: "select_strip", description: "Selects a specific strip (track/bus) in Ardour.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus to select."), "format": String("int32"), "type": String("integer")}, "select_state": Object {"description": String("The desired select state (true to select). Currently, only true (1) is effective for selection."), "type": String("boolean")}}, "required": Array [String("rid"), String("select_state")], "title": String("SelectStripArgs"), "type": String("object")} }, Tool { name: "rec_enable_toggle", description: "Toggles the master record enable or selected track record enable.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "prev_marker", description: "Moves the playhead to the previous location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_trim_db", description: "Sets the trim of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_db": Object {"description": String("The desired trim in dB. Valid range: -20.0 to 20.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_db")], "title": String("SetTrackTrimDBArgs"), "type": String("object")} }, Tool { name: "toggle_all_rec_enables", description: "Toggles the record enable state for ALL tracks.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "undo", description: "Undoes the last action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_solo", description: "Sets the solo state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "solo_st": Object {"description": String("The desired solo state. 0 for solo off, 1 for solo on."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rid"), String("solo_st")], "title": String("SetTrackSoloArgs"), "type": String("object")} }, Tool { name: "transport_stop", description: "Stops Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_gain_abs", description: "Sets the absolute gain of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_abs": Object {"description": String("The desired absolute gain. Valid range: 0.0 to 2.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_abs"), String("rid")], "title": String("SetTrackGainAbsArgs"), "type": String("object")} }, Tool { name: "set_transport_speed", description: "Sets Ardour's transport speed. Valid range: -8.0 to 8.0.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"speed": Object {"description": String("The desired transport speed. Valid range: -8.0 to 8.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("speed")], "title": String("SetTransportSpeedArgs"), "type": String("object")} }, Tool { name: "ffwd", description: "Fast forwards the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "loop_toggle", description: "Toggles loop playback mode.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_gain_db", description: "Sets the gain of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_db": Object {"description": String("The desired gain in dB. Valid range: -400.0 to 6.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_db"), String("rid")], "title": String("SetTrackGainDBArgs"), "type": String("object")} }, Tool { name: "locate", description: "Locates the playhead to a specific sample position and optionally starts playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"roll": Object {"description": String("Whether to start playing after locating. 0 for stop, 1 for play."), "format": String("int32"), "type": String("integer")}, "spos": Object {"description": String("The position in samples to locate to."), "format": String("int64"), "type": String("integer")}}, "required": Array [String("roll"), String("spos")], "title": String("LocateToolArgs"), "type": String("object")} }, Tool { name: "transport_play", description: "Starts Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_rec_enable", description: "Sets the record enable state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rec_st": Object {"description": String("The desired record enable state. 0 for off, 1 for on."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rec_st"), String("rid")], "title": String("SetTrackRecEnableArgs"), "type": String("object")} }, Tool { name: "set_strip_plugin_parameter", description: "Sets a specific parameter of a plugin on a strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"param_id": Object {"description": String("The 1-indexed ID of the parameter within the plugin."), "format": String("int32"), "type": String("integer")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "value": Object {"description": String("The desired parameter value, normalized (0.0 to 1.0)."), "format": String("float"), "type": String("number")}}, "required": Array [String("param_id"), String("plugin_slot"), String("rid"), String("value")], "title": String("SetStripPluginParameterArgs"), "type": String("object")} }, Tool { name: "next_marker", description: "Moves the playhead to the next location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }] }) +2026-06-30T11:11:53.699187Z DEBUG ardour_mcp_server: Listing resources. Count: 2, Content: [Annotated { raw: RawResource { uri: "ardour:/state/playback", name: "Ardour Playback State", description: Some("Current playback state of Ardour (e.g., Playing, Stopped, Unknown)."), mime_type: None, size: None }, annotations: None }, Annotated { raw: RawResource { uri: "ardour:/state/transport_frame", name: "Ardour Transport Frame Position", description: Some("Current playhead position in samples. Returns 'Unknown' if not yet reported by Ardour."), mime_type: None, size: None }, annotations: None }] +2026-06-30T11:11:53.699193Z  INFO rmcp::service: response message id=2 result=ListResourcesResult(ListResourcesResult { next_cursor: None, resources: [Annotated { raw: RawResource { uri: "ardour:/state/playback", name: "Ardour Playback State", description: Some("Current playback state of Ardour (e.g., Playing, Stopped, Unknown)."), mime_type: None, size: None }, annotations: None }, Annotated { raw: RawResource { uri: "ardour:/state/transport_frame", name: "Ardour Transport Frame Position", description: Some("Current playhead position in samples. Returns 'Unknown' if not yet reported by Ardour."), mime_type: None, size: None }, annotations: None }] }) +2026-06-30T11:11:53.699199Z DEBUG rmcp::service: new event evt=ToSink(Response(ListToolsResult(ListToolsResult { next_cursor: None, tools: [Tool { name: "redo", description: "Redoes the last undone action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "access_action", description: "Executes a specified Ardour menu action by its name.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"action_name": Object {"description": String("The name of the Ardour menu action to execute (e.g., 'Editor/zoom-to-session')."), "type": String("string")}}, "required": Array [String("action_name")], "title": String("AccessActionArgs"), "type": String("object")} }, Tool { name: "set_selected_strip_pan_stereo_width", description: "Sets the stereo width for the currently selected stereo strip's panner.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"width": Object {"description": String("The desired stereo width for the currently selected strip. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("width")], "title": String("SetSelectedStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "set_strip_plugin_active", description: "Activates or deactivates a plugin on a specific strip slot.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"active_state": Object {"description": String("The desired activation state (true for active, false for inactive)."), "type": String("boolean")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("active_state"), String("plugin_slot"), String("rid")], "title": String("SetStripPluginActiveArgs"), "type": String("object")} }, Tool { name: "set_track_mute", description: "Sets the mute state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"mute_state": Object {"description": String("The desired mute state (true for mute, false for unmute)."), "type": String("boolean")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("mute_state"), String("rid")], "title": String("SetTrackMuteArgs"), "type": String("object")} }, Tool { name: "set_strip_pan_stereo_width", description: "Sets the stereo width for a panner on a stereo strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the stereo track/bus."), "format": String("int32"), "type": String("integer")}, "width": Object {"description": String("The desired stereo width. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("width")], "title": String("SetStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "rewind", description: "Rewinds the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "toggle_punch_out", description: "Toggles the Punch Out state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "goto_start", description: "Moves the playhead to the session start.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "goto_end", description: "Moves the playhead to the session end.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "toggle_punch_in", description: "Toggles the Punch In state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "add_marker", description: "Adds a location marker at the current playhead position.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "save_state", description: "Saves the current session state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_trim_abs", description: "Sets the absolute trim of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_abs": Object {"description": String("The desired absolute trim. Valid range: 0.1 to 10.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_abs")], "title": String("SetTrackTrimAbsArgs"), "type": String("object")} }, Tool { name: "select_strip", description: "Selects a specific strip (track/bus) in Ardour.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus to select."), "format": String("int32"), "type": String("integer")}, "select_state": Object {"description": String("The desired select state (true to select). Currently, only true (1) is effective for selection."), "type": String("boolean")}}, "required": Array [String("rid"), String("select_state")], "title": String("SelectStripArgs"), "type": String("object")} }, Tool { name: "rec_enable_toggle", description: "Toggles the master record enable or selected track record enable.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "prev_marker", description: "Moves the playhead to the previous location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_trim_db", description: "Sets the trim of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_db": Object {"description": String("The desired trim in dB. Valid range: -20.0 to 20.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_db")], "title": String("SetTrackTrimDBArgs"), "type": String("object")} }, Tool { name: "toggle_all_rec_enables", description: "Toggles the record enable state for ALL tracks.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "undo", description: "Undoes the last action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_solo", description: "Sets the solo state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "solo_st": Object {"description": String("The desired solo state. 0 for solo off, 1 for solo on."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rid"), String("solo_st")], "title": String("SetTrackSoloArgs"), "type": String("object")} }, Tool { name: "transport_stop", description: "Stops Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_gain_abs", description: "Sets the absolute gain of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_abs": Object {"description": String("The desired absolute gain. Valid range: 0.0 to 2.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_abs"), String("rid")], "title": String("SetTrackGainAbsArgs"), "type": String("object")} }, Tool { name: "set_transport_speed", description: "Sets Ardour's transport speed. Valid range: -8.0 to 8.0.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"speed": Object {"description": String("The desired transport speed. Valid range: -8.0 to 8.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("speed")], "title": String("SetTransportSpeedArgs"), "type": String("object")} }, Tool { name: "ffwd", description: "Fast forwards the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "loop_toggle", description: "Toggles loop playback mode.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_gain_db", description: "Sets the gain of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_db": Object {"description": String("The desired gain in dB. Valid range: -400.0 to 6.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_db"), String("rid")], "title": String("SetTrackGainDBArgs"), "type": String("object")} }, Tool { name: "locate", description: "Locates the playhead to a specific sample position and optionally starts playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"roll": Object {"description": String("Whether to start playing after locating. 0 for stop, 1 for play."), "format": String("int32"), "type": String("integer")}, "spos": Object {"description": String("The position in samples to locate to."), "format": String("int64"), "type": String("integer")}}, "required": Array [String("roll"), String("spos")], "title": String("LocateToolArgs"), "type": String("object")} }, Tool { name: "transport_play", description: "Starts Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_rec_enable", description: "Sets the record enable state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rec_st": Object {"description": String("The desired record enable state. 0 for off, 1 for on."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rec_st"), String("rid")], "title": String("SetTrackRecEnableArgs"), "type": String("object")} }, Tool { name: "set_strip_plugin_parameter", description: "Sets a specific parameter of a plugin on a strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"param_id": Object {"description": String("The 1-indexed ID of the parameter within the plugin."), "format": String("int32"), "type": String("integer")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "value": Object {"description": String("The desired parameter value, normalized (0.0 to 1.0)."), "format": String("float"), "type": String("number")}}, "required": Array [String("param_id"), String("plugin_slot"), String("rid"), String("value")], "title": String("SetStripPluginParameterArgs"), "type": String("object")} }, Tool { name: "next_marker", description: "Moves the playhead to the next location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }] }), Number(1))) +2026-06-30T11:11:53.699750Z DEBUG rmcp::service: new event evt=ToSink(Response(ListResourcesResult(ListResourcesResult { next_cursor: None, resources: [Annotated { raw: RawResource { uri: "ardour:/state/playback", name: "Ardour Playback State", description: Some("Current playback state of Ardour (e.g., Playing, Stopped, Unknown)."), mime_type: None, size: None }, annotations: None }, Annotated { raw: RawResource { uri: "ardour:/state/transport_frame", name: "Ardour Transport Frame Position", description: Some("Current playhead position in samples. Returns 'Unknown' if not yet reported by Ardour."), mime_type: None, size: None }, annotations: None }] }), Number(2))) +2026-06-30T11:16:32.624424Z  INFO ardour_mcp_server: +====================================================================== +NEW SERVER RUN: Tue, 30 Jun 2026 13:16:32 +0200 +====================================================================== +2026-06-30T11:16:32.624459Z  INFO ardour_mcp_server: Attempting to create OSC sender for Ardour at 127.0.0.1:3819 +2026-06-30T11:16:32.624487Z  INFO ardour_mcp_server: OSC sender created and connected to Ardour at 127.0.0.1:3819 +2026-06-30T11:16:32.624493Z  INFO ardour_mcp_server: Sending /set_surface to Ardour to enable OSC feedback. +2026-06-30T11:16:32.624498Z  INFO ardour_mcp_server: Targeting feedback port: 9099 +2026-06-30T11:16:32.624719Z  INFO ardour_mcp_server: /set_surface command sent successfully to Ardour. +2026-06-30T11:16:32.643475Z  INFO serve_inner: rmcp::service: Service initialized as server peer_info=InitializeRequestParam { protocol_version: ProtocolVersion("2025-11-25"), capabilities: ClientCapabilities { experimental: None, roots: Some(RootsCapabilities { list_changed: None }), sampling: None }, client_info: Implementation { name: "claude-code", version: "2.1.196" } } +2026-06-30T11:16:32.643538Z  INFO ardour_mcp_server: Ardour MCP server started and waiting for connections... +2026-06-30T11:16:32.643580Z  INFO ardour_mcp_server: Starting OSC listener for Ardour events on 127.0.0.1:9099 +2026-06-30T11:16:32.643591Z DEBUG rmcp::service: new event evt=PeerMessage(Request(ListToolsRequest(Request { method: ListToolsRequestMethod, params: None }), Number(1))) +2026-06-30T11:16:32.643607Z  INFO rmcp::service: received request id=1 request=ListToolsRequest(Request { method: ListToolsRequestMethod, params: None }) +2026-06-30T11:16:32.643631Z DEBUG rmcp::service: new event evt=PeerMessage(Request(ListResourcesRequest(Request { method: ListResourcesRequestMethod, params: None }), Number(2))) +2026-06-30T11:16:32.643635Z  INFO rmcp::service: received request id=2 request=ListResourcesRequest(Request { method: ListResourcesRequestMethod, params: None }) +2026-06-30T11:16:32.643638Z ERROR ardour_mcp_server: OSC listener task failed: Failed to bind Tokio UDP socket for OSC on 127.0.0.1:9099: Address already in use (os error 98) +2026-06-30T11:16:32.643653Z DEBUG ardour_mcp_server: Listing resources. Count: 2, Content: [Annotated { raw: RawResource { uri: "ardour:/state/playback", name: "Ardour Playback State", description: Some("Current playback state of Ardour (e.g., Playing, Stopped, Unknown)."), mime_type: None, size: None }, annotations: None }, Annotated { raw: RawResource { uri: "ardour:/state/transport_frame", name: "Ardour Transport Frame Position", description: Some("Current playhead position in samples. Returns 'Unknown' if not yet reported by Ardour."), mime_type: None, size: None }, annotations: None }] +2026-06-30T11:16:32.643677Z  INFO rmcp::service: response message id=2 result=ListResourcesResult(ListResourcesResult { next_cursor: None, resources: [Annotated { raw: RawResource { uri: "ardour:/state/playback", name: "Ardour Playback State", description: Some("Current playback state of Ardour (e.g., Playing, Stopped, Unknown)."), mime_type: None, size: None }, annotations: None }, Annotated { raw: RawResource { uri: "ardour:/state/transport_frame", name: "Ardour Transport Frame Position", description: Some("Current playhead position in samples. Returns 'Unknown' if not yet reported by Ardour."), mime_type: None, size: None }, annotations: None }] }) +2026-06-30T11:16:32.643690Z DEBUG rmcp::service: new event evt=ToSink(Response(ListResourcesResult(ListResourcesResult { next_cursor: None, resources: [Annotated { raw: RawResource { uri: "ardour:/state/playback", name: "Ardour Playback State", description: Some("Current playback state of Ardour (e.g., Playing, Stopped, Unknown)."), mime_type: None, size: None }, annotations: None }, Annotated { raw: RawResource { uri: "ardour:/state/transport_frame", name: "Ardour Transport Frame Position", description: Some("Current playhead position in samples. Returns 'Unknown' if not yet reported by Ardour."), mime_type: None, size: None }, annotations: None }] }), Number(2))) +2026-06-30T11:16:32.643810Z  INFO rmcp::service: response message id=1 result=ListToolsResult(ListToolsResult { next_cursor: None, tools: [Tool { name: "set_track_trim_db", description: "Sets the trim of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_db": Object {"description": String("The desired trim in dB. Valid range: -20.0 to 20.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_db")], "title": String("SetTrackTrimDBArgs"), "type": String("object")} }, Tool { name: "select_strip", description: "Selects a specific strip (track/bus) in Ardour.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus to select."), "format": String("int32"), "type": String("integer")}, "select_state": Object {"description": String("The desired select state (true to select). Currently, only true (1) is effective for selection."), "type": String("boolean")}}, "required": Array [String("rid"), String("select_state")], "title": String("SelectStripArgs"), "type": String("object")} }, Tool { name: "toggle_punch_in", description: "Toggles the Punch In state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_strip_plugin_active", description: "Activates or deactivates a plugin on a specific strip slot.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"active_state": Object {"description": String("The desired activation state (true for active, false for inactive)."), "type": String("boolean")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("active_state"), String("plugin_slot"), String("rid")], "title": String("SetStripPluginActiveArgs"), "type": String("object")} }, Tool { name: "undo", description: "Undoes the last action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "add_marker", description: "Adds a location marker at the current playhead position.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "prev_marker", description: "Moves the playhead to the previous location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_selected_strip_pan_stereo_width", description: "Sets the stereo width for the currently selected stereo strip's panner.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"width": Object {"description": String("The desired stereo width for the currently selected strip. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("width")], "title": String("SetSelectedStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "set_track_trim_abs", description: "Sets the absolute trim of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_abs": Object {"description": String("The desired absolute trim. Valid range: 0.1 to 10.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_abs")], "title": String("SetTrackTrimAbsArgs"), "type": String("object")} }, Tool { name: "transport_stop", description: "Stops Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_gain_db", description: "Sets the gain of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_db": Object {"description": String("The desired gain in dB. Valid range: -400.0 to 6.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_db"), String("rid")], "title": String("SetTrackGainDBArgs"), "type": String("object")} }, Tool { name: "toggle_all_rec_enables", description: "Toggles the record enable state for ALL tracks.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "rewind", description: "Rewinds the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "redo", description: "Redoes the last undone action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_solo", description: "Sets the solo state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "solo_st": Object {"description": String("The desired solo state. 0 for solo off, 1 for solo on."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rid"), String("solo_st")], "title": String("SetTrackSoloArgs"), "type": String("object")} }, Tool { name: "set_track_rec_enable", description: "Sets the record enable state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rec_st": Object {"description": String("The desired record enable state. 0 for off, 1 for on."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rec_st"), String("rid")], "title": String("SetTrackRecEnableArgs"), "type": String("object")} }, Tool { name: "loop_toggle", description: "Toggles loop playback mode.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "goto_end", description: "Moves the playhead to the session end.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "locate", description: "Locates the playhead to a specific sample position and optionally starts playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"roll": Object {"description": String("Whether to start playing after locating. 0 for stop, 1 for play."), "format": String("int32"), "type": String("integer")}, "spos": Object {"description": String("The position in samples to locate to."), "format": String("int64"), "type": String("integer")}}, "required": Array [String("roll"), String("spos")], "title": String("LocateToolArgs"), "type": String("object")} }, Tool { name: "save_state", description: "Saves the current session state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "access_action", description: "Executes a specified Ardour menu action by its name.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"action_name": Object {"description": String("The name of the Ardour menu action to execute (e.g., 'Editor/zoom-to-session')."), "type": String("string")}}, "required": Array [String("action_name")], "title": String("AccessActionArgs"), "type": String("object")} }, Tool { name: "set_strip_plugin_parameter", description: "Sets a specific parameter of a plugin on a strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"param_id": Object {"description": String("The 1-indexed ID of the parameter within the plugin."), "format": String("int32"), "type": String("integer")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "value": Object {"description": String("The desired parameter value, normalized (0.0 to 1.0)."), "format": String("float"), "type": String("number")}}, "required": Array [String("param_id"), String("plugin_slot"), String("rid"), String("value")], "title": String("SetStripPluginParameterArgs"), "type": String("object")} }, Tool { name: "next_marker", description: "Moves the playhead to the next location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_strip_pan_stereo_width", description: "Sets the stereo width for a panner on a stereo strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the stereo track/bus."), "format": String("int32"), "type": String("integer")}, "width": Object {"description": String("The desired stereo width. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("width")], "title": String("SetStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "toggle_punch_out", description: "Toggles the Punch Out state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "transport_play", description: "Starts Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "rec_enable_toggle", description: "Toggles the master record enable or selected track record enable.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "ffwd", description: "Fast forwards the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_gain_abs", description: "Sets the absolute gain of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_abs": Object {"description": String("The desired absolute gain. Valid range: 0.0 to 2.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_abs"), String("rid")], "title": String("SetTrackGainAbsArgs"), "type": String("object")} }, Tool { name: "set_track_mute", description: "Sets the mute state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"mute_state": Object {"description": String("The desired mute state (true for mute, false for unmute)."), "type": String("boolean")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("mute_state"), String("rid")], "title": String("SetTrackMuteArgs"), "type": String("object")} }, Tool { name: "goto_start", description: "Moves the playhead to the session start.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_transport_speed", description: "Sets Ardour's transport speed. Valid range: -8.0 to 8.0.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"speed": Object {"description": String("The desired transport speed. Valid range: -8.0 to 8.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("speed")], "title": String("SetTransportSpeedArgs"), "type": String("object")} }] }) +2026-06-30T11:16:32.643888Z DEBUG rmcp::service: new event evt=ToSink(Response(ListToolsResult(ListToolsResult { next_cursor: None, tools: [Tool { name: "set_track_trim_db", description: "Sets the trim of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_db": Object {"description": String("The desired trim in dB. Valid range: -20.0 to 20.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_db")], "title": String("SetTrackTrimDBArgs"), "type": String("object")} }, Tool { name: "select_strip", description: "Selects a specific strip (track/bus) in Ardour.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus to select."), "format": String("int32"), "type": String("integer")}, "select_state": Object {"description": String("The desired select state (true to select). Currently, only true (1) is effective for selection."), "type": String("boolean")}}, "required": Array [String("rid"), String("select_state")], "title": String("SelectStripArgs"), "type": String("object")} }, Tool { name: "toggle_punch_in", description: "Toggles the Punch In state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_strip_plugin_active", description: "Activates or deactivates a plugin on a specific strip slot.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"active_state": Object {"description": String("The desired activation state (true for active, false for inactive)."), "type": String("boolean")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("active_state"), String("plugin_slot"), String("rid")], "title": String("SetStripPluginActiveArgs"), "type": String("object")} }, Tool { name: "undo", description: "Undoes the last action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "add_marker", description: "Adds a location marker at the current playhead position.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "prev_marker", description: "Moves the playhead to the previous location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_selected_strip_pan_stereo_width", description: "Sets the stereo width for the currently selected stereo strip's panner.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"width": Object {"description": String("The desired stereo width for the currently selected strip. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("width")], "title": String("SetSelectedStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "set_track_trim_abs", description: "Sets the absolute trim of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_abs": Object {"description": String("The desired absolute trim. Valid range: 0.1 to 10.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_abs")], "title": String("SetTrackTrimAbsArgs"), "type": String("object")} }, Tool { name: "transport_stop", description: "Stops Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_gain_db", description: "Sets the gain of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_db": Object {"description": String("The desired gain in dB. Valid range: -400.0 to 6.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_db"), String("rid")], "title": String("SetTrackGainDBArgs"), "type": String("object")} }, Tool { name: "toggle_all_rec_enables", description: "Toggles the record enable state for ALL tracks.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "rewind", description: "Rewinds the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "redo", description: "Redoes the last undone action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_solo", description: "Sets the solo state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "solo_st": Object {"description": String("The desired solo state. 0 for solo off, 1 for solo on."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rid"), String("solo_st")], "title": String("SetTrackSoloArgs"), "type": String("object")} }, Tool { name: "set_track_rec_enable", description: "Sets the record enable state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rec_st": Object {"description": String("The desired record enable state. 0 for off, 1 for on."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rec_st"), String("rid")], "title": String("SetTrackRecEnableArgs"), "type": String("object")} }, Tool { name: "loop_toggle", description: "Toggles loop playback mode.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "goto_end", description: "Moves the playhead to the session end.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "locate", description: "Locates the playhead to a specific sample position and optionally starts playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"roll": Object {"description": String("Whether to start playing after locating. 0 for stop, 1 for play."), "format": String("int32"), "type": String("integer")}, "spos": Object {"description": String("The position in samples to locate to."), "format": String("int64"), "type": String("integer")}}, "required": Array [String("roll"), String("spos")], "title": String("LocateToolArgs"), "type": String("object")} }, Tool { name: "save_state", description: "Saves the current session state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "access_action", description: "Executes a specified Ardour menu action by its name.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"action_name": Object {"description": String("The name of the Ardour menu action to execute (e.g., 'Editor/zoom-to-session')."), "type": String("string")}}, "required": Array [String("action_name")], "title": String("AccessActionArgs"), "type": String("object")} }, Tool { name: "set_strip_plugin_parameter", description: "Sets a specific parameter of a plugin on a strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"param_id": Object {"description": String("The 1-indexed ID of the parameter within the plugin."), "format": String("int32"), "type": String("integer")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "value": Object {"description": String("The desired parameter value, normalized (0.0 to 1.0)."), "format": String("float"), "type": String("number")}}, "required": Array [String("param_id"), String("plugin_slot"), String("rid"), String("value")], "title": String("SetStripPluginParameterArgs"), "type": String("object")} }, Tool { name: "next_marker", description: "Moves the playhead to the next location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_strip_pan_stereo_width", description: "Sets the stereo width for a panner on a stereo strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the stereo track/bus."), "format": String("int32"), "type": String("integer")}, "width": Object {"description": String("The desired stereo width. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("width")], "title": String("SetStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "toggle_punch_out", description: "Toggles the Punch Out state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "transport_play", description: "Starts Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "rec_enable_toggle", description: "Toggles the master record enable or selected track record enable.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "ffwd", description: "Fast forwards the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_gain_abs", description: "Sets the absolute gain of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_abs": Object {"description": String("The desired absolute gain. Valid range: 0.0 to 2.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_abs"), String("rid")], "title": String("SetTrackGainAbsArgs"), "type": String("object")} }, Tool { name: "set_track_mute", description: "Sets the mute state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"mute_state": Object {"description": String("The desired mute state (true for mute, false for unmute)."), "type": String("boolean")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("mute_state"), String("rid")], "title": String("SetTrackMuteArgs"), "type": String("object")} }, Tool { name: "goto_start", description: "Moves the playhead to the session start.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_transport_speed", description: "Sets Ardour's transport speed. Valid range: -8.0 to 8.0.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"speed": Object {"description": String("The desired transport speed. Valid range: -8.0 to 8.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("speed")], "title": String("SetTransportSpeedArgs"), "type": String("object")} }] }), Number(1))) +2026-06-30T11:16:47.652312Z  INFO ardour_mcp_server: +====================================================================== +NEW SERVER RUN: Tue, 30 Jun 2026 13:16:47 +0200 +====================================================================== +2026-06-30T11:16:47.652351Z  INFO ardour_mcp_server: Attempting to create OSC sender for Ardour at 127.0.0.1:3819 +2026-06-30T11:16:47.652384Z  INFO ardour_mcp_server: OSC sender created and connected to Ardour at 127.0.0.1:3819 +2026-06-30T11:16:47.652389Z  INFO ardour_mcp_server: Sending /set_surface to Ardour to enable OSC feedback. +2026-06-30T11:16:47.652393Z  INFO ardour_mcp_server: Targeting feedback port: 9099 +2026-06-30T11:16:47.652531Z  INFO ardour_mcp_server: /set_surface command sent successfully to Ardour. +2026-06-30T11:16:47.736835Z  INFO serve_inner: rmcp::service: Service initialized as server peer_info=InitializeRequestParam { protocol_version: ProtocolVersion("2025-11-25"), capabilities: ClientCapabilities { experimental: None, roots: Some(RootsCapabilities { list_changed: None }), sampling: None }, client_info: Implementation { name: "claude-code", version: "2.1.196" } } +2026-06-30T11:16:47.736887Z  INFO ardour_mcp_server: Ardour MCP server started and waiting for connections... +2026-06-30T11:16:47.736917Z DEBUG rmcp::service: new event evt=PeerMessage(Request(ListToolsRequest(Request { method: ListToolsRequestMethod, params: None }), Number(1))) +2026-06-30T11:16:47.736924Z  INFO ardour_mcp_server: Starting OSC listener for Ardour events on 127.0.0.1:9099 +2026-06-30T11:16:47.736929Z  INFO rmcp::service: received request id=1 request=ListToolsRequest(Request { method: ListToolsRequestMethod, params: None }) +2026-06-30T11:16:47.736967Z DEBUG rmcp::service: new event evt=PeerMessage(Request(ListResourcesRequest(Request { method: ListResourcesRequestMethod, params: None }), Number(2))) +2026-06-30T11:16:47.736972Z  INFO rmcp::service: received request id=2 request=ListResourcesRequest(Request { method: ListResourcesRequestMethod, params: None }) +2026-06-30T11:16:47.736989Z ERROR ardour_mcp_server: OSC listener task failed: Failed to bind Tokio UDP socket for OSC on 127.0.0.1:9099: Address already in use (os error 98) +2026-06-30T11:16:47.736991Z DEBUG ardour_mcp_server: Listing resources. Count: 2, Content: [Annotated { raw: RawResource { uri: "ardour:/state/playback", name: "Ardour Playback State", description: Some("Current playback state of Ardour (e.g., Playing, Stopped, Unknown)."), mime_type: None, size: None }, annotations: None }, Annotated { raw: RawResource { uri: "ardour:/state/transport_frame", name: "Ardour Transport Frame Position", description: Some("Current playhead position in samples. Returns 'Unknown' if not yet reported by Ardour."), mime_type: None, size: None }, annotations: None }] +2026-06-30T11:16:47.737003Z  INFO rmcp::service: response message id=2 result=ListResourcesResult(ListResourcesResult { next_cursor: None, resources: [Annotated { raw: RawResource { uri: "ardour:/state/playback", name: "Ardour Playback State", description: Some("Current playback state of Ardour (e.g., Playing, Stopped, Unknown)."), mime_type: None, size: None }, annotations: None }, Annotated { raw: RawResource { uri: "ardour:/state/transport_frame", name: "Ardour Transport Frame Position", description: Some("Current playhead position in samples. Returns 'Unknown' if not yet reported by Ardour."), mime_type: None, size: None }, annotations: None }] }) +2026-06-30T11:16:47.737018Z DEBUG rmcp::service: new event evt=ToSink(Response(ListResourcesResult(ListResourcesResult { next_cursor: None, resources: [Annotated { raw: RawResource { uri: "ardour:/state/playback", name: "Ardour Playback State", description: Some("Current playback state of Ardour (e.g., Playing, Stopped, Unknown)."), mime_type: None, size: None }, annotations: None }, Annotated { raw: RawResource { uri: "ardour:/state/transport_frame", name: "Ardour Transport Frame Position", description: Some("Current playhead position in samples. Returns 'Unknown' if not yet reported by Ardour."), mime_type: None, size: None }, annotations: None }] }), Number(2))) +2026-06-30T11:16:47.737137Z  INFO rmcp::service: response message id=1 result=ListToolsResult(ListToolsResult { next_cursor: None, tools: [Tool { name: "goto_end", description: "Moves the playhead to the session end.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "rec_enable_toggle", description: "Toggles the master record enable or selected track record enable.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_strip_plugin_active", description: "Activates or deactivates a plugin on a specific strip slot.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"active_state": Object {"description": String("The desired activation state (true for active, false for inactive)."), "type": String("boolean")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("active_state"), String("plugin_slot"), String("rid")], "title": String("SetStripPluginActiveArgs"), "type": String("object")} }, Tool { name: "loop_toggle", description: "Toggles loop playback mode.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "add_marker", description: "Adds a location marker at the current playhead position.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "prev_marker", description: "Moves the playhead to the previous location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "save_state", description: "Saves the current session state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "locate", description: "Locates the playhead to a specific sample position and optionally starts playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"roll": Object {"description": String("Whether to start playing after locating. 0 for stop, 1 for play."), "format": String("int32"), "type": String("integer")}, "spos": Object {"description": String("The position in samples to locate to."), "format": String("int64"), "type": String("integer")}}, "required": Array [String("roll"), String("spos")], "title": String("LocateToolArgs"), "type": String("object")} }, Tool { name: "set_track_trim_db", description: "Sets the trim of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_db": Object {"description": String("The desired trim in dB. Valid range: -20.0 to 20.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_db")], "title": String("SetTrackTrimDBArgs"), "type": String("object")} }, Tool { name: "goto_start", description: "Moves the playhead to the session start.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "undo", description: "Undoes the last action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "next_marker", description: "Moves the playhead to the next location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "select_strip", description: "Selects a specific strip (track/bus) in Ardour.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus to select."), "format": String("int32"), "type": String("integer")}, "select_state": Object {"description": String("The desired select state (true to select). Currently, only true (1) is effective for selection."), "type": String("boolean")}}, "required": Array [String("rid"), String("select_state")], "title": String("SelectStripArgs"), "type": String("object")} }, Tool { name: "set_track_rec_enable", description: "Sets the record enable state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rec_st": Object {"description": String("The desired record enable state. 0 for off, 1 for on."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rec_st"), String("rid")], "title": String("SetTrackRecEnableArgs"), "type": String("object")} }, Tool { name: "transport_play", description: "Starts Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "toggle_punch_out", description: "Toggles the Punch Out state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_solo", description: "Sets the solo state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "solo_st": Object {"description": String("The desired solo state. 0 for solo off, 1 for solo on."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rid"), String("solo_st")], "title": String("SetTrackSoloArgs"), "type": String("object")} }, Tool { name: "access_action", description: "Executes a specified Ardour menu action by its name.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"action_name": Object {"description": String("The name of the Ardour menu action to execute (e.g., 'Editor/zoom-to-session')."), "type": String("string")}}, "required": Array [String("action_name")], "title": String("AccessActionArgs"), "type": String("object")} }, Tool { name: "toggle_all_rec_enables", description: "Toggles the record enable state for ALL tracks.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_selected_strip_pan_stereo_width", description: "Sets the stereo width for the currently selected stereo strip's panner.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"width": Object {"description": String("The desired stereo width for the currently selected strip. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("width")], "title": String("SetSelectedStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "redo", description: "Redoes the last undone action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "transport_stop", description: "Stops Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "toggle_punch_in", description: "Toggles the Punch In state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_gain_db", description: "Sets the gain of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_db": Object {"description": String("The desired gain in dB. Valid range: -400.0 to 6.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_db"), String("rid")], "title": String("SetTrackGainDBArgs"), "type": String("object")} }, Tool { name: "set_strip_pan_stereo_width", description: "Sets the stereo width for a panner on a stereo strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the stereo track/bus."), "format": String("int32"), "type": String("integer")}, "width": Object {"description": String("The desired stereo width. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("width")], "title": String("SetStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "ffwd", description: "Fast forwards the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_mute", description: "Sets the mute state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"mute_state": Object {"description": String("The desired mute state (true for mute, false for unmute)."), "type": String("boolean")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("mute_state"), String("rid")], "title": String("SetTrackMuteArgs"), "type": String("object")} }, Tool { name: "set_strip_plugin_parameter", description: "Sets a specific parameter of a plugin on a strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"param_id": Object {"description": String("The 1-indexed ID of the parameter within the plugin."), "format": String("int32"), "type": String("integer")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "value": Object {"description": String("The desired parameter value, normalized (0.0 to 1.0)."), "format": String("float"), "type": String("number")}}, "required": Array [String("param_id"), String("plugin_slot"), String("rid"), String("value")], "title": String("SetStripPluginParameterArgs"), "type": String("object")} }, Tool { name: "set_track_trim_abs", description: "Sets the absolute trim of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_abs": Object {"description": String("The desired absolute trim. Valid range: 0.1 to 10.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_abs")], "title": String("SetTrackTrimAbsArgs"), "type": String("object")} }, Tool { name: "set_transport_speed", description: "Sets Ardour's transport speed. Valid range: -8.0 to 8.0.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"speed": Object {"description": String("The desired transport speed. Valid range: -8.0 to 8.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("speed")], "title": String("SetTransportSpeedArgs"), "type": String("object")} }, Tool { name: "set_track_gain_abs", description: "Sets the absolute gain of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_abs": Object {"description": String("The desired absolute gain. Valid range: 0.0 to 2.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_abs"), String("rid")], "title": String("SetTrackGainAbsArgs"), "type": String("object")} }, Tool { name: "rewind", description: "Rewinds the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }] }) +2026-06-30T11:16:47.737200Z DEBUG rmcp::service: new event evt=ToSink(Response(ListToolsResult(ListToolsResult { next_cursor: None, tools: [Tool { name: "goto_end", description: "Moves the playhead to the session end.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "rec_enable_toggle", description: "Toggles the master record enable or selected track record enable.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_strip_plugin_active", description: "Activates or deactivates a plugin on a specific strip slot.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"active_state": Object {"description": String("The desired activation state (true for active, false for inactive)."), "type": String("boolean")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("active_state"), String("plugin_slot"), String("rid")], "title": String("SetStripPluginActiveArgs"), "type": String("object")} }, Tool { name: "loop_toggle", description: "Toggles loop playback mode.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "add_marker", description: "Adds a location marker at the current playhead position.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "prev_marker", description: "Moves the playhead to the previous location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "save_state", description: "Saves the current session state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "locate", description: "Locates the playhead to a specific sample position and optionally starts playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"roll": Object {"description": String("Whether to start playing after locating. 0 for stop, 1 for play."), "format": String("int32"), "type": String("integer")}, "spos": Object {"description": String("The position in samples to locate to."), "format": String("int64"), "type": String("integer")}}, "required": Array [String("roll"), String("spos")], "title": String("LocateToolArgs"), "type": String("object")} }, Tool { name: "set_track_trim_db", description: "Sets the trim of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_db": Object {"description": String("The desired trim in dB. Valid range: -20.0 to 20.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_db")], "title": String("SetTrackTrimDBArgs"), "type": String("object")} }, Tool { name: "goto_start", description: "Moves the playhead to the session start.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "undo", description: "Undoes the last action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "next_marker", description: "Moves the playhead to the next location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "select_strip", description: "Selects a specific strip (track/bus) in Ardour.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus to select."), "format": String("int32"), "type": String("integer")}, "select_state": Object {"description": String("The desired select state (true to select). Currently, only true (1) is effective for selection."), "type": String("boolean")}}, "required": Array [String("rid"), String("select_state")], "title": String("SelectStripArgs"), "type": String("object")} }, Tool { name: "set_track_rec_enable", description: "Sets the record enable state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rec_st": Object {"description": String("The desired record enable state. 0 for off, 1 for on."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rec_st"), String("rid")], "title": String("SetTrackRecEnableArgs"), "type": String("object")} }, Tool { name: "transport_play", description: "Starts Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "toggle_punch_out", description: "Toggles the Punch Out state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_solo", description: "Sets the solo state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "solo_st": Object {"description": String("The desired solo state. 0 for solo off, 1 for solo on."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rid"), String("solo_st")], "title": String("SetTrackSoloArgs"), "type": String("object")} }, Tool { name: "access_action", description: "Executes a specified Ardour menu action by its name.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"action_name": Object {"description": String("The name of the Ardour menu action to execute (e.g., 'Editor/zoom-to-session')."), "type": String("string")}}, "required": Array [String("action_name")], "title": String("AccessActionArgs"), "type": String("object")} }, Tool { name: "toggle_all_rec_enables", description: "Toggles the record enable state for ALL tracks.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_selected_strip_pan_stereo_width", description: "Sets the stereo width for the currently selected stereo strip's panner.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"width": Object {"description": String("The desired stereo width for the currently selected strip. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("width")], "title": String("SetSelectedStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "redo", description: "Redoes the last undone action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "transport_stop", description: "Stops Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "toggle_punch_in", description: "Toggles the Punch In state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_gain_db", description: "Sets the gain of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_db": Object {"description": String("The desired gain in dB. Valid range: -400.0 to 6.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_db"), String("rid")], "title": String("SetTrackGainDBArgs"), "type": String("object")} }, Tool { name: "set_strip_pan_stereo_width", description: "Sets the stereo width for a panner on a stereo strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the stereo track/bus."), "format": String("int32"), "type": String("integer")}, "width": Object {"description": String("The desired stereo width. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("width")], "title": String("SetStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "ffwd", description: "Fast forwards the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_mute", description: "Sets the mute state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"mute_state": Object {"description": String("The desired mute state (true for mute, false for unmute)."), "type": String("boolean")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("mute_state"), String("rid")], "title": String("SetTrackMuteArgs"), "type": String("object")} }, Tool { name: "set_strip_plugin_parameter", description: "Sets a specific parameter of a plugin on a strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"param_id": Object {"description": String("The 1-indexed ID of the parameter within the plugin."), "format": String("int32"), "type": String("integer")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "value": Object {"description": String("The desired parameter value, normalized (0.0 to 1.0)."), "format": String("float"), "type": String("number")}}, "required": Array [String("param_id"), String("plugin_slot"), String("rid"), String("value")], "title": String("SetStripPluginParameterArgs"), "type": String("object")} }, Tool { name: "set_track_trim_abs", description: "Sets the absolute trim of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_abs": Object {"description": String("The desired absolute trim. Valid range: 0.1 to 10.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_abs")], "title": String("SetTrackTrimAbsArgs"), "type": String("object")} }, Tool { name: "set_transport_speed", description: "Sets Ardour's transport speed. Valid range: -8.0 to 8.0.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"speed": Object {"description": String("The desired transport speed. Valid range: -8.0 to 8.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("speed")], "title": String("SetTransportSpeedArgs"), "type": String("object")} }, Tool { name: "set_track_gain_abs", description: "Sets the absolute gain of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_abs": Object {"description": String("The desired absolute gain. Valid range: 0.0 to 2.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_abs"), String("rid")], "title": String("SetTrackGainAbsArgs"), "type": String("object")} }, Tool { name: "rewind", description: "Rewinds the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }] }), Number(1))) +2026-06-30T11:48:57.451021Z  INFO ardour_mcp_server: +====================================================================== +NEW SERVER RUN: Tue, 30 Jun 2026 13:48:57 +0200 +====================================================================== +2026-06-30T11:48:57.451069Z  INFO ardour_mcp_server: Attempting to create OSC sender for Ardour at 127.0.0.1:3819 +2026-06-30T11:48:57.451099Z  INFO ardour_mcp_server: OSC sender created and connected to Ardour at 127.0.0.1:3819 +2026-06-30T11:48:57.451102Z  INFO ardour_mcp_server: Sending /set_surface to Ardour to enable OSC feedback. +2026-06-30T11:48:57.451106Z  INFO ardour_mcp_server: Targeting feedback port: 9099 +2026-06-30T11:48:57.451248Z  INFO ardour_mcp_server: /set_surface command sent successfully to Ardour. +2026-06-30T11:48:57.488115Z  INFO ardour_mcp_server: +====================================================================== +NEW SERVER RUN: Tue, 30 Jun 2026 13:48:57 +0200 +====================================================================== +2026-06-30T11:48:57.488173Z  INFO ardour_mcp_server: Attempting to create OSC sender for Ardour at 127.0.0.1:3819 +2026-06-30T11:48:57.488216Z  INFO ardour_mcp_server: OSC sender created and connected to Ardour at 127.0.0.1:3819 +2026-06-30T11:48:57.488224Z  INFO ardour_mcp_server: Sending /set_surface to Ardour to enable OSC feedback. +2026-06-30T11:48:57.488229Z  INFO ardour_mcp_server: Targeting feedback port: 9099 +2026-06-30T11:48:57.488449Z  INFO ardour_mcp_server: /set_surface command sent successfully to Ardour. +2026-06-30T11:48:57.504010Z  INFO serve_inner: rmcp::service: Service initialized as server peer_info=InitializeRequestParam { protocol_version: ProtocolVersion("2025-11-25"), capabilities: ClientCapabilities { experimental: None, roots: Some(RootsCapabilities { list_changed: None }), sampling: None }, client_info: Implementation { name: "claude-code", version: "2.1.196" } } +2026-06-30T11:48:57.504055Z  INFO ardour_mcp_server: Ardour MCP server started and waiting for connections... +2026-06-30T11:48:57.504077Z  INFO ardour_mcp_server: Starting OSC listener for Ardour events on 127.0.0.1:9099 +2026-06-30T11:48:57.504090Z DEBUG rmcp::service: new event evt=PeerMessage(Request(ListToolsRequest(Request { method: ListToolsRequestMethod, params: None }), Number(1))) +2026-06-30T11:48:57.504100Z  INFO rmcp::service: received request id=1 request=ListToolsRequest(Request { method: ListToolsRequestMethod, params: None }) +2026-06-30T11:48:57.504112Z DEBUG rmcp::service: new event evt=PeerMessage(Request(ListResourcesRequest(Request { method: ListResourcesRequestMethod, params: None }), Number(2))) +2026-06-30T11:48:57.504114Z  INFO rmcp::service: received request id=2 request=ListResourcesRequest(Request { method: ListResourcesRequestMethod, params: None }) +2026-06-30T11:48:57.504123Z ERROR ardour_mcp_server: OSC listener task failed: Failed to bind Tokio UDP socket for OSC on 127.0.0.1:9099: Address already in use (os error 98) +2026-06-30T11:48:57.504132Z DEBUG ardour_mcp_server: Listing resources. Count: 2, Content: [Annotated { raw: RawResource { uri: "ardour:/state/playback", name: "Ardour Playback State", description: Some("Current playback state of Ardour (e.g., Playing, Stopped, Unknown)."), mime_type: None, size: None }, annotations: None }, Annotated { raw: RawResource { uri: "ardour:/state/transport_frame", name: "Ardour Transport Frame Position", description: Some("Current playhead position in samples. Returns 'Unknown' if not yet reported by Ardour."), mime_type: None, size: None }, annotations: None }] +2026-06-30T11:48:57.504141Z  INFO rmcp::service: response message id=2 result=ListResourcesResult(ListResourcesResult { next_cursor: None, resources: [Annotated { raw: RawResource { uri: "ardour:/state/playback", name: "Ardour Playback State", description: Some("Current playback state of Ardour (e.g., Playing, Stopped, Unknown)."), mime_type: None, size: None }, annotations: None }, Annotated { raw: RawResource { uri: "ardour:/state/transport_frame", name: "Ardour Transport Frame Position", description: Some("Current playhead position in samples. Returns 'Unknown' if not yet reported by Ardour."), mime_type: None, size: None }, annotations: None }] }) +2026-06-30T11:48:57.504149Z DEBUG rmcp::service: new event evt=ToSink(Response(ListResourcesResult(ListResourcesResult { next_cursor: None, resources: [Annotated { raw: RawResource { uri: "ardour:/state/playback", name: "Ardour Playback State", description: Some("Current playback state of Ardour (e.g., Playing, Stopped, Unknown)."), mime_type: None, size: None }, annotations: None }, Annotated { raw: RawResource { uri: "ardour:/state/transport_frame", name: "Ardour Transport Frame Position", description: Some("Current playhead position in samples. Returns 'Unknown' if not yet reported by Ardour."), mime_type: None, size: None }, annotations: None }] }), Number(2))) +2026-06-30T11:48:57.504260Z  INFO rmcp::service: response message id=1 result=ListToolsResult(ListToolsResult { next_cursor: None, tools: [Tool { name: "set_track_solo", description: "Sets the solo state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "solo_st": Object {"description": String("The desired solo state. 0 for solo off, 1 for solo on."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rid"), String("solo_st")], "title": String("SetTrackSoloArgs"), "type": String("object")} }, Tool { name: "set_transport_speed", description: "Sets Ardour's transport speed. Valid range: -8.0 to 8.0.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"speed": Object {"description": String("The desired transport speed. Valid range: -8.0 to 8.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("speed")], "title": String("SetTransportSpeedArgs"), "type": String("object")} }, Tool { name: "toggle_all_rec_enables", description: "Toggles the record enable state for ALL tracks.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "toggle_punch_in", description: "Toggles the Punch In state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "select_strip", description: "Selects a specific strip (track/bus) in Ardour.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus to select."), "format": String("int32"), "type": String("integer")}, "select_state": Object {"description": String("The desired select state (true to select). Currently, only true (1) is effective for selection."), "type": String("boolean")}}, "required": Array [String("rid"), String("select_state")], "title": String("SelectStripArgs"), "type": String("object")} }, Tool { name: "set_track_trim_abs", description: "Sets the absolute trim of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_abs": Object {"description": String("The desired absolute trim. Valid range: 0.1 to 10.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_abs")], "title": String("SetTrackTrimAbsArgs"), "type": String("object")} }, Tool { name: "set_strip_plugin_parameter", description: "Sets a specific parameter of a plugin on a strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"param_id": Object {"description": String("The 1-indexed ID of the parameter within the plugin."), "format": String("int32"), "type": String("integer")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "value": Object {"description": String("The desired parameter value, normalized (0.0 to 1.0)."), "format": String("float"), "type": String("number")}}, "required": Array [String("param_id"), String("plugin_slot"), String("rid"), String("value")], "title": String("SetStripPluginParameterArgs"), "type": String("object")} }, Tool { name: "access_action", description: "Executes a specified Ardour menu action by its name.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"action_name": Object {"description": String("The name of the Ardour menu action to execute (e.g., 'Editor/zoom-to-session')."), "type": String("string")}}, "required": Array [String("action_name")], "title": String("AccessActionArgs"), "type": String("object")} }, Tool { name: "locate", description: "Locates the playhead to a specific sample position and optionally starts playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"roll": Object {"description": String("Whether to start playing after locating. 0 for stop, 1 for play."), "format": String("int32"), "type": String("integer")}, "spos": Object {"description": String("The position in samples to locate to."), "format": String("int64"), "type": String("integer")}}, "required": Array [String("roll"), String("spos")], "title": String("LocateToolArgs"), "type": String("object")} }, Tool { name: "rewind", description: "Rewinds the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_trim_db", description: "Sets the trim of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_db": Object {"description": String("The desired trim in dB. Valid range: -20.0 to 20.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_db")], "title": String("SetTrackTrimDBArgs"), "type": String("object")} }, Tool { name: "set_track_gain_abs", description: "Sets the absolute gain of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_abs": Object {"description": String("The desired absolute gain. Valid range: 0.0 to 2.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_abs"), String("rid")], "title": String("SetTrackGainAbsArgs"), "type": String("object")} }, Tool { name: "ffwd", description: "Fast forwards the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_mute", description: "Sets the mute state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"mute_state": Object {"description": String("The desired mute state (true for mute, false for unmute)."), "type": String("boolean")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("mute_state"), String("rid")], "title": String("SetTrackMuteArgs"), "type": String("object")} }, Tool { name: "set_strip_pan_stereo_width", description: "Sets the stereo width for a panner on a stereo strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the stereo track/bus."), "format": String("int32"), "type": String("integer")}, "width": Object {"description": String("The desired stereo width. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("width")], "title": String("SetStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "rec_enable_toggle", description: "Toggles the master record enable or selected track record enable.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "add_marker", description: "Adds a location marker at the current playhead position.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "next_marker", description: "Moves the playhead to the next location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_gain_db", description: "Sets the gain of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_db": Object {"description": String("The desired gain in dB. Valid range: -400.0 to 6.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_db"), String("rid")], "title": String("SetTrackGainDBArgs"), "type": String("object")} }, Tool { name: "save_state", description: "Saves the current session state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "transport_play", description: "Starts Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_selected_strip_pan_stereo_width", description: "Sets the stereo width for the currently selected stereo strip's panner.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"width": Object {"description": String("The desired stereo width for the currently selected strip. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("width")], "title": String("SetSelectedStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "transport_stop", description: "Stops Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_rec_enable", description: "Sets the record enable state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rec_st": Object {"description": String("The desired record enable state. 0 for off, 1 for on."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rec_st"), String("rid")], "title": String("SetTrackRecEnableArgs"), "type": String("object")} }, Tool { name: "undo", description: "Undoes the last action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "goto_start", description: "Moves the playhead to the session start.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_strip_plugin_active", description: "Activates or deactivates a plugin on a specific strip slot.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"active_state": Object {"description": String("The desired activation state (true for active, false for inactive)."), "type": String("boolean")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("active_state"), String("plugin_slot"), String("rid")], "title": String("SetStripPluginActiveArgs"), "type": String("object")} }, Tool { name: "redo", description: "Redoes the last undone action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "loop_toggle", description: "Toggles loop playback mode.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "toggle_punch_out", description: "Toggles the Punch Out state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "goto_end", description: "Moves the playhead to the session end.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "prev_marker", description: "Moves the playhead to the previous location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }] }) +2026-06-30T11:48:57.504331Z DEBUG rmcp::service: new event evt=ToSink(Response(ListToolsResult(ListToolsResult { next_cursor: None, tools: [Tool { name: "set_track_solo", description: "Sets the solo state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "solo_st": Object {"description": String("The desired solo state. 0 for solo off, 1 for solo on."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rid"), String("solo_st")], "title": String("SetTrackSoloArgs"), "type": String("object")} }, Tool { name: "set_transport_speed", description: "Sets Ardour's transport speed. Valid range: -8.0 to 8.0.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"speed": Object {"description": String("The desired transport speed. Valid range: -8.0 to 8.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("speed")], "title": String("SetTransportSpeedArgs"), "type": String("object")} }, Tool { name: "toggle_all_rec_enables", description: "Toggles the record enable state for ALL tracks.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "toggle_punch_in", description: "Toggles the Punch In state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "select_strip", description: "Selects a specific strip (track/bus) in Ardour.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus to select."), "format": String("int32"), "type": String("integer")}, "select_state": Object {"description": String("The desired select state (true to select). Currently, only true (1) is effective for selection."), "type": String("boolean")}}, "required": Array [String("rid"), String("select_state")], "title": String("SelectStripArgs"), "type": String("object")} }, Tool { name: "set_track_trim_abs", description: "Sets the absolute trim of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_abs": Object {"description": String("The desired absolute trim. Valid range: 0.1 to 10.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_abs")], "title": String("SetTrackTrimAbsArgs"), "type": String("object")} }, Tool { name: "set_strip_plugin_parameter", description: "Sets a specific parameter of a plugin on a strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"param_id": Object {"description": String("The 1-indexed ID of the parameter within the plugin."), "format": String("int32"), "type": String("integer")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "value": Object {"description": String("The desired parameter value, normalized (0.0 to 1.0)."), "format": String("float"), "type": String("number")}}, "required": Array [String("param_id"), String("plugin_slot"), String("rid"), String("value")], "title": String("SetStripPluginParameterArgs"), "type": String("object")} }, Tool { name: "access_action", description: "Executes a specified Ardour menu action by its name.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"action_name": Object {"description": String("The name of the Ardour menu action to execute (e.g., 'Editor/zoom-to-session')."), "type": String("string")}}, "required": Array [String("action_name")], "title": String("AccessActionArgs"), "type": String("object")} }, Tool { name: "locate", description: "Locates the playhead to a specific sample position and optionally starts playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"roll": Object {"description": String("Whether to start playing after locating. 0 for stop, 1 for play."), "format": String("int32"), "type": String("integer")}, "spos": Object {"description": String("The position in samples to locate to."), "format": String("int64"), "type": String("integer")}}, "required": Array [String("roll"), String("spos")], "title": String("LocateToolArgs"), "type": String("object")} }, Tool { name: "rewind", description: "Rewinds the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_trim_db", description: "Sets the trim of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_db": Object {"description": String("The desired trim in dB. Valid range: -20.0 to 20.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_db")], "title": String("SetTrackTrimDBArgs"), "type": String("object")} }, Tool { name: "set_track_gain_abs", description: "Sets the absolute gain of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_abs": Object {"description": String("The desired absolute gain. Valid range: 0.0 to 2.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_abs"), String("rid")], "title": String("SetTrackGainAbsArgs"), "type": String("object")} }, Tool { name: "ffwd", description: "Fast forwards the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_mute", description: "Sets the mute state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"mute_state": Object {"description": String("The desired mute state (true for mute, false for unmute)."), "type": String("boolean")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("mute_state"), String("rid")], "title": String("SetTrackMuteArgs"), "type": String("object")} }, Tool { name: "set_strip_pan_stereo_width", description: "Sets the stereo width for a panner on a stereo strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the stereo track/bus."), "format": String("int32"), "type": String("integer")}, "width": Object {"description": String("The desired stereo width. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("width")], "title": String("SetStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "rec_enable_toggle", description: "Toggles the master record enable or selected track record enable.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "add_marker", description: "Adds a location marker at the current playhead position.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "next_marker", description: "Moves the playhead to the next location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_gain_db", description: "Sets the gain of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_db": Object {"description": String("The desired gain in dB. Valid range: -400.0 to 6.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_db"), String("rid")], "title": String("SetTrackGainDBArgs"), "type": String("object")} }, Tool { name: "save_state", description: "Saves the current session state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "transport_play", description: "Starts Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_selected_strip_pan_stereo_width", description: "Sets the stereo width for the currently selected stereo strip's panner.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"width": Object {"description": String("The desired stereo width for the currently selected strip. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("width")], "title": String("SetSelectedStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "transport_stop", description: "Stops Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_rec_enable", description: "Sets the record enable state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rec_st": Object {"description": String("The desired record enable state. 0 for off, 1 for on."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rec_st"), String("rid")], "title": String("SetTrackRecEnableArgs"), "type": String("object")} }, Tool { name: "undo", description: "Undoes the last action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "goto_start", description: "Moves the playhead to the session start.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_strip_plugin_active", description: "Activates or deactivates a plugin on a specific strip slot.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"active_state": Object {"description": String("The desired activation state (true for active, false for inactive)."), "type": String("boolean")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("active_state"), String("plugin_slot"), String("rid")], "title": String("SetStripPluginActiveArgs"), "type": String("object")} }, Tool { name: "redo", description: "Redoes the last undone action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "loop_toggle", description: "Toggles loop playback mode.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "toggle_punch_out", description: "Toggles the Punch Out state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "goto_end", description: "Moves the playhead to the session end.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "prev_marker", description: "Moves the playhead to the previous location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }] }), Number(1))) +2026-06-30T11:48:57.569462Z  INFO serve_inner: rmcp::service: Service initialized as server peer_info=InitializeRequestParam { protocol_version: ProtocolVersion("2025-11-25"), capabilities: ClientCapabilities { experimental: None, roots: Some(RootsCapabilities { list_changed: None }), sampling: None }, client_info: Implementation { name: "claude-code", version: "2.1.196" } } +2026-06-30T11:48:57.569542Z  INFO ardour_mcp_server: Ardour MCP server started and waiting for connections... +2026-06-30T11:48:57.569600Z DEBUG rmcp::service: new event evt=PeerMessage(Request(ListToolsRequest(Request { method: ListToolsRequestMethod, params: None }), Number(1))) +2026-06-30T11:48:57.569613Z  INFO ardour_mcp_server: Starting OSC listener for Ardour events on 127.0.0.1:9099 +2026-06-30T11:48:57.569615Z  INFO rmcp::service: received request id=1 request=ListToolsRequest(Request { method: ListToolsRequestMethod, params: None }) +2026-06-30T11:48:57.569647Z DEBUG rmcp::service: new event evt=PeerMessage(Request(ListResourcesRequest(Request { method: ListResourcesRequestMethod, params: None }), Number(2))) +2026-06-30T11:48:57.569649Z  INFO rmcp::service: received request id=2 request=ListResourcesRequest(Request { method: ListResourcesRequestMethod, params: None }) +2026-06-30T11:48:57.569685Z ERROR ardour_mcp_server: OSC listener task failed: Failed to bind Tokio UDP socket for OSC on 127.0.0.1:9099: Address already in use (os error 98) +2026-06-30T11:48:57.569701Z DEBUG ardour_mcp_server: Listing resources. Count: 2, Content: [Annotated { raw: RawResource { uri: "ardour:/state/playback", name: "Ardour Playback State", description: Some("Current playback state of Ardour (e.g., Playing, Stopped, Unknown)."), mime_type: None, size: None }, annotations: None }, Annotated { raw: RawResource { uri: "ardour:/state/transport_frame", name: "Ardour Transport Frame Position", description: Some("Current playhead position in samples. Returns 'Unknown' if not yet reported by Ardour."), mime_type: None, size: None }, annotations: None }] +2026-06-30T11:48:57.569711Z  INFO rmcp::service: response message id=2 result=ListResourcesResult(ListResourcesResult { next_cursor: None, resources: [Annotated { raw: RawResource { uri: "ardour:/state/playback", name: "Ardour Playback State", description: Some("Current playback state of Ardour (e.g., Playing, Stopped, Unknown)."), mime_type: None, size: None }, annotations: None }, Annotated { raw: RawResource { uri: "ardour:/state/transport_frame", name: "Ardour Transport Frame Position", description: Some("Current playhead position in samples. Returns 'Unknown' if not yet reported by Ardour."), mime_type: None, size: None }, annotations: None }] }) +2026-06-30T11:48:57.569723Z DEBUG rmcp::service: new event evt=ToSink(Response(ListResourcesResult(ListResourcesResult { next_cursor: None, resources: [Annotated { raw: RawResource { uri: "ardour:/state/playback", name: "Ardour Playback State", description: Some("Current playback state of Ardour (e.g., Playing, Stopped, Unknown)."), mime_type: None, size: None }, annotations: None }, Annotated { raw: RawResource { uri: "ardour:/state/transport_frame", name: "Ardour Transport Frame Position", description: Some("Current playhead position in samples. Returns 'Unknown' if not yet reported by Ardour."), mime_type: None, size: None }, annotations: None }] }), Number(2))) +2026-06-30T11:48:57.569863Z  INFO rmcp::service: response message id=1 result=ListToolsResult(ListToolsResult { next_cursor: None, tools: [Tool { name: "set_transport_speed", description: "Sets Ardour's transport speed. Valid range: -8.0 to 8.0.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"speed": Object {"description": String("The desired transport speed. Valid range: -8.0 to 8.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("speed")], "title": String("SetTransportSpeedArgs"), "type": String("object")} }, Tool { name: "select_strip", description: "Selects a specific strip (track/bus) in Ardour.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus to select."), "format": String("int32"), "type": String("integer")}, "select_state": Object {"description": String("The desired select state (true to select). Currently, only true (1) is effective for selection."), "type": String("boolean")}}, "required": Array [String("rid"), String("select_state")], "title": String("SelectStripArgs"), "type": String("object")} }, Tool { name: "set_strip_plugin_active", description: "Activates or deactivates a plugin on a specific strip slot.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"active_state": Object {"description": String("The desired activation state (true for active, false for inactive)."), "type": String("boolean")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("active_state"), String("plugin_slot"), String("rid")], "title": String("SetStripPluginActiveArgs"), "type": String("object")} }, Tool { name: "set_strip_pan_stereo_width", description: "Sets the stereo width for a panner on a stereo strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the stereo track/bus."), "format": String("int32"), "type": String("integer")}, "width": Object {"description": String("The desired stereo width. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("width")], "title": String("SetStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "goto_end", description: "Moves the playhead to the session end.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_trim_db", description: "Sets the trim of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_db": Object {"description": String("The desired trim in dB. Valid range: -20.0 to 20.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_db")], "title": String("SetTrackTrimDBArgs"), "type": String("object")} }, Tool { name: "set_strip_plugin_parameter", description: "Sets a specific parameter of a plugin on a strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"param_id": Object {"description": String("The 1-indexed ID of the parameter within the plugin."), "format": String("int32"), "type": String("integer")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "value": Object {"description": String("The desired parameter value, normalized (0.0 to 1.0)."), "format": String("float"), "type": String("number")}}, "required": Array [String("param_id"), String("plugin_slot"), String("rid"), String("value")], "title": String("SetStripPluginParameterArgs"), "type": String("object")} }, Tool { name: "redo", description: "Redoes the last undone action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "access_action", description: "Executes a specified Ardour menu action by its name.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"action_name": Object {"description": String("The name of the Ardour menu action to execute (e.g., 'Editor/zoom-to-session')."), "type": String("string")}}, "required": Array [String("action_name")], "title": String("AccessActionArgs"), "type": String("object")} }, Tool { name: "rewind", description: "Rewinds the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_mute", description: "Sets the mute state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"mute_state": Object {"description": String("The desired mute state (true for mute, false for unmute)."), "type": String("boolean")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("mute_state"), String("rid")], "title": String("SetTrackMuteArgs"), "type": String("object")} }, Tool { name: "loop_toggle", description: "Toggles loop playback mode.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "rec_enable_toggle", description: "Toggles the master record enable or selected track record enable.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "add_marker", description: "Adds a location marker at the current playhead position.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "transport_play", description: "Starts Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "locate", description: "Locates the playhead to a specific sample position and optionally starts playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"roll": Object {"description": String("Whether to start playing after locating. 0 for stop, 1 for play."), "format": String("int32"), "type": String("integer")}, "spos": Object {"description": String("The position in samples to locate to."), "format": String("int64"), "type": String("integer")}}, "required": Array [String("roll"), String("spos")], "title": String("LocateToolArgs"), "type": String("object")} }, Tool { name: "toggle_punch_in", description: "Toggles the Punch In state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_gain_db", description: "Sets the gain of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_db": Object {"description": String("The desired gain in dB. Valid range: -400.0 to 6.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_db"), String("rid")], "title": String("SetTrackGainDBArgs"), "type": String("object")} }, Tool { name: "undo", description: "Undoes the last action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "toggle_punch_out", description: "Toggles the Punch Out state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "next_marker", description: "Moves the playhead to the next location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "goto_start", description: "Moves the playhead to the session start.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "prev_marker", description: "Moves the playhead to the previous location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "transport_stop", description: "Stops Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_solo", description: "Sets the solo state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "solo_st": Object {"description": String("The desired solo state. 0 for solo off, 1 for solo on."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rid"), String("solo_st")], "title": String("SetTrackSoloArgs"), "type": String("object")} }, Tool { name: "toggle_all_rec_enables", description: "Toggles the record enable state for ALL tracks.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_rec_enable", description: "Sets the record enable state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rec_st": Object {"description": String("The desired record enable state. 0 for off, 1 for on."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rec_st"), String("rid")], "title": String("SetTrackRecEnableArgs"), "type": String("object")} }, Tool { name: "set_selected_strip_pan_stereo_width", description: "Sets the stereo width for the currently selected stereo strip's panner.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"width": Object {"description": String("The desired stereo width for the currently selected strip. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("width")], "title": String("SetSelectedStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "set_track_gain_abs", description: "Sets the absolute gain of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_abs": Object {"description": String("The desired absolute gain. Valid range: 0.0 to 2.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_abs"), String("rid")], "title": String("SetTrackGainAbsArgs"), "type": String("object")} }, Tool { name: "ffwd", description: "Fast forwards the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_trim_abs", description: "Sets the absolute trim of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_abs": Object {"description": String("The desired absolute trim. Valid range: 0.1 to 10.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_abs")], "title": String("SetTrackTrimAbsArgs"), "type": String("object")} }, Tool { name: "save_state", description: "Saves the current session state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }] }) +2026-06-30T11:48:57.569953Z DEBUG rmcp::service: new event evt=ToSink(Response(ListToolsResult(ListToolsResult { next_cursor: None, tools: [Tool { name: "set_transport_speed", description: "Sets Ardour's transport speed. Valid range: -8.0 to 8.0.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"speed": Object {"description": String("The desired transport speed. Valid range: -8.0 to 8.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("speed")], "title": String("SetTransportSpeedArgs"), "type": String("object")} }, Tool { name: "select_strip", description: "Selects a specific strip (track/bus) in Ardour.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus to select."), "format": String("int32"), "type": String("integer")}, "select_state": Object {"description": String("The desired select state (true to select). Currently, only true (1) is effective for selection."), "type": String("boolean")}}, "required": Array [String("rid"), String("select_state")], "title": String("SelectStripArgs"), "type": String("object")} }, Tool { name: "set_strip_plugin_active", description: "Activates or deactivates a plugin on a specific strip slot.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"active_state": Object {"description": String("The desired activation state (true for active, false for inactive)."), "type": String("boolean")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("active_state"), String("plugin_slot"), String("rid")], "title": String("SetStripPluginActiveArgs"), "type": String("object")} }, Tool { name: "set_strip_pan_stereo_width", description: "Sets the stereo width for a panner on a stereo strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the stereo track/bus."), "format": String("int32"), "type": String("integer")}, "width": Object {"description": String("The desired stereo width. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("width")], "title": String("SetStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "goto_end", description: "Moves the playhead to the session end.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_trim_db", description: "Sets the trim of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_db": Object {"description": String("The desired trim in dB. Valid range: -20.0 to 20.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_db")], "title": String("SetTrackTrimDBArgs"), "type": String("object")} }, Tool { name: "set_strip_plugin_parameter", description: "Sets a specific parameter of a plugin on a strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"param_id": Object {"description": String("The 1-indexed ID of the parameter within the plugin."), "format": String("int32"), "type": String("integer")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "value": Object {"description": String("The desired parameter value, normalized (0.0 to 1.0)."), "format": String("float"), "type": String("number")}}, "required": Array [String("param_id"), String("plugin_slot"), String("rid"), String("value")], "title": String("SetStripPluginParameterArgs"), "type": String("object")} }, Tool { name: "redo", description: "Redoes the last undone action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "access_action", description: "Executes a specified Ardour menu action by its name.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"action_name": Object {"description": String("The name of the Ardour menu action to execute (e.g., 'Editor/zoom-to-session')."), "type": String("string")}}, "required": Array [String("action_name")], "title": String("AccessActionArgs"), "type": String("object")} }, Tool { name: "rewind", description: "Rewinds the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_mute", description: "Sets the mute state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"mute_state": Object {"description": String("The desired mute state (true for mute, false for unmute)."), "type": String("boolean")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("mute_state"), String("rid")], "title": String("SetTrackMuteArgs"), "type": String("object")} }, Tool { name: "loop_toggle", description: "Toggles loop playback mode.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "rec_enable_toggle", description: "Toggles the master record enable or selected track record enable.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "add_marker", description: "Adds a location marker at the current playhead position.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "transport_play", description: "Starts Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "locate", description: "Locates the playhead to a specific sample position and optionally starts playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"roll": Object {"description": String("Whether to start playing after locating. 0 for stop, 1 for play."), "format": String("int32"), "type": String("integer")}, "spos": Object {"description": String("The position in samples to locate to."), "format": String("int64"), "type": String("integer")}}, "required": Array [String("roll"), String("spos")], "title": String("LocateToolArgs"), "type": String("object")} }, Tool { name: "toggle_punch_in", description: "Toggles the Punch In state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_gain_db", description: "Sets the gain of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_db": Object {"description": String("The desired gain in dB. Valid range: -400.0 to 6.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_db"), String("rid")], "title": String("SetTrackGainDBArgs"), "type": String("object")} }, Tool { name: "undo", description: "Undoes the last action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "toggle_punch_out", description: "Toggles the Punch Out state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "next_marker", description: "Moves the playhead to the next location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "goto_start", description: "Moves the playhead to the session start.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "prev_marker", description: "Moves the playhead to the previous location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "transport_stop", description: "Stops Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_solo", description: "Sets the solo state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "solo_st": Object {"description": String("The desired solo state. 0 for solo off, 1 for solo on."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rid"), String("solo_st")], "title": String("SetTrackSoloArgs"), "type": String("object")} }, Tool { name: "toggle_all_rec_enables", description: "Toggles the record enable state for ALL tracks.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_rec_enable", description: "Sets the record enable state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rec_st": Object {"description": String("The desired record enable state. 0 for off, 1 for on."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rec_st"), String("rid")], "title": String("SetTrackRecEnableArgs"), "type": String("object")} }, Tool { name: "set_selected_strip_pan_stereo_width", description: "Sets the stereo width for the currently selected stereo strip's panner.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"width": Object {"description": String("The desired stereo width for the currently selected strip. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("width")], "title": String("SetSelectedStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "set_track_gain_abs", description: "Sets the absolute gain of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_abs": Object {"description": String("The desired absolute gain. Valid range: 0.0 to 2.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_abs"), String("rid")], "title": String("SetTrackGainAbsArgs"), "type": String("object")} }, Tool { name: "ffwd", description: "Fast forwards the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_trim_abs", description: "Sets the absolute trim of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_abs": Object {"description": String("The desired absolute trim. Valid range: 0.1 to 10.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_abs")], "title": String("SetTrackTrimAbsArgs"), "type": String("object")} }, Tool { name: "save_state", description: "Saves the current session state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }] }), Number(1))) +2026-06-30T12:08:53.000951Z  INFO ardour_mcp_server: +====================================================================== +NEW SERVER RUN: Tue, 30 Jun 2026 14:08:53 +0200 +====================================================================== +2026-06-30T12:08:53.000996Z  INFO ardour_mcp_server: Attempting to create OSC sender for Ardour at 127.0.0.1:3819 +2026-06-30T12:08:53.001034Z  INFO ardour_mcp_server: OSC sender created and connected to Ardour at 127.0.0.1:3819 +2026-06-30T12:08:53.001038Z  INFO ardour_mcp_server: Sending /set_surface to Ardour to enable OSC feedback. +2026-06-30T12:08:53.001041Z  INFO ardour_mcp_server: Targeting feedback port: 9099 +2026-06-30T12:08:53.001664Z  INFO ardour_mcp_server: /set_surface command sent successfully to Ardour. +2026-06-30T12:08:53.089098Z  INFO serve_inner: rmcp::service: Service initialized as server peer_info=InitializeRequestParam { protocol_version: ProtocolVersion("2025-11-25"), capabilities: ClientCapabilities { experimental: None, roots: Some(RootsCapabilities { list_changed: None }), sampling: None }, client_info: Implementation { name: "claude-code", version: "2.1.196" } } +2026-06-30T12:08:53.089145Z  INFO ardour_mcp_server: Ardour MCP server started and waiting for connections... +2026-06-30T12:08:53.089173Z DEBUG rmcp::service: new event evt=PeerMessage(Request(ListToolsRequest(Request { method: ListToolsRequestMethod, params: None }), Number(1))) +2026-06-30T12:08:53.089184Z  INFO ardour_mcp_server: Starting OSC listener for Ardour events on 127.0.0.1:9099 +2026-06-30T12:08:53.089195Z  INFO rmcp::service: received request id=1 request=ListToolsRequest(Request { method: ListToolsRequestMethod, params: None }) +2026-06-30T12:08:53.089207Z DEBUG rmcp::service: new event evt=PeerMessage(Request(ListResourcesRequest(Request { method: ListResourcesRequestMethod, params: None }), Number(2))) +2026-06-30T12:08:53.089208Z  INFO rmcp::service: received request id=2 request=ListResourcesRequest(Request { method: ListResourcesRequestMethod, params: None }) +2026-06-30T12:08:53.089219Z DEBUG ardour_mcp_server: Listing resources. Count: 2, Content: [Annotated { raw: RawResource { uri: "ardour:/state/playback", name: "Ardour Playback State", description: Some("Current playback state of Ardour (e.g., Playing, Stopped, Unknown)."), mime_type: None, size: None }, annotations: None }, Annotated { raw: RawResource { uri: "ardour:/state/transport_frame", name: "Ardour Transport Frame Position", description: Some("Current playhead position in samples. Returns 'Unknown' if not yet reported by Ardour."), mime_type: None, size: None }, annotations: None }] +2026-06-30T12:08:53.089225Z  INFO rmcp::service: response message id=2 result=ListResourcesResult(ListResourcesResult { next_cursor: None, resources: [Annotated { raw: RawResource { uri: "ardour:/state/playback", name: "Ardour Playback State", description: Some("Current playback state of Ardour (e.g., Playing, Stopped, Unknown)."), mime_type: None, size: None }, annotations: None }, Annotated { raw: RawResource { uri: "ardour:/state/transport_frame", name: "Ardour Transport Frame Position", description: Some("Current playhead position in samples. Returns 'Unknown' if not yet reported by Ardour."), mime_type: None, size: None }, annotations: None }] }) +2026-06-30T12:08:53.089233Z DEBUG rmcp::service: new event evt=ToSink(Response(ListResourcesResult(ListResourcesResult { next_cursor: None, resources: [Annotated { raw: RawResource { uri: "ardour:/state/playback", name: "Ardour Playback State", description: Some("Current playback state of Ardour (e.g., Playing, Stopped, Unknown)."), mime_type: None, size: None }, annotations: None }, Annotated { raw: RawResource { uri: "ardour:/state/transport_frame", name: "Ardour Transport Frame Position", description: Some("Current playhead position in samples. Returns 'Unknown' if not yet reported by Ardour."), mime_type: None, size: None }, annotations: None }] }), Number(2))) +2026-06-30T12:08:53.089239Z ERROR ardour_mcp_server: OSC listener task failed: Failed to bind Tokio UDP socket for OSC on 127.0.0.1:9099: Address already in use (os error 98) +2026-06-30T12:08:53.089358Z  INFO rmcp::service: response message id=1 result=ListToolsResult(ListToolsResult { next_cursor: None, tools: [Tool { name: "toggle_punch_in", description: "Toggles the Punch In state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "add_marker", description: "Adds a location marker at the current playhead position.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "ffwd", description: "Fast forwards the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_strip_pan_stereo_width", description: "Sets the stereo width for a panner on a stereo strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the stereo track/bus."), "format": String("int32"), "type": String("integer")}, "width": Object {"description": String("The desired stereo width. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("width")], "title": String("SetStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "undo", description: "Undoes the last action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "transport_play", description: "Starts Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_solo", description: "Sets the solo state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "solo_st": Object {"description": String("The desired solo state. 0 for solo off, 1 for solo on."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rid"), String("solo_st")], "title": String("SetTrackSoloArgs"), "type": String("object")} }, Tool { name: "locate", description: "Locates the playhead to a specific sample position and optionally starts playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"roll": Object {"description": String("Whether to start playing after locating. 0 for stop, 1 for play."), "format": String("int32"), "type": String("integer")}, "spos": Object {"description": String("The position in samples to locate to."), "format": String("int64"), "type": String("integer")}}, "required": Array [String("roll"), String("spos")], "title": String("LocateToolArgs"), "type": String("object")} }, Tool { name: "transport_stop", description: "Stops Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "next_marker", description: "Moves the playhead to the next location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_transport_speed", description: "Sets Ardour's transport speed. Valid range: -8.0 to 8.0.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"speed": Object {"description": String("The desired transport speed. Valid range: -8.0 to 8.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("speed")], "title": String("SetTransportSpeedArgs"), "type": String("object")} }, Tool { name: "set_selected_strip_pan_stereo_width", description: "Sets the stereo width for the currently selected stereo strip's panner.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"width": Object {"description": String("The desired stereo width for the currently selected strip. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("width")], "title": String("SetSelectedStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "access_action", description: "Executes a specified Ardour menu action by its name.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"action_name": Object {"description": String("The name of the Ardour menu action to execute (e.g., 'Editor/zoom-to-session')."), "type": String("string")}}, "required": Array [String("action_name")], "title": String("AccessActionArgs"), "type": String("object")} }, Tool { name: "toggle_punch_out", description: "Toggles the Punch Out state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_trim_db", description: "Sets the trim of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_db": Object {"description": String("The desired trim in dB. Valid range: -20.0 to 20.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_db")], "title": String("SetTrackTrimDBArgs"), "type": String("object")} }, Tool { name: "select_strip", description: "Selects a specific strip (track/bus) in Ardour.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus to select."), "format": String("int32"), "type": String("integer")}, "select_state": Object {"description": String("The desired select state (true to select). Currently, only true (1) is effective for selection."), "type": String("boolean")}}, "required": Array [String("rid"), String("select_state")], "title": String("SelectStripArgs"), "type": String("object")} }, Tool { name: "set_track_gain_db", description: "Sets the gain of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_db": Object {"description": String("The desired gain in dB. Valid range: -400.0 to 6.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_db"), String("rid")], "title": String("SetTrackGainDBArgs"), "type": String("object")} }, Tool { name: "rec_enable_toggle", description: "Toggles the master record enable or selected track record enable.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_trim_abs", description: "Sets the absolute trim of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_abs": Object {"description": String("The desired absolute trim. Valid range: 0.1 to 10.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_abs")], "title": String("SetTrackTrimAbsArgs"), "type": String("object")} }, Tool { name: "toggle_all_rec_enables", description: "Toggles the record enable state for ALL tracks.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "rewind", description: "Rewinds the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "goto_start", description: "Moves the playhead to the session start.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_mute", description: "Sets the mute state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"mute_state": Object {"description": String("The desired mute state (true for mute, false for unmute)."), "type": String("boolean")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("mute_state"), String("rid")], "title": String("SetTrackMuteArgs"), "type": String("object")} }, Tool { name: "loop_toggle", description: "Toggles loop playback mode.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "goto_end", description: "Moves the playhead to the session end.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "save_state", description: "Saves the current session state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "redo", description: "Redoes the last undone action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_strip_plugin_active", description: "Activates or deactivates a plugin on a specific strip slot.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"active_state": Object {"description": String("The desired activation state (true for active, false for inactive)."), "type": String("boolean")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("active_state"), String("plugin_slot"), String("rid")], "title": String("SetStripPluginActiveArgs"), "type": String("object")} }, Tool { name: "set_strip_plugin_parameter", description: "Sets a specific parameter of a plugin on a strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"param_id": Object {"description": String("The 1-indexed ID of the parameter within the plugin."), "format": String("int32"), "type": String("integer")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "value": Object {"description": String("The desired parameter value, normalized (0.0 to 1.0)."), "format": String("float"), "type": String("number")}}, "required": Array [String("param_id"), String("plugin_slot"), String("rid"), String("value")], "title": String("SetStripPluginParameterArgs"), "type": String("object")} }, Tool { name: "prev_marker", description: "Moves the playhead to the previous location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_rec_enable", description: "Sets the record enable state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rec_st": Object {"description": String("The desired record enable state. 0 for off, 1 for on."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rec_st"), String("rid")], "title": String("SetTrackRecEnableArgs"), "type": String("object")} }, Tool { name: "set_track_gain_abs", description: "Sets the absolute gain of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_abs": Object {"description": String("The desired absolute gain. Valid range: 0.0 to 2.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_abs"), String("rid")], "title": String("SetTrackGainAbsArgs"), "type": String("object")} }] }) +2026-06-30T12:08:53.089425Z DEBUG rmcp::service: new event evt=ToSink(Response(ListToolsResult(ListToolsResult { next_cursor: None, tools: [Tool { name: "toggle_punch_in", description: "Toggles the Punch In state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "add_marker", description: "Adds a location marker at the current playhead position.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "ffwd", description: "Fast forwards the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_strip_pan_stereo_width", description: "Sets the stereo width for a panner on a stereo strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the stereo track/bus."), "format": String("int32"), "type": String("integer")}, "width": Object {"description": String("The desired stereo width. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("width")], "title": String("SetStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "undo", description: "Undoes the last action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "transport_play", description: "Starts Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_solo", description: "Sets the solo state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "solo_st": Object {"description": String("The desired solo state. 0 for solo off, 1 for solo on."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rid"), String("solo_st")], "title": String("SetTrackSoloArgs"), "type": String("object")} }, Tool { name: "locate", description: "Locates the playhead to a specific sample position and optionally starts playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"roll": Object {"description": String("Whether to start playing after locating. 0 for stop, 1 for play."), "format": String("int32"), "type": String("integer")}, "spos": Object {"description": String("The position in samples to locate to."), "format": String("int64"), "type": String("integer")}}, "required": Array [String("roll"), String("spos")], "title": String("LocateToolArgs"), "type": String("object")} }, Tool { name: "transport_stop", description: "Stops Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "next_marker", description: "Moves the playhead to the next location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_transport_speed", description: "Sets Ardour's transport speed. Valid range: -8.0 to 8.0.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"speed": Object {"description": String("The desired transport speed. Valid range: -8.0 to 8.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("speed")], "title": String("SetTransportSpeedArgs"), "type": String("object")} }, Tool { name: "set_selected_strip_pan_stereo_width", description: "Sets the stereo width for the currently selected stereo strip's panner.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"width": Object {"description": String("The desired stereo width for the currently selected strip. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("width")], "title": String("SetSelectedStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "access_action", description: "Executes a specified Ardour menu action by its name.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"action_name": Object {"description": String("The name of the Ardour menu action to execute (e.g., 'Editor/zoom-to-session')."), "type": String("string")}}, "required": Array [String("action_name")], "title": String("AccessActionArgs"), "type": String("object")} }, Tool { name: "toggle_punch_out", description: "Toggles the Punch Out state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_trim_db", description: "Sets the trim of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_db": Object {"description": String("The desired trim in dB. Valid range: -20.0 to 20.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_db")], "title": String("SetTrackTrimDBArgs"), "type": String("object")} }, Tool { name: "select_strip", description: "Selects a specific strip (track/bus) in Ardour.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus to select."), "format": String("int32"), "type": String("integer")}, "select_state": Object {"description": String("The desired select state (true to select). Currently, only true (1) is effective for selection."), "type": String("boolean")}}, "required": Array [String("rid"), String("select_state")], "title": String("SelectStripArgs"), "type": String("object")} }, Tool { name: "set_track_gain_db", description: "Sets the gain of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_db": Object {"description": String("The desired gain in dB. Valid range: -400.0 to 6.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_db"), String("rid")], "title": String("SetTrackGainDBArgs"), "type": String("object")} }, Tool { name: "rec_enable_toggle", description: "Toggles the master record enable or selected track record enable.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_trim_abs", description: "Sets the absolute trim of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_abs": Object {"description": String("The desired absolute trim. Valid range: 0.1 to 10.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_abs")], "title": String("SetTrackTrimAbsArgs"), "type": String("object")} }, Tool { name: "toggle_all_rec_enables", description: "Toggles the record enable state for ALL tracks.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "rewind", description: "Rewinds the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "goto_start", description: "Moves the playhead to the session start.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_mute", description: "Sets the mute state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"mute_state": Object {"description": String("The desired mute state (true for mute, false for unmute)."), "type": String("boolean")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("mute_state"), String("rid")], "title": String("SetTrackMuteArgs"), "type": String("object")} }, Tool { name: "loop_toggle", description: "Toggles loop playback mode.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "goto_end", description: "Moves the playhead to the session end.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "save_state", description: "Saves the current session state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "redo", description: "Redoes the last undone action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_strip_plugin_active", description: "Activates or deactivates a plugin on a specific strip slot.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"active_state": Object {"description": String("The desired activation state (true for active, false for inactive)."), "type": String("boolean")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("active_state"), String("plugin_slot"), String("rid")], "title": String("SetStripPluginActiveArgs"), "type": String("object")} }, Tool { name: "set_strip_plugin_parameter", description: "Sets a specific parameter of a plugin on a strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"param_id": Object {"description": String("The 1-indexed ID of the parameter within the plugin."), "format": String("int32"), "type": String("integer")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "value": Object {"description": String("The desired parameter value, normalized (0.0 to 1.0)."), "format": String("float"), "type": String("number")}}, "required": Array [String("param_id"), String("plugin_slot"), String("rid"), String("value")], "title": String("SetStripPluginParameterArgs"), "type": String("object")} }, Tool { name: "prev_marker", description: "Moves the playhead to the previous location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_rec_enable", description: "Sets the record enable state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rec_st": Object {"description": String("The desired record enable state. 0 for off, 1 for on."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rec_st"), String("rid")], "title": String("SetTrackRecEnableArgs"), "type": String("object")} }, Tool { name: "set_track_gain_abs", description: "Sets the absolute gain of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_abs": Object {"description": String("The desired absolute gain. Valid range: 0.0 to 2.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_abs"), String("rid")], "title": String("SetTrackGainAbsArgs"), "type": String("object")} }] }), Number(1))) +2026-06-30T12:12:17.344926Z  INFO ardour_mcp_server: +====================================================================== +NEW SERVER RUN: Tue, 30 Jun 2026 14:12:17 +0200 +====================================================================== +2026-06-30T12:12:17.344966Z  INFO ardour_mcp_server: Attempting to create OSC sender for Ardour at 127.0.0.1:3819 +2026-06-30T12:12:17.344992Z  INFO ardour_mcp_server: OSC sender created and connected to Ardour at 127.0.0.1:3819 +2026-06-30T12:12:17.344995Z  INFO ardour_mcp_server: Sending /set_surface to Ardour to enable OSC feedback. +2026-06-30T12:12:17.344998Z  INFO ardour_mcp_server: Targeting feedback port: 9099 +2026-06-30T12:12:17.345144Z  INFO ardour_mcp_server: /set_surface command sent successfully to Ardour. +2026-06-30T12:12:17.447367Z  INFO serve_inner: rmcp::service: Service initialized as server peer_info=InitializeRequestParam { protocol_version: ProtocolVersion("2025-11-25"), capabilities: ClientCapabilities { experimental: None, roots: Some(RootsCapabilities { list_changed: None }), sampling: None }, client_info: Implementation { name: "claude-code", version: "2.1.196" } } +2026-06-30T12:12:17.447416Z  INFO ardour_mcp_server: Ardour MCP server started and waiting for connections... +2026-06-30T12:12:17.447450Z DEBUG rmcp::service: new event evt=PeerMessage(Request(ListToolsRequest(Request { method: ListToolsRequestMethod, params: None }), Number(1))) +2026-06-30T12:12:17.447447Z  INFO ardour_mcp_server: Starting OSC listener for Ardour events on 127.0.0.1:9099 +2026-06-30T12:12:17.447463Z  INFO rmcp::service: received request id=1 request=ListToolsRequest(Request { method: ListToolsRequestMethod, params: None }) +2026-06-30T12:12:17.447480Z DEBUG rmcp::service: new event evt=PeerMessage(Request(ListResourcesRequest(Request { method: ListResourcesRequestMethod, params: None }), Number(2))) +2026-06-30T12:12:17.447483Z  INFO rmcp::service: received request id=2 request=ListResourcesRequest(Request { method: ListResourcesRequestMethod, params: None }) +2026-06-30T12:12:17.447494Z DEBUG ardour_mcp_server: Listing resources. Count: 2, Content: [Annotated { raw: RawResource { uri: "ardour:/state/playback", name: "Ardour Playback State", description: Some("Current playback state of Ardour (e.g., Playing, Stopped, Unknown)."), mime_type: None, size: None }, annotations: None }, Annotated { raw: RawResource { uri: "ardour:/state/transport_frame", name: "Ardour Transport Frame Position", description: Some("Current playhead position in samples. Returns 'Unknown' if not yet reported by Ardour."), mime_type: None, size: None }, annotations: None }] +2026-06-30T12:12:17.447511Z  INFO rmcp::service: response message id=2 result=ListResourcesResult(ListResourcesResult { next_cursor: None, resources: [Annotated { raw: RawResource { uri: "ardour:/state/playback", name: "Ardour Playback State", description: Some("Current playback state of Ardour (e.g., Playing, Stopped, Unknown)."), mime_type: None, size: None }, annotations: None }, Annotated { raw: RawResource { uri: "ardour:/state/transport_frame", name: "Ardour Transport Frame Position", description: Some("Current playhead position in samples. Returns 'Unknown' if not yet reported by Ardour."), mime_type: None, size: None }, annotations: None }] }) +2026-06-30T12:12:17.447520Z DEBUG rmcp::service: new event evt=ToSink(Response(ListResourcesResult(ListResourcesResult { next_cursor: None, resources: [Annotated { raw: RawResource { uri: "ardour:/state/playback", name: "Ardour Playback State", description: Some("Current playback state of Ardour (e.g., Playing, Stopped, Unknown)."), mime_type: None, size: None }, annotations: None }, Annotated { raw: RawResource { uri: "ardour:/state/transport_frame", name: "Ardour Transport Frame Position", description: Some("Current playhead position in samples. Returns 'Unknown' if not yet reported by Ardour."), mime_type: None, size: None }, annotations: None }] }), Number(2))) +2026-06-30T12:12:17.447518Z ERROR ardour_mcp_server: OSC listener task failed: Failed to bind Tokio UDP socket for OSC on 127.0.0.1:9099: Address already in use (os error 98) +2026-06-30T12:12:17.447603Z  INFO rmcp::service: response message id=1 result=ListToolsResult(ListToolsResult { next_cursor: None, tools: [Tool { name: "transport_play", description: "Starts Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_strip_plugin_parameter", description: "Sets a specific parameter of a plugin on a strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"param_id": Object {"description": String("The 1-indexed ID of the parameter within the plugin."), "format": String("int32"), "type": String("integer")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "value": Object {"description": String("The desired parameter value, normalized (0.0 to 1.0)."), "format": String("float"), "type": String("number")}}, "required": Array [String("param_id"), String("plugin_slot"), String("rid"), String("value")], "title": String("SetStripPluginParameterArgs"), "type": String("object")} }, Tool { name: "access_action", description: "Executes a specified Ardour menu action by its name.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"action_name": Object {"description": String("The name of the Ardour menu action to execute (e.g., 'Editor/zoom-to-session')."), "type": String("string")}}, "required": Array [String("action_name")], "title": String("AccessActionArgs"), "type": String("object")} }, Tool { name: "next_marker", description: "Moves the playhead to the next location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "undo", description: "Undoes the last action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_selected_strip_pan_stereo_width", description: "Sets the stereo width for the currently selected stereo strip's panner.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"width": Object {"description": String("The desired stereo width for the currently selected strip. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("width")], "title": String("SetSelectedStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "set_strip_plugin_active", description: "Activates or deactivates a plugin on a specific strip slot.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"active_state": Object {"description": String("The desired activation state (true for active, false for inactive)."), "type": String("boolean")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("active_state"), String("plugin_slot"), String("rid")], "title": String("SetStripPluginActiveArgs"), "type": String("object")} }, Tool { name: "rec_enable_toggle", description: "Toggles the master record enable or selected track record enable.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "locate", description: "Locates the playhead to a specific sample position and optionally starts playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"roll": Object {"description": String("Whether to start playing after locating. 0 for stop, 1 for play."), "format": String("int32"), "type": String("integer")}, "spos": Object {"description": String("The position in samples to locate to."), "format": String("int64"), "type": String("integer")}}, "required": Array [String("roll"), String("spos")], "title": String("LocateToolArgs"), "type": String("object")} }, Tool { name: "select_strip", description: "Selects a specific strip (track/bus) in Ardour.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus to select."), "format": String("int32"), "type": String("integer")}, "select_state": Object {"description": String("The desired select state (true to select). Currently, only true (1) is effective for selection."), "type": String("boolean")}}, "required": Array [String("rid"), String("select_state")], "title": String("SelectStripArgs"), "type": String("object")} }, Tool { name: "toggle_all_rec_enables", description: "Toggles the record enable state for ALL tracks.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_transport_speed", description: "Sets Ardour's transport speed. Valid range: -8.0 to 8.0.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"speed": Object {"description": String("The desired transport speed. Valid range: -8.0 to 8.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("speed")], "title": String("SetTransportSpeedArgs"), "type": String("object")} }, Tool { name: "ffwd", description: "Fast forwards the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_mute", description: "Sets the mute state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"mute_state": Object {"description": String("The desired mute state (true for mute, false for unmute)."), "type": String("boolean")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("mute_state"), String("rid")], "title": String("SetTrackMuteArgs"), "type": String("object")} }, Tool { name: "set_track_trim_db", description: "Sets the trim of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_db": Object {"description": String("The desired trim in dB. Valid range: -20.0 to 20.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_db")], "title": String("SetTrackTrimDBArgs"), "type": String("object")} }, Tool { name: "toggle_punch_out", description: "Toggles the Punch Out state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_strip_pan_stereo_width", description: "Sets the stereo width for a panner on a stereo strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the stereo track/bus."), "format": String("int32"), "type": String("integer")}, "width": Object {"description": String("The desired stereo width. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("width")], "title": String("SetStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "loop_toggle", description: "Toggles loop playback mode.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "goto_end", description: "Moves the playhead to the session end.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "transport_stop", description: "Stops Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "goto_start", description: "Moves the playhead to the session start.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "redo", description: "Redoes the last undone action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_solo", description: "Sets the solo state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "solo_st": Object {"description": String("The desired solo state. 0 for solo off, 1 for solo on."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rid"), String("solo_st")], "title": String("SetTrackSoloArgs"), "type": String("object")} }, Tool { name: "rewind", description: "Rewinds the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "save_state", description: "Saves the current session state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_gain_db", description: "Sets the gain of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_db": Object {"description": String("The desired gain in dB. Valid range: -400.0 to 6.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_db"), String("rid")], "title": String("SetTrackGainDBArgs"), "type": String("object")} }, Tool { name: "set_track_gain_abs", description: "Sets the absolute gain of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_abs": Object {"description": String("The desired absolute gain. Valid range: 0.0 to 2.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_abs"), String("rid")], "title": String("SetTrackGainAbsArgs"), "type": String("object")} }, Tool { name: "toggle_punch_in", description: "Toggles the Punch In state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "prev_marker", description: "Moves the playhead to the previous location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_trim_abs", description: "Sets the absolute trim of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_abs": Object {"description": String("The desired absolute trim. Valid range: 0.1 to 10.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_abs")], "title": String("SetTrackTrimAbsArgs"), "type": String("object")} }, Tool { name: "add_marker", description: "Adds a location marker at the current playhead position.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_rec_enable", description: "Sets the record enable state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rec_st": Object {"description": String("The desired record enable state. 0 for off, 1 for on."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rec_st"), String("rid")], "title": String("SetTrackRecEnableArgs"), "type": String("object")} }] }) +2026-06-30T12:12:17.447649Z DEBUG rmcp::service: new event evt=ToSink(Response(ListToolsResult(ListToolsResult { next_cursor: None, tools: [Tool { name: "transport_play", description: "Starts Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_strip_plugin_parameter", description: "Sets a specific parameter of a plugin on a strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"param_id": Object {"description": String("The 1-indexed ID of the parameter within the plugin."), "format": String("int32"), "type": String("integer")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "value": Object {"description": String("The desired parameter value, normalized (0.0 to 1.0)."), "format": String("float"), "type": String("number")}}, "required": Array [String("param_id"), String("plugin_slot"), String("rid"), String("value")], "title": String("SetStripPluginParameterArgs"), "type": String("object")} }, Tool { name: "access_action", description: "Executes a specified Ardour menu action by its name.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"action_name": Object {"description": String("The name of the Ardour menu action to execute (e.g., 'Editor/zoom-to-session')."), "type": String("string")}}, "required": Array [String("action_name")], "title": String("AccessActionArgs"), "type": String("object")} }, Tool { name: "next_marker", description: "Moves the playhead to the next location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "undo", description: "Undoes the last action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_selected_strip_pan_stereo_width", description: "Sets the stereo width for the currently selected stereo strip's panner.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"width": Object {"description": String("The desired stereo width for the currently selected strip. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("width")], "title": String("SetSelectedStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "set_strip_plugin_active", description: "Activates or deactivates a plugin on a specific strip slot.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"active_state": Object {"description": String("The desired activation state (true for active, false for inactive)."), "type": String("boolean")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("active_state"), String("plugin_slot"), String("rid")], "title": String("SetStripPluginActiveArgs"), "type": String("object")} }, Tool { name: "rec_enable_toggle", description: "Toggles the master record enable or selected track record enable.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "locate", description: "Locates the playhead to a specific sample position and optionally starts playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"roll": Object {"description": String("Whether to start playing after locating. 0 for stop, 1 for play."), "format": String("int32"), "type": String("integer")}, "spos": Object {"description": String("The position in samples to locate to."), "format": String("int64"), "type": String("integer")}}, "required": Array [String("roll"), String("spos")], "title": String("LocateToolArgs"), "type": String("object")} }, Tool { name: "select_strip", description: "Selects a specific strip (track/bus) in Ardour.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus to select."), "format": String("int32"), "type": String("integer")}, "select_state": Object {"description": String("The desired select state (true to select). Currently, only true (1) is effective for selection."), "type": String("boolean")}}, "required": Array [String("rid"), String("select_state")], "title": String("SelectStripArgs"), "type": String("object")} }, Tool { name: "toggle_all_rec_enables", description: "Toggles the record enable state for ALL tracks.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_transport_speed", description: "Sets Ardour's transport speed. Valid range: -8.0 to 8.0.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"speed": Object {"description": String("The desired transport speed. Valid range: -8.0 to 8.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("speed")], "title": String("SetTransportSpeedArgs"), "type": String("object")} }, Tool { name: "ffwd", description: "Fast forwards the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_mute", description: "Sets the mute state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"mute_state": Object {"description": String("The desired mute state (true for mute, false for unmute)."), "type": String("boolean")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("mute_state"), String("rid")], "title": String("SetTrackMuteArgs"), "type": String("object")} }, Tool { name: "set_track_trim_db", description: "Sets the trim of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_db": Object {"description": String("The desired trim in dB. Valid range: -20.0 to 20.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_db")], "title": String("SetTrackTrimDBArgs"), "type": String("object")} }, Tool { name: "toggle_punch_out", description: "Toggles the Punch Out state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_strip_pan_stereo_width", description: "Sets the stereo width for a panner on a stereo strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the stereo track/bus."), "format": String("int32"), "type": String("integer")}, "width": Object {"description": String("The desired stereo width. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("width")], "title": String("SetStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "loop_toggle", description: "Toggles loop playback mode.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "goto_end", description: "Moves the playhead to the session end.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "transport_stop", description: "Stops Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "goto_start", description: "Moves the playhead to the session start.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "redo", description: "Redoes the last undone action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_solo", description: "Sets the solo state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "solo_st": Object {"description": String("The desired solo state. 0 for solo off, 1 for solo on."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rid"), String("solo_st")], "title": String("SetTrackSoloArgs"), "type": String("object")} }, Tool { name: "rewind", description: "Rewinds the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "save_state", description: "Saves the current session state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_gain_db", description: "Sets the gain of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_db": Object {"description": String("The desired gain in dB. Valid range: -400.0 to 6.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_db"), String("rid")], "title": String("SetTrackGainDBArgs"), "type": String("object")} }, Tool { name: "set_track_gain_abs", description: "Sets the absolute gain of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_abs": Object {"description": String("The desired absolute gain. Valid range: 0.0 to 2.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_abs"), String("rid")], "title": String("SetTrackGainAbsArgs"), "type": String("object")} }, Tool { name: "toggle_punch_in", description: "Toggles the Punch In state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "prev_marker", description: "Moves the playhead to the previous location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_trim_abs", description: "Sets the absolute trim of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_abs": Object {"description": String("The desired absolute trim. Valid range: 0.1 to 10.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_abs")], "title": String("SetTrackTrimAbsArgs"), "type": String("object")} }, Tool { name: "add_marker", description: "Adds a location marker at the current playhead position.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_rec_enable", description: "Sets the record enable state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rec_st": Object {"description": String("The desired record enable state. 0 for off, 1 for on."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rec_st"), String("rid")], "title": String("SetTrackRecEnableArgs"), "type": String("object")} }] }), Number(1))) +2026-06-30T13:18:27.385586Z  INFO ardour_mcp_server: +====================================================================== +NEW SERVER RUN: Tue, 30 Jun 2026 15:18:27 +0200 +====================================================================== +2026-06-30T13:18:27.385639Z  INFO ardour_mcp_server: Attempting to create OSC sender for Ardour at 127.0.0.1:3819 +2026-06-30T13:18:27.385670Z  INFO ardour_mcp_server: OSC sender created and connected to Ardour at 127.0.0.1:3819 +2026-06-30T13:18:27.385674Z  INFO ardour_mcp_server: Sending /set_surface to Ardour to enable OSC feedback. +2026-06-30T13:18:27.385677Z  INFO ardour_mcp_server: Targeting feedback port: 9099 +2026-06-30T13:18:27.385869Z  INFO ardour_mcp_server: /set_surface command sent successfully to Ardour. +2026-06-30T13:18:27.489761Z  INFO serve_inner: rmcp::service: Service initialized as server peer_info=InitializeRequestParam { protocol_version: ProtocolVersion("2025-11-25"), capabilities: ClientCapabilities { experimental: None, roots: Some(RootsCapabilities { list_changed: None }), sampling: None }, client_info: Implementation { name: "claude-code", version: "2.1.196" } } +2026-06-30T13:18:27.489848Z  INFO ardour_mcp_server: Ardour MCP server started and waiting for connections... +2026-06-30T13:18:27.489898Z DEBUG rmcp::service: new event evt=PeerMessage(Request(ListToolsRequest(Request { method: ListToolsRequestMethod, params: None }), Number(1))) +2026-06-30T13:18:27.489906Z  INFO ardour_mcp_server: Starting OSC listener for Ardour events on 127.0.0.1:9099 +2026-06-30T13:18:27.489914Z  INFO rmcp::service: received request id=1 request=ListToolsRequest(Request { method: ListToolsRequestMethod, params: None }) +2026-06-30T13:18:27.489940Z DEBUG rmcp::service: new event evt=PeerMessage(Request(ListResourcesRequest(Request { method: ListResourcesRequestMethod, params: None }), Number(2))) +2026-06-30T13:18:27.489944Z  INFO rmcp::service: received request id=2 request=ListResourcesRequest(Request { method: ListResourcesRequestMethod, params: None }) +2026-06-30T13:18:27.489968Z DEBUG ardour_mcp_server: Listing resources. Count: 2, Content: [Annotated { raw: RawResource { uri: "ardour:/state/playback", name: "Ardour Playback State", description: Some("Current playback state of Ardour (e.g., Playing, Stopped, Unknown)."), mime_type: None, size: None }, annotations: None }, Annotated { raw: RawResource { uri: "ardour:/state/transport_frame", name: "Ardour Transport Frame Position", description: Some("Current playhead position in samples. Returns 'Unknown' if not yet reported by Ardour."), mime_type: None, size: None }, annotations: None }] +2026-06-30T13:18:27.489982Z  INFO rmcp::service: response message id=2 result=ListResourcesResult(ListResourcesResult { next_cursor: None, resources: [Annotated { raw: RawResource { uri: "ardour:/state/playback", name: "Ardour Playback State", description: Some("Current playback state of Ardour (e.g., Playing, Stopped, Unknown)."), mime_type: None, size: None }, annotations: None }, Annotated { raw: RawResource { uri: "ardour:/state/transport_frame", name: "Ardour Transport Frame Position", description: Some("Current playhead position in samples. Returns 'Unknown' if not yet reported by Ardour."), mime_type: None, size: None }, annotations: None }] }) +2026-06-30T13:18:27.489988Z ERROR ardour_mcp_server: OSC listener task failed: Failed to bind Tokio UDP socket for OSC on 127.0.0.1:9099: Address already in use (os error 98) +2026-06-30T13:18:27.489997Z DEBUG rmcp::service: new event evt=ToSink(Response(ListResourcesResult(ListResourcesResult { next_cursor: None, resources: [Annotated { raw: RawResource { uri: "ardour:/state/playback", name: "Ardour Playback State", description: Some("Current playback state of Ardour (e.g., Playing, Stopped, Unknown)."), mime_type: None, size: None }, annotations: None }, Annotated { raw: RawResource { uri: "ardour:/state/transport_frame", name: "Ardour Transport Frame Position", description: Some("Current playhead position in samples. Returns 'Unknown' if not yet reported by Ardour."), mime_type: None, size: None }, annotations: None }] }), Number(2))) +2026-06-30T13:18:27.490247Z  INFO rmcp::service: response message id=1 result=ListToolsResult(ListToolsResult { next_cursor: None, tools: [Tool { name: "goto_end", description: "Moves the playhead to the session end.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "prev_marker", description: "Moves the playhead to the previous location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "rewind", description: "Rewinds the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_rec_enable", description: "Sets the record enable state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rec_st": Object {"description": String("The desired record enable state. 0 for off, 1 for on."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rec_st"), String("rid")], "title": String("SetTrackRecEnableArgs"), "type": String("object")} }, Tool { name: "transport_stop", description: "Stops Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "add_marker", description: "Adds a location marker at the current playhead position.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "access_action", description: "Executes a specified Ardour menu action by its name.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"action_name": Object {"description": String("The name of the Ardour menu action to execute (e.g., 'Editor/zoom-to-session')."), "type": String("string")}}, "required": Array [String("action_name")], "title": String("AccessActionArgs"), "type": String("object")} }, Tool { name: "redo", description: "Redoes the last undone action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "toggle_punch_in", description: "Toggles the Punch In state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "locate", description: "Locates the playhead to a specific sample position and optionally starts playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"roll": Object {"description": String("Whether to start playing after locating. 0 for stop, 1 for play."), "format": String("int32"), "type": String("integer")}, "spos": Object {"description": String("The position in samples to locate to."), "format": String("int64"), "type": String("integer")}}, "required": Array [String("roll"), String("spos")], "title": String("LocateToolArgs"), "type": String("object")} }, Tool { name: "set_transport_speed", description: "Sets Ardour's transport speed. Valid range: -8.0 to 8.0.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"speed": Object {"description": String("The desired transport speed. Valid range: -8.0 to 8.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("speed")], "title": String("SetTransportSpeedArgs"), "type": String("object")} }, Tool { name: "set_track_gain_abs", description: "Sets the absolute gain of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_abs": Object {"description": String("The desired absolute gain. Valid range: 0.0 to 2.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_abs"), String("rid")], "title": String("SetTrackGainAbsArgs"), "type": String("object")} }, Tool { name: "set_track_gain_db", description: "Sets the gain of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_db": Object {"description": String("The desired gain in dB. Valid range: -400.0 to 6.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_db"), String("rid")], "title": String("SetTrackGainDBArgs"), "type": String("object")} }, Tool { name: "undo", description: "Undoes the last action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "toggle_all_rec_enables", description: "Toggles the record enable state for ALL tracks.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "next_marker", description: "Moves the playhead to the next location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "toggle_punch_out", description: "Toggles the Punch Out state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_strip_plugin_parameter", description: "Sets a specific parameter of a plugin on a strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"param_id": Object {"description": String("The 1-indexed ID of the parameter within the plugin."), "format": String("int32"), "type": String("integer")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "value": Object {"description": String("The desired parameter value, normalized (0.0 to 1.0)."), "format": String("float"), "type": String("number")}}, "required": Array [String("param_id"), String("plugin_slot"), String("rid"), String("value")], "title": String("SetStripPluginParameterArgs"), "type": String("object")} }, Tool { name: "rec_enable_toggle", description: "Toggles the master record enable or selected track record enable.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "ffwd", description: "Fast forwards the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_trim_abs", description: "Sets the absolute trim of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_abs": Object {"description": String("The desired absolute trim. Valid range: 0.1 to 10.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_abs")], "title": String("SetTrackTrimAbsArgs"), "type": String("object")} }, Tool { name: "goto_start", description: "Moves the playhead to the session start.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "loop_toggle", description: "Toggles loop playback mode.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_mute", description: "Sets the mute state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"mute_state": Object {"description": String("The desired mute state (true for mute, false for unmute)."), "type": String("boolean")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("mute_state"), String("rid")], "title": String("SetTrackMuteArgs"), "type": String("object")} }, Tool { name: "set_strip_plugin_active", description: "Activates or deactivates a plugin on a specific strip slot.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"active_state": Object {"description": String("The desired activation state (true for active, false for inactive)."), "type": String("boolean")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("active_state"), String("plugin_slot"), String("rid")], "title": String("SetStripPluginActiveArgs"), "type": String("object")} }, Tool { name: "set_track_solo", description: "Sets the solo state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "solo_st": Object {"description": String("The desired solo state. 0 for solo off, 1 for solo on."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rid"), String("solo_st")], "title": String("SetTrackSoloArgs"), "type": String("object")} }, Tool { name: "set_strip_pan_stereo_width", description: "Sets the stereo width for a panner on a stereo strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the stereo track/bus."), "format": String("int32"), "type": String("integer")}, "width": Object {"description": String("The desired stereo width. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("width")], "title": String("SetStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "select_strip", description: "Selects a specific strip (track/bus) in Ardour.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus to select."), "format": String("int32"), "type": String("integer")}, "select_state": Object {"description": String("The desired select state (true to select). Currently, only true (1) is effective for selection."), "type": String("boolean")}}, "required": Array [String("rid"), String("select_state")], "title": String("SelectStripArgs"), "type": String("object")} }, Tool { name: "transport_play", description: "Starts Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_trim_db", description: "Sets the trim of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_db": Object {"description": String("The desired trim in dB. Valid range: -20.0 to 20.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_db")], "title": String("SetTrackTrimDBArgs"), "type": String("object")} }, Tool { name: "set_selected_strip_pan_stereo_width", description: "Sets the stereo width for the currently selected stereo strip's panner.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"width": Object {"description": String("The desired stereo width for the currently selected strip. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("width")], "title": String("SetSelectedStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "save_state", description: "Saves the current session state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }] }) +2026-06-30T13:18:27.490334Z DEBUG rmcp::service: new event evt=ToSink(Response(ListToolsResult(ListToolsResult { next_cursor: None, tools: [Tool { name: "goto_end", description: "Moves the playhead to the session end.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "prev_marker", description: "Moves the playhead to the previous location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "rewind", description: "Rewinds the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_rec_enable", description: "Sets the record enable state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rec_st": Object {"description": String("The desired record enable state. 0 for off, 1 for on."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rec_st"), String("rid")], "title": String("SetTrackRecEnableArgs"), "type": String("object")} }, Tool { name: "transport_stop", description: "Stops Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "add_marker", description: "Adds a location marker at the current playhead position.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "access_action", description: "Executes a specified Ardour menu action by its name.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"action_name": Object {"description": String("The name of the Ardour menu action to execute (e.g., 'Editor/zoom-to-session')."), "type": String("string")}}, "required": Array [String("action_name")], "title": String("AccessActionArgs"), "type": String("object")} }, Tool { name: "redo", description: "Redoes the last undone action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "toggle_punch_in", description: "Toggles the Punch In state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "locate", description: "Locates the playhead to a specific sample position and optionally starts playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"roll": Object {"description": String("Whether to start playing after locating. 0 for stop, 1 for play."), "format": String("int32"), "type": String("integer")}, "spos": Object {"description": String("The position in samples to locate to."), "format": String("int64"), "type": String("integer")}}, "required": Array [String("roll"), String("spos")], "title": String("LocateToolArgs"), "type": String("object")} }, Tool { name: "set_transport_speed", description: "Sets Ardour's transport speed. Valid range: -8.0 to 8.0.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"speed": Object {"description": String("The desired transport speed. Valid range: -8.0 to 8.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("speed")], "title": String("SetTransportSpeedArgs"), "type": String("object")} }, Tool { name: "set_track_gain_abs", description: "Sets the absolute gain of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_abs": Object {"description": String("The desired absolute gain. Valid range: 0.0 to 2.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_abs"), String("rid")], "title": String("SetTrackGainAbsArgs"), "type": String("object")} }, Tool { name: "set_track_gain_db", description: "Sets the gain of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_db": Object {"description": String("The desired gain in dB. Valid range: -400.0 to 6.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_db"), String("rid")], "title": String("SetTrackGainDBArgs"), "type": String("object")} }, Tool { name: "undo", description: "Undoes the last action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "toggle_all_rec_enables", description: "Toggles the record enable state for ALL tracks.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "next_marker", description: "Moves the playhead to the next location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "toggle_punch_out", description: "Toggles the Punch Out state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_strip_plugin_parameter", description: "Sets a specific parameter of a plugin on a strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"param_id": Object {"description": String("The 1-indexed ID of the parameter within the plugin."), "format": String("int32"), "type": String("integer")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "value": Object {"description": String("The desired parameter value, normalized (0.0 to 1.0)."), "format": String("float"), "type": String("number")}}, "required": Array [String("param_id"), String("plugin_slot"), String("rid"), String("value")], "title": String("SetStripPluginParameterArgs"), "type": String("object")} }, Tool { name: "rec_enable_toggle", description: "Toggles the master record enable or selected track record enable.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "ffwd", description: "Fast forwards the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_trim_abs", description: "Sets the absolute trim of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_abs": Object {"description": String("The desired absolute trim. Valid range: 0.1 to 10.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_abs")], "title": String("SetTrackTrimAbsArgs"), "type": String("object")} }, Tool { name: "goto_start", description: "Moves the playhead to the session start.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "loop_toggle", description: "Toggles loop playback mode.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_mute", description: "Sets the mute state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"mute_state": Object {"description": String("The desired mute state (true for mute, false for unmute)."), "type": String("boolean")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("mute_state"), String("rid")], "title": String("SetTrackMuteArgs"), "type": String("object")} }, Tool { name: "set_strip_plugin_active", description: "Activates or deactivates a plugin on a specific strip slot.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"active_state": Object {"description": String("The desired activation state (true for active, false for inactive)."), "type": String("boolean")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("active_state"), String("plugin_slot"), String("rid")], "title": String("SetStripPluginActiveArgs"), "type": String("object")} }, Tool { name: "set_track_solo", description: "Sets the solo state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "solo_st": Object {"description": String("The desired solo state. 0 for solo off, 1 for solo on."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rid"), String("solo_st")], "title": String("SetTrackSoloArgs"), "type": String("object")} }, Tool { name: "set_strip_pan_stereo_width", description: "Sets the stereo width for a panner on a stereo strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the stereo track/bus."), "format": String("int32"), "type": String("integer")}, "width": Object {"description": String("The desired stereo width. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("width")], "title": String("SetStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "select_strip", description: "Selects a specific strip (track/bus) in Ardour.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus to select."), "format": String("int32"), "type": String("integer")}, "select_state": Object {"description": String("The desired select state (true to select). Currently, only true (1) is effective for selection."), "type": String("boolean")}}, "required": Array [String("rid"), String("select_state")], "title": String("SelectStripArgs"), "type": String("object")} }, Tool { name: "transport_play", description: "Starts Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_trim_db", description: "Sets the trim of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_db": Object {"description": String("The desired trim in dB. Valid range: -20.0 to 20.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_db")], "title": String("SetTrackTrimDBArgs"), "type": String("object")} }, Tool { name: "set_selected_strip_pan_stereo_width", description: "Sets the stereo width for the currently selected stereo strip's panner.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"width": Object {"description": String("The desired stereo width for the currently selected strip. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("width")], "title": String("SetSelectedStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "save_state", description: "Saves the current session state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }] }), Number(1))) +2026-06-30T13:18:27.535976Z  INFO ardour_mcp_server: +====================================================================== +NEW SERVER RUN: Tue, 30 Jun 2026 15:18:27 +0200 +====================================================================== +2026-06-30T13:18:27.536034Z  INFO ardour_mcp_server: Attempting to create OSC sender for Ardour at 127.0.0.1:3819 +2026-06-30T13:18:27.536096Z  INFO ardour_mcp_server: OSC sender created and connected to Ardour at 127.0.0.1:3819 +2026-06-30T13:18:27.536101Z  INFO ardour_mcp_server: Sending /set_surface to Ardour to enable OSC feedback. +2026-06-30T13:18:27.536106Z  INFO ardour_mcp_server: Targeting feedback port: 9099 +2026-06-30T13:18:27.536334Z  INFO ardour_mcp_server: /set_surface command sent successfully to Ardour. +2026-06-30T13:18:27.588516Z  INFO serve_inner: rmcp::service: Service initialized as server peer_info=InitializeRequestParam { protocol_version: ProtocolVersion("2025-11-25"), capabilities: ClientCapabilities { experimental: None, roots: Some(RootsCapabilities { list_changed: None }), sampling: None }, client_info: Implementation { name: "claude-code", version: "2.1.196" } } +2026-06-30T13:18:27.588623Z  INFO ardour_mcp_server: Ardour MCP server started and waiting for connections... +2026-06-30T13:18:27.588694Z  INFO ardour_mcp_server: Starting OSC listener for Ardour events on 127.0.0.1:9099 +2026-06-30T13:18:27.588734Z DEBUG rmcp::service: new event evt=PeerMessage(Request(ListToolsRequest(Request { method: ListToolsRequestMethod, params: None }), Number(1))) +2026-06-30T13:18:27.588751Z  INFO rmcp::service: received request id=1 request=ListToolsRequest(Request { method: ListToolsRequestMethod, params: None }) +2026-06-30T13:18:27.588773Z DEBUG rmcp::service: new event evt=PeerMessage(Request(ListResourcesRequest(Request { method: ListResourcesRequestMethod, params: None }), Number(2))) +2026-06-30T13:18:27.588777Z  INFO rmcp::service: received request id=2 request=ListResourcesRequest(Request { method: ListResourcesRequestMethod, params: None }) +2026-06-30T13:18:27.588779Z ERROR ardour_mcp_server: OSC listener task failed: Failed to bind Tokio UDP socket for OSC on 127.0.0.1:9099: Address already in use (os error 98) +2026-06-30T13:18:27.588798Z DEBUG ardour_mcp_server: Listing resources. Count: 2, Content: [Annotated { raw: RawResource { uri: "ardour:/state/playback", name: "Ardour Playback State", description: Some("Current playback state of Ardour (e.g., Playing, Stopped, Unknown)."), mime_type: None, size: None }, annotations: None }, Annotated { raw: RawResource { uri: "ardour:/state/transport_frame", name: "Ardour Transport Frame Position", description: Some("Current playhead position in samples. Returns 'Unknown' if not yet reported by Ardour."), mime_type: None, size: None }, annotations: None }] +2026-06-30T13:18:27.588812Z  INFO rmcp::service: response message id=2 result=ListResourcesResult(ListResourcesResult { next_cursor: None, resources: [Annotated { raw: RawResource { uri: "ardour:/state/playback", name: "Ardour Playback State", description: Some("Current playback state of Ardour (e.g., Playing, Stopped, Unknown)."), mime_type: None, size: None }, annotations: None }, Annotated { raw: RawResource { uri: "ardour:/state/transport_frame", name: "Ardour Transport Frame Position", description: Some("Current playhead position in samples. Returns 'Unknown' if not yet reported by Ardour."), mime_type: None, size: None }, annotations: None }] }) +2026-06-30T13:18:27.588824Z DEBUG rmcp::service: new event evt=ToSink(Response(ListResourcesResult(ListResourcesResult { next_cursor: None, resources: [Annotated { raw: RawResource { uri: "ardour:/state/playback", name: "Ardour Playback State", description: Some("Current playback state of Ardour (e.g., Playing, Stopped, Unknown)."), mime_type: None, size: None }, annotations: None }, Annotated { raw: RawResource { uri: "ardour:/state/transport_frame", name: "Ardour Transport Frame Position", description: Some("Current playhead position in samples. Returns 'Unknown' if not yet reported by Ardour."), mime_type: None, size: None }, annotations: None }] }), Number(2))) +2026-06-30T13:18:27.589018Z  INFO rmcp::service: response message id=1 result=ListToolsResult(ListToolsResult { next_cursor: None, tools: [Tool { name: "access_action", description: "Executes a specified Ardour menu action by its name.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"action_name": Object {"description": String("The name of the Ardour menu action to execute (e.g., 'Editor/zoom-to-session')."), "type": String("string")}}, "required": Array [String("action_name")], "title": String("AccessActionArgs"), "type": String("object")} }, Tool { name: "next_marker", description: "Moves the playhead to the next location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "goto_start", description: "Moves the playhead to the session start.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "transport_play", description: "Starts Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "rewind", description: "Rewinds the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "prev_marker", description: "Moves the playhead to the previous location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_gain_abs", description: "Sets the absolute gain of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_abs": Object {"description": String("The desired absolute gain. Valid range: 0.0 to 2.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_abs"), String("rid")], "title": String("SetTrackGainAbsArgs"), "type": String("object")} }, Tool { name: "set_track_trim_db", description: "Sets the trim of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_db": Object {"description": String("The desired trim in dB. Valid range: -20.0 to 20.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_db")], "title": String("SetTrackTrimDBArgs"), "type": String("object")} }, Tool { name: "set_strip_plugin_parameter", description: "Sets a specific parameter of a plugin on a strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"param_id": Object {"description": String("The 1-indexed ID of the parameter within the plugin."), "format": String("int32"), "type": String("integer")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "value": Object {"description": String("The desired parameter value, normalized (0.0 to 1.0)."), "format": String("float"), "type": String("number")}}, "required": Array [String("param_id"), String("plugin_slot"), String("rid"), String("value")], "title": String("SetStripPluginParameterArgs"), "type": String("object")} }, Tool { name: "save_state", description: "Saves the current session state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "redo", description: "Redoes the last undone action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "undo", description: "Undoes the last action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_rec_enable", description: "Sets the record enable state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rec_st": Object {"description": String("The desired record enable state. 0 for off, 1 for on."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rec_st"), String("rid")], "title": String("SetTrackRecEnableArgs"), "type": String("object")} }, Tool { name: "transport_stop", description: "Stops Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "ffwd", description: "Fast forwards the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "locate", description: "Locates the playhead to a specific sample position and optionally starts playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"roll": Object {"description": String("Whether to start playing after locating. 0 for stop, 1 for play."), "format": String("int32"), "type": String("integer")}, "spos": Object {"description": String("The position in samples to locate to."), "format": String("int64"), "type": String("integer")}}, "required": Array [String("roll"), String("spos")], "title": String("LocateToolArgs"), "type": String("object")} }, Tool { name: "set_strip_plugin_active", description: "Activates or deactivates a plugin on a specific strip slot.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"active_state": Object {"description": String("The desired activation state (true for active, false for inactive)."), "type": String("boolean")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("active_state"), String("plugin_slot"), String("rid")], "title": String("SetStripPluginActiveArgs"), "type": String("object")} }, Tool { name: "toggle_punch_in", description: "Toggles the Punch In state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "toggle_punch_out", description: "Toggles the Punch Out state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "goto_end", description: "Moves the playhead to the session end.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "add_marker", description: "Adds a location marker at the current playhead position.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_mute", description: "Sets the mute state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"mute_state": Object {"description": String("The desired mute state (true for mute, false for unmute)."), "type": String("boolean")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("mute_state"), String("rid")], "title": String("SetTrackMuteArgs"), "type": String("object")} }, Tool { name: "set_track_solo", description: "Sets the solo state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "solo_st": Object {"description": String("The desired solo state. 0 for solo off, 1 for solo on."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rid"), String("solo_st")], "title": String("SetTrackSoloArgs"), "type": String("object")} }, Tool { name: "toggle_all_rec_enables", description: "Toggles the record enable state for ALL tracks.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_gain_db", description: "Sets the gain of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_db": Object {"description": String("The desired gain in dB. Valid range: -400.0 to 6.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_db"), String("rid")], "title": String("SetTrackGainDBArgs"), "type": String("object")} }, Tool { name: "rec_enable_toggle", description: "Toggles the master record enable or selected track record enable.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_transport_speed", description: "Sets Ardour's transport speed. Valid range: -8.0 to 8.0.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"speed": Object {"description": String("The desired transport speed. Valid range: -8.0 to 8.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("speed")], "title": String("SetTransportSpeedArgs"), "type": String("object")} }, Tool { name: "loop_toggle", description: "Toggles loop playback mode.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_selected_strip_pan_stereo_width", description: "Sets the stereo width for the currently selected stereo strip's panner.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"width": Object {"description": String("The desired stereo width for the currently selected strip. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("width")], "title": String("SetSelectedStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "set_track_trim_abs", description: "Sets the absolute trim of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_abs": Object {"description": String("The desired absolute trim. Valid range: 0.1 to 10.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_abs")], "title": String("SetTrackTrimAbsArgs"), "type": String("object")} }, Tool { name: "select_strip", description: "Selects a specific strip (track/bus) in Ardour.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus to select."), "format": String("int32"), "type": String("integer")}, "select_state": Object {"description": String("The desired select state (true to select). Currently, only true (1) is effective for selection."), "type": String("boolean")}}, "required": Array [String("rid"), String("select_state")], "title": String("SelectStripArgs"), "type": String("object")} }, Tool { name: "set_strip_pan_stereo_width", description: "Sets the stereo width for a panner on a stereo strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the stereo track/bus."), "format": String("int32"), "type": String("integer")}, "width": Object {"description": String("The desired stereo width. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("width")], "title": String("SetStripPanStereoWidthArgs"), "type": String("object")} }] }) +2026-06-30T13:18:27.589120Z DEBUG rmcp::service: new event evt=ToSink(Response(ListToolsResult(ListToolsResult { next_cursor: None, tools: [Tool { name: "access_action", description: "Executes a specified Ardour menu action by its name.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"action_name": Object {"description": String("The name of the Ardour menu action to execute (e.g., 'Editor/zoom-to-session')."), "type": String("string")}}, "required": Array [String("action_name")], "title": String("AccessActionArgs"), "type": String("object")} }, Tool { name: "next_marker", description: "Moves the playhead to the next location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "goto_start", description: "Moves the playhead to the session start.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "transport_play", description: "Starts Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "rewind", description: "Rewinds the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "prev_marker", description: "Moves the playhead to the previous location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_gain_abs", description: "Sets the absolute gain of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_abs": Object {"description": String("The desired absolute gain. Valid range: 0.0 to 2.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_abs"), String("rid")], "title": String("SetTrackGainAbsArgs"), "type": String("object")} }, Tool { name: "set_track_trim_db", description: "Sets the trim of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_db": Object {"description": String("The desired trim in dB. Valid range: -20.0 to 20.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_db")], "title": String("SetTrackTrimDBArgs"), "type": String("object")} }, Tool { name: "set_strip_plugin_parameter", description: "Sets a specific parameter of a plugin on a strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"param_id": Object {"description": String("The 1-indexed ID of the parameter within the plugin."), "format": String("int32"), "type": String("integer")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "value": Object {"description": String("The desired parameter value, normalized (0.0 to 1.0)."), "format": String("float"), "type": String("number")}}, "required": Array [String("param_id"), String("plugin_slot"), String("rid"), String("value")], "title": String("SetStripPluginParameterArgs"), "type": String("object")} }, Tool { name: "save_state", description: "Saves the current session state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "redo", description: "Redoes the last undone action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "undo", description: "Undoes the last action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_rec_enable", description: "Sets the record enable state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rec_st": Object {"description": String("The desired record enable state. 0 for off, 1 for on."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rec_st"), String("rid")], "title": String("SetTrackRecEnableArgs"), "type": String("object")} }, Tool { name: "transport_stop", description: "Stops Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "ffwd", description: "Fast forwards the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "locate", description: "Locates the playhead to a specific sample position and optionally starts playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"roll": Object {"description": String("Whether to start playing after locating. 0 for stop, 1 for play."), "format": String("int32"), "type": String("integer")}, "spos": Object {"description": String("The position in samples to locate to."), "format": String("int64"), "type": String("integer")}}, "required": Array [String("roll"), String("spos")], "title": String("LocateToolArgs"), "type": String("object")} }, Tool { name: "set_strip_plugin_active", description: "Activates or deactivates a plugin on a specific strip slot.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"active_state": Object {"description": String("The desired activation state (true for active, false for inactive)."), "type": String("boolean")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("active_state"), String("plugin_slot"), String("rid")], "title": String("SetStripPluginActiveArgs"), "type": String("object")} }, Tool { name: "toggle_punch_in", description: "Toggles the Punch In state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "toggle_punch_out", description: "Toggles the Punch Out state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "goto_end", description: "Moves the playhead to the session end.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "add_marker", description: "Adds a location marker at the current playhead position.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_mute", description: "Sets the mute state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"mute_state": Object {"description": String("The desired mute state (true for mute, false for unmute)."), "type": String("boolean")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("mute_state"), String("rid")], "title": String("SetTrackMuteArgs"), "type": String("object")} }, Tool { name: "set_track_solo", description: "Sets the solo state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "solo_st": Object {"description": String("The desired solo state. 0 for solo off, 1 for solo on."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rid"), String("solo_st")], "title": String("SetTrackSoloArgs"), "type": String("object")} }, Tool { name: "toggle_all_rec_enables", description: "Toggles the record enable state for ALL tracks.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_gain_db", description: "Sets the gain of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_db": Object {"description": String("The desired gain in dB. Valid range: -400.0 to 6.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_db"), String("rid")], "title": String("SetTrackGainDBArgs"), "type": String("object")} }, Tool { name: "rec_enable_toggle", description: "Toggles the master record enable or selected track record enable.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_transport_speed", description: "Sets Ardour's transport speed. Valid range: -8.0 to 8.0.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"speed": Object {"description": String("The desired transport speed. Valid range: -8.0 to 8.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("speed")], "title": String("SetTransportSpeedArgs"), "type": String("object")} }, Tool { name: "loop_toggle", description: "Toggles loop playback mode.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_selected_strip_pan_stereo_width", description: "Sets the stereo width for the currently selected stereo strip's panner.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"width": Object {"description": String("The desired stereo width for the currently selected strip. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("width")], "title": String("SetSelectedStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "set_track_trim_abs", description: "Sets the absolute trim of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_abs": Object {"description": String("The desired absolute trim. Valid range: 0.1 to 10.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_abs")], "title": String("SetTrackTrimAbsArgs"), "type": String("object")} }, Tool { name: "select_strip", description: "Selects a specific strip (track/bus) in Ardour.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus to select."), "format": String("int32"), "type": String("integer")}, "select_state": Object {"description": String("The desired select state (true to select). Currently, only true (1) is effective for selection."), "type": String("boolean")}}, "required": Array [String("rid"), String("select_state")], "title": String("SelectStripArgs"), "type": String("object")} }, Tool { name: "set_strip_pan_stereo_width", description: "Sets the stereo width for a panner on a stereo strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the stereo track/bus."), "format": String("int32"), "type": String("integer")}, "width": Object {"description": String("The desired stereo width. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("width")], "title": String("SetStripPanStereoWidthArgs"), "type": String("object")} }] }), Number(1))) +2026-06-30T13:22:44.935550Z  INFO ardour_mcp_server: +====================================================================== +NEW SERVER RUN: Tue, 30 Jun 2026 15:22:44 +0200 +====================================================================== +2026-06-30T13:22:44.935581Z  INFO ardour_mcp_server: Attempting to create OSC sender for Ardour at 127.0.0.1:3819 +2026-06-30T13:22:44.935607Z  INFO ardour_mcp_server: OSC sender created and connected to Ardour at 127.0.0.1:3819 +2026-06-30T13:22:44.935609Z  INFO ardour_mcp_server: Sending /set_surface to Ardour to enable OSC feedback. +2026-06-30T13:22:44.935612Z  INFO ardour_mcp_server: Targeting feedback port: 9099 +2026-06-30T13:22:44.935712Z  INFO ardour_mcp_server: /set_surface command sent successfully to Ardour. +2026-06-30T13:22:44.995833Z  INFO serve_inner: rmcp::service: Service initialized as server peer_info=InitializeRequestParam { protocol_version: ProtocolVersion("2025-11-25"), capabilities: ClientCapabilities { experimental: None, roots: Some(RootsCapabilities { list_changed: None }), sampling: None }, client_info: Implementation { name: "claude-code", version: "2.1.196" } } +2026-06-30T13:22:44.995883Z  INFO ardour_mcp_server: Ardour MCP server started and waiting for connections... +2026-06-30T13:22:44.995911Z DEBUG rmcp::service: new event evt=PeerMessage(Request(ListToolsRequest(Request { method: ListToolsRequestMethod, params: None }), Number(1))) +2026-06-30T13:22:44.995921Z  INFO ardour_mcp_server: Starting OSC listener for Ardour events on 127.0.0.1:9099 +2026-06-30T13:22:44.995924Z  INFO rmcp::service: received request id=1 request=ListToolsRequest(Request { method: ListToolsRequestMethod, params: None }) +2026-06-30T13:22:44.995950Z DEBUG rmcp::service: new event evt=PeerMessage(Request(ListResourcesRequest(Request { method: ListResourcesRequestMethod, params: None }), Number(2))) +2026-06-30T13:22:44.995952Z  INFO rmcp::service: received request id=2 request=ListResourcesRequest(Request { method: ListResourcesRequestMethod, params: None }) +2026-06-30T13:22:44.995963Z DEBUG ardour_mcp_server: Listing resources. Count: 2, Content: [Annotated { raw: RawResource { uri: "ardour:/state/playback", name: "Ardour Playback State", description: Some("Current playback state of Ardour (e.g., Playing, Stopped, Unknown)."), mime_type: None, size: None }, annotations: None }, Annotated { raw: RawResource { uri: "ardour:/state/transport_frame", name: "Ardour Transport Frame Position", description: Some("Current playhead position in samples. Returns 'Unknown' if not yet reported by Ardour."), mime_type: None, size: None }, annotations: None }] +2026-06-30T13:22:44.995976Z  INFO rmcp::service: response message id=2 result=ListResourcesResult(ListResourcesResult { next_cursor: None, resources: [Annotated { raw: RawResource { uri: "ardour:/state/playback", name: "Ardour Playback State", description: Some("Current playback state of Ardour (e.g., Playing, Stopped, Unknown)."), mime_type: None, size: None }, annotations: None }, Annotated { raw: RawResource { uri: "ardour:/state/transport_frame", name: "Ardour Transport Frame Position", description: Some("Current playhead position in samples. Returns 'Unknown' if not yet reported by Ardour."), mime_type: None, size: None }, annotations: None }] }) +2026-06-30T13:22:44.995983Z DEBUG rmcp::service: new event evt=ToSink(Response(ListResourcesResult(ListResourcesResult { next_cursor: None, resources: [Annotated { raw: RawResource { uri: "ardour:/state/playback", name: "Ardour Playback State", description: Some("Current playback state of Ardour (e.g., Playing, Stopped, Unknown)."), mime_type: None, size: None }, annotations: None }, Annotated { raw: RawResource { uri: "ardour:/state/transport_frame", name: "Ardour Transport Frame Position", description: Some("Current playhead position in samples. Returns 'Unknown' if not yet reported by Ardour."), mime_type: None, size: None }, annotations: None }] }), Number(2))) +2026-06-30T13:22:44.995982Z ERROR ardour_mcp_server: OSC listener task failed: Failed to bind Tokio UDP socket for OSC on 127.0.0.1:9099: Address already in use (os error 98) +2026-06-30T13:22:44.996110Z  INFO rmcp::service: response message id=1 result=ListToolsResult(ListToolsResult { next_cursor: None, tools: [Tool { name: "access_action", description: "Executes a specified Ardour menu action by its name.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"action_name": Object {"description": String("The name of the Ardour menu action to execute (e.g., 'Editor/zoom-to-session')."), "type": String("string")}}, "required": Array [String("action_name")], "title": String("AccessActionArgs"), "type": String("object")} }, Tool { name: "select_strip", description: "Selects a specific strip (track/bus) in Ardour.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus to select."), "format": String("int32"), "type": String("integer")}, "select_state": Object {"description": String("The desired select state (true to select). Currently, only true (1) is effective for selection."), "type": String("boolean")}}, "required": Array [String("rid"), String("select_state")], "title": String("SelectStripArgs"), "type": String("object")} }, Tool { name: "set_strip_plugin_parameter", description: "Sets a specific parameter of a plugin on a strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"param_id": Object {"description": String("The 1-indexed ID of the parameter within the plugin."), "format": String("int32"), "type": String("integer")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "value": Object {"description": String("The desired parameter value, normalized (0.0 to 1.0)."), "format": String("float"), "type": String("number")}}, "required": Array [String("param_id"), String("plugin_slot"), String("rid"), String("value")], "title": String("SetStripPluginParameterArgs"), "type": String("object")} }, Tool { name: "next_marker", description: "Moves the playhead to the next location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_mute", description: "Sets the mute state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"mute_state": Object {"description": String("The desired mute state (true for mute, false for unmute)."), "type": String("boolean")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("mute_state"), String("rid")], "title": String("SetTrackMuteArgs"), "type": String("object")} }, Tool { name: "set_track_gain_abs", description: "Sets the absolute gain of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_abs": Object {"description": String("The desired absolute gain. Valid range: 0.0 to 2.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_abs"), String("rid")], "title": String("SetTrackGainAbsArgs"), "type": String("object")} }, Tool { name: "locate", description: "Locates the playhead to a specific sample position and optionally starts playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"roll": Object {"description": String("Whether to start playing after locating. 0 for stop, 1 for play."), "format": String("int32"), "type": String("integer")}, "spos": Object {"description": String("The position in samples to locate to."), "format": String("int64"), "type": String("integer")}}, "required": Array [String("roll"), String("spos")], "title": String("LocateToolArgs"), "type": String("object")} }, Tool { name: "toggle_all_rec_enables", description: "Toggles the record enable state for ALL tracks.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "prev_marker", description: "Moves the playhead to the previous location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "add_marker", description: "Adds a location marker at the current playhead position.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "transport_play", description: "Starts Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_trim_abs", description: "Sets the absolute trim of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_abs": Object {"description": String("The desired absolute trim. Valid range: 0.1 to 10.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_abs")], "title": String("SetTrackTrimAbsArgs"), "type": String("object")} }, Tool { name: "set_strip_plugin_active", description: "Activates or deactivates a plugin on a specific strip slot.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"active_state": Object {"description": String("The desired activation state (true for active, false for inactive)."), "type": String("boolean")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("active_state"), String("plugin_slot"), String("rid")], "title": String("SetStripPluginActiveArgs"), "type": String("object")} }, Tool { name: "redo", description: "Redoes the last undone action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "toggle_punch_out", description: "Toggles the Punch Out state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "rewind", description: "Rewinds the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "rec_enable_toggle", description: "Toggles the master record enable or selected track record enable.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_strip_pan_stereo_width", description: "Sets the stereo width for a panner on a stereo strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the stereo track/bus."), "format": String("int32"), "type": String("integer")}, "width": Object {"description": String("The desired stereo width. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("width")], "title": String("SetStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "set_track_rec_enable", description: "Sets the record enable state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rec_st": Object {"description": String("The desired record enable state. 0 for off, 1 for on."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rec_st"), String("rid")], "title": String("SetTrackRecEnableArgs"), "type": String("object")} }, Tool { name: "undo", description: "Undoes the last action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_selected_strip_pan_stereo_width", description: "Sets the stereo width for the currently selected stereo strip's panner.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"width": Object {"description": String("The desired stereo width for the currently selected strip. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("width")], "title": String("SetSelectedStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "set_track_solo", description: "Sets the solo state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "solo_st": Object {"description": String("The desired solo state. 0 for solo off, 1 for solo on."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rid"), String("solo_st")], "title": String("SetTrackSoloArgs"), "type": String("object")} }, Tool { name: "save_state", description: "Saves the current session state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_gain_db", description: "Sets the gain of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_db": Object {"description": String("The desired gain in dB. Valid range: -400.0 to 6.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_db"), String("rid")], "title": String("SetTrackGainDBArgs"), "type": String("object")} }, Tool { name: "set_transport_speed", description: "Sets Ardour's transport speed. Valid range: -8.0 to 8.0.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"speed": Object {"description": String("The desired transport speed. Valid range: -8.0 to 8.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("speed")], "title": String("SetTransportSpeedArgs"), "type": String("object")} }, Tool { name: "goto_end", description: "Moves the playhead to the session end.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "transport_stop", description: "Stops Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "goto_start", description: "Moves the playhead to the session start.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "loop_toggle", description: "Toggles loop playback mode.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "toggle_punch_in", description: "Toggles the Punch In state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_trim_db", description: "Sets the trim of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_db": Object {"description": String("The desired trim in dB. Valid range: -20.0 to 20.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_db")], "title": String("SetTrackTrimDBArgs"), "type": String("object")} }, Tool { name: "ffwd", description: "Fast forwards the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }] }) +2026-06-30T13:22:44.996180Z DEBUG rmcp::service: new event evt=ToSink(Response(ListToolsResult(ListToolsResult { next_cursor: None, tools: [Tool { name: "access_action", description: "Executes a specified Ardour menu action by its name.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"action_name": Object {"description": String("The name of the Ardour menu action to execute (e.g., 'Editor/zoom-to-session')."), "type": String("string")}}, "required": Array [String("action_name")], "title": String("AccessActionArgs"), "type": String("object")} }, Tool { name: "select_strip", description: "Selects a specific strip (track/bus) in Ardour.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus to select."), "format": String("int32"), "type": String("integer")}, "select_state": Object {"description": String("The desired select state (true to select). Currently, only true (1) is effective for selection."), "type": String("boolean")}}, "required": Array [String("rid"), String("select_state")], "title": String("SelectStripArgs"), "type": String("object")} }, Tool { name: "set_strip_plugin_parameter", description: "Sets a specific parameter of a plugin on a strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"param_id": Object {"description": String("The 1-indexed ID of the parameter within the plugin."), "format": String("int32"), "type": String("integer")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "value": Object {"description": String("The desired parameter value, normalized (0.0 to 1.0)."), "format": String("float"), "type": String("number")}}, "required": Array [String("param_id"), String("plugin_slot"), String("rid"), String("value")], "title": String("SetStripPluginParameterArgs"), "type": String("object")} }, Tool { name: "next_marker", description: "Moves the playhead to the next location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_mute", description: "Sets the mute state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"mute_state": Object {"description": String("The desired mute state (true for mute, false for unmute)."), "type": String("boolean")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("mute_state"), String("rid")], "title": String("SetTrackMuteArgs"), "type": String("object")} }, Tool { name: "set_track_gain_abs", description: "Sets the absolute gain of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_abs": Object {"description": String("The desired absolute gain. Valid range: 0.0 to 2.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_abs"), String("rid")], "title": String("SetTrackGainAbsArgs"), "type": String("object")} }, Tool { name: "locate", description: "Locates the playhead to a specific sample position and optionally starts playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"roll": Object {"description": String("Whether to start playing after locating. 0 for stop, 1 for play."), "format": String("int32"), "type": String("integer")}, "spos": Object {"description": String("The position in samples to locate to."), "format": String("int64"), "type": String("integer")}}, "required": Array [String("roll"), String("spos")], "title": String("LocateToolArgs"), "type": String("object")} }, Tool { name: "toggle_all_rec_enables", description: "Toggles the record enable state for ALL tracks.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "prev_marker", description: "Moves the playhead to the previous location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "add_marker", description: "Adds a location marker at the current playhead position.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "transport_play", description: "Starts Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_trim_abs", description: "Sets the absolute trim of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_abs": Object {"description": String("The desired absolute trim. Valid range: 0.1 to 10.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_abs")], "title": String("SetTrackTrimAbsArgs"), "type": String("object")} }, Tool { name: "set_strip_plugin_active", description: "Activates or deactivates a plugin on a specific strip slot.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"active_state": Object {"description": String("The desired activation state (true for active, false for inactive)."), "type": String("boolean")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("active_state"), String("plugin_slot"), String("rid")], "title": String("SetStripPluginActiveArgs"), "type": String("object")} }, Tool { name: "redo", description: "Redoes the last undone action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "toggle_punch_out", description: "Toggles the Punch Out state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "rewind", description: "Rewinds the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "rec_enable_toggle", description: "Toggles the master record enable or selected track record enable.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_strip_pan_stereo_width", description: "Sets the stereo width for a panner on a stereo strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the stereo track/bus."), "format": String("int32"), "type": String("integer")}, "width": Object {"description": String("The desired stereo width. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("width")], "title": String("SetStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "set_track_rec_enable", description: "Sets the record enable state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rec_st": Object {"description": String("The desired record enable state. 0 for off, 1 for on."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rec_st"), String("rid")], "title": String("SetTrackRecEnableArgs"), "type": String("object")} }, Tool { name: "undo", description: "Undoes the last action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_selected_strip_pan_stereo_width", description: "Sets the stereo width for the currently selected stereo strip's panner.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"width": Object {"description": String("The desired stereo width for the currently selected strip. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("width")], "title": String("SetSelectedStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "set_track_solo", description: "Sets the solo state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "solo_st": Object {"description": String("The desired solo state. 0 for solo off, 1 for solo on."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rid"), String("solo_st")], "title": String("SetTrackSoloArgs"), "type": String("object")} }, Tool { name: "save_state", description: "Saves the current session state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_gain_db", description: "Sets the gain of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_db": Object {"description": String("The desired gain in dB. Valid range: -400.0 to 6.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_db"), String("rid")], "title": String("SetTrackGainDBArgs"), "type": String("object")} }, Tool { name: "set_transport_speed", description: "Sets Ardour's transport speed. Valid range: -8.0 to 8.0.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"speed": Object {"description": String("The desired transport speed. Valid range: -8.0 to 8.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("speed")], "title": String("SetTransportSpeedArgs"), "type": String("object")} }, Tool { name: "goto_end", description: "Moves the playhead to the session end.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "transport_stop", description: "Stops Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "goto_start", description: "Moves the playhead to the session start.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "loop_toggle", description: "Toggles loop playback mode.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "toggle_punch_in", description: "Toggles the Punch In state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_trim_db", description: "Sets the trim of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_db": Object {"description": String("The desired trim in dB. Valid range: -20.0 to 20.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_db")], "title": String("SetTrackTrimDBArgs"), "type": String("object")} }, Tool { name: "ffwd", description: "Fast forwards the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }] }), Number(1))) +2026-06-30T13:40:56.928934Z  INFO ardour_mcp_server: +====================================================================== +NEW SERVER RUN: Tue, 30 Jun 2026 15:40:56 +0200 +====================================================================== +2026-06-30T13:40:56.928978Z  INFO ardour_mcp_server: Attempting to create OSC sender for Ardour at 127.0.0.1:3819 +2026-06-30T13:40:56.929007Z  INFO ardour_mcp_server: OSC sender created and connected to Ardour at 127.0.0.1:3819 +2026-06-30T13:40:56.929009Z  INFO ardour_mcp_server: Sending /set_surface to Ardour to enable OSC feedback. +2026-06-30T13:40:56.929012Z  INFO ardour_mcp_server: Targeting feedback port: 9099 +2026-06-30T13:40:56.929130Z  INFO ardour_mcp_server: /set_surface command sent successfully to Ardour. +2026-06-30T13:40:57.025963Z  INFO serve_inner: rmcp::service: Service initialized as server peer_info=InitializeRequestParam { protocol_version: ProtocolVersion("2025-11-25"), capabilities: ClientCapabilities { experimental: None, roots: Some(RootsCapabilities { list_changed: None }), sampling: None }, client_info: Implementation { name: "claude-code", version: "2.1.196" } } +2026-06-30T13:40:57.026009Z  INFO ardour_mcp_server: Ardour MCP server started and waiting for connections... +2026-06-30T13:40:57.026037Z DEBUG rmcp::service: new event evt=PeerMessage(Request(ListToolsRequest(Request { method: ListToolsRequestMethod, params: None }), Number(1))) +2026-06-30T13:40:57.026039Z  INFO ardour_mcp_server: Starting OSC listener for Ardour events on 127.0.0.1:9099 +2026-06-30T13:40:57.026052Z  INFO rmcp::service: received request id=1 request=ListToolsRequest(Request { method: ListToolsRequestMethod, params: None }) +2026-06-30T13:40:57.026077Z DEBUG rmcp::service: new event evt=PeerMessage(Request(ListResourcesRequest(Request { method: ListResourcesRequestMethod, params: None }), Number(2))) +2026-06-30T13:40:57.026080Z  INFO rmcp::service: received request id=2 request=ListResourcesRequest(Request { method: ListResourcesRequestMethod, params: None }) +2026-06-30T13:40:57.026090Z DEBUG ardour_mcp_server: Listing resources. Count: 2, Content: [Annotated { raw: RawResource { uri: "ardour:/state/playback", name: "Ardour Playback State", description: Some("Current playback state of Ardour (e.g., Playing, Stopped, Unknown)."), mime_type: None, size: None }, annotations: None }, Annotated { raw: RawResource { uri: "ardour:/state/transport_frame", name: "Ardour Transport Frame Position", description: Some("Current playhead position in samples. Returns 'Unknown' if not yet reported by Ardour."), mime_type: None, size: None }, annotations: None }] +2026-06-30T13:40:57.026097Z  INFO rmcp::service: response message id=2 result=ListResourcesResult(ListResourcesResult { next_cursor: None, resources: [Annotated { raw: RawResource { uri: "ardour:/state/playback", name: "Ardour Playback State", description: Some("Current playback state of Ardour (e.g., Playing, Stopped, Unknown)."), mime_type: None, size: None }, annotations: None }, Annotated { raw: RawResource { uri: "ardour:/state/transport_frame", name: "Ardour Transport Frame Position", description: Some("Current playhead position in samples. Returns 'Unknown' if not yet reported by Ardour."), mime_type: None, size: None }, annotations: None }] }) +2026-06-30T13:40:57.026106Z ERROR ardour_mcp_server: OSC listener task failed: Failed to bind Tokio UDP socket for OSC on 127.0.0.1:9099: Address already in use (os error 98) +2026-06-30T13:40:57.026112Z DEBUG rmcp::service: new event evt=ToSink(Response(ListResourcesResult(ListResourcesResult { next_cursor: None, resources: [Annotated { raw: RawResource { uri: "ardour:/state/playback", name: "Ardour Playback State", description: Some("Current playback state of Ardour (e.g., Playing, Stopped, Unknown)."), mime_type: None, size: None }, annotations: None }, Annotated { raw: RawResource { uri: "ardour:/state/transport_frame", name: "Ardour Transport Frame Position", description: Some("Current playhead position in samples. Returns 'Unknown' if not yet reported by Ardour."), mime_type: None, size: None }, annotations: None }] }), Number(2))) +2026-06-30T13:40:57.026187Z  INFO rmcp::service: response message id=1 result=ListToolsResult(ListToolsResult { next_cursor: None, tools: [Tool { name: "set_track_trim_db", description: "Sets the trim of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_db": Object {"description": String("The desired trim in dB. Valid range: -20.0 to 20.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_db")], "title": String("SetTrackTrimDBArgs"), "type": String("object")} }, Tool { name: "select_strip", description: "Selects a specific strip (track/bus) in Ardour.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus to select."), "format": String("int32"), "type": String("integer")}, "select_state": Object {"description": String("The desired select state (true to select). Currently, only true (1) is effective for selection."), "type": String("boolean")}}, "required": Array [String("rid"), String("select_state")], "title": String("SelectStripArgs"), "type": String("object")} }, Tool { name: "access_action", description: "Executes a specified Ardour menu action by its name.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"action_name": Object {"description": String("The name of the Ardour menu action to execute (e.g., 'Editor/zoom-to-session')."), "type": String("string")}}, "required": Array [String("action_name")], "title": String("AccessActionArgs"), "type": String("object")} }, Tool { name: "goto_end", description: "Moves the playhead to the session end.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_mute", description: "Sets the mute state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"mute_state": Object {"description": String("The desired mute state (true for mute, false for unmute)."), "type": String("boolean")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("mute_state"), String("rid")], "title": String("SetTrackMuteArgs"), "type": String("object")} }, Tool { name: "set_strip_plugin_active", description: "Activates or deactivates a plugin on a specific strip slot.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"active_state": Object {"description": String("The desired activation state (true for active, false for inactive)."), "type": String("boolean")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("active_state"), String("plugin_slot"), String("rid")], "title": String("SetStripPluginActiveArgs"), "type": String("object")} }, Tool { name: "rewind", description: "Rewinds the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "toggle_punch_in", description: "Toggles the Punch In state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_selected_strip_pan_stereo_width", description: "Sets the stereo width for the currently selected stereo strip's panner.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"width": Object {"description": String("The desired stereo width for the currently selected strip. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("width")], "title": String("SetSelectedStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "rec_enable_toggle", description: "Toggles the master record enable or selected track record enable.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "add_marker", description: "Adds a location marker at the current playhead position.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_trim_abs", description: "Sets the absolute trim of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_abs": Object {"description": String("The desired absolute trim. Valid range: 0.1 to 10.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_abs")], "title": String("SetTrackTrimAbsArgs"), "type": String("object")} }, Tool { name: "set_track_gain_db", description: "Sets the gain of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_db": Object {"description": String("The desired gain in dB. Valid range: -400.0 to 6.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_db"), String("rid")], "title": String("SetTrackGainDBArgs"), "type": String("object")} }, Tool { name: "set_track_gain_abs", description: "Sets the absolute gain of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_abs": Object {"description": String("The desired absolute gain. Valid range: 0.0 to 2.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_abs"), String("rid")], "title": String("SetTrackGainAbsArgs"), "type": String("object")} }, Tool { name: "prev_marker", description: "Moves the playhead to the previous location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "goto_start", description: "Moves the playhead to the session start.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "toggle_all_rec_enables", description: "Toggles the record enable state for ALL tracks.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_transport_speed", description: "Sets Ardour's transport speed. Valid range: -8.0 to 8.0.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"speed": Object {"description": String("The desired transport speed. Valid range: -8.0 to 8.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("speed")], "title": String("SetTransportSpeedArgs"), "type": String("object")} }, Tool { name: "transport_play", description: "Starts Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "toggle_punch_out", description: "Toggles the Punch Out state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "ffwd", description: "Fast forwards the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "save_state", description: "Saves the current session state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "undo", description: "Undoes the last action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "transport_stop", description: "Stops Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_solo", description: "Sets the solo state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "solo_st": Object {"description": String("The desired solo state. 0 for solo off, 1 for solo on."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rid"), String("solo_st")], "title": String("SetTrackSoloArgs"), "type": String("object")} }, Tool { name: "loop_toggle", description: "Toggles loop playback mode.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "redo", description: "Redoes the last undone action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_rec_enable", description: "Sets the record enable state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rec_st": Object {"description": String("The desired record enable state. 0 for off, 1 for on."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rec_st"), String("rid")], "title": String("SetTrackRecEnableArgs"), "type": String("object")} }, Tool { name: "next_marker", description: "Moves the playhead to the next location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "locate", description: "Locates the playhead to a specific sample position and optionally starts playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"roll": Object {"description": String("Whether to start playing after locating. 0 for stop, 1 for play."), "format": String("int32"), "type": String("integer")}, "spos": Object {"description": String("The position in samples to locate to."), "format": String("int64"), "type": String("integer")}}, "required": Array [String("roll"), String("spos")], "title": String("LocateToolArgs"), "type": String("object")} }, Tool { name: "set_strip_plugin_parameter", description: "Sets a specific parameter of a plugin on a strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"param_id": Object {"description": String("The 1-indexed ID of the parameter within the plugin."), "format": String("int32"), "type": String("integer")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "value": Object {"description": String("The desired parameter value, normalized (0.0 to 1.0)."), "format": String("float"), "type": String("number")}}, "required": Array [String("param_id"), String("plugin_slot"), String("rid"), String("value")], "title": String("SetStripPluginParameterArgs"), "type": String("object")} }, Tool { name: "set_strip_pan_stereo_width", description: "Sets the stereo width for a panner on a stereo strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the stereo track/bus."), "format": String("int32"), "type": String("integer")}, "width": Object {"description": String("The desired stereo width. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("width")], "title": String("SetStripPanStereoWidthArgs"), "type": String("object")} }] }) +2026-06-30T13:40:57.026266Z DEBUG rmcp::service: new event evt=ToSink(Response(ListToolsResult(ListToolsResult { next_cursor: None, tools: [Tool { name: "set_track_trim_db", description: "Sets the trim of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_db": Object {"description": String("The desired trim in dB. Valid range: -20.0 to 20.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_db")], "title": String("SetTrackTrimDBArgs"), "type": String("object")} }, Tool { name: "select_strip", description: "Selects a specific strip (track/bus) in Ardour.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus to select."), "format": String("int32"), "type": String("integer")}, "select_state": Object {"description": String("The desired select state (true to select). Currently, only true (1) is effective for selection."), "type": String("boolean")}}, "required": Array [String("rid"), String("select_state")], "title": String("SelectStripArgs"), "type": String("object")} }, Tool { name: "access_action", description: "Executes a specified Ardour menu action by its name.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"action_name": Object {"description": String("The name of the Ardour menu action to execute (e.g., 'Editor/zoom-to-session')."), "type": String("string")}}, "required": Array [String("action_name")], "title": String("AccessActionArgs"), "type": String("object")} }, Tool { name: "goto_end", description: "Moves the playhead to the session end.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_mute", description: "Sets the mute state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"mute_state": Object {"description": String("The desired mute state (true for mute, false for unmute)."), "type": String("boolean")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("mute_state"), String("rid")], "title": String("SetTrackMuteArgs"), "type": String("object")} }, Tool { name: "set_strip_plugin_active", description: "Activates or deactivates a plugin on a specific strip slot.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"active_state": Object {"description": String("The desired activation state (true for active, false for inactive)."), "type": String("boolean")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("active_state"), String("plugin_slot"), String("rid")], "title": String("SetStripPluginActiveArgs"), "type": String("object")} }, Tool { name: "rewind", description: "Rewinds the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "toggle_punch_in", description: "Toggles the Punch In state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_selected_strip_pan_stereo_width", description: "Sets the stereo width for the currently selected stereo strip's panner.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"width": Object {"description": String("The desired stereo width for the currently selected strip. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("width")], "title": String("SetSelectedStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "rec_enable_toggle", description: "Toggles the master record enable or selected track record enable.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "add_marker", description: "Adds a location marker at the current playhead position.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_trim_abs", description: "Sets the absolute trim of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_abs": Object {"description": String("The desired absolute trim. Valid range: 0.1 to 10.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_abs")], "title": String("SetTrackTrimAbsArgs"), "type": String("object")} }, Tool { name: "set_track_gain_db", description: "Sets the gain of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_db": Object {"description": String("The desired gain in dB. Valid range: -400.0 to 6.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_db"), String("rid")], "title": String("SetTrackGainDBArgs"), "type": String("object")} }, Tool { name: "set_track_gain_abs", description: "Sets the absolute gain of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_abs": Object {"description": String("The desired absolute gain. Valid range: 0.0 to 2.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_abs"), String("rid")], "title": String("SetTrackGainAbsArgs"), "type": String("object")} }, Tool { name: "prev_marker", description: "Moves the playhead to the previous location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "goto_start", description: "Moves the playhead to the session start.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "toggle_all_rec_enables", description: "Toggles the record enable state for ALL tracks.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_transport_speed", description: "Sets Ardour's transport speed. Valid range: -8.0 to 8.0.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"speed": Object {"description": String("The desired transport speed. Valid range: -8.0 to 8.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("speed")], "title": String("SetTransportSpeedArgs"), "type": String("object")} }, Tool { name: "transport_play", description: "Starts Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "toggle_punch_out", description: "Toggles the Punch Out state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "ffwd", description: "Fast forwards the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "save_state", description: "Saves the current session state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "undo", description: "Undoes the last action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "transport_stop", description: "Stops Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_solo", description: "Sets the solo state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "solo_st": Object {"description": String("The desired solo state. 0 for solo off, 1 for solo on."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rid"), String("solo_st")], "title": String("SetTrackSoloArgs"), "type": String("object")} }, Tool { name: "loop_toggle", description: "Toggles loop playback mode.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "redo", description: "Redoes the last undone action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_rec_enable", description: "Sets the record enable state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rec_st": Object {"description": String("The desired record enable state. 0 for off, 1 for on."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rec_st"), String("rid")], "title": String("SetTrackRecEnableArgs"), "type": String("object")} }, Tool { name: "next_marker", description: "Moves the playhead to the next location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "locate", description: "Locates the playhead to a specific sample position and optionally starts playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"roll": Object {"description": String("Whether to start playing after locating. 0 for stop, 1 for play."), "format": String("int32"), "type": String("integer")}, "spos": Object {"description": String("The position in samples to locate to."), "format": String("int64"), "type": String("integer")}}, "required": Array [String("roll"), String("spos")], "title": String("LocateToolArgs"), "type": String("object")} }, Tool { name: "set_strip_plugin_parameter", description: "Sets a specific parameter of a plugin on a strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"param_id": Object {"description": String("The 1-indexed ID of the parameter within the plugin."), "format": String("int32"), "type": String("integer")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "value": Object {"description": String("The desired parameter value, normalized (0.0 to 1.0)."), "format": String("float"), "type": String("number")}}, "required": Array [String("param_id"), String("plugin_slot"), String("rid"), String("value")], "title": String("SetStripPluginParameterArgs"), "type": String("object")} }, Tool { name: "set_strip_pan_stereo_width", description: "Sets the stereo width for a panner on a stereo strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the stereo track/bus."), "format": String("int32"), "type": String("integer")}, "width": Object {"description": String("The desired stereo width. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("width")], "title": String("SetStripPanStereoWidthArgs"), "type": String("object")} }] }), Number(1))) +2026-06-30T14:55:29.328306Z  INFO ardour_mcp_server: +====================================================================== +NEW SERVER RUN: Tue, 30 Jun 2026 16:55:29 +0200 +====================================================================== +2026-06-30T14:55:29.328347Z  INFO ardour_mcp_server: Attempting to create OSC sender for Ardour at 127.0.0.1:3819 +2026-06-30T14:55:29.328382Z  INFO ardour_mcp_server: OSC sender created and connected to Ardour at 127.0.0.1:3819 +2026-06-30T14:55:29.328385Z  INFO ardour_mcp_server: Sending /set_surface to Ardour to enable OSC feedback. +2026-06-30T14:55:29.328389Z  INFO ardour_mcp_server: Targeting feedback port: 9099 +2026-06-30T14:55:29.328556Z  INFO ardour_mcp_server: /set_surface command sent successfully to Ardour. +2026-06-30T14:55:29.408351Z  INFO serve_inner: rmcp::service: Service initialized as server peer_info=InitializeRequestParam { protocol_version: ProtocolVersion("2025-11-25"), capabilities: ClientCapabilities { experimental: None, roots: Some(RootsCapabilities { list_changed: None }), sampling: None }, client_info: Implementation { name: "claude-code", version: "2.1.196" } } +2026-06-30T14:55:29.408404Z  INFO ardour_mcp_server: Ardour MCP server started and waiting for connections... +2026-06-30T14:55:29.408453Z  INFO ardour_mcp_server: Starting OSC listener for Ardour events on 127.0.0.1:9099 +2026-06-30T14:55:29.408471Z DEBUG rmcp::service: new event evt=PeerMessage(Request(ListToolsRequest(Request { method: ListToolsRequestMethod, params: None }), Number(1))) +2026-06-30T14:55:29.408482Z  INFO rmcp::service: received request id=1 request=ListToolsRequest(Request { method: ListToolsRequestMethod, params: None }) +2026-06-30T14:55:29.408492Z ERROR ardour_mcp_server: OSC listener task failed: Failed to bind Tokio UDP socket for OSC on 127.0.0.1:9099: Address already in use (os error 98) +2026-06-30T14:55:29.408500Z DEBUG rmcp::service: new event evt=PeerMessage(Request(ListResourcesRequest(Request { method: ListResourcesRequestMethod, params: None }), Number(2))) +2026-06-30T14:55:29.408504Z  INFO rmcp::service: received request id=2 request=ListResourcesRequest(Request { method: ListResourcesRequestMethod, params: None }) +2026-06-30T14:55:29.408521Z DEBUG ardour_mcp_server: Listing resources. Count: 2, Content: [Annotated { raw: RawResource { uri: "ardour:/state/playback", name: "Ardour Playback State", description: Some("Current playback state of Ardour (e.g., Playing, Stopped, Unknown)."), mime_type: None, size: None }, annotations: None }, Annotated { raw: RawResource { uri: "ardour:/state/transport_frame", name: "Ardour Transport Frame Position", description: Some("Current playhead position in samples. Returns 'Unknown' if not yet reported by Ardour."), mime_type: None, size: None }, annotations: None }] +2026-06-30T14:55:29.408528Z  INFO rmcp::service: response message id=2 result=ListResourcesResult(ListResourcesResult { next_cursor: None, resources: [Annotated { raw: RawResource { uri: "ardour:/state/playback", name: "Ardour Playback State", description: Some("Current playback state of Ardour (e.g., Playing, Stopped, Unknown)."), mime_type: None, size: None }, annotations: None }, Annotated { raw: RawResource { uri: "ardour:/state/transport_frame", name: "Ardour Transport Frame Position", description: Some("Current playhead position in samples. Returns 'Unknown' if not yet reported by Ardour."), mime_type: None, size: None }, annotations: None }] }) +2026-06-30T14:55:29.408539Z DEBUG rmcp::service: new event evt=ToSink(Response(ListResourcesResult(ListResourcesResult { next_cursor: None, resources: [Annotated { raw: RawResource { uri: "ardour:/state/playback", name: "Ardour Playback State", description: Some("Current playback state of Ardour (e.g., Playing, Stopped, Unknown)."), mime_type: None, size: None }, annotations: None }, Annotated { raw: RawResource { uri: "ardour:/state/transport_frame", name: "Ardour Transport Frame Position", description: Some("Current playhead position in samples. Returns 'Unknown' if not yet reported by Ardour."), mime_type: None, size: None }, annotations: None }] }), Number(2))) +2026-06-30T14:55:29.408601Z  INFO rmcp::service: response message id=1 result=ListToolsResult(ListToolsResult { next_cursor: None, tools: [Tool { name: "undo", description: "Undoes the last action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "loop_toggle", description: "Toggles loop playback mode.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "goto_end", description: "Moves the playhead to the session end.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_gain_abs", description: "Sets the absolute gain of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_abs": Object {"description": String("The desired absolute gain. Valid range: 0.0 to 2.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_abs"), String("rid")], "title": String("SetTrackGainAbsArgs"), "type": String("object")} }, Tool { name: "set_transport_speed", description: "Sets Ardour's transport speed. Valid range: -8.0 to 8.0.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"speed": Object {"description": String("The desired transport speed. Valid range: -8.0 to 8.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("speed")], "title": String("SetTransportSpeedArgs"), "type": String("object")} }, Tool { name: "rec_enable_toggle", description: "Toggles the master record enable or selected track record enable.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_strip_plugin_parameter", description: "Sets a specific parameter of a plugin on a strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"param_id": Object {"description": String("The 1-indexed ID of the parameter within the plugin."), "format": String("int32"), "type": String("integer")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "value": Object {"description": String("The desired parameter value, normalized (0.0 to 1.0)."), "format": String("float"), "type": String("number")}}, "required": Array [String("param_id"), String("plugin_slot"), String("rid"), String("value")], "title": String("SetStripPluginParameterArgs"), "type": String("object")} }, Tool { name: "ffwd", description: "Fast forwards the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_selected_strip_pan_stereo_width", description: "Sets the stereo width for the currently selected stereo strip's panner.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"width": Object {"description": String("The desired stereo width for the currently selected strip. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("width")], "title": String("SetSelectedStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "set_track_rec_enable", description: "Sets the record enable state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rec_st": Object {"description": String("The desired record enable state. 0 for off, 1 for on."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rec_st"), String("rid")], "title": String("SetTrackRecEnableArgs"), "type": String("object")} }, Tool { name: "set_strip_plugin_active", description: "Activates or deactivates a plugin on a specific strip slot.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"active_state": Object {"description": String("The desired activation state (true for active, false for inactive)."), "type": String("boolean")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("active_state"), String("plugin_slot"), String("rid")], "title": String("SetStripPluginActiveArgs"), "type": String("object")} }, Tool { name: "set_track_solo", description: "Sets the solo state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "solo_st": Object {"description": String("The desired solo state. 0 for solo off, 1 for solo on."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rid"), String("solo_st")], "title": String("SetTrackSoloArgs"), "type": String("object")} }, Tool { name: "set_track_trim_db", description: "Sets the trim of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_db": Object {"description": String("The desired trim in dB. Valid range: -20.0 to 20.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_db")], "title": String("SetTrackTrimDBArgs"), "type": String("object")} }, Tool { name: "next_marker", description: "Moves the playhead to the next location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "locate", description: "Locates the playhead to a specific sample position and optionally starts playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"roll": Object {"description": String("Whether to start playing after locating. 0 for stop, 1 for play."), "format": String("int32"), "type": String("integer")}, "spos": Object {"description": String("The position in samples to locate to."), "format": String("int64"), "type": String("integer")}}, "required": Array [String("roll"), String("spos")], "title": String("LocateToolArgs"), "type": String("object")} }, Tool { name: "set_track_trim_abs", description: "Sets the absolute trim of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_abs": Object {"description": String("The desired absolute trim. Valid range: 0.1 to 10.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_abs")], "title": String("SetTrackTrimAbsArgs"), "type": String("object")} }, Tool { name: "transport_play", description: "Starts Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "goto_start", description: "Moves the playhead to the session start.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "add_marker", description: "Adds a location marker at the current playhead position.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "prev_marker", description: "Moves the playhead to the previous location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "transport_stop", description: "Stops Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "access_action", description: "Executes a specified Ardour menu action by its name.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"action_name": Object {"description": String("The name of the Ardour menu action to execute (e.g., 'Editor/zoom-to-session')."), "type": String("string")}}, "required": Array [String("action_name")], "title": String("AccessActionArgs"), "type": String("object")} }, Tool { name: "rewind", description: "Rewinds the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_strip_pan_stereo_width", description: "Sets the stereo width for a panner on a stereo strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the stereo track/bus."), "format": String("int32"), "type": String("integer")}, "width": Object {"description": String("The desired stereo width. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("width")], "title": String("SetStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "set_track_gain_db", description: "Sets the gain of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_db": Object {"description": String("The desired gain in dB. Valid range: -400.0 to 6.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_db"), String("rid")], "title": String("SetTrackGainDBArgs"), "type": String("object")} }, Tool { name: "select_strip", description: "Selects a specific strip (track/bus) in Ardour.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus to select."), "format": String("int32"), "type": String("integer")}, "select_state": Object {"description": String("The desired select state (true to select). Currently, only true (1) is effective for selection."), "type": String("boolean")}}, "required": Array [String("rid"), String("select_state")], "title": String("SelectStripArgs"), "type": String("object")} }, Tool { name: "toggle_punch_out", description: "Toggles the Punch Out state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "redo", description: "Redoes the last undone action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "save_state", description: "Saves the current session state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "toggle_punch_in", description: "Toggles the Punch In state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_mute", description: "Sets the mute state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"mute_state": Object {"description": String("The desired mute state (true for mute, false for unmute)."), "type": String("boolean")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("mute_state"), String("rid")], "title": String("SetTrackMuteArgs"), "type": String("object")} }, Tool { name: "toggle_all_rec_enables", description: "Toggles the record enable state for ALL tracks.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }] }) +2026-06-30T14:55:29.408712Z DEBUG rmcp::service: new event evt=ToSink(Response(ListToolsResult(ListToolsResult { next_cursor: None, tools: [Tool { name: "undo", description: "Undoes the last action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "loop_toggle", description: "Toggles loop playback mode.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "goto_end", description: "Moves the playhead to the session end.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_gain_abs", description: "Sets the absolute gain of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_abs": Object {"description": String("The desired absolute gain. Valid range: 0.0 to 2.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_abs"), String("rid")], "title": String("SetTrackGainAbsArgs"), "type": String("object")} }, Tool { name: "set_transport_speed", description: "Sets Ardour's transport speed. Valid range: -8.0 to 8.0.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"speed": Object {"description": String("The desired transport speed. Valid range: -8.0 to 8.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("speed")], "title": String("SetTransportSpeedArgs"), "type": String("object")} }, Tool { name: "rec_enable_toggle", description: "Toggles the master record enable or selected track record enable.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_strip_plugin_parameter", description: "Sets a specific parameter of a plugin on a strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"param_id": Object {"description": String("The 1-indexed ID of the parameter within the plugin."), "format": String("int32"), "type": String("integer")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "value": Object {"description": String("The desired parameter value, normalized (0.0 to 1.0)."), "format": String("float"), "type": String("number")}}, "required": Array [String("param_id"), String("plugin_slot"), String("rid"), String("value")], "title": String("SetStripPluginParameterArgs"), "type": String("object")} }, Tool { name: "ffwd", description: "Fast forwards the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_selected_strip_pan_stereo_width", description: "Sets the stereo width for the currently selected stereo strip's panner.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"width": Object {"description": String("The desired stereo width for the currently selected strip. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("width")], "title": String("SetSelectedStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "set_track_rec_enable", description: "Sets the record enable state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rec_st": Object {"description": String("The desired record enable state. 0 for off, 1 for on."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rec_st"), String("rid")], "title": String("SetTrackRecEnableArgs"), "type": String("object")} }, Tool { name: "set_strip_plugin_active", description: "Activates or deactivates a plugin on a specific strip slot.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"active_state": Object {"description": String("The desired activation state (true for active, false for inactive)."), "type": String("boolean")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("active_state"), String("plugin_slot"), String("rid")], "title": String("SetStripPluginActiveArgs"), "type": String("object")} }, Tool { name: "set_track_solo", description: "Sets the solo state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "solo_st": Object {"description": String("The desired solo state. 0 for solo off, 1 for solo on."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rid"), String("solo_st")], "title": String("SetTrackSoloArgs"), "type": String("object")} }, Tool { name: "set_track_trim_db", description: "Sets the trim of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_db": Object {"description": String("The desired trim in dB. Valid range: -20.0 to 20.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_db")], "title": String("SetTrackTrimDBArgs"), "type": String("object")} }, Tool { name: "next_marker", description: "Moves the playhead to the next location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "locate", description: "Locates the playhead to a specific sample position and optionally starts playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"roll": Object {"description": String("Whether to start playing after locating. 0 for stop, 1 for play."), "format": String("int32"), "type": String("integer")}, "spos": Object {"description": String("The position in samples to locate to."), "format": String("int64"), "type": String("integer")}}, "required": Array [String("roll"), String("spos")], "title": String("LocateToolArgs"), "type": String("object")} }, Tool { name: "set_track_trim_abs", description: "Sets the absolute trim of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_abs": Object {"description": String("The desired absolute trim. Valid range: 0.1 to 10.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_abs")], "title": String("SetTrackTrimAbsArgs"), "type": String("object")} }, Tool { name: "transport_play", description: "Starts Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "goto_start", description: "Moves the playhead to the session start.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "add_marker", description: "Adds a location marker at the current playhead position.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "prev_marker", description: "Moves the playhead to the previous location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "transport_stop", description: "Stops Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "access_action", description: "Executes a specified Ardour menu action by its name.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"action_name": Object {"description": String("The name of the Ardour menu action to execute (e.g., 'Editor/zoom-to-session')."), "type": String("string")}}, "required": Array [String("action_name")], "title": String("AccessActionArgs"), "type": String("object")} }, Tool { name: "rewind", description: "Rewinds the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_strip_pan_stereo_width", description: "Sets the stereo width for a panner on a stereo strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the stereo track/bus."), "format": String("int32"), "type": String("integer")}, "width": Object {"description": String("The desired stereo width. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("width")], "title": String("SetStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "set_track_gain_db", description: "Sets the gain of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_db": Object {"description": String("The desired gain in dB. Valid range: -400.0 to 6.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_db"), String("rid")], "title": String("SetTrackGainDBArgs"), "type": String("object")} }, Tool { name: "select_strip", description: "Selects a specific strip (track/bus) in Ardour.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus to select."), "format": String("int32"), "type": String("integer")}, "select_state": Object {"description": String("The desired select state (true to select). Currently, only true (1) is effective for selection."), "type": String("boolean")}}, "required": Array [String("rid"), String("select_state")], "title": String("SelectStripArgs"), "type": String("object")} }, Tool { name: "toggle_punch_out", description: "Toggles the Punch Out state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "redo", description: "Redoes the last undone action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "save_state", description: "Saves the current session state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "toggle_punch_in", description: "Toggles the Punch In state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_mute", description: "Sets the mute state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"mute_state": Object {"description": String("The desired mute state (true for mute, false for unmute)."), "type": String("boolean")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("mute_state"), String("rid")], "title": String("SetTrackMuteArgs"), "type": String("object")} }, Tool { name: "toggle_all_rec_enables", description: "Toggles the record enable state for ALL tracks.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }] }), Number(1))) +2026-06-30T15:23:06.511508Z  INFO ardour_mcp_server: +====================================================================== +NEW SERVER RUN: Tue, 30 Jun 2026 17:23:06 +0200 +====================================================================== +2026-06-30T15:23:06.511597Z  INFO ardour_mcp_server: Attempting to create OSC sender for Ardour at 127.0.0.1:3819 +2026-06-30T15:23:06.511637Z  INFO ardour_mcp_server: OSC sender created and connected to Ardour at 127.0.0.1:3819 +2026-06-30T15:23:06.511642Z  INFO ardour_mcp_server: Sending /set_surface to Ardour to enable OSC feedback. +2026-06-30T15:23:06.511649Z  INFO ardour_mcp_server: Targeting feedback port: 9099 +2026-06-30T15:23:06.511865Z  INFO ardour_mcp_server: /set_surface command sent successfully to Ardour. +2026-06-30T15:23:06.594065Z  INFO ardour_mcp_server: +====================================================================== +NEW SERVER RUN: Tue, 30 Jun 2026 17:23:06 +0200 +====================================================================== +2026-06-30T15:23:06.594098Z  INFO ardour_mcp_server: Attempting to create OSC sender for Ardour at 127.0.0.1:3819 +2026-06-30T15:23:06.594130Z  INFO ardour_mcp_server: OSC sender created and connected to Ardour at 127.0.0.1:3819 +2026-06-30T15:23:06.594133Z  INFO ardour_mcp_server: Sending /set_surface to Ardour to enable OSC feedback. +2026-06-30T15:23:06.594136Z  INFO ardour_mcp_server: Targeting feedback port: 9099 +2026-06-30T15:23:06.594288Z  INFO ardour_mcp_server: /set_surface command sent successfully to Ardour. +2026-06-30T15:23:06.621517Z  INFO serve_inner: rmcp::service: Service initialized as server peer_info=InitializeRequestParam { protocol_version: ProtocolVersion("2025-11-25"), capabilities: ClientCapabilities { experimental: None, roots: Some(RootsCapabilities { list_changed: None }), sampling: None }, client_info: Implementation { name: "claude-code", version: "2.1.196" } } +2026-06-30T15:23:06.621604Z  INFO ardour_mcp_server: Ardour MCP server started and waiting for connections... +2026-06-30T15:23:06.621677Z  INFO ardour_mcp_server: Starting OSC listener for Ardour events on 127.0.0.1:9099 +2026-06-30T15:23:06.621687Z DEBUG rmcp::service: new event evt=PeerMessage(Request(ListToolsRequest(Request { method: ListToolsRequestMethod, params: None }), Number(1))) +2026-06-30T15:23:06.621711Z  INFO rmcp::service: received request id=1 request=ListToolsRequest(Request { method: ListToolsRequestMethod, params: None }) +2026-06-30T15:23:06.621725Z DEBUG rmcp::service: new event evt=PeerMessage(Request(ListResourcesRequest(Request { method: ListResourcesRequestMethod, params: None }), Number(2))) +2026-06-30T15:23:06.621728Z  INFO rmcp::service: received request id=2 request=ListResourcesRequest(Request { method: ListResourcesRequestMethod, params: None }) +2026-06-30T15:23:06.621740Z DEBUG ardour_mcp_server: Listing resources. Count: 2, Content: [Annotated { raw: RawResource { uri: "ardour:/state/playback", name: "Ardour Playback State", description: Some("Current playback state of Ardour (e.g., Playing, Stopped, Unknown)."), mime_type: None, size: None }, annotations: None }, Annotated { raw: RawResource { uri: "ardour:/state/transport_frame", name: "Ardour Transport Frame Position", description: Some("Current playhead position in samples. Returns 'Unknown' if not yet reported by Ardour."), mime_type: None, size: None }, annotations: None }] +2026-06-30T15:23:06.621748Z  INFO rmcp::service: response message id=2 result=ListResourcesResult(ListResourcesResult { next_cursor: None, resources: [Annotated { raw: RawResource { uri: "ardour:/state/playback", name: "Ardour Playback State", description: Some("Current playback state of Ardour (e.g., Playing, Stopped, Unknown)."), mime_type: None, size: None }, annotations: None }, Annotated { raw: RawResource { uri: "ardour:/state/transport_frame", name: "Ardour Transport Frame Position", description: Some("Current playhead position in samples. Returns 'Unknown' if not yet reported by Ardour."), mime_type: None, size: None }, annotations: None }] }) +2026-06-30T15:23:06.621756Z DEBUG rmcp::service: new event evt=ToSink(Response(ListResourcesResult(ListResourcesResult { next_cursor: None, resources: [Annotated { raw: RawResource { uri: "ardour:/state/playback", name: "Ardour Playback State", description: Some("Current playback state of Ardour (e.g., Playing, Stopped, Unknown)."), mime_type: None, size: None }, annotations: None }, Annotated { raw: RawResource { uri: "ardour:/state/transport_frame", name: "Ardour Transport Frame Position", description: Some("Current playhead position in samples. Returns 'Unknown' if not yet reported by Ardour."), mime_type: None, size: None }, annotations: None }] }), Number(2))) +2026-06-30T15:23:06.621777Z ERROR ardour_mcp_server: OSC listener task failed: Failed to bind Tokio UDP socket for OSC on 127.0.0.1:9099: Address already in use (os error 98) +2026-06-30T15:23:06.621861Z  INFO rmcp::service: response message id=1 result=ListToolsResult(ListToolsResult { next_cursor: None, tools: [Tool { name: "next_marker", description: "Moves the playhead to the next location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "prev_marker", description: "Moves the playhead to the previous location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "save_state", description: "Saves the current session state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "redo", description: "Redoes the last undone action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "locate", description: "Locates the playhead to a specific sample position and optionally starts playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"roll": Object {"description": String("Whether to start playing after locating. 0 for stop, 1 for play."), "format": String("int32"), "type": String("integer")}, "spos": Object {"description": String("The position in samples to locate to."), "format": String("int64"), "type": String("integer")}}, "required": Array [String("roll"), String("spos")], "title": String("LocateToolArgs"), "type": String("object")} }, Tool { name: "set_track_gain_abs", description: "Sets the absolute gain of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_abs": Object {"description": String("The desired absolute gain. Valid range: 0.0 to 2.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_abs"), String("rid")], "title": String("SetTrackGainAbsArgs"), "type": String("object")} }, Tool { name: "set_track_gain_db", description: "Sets the gain of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_db": Object {"description": String("The desired gain in dB. Valid range: -400.0 to 6.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_db"), String("rid")], "title": String("SetTrackGainDBArgs"), "type": String("object")} }, Tool { name: "set_strip_plugin_active", description: "Activates or deactivates a plugin on a specific strip slot.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"active_state": Object {"description": String("The desired activation state (true for active, false for inactive)."), "type": String("boolean")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("active_state"), String("plugin_slot"), String("rid")], "title": String("SetStripPluginActiveArgs"), "type": String("object")} }, Tool { name: "set_strip_plugin_parameter", description: "Sets a specific parameter of a plugin on a strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"param_id": Object {"description": String("The 1-indexed ID of the parameter within the plugin."), "format": String("int32"), "type": String("integer")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "value": Object {"description": String("The desired parameter value, normalized (0.0 to 1.0)."), "format": String("float"), "type": String("number")}}, "required": Array [String("param_id"), String("plugin_slot"), String("rid"), String("value")], "title": String("SetStripPluginParameterArgs"), "type": String("object")} }, Tool { name: "set_track_solo", description: "Sets the solo state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "solo_st": Object {"description": String("The desired solo state. 0 for solo off, 1 for solo on."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rid"), String("solo_st")], "title": String("SetTrackSoloArgs"), "type": String("object")} }, Tool { name: "toggle_punch_out", description: "Toggles the Punch Out state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_selected_strip_pan_stereo_width", description: "Sets the stereo width for the currently selected stereo strip's panner.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"width": Object {"description": String("The desired stereo width for the currently selected strip. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("width")], "title": String("SetSelectedStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "goto_end", description: "Moves the playhead to the session end.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "toggle_punch_in", description: "Toggles the Punch In state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "rewind", description: "Rewinds the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "toggle_all_rec_enables", description: "Toggles the record enable state for ALL tracks.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "goto_start", description: "Moves the playhead to the session start.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "transport_stop", description: "Stops Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_trim_abs", description: "Sets the absolute trim of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_abs": Object {"description": String("The desired absolute trim. Valid range: 0.1 to 10.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_abs")], "title": String("SetTrackTrimAbsArgs"), "type": String("object")} }, Tool { name: "rec_enable_toggle", description: "Toggles the master record enable or selected track record enable.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "access_action", description: "Executes a specified Ardour menu action by its name.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"action_name": Object {"description": String("The name of the Ardour menu action to execute (e.g., 'Editor/zoom-to-session')."), "type": String("string")}}, "required": Array [String("action_name")], "title": String("AccessActionArgs"), "type": String("object")} }, Tool { name: "set_strip_pan_stereo_width", description: "Sets the stereo width for a panner on a stereo strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the stereo track/bus."), "format": String("int32"), "type": String("integer")}, "width": Object {"description": String("The desired stereo width. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("width")], "title": String("SetStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "select_strip", description: "Selects a specific strip (track/bus) in Ardour.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus to select."), "format": String("int32"), "type": String("integer")}, "select_state": Object {"description": String("The desired select state (true to select). Currently, only true (1) is effective for selection."), "type": String("boolean")}}, "required": Array [String("rid"), String("select_state")], "title": String("SelectStripArgs"), "type": String("object")} }, Tool { name: "ffwd", description: "Fast forwards the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "add_marker", description: "Adds a location marker at the current playhead position.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_trim_db", description: "Sets the trim of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_db": Object {"description": String("The desired trim in dB. Valid range: -20.0 to 20.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_db")], "title": String("SetTrackTrimDBArgs"), "type": String("object")} }, Tool { name: "set_transport_speed", description: "Sets Ardour's transport speed. Valid range: -8.0 to 8.0.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"speed": Object {"description": String("The desired transport speed. Valid range: -8.0 to 8.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("speed")], "title": String("SetTransportSpeedArgs"), "type": String("object")} }, Tool { name: "set_track_rec_enable", description: "Sets the record enable state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rec_st": Object {"description": String("The desired record enable state. 0 for off, 1 for on."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rec_st"), String("rid")], "title": String("SetTrackRecEnableArgs"), "type": String("object")} }, Tool { name: "loop_toggle", description: "Toggles loop playback mode.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_mute", description: "Sets the mute state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"mute_state": Object {"description": String("The desired mute state (true for mute, false for unmute)."), "type": String("boolean")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("mute_state"), String("rid")], "title": String("SetTrackMuteArgs"), "type": String("object")} }, Tool { name: "undo", description: "Undoes the last action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "transport_play", description: "Starts Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }] }) +2026-06-30T15:23:06.621915Z DEBUG rmcp::service: new event evt=ToSink(Response(ListToolsResult(ListToolsResult { next_cursor: None, tools: [Tool { name: "next_marker", description: "Moves the playhead to the next location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "prev_marker", description: "Moves the playhead to the previous location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "save_state", description: "Saves the current session state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "redo", description: "Redoes the last undone action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "locate", description: "Locates the playhead to a specific sample position and optionally starts playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"roll": Object {"description": String("Whether to start playing after locating. 0 for stop, 1 for play."), "format": String("int32"), "type": String("integer")}, "spos": Object {"description": String("The position in samples to locate to."), "format": String("int64"), "type": String("integer")}}, "required": Array [String("roll"), String("spos")], "title": String("LocateToolArgs"), "type": String("object")} }, Tool { name: "set_track_gain_abs", description: "Sets the absolute gain of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_abs": Object {"description": String("The desired absolute gain. Valid range: 0.0 to 2.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_abs"), String("rid")], "title": String("SetTrackGainAbsArgs"), "type": String("object")} }, Tool { name: "set_track_gain_db", description: "Sets the gain of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_db": Object {"description": String("The desired gain in dB. Valid range: -400.0 to 6.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_db"), String("rid")], "title": String("SetTrackGainDBArgs"), "type": String("object")} }, Tool { name: "set_strip_plugin_active", description: "Activates or deactivates a plugin on a specific strip slot.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"active_state": Object {"description": String("The desired activation state (true for active, false for inactive)."), "type": String("boolean")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("active_state"), String("plugin_slot"), String("rid")], "title": String("SetStripPluginActiveArgs"), "type": String("object")} }, Tool { name: "set_strip_plugin_parameter", description: "Sets a specific parameter of a plugin on a strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"param_id": Object {"description": String("The 1-indexed ID of the parameter within the plugin."), "format": String("int32"), "type": String("integer")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "value": Object {"description": String("The desired parameter value, normalized (0.0 to 1.0)."), "format": String("float"), "type": String("number")}}, "required": Array [String("param_id"), String("plugin_slot"), String("rid"), String("value")], "title": String("SetStripPluginParameterArgs"), "type": String("object")} }, Tool { name: "set_track_solo", description: "Sets the solo state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "solo_st": Object {"description": String("The desired solo state. 0 for solo off, 1 for solo on."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rid"), String("solo_st")], "title": String("SetTrackSoloArgs"), "type": String("object")} }, Tool { name: "toggle_punch_out", description: "Toggles the Punch Out state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_selected_strip_pan_stereo_width", description: "Sets the stereo width for the currently selected stereo strip's panner.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"width": Object {"description": String("The desired stereo width for the currently selected strip. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("width")], "title": String("SetSelectedStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "goto_end", description: "Moves the playhead to the session end.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "toggle_punch_in", description: "Toggles the Punch In state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "rewind", description: "Rewinds the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "toggle_all_rec_enables", description: "Toggles the record enable state for ALL tracks.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "goto_start", description: "Moves the playhead to the session start.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "transport_stop", description: "Stops Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_trim_abs", description: "Sets the absolute trim of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_abs": Object {"description": String("The desired absolute trim. Valid range: 0.1 to 10.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_abs")], "title": String("SetTrackTrimAbsArgs"), "type": String("object")} }, Tool { name: "rec_enable_toggle", description: "Toggles the master record enable or selected track record enable.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "access_action", description: "Executes a specified Ardour menu action by its name.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"action_name": Object {"description": String("The name of the Ardour menu action to execute (e.g., 'Editor/zoom-to-session')."), "type": String("string")}}, "required": Array [String("action_name")], "title": String("AccessActionArgs"), "type": String("object")} }, Tool { name: "set_strip_pan_stereo_width", description: "Sets the stereo width for a panner on a stereo strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the stereo track/bus."), "format": String("int32"), "type": String("integer")}, "width": Object {"description": String("The desired stereo width. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("width")], "title": String("SetStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "select_strip", description: "Selects a specific strip (track/bus) in Ardour.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus to select."), "format": String("int32"), "type": String("integer")}, "select_state": Object {"description": String("The desired select state (true to select). Currently, only true (1) is effective for selection."), "type": String("boolean")}}, "required": Array [String("rid"), String("select_state")], "title": String("SelectStripArgs"), "type": String("object")} }, Tool { name: "ffwd", description: "Fast forwards the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "add_marker", description: "Adds a location marker at the current playhead position.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_trim_db", description: "Sets the trim of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_db": Object {"description": String("The desired trim in dB. Valid range: -20.0 to 20.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_db")], "title": String("SetTrackTrimDBArgs"), "type": String("object")} }, Tool { name: "set_transport_speed", description: "Sets Ardour's transport speed. Valid range: -8.0 to 8.0.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"speed": Object {"description": String("The desired transport speed. Valid range: -8.0 to 8.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("speed")], "title": String("SetTransportSpeedArgs"), "type": String("object")} }, Tool { name: "set_track_rec_enable", description: "Sets the record enable state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rec_st": Object {"description": String("The desired record enable state. 0 for off, 1 for on."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rec_st"), String("rid")], "title": String("SetTrackRecEnableArgs"), "type": String("object")} }, Tool { name: "loop_toggle", description: "Toggles loop playback mode.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_mute", description: "Sets the mute state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"mute_state": Object {"description": String("The desired mute state (true for mute, false for unmute)."), "type": String("boolean")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("mute_state"), String("rid")], "title": String("SetTrackMuteArgs"), "type": String("object")} }, Tool { name: "undo", description: "Undoes the last action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "transport_play", description: "Starts Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }] }), Number(1))) +2026-06-30T15:23:06.656849Z  INFO serve_inner: rmcp::service: Service initialized as server peer_info=InitializeRequestParam { protocol_version: ProtocolVersion("2025-11-25"), capabilities: ClientCapabilities { experimental: None, roots: Some(RootsCapabilities { list_changed: None }), sampling: None }, client_info: Implementation { name: "claude-code", version: "2.1.196" } } +2026-06-30T15:23:06.656898Z  INFO ardour_mcp_server: Ardour MCP server started and waiting for connections... +2026-06-30T15:23:06.656931Z DEBUG rmcp::service: new event evt=PeerMessage(Request(ListToolsRequest(Request { method: ListToolsRequestMethod, params: None }), Number(1))) +2026-06-30T15:23:06.656939Z  INFO ardour_mcp_server: Starting OSC listener for Ardour events on 127.0.0.1:9099 +2026-06-30T15:23:06.656942Z  INFO rmcp::service: received request id=1 request=ListToolsRequest(Request { method: ListToolsRequestMethod, params: None }) +2026-06-30T15:23:06.656971Z DEBUG rmcp::service: new event evt=PeerMessage(Request(ListResourcesRequest(Request { method: ListResourcesRequestMethod, params: None }), Number(2))) +2026-06-30T15:23:06.656973Z  INFO rmcp::service: received request id=2 request=ListResourcesRequest(Request { method: ListResourcesRequestMethod, params: None }) +2026-06-30T15:23:06.656987Z DEBUG ardour_mcp_server: Listing resources. Count: 2, Content: [Annotated { raw: RawResource { uri: "ardour:/state/playback", name: "Ardour Playback State", description: Some("Current playback state of Ardour (e.g., Playing, Stopped, Unknown)."), mime_type: None, size: None }, annotations: None }, Annotated { raw: RawResource { uri: "ardour:/state/transport_frame", name: "Ardour Transport Frame Position", description: Some("Current playhead position in samples. Returns 'Unknown' if not yet reported by Ardour."), mime_type: None, size: None }, annotations: None }] +2026-06-30T15:23:06.656995Z  INFO rmcp::service: response message id=2 result=ListResourcesResult(ListResourcesResult { next_cursor: None, resources: [Annotated { raw: RawResource { uri: "ardour:/state/playback", name: "Ardour Playback State", description: Some("Current playback state of Ardour (e.g., Playing, Stopped, Unknown)."), mime_type: None, size: None }, annotations: None }, Annotated { raw: RawResource { uri: "ardour:/state/transport_frame", name: "Ardour Transport Frame Position", description: Some("Current playhead position in samples. Returns 'Unknown' if not yet reported by Ardour."), mime_type: None, size: None }, annotations: None }] }) +2026-06-30T15:23:06.657012Z DEBUG rmcp::service: new event evt=ToSink(Response(ListResourcesResult(ListResourcesResult { next_cursor: None, resources: [Annotated { raw: RawResource { uri: "ardour:/state/playback", name: "Ardour Playback State", description: Some("Current playback state of Ardour (e.g., Playing, Stopped, Unknown)."), mime_type: None, size: None }, annotations: None }, Annotated { raw: RawResource { uri: "ardour:/state/transport_frame", name: "Ardour Transport Frame Position", description: Some("Current playhead position in samples. Returns 'Unknown' if not yet reported by Ardour."), mime_type: None, size: None }, annotations: None }] }), Number(2))) +2026-06-30T15:23:06.657009Z ERROR ardour_mcp_server: OSC listener task failed: Failed to bind Tokio UDP socket for OSC on 127.0.0.1:9099: Address already in use (os error 98) +2026-06-30T15:23:06.657102Z  INFO rmcp::service: response message id=1 result=ListToolsResult(ListToolsResult { next_cursor: None, tools: [Tool { name: "transport_stop", description: "Stops Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "goto_start", description: "Moves the playhead to the session start.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "goto_end", description: "Moves the playhead to the session end.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "rewind", description: "Rewinds the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "next_marker", description: "Moves the playhead to the next location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "rec_enable_toggle", description: "Toggles the master record enable or selected track record enable.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "prev_marker", description: "Moves the playhead to the previous location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "redo", description: "Redoes the last undone action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "save_state", description: "Saves the current session state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_strip_pan_stereo_width", description: "Sets the stereo width for a panner on a stereo strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the stereo track/bus."), "format": String("int32"), "type": String("integer")}, "width": Object {"description": String("The desired stereo width. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("width")], "title": String("SetStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "toggle_punch_in", description: "Toggles the Punch In state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_transport_speed", description: "Sets Ardour's transport speed. Valid range: -8.0 to 8.0.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"speed": Object {"description": String("The desired transport speed. Valid range: -8.0 to 8.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("speed")], "title": String("SetTransportSpeedArgs"), "type": String("object")} }, Tool { name: "select_strip", description: "Selects a specific strip (track/bus) in Ardour.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus to select."), "format": String("int32"), "type": String("integer")}, "select_state": Object {"description": String("The desired select state (true to select). Currently, only true (1) is effective for selection."), "type": String("boolean")}}, "required": Array [String("rid"), String("select_state")], "title": String("SelectStripArgs"), "type": String("object")} }, Tool { name: "loop_toggle", description: "Toggles loop playback mode.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_strip_plugin_active", description: "Activates or deactivates a plugin on a specific strip slot.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"active_state": Object {"description": String("The desired activation state (true for active, false for inactive)."), "type": String("boolean")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("active_state"), String("plugin_slot"), String("rid")], "title": String("SetStripPluginActiveArgs"), "type": String("object")} }, Tool { name: "toggle_all_rec_enables", description: "Toggles the record enable state for ALL tracks.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "ffwd", description: "Fast forwards the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_mute", description: "Sets the mute state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"mute_state": Object {"description": String("The desired mute state (true for mute, false for unmute)."), "type": String("boolean")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("mute_state"), String("rid")], "title": String("SetTrackMuteArgs"), "type": String("object")} }, Tool { name: "set_track_gain_db", description: "Sets the gain of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_db": Object {"description": String("The desired gain in dB. Valid range: -400.0 to 6.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_db"), String("rid")], "title": String("SetTrackGainDBArgs"), "type": String("object")} }, Tool { name: "set_track_solo", description: "Sets the solo state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "solo_st": Object {"description": String("The desired solo state. 0 for solo off, 1 for solo on."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rid"), String("solo_st")], "title": String("SetTrackSoloArgs"), "type": String("object")} }, Tool { name: "undo", description: "Undoes the last action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "toggle_punch_out", description: "Toggles the Punch Out state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_trim_abs", description: "Sets the absolute trim of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_abs": Object {"description": String("The desired absolute trim. Valid range: 0.1 to 10.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_abs")], "title": String("SetTrackTrimAbsArgs"), "type": String("object")} }, Tool { name: "set_selected_strip_pan_stereo_width", description: "Sets the stereo width for the currently selected stereo strip's panner.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"width": Object {"description": String("The desired stereo width for the currently selected strip. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("width")], "title": String("SetSelectedStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "set_track_rec_enable", description: "Sets the record enable state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rec_st": Object {"description": String("The desired record enable state. 0 for off, 1 for on."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rec_st"), String("rid")], "title": String("SetTrackRecEnableArgs"), "type": String("object")} }, Tool { name: "add_marker", description: "Adds a location marker at the current playhead position.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_gain_abs", description: "Sets the absolute gain of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_abs": Object {"description": String("The desired absolute gain. Valid range: 0.0 to 2.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_abs"), String("rid")], "title": String("SetTrackGainAbsArgs"), "type": String("object")} }, Tool { name: "transport_play", description: "Starts Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_trim_db", description: "Sets the trim of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_db": Object {"description": String("The desired trim in dB. Valid range: -20.0 to 20.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_db")], "title": String("SetTrackTrimDBArgs"), "type": String("object")} }, Tool { name: "set_strip_plugin_parameter", description: "Sets a specific parameter of a plugin on a strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"param_id": Object {"description": String("The 1-indexed ID of the parameter within the plugin."), "format": String("int32"), "type": String("integer")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "value": Object {"description": String("The desired parameter value, normalized (0.0 to 1.0)."), "format": String("float"), "type": String("number")}}, "required": Array [String("param_id"), String("plugin_slot"), String("rid"), String("value")], "title": String("SetStripPluginParameterArgs"), "type": String("object")} }, Tool { name: "locate", description: "Locates the playhead to a specific sample position and optionally starts playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"roll": Object {"description": String("Whether to start playing after locating. 0 for stop, 1 for play."), "format": String("int32"), "type": String("integer")}, "spos": Object {"description": String("The position in samples to locate to."), "format": String("int64"), "type": String("integer")}}, "required": Array [String("roll"), String("spos")], "title": String("LocateToolArgs"), "type": String("object")} }, Tool { name: "access_action", description: "Executes a specified Ardour menu action by its name.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"action_name": Object {"description": String("The name of the Ardour menu action to execute (e.g., 'Editor/zoom-to-session')."), "type": String("string")}}, "required": Array [String("action_name")], "title": String("AccessActionArgs"), "type": String("object")} }] }) +2026-06-30T15:23:06.657158Z DEBUG rmcp::service: new event evt=ToSink(Response(ListToolsResult(ListToolsResult { next_cursor: None, tools: [Tool { name: "transport_stop", description: "Stops Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "goto_start", description: "Moves the playhead to the session start.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "goto_end", description: "Moves the playhead to the session end.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "rewind", description: "Rewinds the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "next_marker", description: "Moves the playhead to the next location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "rec_enable_toggle", description: "Toggles the master record enable or selected track record enable.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "prev_marker", description: "Moves the playhead to the previous location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "redo", description: "Redoes the last undone action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "save_state", description: "Saves the current session state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_strip_pan_stereo_width", description: "Sets the stereo width for a panner on a stereo strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the stereo track/bus."), "format": String("int32"), "type": String("integer")}, "width": Object {"description": String("The desired stereo width. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("width")], "title": String("SetStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "toggle_punch_in", description: "Toggles the Punch In state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_transport_speed", description: "Sets Ardour's transport speed. Valid range: -8.0 to 8.0.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"speed": Object {"description": String("The desired transport speed. Valid range: -8.0 to 8.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("speed")], "title": String("SetTransportSpeedArgs"), "type": String("object")} }, Tool { name: "select_strip", description: "Selects a specific strip (track/bus) in Ardour.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus to select."), "format": String("int32"), "type": String("integer")}, "select_state": Object {"description": String("The desired select state (true to select). Currently, only true (1) is effective for selection."), "type": String("boolean")}}, "required": Array [String("rid"), String("select_state")], "title": String("SelectStripArgs"), "type": String("object")} }, Tool { name: "loop_toggle", description: "Toggles loop playback mode.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_strip_plugin_active", description: "Activates or deactivates a plugin on a specific strip slot.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"active_state": Object {"description": String("The desired activation state (true for active, false for inactive)."), "type": String("boolean")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("active_state"), String("plugin_slot"), String("rid")], "title": String("SetStripPluginActiveArgs"), "type": String("object")} }, Tool { name: "toggle_all_rec_enables", description: "Toggles the record enable state for ALL tracks.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "ffwd", description: "Fast forwards the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_mute", description: "Sets the mute state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"mute_state": Object {"description": String("The desired mute state (true for mute, false for unmute)."), "type": String("boolean")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("mute_state"), String("rid")], "title": String("SetTrackMuteArgs"), "type": String("object")} }, Tool { name: "set_track_gain_db", description: "Sets the gain of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_db": Object {"description": String("The desired gain in dB. Valid range: -400.0 to 6.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_db"), String("rid")], "title": String("SetTrackGainDBArgs"), "type": String("object")} }, Tool { name: "set_track_solo", description: "Sets the solo state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "solo_st": Object {"description": String("The desired solo state. 0 for solo off, 1 for solo on."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rid"), String("solo_st")], "title": String("SetTrackSoloArgs"), "type": String("object")} }, Tool { name: "undo", description: "Undoes the last action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "toggle_punch_out", description: "Toggles the Punch Out state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_trim_abs", description: "Sets the absolute trim of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_abs": Object {"description": String("The desired absolute trim. Valid range: 0.1 to 10.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_abs")], "title": String("SetTrackTrimAbsArgs"), "type": String("object")} }, Tool { name: "set_selected_strip_pan_stereo_width", description: "Sets the stereo width for the currently selected stereo strip's panner.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"width": Object {"description": String("The desired stereo width for the currently selected strip. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("width")], "title": String("SetSelectedStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "set_track_rec_enable", description: "Sets the record enable state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rec_st": Object {"description": String("The desired record enable state. 0 for off, 1 for on."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rec_st"), String("rid")], "title": String("SetTrackRecEnableArgs"), "type": String("object")} }, Tool { name: "add_marker", description: "Adds a location marker at the current playhead position.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_gain_abs", description: "Sets the absolute gain of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_abs": Object {"description": String("The desired absolute gain. Valid range: 0.0 to 2.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_abs"), String("rid")], "title": String("SetTrackGainAbsArgs"), "type": String("object")} }, Tool { name: "transport_play", description: "Starts Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_trim_db", description: "Sets the trim of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_db": Object {"description": String("The desired trim in dB. Valid range: -20.0 to 20.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_db")], "title": String("SetTrackTrimDBArgs"), "type": String("object")} }, Tool { name: "set_strip_plugin_parameter", description: "Sets a specific parameter of a plugin on a strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"param_id": Object {"description": String("The 1-indexed ID of the parameter within the plugin."), "format": String("int32"), "type": String("integer")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "value": Object {"description": String("The desired parameter value, normalized (0.0 to 1.0)."), "format": String("float"), "type": String("number")}}, "required": Array [String("param_id"), String("plugin_slot"), String("rid"), String("value")], "title": String("SetStripPluginParameterArgs"), "type": String("object")} }, Tool { name: "locate", description: "Locates the playhead to a specific sample position and optionally starts playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"roll": Object {"description": String("Whether to start playing after locating. 0 for stop, 1 for play."), "format": String("int32"), "type": String("integer")}, "spos": Object {"description": String("The position in samples to locate to."), "format": String("int64"), "type": String("integer")}}, "required": Array [String("roll"), String("spos")], "title": String("LocateToolArgs"), "type": String("object")} }, Tool { name: "access_action", description: "Executes a specified Ardour menu action by its name.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"action_name": Object {"description": String("The name of the Ardour menu action to execute (e.g., 'Editor/zoom-to-session')."), "type": String("string")}}, "required": Array [String("action_name")], "title": String("AccessActionArgs"), "type": String("object")} }] }), Number(1))) +2026-06-30T15:27:12.616760Z  INFO ardour_mcp_server: +====================================================================== +NEW SERVER RUN: Tue, 30 Jun 2026 17:27:12 +0200 +====================================================================== +2026-06-30T15:27:12.616846Z  INFO ardour_mcp_server: Attempting to create OSC sender for Ardour at 127.0.0.1:3819 +2026-06-30T15:27:12.616899Z  INFO ardour_mcp_server: OSC sender created and connected to Ardour at 127.0.0.1:3819 +2026-06-30T15:27:12.616906Z  INFO ardour_mcp_server: Sending /set_surface to Ardour to enable OSC feedback. +2026-06-30T15:27:12.616932Z  INFO ardour_mcp_server: Targeting feedback port: 9099 +2026-06-30T15:27:12.617359Z  INFO ardour_mcp_server: /set_surface command sent successfully to Ardour. +2026-06-30T15:27:12.718764Z  INFO serve_inner: rmcp::service: Service initialized as server peer_info=InitializeRequestParam { protocol_version: ProtocolVersion("2025-11-25"), capabilities: ClientCapabilities { experimental: None, roots: Some(RootsCapabilities { list_changed: None }), sampling: None }, client_info: Implementation { name: "claude-code", version: "2.1.196" } } +2026-06-30T15:27:12.718841Z  INFO ardour_mcp_server: Ardour MCP server started and waiting for connections... +2026-06-30T15:27:12.718882Z DEBUG rmcp::service: new event evt=PeerMessage(Request(ListToolsRequest(Request { method: ListToolsRequestMethod, params: None }), Number(1))) +2026-06-30T15:27:12.718882Z  INFO ardour_mcp_server: Starting OSC listener for Ardour events on 127.0.0.1:9099 +2026-06-30T15:27:12.718894Z  INFO rmcp::service: received request id=1 request=ListToolsRequest(Request { method: ListToolsRequestMethod, params: None }) +2026-06-30T15:27:12.718910Z DEBUG rmcp::service: new event evt=PeerMessage(Request(ListResourcesRequest(Request { method: ListResourcesRequestMethod, params: None }), Number(2))) +2026-06-30T15:27:12.718912Z  INFO rmcp::service: received request id=2 request=ListResourcesRequest(Request { method: ListResourcesRequestMethod, params: None }) +2026-06-30T15:27:12.718931Z DEBUG ardour_mcp_server: Listing resources. Count: 2, Content: [Annotated { raw: RawResource { uri: "ardour:/state/playback", name: "Ardour Playback State", description: Some("Current playback state of Ardour (e.g., Playing, Stopped, Unknown)."), mime_type: None, size: None }, annotations: None }, Annotated { raw: RawResource { uri: "ardour:/state/transport_frame", name: "Ardour Transport Frame Position", description: Some("Current playhead position in samples. Returns 'Unknown' if not yet reported by Ardour."), mime_type: None, size: None }, annotations: None }] +2026-06-30T15:27:12.718941Z  INFO rmcp::service: response message id=2 result=ListResourcesResult(ListResourcesResult { next_cursor: None, resources: [Annotated { raw: RawResource { uri: "ardour:/state/playback", name: "Ardour Playback State", description: Some("Current playback state of Ardour (e.g., Playing, Stopped, Unknown)."), mime_type: None, size: None }, annotations: None }, Annotated { raw: RawResource { uri: "ardour:/state/transport_frame", name: "Ardour Transport Frame Position", description: Some("Current playhead position in samples. Returns 'Unknown' if not yet reported by Ardour."), mime_type: None, size: None }, annotations: None }] }) +2026-06-30T15:27:12.718954Z DEBUG rmcp::service: new event evt=ToSink(Response(ListResourcesResult(ListResourcesResult { next_cursor: None, resources: [Annotated { raw: RawResource { uri: "ardour:/state/playback", name: "Ardour Playback State", description: Some("Current playback state of Ardour (e.g., Playing, Stopped, Unknown)."), mime_type: None, size: None }, annotations: None }, Annotated { raw: RawResource { uri: "ardour:/state/transport_frame", name: "Ardour Transport Frame Position", description: Some("Current playhead position in samples. Returns 'Unknown' if not yet reported by Ardour."), mime_type: None, size: None }, annotations: None }] }), Number(2))) +2026-06-30T15:27:12.718959Z ERROR ardour_mcp_server: OSC listener task failed: Failed to bind Tokio UDP socket for OSC on 127.0.0.1:9099: Address already in use (os error 98) +2026-06-30T15:27:12.719096Z  INFO rmcp::service: response message id=1 result=ListToolsResult(ListToolsResult { next_cursor: None, tools: [Tool { name: "access_action", description: "Executes a specified Ardour menu action by its name.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"action_name": Object {"description": String("The name of the Ardour menu action to execute (e.g., 'Editor/zoom-to-session')."), "type": String("string")}}, "required": Array [String("action_name")], "title": String("AccessActionArgs"), "type": String("object")} }, Tool { name: "set_strip_plugin_active", description: "Activates or deactivates a plugin on a specific strip slot.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"active_state": Object {"description": String("The desired activation state (true for active, false for inactive)."), "type": String("boolean")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("active_state"), String("plugin_slot"), String("rid")], "title": String("SetStripPluginActiveArgs"), "type": String("object")} }, Tool { name: "add_marker", description: "Adds a location marker at the current playhead position.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_strip_pan_stereo_width", description: "Sets the stereo width for a panner on a stereo strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the stereo track/bus."), "format": String("int32"), "type": String("integer")}, "width": Object {"description": String("The desired stereo width. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("width")], "title": String("SetStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "rewind", description: "Rewinds the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "undo", description: "Undoes the last action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_rec_enable", description: "Sets the record enable state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rec_st": Object {"description": String("The desired record enable state. 0 for off, 1 for on."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rec_st"), String("rid")], "title": String("SetTrackRecEnableArgs"), "type": String("object")} }, Tool { name: "transport_play", description: "Starts Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_gain_abs", description: "Sets the absolute gain of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_abs": Object {"description": String("The desired absolute gain. Valid range: 0.0 to 2.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_abs"), String("rid")], "title": String("SetTrackGainAbsArgs"), "type": String("object")} }, Tool { name: "set_track_mute", description: "Sets the mute state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"mute_state": Object {"description": String("The desired mute state (true for mute, false for unmute)."), "type": String("boolean")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("mute_state"), String("rid")], "title": String("SetTrackMuteArgs"), "type": String("object")} }, Tool { name: "toggle_punch_out", description: "Toggles the Punch Out state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "goto_start", description: "Moves the playhead to the session start.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "transport_stop", description: "Stops Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "toggle_all_rec_enables", description: "Toggles the record enable state for ALL tracks.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "ffwd", description: "Fast forwards the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "goto_end", description: "Moves the playhead to the session end.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "toggle_punch_in", description: "Toggles the Punch In state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_transport_speed", description: "Sets Ardour's transport speed. Valid range: -8.0 to 8.0.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"speed": Object {"description": String("The desired transport speed. Valid range: -8.0 to 8.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("speed")], "title": String("SetTransportSpeedArgs"), "type": String("object")} }, Tool { name: "set_track_solo", description: "Sets the solo state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "solo_st": Object {"description": String("The desired solo state. 0 for solo off, 1 for solo on."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rid"), String("solo_st")], "title": String("SetTrackSoloArgs"), "type": String("object")} }, Tool { name: "select_strip", description: "Selects a specific strip (track/bus) in Ardour.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus to select."), "format": String("int32"), "type": String("integer")}, "select_state": Object {"description": String("The desired select state (true to select). Currently, only true (1) is effective for selection."), "type": String("boolean")}}, "required": Array [String("rid"), String("select_state")], "title": String("SelectStripArgs"), "type": String("object")} }, Tool { name: "set_track_trim_abs", description: "Sets the absolute trim of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_abs": Object {"description": String("The desired absolute trim. Valid range: 0.1 to 10.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_abs")], "title": String("SetTrackTrimAbsArgs"), "type": String("object")} }, Tool { name: "set_track_trim_db", description: "Sets the trim of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_db": Object {"description": String("The desired trim in dB. Valid range: -20.0 to 20.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_db")], "title": String("SetTrackTrimDBArgs"), "type": String("object")} }, Tool { name: "set_strip_plugin_parameter", description: "Sets a specific parameter of a plugin on a strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"param_id": Object {"description": String("The 1-indexed ID of the parameter within the plugin."), "format": String("int32"), "type": String("integer")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "value": Object {"description": String("The desired parameter value, normalized (0.0 to 1.0)."), "format": String("float"), "type": String("number")}}, "required": Array [String("param_id"), String("plugin_slot"), String("rid"), String("value")], "title": String("SetStripPluginParameterArgs"), "type": String("object")} }, Tool { name: "locate", description: "Locates the playhead to a specific sample position and optionally starts playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"roll": Object {"description": String("Whether to start playing after locating. 0 for stop, 1 for play."), "format": String("int32"), "type": String("integer")}, "spos": Object {"description": String("The position in samples to locate to."), "format": String("int64"), "type": String("integer")}}, "required": Array [String("roll"), String("spos")], "title": String("LocateToolArgs"), "type": String("object")} }, Tool { name: "set_selected_strip_pan_stereo_width", description: "Sets the stereo width for the currently selected stereo strip's panner.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"width": Object {"description": String("The desired stereo width for the currently selected strip. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("width")], "title": String("SetSelectedStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "next_marker", description: "Moves the playhead to the next location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "prev_marker", description: "Moves the playhead to the previous location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "rec_enable_toggle", description: "Toggles the master record enable or selected track record enable.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "save_state", description: "Saves the current session state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_gain_db", description: "Sets the gain of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_db": Object {"description": String("The desired gain in dB. Valid range: -400.0 to 6.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_db"), String("rid")], "title": String("SetTrackGainDBArgs"), "type": String("object")} }, Tool { name: "redo", description: "Redoes the last undone action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "loop_toggle", description: "Toggles loop playback mode.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }] }) +2026-06-30T15:27:12.719177Z DEBUG rmcp::service: new event evt=ToSink(Response(ListToolsResult(ListToolsResult { next_cursor: None, tools: [Tool { name: "access_action", description: "Executes a specified Ardour menu action by its name.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"action_name": Object {"description": String("The name of the Ardour menu action to execute (e.g., 'Editor/zoom-to-session')."), "type": String("string")}}, "required": Array [String("action_name")], "title": String("AccessActionArgs"), "type": String("object")} }, Tool { name: "set_strip_plugin_active", description: "Activates or deactivates a plugin on a specific strip slot.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"active_state": Object {"description": String("The desired activation state (true for active, false for inactive)."), "type": String("boolean")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("active_state"), String("plugin_slot"), String("rid")], "title": String("SetStripPluginActiveArgs"), "type": String("object")} }, Tool { name: "add_marker", description: "Adds a location marker at the current playhead position.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_strip_pan_stereo_width", description: "Sets the stereo width for a panner on a stereo strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the stereo track/bus."), "format": String("int32"), "type": String("integer")}, "width": Object {"description": String("The desired stereo width. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("width")], "title": String("SetStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "rewind", description: "Rewinds the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "undo", description: "Undoes the last action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_rec_enable", description: "Sets the record enable state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rec_st": Object {"description": String("The desired record enable state. 0 for off, 1 for on."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rec_st"), String("rid")], "title": String("SetTrackRecEnableArgs"), "type": String("object")} }, Tool { name: "transport_play", description: "Starts Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_gain_abs", description: "Sets the absolute gain of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_abs": Object {"description": String("The desired absolute gain. Valid range: 0.0 to 2.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_abs"), String("rid")], "title": String("SetTrackGainAbsArgs"), "type": String("object")} }, Tool { name: "set_track_mute", description: "Sets the mute state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"mute_state": Object {"description": String("The desired mute state (true for mute, false for unmute)."), "type": String("boolean")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("mute_state"), String("rid")], "title": String("SetTrackMuteArgs"), "type": String("object")} }, Tool { name: "toggle_punch_out", description: "Toggles the Punch Out state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "goto_start", description: "Moves the playhead to the session start.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "transport_stop", description: "Stops Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "toggle_all_rec_enables", description: "Toggles the record enable state for ALL tracks.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "ffwd", description: "Fast forwards the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "goto_end", description: "Moves the playhead to the session end.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "toggle_punch_in", description: "Toggles the Punch In state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_transport_speed", description: "Sets Ardour's transport speed. Valid range: -8.0 to 8.0.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"speed": Object {"description": String("The desired transport speed. Valid range: -8.0 to 8.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("speed")], "title": String("SetTransportSpeedArgs"), "type": String("object")} }, Tool { name: "set_track_solo", description: "Sets the solo state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "solo_st": Object {"description": String("The desired solo state. 0 for solo off, 1 for solo on."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rid"), String("solo_st")], "title": String("SetTrackSoloArgs"), "type": String("object")} }, Tool { name: "select_strip", description: "Selects a specific strip (track/bus) in Ardour.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus to select."), "format": String("int32"), "type": String("integer")}, "select_state": Object {"description": String("The desired select state (true to select). Currently, only true (1) is effective for selection."), "type": String("boolean")}}, "required": Array [String("rid"), String("select_state")], "title": String("SelectStripArgs"), "type": String("object")} }, Tool { name: "set_track_trim_abs", description: "Sets the absolute trim of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_abs": Object {"description": String("The desired absolute trim. Valid range: 0.1 to 10.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_abs")], "title": String("SetTrackTrimAbsArgs"), "type": String("object")} }, Tool { name: "set_track_trim_db", description: "Sets the trim of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_db": Object {"description": String("The desired trim in dB. Valid range: -20.0 to 20.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_db")], "title": String("SetTrackTrimDBArgs"), "type": String("object")} }, Tool { name: "set_strip_plugin_parameter", description: "Sets a specific parameter of a plugin on a strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"param_id": Object {"description": String("The 1-indexed ID of the parameter within the plugin."), "format": String("int32"), "type": String("integer")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "value": Object {"description": String("The desired parameter value, normalized (0.0 to 1.0)."), "format": String("float"), "type": String("number")}}, "required": Array [String("param_id"), String("plugin_slot"), String("rid"), String("value")], "title": String("SetStripPluginParameterArgs"), "type": String("object")} }, Tool { name: "locate", description: "Locates the playhead to a specific sample position and optionally starts playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"roll": Object {"description": String("Whether to start playing after locating. 0 for stop, 1 for play."), "format": String("int32"), "type": String("integer")}, "spos": Object {"description": String("The position in samples to locate to."), "format": String("int64"), "type": String("integer")}}, "required": Array [String("roll"), String("spos")], "title": String("LocateToolArgs"), "type": String("object")} }, Tool { name: "set_selected_strip_pan_stereo_width", description: "Sets the stereo width for the currently selected stereo strip's panner.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"width": Object {"description": String("The desired stereo width for the currently selected strip. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("width")], "title": String("SetSelectedStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "next_marker", description: "Moves the playhead to the next location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "prev_marker", description: "Moves the playhead to the previous location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "rec_enable_toggle", description: "Toggles the master record enable or selected track record enable.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "save_state", description: "Saves the current session state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_gain_db", description: "Sets the gain of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_db": Object {"description": String("The desired gain in dB. Valid range: -400.0 to 6.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_db"), String("rid")], "title": String("SetTrackGainDBArgs"), "type": String("object")} }, Tool { name: "redo", description: "Redoes the last undone action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "loop_toggle", description: "Toggles loop playback mode.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }] }), Number(1))) +2026-06-30T15:27:13.809133Z  INFO ardour_mcp_server: +====================================================================== +NEW SERVER RUN: Tue, 30 Jun 2026 17:27:13 +0200 +====================================================================== +2026-06-30T15:27:13.809174Z  INFO ardour_mcp_server: Attempting to create OSC sender for Ardour at 127.0.0.1:3819 +2026-06-30T15:27:13.809207Z  INFO ardour_mcp_server: OSC sender created and connected to Ardour at 127.0.0.1:3819 +2026-06-30T15:27:13.809211Z  INFO ardour_mcp_server: Sending /set_surface to Ardour to enable OSC feedback. +2026-06-30T15:27:13.809214Z  INFO ardour_mcp_server: Targeting feedback port: 9099 +2026-06-30T15:27:13.809830Z  INFO ardour_mcp_server: /set_surface command sent successfully to Ardour. +2026-06-30T15:27:13.896776Z  INFO serve_inner: rmcp::service: Service initialized as server peer_info=InitializeRequestParam { protocol_version: ProtocolVersion("2025-11-25"), capabilities: ClientCapabilities { experimental: None, roots: Some(RootsCapabilities { list_changed: None }), sampling: None }, client_info: Implementation { name: "claude-code", version: "2.1.196" } } +2026-06-30T15:27:13.896812Z  INFO ardour_mcp_server: Ardour MCP server started and waiting for connections... +2026-06-30T15:27:13.896850Z DEBUG rmcp::service: new event evt=PeerMessage(Request(ListToolsRequest(Request { method: ListToolsRequestMethod, params: None }), Number(1))) +2026-06-30T15:27:13.896848Z  INFO ardour_mcp_server: Starting OSC listener for Ardour events on 127.0.0.1:9099 +2026-06-30T15:27:13.896860Z  INFO rmcp::service: received request id=1 request=ListToolsRequest(Request { method: ListToolsRequestMethod, params: None }) +2026-06-30T15:27:13.896885Z DEBUG rmcp::service: new event evt=PeerMessage(Request(ListResourcesRequest(Request { method: ListResourcesRequestMethod, params: None }), Number(2))) +2026-06-30T15:27:13.896887Z  INFO rmcp::service: received request id=2 request=ListResourcesRequest(Request { method: ListResourcesRequestMethod, params: None }) +2026-06-30T15:27:13.896902Z DEBUG ardour_mcp_server: Listing resources. Count: 2, Content: [Annotated { raw: RawResource { uri: "ardour:/state/playback", name: "Ardour Playback State", description: Some("Current playback state of Ardour (e.g., Playing, Stopped, Unknown)."), mime_type: None, size: None }, annotations: None }, Annotated { raw: RawResource { uri: "ardour:/state/transport_frame", name: "Ardour Transport Frame Position", description: Some("Current playhead position in samples. Returns 'Unknown' if not yet reported by Ardour."), mime_type: None, size: None }, annotations: None }] +2026-06-30T15:27:13.896910Z  INFO rmcp::service: response message id=2 result=ListResourcesResult(ListResourcesResult { next_cursor: None, resources: [Annotated { raw: RawResource { uri: "ardour:/state/playback", name: "Ardour Playback State", description: Some("Current playback state of Ardour (e.g., Playing, Stopped, Unknown)."), mime_type: None, size: None }, annotations: None }, Annotated { raw: RawResource { uri: "ardour:/state/transport_frame", name: "Ardour Transport Frame Position", description: Some("Current playhead position in samples. Returns 'Unknown' if not yet reported by Ardour."), mime_type: None, size: None }, annotations: None }] }) +2026-06-30T15:27:13.896915Z ERROR ardour_mcp_server: OSC listener task failed: Failed to bind Tokio UDP socket for OSC on 127.0.0.1:9099: Address already in use (os error 98) +2026-06-30T15:27:13.896920Z DEBUG rmcp::service: new event evt=ToSink(Response(ListResourcesResult(ListResourcesResult { next_cursor: None, resources: [Annotated { raw: RawResource { uri: "ardour:/state/playback", name: "Ardour Playback State", description: Some("Current playback state of Ardour (e.g., Playing, Stopped, Unknown)."), mime_type: None, size: None }, annotations: None }, Annotated { raw: RawResource { uri: "ardour:/state/transport_frame", name: "Ardour Transport Frame Position", description: Some("Current playhead position in samples. Returns 'Unknown' if not yet reported by Ardour."), mime_type: None, size: None }, annotations: None }] }), Number(2))) +2026-06-30T15:27:13.897031Z  INFO rmcp::service: response message id=1 result=ListToolsResult(ListToolsResult { next_cursor: None, tools: [Tool { name: "next_marker", description: "Moves the playhead to the next location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "toggle_punch_out", description: "Toggles the Punch Out state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_gain_abs", description: "Sets the absolute gain of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_abs": Object {"description": String("The desired absolute gain. Valid range: 0.0 to 2.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_abs"), String("rid")], "title": String("SetTrackGainAbsArgs"), "type": String("object")} }, Tool { name: "set_strip_pan_stereo_width", description: "Sets the stereo width for a panner on a stereo strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the stereo track/bus."), "format": String("int32"), "type": String("integer")}, "width": Object {"description": String("The desired stereo width. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("width")], "title": String("SetStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "goto_start", description: "Moves the playhead to the session start.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "loop_toggle", description: "Toggles loop playback mode.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "add_marker", description: "Adds a location marker at the current playhead position.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "rec_enable_toggle", description: "Toggles the master record enable or selected track record enable.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "rewind", description: "Rewinds the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_strip_plugin_parameter", description: "Sets a specific parameter of a plugin on a strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"param_id": Object {"description": String("The 1-indexed ID of the parameter within the plugin."), "format": String("int32"), "type": String("integer")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "value": Object {"description": String("The desired parameter value, normalized (0.0 to 1.0)."), "format": String("float"), "type": String("number")}}, "required": Array [String("param_id"), String("plugin_slot"), String("rid"), String("value")], "title": String("SetStripPluginParameterArgs"), "type": String("object")} }, Tool { name: "set_track_solo", description: "Sets the solo state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "solo_st": Object {"description": String("The desired solo state. 0 for solo off, 1 for solo on."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rid"), String("solo_st")], "title": String("SetTrackSoloArgs"), "type": String("object")} }, Tool { name: "transport_play", description: "Starts Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_rec_enable", description: "Sets the record enable state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rec_st": Object {"description": String("The desired record enable state. 0 for off, 1 for on."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rec_st"), String("rid")], "title": String("SetTrackRecEnableArgs"), "type": String("object")} }, Tool { name: "set_selected_strip_pan_stereo_width", description: "Sets the stereo width for the currently selected stereo strip's panner.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"width": Object {"description": String("The desired stereo width for the currently selected strip. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("width")], "title": String("SetSelectedStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "locate", description: "Locates the playhead to a specific sample position and optionally starts playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"roll": Object {"description": String("Whether to start playing after locating. 0 for stop, 1 for play."), "format": String("int32"), "type": String("integer")}, "spos": Object {"description": String("The position in samples to locate to."), "format": String("int64"), "type": String("integer")}}, "required": Array [String("roll"), String("spos")], "title": String("LocateToolArgs"), "type": String("object")} }, Tool { name: "toggle_punch_in", description: "Toggles the Punch In state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_trim_db", description: "Sets the trim of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_db": Object {"description": String("The desired trim in dB. Valid range: -20.0 to 20.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_db")], "title": String("SetTrackTrimDBArgs"), "type": String("object")} }, Tool { name: "set_strip_plugin_active", description: "Activates or deactivates a plugin on a specific strip slot.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"active_state": Object {"description": String("The desired activation state (true for active, false for inactive)."), "type": String("boolean")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("active_state"), String("plugin_slot"), String("rid")], "title": String("SetStripPluginActiveArgs"), "type": String("object")} }, Tool { name: "transport_stop", description: "Stops Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "access_action", description: "Executes a specified Ardour menu action by its name.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"action_name": Object {"description": String("The name of the Ardour menu action to execute (e.g., 'Editor/zoom-to-session')."), "type": String("string")}}, "required": Array [String("action_name")], "title": String("AccessActionArgs"), "type": String("object")} }, Tool { name: "prev_marker", description: "Moves the playhead to the previous location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "save_state", description: "Saves the current session state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "goto_end", description: "Moves the playhead to the session end.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "undo", description: "Undoes the last action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_trim_abs", description: "Sets the absolute trim of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_abs": Object {"description": String("The desired absolute trim. Valid range: 0.1 to 10.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_abs")], "title": String("SetTrackTrimAbsArgs"), "type": String("object")} }, Tool { name: "redo", description: "Redoes the last undone action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "ffwd", description: "Fast forwards the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "select_strip", description: "Selects a specific strip (track/bus) in Ardour.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus to select."), "format": String("int32"), "type": String("integer")}, "select_state": Object {"description": String("The desired select state (true to select). Currently, only true (1) is effective for selection."), "type": String("boolean")}}, "required": Array [String("rid"), String("select_state")], "title": String("SelectStripArgs"), "type": String("object")} }, Tool { name: "set_transport_speed", description: "Sets Ardour's transport speed. Valid range: -8.0 to 8.0.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"speed": Object {"description": String("The desired transport speed. Valid range: -8.0 to 8.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("speed")], "title": String("SetTransportSpeedArgs"), "type": String("object")} }, Tool { name: "set_track_gain_db", description: "Sets the gain of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_db": Object {"description": String("The desired gain in dB. Valid range: -400.0 to 6.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_db"), String("rid")], "title": String("SetTrackGainDBArgs"), "type": String("object")} }, Tool { name: "toggle_all_rec_enables", description: "Toggles the record enable state for ALL tracks.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_mute", description: "Sets the mute state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"mute_state": Object {"description": String("The desired mute state (true for mute, false for unmute)."), "type": String("boolean")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("mute_state"), String("rid")], "title": String("SetTrackMuteArgs"), "type": String("object")} }] }) +2026-06-30T15:27:13.897092Z DEBUG rmcp::service: new event evt=ToSink(Response(ListToolsResult(ListToolsResult { next_cursor: None, tools: [Tool { name: "next_marker", description: "Moves the playhead to the next location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "toggle_punch_out", description: "Toggles the Punch Out state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_gain_abs", description: "Sets the absolute gain of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_abs": Object {"description": String("The desired absolute gain. Valid range: 0.0 to 2.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_abs"), String("rid")], "title": String("SetTrackGainAbsArgs"), "type": String("object")} }, Tool { name: "set_strip_pan_stereo_width", description: "Sets the stereo width for a panner on a stereo strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the stereo track/bus."), "format": String("int32"), "type": String("integer")}, "width": Object {"description": String("The desired stereo width. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("width")], "title": String("SetStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "goto_start", description: "Moves the playhead to the session start.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "loop_toggle", description: "Toggles loop playback mode.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "add_marker", description: "Adds a location marker at the current playhead position.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "rec_enable_toggle", description: "Toggles the master record enable or selected track record enable.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "rewind", description: "Rewinds the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_strip_plugin_parameter", description: "Sets a specific parameter of a plugin on a strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"param_id": Object {"description": String("The 1-indexed ID of the parameter within the plugin."), "format": String("int32"), "type": String("integer")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "value": Object {"description": String("The desired parameter value, normalized (0.0 to 1.0)."), "format": String("float"), "type": String("number")}}, "required": Array [String("param_id"), String("plugin_slot"), String("rid"), String("value")], "title": String("SetStripPluginParameterArgs"), "type": String("object")} }, Tool { name: "set_track_solo", description: "Sets the solo state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "solo_st": Object {"description": String("The desired solo state. 0 for solo off, 1 for solo on."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rid"), String("solo_st")], "title": String("SetTrackSoloArgs"), "type": String("object")} }, Tool { name: "transport_play", description: "Starts Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_rec_enable", description: "Sets the record enable state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rec_st": Object {"description": String("The desired record enable state. 0 for off, 1 for on."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rec_st"), String("rid")], "title": String("SetTrackRecEnableArgs"), "type": String("object")} }, Tool { name: "set_selected_strip_pan_stereo_width", description: "Sets the stereo width for the currently selected stereo strip's panner.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"width": Object {"description": String("The desired stereo width for the currently selected strip. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("width")], "title": String("SetSelectedStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "locate", description: "Locates the playhead to a specific sample position and optionally starts playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"roll": Object {"description": String("Whether to start playing after locating. 0 for stop, 1 for play."), "format": String("int32"), "type": String("integer")}, "spos": Object {"description": String("The position in samples to locate to."), "format": String("int64"), "type": String("integer")}}, "required": Array [String("roll"), String("spos")], "title": String("LocateToolArgs"), "type": String("object")} }, Tool { name: "toggle_punch_in", description: "Toggles the Punch In state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_trim_db", description: "Sets the trim of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_db": Object {"description": String("The desired trim in dB. Valid range: -20.0 to 20.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_db")], "title": String("SetTrackTrimDBArgs"), "type": String("object")} }, Tool { name: "set_strip_plugin_active", description: "Activates or deactivates a plugin on a specific strip slot.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"active_state": Object {"description": String("The desired activation state (true for active, false for inactive)."), "type": String("boolean")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("active_state"), String("plugin_slot"), String("rid")], "title": String("SetStripPluginActiveArgs"), "type": String("object")} }, Tool { name: "transport_stop", description: "Stops Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "access_action", description: "Executes a specified Ardour menu action by its name.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"action_name": Object {"description": String("The name of the Ardour menu action to execute (e.g., 'Editor/zoom-to-session')."), "type": String("string")}}, "required": Array [String("action_name")], "title": String("AccessActionArgs"), "type": String("object")} }, Tool { name: "prev_marker", description: "Moves the playhead to the previous location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "save_state", description: "Saves the current session state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "goto_end", description: "Moves the playhead to the session end.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "undo", description: "Undoes the last action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_trim_abs", description: "Sets the absolute trim of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_abs": Object {"description": String("The desired absolute trim. Valid range: 0.1 to 10.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_abs")], "title": String("SetTrackTrimAbsArgs"), "type": String("object")} }, Tool { name: "redo", description: "Redoes the last undone action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "ffwd", description: "Fast forwards the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "select_strip", description: "Selects a specific strip (track/bus) in Ardour.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus to select."), "format": String("int32"), "type": String("integer")}, "select_state": Object {"description": String("The desired select state (true to select). Currently, only true (1) is effective for selection."), "type": String("boolean")}}, "required": Array [String("rid"), String("select_state")], "title": String("SelectStripArgs"), "type": String("object")} }, Tool { name: "set_transport_speed", description: "Sets Ardour's transport speed. Valid range: -8.0 to 8.0.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"speed": Object {"description": String("The desired transport speed. Valid range: -8.0 to 8.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("speed")], "title": String("SetTransportSpeedArgs"), "type": String("object")} }, Tool { name: "set_track_gain_db", description: "Sets the gain of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_db": Object {"description": String("The desired gain in dB. Valid range: -400.0 to 6.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_db"), String("rid")], "title": String("SetTrackGainDBArgs"), "type": String("object")} }, Tool { name: "toggle_all_rec_enables", description: "Toggles the record enable state for ALL tracks.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_mute", description: "Sets the mute state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"mute_state": Object {"description": String("The desired mute state (true for mute, false for unmute)."), "type": String("boolean")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("mute_state"), String("rid")], "title": String("SetTrackMuteArgs"), "type": String("object")} }] }), Number(1))) +2026-06-30T16:04:20.541207Z  INFO ardour_mcp_server: +====================================================================== +NEW SERVER RUN: Tue, 30 Jun 2026 18:04:20 +0200 +====================================================================== +2026-06-30T16:04:20.541267Z  INFO ardour_mcp_server: Attempting to create OSC sender for Ardour at 127.0.0.1:3819 +2026-06-30T16:04:20.541321Z  INFO ardour_mcp_server: OSC sender created and connected to Ardour at 127.0.0.1:3819 +2026-06-30T16:04:20.541325Z  INFO ardour_mcp_server: Sending /set_surface to Ardour to enable OSC feedback. +2026-06-30T16:04:20.541330Z  INFO ardour_mcp_server: Targeting feedback port: 9099 +2026-06-30T16:04:20.570040Z  INFO ardour_mcp_server: /set_surface command sent successfully to Ardour. +2026-06-30T16:04:20.587361Z  INFO ardour_mcp_server: +====================================================================== +NEW SERVER RUN: Tue, 30 Jun 2026 18:04:20 +0200 +====================================================================== +2026-06-30T16:04:20.587439Z  INFO ardour_mcp_server: Attempting to create OSC sender for Ardour at 127.0.0.1:3819 +2026-06-30T16:04:20.587508Z  INFO ardour_mcp_server: OSC sender created and connected to Ardour at 127.0.0.1:3819 +2026-06-30T16:04:20.587517Z  INFO ardour_mcp_server: Sending /set_surface to Ardour to enable OSC feedback. +2026-06-30T16:04:20.587524Z  INFO ardour_mcp_server: Targeting feedback port: 9099 +2026-06-30T16:04:20.587821Z  INFO ardour_mcp_server: /set_surface command sent successfully to Ardour. +2026-06-30T16:04:20.707625Z  INFO serve_inner: rmcp::service: Service initialized as server peer_info=InitializeRequestParam { protocol_version: ProtocolVersion("2025-11-25"), capabilities: ClientCapabilities { experimental: None, roots: Some(RootsCapabilities { list_changed: None }), sampling: None }, client_info: Implementation { name: "claude-code", version: "2.1.196" } } +2026-06-30T16:04:20.707714Z  INFO ardour_mcp_server: Ardour MCP server started and waiting for connections... +2026-06-30T16:04:20.707764Z  INFO ardour_mcp_server: Starting OSC listener for Ardour events on 127.0.0.1:9099 +2026-06-30T16:04:20.707773Z DEBUG rmcp::service: new event evt=PeerMessage(Request(ListToolsRequest(Request { method: ListToolsRequestMethod, params: None }), Number(1))) +2026-06-30T16:04:20.707799Z  INFO rmcp::service: received request id=1 request=ListToolsRequest(Request { method: ListToolsRequestMethod, params: None }) +2026-06-30T16:04:20.707843Z DEBUG rmcp::service: new event evt=PeerMessage(Request(ListResourcesRequest(Request { method: ListResourcesRequestMethod, params: None }), Number(2))) +2026-06-30T16:04:20.707846Z  INFO rmcp::service: received request id=2 request=ListResourcesRequest(Request { method: ListResourcesRequestMethod, params: None }) +2026-06-30T16:04:20.707860Z ERROR ardour_mcp_server: OSC listener task failed: Failed to bind Tokio UDP socket for OSC on 127.0.0.1:9099: Address already in use (os error 98) +2026-06-30T16:04:20.707869Z DEBUG ardour_mcp_server: Listing resources. Count: 2, Content: [Annotated { raw: RawResource { uri: "ardour:/state/playback", name: "Ardour Playback State", description: Some("Current playback state of Ardour (e.g., Playing, Stopped, Unknown)."), mime_type: None, size: None }, annotations: None }, Annotated { raw: RawResource { uri: "ardour:/state/transport_frame", name: "Ardour Transport Frame Position", description: Some("Current playhead position in samples. Returns 'Unknown' if not yet reported by Ardour."), mime_type: None, size: None }, annotations: None }] +2026-06-30T16:04:20.707880Z  INFO rmcp::service: response message id=2 result=ListResourcesResult(ListResourcesResult { next_cursor: None, resources: [Annotated { raw: RawResource { uri: "ardour:/state/playback", name: "Ardour Playback State", description: Some("Current playback state of Ardour (e.g., Playing, Stopped, Unknown)."), mime_type: None, size: None }, annotations: None }, Annotated { raw: RawResource { uri: "ardour:/state/transport_frame", name: "Ardour Transport Frame Position", description: Some("Current playhead position in samples. Returns 'Unknown' if not yet reported by Ardour."), mime_type: None, size: None }, annotations: None }] }) +2026-06-30T16:04:20.707892Z DEBUG rmcp::service: new event evt=ToSink(Response(ListResourcesResult(ListResourcesResult { next_cursor: None, resources: [Annotated { raw: RawResource { uri: "ardour:/state/playback", name: "Ardour Playback State", description: Some("Current playback state of Ardour (e.g., Playing, Stopped, Unknown)."), mime_type: None, size: None }, annotations: None }, Annotated { raw: RawResource { uri: "ardour:/state/transport_frame", name: "Ardour Transport Frame Position", description: Some("Current playhead position in samples. Returns 'Unknown' if not yet reported by Ardour."), mime_type: None, size: None }, annotations: None }] }), Number(2))) +2026-06-30T16:04:20.708050Z  INFO rmcp::service: response message id=1 result=ListToolsResult(ListToolsResult { next_cursor: None, tools: [Tool { name: "goto_end", description: "Moves the playhead to the session end.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_mute", description: "Sets the mute state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"mute_state": Object {"description": String("The desired mute state (true for mute, false for unmute)."), "type": String("boolean")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("mute_state"), String("rid")], "title": String("SetTrackMuteArgs"), "type": String("object")} }, Tool { name: "set_track_gain_db", description: "Sets the gain of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_db": Object {"description": String("The desired gain in dB. Valid range: -400.0 to 6.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_db"), String("rid")], "title": String("SetTrackGainDBArgs"), "type": String("object")} }, Tool { name: "toggle_all_rec_enables", description: "Toggles the record enable state for ALL tracks.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_trim_abs", description: "Sets the absolute trim of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_abs": Object {"description": String("The desired absolute trim. Valid range: 0.1 to 10.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_abs")], "title": String("SetTrackTrimAbsArgs"), "type": String("object")} }, Tool { name: "redo", description: "Redoes the last undone action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "toggle_punch_out", description: "Toggles the Punch Out state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "locate", description: "Locates the playhead to a specific sample position and optionally starts playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"roll": Object {"description": String("Whether to start playing after locating. 0 for stop, 1 for play."), "format": String("int32"), "type": String("integer")}, "spos": Object {"description": String("The position in samples to locate to."), "format": String("int64"), "type": String("integer")}}, "required": Array [String("roll"), String("spos")], "title": String("LocateToolArgs"), "type": String("object")} }, Tool { name: "prev_marker", description: "Moves the playhead to the previous location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "rewind", description: "Rewinds the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_strip_pan_stereo_width", description: "Sets the stereo width for a panner on a stereo strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the stereo track/bus."), "format": String("int32"), "type": String("integer")}, "width": Object {"description": String("The desired stereo width. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("width")], "title": String("SetStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "set_selected_strip_pan_stereo_width", description: "Sets the stereo width for the currently selected stereo strip's panner.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"width": Object {"description": String("The desired stereo width for the currently selected strip. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("width")], "title": String("SetSelectedStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "set_strip_plugin_active", description: "Activates or deactivates a plugin on a specific strip slot.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"active_state": Object {"description": String("The desired activation state (true for active, false for inactive)."), "type": String("boolean")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("active_state"), String("plugin_slot"), String("rid")], "title": String("SetStripPluginActiveArgs"), "type": String("object")} }, Tool { name: "toggle_punch_in", description: "Toggles the Punch In state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "add_marker", description: "Adds a location marker at the current playhead position.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_strip_plugin_parameter", description: "Sets a specific parameter of a plugin on a strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"param_id": Object {"description": String("The 1-indexed ID of the parameter within the plugin."), "format": String("int32"), "type": String("integer")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "value": Object {"description": String("The desired parameter value, normalized (0.0 to 1.0)."), "format": String("float"), "type": String("number")}}, "required": Array [String("param_id"), String("plugin_slot"), String("rid"), String("value")], "title": String("SetStripPluginParameterArgs"), "type": String("object")} }, Tool { name: "transport_play", description: "Starts Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "loop_toggle", description: "Toggles loop playback mode.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "rec_enable_toggle", description: "Toggles the master record enable or selected track record enable.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_gain_abs", description: "Sets the absolute gain of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_abs": Object {"description": String("The desired absolute gain. Valid range: 0.0 to 2.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_abs"), String("rid")], "title": String("SetTrackGainAbsArgs"), "type": String("object")} }, Tool { name: "access_action", description: "Executes a specified Ardour menu action by its name.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"action_name": Object {"description": String("The name of the Ardour menu action to execute (e.g., 'Editor/zoom-to-session')."), "type": String("string")}}, "required": Array [String("action_name")], "title": String("AccessActionArgs"), "type": String("object")} }, Tool { name: "transport_stop", description: "Stops Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_solo", description: "Sets the solo state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "solo_st": Object {"description": String("The desired solo state. 0 for solo off, 1 for solo on."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rid"), String("solo_st")], "title": String("SetTrackSoloArgs"), "type": String("object")} }, Tool { name: "goto_start", description: "Moves the playhead to the session start.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "ffwd", description: "Fast forwards the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "undo", description: "Undoes the last action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "next_marker", description: "Moves the playhead to the next location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_rec_enable", description: "Sets the record enable state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rec_st": Object {"description": String("The desired record enable state. 0 for off, 1 for on."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rec_st"), String("rid")], "title": String("SetTrackRecEnableArgs"), "type": String("object")} }, Tool { name: "save_state", description: "Saves the current session state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_trim_db", description: "Sets the trim of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_db": Object {"description": String("The desired trim in dB. Valid range: -20.0 to 20.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_db")], "title": String("SetTrackTrimDBArgs"), "type": String("object")} }, Tool { name: "select_strip", description: "Selects a specific strip (track/bus) in Ardour.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus to select."), "format": String("int32"), "type": String("integer")}, "select_state": Object {"description": String("The desired select state (true to select). Currently, only true (1) is effective for selection."), "type": String("boolean")}}, "required": Array [String("rid"), String("select_state")], "title": String("SelectStripArgs"), "type": String("object")} }, Tool { name: "set_transport_speed", description: "Sets Ardour's transport speed. Valid range: -8.0 to 8.0.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"speed": Object {"description": String("The desired transport speed. Valid range: -8.0 to 8.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("speed")], "title": String("SetTransportSpeedArgs"), "type": String("object")} }] }) +2026-06-30T16:04:20.708106Z DEBUG rmcp::service: new event evt=ToSink(Response(ListToolsResult(ListToolsResult { next_cursor: None, tools: [Tool { name: "goto_end", description: "Moves the playhead to the session end.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_mute", description: "Sets the mute state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"mute_state": Object {"description": String("The desired mute state (true for mute, false for unmute)."), "type": String("boolean")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("mute_state"), String("rid")], "title": String("SetTrackMuteArgs"), "type": String("object")} }, Tool { name: "set_track_gain_db", description: "Sets the gain of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_db": Object {"description": String("The desired gain in dB. Valid range: -400.0 to 6.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_db"), String("rid")], "title": String("SetTrackGainDBArgs"), "type": String("object")} }, Tool { name: "toggle_all_rec_enables", description: "Toggles the record enable state for ALL tracks.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_trim_abs", description: "Sets the absolute trim of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_abs": Object {"description": String("The desired absolute trim. Valid range: 0.1 to 10.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_abs")], "title": String("SetTrackTrimAbsArgs"), "type": String("object")} }, Tool { name: "redo", description: "Redoes the last undone action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "toggle_punch_out", description: "Toggles the Punch Out state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "locate", description: "Locates the playhead to a specific sample position and optionally starts playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"roll": Object {"description": String("Whether to start playing after locating. 0 for stop, 1 for play."), "format": String("int32"), "type": String("integer")}, "spos": Object {"description": String("The position in samples to locate to."), "format": String("int64"), "type": String("integer")}}, "required": Array [String("roll"), String("spos")], "title": String("LocateToolArgs"), "type": String("object")} }, Tool { name: "prev_marker", description: "Moves the playhead to the previous location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "rewind", description: "Rewinds the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_strip_pan_stereo_width", description: "Sets the stereo width for a panner on a stereo strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the stereo track/bus."), "format": String("int32"), "type": String("integer")}, "width": Object {"description": String("The desired stereo width. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("width")], "title": String("SetStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "set_selected_strip_pan_stereo_width", description: "Sets the stereo width for the currently selected stereo strip's panner.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"width": Object {"description": String("The desired stereo width for the currently selected strip. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("width")], "title": String("SetSelectedStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "set_strip_plugin_active", description: "Activates or deactivates a plugin on a specific strip slot.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"active_state": Object {"description": String("The desired activation state (true for active, false for inactive)."), "type": String("boolean")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("active_state"), String("plugin_slot"), String("rid")], "title": String("SetStripPluginActiveArgs"), "type": String("object")} }, Tool { name: "toggle_punch_in", description: "Toggles the Punch In state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "add_marker", description: "Adds a location marker at the current playhead position.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_strip_plugin_parameter", description: "Sets a specific parameter of a plugin on a strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"param_id": Object {"description": String("The 1-indexed ID of the parameter within the plugin."), "format": String("int32"), "type": String("integer")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "value": Object {"description": String("The desired parameter value, normalized (0.0 to 1.0)."), "format": String("float"), "type": String("number")}}, "required": Array [String("param_id"), String("plugin_slot"), String("rid"), String("value")], "title": String("SetStripPluginParameterArgs"), "type": String("object")} }, Tool { name: "transport_play", description: "Starts Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "loop_toggle", description: "Toggles loop playback mode.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "rec_enable_toggle", description: "Toggles the master record enable or selected track record enable.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_gain_abs", description: "Sets the absolute gain of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_abs": Object {"description": String("The desired absolute gain. Valid range: 0.0 to 2.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_abs"), String("rid")], "title": String("SetTrackGainAbsArgs"), "type": String("object")} }, Tool { name: "access_action", description: "Executes a specified Ardour menu action by its name.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"action_name": Object {"description": String("The name of the Ardour menu action to execute (e.g., 'Editor/zoom-to-session')."), "type": String("string")}}, "required": Array [String("action_name")], "title": String("AccessActionArgs"), "type": String("object")} }, Tool { name: "transport_stop", description: "Stops Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_solo", description: "Sets the solo state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "solo_st": Object {"description": String("The desired solo state. 0 for solo off, 1 for solo on."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rid"), String("solo_st")], "title": String("SetTrackSoloArgs"), "type": String("object")} }, Tool { name: "goto_start", description: "Moves the playhead to the session start.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "ffwd", description: "Fast forwards the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "undo", description: "Undoes the last action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "next_marker", description: "Moves the playhead to the next location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_rec_enable", description: "Sets the record enable state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rec_st": Object {"description": String("The desired record enable state. 0 for off, 1 for on."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rec_st"), String("rid")], "title": String("SetTrackRecEnableArgs"), "type": String("object")} }, Tool { name: "save_state", description: "Saves the current session state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_trim_db", description: "Sets the trim of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_db": Object {"description": String("The desired trim in dB. Valid range: -20.0 to 20.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_db")], "title": String("SetTrackTrimDBArgs"), "type": String("object")} }, Tool { name: "select_strip", description: "Selects a specific strip (track/bus) in Ardour.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus to select."), "format": String("int32"), "type": String("integer")}, "select_state": Object {"description": String("The desired select state (true to select). Currently, only true (1) is effective for selection."), "type": String("boolean")}}, "required": Array [String("rid"), String("select_state")], "title": String("SelectStripArgs"), "type": String("object")} }, Tool { name: "set_transport_speed", description: "Sets Ardour's transport speed. Valid range: -8.0 to 8.0.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"speed": Object {"description": String("The desired transport speed. Valid range: -8.0 to 8.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("speed")], "title": String("SetTransportSpeedArgs"), "type": String("object")} }] }), Number(1))) +2026-06-30T16:04:20.757286Z  INFO serve_inner: rmcp::service: Service initialized as server peer_info=InitializeRequestParam { protocol_version: ProtocolVersion("2025-11-25"), capabilities: ClientCapabilities { experimental: None, roots: Some(RootsCapabilities { list_changed: None }), sampling: None }, client_info: Implementation { name: "claude-code", version: "2.1.196" } } +2026-06-30T16:04:20.757372Z  INFO ardour_mcp_server: Ardour MCP server started and waiting for connections... +2026-06-30T16:04:20.757419Z DEBUG rmcp::service: new event evt=PeerMessage(Request(ListToolsRequest(Request { method: ListToolsRequestMethod, params: None }), Number(1))) +2026-06-30T16:04:20.757422Z  INFO ardour_mcp_server: Starting OSC listener for Ardour events on 127.0.0.1:9099 +2026-06-30T16:04:20.757434Z  INFO rmcp::service: received request id=1 request=ListToolsRequest(Request { method: ListToolsRequestMethod, params: None }) +2026-06-30T16:04:20.757464Z DEBUG rmcp::service: new event evt=PeerMessage(Request(ListResourcesRequest(Request { method: ListResourcesRequestMethod, params: None }), Number(2))) +2026-06-30T16:04:20.757468Z  INFO rmcp::service: received request id=2 request=ListResourcesRequest(Request { method: ListResourcesRequestMethod, params: None }) +2026-06-30T16:04:20.757489Z DEBUG ardour_mcp_server: Listing resources. Count: 2, Content: [Annotated { raw: RawResource { uri: "ardour:/state/playback", name: "Ardour Playback State", description: Some("Current playback state of Ardour (e.g., Playing, Stopped, Unknown)."), mime_type: None, size: None }, annotations: None }, Annotated { raw: RawResource { uri: "ardour:/state/transport_frame", name: "Ardour Transport Frame Position", description: Some("Current playhead position in samples. Returns 'Unknown' if not yet reported by Ardour."), mime_type: None, size: None }, annotations: None }] +2026-06-30T16:04:20.757501Z  INFO rmcp::service: response message id=2 result=ListResourcesResult(ListResourcesResult { next_cursor: None, resources: [Annotated { raw: RawResource { uri: "ardour:/state/playback", name: "Ardour Playback State", description: Some("Current playback state of Ardour (e.g., Playing, Stopped, Unknown)."), mime_type: None, size: None }, annotations: None }, Annotated { raw: RawResource { uri: "ardour:/state/transport_frame", name: "Ardour Transport Frame Position", description: Some("Current playhead position in samples. Returns 'Unknown' if not yet reported by Ardour."), mime_type: None, size: None }, annotations: None }] }) +2026-06-30T16:04:20.757502Z ERROR ardour_mcp_server: OSC listener task failed: Failed to bind Tokio UDP socket for OSC on 127.0.0.1:9099: Address already in use (os error 98) +2026-06-30T16:04:20.757514Z DEBUG rmcp::service: new event evt=ToSink(Response(ListResourcesResult(ListResourcesResult { next_cursor: None, resources: [Annotated { raw: RawResource { uri: "ardour:/state/playback", name: "Ardour Playback State", description: Some("Current playback state of Ardour (e.g., Playing, Stopped, Unknown)."), mime_type: None, size: None }, annotations: None }, Annotated { raw: RawResource { uri: "ardour:/state/transport_frame", name: "Ardour Transport Frame Position", description: Some("Current playhead position in samples. Returns 'Unknown' if not yet reported by Ardour."), mime_type: None, size: None }, annotations: None }] }), Number(2))) +2026-06-30T16:04:20.757648Z  INFO rmcp::service: response message id=1 result=ListToolsResult(ListToolsResult { next_cursor: None, tools: [Tool { name: "loop_toggle", description: "Toggles loop playback mode.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "toggle_punch_in", description: "Toggles the Punch In state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "prev_marker", description: "Moves the playhead to the previous location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "toggle_punch_out", description: "Toggles the Punch Out state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_gain_abs", description: "Sets the absolute gain of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_abs": Object {"description": String("The desired absolute gain. Valid range: 0.0 to 2.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_abs"), String("rid")], "title": String("SetTrackGainAbsArgs"), "type": String("object")} }, Tool { name: "set_track_trim_abs", description: "Sets the absolute trim of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_abs": Object {"description": String("The desired absolute trim. Valid range: 0.1 to 10.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_abs")], "title": String("SetTrackTrimAbsArgs"), "type": String("object")} }, Tool { name: "select_strip", description: "Selects a specific strip (track/bus) in Ardour.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus to select."), "format": String("int32"), "type": String("integer")}, "select_state": Object {"description": String("The desired select state (true to select). Currently, only true (1) is effective for selection."), "type": String("boolean")}}, "required": Array [String("rid"), String("select_state")], "title": String("SelectStripArgs"), "type": String("object")} }, Tool { name: "set_track_gain_db", description: "Sets the gain of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_db": Object {"description": String("The desired gain in dB. Valid range: -400.0 to 6.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_db"), String("rid")], "title": String("SetTrackGainDBArgs"), "type": String("object")} }, Tool { name: "next_marker", description: "Moves the playhead to the next location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "transport_play", description: "Starts Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_mute", description: "Sets the mute state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"mute_state": Object {"description": String("The desired mute state (true for mute, false for unmute)."), "type": String("boolean")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("mute_state"), String("rid")], "title": String("SetTrackMuteArgs"), "type": String("object")} }, Tool { name: "set_selected_strip_pan_stereo_width", description: "Sets the stereo width for the currently selected stereo strip's panner.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"width": Object {"description": String("The desired stereo width for the currently selected strip. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("width")], "title": String("SetSelectedStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "ffwd", description: "Fast forwards the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_trim_db", description: "Sets the trim of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_db": Object {"description": String("The desired trim in dB. Valid range: -20.0 to 20.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_db")], "title": String("SetTrackTrimDBArgs"), "type": String("object")} }, Tool { name: "set_track_solo", description: "Sets the solo state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "solo_st": Object {"description": String("The desired solo state. 0 for solo off, 1 for solo on."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rid"), String("solo_st")], "title": String("SetTrackSoloArgs"), "type": String("object")} }, Tool { name: "undo", description: "Undoes the last action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "rewind", description: "Rewinds the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "transport_stop", description: "Stops Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "redo", description: "Redoes the last undone action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_strip_plugin_active", description: "Activates or deactivates a plugin on a specific strip slot.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"active_state": Object {"description": String("The desired activation state (true for active, false for inactive)."), "type": String("boolean")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("active_state"), String("plugin_slot"), String("rid")], "title": String("SetStripPluginActiveArgs"), "type": String("object")} }, Tool { name: "set_transport_speed", description: "Sets Ardour's transport speed. Valid range: -8.0 to 8.0.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"speed": Object {"description": String("The desired transport speed. Valid range: -8.0 to 8.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("speed")], "title": String("SetTransportSpeedArgs"), "type": String("object")} }, Tool { name: "goto_start", description: "Moves the playhead to the session start.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "rec_enable_toggle", description: "Toggles the master record enable or selected track record enable.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "access_action", description: "Executes a specified Ardour menu action by its name.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"action_name": Object {"description": String("The name of the Ardour menu action to execute (e.g., 'Editor/zoom-to-session')."), "type": String("string")}}, "required": Array [String("action_name")], "title": String("AccessActionArgs"), "type": String("object")} }, Tool { name: "toggle_all_rec_enables", description: "Toggles the record enable state for ALL tracks.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "save_state", description: "Saves the current session state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "goto_end", description: "Moves the playhead to the session end.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "add_marker", description: "Adds a location marker at the current playhead position.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_strip_plugin_parameter", description: "Sets a specific parameter of a plugin on a strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"param_id": Object {"description": String("The 1-indexed ID of the parameter within the plugin."), "format": String("int32"), "type": String("integer")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "value": Object {"description": String("The desired parameter value, normalized (0.0 to 1.0)."), "format": String("float"), "type": String("number")}}, "required": Array [String("param_id"), String("plugin_slot"), String("rid"), String("value")], "title": String("SetStripPluginParameterArgs"), "type": String("object")} }, Tool { name: "set_strip_pan_stereo_width", description: "Sets the stereo width for a panner on a stereo strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the stereo track/bus."), "format": String("int32"), "type": String("integer")}, "width": Object {"description": String("The desired stereo width. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("width")], "title": String("SetStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "set_track_rec_enable", description: "Sets the record enable state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rec_st": Object {"description": String("The desired record enable state. 0 for off, 1 for on."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rec_st"), String("rid")], "title": String("SetTrackRecEnableArgs"), "type": String("object")} }, Tool { name: "locate", description: "Locates the playhead to a specific sample position and optionally starts playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"roll": Object {"description": String("Whether to start playing after locating. 0 for stop, 1 for play."), "format": String("int32"), "type": String("integer")}, "spos": Object {"description": String("The position in samples to locate to."), "format": String("int64"), "type": String("integer")}}, "required": Array [String("roll"), String("spos")], "title": String("LocateToolArgs"), "type": String("object")} }] }) +2026-06-30T16:04:20.757741Z DEBUG rmcp::service: new event evt=ToSink(Response(ListToolsResult(ListToolsResult { next_cursor: None, tools: [Tool { name: "loop_toggle", description: "Toggles loop playback mode.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "toggle_punch_in", description: "Toggles the Punch In state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "prev_marker", description: "Moves the playhead to the previous location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "toggle_punch_out", description: "Toggles the Punch Out state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_gain_abs", description: "Sets the absolute gain of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_abs": Object {"description": String("The desired absolute gain. Valid range: 0.0 to 2.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_abs"), String("rid")], "title": String("SetTrackGainAbsArgs"), "type": String("object")} }, Tool { name: "set_track_trim_abs", description: "Sets the absolute trim of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_abs": Object {"description": String("The desired absolute trim. Valid range: 0.1 to 10.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_abs")], "title": String("SetTrackTrimAbsArgs"), "type": String("object")} }, Tool { name: "select_strip", description: "Selects a specific strip (track/bus) in Ardour.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus to select."), "format": String("int32"), "type": String("integer")}, "select_state": Object {"description": String("The desired select state (true to select). Currently, only true (1) is effective for selection."), "type": String("boolean")}}, "required": Array [String("rid"), String("select_state")], "title": String("SelectStripArgs"), "type": String("object")} }, Tool { name: "set_track_gain_db", description: "Sets the gain of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_db": Object {"description": String("The desired gain in dB. Valid range: -400.0 to 6.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_db"), String("rid")], "title": String("SetTrackGainDBArgs"), "type": String("object")} }, Tool { name: "next_marker", description: "Moves the playhead to the next location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "transport_play", description: "Starts Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_mute", description: "Sets the mute state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"mute_state": Object {"description": String("The desired mute state (true for mute, false for unmute)."), "type": String("boolean")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("mute_state"), String("rid")], "title": String("SetTrackMuteArgs"), "type": String("object")} }, Tool { name: "set_selected_strip_pan_stereo_width", description: "Sets the stereo width for the currently selected stereo strip's panner.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"width": Object {"description": String("The desired stereo width for the currently selected strip. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("width")], "title": String("SetSelectedStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "ffwd", description: "Fast forwards the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_trim_db", description: "Sets the trim of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_db": Object {"description": String("The desired trim in dB. Valid range: -20.0 to 20.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_db")], "title": String("SetTrackTrimDBArgs"), "type": String("object")} }, Tool { name: "set_track_solo", description: "Sets the solo state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "solo_st": Object {"description": String("The desired solo state. 0 for solo off, 1 for solo on."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rid"), String("solo_st")], "title": String("SetTrackSoloArgs"), "type": String("object")} }, Tool { name: "undo", description: "Undoes the last action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "rewind", description: "Rewinds the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "transport_stop", description: "Stops Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "redo", description: "Redoes the last undone action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_strip_plugin_active", description: "Activates or deactivates a plugin on a specific strip slot.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"active_state": Object {"description": String("The desired activation state (true for active, false for inactive)."), "type": String("boolean")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("active_state"), String("plugin_slot"), String("rid")], "title": String("SetStripPluginActiveArgs"), "type": String("object")} }, Tool { name: "set_transport_speed", description: "Sets Ardour's transport speed. Valid range: -8.0 to 8.0.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"speed": Object {"description": String("The desired transport speed. Valid range: -8.0 to 8.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("speed")], "title": String("SetTransportSpeedArgs"), "type": String("object")} }, Tool { name: "goto_start", description: "Moves the playhead to the session start.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "rec_enable_toggle", description: "Toggles the master record enable or selected track record enable.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "access_action", description: "Executes a specified Ardour menu action by its name.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"action_name": Object {"description": String("The name of the Ardour menu action to execute (e.g., 'Editor/zoom-to-session')."), "type": String("string")}}, "required": Array [String("action_name")], "title": String("AccessActionArgs"), "type": String("object")} }, Tool { name: "toggle_all_rec_enables", description: "Toggles the record enable state for ALL tracks.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "save_state", description: "Saves the current session state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "goto_end", description: "Moves the playhead to the session end.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "add_marker", description: "Adds a location marker at the current playhead position.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_strip_plugin_parameter", description: "Sets a specific parameter of a plugin on a strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"param_id": Object {"description": String("The 1-indexed ID of the parameter within the plugin."), "format": String("int32"), "type": String("integer")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "value": Object {"description": String("The desired parameter value, normalized (0.0 to 1.0)."), "format": String("float"), "type": String("number")}}, "required": Array [String("param_id"), String("plugin_slot"), String("rid"), String("value")], "title": String("SetStripPluginParameterArgs"), "type": String("object")} }, Tool { name: "set_strip_pan_stereo_width", description: "Sets the stereo width for a panner on a stereo strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the stereo track/bus."), "format": String("int32"), "type": String("integer")}, "width": Object {"description": String("The desired stereo width. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("width")], "title": String("SetStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "set_track_rec_enable", description: "Sets the record enable state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rec_st": Object {"description": String("The desired record enable state. 0 for off, 1 for on."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rec_st"), String("rid")], "title": String("SetTrackRecEnableArgs"), "type": String("object")} }, Tool { name: "locate", description: "Locates the playhead to a specific sample position and optionally starts playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"roll": Object {"description": String("Whether to start playing after locating. 0 for stop, 1 for play."), "format": String("int32"), "type": String("integer")}, "spos": Object {"description": String("The position in samples to locate to."), "format": String("int64"), "type": String("integer")}}, "required": Array [String("roll"), String("spos")], "title": String("LocateToolArgs"), "type": String("object")} }] }), Number(1))) +2026-06-30T16:20:17.883897Z  INFO ardour_mcp_server: +====================================================================== +NEW SERVER RUN: Tue, 30 Jun 2026 18:20:17 +0200 +====================================================================== +2026-06-30T16:20:17.883936Z  INFO ardour_mcp_server: Attempting to create OSC sender for Ardour at 127.0.0.1:3819 +2026-06-30T16:20:17.883969Z  INFO ardour_mcp_server: OSC sender created and connected to Ardour at 127.0.0.1:3819 +2026-06-30T16:20:17.883972Z  INFO ardour_mcp_server: Sending /set_surface to Ardour to enable OSC feedback. +2026-06-30T16:20:17.883976Z  INFO ardour_mcp_server: Targeting feedback port: 9099 +2026-06-30T16:20:17.884123Z  INFO ardour_mcp_server: /set_surface command sent successfully to Ardour. +2026-06-30T16:20:18.051393Z  INFO ardour_mcp_server: +====================================================================== +NEW SERVER RUN: Tue, 30 Jun 2026 18:20:18 +0200 +====================================================================== +2026-06-30T16:20:18.051456Z  INFO ardour_mcp_server: Attempting to create OSC sender for Ardour at 127.0.0.1:3819 +2026-06-30T16:20:18.051508Z  INFO ardour_mcp_server: OSC sender created and connected to Ardour at 127.0.0.1:3819 +2026-06-30T16:20:18.051528Z  INFO ardour_mcp_server: Sending /set_surface to Ardour to enable OSC feedback. +2026-06-30T16:20:18.051533Z  INFO ardour_mcp_server: Targeting feedback port: 9099 +2026-06-30T16:20:18.051968Z  INFO ardour_mcp_server: /set_surface command sent successfully to Ardour. +2026-06-30T16:20:18.111103Z  INFO serve_inner: rmcp::service: Service initialized as server peer_info=InitializeRequestParam { protocol_version: ProtocolVersion("2025-11-25"), capabilities: ClientCapabilities { experimental: None, roots: Some(RootsCapabilities { list_changed: None }), sampling: None }, client_info: Implementation { name: "claude-code", version: "2.1.196" } } +2026-06-30T16:20:18.111185Z  INFO ardour_mcp_server: Ardour MCP server started and waiting for connections... +2026-06-30T16:20:18.111251Z  INFO ardour_mcp_server: Starting OSC listener for Ardour events on 127.0.0.1:9099 +2026-06-30T16:20:18.111327Z ERROR ardour_mcp_server: OSC listener task failed: Failed to bind Tokio UDP socket for OSC on 127.0.0.1:9099: Address already in use (os error 98) +2026-06-30T16:20:18.111584Z DEBUG rmcp::service: new event evt=PeerMessage(Request(ListToolsRequest(Request { method: ListToolsRequestMethod, params: None }), Number(1))) +2026-06-30T16:20:18.111597Z  INFO rmcp::service: received request id=1 request=ListToolsRequest(Request { method: ListToolsRequestMethod, params: None }) +2026-06-30T16:20:18.111616Z DEBUG rmcp::service: new event evt=PeerMessage(Request(ListResourcesRequest(Request { method: ListResourcesRequestMethod, params: None }), Number(2))) +2026-06-30T16:20:18.111620Z  INFO rmcp::service: received request id=2 request=ListResourcesRequest(Request { method: ListResourcesRequestMethod, params: None }) +2026-06-30T16:20:18.111641Z DEBUG ardour_mcp_server: Listing resources. Count: 2, Content: [Annotated { raw: RawResource { uri: "ardour:/state/playback", name: "Ardour Playback State", description: Some("Current playback state of Ardour (e.g., Playing, Stopped, Unknown)."), mime_type: None, size: None }, annotations: None }, Annotated { raw: RawResource { uri: "ardour:/state/transport_frame", name: "Ardour Transport Frame Position", description: Some("Current playhead position in samples. Returns 'Unknown' if not yet reported by Ardour."), mime_type: None, size: None }, annotations: None }] +2026-06-30T16:20:18.111654Z  INFO rmcp::service: response message id=2 result=ListResourcesResult(ListResourcesResult { next_cursor: None, resources: [Annotated { raw: RawResource { uri: "ardour:/state/playback", name: "Ardour Playback State", description: Some("Current playback state of Ardour (e.g., Playing, Stopped, Unknown)."), mime_type: None, size: None }, annotations: None }, Annotated { raw: RawResource { uri: "ardour:/state/transport_frame", name: "Ardour Transport Frame Position", description: Some("Current playhead position in samples. Returns 'Unknown' if not yet reported by Ardour."), mime_type: None, size: None }, annotations: None }] }) +2026-06-30T16:20:18.111719Z DEBUG rmcp::service: new event evt=ToSink(Response(ListResourcesResult(ListResourcesResult { next_cursor: None, resources: [Annotated { raw: RawResource { uri: "ardour:/state/playback", name: "Ardour Playback State", description: Some("Current playback state of Ardour (e.g., Playing, Stopped, Unknown)."), mime_type: None, size: None }, annotations: None }, Annotated { raw: RawResource { uri: "ardour:/state/transport_frame", name: "Ardour Transport Frame Position", description: Some("Current playhead position in samples. Returns 'Unknown' if not yet reported by Ardour."), mime_type: None, size: None }, annotations: None }] }), Number(2))) +2026-06-30T16:20:18.111999Z  INFO rmcp::service: response message id=1 result=ListToolsResult(ListToolsResult { next_cursor: None, tools: [Tool { name: "set_track_gain_abs", description: "Sets the absolute gain of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_abs": Object {"description": String("The desired absolute gain. Valid range: 0.0 to 2.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_abs"), String("rid")], "title": String("SetTrackGainAbsArgs"), "type": String("object")} }, Tool { name: "access_action", description: "Executes a specified Ardour menu action by its name.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"action_name": Object {"description": String("The name of the Ardour menu action to execute (e.g., 'Editor/zoom-to-session')."), "type": String("string")}}, "required": Array [String("action_name")], "title": String("AccessActionArgs"), "type": String("object")} }, Tool { name: "transport_play", description: "Starts Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_strip_plugin_parameter", description: "Sets a specific parameter of a plugin on a strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"param_id": Object {"description": String("The 1-indexed ID of the parameter within the plugin."), "format": String("int32"), "type": String("integer")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "value": Object {"description": String("The desired parameter value, normalized (0.0 to 1.0)."), "format": String("float"), "type": String("number")}}, "required": Array [String("param_id"), String("plugin_slot"), String("rid"), String("value")], "title": String("SetStripPluginParameterArgs"), "type": String("object")} }, Tool { name: "set_transport_speed", description: "Sets Ardour's transport speed. Valid range: -8.0 to 8.0.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"speed": Object {"description": String("The desired transport speed. Valid range: -8.0 to 8.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("speed")], "title": String("SetTransportSpeedArgs"), "type": String("object")} }, Tool { name: "rec_enable_toggle", description: "Toggles the master record enable or selected track record enable.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "toggle_punch_out", description: "Toggles the Punch Out state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "save_state", description: "Saves the current session state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_solo", description: "Sets the solo state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "solo_st": Object {"description": String("The desired solo state. 0 for solo off, 1 for solo on."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rid"), String("solo_st")], "title": String("SetTrackSoloArgs"), "type": String("object")} }, Tool { name: "set_track_rec_enable", description: "Sets the record enable state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rec_st": Object {"description": String("The desired record enable state. 0 for off, 1 for on."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rec_st"), String("rid")], "title": String("SetTrackRecEnableArgs"), "type": String("object")} }, Tool { name: "locate", description: "Locates the playhead to a specific sample position and optionally starts playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"roll": Object {"description": String("Whether to start playing after locating. 0 for stop, 1 for play."), "format": String("int32"), "type": String("integer")}, "spos": Object {"description": String("The position in samples to locate to."), "format": String("int64"), "type": String("integer")}}, "required": Array [String("roll"), String("spos")], "title": String("LocateToolArgs"), "type": String("object")} }, Tool { name: "undo", description: "Undoes the last action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "toggle_all_rec_enables", description: "Toggles the record enable state for ALL tracks.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "select_strip", description: "Selects a specific strip (track/bus) in Ardour.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus to select."), "format": String("int32"), "type": String("integer")}, "select_state": Object {"description": String("The desired select state (true to select). Currently, only true (1) is effective for selection."), "type": String("boolean")}}, "required": Array [String("rid"), String("select_state")], "title": String("SelectStripArgs"), "type": String("object")} }, Tool { name: "add_marker", description: "Adds a location marker at the current playhead position.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_mute", description: "Sets the mute state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"mute_state": Object {"description": String("The desired mute state (true for mute, false for unmute)."), "type": String("boolean")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("mute_state"), String("rid")], "title": String("SetTrackMuteArgs"), "type": String("object")} }, Tool { name: "goto_end", description: "Moves the playhead to the session end.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "redo", description: "Redoes the last undone action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "toggle_punch_in", description: "Toggles the Punch In state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "prev_marker", description: "Moves the playhead to the previous location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "loop_toggle", description: "Toggles loop playback mode.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_strip_plugin_active", description: "Activates or deactivates a plugin on a specific strip slot.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"active_state": Object {"description": String("The desired activation state (true for active, false for inactive)."), "type": String("boolean")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("active_state"), String("plugin_slot"), String("rid")], "title": String("SetStripPluginActiveArgs"), "type": String("object")} }, Tool { name: "ffwd", description: "Fast forwards the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "goto_start", description: "Moves the playhead to the session start.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "rewind", description: "Rewinds the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_selected_strip_pan_stereo_width", description: "Sets the stereo width for the currently selected stereo strip's panner.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"width": Object {"description": String("The desired stereo width for the currently selected strip. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("width")], "title": String("SetSelectedStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "transport_stop", description: "Stops Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_trim_abs", description: "Sets the absolute trim of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_abs": Object {"description": String("The desired absolute trim. Valid range: 0.1 to 10.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_abs")], "title": String("SetTrackTrimAbsArgs"), "type": String("object")} }, Tool { name: "set_strip_pan_stereo_width", description: "Sets the stereo width for a panner on a stereo strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the stereo track/bus."), "format": String("int32"), "type": String("integer")}, "width": Object {"description": String("The desired stereo width. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("width")], "title": String("SetStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "set_track_gain_db", description: "Sets the gain of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_db": Object {"description": String("The desired gain in dB. Valid range: -400.0 to 6.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_db"), String("rid")], "title": String("SetTrackGainDBArgs"), "type": String("object")} }, Tool { name: "set_track_trim_db", description: "Sets the trim of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_db": Object {"description": String("The desired trim in dB. Valid range: -20.0 to 20.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_db")], "title": String("SetTrackTrimDBArgs"), "type": String("object")} }, Tool { name: "next_marker", description: "Moves the playhead to the next location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }] }) +2026-06-30T16:20:18.112095Z DEBUG rmcp::service: new event evt=ToSink(Response(ListToolsResult(ListToolsResult { next_cursor: None, tools: [Tool { name: "set_track_gain_abs", description: "Sets the absolute gain of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_abs": Object {"description": String("The desired absolute gain. Valid range: 0.0 to 2.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_abs"), String("rid")], "title": String("SetTrackGainAbsArgs"), "type": String("object")} }, Tool { name: "access_action", description: "Executes a specified Ardour menu action by its name.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"action_name": Object {"description": String("The name of the Ardour menu action to execute (e.g., 'Editor/zoom-to-session')."), "type": String("string")}}, "required": Array [String("action_name")], "title": String("AccessActionArgs"), "type": String("object")} }, Tool { name: "transport_play", description: "Starts Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_strip_plugin_parameter", description: "Sets a specific parameter of a plugin on a strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"param_id": Object {"description": String("The 1-indexed ID of the parameter within the plugin."), "format": String("int32"), "type": String("integer")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "value": Object {"description": String("The desired parameter value, normalized (0.0 to 1.0)."), "format": String("float"), "type": String("number")}}, "required": Array [String("param_id"), String("plugin_slot"), String("rid"), String("value")], "title": String("SetStripPluginParameterArgs"), "type": String("object")} }, Tool { name: "set_transport_speed", description: "Sets Ardour's transport speed. Valid range: -8.0 to 8.0.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"speed": Object {"description": String("The desired transport speed. Valid range: -8.0 to 8.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("speed")], "title": String("SetTransportSpeedArgs"), "type": String("object")} }, Tool { name: "rec_enable_toggle", description: "Toggles the master record enable or selected track record enable.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "toggle_punch_out", description: "Toggles the Punch Out state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "save_state", description: "Saves the current session state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_solo", description: "Sets the solo state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "solo_st": Object {"description": String("The desired solo state. 0 for solo off, 1 for solo on."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rid"), String("solo_st")], "title": String("SetTrackSoloArgs"), "type": String("object")} }, Tool { name: "set_track_rec_enable", description: "Sets the record enable state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rec_st": Object {"description": String("The desired record enable state. 0 for off, 1 for on."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rec_st"), String("rid")], "title": String("SetTrackRecEnableArgs"), "type": String("object")} }, Tool { name: "locate", description: "Locates the playhead to a specific sample position and optionally starts playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"roll": Object {"description": String("Whether to start playing after locating. 0 for stop, 1 for play."), "format": String("int32"), "type": String("integer")}, "spos": Object {"description": String("The position in samples to locate to."), "format": String("int64"), "type": String("integer")}}, "required": Array [String("roll"), String("spos")], "title": String("LocateToolArgs"), "type": String("object")} }, Tool { name: "undo", description: "Undoes the last action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "toggle_all_rec_enables", description: "Toggles the record enable state for ALL tracks.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "select_strip", description: "Selects a specific strip (track/bus) in Ardour.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus to select."), "format": String("int32"), "type": String("integer")}, "select_state": Object {"description": String("The desired select state (true to select). Currently, only true (1) is effective for selection."), "type": String("boolean")}}, "required": Array [String("rid"), String("select_state")], "title": String("SelectStripArgs"), "type": String("object")} }, Tool { name: "add_marker", description: "Adds a location marker at the current playhead position.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_mute", description: "Sets the mute state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"mute_state": Object {"description": String("The desired mute state (true for mute, false for unmute)."), "type": String("boolean")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("mute_state"), String("rid")], "title": String("SetTrackMuteArgs"), "type": String("object")} }, Tool { name: "goto_end", description: "Moves the playhead to the session end.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "redo", description: "Redoes the last undone action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "toggle_punch_in", description: "Toggles the Punch In state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "prev_marker", description: "Moves the playhead to the previous location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "loop_toggle", description: "Toggles loop playback mode.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_strip_plugin_active", description: "Activates or deactivates a plugin on a specific strip slot.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"active_state": Object {"description": String("The desired activation state (true for active, false for inactive)."), "type": String("boolean")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("active_state"), String("plugin_slot"), String("rid")], "title": String("SetStripPluginActiveArgs"), "type": String("object")} }, Tool { name: "ffwd", description: "Fast forwards the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "goto_start", description: "Moves the playhead to the session start.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "rewind", description: "Rewinds the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_selected_strip_pan_stereo_width", description: "Sets the stereo width for the currently selected stereo strip's panner.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"width": Object {"description": String("The desired stereo width for the currently selected strip. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("width")], "title": String("SetSelectedStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "transport_stop", description: "Stops Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_trim_abs", description: "Sets the absolute trim of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_abs": Object {"description": String("The desired absolute trim. Valid range: 0.1 to 10.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_abs")], "title": String("SetTrackTrimAbsArgs"), "type": String("object")} }, Tool { name: "set_strip_pan_stereo_width", description: "Sets the stereo width for a panner on a stereo strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the stereo track/bus."), "format": String("int32"), "type": String("integer")}, "width": Object {"description": String("The desired stereo width. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("width")], "title": String("SetStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "set_track_gain_db", description: "Sets the gain of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_db": Object {"description": String("The desired gain in dB. Valid range: -400.0 to 6.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_db"), String("rid")], "title": String("SetTrackGainDBArgs"), "type": String("object")} }, Tool { name: "set_track_trim_db", description: "Sets the trim of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_db": Object {"description": String("The desired trim in dB. Valid range: -20.0 to 20.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_db")], "title": String("SetTrackTrimDBArgs"), "type": String("object")} }, Tool { name: "next_marker", description: "Moves the playhead to the next location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }] }), Number(1))) +2026-06-30T16:20:18.235682Z  INFO serve_inner: rmcp::service: Service initialized as server peer_info=InitializeRequestParam { protocol_version: ProtocolVersion("2025-11-25"), capabilities: ClientCapabilities { experimental: None, roots: Some(RootsCapabilities { list_changed: None }), sampling: None }, client_info: Implementation { name: "claude-code", version: "2.1.196" } } +2026-06-30T16:20:18.235733Z  INFO ardour_mcp_server: Ardour MCP server started and waiting for connections... +2026-06-30T16:20:18.235767Z DEBUG rmcp::service: new event evt=PeerMessage(Request(ListToolsRequest(Request { method: ListToolsRequestMethod, params: None }), Number(1))) +2026-06-30T16:20:18.235768Z  INFO ardour_mcp_server: Starting OSC listener for Ardour events on 127.0.0.1:9099 +2026-06-30T16:20:18.235774Z  INFO rmcp::service: received request id=1 request=ListToolsRequest(Request { method: ListToolsRequestMethod, params: None }) +2026-06-30T16:20:18.235800Z DEBUG rmcp::service: new event evt=PeerMessage(Request(ListResourcesRequest(Request { method: ListResourcesRequestMethod, params: None }), Number(2))) +2026-06-30T16:20:18.235802Z  INFO rmcp::service: received request id=2 request=ListResourcesRequest(Request { method: ListResourcesRequestMethod, params: None }) +2026-06-30T16:20:18.235814Z DEBUG ardour_mcp_server: Listing resources. Count: 2, Content: [Annotated { raw: RawResource { uri: "ardour:/state/playback", name: "Ardour Playback State", description: Some("Current playback state of Ardour (e.g., Playing, Stopped, Unknown)."), mime_type: None, size: None }, annotations: None }, Annotated { raw: RawResource { uri: "ardour:/state/transport_frame", name: "Ardour Transport Frame Position", description: Some("Current playhead position in samples. Returns 'Unknown' if not yet reported by Ardour."), mime_type: None, size: None }, annotations: None }] +2026-06-30T16:20:18.235838Z ERROR ardour_mcp_server: OSC listener task failed: Failed to bind Tokio UDP socket for OSC on 127.0.0.1:9099: Address already in use (os error 98) +2026-06-30T16:20:18.235843Z  INFO rmcp::service: response message id=2 result=ListResourcesResult(ListResourcesResult { next_cursor: None, resources: [Annotated { raw: RawResource { uri: "ardour:/state/playback", name: "Ardour Playback State", description: Some("Current playback state of Ardour (e.g., Playing, Stopped, Unknown)."), mime_type: None, size: None }, annotations: None }, Annotated { raw: RawResource { uri: "ardour:/state/transport_frame", name: "Ardour Transport Frame Position", description: Some("Current playhead position in samples. Returns 'Unknown' if not yet reported by Ardour."), mime_type: None, size: None }, annotations: None }] }) +2026-06-30T16:20:18.235852Z DEBUG rmcp::service: new event evt=ToSink(Response(ListResourcesResult(ListResourcesResult { next_cursor: None, resources: [Annotated { raw: RawResource { uri: "ardour:/state/playback", name: "Ardour Playback State", description: Some("Current playback state of Ardour (e.g., Playing, Stopped, Unknown)."), mime_type: None, size: None }, annotations: None }, Annotated { raw: RawResource { uri: "ardour:/state/transport_frame", name: "Ardour Transport Frame Position", description: Some("Current playhead position in samples. Returns 'Unknown' if not yet reported by Ardour."), mime_type: None, size: None }, annotations: None }] }), Number(2))) +2026-06-30T16:20:18.235929Z  INFO rmcp::service: response message id=1 result=ListToolsResult(ListToolsResult { next_cursor: None, tools: [Tool { name: "set_track_trim_db", description: "Sets the trim of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_db": Object {"description": String("The desired trim in dB. Valid range: -20.0 to 20.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_db")], "title": String("SetTrackTrimDBArgs"), "type": String("object")} }, Tool { name: "select_strip", description: "Selects a specific strip (track/bus) in Ardour.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus to select."), "format": String("int32"), "type": String("integer")}, "select_state": Object {"description": String("The desired select state (true to select). Currently, only true (1) is effective for selection."), "type": String("boolean")}}, "required": Array [String("rid"), String("select_state")], "title": String("SelectStripArgs"), "type": String("object")} }, Tool { name: "rewind", description: "Rewinds the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "rec_enable_toggle", description: "Toggles the master record enable or selected track record enable.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "redo", description: "Redoes the last undone action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_strip_pan_stereo_width", description: "Sets the stereo width for a panner on a stereo strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the stereo track/bus."), "format": String("int32"), "type": String("integer")}, "width": Object {"description": String("The desired stereo width. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("width")], "title": String("SetStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "toggle_punch_in", description: "Toggles the Punch In state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "goto_start", description: "Moves the playhead to the session start.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_mute", description: "Sets the mute state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"mute_state": Object {"description": String("The desired mute state (true for mute, false for unmute)."), "type": String("boolean")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("mute_state"), String("rid")], "title": String("SetTrackMuteArgs"), "type": String("object")} }, Tool { name: "set_track_rec_enable", description: "Sets the record enable state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rec_st": Object {"description": String("The desired record enable state. 0 for off, 1 for on."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rec_st"), String("rid")], "title": String("SetTrackRecEnableArgs"), "type": String("object")} }, Tool { name: "transport_stop", description: "Stops Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "access_action", description: "Executes a specified Ardour menu action by its name.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"action_name": Object {"description": String("The name of the Ardour menu action to execute (e.g., 'Editor/zoom-to-session')."), "type": String("string")}}, "required": Array [String("action_name")], "title": String("AccessActionArgs"), "type": String("object")} }, Tool { name: "toggle_punch_out", description: "Toggles the Punch Out state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "next_marker", description: "Moves the playhead to the next location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_trim_abs", description: "Sets the absolute trim of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_abs": Object {"description": String("The desired absolute trim. Valid range: 0.1 to 10.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_abs")], "title": String("SetTrackTrimAbsArgs"), "type": String("object")} }, Tool { name: "set_strip_plugin_active", description: "Activates or deactivates a plugin on a specific strip slot.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"active_state": Object {"description": String("The desired activation state (true for active, false for inactive)."), "type": String("boolean")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("active_state"), String("plugin_slot"), String("rid")], "title": String("SetStripPluginActiveArgs"), "type": String("object")} }, Tool { name: "set_strip_plugin_parameter", description: "Sets a specific parameter of a plugin on a strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"param_id": Object {"description": String("The 1-indexed ID of the parameter within the plugin."), "format": String("int32"), "type": String("integer")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "value": Object {"description": String("The desired parameter value, normalized (0.0 to 1.0)."), "format": String("float"), "type": String("number")}}, "required": Array [String("param_id"), String("plugin_slot"), String("rid"), String("value")], "title": String("SetStripPluginParameterArgs"), "type": String("object")} }, Tool { name: "goto_end", description: "Moves the playhead to the session end.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_solo", description: "Sets the solo state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "solo_st": Object {"description": String("The desired solo state. 0 for solo off, 1 for solo on."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rid"), String("solo_st")], "title": String("SetTrackSoloArgs"), "type": String("object")} }, Tool { name: "toggle_all_rec_enables", description: "Toggles the record enable state for ALL tracks.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "locate", description: "Locates the playhead to a specific sample position and optionally starts playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"roll": Object {"description": String("Whether to start playing after locating. 0 for stop, 1 for play."), "format": String("int32"), "type": String("integer")}, "spos": Object {"description": String("The position in samples to locate to."), "format": String("int64"), "type": String("integer")}}, "required": Array [String("roll"), String("spos")], "title": String("LocateToolArgs"), "type": String("object")} }, Tool { name: "add_marker", description: "Adds a location marker at the current playhead position.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "prev_marker", description: "Moves the playhead to the previous location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "save_state", description: "Saves the current session state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "undo", description: "Undoes the last action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_transport_speed", description: "Sets Ardour's transport speed. Valid range: -8.0 to 8.0.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"speed": Object {"description": String("The desired transport speed. Valid range: -8.0 to 8.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("speed")], "title": String("SetTransportSpeedArgs"), "type": String("object")} }, Tool { name: "transport_play", description: "Starts Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "ffwd", description: "Fast forwards the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "loop_toggle", description: "Toggles loop playback mode.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_selected_strip_pan_stereo_width", description: "Sets the stereo width for the currently selected stereo strip's panner.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"width": Object {"description": String("The desired stereo width for the currently selected strip. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("width")], "title": String("SetSelectedStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "set_track_gain_abs", description: "Sets the absolute gain of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_abs": Object {"description": String("The desired absolute gain. Valid range: 0.0 to 2.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_abs"), String("rid")], "title": String("SetTrackGainAbsArgs"), "type": String("object")} }, Tool { name: "set_track_gain_db", description: "Sets the gain of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_db": Object {"description": String("The desired gain in dB. Valid range: -400.0 to 6.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_db"), String("rid")], "title": String("SetTrackGainDBArgs"), "type": String("object")} }] }) +2026-06-30T16:20:18.235985Z DEBUG rmcp::service: new event evt=ToSink(Response(ListToolsResult(ListToolsResult { next_cursor: None, tools: [Tool { name: "set_track_trim_db", description: "Sets the trim of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_db": Object {"description": String("The desired trim in dB. Valid range: -20.0 to 20.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_db")], "title": String("SetTrackTrimDBArgs"), "type": String("object")} }, Tool { name: "select_strip", description: "Selects a specific strip (track/bus) in Ardour.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus to select."), "format": String("int32"), "type": String("integer")}, "select_state": Object {"description": String("The desired select state (true to select). Currently, only true (1) is effective for selection."), "type": String("boolean")}}, "required": Array [String("rid"), String("select_state")], "title": String("SelectStripArgs"), "type": String("object")} }, Tool { name: "rewind", description: "Rewinds the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "rec_enable_toggle", description: "Toggles the master record enable or selected track record enable.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "redo", description: "Redoes the last undone action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_strip_pan_stereo_width", description: "Sets the stereo width for a panner on a stereo strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the stereo track/bus."), "format": String("int32"), "type": String("integer")}, "width": Object {"description": String("The desired stereo width. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("width")], "title": String("SetStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "toggle_punch_in", description: "Toggles the Punch In state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "goto_start", description: "Moves the playhead to the session start.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_mute", description: "Sets the mute state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"mute_state": Object {"description": String("The desired mute state (true for mute, false for unmute)."), "type": String("boolean")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("mute_state"), String("rid")], "title": String("SetTrackMuteArgs"), "type": String("object")} }, Tool { name: "set_track_rec_enable", description: "Sets the record enable state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rec_st": Object {"description": String("The desired record enable state. 0 for off, 1 for on."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rec_st"), String("rid")], "title": String("SetTrackRecEnableArgs"), "type": String("object")} }, Tool { name: "transport_stop", description: "Stops Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "access_action", description: "Executes a specified Ardour menu action by its name.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"action_name": Object {"description": String("The name of the Ardour menu action to execute (e.g., 'Editor/zoom-to-session')."), "type": String("string")}}, "required": Array [String("action_name")], "title": String("AccessActionArgs"), "type": String("object")} }, Tool { name: "toggle_punch_out", description: "Toggles the Punch Out state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "next_marker", description: "Moves the playhead to the next location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_trim_abs", description: "Sets the absolute trim of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_abs": Object {"description": String("The desired absolute trim. Valid range: 0.1 to 10.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_abs")], "title": String("SetTrackTrimAbsArgs"), "type": String("object")} }, Tool { name: "set_strip_plugin_active", description: "Activates or deactivates a plugin on a specific strip slot.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"active_state": Object {"description": String("The desired activation state (true for active, false for inactive)."), "type": String("boolean")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("active_state"), String("plugin_slot"), String("rid")], "title": String("SetStripPluginActiveArgs"), "type": String("object")} }, Tool { name: "set_strip_plugin_parameter", description: "Sets a specific parameter of a plugin on a strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"param_id": Object {"description": String("The 1-indexed ID of the parameter within the plugin."), "format": String("int32"), "type": String("integer")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "value": Object {"description": String("The desired parameter value, normalized (0.0 to 1.0)."), "format": String("float"), "type": String("number")}}, "required": Array [String("param_id"), String("plugin_slot"), String("rid"), String("value")], "title": String("SetStripPluginParameterArgs"), "type": String("object")} }, Tool { name: "goto_end", description: "Moves the playhead to the session end.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_solo", description: "Sets the solo state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "solo_st": Object {"description": String("The desired solo state. 0 for solo off, 1 for solo on."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rid"), String("solo_st")], "title": String("SetTrackSoloArgs"), "type": String("object")} }, Tool { name: "toggle_all_rec_enables", description: "Toggles the record enable state for ALL tracks.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "locate", description: "Locates the playhead to a specific sample position and optionally starts playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"roll": Object {"description": String("Whether to start playing after locating. 0 for stop, 1 for play."), "format": String("int32"), "type": String("integer")}, "spos": Object {"description": String("The position in samples to locate to."), "format": String("int64"), "type": String("integer")}}, "required": Array [String("roll"), String("spos")], "title": String("LocateToolArgs"), "type": String("object")} }, Tool { name: "add_marker", description: "Adds a location marker at the current playhead position.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "prev_marker", description: "Moves the playhead to the previous location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "save_state", description: "Saves the current session state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "undo", description: "Undoes the last action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_transport_speed", description: "Sets Ardour's transport speed. Valid range: -8.0 to 8.0.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"speed": Object {"description": String("The desired transport speed. Valid range: -8.0 to 8.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("speed")], "title": String("SetTransportSpeedArgs"), "type": String("object")} }, Tool { name: "transport_play", description: "Starts Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "ffwd", description: "Fast forwards the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "loop_toggle", description: "Toggles loop playback mode.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_selected_strip_pan_stereo_width", description: "Sets the stereo width for the currently selected stereo strip's panner.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"width": Object {"description": String("The desired stereo width for the currently selected strip. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("width")], "title": String("SetSelectedStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "set_track_gain_abs", description: "Sets the absolute gain of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_abs": Object {"description": String("The desired absolute gain. Valid range: 0.0 to 2.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_abs"), String("rid")], "title": String("SetTrackGainAbsArgs"), "type": String("object")} }, Tool { name: "set_track_gain_db", description: "Sets the gain of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_db": Object {"description": String("The desired gain in dB. Valid range: -400.0 to 6.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_db"), String("rid")], "title": String("SetTrackGainDBArgs"), "type": String("object")} }] }), Number(1))) +2026-06-30T16:34:00.424523Z  INFO ardour_mcp_server: +====================================================================== +NEW SERVER RUN: Tue, 30 Jun 2026 18:34:00 +0200 +====================================================================== +2026-06-30T16:34:00.424577Z  INFO ardour_mcp_server: Attempting to create OSC sender for Ardour at 127.0.0.1:3819 +2026-06-30T16:34:00.424618Z  INFO ardour_mcp_server: OSC sender created and connected to Ardour at 127.0.0.1:3819 +2026-06-30T16:34:00.424625Z  INFO ardour_mcp_server: Sending /set_surface to Ardour to enable OSC feedback. +2026-06-30T16:34:00.424631Z  INFO ardour_mcp_server: Targeting feedback port: 9099 +2026-06-30T16:34:00.424820Z  INFO ardour_mcp_server: /set_surface command sent successfully to Ardour. +2026-06-30T16:34:00.481128Z  INFO serve_inner: rmcp::service: Service initialized as server peer_info=InitializeRequestParam { protocol_version: ProtocolVersion("2025-11-25"), capabilities: ClientCapabilities { experimental: None, roots: Some(RootsCapabilities { list_changed: None }), sampling: None }, client_info: Implementation { name: "claude-code", version: "2.1.196" } } +2026-06-30T16:34:00.481181Z  INFO ardour_mcp_server: Ardour MCP server started and waiting for connections... +2026-06-30T16:34:00.481211Z  INFO ardour_mcp_server: Starting OSC listener for Ardour events on 127.0.0.1:9099 +2026-06-30T16:34:00.481214Z DEBUG rmcp::service: new event evt=PeerMessage(Request(ListToolsRequest(Request { method: ListToolsRequestMethod, params: None }), Number(1))) +2026-06-30T16:34:00.481239Z  INFO rmcp::service: received request id=1 request=ListToolsRequest(Request { method: ListToolsRequestMethod, params: None }) +2026-06-30T16:34:00.481263Z DEBUG rmcp::service: new event evt=PeerMessage(Request(ListResourcesRequest(Request { method: ListResourcesRequestMethod, params: None }), Number(2))) +2026-06-30T16:34:00.481268Z  INFO rmcp::service: received request id=2 request=ListResourcesRequest(Request { method: ListResourcesRequestMethod, params: None }) +2026-06-30T16:34:00.481280Z ERROR ardour_mcp_server: OSC listener task failed: Failed to bind Tokio UDP socket for OSC on 127.0.0.1:9099: Address already in use (os error 98) +2026-06-30T16:34:00.481290Z DEBUG ardour_mcp_server: Listing resources. Count: 2, Content: [Annotated { raw: RawResource { uri: "ardour:/state/playback", name: "Ardour Playback State", description: Some("Current playback state of Ardour (e.g., Playing, Stopped, Unknown)."), mime_type: None, size: None }, annotations: None }, Annotated { raw: RawResource { uri: "ardour:/state/transport_frame", name: "Ardour Transport Frame Position", description: Some("Current playhead position in samples. Returns 'Unknown' if not yet reported by Ardour."), mime_type: None, size: None }, annotations: None }] +2026-06-30T16:34:00.481301Z  INFO rmcp::service: response message id=2 result=ListResourcesResult(ListResourcesResult { next_cursor: None, resources: [Annotated { raw: RawResource { uri: "ardour:/state/playback", name: "Ardour Playback State", description: Some("Current playback state of Ardour (e.g., Playing, Stopped, Unknown)."), mime_type: None, size: None }, annotations: None }, Annotated { raw: RawResource { uri: "ardour:/state/transport_frame", name: "Ardour Transport Frame Position", description: Some("Current playhead position in samples. Returns 'Unknown' if not yet reported by Ardour."), mime_type: None, size: None }, annotations: None }] }) +2026-06-30T16:34:00.481339Z DEBUG rmcp::service: new event evt=ToSink(Response(ListResourcesResult(ListResourcesResult { next_cursor: None, resources: [Annotated { raw: RawResource { uri: "ardour:/state/playback", name: "Ardour Playback State", description: Some("Current playback state of Ardour (e.g., Playing, Stopped, Unknown)."), mime_type: None, size: None }, annotations: None }, Annotated { raw: RawResource { uri: "ardour:/state/transport_frame", name: "Ardour Transport Frame Position", description: Some("Current playhead position in samples. Returns 'Unknown' if not yet reported by Ardour."), mime_type: None, size: None }, annotations: None }] }), Number(2))) +2026-06-30T16:34:00.481423Z  INFO rmcp::service: response message id=1 result=ListToolsResult(ListToolsResult { next_cursor: None, tools: [Tool { name: "toggle_all_rec_enables", description: "Toggles the record enable state for ALL tracks.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "save_state", description: "Saves the current session state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "toggle_punch_out", description: "Toggles the Punch Out state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "transport_play", description: "Starts Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "goto_end", description: "Moves the playhead to the session end.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "add_marker", description: "Adds a location marker at the current playhead position.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "next_marker", description: "Moves the playhead to the next location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "goto_start", description: "Moves the playhead to the session start.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_trim_abs", description: "Sets the absolute trim of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_abs": Object {"description": String("The desired absolute trim. Valid range: 0.1 to 10.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_abs")], "title": String("SetTrackTrimAbsArgs"), "type": String("object")} }, Tool { name: "set_track_trim_db", description: "Sets the trim of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_db": Object {"description": String("The desired trim in dB. Valid range: -20.0 to 20.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_db")], "title": String("SetTrackTrimDBArgs"), "type": String("object")} }, Tool { name: "set_selected_strip_pan_stereo_width", description: "Sets the stereo width for the currently selected stereo strip's panner.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"width": Object {"description": String("The desired stereo width for the currently selected strip. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("width")], "title": String("SetSelectedStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "loop_toggle", description: "Toggles loop playback mode.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "access_action", description: "Executes a specified Ardour menu action by its name.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"action_name": Object {"description": String("The name of the Ardour menu action to execute (e.g., 'Editor/zoom-to-session')."), "type": String("string")}}, "required": Array [String("action_name")], "title": String("AccessActionArgs"), "type": String("object")} }, Tool { name: "set_track_gain_db", description: "Sets the gain of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_db": Object {"description": String("The desired gain in dB. Valid range: -400.0 to 6.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_db"), String("rid")], "title": String("SetTrackGainDBArgs"), "type": String("object")} }, Tool { name: "undo", description: "Undoes the last action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "transport_stop", description: "Stops Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "ffwd", description: "Fast forwards the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "rewind", description: "Rewinds the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_transport_speed", description: "Sets Ardour's transport speed. Valid range: -8.0 to 8.0.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"speed": Object {"description": String("The desired transport speed. Valid range: -8.0 to 8.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("speed")], "title": String("SetTransportSpeedArgs"), "type": String("object")} }, Tool { name: "rec_enable_toggle", description: "Toggles the master record enable or selected track record enable.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "prev_marker", description: "Moves the playhead to the previous location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_solo", description: "Sets the solo state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "solo_st": Object {"description": String("The desired solo state. 0 for solo off, 1 for solo on."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rid"), String("solo_st")], "title": String("SetTrackSoloArgs"), "type": String("object")} }, Tool { name: "set_strip_pan_stereo_width", description: "Sets the stereo width for a panner on a stereo strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the stereo track/bus."), "format": String("int32"), "type": String("integer")}, "width": Object {"description": String("The desired stereo width. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("width")], "title": String("SetStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "select_strip", description: "Selects a specific strip (track/bus) in Ardour.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus to select."), "format": String("int32"), "type": String("integer")}, "select_state": Object {"description": String("The desired select state (true to select). Currently, only true (1) is effective for selection."), "type": String("boolean")}}, "required": Array [String("rid"), String("select_state")], "title": String("SelectStripArgs"), "type": String("object")} }, Tool { name: "toggle_punch_in", description: "Toggles the Punch In state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "locate", description: "Locates the playhead to a specific sample position and optionally starts playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"roll": Object {"description": String("Whether to start playing after locating. 0 for stop, 1 for play."), "format": String("int32"), "type": String("integer")}, "spos": Object {"description": String("The position in samples to locate to."), "format": String("int64"), "type": String("integer")}}, "required": Array [String("roll"), String("spos")], "title": String("LocateToolArgs"), "type": String("object")} }, Tool { name: "set_track_mute", description: "Sets the mute state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"mute_state": Object {"description": String("The desired mute state (true for mute, false for unmute)."), "type": String("boolean")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("mute_state"), String("rid")], "title": String("SetTrackMuteArgs"), "type": String("object")} }, Tool { name: "set_strip_plugin_active", description: "Activates or deactivates a plugin on a specific strip slot.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"active_state": Object {"description": String("The desired activation state (true for active, false for inactive)."), "type": String("boolean")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("active_state"), String("plugin_slot"), String("rid")], "title": String("SetStripPluginActiveArgs"), "type": String("object")} }, Tool { name: "set_track_gain_abs", description: "Sets the absolute gain of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_abs": Object {"description": String("The desired absolute gain. Valid range: 0.0 to 2.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_abs"), String("rid")], "title": String("SetTrackGainAbsArgs"), "type": String("object")} }, Tool { name: "set_strip_plugin_parameter", description: "Sets a specific parameter of a plugin on a strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"param_id": Object {"description": String("The 1-indexed ID of the parameter within the plugin."), "format": String("int32"), "type": String("integer")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "value": Object {"description": String("The desired parameter value, normalized (0.0 to 1.0)."), "format": String("float"), "type": String("number")}}, "required": Array [String("param_id"), String("plugin_slot"), String("rid"), String("value")], "title": String("SetStripPluginParameterArgs"), "type": String("object")} }, Tool { name: "set_track_rec_enable", description: "Sets the record enable state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rec_st": Object {"description": String("The desired record enable state. 0 for off, 1 for on."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rec_st"), String("rid")], "title": String("SetTrackRecEnableArgs"), "type": String("object")} }, Tool { name: "redo", description: "Redoes the last undone action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }] }) +2026-06-30T16:34:00.481515Z DEBUG rmcp::service: new event evt=ToSink(Response(ListToolsResult(ListToolsResult { next_cursor: None, tools: [Tool { name: "toggle_all_rec_enables", description: "Toggles the record enable state for ALL tracks.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "save_state", description: "Saves the current session state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "toggle_punch_out", description: "Toggles the Punch Out state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "transport_play", description: "Starts Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "goto_end", description: "Moves the playhead to the session end.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "add_marker", description: "Adds a location marker at the current playhead position.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "next_marker", description: "Moves the playhead to the next location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "goto_start", description: "Moves the playhead to the session start.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_trim_abs", description: "Sets the absolute trim of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_abs": Object {"description": String("The desired absolute trim. Valid range: 0.1 to 10.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_abs")], "title": String("SetTrackTrimAbsArgs"), "type": String("object")} }, Tool { name: "set_track_trim_db", description: "Sets the trim of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_db": Object {"description": String("The desired trim in dB. Valid range: -20.0 to 20.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_db")], "title": String("SetTrackTrimDBArgs"), "type": String("object")} }, Tool { name: "set_selected_strip_pan_stereo_width", description: "Sets the stereo width for the currently selected stereo strip's panner.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"width": Object {"description": String("The desired stereo width for the currently selected strip. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("width")], "title": String("SetSelectedStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "loop_toggle", description: "Toggles loop playback mode.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "access_action", description: "Executes a specified Ardour menu action by its name.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"action_name": Object {"description": String("The name of the Ardour menu action to execute (e.g., 'Editor/zoom-to-session')."), "type": String("string")}}, "required": Array [String("action_name")], "title": String("AccessActionArgs"), "type": String("object")} }, Tool { name: "set_track_gain_db", description: "Sets the gain of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_db": Object {"description": String("The desired gain in dB. Valid range: -400.0 to 6.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_db"), String("rid")], "title": String("SetTrackGainDBArgs"), "type": String("object")} }, Tool { name: "undo", description: "Undoes the last action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "transport_stop", description: "Stops Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "ffwd", description: "Fast forwards the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "rewind", description: "Rewinds the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_transport_speed", description: "Sets Ardour's transport speed. Valid range: -8.0 to 8.0.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"speed": Object {"description": String("The desired transport speed. Valid range: -8.0 to 8.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("speed")], "title": String("SetTransportSpeedArgs"), "type": String("object")} }, Tool { name: "rec_enable_toggle", description: "Toggles the master record enable or selected track record enable.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "prev_marker", description: "Moves the playhead to the previous location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_solo", description: "Sets the solo state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "solo_st": Object {"description": String("The desired solo state. 0 for solo off, 1 for solo on."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rid"), String("solo_st")], "title": String("SetTrackSoloArgs"), "type": String("object")} }, Tool { name: "set_strip_pan_stereo_width", description: "Sets the stereo width for a panner on a stereo strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the stereo track/bus."), "format": String("int32"), "type": String("integer")}, "width": Object {"description": String("The desired stereo width. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("width")], "title": String("SetStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "select_strip", description: "Selects a specific strip (track/bus) in Ardour.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus to select."), "format": String("int32"), "type": String("integer")}, "select_state": Object {"description": String("The desired select state (true to select). Currently, only true (1) is effective for selection."), "type": String("boolean")}}, "required": Array [String("rid"), String("select_state")], "title": String("SelectStripArgs"), "type": String("object")} }, Tool { name: "toggle_punch_in", description: "Toggles the Punch In state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "locate", description: "Locates the playhead to a specific sample position and optionally starts playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"roll": Object {"description": String("Whether to start playing after locating. 0 for stop, 1 for play."), "format": String("int32"), "type": String("integer")}, "spos": Object {"description": String("The position in samples to locate to."), "format": String("int64"), "type": String("integer")}}, "required": Array [String("roll"), String("spos")], "title": String("LocateToolArgs"), "type": String("object")} }, Tool { name: "set_track_mute", description: "Sets the mute state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"mute_state": Object {"description": String("The desired mute state (true for mute, false for unmute)."), "type": String("boolean")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("mute_state"), String("rid")], "title": String("SetTrackMuteArgs"), "type": String("object")} }, Tool { name: "set_strip_plugin_active", description: "Activates or deactivates a plugin on a specific strip slot.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"active_state": Object {"description": String("The desired activation state (true for active, false for inactive)."), "type": String("boolean")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("active_state"), String("plugin_slot"), String("rid")], "title": String("SetStripPluginActiveArgs"), "type": String("object")} }, Tool { name: "set_track_gain_abs", description: "Sets the absolute gain of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_abs": Object {"description": String("The desired absolute gain. Valid range: 0.0 to 2.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_abs"), String("rid")], "title": String("SetTrackGainAbsArgs"), "type": String("object")} }, Tool { name: "set_strip_plugin_parameter", description: "Sets a specific parameter of a plugin on a strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"param_id": Object {"description": String("The 1-indexed ID of the parameter within the plugin."), "format": String("int32"), "type": String("integer")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "value": Object {"description": String("The desired parameter value, normalized (0.0 to 1.0)."), "format": String("float"), "type": String("number")}}, "required": Array [String("param_id"), String("plugin_slot"), String("rid"), String("value")], "title": String("SetStripPluginParameterArgs"), "type": String("object")} }, Tool { name: "set_track_rec_enable", description: "Sets the record enable state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rec_st": Object {"description": String("The desired record enable state. 0 for off, 1 for on."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rec_st"), String("rid")], "title": String("SetTrackRecEnableArgs"), "type": String("object")} }, Tool { name: "redo", description: "Redoes the last undone action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }] }), Number(1))) +2026-06-30T17:56:39.898812Z  INFO ardour_mcp_server: +====================================================================== +NEW SERVER RUN: Tue, 30 Jun 2026 19:56:39 +0200 +====================================================================== +2026-06-30T17:56:39.898848Z  INFO ardour_mcp_server: Attempting to create OSC sender for Ardour at 127.0.0.1:3819 +2026-06-30T17:56:39.898876Z  INFO ardour_mcp_server: OSC sender created and connected to Ardour at 127.0.0.1:3819 +2026-06-30T17:56:39.898879Z  INFO ardour_mcp_server: Sending /set_surface to Ardour to enable OSC feedback. +2026-06-30T17:56:39.898881Z  INFO ardour_mcp_server: Targeting feedback port: 9099 +2026-06-30T17:56:39.898999Z  INFO ardour_mcp_server: /set_surface command sent successfully to Ardour. +2026-06-30T17:56:39.985596Z  INFO serve_inner: rmcp::service: Service initialized as server peer_info=InitializeRequestParam { protocol_version: ProtocolVersion("2025-11-25"), capabilities: ClientCapabilities { experimental: None, roots: Some(RootsCapabilities { list_changed: None }), sampling: None }, client_info: Implementation { name: "claude-code", version: "2.1.196" } } +2026-06-30T17:56:39.985643Z  INFO ardour_mcp_server: Ardour MCP server started and waiting for connections... +2026-06-30T17:56:39.985679Z DEBUG rmcp::service: new event evt=PeerMessage(Request(ListToolsRequest(Request { method: ListToolsRequestMethod, params: None }), Number(1))) +2026-06-30T17:56:39.985693Z  INFO rmcp::service: received request id=1 request=ListToolsRequest(Request { method: ListToolsRequestMethod, params: None }) +2026-06-30T17:56:39.985696Z  INFO ardour_mcp_server: Starting OSC listener for Ardour events on 127.0.0.1:9099 +2026-06-30T17:56:39.985713Z DEBUG rmcp::service: new event evt=PeerMessage(Request(ListResourcesRequest(Request { method: ListResourcesRequestMethod, params: None }), Number(2))) +2026-06-30T17:56:39.985715Z  INFO rmcp::service: received request id=2 request=ListResourcesRequest(Request { method: ListResourcesRequestMethod, params: None }) +2026-06-30T17:56:39.985725Z DEBUG ardour_mcp_server: Listing resources. Count: 2, Content: [Annotated { raw: RawResource { uri: "ardour:/state/playback", name: "Ardour Playback State", description: Some("Current playback state of Ardour (e.g., Playing, Stopped, Unknown)."), mime_type: None, size: None }, annotations: None }, Annotated { raw: RawResource { uri: "ardour:/state/transport_frame", name: "Ardour Transport Frame Position", description: Some("Current playhead position in samples. Returns 'Unknown' if not yet reported by Ardour."), mime_type: None, size: None }, annotations: None }] +2026-06-30T17:56:39.985731Z  INFO rmcp::service: response message id=2 result=ListResourcesResult(ListResourcesResult { next_cursor: None, resources: [Annotated { raw: RawResource { uri: "ardour:/state/playback", name: "Ardour Playback State", description: Some("Current playback state of Ardour (e.g., Playing, Stopped, Unknown)."), mime_type: None, size: None }, annotations: None }, Annotated { raw: RawResource { uri: "ardour:/state/transport_frame", name: "Ardour Transport Frame Position", description: Some("Current playhead position in samples. Returns 'Unknown' if not yet reported by Ardour."), mime_type: None, size: None }, annotations: None }] }) +2026-06-30T17:56:39.985738Z DEBUG rmcp::service: new event evt=ToSink(Response(ListResourcesResult(ListResourcesResult { next_cursor: None, resources: [Annotated { raw: RawResource { uri: "ardour:/state/playback", name: "Ardour Playback State", description: Some("Current playback state of Ardour (e.g., Playing, Stopped, Unknown)."), mime_type: None, size: None }, annotations: None }, Annotated { raw: RawResource { uri: "ardour:/state/transport_frame", name: "Ardour Transport Frame Position", description: Some("Current playhead position in samples. Returns 'Unknown' if not yet reported by Ardour."), mime_type: None, size: None }, annotations: None }] }), Number(2))) +2026-06-30T17:56:39.985754Z ERROR ardour_mcp_server: OSC listener task failed: Failed to bind Tokio UDP socket for OSC on 127.0.0.1:9099: Address already in use (os error 98) +2026-06-30T17:56:39.985884Z  INFO rmcp::service: response message id=1 result=ListToolsResult(ListToolsResult { next_cursor: None, tools: [Tool { name: "redo", description: "Redoes the last undone action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "goto_end", description: "Moves the playhead to the session end.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_strip_plugin_active", description: "Activates or deactivates a plugin on a specific strip slot.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"active_state": Object {"description": String("The desired activation state (true for active, false for inactive)."), "type": String("boolean")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("active_state"), String("plugin_slot"), String("rid")], "title": String("SetStripPluginActiveArgs"), "type": String("object")} }, Tool { name: "transport_stop", description: "Stops Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_rec_enable", description: "Sets the record enable state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rec_st": Object {"description": String("The desired record enable state. 0 for off, 1 for on."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rec_st"), String("rid")], "title": String("SetTrackRecEnableArgs"), "type": String("object")} }, Tool { name: "set_strip_plugin_parameter", description: "Sets a specific parameter of a plugin on a strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"param_id": Object {"description": String("The 1-indexed ID of the parameter within the plugin."), "format": String("int32"), "type": String("integer")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "value": Object {"description": String("The desired parameter value, normalized (0.0 to 1.0)."), "format": String("float"), "type": String("number")}}, "required": Array [String("param_id"), String("plugin_slot"), String("rid"), String("value")], "title": String("SetStripPluginParameterArgs"), "type": String("object")} }, Tool { name: "toggle_punch_out", description: "Toggles the Punch Out state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "ffwd", description: "Fast forwards the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "toggle_all_rec_enables", description: "Toggles the record enable state for ALL tracks.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "undo", description: "Undoes the last action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_strip_pan_stereo_width", description: "Sets the stereo width for a panner on a stereo strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the stereo track/bus."), "format": String("int32"), "type": String("integer")}, "width": Object {"description": String("The desired stereo width. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("width")], "title": String("SetStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "goto_start", description: "Moves the playhead to the session start.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_transport_speed", description: "Sets Ardour's transport speed. Valid range: -8.0 to 8.0.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"speed": Object {"description": String("The desired transport speed. Valid range: -8.0 to 8.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("speed")], "title": String("SetTransportSpeedArgs"), "type": String("object")} }, Tool { name: "loop_toggle", description: "Toggles loop playback mode.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "next_marker", description: "Moves the playhead to the next location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_solo", description: "Sets the solo state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "solo_st": Object {"description": String("The desired solo state. 0 for solo off, 1 for solo on."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rid"), String("solo_st")], "title": String("SetTrackSoloArgs"), "type": String("object")} }, Tool { name: "set_track_gain_db", description: "Sets the gain of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_db": Object {"description": String("The desired gain in dB. Valid range: -400.0 to 6.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_db"), String("rid")], "title": String("SetTrackGainDBArgs"), "type": String("object")} }, Tool { name: "save_state", description: "Saves the current session state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "select_strip", description: "Selects a specific strip (track/bus) in Ardour.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus to select."), "format": String("int32"), "type": String("integer")}, "select_state": Object {"description": String("The desired select state (true to select). Currently, only true (1) is effective for selection."), "type": String("boolean")}}, "required": Array [String("rid"), String("select_state")], "title": String("SelectStripArgs"), "type": String("object")} }, Tool { name: "set_track_gain_abs", description: "Sets the absolute gain of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_abs": Object {"description": String("The desired absolute gain. Valid range: 0.0 to 2.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_abs"), String("rid")], "title": String("SetTrackGainAbsArgs"), "type": String("object")} }, Tool { name: "set_selected_strip_pan_stereo_width", description: "Sets the stereo width for the currently selected stereo strip's panner.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"width": Object {"description": String("The desired stereo width for the currently selected strip. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("width")], "title": String("SetSelectedStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "transport_play", description: "Starts Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "rewind", description: "Rewinds the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_trim_abs", description: "Sets the absolute trim of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_abs": Object {"description": String("The desired absolute trim. Valid range: 0.1 to 10.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_abs")], "title": String("SetTrackTrimAbsArgs"), "type": String("object")} }, Tool { name: "toggle_punch_in", description: "Toggles the Punch In state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "add_marker", description: "Adds a location marker at the current playhead position.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_trim_db", description: "Sets the trim of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_db": Object {"description": String("The desired trim in dB. Valid range: -20.0 to 20.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_db")], "title": String("SetTrackTrimDBArgs"), "type": String("object")} }, Tool { name: "locate", description: "Locates the playhead to a specific sample position and optionally starts playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"roll": Object {"description": String("Whether to start playing after locating. 0 for stop, 1 for play."), "format": String("int32"), "type": String("integer")}, "spos": Object {"description": String("The position in samples to locate to."), "format": String("int64"), "type": String("integer")}}, "required": Array [String("roll"), String("spos")], "title": String("LocateToolArgs"), "type": String("object")} }, Tool { name: "set_track_mute", description: "Sets the mute state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"mute_state": Object {"description": String("The desired mute state (true for mute, false for unmute)."), "type": String("boolean")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("mute_state"), String("rid")], "title": String("SetTrackMuteArgs"), "type": String("object")} }, Tool { name: "rec_enable_toggle", description: "Toggles the master record enable or selected track record enable.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "prev_marker", description: "Moves the playhead to the previous location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "access_action", description: "Executes a specified Ardour menu action by its name.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"action_name": Object {"description": String("The name of the Ardour menu action to execute (e.g., 'Editor/zoom-to-session')."), "type": String("string")}}, "required": Array [String("action_name")], "title": String("AccessActionArgs"), "type": String("object")} }] }) +2026-06-30T17:56:39.985961Z DEBUG rmcp::service: new event evt=ToSink(Response(ListToolsResult(ListToolsResult { next_cursor: None, tools: [Tool { name: "redo", description: "Redoes the last undone action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "goto_end", description: "Moves the playhead to the session end.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_strip_plugin_active", description: "Activates or deactivates a plugin on a specific strip slot.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"active_state": Object {"description": String("The desired activation state (true for active, false for inactive)."), "type": String("boolean")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("active_state"), String("plugin_slot"), String("rid")], "title": String("SetStripPluginActiveArgs"), "type": String("object")} }, Tool { name: "transport_stop", description: "Stops Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_rec_enable", description: "Sets the record enable state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rec_st": Object {"description": String("The desired record enable state. 0 for off, 1 for on."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rec_st"), String("rid")], "title": String("SetTrackRecEnableArgs"), "type": String("object")} }, Tool { name: "set_strip_plugin_parameter", description: "Sets a specific parameter of a plugin on a strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"param_id": Object {"description": String("The 1-indexed ID of the parameter within the plugin."), "format": String("int32"), "type": String("integer")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "value": Object {"description": String("The desired parameter value, normalized (0.0 to 1.0)."), "format": String("float"), "type": String("number")}}, "required": Array [String("param_id"), String("plugin_slot"), String("rid"), String("value")], "title": String("SetStripPluginParameterArgs"), "type": String("object")} }, Tool { name: "toggle_punch_out", description: "Toggles the Punch Out state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "ffwd", description: "Fast forwards the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "toggle_all_rec_enables", description: "Toggles the record enable state for ALL tracks.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "undo", description: "Undoes the last action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_strip_pan_stereo_width", description: "Sets the stereo width for a panner on a stereo strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the stereo track/bus."), "format": String("int32"), "type": String("integer")}, "width": Object {"description": String("The desired stereo width. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("width")], "title": String("SetStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "goto_start", description: "Moves the playhead to the session start.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_transport_speed", description: "Sets Ardour's transport speed. Valid range: -8.0 to 8.0.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"speed": Object {"description": String("The desired transport speed. Valid range: -8.0 to 8.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("speed")], "title": String("SetTransportSpeedArgs"), "type": String("object")} }, Tool { name: "loop_toggle", description: "Toggles loop playback mode.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "next_marker", description: "Moves the playhead to the next location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_solo", description: "Sets the solo state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "solo_st": Object {"description": String("The desired solo state. 0 for solo off, 1 for solo on."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rid"), String("solo_st")], "title": String("SetTrackSoloArgs"), "type": String("object")} }, Tool { name: "set_track_gain_db", description: "Sets the gain of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_db": Object {"description": String("The desired gain in dB. Valid range: -400.0 to 6.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_db"), String("rid")], "title": String("SetTrackGainDBArgs"), "type": String("object")} }, Tool { name: "save_state", description: "Saves the current session state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "select_strip", description: "Selects a specific strip (track/bus) in Ardour.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus to select."), "format": String("int32"), "type": String("integer")}, "select_state": Object {"description": String("The desired select state (true to select). Currently, only true (1) is effective for selection."), "type": String("boolean")}}, "required": Array [String("rid"), String("select_state")], "title": String("SelectStripArgs"), "type": String("object")} }, Tool { name: "set_track_gain_abs", description: "Sets the absolute gain of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_abs": Object {"description": String("The desired absolute gain. Valid range: 0.0 to 2.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_abs"), String("rid")], "title": String("SetTrackGainAbsArgs"), "type": String("object")} }, Tool { name: "set_selected_strip_pan_stereo_width", description: "Sets the stereo width for the currently selected stereo strip's panner.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"width": Object {"description": String("The desired stereo width for the currently selected strip. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("width")], "title": String("SetSelectedStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "transport_play", description: "Starts Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "rewind", description: "Rewinds the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_trim_abs", description: "Sets the absolute trim of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_abs": Object {"description": String("The desired absolute trim. Valid range: 0.1 to 10.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_abs")], "title": String("SetTrackTrimAbsArgs"), "type": String("object")} }, Tool { name: "toggle_punch_in", description: "Toggles the Punch In state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "add_marker", description: "Adds a location marker at the current playhead position.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_trim_db", description: "Sets the trim of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_db": Object {"description": String("The desired trim in dB. Valid range: -20.0 to 20.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_db")], "title": String("SetTrackTrimDBArgs"), "type": String("object")} }, Tool { name: "locate", description: "Locates the playhead to a specific sample position and optionally starts playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"roll": Object {"description": String("Whether to start playing after locating. 0 for stop, 1 for play."), "format": String("int32"), "type": String("integer")}, "spos": Object {"description": String("The position in samples to locate to."), "format": String("int64"), "type": String("integer")}}, "required": Array [String("roll"), String("spos")], "title": String("LocateToolArgs"), "type": String("object")} }, Tool { name: "set_track_mute", description: "Sets the mute state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"mute_state": Object {"description": String("The desired mute state (true for mute, false for unmute)."), "type": String("boolean")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("mute_state"), String("rid")], "title": String("SetTrackMuteArgs"), "type": String("object")} }, Tool { name: "rec_enable_toggle", description: "Toggles the master record enable or selected track record enable.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "prev_marker", description: "Moves the playhead to the previous location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "access_action", description: "Executes a specified Ardour menu action by its name.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"action_name": Object {"description": String("The name of the Ardour menu action to execute (e.g., 'Editor/zoom-to-session')."), "type": String("string")}}, "required": Array [String("action_name")], "title": String("AccessActionArgs"), "type": String("object")} }] }), Number(1))) +2026-06-30T18:12:48.720223Z  INFO ardour_mcp_server: +====================================================================== +NEW SERVER RUN: Tue, 30 Jun 2026 20:12:48 +0200 +====================================================================== +2026-06-30T18:12:48.720287Z  INFO ardour_mcp_server: Attempting to create OSC sender for Ardour at 127.0.0.1:3819 +2026-06-30T18:12:48.720340Z  INFO ardour_mcp_server: OSC sender created and connected to Ardour at 127.0.0.1:3819 +2026-06-30T18:12:48.720346Z  INFO ardour_mcp_server: Sending /set_surface to Ardour to enable OSC feedback. +2026-06-30T18:12:48.720351Z  INFO ardour_mcp_server: Targeting feedback port: 9099 +2026-06-30T18:12:48.721790Z  INFO ardour_mcp_server: /set_surface command sent successfully to Ardour. +2026-06-30T18:12:48.762311Z  INFO serve_inner: rmcp::service: Service initialized as server peer_info=InitializeRequestParam { protocol_version: ProtocolVersion("2025-11-25"), capabilities: ClientCapabilities { experimental: None, roots: Some(RootsCapabilities { list_changed: None }), sampling: None }, client_info: Implementation { name: "claude-code", version: "2.1.197" } } +2026-06-30T18:12:48.762347Z  INFO ardour_mcp_server: Ardour MCP server started and waiting for connections... +2026-06-30T18:12:48.762370Z  INFO ardour_mcp_server: Starting OSC listener for Ardour events on 127.0.0.1:9099 +2026-06-30T18:12:48.762373Z DEBUG rmcp::service: new event evt=PeerMessage(Request(ListToolsRequest(Request { method: ListToolsRequestMethod, params: None }), Number(1))) +2026-06-30T18:12:48.762392Z  INFO rmcp::service: received request id=1 request=ListToolsRequest(Request { method: ListToolsRequestMethod, params: None }) +2026-06-30T18:12:48.762405Z DEBUG rmcp::service: new event evt=PeerMessage(Request(ListResourcesRequest(Request { method: ListResourcesRequestMethod, params: None }), Number(2))) +2026-06-30T18:12:48.762416Z  INFO rmcp::service: received request id=2 request=ListResourcesRequest(Request { method: ListResourcesRequestMethod, params: None }) +2026-06-30T18:12:48.762420Z ERROR ardour_mcp_server: OSC listener task failed: Failed to bind Tokio UDP socket for OSC on 127.0.0.1:9099: Address already in use (os error 98) +2026-06-30T18:12:48.762429Z DEBUG ardour_mcp_server: Listing resources. Count: 2, Content: [Annotated { raw: RawResource { uri: "ardour:/state/playback", name: "Ardour Playback State", description: Some("Current playback state of Ardour (e.g., Playing, Stopped, Unknown)."), mime_type: None, size: None }, annotations: None }, Annotated { raw: RawResource { uri: "ardour:/state/transport_frame", name: "Ardour Transport Frame Position", description: Some("Current playhead position in samples. Returns 'Unknown' if not yet reported by Ardour."), mime_type: None, size: None }, annotations: None }] +2026-06-30T18:12:48.762436Z  INFO rmcp::service: response message id=2 result=ListResourcesResult(ListResourcesResult { next_cursor: None, resources: [Annotated { raw: RawResource { uri: "ardour:/state/playback", name: "Ardour Playback State", description: Some("Current playback state of Ardour (e.g., Playing, Stopped, Unknown)."), mime_type: None, size: None }, annotations: None }, Annotated { raw: RawResource { uri: "ardour:/state/transport_frame", name: "Ardour Transport Frame Position", description: Some("Current playhead position in samples. Returns 'Unknown' if not yet reported by Ardour."), mime_type: None, size: None }, annotations: None }] }) +2026-06-30T18:12:48.762444Z DEBUG rmcp::service: new event evt=ToSink(Response(ListResourcesResult(ListResourcesResult { next_cursor: None, resources: [Annotated { raw: RawResource { uri: "ardour:/state/playback", name: "Ardour Playback State", description: Some("Current playback state of Ardour (e.g., Playing, Stopped, Unknown)."), mime_type: None, size: None }, annotations: None }, Annotated { raw: RawResource { uri: "ardour:/state/transport_frame", name: "Ardour Transport Frame Position", description: Some("Current playhead position in samples. Returns 'Unknown' if not yet reported by Ardour."), mime_type: None, size: None }, annotations: None }] }), Number(2))) +2026-06-30T18:12:48.762556Z  INFO rmcp::service: response message id=1 result=ListToolsResult(ListToolsResult { next_cursor: None, tools: [Tool { name: "transport_play", description: "Starts Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "save_state", description: "Saves the current session state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_gain_abs", description: "Sets the absolute gain of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_abs": Object {"description": String("The desired absolute gain. Valid range: 0.0 to 2.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_abs"), String("rid")], "title": String("SetTrackGainAbsArgs"), "type": String("object")} }, Tool { name: "add_marker", description: "Adds a location marker at the current playhead position.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "toggle_all_rec_enables", description: "Toggles the record enable state for ALL tracks.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_mute", description: "Sets the mute state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"mute_state": Object {"description": String("The desired mute state (true for mute, false for unmute)."), "type": String("boolean")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("mute_state"), String("rid")], "title": String("SetTrackMuteArgs"), "type": String("object")} }, Tool { name: "set_transport_speed", description: "Sets Ardour's transport speed. Valid range: -8.0 to 8.0.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"speed": Object {"description": String("The desired transport speed. Valid range: -8.0 to 8.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("speed")], "title": String("SetTransportSpeedArgs"), "type": String("object")} }, Tool { name: "set_strip_plugin_parameter", description: "Sets a specific parameter of a plugin on a strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"param_id": Object {"description": String("The 1-indexed ID of the parameter within the plugin."), "format": String("int32"), "type": String("integer")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "value": Object {"description": String("The desired parameter value, normalized (0.0 to 1.0)."), "format": String("float"), "type": String("number")}}, "required": Array [String("param_id"), String("plugin_slot"), String("rid"), String("value")], "title": String("SetStripPluginParameterArgs"), "type": String("object")} }, Tool { name: "access_action", description: "Executes a specified Ardour menu action by its name.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"action_name": Object {"description": String("The name of the Ardour menu action to execute (e.g., 'Editor/zoom-to-session')."), "type": String("string")}}, "required": Array [String("action_name")], "title": String("AccessActionArgs"), "type": String("object")} }, Tool { name: "rec_enable_toggle", description: "Toggles the master record enable or selected track record enable.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "locate", description: "Locates the playhead to a specific sample position and optionally starts playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"roll": Object {"description": String("Whether to start playing after locating. 0 for stop, 1 for play."), "format": String("int32"), "type": String("integer")}, "spos": Object {"description": String("The position in samples to locate to."), "format": String("int64"), "type": String("integer")}}, "required": Array [String("roll"), String("spos")], "title": String("LocateToolArgs"), "type": String("object")} }, Tool { name: "select_strip", description: "Selects a specific strip (track/bus) in Ardour.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus to select."), "format": String("int32"), "type": String("integer")}, "select_state": Object {"description": String("The desired select state (true to select). Currently, only true (1) is effective for selection."), "type": String("boolean")}}, "required": Array [String("rid"), String("select_state")], "title": String("SelectStripArgs"), "type": String("object")} }, Tool { name: "prev_marker", description: "Moves the playhead to the previous location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "rewind", description: "Rewinds the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "loop_toggle", description: "Toggles loop playback mode.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "toggle_punch_in", description: "Toggles the Punch In state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "goto_end", description: "Moves the playhead to the session end.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "toggle_punch_out", description: "Toggles the Punch Out state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "transport_stop", description: "Stops Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_rec_enable", description: "Sets the record enable state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rec_st": Object {"description": String("The desired record enable state. 0 for off, 1 for on."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rec_st"), String("rid")], "title": String("SetTrackRecEnableArgs"), "type": String("object")} }, Tool { name: "set_selected_strip_pan_stereo_width", description: "Sets the stereo width for the currently selected stereo strip's panner.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"width": Object {"description": String("The desired stereo width for the currently selected strip. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("width")], "title": String("SetSelectedStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "undo", description: "Undoes the last action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_trim_db", description: "Sets the trim of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_db": Object {"description": String("The desired trim in dB. Valid range: -20.0 to 20.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_db")], "title": String("SetTrackTrimDBArgs"), "type": String("object")} }, Tool { name: "set_strip_plugin_active", description: "Activates or deactivates a plugin on a specific strip slot.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"active_state": Object {"description": String("The desired activation state (true for active, false for inactive)."), "type": String("boolean")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("active_state"), String("plugin_slot"), String("rid")], "title": String("SetStripPluginActiveArgs"), "type": String("object")} }, Tool { name: "set_track_solo", description: "Sets the solo state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "solo_st": Object {"description": String("The desired solo state. 0 for solo off, 1 for solo on."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rid"), String("solo_st")], "title": String("SetTrackSoloArgs"), "type": String("object")} }, Tool { name: "set_strip_pan_stereo_width", description: "Sets the stereo width for a panner on a stereo strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the stereo track/bus."), "format": String("int32"), "type": String("integer")}, "width": Object {"description": String("The desired stereo width. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("width")], "title": String("SetStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "next_marker", description: "Moves the playhead to the next location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_gain_db", description: "Sets the gain of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_db": Object {"description": String("The desired gain in dB. Valid range: -400.0 to 6.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_db"), String("rid")], "title": String("SetTrackGainDBArgs"), "type": String("object")} }, Tool { name: "goto_start", description: "Moves the playhead to the session start.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_trim_abs", description: "Sets the absolute trim of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_abs": Object {"description": String("The desired absolute trim. Valid range: 0.1 to 10.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_abs")], "title": String("SetTrackTrimAbsArgs"), "type": String("object")} }, Tool { name: "ffwd", description: "Fast forwards the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "redo", description: "Redoes the last undone action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }] }) +2026-06-30T18:12:48.762603Z DEBUG rmcp::service: new event evt=ToSink(Response(ListToolsResult(ListToolsResult { next_cursor: None, tools: [Tool { name: "transport_play", description: "Starts Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "save_state", description: "Saves the current session state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_gain_abs", description: "Sets the absolute gain of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_abs": Object {"description": String("The desired absolute gain. Valid range: 0.0 to 2.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_abs"), String("rid")], "title": String("SetTrackGainAbsArgs"), "type": String("object")} }, Tool { name: "add_marker", description: "Adds a location marker at the current playhead position.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "toggle_all_rec_enables", description: "Toggles the record enable state for ALL tracks.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_mute", description: "Sets the mute state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"mute_state": Object {"description": String("The desired mute state (true for mute, false for unmute)."), "type": String("boolean")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("mute_state"), String("rid")], "title": String("SetTrackMuteArgs"), "type": String("object")} }, Tool { name: "set_transport_speed", description: "Sets Ardour's transport speed. Valid range: -8.0 to 8.0.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"speed": Object {"description": String("The desired transport speed. Valid range: -8.0 to 8.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("speed")], "title": String("SetTransportSpeedArgs"), "type": String("object")} }, Tool { name: "set_strip_plugin_parameter", description: "Sets a specific parameter of a plugin on a strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"param_id": Object {"description": String("The 1-indexed ID of the parameter within the plugin."), "format": String("int32"), "type": String("integer")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "value": Object {"description": String("The desired parameter value, normalized (0.0 to 1.0)."), "format": String("float"), "type": String("number")}}, "required": Array [String("param_id"), String("plugin_slot"), String("rid"), String("value")], "title": String("SetStripPluginParameterArgs"), "type": String("object")} }, Tool { name: "access_action", description: "Executes a specified Ardour menu action by its name.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"action_name": Object {"description": String("The name of the Ardour menu action to execute (e.g., 'Editor/zoom-to-session')."), "type": String("string")}}, "required": Array [String("action_name")], "title": String("AccessActionArgs"), "type": String("object")} }, Tool { name: "rec_enable_toggle", description: "Toggles the master record enable or selected track record enable.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "locate", description: "Locates the playhead to a specific sample position and optionally starts playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"roll": Object {"description": String("Whether to start playing after locating. 0 for stop, 1 for play."), "format": String("int32"), "type": String("integer")}, "spos": Object {"description": String("The position in samples to locate to."), "format": String("int64"), "type": String("integer")}}, "required": Array [String("roll"), String("spos")], "title": String("LocateToolArgs"), "type": String("object")} }, Tool { name: "select_strip", description: "Selects a specific strip (track/bus) in Ardour.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus to select."), "format": String("int32"), "type": String("integer")}, "select_state": Object {"description": String("The desired select state (true to select). Currently, only true (1) is effective for selection."), "type": String("boolean")}}, "required": Array [String("rid"), String("select_state")], "title": String("SelectStripArgs"), "type": String("object")} }, Tool { name: "prev_marker", description: "Moves the playhead to the previous location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "rewind", description: "Rewinds the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "loop_toggle", description: "Toggles loop playback mode.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "toggle_punch_in", description: "Toggles the Punch In state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "goto_end", description: "Moves the playhead to the session end.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "toggle_punch_out", description: "Toggles the Punch Out state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "transport_stop", description: "Stops Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_rec_enable", description: "Sets the record enable state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rec_st": Object {"description": String("The desired record enable state. 0 for off, 1 for on."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rec_st"), String("rid")], "title": String("SetTrackRecEnableArgs"), "type": String("object")} }, Tool { name: "set_selected_strip_pan_stereo_width", description: "Sets the stereo width for the currently selected stereo strip's panner.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"width": Object {"description": String("The desired stereo width for the currently selected strip. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("width")], "title": String("SetSelectedStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "undo", description: "Undoes the last action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_trim_db", description: "Sets the trim of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_db": Object {"description": String("The desired trim in dB. Valid range: -20.0 to 20.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_db")], "title": String("SetTrackTrimDBArgs"), "type": String("object")} }, Tool { name: "set_strip_plugin_active", description: "Activates or deactivates a plugin on a specific strip slot.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"active_state": Object {"description": String("The desired activation state (true for active, false for inactive)."), "type": String("boolean")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("active_state"), String("plugin_slot"), String("rid")], "title": String("SetStripPluginActiveArgs"), "type": String("object")} }, Tool { name: "set_track_solo", description: "Sets the solo state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "solo_st": Object {"description": String("The desired solo state. 0 for solo off, 1 for solo on."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rid"), String("solo_st")], "title": String("SetTrackSoloArgs"), "type": String("object")} }, Tool { name: "set_strip_pan_stereo_width", description: "Sets the stereo width for a panner on a stereo strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the stereo track/bus."), "format": String("int32"), "type": String("integer")}, "width": Object {"description": String("The desired stereo width. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("width")], "title": String("SetStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "next_marker", description: "Moves the playhead to the next location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_gain_db", description: "Sets the gain of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_db": Object {"description": String("The desired gain in dB. Valid range: -400.0 to 6.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_db"), String("rid")], "title": String("SetTrackGainDBArgs"), "type": String("object")} }, Tool { name: "goto_start", description: "Moves the playhead to the session start.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_trim_abs", description: "Sets the absolute trim of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_abs": Object {"description": String("The desired absolute trim. Valid range: 0.1 to 10.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_abs")], "title": String("SetTrackTrimAbsArgs"), "type": String("object")} }, Tool { name: "ffwd", description: "Fast forwards the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "redo", description: "Redoes the last undone action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }] }), Number(1))) +2026-06-30T20:13:06.691370Z  INFO ardour_mcp_server: +====================================================================== +NEW SERVER RUN: Tue, 30 Jun 2026 22:13:06 +0200 +====================================================================== +2026-06-30T20:13:06.691418Z  INFO ardour_mcp_server: Attempting to create OSC sender for Ardour at 127.0.0.1:3819 +2026-06-30T20:13:06.691451Z  INFO ardour_mcp_server: OSC sender created and connected to Ardour at 127.0.0.1:3819 +2026-06-30T20:13:06.691454Z  INFO ardour_mcp_server: Sending /set_surface to Ardour to enable OSC feedback. +2026-06-30T20:13:06.691458Z  INFO ardour_mcp_server: Targeting feedback port: 9099 +2026-06-30T20:13:06.691620Z  INFO ardour_mcp_server: /set_surface command sent successfully to Ardour. +2026-06-30T20:13:06.767092Z  INFO serve_inner: rmcp::service: Service initialized as server peer_info=InitializeRequestParam { protocol_version: ProtocolVersion("2025-11-25"), capabilities: ClientCapabilities { experimental: None, roots: Some(RootsCapabilities { list_changed: None }), sampling: None }, client_info: Implementation { name: "claude-code", version: "2.1.197" } } +2026-06-30T20:13:06.767160Z  INFO ardour_mcp_server: Ardour MCP server started and waiting for connections... +2026-06-30T20:13:06.767189Z  INFO ardour_mcp_server: Starting OSC listener for Ardour events on 127.0.0.1:9099 +2026-06-30T20:13:06.767212Z DEBUG rmcp::service: new event evt=PeerMessage(Request(ListToolsRequest(Request { method: ListToolsRequestMethod, params: None }), Number(1))) +2026-06-30T20:13:06.767238Z  INFO rmcp::service: received request id=1 request=ListToolsRequest(Request { method: ListToolsRequestMethod, params: None }) +2026-06-30T20:13:06.767248Z ERROR ardour_mcp_server: OSC listener task failed: Failed to bind Tokio UDP socket for OSC on 127.0.0.1:9099: Address already in use (os error 98) +2026-06-30T20:13:06.767263Z DEBUG rmcp::service: new event evt=PeerMessage(Request(ListResourcesRequest(Request { method: ListResourcesRequestMethod, params: None }), Number(2))) +2026-06-30T20:13:06.767269Z  INFO rmcp::service: received request id=2 request=ListResourcesRequest(Request { method: ListResourcesRequestMethod, params: None }) +2026-06-30T20:13:06.767289Z DEBUG ardour_mcp_server: Listing resources. Count: 2, Content: [Annotated { raw: RawResource { uri: "ardour:/state/playback", name: "Ardour Playback State", description: Some("Current playback state of Ardour (e.g., Playing, Stopped, Unknown)."), mime_type: None, size: None }, annotations: None }, Annotated { raw: RawResource { uri: "ardour:/state/transport_frame", name: "Ardour Transport Frame Position", description: Some("Current playhead position in samples. Returns 'Unknown' if not yet reported by Ardour."), mime_type: None, size: None }, annotations: None }] +2026-06-30T20:13:06.767302Z  INFO rmcp::service: response message id=2 result=ListResourcesResult(ListResourcesResult { next_cursor: None, resources: [Annotated { raw: RawResource { uri: "ardour:/state/playback", name: "Ardour Playback State", description: Some("Current playback state of Ardour (e.g., Playing, Stopped, Unknown)."), mime_type: None, size: None }, annotations: None }, Annotated { raw: RawResource { uri: "ardour:/state/transport_frame", name: "Ardour Transport Frame Position", description: Some("Current playhead position in samples. Returns 'Unknown' if not yet reported by Ardour."), mime_type: None, size: None }, annotations: None }] }) +2026-06-30T20:13:06.767324Z DEBUG rmcp::service: new event evt=ToSink(Response(ListResourcesResult(ListResourcesResult { next_cursor: None, resources: [Annotated { raw: RawResource { uri: "ardour:/state/playback", name: "Ardour Playback State", description: Some("Current playback state of Ardour (e.g., Playing, Stopped, Unknown)."), mime_type: None, size: None }, annotations: None }, Annotated { raw: RawResource { uri: "ardour:/state/transport_frame", name: "Ardour Transport Frame Position", description: Some("Current playhead position in samples. Returns 'Unknown' if not yet reported by Ardour."), mime_type: None, size: None }, annotations: None }] }), Number(2))) +2026-06-30T20:13:06.767384Z  INFO rmcp::service: response message id=1 result=ListToolsResult(ListToolsResult { next_cursor: None, tools: [Tool { name: "set_track_rec_enable", description: "Sets the record enable state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rec_st": Object {"description": String("The desired record enable state. 0 for off, 1 for on."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rec_st"), String("rid")], "title": String("SetTrackRecEnableArgs"), "type": String("object")} }, Tool { name: "save_state", description: "Saves the current session state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "add_marker", description: "Adds a location marker at the current playhead position.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_selected_strip_pan_stereo_width", description: "Sets the stereo width for the currently selected stereo strip's panner.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"width": Object {"description": String("The desired stereo width for the currently selected strip. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("width")], "title": String("SetSelectedStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "select_strip", description: "Selects a specific strip (track/bus) in Ardour.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus to select."), "format": String("int32"), "type": String("integer")}, "select_state": Object {"description": String("The desired select state (true to select). Currently, only true (1) is effective for selection."), "type": String("boolean")}}, "required": Array [String("rid"), String("select_state")], "title": String("SelectStripArgs"), "type": String("object")} }, Tool { name: "toggle_punch_in", description: "Toggles the Punch In state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "access_action", description: "Executes a specified Ardour menu action by its name.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"action_name": Object {"description": String("The name of the Ardour menu action to execute (e.g., 'Editor/zoom-to-session')."), "type": String("string")}}, "required": Array [String("action_name")], "title": String("AccessActionArgs"), "type": String("object")} }, Tool { name: "set_track_gain_db", description: "Sets the gain of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_db": Object {"description": String("The desired gain in dB. Valid range: -400.0 to 6.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_db"), String("rid")], "title": String("SetTrackGainDBArgs"), "type": String("object")} }, Tool { name: "rec_enable_toggle", description: "Toggles the master record enable or selected track record enable.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "redo", description: "Redoes the last undone action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "prev_marker", description: "Moves the playhead to the previous location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "undo", description: "Undoes the last action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_trim_abs", description: "Sets the absolute trim of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_abs": Object {"description": String("The desired absolute trim. Valid range: 0.1 to 10.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_abs")], "title": String("SetTrackTrimAbsArgs"), "type": String("object")} }, Tool { name: "loop_toggle", description: "Toggles loop playback mode.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_strip_plugin_active", description: "Activates or deactivates a plugin on a specific strip slot.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"active_state": Object {"description": String("The desired activation state (true for active, false for inactive)."), "type": String("boolean")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("active_state"), String("plugin_slot"), String("rid")], "title": String("SetStripPluginActiveArgs"), "type": String("object")} }, Tool { name: "rewind", description: "Rewinds the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_trim_db", description: "Sets the trim of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_db": Object {"description": String("The desired trim in dB. Valid range: -20.0 to 20.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_db")], "title": String("SetTrackTrimDBArgs"), "type": String("object")} }, Tool { name: "set_strip_plugin_parameter", description: "Sets a specific parameter of a plugin on a strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"param_id": Object {"description": String("The 1-indexed ID of the parameter within the plugin."), "format": String("int32"), "type": String("integer")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "value": Object {"description": String("The desired parameter value, normalized (0.0 to 1.0)."), "format": String("float"), "type": String("number")}}, "required": Array [String("param_id"), String("plugin_slot"), String("rid"), String("value")], "title": String("SetStripPluginParameterArgs"), "type": String("object")} }, Tool { name: "set_strip_pan_stereo_width", description: "Sets the stereo width for a panner on a stereo strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the stereo track/bus."), "format": String("int32"), "type": String("integer")}, "width": Object {"description": String("The desired stereo width. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("width")], "title": String("SetStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "set_track_mute", description: "Sets the mute state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"mute_state": Object {"description": String("The desired mute state (true for mute, false for unmute)."), "type": String("boolean")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("mute_state"), String("rid")], "title": String("SetTrackMuteArgs"), "type": String("object")} }, Tool { name: "set_track_gain_abs", description: "Sets the absolute gain of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_abs": Object {"description": String("The desired absolute gain. Valid range: 0.0 to 2.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_abs"), String("rid")], "title": String("SetTrackGainAbsArgs"), "type": String("object")} }, Tool { name: "toggle_punch_out", description: "Toggles the Punch Out state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_solo", description: "Sets the solo state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "solo_st": Object {"description": String("The desired solo state. 0 for solo off, 1 for solo on."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rid"), String("solo_st")], "title": String("SetTrackSoloArgs"), "type": String("object")} }, Tool { name: "transport_stop", description: "Stops Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_transport_speed", description: "Sets Ardour's transport speed. Valid range: -8.0 to 8.0.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"speed": Object {"description": String("The desired transport speed. Valid range: -8.0 to 8.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("speed")], "title": String("SetTransportSpeedArgs"), "type": String("object")} }, Tool { name: "goto_start", description: "Moves the playhead to the session start.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "goto_end", description: "Moves the playhead to the session end.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "transport_play", description: "Starts Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "next_marker", description: "Moves the playhead to the next location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "locate", description: "Locates the playhead to a specific sample position and optionally starts playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"roll": Object {"description": String("Whether to start playing after locating. 0 for stop, 1 for play."), "format": String("int32"), "type": String("integer")}, "spos": Object {"description": String("The position in samples to locate to."), "format": String("int64"), "type": String("integer")}}, "required": Array [String("roll"), String("spos")], "title": String("LocateToolArgs"), "type": String("object")} }, Tool { name: "ffwd", description: "Fast forwards the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "toggle_all_rec_enables", description: "Toggles the record enable state for ALL tracks.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }] }) +2026-06-30T20:13:06.767491Z DEBUG rmcp::service: new event evt=ToSink(Response(ListToolsResult(ListToolsResult { next_cursor: None, tools: [Tool { name: "set_track_rec_enable", description: "Sets the record enable state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rec_st": Object {"description": String("The desired record enable state. 0 for off, 1 for on."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rec_st"), String("rid")], "title": String("SetTrackRecEnableArgs"), "type": String("object")} }, Tool { name: "save_state", description: "Saves the current session state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "add_marker", description: "Adds a location marker at the current playhead position.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_selected_strip_pan_stereo_width", description: "Sets the stereo width for the currently selected stereo strip's panner.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"width": Object {"description": String("The desired stereo width for the currently selected strip. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("width")], "title": String("SetSelectedStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "select_strip", description: "Selects a specific strip (track/bus) in Ardour.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus to select."), "format": String("int32"), "type": String("integer")}, "select_state": Object {"description": String("The desired select state (true to select). Currently, only true (1) is effective for selection."), "type": String("boolean")}}, "required": Array [String("rid"), String("select_state")], "title": String("SelectStripArgs"), "type": String("object")} }, Tool { name: "toggle_punch_in", description: "Toggles the Punch In state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "access_action", description: "Executes a specified Ardour menu action by its name.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"action_name": Object {"description": String("The name of the Ardour menu action to execute (e.g., 'Editor/zoom-to-session')."), "type": String("string")}}, "required": Array [String("action_name")], "title": String("AccessActionArgs"), "type": String("object")} }, Tool { name: "set_track_gain_db", description: "Sets the gain of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_db": Object {"description": String("The desired gain in dB. Valid range: -400.0 to 6.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_db"), String("rid")], "title": String("SetTrackGainDBArgs"), "type": String("object")} }, Tool { name: "rec_enable_toggle", description: "Toggles the master record enable or selected track record enable.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "redo", description: "Redoes the last undone action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "prev_marker", description: "Moves the playhead to the previous location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "undo", description: "Undoes the last action.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_trim_abs", description: "Sets the absolute trim of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_abs": Object {"description": String("The desired absolute trim. Valid range: 0.1 to 10.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_abs")], "title": String("SetTrackTrimAbsArgs"), "type": String("object")} }, Tool { name: "loop_toggle", description: "Toggles loop playback mode.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_strip_plugin_active", description: "Activates or deactivates a plugin on a specific strip slot.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"active_state": Object {"description": String("The desired activation state (true for active, false for inactive)."), "type": String("boolean")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("active_state"), String("plugin_slot"), String("rid")], "title": String("SetStripPluginActiveArgs"), "type": String("object")} }, Tool { name: "rewind", description: "Rewinds the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_trim_db", description: "Sets the trim of a specific track in dB.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "trim_db": Object {"description": String("The desired trim in dB. Valid range: -20.0 to 20.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("trim_db")], "title": String("SetTrackTrimDBArgs"), "type": String("object")} }, Tool { name: "set_strip_plugin_parameter", description: "Sets a specific parameter of a plugin on a strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"param_id": Object {"description": String("The 1-indexed ID of the parameter within the plugin."), "format": String("int32"), "type": String("integer")}, "plugin_slot": Object {"description": String("The 1-indexed slot of the plugin on the strip."), "format": String("int32"), "type": String("integer")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "value": Object {"description": String("The desired parameter value, normalized (0.0 to 1.0)."), "format": String("float"), "type": String("number")}}, "required": Array [String("param_id"), String("plugin_slot"), String("rid"), String("value")], "title": String("SetStripPluginParameterArgs"), "type": String("object")} }, Tool { name: "set_strip_pan_stereo_width", description: "Sets the stereo width for a panner on a stereo strip.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the stereo track/bus."), "format": String("int32"), "type": String("integer")}, "width": Object {"description": String("The desired stereo width. Valid range: 0.0 to 1.0. Default is 1.0 (full width)."), "format": String("float"), "type": String("number")}}, "required": Array [String("rid"), String("width")], "title": String("SetStripPanStereoWidthArgs"), "type": String("object")} }, Tool { name: "set_track_mute", description: "Sets the mute state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"mute_state": Object {"description": String("The desired mute state (true for mute, false for unmute)."), "type": String("boolean")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("mute_state"), String("rid")], "title": String("SetTrackMuteArgs"), "type": String("object")} }, Tool { name: "set_track_gain_abs", description: "Sets the absolute gain of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"gain_abs": Object {"description": String("The desired absolute gain. Valid range: 0.0 to 2.0."), "format": String("float"), "type": String("number")}, "rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("gain_abs"), String("rid")], "title": String("SetTrackGainAbsArgs"), "type": String("object")} }, Tool { name: "toggle_punch_out", description: "Toggles the Punch Out state.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_track_solo", description: "Sets the solo state of a specific track.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"rid": Object {"description": String("The Router ID (rid) of the track/bus."), "format": String("int32"), "type": String("integer")}, "solo_st": Object {"description": String("The desired solo state. 0 for solo off, 1 for solo on."), "format": String("int32"), "type": String("integer")}}, "required": Array [String("rid"), String("solo_st")], "title": String("SetTrackSoloArgs"), "type": String("object")} }, Tool { name: "transport_stop", description: "Stops Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "set_transport_speed", description: "Sets Ardour's transport speed. Valid range: -8.0 to 8.0.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"speed": Object {"description": String("The desired transport speed. Valid range: -8.0 to 8.0."), "format": String("float"), "type": String("number")}}, "required": Array [String("speed")], "title": String("SetTransportSpeedArgs"), "type": String("object")} }, Tool { name: "goto_start", description: "Moves the playhead to the session start.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "goto_end", description: "Moves the playhead to the session end.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "transport_play", description: "Starts Ardour playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "next_marker", description: "Moves the playhead to the next location marker.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "locate", description: "Locates the playhead to a specific sample position and optionally starts playback.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "properties": Object {"roll": Object {"description": String("Whether to start playing after locating. 0 for stop, 1 for play."), "format": String("int32"), "type": String("integer")}, "spos": Object {"description": String("The position in samples to locate to."), "format": String("int64"), "type": String("integer")}}, "required": Array [String("roll"), String("spos")], "title": String("LocateToolArgs"), "type": String("object")} }, Tool { name: "ffwd", description: "Fast forwards the transport.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }, Tool { name: "toggle_all_rec_enables", description: "Toggles the record enable state for ALL tracks.", input_schema: {"$schema": String("http://json-schema.org/draft-07/schema#"), "title": String("EmptyObject"), "type": String("object")} }] }), Number(1)))