Compare commits
7 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 6e3c3cf2a2 | |||
| a1e2e3567c | |||
| 833597c831 | |||
| 7158be8142 | |||
| 9be84a48ea | |||
| fd63261444 | |||
| 4099d88eaf |
@@ -54,6 +54,13 @@ reports/*
|
||||
!reports/.gitkeep
|
||||
projects/*/reports/
|
||||
|
||||
# Papers — artefacto local: papers académicos reproducibles. En fase interna viven
|
||||
# local y gitignored (como los reports); al promocionar a fase publishable se
|
||||
# vuelven sub-repo Gitea propio (como apps/analyses). Solo el marcador .gitkeep se
|
||||
# versiona. Convención: docs/capabilities/papers.md
|
||||
papers/*
|
||||
!papers/.gitkeep
|
||||
|
||||
# Node / pnpm
|
||||
**/node_modules/
|
||||
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
---
|
||||
name: next_numbered_dir
|
||||
kind: function
|
||||
lang: bash
|
||||
domain: io
|
||||
version: "1.0.0"
|
||||
purity: impure
|
||||
signature: "next_numbered_dir(parent_dir: string, [width: int]) -> string"
|
||||
description: "Calcula el siguiente prefijo numerico NNNN- para un directorio numerado incremental. Escanea los subdirectorios directos de parent_dir cuyo nombre empiece por NNNN- (4+ digitos seguidos de guion), toma el maximo, le suma 1 y lo imprime con zero-padding al ancho width (default 4). Si parent_dir no existe o no tiene subdirs que matcheen, imprime 0001."
|
||||
tags: [papers, io, scaffold]
|
||||
uses_functions: []
|
||||
uses_types: []
|
||||
returns: []
|
||||
returns_optional: false
|
||||
error_type: "error_go_core"
|
||||
imports: []
|
||||
params:
|
||||
- name: parent_dir
|
||||
desc: "directorio padre cuyos subdirectorios numerados (NNNN-...) se escanean; obligatorio"
|
||||
- name: width
|
||||
desc: "ancho del zero-padding del numero impreso (default 4); opcional"
|
||||
output: "el siguiente numero como string con zero-padding a width digitos a stdout (ej. 0003); usage a stderr y exit 1 si falta parent_dir"
|
||||
tested: false
|
||||
tests: []
|
||||
test_file_path: ""
|
||||
file_path: "bash/functions/io/next_numbered_dir.sh"
|
||||
---
|
||||
|
||||
## Ejemplo
|
||||
|
||||
```bash
|
||||
source bash/functions/io/next_numbered_dir.sh
|
||||
|
||||
# Sobre un papers/ que ya contiene 0001-foo y 0002-bar
|
||||
mkdir -p /tmp/papers/{0001-foo,0002-bar}
|
||||
next_numbered_dir /tmp/papers
|
||||
# -> 0003
|
||||
|
||||
# Directorio vacio o inexistente -> primer numero
|
||||
next_numbered_dir /tmp/papers_nuevo
|
||||
# -> 0001
|
||||
|
||||
# Ancho de padding distinto
|
||||
next_numbered_dir /tmp/papers 6
|
||||
# -> 000003
|
||||
```
|
||||
|
||||
## Cuando usarla
|
||||
|
||||
Cuando scaffoldees un artefacto numerado incremental (papers/, reports/, issues/) y necesites el siguiente NNNN sin colision: escanea lo que ya existe en disco y te da el numero libre listo para crear `<NNNN>-<slug>`.
|
||||
|
||||
## Gotchas
|
||||
|
||||
- **Impura**: lee el filesystem (estado del directorio en el momento de la llamada). No crea nada — solo calcula e imprime el numero.
|
||||
- **Octal**: los numeros con cero a la izquierda (`08`, `09`) se interpretan como octal en aritmetica bash y romperian el calculo. La funcion fuerza base 10 con `10#$num` para evitarlo.
|
||||
- **Solo subdirectorios**: cuenta unicamente subdirs directos. Archivos sueltos (`.gitkeep`, `notas.md`) y subdirs que no matcheen el patron se ignoran. No es recursivo.
|
||||
- **Patron estricto**: el prefijo debe ser `NNNN-` (minimo 4 digitos seguidos de guion). Un subdir `12-foo` o `0001foo` (sin guion) NO se cuenta.
|
||||
- No hay deteccion de huecos: devuelve `max+1`, no el primer numero libre intermedio. Si tienes `0001` y `0003`, devuelve `0004`, no `0002`.
|
||||
@@ -0,0 +1,46 @@
|
||||
#!/usr/bin/env bash
|
||||
# next_numbered_dir — Compute the next NNNN- prefix for a numbered directory.
|
||||
#
|
||||
# Scans the DIRECT subdirectories of <parent_dir> whose names start with a
|
||||
# numeric prefix of the form `NNNN-` (4+ digits followed by a hyphen), takes
|
||||
# the maximum number, adds 1, and prints it zero-padded to <width> (default 4).
|
||||
# If <parent_dir> does not exist or contains no matching subdir, prints the
|
||||
# first number (0001 at default width).
|
||||
|
||||
next_numbered_dir() {
|
||||
local parent_dir="${1:-}"
|
||||
local width="${2:-4}"
|
||||
|
||||
if [[ -z "$parent_dir" ]]; then
|
||||
echo "usage: next_numbered_dir <parent_dir> [width]" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
local max=0
|
||||
local entry base num
|
||||
|
||||
if [[ -d "$parent_dir" ]]; then
|
||||
# Iterate only over direct subdirectories. The trailing slash in the
|
||||
# glob ensures files (e.g. .gitkeep) are skipped — only dirs match.
|
||||
for entry in "$parent_dir"/*/; do
|
||||
# If the glob matched nothing it stays literal; guard with -d.
|
||||
[[ -d "$entry" ]] || continue
|
||||
base="$(basename "$entry")"
|
||||
# Require a prefix of 4+ digits followed by a hyphen.
|
||||
if [[ "$base" =~ ^([0-9]{4,})- ]]; then
|
||||
num="${BASH_REMATCH[1]}"
|
||||
# Force base 10 so leading zeros (08, 09) are not read as octal.
|
||||
num=$((10#$num))
|
||||
if (( num > max )); then
|
||||
max=$num
|
||||
fi
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
printf "%0*d\n" "$width" $(( max + 1 ))
|
||||
}
|
||||
|
||||
if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
|
||||
next_numbered_dir "$@"
|
||||
fi
|
||||
@@ -0,0 +1,69 @@
|
||||
---
|
||||
name: init_paper
|
||||
kind: pipeline
|
||||
lang: bash
|
||||
domain: pipelines
|
||||
version: "1.0.0"
|
||||
purity: impure
|
||||
signature: "init_paper(slug: string, [--title <t>] [--domain <d>] [--tags <csv>]) -> void"
|
||||
description: "Scaffold de un paper académico reproducible en papers/<NNNN-slug>/. Calcula el siguiente número incremental escaneando papers/, crea las subcarpetas (experiments data figures reviews out), copia las plantillas paper.md (IMRaD) + preregistration.md (anti-HARKing) rellenando el frontmatter (title, slug, date de hoy, phase=question, status=draft) y crea references.md. NO hace git init: el paper arranca en fase interna local (papers/ gitignored). Grupo de capacidad papers."
|
||||
tags: [papers, scaffold, paper, pipeline, bash, launcher]
|
||||
uses_functions:
|
||||
- next_numbered_dir_bash_io
|
||||
- slugify_ascii_py_core
|
||||
uses_types: []
|
||||
returns: []
|
||||
returns_optional: false
|
||||
error_type: "error_go_core"
|
||||
imports: []
|
||||
params:
|
||||
- name: slug
|
||||
desc: "identificador legible del paper; se slugifica a ASCII (espacios/acentos se normalizan) y se prefija con el siguiente NNNN incremental"
|
||||
- name: "--title"
|
||||
desc: "título del paper (string); si se omite, usa el slug limpio. No debe contener el carácter '|'"
|
||||
- name: "--domain"
|
||||
desc: "dominio del paper escrito en el frontmatter (default datascience)"
|
||||
- name: "--tags"
|
||||
desc: "tags CSV que se escriben en el frontmatter de paper.md (opcional)"
|
||||
output: "sin salida directa; crea papers/<NNNN-slug>/ con paper.md, preregistration.md, references.md y las subcarpetas experiments/ data/ figures/ reviews/ out/. Imprime el resumen y los pasos siguientes a stdout."
|
||||
tested: false
|
||||
tests: []
|
||||
test_file_path: ""
|
||||
file_path: "bash/functions/pipelines/init_paper.sh"
|
||||
---
|
||||
|
||||
## Ejemplo
|
||||
|
||||
```bash
|
||||
# Scaffold de un paper nuevo (numera 0001, 0002, ... automáticamente)
|
||||
fn run init_paper mi-primer-paper --title "Mi primer paper"
|
||||
fn run init_paper reactive-loop-calls --domain datascience --tags registry,telemetria
|
||||
|
||||
# El slug se slugifica: "Áreas de Mejora" -> papers/0003-areas-de-mejora/
|
||||
fn run init_paper "Áreas de Mejora"
|
||||
```
|
||||
|
||||
## Cuando usarla
|
||||
|
||||
Cuando empiezas un paper académico nuevo dentro de `fn_registry` y necesitas el esqueleto del artefacto (`papers/<NNNN-slug>/`) con las plantillas IMRaD y de pre-registro listas para rellenar. Es el paso 1 del grupo de capacidad `papers` (ver `docs/capabilities/papers.md`), antes de la revisión de literatura y del pre-registro de la hipótesis.
|
||||
|
||||
## Flujo
|
||||
|
||||
1. Parsea `<slug>` (posicional) + flags `--title` / `--domain` / `--tags`. Falla con exit ≠ 0 si falta el slug.
|
||||
2. `slugify_ascii` — normaliza el slug a ASCII lowercase sin diacríticos (reutiliza la función del registry, solo stdlib).
|
||||
3. `next_numbered_dir papers/` — calcula el siguiente NNNN de 4 dígitos sin colisión.
|
||||
4. Crea `papers/<NNNN-slug>/` con las subcarpetas `experiments/ data/ figures/ reviews/ out/`.
|
||||
5. Copia `docs/templates/paper.md` + `docs/templates/preregistration.md` y rellena el frontmatter por clave de línea (title, slug, date de hoy, domain, tags; phase=question y status=draft vienen de la plantilla).
|
||||
6. Crea `references.md` vacío.
|
||||
|
||||
## Gotchas
|
||||
|
||||
- **NO hace `git init`.** El paper arranca en fase interna local; `papers/` está gitignored en el repo padre (solo `papers/.gitkeep` se versiona). Promocionar a sub-repo Gitea (fase publishable) es manual.
|
||||
- **El `--title` no debe contener el carácter `|`** (se usa como delimitador de sed al rellenar el frontmatter; los `&` y `\` sí se escapan).
|
||||
- **No indexa el paper en `registry.db`** — los artefactos `papers/<slug>/` no se indexan en esta fase (KISS); sí se indexa este pipeline.
|
||||
- Requiere `python3` (del venv del registry o del sistema) para slugificar; `slugify_ascii` solo usa stdlib, así que el venv no es obligatorio.
|
||||
- Idempotencia: si el directorio destino ya existiera, aborta con exit ≠ 0 en vez de sobrescribir.
|
||||
|
||||
## Notas
|
||||
|
||||
Cada paper es un artefacto independiente (mismo patrón que `apps/` y `analysis/`, pero para investigación). El pipeline usa `set -euo pipefail`: cualquier fallo detiene la ejecución. Parte del grupo de capacidad `papers` — diseño completo en `reports/0001-2026-06-30-papers-system-design.md`.
|
||||
@@ -0,0 +1,177 @@
|
||||
#!/usr/bin/env bash
|
||||
# init_paper
|
||||
# ----------
|
||||
# Scaffold de un paper académico reproducible en papers/<NNNN-slug>/.
|
||||
#
|
||||
# Calcula el siguiente número incremental escaneando papers/, crea el
|
||||
# directorio con todas las subcarpetas (experiments data figures reviews out),
|
||||
# copia las plantillas paper.md + preregistration.md rellenando el frontmatter
|
||||
# (title, slug, date de hoy, phase=question, status=draft) y crea references.md.
|
||||
#
|
||||
# NO hace `git init`: el paper arranca en fase interna local (papers/ está
|
||||
# gitignored en el repo padre, solo .gitkeep se versiona). La promoción a
|
||||
# sub-repo Gitea (fase publishable) es un paso posterior MANUAL.
|
||||
#
|
||||
# Compone: next_numbered_dir (helper de numeración del registry) +
|
||||
# slugify_ascii (slug ASCII del registry).
|
||||
#
|
||||
# USO:
|
||||
# ./init_paper.sh <slug> [--title "..."] [--domain <d>] [--tags a,b,c]
|
||||
#
|
||||
# EJEMPLOS:
|
||||
# ./init_paper.sh mi-primer-paper --title "Mi primer paper"
|
||||
# ./init_paper.sh reactive-loop-calls --domain datascience --tags registry,telemetria
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REGISTRY_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)"
|
||||
|
||||
# Funciones atómicas del registry
|
||||
source "$REGISTRY_ROOT/bash/functions/io/next_numbered_dir.sh"
|
||||
|
||||
# ── Parsing de argumentos ────────────────────────────────────
|
||||
|
||||
SLUG_RAW=""
|
||||
TITLE=""
|
||||
DOMAIN="datascience"
|
||||
TAGS=""
|
||||
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--title)
|
||||
TITLE="$2"; shift 2 ;;
|
||||
--domain)
|
||||
DOMAIN="$2"; shift 2 ;;
|
||||
--tags)
|
||||
TAGS="$2"; shift 2 ;;
|
||||
-h|--help)
|
||||
grep "^#" "$0" | sed 's/^# \?//' ; exit 0 ;;
|
||||
-*)
|
||||
echo "Flag desconocido: $1" >&2 ; exit 1 ;;
|
||||
*)
|
||||
if [ -z "$SLUG_RAW" ]; then
|
||||
SLUG_RAW="$1"
|
||||
else
|
||||
echo "ERROR: argumento posicional inesperado: '$1' (solo se admite un <slug>)." >&2
|
||||
exit 1
|
||||
fi
|
||||
shift ;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [ -z "$SLUG_RAW" ]; then
|
||||
echo "ERROR: falta el argumento <slug>." >&2
|
||||
echo "Uso: $0 <slug> [--title \"...\"] [--domain <d>] [--tags a,b,c]" >&2
|
||||
echo " Ejemplo: $0 mi-primer-paper --title \"Mi primer paper\"" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# ── Slugificar (reutiliza slugify_ascii del registry; solo stdlib) ──
|
||||
|
||||
PYBIN="$REGISTRY_ROOT/python/.venv/bin/python3"
|
||||
[ -x "$PYBIN" ] || PYBIN="$(command -v python3 || true)"
|
||||
if [ -z "$PYBIN" ]; then
|
||||
echo "ERROR: no se encontró python3 para slugificar el slug." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
SLUG_CLEAN=$("$PYBIN" -c '
|
||||
import sys, os
|
||||
sys.path.insert(0, os.path.join(sys.argv[2], "python", "functions"))
|
||||
from core.slugify_ascii import slugify_ascii
|
||||
print(slugify_ascii(sys.argv[1], default="paper"))
|
||||
' "$SLUG_RAW" "$REGISTRY_ROOT")
|
||||
|
||||
# ── Resolver número incremental y directorio destino ─────────
|
||||
|
||||
PAPERS_DIR="$REGISTRY_ROOT/papers"
|
||||
mkdir -p "$PAPERS_DIR"
|
||||
|
||||
NUM=$(next_numbered_dir "$PAPERS_DIR")
|
||||
SLUG_FULL="${NUM}-${SLUG_CLEAN}"
|
||||
PAPER_DIR="$PAPERS_DIR/$SLUG_FULL"
|
||||
|
||||
if [ -d "$PAPER_DIR" ]; then
|
||||
echo "ERROR: el directorio del paper ya existe: $PAPER_DIR" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
TODAY=$(date +%Y-%m-%d)
|
||||
[ -n "$TITLE" ] || TITLE="$SLUG_CLEAN"
|
||||
|
||||
TAGS_YAML="[]"
|
||||
if [ -n "$TAGS" ]; then
|
||||
TAGS_YAML="[$(echo "$TAGS" | sed 's/,/, /g')]"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "════════════════════════════════════════════════════════════"
|
||||
echo " INIT PAPER: ${SLUG_FULL}"
|
||||
echo " Título: ${TITLE}"
|
||||
echo " Directorio: ${PAPER_DIR}"
|
||||
echo "════════════════════════════════════════════════════════════"
|
||||
echo ""
|
||||
|
||||
# ── Crear estructura ─────────────────────────────────────────
|
||||
|
||||
echo "[1/3] Creando estructura..."
|
||||
mkdir -p "$PAPER_DIR"/experiments "$PAPER_DIR"/data "$PAPER_DIR"/figures \
|
||||
"$PAPER_DIR"/reviews "$PAPER_DIR"/out
|
||||
echo " experiments/ data/ figures/ reviews/ out/"
|
||||
|
||||
# ── Copiar plantillas + rellenar frontmatter ─────────────────
|
||||
|
||||
echo "[2/3] Escribiendo paper.md + preregistration.md..."
|
||||
|
||||
# Escapa caracteres especiales del RHS de sed (delimitador |)
|
||||
sed_escape() { printf '%s' "$1" | sed -e 's/[\\&|]/\\&/g'; }
|
||||
TITLE_ESC="$(sed_escape "$TITLE")"
|
||||
DOMAIN_ESC="$(sed_escape "$DOMAIN")"
|
||||
|
||||
PAPER_MD="$PAPER_DIR/paper.md"
|
||||
PREREG_MD="$PAPER_DIR/preregistration.md"
|
||||
|
||||
cp "$REGISTRY_ROOT/docs/templates/paper.md" "$PAPER_MD"
|
||||
cp "$REGISTRY_ROOT/docs/templates/preregistration.md" "$PREREG_MD"
|
||||
|
||||
sed -i \
|
||||
-e "s|^title:.*|title: \"${TITLE_ESC}\"|" \
|
||||
-e "s|^slug:.*|slug: ${SLUG_FULL}|" \
|
||||
-e "s|^date:.*|date: ${TODAY}|" \
|
||||
-e "s|^domain:.*|domain: ${DOMAIN_ESC}|" \
|
||||
-e "s|^tags:.*|tags: ${TAGS_YAML}|" \
|
||||
"$PAPER_MD"
|
||||
|
||||
sed -i \
|
||||
-e "s|^paper_slug:.*|paper_slug: ${SLUG_FULL}|" \
|
||||
"$PREREG_MD"
|
||||
|
||||
echo " $PAPER_MD"
|
||||
echo " $PREREG_MD"
|
||||
|
||||
# ── references.md ────────────────────────────────────────────
|
||||
|
||||
echo "[3/3] Escribiendo references.md..."
|
||||
cat > "$PAPER_DIR/references.md" << EOF
|
||||
# References — ${TITLE}
|
||||
|
||||
<!-- Una entrada por referencia. Formato libre (o BibTeX) hasta promocionar a publishable. -->
|
||||
EOF
|
||||
echo " $PAPER_DIR/references.md"
|
||||
|
||||
# ── Resumen ──────────────────────────────────────────────────
|
||||
|
||||
echo ""
|
||||
echo "════════════════════════════════════════════════════════════"
|
||||
echo " PAPER '${SLUG_FULL}' LISTO (fase: question, status: draft)"
|
||||
echo "════════════════════════════════════════════════════════════"
|
||||
echo ""
|
||||
echo " Pasos siguientes:"
|
||||
echo " 1. Revisión de literatura (skill /deep-research) → Related work."
|
||||
echo " 2. Pre-registro: congela H0/H1 + plan en preregistration.md (preregister_hypothesis)."
|
||||
echo " 3. Experimentos en experiments/ → análisis (grupo eda) → escritura IMRaD en paper.md."
|
||||
echo " 4. render_paper_pdf → out/paper.pdf. Peer review adversarial → reviews/."
|
||||
echo ""
|
||||
echo " papers/ está gitignored: este paper vive local hasta promocionar a publishable."
|
||||
echo ""
|
||||
@@ -39,6 +39,7 @@ Indice de grupos de capacidades del registry. Cada grupo agrupa >=3 funciones qu
|
||||
| [cpp-tables](tql.md) | 9 | Table Query Language C++ puro: filter, group, agg, sort, join, stats, formulas Lua, round-trip emit/apply |
|
||||
| [data-table-renderers](data_table_renderers.md) | 1 | API declarativa de cell renderers para data_table: Badge, Progress, Duration, Icon via TableInput.column_specs |
|
||||
| [scheduler](scheduler.md) | 4 | Cron expression parsing, matching, next-run y traduccion humana (consume `apps/dag_engine`) |
|
||||
| [papers](papers.md) | — | Papers académicos reproducibles en `papers/<NNNN-slug>/`: scaffold del artefacto (`init_paper` + helper `next_numbered_dir`), plantillas IMRaD + pre-registro anti-HARKing, y (en construcción por la flota) congelar hipótesis, funciones estadísticas (effect size/CI/corrección múltiple), render md→PDF y peer-review adversarial. Reutiliza `deep-research`, grupo `eda` y el motor PDF de `datascience`. Diseño: `reports/0001-2026-06-30-papers-system-design.md` |
|
||||
| [extractor](extractor.md) | 15 | Funciones que leen datos de fuentes externas (BD, API, archivos, web). Nodos input de `data_factory` |
|
||||
| [transformer](transformer.md) | 15 | Funciones que clean/dedup/aggregate/feature-engineer datos. Nodos intermedios de `data_factory` |
|
||||
| [sink](sink.md) | 11 | Funciones que escriben datos a destino externo (BD, dashboard, alerta, email). Nodos output |
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
# papers — papers académicos reproducibles
|
||||
|
||||
Grupo de capacidad para producir **papers académicos** dentro de `fn_registry`: investigación con hipótesis falsables, experimentos reproducibles, análisis estadístico honesto y escritura en formato IMRaD. Cada paper es un artefacto nuevo en `papers/<NNNN-slug>/` que reutiliza infraestructura existente (skill `deep-research` para la revisión de literatura, grupo `eda` para el análisis, motor md→PDF de `datascience`, patrón de verificación adversarial del orquestador) y añade lo que falta como funciones del registry.
|
||||
|
||||
Diseño completo y decisiones: `reports/0001-2026-06-30-papers-system-design.md`.
|
||||
|
||||
> **Regla de oro anti paper-mill:** una hipótesis que **podía** fallar + un experimento con riesgo real de refutación + estadística que no es teatro. Si no hay riesgo de refutación, no es un paper. Los claims nunca superan a la evidencia. El antídoto al HARKing es el **pre-registro**: el plan de análisis se congela *antes* de mirar los datos.
|
||||
|
||||
## Estructura del artefacto
|
||||
|
||||
```
|
||||
papers/0001-mi-paper/
|
||||
paper.md # frontmatter (title, slug, authors, date, status, phase, tags, domain, hypothesis_id) + cuerpo IMRaD
|
||||
preregistration.md # H0/H1 + plan de análisis CONGELADO (frozen_at + content_hash) antes de correr
|
||||
references.md # bibliografía
|
||||
experiments/ # código / notebooks por experimento (exp01_*, exp02_*)
|
||||
data/ # crudos + procesados (gitignored si pesa)
|
||||
figures/ # gráficos generados
|
||||
reviews/ # outputs del peer-review adversarial
|
||||
out/ # paper.pdf — entregable final
|
||||
.git/ # SOLO cuando promociona a fase publishable (sub-repo Gitea)
|
||||
```
|
||||
|
||||
`papers/` está gitignored en el repo padre (solo `papers/.gitkeep` se versiona): un paper en fase interna no contamina el repo. Al promocionar a `status: publishable` se vuelve sub-repo Gitea `dataforge/<slug>` (como apps y analyses).
|
||||
|
||||
### Fases (campo `phase` de `paper.md`)
|
||||
|
||||
```
|
||||
question → review → hypothesis → design → running → analysis → writing → internal-review
|
||||
→ [DONE interno] → polish → submitted [solo en fase publishable]
|
||||
```
|
||||
|
||||
## Funciones
|
||||
|
||||
| ID | Pureza | Estado | Qué hace |
|
||||
|---|---|---|---|
|
||||
| `init_paper_bash_pipelines` | impure | ✅ disponible | Scaffold de `papers/<NNNN-slug>/`: calcula el siguiente NNNN, crea las subcarpetas, copia `paper.md` + `preregistration.md` con el frontmatter relleno (slug, title, date de hoy, `phase: question`, `status: draft`) y `references.md` vacío. NO hace `git init` (el paper arranca en fase interna local). |
|
||||
| `next_numbered_dir_bash_io` | impure | ✅ disponible | Dado un directorio, devuelve el siguiente número incremental de 4 dígitos (`0001`, `0002`, …) escaneando los subdirs con prefijo `NNNN-`. Helper de numeración de `init_paper` (reutilizable por reports/issues). |
|
||||
| `preregister_hypothesis` | impure | 🚧 en construcción (flota) | Congela el `preregistration.md` (H0/H1 + plan de análisis) con `frozen_at` + `content_hash`, pasa `status` a `frozen` y escribe `hypothesis_id` en `paper.md`. Mata el HARKing: tras congelar, el plan no se edita. |
|
||||
| `cohens_d` (effect size) | pure | 🚧 en construcción (flota) | Tamaño del efecto (Cohen's d) entre dos grupos. Reporta magnitud, no solo significancia. |
|
||||
| `confidence_interval` | pure | 🚧 en construcción (flota) | Intervalo de confianza de una métrica (media/diferencia). |
|
||||
| `holm_bonferroni` | pure | 🚧 en construcción (flota) | Corrección de comparaciones múltiples (Holm-Bonferroni / FWER) para el plan de análisis. |
|
||||
| `render_paper_pdf` | impure | 🚧 en construcción (flota) | Markdown IMRaD (`paper.md` + figuras) → `out/paper.pdf`, reutilizando el motor md→PDF del grupo `eda`/`datascience`. |
|
||||
|
||||
> Las funciones estadísticas reutilizan lo que ya exista en `datascience` (p.ej. `fdr_correction_py_datascience` cubre la corrección de comparaciones múltiples por FDR; el agente del rigor experimental decide si añade Holm-Bonferroni o reusa lo existente). Buscar antes de duplicar: `mcp__registry__fn_search query="effect size" domain="datascience"`.
|
||||
|
||||
### Peer review (no es función del registry)
|
||||
|
||||
El agente adversarial `.claude/agents/paper-reviewer.md` (🚧 en construcción por la flota) puntúa novedad, rigor, reproducibilidad y validez, e intenta **refutar** cada claim. Default a "failed" si la evidencia no soporta. Escribe su veredicto en `reviews/`. Es el equivalente al verificador adversarial del orquestador aplicado al paper.
|
||||
|
||||
## Ejemplo canónico (end-to-end)
|
||||
|
||||
```bash
|
||||
# 1. Scaffold del paper (fase question, local). Crea papers/0001-mi-paper/.
|
||||
./fn run init_paper mi-paper --title "¿El bucle reactivo reduce las calls inline?" --domain datascience --tags registry,telemetria
|
||||
|
||||
# 2. Revisión de literatura → llena Related work (skill deep-research, fase review).
|
||||
# /deep-research "..."
|
||||
|
||||
# 3. Pre-registro: congela H0/H1 + plan de análisis ANTES de mirar datos (fase hypothesis).
|
||||
./fn run preregister_hypothesis papers/0001-mi-paper # 🚧 en construcción
|
||||
|
||||
# 4. Experimentos en papers/0001-mi-paper/experiments/ (fase running) →
|
||||
# análisis con el grupo `eda` + funciones de effect size / CI / corrección múltiple (fase analysis).
|
||||
|
||||
# 5. Escritura IMRaD en paper.md (fase writing) → render del entregable PDF.
|
||||
./fn run render_paper_pdf papers/0001-mi-paper # 🚧 en construcción → out/paper.pdf
|
||||
|
||||
# 6. Peer review adversarial (fase internal-review).
|
||||
# Agent(subagent_type="paper-reviewer", prompt="Revisa papers/0001-mi-paper ...") # 🚧 en construcción
|
||||
```
|
||||
|
||||
## Fronteras
|
||||
|
||||
- **NO es para reports de trabajo.** Un report (`reports/`) es el entregable escrito de una tarea (resumen + evidencia + gaps); un paper es investigación con hipótesis falsable y experimento. Ver `.claude/rules/reports.md`.
|
||||
- **NO se indexa en `registry.db` en esta fase.** No hay tabla `papers` ni `entity_type` `paper` (KISS); se añadiría con migración propia si se decide. Las *funciones* del grupo sí se indexan (viven en `bash/functions/`, `python/functions/`), pero los artefactos `papers/<slug>/` no.
|
||||
- **NO hace `git init` en el scaffold.** El paper arranca en fase interna local y gitignored. La promoción a sub-repo Gitea (fase publishable) es un paso manual posterior.
|
||||
- **NO soporta LaTeX/arXiv todavía.** Formato elegido: Markdown como fuente + PDF como entregable. El soporte LaTeX se añadiría al promocionar un paper a fase publishable.
|
||||
|
||||
## Estado
|
||||
|
||||
Fase de scaffolding. Disponible: estructura del artefacto, plantillas (`docs/templates/paper.md`, `docs/templates/preregistration.md`), pipeline `init_paper` + helper `next_numbered_dir`, esta página y el bloque gitignore de `papers/`. En construcción por la flota: `preregister_hypothesis`, funciones estadísticas (effect size / CI / corrección múltiple), `render_paper_pdf` y el agente `paper-reviewer`. Validación end-to-end con un paper piloto real: pendiente.
|
||||
Vendored
+94
@@ -0,0 +1,94 @@
|
||||
---
|
||||
title: "TITULO DEL PAPER"
|
||||
slug: NNNN-slug
|
||||
authors: [Enmanuel]
|
||||
date: 2026-01-01
|
||||
status: draft # draft | internal | publishable
|
||||
phase: question # question -> review -> hypothesis -> design -> running -> analysis -> writing -> internal-review -> polish -> submitted
|
||||
tags: []
|
||||
domain: datascience
|
||||
hypothesis_id: "" # lo rellena preregister_hypothesis al congelar el preregistro
|
||||
---
|
||||
|
||||
<!--
|
||||
Paper académico reproducible (formato IMRaD). Esta es la FUENTE editable en Markdown;
|
||||
el entregable PDF se genera con render_paper_pdf (grupo `papers`).
|
||||
|
||||
Regla de oro anti paper-mill: una hipótesis que PODÍA fallar + un experimento con
|
||||
riesgo real de refutación + estadística que no es teatro. Si no hay riesgo de
|
||||
refutación, no es un paper. Los claims nunca superan a la evidencia.
|
||||
-->
|
||||
|
||||
# {{título del paper}}
|
||||
|
||||
## Abstract
|
||||
|
||||
<!--
|
||||
Resumen estructurado en 4-6 frases: contexto -> gap -> método -> resultados -> conclusión.
|
||||
Sin citas, sin abreviaturas sin definir. Es lo único que mucha gente leerá: que se sostenga solo.
|
||||
-->
|
||||
|
||||
## 1. Introduction
|
||||
|
||||
<!--
|
||||
Embudo en cuatro movimientos:
|
||||
1. Contexto — el área y por qué importa.
|
||||
2. Gap — qué NO se sabe todavía (el hueco que este paper llena).
|
||||
3. Pregunta / hipótesis — formulada de forma falsable (ver preregistration.md).
|
||||
4. Contribución — lista explícita de lo que aporta este trabajo ("Contributions:").
|
||||
-->
|
||||
|
||||
## 2. Related work
|
||||
|
||||
<!--
|
||||
Qué existe ya y por qué no basta. Agrupa por enfoque, no por autor. Cada cita debe
|
||||
justificar por qué el gap sigue abierto. Output de la fase de revisión (skill deep-research).
|
||||
-->
|
||||
|
||||
## 3. Methods
|
||||
|
||||
<!--
|
||||
Diseño REPRODUCIBLE: otra persona lo corre y obtiene lo mismo.
|
||||
- Variables: independiente(s), dependiente(s), control.
|
||||
- Diseño: N, condiciones, muestreo, aleatorización.
|
||||
- Métricas y cómo se miden.
|
||||
- Protocolo paso a paso + dónde vive el código (experiments/) y los datos (data/).
|
||||
Debe ser coherente con el preregistration.md congelado (no se cambia el plan tras ver datos).
|
||||
-->
|
||||
|
||||
## 4. Results
|
||||
|
||||
<!--
|
||||
Datos SIN interpretar. Tablas y figuras (figures/) con su lectura literal.
|
||||
Reporta effect size + intervalos de confianza, no solo p-valores.
|
||||
Incluye también los resultados negativos / no significativos (anti cherry-picking).
|
||||
-->
|
||||
|
||||
## 5. Discussion
|
||||
|
||||
<!--
|
||||
Interpretación de los resultados a la luz de la pregunta. Claims <= evidencia.
|
||||
-->
|
||||
|
||||
### 5.1 Limitaciones
|
||||
|
||||
<!-- Qué no cubre el estudio, supuestos, datos faltantes. Honestidad explícita. -->
|
||||
|
||||
### 5.2 Amenazas a la validez
|
||||
|
||||
<!--
|
||||
- Validez interna — ¿la causa es lo que decimos o hay confusores?
|
||||
- Validez externa — ¿generaliza fuera de esta muestra/condiciones?
|
||||
- Validez de constructo — ¿la métrica mide lo que dice medir?
|
||||
- Validez estadística — ¿N suficiente, supuestos del test cumplidos, comparaciones múltiples corregidas?
|
||||
-->
|
||||
|
||||
## 6. Conclusion + Future work
|
||||
|
||||
<!--
|
||||
Cierre en 2-4 frases: qué se aprendió (sin overclaiming) + las siguientes preguntas que abre.
|
||||
-->
|
||||
|
||||
## References
|
||||
|
||||
<!-- Ver references.md. -->
|
||||
Vendored
+59
@@ -0,0 +1,59 @@
|
||||
---
|
||||
paper_slug: NNNN-slug
|
||||
frozen_at: "" # timestamp ISO — lo rellena preregister_hypothesis al congelar
|
||||
content_hash: "" # hash del contenido congelado — lo rellena preregister_hypothesis
|
||||
status: draft # draft -> frozen (preregister_hypothesis lo pasa a frozen; tras congelar NO se edita)
|
||||
---
|
||||
|
||||
> **⚠️ ESTE DOCUMENTO SE CONGELA ANTES DE MIRAR LOS DATOS (anti-HARKing).**
|
||||
> El plan de análisis se fija aquí *antes* de ejecutar el experimento. Una vez congelado
|
||||
> (`status: frozen`, con `frozen_at` + `content_hash`), **no se edita**. Inventar o ajustar
|
||||
> la hipótesis después de ver los resultados (HARKing) invalida el paper. Si el plan cambia
|
||||
> tras ver datos, eso es análisis exploratorio y se reporta como tal, no como confirmatorio.
|
||||
|
||||
# Pre-registro — {{título del paper}}
|
||||
|
||||
## 1. Pregunta de investigación
|
||||
|
||||
<!-- La pregunta concreta, en una frase. Debe poder responderse con un experimento. -->
|
||||
|
||||
## 2. Hipótesis
|
||||
|
||||
<!-- Falsable (Popper): una predicción que PODRÍA fallar. -->
|
||||
|
||||
- **H0 (nula):** <!-- no hay efecto / no hay diferencia. Es lo que el test intenta rechazar. -->
|
||||
- **H1 (alternativa):** <!-- el efecto esperado, con dirección si la hay. -->
|
||||
|
||||
## 3. Variables
|
||||
|
||||
- **Independiente(s):** <!-- lo que se manipula. -->
|
||||
- **Dependiente(s):** <!-- lo que se mide (la métrica de resultado). -->
|
||||
- **Control:** <!-- lo que se mantiene fijo / se cubre estadísticamente. -->
|
||||
|
||||
## 4. Diseño
|
||||
|
||||
<!--
|
||||
- N: tamaño de muestra (y justificación / power analysis si aplica).
|
||||
- Condiciones / grupos.
|
||||
- Muestreo y aleatorización.
|
||||
- Criterios de inclusión / exclusión de datos (definidos AHORA, no después).
|
||||
-->
|
||||
|
||||
## 5. Plan de análisis
|
||||
|
||||
<!--
|
||||
El plan estadístico EXACTO, decidido antes de ver los datos:
|
||||
- Test estadístico concreto (p.ej. t-test de Welch, Mann-Whitney U, regresión...).
|
||||
- Métrica de effect size (p.ej. Cohen's d, diferencia de medias, odds ratio).
|
||||
- Criterio de decisión (umbral alpha, qué resultado confirma/refuta H1).
|
||||
- Corrección por comparaciones múltiples (p.ej. Holm-Bonferroni) si hay >1 contraste.
|
||||
- Manejo de supuestos (normalidad, varianzas) y qué se hace si no se cumplen.
|
||||
-->
|
||||
|
||||
## 6. Predicción cuantitativa
|
||||
|
||||
<!--
|
||||
La predicción numérica concreta que el experimento pondrá a prueba.
|
||||
P.ej. "esperamos d >= 0.5 con IC95% que no cruza 0" o "una reducción >= 15% en la métrica X".
|
||||
Cuanto más específica, más falsable.
|
||||
-->
|
||||
@@ -561,13 +561,11 @@ def _intro_blocks(gloss=None, mark_term: bool = False) -> list:
|
||||
t_groupby = _term(mark_term, "groupby", "**por grupos** (split-apply-combine)")
|
||||
t_pivot = _term(mark_term, "pivot_table", "**tablas dinámicas** (pivot)")
|
||||
text = (
|
||||
f"Este capítulo analiza la tabla {t_groupby}: "
|
||||
"elige las columnas categóricas más informativas — por su cardinalidad "
|
||||
"y relevancia, no todas contra todas, para no inflar comparaciones "
|
||||
"espurias — y resume las variables numéricas dentro de cada grupo "
|
||||
f"(conteo, media, mediana, desviación). Las {t_pivot} "
|
||||
"cruzan dos categóricas sobre una medida, y los **gráficos de barras** "
|
||||
"(siempre desde cero) comparan los grupos de un vistazo."
|
||||
f"Este capítulo analiza la tabla {t_groupby}: elige las columnas "
|
||||
"categóricas más informativas (por cardinalidad y relevancia, no todas "
|
||||
"contra todas) y resume las variables numéricas dentro de cada grupo "
|
||||
f"(conteo, media, mediana, desviación). Se añaden {t_pivot} y "
|
||||
"**gráficos de barras** (siempre desde cero) para comparar los grupos."
|
||||
)
|
||||
return [model.Heading(text=CHAPTER_TITLE, level=1),
|
||||
model.Markdown(text=text)]
|
||||
|
||||
@@ -3,12 +3,13 @@
|
||||
Builds the quality chapter from a ``TableProfile`` of the ``eda`` group. The
|
||||
chapter implements the quality model of report 2046:
|
||||
|
||||
1. **En qué se basa la calidad** — an intro paragraph explaining the two scored
|
||||
1. **En qué se basa la calidad** — a concise intro naming the two scored
|
||||
dimensions and their weights (completitud 60%, validez 40%) plus the
|
||||
table-level row uniqueness, BEFORE any number, and stating explicitly that
|
||||
outliers are reported as observations and do **not** lower the score. The
|
||||
criteria terms (calidad de datos, completitud, validez, unicidad de registro)
|
||||
are hooked into the shared glossary as clickable jumps.
|
||||
table-level row uniqueness, BEFORE any number, and stating that outliers are
|
||||
reported as observations and do **not** lower the score. The criteria terms
|
||||
(calidad de datos, completitud, validez, unicidad de registro) are hooked
|
||||
into the shared glossary as clickable jumps; their full definitions live in
|
||||
the GLOSARIO chapter, not inline here.
|
||||
2. **Scores por columna** — a table with, per column, the total quality score and
|
||||
its breakdown into completeness / validity (no consistency dimension).
|
||||
3. **Problemas de calidad** — a table listing ONLY real quality defects
|
||||
@@ -309,30 +310,22 @@ def _term(key: str, label: str, mark: bool) -> str:
|
||||
|
||||
|
||||
def _criteria_intro(mark: bool) -> str:
|
||||
"""Intro paragraph explaining the two scored dimensions and the principle."""
|
||||
"""Intro: how the score is composed, with every term marked clickable.
|
||||
|
||||
Concise on purpose: the definitions of each term (calidad de datos,
|
||||
completitud, validez, unicidad de registro) now live in the GLOSARIO
|
||||
chapter, so the body no longer repeats them — it only states how the score
|
||||
is composed and keeps each term marked so it stays a clickable jump.
|
||||
"""
|
||||
calidad = _term("calidad_datos", "calidad de datos", mark)
|
||||
completitud = _term("completitud", "Completitud (peso 60%)", mark)
|
||||
validez = _term("validez", "Validez (peso 40%, cuando es medible)", mark)
|
||||
completitud = _term("completitud", "completitud", mark)
|
||||
validez = _term("validez", "validez", mark)
|
||||
unicidad = _term("unicidad_registro", "unicidad de registro", mark)
|
||||
return (
|
||||
f"La {calidad} de cada columna es un score de 0 a 100 que combina solo "
|
||||
"dimensiones medibles desde el perfil de la tabla, sin fuente externa "
|
||||
"de verdad:\n\n"
|
||||
f"- {completitud}: proporción de valores presentes (1 − % de nulos; en "
|
||||
"texto, las celdas vacías cuentan como faltantes). Los nulos y vacíos "
|
||||
"bajan el score.\n"
|
||||
f"- {validez}: proporción de valores que encajan con su tipo o formato "
|
||||
"(un número que parsea, una fecha legible, un email con forma de email). "
|
||||
"Si una columna es texto libre sin formato esperado, la validez no se "
|
||||
"mide y el score se basa solo en la completitud.\n\n"
|
||||
f"Score de columna = 100 × (0,6·completitud + 0,4·validez), "
|
||||
"renormalizado cuando la validez no aplica. A nivel de tabla se añade "
|
||||
f"la {unicidad} (1 − % de filas duplicadas).\n\n"
|
||||
"**Los valores atípicos (outliers) NO bajan la calidad.** Un valor "
|
||||
"extremo puede ser real y correcto; detectar atípicos es parte del "
|
||||
"análisis de la distribución, no un juicio de corrección. Por eso, junto "
|
||||
"con las columnas constantes y los identificadores, se listan aparte "
|
||||
"como **observaciones analíticas** que no afectan al score."
|
||||
f"La {calidad} de cada columna es un score de 0 a 100 que combina "
|
||||
f"{completitud} (peso 60%) y {validez} (peso 40%, cuando es medible); "
|
||||
f"a nivel de tabla se añade la {unicidad}. Los valores atípicos no "
|
||||
"bajan el score: se listan aparte como **observaciones analíticas**."
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -72,14 +72,16 @@ def test_golden_chapter_estructura_y_version():
|
||||
assert "markdown" in kinds and "kv_table" in kinds and "data_table" in kinds
|
||||
|
||||
|
||||
def test_golden_intro_explica_dos_dimensiones_y_pesos():
|
||||
def test_golden_intro_nombra_dos_dimensiones_y_pesos():
|
||||
# La intro nombra las dos dimensiones, sus pesos y la unicidad, pero ya NO
|
||||
# repite sus definiciones largas: estas viven ahora en el capítulo GLOSARIO.
|
||||
ch = build_calidad(_profile(), {})
|
||||
intro = [b for b in ch.blocks if b.kind == "markdown"][0].text
|
||||
for needle in ("Completitud", "Validez", "60%", "40%",
|
||||
for needle in ("completitud", "validez", "60%", "40%",
|
||||
"unicidad de registro"):
|
||||
assert needle in intro, f"falta {needle!r} en la intro de criterios"
|
||||
# El principio: los outliers NO bajan la calidad.
|
||||
assert "atípicos" in intro and "NO bajan" in intro
|
||||
assert "atípicos" in intro and "no bajan" in intro
|
||||
# Ya no se menciona la dimensión consistencia eliminada.
|
||||
assert "20%" not in intro
|
||||
|
||||
|
||||
@@ -1,19 +1,25 @@
|
||||
"""Categorical distributions chapter (CAT DISTR).
|
||||
|
||||
Third reference chapter for AutomaticEDA. For every categorical column it shows,
|
||||
fulfilling the user's request:
|
||||
Third reference chapter for AutomaticEDA. Each categorical column gets **its own
|
||||
page (PDF) / slide (PPTX)**: every column is wrapped in a keep-together
|
||||
``model.Group`` with ``page_break_before=True`` (except the first, which may share
|
||||
the intro's page), so its chart sits next to its tables and no column is split.
|
||||
|
||||
1. A short opening explanation of **Shannon entropy** (what it measures, its 0
|
||||
and log2(k) bounds, the normalized 0–1 version) and the dataset row total used
|
||||
as a comparison baseline.
|
||||
2. Per column, a cardinality key/value table: distinct values, ``% distinct``
|
||||
(distinct / total rows), total dataset rows, singleton values (frequency 1),
|
||||
entropy with its theoretical maximum and the normalized ratio, mode, imbalance
|
||||
and string-length stats.
|
||||
3. A short note flagging problematic cardinality (id-like ≈100% distinct, or a
|
||||
A short intro names the clickable **[[term:entropia]]entropía[[/term]]** term —
|
||||
the full definition lives in the GLOSARIO chapter, so it is NOT repeated inline
|
||||
here (one click jumps to the glossary entry). The intro also carries the dataset
|
||||
row total used as a comparison baseline.
|
||||
|
||||
Per column the Group contains, in order:
|
||||
|
||||
1. A cardinality key/value table: distinct values, ``% distinct`` (distinct /
|
||||
total rows), total dataset rows, singleton values (frequency 1), entropy with
|
||||
its theoretical maximum and the normalized ratio, mode, imbalance and
|
||||
string-length stats.
|
||||
2. A short note flagging problematic cardinality (id-like ≈100% distinct, or a
|
||||
single dominating category).
|
||||
4. A ``top-k`` table (value / count / %).
|
||||
5. A **donut pie chart** of the most common categories (top-k + an "Otros"
|
||||
3. A ``top-k`` table (value / count / %).
|
||||
4. A **donut pie chart** of the most common categories (top-k + an "Otros"
|
||||
bucket), drawn lazily so the renderers scale it to fit entirely.
|
||||
|
||||
Data comes from the ``eda`` group: each ``columns[i]['categorical']`` is the
|
||||
@@ -33,7 +39,7 @@ import math
|
||||
|
||||
from .. import model
|
||||
|
||||
CHAPTER_VERSION = "1.1.0"
|
||||
CHAPTER_VERSION = "1.2.0"
|
||||
CHAPTER_ID = "cat_distr"
|
||||
CHAPTER_TITLE = "Distribuciones categóricas"
|
||||
|
||||
@@ -53,11 +59,17 @@ _TERM_ENTROPIA_DEF = (
|
||||
# Cap the number of categorical columns rendered to keep the document bounded;
|
||||
# the rest are summarized in a closing note (no silent truncation).
|
||||
MAX_COLS = 40
|
||||
# Rows shown in each top-k table and explicit slices in the pie.
|
||||
TOP_TABLE_ROWS = 15
|
||||
# Rows shown in each top-k table and explicit slices in the pie. Kept moderate so
|
||||
# the whole column — cardinality table + top-k table + donut — fits on ONE
|
||||
# page/slide with the chart next to its tables; the table note still reports
|
||||
# "top N of M" so nothing is silently hidden. For id-like columns (≈100%
|
||||
# distinct) the top-k table is dropped entirely (it would be a list of unique
|
||||
# values — pure noise), which also frees the room the donut needs (see build).
|
||||
TOP_TABLE_ROWS = 8
|
||||
PIE_TOP_K = 6
|
||||
# Truncate very long category labels in tables (the renderer also wraps).
|
||||
LABEL_MAX = 48
|
||||
# Truncate very long category labels in tables (the renderer also wraps). Kept
|
||||
# tight so a column with long id-like values (names, tickets) still fits its page.
|
||||
LABEL_MAX = 28
|
||||
|
||||
|
||||
def _fmt_int(value) -> str:
|
||||
@@ -267,45 +279,55 @@ def _normalize_card(card: dict) -> dict:
|
||||
|
||||
|
||||
def _cardinality_block(card: dict):
|
||||
"""KVTable with the cardinality / entropy metrics for one column."""
|
||||
"""KVTable with the cardinality / entropy metrics for one column.
|
||||
|
||||
Related metrics are grouped onto a single row each (distinct/%/unique;
|
||||
entropy bits/max/normalized; length min/mean/max) so the whole column —
|
||||
table + chart — fits one page/slide without dropping any datum; the short
|
||||
16:9 PPTX slide does not fit one metric per row plus a chart otherwise."""
|
||||
n_singletons = card.get("n_singletons")
|
||||
if n_singletons is not None and card.get("n_singletons_partial"):
|
||||
singletons = f"≥{_fmt_int(n_singletons)} (en top mostrado)"
|
||||
singletons = f"≥{_fmt_int(n_singletons)}"
|
||||
elif n_singletons is not None:
|
||||
singletons = _fmt_int(n_singletons)
|
||||
else:
|
||||
singletons = "—"
|
||||
|
||||
entropy_ref = _fmt_num(card.get("entropy"))
|
||||
emax = card.get("entropy_max")
|
||||
if emax is not None:
|
||||
entropy_ref = f"{entropy_ref} (máx {_fmt_num(emax)})"
|
||||
# Distinct count · % distinct · unique (frequency 1) on one row.
|
||||
distinct_combo = (f"{_fmt_int(card.get('n_distinct'))} · "
|
||||
f"{_fmt_pct_value(card.get('pct_distinct'))} · "
|
||||
f"{singletons} únicos")
|
||||
|
||||
# Entropy bits · theoretical max · normalized 0–1 on one row.
|
||||
entropy_combo = (f"{_fmt_num(card.get('entropy'))} bits · "
|
||||
f"máx {_fmt_num(card.get('entropy_max'))} · "
|
||||
f"norm {_fmt_num(card.get('entropy_norm'))}")
|
||||
|
||||
mode = card.get("mode")
|
||||
mode_pct = card.get("mode_pct")
|
||||
mode_str = "—" if mode is None else model._safe_str(mode)
|
||||
mode_str = "—" if mode is None else _truncate(mode, 32)
|
||||
if mode is not None and mode_pct is not None:
|
||||
mode_str = f"{mode_str} ({_fmt_pct_value(mode_pct)})"
|
||||
|
||||
rows = [
|
||||
("Valores distintos", _fmt_int(card.get("n_distinct"))),
|
||||
("% distintos", _fmt_pct_value(card.get("pct_distinct"))),
|
||||
("Distintos · % · únicos", distinct_combo),
|
||||
("Total filas (dataset)", _fmt_int(card.get("n_rows"))),
|
||||
("Valores únicos (frecuencia 1)", singletons),
|
||||
("Entropía (bits)", entropy_ref),
|
||||
("Entropía normalizada (0–1)", _fmt_num(card.get("entropy_norm"))),
|
||||
("Entropía (bits · máx · norm)", entropy_combo),
|
||||
("Moda", mode_str),
|
||||
]
|
||||
imbalance = card.get("imbalance")
|
||||
if imbalance is not None:
|
||||
rows.append(("Desbalance", _fmt_num(imbalance)))
|
||||
lm = card.get("len_min")
|
||||
lmean = card.get("len_mean")
|
||||
lmax = card.get("len_max")
|
||||
# Imbalance and string length (both secondary) share one closing row.
|
||||
extras = []
|
||||
if imbalance is not None:
|
||||
extras.append(f"desbalance {_fmt_num(imbalance)}")
|
||||
if any(v is not None for v in (lm, lmean, lmax)):
|
||||
rows.append((
|
||||
"Longitud (mín/media/máx)",
|
||||
f"{_fmt_num(lm)} / {_fmt_num(lmean)} / {_fmt_num(lmax)}"))
|
||||
extras.append(
|
||||
f"long. {_fmt_num(lm)}/{_fmt_num(lmean)}/{_fmt_num(lmax)}")
|
||||
if extras:
|
||||
rows.append(("Desbalance · longitud", " · ".join(extras)))
|
||||
return model.KVTable(rows=rows, title="Cardinalidad")
|
||||
|
||||
|
||||
@@ -315,7 +337,8 @@ def _flag_note(card: dict):
|
||||
return model.Note(
|
||||
"Casi todos los valores son distintos (≈100% distintos): la columna "
|
||||
"se comporta como un identificador y aporta poco para agrupar o "
|
||||
"comparar categorías.")
|
||||
"comparar categorías. No se lista el top de categorías (serían "
|
||||
"valores casi todos únicos).")
|
||||
if card.get("dominated"):
|
||||
mp = card.get("mode_pct")
|
||||
mp_str = _fmt_pct_value(mp) if mp is not None else "muy alta"
|
||||
@@ -335,7 +358,7 @@ def _topk_table(cat: dict):
|
||||
if not isinstance(t, dict):
|
||||
continue
|
||||
rows.append([
|
||||
model._safe_str(t.get("value")),
|
||||
_truncate(t.get("value")),
|
||||
_fmt_int(t.get("count")),
|
||||
_pct_from_maybe_fraction(t.get("pct")),
|
||||
])
|
||||
@@ -353,20 +376,16 @@ def _topk_table(cat: dict):
|
||||
def _intro_blocks(n_rows, mark_term: bool = False):
|
||||
total = _fmt_int(n_rows)
|
||||
# Mark the first appearance of the term as a clickable glossary jump when the
|
||||
# term was registered (mark_term). The visible text is identical either way.
|
||||
entropia = ("[[term:entropia]]**entropía de Shannon**[[/term]]" if mark_term
|
||||
else "**entropía de Shannon**")
|
||||
# term was registered (mark_term). The full definition of entropy lives in the
|
||||
# GLOSARIO chapter, so the intro only names the clickable term here instead of
|
||||
# repeating the long explanation (avoids the redundancy with the glossary).
|
||||
entropia = ("[[term:entropia]]entropía[[/term]]" if mark_term
|
||||
else "entropía")
|
||||
text = (
|
||||
f"La {entropia} mide cómo de repartidos están los valores de "
|
||||
"una columna categórica, en bits. Vale 0 cuando una sola categoría "
|
||||
"concentra todas las filas (máxima previsibilidad) y alcanza su máximo, "
|
||||
"log2(k) para k categorías distintas, cuando todas aparecen por igual "
|
||||
"(máxima diversidad). La **entropía normalizada** (entropía dividida por "
|
||||
"su máximo) la lleva al rango 0–1 para comparar columnas con distinto "
|
||||
"número de categorías. Para cada columna se muestran los valores "
|
||||
"distintos, el porcentaje que representan sobre el total de filas, los "
|
||||
"valores únicos (que aparecen una sola vez), la tabla de las categorías "
|
||||
"más frecuentes y un gráfico de tarta (donut) de las más comunes."
|
||||
f"Cada columna categórica ocupa su propia página: sus métricas de "
|
||||
f"cardinalidad —incluida la {entropia}—, una nota que señala cardinalidad "
|
||||
"problemática, la tabla de las categorías más frecuentes y un gráfico de "
|
||||
"tarta (donut) de las más comunes, todo junto."
|
||||
)
|
||||
if n_rows is not None:
|
||||
text += f" El dataset tiene {total} filas en total como referencia."
|
||||
@@ -398,24 +417,37 @@ def build_cat_distr(profile: dict, ctx: dict):
|
||||
blocks = list(_intro_blocks(n_rows, mark_term=mark_term))
|
||||
|
||||
rendered = cat_cols[:MAX_COLS]
|
||||
for col in rendered:
|
||||
for idx, col in enumerate(rendered):
|
||||
name = col.get("name") or "(columna)"
|
||||
cat = col.get("categorical") or {}
|
||||
card = _normalize_card(_cardinality(cat, n_rows))
|
||||
|
||||
blocks.append(model.Heading(text=str(name), level=2))
|
||||
blocks.append(_cardinality_block(card))
|
||||
# One Group per categorical column: heading + cardinality table + flag
|
||||
# note + top-k table + donut figure are kept together and the renderer
|
||||
# starts each on a fresh page/slide (page_break_before) so every column
|
||||
# gets its own page with its chart next to its tables. The first column
|
||||
# may share the intro's page (no forced break) to avoid a near-empty page.
|
||||
col_blocks = [
|
||||
model.Heading(text=str(name), level=2),
|
||||
_cardinality_block(card),
|
||||
]
|
||||
note = _flag_note(card)
|
||||
if note is not None:
|
||||
blocks.append(note)
|
||||
topk = _topk_table(cat)
|
||||
if topk is not None:
|
||||
blocks.append(topk)
|
||||
blocks.append(model.Figure(
|
||||
col_blocks.append(note)
|
||||
# For id-like columns (≈100% distinct) the top-k is a list of unique
|
||||
# values — pure noise; skip it (the flag note already explains why) and
|
||||
# let the donut take that room so the whole column fits one page/slide.
|
||||
if not card.get("id_like"):
|
||||
topk = _topk_table(cat)
|
||||
if topk is not None:
|
||||
col_blocks.append(topk)
|
||||
col_blocks.append(model.Figure(
|
||||
make=_pie_make(cat.get("top") or [], card.get("n_distinct"),
|
||||
str(name), n_rows),
|
||||
caption=(f"Categorías más comunes de «{_truncate(name, 32)}» "
|
||||
"(donut: top-k + «Otros»)")))
|
||||
blocks.append(model.Group(blocks=col_blocks,
|
||||
page_break_before=(idx > 0)))
|
||||
|
||||
if len(cat_cols) > len(rendered):
|
||||
omitted = len(cat_cols) - len(rendered)
|
||||
|
||||
@@ -2,11 +2,14 @@
|
||||
|
||||
Self-contained: builds synthetic TableProfiles (no DuckDB) so the suite is fast
|
||||
and deterministic. Verifies that ``build_cat_distr`` emits the blocks the user
|
||||
asked for (entropy intro, distinct/total/%-distinct/unique metrics, top-k table
|
||||
and a donut figure), that the chapter renders inside the full document to both
|
||||
PDF and PPTX showing that content, that a profile with no categorical columns
|
||||
yields ``None`` without raising, and that long labels / many columns are never
|
||||
cut in either output.
|
||||
asked for (distinct/total/%-distinct/unique metrics, top-k table and a donut
|
||||
figure), that EACH categorical column is wrapped in its own keep-together
|
||||
``Group`` that starts on a fresh page/slide (one column per page, chart next to
|
||||
its tables), that the long entropy explanation is NOT repeated inline (it lives
|
||||
in the glossary — only the clickable term is kept), that the chapter renders
|
||||
inside the full document to both PDF and PPTX showing that content, that a
|
||||
profile with no categorical columns yields ``None`` without raising, and that
|
||||
long labels / many columns are never cut in either output.
|
||||
"""
|
||||
|
||||
import os
|
||||
@@ -17,7 +20,8 @@ from pypdf import PdfReader
|
||||
from pptx import Presentation
|
||||
|
||||
from datascience.automatic_eda.model import (
|
||||
DataTable, Figure, Heading, KVTable, Note,
|
||||
DataTable, Figure, GlossaryCollector, Group, Heading, KVTable, Markdown,
|
||||
Note,
|
||||
)
|
||||
from datascience.automatic_eda.chapters.cat_distr import (
|
||||
CHAPTER_ID, CHAPTER_VERSION, build_cat_distr,
|
||||
@@ -81,8 +85,20 @@ def _pptx_text(path: str) -> str:
|
||||
return re.sub(r"\s+", " ", " ".join(parts))
|
||||
|
||||
|
||||
def _kinds(chapter):
|
||||
return [b.kind for b in chapter.blocks]
|
||||
def _flatten(blocks):
|
||||
"""Expand keep-together Groups so the per-column heading/table/figure are
|
||||
inspectable as a flat block list (the chapter wraps each column in a Group)."""
|
||||
out = []
|
||||
for b in blocks:
|
||||
if getattr(b, "kind", "") == "group":
|
||||
out.extend(_flatten(getattr(b, "blocks", []) or []))
|
||||
else:
|
||||
out.append(b)
|
||||
return out
|
||||
|
||||
|
||||
def _column_groups(chapter):
|
||||
return [b for b in chapter.blocks if isinstance(b, Group)]
|
||||
|
||||
|
||||
def test_golden_build_cat_distr_emite_bloques_pedidos():
|
||||
@@ -90,36 +106,101 @@ def test_golden_build_cat_distr_emite_bloques_pedidos():
|
||||
assert ch is not None
|
||||
assert ch.id == CHAPTER_ID
|
||||
assert ch.version == CHAPTER_VERSION
|
||||
kinds = _kinds(ch)
|
||||
# Entropy intro present.
|
||||
|
||||
# Entropy intro present, but the long explanation is gone (it lives in the
|
||||
# glossary now): only the term is named, no log2/normalizada walkthrough.
|
||||
headings = [b.text for b in ch.blocks if isinstance(b, Heading)]
|
||||
assert any("Entrop" in h for h in headings)
|
||||
md = next(b for b in ch.blocks if b.kind == "markdown")
|
||||
assert "entropía" in md.text.lower() and "log2" in md.text
|
||||
# Cardinality metrics: distinct, total rows, %-distinct, unique values.
|
||||
kv = next(b for b in ch.blocks if isinstance(b, KVTable))
|
||||
md = next(b for b in ch.blocks if isinstance(b, Markdown))
|
||||
assert "entropía" in md.text.lower()
|
||||
assert "log2" not in md.text # redundant explanation removed.
|
||||
assert "máxima diversidad" not in md.text
|
||||
|
||||
# Per-column blocks are wrapped in keep-together Groups: flatten to inspect.
|
||||
flat = _flatten(ch.blocks)
|
||||
kv = next(b for b in flat if isinstance(b, KVTable))
|
||||
labels = [r[0] for r in kv.rows]
|
||||
assert "Valores distintos" in labels
|
||||
assert "% distintos" in labels
|
||||
values = " ".join(str(r[1]) for r in kv.rows)
|
||||
# Cardinality metrics: distinct count, %-distinct, unique values and total
|
||||
# rows are present (grouped onto compact rows so the chart fits the page).
|
||||
assert "Distintos · % · únicos" in labels
|
||||
assert "Total filas (dataset)" in labels
|
||||
assert "Valores únicos (frecuencia 1)" in labels
|
||||
assert any("Entropía" in lbl for lbl in labels)
|
||||
assert "únicos" in values and "%" in values
|
||||
assert "bits" in values and "norm" in values # entropy + max + normalized.
|
||||
# Top-k table + pie figure.
|
||||
dt = next(b for b in ch.blocks if isinstance(b, DataTable))
|
||||
dt = next(b for b in flat if isinstance(b, DataTable))
|
||||
assert dt.header == ["Valor", "Conteo", "%"]
|
||||
assert any("neumaticos" in str(cell) for row in dt.rows for cell in row)
|
||||
assert any(isinstance(b, Figure) for b in ch.blocks)
|
||||
# id-like column flagged with a Note.
|
||||
assert any(isinstance(b, Note) and "identificador" in b.text
|
||||
for b in ch.blocks)
|
||||
assert any(isinstance(b, Figure) for b in flat)
|
||||
# id-like column flagged with a Note that also explains the top-k is dropped.
|
||||
idnote = next((b for b in flat
|
||||
if isinstance(b, Note) and "identificador" in b.text), None)
|
||||
assert idnote is not None
|
||||
assert "No se lista el top" in idnote.text
|
||||
|
||||
|
||||
def test_golden_render_pdf_muestra_categoricas():
|
||||
def test_golden_idlike_omite_topk_y_conserva_donut():
|
||||
# The id-like column (uuid, 100% distinct) must NOT carry a top-k DataTable
|
||||
# (it would be a list of unique values), but must still keep its donut Figure
|
||||
# and its cardinality table so it stays a full per-column page.
|
||||
ch = build_cat_distr(_profile(), {})
|
||||
groups = _column_groups(ch)
|
||||
uuid_group = next(g for g in groups
|
||||
if any(getattr(b, "text", "") == "uuid" for b in g.blocks))
|
||||
kinds = [b.kind for b in uuid_group.blocks]
|
||||
assert "data_table" not in kinds # top-k of unique values dropped.
|
||||
assert "kv_table" in kinds # cardinality kept.
|
||||
assert "figure" in kinds # donut kept (chart per column).
|
||||
# A non-id-like column keeps its top-k table.
|
||||
cat_group = next(g for g in groups
|
||||
if any(getattr(b, "text", "") == "categoria"
|
||||
for b in g.blocks))
|
||||
assert "data_table" in [b.kind for b in cat_group.blocks]
|
||||
|
||||
|
||||
def test_golden_una_pagina_por_columna_groups():
|
||||
ch = build_cat_distr(_profile(), {})
|
||||
groups = _column_groups(ch)
|
||||
# Two categorical columns -> two column Groups (numeric column excluded).
|
||||
assert len(groups) == 2
|
||||
# Each Group carries one column: a heading + its cardinality table + figure.
|
||||
for g in groups:
|
||||
kinds = [b.kind for b in g.blocks]
|
||||
assert kinds[0] == "heading"
|
||||
assert "kv_table" in kinds
|
||||
assert "figure" in kinds
|
||||
# The first column may share the intro page (no forced break); every later
|
||||
# column starts on a fresh page/slide so each column gets its own page.
|
||||
assert groups[0].page_break_before is False
|
||||
assert all(g.page_break_before is True for g in groups[1:])
|
||||
|
||||
|
||||
def test_golden_entropia_clicable_y_definicion_en_glosario():
|
||||
# With a glossary collector the intro marks the clickable term and the FULL
|
||||
# definition (the long explanation removed from the intro) lands in the
|
||||
# glossary, not inline — no data lost, just relocated.
|
||||
gc = GlossaryCollector()
|
||||
ch = build_cat_distr(_profile(), {"glossary": gc})
|
||||
md = next(b for b in ch.blocks if isinstance(b, Markdown))
|
||||
assert "[[term:entropia]]entropía[[/term]]" in md.text
|
||||
assert gc.has("entropia")
|
||||
entry = gc.get("entropia")
|
||||
assert entry is not None
|
||||
# The definition kept in the glossary still carries the detail removed inline.
|
||||
assert "log2" in entry["definition"]
|
||||
assert "normalizada" in entry["definition"].lower()
|
||||
|
||||
|
||||
def test_golden_render_pdf_una_pagina_por_columna():
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
out = os.path.join(d, "eda.pdf")
|
||||
res = render_automatic_eda_pdf(_profile(), out, {"title": "EDA"})
|
||||
assert res["path"] == out and os.path.exists(out)
|
||||
assert CHAPTER_ID in [c["id"] for c in res["chapters"]]
|
||||
cat_meta = next(c for c in res["chapters"] if c["id"] == CHAPTER_ID)
|
||||
# Two categorical columns, each on its own page -> >= 2 pages for the
|
||||
# chapter (intro shares the first column's page).
|
||||
assert cat_meta["n_pages"] >= 2
|
||||
txt = _pdf_text(out)
|
||||
assert "Entrop" in txt
|
||||
assert "distintos" in txt
|
||||
@@ -133,13 +214,91 @@ def test_golden_render_pptx_muestra_categoricas():
|
||||
out = os.path.join(d, "eda.pptx")
|
||||
res = render_automatic_eda_pptx(_profile(), out, {"title": "EDA"})
|
||||
assert res["path"] == out and os.path.exists(out)
|
||||
assert CHAPTER_ID in [c["id"] for c in res["chapters"]]
|
||||
cat_meta = next(c for c in res["chapters"] if c["id"] == CHAPTER_ID)
|
||||
assert cat_meta["n_slides"] >= 2 # one slide per categorical column.
|
||||
txt = _pptx_text(out)
|
||||
assert "Entrop" in txt
|
||||
assert "categoria" in txt and "neumaticos" in txt
|
||||
assert "distintos" in txt
|
||||
|
||||
|
||||
def _profile_high_card() -> dict:
|
||||
"""Profile with a high-cardinality NON-id-like categorical column whose top-k
|
||||
of long values would split from its donut on a short 16:9 slide unless the
|
||||
renderer trims the table — the exact case the adversarial check flagged
|
||||
(Ticket / Cabin)."""
|
||||
long_vals = [f"Valor largo de categoria numero {i:02d} con texto extra"
|
||||
for i in range(40)]
|
||||
top = [{"value": v, "count": 60 - i, "pct": (60 - i) / 5000.0}
|
||||
for i, v in enumerate(long_vals)]
|
||||
return {
|
||||
"table": "t", "source": "t.csv", "n_rows": 5000, "n_cols": 3,
|
||||
"quality_score": 80.0,
|
||||
"columns": [
|
||||
{"name": "precio", "inferred_type": "numeric", "null_pct": 0.0,
|
||||
"numeric": {"mean": 1.0, "median": 1.0, "min": 0.0, "max": 2.0,
|
||||
"std": 0.5}},
|
||||
# 40 distinct over 5000 rows = 0.8% distinct -> NOT id-like, keeps
|
||||
# its (long) top-k table; the tall table must not push the donut off.
|
||||
{"name": "alta_card_col", "inferred_type": "categorical",
|
||||
"null_pct": 0.0, "distinct_count": 40,
|
||||
"categorical": {"top": top, "mode": long_vals[0], "n_distinct": 40,
|
||||
"entropy": 5.2, "imbalance": 1.2, "len_min": 40,
|
||||
"len_mean": 45, "len_max": 50}},
|
||||
{"name": "baja_card_col", "inferred_type": "categorical",
|
||||
"null_pct": 0.0, "distinct_count": 4,
|
||||
"categorical": {
|
||||
"top": [{"value": "norte", "count": 2000, "pct": 0.4},
|
||||
{"value": "sur", "count": 1500, "pct": 0.3},
|
||||
{"value": "este", "count": 1000, "pct": 0.2},
|
||||
{"value": "oeste", "count": 500, "pct": 0.1}],
|
||||
"mode": "norte", "n_distinct": 4, "entropy": 1.8}},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def test_golden_pptx_una_slide_por_columna_con_su_grafico():
|
||||
"""Each categorical column occupies EXACTLY ONE cat_distr slide that carries
|
||||
BOTH its cardinality table and its donut figure (picture) — i.e. the chart is
|
||||
never separated from its table, even for a high-cardinality column."""
|
||||
from pptx.enum.shapes import MSO_SHAPE_TYPE
|
||||
|
||||
prof = _profile_high_card()
|
||||
cat_names = ["alta_card_col", "baja_card_col"]
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
out = os.path.join(d, "eda.pptx")
|
||||
res = render_automatic_eda_pptx(prof, out, {"title": "EDA"})
|
||||
assert res["path"] == out and os.path.exists(out)
|
||||
prs = Presentation(out)
|
||||
|
||||
# Per column: the cat_distr slides whose text mentions it, and whether the
|
||||
# owning slide also has the donut caption + an actual picture shape.
|
||||
slides_with_col = {n: [] for n in cat_names}
|
||||
owner_has_chart = {n: False for n in cat_names}
|
||||
for i, sl in enumerate(prs.slides):
|
||||
texts, has_pic = [], False
|
||||
for sh in sl.shapes:
|
||||
if sh.has_text_frame:
|
||||
texts.append(sh.text_frame.text)
|
||||
if sh.shape_type == MSO_SHAPE_TYPE.PICTURE:
|
||||
has_pic = True
|
||||
txt = re.sub(r"\s+", " ", " ".join(texts))
|
||||
if "Distribuciones categ" not in txt: # footer stamp of the chapter.
|
||||
continue
|
||||
for n in cat_names:
|
||||
if n in txt:
|
||||
slides_with_col[n].append(i)
|
||||
has_table = "Cardinalidad" in txt or "distintos" in txt
|
||||
if has_pic and "donut" in txt and has_table:
|
||||
owner_has_chart[n] = True
|
||||
|
||||
for n in cat_names:
|
||||
# Exactly one slide carries the column (not split across slides).
|
||||
assert len(slides_with_col[n]) == 1, (n, slides_with_col[n])
|
||||
# That single slide also holds its table AND its donut picture.
|
||||
assert owner_has_chart[n], (n, "tabla y donut no están en el mismo slide")
|
||||
|
||||
|
||||
def test_edge_sin_categoricas_devuelve_none():
|
||||
only_numeric = {
|
||||
"n_rows": 10, "columns": [
|
||||
@@ -170,11 +329,15 @@ def test_anti_corte_label_largo_y_muchas_columnas():
|
||||
|
||||
ch = build_cat_distr(profile, {})
|
||||
assert ch is not None
|
||||
# One Group per column, each forcing its own page (except the first).
|
||||
groups = _column_groups(ch)
|
||||
assert len(groups) == 30
|
||||
assert sum(1 for g in groups if g.page_break_before) == 29
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
pdf = os.path.join(d, "anti.pdf")
|
||||
res = render_automatic_eda_pdf(profile, pdf, {"write_manifest": False})
|
||||
assert res["path"] == pdf
|
||||
assert res["n_pages"] > 1 # many columns spilled across pages, OK.
|
||||
assert res["n_pages"] > 1 # one page per column, OK.
|
||||
txt = _pdf_text(pdf)
|
||||
# Long label wrapped (not truncated): every word survives.
|
||||
for word in ("Lorem", "incididunt", "reprehenderit", "voluptate"):
|
||||
|
||||
@@ -356,12 +356,11 @@ def build_correlacion(profile: dict, ctx: dict):
|
||||
t_cramers = _term(mark_term, "cramers_v", "Cramér's V")
|
||||
t_corr_ratio = _term(mark_term, "correlation_ratio", "razón de correlación")
|
||||
blocks.append(model.Markdown(text=(
|
||||
"Asociación entre columnas. Cada par se evalúa con la métrica adecuada a "
|
||||
f"sus tipos ({t_pearson}/{t_spearman} entre numéricas — con **signo**; "
|
||||
f"{t_cramers} entre categóricas; {t_corr_ratio} num-categórica; "
|
||||
"información mutua como medida común no lineal). Sólo las correlaciones "
|
||||
"**num-num** tienen dirección: por eso los pares **negativos** son siempre "
|
||||
"num-num.")))
|
||||
"Asociación entre columnas. Cada par se evalúa con la métrica adecuada "
|
||||
f"a sus tipos: {t_pearson}/{t_spearman} (numéricas), {t_cramers} "
|
||||
f"(categóricas), {t_corr_ratio} (num-categórica) e información mutua. "
|
||||
"Sólo las correlaciones **num-num** llevan **signo** (dirección): por "
|
||||
"eso los pares **negativos** son siempre num-num.")))
|
||||
|
||||
# 1) Association matrix (heatmap).
|
||||
labels, trimmed = _ordered_labels(pairs)
|
||||
|
||||
@@ -6,15 +6,16 @@ normality}``). It renders, as structured markdown/tables/figures that the core
|
||||
paginator never cuts:
|
||||
|
||||
1. **Normalization note** — every multivariate model below standardizes the
|
||||
columns with z-score first; the chapter explains why (different scales would
|
||||
otherwise dominate distance/variance).
|
||||
columns with z-score first (the term is marked clickable; its definition
|
||||
lives in the GLOSARIO chapter, not inline).
|
||||
2. **PCA** — a scree plot (explained + cumulative variance, single Y axis) plus
|
||||
variance and top-loadings tables.
|
||||
3. **KMeans segments** — a PCA scatter **coloured by cluster** (its own
|
||||
page/slide), the cluster-size table, and a per-cluster LLM micro-analysis
|
||||
with a title for each segment.
|
||||
4. **Isolation Forest outliers** — a short explanation of how anomalous rows are
|
||||
isolated multivariately and how the threshold is chosen, plus the counts.
|
||||
4. **Isolation Forest outliers** — the multivariate anomaly counts and decision
|
||||
threshold (the method is marked clickable; its definition lives in the
|
||||
GLOSARIO chapter, not inline).
|
||||
5. **Normality** — per-column Jarque-Bera / D'Agostino / Shapiro verdicts.
|
||||
|
||||
The raw numeric data needed to colour the cluster scatter is **not** in the
|
||||
@@ -314,12 +315,8 @@ def _normalization_intro(gloss=None, mark_term: bool = False) -> list:
|
||||
text = (
|
||||
"Estos modelos son **no supervisados**: buscan estructura latente sin "
|
||||
"una variable objetivo. Antes de aplicarlos, todas las columnas "
|
||||
f"numéricas se {zscore} (cada valor menos la media, dividido por la "
|
||||
"desviación típica). Sin esta normalización, una variable con escala "
|
||||
"grande (p.ej. ingresos en euros) dominaría las distancias y la varianza "
|
||||
"frente a otra de escala pequeña (p.ej. un ratio entre 0 y 1), sesgando "
|
||||
"tanto el PCA como el KMeans. Tras la estandarización todas las variables "
|
||||
"pesan por igual."
|
||||
f"numéricas se {zscore}, para que todas pesen por igual con "
|
||||
"independencia de su escala."
|
||||
)
|
||||
return [model.Heading(text="Modelos no supervisados", level=1),
|
||||
model.Markdown(text=text)]
|
||||
@@ -334,11 +331,11 @@ def _pca_section(pca: dict, gloss=None, mark_term: bool = False) -> list:
|
||||
n_used = pca.get("n_rows_used")
|
||||
n_feat = pca.get("n_features")
|
||||
intro = (
|
||||
f"El {_term(mark_term, 'pca', 'PCA')} resume {_fmt_num(n_feat)} variables "
|
||||
"numéricas en componentes ortogonales ordenados por la varianza que "
|
||||
f"capturan ({_fmt_num(n_used)} filas usadas tras eliminar nulos). El "
|
||||
"gráfico de sedimentación (scree) muestra cuánta varianza aporta cada "
|
||||
"componente y su acumulado: un codo marca cuántos componentes bastan."
|
||||
f"El {_term(mark_term, 'pca', 'PCA')} se aplica sobre "
|
||||
f"{_fmt_num(n_feat)} variables numéricas ({_fmt_num(n_used)} filas "
|
||||
"usadas tras eliminar nulos). El gráfico de sedimentación (scree) "
|
||||
"muestra cuánta varianza aporta cada componente y su acumulado: un "
|
||||
"codo marca cuántos componentes bastan."
|
||||
)
|
||||
blocks.append(model.Markdown(text=intro))
|
||||
|
||||
@@ -403,9 +400,8 @@ def _kmeans_section(kmeans: dict, projection: dict, titles,
|
||||
t_sil = _term(mark_term, "silhouette", "*silhouette*")
|
||||
intro = (
|
||||
f"{t_kmeans} agrupa las filas en **{_fmt_num(best_k)} segmentos** "
|
||||
f"elegidos automáticamente maximizando el coeficiente de {t_sil} "
|
||||
f"(**{_fmt_num(sil)}**, rango −1 a 1: cuanto más alto, segmentos más "
|
||||
"compactos y separados). Los segmentos se proyectan sobre el plano de "
|
||||
f"elegidos automáticamente por el coeficiente de {t_sil} "
|
||||
f"(**{_fmt_num(sil)}**). Los segmentos se proyectan sobre el plano de "
|
||||
"los dos primeros componentes principales para visualizarlos."
|
||||
)
|
||||
blocks.append(model.Markdown(text=intro))
|
||||
@@ -469,14 +465,10 @@ def _outliers_section(outliers: dict, gloss=None, mark_term: bool = False) -> li
|
||||
level=2)]
|
||||
isof = _term(mark_term, "isolation_forest", "**Isolation Forest**")
|
||||
explain = (
|
||||
f"{isof} detecta filas anómalas de forma *multivariante*: "
|
||||
"construye árboles que parten el espacio con cortes aleatorios y mide "
|
||||
"cuántos cortes hacen falta para aislar cada fila. Las filas raras "
|
||||
"(combinaciones de valores poco frecuentes considerando **todas las "
|
||||
"columnas a la vez**, no una sola) se aíslan con muy pocos cortes y "
|
||||
"obtienen un score bajo. El **umbral** de decisión separa las filas "
|
||||
"normales de las anómalas según la contaminación esperada del modelo: "
|
||||
"una fila es outlier cuando su score queda por debajo de ese umbral."
|
||||
f"{isof} marca filas anómalas de forma *multivariante*: combinaciones "
|
||||
"de valores poco frecuentes considerando **todas las columnas a la "
|
||||
"vez**, no una sola. La tabla resume cuántas se detectaron y el umbral "
|
||||
"de decisión empleado."
|
||||
)
|
||||
blocks.append(model.Markdown(text=explain))
|
||||
blocks.append(model.KVTable(rows=[
|
||||
|
||||
@@ -256,14 +256,14 @@ def _pk_candidates_section(profile: dict, mark: bool) -> list:
|
||||
pk = ("[[term:pk]]**clave primaria**[[/term]]" if mark
|
||||
else "**clave primaria**")
|
||||
intro = (
|
||||
f"Estas columnas son **candidatas a {pk}**: su "
|
||||
"[[term:cardinalidad]]cardinalidad[[/term]] iguala al número de filas y no "
|
||||
"tienen nulos, así que cada valor identifica una fila distinta. Son "
|
||||
"candidatas, no una clave declarada: la base no las marca como tal."
|
||||
f"Columnas **candidatas a {pk}**: su "
|
||||
"[[term:cardinalidad]]cardinalidad[[/term]] iguala al número de filas y "
|
||||
"no tienen nulos. Son candidatas, no una clave declarada: la base no "
|
||||
"las marca como tal."
|
||||
if mark else
|
||||
"Estas columnas son **candidatas a clave primaria**: su cardinalidad "
|
||||
"iguala al número de filas y no tienen nulos, así que cada valor "
|
||||
"identifica una fila distinta.")
|
||||
"Columnas **candidatas a clave primaria**: su cardinalidad iguala al "
|
||||
"número de filas y no tienen nulos. Son candidatas, no una clave "
|
||||
"declarada.")
|
||||
|
||||
rows = []
|
||||
for name in keys:
|
||||
@@ -320,10 +320,10 @@ def _inter_table_section(db_path: str, tables: list, mark: bool) -> list:
|
||||
blocks = [
|
||||
model.Heading(text="Claves foráneas candidatas (inter-tabla)", level=2),
|
||||
model.Markdown(text=(
|
||||
f"La fuente tiene varias tablas. Estas {fk_term} candidatas se infieren "
|
||||
f"por señal de nombre y por {containment}: una columna de una tabla cuyos "
|
||||
"valores están contenidos en la clave de otra. No están declaradas por "
|
||||
"la base; son la relación más probable según los datos.")),
|
||||
f"La fuente tiene varias tablas. Estas {fk_term} candidatas se "
|
||||
f"infieren por señal de nombre y por {containment}. No están "
|
||||
"declaradas por la base; son la relación más probable según los "
|
||||
"datos.")),
|
||||
]
|
||||
|
||||
shown = candidates[:MAX_FK_ROWS]
|
||||
@@ -441,13 +441,12 @@ def _intro_blocks(mark: bool) -> list:
|
||||
pk = "[[term:pk]]clave primaria[[/term]]" if mark else "clave primaria"
|
||||
fk = "[[term:fk]]clave foránea[[/term]]" if mark else "clave foránea"
|
||||
text = (
|
||||
f"Este capítulo analiza las **relaciones de clave** de la tabla: qué columna "
|
||||
f"identifica cada fila (la {pk}) y qué columnas referencian a otra tabla (las "
|
||||
f"{fk}). Cuando la base las **declara** como restricciones del esquema, se "
|
||||
"muestran tal cual; cuando no, se proponen las más probables a partir de los "
|
||||
"datos —por inclusión de valores entre tablas (containment) o, en una sola "
|
||||
"tabla, por una heurística de nombre y cardinalidad— siempre marcadas como "
|
||||
"candidatas, nunca como hechos.")
|
||||
f"Este capítulo analiza las **relaciones de clave** de la tabla: cuál es "
|
||||
f"la {pk} y cuáles son las {fk}. Cuando la base las **declara** como "
|
||||
"restricciones del esquema, se muestran tal cual; cuando no, se proponen "
|
||||
"las más probables a partir de los datos —por containment entre tablas o, "
|
||||
"en una sola tabla, por una heurística de nombre y cardinalidad— siempre "
|
||||
"marcadas como candidatas, nunca como hechos.")
|
||||
return [model.Heading(text=CHAPTER_TITLE, level=1), model.Markdown(text=text)]
|
||||
|
||||
|
||||
|
||||
@@ -139,10 +139,17 @@ class Group:
|
||||
it starts on a fresh page and flows (honest degradation, never cut). Use it to
|
||||
bind ``Heading`` + ``Markdown`` + ``Figure`` of one idea together (see the
|
||||
DISTR NUM / AGREGACION chapters).
|
||||
|
||||
When ``page_break_before`` is True the renderer additionally forces the group
|
||||
to *start* on a fresh page/slide (unless the current one is already empty), so
|
||||
a chapter can give each unit its own page — e.g. one categorical column per
|
||||
page (see CAT DISTR). It is purely additive: the default False keeps the plain
|
||||
keep-together behaviour for every existing chapter.
|
||||
"""
|
||||
|
||||
blocks: list = field(default_factory=list)
|
||||
title: Optional[str] = None
|
||||
page_break_before: bool = False
|
||||
kind: str = field(default="group", init=False)
|
||||
|
||||
|
||||
@@ -228,7 +235,9 @@ def as_block(obj: Any):
|
||||
return Note(text=_safe_str(obj.get("text")))
|
||||
if cls is Group:
|
||||
return Group(blocks=as_blocks(obj.get("blocks")),
|
||||
title=obj.get("title"))
|
||||
title=obj.get("title"),
|
||||
page_break_before=bool(
|
||||
obj.get("page_break_before", False)))
|
||||
if cls is GlossaryEntry:
|
||||
return GlossaryEntry(key=_safe_str(obj.get("key")),
|
||||
label=_safe_str(obj.get("label")),
|
||||
|
||||
@@ -675,6 +675,61 @@ def _measure_figure_like(block) -> float:
|
||||
return target_h + 0.04 + cap_h + _GAP
|
||||
|
||||
|
||||
def _measure_kv_table(block) -> float:
|
||||
"""Faithful height of a KVTable — matches ``_place_kv_table``.
|
||||
|
||||
Counts the optional title heading and, per row, the wrapped VALUE column
|
||||
(the label column never wraps in the placer). The previous estimate assumed
|
||||
one line per row and ignored the title, so a column's keep-together Group
|
||||
under-budgeted the figure and the chart spilled to the next page. Keep this in
|
||||
sync with ``_place_kv_table``."""
|
||||
h = 0.0
|
||||
title = getattr(block, "title", None)
|
||||
if title:
|
||||
h += _measure_heading_text(title, 2)
|
||||
rows = getattr(block, "rows", []) or []
|
||||
key_w = 1.9
|
||||
val_chars = tl.chars_per_line(_USABLE_W - key_w - 0.1, _FS_BODY)
|
||||
lh = tl.line_height_in(_FS_BODY)
|
||||
for row in rows:
|
||||
try:
|
||||
value = row[1]
|
||||
except Exception: # noqa: BLE001
|
||||
value = ""
|
||||
v_lines = tl.wrap(model._safe_str(value), val_chars)
|
||||
h += lh * len(v_lines) + _ROW_VPAD
|
||||
return h + _GAP
|
||||
|
||||
|
||||
def _measure_data_table(block) -> float:
|
||||
"""Faithful height of a DataTable — matches ``_place_data_table``.
|
||||
|
||||
Counts the optional title heading, the wrapped header row, every wrapped data
|
||||
row (per-column wrap via the same ``_col_widths``/``_wrap_row`` the placer
|
||||
uses) and the optional note. Keep this in sync with ``_place_data_table``."""
|
||||
h = 0.0
|
||||
title = getattr(block, "title", None)
|
||||
if title:
|
||||
h += _measure_heading_text(title, 2)
|
||||
header = list(getattr(block, "header", []) or [])
|
||||
rows = list(getattr(block, "rows", []) or [])
|
||||
fs = _FS_CELL
|
||||
widths = _col_widths(header, rows, fs)
|
||||
lh = tl.line_height_in(fs)
|
||||
if header:
|
||||
header_lines = _wrap_row(header, widths, fs)
|
||||
h += lh * max((len(c) for c in header_lines), default=1) + _ROW_VPAD * 2
|
||||
for r in rows:
|
||||
cells_lines = _wrap_row(r, widths, fs)
|
||||
h += lh * max((len(c) for c in cells_lines), default=1) + _ROW_VPAD * 2
|
||||
note = getattr(block, "note", None)
|
||||
if note:
|
||||
nlines = tl.wrap(model._safe_str(note),
|
||||
tl.chars_per_line(_USABLE_W, _FS_NOTE))
|
||||
h += tl.line_height_in(_FS_NOTE) * len(nlines)
|
||||
return h + _GAP
|
||||
|
||||
|
||||
def _measure_block(st: _PdfState, block) -> float:
|
||||
kind = getattr(block, "kind", "")
|
||||
try:
|
||||
@@ -690,13 +745,9 @@ def _measure_block(st: _PdfState, block) -> float:
|
||||
tl.chars_per_line(_USABLE_W, _FS_NOTE))
|
||||
return tl.line_height_in(_FS_NOTE) * len(lines) + _GAP
|
||||
if kind == "kv_table":
|
||||
rows = getattr(block, "rows", []) or []
|
||||
return (tl.line_height_in(_FS_BODY) + _ROW_VPAD) * (len(rows) + 1) \
|
||||
+ _GAP
|
||||
return _measure_kv_table(block)
|
||||
if kind == "data_table":
|
||||
rows = getattr(block, "rows", []) or []
|
||||
return (tl.line_height_in(_FS_CELL) + _ROW_VPAD * 2) \
|
||||
* (len(rows) + 1) + _GAP
|
||||
return _measure_data_table(block)
|
||||
if kind == "group":
|
||||
return sum(_measure_block(st, b)
|
||||
for b in (getattr(block, "blocks", []) or []))
|
||||
@@ -735,6 +786,10 @@ def _place_group(st: _PdfState, block) -> None:
|
||||
blocks = getattr(block, "blocks", []) or []
|
||||
if not blocks:
|
||||
return
|
||||
# Opt-in page break: start this group on a fresh page unless the current one
|
||||
# is still empty (so a chapter can give each unit its own page).
|
||||
if getattr(block, "page_break_before", False) and st.y > _CONTENT_TOP + 1e-6:
|
||||
_new_page(st)
|
||||
avail_full = _CONTENT_BOTTOM - _CONTENT_TOP
|
||||
_shrink_group_figures(st, blocks, avail_full)
|
||||
total = sum(_measure_block(st, b) for b in blocks)
|
||||
|
||||
@@ -625,6 +625,55 @@ def _measure_figure_like(block) -> float:
|
||||
return target_h + 0.05 + cap_h + _GAP
|
||||
|
||||
|
||||
def _measure_kv_table(block) -> float:
|
||||
"""Faithful KVTable height — matches ``_place_kv_table`` (rendered as a
|
||||
Campo/Valor data table with wrapped cells). The previous estimate assumed one
|
||||
line per row and ignored the title, so a keep-together Group under-budgeted
|
||||
the figure and the chart spilled to the next slide. Keep in sync."""
|
||||
h = 0.0
|
||||
title = getattr(block, "title", None)
|
||||
if title:
|
||||
h += _measure_heading_text(title, 2)
|
||||
rows = getattr(block, "rows", []) or []
|
||||
data_rows = []
|
||||
for row in rows:
|
||||
try:
|
||||
label, value = row[0], row[1]
|
||||
except Exception: # noqa: BLE001
|
||||
label, value = str(row), ""
|
||||
data_rows.append([model._safe_str(label), model._safe_str(value)])
|
||||
header = ["Campo", "Valor"]
|
||||
widths = _col_widths(header, data_rows)
|
||||
fs = _FS_CELL
|
||||
h += _row_height_in(header, widths, fs)
|
||||
for r in data_rows:
|
||||
h += _row_height_in(r, widths, fs)
|
||||
return h + _GAP
|
||||
|
||||
|
||||
def _measure_data_table(block) -> float:
|
||||
"""Faithful DataTable height — matches ``_place_data_table`` (title heading +
|
||||
wrapped header + every wrapped row + optional note). Keep in sync."""
|
||||
h = 0.0
|
||||
title = getattr(block, "title", None)
|
||||
if title:
|
||||
h += _measure_heading_text(title, 2)
|
||||
header = list(getattr(block, "header", []) or [])
|
||||
rows = list(getattr(block, "rows", []) or [])
|
||||
fs = _FS_CELL
|
||||
widths = _col_widths(header, rows)
|
||||
if header:
|
||||
h += _row_height_in(header, widths, fs)
|
||||
for r in rows:
|
||||
h += _row_height_in(r, widths, fs)
|
||||
note = getattr(block, "note", None)
|
||||
if note:
|
||||
nlines = tl.wrap(model._safe_str(note),
|
||||
tl.chars_per_line(_USABLE_W, _FS_NOTE))
|
||||
h += tl.line_height_in(_FS_NOTE) * len(nlines) + 0.05
|
||||
return h + _GAP
|
||||
|
||||
|
||||
def _measure_block(st: _PptxState, block) -> float:
|
||||
kind = getattr(block, "kind", "")
|
||||
try:
|
||||
@@ -639,9 +688,10 @@ def _measure_block(st: _PptxState, block) -> float:
|
||||
lines = tl.wrap(getattr(block, "text", ""),
|
||||
tl.chars_per_line(_USABLE_W, _FS_NOTE))
|
||||
return tl.line_height_in(_FS_NOTE) * len(lines) + 0.05 + _GAP
|
||||
if kind in ("kv_table", "data_table"):
|
||||
rows = getattr(block, "rows", []) or []
|
||||
return (tl.line_height_in(_FS_CELL) + 0.10) * (len(rows) + 1) + _GAP
|
||||
if kind == "kv_table":
|
||||
return _measure_kv_table(block)
|
||||
if kind == "data_table":
|
||||
return _measure_data_table(block)
|
||||
if kind == "group":
|
||||
return sum(_measure_block(st, b)
|
||||
for b in (getattr(block, "blocks", []) or []))
|
||||
@@ -664,10 +714,14 @@ def _shrink_group_figures(st: _PptxState, blocks: list, avail_full: float) -> No
|
||||
if getattr(b, "kind", "") not in ("figure", "image"))
|
||||
fig_overhead = tl.line_height_in(_FS_NOTE) + 0.05 + 0.05 + _GAP
|
||||
budget = avail_full - nonfig_h - 0.10 * len(fig_blocks)
|
||||
if budget <= 1.0:
|
||||
# Low thresholds: a 16:9 slide is short, so a content-heavy column (cardinality
|
||||
# table + top-k + chart) only fits if the chart is allowed to shrink small.
|
||||
# Prefer a small-but-present chart on the SAME slide over splitting the column
|
||||
# across slides (matches the PDF renderer's keep-together philosophy).
|
||||
if budget <= 0.6:
|
||||
return # not enough room to keep together; let it flow (degrade).
|
||||
per = budget / len(fig_blocks) - fig_overhead
|
||||
if per <= 0.8:
|
||||
if per <= 0.35:
|
||||
return
|
||||
for fb in fig_blocks:
|
||||
cur = getattr(fb, "height_in", None)
|
||||
@@ -675,12 +729,90 @@ def _shrink_group_figures(st: _PptxState, blocks: list, avail_full: float) -> No
|
||||
if isinstance(cur, (int, float)) and cur > 0 else per)
|
||||
|
||||
|
||||
# Minimum height (inches) reserved for a figure inside a keep-together group on
|
||||
# the short 16:9 slide. When a high-cardinality column's table(s) would otherwise
|
||||
# leave no room, the data table is trimmed (with an honest note) so the chart
|
||||
# stays on the SAME slide next to its table instead of spilling to the next one.
|
||||
_GROUP_MIN_FIG_H = 1.3
|
||||
|
||||
|
||||
def _trim_data_table_to_budget(block, budget: float):
|
||||
"""Return a copy of a DataTable whose rows fit within ``budget`` inches.
|
||||
|
||||
Keeps the title, header, as many leading rows as fit (at least one) and an
|
||||
honest note reporting how many of the original rows are shown. NEVER mutates
|
||||
the original block — the same Chapter blocks are rendered by the PDF renderer,
|
||||
which keeps the full table (an A5 page fits it)."""
|
||||
header = list(getattr(block, "header", []) or [])
|
||||
rows = list(getattr(block, "rows", []) or [])
|
||||
title = getattr(block, "title", None)
|
||||
fs = _FS_CELL
|
||||
widths = _col_widths(header, rows)
|
||||
fixed = 0.0
|
||||
if title:
|
||||
fixed += _measure_heading_text(title, 2)
|
||||
if header:
|
||||
fixed += _row_height_in(header, widths, fs)
|
||||
note_h = tl.line_height_in(_FS_NOTE) + 0.05
|
||||
avail_rows = budget - fixed - note_h - _GAP
|
||||
kept = []
|
||||
used = 0.0
|
||||
for r in rows:
|
||||
rh = _row_height_in(r, widths, fs)
|
||||
if used + rh > avail_rows and kept:
|
||||
break
|
||||
kept.append(r)
|
||||
used += rh
|
||||
if len(kept) >= len(rows):
|
||||
return block # already fits; keep the original (with its own note).
|
||||
note = (f"top {len(kept)} de {len(rows)} categorías mostradas "
|
||||
"(recortado para caber en el slide; el PDF muestra más)")
|
||||
return model.DataTable(header=header, rows=kept, title=title, note=note)
|
||||
|
||||
|
||||
def _fit_group_blocks(st: _PptxState, blocks: list, avail_full: float) -> list:
|
||||
"""Return a slide-fitting copy of a keep-together group's blocks.
|
||||
|
||||
On the short 16:9 slide a high-cardinality column's top-k table plus its
|
||||
chart can overflow. Reserve ``_GROUP_MIN_FIG_H`` for the (later shrunk) figure
|
||||
and trim the data table(s) to what is left, so every column keeps its chart
|
||||
next to its table on ONE slide. No-op when the group has no figure+table pair
|
||||
(e.g. id-like columns already drop the top-k upstream, or it already fits)."""
|
||||
has_fig = any(getattr(b, "kind", "") in ("figure", "image") for b in blocks)
|
||||
tbls = [b for b in blocks if getattr(b, "kind", "") == "data_table"]
|
||||
if not (has_fig and tbls):
|
||||
return blocks
|
||||
fixed_h = sum(_measure_block(st, b) for b in blocks
|
||||
if getattr(b, "kind", "") not in ("figure", "image",
|
||||
"data_table"))
|
||||
tables_h = sum(_measure_block(st, b) for b in tbls)
|
||||
budget_tables = avail_full - fixed_h - _GROUP_MIN_FIG_H
|
||||
if tables_h <= budget_tables:
|
||||
return blocks # already fits next to a min-height figure; leave intact.
|
||||
out = []
|
||||
for b in blocks:
|
||||
if getattr(b, "kind", "") != "data_table":
|
||||
out.append(b)
|
||||
continue
|
||||
trimmed = _trim_data_table_to_budget(b, max(budget_tables, 0.8))
|
||||
out.append(trimmed)
|
||||
budget_tables -= _measure_data_table(trimmed)
|
||||
return out
|
||||
|
||||
|
||||
def _place_group(st: _PptxState, block) -> None:
|
||||
"""Render a keep-together Group: move it whole to the next slide if needed."""
|
||||
blocks = getattr(block, "blocks", []) or []
|
||||
if not blocks:
|
||||
return
|
||||
# Opt-in slide break: start this group on a fresh slide unless the current one
|
||||
# is still empty (so a chapter can give each unit its own slide).
|
||||
if getattr(block, "page_break_before", False) and st.y > _CONTENT_TOP + 1e-6:
|
||||
_new_slide(st, cont=True)
|
||||
avail_full = _CONTENT_BOTTOM - _CONTENT_TOP
|
||||
# Trim oversized tables first (keeps the chart on the same slide), then shrink
|
||||
# the figure to share the remaining room.
|
||||
blocks = _fit_group_blocks(st, blocks, avail_full)
|
||||
_shrink_group_figures(st, blocks, avail_full)
|
||||
total = sum(_measure_block(st, b) for b in blocks)
|
||||
if total <= avail_full:
|
||||
|
||||
Reference in New Issue
Block a user