Files
fn_registry/apps/auto_metabase/payload.py
T
egutierrez 310b409ae0 feat(auto_metabase): push-all + describe/sql + auto-inject de dashcards
- push_all(): pushea todos los YAMLs de un proyecto (cards primero,
  dashboards despues), solo CREATE/UPDATE, resiliente a fallos por item
- explore.py: comandos describe (schema de DB) y sql (query ad-hoc con
  limite, cap 5MB, bloqueo de escrituras destructivas)
- payload.py: auto-inyecta id:-N, visualization_settings:{} y
  parameter_mappings:[] en dashcards nuevas para evitar 500 en push
- test_local: 11 cards + 3 dashboards sobre Sample Database de Metabase
- registry.db regenerado con auto_metabase_py_analytics indexada

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-04-13 13:14:05 +02:00

221 lines
7.9 KiB
Python

"""Construccion del payload final que se envia a Metabase.
Toma el YAML de un item local (con _meta + _refs + payload) y devuelve el
payload listo para POST/PUT, con slugs reemplazados por IDs reales del index.
Funciones puras — sin red, sin escritura, deterministicas.
"""
from __future__ import annotations
import copy
from pathlib import Path
from typing import Any
import yaml
# ---------------------------------------------------------------- Carga de YAMLs
def load_item_yaml(path: Path) -> dict:
"""Lee un YAML de item (card/dashboard/database/collection)."""
with path.open() as f:
doc = yaml.safe_load(f) or {}
if not isinstance(doc, dict):
raise ValueError(f"{path}: contenido no es un dict YAML")
for key in ("_meta", "_refs", "payload"):
if key not in doc:
raise ValueError(f"{path}: falta bloque '{key}'")
return doc
def assert_meta(doc: dict, expected_kind: str, expected_slug: str, path: Path) -> None:
"""Aborta si _meta no coincide con kind/slug esperados (regla R9)."""
meta = doc.get("_meta", {})
if meta.get("kind") != expected_kind:
raise ValueError(
f"{path}: _meta.kind='{meta.get('kind')}' pero esperaba '{expected_kind}'"
)
if meta.get("slug") != expected_slug:
raise ValueError(
f"{path}: _meta.slug='{meta.get('slug')}' pero esperaba '{expected_slug}'"
)
def assert_id_matches_index(doc: dict, kind: str, slug: str, index: dict, path: Path) -> None:
"""Aborta si _meta.id difiere del id del index (regla R11)."""
meta_id = doc.get("_meta", {}).get("id")
idx_id = index.get(kind + "s", {}).get(slug) # cards/dashboards/...
if meta_id is None and idx_id is None:
return # item nuevo, no hay id que comparar
if meta_id != idx_id:
raise ValueError(
f"{path}: _meta.id={meta_id} no coincide con index ({idx_id}). "
f"Posible YAML duplicado con slug renombrado pero id sin actualizar."
)
# ---------------------------------------------------------------- Resolucion de refs
def _resolve_slug(slug: str | None, kind_plural: str, index: dict) -> int | None:
"""Slug -> id Metabase. None -> None. Slug desconocido -> ValueError."""
if slug is None:
return None
mapping = index.get(kind_plural, {})
if slug not in mapping:
raise ValueError(
f"slug '{slug}' (tipo {kind_plural}) no existe en index. "
f"Conocidos: {sorted(mapping.keys())}"
)
return mapping[slug]
# ---------------------------------------------------------------- Builders por kind
def build_card_payload(doc: dict, index: dict) -> dict:
"""Resuelve refs y devuelve el payload listo para POST/PUT a /api/card.
Sin lecturas de Metabase. Sin merges con estado remoto. Solo lo que
tiene el YAML (regla R8).
"""
refs = doc.get("_refs", {}) or {}
payload = copy.deepcopy(doc.get("payload", {}) or {})
# database (obligatorio en cards)
db_slug = refs.get("database")
if db_slug is None:
raise ValueError("card payload: falta _refs.database")
db_id = _resolve_slug(db_slug, "databases", index)
payload["database_id"] = db_id
# dataset_query.database (mismo id, tambien debe ir resuelto)
dq = payload.get("dataset_query")
if isinstance(dq, dict) and "database" in dq:
dq["database"] = db_id
# collection (opcional; puede ser None = root)
if "collection" in refs:
coll_slug = refs["collection"]
payload["collection_id"] = _resolve_slug(coll_slug, "collections", index)
return payload
def build_dashboard_payload(doc: dict, index: dict) -> dict:
"""Resuelve refs y devuelve el payload listo para POST/PUT a /api/dashboard.
Para dashcards: cada `card` slug -> `card_id` int. Series tambien.
Mantiene la lista de dashcards COMPLETA tal como esta en el YAML
(Metabase la trata como estado deseado).
Auto-inyeccion para dashcards nuevas (sin `id`):
- `id`: asigna negativo unico (-1, -2, ...) en orden de aparicion.
- `visualization_settings`: {} si falta.
- `parameter_mappings`: [] si falta.
Si la dashcard ya trae `id` (positivo o negativo), no se toca.
"""
refs = doc.get("_refs", {}) or {}
payload = copy.deepcopy(doc.get("payload", {}) or {})
# collection (opcional)
if "collection" in refs:
coll_slug = refs["collection"]
payload["collection_id"] = _resolve_slug(coll_slug, "collections", index)
# dashcards: card slug -> card_id int + auto-inyeccion de campos nuevos
dashcards = payload.get("dashcards", []) or []
new_dashcards = []
next_neg_id = -1 # contador de ids negativos auto-asignados
used_neg_ids = {dc["id"] for dc in dashcards if isinstance(dc.get("id"), int) and dc["id"] < 0}
for i, dc in enumerate(dashcards):
dc = dict(dc)
card_slug = dc.pop("card", None)
if card_slug is not None:
try:
dc["card_id"] = _resolve_slug(card_slug, "cards", index)
except ValueError as e:
raise ValueError(f"dashcard #{i}: {e}") from None
# series: lista de slugs -> lista de {id}
series_slugs = dc.get("series") or []
if series_slugs:
dc["series"] = [
{"id": _resolve_slug(s, "cards", index)} for s in series_slugs
]
# auto-inyeccion: id negativo si no hay id (CREATE de dashcard)
if "id" not in dc:
while next_neg_id in used_neg_ids:
next_neg_id -= 1
dc["id"] = next_neg_id
used_neg_ids.add(next_neg_id)
next_neg_id -= 1
# auto-inyeccion: viz_settings y parameter_mappings vacios si faltan
dc.setdefault("visualization_settings", {})
dc.setdefault("parameter_mappings", [])
new_dashcards.append(dc)
payload["dashcards"] = new_dashcards
return payload
def build_database_payload(doc: dict, index: dict, env: dict) -> dict:
"""Construye payload de database. Resuelve passwords desde env vars.
Si details.password es ${VAR}, lo sustituye por env[VAR]. Si no esta,
deja el placeholder (push fallara con error claro).
"""
payload = copy.deepcopy(doc.get("payload", {}) or {})
details = payload.get("details", {}) or {}
pwd = details.get("password")
if isinstance(pwd, str) and pwd.startswith("${") and pwd.endswith("}"):
var_name = pwd[2:-1]
if var_name in env:
details["password"] = env[var_name]
# si no esta, queda el placeholder y push fallara
payload["details"] = details
return payload
def build_collection_payload(doc: dict, index: dict) -> dict:
refs = doc.get("_refs", {}) or {}
payload = copy.deepcopy(doc.get("payload", {}) or {})
if "parent" in refs:
parent_slug = refs["parent"]
payload["parent_id"] = _resolve_slug(parent_slug, "collections", index)
return payload
# ---------------------------------------------------------------- Dispatch
_BUILDERS = {
"card": build_card_payload,
"dashboard": build_dashboard_payload,
"database": build_database_payload,
"collection": build_collection_payload,
}
def build_payload(kind: str, doc: dict, index: dict, env: dict | None = None) -> dict:
"""Punto de entrada: dispatch por kind."""
if kind not in _BUILDERS:
raise ValueError(f"kind '{kind}' desconocido. Validos: {sorted(_BUILDERS)}")
if kind == "database":
return _BUILDERS[kind](doc, index, env or {})
return _BUILDERS[kind](doc, index)
def known_card_ids(index: dict) -> set[int]:
"""Set de ids de cards conocidas (para validacion de dashcards)."""
return set(index.get("cards", {}).values())
# ---------------------------------------------------------------- Lookup paths
def item_path(project_dir: Path, kind: str, slug: str) -> Path:
"""Path al YAML de un item: cards/foo.yaml, dashboards/bar.yaml, etc."""
return project_dir / (kind + "s") / f"{slug}.yaml"