310b409ae0
- push_all(): pushea todos los YAMLs de un proyecto (cards primero,
dashboards despues), solo CREATE/UPDATE, resiliente a fallos por item
- explore.py: comandos describe (schema de DB) y sql (query ad-hoc con
limite, cap 5MB, bloqueo de escrituras destructivas)
- payload.py: auto-inyecta id:-N, visualization_settings:{} y
parameter_mappings:[] en dashcards nuevas para evitar 500 en push
- test_local: 11 cards + 3 dashboards sobre Sample Database de Metabase
- registry.db regenerado con auto_metabase_py_analytics indexada
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
221 lines
7.9 KiB
Python
221 lines
7.9 KiB
Python
"""Construccion del payload final que se envia a Metabase.
|
|
|
|
Toma el YAML de un item local (con _meta + _refs + payload) y devuelve el
|
|
payload listo para POST/PUT, con slugs reemplazados por IDs reales del index.
|
|
|
|
Funciones puras — sin red, sin escritura, deterministicas.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import copy
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import yaml
|
|
|
|
|
|
# ---------------------------------------------------------------- Carga de YAMLs
|
|
|
|
|
|
def load_item_yaml(path: Path) -> dict:
|
|
"""Lee un YAML de item (card/dashboard/database/collection)."""
|
|
with path.open() as f:
|
|
doc = yaml.safe_load(f) or {}
|
|
if not isinstance(doc, dict):
|
|
raise ValueError(f"{path}: contenido no es un dict YAML")
|
|
for key in ("_meta", "_refs", "payload"):
|
|
if key not in doc:
|
|
raise ValueError(f"{path}: falta bloque '{key}'")
|
|
return doc
|
|
|
|
|
|
def assert_meta(doc: dict, expected_kind: str, expected_slug: str, path: Path) -> None:
|
|
"""Aborta si _meta no coincide con kind/slug esperados (regla R9)."""
|
|
meta = doc.get("_meta", {})
|
|
if meta.get("kind") != expected_kind:
|
|
raise ValueError(
|
|
f"{path}: _meta.kind='{meta.get('kind')}' pero esperaba '{expected_kind}'"
|
|
)
|
|
if meta.get("slug") != expected_slug:
|
|
raise ValueError(
|
|
f"{path}: _meta.slug='{meta.get('slug')}' pero esperaba '{expected_slug}'"
|
|
)
|
|
|
|
|
|
def assert_id_matches_index(doc: dict, kind: str, slug: str, index: dict, path: Path) -> None:
|
|
"""Aborta si _meta.id difiere del id del index (regla R11)."""
|
|
meta_id = doc.get("_meta", {}).get("id")
|
|
idx_id = index.get(kind + "s", {}).get(slug) # cards/dashboards/...
|
|
if meta_id is None and idx_id is None:
|
|
return # item nuevo, no hay id que comparar
|
|
if meta_id != idx_id:
|
|
raise ValueError(
|
|
f"{path}: _meta.id={meta_id} no coincide con index ({idx_id}). "
|
|
f"Posible YAML duplicado con slug renombrado pero id sin actualizar."
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------- Resolucion de refs
|
|
|
|
|
|
def _resolve_slug(slug: str | None, kind_plural: str, index: dict) -> int | None:
|
|
"""Slug -> id Metabase. None -> None. Slug desconocido -> ValueError."""
|
|
if slug is None:
|
|
return None
|
|
mapping = index.get(kind_plural, {})
|
|
if slug not in mapping:
|
|
raise ValueError(
|
|
f"slug '{slug}' (tipo {kind_plural}) no existe en index. "
|
|
f"Conocidos: {sorted(mapping.keys())}"
|
|
)
|
|
return mapping[slug]
|
|
|
|
|
|
# ---------------------------------------------------------------- Builders por kind
|
|
|
|
|
|
def build_card_payload(doc: dict, index: dict) -> dict:
|
|
"""Resuelve refs y devuelve el payload listo para POST/PUT a /api/card.
|
|
|
|
Sin lecturas de Metabase. Sin merges con estado remoto. Solo lo que
|
|
tiene el YAML (regla R8).
|
|
"""
|
|
refs = doc.get("_refs", {}) or {}
|
|
payload = copy.deepcopy(doc.get("payload", {}) or {})
|
|
|
|
# database (obligatorio en cards)
|
|
db_slug = refs.get("database")
|
|
if db_slug is None:
|
|
raise ValueError("card payload: falta _refs.database")
|
|
db_id = _resolve_slug(db_slug, "databases", index)
|
|
payload["database_id"] = db_id
|
|
|
|
# dataset_query.database (mismo id, tambien debe ir resuelto)
|
|
dq = payload.get("dataset_query")
|
|
if isinstance(dq, dict) and "database" in dq:
|
|
dq["database"] = db_id
|
|
|
|
# collection (opcional; puede ser None = root)
|
|
if "collection" in refs:
|
|
coll_slug = refs["collection"]
|
|
payload["collection_id"] = _resolve_slug(coll_slug, "collections", index)
|
|
|
|
return payload
|
|
|
|
|
|
def build_dashboard_payload(doc: dict, index: dict) -> dict:
|
|
"""Resuelve refs y devuelve el payload listo para POST/PUT a /api/dashboard.
|
|
|
|
Para dashcards: cada `card` slug -> `card_id` int. Series tambien.
|
|
Mantiene la lista de dashcards COMPLETA tal como esta en el YAML
|
|
(Metabase la trata como estado deseado).
|
|
|
|
Auto-inyeccion para dashcards nuevas (sin `id`):
|
|
- `id`: asigna negativo unico (-1, -2, ...) en orden de aparicion.
|
|
- `visualization_settings`: {} si falta.
|
|
- `parameter_mappings`: [] si falta.
|
|
Si la dashcard ya trae `id` (positivo o negativo), no se toca.
|
|
"""
|
|
refs = doc.get("_refs", {}) or {}
|
|
payload = copy.deepcopy(doc.get("payload", {}) or {})
|
|
|
|
# collection (opcional)
|
|
if "collection" in refs:
|
|
coll_slug = refs["collection"]
|
|
payload["collection_id"] = _resolve_slug(coll_slug, "collections", index)
|
|
|
|
# dashcards: card slug -> card_id int + auto-inyeccion de campos nuevos
|
|
dashcards = payload.get("dashcards", []) or []
|
|
new_dashcards = []
|
|
next_neg_id = -1 # contador de ids negativos auto-asignados
|
|
used_neg_ids = {dc["id"] for dc in dashcards if isinstance(dc.get("id"), int) and dc["id"] < 0}
|
|
for i, dc in enumerate(dashcards):
|
|
dc = dict(dc)
|
|
card_slug = dc.pop("card", None)
|
|
if card_slug is not None:
|
|
try:
|
|
dc["card_id"] = _resolve_slug(card_slug, "cards", index)
|
|
except ValueError as e:
|
|
raise ValueError(f"dashcard #{i}: {e}") from None
|
|
# series: lista de slugs -> lista de {id}
|
|
series_slugs = dc.get("series") or []
|
|
if series_slugs:
|
|
dc["series"] = [
|
|
{"id": _resolve_slug(s, "cards", index)} for s in series_slugs
|
|
]
|
|
# auto-inyeccion: id negativo si no hay id (CREATE de dashcard)
|
|
if "id" not in dc:
|
|
while next_neg_id in used_neg_ids:
|
|
next_neg_id -= 1
|
|
dc["id"] = next_neg_id
|
|
used_neg_ids.add(next_neg_id)
|
|
next_neg_id -= 1
|
|
# auto-inyeccion: viz_settings y parameter_mappings vacios si faltan
|
|
dc.setdefault("visualization_settings", {})
|
|
dc.setdefault("parameter_mappings", [])
|
|
new_dashcards.append(dc)
|
|
payload["dashcards"] = new_dashcards
|
|
|
|
return payload
|
|
|
|
|
|
def build_database_payload(doc: dict, index: dict, env: dict) -> dict:
|
|
"""Construye payload de database. Resuelve passwords desde env vars.
|
|
|
|
Si details.password es ${VAR}, lo sustituye por env[VAR]. Si no esta,
|
|
deja el placeholder (push fallara con error claro).
|
|
"""
|
|
payload = copy.deepcopy(doc.get("payload", {}) or {})
|
|
details = payload.get("details", {}) or {}
|
|
pwd = details.get("password")
|
|
if isinstance(pwd, str) and pwd.startswith("${") and pwd.endswith("}"):
|
|
var_name = pwd[2:-1]
|
|
if var_name in env:
|
|
details["password"] = env[var_name]
|
|
# si no esta, queda el placeholder y push fallara
|
|
payload["details"] = details
|
|
return payload
|
|
|
|
|
|
def build_collection_payload(doc: dict, index: dict) -> dict:
|
|
refs = doc.get("_refs", {}) or {}
|
|
payload = copy.deepcopy(doc.get("payload", {}) or {})
|
|
if "parent" in refs:
|
|
parent_slug = refs["parent"]
|
|
payload["parent_id"] = _resolve_slug(parent_slug, "collections", index)
|
|
return payload
|
|
|
|
|
|
# ---------------------------------------------------------------- Dispatch
|
|
|
|
|
|
_BUILDERS = {
|
|
"card": build_card_payload,
|
|
"dashboard": build_dashboard_payload,
|
|
"database": build_database_payload,
|
|
"collection": build_collection_payload,
|
|
}
|
|
|
|
|
|
def build_payload(kind: str, doc: dict, index: dict, env: dict | None = None) -> dict:
|
|
"""Punto de entrada: dispatch por kind."""
|
|
if kind not in _BUILDERS:
|
|
raise ValueError(f"kind '{kind}' desconocido. Validos: {sorted(_BUILDERS)}")
|
|
if kind == "database":
|
|
return _BUILDERS[kind](doc, index, env or {})
|
|
return _BUILDERS[kind](doc, index)
|
|
|
|
|
|
def known_card_ids(index: dict) -> set[int]:
|
|
"""Set de ids de cards conocidas (para validacion de dashcards)."""
|
|
return set(index.get("cards", {}).values())
|
|
|
|
|
|
# ---------------------------------------------------------------- Lookup paths
|
|
|
|
|
|
def item_path(project_dir: Path, kind: str, slug: str) -> Path:
|
|
"""Path al YAML de un item: cards/foo.yaml, dashboards/bar.yaml, etc."""
|
|
return project_dir / (kind + "s") / f"{slug}.yaml"
|