feat(auto_metabase): push-all + describe/sql + auto-inject de dashcards
- push_all(): pushea todos los YAMLs de un proyecto (cards primero,
dashboards despues), solo CREATE/UPDATE, resiliente a fallos por item
- explore.py: comandos describe (schema de DB) y sql (query ad-hoc con
limite, cap 5MB, bloqueo de escrituras destructivas)
- payload.py: auto-inyecta id:-N, visualization_settings:{} y
parameter_mappings:[] en dashcards nuevas para evitar 500 en push
- test_local: 11 cards + 3 dashboards sobre Sample Database de Metabase
- registry.db regenerado con auto_metabase_py_analytics indexada
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,157 @@
|
||||
"""Restore desde backup.
|
||||
|
||||
Estructura de backups:
|
||||
state/backups/{YYYY-MM-DD_HHMMSS}/{cards|dashboards|...}/{slug}.yaml
|
||||
|
||||
Cada backup es un YAML con:
|
||||
_backup_of: {kind, slug, ts}
|
||||
remote_state: <payload tal cual estaba en Metabase justo antes del push>
|
||||
|
||||
Restore NO aplica automaticamente a Metabase. Solo escribe el remote_state
|
||||
de vuelta al YAML activo del item, en formato local (con _meta + _refs +
|
||||
payload). Despues el usuario debe hacer `push <kind> <slug> --apply` para
|
||||
aplicar.
|
||||
|
||||
Esto deja al usuario inspeccionar el resultado antes de aplicar.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as dt
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
from payload import item_path
|
||||
|
||||
|
||||
def list_backups(project, kind: str, slug: str) -> list[Path]:
|
||||
"""Lista todos los backups disponibles para un item, ordenados (mas recientes primero)."""
|
||||
backups_root = project.state_dir / "backups"
|
||||
if not backups_root.exists():
|
||||
return []
|
||||
candidates = []
|
||||
for ts_dir in sorted(backups_root.iterdir(), reverse=True):
|
||||
if not ts_dir.is_dir():
|
||||
continue
|
||||
bp = ts_dir / (kind + "s") / f"{slug}.yaml"
|
||||
if bp.exists():
|
||||
candidates.append(bp)
|
||||
return candidates
|
||||
|
||||
|
||||
def _utc_now_iso() -> str:
|
||||
return dt.datetime.now(dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
|
||||
def _id_to_slug(id_: int | None, mapping: dict[str, int]) -> str | None:
|
||||
if id_ is None:
|
||||
return None
|
||||
for slug, mid in mapping.items():
|
||||
if mid == id_:
|
||||
return slug
|
||||
return None
|
||||
|
||||
|
||||
def _strip_remote_to_local(kind: str, remote: dict, project) -> dict:
|
||||
"""Convierte un payload remoto en formato YAML local (_meta + _refs + payload).
|
||||
|
||||
Reusa la misma logica de transformacion que sync_pull.pull_one. Lo mas
|
||||
simple es importar las funciones de strip y reconstruir el doc.
|
||||
"""
|
||||
from sync_pull import _id_to_slug as _id_to_slug_fn, _strip_volatile
|
||||
|
||||
index = project.load_index()
|
||||
payload = _strip_volatile(remote)
|
||||
payload.pop("collection", None)
|
||||
|
||||
body: dict = {"_meta": {}, "_refs": {}, "payload": payload}
|
||||
body["_meta"] = {
|
||||
"kind": kind,
|
||||
"id": remote.get("id"),
|
||||
"slug": None, # se rellena despues
|
||||
"synced_at": _utc_now_iso(),
|
||||
"remote_updated_at": remote.get("updated_at"),
|
||||
"restored_from_backup": True,
|
||||
}
|
||||
|
||||
if kind == "card":
|
||||
db_slug = _id_to_slug_fn(remote.get("database_id"), index.get("databases", {}))
|
||||
coll_slug = _id_to_slug_fn(remote.get("collection_id"), index.get("collections", {}))
|
||||
body["_refs"] = {"database": db_slug, "collection": coll_slug}
|
||||
payload.pop("database_id", None)
|
||||
payload.pop("collection_id", None)
|
||||
if isinstance(payload.get("dataset_query"), dict) and "database" in payload["dataset_query"]:
|
||||
payload["dataset_query"]["database"] = db_slug
|
||||
|
||||
elif kind == "dashboard":
|
||||
coll_slug = _id_to_slug_fn(remote.get("collection_id"), index.get("collections", {}))
|
||||
body["_refs"] = {"collection": coll_slug}
|
||||
payload.pop("collection_id", None)
|
||||
# dashcards: card_id -> card slug
|
||||
cards_idx = index.get("cards", {})
|
||||
clean = []
|
||||
for dc in payload.get("dashcards", []) or []:
|
||||
dc = dict(dc)
|
||||
cid = dc.pop("card_id", None)
|
||||
dc.pop("card", None)
|
||||
dc.pop("dashboard_id", None)
|
||||
dc["card"] = _id_to_slug_fn(cid, cards_idx)
|
||||
series = dc.get("series") or []
|
||||
if series:
|
||||
dc["series"] = [_id_to_slug_fn(s.get("id") if isinstance(s, dict) else s, cards_idx) for s in series]
|
||||
clean.append({k: v for k, v in dc.items() if v not in (None, [], {})})
|
||||
payload["dashcards"] = clean
|
||||
body["_meta"]["dashcards_count"] = len(clean)
|
||||
body["_meta"]["tabs_count"] = len(payload.get("tabs", []) or [])
|
||||
body["_meta"]["parameters_count"] = len(payload.get("parameters", []) or [])
|
||||
|
||||
return body
|
||||
|
||||
|
||||
def restore_one(project, kind: str, slug: str, *, from_ts: str | None = None) -> Path:
|
||||
"""Restaura el YAML local desde un backup.
|
||||
|
||||
NO aplica a Metabase. Solo escribe el archivo de disco para que el
|
||||
usuario inspeccione y haga push --apply manualmente.
|
||||
"""
|
||||
backups = list_backups(project, kind, slug)
|
||||
if not backups:
|
||||
raise SystemExit(
|
||||
f"No hay backups para {kind} {slug} en {project.state_dir / 'backups'}"
|
||||
)
|
||||
|
||||
if from_ts is None:
|
||||
chosen = backups[0] # mas reciente
|
||||
else:
|
||||
matches = [b for b in backups if from_ts in str(b)]
|
||||
if not matches:
|
||||
raise SystemExit(
|
||||
f"No hay backup con timestamp '{from_ts}'. Disponibles:\n "
|
||||
+ "\n ".join(str(b.relative_to(project.dir.parent.parent)) for b in backups)
|
||||
)
|
||||
chosen = matches[0]
|
||||
|
||||
print(f"[{project.name}] restore {kind} {slug}")
|
||||
print(f" desde: {chosen.relative_to(project.dir.parent.parent)}")
|
||||
|
||||
with chosen.open() as f:
|
||||
backup_doc = yaml.safe_load(f) or {}
|
||||
remote_state = backup_doc.get("remote_state")
|
||||
if not remote_state:
|
||||
raise SystemExit(f"Backup corrupto: falta 'remote_state' en {chosen}")
|
||||
|
||||
body = _strip_remote_to_local(kind, remote_state, project)
|
||||
body["_meta"]["slug"] = slug
|
||||
body["_meta"]["restored_from"] = str(chosen.relative_to(project.dir.parent.parent))
|
||||
|
||||
target = item_path(project.dir, kind, slug)
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
with target.open("w") as f:
|
||||
yaml.safe_dump(body, f, sort_keys=False, allow_unicode=True, default_flow_style=False, width=120)
|
||||
|
||||
print(f" escrito en: {target.relative_to(project.dir.parent.parent)}")
|
||||
print(f"\n El backup quedo restaurado al disco. Para aplicarlo a Metabase:")
|
||||
print(f" python main.py push {kind} {slug} --apply")
|
||||
print(f" Antes, te recomiendo: cat {target.relative_to(project.dir.parent.parent)}")
|
||||
return target
|
||||
Reference in New Issue
Block a user