From 9095fe8c657fee0adc79ead3ac54644a4ac045a1 Mon Sep 17 00:00:00 2001 From: Egutierrez Date: Sun, 29 Mar 2026 00:54:18 +0100 Subject: [PATCH 1/3] feat: dashboard apps y mejora layout del dashboard Overview MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Dashboard fn-registry Apps con 10 cards: KPIs por lenguaje, dominio, framework, dependencias y catálogo completo. Cards del Overview ampliadas a grid de 24 columnas con tamaños más legibles. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../create_apps_dashboard.py | 219 ++++++++++++++++++ .../create_registry_dashboard.py | 43 ++-- 2 files changed, 247 insertions(+), 15 deletions(-) create mode 100644 apps/metabase_registry/create_apps_dashboard.py diff --git a/apps/metabase_registry/create_apps_dashboard.py b/apps/metabase_registry/create_apps_dashboard.py new file mode 100644 index 00000000..f9d0055d --- /dev/null +++ b/apps/metabase_registry/create_apps_dashboard.py @@ -0,0 +1,219 @@ +"""Crea un dashboard en Metabase con metricas de la tabla apps del fn-registry.""" + +import sys +import os + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "python", "functions")) + +from metabase.client import metabase_auth +from metabase import ( + metabase_list_databases, + metabase_create_card, + metabase_create_dashboard, + metabase_update_dashboard, + metabase_list_dashboards, +) + +# --- Config --- +METABASE_URL = "http://localhost:3000" +EMAIL = "admin@fnregistry.local" +PASSWORD = "FnRegistry2024!" + +# --- Layout --- +# Grid de 24 unidades de ancho (estandar Metabase). +# Fila 0 (h=5): 4 scalars de 6 unidades → KPIs +# Fila 5 (h=8): 3 graficas de 8 unidades → distribucion general +# Fila 13 (h=9): 2 graficas de 12 unidades → uso de funciones + frameworks +# Fila 22 (h=8): tabla completa de apps → catalogo completo + +CARDS = [ + # ---- Fila 0: KPIs escalares (h=5) ---- + { + "name": "Total de Apps", + "display": "scalar", + "sql": "SELECT COUNT(*) AS total FROM apps;", + "size_x": 6, "size_y": 5, "col": 0, "row": 0, + }, + { + "name": "Apps Go", + "display": "scalar", + "sql": "SELECT COUNT(*) AS apps_go FROM apps WHERE lang = 'go';", + "size_x": 6, "size_y": 5, "col": 6, "row": 0, + }, + { + "name": "Apps Python", + "display": "scalar", + "sql": "SELECT COUNT(*) AS apps_python FROM apps WHERE lang = 'py';", + "size_x": 6, "size_y": 5, "col": 12, "row": 0, + }, + { + "name": "Dominios con Apps", + "display": "scalar", + "sql": "SELECT COUNT(DISTINCT domain) AS dominios FROM apps;", + "size_x": 6, "size_y": 5, "col": 18, "row": 0, + }, + # ---- Fila 5: Distribucion general (h=8) ---- + { + "name": "Apps por Lenguaje", + "display": "bar", + "sql": "SELECT lang, COUNT(*) AS cantidad FROM apps GROUP BY lang ORDER BY cantidad DESC;", + "size_x": 8, "size_y": 8, "col": 0, "row": 5, + }, + { + "name": "Apps por Dominio", + "display": "pie", + "sql": "SELECT domain, COUNT(*) AS cantidad FROM apps GROUP BY domain ORDER BY cantidad DESC;", + "size_x": 8, "size_y": 8, "col": 8, "row": 5, + }, + { + "name": "Apps con Framework", + "display": "bar", + "sql": """ + SELECT + CASE WHEN framework = '' OR framework IS NULL THEN '(sin framework)' ELSE framework END AS framework, + COUNT(*) AS cantidad + FROM apps + GROUP BY framework + ORDER BY cantidad DESC; + """, + "size_x": 8, "size_y": 8, "col": 16, "row": 5, + }, + # ---- Fila 13: Analisis de dependencias (h=9) ---- + { + "name": "Apps con Mas Dependencias de Funciones", + "display": "row", + "sql": """ + SELECT + name || ' (' || lang || ')' AS app, + (LENGTH(uses_functions) - LENGTH(REPLACE(uses_functions, ',', '')) + + CASE WHEN uses_functions != '[]' AND uses_functions != '' THEN 1 ELSE 0 END) AS num_funciones_usadas + FROM apps + WHERE uses_functions != '[]' AND uses_functions != '' + ORDER BY num_funciones_usadas DESC + LIMIT 15; + """, + "size_x": 12, "size_y": 9, "col": 0, "row": 13, + }, + { + "name": "Funciones del Registry Mas Usadas en Apps", + "display": "row", + "sql": """ + WITH RECURSIVE split_uses(app_id, rest, val) AS ( + SELECT id, uses_functions || ',', NULL + FROM apps + WHERE uses_functions != '[]' AND uses_functions != '' + UNION ALL + SELECT app_id, + SUBSTR(rest, INSTR(rest, ',') + 1), + TRIM(SUBSTR(rest, 1, INSTR(rest, ',') - 1), ' "[]') + FROM split_uses WHERE rest != '' + ) + SELECT val AS funcion_usada, COUNT(*) AS veces_usada + FROM split_uses + WHERE val IS NOT NULL AND val != '' AND val != ']' + GROUP BY val + ORDER BY veces_usada DESC + LIMIT 15; + """, + "size_x": 12, "size_y": 9, "col": 12, "row": 13, + }, + # ---- Fila 22: Catalogo completo (h=9) ---- + { + "name": "Catalogo de Apps", + "display": "table", + "sql": """ + SELECT + id, + name, + lang, + domain, + CASE WHEN framework = '' THEN '-' ELSE framework END AS framework, + description, + entry_point, + updated_at + FROM apps + ORDER BY domain, name; + """, + "size_x": 24, "size_y": 9, "col": 0, "row": 22, + }, +] + + +def main(): + print("Autenticando en Metabase...") + client = metabase_auth(METABASE_URL, EMAIL, PASSWORD) + + # Encontrar la database registry.db + dbs = metabase_list_databases(client) + registry_db_id = None + for db in dbs: + if "registry" in db.get("name", "").lower() or ( + db.get("engine") == "sqlite" + and "registry" in db.get("details", {}).get("db", "") + ): + registry_db_id = db["id"] + print(f" Database encontrada: {db['name']} (id={db['id']})") + break + + if not registry_db_id: + print("ERROR: No se encontro registry.db en Metabase.") + print("Databases disponibles:") + for db in dbs: + print(f" - {db['id']}: {db['name']} ({db['engine']})") + sys.exit(1) + + # Verificar si ya existe un dashboard con este nombre + existing = metabase_list_dashboards(client) + for d in existing: + if d.get("name") == "fn-registry Apps": + print(f" Dashboard ya existe (id={d['id']}), recreando...") + from metabase import metabase_delete_dashboard + metabase_delete_dashboard(client, d["id"]) + + # Crear cards + print("Creando cards...") + created_cards = [] + for i, card_def in enumerate(CARDS): + card = metabase_create_card( + client, + name=card_def["name"], + dataset_query={ + "database": registry_db_id, + "type": "native", + "native": {"query": card_def["sql"]}, + }, + display=card_def["display"], + description=f"fn-registry apps: {card_def['name']}", + ) + created_cards.append((card, card_def)) + print(f" [{i+1}/{len(CARDS)}] {card_def['name']} (id={card['id']})") + + # Crear dashboard + print("Creando dashboard...") + dashboard = metabase_create_dashboard( + client, + name="fn-registry Apps", + description="Dashboard de apps del registry: distribucion por lenguaje, dominio, dependencias y catalogo completo.", + ) + dash_id = dashboard["id"] + print(f" Dashboard creado: id={dash_id}") + + # Agregar cards al dashboard con posiciones + dashcards = [] + for idx, (card, card_def) in enumerate(created_cards): + dashcards.append({ + "id": -(idx + 1), + "card_id": card["id"], + "size_x": card_def["size_x"], + "size_y": card_def["size_y"], + "col": card_def["col"], + "row": card_def["row"], + }) + + metabase_update_dashboard(client, dash_id, dashcards=dashcards) + print(f"\nDashboard listo: {METABASE_URL}/dashboard/{dash_id}") + client.close() + + +if __name__ == "__main__": + main() diff --git a/apps/metabase_registry/create_registry_dashboard.py b/apps/metabase_registry/create_registry_dashboard.py index 5cf556d5..fc2f6154 100644 --- a/apps/metabase_registry/create_registry_dashboard.py +++ b/apps/metabase_registry/create_registry_dashboard.py @@ -19,61 +19,72 @@ METABASE_URL = "http://localhost:3000" EMAIL = "admin@fnregistry.local" PASSWORD = "FnRegistry2024!" +# --- Layout --- +# Grid de 24 unidades de ancho (estandar Metabase). +# Fila 0 (h=5): 5 scalars de 4-5 unidades cada uno → fila de KPIs +# Fila 5 (h=8): 3 graficas de 8 unidades → distribucion general +# Fila 13 (h=9): 3 graficas de 8 unidades → analisis profundo +# Fila 22 (h=8): 2 tablas de 12 unidades → cobertura + x lenguaje +# Fila 30 (h=8): 2 tablas de 12 unidades → tipos + recientes + # --- SQL Queries --- CARDS = [ + # ---- Fila 0: KPIs escalares (h=5) ---- { "name": "Total de Funciones", "display": "scalar", "sql": "SELECT COUNT(*) AS total FROM functions;", - "size_x": 4, "size_y": 3, "col": 0, "row": 0, + "size_x": 5, "size_y": 5, "col": 0, "row": 0, }, { "name": "Funciones con Tests", "display": "scalar", "sql": "SELECT COUNT(*) AS con_tests FROM functions WHERE tested = 1;", - "size_x": 4, "size_y": 3, "col": 4, "row": 0, + "size_x": 5, "size_y": 5, "col": 5, "row": 0, }, { "name": "Funciones sin Tests", "display": "scalar", "sql": "SELECT COUNT(*) AS sin_tests FROM functions WHERE tested = 0;", - "size_x": 4, "size_y": 3, "col": 8, "row": 0, + "size_x": 4, "size_y": 5, "col": 10, "row": 0, }, { "name": "Total de Tipos", "display": "scalar", "sql": "SELECT COUNT(*) AS total FROM types;", - "size_x": 3, "size_y": 3, "col": 12, "row": 0, + "size_x": 5, "size_y": 5, "col": 14, "row": 0, }, { "name": "Proposals Pendientes", "display": "scalar", "sql": "SELECT COUNT(*) AS pendientes FROM proposals WHERE status = 'pending';", - "size_x": 3, "size_y": 3, "col": 15, "row": 0, + "size_x": 5, "size_y": 5, "col": 19, "row": 0, }, + # ---- Fila 5: Distribucion general (h=8) ---- { "name": "Funciones por Lenguaje", "display": "bar", "sql": "SELECT lang, COUNT(*) AS cantidad FROM functions GROUP BY lang ORDER BY cantidad DESC;", - "size_x": 6, "size_y": 5, "col": 0, "row": 3, + "size_x": 8, "size_y": 8, "col": 0, "row": 5, }, { "name": "Funciones por Dominio", "display": "pie", "sql": "SELECT domain, COUNT(*) AS cantidad FROM functions GROUP BY domain ORDER BY cantidad DESC;", - "size_x": 6, "size_y": 5, "col": 6, "row": 3, + "size_x": 8, "size_y": 8, "col": 8, "row": 5, }, { "name": "Funciones por Kind", "display": "bar", "sql": "SELECT kind, COUNT(*) AS cantidad FROM functions GROUP BY kind ORDER BY cantidad DESC;", - "size_x": 6, "size_y": 5, "col": 12, "row": 3, + "size_x": 8, "size_y": 8, "col": 16, "row": 5, }, + # ---- Fila 13: Analisis profundo (h=9) ---- { "name": "Puras vs Impuras", "display": "pie", "sql": "SELECT purity, COUNT(*) AS cantidad FROM functions GROUP BY purity ORDER BY cantidad DESC;", - "size_x": 6, "size_y": 5, "col": 0, "row": 8, + "size_x": 8, "size_y": 9, "col": 0, "row": 13, }, { "name": "Funciones Mas Usadas por Otras", @@ -94,7 +105,7 @@ CARDS = [ ORDER BY veces_usada DESC LIMIT 15; """, - "size_x": 6, "size_y": 5, "col": 6, "row": 8, + "size_x": 8, "size_y": 9, "col": 8, "row": 13, }, { "name": "Funciones Mas Complejas (mas dependencias)", @@ -111,8 +122,9 @@ CARDS = [ ORDER BY num_dependencias DESC LIMIT 15; """, - "size_x": 6, "size_y": 5, "col": 12, "row": 8, + "size_x": 8, "size_y": 9, "col": 16, "row": 13, }, + # ---- Fila 22: Cobertura + cross-table (h=8) ---- { "name": "Cobertura de Tests por Dominio", "display": "bar", @@ -125,7 +137,7 @@ CARDS = [ GROUP BY domain ORDER BY domain; """, - "size_x": 9, "size_y": 5, "col": 0, "row": 13, + "size_x": 12, "size_y": 8, "col": 0, "row": 22, }, { "name": "Funciones por Lenguaje y Dominio", @@ -142,8 +154,9 @@ CARDS = [ GROUP BY domain ORDER BY total DESC; """, - "size_x": 9, "size_y": 5, "col": 9, "row": 13, + "size_x": 12, "size_y": 8, "col": 12, "row": 22, }, + # ---- Fila 30: Tipos + recientes (h=8) ---- { "name": "Tipos por Dominio y Algebraic", "display": "table", @@ -156,7 +169,7 @@ CARDS = [ GROUP BY domain, algebraic ORDER BY domain, cantidad DESC; """, - "size_x": 9, "size_y": 4, "col": 0, "row": 18, + "size_x": 12, "size_y": 8, "col": 0, "row": 30, }, { "name": "Funciones Recientes (ultimas 20 indexadas)", @@ -167,7 +180,7 @@ CARDS = [ ORDER BY created_at DESC LIMIT 20; """, - "size_x": 9, "size_y": 4, "col": 9, "row": 18, + "size_x": 12, "size_y": 8, "col": 12, "row": 30, }, ] From dd324b7785e1543cc9c458fccd00fe8b71f8ccc9 Mon Sep 17 00:00:00 2001 From: Egutierrez Date: Sun, 29 Mar 2026 00:54:24 +0100 Subject: [PATCH 2/3] =?UTF-8?q?feat:=20pipelines=20Metabase=20=E2=80=94=20?= =?UTF-8?q?add=20ops=20db,=20create=20ops=20dashboard,=20fix=20permissions?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tres pipelines Python para gestionar operations.db en Metabase: - metabase_add_ops_db: registra la operations.db de una app como database SQLite - metabase_create_ops_dashboard: genera dashboard operativo con 14 cards (KPIs, distribución, executions, assertions) para cualquier app - metabase_fix_permissions: arregla SQLITE_READONLY_DIRECTORY haciendo chmod 777/666 sin chown (que se propaga al host via bind mount) Co-Authored-By: Claude Opus 4.6 (1M context) --- python/functions/pipelines/__init__.py | 0 .../pipelines/metabase_add_ops_db.md | 59 ++++ .../pipelines/metabase_add_ops_db.py | 116 +++++++ .../metabase_create_ops_dashboard.md | 68 ++++ .../metabase_create_ops_dashboard.py | 291 ++++++++++++++++++ .../pipelines/metabase_fix_permissions.md | 58 ++++ .../pipelines/metabase_fix_permissions.py | 152 +++++++++ 7 files changed, 744 insertions(+) create mode 100644 python/functions/pipelines/__init__.py create mode 100644 python/functions/pipelines/metabase_add_ops_db.md create mode 100644 python/functions/pipelines/metabase_add_ops_db.py create mode 100644 python/functions/pipelines/metabase_create_ops_dashboard.md create mode 100644 python/functions/pipelines/metabase_create_ops_dashboard.py create mode 100644 python/functions/pipelines/metabase_fix_permissions.md create mode 100644 python/functions/pipelines/metabase_fix_permissions.py diff --git a/python/functions/pipelines/__init__.py b/python/functions/pipelines/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/functions/pipelines/metabase_add_ops_db.md b/python/functions/pipelines/metabase_add_ops_db.md new file mode 100644 index 00000000..7914967a --- /dev/null +++ b/python/functions/pipelines/metabase_add_ops_db.md @@ -0,0 +1,59 @@ +--- +name: metabase_add_ops_db +kind: pipeline +lang: py +domain: pipelines +version: "1.0.0" +purity: impure +signature: "metabase_add_ops_db(app_name: str) -> None" +description: "Registra la operations.db de una app en Metabase como database SQLite. Verifica duplicados y muestra el mount necesario para el contenedor Docker." +tags: [metabase, operations, sqlite, docker, pipeline, infra, launcher] +uses_functions: + - metabase_auth_py_infra + - metabase_list_databases_py_infra + - metabase_add_database_py_infra +uses_types: [] +returns: [] +returns_optional: false +error_type: "error_go_core" +imports: [] +tested: false +tests: [] +test_file_path: "" +file_path: "python/functions/pipelines/metabase_add_ops_db.py" +--- + +## Ejemplo + +```bash +# Registrar una app +python python/functions/pipelines/metabase_add_ops_db.py docker_tui + +# Listar estado de todas las apps +python python/functions/pipelines/metabase_add_ops_db.py --list +``` + +## Flujo + +1. `metabase_auth` - autentica contra Metabase +2. `metabase_list_databases` - verifica si la database ya existe +3. `metabase_add_database` - registra la operations.db como SQLite + +## Requisitos + +El contenedor Metabase debe tener montado el directorio de la app como volumen RW: + +``` +-v /home/lucas/fn_registry/apps/:/data/ops- +``` + +Ademas, el directorio debe tener permisos para el usuario metabase (UID 2000): + +```bash +docker exec -u root chown -R metabase:metabase /data/ops-/ +``` + +## Convencion de nombres + +- Database en Metabase: `ops-` (guiones en vez de underscores) +- Path en contenedor: `/data/ops-/operations.db` diff --git a/python/functions/pipelines/metabase_add_ops_db.py b/python/functions/pipelines/metabase_add_ops_db.py new file mode 100644 index 00000000..837b7fff --- /dev/null +++ b/python/functions/pipelines/metabase_add_ops_db.py @@ -0,0 +1,116 @@ +"""Pipeline: registra la operations.db de una app en Metabase. + +Compone: metabase_auth + metabase_list_databases + metabase_add_database. + +Requiere que el contenedor Metabase tenga montado el directorio de la app +como volumen RW para que SQLite pueda crear journal files. + +Uso: + python metabase_add_ops_db.py + python metabase_add_ops_db.py docker_tui + python metabase_add_ops_db.py --list +""" + +import argparse +import os +import sys + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) + +from metabase.client import metabase_auth +from metabase import metabase_list_databases, metabase_add_database + +METABASE_URL = os.environ.get("METABASE_URL", "http://localhost:3000") +EMAIL = os.environ.get("METABASE_ADMIN_EMAIL", "admin@fnregistry.local") +PASSWORD = os.environ.get("METABASE_ADMIN_PASSWORD", "FnRegistry2024!") + +CONTAINER_DATA_PREFIX = "/data/ops-" + + +def find_apps(registry_root: str) -> list[str]: + """Lista apps que tienen operations.db.""" + apps_dir = os.path.join(registry_root, "apps") + if not os.path.isdir(apps_dir): + return [] + return sorted( + d + for d in os.listdir(apps_dir) + if os.path.isfile(os.path.join(apps_dir, d, "operations.db")) + ) + + +def db_name_for_app(app_name: str) -> str: + return f"ops-{app_name.replace('_', '-')}" + + +def container_path_for_app(app_name: str) -> str: + return f"{CONTAINER_DATA_PREFIX}{app_name.replace('_', '-')}/operations.db" + + +def main(): + parser = argparse.ArgumentParser(description="Registra operations.db de una app en Metabase") + parser.add_argument("app_name", nargs="?", help="Nombre de la app (directorio en apps/)") + parser.add_argument("--list", action="store_true", help="Lista apps disponibles y su estado en Metabase") + args = parser.parse_args() + + registry_root = os.environ.get( + "FN_REGISTRY_ROOT", + os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")), + ) + + if args.list: + client = metabase_auth(METABASE_URL, EMAIL, PASSWORD) + dbs = metabase_list_databases(client) + db_names = {d["name"] for d in dbs} + apps = find_apps(registry_root) + print(f"Apps con operations.db ({len(apps)}):\n") + for app in apps: + name = db_name_for_app(app) + status = "registrada" if name in db_names else "NO registrada" + print(f" {app:30s} -> {name:30s} [{status}]") + client.close() + return + + if not args.app_name: + parser.error("Se requiere app_name o --list") + + app_name = args.app_name + ops_path = os.path.join(registry_root, "apps", app_name, "operations.db") + if not os.path.isfile(ops_path): + print(f"ERROR: No existe {ops_path}") + print(f"Apps disponibles: {', '.join(find_apps(registry_root))}") + sys.exit(1) + + name = db_name_for_app(app_name) + container_path = container_path_for_app(app_name) + + print(f"Registrando {app_name} en Metabase...") + print(f" Nombre: {name}") + print(f" Path: {container_path}") + + client = metabase_auth(METABASE_URL, EMAIL, PASSWORD) + + # Verificar si ya existe + dbs = metabase_list_databases(client) + for db in dbs: + if db["name"] == name: + print(f" Ya existe con id={db['id']}, nada que hacer.") + client.close() + return + + # Registrar + result = metabase_add_database(client, name, "sqlite", {"db": container_path}) + db_id = result["id"] + print(f" Registrada con id={db_id}") + + # Recordar: el contenedor necesita el volumen montado + print(f"\nIMPORTANTE: El contenedor Metabase debe tener montado:") + print(f" -v {os.path.join(registry_root, 'apps', app_name)}:{CONTAINER_DATA_PREFIX}{app_name.replace('_', '-')}") + print(f"\nSi el mount no existe, hay que recrear el contenedor.") + print(f"\nDatabase disponible en: {METABASE_URL}/question#new?database={db_id}") + + client.close() + + +if __name__ == "__main__": + main() diff --git a/python/functions/pipelines/metabase_create_ops_dashboard.md b/python/functions/pipelines/metabase_create_ops_dashboard.md new file mode 100644 index 00000000..f6424c66 --- /dev/null +++ b/python/functions/pipelines/metabase_create_ops_dashboard.md @@ -0,0 +1,68 @@ +--- +name: metabase_create_ops_dashboard +kind: pipeline +lang: py +domain: pipelines +version: "1.0.0" +purity: impure +signature: "metabase_create_ops_dashboard(app_name: str) -> None" +description: "Crea dashboard operativo en Metabase para una app: KPIs de entities/relations/executions/assertions, distribucion por status y tipo, relaciones frecuentes, resultados de ejecuciones y assertions." +tags: [metabase, operations, dashboard, pipeline, infra, launcher] +uses_functions: + - metabase_auth_py_infra + - metabase_list_databases_py_infra + - metabase_create_card_py_infra + - metabase_create_dashboard_py_infra + - metabase_update_dashboard_py_infra + - metabase_list_dashboards_py_infra + - metabase_delete_dashboard_py_infra +uses_types: [] +returns: [] +returns_optional: false +error_type: "error_go_core" +imports: [] +tested: false +tests: [] +test_file_path: "" +file_path: "python/functions/pipelines/metabase_create_ops_dashboard.py" +--- + +## Ejemplo + +```bash +# Dashboard para una app +python python/functions/pipelines/metabase_create_ops_dashboard.py docker_tui + +# Dashboards para todas las apps registradas +python python/functions/pipelines/metabase_create_ops_dashboard.py --all +``` + +## Cards del dashboard + +| Fila | Cards | Tipo | +|------|-------|------| +| 0 | Entities, Relations, Executions, Assertions | scalar (KPIs) | +| 5 | Entities por Status, Entities por Tipo, Relations por Status | pie/bar | +| 13 | Executions Success/Failure, Assertions por Severity, Assertion Results | pie/bar | +| 22 | Top Relaciones (via), Executions Recientes | row/table | +| 30 | Lista de Entities | table | +| 38 | Assertion Results Detalle | table | + +## Flujo + +1. `metabase_auth` - autentica contra Metabase +2. `metabase_list_databases` - busca la database operacional de la app +3. `metabase_create_card` x 14 - crea las cards con queries SQL nativas +4. `metabase_create_dashboard` - crea el dashboard +5. `metabase_update_dashboard` - posiciona las cards en el grid + +## Requisitos + +La database operacional debe estar registrada previamente con `metabase_add_ops_db`. + +Las tablas `executions`, `assertions` y `assertion_results` requieren la migracion 002. Si no existen, las cards correspondientes mostraran error (no rompen el dashboard). + +## Convencion + +- Dashboard name: `ops: ` +- Si el dashboard ya existe, se elimina y recrea. diff --git a/python/functions/pipelines/metabase_create_ops_dashboard.py b/python/functions/pipelines/metabase_create_ops_dashboard.py new file mode 100644 index 00000000..b3a5f2ff --- /dev/null +++ b/python/functions/pipelines/metabase_create_ops_dashboard.py @@ -0,0 +1,291 @@ +"""Pipeline: crea dashboard operativo en Metabase para una app. + +Compone: metabase_auth + metabase_list_databases + metabase_create_card + + metabase_create_dashboard + metabase_update_dashboard. + +Genera un dashboard estandar con KPIs, distribucion y tablas sobre la +operations.db de una app: entities, relations, executions, assertions. + +Uso: + python metabase_create_ops_dashboard.py + python metabase_create_ops_dashboard.py docker_tui + python metabase_create_ops_dashboard.py --all +""" + +import argparse +import os +import sys + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) + +from metabase.client import metabase_auth +from metabase import ( + metabase_list_databases, + metabase_create_card, + metabase_create_dashboard, + metabase_update_dashboard, + metabase_list_dashboards, + metabase_delete_dashboard, +) + +METABASE_URL = os.environ.get("METABASE_URL", "http://localhost:3000") +EMAIL = os.environ.get("METABASE_ADMIN_EMAIL", "admin@fnregistry.local") +PASSWORD = os.environ.get("METABASE_ADMIN_PASSWORD", "FnRegistry2024!") + + +def dashboard_name(app_name: str) -> str: + return f"ops: {app_name.replace('_', '-')}" + + +def db_name_for_app(app_name: str) -> str: + return f"ops-{app_name.replace('_', '-')}" + + +def build_cards(app_name: str) -> list[dict]: + """Define las cards del dashboard operativo.""" + return [ + # ---- Fila 0: KPIs (h=5) ---- + { + "name": f"[{app_name}] Entities", + "display": "scalar", + "sql": "SELECT COUNT(*) FROM entities;", + "size_x": 6, "size_y": 5, "col": 0, "row": 0, + }, + { + "name": f"[{app_name}] Relations", + "display": "scalar", + "sql": "SELECT COUNT(*) FROM relations;", + "size_x": 6, "size_y": 5, "col": 6, "row": 0, + }, + { + "name": f"[{app_name}] Executions", + "display": "scalar", + "sql": "SELECT COUNT(*) FROM executions;", + "size_x": 6, "size_y": 5, "col": 12, "row": 0, + }, + { + "name": f"[{app_name}] Assertions", + "display": "scalar", + "sql": "SELECT COUNT(*) FROM assertions;", + "size_x": 6, "size_y": 5, "col": 18, "row": 0, + }, + # ---- Fila 5: Distribucion (h=8) ---- + { + "name": f"[{app_name}] Entities por Status", + "display": "pie", + "sql": "SELECT status, COUNT(*) AS cantidad FROM entities GROUP BY status ORDER BY cantidad DESC;", + "size_x": 8, "size_y": 8, "col": 0, "row": 5, + }, + { + "name": f"[{app_name}] Entities por Tipo", + "display": "bar", + "sql": "SELECT type_ref, COUNT(*) AS cantidad FROM entities GROUP BY type_ref ORDER BY cantidad DESC;", + "size_x": 8, "size_y": 8, "col": 8, "row": 5, + }, + { + "name": f"[{app_name}] Relations por Status", + "display": "pie", + "sql": "SELECT status, COUNT(*) AS cantidad FROM relations GROUP BY status ORDER BY cantidad DESC;", + "size_x": 8, "size_y": 8, "col": 16, "row": 5, + }, + # ---- Fila 13: Executions y Assertions (h=9) ---- + { + "name": f"[{app_name}] Executions: Success vs Failure", + "display": "pie", + "sql": "SELECT status, COUNT(*) AS cantidad FROM executions GROUP BY status ORDER BY cantidad DESC;", + "size_x": 8, "size_y": 9, "col": 0, "row": 13, + }, + { + "name": f"[{app_name}] Assertions por Severity", + "display": "bar", + "sql": "SELECT severity, COUNT(*) AS cantidad FROM assertions GROUP BY severity ORDER BY cantidad DESC;", + "size_x": 8, "size_y": 9, "col": 8, "row": 13, + }, + { + "name": f"[{app_name}] Assertion Results", + "display": "pie", + "sql": "SELECT status, COUNT(*) AS cantidad FROM assertion_results GROUP BY status ORDER BY cantidad DESC;", + "size_x": 8, "size_y": 9, "col": 16, "row": 13, + }, + # ---- Fila 22: Relaciones frecuentes (h=8) ---- + { + "name": f"[{app_name}] Top Relaciones (via)", + "display": "row", + "sql": """ + SELECT + CASE WHEN via = '' THEN '(directo)' ELSE via END AS via_funcion, + COUNT(*) AS cantidad + FROM relations + GROUP BY via + ORDER BY cantidad DESC + LIMIT 15; + """, + "size_x": 12, "size_y": 8, "col": 0, "row": 22, + }, + { + "name": f"[{app_name}] Executions Recientes", + "display": "table", + "sql": """ + SELECT + id, + pipeline_id, + status, + duration_ms, + records_in, + records_out, + CASE WHEN error = '' THEN '-' ELSE error END AS error, + started_at + FROM executions + ORDER BY started_at DESC + LIMIT 20; + """, + "size_x": 12, "size_y": 8, "col": 12, "row": 22, + }, + # ---- Fila 30: Tablas detalle (h=8) ---- + { + "name": f"[{app_name}] Lista de Entities", + "display": "table", + "sql": """ + SELECT + id, + name, + type_ref, + status, + domain, + source, + created_at, + updated_at + FROM entities + ORDER BY updated_at DESC; + """, + "size_x": 24, "size_y": 8, "col": 0, "row": 30, + }, + { + "name": f"[{app_name}] Assertion Results Detalle", + "display": "table", + "sql": """ + SELECT + ar.id, + a.name AS assertion, + a.kind, + a.severity, + ar.status, + ar.value, + ar.message, + ar.evaluated_at + FROM assertion_results ar + JOIN assertions a ON a.id = ar.assertion_id + ORDER BY ar.evaluated_at DESC + LIMIT 50; + """, + "size_x": 24, "size_y": 8, "col": 0, "row": 38, + }, + ] + + +def create_ops_dashboard(client, app_name: str, db_id: int) -> int: + """Crea el dashboard operativo para una app. Retorna dashboard_id.""" + dash_name = dashboard_name(app_name) + + # Eliminar dashboard existente si hay + for d in metabase_list_dashboards(client): + if d.get("name") == dash_name: + print(f" Dashboard existente (id={d['id']}), recreando...") + metabase_delete_dashboard(client, d["id"]) + + cards_def = build_cards(app_name) + + print(f" Creando {len(cards_def)} cards...") + created = [] + for i, card_def in enumerate(cards_def): + card = metabase_create_card( + client, + name=card_def["name"], + dataset_query={ + "database": db_id, + "type": "native", + "native": {"query": card_def["sql"]}, + }, + display=card_def["display"], + description=f"ops dashboard: {app_name}", + ) + created.append((card, card_def)) + print(f" [{i+1}/{len(cards_def)}] {card_def['name']} (id={card['id']})") + + print(" Creando dashboard...") + dashboard = metabase_create_dashboard( + client, + name=dash_name, + description=f"Dashboard operativo de {app_name}: entities, relations, executions, assertions.", + ) + dash_id = dashboard["id"] + + dashcards = [] + for idx, (card, card_def) in enumerate(created): + dashcards.append({ + "id": -(idx + 1), + "card_id": card["id"], + "size_x": card_def["size_x"], + "size_y": card_def["size_y"], + "col": card_def["col"], + "row": card_def["row"], + }) + + metabase_update_dashboard(client, dash_id, dashcards=dashcards) + return dash_id + + +def main(): + parser = argparse.ArgumentParser(description="Crea dashboard operativo en Metabase para una app") + parser.add_argument("app_name", nargs="?", help="Nombre de la app") + parser.add_argument("--all", action="store_true", help="Crea dashboards para todas las apps registradas en Metabase") + args = parser.parse_args() + + if not args.app_name and not args.all: + parser.error("Se requiere app_name o --all") + + print("Autenticando en Metabase...") + client = metabase_auth(METABASE_URL, EMAIL, PASSWORD) + dbs = metabase_list_databases(client) + + if args.all: + ops_dbs = [ + (db["name"].replace("ops-", "").replace("-", "_"), db["id"]) + for db in dbs + if db["name"].startswith("ops-") + ] + if not ops_dbs: + print("No hay databases operacionales registradas en Metabase.") + sys.exit(1) + + print(f"Creando dashboards para {len(ops_dbs)} apps...\n") + for app_name, db_id in ops_dbs: + print(f"--- {app_name} (db_id={db_id}) ---") + dash_id = create_ops_dashboard(client, app_name, db_id) + print(f" Dashboard: {METABASE_URL}/dashboard/{dash_id}\n") + else: + app_name = args.app_name + expected_db_name = f"ops-{app_name.replace('_', '-')}" + db_id = None + for db in dbs: + if db["name"] == expected_db_name: + db_id = db["id"] + break + + if not db_id: + print(f"ERROR: Database '{expected_db_name}' no encontrada en Metabase.") + print("Databases disponibles:") + for db in dbs: + print(f" - {db['name']} (id={db['id']})") + print(f"\nRegistra primero con: python metabase_add_ops_db.py {app_name}") + sys.exit(1) + + print(f"App: {app_name} (db_id={db_id})") + dash_id = create_ops_dashboard(client, app_name, db_id) + print(f"\nDashboard listo: {METABASE_URL}/dashboard/{dash_id}") + + client.close() + + +if __name__ == "__main__": + main() diff --git a/python/functions/pipelines/metabase_fix_permissions.md b/python/functions/pipelines/metabase_fix_permissions.md new file mode 100644 index 00000000..52801630 --- /dev/null +++ b/python/functions/pipelines/metabase_fix_permissions.md @@ -0,0 +1,58 @@ +--- +name: metabase_fix_permissions +kind: pipeline +lang: py +domain: pipelines +version: "1.0.0" +purity: impure +signature: "metabase_fix_permissions() -> None" +description: "Arregla permisos SQLITE_READONLY_DIRECTORY en el contenedor Metabase. Hace chmod 777/666 en directorios y archivos .db bajo /data/ para que el usuario metabase (UID 2000) pueda crear journal files." +tags: [metabase, sqlite, permissions, docker, pipeline, infra, launcher] +uses_functions: + - metabase_auth_py_infra + - metabase_list_databases_py_infra +uses_types: [] +returns: [] +returns_optional: false +error_type: "error_go_core" +imports: [] +tested: false +tests: [] +test_file_path: "" +file_path: "python/functions/pipelines/metabase_fix_permissions.py" +--- + +## Ejemplo + +```bash +# Fix permisos (usa contenedor por defecto: fn_registry-metabase) +python python/functions/pipelines/metabase_fix_permissions.py + +# Contenedor custom +python python/functions/pipelines/metabase_fix_permissions.py --container mi-metabase +``` + +## Problema + +Metabase corre Java como UID 2000 (usuario `metabase`). SQLite necesita crear +journal/WAL files en el mismo directorio que la BD. Si el directorio no es +escribible por UID 2000, Metabase devuelve: + +``` +SQLITE_READONLY_DIRECTORY: Process does not have permission to create a +journal file in the same directory as the database +``` + +## Solucion + +El pipeline ejecuta dentro del contenedor (como root): +- `chmod 777` en cada directorio que contiene un `.db` +- `chmod 666` en cada archivo `.db`, `-wal`, `-shm`, `-journal` + +Esto NO cambia ownership (evita que `chown` se propague al host via bind mount). + +## Cuando ejecutar + +- Despues de recrear el contenedor Metabase +- Despues de añadir una nueva database con `metabase_add_ops_db` +- Cuando Metabase muestra error SQLITE_READONLY_DIRECTORY diff --git a/python/functions/pipelines/metabase_fix_permissions.py b/python/functions/pipelines/metabase_fix_permissions.py new file mode 100644 index 00000000..60eef2ee --- /dev/null +++ b/python/functions/pipelines/metabase_fix_permissions.py @@ -0,0 +1,152 @@ +"""Pipeline: arregla permisos de SQLite en el contenedor Metabase. + +Compone: metabase_auth + metabase_list_databases (para verificar). + +SQLite necesita escribir journal/WAL files en el mismo directorio que la BD. +El contenedor Metabase corre Java como UID 2000 (usuario metabase). +Si los directorios o archivos no son escribibles por ese UID, Metabase +devuelve SQLITE_READONLY_DIRECTORY. + +Este pipeline: +1. Detecta el contenedor Metabase +2. Encuentra todos los directorios /data/ que contienen .db +3. Hace chmod 777 en directorios y chmod 666 en archivos .db +4. Verifica que Metabase puede leer cada database + +Uso: + python metabase_fix_permissions.py + python metabase_fix_permissions.py --container fn_registry-metabase +""" + +import argparse +import os +import subprocess +import sys + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) + +from metabase.client import metabase_auth +from metabase import metabase_list_databases + +METABASE_URL = os.environ.get("METABASE_URL", "http://localhost:3000") +EMAIL = os.environ.get("METABASE_ADMIN_EMAIL", "admin@fnregistry.local") +PASSWORD = os.environ.get("METABASE_ADMIN_PASSWORD", "FnRegistry2024!") +DEFAULT_CONTAINER = "fn_registry-metabase" + + +def docker_exec(container: str, cmd: list[str], user: str = "root") -> str: + """Ejecuta comando dentro del contenedor.""" + result = subprocess.run( + ["docker", "exec", "-u", user, container] + cmd, + capture_output=True, text=True, + ) + return result.stdout.strip() + + +def fix_container_permissions(container: str) -> list[str]: + """Arregla permisos de todos los .db en /data/ del contenedor. + + Returns: + Lista de paths arreglados. + """ + # Encontrar todos los .db bajo /data/ + find_output = docker_exec(container, ["find", "/data", "-name", "*.db", "-type", "f"]) + if not find_output: + print(" No se encontraron archivos .db en /data/") + return [] + + db_files = [f.strip() for f in find_output.splitlines() if f.strip()] + fixed = [] + + for db_path in db_files: + db_dir = os.path.dirname(db_path) + + # Fix directorio: 777 para que UID 2000 pueda crear journal + docker_exec(container, ["chmod", "777", db_dir]) + + # Fix archivo: 666 para que UID 2000 pueda leer/escribir + docker_exec(container, ["chmod", "666", db_path]) + + # Fix WAL/SHM si existen + for suffix in ["-wal", "-shm", "-journal"]: + wal_path = db_path + suffix + docker_exec(container, ["chmod", "666", wal_path]) + + fixed.append(db_path) + print(f" Fixed: {db_path} (dir: {db_dir})") + + return fixed + + +def verify_databases(client) -> bool: + """Verifica que todas las databases SQLite responden.""" + import httpx + + dbs = metabase_list_databases(client) + all_ok = True + + for db in dbs: + if db.get("engine") != "sqlite": + continue + + name = db["name"] + db_id = db["id"] + + try: + resp = client._http.request("POST", "/api/dataset", json={ + "database": db_id, + "type": "native", + "native": {"query": "SELECT 1"}, + }, extensions={"timeout": {"read": 60, "write": 10, "connect": 10, "pool": 10}}) + + if resp.status_code in (200, 202): + print(f" OK: {name} (id={db_id})") + else: + error = resp.text[:150] + print(f" FAIL: {name} (id={db_id}) -> {error}") + all_ok = False + except httpx.ReadTimeout: + print(f" TIMEOUT: {name} (id={db_id}) -> Metabase aun sincronizando, reintentar en unos segundos") + all_ok = False + + return all_ok + + +def main(): + parser = argparse.ArgumentParser(description="Arregla permisos SQLite en contenedor Metabase") + parser.add_argument("--container", default=DEFAULT_CONTAINER, help="Nombre del contenedor Metabase") + args = parser.parse_args() + + container = args.container + + # Verificar que el contenedor existe + result = subprocess.run( + ["docker", "inspect", container, "--format", "{{.State.Running}}"], + capture_output=True, text=True, + ) + if result.returncode != 0 or result.stdout.strip() != "true": + print(f"ERROR: Contenedor '{container}' no esta corriendo.") + sys.exit(1) + + print(f"Contenedor: {container}") + print(f"\n1. Arreglando permisos en /data/...") + fixed = fix_container_permissions(container) + + if not fixed: + print(" Nada que arreglar.") + return + + print(f"\n2. Verificando databases en Metabase...") + client = metabase_auth(METABASE_URL, EMAIL, PASSWORD) + ok = verify_databases(client) + client.close() + + if ok: + print(f"\nTodas las databases responden correctamente.") + else: + print(f"\nAlgunas databases siguen fallando. Revisar logs del contenedor:") + print(f" docker logs {container} --tail 50") + + +if __name__ == "__main__": + main() From c2528c6ea48b2b2ab04b297d0f742c2a535b46f8 Mon Sep 17 00:00:00 2001 From: Egutierrez Date: Sun, 29 Mar 2026 00:54:28 +0100 Subject: [PATCH 3/3] =?UTF-8?q?docs:=20documentaci=C3=B3n=20completa=20de?= =?UTF-8?q?=20metabase=5Fregistry?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Arquitectura de mounts Docker, tabla de databases, permisos SQLite (nunca chown, siempre chmod), flujo para app nueva paso a paso, y referencia a los 3 pipelines relacionados. Co-Authored-By: Claude Opus 4.6 (1M context) --- apps/metabase_registry/app.md | 88 +++++++++++++++++++++++++++++++++-- 1 file changed, 84 insertions(+), 4 deletions(-) diff --git a/apps/metabase_registry/app.md b/apps/metabase_registry/app.md index 38ae41b8..723fa24b 100644 --- a/apps/metabase_registry/app.md +++ b/apps/metabase_registry/app.md @@ -2,8 +2,8 @@ name: metabase_registry lang: py domain: analytics -description: "Setup y dashboards automaticos de Metabase para visualizar metricas del fn-registry." -tags: [metabase, dashboard, analytics, visualization] +description: "Setup y dashboards automaticos de Metabase para visualizar metricas del fn-registry y operations.db de cada app." +tags: [metabase, dashboard, analytics, visualization, operations] uses_functions: - metabase_auth_py_infra - metabase_create_card_py_infra @@ -12,6 +12,7 @@ uses_functions: - metabase_list_databases_py_infra - metabase_add_database_py_infra - metabase_list_dashboards_py_infra + - metabase_delete_dashboard_py_infra - metabase_create_user_py_infra uses_types: [] framework: httpx @@ -19,6 +20,85 @@ entry_point: "main.py" dir_path: "apps/metabase_registry" --- -## Notas +## Arquitectura -Scripts Python que conectan con la API REST de Metabase para crear datasources, cards SQL y dashboards automaticamente. Usa las funciones del paquete python/functions/metabase/ del registry. Credenciales en .env local. +Metabase corre en Docker (`fn_registry-metabase`) con Postgres como backend interno. +Las bases de datos SQLite del proyecto se montan como bind mounts RW en `/data/`: + +| Database | Mount en container | Contenido | +|----------|-------------------|-----------| +| registry.db | `/data/registry/registry.db` | functions, types, proposals, apps | +| ops-docker-tui | `/data/ops-docker-tui/operations.db` | entities, relations, executions | +| ops-metabase-registry | `/data/ops-metabase-registry/operations.db` | entities, relations, executions | +| ops-pipeline-launcher | `/data/ops-pipeline-launcher/operations.db` | entities, relations, executions | + +## Dashboards + +| Dashboard | Contenido | +|-----------|-----------| +| fn-registry Overview | KPIs, distribucion y analisis del registry | +| fn-registry Apps | Apps por lenguaje, dominio, dependencias | +| ops: \ | Dashboard operativo por app (entities, relations, executions, assertions) | + +## Permisos SQLite en Docker + +Metabase corre Java como UID 2000 (usuario `metabase`). SQLite necesita crear journal/WAL +files en el mismo directorio que la BD. Reglas: + +- **NUNCA** hacer `chown` dentro del container: se propaga al host via bind mount y rompe permisos locales. +- **Usar `chmod`**: `chmod 777` en directorios, `chmod 666` en archivos `.db`. +- **Pipeline automatico**: `./fn run metabase_fix_permissions` arregla todos los permisos. +- **Ejecutar despues de**: recrear container, añadir nueva database, o ver error `SQLITE_READONLY_DIRECTORY`. + +## Flujo para app nueva + +```bash +# 1. Crear operations.db +./fn ops init apps/nueva_app + +# 2. Recrear container con el nuevo mount +docker stop fn_registry-metabase && docker rm fn_registry-metabase +docker run -d \ + --name fn_registry-metabase \ + --network fn_registry-net \ + -p 3000:3000 \ + -e MB_DB_TYPE=postgres -e MB_DB_DBNAME=metabase \ + -e MB_DB_PORT=5432 -e MB_DB_USER=metabase \ + -e MB_DB_PASS=metabase -e MB_DB_HOST=fn_registry-postgres \ + -v /home/lucas/fn_registry:/registry:ro \ + -v /home/lucas/fn_registry/registry.db:/data/registry/registry.db \ + -v /home/lucas/fn_registry/apps/docker_tui:/data/ops-docker-tui \ + -v /home/lucas/fn_registry/apps/metabase_registry:/data/ops-metabase-registry \ + -v /home/lucas/fn_registry/apps/pipeline_launcher:/data/ops-pipeline-launcher \ + -v /home/lucas/fn_registry/apps/nueva_app:/data/ops-nueva-app \ + metabase/metabase:latest + +# 3. Fix permisos +./fn run metabase_fix_permissions + +# 4. Registrar database en Metabase +./fn run metabase_add_ops_db nueva_app + +# 5. Crear dashboard operativo +./fn run metabase_create_ops_dashboard nueva_app +``` + +## Scripts + +| Script | Funcion | +|--------|---------| +| `main.py` | Setup inicial: datasource + cards basicas + dashboard Overview | +| `create_registry_dashboard.py` | Dashboard fn-registry Overview (18 cards) | +| `create_apps_dashboard.py` | Dashboard fn-registry Apps (10 cards) | + +## Pipelines relacionados + +| Pipeline | ID | Funcion | +|----------|-----|---------| +| `metabase_add_ops_db` | `metabase_add_ops_db_py_pipelines` | Registra operations.db de una app | +| `metabase_create_ops_dashboard` | `metabase_create_ops_dashboard_py_pipelines` | Crea dashboard operativo para una app | +| `metabase_fix_permissions` | `metabase_fix_permissions_py_pipelines` | Arregla SQLITE_READONLY_DIRECTORY | + +## Credenciales + +En `.env` local (NO commitear).