From 6ad82167bbc713f2c28f66f71d4674563965cfe1 Mon Sep 17 00:00:00 2001 From: Egutierrez Date: Sun, 17 May 2026 00:07:03 +0200 Subject: [PATCH] docs(flows): DoD obligatorio con user-facing surface + abrir issues 0100-0103 (taxonomia, frontmatter migration, dev_console, work dashboard) Co-Authored-By: Claude Opus 4.7 (1M context) --- .claude/rules/cpp_apps.md | 63 ++++- .gitignore | 4 + CHANGELOG.md | 11 + .../functions/infra/launch_cpp_app_windows.md | 12 +- .../functions/infra/launch_cpp_app_windows.sh | 14 +- .../infra/refresh_windows_icon_cache.md | 58 +++++ .../infra/refresh_windows_icon_cache.sh | 27 +++ .../pipelines/redeploy_cpp_app_windows.md | 6 +- .../pipelines/redeploy_cpp_app_windows.sh | 7 + cmd/fn/doctor.go | 49 ++++ cmd/fn/main.go | 39 ++- cmd/fn/sync.go | 14 ++ cmd/launch_chrome/main.go | 29 +++ cpp/CMakeLists.txt | 105 +++++--- cpp/framework/app_base.cpp | 68 +++++- cpp/framework/app_base.h | 5 + cpp/framework/app_modules.h | 32 +++ cpp/functions/core/app_about.cpp | 36 +++ cpp/functions/core/data_table_types.h | 53 ++++- cpp/functions/core/tql_apply.cpp | 65 ++++- cpp/functions/core/tql_emit.cpp | 52 +++- cpp/tests/CMakeLists.txt | 14 +- cpp/tests/test_column_specs.cpp | 216 ++++++++++++++++- cpp/tests/test_fn_table_viz_smoke.cpp | 2 +- dev/flows/0001-hn-top-stories.md | 27 +++ dev/flows/0002-aemet-madrid.md | 27 +++ dev/flows/0003-bbva-movimientos.md | 30 +++ dev/flows/0004-gitea-releases-monitor.md | 28 +++ dev/flows/0005-osint-person-lookup.md | 29 +++ dev/flows/0006-metabase-versioning.md | 29 +++ dev/flows/0007-matrix-telemetry-bot.md | 28 +++ dev/flows/AGENT_GUIDE.md | 7 + dev/flows/INDEX.md | 20 +- dev/flows/README.md | 45 ++++ dev/flows/template.md | 24 ++ dev/gen_app_icons.py | 118 --------- .../0100-issues-frontmatter-migration.md | 105 ++++++++ dev/issues/0101-dev-console-binary.md | 100 ++++++++ dev/issues/0102-work-dashboard-tab.md | 77 ++++++ .../0103-taxonomy-and-slash-commands.md | 123 ++++++++++ docs/diary/2026-05-16.md | 15 ++ functions/browser/chrome_launch.go | 146 ++++++++++-- functions/browser/chrome_launch.md | 48 +++- functions/browser/chrome_launch_test.go | 75 +++++- functions/infra/audit_modules_drift.go | 225 ++++++++++++++++++ functions/infra/audit_modules_drift.md | 57 +++++ modules/data_table/CMakeLists.txt | 54 +++++ .../viz => modules/data_table}/data_table.cpp | 150 +++++++++++- .../viz => modules/data_table}/data_table.h | 0 .../viz => modules/data_table}/data_table.md | 16 +- modules/data_table/module.md | 57 +++++ modules/framework/module.md | 52 ++++ python/functions/core/validate_recipe_yaml.py | 14 +- python/functions/infra/claude_cli_prompt.py | 29 ++- python/functions/infra/codegen_app_modules.md | 75 ++++++ python/functions/infra/codegen_app_modules.py | 149 ++++++++++++ python/functions/infra/export_hub_manifest.md | 72 ++++++ python/functions/infra/export_hub_manifest.py | 142 +++++++++++ .../functions/pipelines/cdp_extract_recipe.md | 8 +- .../functions/pipelines/cdp_extract_recipe.py | 146 ++++++++++-- .../pipelines/dedup_duckdb_table_by_hash.md | 60 +++++ .../pipelines/dedup_duckdb_table_by_hash.py | 141 +++++++++++ .../dedup_duckdb_table_by_hash_test.py | 95 ++++++++ .../pipelines/regenerate_app_icons.md | 66 +++++ .../pipelines/regenerate_app_icons.py | 97 ++++++++ registry/hash.go | 19 +- registry/indexer.go | 61 ++++- registry/migrations/013_modules.sql | 57 +++++ registry/models.go | 29 +++ registry/module_codegen.go | 60 +++++ registry/parser.go | 65 +++++ registry/store.go | 145 ++++++++++- 72 files changed, 3920 insertions(+), 303 deletions(-) create mode 100644 bash/functions/infra/refresh_windows_icon_cache.md create mode 100644 bash/functions/infra/refresh_windows_icon_cache.sh create mode 100644 cmd/launch_chrome/main.go create mode 100644 cpp/framework/app_modules.h delete mode 100644 dev/gen_app_icons.py create mode 100644 dev/issues/0100-issues-frontmatter-migration.md create mode 100644 dev/issues/0101-dev-console-binary.md create mode 100644 dev/issues/0102-work-dashboard-tab.md create mode 100644 dev/issues/0103-taxonomy-and-slash-commands.md create mode 100644 functions/infra/audit_modules_drift.go create mode 100644 functions/infra/audit_modules_drift.md create mode 100644 modules/data_table/CMakeLists.txt rename {cpp/functions/viz => modules/data_table}/data_table.cpp (96%) rename {cpp/functions/viz => modules/data_table}/data_table.h (100%) rename {cpp/functions/viz => modules/data_table}/data_table.md (84%) create mode 100644 modules/data_table/module.md create mode 100644 modules/framework/module.md create mode 100644 python/functions/infra/codegen_app_modules.md create mode 100644 python/functions/infra/codegen_app_modules.py create mode 100644 python/functions/infra/export_hub_manifest.md create mode 100644 python/functions/infra/export_hub_manifest.py create mode 100644 python/functions/pipelines/dedup_duckdb_table_by_hash.md create mode 100644 python/functions/pipelines/dedup_duckdb_table_by_hash.py create mode 100644 python/functions/pipelines/dedup_duckdb_table_by_hash_test.py create mode 100644 python/functions/pipelines/regenerate_app_icons.md create mode 100644 python/functions/pipelines/regenerate_app_icons.py create mode 100644 registry/migrations/013_modules.sql create mode 100644 registry/module_codegen.go diff --git a/.claude/rules/cpp_apps.md b/.claude/rules/cpp_apps.md index 6a03ed58..515ebf49 100644 --- a/.claude/rules/cpp_apps.md +++ b/.claude/rules/cpp_apps.md @@ -382,11 +382,22 @@ generate_app_icon( ) ``` -Mapping inicial (2026-05-16) en `dev/gen_app_icons.py` — script reproducible -que regenera los 11 `.ico` de un golpe leyendo la tabla `APPS`. Anadir app -nueva: una fila `(app_id, dir, phosphor_icon, accent_hex)` en `APPS` y -`/tmp/iconenv/bin/python dev/gen_app_icons.py` (o el venv del registry, ya -trae `cairosvg` + `Pillow`). +Mapping vive en el frontmatter de cada `app.md` C++: + +```yaml +icon: + phosphor: "chart-bar" + accent: "#0ea5e9" +``` + +Regeneracion batch via pipeline del registry — escanea `app.md`s y compone +`generate_app_icon` por app. Anadir app nueva: declarar `icon:` en su +`app.md` y lanzar: + +```bash +./fn run regenerate_app_icons # todas +./fn run regenerate_app_icons chart_demo # solo una +``` Convenciones: - **Glyph weight**: `fill` (mas legible a 16px que `regular` o `bold`). @@ -401,9 +412,9 @@ con `ls | grep ` antes de inventar — 1512 disponibles. #### Re-deploy tras cambiar icono ```bash -# 1. Regenerar .ico -./fn run generate_app_icon "chart-bar" "#0ea5e9" "apps/chart_demo/appicon.ico" -# (o editar dev/gen_app_icons.py + relanzar) +# 1. Editar icon: en apps/chart_demo/app.md y regenerar +./fn run regenerate_app_icons chart_demo +# (o ./fn run generate_app_icon "chart-bar" "#0ea5e9" "apps/chart_demo/appicon.ico" para uno suelto sin tocar app.md) # 2. Rebuild + redeploy (build dispara windres → nuevo .rsrc) ./fn run redeploy_cpp_app_windows chart_demo apps/chart_demo --build @@ -411,3 +422,39 @@ con `ls | grep ` antes de inventar — 1512 disponibles. Windows cachea iconos en `iconcache.db`. Si el nuevo icono no aparece tras desplegar, refresh con `ie4uinit.exe -show` o reiniciar Explorer. + +#### Runtime attach: taskbar + title bar + Alt+Tab (2026-05-16) + +Embeber `.ico` en el `.exe` (windres) basta para File Explorer / shortcuts — +pero GLFW crea su WNDCLASS sin icono, asi que la **barra de tareas**, el +**header de la ventana** y **Alt+Tab** muestran el icono GLFW por defecto a +menos que adjuntemos el recurso al HWND en runtime. + +`fn::run_app` lo hace automaticamente, sin opt-in. Tras `glfwCreateWindow`: + +```cpp +HICON hSmall = LoadImageW(GetModuleHandleW(NULL), MAKEINTRESOURCEW(101), + IMAGE_ICON, GetSystemMetrics(SM_CXSMICON), + GetSystemMetrics(SM_CYSMICON), LR_SHARED); +HICON hBig = LoadImageW(..., SM_CXICON, SM_CYICON, LR_SHARED); +SendMessageW(hwnd, WM_SETICON, ICON_SMALL, (LPARAM)hSmall); // title bar +SendMessageW(hwnd, WM_SETICON, ICON_BIG, (LPARAM)hBig); // taskbar +SetClassLongPtrW(hwnd, GCLP_HICONSM, (LONG_PTR)hSmall); +SetClassLongPtrW(hwnd, GCLP_HICON, (LONG_PTR)hBig); +``` + +Resource ID `101` lo emite `add_imgui_app` en el `.rc` generado +(`101 ICON "/appicon.ico"`). Si la app no tiene `appicon.ico`, el +`.rc` no se genera, `LoadImageW` devuelve NULL y el HWND queda con el icono +GLFW por defecto (sin error). + +Cobertura multi-viewport: el per-frame scan de `pio.Viewports` (mismo que +instala el sizemove subclass) tambien llama `attach_app_icon_to_hwnd` sobre +cada HWND secundario nuevo. Floating panels dragged-out heredan el icono +sin codigo extra en la app. + +Cache shell: el pipeline `redeploy_cpp_app_windows` llama +`refresh_windows_icon_cache_bash_infra` tras copiar el .exe — invoca +`ie4uinit.exe -show` para que Explorer recargue `iconcache.db` sin esperar +a que detecte el cambio por timestamp. Si Explorer sigue mostrando el +icono viejo: borrar `%LOCALAPPDATA%\IconCache.db` + reiniciar Explorer. diff --git a/.gitignore b/.gitignore index e9feeedc..0f36ec04 100644 --- a/.gitignore +++ b/.gitignore @@ -81,3 +81,7 @@ broken_paths.txt imgui.ini prompts/ kotlin/functions/ui/ + +# Module versioning auto-generated headers (written by `fn index`, issue 0097) +**/version_generated.h +**/app_modules_generated.h diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a1a05f7..7bf07a32 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,17 @@ Para contexto detallado del trabajo diario ver `docs/diary/`. Para decisiones ar ### Added +- **Panel "Logs" en `dag_engine` RunDetail** — `apps/dag_engine/frontend/src/pages/RunDetail.tsx` anade `` final con `` scrollable + `CopyButton` de Mantine. Helper `buildLogText(run, steps)` compone texto plano (metadata del run + por-step status/exit/duration/stdout/stderr indentado) para pegar entero al LLM sin abrir los `Collapse` del `StepTimeline`. + +### Fixed + +- **`dag_engine` steps `function:` fallando con `error: function "" not found (tried as ID and name)`** — tres DAGs nocturnos (`fn_backup` x2, `daily-registry-audit`) fallaron 2026-05-15/16 porque el binario `fn` resolvia una copia stale `apps/dag_engine/registry.db` (May 15, 262 KB) en vez del `registry.db` raiz. Raiz: el systemd unit `dag_engine.service` tiene `WorkingDirectory=apps/dag_engine/` y no exportaba `FN_REGISTRY_ROOT`; `cmd/fn/ops.go::tryOpenRegistryDB` cae al walk-up `go.mod` (devuelve `apps/dag_engine/`). Fix: + - Borrado `apps/dag_engine/registry.db` stale (violaba `.claude/rules/db_locations.md`). + - `~/.config/systemd/user/dag_engine.service`: anadido `Environment=FN_REGISTRY_ROOT`, `FN_BIN`, `PATH` (con `/usr/local/go/bin` para steps `function:` Go sin tests que invocan `go vet`), `HOME`. + - `apps/dag_engine/executor.go`: steps `function:` exportan `FN_REGISTRY_ROOT=` en env y default `dir = fnRegistryRoot` si `step.Dir`/`dag.WorkingDir` vacios. Steps `command:`/`script:` sin cambio. + +### Added + - **Iconos `.ico` Windows para apps C++** — 11 apps GUI (`chart_demo`, `dag_engine_ui`, `data_factory`, `graph_explorer`, `navegator_dashboard`, `odr_console`, `primitives_gallery`, `registry_dashboard`, `shaders_lab`, `text_editor_smoke`, `altsnap_jitter_test`) ahora tienen icono propio en el `.exe` y en `` desplegado. - Glyphs: **Phosphor Icons** (`fill` weight), clonado en `sources/phosphor-core/` (1512 SVGs disponibles). Cada app usa un `accent_hex` distinto (Tailwind 500-700) para distinguirse en taskbar/desktop. - Mapping inicial en `dev/gen_app_icons.py` (script reproducible). Cada `.ico` multi-resolucion (16/24/32/48/64/128/256). diff --git a/bash/functions/infra/launch_cpp_app_windows.md b/bash/functions/infra/launch_cpp_app_windows.md index f8e3bc87..742ce4f7 100644 --- a/bash/functions/infra/launch_cpp_app_windows.md +++ b/bash/functions/infra/launch_cpp_app_windows.md @@ -3,7 +3,7 @@ name: launch_cpp_app_windows kind: function lang: bash domain: infra -version: "1.0.0" +version: "1.1.0" purity: impure signature: "launch_cpp_app_windows(app_name: string, [desktop_dir: string]) -> void" description: "Lanza un binario .exe en Windows desde WSL2. Asume que deploy_cpp_exe_to_windows ya copió el exe a Desktop/apps//. Usa cmd.exe /c start para desacoplar el proceso y retornar inmediatamente." @@ -68,3 +68,13 @@ launch_cpp_app_windows "registry_dashboard" ``` No se incluyen tests automatizados porque requieren entorno WSL2 con Windows activo y no son automatizables en CI. + +## Gotchas + +- Si `FN_REGISTRY_ROOT_WSL` no es tu ruta default de fn_registry (`/home//fn_registry`), setea la variable antes de invocar esta función: `FN_REGISTRY_ROOT_WSL=/ruta/custom launch_cpp_app_windows `. +- El proceso hijo hereda `FN_REGISTRY_ROOT` como path Windows (backslashes) y `FN_REGISTRY_ROOT_WSL` como path Linux. En el exe C++, `py_resolve_interpreter()` usa `FN_REGISTRY_ROOT_WSL` para construir el invocation `wsl.exe -- /path/python3`. +- PowerShell escapa `$` con `\$` para evitar expansión de variables en el string del comando. + +## Capability growth log + +- v1.1.0 (2026-05-16) — auto-propaga `FN_REGISTRY_ROOT` (Windows path) + `FN_REGISTRY_ROOT_WSL` (Linux path) al proceso hijo para que pueda invocar WSL python via `wsl.exe`. diff --git a/bash/functions/infra/launch_cpp_app_windows.sh b/bash/functions/infra/launch_cpp_app_windows.sh index d2ace91d..630300ad 100644 --- a/bash/functions/infra/launch_cpp_app_windows.sh +++ b/bash/functions/infra/launch_cpp_app_windows.sh @@ -1,6 +1,8 @@ #!/usr/bin/env bash -# launch_cpp_app_windows — Lanza un .exe en Windows desde WSL2 via cmd.exe /c start. +# launch_cpp_app_windows v1.1.0 — Lanza un .exe en Windows desde WSL2 via PowerShell. # Asume que el exe ya fue copiado por deploy_cpp_exe_to_windows al escritorio. +# v1.1.0: propaga FN_REGISTRY_ROOT (Windows path) y FN_REGISTRY_ROOT_WSL (Linux path) +# al proceso hijo para que pueda invocar WSL python via wsl.exe. launch_cpp_app_windows() { local app="${1:-}" @@ -26,10 +28,18 @@ launch_cpp_app_windows() { win_app_dir=$(wslpath -w "$desktop_dir/apps/$app") win_exe="$win_app_dir\\$app.exe" + # Deducir raiz del registry en Linux (WSL) y traducir a Windows path. + # FN_REGISTRY_ROOT_WSL puede sobreescribirse en el entorno del llamante. + local linux_root win_root + linux_root="${FN_REGISTRY_ROOT_WSL:-$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)}" + win_root=$(wslpath -w "$linux_root") + # Start-Process detacha (equivale a `start` de cmd) y respeta -WorkingDirectory. # Las comillas simples en PowerShell son literales — no procesa \ ni $. + # Se inyectan FN_REGISTRY_ROOT (Windows path) y FN_REGISTRY_ROOT_WSL (Linux path) + # para que el exe pueda localizar el venv WSL y hacer: wsl.exe -- python3 ... powershell.exe -NoProfile -Command \ - "Start-Process -FilePath '$win_exe' -WorkingDirectory '$win_app_dir'" \ + "\$env:FN_REGISTRY_ROOT='$win_root'; \$env:FN_REGISTRY_ROOT_WSL='$linux_root'; Start-Process -FilePath '$win_exe' -WorkingDirectory '$win_app_dir'" \ >/dev/null 2>&1 local ts diff --git a/bash/functions/infra/refresh_windows_icon_cache.md b/bash/functions/infra/refresh_windows_icon_cache.md new file mode 100644 index 00000000..28a6c5d0 --- /dev/null +++ b/bash/functions/infra/refresh_windows_icon_cache.md @@ -0,0 +1,58 @@ +--- +name: refresh_windows_icon_cache +kind: function +lang: bash +domain: infra +version: "1.0.0" +purity: impure +signature: "refresh_windows_icon_cache() -> void" +description: "Fuerza a Windows Explorer a recargar la cache de iconos desde WSL2 via ie4uinit.exe. Best-effort: nunca aborta, retorna 0 si alguna estrategia tuvo exito." +tags: [windows, wsl, deploy, shell, icons, cpp-windows] +uses_functions: [] +uses_types: [] +returns: [] +returns_optional: false +error_type: "error_go_core" +imports: [] +tested: false +tests: [] +test_file_path: "" +file_path: "bash/functions/infra/refresh_windows_icon_cache.sh" +params: [] +output: "0 si al menos una estrategia tuvo exito, non-zero si todas fallaron. Imprime una linea de estado en stdout." +--- + +## Ejemplo + +```bash +source bash/functions/infra/refresh_windows_icon_cache.sh +refresh_windows_icon_cache +# icon cache refresh: ok via ie4uinit -show +``` + +O directamente via `fn run`: + +```bash +./fn run refresh_windows_icon_cache_bash_infra +``` + +Uso tipico en un pipeline de redeploy tras reconstruir el `.exe`: + +```bash +source bash/functions/infra/deploy_cpp_exe_to_windows.sh +source bash/functions/infra/refresh_windows_icon_cache.sh + +deploy_cpp_exe_to_windows "registry_dashboard" "apps/registry_dashboard" +refresh_windows_icon_cache +``` + +## Cuando usarla + +Despues de redeployar un `.exe` Windows cuyo `appicon.ico` cambio (via windres embebido en el build), antes de que Windows muestre el icono nuevo en taskbar, Alt+Tab y File Explorer. Sin esta llamada Windows puede tardar minutos en reflejar el icono actualizado, o no actualizarlo hasta reiniciar Explorer. + +## Gotchas + +- `ie4uinit.exe` debe estar en el PATH de WSL2 (normalmente via `/mnt/c/Windows/System32/`). Si Windows esta muy roto puede no encontrarse — la funcion retornara 1 con mensaje de error. +- El cambio puede tardar 1-2 segundos en propagarse visualmente despues de que la funcion retorne. +- Algunos casos extremos (icono cacheado en el dockable taskbar previamente fijado) requieren desanclar y volver a anclar el ejecutable, o reiniciar `explorer.exe`. Esta funcion no mata Explorer — seria demasiado disruptivo. +- Solo funciona desde WSL2 con acceso a herramientas Windows (`/mnt/c/Windows/System32/` en PATH). No tiene efecto en Linux nativo. diff --git a/bash/functions/infra/refresh_windows_icon_cache.sh b/bash/functions/infra/refresh_windows_icon_cache.sh new file mode 100644 index 00000000..f71b79e5 --- /dev/null +++ b/bash/functions/infra/refresh_windows_icon_cache.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# refresh_windows_icon_cache — Fuerza a Windows Explorer a recargar la cache +# de iconos desde WSL2. Best-effort: nunca aborta, retorna 0 si alguna +# estrategia tuvo exito. + +refresh_windows_icon_cache() { + # Estrategia 1: ie4uinit.exe -show (Windows 10/11 — emite SHCNE_ASSOCCHANGED) + if command -v ie4uinit.exe >/dev/null 2>&1; then + if ie4uinit.exe -show >/dev/null 2>&1; then + echo "icon cache refresh: ok via ie4uinit -show" + return 0 + fi + + # Estrategia 2: ie4uinit.exe -ClearIconCache (fallback para builds viejos) + if ie4uinit.exe -ClearIconCache >/dev/null 2>&1; then + echo "icon cache refresh: ok via ie4uinit -ClearIconCache" + return 0 + fi + fi + + echo "icon cache refresh: failed (ie4uinit.exe not found or all strategies failed)" + return 1 +} + +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + refresh_windows_icon_cache "$@" +fi diff --git a/bash/functions/pipelines/redeploy_cpp_app_windows.md b/bash/functions/pipelines/redeploy_cpp_app_windows.md index cdc2250b..377ec012 100644 --- a/bash/functions/pipelines/redeploy_cpp_app_windows.md +++ b/bash/functions/pipelines/redeploy_cpp_app_windows.md @@ -3,16 +3,17 @@ name: redeploy_cpp_app_windows kind: pipeline lang: bash domain: pipelines -version: "1.0.0" +version: "1.1.0" purity: impure signature: "redeploy_cpp_app_windows(app_name: string, app_dir: string, [--build]) -> void" -description: "Pipeline orquestador para redeployar una app C++ en Windows desde WSL2 en un solo comando. Reemplaza la secuencia manual taskkill+copy+launch+verify." +description: "Pipeline orquestador para redeployar una app C++ en Windows desde WSL2 en un solo comando. Reemplaza la secuencia manual taskkill+copy+launch+verify e incluye refresh del icon cache del shell." tags: [cpp, windows, redeploy, pipeline, wsl, launcher, cpp-windows] uses_functions: - build_cpp_windows_bash_infra - deploy_cpp_exe_to_windows_bash_infra - launch_cpp_app_windows_bash_infra - is_cpp_app_running_windows_bash_infra + - refresh_windows_icon_cache_bash_infra uses_types: [] returns: [] returns_optional: false @@ -47,6 +48,7 @@ redeploy_cpp_app_windows "chart_demo" "/home/lucas/fn_registry/cpp/apps/chart_de 1. **Parsear flag `--build`** (default off, opt-in). 2. **Si `--build`**: invocar `build_cpp_windows ` para compilar `cpp/build/windows/apps//.exe`. Si falla, exit 1 sin tocar el Desktop. 3. **Deploy**: invocar `deploy_cpp_exe_to_windows "" ""`. Esta función mata el proceso si está vivo (taskkill.exe pre-autorizado), copia exe + DLLs + assets + runtime + enrichers, y preserva `local_files/`. +3b. **Refresh icon cache** (v1.1.0+): invocar `refresh_windows_icon_cache` (best-effort). Llama `ie4uinit.exe -show` para que Explorer recargue `iconcache.db` sin esperar al timestamp. Si falla, no aborta el pipeline. 4. **Launch**: invocar `launch_cpp_app_windows ""` para arrancar la app en Windows. 5. **Wait**: `sleep 1` — espera arranque corto. 6. **Verify**: invocar `is_cpp_app_running_windows ""`. Si NO está vivo → exit 1 con mensaje claro. diff --git a/bash/functions/pipelines/redeploy_cpp_app_windows.sh b/bash/functions/pipelines/redeploy_cpp_app_windows.sh index f6986a75..57af7efc 100644 --- a/bash/functions/pipelines/redeploy_cpp_app_windows.sh +++ b/bash/functions/pipelines/redeploy_cpp_app_windows.sh @@ -7,6 +7,7 @@ source "$SCRIPT_DIR/../infra/build_cpp_windows.sh" source "$SCRIPT_DIR/../infra/deploy_cpp_exe_to_windows.sh" source "$SCRIPT_DIR/../infra/launch_cpp_app_windows.sh" source "$SCRIPT_DIR/../infra/is_cpp_app_running_windows.sh" +source "$SCRIPT_DIR/../infra/refresh_windows_icon_cache.sh" redeploy_cpp_app_windows() { local app_name="" @@ -63,6 +64,12 @@ redeploy_cpp_app_windows() { fi echo "[2/4] Deploy OK" + # Refrescar cache de iconos del shell. Sin esto el .exe nuevo puede salir + # con el icono generico (Windows cachea por timestamp/path en iconcache.db + # y a veces no detecta el cambio inmediatamente). Best-effort: si falla + # no abortamos el redeploy. + refresh_windows_icon_cache || true + # Paso 3: lanzar la app echo "[3/4] Launching $app_name..." if ! launch_cpp_app_windows "$app_name"; then diff --git a/cmd/fn/doctor.go b/cmd/fn/doctor.go index 438cd1e3..0c24bfc7 100644 --- a/cmd/fn/doctor.go +++ b/cmd/fn/doctor.go @@ -61,6 +61,8 @@ func cmdDoctor(args []string) { } case "app-location": doctorAppLocation(r, jsonOut) + case "modules": + doctorModules(r, jsonOut) default: fmt.Fprintf(os.Stderr, "unknown doctor subcommand: %s\n", sub) doctorUsage() @@ -87,6 +89,7 @@ Subcommands: copied-code Detecta cuerpos de funcion del registry copiados en apps sin import (issue 0085k) capabilities Drift entre docs/capabilities/INDEX.md, tags de funciones, y paginas .md (issue 0086) app-location Detecta artefactos (apps/analysis) en carpetas de lenguaje (cpp/apps/, etc.) - issue 0096 + modules Drift entre uses_modules (app.md) y fn_module_ link calls (CMakeLists.txt) - issue 0097 Flags: --json Salida JSON (para scripting/agentes) @@ -539,3 +542,49 @@ func doctorAppLocation(root string, jsonOut bool) { w.Flush() fmt.Printf("\n%d violation(s): move artefact to apps// or projects/

/apps// (issue 0096).\n", len(violations)) } + +func doctorModules(root string, jsonOut bool) { + checks, err := infra.AuditModulesDrift(root) + if err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + os.Exit(1) + } + + if jsonOut { + emit(checks) + return + } + + bad := 0 + w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0) + fmt.Fprintln(w, "STATUS\tAPP\tDECLARED\tLINKED\tMISSING\tEXTRA") + for _, c := range checks { + status := "OK" + if !c.OK { + status = "DRIFT" + bad++ + } + decl := strings.Join(c.Declared, ",") + if decl == "" { + decl = "-" + } + link := strings.Join(c.Linked, ",") + if link == "" { + link = "-" + } + missing := strings.Join(c.MissingLinks, ",") + if missing == "" { + missing = "-" + } + extra := strings.Join(c.ExtraLinks, ",") + if extra == "" { + extra = "-" + } + fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\n", status, c.AppID, decl, link, missing, extra) + } + w.Flush() + fmt.Printf("\n%d/%d apps with module drift.\n", bad, len(checks)) + if bad > 0 { + fmt.Println("Fix: align uses_modules in app.md with target_link_libraries(fn_module_*) in CMakeLists.txt.") + } +} diff --git a/cmd/fn/main.go b/cmd/fn/main.go index 6a4a881f..2ec42d8e 100644 --- a/cmd/fn/main.go +++ b/cmd/fn/main.go @@ -143,8 +143,8 @@ func cmdIndex() { } } - fmt.Printf("Indexed %d functions, %d types, %d apps, %d analysis, %d projects, %d vaults, %d unit_tests\n", - result.Functions, result.Types, result.Apps, result.Analysis, result.Projects, result.Vaults, result.UnitTests) + fmt.Printf("Indexed %d functions, %d types, %d apps, %d analysis, %d projects, %d vaults, %d modules, %d unit_tests\n", + result.Functions, result.Types, result.Apps, result.Analysis, result.Projects, result.Vaults, result.Modules, result.UnitTests) for _, e := range result.ValidationErrors { fmt.Fprintf(os.Stderr, " INVALID: %s\n", e) } @@ -420,10 +420,42 @@ func cmdShow(args []string) { return } + m, errM := db.GetModule(id) + if errM == nil { + printModule(m) + return + } + fmt.Fprintf(os.Stderr, "not found: %s\n", id) os.Exit(1) } +func printModule(m *registry.Module) { + fmt.Printf("ID: %s\n", m.ID) + fmt.Printf("Name: %s\n", m.Name) + fmt.Printf("Version: %s\n", m.Version) + fmt.Printf("Lang: %s\n", m.Lang) + fmt.Printf("Description: %s\n", m.Description) + if len(m.Members) > 0 { + fmt.Printf("Members: %s\n", strings.Join(m.Members, ", ")) + } + if len(m.Tags) > 0 { + fmt.Printf("Tags: %s\n", strings.Join(m.Tags, ", ")) + } + if m.DirPath != "" { + fmt.Printf("DirPath: %s\n", m.DirPath) + } + if m.RepoURL != "" { + fmt.Printf("RepoURL: %s\n", m.RepoURL) + } + if m.Documentation != "" { + fmt.Printf("\nDocumentation:\n%s\n", m.Documentation) + } + if m.Notes != "" { + fmt.Printf("\nNotes:\n%s\n", m.Notes) + } +} + func printFunction(f *registry.Function) { fmt.Printf("ID: %s\n", f.ID) fmt.Printf("Name: %s\n", f.Name) @@ -540,6 +572,9 @@ func printApp(a *registry.App) { if len(a.UsesTypes) > 0 { fmt.Printf("Uses types: %s\n", strings.Join(a.UsesTypes, ", ")) } + if len(a.UsesModules) > 0 { + fmt.Printf("Uses mods: %s\n", strings.Join(a.UsesModules, ", ")) + } if a.Notes != "" { fmt.Printf("\nNotes:\n%s\n", a.Notes) } diff --git a/cmd/fn/sync.go b/cmd/fn/sync.go index 0432f149..10d291e4 100644 --- a/cmd/fn/sync.go +++ b/cmd/fn/sync.go @@ -27,6 +27,7 @@ type syncRequest struct { Analysis []registry.Analysis `json:"analysis"` Projects []registry.Project `json:"projects"` Vaults []registry.Vault `json:"vaults"` + Modules []registry.Module `json:"modules"` Proposals []registry.Proposal `json:"proposals"` Locations []registry.PcLocation `json:"locations"` } @@ -37,6 +38,7 @@ type syncResponse struct { Analysis []registry.Analysis `json:"analysis"` Projects []registry.Project `json:"projects"` Vaults []registry.Vault `json:"vaults"` + Modules []registry.Module `json:"modules"` Proposals []registry.Proposal `json:"proposals"` Locations []registry.PcLocation `json:"locations"` Stats struct { @@ -100,6 +102,7 @@ func syncPushPull() { analysis, _ := db.AllAnalysis() projects, _ := db.ListAllProjects() vaults, _ := db.AllVaults() + modules, _ := db.AllModules() proposals, _ := db.AllProposals() // 2. Scan local directories and build pc_locations @@ -112,6 +115,7 @@ func syncPushPull() { Analysis: analysis, Projects: projects, Vaults: vaults, + Modules: modules, Proposals: proposals, Locations: locations, } @@ -203,6 +207,14 @@ func applySync(db *registry.DB, resp syncResponse) int { } } + for _, m := range resp.Modules { + existing, err := db.GetModule(m.ID) + if err != nil || m.UpdatedAt.After(existing.UpdatedAt) { + db.InsertModule(&m) + imported++ + } + } + for _, p := range resp.Proposals { existing, err := db.GetProposal(p.ID) if err != nil || p.UpdatedAt.After(existing.UpdatedAt) { @@ -329,6 +341,7 @@ func syncStatus() { analysis, _ := db.AllAnalysis() projects, _ := db.ListAllProjects() vaults, _ := db.AllVaults() + modules, _ := db.AllModules() proposals, _ := db.AllProposals() locs, _ := db.ListAllPcLocations() @@ -337,6 +350,7 @@ func syncStatus() { fmt.Printf(" analysis: %d\n", len(analysis)) fmt.Printf(" projects: %d\n", len(projects)) fmt.Printf(" vaults: %d\n", len(vaults)) + fmt.Printf(" modules: %d\n", len(modules)) fmt.Printf(" proposals: %d\n", len(proposals)) fmt.Printf(" locations: %d\n", len(locs)) diff --git a/cmd/launch_chrome/main.go b/cmd/launch_chrome/main.go new file mode 100644 index 00000000..a2875764 --- /dev/null +++ b/cmd/launch_chrome/main.go @@ -0,0 +1,29 @@ +package main + +import ( + "flag" + "fmt" + "os" + + "fn-registry/functions/browser" +) + +func main() { + port := flag.Int("port", 9222, "CDP debug port") + headless := flag.Bool("headless", false, "headless mode") + chromePath := flag.String("chrome-path", "", "explicit chrome.exe path (optional)") + userDataDir := flag.String("user-data-dir", "", "user-data-dir (optional; WSL2 auto-translates)") + flag.Parse() + + pid, err := browser.ChromeLaunch(browser.ChromeLaunchOpts{ + Port: *port, + Headless: *headless, + ChromePath: *chromePath, + UserDataDir: *userDataDir, + }) + if err != nil { + fmt.Fprintf(os.Stderr, "chrome_launch failed: %v\n", err) + os.Exit(1) + } + fmt.Printf("OK pid=%d port=%d\n", pid, *port) +} diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 24fd2564..d3ce8d00 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -248,13 +248,67 @@ function(add_imgui_app target) set(_rc_file ${CMAKE_CURRENT_BINARY_DIR}/${target}_appicon.rc) # Forward slashes para que windres no se confunda con escapes. file(TO_CMAKE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/appicon.ico _ico_path) - file(WRITE ${_rc_file} "IDI_ICON1 ICON \"${_ico_path}\"\n") + # Numeric ID 101 = FN_APP_ICON_ID (ver cpp/framework/app_base.cpp). + # Usamos ID numerico (no string "IDI_ICON1") para que LoadImageW + # pueda recuperarlo en runtime y attacharlo al HWND (WM_SETICON). + file(WRITE ${_rc_file} "101 ICON \"${_ico_path}\"\n") list(APPEND _extra_sources ${_rc_file}) endif() + + # Modules manifest (issue 0097): siempre generamos _modules_generated.cpp. + # Si la app tiene app.md con uses_modules, el .cpp resultante define + # fn::app_modules_array[] con sus modulos. Si no, genera un stub vacio + # (apps sin app.md no rompen el linkage de framework's app_about). + set(_modules_gen ${CMAKE_CURRENT_BINARY_DIR}/${target}_modules_generated.cpp) + set(_codegen_script ${FN_CPP_ROOT_DIR}/../python/functions/infra/codegen_app_modules.py) + set(_modules_root ${FN_CPP_ROOT_DIR}/../modules) + set(_app_md ${CMAKE_CURRENT_SOURCE_DIR}/app.md) + if(NOT EXISTS ${_app_md}) + # No app.md: emit empty stub directamente (sin invocar Python). + file(WRITE ${_modules_gen} +"// Auto-generated stub (no app.md). +#include \"app_modules.h\" +namespace fn { +const ModuleInfo app_modules_array[1] = { { nullptr, nullptr, nullptr } }; +const unsigned long app_modules_count = 0; +} +") + else() + find_package(Python3 QUIET COMPONENTS Interpreter) + if(Python3_FOUND AND EXISTS ${_codegen_script}) + execute_process( + COMMAND ${Python3_EXECUTABLE} ${_codegen_script} + --app-md ${_app_md} + --modules-root ${_modules_root} + --app-name ${target} + --out ${_modules_gen} + RESULT_VARIABLE _codegen_rc + OUTPUT_VARIABLE _codegen_out + ERROR_VARIABLE _codegen_err + ) + if(NOT _codegen_rc EQUAL 0 AND NOT _codegen_rc EQUAL 2) + message(WARNING "codegen_app_modules failed for ${target}: ${_codegen_err}") + endif() + endif() + # Si python falla o el script no esta, emit stub vacio. + if(NOT EXISTS ${_modules_gen}) + file(WRITE ${_modules_gen} +"// Auto-generated stub (codegen unavailable). +#include \"app_modules.h\" +namespace fn { +const ModuleInfo app_modules_array[1] = { { nullptr, nullptr, nullptr } }; +const unsigned long app_modules_count = 0; +} +") + endif() + endif() + list(APPEND _extra_sources ${_modules_gen}) + add_executable(${target} ${ARGN} ${_extra_sources}) target_link_libraries(${target} PRIVATE fn_framework) target_include_directories(${target} PRIVATE ${FN_CPP_ROOT_DIR}/functions + ${FN_CPP_ROOT_DIR}/framework ) # Convencion de layout (cpp_apps.md §7): # /.exe + .dll (binario + DLLs Windows convention) @@ -289,44 +343,13 @@ endfunction() # Functions are compiled as part of apps that use them via add_imgui_app. # Each function is a .h/.cpp pair included by the app's CMakeLists.txt. -# --- fn_table_viz: static lib bundling all Wave 1+2 tables-stack functions --- -# Issue 0081-I. Apps consumidores: target_link_libraries( PRIVATE fn_table_viz). -# data_table.cpp references playground-local headers (llm_anthropic.h, tql_to_sql.h, -# tql.h, data_table_logic.h). These are NOT available in the registry build — they -# live in the playground. fn_table_viz excludes data_table.cpp intentionally until -# those playground dependencies are promoted to the registry (Wave 4 deuda). -# The remaining 9 .cpp files compile cleanly with only registry headers. +# --- fn_module_data_table (issue 0097 modules) --- +# Static lib defined in modules/data_table/CMakeLists.txt. Replaces former +# fn_module_data_table target. Apps opt-in via: +# target_link_libraries( PRIVATE fn_module_data_table) +# Lua is a hard dep — only build the module when the vendored lua tree exists. if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/vendor/lua/CMakeLists.txt) -add_library(fn_table_viz STATIC - ${CMAKE_CURRENT_SOURCE_DIR}/functions/core/compute_stage.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/functions/core/compute_pipeline.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/functions/core/tql_emit.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/functions/core/tql_helpers.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/functions/core/tql_apply.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/functions/core/tql_to_sql.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/functions/core/lua_engine.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/functions/core/join_tables.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/functions/core/auto_detect_type.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/functions/core/compute_column_stats.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/functions/core/llm_anthropic.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/functions/viz/viz_render.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/functions/viz/data_table.cpp -) -target_include_directories(fn_table_viz PUBLIC - ${CMAKE_CURRENT_SOURCE_DIR}/functions -) -target_include_directories(fn_table_viz PRIVATE - ${CMAKE_CURRENT_SOURCE_DIR}/framework -) -target_compile_definitions(fn_table_viz PUBLIC FN_LLM_ANTHROPIC=1) -target_link_libraries(fn_table_viz PUBLIC - imgui - implot - lua54 -) -# fn::local_path() used by data_table.cpp (Ask AI export path + TQL save/load). -# fn_framework provides the implementation; link it here. -target_link_libraries(fn_table_viz PRIVATE fn_framework) + add_subdirectory(${CMAKE_SOURCE_DIR}/../modules/data_table ${CMAKE_BINARY_DIR}/modules/data_table) endif() # --- Demo app (lives in apps/, issue 0096 standardization) --- @@ -464,3 +487,9 @@ set(_DATA_FACTORY_DIR ${CMAKE_SOURCE_DIR}/../apps/data_factory) if(EXISTS ${_DATA_FACTORY_DIR}/CMakeLists.txt) add_subdirectory(${_DATA_FACTORY_DIR} ${CMAKE_BINARY_DIR}/apps/data_factory) endif() + +# --- app_hub_launcher (lives in apps/, issue 0096) --- +set(_APP_HUB_LAUNCHER_DIR ${CMAKE_SOURCE_DIR}/../apps/app_hub_launcher) +if(EXISTS ${_APP_HUB_LAUNCHER_DIR}/CMakeLists.txt) + add_subdirectory(${_APP_HUB_LAUNCHER_DIR} ${CMAKE_BINARY_DIR}/apps/app_hub_launcher) +endif() diff --git a/cpp/framework/app_base.cpp b/cpp/framework/app_base.cpp index 24b08e72..945637e7 100644 --- a/cpp/framework/app_base.cpp +++ b/cpp/framework/app_base.cpp @@ -1,4 +1,5 @@ #include "app_base.h" +#include "version_generated.h" #include "imgui.h" #include "imgui_impl_glfw.h" @@ -24,6 +25,7 @@ #include #include #include +#include #ifdef _WIN32 #ifndef WIN32_LEAN_AND_MEAN @@ -178,6 +180,50 @@ static void install_sizemove_subclass_hwnd(HWND hwnd) { g_subclassed[hwnd] = orig; } +// Resource ID generado por cpp/CMakeLists.txt en _appicon.rc: +// 101 ICON "/appicon.ico" +// Si la app no tiene appicon.ico el .rc no se genera y LoadImageW devuelve +// NULL — no error visible, los HWND quedan con el icono GLFW por defecto. +#define FN_APP_ICON_RES_ID 101 + +// Carga el icono embebido al tamaño OS-recomendado para small (title bar) y +// big (Alt+Tab / taskbar). LR_SHARED -> Windows gestiona el handle; no hay +// que DestroyIcon. Cacheado por HMODULE+ID+size. +static HICON load_app_icon(int cx, int cy) { + HMODULE mod = GetModuleHandleW(nullptr); + return (HICON)LoadImageW(mod, MAKEINTRESOURCEW(FN_APP_ICON_RES_ID), + IMAGE_ICON, cx, cy, LR_SHARED | LR_DEFAULTCOLOR); +} + +// Adjunta el icono embebido al HWND: +// WM_SETICON ICON_SMALL -> title bar (16x16) y Alt+Tab small variant. +// WM_SETICON ICON_BIG -> taskbar (32x32) y Alt+Tab big variant. +// SetClassLongPtrW propaga el icono al WNDCLASS para que nuevos HWNDs de la +// misma clase lo hereden (no critico — el per-frame scan ya cubre cada +// viewport secundario via su HWND propio, que puede tener WNDCLASS distinta). +static std::unordered_set g_icon_attached; +static void attach_app_icon_to_hwnd(HWND hwnd) { + if (!hwnd) return; + if (g_icon_attached.count(hwnd)) return; // idempotent + HICON hSmall = load_app_icon(GetSystemMetrics(SM_CXSMICON), + GetSystemMetrics(SM_CYSMICON)); + HICON hBig = load_app_icon(GetSystemMetrics(SM_CXICON), + GetSystemMetrics(SM_CYICON)); + if (!hSmall && !hBig) return; // no appicon.ico embebido — nada que hacer + if (hSmall) SendMessageW(hwnd, WM_SETICON, ICON_SMALL, (LPARAM)hSmall); + if (hBig) SendMessageW(hwnd, WM_SETICON, ICON_BIG, (LPARAM)hBig); + if (hSmall) SetClassLongPtrW(hwnd, GCLP_HICONSM, (LONG_PTR)hSmall); + if (hBig) SetClassLongPtrW(hwnd, GCLP_HICON, (LONG_PTR)hBig); + g_icon_attached.insert(hwnd); +} + +static void prune_dead_icon_attached() { + for (auto it = g_icon_attached.begin(); it != g_icon_attached.end();) { + if (!IsWindow(*it)) it = g_icon_attached.erase(it); + else ++it; + } +} + static void install_sizemove_subclass(GLFWwindow* w) { if (!w) return; install_sizemove_subclass_hwnd(glfwGetWin32Window(w)); @@ -337,6 +383,14 @@ void migrate_to_local_files(const char* const* names, std::size_t n) { } } +const char* framework_version() { + return FN_MODULE_FRAMEWORK_VERSION; +} + +const char* framework_description() { + return FN_MODULE_FRAMEWORK_DESCRIPTION; +} + int run_app(AppConfig config, std::function render_fn) { // Logger primero para capturar fallos del propio init (GLFW, ventana, GL). if (config.log.file_path != nullptr) { @@ -401,6 +455,11 @@ int run_app(AppConfig config, std::function render_fn) { // thread; we observe them and skip render+swap so the compositor moves // the existing buffer (same contract as native title-bar drag). install_sizemove_subclass(window); + + // Adjuntar appicon embebido al HWND principal para que aparezca en la + // barra de tareas, Alt+Tab y title bar (GLFW no propaga el icono de + // recursos del .exe a su WNDCLASS por defecto). + attach_app_icon_to_hwnd(glfwGetWin32Window(window)); #endif // Carga punteros a funciones GL >= 2.0 si la app lo pide. En Linux es @@ -565,11 +624,18 @@ int run_app(AppConfig config, std::function render_fn) { // their very first frame onwards. if (io.ConfigFlags & ImGuiConfigFlags_ViewportsEnable) { prune_dead_subclassed(); + prune_dead_icon_attached(); ImGuiPlatformIO& pio_sub = ImGui::GetPlatformIO(); for (int i = 0; i < pio_sub.Viewports.Size; ++i) { ImGuiViewport* vp = pio_sub.Viewports[i]; if (!vp || !vp->PlatformHandle) continue; - install_sizemove_subclass((GLFWwindow*)vp->PlatformHandle); + GLFWwindow* gw = (GLFWwindow*)vp->PlatformHandle; + install_sizemove_subclass(gw); + // Floating panels = secondary HWNDs creados por el backend + // GLFW. WNDCLASS distinta de la main -> no heredan icono via + // SetClassLongPtrW. WM_SETICON per-HWND es la unica forma de + // que el taskbar/titlebar muestren el icono. + attach_app_icon_to_hwnd(glfwGetWin32Window(gw)); } } #endif diff --git a/cpp/framework/app_base.h b/cpp/framework/app_base.h index e31f9792..e2f54dfb 100644 --- a/cpp/framework/app_base.h +++ b/cpp/framework/app_base.h @@ -82,6 +82,11 @@ const char* asset_path(const char* name); // apps lo llaman al iniciar para migrar instalaciones viejas. void migrate_to_local_files(const char* const* names, std::size_t n); +// Framework metadata (auto-generated from modules/framework/module.md via +// `fn index`). About panel reads these. +const char* framework_version(); +const char* framework_description(); + // Modos de tema para run_app. enum class ThemeMode { FnDark, // Identidad del registry (Mantine v9 dark + indigo). DEFAULT. diff --git a/cpp/framework/app_modules.h b/cpp/framework/app_modules.h new file mode 100644 index 00000000..30e43965 --- /dev/null +++ b/cpp/framework/app_modules.h @@ -0,0 +1,32 @@ +// Module manifest visible to fn_framework's About panel. +// +// Each app gets an auto-generated _modules_generated.cpp (codegen via +// python/functions/infra/codegen_app_modules.py, invoked by add_imgui_app at +// CMake configure time) that defines the array + count below from the app's +// `uses_modules:` declaration in its app.md. +// +// Apps without uses_modules still get a stub array of length 0 — links cleanly. +// +// Framework reads via: +// +// for (size_t i = 0; i < fn::app_modules_count; ++i) { +// const auto& m = fn::app_modules_array[i]; +// ImGui::Text("%s v%s — %s", m.name, m.version, m.description); +// } + +#pragma once + +#include + +namespace fn { + +struct ModuleInfo { + const char* name; + const char* version; + const char* description; +}; + +extern const ModuleInfo app_modules_array[]; +extern const unsigned long app_modules_count; + +} // namespace fn diff --git a/cpp/functions/core/app_about.cpp b/cpp/functions/core/app_about.cpp index 35b58207..be18e6bb 100644 --- a/cpp/functions/core/app_about.cpp +++ b/cpp/functions/core/app_about.cpp @@ -1,5 +1,7 @@ #include "core/app_about.h" +#include "app_base.h" +#include "app_modules.h" #include "imgui.h" #include @@ -58,6 +60,40 @@ void about_window_render() { ImGui::TextWrapped("%s", g_description.c_str()); } + ImGui::Spacing(); + ImGui::Separator(); + ImGui::Spacing(); + + // --- Framework version (issue 0097) --- + ImGui::Text("Framework"); + ImGui::SameLine(); + ImGui::TextDisabled("v%s", fn::framework_version()); + + // --- Modules consumidos por la app (issue 0097) --- + if (fn::app_modules_count > 0) { + ImGui::Spacing(); + ImGui::Text("Modules (%lu)", fn::app_modules_count); + if (ImGui::BeginTable("##fn_modules_table", 2, + ImGuiTableFlags_RowBg | ImGuiTableFlags_BordersInnerH | + ImGuiTableFlags_SizingStretchProp)) { + ImGui::TableSetupColumn("Name", ImGuiTableColumnFlags_WidthFixed, 140.0f); + ImGui::TableSetupColumn("Version", ImGuiTableColumnFlags_WidthFixed, 80.0f); + for (unsigned long i = 0; i < fn::app_modules_count; ++i) { + const auto& m = fn::app_modules_array[i]; + if (!m.name) continue; + ImGui::TableNextRow(); + ImGui::TableSetColumnIndex(0); + ImGui::TextUnformatted(m.name); + if (m.description && *m.description && ImGui::IsItemHovered()) { + ImGui::SetTooltip("%s", m.description); + } + ImGui::TableSetColumnIndex(1); + ImGui::TextDisabled("v%s", m.version ? m.version : "?"); + } + ImGui::EndTable(); + } + } + ImGui::Spacing(); ImGui::Separator(); ImGui::TextDisabled("fn_registry"); diff --git a/cpp/functions/core/data_table_types.h b/cpp/functions/core/data_table_types.h index 90bb6da5..fad5cdfb 100644 --- a/cpp/functions/core/data_table_types.h +++ b/cpp/functions/core/data_table_types.h @@ -2,6 +2,7 @@ // Promovido al registry desde cpp/apps/primitives_gallery/playground/tables/. // Ver issue 0081 + docs/TQL.md. Pure value types + enums. // Issue 0081-N: CellRenderer / ColumnSpec / BadgeRule / IconMapEntry (v1.1.0). +// v1.4.0: ChipRule / ColorStop / CategoricalChip / ColorScale renderers. #pragma once #include @@ -131,16 +132,19 @@ enum class JoinStrategy { Left, Inner, Right, Full }; // CellRenderer: declarative rendering mode per column (issue 0081-N, v1.1.0). // Phase 2 (issue 0081-O, v1.2.0): Button=5 added. // Phase 2.5 (issue 0081-O.5, v1.3.0): Dots=8 added (inline status timeline). +// v1.4.0: CategoricalChip=9, ColorScale=10. // ---------------------------------------------------------------------------- enum class CellRenderer : uint8_t { - Text = 0, // default — current behavior - Badge = 1, // colored badge per-value - Progress = 2, // progress bar (0..1 or 0..100) - Duration = 3, // milliseconds with color gradient - Icon = 4, // icon lookup by value string - Button = 5, // clickable button; emits TableEvent::ButtonClick + Text = 0, // default — current behavior + Badge = 1, // colored badge per-value + Progress = 2, // progress bar (0..1 or 0..100) + Duration = 3, // milliseconds with color gradient + Icon = 4, // icon lookup by value string + Button = 5, // clickable button; emits TableEvent::ButtonClick // 6, 7: reserved for Phase 3 (TextInput, Custom). - Dots = 8, // inline dots sparkline; cell = separator-delimited tokens + Dots = 8, // inline dots sparkline; cell = separator-delimited tokens + CategoricalChip = 9, // filled circle (8px) to left of text; color by value match + ColorScale = 10, // continuous N-color gradient tint on cell background }; // ---------------------------------------------------------------------------- @@ -178,6 +182,19 @@ struct IconMapEntry { std::string color_hex; // optional; "" -> default text color }; +// ChipRule: maps a cell value to a dot color for CategoricalChip renderer (v1.4.0). +// If no rule matches, no dot is drawn (fallback: plain text only). +struct ChipRule { + std::string match; // exact match (case-sensitive) against cell value + std::string color; // "#rrggbb" hex color for the filled circle +}; + +// ColorStop: one stop in an N-color gradient for ColorScale renderer (v1.4.0). +struct ColorStop { + float position; // 0.0 (leftmost/min) to 1.0 (rightmost/max) + std::string color; // "#rrggbb" hex color at this stop +}; + // ColumnSpec: rendering spec for one column. Indexed by column position. struct ColumnSpec { std::string id; // stable id, used in TQL @@ -215,6 +232,20 @@ struct ColumnSpec { float dots_glyph_size = 0.0f; // glyph size px; 0 = default font size int dots_max = 0; // hard limit on dots shown; 0 = no limit bool dots_show_count = false; // if true, appends " (N)" after dots + + // CategoricalChip (v1.4.0): CellRenderer::CategoricalChip. + // Draws a filled circle (radius ~4px) to the left of the cell text. + // Color is determined by matching cell value against `chips` rules. + // Always visible (not hover-only). If no rule matches, no dot is drawn. + std::vector chips; // value → color rules + + // ColorScale (v1.4.0): CellRenderer::ColorScale. + // Maps numeric cell value to a background tint via N-color gradient LERP. + // Low alpha so text remains legible. + double range_min = 0.0; // value at t=0.0 + double range_max = 1.0; // value at t=1.0 + float range_alpha = 0.25f; // [0..1]; background tint opacity + std::vector range_stops; // N≥2 stops; empty → default green→amber→red }; // ---------------------------------------------------------------------------- @@ -292,6 +323,14 @@ struct State { // Caller-provided column_specs take precedence over aux_column_specs. std::vector> aux_column_specs; + // Per-table "Show UI" toggle. Moved from global UiCache to per-State so each + // table's chrome (chips bar) can be toggled independently (issue: multiple + // tables on screen, "Show UI" used to flip all at once). + // Defaults: user_set=true + visible=false => chrome closed by default, ignoring + // the API arg show_chrome from frame 1 (preserves legacy behavior). + bool chrome_user_set = true; + bool chrome_user_visible = false; + // Helpers (definidos en compute_stage.cpp). Stage& raw(); const Stage& raw() const; diff --git a/cpp/functions/core/tql_apply.cpp b/cpp/functions/core/tql_apply.cpp index 8875dba5..4cc2ef2b 100644 --- a/cpp/functions/core/tql_apply.cpp +++ b/cpp/functions/core/tql_apply.cpp @@ -540,12 +540,14 @@ ApplyResult apply(const std::string& lua_text, lua_getfield(L, -1, "renderer"); if (lua_isstring(L, -1)) { std::string rn = lua_tostring(L, -1); - if (rn == "badge") cs.renderer = data_table::CellRenderer::Badge; - else if (rn == "progress") cs.renderer = data_table::CellRenderer::Progress; - else if (rn == "duration") cs.renderer = data_table::CellRenderer::Duration; - else if (rn == "icon") cs.renderer = data_table::CellRenderer::Icon; - else if (rn == "button") cs.renderer = data_table::CellRenderer::Button; - else if (rn == "dots") cs.renderer = data_table::CellRenderer::Dots; + if (rn == "badge") cs.renderer = data_table::CellRenderer::Badge; + else if (rn == "progress") cs.renderer = data_table::CellRenderer::Progress; + else if (rn == "duration") cs.renderer = data_table::CellRenderer::Duration; + else if (rn == "icon") cs.renderer = data_table::CellRenderer::Icon; + else if (rn == "button") cs.renderer = data_table::CellRenderer::Button; + else if (rn == "dots") cs.renderer = data_table::CellRenderer::Dots; + else if (rn == "categorical_chip") cs.renderer = data_table::CellRenderer::CategoricalChip; + else if (rn == "color_scale") cs.renderer = data_table::CellRenderer::ColorScale; else cs.renderer = data_table::CellRenderer::Text; } lua_pop(L, 1); @@ -642,6 +644,57 @@ ApplyResult apply(const std::string& lua_text, if (lua_isnumber(L, -1)) cs.dots_glyph_size = (float)lua_tonumber(L, -1); lua_pop(L, 1); + // CategoricalChip (v1.4.0) + lua_getfield(L, -1, "chips"); + if (lua_istable(L, -1)) { + int nc = (int)lua_rawlen(L, -1); + for (int j = 1; j <= nc; ++j) { + lua_rawgeti(L, -1, j); + if (lua_istable(L, -1)) { + data_table::ChipRule cr; + lua_getfield(L, -1, "match"); + if (lua_isstring(L, -1)) cr.match = lua_tostring(L, -1); + lua_pop(L, 1); + lua_getfield(L, -1, "color"); + if (lua_isstring(L, -1)) cr.color = lua_tostring(L, -1); + lua_pop(L, 1); + cs.chips.push_back(std::move(cr)); + } + lua_pop(L, 1); + } + } + lua_pop(L, 1); // chips + + // ColorScale (v1.4.0) + lua_getfield(L, -1, "range_min"); + if (lua_isnumber(L, -1)) cs.range_min = (double)lua_tonumber(L, -1); + lua_pop(L, 1); + lua_getfield(L, -1, "range_max"); + if (lua_isnumber(L, -1)) cs.range_max = (double)lua_tonumber(L, -1); + lua_pop(L, 1); + lua_getfield(L, -1, "range_alpha"); + if (lua_isnumber(L, -1)) cs.range_alpha = (float)lua_tonumber(L, -1); + lua_pop(L, 1); + lua_getfield(L, -1, "range_stops"); + if (lua_istable(L, -1)) { + int ns = (int)lua_rawlen(L, -1); + for (int j = 1; j <= ns; ++j) { + lua_rawgeti(L, -1, j); + if (lua_istable(L, -1)) { + data_table::ColorStop stop; + lua_getfield(L, -1, "position"); + if (lua_isnumber(L, -1)) stop.position = (float)lua_tonumber(L, -1); + lua_pop(L, 1); + lua_getfield(L, -1, "color"); + if (lua_isstring(L, -1)) stop.color = lua_tostring(L, -1); + lua_pop(L, 1); + cs.range_stops.push_back(std::move(stop)); + } + lua_pop(L, 1); + } + } + lua_pop(L, 1); // range_stops + // Tooltip lua_getfield(L, -1, "tooltip"); if (lua_isstring(L, -1)) cs.tooltip = lua_tostring(L, -1); diff --git a/cpp/functions/core/tql_emit.cpp b/cpp/functions/core/tql_emit.cpp index fc4f90cd..3ca3f9f3 100644 --- a/cpp/functions/core/tql_emit.cpp +++ b/cpp/functions/core/tql_emit.cpp @@ -283,8 +283,7 @@ std::string emit(const State& state, // Emit the block only if at least one spec has a non-default renderer OR tooltip. bool any_renderable = false; for (const auto& cs : specs) { - if (cs.renderer != data_table::CellRenderer::Text || cs.tooltip_on_hover || - cs.renderer == data_table::CellRenderer::Dots) { + if (cs.renderer != data_table::CellRenderer::Text || cs.tooltip_on_hover) { any_renderable = true; break; } } @@ -297,12 +296,14 @@ std::string emit(const State& state, // renderer const char* rname = "text"; switch (cs.renderer) { - case data_table::CellRenderer::Badge: rname = "badge"; break; - case data_table::CellRenderer::Progress: rname = "progress"; break; - case data_table::CellRenderer::Duration: rname = "duration"; break; - case data_table::CellRenderer::Icon: rname = "icon"; break; - case data_table::CellRenderer::Button: rname = "button"; break; - case data_table::CellRenderer::Dots: rname = "dots"; break; + case data_table::CellRenderer::Badge: rname = "badge"; break; + case data_table::CellRenderer::Progress: rname = "progress"; break; + case data_table::CellRenderer::Duration: rname = "duration"; break; + case data_table::CellRenderer::Icon: rname = "icon"; break; + case data_table::CellRenderer::Button: rname = "button"; break; + case data_table::CellRenderer::Dots: rname = "dots"; break; + case data_table::CellRenderer::CategoricalChip: rname = "categorical_chip"; break; + case data_table::CellRenderer::ColorScale: rname = "color_scale"; break; default: break; } out += ", renderer = " + lua_string_literal(rname); @@ -370,6 +371,41 @@ std::string emit(const State& state, out += std::string(", dots_glyph_size = ") + buf; } } + // CategoricalChip (v1.4.0) + if (cs.renderer == data_table::CellRenderer::CategoricalChip) { + if (!cs.chips.empty()) { + out += ", chips = {\n"; + for (const auto& cr : cs.chips) { + out += " { match = " + lua_string_literal(cr.match); + out += ", color = " + lua_string_literal(cr.color); + out += " },\n"; + } + out += " }"; + } + } + // ColorScale (v1.4.0) + if (cs.renderer == data_table::CellRenderer::ColorScale) { + char buf[64]; + std::snprintf(buf, sizeof(buf), "%g", cs.range_min); + out += std::string(", range_min = ") + buf; + std::snprintf(buf, sizeof(buf), "%g", cs.range_max); + out += std::string(", range_max = ") + buf; + if (cs.range_alpha != 0.25f) { + std::snprintf(buf, sizeof(buf), "%g", (double)cs.range_alpha); + out += std::string(", range_alpha = ") + buf; + } + if (!cs.range_stops.empty()) { + out += ", range_stops = {\n"; + for (const auto& stop : cs.range_stops) { + std::snprintf(buf, sizeof(buf), "%g", (double)stop.position); + out += " { position = "; + out += buf; + out += ", color = " + lua_string_literal(stop.color); + out += " },\n"; + } + out += " }"; + } + } // Tooltip if (cs.tooltip_on_hover) { out += ", tooltip = " + lua_string_literal(cs.tooltip.empty() ? "auto" : cs.tooltip); diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index 084a74cd..1afb8240 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -145,7 +145,7 @@ endif() # --- Issue 0081-B — compute_stage + compute_pipeline (TQL pure logic) ------- # tql_helpers.cpp added (issue 0081-I): compute_stage.cpp now delegates -# aggregation_alias to tql_helpers to avoid ODR conflict in fn_table_viz lib. +# aggregation_alias to tql_helpers to avoid ODR conflict in fn_module_data_table lib. add_fn_test(test_compute_stage test_compute_stage.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../functions/core/compute_stage.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../functions/core/tql_helpers.cpp) @@ -214,32 +214,32 @@ target_include_directories(tql_apply_test PRIVATE target_link_libraries(tql_apply_test PRIVATE lua54) add_test(NAME tql_apply_test COMMAND tql_apply_test) -# --- Issue 0081-I — fn_table_viz static lib smoke test --------------------- -# Linker test: verifies that all 9 registry .cpp files in fn_table_viz resolve +# --- Issue 0081-I — fn_module_data_table static lib smoke test --------------------- +# Linker test: verifies that all 9 registry .cpp files in fn_module_data_table resolve # symbols correctly when linked as a static lib. Does NOT call data_table::render # (requires ImGui context + playground headers). Uses its own main(). -if(TARGET fn_table_viz) +if(TARGET fn_module_data_table) add_executable(test_fn_table_viz_smoke ${CMAKE_CURRENT_SOURCE_DIR}/test_fn_table_viz_smoke.cpp) target_include_directories(test_fn_table_viz_smoke PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../functions ${CMAKE_CURRENT_SOURCE_DIR}/../vendor/imgui ${CMAKE_CURRENT_SOURCE_DIR}/../vendor/implot) - target_link_libraries(test_fn_table_viz_smoke PRIVATE fn_table_viz) + target_link_libraries(test_fn_table_viz_smoke PRIVATE fn_module_data_table) add_test(NAME test_fn_table_viz_smoke COMMAND test_fn_table_viz_smoke) endif() # --- Issue 0081-N — declarative CellRenderer (Badge/Progress/Duration/Icon) -- # Smoke + back-compat tests for TableInput.column_specs (v1.1.0). # Verifies type construction + link resolution; does NOT call render() (ImGui). -if(TARGET fn_table_viz) +if(TARGET fn_module_data_table) add_executable(test_column_specs ${CMAKE_CURRENT_SOURCE_DIR}/test_column_specs.cpp) target_include_directories(test_column_specs PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../functions ${CMAKE_CURRENT_SOURCE_DIR}/../vendor/imgui ${CMAKE_CURRENT_SOURCE_DIR}/../vendor/implot) - target_link_libraries(test_column_specs PRIVATE fn_table_viz) + target_link_libraries(test_column_specs PRIVATE fn_module_data_table) add_test(NAME test_column_specs COMMAND test_column_specs) endif() diff --git a/cpp/tests/test_column_specs.cpp b/cpp/tests/test_column_specs.cpp index 77de4f77..6c20e5eb 100644 --- a/cpp/tests/test_column_specs.cpp +++ b/cpp/tests/test_column_specs.cpp @@ -1,6 +1,9 @@ // test_column_specs.cpp — Smoke / back-compat tests for declarative cell renderers. // Issue 0081-N, v1.1.0. Phase 2 (issue 0081-O, v1.2.0). // Phase 2.5 (issue 0081-O.5, v1.3.0): Dots renderer. +// v1.4.0: CategoricalChip + ColorScale renderers (TestCategoricalChipRule, +// TestColorScaleLerpTwoStops, TestColorScaleLerpThreeStops, +// TestColorScaleOutOfRange). // // These tests verify: // 1. TableInput without column_specs compiles and links (back-compat). @@ -9,6 +12,11 @@ // 7. Tooltip field: ColumnSpec with tooltip_on_hover=true compiles and links. // 8. render() overload with events_out=nullptr back-compat (symbol resolution only). // 9. Dots renderer: ColumnSpec with CellRenderer::Dots + badges constructs correctly. +// 10. Dots TQL roundtrip: State::aux_column_specs accepts Dots spec. +// 11. TestCategoricalChipRule: ChipRule with match="success" produces correct color. +// 12. TestColorScaleLerpTwoStops: t=0→first color, t=1→last color, t=0.5→midpoint. +// 13. TestColorScaleLerpThreeStops: t=0.25 lies between stop0 and stop1. +// 14. TestColorScaleOutOfRange: t<0 saturates at first; t>1 saturates at last. // // None of these tests call data_table::render() (requires ImGui context). // They only verify that the new types are usable and that the symbols from @@ -18,7 +26,7 @@ // Run: ./cpp/build/linux/tests/test_column_specs #include "core/data_table_types.h" -#include "viz/data_table.h" +#include "data_table/data_table.h" #include #include @@ -396,6 +404,206 @@ static void test_dots_tql_roundtrip() { "(State::aux_column_specs accepts Dots spec)\n"); } +// --------------------------------------------------------------------------- +// Test 11: TestCategoricalChipRule — ChipRule with match="success" correct color. +// Verifies ChipRule struct construction + ColumnSpec.chips field accessible. +// --------------------------------------------------------------------------- +static void test_categorical_chip_rule() { + ColumnSpec cs; + cs.id = "state"; + cs.renderer = CellRenderer::CategoricalChip; + cs.chips = { + ChipRule{"success", "#22c55e"}, + ChipRule{"failure", "#ef4444"}, + ChipRule{"pending", "#f59e0b"}, + }; + + assert(cs.renderer == CellRenderer::CategoricalChip); + assert(cs.chips.size() == 3); + assert(cs.chips[0].match == "success"); + assert(cs.chips[0].color == "#22c55e"); + assert(cs.chips[1].match == "failure"); + assert(cs.chips[1].color == "#ef4444"); + assert(cs.chips[2].match == "pending"); + + // No matching rule for "unknown" — chips lookup returns nullptr (logic check). + const ChipRule* found = nullptr; + const char* test_val = "unknown"; + for (const auto& cr : cs.chips) { + if (cr.match == test_val) { found = &cr; break; } + } + assert(found == nullptr && "no rule should match 'unknown'"); + + // Match "success" should find first rule. + const ChipRule* found2 = nullptr; + for (const auto& cr : cs.chips) { + if (cr.match == std::string("success")) { found2 = &cr; break; } + } + assert(found2 != nullptr && found2->color == "#22c55e"); + + std::printf("PASS: TestCategoricalChipRule " + "(3 chip rules, match/no-match logic correct)\n"); +} + +// --------------------------------------------------------------------------- +// Headless color lerp helpers (mirrors the static functions in data_table.cpp, +// replicated here so tests run without ImGui context). +// Uses a plain struct RGB3 instead of std::tuple to avoid extra includes. +// --------------------------------------------------------------------------- +struct RGB3 { float r, g, b; }; + +static float lerp_f(float a, float b, float t) { return a + t * (b - a); } + +// Parse "#rrggbb" -> RGB3 floats in [0,1]. Returns {-1,-1,-1} on failure. +static RGB3 parse_rgb(const std::string& hex) { + const char* p = hex.c_str(); + if (*p == '#') ++p; + unsigned int r = 0, g = 0, b = 0; + if (std::sscanf(p, "%02x%02x%02x", &r, &g, &b) != 3) + return {-1.f, -1.f, -1.f}; + return {r / 255.f, g / 255.f, b / 255.f}; +} + +// Lerp between two ColorStop RGB colors at a given global t. +static RGB3 lerp_between(const ColorStop& lo, const ColorStop& hi, float t_global) { + float span = hi.position - lo.position; + float f = (span > 1e-6f) ? (t_global - lo.position) / span : 0.f; + RGB3 ca = parse_rgb(lo.color); + RGB3 cb = parse_rgb(hi.color); + return {lerp_f(ca.r,cb.r,f), lerp_f(ca.g,cb.g,f), lerp_f(ca.b,cb.b,f)}; +} + +// lerp_stops: full N-stop lerp (same logic as lerp_color_along_stops in data_table.cpp). +static RGB3 lerp_stops(const std::vector& stops, float t) { + static const ColorStop kDefault[] = { + {0.0f, "#22c55e"}, {0.5f, "#f59e0b"}, {1.0f, "#ef4444"} + }; + static const int kDefaultN = 3; + + // Build a working sorted copy. + std::vector s; + if (stops.empty()) { + for (int i = 0; i < kDefaultN; ++i) s.push_back(kDefault[i]); + } else { + s = stops; + } + // Simple insertion sort (N is tiny, avoids std::sort include). + for (size_t i = 1; i < s.size(); ++i) { + ColorStop key = s[i]; + int j = (int)i - 1; + while (j >= 0 && s[j].position > key.position) { s[j+1] = s[j]; --j; } + s[j+1] = key; + } + + t = t < 0.f ? 0.f : (t > 1.f ? 1.f : t); + if (t <= s.front().position) return parse_rgb(s.front().color); + if (t >= s.back().position) return parse_rgb(s.back().color); + for (size_t i = 0; i + 1 < s.size(); ++i) { + if (t >= s[i].position && t <= s[i+1].position) + return lerp_between(s[i], s[i+1], t); + } + return parse_rgb(s.back().color); +} + +// --------------------------------------------------------------------------- +// Test 12: TestColorScaleLerpTwoStops — t=0→first, t=1→last, t=0.5→midpoint. +// --------------------------------------------------------------------------- +static void test_color_scale_lerp_two_stops() { + std::vector stops = { + {0.0f, "#000000"}, // black + {1.0f, "#ffffff"}, // white + }; + ColumnSpec cs; + cs.renderer = CellRenderer::ColorScale; + cs.range_min = 0.0; + cs.range_max = 1.0; + cs.range_stops = stops; + cs.range_alpha = 0.25f; + + assert(cs.renderer == CellRenderer::ColorScale); + assert(cs.range_stops.size() == 2); + + // t=0.0 → black (0,0,0) + RGB3 c0 = lerp_stops(stops, 0.0f); + assert(c0.r < 0.01f && c0.g < 0.01f && c0.b < 0.01f); + + // t=1.0 → white (1,1,1) + RGB3 c1 = lerp_stops(stops, 1.0f); + assert(c1.r > 0.99f && c1.g > 0.99f && c1.b > 0.99f); + + // t=0.5 → midpoint (0.5, 0.5, 0.5) within floating-point tolerance + RGB3 c5 = lerp_stops(stops, 0.5f); + assert(c5.r > 0.49f && c5.r < 0.51f); + assert(c5.g > 0.49f && c5.g < 0.51f); + assert(c5.b > 0.49f && c5.b < 0.51f); + + std::printf("PASS: TestColorScaleLerpTwoStops " + "(t=0→black, t=1→white, t=0.5→mid-grey)\n"); +} + +// --------------------------------------------------------------------------- +// Test 13: TestColorScaleLerpThreeStops — t=0.25 between stop0 and stop1. +// Stops: {0.0,red}, {0.5,green}, {1.0,blue}. +// At t=0.25 we expect halfway between red and green. +// --------------------------------------------------------------------------- +static void test_color_scale_lerp_three_stops() { + // red=#ff0000, green=#00ff00, blue=#0000ff + std::vector stops = { + {0.0f, "#ff0000"}, // red + {0.5f, "#00ff00"}, // green + {1.0f, "#0000ff"}, // blue + }; + + // t=0.25 is halfway between stop0 (t=0) and stop1 (t=0.5). + // Lerp factor f = (0.25 - 0.0) / (0.5 - 0.0) = 0.5. + // Expected: R = lerp(1,0,0.5)=0.5, G = lerp(0,1,0.5)=0.5, B = lerp(0,0,0.5)=0. + RGB3 ca = lerp_stops(stops, 0.25f); + assert(ca.r > 0.49f && ca.r < 0.51f && "R should be ~0.5 at t=0.25"); + assert(ca.g > 0.49f && ca.g < 0.51f && "G should be ~0.5 at t=0.25"); + assert(ca.b < 0.01f && "B should be ~0 at t=0.25"); + + // t=0.75 is halfway between stop1 (t=0.5) and stop2 (t=1.0). + // Expected: R=0, G=0.5, B=0.5. + RGB3 cb = lerp_stops(stops, 0.75f); + assert(cb.r < 0.01f && "R should be ~0 at t=0.75"); + assert(cb.g > 0.49f && cb.g < 0.51f && "G should be ~0.5 at t=0.75"); + assert(cb.b > 0.49f && cb.b < 0.51f && "B should be ~0.5 at t=0.75"); + + std::printf("PASS: TestColorScaleLerpThreeStops " + "(t=0.25 between stop0/stop1, t=0.75 between stop1/stop2)\n"); +} + +// --------------------------------------------------------------------------- +// Test 14: TestColorScaleOutOfRange — t<0 saturates at first; t>1 at last. +// --------------------------------------------------------------------------- +static void test_color_scale_out_of_range() { + std::vector stops = { + {0.0f, "#ff0000"}, // red at t=0 + {1.0f, "#0000ff"}, // blue at t=1 + }; + + // t=-0.5 → clamp to 0 → red + RGB3 cu = lerp_stops(stops, -0.5f); + assert(cu.r > 0.99f && "under-range should saturate at first stop (red)"); + assert(cu.b < 0.01f); + + // t=1.5 → clamp to 1 → blue + RGB3 co = lerp_stops(stops, 1.5f); + assert(co.r < 0.01f && "over-range should saturate at last stop (blue)"); + assert(co.b > 0.99f); + + // ColumnSpec struct fields accessible and defaults sensible. + ColumnSpec cs; + cs.renderer = CellRenderer::ColorScale; + cs.range_min = -10.0; + cs.range_max = 10.0; + assert(cs.range_alpha == 0.25f && "default range_alpha should be 0.25"); + assert(cs.range_stops.empty() && "default range_stops should be empty (→ use default gradient)"); + + std::printf("PASS: TestColorScaleOutOfRange " + "(t<0 saturates at first stop, t>1 saturates at last stop)\n"); +} + // --------------------------------------------------------------------------- // main // --------------------------------------------------------------------------- @@ -411,6 +619,10 @@ int main() { test_render_backcompat_overload(); test_dots_column_spec(); test_dots_tql_roundtrip(); - std::printf("=== ALL TESTS PASSED (10/10) ===\n"); + test_categorical_chip_rule(); + test_color_scale_lerp_two_stops(); + test_color_scale_lerp_three_stops(); + test_color_scale_out_of_range(); + std::printf("=== ALL TESTS PASSED (14/14) ===\n"); return 0; } diff --git a/cpp/tests/test_fn_table_viz_smoke.cpp b/cpp/tests/test_fn_table_viz_smoke.cpp index e4d34469..f4e867c2 100644 --- a/cpp/tests/test_fn_table_viz_smoke.cpp +++ b/cpp/tests/test_fn_table_viz_smoke.cpp @@ -14,7 +14,7 @@ #include "core/auto_detect_type.h" #include "core/compute_column_stats.h" #include "viz/viz_render.h" -#include "viz/data_table.h" +#include "data_table/data_table.h" #include #include diff --git a/dev/flows/0001-hn-top-stories.md b/dev/flows/0001-hn-top-stories.md index 1cf2c8a6..6ddf84bd 100644 --- a/dev/flows/0001-hn-top-stories.md +++ b/dev/flows/0001-hn-top-stories.md @@ -68,6 +68,33 @@ Probar end-to-end el stack: navegator AutoExtract -> recipe -> dag_engine schedu - `dag_engine.dag_step_results`: step `extract` con `function_id='cdp_extract_recipe_py_pipelines'`. - `call_monitor.calls`: chain function call. +## Definition of Done + +Ver `README.md` seccion DoD + user-facing. + +### Generico + +- [ ] **Repetibilidad**: corre 3 veces consecutivas via cron sin intervencion. +- [ ] **Observabilidad**: `call_monitor.calls` registra `cdp_extract_recipe_py_pipelines` + `data_factory.runs` muestra `node_id=hn_top_stories`. +- [ ] **Error-path**: si Chrome :9222 cae, el step falla con mensaje claro (no crash silencioso del DAG). +- [ ] **Idempotencia**: dedup `dedup_duckdb_table_by_hash_py_pipelines` corre tras extract; mismo HTML 2x = 0 filas nuevas. +- [ ] **Secrets**: N/A (HN publico). +- [ ] **Docs**: `## Notas` con comandos para reproducir + onboarding. +- [ ] **Registry-first**: extract sin codigo inline en el DAG. +- [ ] **INDEX + status**: `status: done` + `INDEX.md` + movido a `completed/`. + +### User-facing + +- [ ] **User-facing**: usuario abre `data_factory.exe` → tab "All Runs" filtra `node_id=hn_top_stories` → ve >=30 filas con rank/title/url/points. +- [ ] **User-facing repeat**: vuelve manana al mismo tab, ve runs frescos (cada 30 min) y tabla actualizada. +- [ ] **User-facing onboarding**: parrafo en `## Notas`: "Para ver HN top: lanzar `data_factory.exe` → tab Extractors → `hn_top_stories`. DuckDB en `apps/data_factory/data/hn_top_stories.duckdb` tabla `hn_stories`." +- [ ] **User-facing latencia**: cron `*/30 * * * *` → datos frescos en <31 min p95. + +### Custom + +- [ ] 7/7 campos cubiertos en TODOS los runs ultimas 24h (rank/title/url/points/author/age/comments). +- [ ] Latencia extract <30s p95 (cdp_extract_recipe + render). + ## Notas (rellenas tras correr) diff --git a/dev/flows/0002-aemet-madrid.md b/dev/flows/0002-aemet-madrid.md index 92774910..57ff1c31 100644 --- a/dev/flows/0002-aemet-madrid.md +++ b/dev/flows/0002-aemet-madrid.md @@ -63,6 +63,33 @@ Probar path HTTP-only (sin Chrome/CDP). Extractor REST -> data_factory -> sink g - `data_factory.runs`: 24 entries/dia. - `data_factory.databases.last_seen_at` actualizado por sink. +## Definition of Done + +Ver `README.md` seccion DoD + user-facing. + +### Generico + +- [ ] **Repetibilidad**: cron `0 * * * *` corre 3h consecutivas sin error. +- [ ] **Observabilidad**: extractor en `call_monitor.calls`, runs en `data_factory.runs`, fila en `databases.last_seen_at`. +- [ ] **Error-path**: AEMET 5xx → 3 reintentos exp-backoff, despues marca run failed (no crash). +- [ ] **Idempotencia**: re-run mismo timestamp = upsert PostGIS, sin duplicar puntos. +- [ ] **Secrets**: API key AEMET en `pass aemet/api-key`, nunca en el DAG. +- [ ] **Docs**: `## Notas` con comandos + onboarding. +- [ ] **Registry-first**: extractor AEMET creado como funcion del registry (`aemet_get_madrid_observations_py_*` o reuso de `http_get_json_*`), nada inline. +- [ ] **INDEX + status**: `status: done` + INDEX + movido. + +### User-facing + +- [ ] **User-facing**: usuario abre `footprint_geo_stack` → preset `madrid-weather` → ve overlay tiles con puntos meteo + tooltip (temp/humidity). +- [ ] **User-facing repeat**: mismo preset manana muestra datos refrescados; tooltip ultima hora. +- [ ] **User-facing onboarding**: parrafo en `## Notas`: "Para ver weather Madrid: `footprint_geo_stack.exe` → File → Open preset `madrid-weather`. Tile server local en :3000." +- [ ] **User-facing latencia**: cron 1h → mapa refleja datos en <61 min. + +### Custom + +- [ ] PostGIS schema via `migrations/NNN_*.sql` (no `CREATE TABLE` inline). +- [ ] Tile overlay sirve en <3s desde click. + ## Notas - Sin LLM/CDP. Mas barato que flow 0001. diff --git a/dev/flows/0003-bbva-movimientos.md b/dev/flows/0003-bbva-movimientos.md index 77fe6893..8e377e51 100644 --- a/dev/flows/0003-bbva-movimientos.md +++ b/dev/flows/0003-bbva-movimientos.md @@ -61,6 +61,36 @@ Caso de uso REAL con auth + datos sensibles. Probar persistencia local (duckdb e - `data_factory.runs`: 1 entry status=success. - `auto_metabase`: 1 card creado. +## Definition of Done + +Ver `README.md` seccion DoD + user-facing. **Risk=high** -> DoD strict obligatorio. + +### Generico + +- [ ] **Repetibilidad**: re-login + extraccion mensual reproducible (no flaky por DOM changes inesperados). +- [ ] **Observabilidad**: `call_monitor.calls` muestra ejecucion sin valores; `data_factory.runs` registra ambos nodos. +- [ ] **Error-path**: sesion expirada → mensaje claro al usuario para re-login (no datos corruptos). +- [ ] **Idempotencia**: re-extraer mismo mes = upsert por `movimiento_id`, 0 duplicados. +- [ ] **Secrets**: credenciales BBVA solo en `pass bbva/login`; vault `~/vaults/finanzas/` gitignored verificado. +- [ ] **Docs**: `## Notas` con onboarding + procedimiento de rotacion mensual. +- [ ] **Registry-first**: recipe + persistencia duckdb usan funciones del registry. +- [ ] **INDEX + status**: `status: done` + INDEX + movido. + +### User-facing + +- [ ] **User-facing**: usuario abre Metabase LOCAL :3000 → dashboard `Finanzas personales` → card `Gasto mensual` con grafico actualizado. +- [ ] **User-facing repeat**: misma URL manana muestra movimientos del mes hasta hoy; despues de re-login mensual, mes nuevo aparece automatico. +- [ ] **User-facing onboarding**: parrafo en `## Notas`: "Para revisar gasto: abrir http://localhost:3000 (creds en `pass metabase/local`) → dashboard `Finanzas personales`. Re-login BBVA: lanzar navegator → recipe `bbva_movimientos` → click Run." +- [ ] **User-facing latencia**: tras re-login + run manual, card actualizada en <2 min. + +### Custom (risk=high) + +- [ ] **No-leak**: `fn sync` NO sube duckdb (verificado: `pc_locations` registra path, sync no transmite bytes). +- [ ] **No-leak**: recipe extrae solo campos minimos (fecha, concepto, importe, categoria); NO DNI, NO saldo, NO IBAN completo. +- [ ] **No-leak**: Metabase corre LOCAL; verificar `auto_metabase.app.md` declara `tags: [local-only]`. +- [ ] **Rotacion**: re-login mensual probado sin perder datos historicos. +- [ ] **Red-team**: ningun log/screenshot/traza del flow contiene valores sensibles (grep IBAN/saldo en `call_monitor.calls`, `data_factory.runs`, `~/.cache/`). + ## Notas - **NO commitear** `~/vaults/finanzas/` (gitignored por defecto). diff --git a/dev/flows/0004-gitea-releases-monitor.md b/dev/flows/0004-gitea-releases-monitor.md index 37b563b6..e02059b6 100644 --- a/dev/flows/0004-gitea-releases-monitor.md +++ b/dev/flows/0004-gitea-releases-monitor.md @@ -61,6 +61,34 @@ Probar webhooks como trigger (no cron, no manual). Cada push a un repo `dataforg - Por repo: 1 nodo extractor. - Matrix: 1 msg por push. +## Definition of Done + +Ver `README.md` seccion DoD + user-facing. + +### Generico + +- [ ] **Repetibilidad**: 3 pushes test distintos disparan 3 mensajes Matrix sin intervencion. +- [ ] **Observabilidad**: `data_factory.runs` con `trigger=webhook` + `call_monitor.calls` chain por push. +- [ ] **Error-path**: payload invalido → 4xx + entry en log + NO crash receptor. +- [ ] **Idempotencia**: recepcion duplicada (Gitea retry) → 1 mensaje, no N. +- [ ] **Secrets**: webhook secret en `pass gitea/webhook-secret`; HMAC verificado por receptor. +- [ ] **Docs**: `## Notas` con setup webhook + onboarding. +- [ ] **Registry-first**: receptor reusa `http_post_json_*` + `matrix_send_message_*`. +- [ ] **INDEX + status**: `status: done` + INDEX + movido. + +### User-facing + +- [ ] **User-facing**: usuario lee mensaje en sala Matrix `#fn-registry-news` con formato `[] pushed commits to ` + link al commit. +- [ ] **User-facing repeat**: cada push real a `dataforge/*` dispara mensaje; sala es la fuente diaria de actividad multi-repo. +- [ ] **User-facing onboarding**: parrafo en `## Notas`: "Para enterarse de pushes: unirse a sala Matrix `#fn-registry-news`. Para anadir un repo nuevo: `gitea_create_webhook_bash_infra `." +- [ ] **User-facing latencia**: push → mensaje en <5s p95. + +### Custom + +- [ ] >=3 repos cubiertos (no solo 1). +- [ ] Rate-limit: max 1 mensaje/repo/minuto (no flood si N pushes seguidos). +- [ ] Health endpoint `/webhook/health` retorna 200 + lista repos suscritos. + ## Notas - Webhook secret debe estar en `pass gitea/webhook-secret` o env var. diff --git a/dev/flows/0005-osint-person-lookup.md b/dev/flows/0005-osint-person-lookup.md index 09cedfaa..cb7f495f 100644 --- a/dev/flows/0005-osint-person-lookup.md +++ b/dev/flows/0005-osint-person-lookup.md @@ -58,6 +58,35 @@ Probar paralelismo (multiples scraping jobs concurrentes) + agregacion a grafo. - `operations.db` de osint_graph: entities += N, relations += N. - `function_stats.claude_cli_prompt_py_infra`: calls += 1. +## Definition of Done + +Ver `README.md` seccion DoD + user-facing. **Risk=medium** -> attention en datos personales. + +### Generico + +- [ ] **Repetibilidad**: 3 lookups distintos (3 personas test) producen reports completos sin re-config. +- [ ] **Observabilidad**: 3 jobs visibles en `odr_console.operations.db` + `call_monitor.calls` chain por job. +- [ ] **Error-path**: si LinkedIn devuelve 429 → job marcado failed, otros 2 continuan (no aborta el flow entero). +- [ ] **Idempotencia**: re-lookup misma persona → upsert por `snippet_hash`, no duplica nodos Person. +- [ ] **Secrets**: creds Twitter/GitHub en `pass`; LinkedIn usa sesion del navegador (cookie via navegator). +- [ ] **Docs**: `## Notas` con onboarding + check legal. +- [ ] **Registry-first**: recipes + agregacion + render reusan funciones registry. +- [ ] **INDEX + status**: `status: done` + INDEX + movido. + +### User-facing + +- [ ] **User-facing**: usuario abre `graph_explorer.exe` → File → Load dataset `osint/` → ve grafo Person + N Snippets navegable (zoom, click → snippet content). +- [ ] **User-facing repeat**: persona nueva → comando lanza job, dataset aparece en lista de graph_explorer en <5min. +- [ ] **User-facing onboarding**: parrafo en `## Notas`: "Para investigar persona: `/flow run 0005 --target ''` (o `odr_console.exe` → New Job → 3 recipes). Esperar ~5min. Abrir `graph_explorer.exe` → Load `osint/`. Resumen LLM en `report.md` del repo." +- [ ] **User-facing latencia**: job lanzado → grafo listo en <5min (3 jobs paralelos). + +### Custom + +- [ ] Paralelismo medido: 3 jobs concurrentes <60s wall vs ~180s en serie. +- [ ] Race-condition test: 2 corridas simultaneas del flow no corrompen operations.db. +- [ ] Red-team: nada de menores/info no publica en snippets capturados. +- [ ] Report `.md` firmado por commit en repo `osint_graph`. + ## Notas - Consideracion legal: extracciones publicas (perfiles abiertos). NO bypassear paywalls/captchas. diff --git a/dev/flows/0006-metabase-versioning.md b/dev/flows/0006-metabase-versioning.md index cea660c5..87fdb0c8 100644 --- a/dev/flows/0006-metabase-versioning.md +++ b/dev/flows/0006-metabase-versioning.md @@ -72,6 +72,35 @@ Probar flujo INVERSO al tipico: extraer estado de un servicio interno (Metabase) - 1 run/dia en data_factory. - 7 commits en metabase_registry repo (1 semana baseline). +## Definition of Done + +Ver `README.md` seccion DoD + user-facing. + +### Generico + +- [ ] **Repetibilidad**: cron diario 02:00 corre 7 dias consecutivos sin error. +- [ ] **Observabilidad**: `data_factory.runs` + 1 commit en repo `metabase_registry` por dia (o `NO_CHANGES`). +- [ ] **Error-path**: token Metabase expirado → healthcheck pre-pull falla con mensaje claro, no silencio. +- [ ] **Idempotencia**: NO_CHANGES no genera commit vacio en git. +- [ ] **Secrets**: token Metabase en `pass metabase/api-token`. +- [ ] **Docs**: `## Notas` con onboarding + rollback procedure. +- [ ] **Registry-first**: pull/diff/push reusan funciones registry. +- [ ] **INDEX + status**: `status: done` + INDEX + movido. + +### User-facing + +- [ ] **User-facing**: usuario navega a `https://gitea.../dataforge/metabase_registry/commits/master` → ve commits diarios con diff YAML de dashboards/cards. +- [ ] **User-facing repeat**: misma URL manana muestra commit nuevo (o `NO_CHANGES` skip); rollback con click derecho en commit → restore. +- [ ] **User-facing onboarding**: parrafo en `## Notas`: "Para auditar cambios Metabase: abrir Gitea repo `dataforge/metabase_registry`. Rollback: revertir commit en Gitea → push trigger DAG manual → Metabase restaurado. Matrix bot diario en `#fn-registry-ops` a las 09:00." +- [ ] **User-facing latencia**: cambio manual en Metabase → commit visible al dia siguiente 02:00. + +### Custom + +- [ ] Rollback E2E probado: revertir commit → siguiente run aplica YAML viejo → Metabase restaura dashboard. +- [ ] Diff YAML estable: keys ordenadas, no churn aleatorio. +- [ ] Dashboards eliminados → commit `DELETED:`, no tombstone huerfano. +- [ ] Backup adicional a vault (no solo git). + ## Notas - Riesgo: si Metabase token expira, el DAG falla silenciosamente. Anadir healthcheck pre-pull. diff --git a/dev/flows/0007-matrix-telemetry-bot.md b/dev/flows/0007-matrix-telemetry-bot.md index 7692a473..dfaf45bb 100644 --- a/dev/flows/0007-matrix-telemetry-bot.md +++ b/dev/flows/0007-matrix-telemetry-bot.md @@ -71,6 +71,34 @@ Tres triggers distintos, mismo sink. - `function_stats.matrix_send_message_*`: calls dependientes de eventos reales. - Sala Matrix recibe mensajes de los 3 origenes. +## Definition of Done + +Ver `README.md` seccion DoD + user-facing. + +### Generico + +- [ ] **Repetibilidad**: 3 triggers (node fail / DAG fail / violations) disparan mensaje cada uno, 100x sin perdida. +- [ ] **Observabilidad**: cada envio en `call_monitor.calls`; cola persistente registra envios pendientes si Matrix down. +- [ ] **Error-path**: Matrix down → cola en operations.db; al reconectar drena en orden. +- [ ] **Idempotencia**: dedup: misma alerta 10x en 1min → 1 mensaje agregado, no flood. +- [ ] **Secrets**: bot token en `pass matrix/bot-token`. +- [ ] **Docs**: `## Notas` con onboarding + comandos para provocar trigger de prueba. +- [ ] **Registry-first**: `matrix_send_message_py_infra` registrado + reusado. +- [ ] **INDEX + status**: `status: done` + INDEX + movido. + +### User-facing + +- [ ] **User-facing**: usuario lee alerta en sala Matrix `#fn-registry-ops` con prefix emoji severidad + nombre app + link al dashboard de la app fallida. +- [ ] **User-facing repeat**: cada fallo real dispara mensaje en la sala; sala es el feed de salud diario del stack. +- [ ] **User-facing onboarding**: parrafo en `## Notas`: "Para enterarse de fallos: unirse a `#fn-registry-ops` (creds Matrix en `pass matrix/user`). Heartbeat 09:00 confirma bot vivo. Probar trigger: `./fn run inject_synthetic_violation`." +- [ ] **User-facing latencia**: evento → mensaje en <3s p95 (medido sobre 100 envios). + +### Custom + +- [ ] Severity routing: `critical` → `#fn-registry-ops`; `warning` → `#fn-registry-dev`. +- [ ] Self-test diario 09:00: bot envia heartbeat si vivo; ausencia heartbeat = alerta meta. +- [ ] Mensaje formateado con link al dashboard (no solo texto plano). + ## Notas - Throttling: max 1 mensaje/minuto por origen para evitar spam. diff --git a/dev/flows/AGENT_GUIDE.md b/dev/flows/AGENT_GUIDE.md index 11ef0174..a6ce8a53 100644 --- a/dev/flows/AGENT_GUIDE.md +++ b/dev/flows/AGENT_GUIDE.md @@ -21,6 +21,13 @@ Al recibir "crea flow para " o `/flow create `: 4. **Marca riesgo** (low/medium/high) por sensibilidad de datos. 5. **Sugiere schedule** (cron / webhook / manual) basado en el tipo de fuente. 6. **Sugiere apps** del stack que encajan, sin inflar — solo las que realmente tocara. +7. **REDACTA `## Definition of Done` OBLIGATORIO**. No scaffold sin DoD. Empieza por la plantilla minima del `README.md` y anade DoD especificos al dominio del flow (ej. "datos NO viajan a registry.organic-machine", "geo: tiles sirven en <3s", "matrix bot tarda <5s en mensaje"). Acceptance != DoD: Acceptance = "corre"; DoD = "esta listo para vivir solo". +8. **DECLARA USER-FACING SURFACE**. Dentro del DoD, los 4 checks `User-facing` son OBLIGATORIOS y concretos. Responde antes de scaffold: + - **donde** lo ve el humano? (app concreta + tab/panel, sala Matrix, dashboard URL, Metabase card, repo Gitea con commits, archivo en vault). NUNCA "en la BD" o "en un log". + - **cuanto tarda** en aparecer? (declara latencia en segundos/minutos). + - **como vuelve** a verlo manana? (URL bookmark? slash command? cron + dashboard?). + - **que parrafo onboarding** ira en `## Notas` para que un humano nuevo lo use sin leer el flow. + Si la unica respuesta es "lo consume otro flow/app", devuelve el flow a borrador — falta superficie humana. ## Mapa de discovery — donde mirar para cada decision diff --git a/dev/flows/INDEX.md b/dev/flows/INDEX.md index 4a2e11a7..f1232c45 100644 --- a/dev/flows/INDEX.md +++ b/dev/flows/INDEX.md @@ -2,20 +2,22 @@ Tabla de casos de uso multi-app. Mantenida por `/flow create` y `/flow done`. -| ID | Slug | Apps | Status | Risk | Updated | -|----|------|------|--------|------|---------| -| [0001](0001-hn-top-stories.md) | hn-top-stories | navegator_dashboard, dag_engine, data_factory, agents_and_robots | pending | low | 2026-05-16 | -| [0002](0002-aemet-madrid.md) | aemet-madrid | dag_engine, data_factory, footprint_geo_stack | pending | low | 2026-05-16 | -| [0003](0003-bbva-movimientos.md) | bbva-movimientos | navegator_dashboard, dag_engine, data_factory, auto_metabase | pending | high | 2026-05-16 | -| [0004](0004-gitea-releases-monitor.md) | gitea-releases-monitor | registry_api, data_factory, agents_and_robots | pending | low | 2026-05-16 | -| [0005](0005-osint-person-lookup.md) | osint-person-lookup | navegator_dashboard, odr_console, graph_explorer | pending | medium | 2026-05-16 | -| [0006](0006-metabase-versioning.md) | metabase-versioning | auto_metabase, dag_engine | pending | medium | 2026-05-16 | -| [0007](0007-matrix-telemetry-bot.md) | matrix-telemetry-bot | data_factory, dag_engine, call_monitor, agents_and_robots | pending | low | 2026-05-16 | +| ID | Slug | Pattern | Apps | Status | Risk | DoD % | Updated | +|----|------|---------|------|--------|------|-------|---------| +| [0001](0001-hn-top-stories.md) | hn-top-stories | smoke-cron | navegator_dashboard, dag_engine, data_factory, agents_and_robots | pending | low | 0% | 2026-05-16 | +| [0002](0002-aemet-madrid.md) | aemet-madrid | smoke-cron | dag_engine, data_factory, footprint_geo_stack | pending | low | 0% | 2026-05-16 | +| [0003](0003-bbva-movimientos.md) | bbva-movimientos | prod-data | navegator_dashboard, dag_engine, data_factory, auto_metabase | pending | high | 0% | 2026-05-16 | +| [0004](0004-gitea-releases-monitor.md) | gitea-releases-monitor | event-driven | registry_api, data_factory, agents_and_robots | pending | low | 0% | 2026-05-16 | +| [0005](0005-osint-person-lookup.md) | osint-person-lookup | manual-deep | navegator_dashboard, odr_console, graph_explorer | pending | medium | 0% | 2026-05-16 | +| [0006](0006-metabase-versioning.md) | metabase-versioning | gitops | auto_metabase, dag_engine | pending | medium | 0% | 2026-05-16 | +| [0007](0007-matrix-telemetry-bot.md) | matrix-telemetry-bot | event-driven | data_factory, dag_engine, call_monitor, agents_and_robots | pending | low | 0% | 2026-05-16 | ## Leyenda - **Status**: `pending` (no arrancado) / `running` / `done` / `failed` / `deferred`. - **Risk**: `low` (datos publicos), `medium` (auth pero no sensible), `high` (datos personales/financieros). +- **Pattern**: `smoke-cron` / `prod-data` / `event-driven` / `manual-deep` / `gitops` / `realtime-loop` — ver `README.md`. +- **DoD %**: ratio de checks `[x]` en el bloque `## Definition of Done` del flow. `/flow done` exige 100%. ## Completados diff --git a/dev/flows/README.md b/dev/flows/README.md index f9bdf469..5c6c9569 100644 --- a/dev/flows/README.md +++ b/dev/flows/README.md @@ -9,8 +9,53 @@ Un flow describe una secuencia de pasos que atraviesa varias apps (`navegator_da - Archivo por flow: `NNNN-.md` (numeracion zero-padded propia, NO comparte con `dev/issues/`). - Estado vivo en frontmatter (`status`). - Acceptance checkboxes `[ ]` en el body — `/flow status` calcula % completado. +- **Definition of Done OBLIGATORIA** — ver seccion abajo. Sin DoD el flow NO puede crearse. - Cerrados se mueven a `completed/`. +## Definition of Done (OBLIGATORIA) + +Cada flow al crearse DEBE declarar un bloque `## Definition of Done` distinto de `## Acceptance`. Sin el, `/flow create` rechaza el scaffold y `/flow done` rechaza el cierre. + +**Diferencia:** + +| `## Acceptance` | `## Definition of Done` | +|---|---| +| Checks task-level del flow (ejecucion concreta) | Contrato global de calidad para considerar el flow CERRADO | +| Pueden quedar `[ ]` mientras iteras | TODOS deben estar `[x]` antes de mover a `completed/` | +| Verifica que el flow CORRE | Verifica que el flow es REPETIBLE, OBSERVABLE y MANTENIBLE | + +**Plantilla minima de DoD** (anadir/ajustar segun flow): + +```markdown +## Definition of Done + +- [ ] **Repetibilidad**: el flow corre N veces consecutivas (N declarado en el flow, default 3) sin intervencion manual. +- [ ] **Observabilidad**: queda trazado en `call_monitor.calls` + `data_factory.runs` + dashboard correspondiente. +- [ ] **Error-path**: al menos 1 modo de fallo probado y manejado (no crash silencioso). +- [ ] **Idempotencia**: re-ejecutar no duplica datos ni rompe estado (clave en sinks). +- [ ] **Secrets**: cero credenciales en disco fuera de `pass`/vaults; cero datos sensibles fuera de `risk` declarado. +- [ ] **Docs**: `## Notas` rellenado con hallazgos reales + comandos para reproducir. +- [ ] **Registry-first**: todas las piezas reutilizables existen como funciones del registry (no inline en apps). +- [ ] **INDEX + status**: `status: done` en frontmatter + fila actualizada en `INDEX.md` + archivo movido a `completed/`. +``` + +Cada flow puede anadir DoD especificos al dominio (ej. `bbva-movimientos`: "datos NUNCA cruzan a registry.organic-machine"). El bloque DoD se **versiona con el flow** — un cambio de DoD requiere bump de `updated:` en frontmatter. + +### User-facing surface (sub-bloque OBLIGATORIO dentro de DoD) + +"DoD verde" sin valor visible al humano = plumbing limpio sin razon de existir. Cada DoD DEBE incluir, al menos, estos cuatro checks tipo `User-facing`: + +```markdown +- [ ] **User-facing**: . +- [ ] **User-facing repeat**: el humano vuelve manana al mismo lugar y ve datos frescos sin conocer el flow. +- [ ] **User-facing onboarding**: parrafo en `## Notas` que explica "para ver/usar esto: hacer X" — sin leer el flow. +- [ ] **User-facing latencia**: el humano percibe el cambio en tras el evento (X declarado por flow). +``` + +Regla: si la respuesta a "donde lo ves" es "en una BD" o "en un log" -> NO vale. Tiene que ser una superficie usada por el humano (UI de una app, sala Matrix, dashboard, Metabase card, repo Gitea, archivo en vault abierto a mano). Si el output solo lo consume otra app/flow, esa app/flow es quien debe declarar su propia user-facing surface. + +`/flow done` rechaza el cierre si falta alguno de los 4 user-facing checks o si `## Notas` no contiene parrafo onboarding. + ## Para agentes / LLMs Antes de crear o editar un flow, lee `AGENT_GUIDE.md`. Define: diff --git a/dev/flows/template.md b/dev/flows/template.md index ae27cdff..fc8b1761 100644 --- a/dev/flows/template.md +++ b/dev/flows/template.md @@ -71,6 +71,30 @@ Pasos numerados. Cada paso puede ser: - [ ] Criterio 1. - [ ] Criterio 2. +## Definition of Done + +Contrato global de cierre. TODOS marcados antes de mover a `completed/`. Ver README.md seccion "Definition of Done". + +- [ ] **Repetibilidad**: corre 3 veces consecutivas sin intervencion manual. +- [ ] **Observabilidad**: trazado en `call_monitor.calls` + `data_factory.runs` + dashboard relevante. +- [ ] **Error-path**: >=1 modo de fallo probado y manejado. +- [ ] **Idempotencia**: re-ejecucion no duplica ni corrompe sinks. +- [ ] **Secrets**: cero credenciales fuera de `pass`/vaults; risk declarado coincide con datos reales. +- [ ] **Docs**: `## Notas` con hallazgos + comandos reproducibles. +- [ ] **Registry-first**: piezas reutilizables viven como funciones del registry. +- [ ] **INDEX + status**: `status: done` + `INDEX.md` actualizado + movido a `completed/`. + +### User-facing (obligatorio) + +- [ ] **User-facing**: . +- [ ] **User-facing repeat**: humano vuelve manana al mismo lugar, ve datos frescos sin conocer el flow. +- [ ] **User-facing onboarding**: parrafo en `## Notas` explica "para ver/usar esto: hacer X" sin leer el flow. +- [ ] **User-facing latencia**: humano percibe el cambio en tras el evento (X declarado). + +### Custom (opcional, dominio-especifico) + +- [ ] _(custom)_ . + ## Telemetria esperada - `call_monitor.calls`: que aparece. diff --git a/dev/gen_app_icons.py b/dev/gen_app_icons.py deleted file mode 100644 index 4d56c415..00000000 --- a/dev/gen_app_icons.py +++ /dev/null @@ -1,118 +0,0 @@ -#!/usr/bin/env python3 -""" -Generador de iconos .ico para apps C++ del registry. -Toma SVG phosphor → renderiza con cairosvg → compone fondo redondeado + glyph -blanco → exporta .ico multi-resolucion (16,24,32,48,64,128,256) en -/appicon.ico. - -Mapping: APPS = [(app_id, dir, phosphor_icon, accent_hex)] -""" -import io -import os -import sys -from pathlib import Path -import cairosvg -from PIL import Image, ImageDraw - -REGISTRY_ROOT = Path(__file__).resolve().parent.parent -PHOSPHOR_FILL = REGISTRY_ROOT / "sources/phosphor-core/assets/fill" - -APPS = [ - ("altsnap_jitter_test", "apps/altsnap_jitter_test", "arrows-clockwise", "#dc2626"), - ("chart_demo", "apps/chart_demo", "chart-bar", "#0ea5e9"), - ("dag_engine_ui", "apps/dag_engine_ui", "tree-structure", "#7c3aed"), - ("data_factory", "apps/data_factory", "factory", "#f97316"), - ("engine_smoke", "apps/engine_smoke", "game-controller", "#16a34a"), - ("graph_explorer", "projects/osint_graph/apps/graph_explorer", "graph", "#0891b2"), - ("navegator_dashboard", "projects/navegator/apps/navegator_dashboard", "compass", "#2563eb"), - ("odr_console", "projects/online_data_recopilation/apps/odr_console", "terminal-window", "#52525b"), - ("primitives_gallery", "apps/primitives_gallery", "shapes", "#db2777"), - ("registry_dashboard", "projects/fn_monitoring/apps/registry_dashboard", "gauge", "#059669"), - ("runtime_test", "apps/runtime_test", "flask", "#9333ea"), - ("shaders_lab", "apps/shaders_lab", "palette", "#ea580c"), - ("text_editor_smoke", "apps/text_editor_smoke", "note-pencil", "#0d9488"), -] - -ICON_SIZES = [16, 24, 32, 48, 64, 128, 256] -RENDER_SIZE = 256 # canvas of reference, downscaled to each .ico size - - -def hex_to_rgb(h: str) -> tuple[int, int, int]: - h = h.lstrip("#") - return tuple(int(h[i : i + 2], 16) for i in (0, 2, 4)) - - -def render_glyph_white(svg_path: Path, size: int) -> Image.Image: - """Render phosphor SVG as white-on-transparent at given size.""" - svg = svg_path.read_text() - # phosphor uses fill="currentColor". Force white. - svg = svg.replace('fill="currentColor"', 'fill="#ffffff"') - png_bytes = cairosvg.svg2png( - bytestring=svg.encode("utf-8"), - output_width=size, - output_height=size, - ) - return Image.open(io.BytesIO(png_bytes)).convert("RGBA") - - -def make_icon_image(svg_path: Path, accent_hex: str, size: int) -> Image.Image: - """Compose: rounded-square accent background + centered white glyph.""" - bg_color = hex_to_rgb(accent_hex) + (255,) - canvas = Image.new("RGBA", (size, size), (0, 0, 0, 0)) - draw = ImageDraw.Draw(canvas) - radius = max(2, size // 6) # ~16% rounded corners - draw.rounded_rectangle( - [(0, 0), (size - 1, size - 1)], - radius=radius, - fill=bg_color, - ) - # Glyph occupies inner ~70% (padding ~15% all around). - glyph_size = int(size * 0.7) - if glyph_size < 8: - glyph_size = max(8, size - 2) - glyph = render_glyph_white(svg_path, glyph_size) - off = ((size - glyph_size) // 2, (size - glyph_size) // 2) - canvas.alpha_composite(glyph, dest=off) - return canvas - - -def build_ico(app_id: str, app_dir: Path, phosphor_name: str, accent_hex: str) -> Path: - svg_file = PHOSPHOR_FILL / f"{phosphor_name}-fill.svg" - if not svg_file.exists(): - raise FileNotFoundError(f"phosphor icon not found: {svg_file}") - # Render the highest-quality image (256) and let Pillow downscale via `sizes`. - # Using append_images with custom-rendered per-size variants preserves - # crispness of the phosphor glyph at small sizes (16/24). - images = {s: make_icon_image(svg_file, accent_hex, s) for s in ICON_SIZES} - out = app_dir / "appicon.ico" - out.parent.mkdir(parents=True, exist_ok=True) - biggest = images[max(ICON_SIZES)] - others = [images[s] for s in ICON_SIZES if s != max(ICON_SIZES)] - biggest.save( - out, - format="ICO", - sizes=[(s, s) for s in ICON_SIZES], - append_images=others, - ) - return out - - -def main() -> int: - errors = 0 - for app_id, rel_dir, phosphor_name, accent_hex in APPS: - app_dir = REGISTRY_ROOT / rel_dir - if not app_dir.exists(): - print(f"SKIP {app_id}: dir not found ({rel_dir})", file=sys.stderr) - errors += 1 - continue - try: - out = build_ico(app_id, app_dir, phosphor_name, accent_hex) - print(f"OK {app_id:25s} -> {out.relative_to(REGISTRY_ROOT)} ({phosphor_name}, {accent_hex})") - except Exception as e: # pragma: no cover - reporting only - print(f"FAIL {app_id}: {e}", file=sys.stderr) - errors += 1 - return 1 if errors else 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/dev/issues/0100-issues-frontmatter-migration.md b/dev/issues/0100-issues-frontmatter-migration.md new file mode 100644 index 00000000..156b4b17 --- /dev/null +++ b/dev/issues/0100-issues-frontmatter-migration.md @@ -0,0 +1,105 @@ +# 0100 — Migrar frontmatter inline a YAML canonico en dev/issues/ + +**Status:** pendiente +**Created:** 2026-05-16 +**Type:** chore +**Priority:** alta +**Domain:** registry-quality +**Scope:** registry-only +**Depends:** — +**Blocks:** 0102 (work dashboard tab necesita filtros frontmatter) +**Related:** 0103 (taxonomia + slash commands) + +## Problema + +Hoy los 71 archivos `dev/issues/*.md` declaran metadata como markdown inline: + +``` +# 0099 — datahub app (launcher central) + +**Status:** pendiente +**Created:** 2026-05-16 +**Type:** app +**Priority:** alta +**Depends:** 0096 — DONE +**Blocks:** — +``` + +Imposible filtrar/agrupar sin parsers ad-hoc por linea. Issues antiguos (0027-0070) ni siquiera tienen `Type` ni `Priority`. Resultado: `/issue list` no existe; humano lee `README.md` (tabla manual) que se queda obsoleta. + +## Objetivo + +Frontmatter YAML canonico al inicio de cada issue, igual modelo que `dev/flows/`. Mantener el contenido humano intacto debajo. + +```yaml +--- +id: 0099 +title: datahub app launcher central +status: pendiente # pendiente | in-progress | bloqueado | completado | deferred +type: app # app | feature | bugfix | refactor | chore | docs | spike | epic | infra +domain: [apps-infra, cpp-stack] +scope: app-scoped # registry-only | app-scoped | multi-app | cross-stack +priority: alta # alta | media | baja +depends: [0096] +blocks: [] +related: [0095, 0097] +created: 2026-05-16 +updated: 2026-05-16 +tags: [] +--- + +# 0099 — datahub app launcher central + +(cuerpo original sin tocar) +``` + +## Pipeline propuesto + +`migrate_issues_frontmatter_bash_pipelines` (o python, lo que encaje mejor). Idempotente. + +1. Para cada `dev/issues/*.md`: + - Si ya tiene frontmatter YAML (`---` en linea 1): merge campos faltantes solo, no sobreescribe. + - Si no: parsea las lineas `**Key:** value` debajo del H1, extrae a YAML. + - Si `Type` / `Priority` ausentes: deja vacios + log warning para revision manual. + - `domain` y `scope` se infieren con heuristica por nombre/contenido (ej. `cpp-*` -> `cpp-stack`, `kanban-*` -> `kanban`, `trading-*` -> `trading`). +2. Backup en `dev/issues/.backup_pre_0100/` antes de cualquier escritura. +3. Output final: tabla de issues sin clasificar para review humano. + +## Dominios canonicos (allowlist) + +``` +meta, cpp-stack, kanban, trading, gamedev, osint, data-ingest, +registry-quality, notify, imagegen, apps-infra, dev-ux, deploy, +frontend, mcp, browser, telemetry, docs +``` + +Cualquier issue con `domain:` fuera de esta lista hace fallar el indexer. + +## Acceptance + +- [ ] Pipeline existe en `bash/functions/pipelines/` o `python/functions/pipelines/`. +- [ ] 71 issues migrados sin perder contenido (diff vs backup solo en cabecera). +- [ ] `dev/issues/README.md` ya no es fuente de verdad — se genera desde frontmatter via subcomando `/issue list` o cron diario. +- [ ] Issues completados en `dev/issues/completed/` tambien migrados. +- [ ] `fn doctor issues` (subcomando nuevo) reporta issues sin Type/Priority/Domain/Scope. +- [ ] Pipeline idempotente (segunda corrida = 0 cambios). + +## Definition of Done + +### Generico + +- [ ] **Repetibilidad**: pipeline corre N veces sin diff. +- [ ] **Observabilidad**: log de campos inferidos vs por defecto. +- [ ] **Error-path**: archivo malformado -> skip + log + exit code != 0. +- [ ] **Idempotencia**: archivo ya migrado -> 0 cambios. +- [ ] **Secrets**: N/A. +- [ ] **Docs**: README de `dev/issues/` actualizado para apuntar al frontmatter. +- [ ] **Registry-first**: pipeline reusa `parse_yaml_frontmatter_*` (crear si no existe). +- [ ] **INDEX + status**: issue cerrado + movido a `completed/`. + +### User-facing + +- [ ] **User-facing**: tras correr el pipeline, `head -20 dev/issues/0099-datahub-app-launcher.md` muestra YAML legible + `/issue show 0099` (cuando exista) imprime tabla limpia. +- [ ] **User-facing repeat**: cada issue nuevo creado con `/issue create` (issue 0101) hereda el formato. +- [ ] **User-facing onboarding**: parrafo en `dev/issues/README.md`: "Cada issue empieza con frontmatter YAML. Para ver/filtrar: `/issue list --domain trading --status pendiente`." +- [ ] **User-facing latencia**: migracion completa en <60s sobre 71 archivos. diff --git a/dev/issues/0101-dev-console-binary.md b/dev/issues/0101-dev-console-binary.md new file mode 100644 index 00000000..07361302 --- /dev/null +++ b/dev/issues/0101-dev-console-binary.md @@ -0,0 +1,100 @@ +# 0101 — dev_console Go binario: /issue /flow /work unificados + +**Status:** pendiente +**Created:** 2026-05-16 +**Type:** app +**Priority:** alta +**Domain:** meta +**Scope:** registry-only +**Depends:** 0100 (frontmatter migration) +**Blocks:** 0102 (work dashboard tab consume `dev_console --json`) +**Related:** 0103 (slash commands llaman al binario) + +## Problema + +Issues y flows hoy se gestionan a ojo: `ls dev/issues/`, `grep`, edit manual de tablas en `README.md` / `INDEX.md`. Sin un comando unificado: + +- No hay `/issue list --domain trading --status pendiente`. +- No hay `/flow status 0001` que cuente checkboxes + DoD %. +- No hay vista cross-cutting "que hacer hoy" mezclando issues + flows. + +Necesitamos un binario unico (`dev_console`) con la misma forma que `fn`: subcomandos consistentes, output texto + `--json`, latencia <200ms. + +## Objetivo v1 + +App Go en `apps/dev_console/` con subcomandos: + +### issue + +| Subcomando | Que hace | +|---|---| +| `dev_console issue list [--domain X] [--type Y] [--status Z] [--prio P] [--epic NNNN]` | tabla filtrable + DoD % | +| `dev_console issue show NNNN` | imprime archivo | +| `dev_console issue status NNNN` | % acceptance + estado deps (resuelto si todos los `depends` estan `completado`) | +| `dev_console issue board` | output TUI o tabla columnas pendiente/in-progress/bloqueado/done | +| `dev_console issue dep NNNN` | arbol bloquea/depende navegable | +| `dev_console issue roadmap NNNN` | epic + sub-IDs (auto-detecta `NNNNa`, `NNNNb`, ...) | +| `dev_console issue tag NNNN +X -Y` | mantenimiento tags | +| `dev_console issue done NNNN` | mueve a `completed/`, valida deps, actualiza README | +| `dev_console issue stale [--days 30]` | sin update >N dias | +| `dev_console issue create --type T --domain D` | scaffold con frontmatter canonico | + +### flow + +| Subcomando | Que hace | +|---|---| +| `dev_console flow list [--app X] [--pattern P] [--risk R]` | tabla filtrable + DoD % | +| `dev_console flow create ` | scaffold (rechaza si falta DoD user-facing) | +| `dev_console flow show NNNN` | imprime archivo | +| `dev_console flow status NNNN` | Acceptance % + DoD % separados + checks user-facing destacados | +| `dev_console flow dod NNNN` | solo bloque DoD + checklist live | +| `dev_console flow trace NNNN` | join `call_monitor.calls` + `data_factory.runs` filtrados por funciones/apps del flow | +| `dev_console flow user-test NNNN` | abre superficie usuario declarada en DoD (URL, lanza .exe, abre tab) | +| `dev_console flow run NNNN` | fase 2 — ejecuta steps con `function:` | +| `dev_console flow chain N M` | declara composicion N -> M | +| `dev_console flow done NNNN` | exige DoD 100% (incluyendo user-facing) antes de mover | + +### work (cross-cutting) + +| Subcomando | Que hace | +|---|---| +| `dev_console work today` | top items prio alta + deps satisfechas (issues + flows) | +| `dev_console work weekly` | review semanal: closed vs planeados (lookup en git log + completed/) | +| `dev_console work search "texto"` | FTS sobre issues + flows + completed | +| `dev_console work dashboard` | imprime JSON consumible por tab Work (issue 0102) | + +## Reglas tecnicas + +- Go + parser YAML (gopkg.in/yaml.v3) + tabwriter. Sin DB propia — fuente de verdad = archivos `.md`. +- Cache opcional en `~/.cache/dev_console/index.json` invalidada por mtime. +- `--json` en TODOS los subcomandos para consumo por dashboards/agentes. +- Latencia objetivo <200ms en lookup, <500ms en list (71 issues + 7 flows). +- Build canonico: `CGO_ENABLED=0 go build -tags fts5 -o dev_console .` + +## Acceptance + +- [ ] `dev_console issue list --status pendiente` lista los issues abiertos. +- [ ] `dev_console flow status 0001` muestra Acceptance + DoD + user-facing %. +- [ ] `dev_console work today` produce lista util (no vacia, no flood). +- [ ] `dev_console flow done 0001` rechaza si DoD <100%. +- [ ] Tests con fixtures en `apps/dev_console/testdata/`. + +## Definition of Done + +### Generico + +- [ ] **Repetibilidad**: tests verdes 3x; latencia consistente. +- [ ] **Observabilidad**: cada invocacion registrada en `call_monitor.calls` (hook PostToolUse Bash detecta `dev_console *`). +- [ ] **Error-path**: archivo malformado -> mensaje claro + exit code != 0. +- [ ] **Idempotencia**: `done` 2x sobre mismo issue = 0 cambios la segunda. +- [ ] **Secrets**: N/A. +- [ ] **Docs**: `apps/dev_console/app.md` + `README.md` con ejemplos. +- [ ] **Registry-first**: reusa `parse_yaml_frontmatter_*`, `checklist_count_*`, etc. +- [ ] **INDEX + status**: issue cerrado. + +### User-facing + +- [ ] **User-facing**: usuario teclea `/issue list` en Claude Code o `dev_console issue list` en terminal y ve tabla limpia con prio/domain/status. +- [ ] **User-facing repeat**: comando responde igual cada vez, sub-segundo, sin reset de estado. +- [ ] **User-facing onboarding**: `apps/dev_console/app.md` lista comandos canonicos + casos comunes. +- [ ] **User-facing latencia**: <500ms p95 para list, <200ms para show. diff --git a/dev/issues/0102-work-dashboard-tab.md b/dev/issues/0102-work-dashboard-tab.md new file mode 100644 index 00000000..7618cdec --- /dev/null +++ b/dev/issues/0102-work-dashboard-tab.md @@ -0,0 +1,77 @@ +# 0102 — Tab Work en registry_dashboard (issues + flows + telemetria) + +**Status:** pendiente +**Created:** 2026-05-16 +**Type:** feature +**Priority:** media +**Domain:** meta +**Scope:** app-scoped +**Depends:** 0100 (frontmatter migration), 0101 (dev_console --json) +**Blocks:** — +**Related:** 0103 (slash commands) + +## Problema + +Hoy para ver "estado global del trabajo" hay que: + +1. `ls dev/issues/*.md` + leer cabeceras. +2. `cat dev/flows/INDEX.md` + abrir flow por flow. +3. `sqlite3 call_monitor.db` para metricas. +4. Cruzar a mano que issues bloquean que flow. + +Cero visibilidad cross-cutting. Y nada me dice "abre el flow 0001 ya, todos sus checks user-facing estan listos" o "issue 0099 esta verde pero su dependencia 0096 esta marcada como DONE incorrectamente". + +## Objetivo + +Tab nueva `Work` en `projects/fn_monitoring/apps/registry_dashboard` (C++ ImGui). Tres paneles: + +### Panel 1 — Kanban issues + +Columnas: `pendiente | in-progress | bloqueado | completado-hoy`. Filtros (combo): domain, type, priority. Card por issue muestra: id, title, prio, deps no resueltas (en rojo si las hay). + +Drag entre columnas -> llama `dev_console issue tag NNNN --status X` por debajo. + +### Panel 2 — Flows table + +Tabla con columnas: id, slug, pattern, status, Acceptance %, DoD %, **DoD user-facing %**, ultima run en `data_factory.runs`. Click en fila -> abre archivo .md (o panel detalle al lado). + +Boton `User-test` por fila -> lanza `dev_console flow user-test NNNN` (abre URL/app/sala Matrix declarada). + +### Panel 3 — Telemetria (resumen call_monitor) + +KPIs ultimas 24h: `calls_24h`, `violations_24h`, `pending_proposals`, `Reg %`. Sparkline 7d por KPI. Misma fuente que el hook UserPromptSubmit. + +## Reglas + +- ImGui + `data_table_cpp_viz` para tablas (registry-first). +- Datos vienen de `dev_console work dashboard --json` (call cada 5s en debug, cada 30s en prod). +- Si `dev_console` no esta instalado: panel muestra placeholder + comando para instalar (sin crash). +- Tab carga en <300ms (issue 0101 garantiza el binario). + +## Acceptance + +- [ ] Tab Work aparece en `registry_dashboard` con los 3 paneles. +- [ ] Filtros funcionan (domain, type, priority, pattern). +- [ ] Drag de issue actualiza disco. +- [ ] User-test boton abre superficie usuario. +- [ ] Refresh manual + auto cada 30s. + +## Definition of Done + +### Generico + +- [ ] **Repetibilidad**: tab abre 10x sin leak handles ni memoria. +- [ ] **Observabilidad**: cada accion (drag, click User-test) loguea via `fn_log`. +- [ ] **Error-path**: `dev_console` falla -> tab muestra error formateado, no crash. +- [ ] **Idempotencia**: refresh 100x = misma tabla. +- [ ] **Secrets**: N/A. +- [ ] **Docs**: `registry_dashboard.app.md` lista la tab + casos de uso. +- [ ] **Registry-first**: reusa `data_table_cpp_viz`, `selectable_text`, `fn_log`. +- [ ] **INDEX + status**: issue cerrado. + +### User-facing + +- [ ] **User-facing**: usuario abre `registry_dashboard.exe` -> tab Work -> ve issues kanban + flows table + KPIs todo en una pantalla. +- [ ] **User-facing repeat**: mismo dashboard manana muestra estado actualizado sin reset (deps resueltas se reflejan). +- [ ] **User-facing onboarding**: parrafo en `app.md`: "Para el estado del trabajo: lanzar `registry_dashboard.exe` -> tab Work. Boton User-test abre la superficie usuario del flow." +- [ ] **User-facing latencia**: refresh <300ms; cambio en disco visible en <30s (auto-refresh). diff --git a/dev/issues/0103-taxonomy-and-slash-commands.md b/dev/issues/0103-taxonomy-and-slash-commands.md new file mode 100644 index 00000000..9f432e23 --- /dev/null +++ b/dev/issues/0103-taxonomy-and-slash-commands.md @@ -0,0 +1,123 @@ +# 0103 — Taxonomia + slash commands /issue /flow /work + +**Status:** pendiente +**Created:** 2026-05-16 +**Type:** feature +**Priority:** alta +**Domain:** meta +**Scope:** registry-only +**Depends:** 0100 (frontmatter ya canonico), 0101 (dev_console binary) +**Blocks:** 0102 (work dashboard usa los slash desde la tab) +**Related:** todos los issues + flows + +## Problema + +Sin taxonomia formal, todo issue/flow se mezcla en un saco. `dev_console` (issue 0101) necesita un schema concreto para filtros: que dominios existen, que tipos son validos, que estados, que scopes. Y los slash commands `/issue *` / `/flow *` / `/work *` necesitan existir como archivos en `.claude/commands/` para que Claude Code los reconozca. + +## Objetivo + +### A) Taxonomia documentada + +Crear `dev/TAXONOMY.md` con la lista canonica: + +**Dominios** (allowlist): +``` +meta, cpp-stack, kanban, trading, gamedev, osint, data-ingest, +registry-quality, notify, imagegen, apps-infra, dev-ux, deploy, +frontend, mcp, browser, telemetry, docs +``` + +**Tipos**: +``` +app | feature | bugfix | refactor | chore | docs | spike | epic | infra | planning +``` + +**Estados**: +``` +pendiente | in-progress | bloqueado | completado | deferred +``` + +**Scopes**: +``` +registry-only | app-scoped | multi-app | cross-stack +``` + +**Prioridades**: +``` +alta | media | baja +``` + +**Flow patterns**: +``` +smoke-cron | prod-data | event-driven | manual-deep | gitops | realtime-loop +``` + +### B) Slash commands + +Crear `.claude/commands/issue.md`, `flow.md`, `work.md`. Cada uno con frontmatter que define `tool: Bash` + un `command:` que llama a `dev_console "$ARGS"`. Mientras 0101 no este listo: stub que avisa. + +```yaml +# .claude/commands/issue.md +--- +description: Gestiona issues del registry (list, show, status, board, done, ...) +allowed-tools: [Bash] +--- + +Usage: /issue [args] + +Subcomandos: +- list [--domain X] [--status Y] [--prio P] +- show NNNN +- status NNNN +- board +- dep NNNN +- roadmap NNNN +- tag NNNN +X -Y +- done NNNN +- stale [--days N] +- create --type T --domain D + +Run: +!`./apps/dev_console/dev_console issue $ARGUMENTS` +``` + +### C) Aplicar tags retroactivos + +Pipeline `tag_existing_issues_bash_pipelines` que, basado en heuristicas (nombre del archivo, contenido), propone `domain` y `scope` para los 71 issues. Output: lista para review humano (no escribe sin confirmacion). + +Heuristicas iniciales: +- `cpp-*` -> domain `cpp-stack` +- `kanban-*` -> domain `kanban` +- `trading-*` -> domain `trading` +- `gamedev-*` -> domain `gamedev` +- `osint-*`, `odr-*` -> domain `osint` +- `cpp-app-*`, `apps-*`, `init-*-app` -> scope `app-scoped` +- `roadmap` en el title -> type `epic` + +## Acceptance + +- [ ] `dev/TAXONOMY.md` creado con todas las listas + descripcion 1-frase por valor. +- [ ] `.claude/commands/{issue,flow,work}.md` existen y son visibles a Claude Code. +- [ ] `fn doctor issues` (subcomando de 0100) valida `domain` y `scope` contra la allowlist. +- [ ] Pipeline de tags retroactivos corre + produce reporte. +- [ ] >=80% de los 71 issues quedan clasificados sin intervencion humana. + +## Definition of Done + +### Generico + +- [ ] **Repetibilidad**: pipeline + slash commands estables; no varian salida. +- [ ] **Observabilidad**: cada slash command pasa por hook PostToolUse -> `call_monitor.calls`. +- [ ] **Error-path**: dominio invalido -> error claro + sugerencia ("did you mean ...?"). +- [ ] **Idempotencia**: pipeline 2x = 0 cambios despues de primera pasada. +- [ ] **Secrets**: N/A. +- [ ] **Docs**: TAXONOMY referenciado desde `.claude/rules/INDEX.md`. +- [ ] **Registry-first**: pipeline reusa parsers existentes. +- [ ] **INDEX + status**: issue cerrado. + +### User-facing + +- [ ] **User-facing**: usuario teclea `/issue list --domain trading` en Claude Code y ve los 10 sub-issues del roadmap trading. +- [ ] **User-facing repeat**: comandos disponibles desde cualquier sesion, no estado por sesion. +- [ ] **User-facing onboarding**: `.claude/commands/issue.md` autodescribe los subcomandos (Claude Code los muestra al tipear `/issue` + tab). +- [ ] **User-facing latencia**: <500ms por slash command. diff --git a/docs/diary/2026-05-16.md b/docs/diary/2026-05-16.md index e018e2be..5627adb9 100644 --- a/docs/diary/2026-05-16.md +++ b/docs/diary/2026-05-16.md @@ -122,3 +122,18 @@ ls sources/phosphor-core/assets/fill/ | grep - Anadir icono tambien a `engine_smoke` y `runtime_test` si se promueven a apps user-facing (hoy son smoke tests). - Considerar `fn doctor cpp-apps` check: app C++ sin `appicon.ico` → warning. - Si aparecen iconos antiguos en Explorer tras redeploy, anadir `ie4uinit.exe -show` al final de `deploy_cpp_exe_to_windows`. + +## 20:54 — dag_engine — fix function-not-found nocturno + panel Logs en RunDetail + +- Hecho: diagnostico de 3 fallos nocturnos (`fn_backup` x2 2026-05-15/16, `daily-registry-audit` 2026-05-16) que reportaban `error: function "" not found (tried as ID and name)` aunque `audit_capability_groups_go_infra`, `backup_all_bash_pipelines` y `cdp_extract_recipe_py_pipelines` existen en `registry.db` raiz. +- Hecho: raiz identificada en `cmd/fn/ops.go:1597-1628 tryOpenRegistryDB` — sin `FN_REGISTRY_ROOT` el resolver cae al walk-up `go.mod` y devuelve `apps/dag_engine/` donde habia una copia stale `apps/dag_engine/registry.db` (262 KB, May 15) que violaba `.claude/rules/db_locations.md` (registry.db SOLO en raiz). +- Hecho: stale db borrada. +- Hecho: `~/.config/systemd/user/dag_engine.service` ampliado con `Environment=FN_REGISTRY_ROOT=/home/lucas/fn_registry`, `FN_BIN`, `PATH=/usr/local/go/bin:...` (sin PATH `go vet` fallaba con `exec: "go": executable file not found`), `HOME`. `daemon-reload` + `restart`. +- Hecho: `apps/dag_engine/executor.go` belt-and-suspenders — steps `function:` exportan `FN_REGISTRY_ROOT` en env del spawn y default `dir = fnRegistryRoot` si `step.Dir`/`dag.WorkingDir` vacios; rebuild `CGO_ENABLED=1 go build -tags fts5 -o dag_engine .`. +- Hecho: smoke test `POST /api/dags/daily-registry-audit/run` -> step `audit_capabilities` SUCCESS 387 ms (era el step que fallaba con not-found). `audit_unused` SUCCESS, `audit_artefacts` falla con exit 1 (bug aparte) y `fn_backup` `run_backup_all` exit 4 sin respetar `continue_on.exit_code` (bug aparte). +- Hecho: panel "Logs" anadido a `apps/dag_engine/frontend/src/pages/RunDetail.tsx` — `` final con `` (max-h 480 scrollable) + `CopyButton` Mantine (icono toggle copy/check teal 1.5s). Helper `buildLogText(run, steps)` compone texto plano con metadata del run (dag/path/status/trigger/started/finished/duration/error) + por-step (`[status] name exit=N Nms`, stdout/stderr indentado 4 espacios). Type-checkea limpio. +- Hecho: documentadas BBDDs canonicas — `dag_engine.db` en `apps/dag_engine/`, `data_factory.db` en `apps/data_factory/` (tablas nodes/connections/runs/databases), `navegator_dashboard` NO tiene BD propia (solo `layouts.db` del framework via `fn::local_path`). +- Hecho: append en `apps/dag_engine/app.md` (seccion fechada + "Lo siguiente que pega"), `apps/dag_engine/README.md` (Function steps: aviso sobre `FN_REGISTRY_ROOT` y `PATH` en systemd), `CHANGELOG.md` (Added panel Logs + Fixed function-not-found bug). +- Pendiente: investigar exit 1 real de `audit_artefacts` en `daily-registry-audit` (probable artefacto huerfano o git drift). +- Pendiente: bug en executor — `continue_on.exit_code: [4]` no se respeta; solo se mira `step.ContinueOn.Failure` (bool). Parsear `ContinueOn.ExitCode []int` y comparar con `result.ExitCode` antes de marcar `runFailed=true`. +- Pendiente: frontend `pnpm build` roto por API drift de Mantine (`` en `StepTimeline.tsx:49`) y CSS type import en `main.tsx:1`. Bloquea ver el panel Logs en vivo; type-check ya pasa. diff --git a/functions/browser/chrome_launch.go b/functions/browser/chrome_launch.go index 87030039..9a118ecc 100644 --- a/functions/browser/chrome_launch.go +++ b/functions/browser/chrome_launch.go @@ -5,6 +5,8 @@ import ( "net" "os" "os/exec" + "regexp" + "strings" "time" ) @@ -13,6 +15,9 @@ type ChromeLaunchOpts struct { // Port es el puerto de remote debugging. Por defecto 9222. Port int // UserDataDir es el directorio de perfil de Chrome. Por defecto /tmp/chrome-cdp-profile. + // En WSL2 con chrome.exe, si el valor comienza con /tmp/ o /home/ se traduce + // automaticamente a una ruta Windows via wslpath. Pasar una ruta Windows + // (ej: C:\Users\...) la deja intacta. UserDataDir string // Headless activa el modo headless (--headless=new). Por defecto false. Headless bool @@ -22,6 +27,53 @@ type ChromeLaunchOpts struct { ExtraArgs []string } +// reWindowsPath coincide con rutas absolutas de Windows (C:\... D:\... etc.). +var reWindowsPath = regexp.MustCompile(`(?i)^[A-Z]:\\`) + +// isWSL2 devuelve true si el proceso corre dentro de WSL2. +// Lee /proc/version y busca "microsoft" o "WSL" (case-insensitive). +func isWSL2() bool { + b, err := os.ReadFile("/proc/version") + if err != nil { + return false + } + lower := strings.ToLower(string(b)) + return strings.Contains(lower, "microsoft") || strings.Contains(lower, "wsl") +} + +// isWindowsExe devuelve true si la ruta del ejecutable corresponde a un .exe +// (chrome.exe en Windows via WSL2: puede ser /mnt/c/... o resolverse en PATH). +func isWindowsExe(path string) bool { + return strings.HasSuffix(strings.ToLower(path), ".exe") +} + +// translateUserDataDirForWindows convierte una ruta Linux a ruta Windows via wslpath. +// Ejemplo: "/tmp/foo" -> "C:\Users\lucas\AppData\Local\Temp\foo" (depende del sistema). +// Devuelve error si wslpath no esta disponible. +func translateUserDataDirForWindows(linuxPath string) (string, error) { + out, err := exec.Command("wslpath", "-w", linuxPath).Output() + if err != nil { + return "", fmt.Errorf("wslpath -w %q: %w", linuxPath, err) + } + return strings.TrimSpace(string(out)), nil +} + +// defaultWindowsUserDataDir devuelve la ruta Windows del perfil CDP por defecto, +// usando la variable de entorno USERNAME de Windows si esta disponible. +func defaultWindowsUserDataDir() (string, error) { + // Intentar leer el home de Windows via wslpath del home de usuario + // Si falla, usar C:\Users\Public\fn-chrome-cdp-profile como fallback. + user := os.Getenv("USERNAME") // Windows user via WSL env passthrough + if user == "" { + user = os.Getenv("USER") + } + if user == "" { + user = "Public" + } + linuxPath := fmt.Sprintf("/mnt/c/Users/%s/AppData/Local/fn-chrome-cdp-profile", user) + return translateUserDataDirForWindows(linuxPath) +} + // chromePaths lista los ejecutables de Chrome conocidos en WSL2/Linux. var chromePaths = []string{ "chrome.exe", @@ -47,13 +99,13 @@ func findChrome() (string, error) { } // waitCDPReady espera hasta que el puerto CDP responda conexiones TCP. -// host puede estar vacio (usa "localhost"). +// host puede estar vacio (usa "127.0.0.1"). func waitCDPReady(host string, port int, timeout time.Duration) error { if host == "" { - host = "localhost" + host = "127.0.0.1" } deadline := time.Now().Add(timeout) - addr := fmt.Sprintf("%s:%d", host, port) + addr := net.JoinHostPort(host, fmt.Sprintf("%d", port)) for time.Now().Before(deadline) { conn, err := net.DialTimeout("tcp", addr, 200*time.Millisecond) if err == nil { @@ -62,19 +114,22 @@ func waitCDPReady(host string, port int, timeout time.Duration) error { } time.Sleep(200 * time.Millisecond) } - return fmt.Errorf("chrome: puerto CDP %s:%d no disponible despues de %s", host, port, timeout) + return fmt.Errorf("chrome: puerto CDP %s no disponible despues de %s", addr, timeout) } // ChromeLaunch lanza Google Chrome con remote debugging habilitado en el puerto indicado. // Retorna el PID del proceso Chrome. Espera hasta 15s a que el puerto CDP este listo. // Busca chrome.exe en PATH (WSL2) o en rutas conocidas de Windows. +// +// WSL2 + chrome.exe: cuando se detecta WSL2 y el ejecutable es un .exe, +// - El UserDataDir se traduce automaticamente a ruta Windows via wslpath +// (si esta vacio o comienza con /tmp/ o /home/). +// - Se inyecta --remote-debugging-address=0.0.0.0 para que Chrome sea +// alcanzable desde WSL2 via 127.0.0.1 (el WSL networking reenvía localhost). func ChromeLaunch(opts ChromeLaunchOpts) (int, error) { if opts.Port == 0 { opts.Port = 9222 } - if opts.UserDataDir == "" { - opts.UserDataDir = "/tmp/chrome-cdp-profile" - } chromePath := opts.ChromePath if chromePath == "" { @@ -85,9 +140,48 @@ func ChromeLaunch(opts ChromeLaunchOpts) (int, error) { } } + // Detectar si estamos en WSL2 lanzando un exe de Windows + wsl2WindowsMode := isWSL2() && isWindowsExe(chromePath) + + // Resolver UserDataDir + userDataDir := opts.UserDataDir + if wsl2WindowsMode { + switch { + case userDataDir == "" || + strings.HasPrefix(userDataDir, "/tmp/") || + strings.HasPrefix(userDataDir, "/home/"): + // Traducir a ruta Windows + if userDataDir == "" { + var err error + userDataDir, err = defaultWindowsUserDataDir() + if err != nil { + // Fallback seguro: usar una ruta Windows fija + userDataDir = `C:\Users\Public\fn-chrome-cdp-profile` + } + } else { + translated, err := translateUserDataDirForWindows(userDataDir) + if err != nil { + return 0, fmt.Errorf("chrome: traducir user-data-dir para Windows: %w", err) + } + userDataDir = translated + } + case reWindowsPath.MatchString(userDataDir): + // Ya es una ruta Windows absoluta (C:\...), dejar intacta + default: + // Ruta que no es ni /tmp/ ni /home/ ni Windows absoluta: + // intentar traducir igualmente. + translated, err := translateUserDataDirForWindows(userDataDir) + if err == nil { + userDataDir = translated + } + } + } else if userDataDir == "" { + userDataDir = "/tmp/chrome-cdp-profile" + } + args := []string{ fmt.Sprintf("--remote-debugging-port=%d", opts.Port), - fmt.Sprintf("--user-data-dir=%s", opts.UserDataDir), + fmt.Sprintf("--user-data-dir=%s", userDataDir), "--no-first-run", "--no-default-browser-check", "--disable-background-networking", @@ -101,12 +195,26 @@ func ChromeLaunch(opts ChromeLaunchOpts) (int, error) { "--disable-translate", "--metrics-recording-only", "--safebrowsing-disable-auto-update", + "--remote-allow-origins=*", } if opts.Headless { args = append(args, "--headless=new", "--disable-gpu") } + // En WSL2+Windows: inyectar --remote-debugging-address=0.0.0.0 si no esta ya presente + hasBindAll := false + for _, a := range opts.ExtraArgs { + if a == "--remote-debugging-address=0.0.0.0" { + hasBindAll = true + break + } + } + if wsl2WindowsMode && !hasBindAll { + args = append(args, "--remote-debugging-address=0.0.0.0") + hasBindAll = true + } + args = append(args, opts.ExtraArgs...) cmd := exec.Command(chromePath, args...) @@ -120,20 +228,14 @@ func ChromeLaunch(opts ChromeLaunchOpts) (int, error) { pid := cmd.Process.Pid - // Esperar a que el puerto CDP este listo - // Si Chrome escucha en 0.0.0.0 (ej: WSL2 -> Windows), el caller se encarga del wait - skipWait := false - for _, a := range opts.ExtraArgs { - if a == "--remote-debugging-address=0.0.0.0" { - skipWait = true - break - } - } - if !skipWait { - if err := waitCDPReady("localhost", opts.Port, 15*time.Second); err != nil { - cmd.Process.Kill() - return 0, err - } + // Esperar a que el puerto CDP este listo. + // Siempre esperamos, incluyendo el caso WSL2+Windows donde Chrome escucha en + // 0.0.0.0 — el WSL networking reenvía localhost:9222 → Windows:9222. + // Usamos 127.0.0.1 explicitamente para evitar resolución IPv6 en algunos entornos. + _ = hasBindAll // ya no se usa para skipWait + if err := waitCDPReady("127.0.0.1", opts.Port, 15*time.Second); err != nil { + cmd.Process.Kill() + return 0, err } return pid, nil diff --git a/functions/browser/chrome_launch.md b/functions/browser/chrome_launch.md index 693a9d36..7a1f67cd 100644 --- a/functions/browser/chrome_launch.md +++ b/functions/browser/chrome_launch.md @@ -3,23 +3,23 @@ name: chrome_launch kind: function lang: go domain: browser -version: "1.0.0" +version: "1.1.0" purity: impure signature: "func ChromeLaunch(opts ChromeLaunchOpts) (int, error)" -description: "Lanza Google Chrome con remote debugging habilitado en el puerto indicado. Busca chrome.exe en PATH (WSL2) o en rutas conocidas de Windows. Espera hasta 15s a que el puerto CDP este listo antes de retornar. Retorna el PID del proceso." -tags: [chrome, cdp, browser, automation, wsl2, headless] +description: "Lanza Google Chrome con remote debugging habilitado en el puerto indicado. Busca chrome.exe en PATH (WSL2) o en rutas conocidas de Windows. En WSL2+chrome.exe, traduce UserDataDir a ruta Windows via wslpath e inyecta --remote-debugging-address=0.0.0.0 automaticamente. Espera hasta 15s a que el puerto CDP este listo antes de retornar. Retorna el PID del proceso." +tags: [chrome, cdp, browser, automation, wsl2, headless, navegator] uses_functions: [] uses_types: [] returns: [] returns_optional: false error_type: "error_go_core" -imports: [fmt, net, os, os/exec, time] +imports: [fmt, net, os, os/exec, regexp, strings, time] params: - name: opts - desc: "opciones de lanzamiento: Port, UserDataDir, Headless" + desc: "opciones de lanzamiento: Port (defecto 9222), UserDataDir (defecto /tmp/chrome-cdp-profile en Linux, C:\\Users\\\\AppData\\Local\\fn-chrome-cdp-profile en WSL2+exe), Headless, ChromePath, ExtraArgs" output: "int: PID del proceso Chrome lanzado" tested: true -tests: ["TestFindChrome", "TestChromeLaunchAndConnect"] +tests: ["TestIsWSL2", "TestTranslateUserDataDirForWindows", "TestIsWindowsExe", "TestFindChrome", "TestChromeLaunchAndConnect"] test_file_path: "functions/browser/chrome_launch_test.go" file_path: "functions/browser/chrome_launch.go" --- @@ -27,10 +27,10 @@ file_path: "functions/browser/chrome_launch.go" ## Ejemplo ```go +// Linux nativo (sin WSL2 o con Linux Chrome) pid, err := ChromeLaunch(ChromeLaunchOpts{ - Port: 9222, - UserDataDir: "/tmp/chrome-cdp", - Headless: true, + Port: 9222, + Headless: true, }) if err != nil { log.Fatal(err) @@ -38,6 +38,32 @@ if err != nil { defer CdpClose(nil, pid) ``` +```go +// WSL2 → chrome.exe Windows: cero configuracion, todo automatico +// ChromeLaunch detecta WSL2+.exe, traduce user-data-dir y bind 0.0.0.0 +pid, err := ChromeLaunch(ChromeLaunchOpts{}) +if err != nil { + log.Fatal(err) +} +// CDP listo en 127.0.0.1:9222 desde WSL2 +conn, err := CdpConnect(9222) +``` + +## Cuando usarla + +Cuando necesites lanzar Chrome con CDP desde Go para automatizacion (scraping, tests, capturas). Usar antes de `CdpConnect` / `CdpNavigate` / `CdpScreenshot`. Funciona tanto en Linux nativo como en WSL2 apuntando al chrome.exe de Windows. + +## Gotchas + +- **WSL2 + chrome.exe**: la funcion detecta automaticamente WSL2 (`/proc/version` contiene "microsoft"/"WSL") y que el ejecutable es `.exe`. En ese caso: + - `UserDataDir` vacio o con prefijo `/tmp/` o `/home/` se traduce via `wslpath -w` a ruta Windows. Por defecto: `C:\Users\\AppData\Local\fn-chrome-cdp-profile`. + - Se inyecta `--remote-debugging-address=0.0.0.0` para que Chrome sea accesible desde WSL2 vía `127.0.0.1:`. + - `waitCDPReady` siempre espera usando `127.0.0.1` (WSL networking reenvía localhost → Windows). +- **`wslpath` debe estar disponible**: se invoca como subproceso. Si falla, `ChromeLaunch` retorna error. `wslpath` es estándar en WSL2 desde Windows 10 1903+. +- **Chrome no cierra solo**: el PID devuelto es el proceso Chrome. Usar `CdpClose(nil, pid)` o `os.FindProcess(pid).Kill()` para terminarlo. +- **Puerto ocupado**: si el puerto ya está en uso por otra instancia de Chrome, `waitCDPReady` puede conectar al proceso previo. Usar puertos distintos por sesión. +- **Headless en Windows via WSL2**: `--headless=new --disable-gpu` funciona bien con chrome.exe. + ## Notas Busca Chrome en este orden: @@ -49,3 +75,7 @@ Busca Chrome en este orden: Los flags aplicados desactivan funcionalidades de red y actualizacion en segundo plano para entornos de automatizacion. En modo headless se agrega `--headless=new --disable-gpu`. El struct `ChromeLaunchOpts` se define en el mismo archivo. + +## Capability growth log + +- v1.1.0 (2026-05-16) — auto-handle WSL2→Windows chrome.exe: translate user-data-dir via wslpath + inject --remote-debugging-address=0.0.0.0 diff --git a/functions/browser/chrome_launch_test.go b/functions/browser/chrome_launch_test.go index 31238ea8..8ff0ce29 100644 --- a/functions/browser/chrome_launch_test.go +++ b/functions/browser/chrome_launch_test.go @@ -2,11 +2,66 @@ package browser import ( "os" + "regexp" "strings" "testing" "time" ) +// TestIsWSL2 verifica que isWSL2 detecta el entorno correctamente leyendo /proc/version. +func TestIsWSL2(t *testing.T) { + b, err := os.ReadFile("/proc/version") + if err != nil { + t.Skip("/proc/version no disponible (no es Linux)") + } + lower := strings.ToLower(string(b)) + expectWSL2 := strings.Contains(lower, "microsoft") || strings.Contains(lower, "wsl") + got := isWSL2() + if got != expectWSL2 { + t.Errorf("isWSL2() = %v, want %v (contenido: %q)", got, expectWSL2, string(b)[:min(120, len(b))]) + } + t.Logf("isWSL2() = %v (entorno: %s)", got, string(b)[:min(80, len(b))]) +} + +// TestTranslateUserDataDirForWindows verifica la traduccion de rutas Linux a Windows. +// Solo corre si wslpath esta disponible (WSL2). +func TestTranslateUserDataDirForWindows(t *testing.T) { + if !isWSL2() { + t.Skip("solo aplica en WSL2") + } + result, err := translateUserDataDirForWindows("/tmp/test-chrome-profile") + if err != nil { + t.Fatalf("translateUserDataDirForWindows: %v", err) + } + // El resultado debe contener backslash (ruta Windows) o empezar con [A-Z]: + reWin := regexp.MustCompile(`(?i)^[A-Z]:\\|\\`) + if !reWin.MatchString(result) { + t.Errorf("resultado no parece ruta Windows: %q", result) + } + t.Logf("translateUserDataDirForWindows('/tmp/test-chrome-profile') = %q", result) +} + +// TestIsWindowsExe verifica que isWindowsExe detecta ejecutables .exe. +func TestIsWindowsExe(t *testing.T) { + cases := []struct { + path string + want bool + }{ + {"/mnt/c/Program Files/Google/Chrome/Application/chrome.exe", true}, + {"chrome.exe", true}, + {"CHROME.EXE", true}, + {"/usr/bin/google-chrome", false}, + {"chromium", false}, + {"/mnt/c/Windows/System32/cmd.exe", true}, + } + for _, c := range cases { + got := isWindowsExe(c.path) + if got != c.want { + t.Errorf("isWindowsExe(%q) = %v, want %v", c.path, got, c.want) + } + } +} + // TestFindChrome verifica que el ejecutable de Chrome es localizable. func TestFindChrome(t *testing.T) { path, err := findChrome() @@ -20,9 +75,10 @@ func TestFindChrome(t *testing.T) { } // TestChromeLaunchAndConnect lanza Chrome, conecta CDP, navega a about:blank y cierra. +// Requiere CHROME_E2E=1 (integración real con Chrome). func TestChromeLaunchAndConnect(t *testing.T) { - if testing.Short() { - t.Skip("skip: requiere Chrome real (use -short=false para ejecutar)") + if os.Getenv("CHROME_E2E") != "1" { + t.Skip("skip: requiere CHROME_E2E=1 y Chrome real") } // Verificar que Chrome esta disponible @@ -67,9 +123,10 @@ func TestChromeLaunchAndConnect(t *testing.T) { } // TestCdpEvaluate ejecuta JS simple en Chrome y verifica el resultado. +// Requiere CHROME_E2E=1. func TestCdpEvaluate(t *testing.T) { - if testing.Short() { - t.Skip("skip: requiere Chrome real (use -short=false para ejecutar)") + if os.Getenv("CHROME_E2E") != "1" { + t.Skip("skip: requiere CHROME_E2E=1 y Chrome real") } if _, err := findChrome(); err != nil { @@ -130,9 +187,10 @@ func TestCdpEvaluate(t *testing.T) { } // TestCdpGetHTML obtiene el HTML de about:blank y verifica que contiene elementos basicos. +// Requiere CHROME_E2E=1. func TestCdpGetHTML(t *testing.T) { - if testing.Short() { - t.Skip("skip: requiere Chrome real (use -short=false para ejecutar)") + if os.Getenv("CHROME_E2E") != "1" { + t.Skip("skip: requiere CHROME_E2E=1 y Chrome real") } if _, err := findChrome(); err != nil { @@ -178,9 +236,10 @@ func TestCdpGetHTML(t *testing.T) { } // TestCdpScreenshot toma un screenshot de about:blank y verifica que se crea el archivo PNG. +// Requiere CHROME_E2E=1. func TestCdpScreenshot(t *testing.T) { - if testing.Short() { - t.Skip("skip: requiere Chrome real (use -short=false para ejecutar)") + if os.Getenv("CHROME_E2E") != "1" { + t.Skip("skip: requiere CHROME_E2E=1 y Chrome real") } if _, err := findChrome(); err != nil { diff --git a/functions/infra/audit_modules_drift.go b/functions/infra/audit_modules_drift.go new file mode 100644 index 00000000..dc501342 --- /dev/null +++ b/functions/infra/audit_modules_drift.go @@ -0,0 +1,225 @@ +package infra + +import ( + "fmt" + "os" + "path/filepath" + "regexp" + "sort" + "strings" + + "gopkg.in/yaml.v3" +) + +// ModuleDriftCheck describes per-app drift between app.md uses_modules and +// CMakeLists.txt fn_module_* link calls. +type ModuleDriftCheck struct { + AppID string `json:"app_id"` + AppMD string `json:"app_md"` + CMakeLists string `json:"cmake_lists"` + Declared []string `json:"declared"` // module IDs from uses_modules + Linked []string `json:"linked"` // module names from fn_module_ + MissingLinks []string `json:"missing_links"` // declared but not linked + ExtraLinks []string `json:"extra_links"` // linked but not declared + OK bool `json:"ok"` +} + +var ( + cmakeLinkRE = regexp.MustCompile(`\bfn_module_([a-z0-9_]+)\b`) +) + +// AuditModulesDrift scans apps/*/app.md, projects/*/apps/*/app.md, cpp/apps/*/app.md +// and compares uses_modules in the frontmatter against fn_module_ link calls +// in the adjacent CMakeLists.txt. +// +// An app is OK when: +// - It has no CMakeLists.txt (non-C++ app) — drift check N/A; skipped silently. +// - declared (modulo `_` suffix) == linked. +func AuditModulesDrift(root string) ([]ModuleDriftCheck, error) { + candidates, err := findAppDirs(root) + if err != nil { + return nil, err + } + + var result []ModuleDriftCheck + for _, dir := range candidates { + appMD := filepath.Join(dir, "app.md") + cmakeLists := filepath.Join(dir, "CMakeLists.txt") + + if _, err := os.Stat(cmakeLists); err != nil { + // Non-C++ app or app without CMakeLists. Skip drift check. + continue + } + + declared, appID, err := readUsesModules(appMD) + if err != nil { + continue + } + + linked, err := readLinkedModules(cmakeLists) + if err != nil { + continue + } + + // Normalize declared (module IDs like "data_table_cpp") to module names + // for comparison with link target names ("fn_module_data_table" -> "data_table"). + declaredNames := make([]string, 0, len(declared)) + for _, d := range declared { + declaredNames = append(declaredNames, stripLangSuffix(d)) + } + + missing := diffStrings(declaredNames, linked) + extra := diffStrings(linked, declaredNames) + + relMD, _ := filepath.Rel(root, appMD) + relCM, _ := filepath.Rel(root, cmakeLists) + + result = append(result, ModuleDriftCheck{ + AppID: appID, + AppMD: relMD, + CMakeLists: relCM, + Declared: declaredNames, + Linked: linked, + MissingLinks: missing, + ExtraLinks: extra, + OK: len(missing) == 0 && len(extra) == 0, + }) + } + + sort.Slice(result, func(i, j int) bool { return result[i].AppID < result[j].AppID }) + return result, nil +} + +// findAppDirs returns directories that contain an app.md file: +// - /apps/*/ +// - /projects/*/apps/*/ +func findAppDirs(root string) ([]string, error) { + var dirs []string + + // /apps/*/ + appsRoot := filepath.Join(root, "apps") + if entries, err := os.ReadDir(appsRoot); err == nil { + for _, e := range entries { + if !e.IsDir() { + continue + } + candidate := filepath.Join(appsRoot, e.Name()) + if _, err := os.Stat(filepath.Join(candidate, "app.md")); err == nil { + dirs = append(dirs, candidate) + } + } + } + + // /projects/*/apps/*/ + projectsRoot := filepath.Join(root, "projects") + if projEntries, err := os.ReadDir(projectsRoot); err == nil { + for _, pe := range projEntries { + if !pe.IsDir() { + continue + } + projAppsDir := filepath.Join(projectsRoot, pe.Name(), "apps") + if appEntries, err := os.ReadDir(projAppsDir); err == nil { + for _, ae := range appEntries { + if !ae.IsDir() { + continue + } + candidate := filepath.Join(projAppsDir, ae.Name()) + if _, err := os.Stat(filepath.Join(candidate, "app.md")); err == nil { + dirs = append(dirs, candidate) + } + } + } + } + } + + return dirs, nil +} + +type appFrontmatter struct { + Name string `yaml:"name"` + Lang string `yaml:"lang"` + Domain string `yaml:"domain"` + UsesModules []string `yaml:"uses_modules"` +} + +func readUsesModules(path string) (modules []string, appID string, err error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, "", err + } + // Extract YAML frontmatter between leading "---" markers. + if !strings.HasPrefix(string(data), "---") { + return nil, "", fmt.Errorf("missing frontmatter in %s", path) + } + rest := string(data)[4:] + end := strings.Index(rest, "\n---") + if end < 0 { + return nil, "", fmt.Errorf("missing closing --- in %s", path) + } + fm := rest[:end] + + var raw appFrontmatter + if err := yaml.Unmarshal([]byte(fm), &raw); err != nil { + return nil, "", err + } + + if raw.Name == "" { + return nil, "", fmt.Errorf("no name in %s", path) + } + id := raw.Name + if raw.Lang != "" { + id += "_" + raw.Lang + } + if raw.Domain != "" { + id += "_" + raw.Domain + } + return raw.UsesModules, id, nil +} + +func readLinkedModules(cmakePath string) ([]string, error) { + data, err := os.ReadFile(cmakePath) + if err != nil { + return nil, err + } + matches := cmakeLinkRE.FindAllStringSubmatch(string(data), -1) + seen := map[string]bool{} + var out []string + for _, m := range matches { + if len(m) < 2 { + continue + } + if !seen[m[1]] { + seen[m[1]] = true + out = append(out, m[1]) + } + } + sort.Strings(out) + return out, nil +} + +// stripLangSuffix removes a trailing _ suffix for matching purposes. +// "data_table_cpp" -> "data_table". +func stripLangSuffix(id string) string { + for _, suf := range []string{"_cpp", "_py", "_ts", "_bash", "_go"} { + if strings.HasSuffix(id, suf) { + return id[:len(id)-len(suf)] + } + } + return id +} + +// diffStrings returns elements in a that are not in b. +func diffStrings(a, b []string) []string { + bset := map[string]bool{} + for _, x := range b { + bset[x] = true + } + var out []string + for _, x := range a { + if !bset[x] { + out = append(out, x) + } + } + sort.Strings(out) + return out +} diff --git a/functions/infra/audit_modules_drift.md b/functions/infra/audit_modules_drift.md new file mode 100644 index 00000000..33078155 --- /dev/null +++ b/functions/infra/audit_modules_drift.md @@ -0,0 +1,57 @@ +--- +name: audit_modules_drift +kind: function +lang: go +domain: infra +version: "1.0.0" +purity: impure +signature: "AuditModulesDrift(root string) ([]ModuleDriftCheck, error)" +description: "Detecta drift entre app.md uses_modules y CMakeLists.txt fn_module_ link calls. Para cada app C++ con CMakeLists.txt: parsea uses_modules + regex sobre target_link_libraries. Devuelve por-app: declared/linked/missing/extra/OK." +tags: [audit, modules, cmake, drift, doctor, cpp] +uses_functions: [] +uses_types: [] +returns: [] +returns_optional: false +error_type: "error_go_core" +imports: + - gopkg.in/yaml.v3 +file_path: "functions/infra/audit_modules_drift.go" +params: + - name: root + desc: "Raiz del repositorio fn_registry. Se escanean apps/*, projects/*/apps/*." +output: "Slice de ModuleDriftCheck (uno por app C++ con CMakeLists.txt). Apps sin CMakeLists son saltadas." +--- + +## Ejemplo + +```go +import "fn-registry/functions/infra" + +checks, err := infra.AuditModulesDrift("/home/lucas/fn_registry") +if err != nil { panic(err) } +for _, c := range checks { + if !c.OK { + fmt.Printf("DRIFT %s: missing=%v extra=%v\n", c.AppID, c.MissingLinks, c.ExtraLinks) + } +} +``` + +Tambien expuesto via CLI: + +```bash +fn doctor modules # tabla legible +fn doctor modules --json # JSON para agentes +``` + +## Cuando usarla + +Tras anadir/quitar un modulo a la app: +- Verifica que el `uses_modules` del `app.md` y `target_link_libraries(... PRIVATE fn_module_*)` del CMakeLists.txt coinciden. +- Tras renombrar un modulo, detecta apps que quedaron con la version antigua. +- Como gate en `/full-git-push` antes de mergear cambios de modulos. + +## Gotchas + +- Apps sin `CMakeLists.txt` (Python, bash, etc.) se saltan — el drift check no aplica. +- Modulos IDs en `uses_modules` llevan sufijo `_` (ej. `data_table_cpp`); los link targets son `fn_module_` (sin sufijo). La funcion strippa el sufijo antes de comparar. +- Regex acepta `fn_module_` en cualquier parte del CMakeLists — comentarios incluidos. Si un comentario referencia un modulo no usado, se reporta como `extra_links` (falso positivo aceptable). diff --git a/modules/data_table/CMakeLists.txt b/modules/data_table/CMakeLists.txt new file mode 100644 index 00000000..d165c752 --- /dev/null +++ b/modules/data_table/CMakeLists.txt @@ -0,0 +1,54 @@ +# --- fn_module_data_table --- +# Static lib bundling the data_table module: TQL pipeline + Lua engine + +# viz_render + data_table.cpp entrypoint. +# +# Apps opt-in via: +# target_link_libraries( PRIVATE fn_module_data_table) +# +# Header access: +# #include "data_table/data_table.h" +# #include "core/data_table_types.h" +# +# Replaces former fn_table_viz target (renamed 2026-05-16 as part of the +# modules system rollout — issue 0097 modules). + +cmake_minimum_required(VERSION 3.16) + +# Module sources: the entrypoint lives here; members live in cpp/functions/. +set(_FN_CPP_ROOT ${CMAKE_SOURCE_DIR}/../cpp) + +add_library(fn_module_data_table STATIC + ${CMAKE_CURRENT_SOURCE_DIR}/data_table.cpp + ${_FN_CPP_ROOT}/functions/core/compute_stage.cpp + ${_FN_CPP_ROOT}/functions/core/compute_pipeline.cpp + ${_FN_CPP_ROOT}/functions/core/tql_emit.cpp + ${_FN_CPP_ROOT}/functions/core/tql_helpers.cpp + ${_FN_CPP_ROOT}/functions/core/tql_apply.cpp + ${_FN_CPP_ROOT}/functions/core/tql_to_sql.cpp + ${_FN_CPP_ROOT}/functions/core/lua_engine.cpp + ${_FN_CPP_ROOT}/functions/core/join_tables.cpp + ${_FN_CPP_ROOT}/functions/core/auto_detect_type.cpp + ${_FN_CPP_ROOT}/functions/core/compute_column_stats.cpp + ${_FN_CPP_ROOT}/functions/core/llm_anthropic.cpp + ${_FN_CPP_ROOT}/functions/viz/viz_render.cpp +) + +# PUBLIC: consumers `#include "data_table/data_table.h"` and "core/..." via cpp/functions. +target_include_directories(fn_module_data_table PUBLIC + ${CMAKE_SOURCE_DIR}/../modules + ${_FN_CPP_ROOT}/functions +) +target_include_directories(fn_module_data_table PRIVATE + ${_FN_CPP_ROOT}/framework +) + +target_compile_definitions(fn_module_data_table PUBLIC FN_LLM_ANTHROPIC=1) + +target_link_libraries(fn_module_data_table PUBLIC + imgui + implot + lua54 +) + +# fn::local_path() (Ask AI export path + TQL save/load) needs fn_framework. +target_link_libraries(fn_module_data_table PRIVATE fn_framework) diff --git a/cpp/functions/viz/data_table.cpp b/modules/data_table/data_table.cpp similarity index 96% rename from cpp/functions/viz/data_table.cpp rename to modules/data_table/data_table.cpp index 9b141d70..0b80f9f6 100644 --- a/cpp/functions/viz/data_table.cpp +++ b/modules/data_table/data_table.cpp @@ -24,7 +24,7 @@ // - tql_to_sql (SQL transpile): incluido desde el playground. Pendiente: registry Wave 4. // - tql_duckdb (FN_TQL_DUCKDB): opcional, sin wrapper en registry todavia. -#include "viz/data_table.h" +#include "data_table/data_table.h" // Framework ImGui (via fn_framework) #include "imgui.h" @@ -138,6 +138,81 @@ static ImVec4 hex_to_imcolor(const std::string& hex) { return ImVec4(r / 255.f, g / 255.f, b / 255.f, 1.f); } +// parse_hex_color: parses "#rrggbb" or "#rrggbbaa" -> ImU32 with explicit alpha. +// Returns IM_COL32(128,128,128,255) on failure (visible fallback). +// v1.4.0 helper for CategoricalChip and ColorScale renderers. +// --------------------------------------------------------------------------- +static ImU32 parse_hex_color(const std::string& hex, float alpha = 1.0f) { + const char* p = hex.c_str(); + if (*p == '#') ++p; + unsigned int r = 0, g = 0, b = 0, a = 255; + int parsed = std::sscanf(p, "%02x%02x%02x%02x", &r, &g, &b, &a); + if (parsed < 3) return IM_COL32(128, 128, 128, 255); + if (parsed == 3) { + // alpha parameter overrides when no alpha in hex string + a = (unsigned int)(alpha * 255.f + 0.5f); + } + return IM_COL32(r, g, b, a); +} + +// lerp_color_along_stops: LERP between N color stops based on t in [0,1]. +// Stops need not be sorted; function sorts a local copy first. +// If stops is empty, uses default green→amber→red gradient. +// alpha overrides the per-channel alpha of the result. +// v1.4.0 helper for ColorScale renderer. +// --------------------------------------------------------------------------- +static ImU32 lerp_color_along_stops( + const std::vector& stops, float t, float alpha) +{ + // Default green→amber→red when caller provides no stops. + static const std::vector kDefault = { + {0.0f, "#22c55e"}, + {0.5f, "#f59e0b"}, + {1.0f, "#ef4444"}, + }; + const auto& sv = stops.empty() ? kDefault : stops; + + // Sort by position (copy; usually already sorted). + std::vector sorted_sv = sv; + std::sort(sorted_sv.begin(), sorted_sv.end(), + [](const data_table::ColorStop& a, const data_table::ColorStop& b){ + return a.position < b.position; + }); + + // Clamp t. + t = t < 0.f ? 0.f : (t > 1.f ? 1.f : t); + + // Edge cases: before first stop or after last stop. + if (t <= sorted_sv.front().position) + return parse_hex_color(sorted_sv.front().color, alpha); + if (t >= sorted_sv.back().position) + return parse_hex_color(sorted_sv.back().color, alpha); + + // Find surrounding stops. + for (size_t i = 0; i + 1 < sorted_sv.size(); ++i) { + const auto& lo = sorted_sv[i]; + const auto& hi = sorted_sv[i + 1]; + if (t >= lo.position && t <= hi.position) { + float span = hi.position - lo.position; + float f = (span > 1e-6f) ? (t - lo.position) / span : 0.f; + ImVec4 ca = hex_to_imcolor(lo.color); + ImVec4 cb = hex_to_imcolor(hi.color); + if (ca.x < 0.f) ca = ImVec4(0.5f, 0.5f, 0.5f, 1.f); + if (cb.x < 0.f) cb = ImVec4(0.5f, 0.5f, 0.5f, 1.f); + float r = ca.x + f * (cb.x - ca.x); + float g = ca.y + f * (cb.y - ca.y); + float b = ca.z + f * (cb.z - ca.z); + unsigned int ri = (unsigned int)(r * 255.f + 0.5f); + unsigned int gi = (unsigned int)(g * 255.f + 0.5f); + unsigned int bi = (unsigned int)(b * 255.f + 0.5f); + unsigned int ai = (unsigned int)(alpha * 255.f + 0.5f); + return IM_COL32(ri, gi, bi, ai); + } + } + // Fallback (should not reach here). + return parse_hex_color(sorted_sv.back().color, alpha); +} + // --------------------------------------------------------------------------- // icon_name_to_glyph: static lookup of icon_name string -> Tabler glyph. // Covers the ~30 most-used icons. Returns nullptr if not found. @@ -392,6 +467,64 @@ static void draw_cell_custom(const ColumnSpec& spec, const char* value, break; } + case CellRenderer::CategoricalChip: { + // Draw a filled circle to the LEFT of the cell text. + // Color determined by matching value against chips rules. + // Always visible (not hover-only). If no rule matches, no dot. + // v1.4.0. + const ChipRule* matched_chip = nullptr; + for (const auto& cr : spec.chips) { + if (cr.match == value) { matched_chip = &cr; break; } + } + if (matched_chip) { + float font_h = ImGui::GetTextLineHeight(); + ImVec2 cursor = ImGui::GetCursorScreenPos(); + float radius = 4.0f; + float cy = cursor.y + font_h * 0.5f; + float cx = cursor.x + radius; + ImDrawList* dl = ImGui::GetWindowDrawList(); + ImU32 chip_col = parse_hex_color(matched_chip->color, 1.0f); + dl->AddCircleFilled(ImVec2(cx, cy), radius, chip_col, 18); + // Advance cursor past the dot + 4px gap. + ImGui::Dummy(ImVec2(radius * 2.0f + 4.0f, font_h)); + ImGui::SameLine(0, 0); + } + ImGui::TextUnformatted(value); + break; + } + + case CellRenderer::ColorScale: { + // Paint cell background with an interpolated color from N-stop gradient. + // Numeric value mapped to t = (val - range_min) / (range_max - range_min). + // Clamped to [0,1]. Non-parseable values render as plain text. + // v1.4.0. + double v_raw = 0.0; + if (!parse_number(value, v_raw)) { + ImGui::TextUnformatted(value); + break; + } + double span = spec.range_max - spec.range_min; + float t = 0.f; + if (span > 1e-12) { + t = (float)((v_raw - spec.range_min) / span); + } + // Clamp. + t = t < 0.f ? 0.f : (t > 1.f ? 1.f : t); + + // Paint background rect covering the full cell area. + ImU32 bg_col = lerp_color_along_stops(spec.range_stops, t, spec.range_alpha); + ImVec2 cell_min = ImGui::GetCursorScreenPos(); + // Use cell size: full column width × row height. + float row_h = ImGui::GetTextLineHeight(); + float col_w = ImGui::GetContentRegionAvail().x; + ImVec2 cell_max = ImVec2(cell_min.x + col_w, cell_min.y + row_h); + ImGui::GetWindowDrawList()->AddRectFilled(cell_min, cell_max, bg_col); + + // Draw text on top. + ImGui::TextUnformatted(value); + break; + } + default: // CellRenderer::Text or unknown — plain text. ImGui::TextUnformatted(value); @@ -979,11 +1112,7 @@ struct UiState { int prev_viz_stage = 0; size_t prev_viz_cfg_h = 0; - // show_chrome user override. Default: chips bar closed — user opens via - // "Show UI" button. Cached as user-set so the API arg show_chrome is - // bypassed from frame 1. - bool chrome_user_set = true; - bool chrome_user_visible = false; + // (chrome_user_set / chrome_user_visible moved to State — per-table now.) // Toggle Table <-> View: remember last non-table display. ViewMode last_non_table_main = ViewMode::Bar; @@ -2618,16 +2747,17 @@ void render(const char* id, } const std::vector* joinables = joinables_v.empty() ? nullptr : &joinables_v; - auto& U_chrome = ui(); - bool chrome_visible = U_chrome.chrome_user_set ? U_chrome.chrome_user_visible : show_chrome; + // Per-table chrome visibility (issue: previously global in UiCache → flipping + // one table's "Show UI" affected all tables on screen). Now lives in State. + bool chrome_visible = st.chrome_user_set ? st.chrome_user_visible : show_chrome; // Toggle Hide/Show UI siempre visible (botoncito arriba a la derecha). { float right = ImGui::GetWindowContentRegionMax().x; ImGui::SetCursorPosX(right - 90.0f); if (ImGui::SmallButton(chrome_visible ? "Hide UI##chrome" : "Show UI##chrome")) { - U_chrome.chrome_user_set = true; - U_chrome.chrome_user_visible = !chrome_visible; + st.chrome_user_set = true; + st.chrome_user_visible = !chrome_visible; } } diff --git a/cpp/functions/viz/data_table.h b/modules/data_table/data_table.h similarity index 100% rename from cpp/functions/viz/data_table.h rename to modules/data_table/data_table.h diff --git a/cpp/functions/viz/data_table.md b/modules/data_table/data_table.md similarity index 84% rename from cpp/functions/viz/data_table.md rename to modules/data_table/data_table.md index d02793e6..0310eac1 100644 --- a/cpp/functions/viz/data_table.md +++ b/modules/data_table/data_table.md @@ -3,10 +3,10 @@ name: data_table kind: function lang: cpp domain: viz -version: "1.3.6" +version: "1.4.0" purity: impure signature: "void data_table::render(const char* id, const std::vector& tables, State& st, std::vector* events_out = nullptr, bool show_chrome = true)" -description: "Render UI completa de tabla TQL: chips bar, tabla, viz panels, column-stats inline, drill, color rules, joins, TQL editor, Ask AI, Button renderer, event sink (ButtonClick/RowDoubleClick/RowRightClick), tooltip per-cell, column_specs persisted in TQL. Dots renderer para sparkline-like de status (v1.3.0). Entry-point publica del stack data_table. Muta State segun interaccion del usuario." +description: "Render UI completa de tabla TQL: chips bar, tabla, viz panels, column-stats inline, drill, color rules, joins, TQL editor, Ask AI, Button renderer, event sink (ButtonClick/RowDoubleClick/RowRightClick), tooltip per-cell, column_specs persisted in TQL. Dots renderer para sparkline-like de status (v1.3.0). CategoricalChip (dot izquierda + text, siempre visible) y ColorScale (gradient N-color en fondo de celda) en v1.4.0. Entry-point publica del stack data_table. Muta State segun interaccion del usuario." tags: [tables, viz, ui, imgui, tql, cpp-tables] uses_functions: - compute_stage_cpp_core @@ -72,8 +72,12 @@ tests: - "Back-compat: both render() signatures (with/without events_out) link" - "Dots: ColumnSpec with CellRenderer::Dots + badges constructs correctly" - "Dots TQL roundtrip: State::aux_column_specs accepts Dots spec" + - "TestCategoricalChipRule: chip rule with match='success' produces correct color" + - "TestColorScaleLerpTwoStops: t=0→first color, t=1→last color, t=0.5→midpoint" + - "TestColorScaleLerpThreeStops: t=0.25 lies between stop0 and stop1" + - "TestColorScaleOutOfRange: t<0 saturates at first; t>1 saturates at last" test_file_path: "cpp/tests/test_column_specs.cpp" -file_path: "cpp/functions/viz/data_table.cpp" +file_path: "modules/data_table/data_table.cpp" params: - name: id desc: "ID unico ImGui para esta instancia, ej. '##orders_table'. Debe ser estable entre frames." @@ -143,6 +147,8 @@ for (const auto& ev : events) { Cuando una app necesita tabla con filtros + agregaciones + viz + joins sobre datos en memoria. Reemplaza `ImGui::BeginTable` inline + toda la logica TQL manual. Sustituye directamente el include del playground (`tables/data_table.h`) cambiando solo el path a `viz/data_table.h`. +Usar `CategoricalChip` cuando quieras un indicador visual (dot de color) siempre visible a la izquierda del texto de la celda, para columnas categoricas (estado, tipo, severidad). Mas discreto que Badge y sin hover-only. Usar `ColorScale` cuando la columna sea numerica continua y quieras dar contexto visual de "alto/bajo/medio" con un fondo tintado proporcional al valor — util para latencias, scores, porcentajes, metricas. + ## Gotchas - **ImGui + ImPlot context activos**: `render()` llama a APIs de ambas librerias. Llamar fuera de un frame activo causa UB. @@ -153,6 +159,8 @@ Cuando una app necesita tabla con filtros + agregaciones + viz + joins sobre dat - **events_out no se limpia**: `render()` solo hace `push_back`. El caller debe llamar `events.clear()` antes de cada frame o acumulara eventos de frames anteriores. - **Button + celda vacia**: si el cell value es vacio, el boton NO se dibuja. La app controla cuando mostrar el boton poniendo un value no vacio (ej. "1" o el ID de la fila). - **RowRightClick emite evento Y abre popup interno**: la tabla de stages (stage>0) sigue abriendo su popup de drill. En el raw table (stage 0), se emite el evento pero el popup de drill antiguo tambien puede abrirse via `U.open_cell_popup`. El caller puede ignorar el popup interno y gestionar su propio menu al detectar `RowRightClick`. +- **CategoricalChip sin regla coincidente → sin dot**: si ninguna `ChipRule.match` coincide con el valor de la celda, solo se renderiza el texto. Definir una regla de fallback explicita si se necesita dot para valores no mapeados. +- **ColorScale clampa fuera de rango**: valores por debajo de `range_min` se tratan como t=0 (primer stop) y valores por encima de `range_max` como t=1 (ultimo stop). Definir `range_min`/`range_max` sensatos para que el gradiente sea informativo; valores muy alejados de la mayoria hacen que todo el gradiente aparezca en un extremo. - **aux_column_specs merge**: si `TableInput.column_specs` esta vacio pero `State.aux_column_specs[0]` no, `render()` los aplica automaticamente. Si el caller pasa column_specs no vacios, ganan sobre los del State. - **Ask AI modal (llm_anthropic)**: el boton "Ask AI" usa un stub interno de `llm_anthropic` que retorna error por defecto. Para activar la feature real, compilar con `-DFN_LLM_ANTHROPIC=1` y proveer `infra/llm_anthropic.h` en el include path. Pendiente Wave 4: promover al registry. - **FN_TQL_DUCKDB**: modo SQL del Ask AI requiere compilar con `-DFN_TQL_DUCKDB=1` y la libreria DuckDB disponible. @@ -196,5 +204,7 @@ v1.3.5 (2026-05-15) — Cell hover paints via TableSetBgColor (covers entire cel v1.3.6 (2026-05-15) — Selection (drag-range) also paints via TableSetBgColor — same edge-to-edge coverage as hover. Header/HeaderHovered/HeaderActive colors set to fully transparent so Selectable doesn't paint anything; all cell bg states (hover, selected, selected+hover) go through TableSetBgColor uniformly. +v1.4.0 (2026-05-16) — new renderers: `CategoricalChip` (dot izquierda + text, always visible, replaces hover-only color-on-text for categorical) + `ColorScale` (continuous N-color LERP gradient for numeric cells, configurable `range_min`/`range_max`/`range_stops`/`range_alpha`). New types: `ChipRule{match,color}` + `ColorStop{position,color}` in `data_table_types.h`. TQL roundtrip (emit+apply) for both renderers. 4 headless tests added to `test_column_specs.cpp`. + --- Promovido desde `cpp/apps/primitives_gallery/playground/tables/data_table.{h,cpp}` — issue 0081-H. diff --git a/modules/data_table/module.md b/modules/data_table/module.md new file mode 100644 index 00000000..d5e3b68b --- /dev/null +++ b/modules/data_table/module.md @@ -0,0 +1,57 @@ +--- +name: data_table +version: 1.4.0 +lang: cpp +description: "Reusable C++ ImGui module to render a full TQL-aware data table: chips bar, table grid, viz panels, column-stats inline, drill, color rules, joins, TQL editor, Ask AI, Button renderer, event sink, tooltip per-cell. Bundles compute pipeline + TQL stack + Lua engine + viz_render." +members: + - data_table_cpp_viz + - compute_stage_cpp_core + - compute_pipeline_cpp_core + - compute_column_stats_cpp_core + - tql_emit_cpp_core + - tql_helpers_cpp_core + - tql_apply_cpp_core + - tql_to_sql_cpp_core + - lua_engine_cpp_core + - join_tables_cpp_core + - auto_detect_type_cpp_core + - llm_anthropic_cpp_core + - viz_render_cpp_viz +tags: [tables, viz, ui, imgui, tql, cpp] +dir_path: modules/data_table +--- + +## Documentation + +C++ ImGui module to render a full data table with TQL pipeline, viz panels, joins, color rules, declarative cell renderers (Badge, Progress, Duration, Icon, Button, Dots, CategoricalChip, ColorScale), drill, Ask AI and event sink. + +Entry-point: `data_table::render(id, tables, state, events_out, show_chrome)`. + +### Opt-in en una app + +1. `app.md`: anadir `uses_modules: [data_table_cpp]`. +2. `CMakeLists.txt`: `target_link_libraries( PRIVATE fn_module_data_table)`. +3. Header: `#include "data_table/data_table.h"` y `#include "core/data_table_types.h"`. +4. Reservar `data_table::State` persistente entre frames y llamar `data_table::render(...)` cada frame. + +### Funciones miembro + +Cada ID en `members` es una funcion del registry que el modulo bundla en su static lib. Cuando una app declara `uses_modules: [data_table_cpp]`, automaticamente "usa" estas funciones a traves del modulo — no hace falta listarlas otra vez en `uses_functions`. + +### Version policy + +Semver. Bumps de version se documentan en `## Capability growth log`. Cambios en API publica (`data_table.h`) = major. Adicion de funcionalidad opt-in = minor. Bugfix = patch. + +## Capability growth log + +- v1.4.0 (2026-05-16) — CategoricalChip (dot izquierda + text) + ColorScale (gradient N-color en fondo de celda) +- v1.3.1 (anterior) — Dots renderer via ImDrawList (font-independent) +- v1.3.0 — Dots renderer para sparkline-like de status timelines +- v1.2.0 — Joins, drill, color rules, tooltip per-cell, Button event sink +- v1.0.0 — Initial table + TQL pipeline + chips bar + +## Notes + +- El modulo se compila como static lib `fn_module_data_table` (cmake target). Static lib bundla todos los miembros — apps consumidoras solo enlazan UN target. +- Replaces former `fn_table_viz` target (2026-05-16). +- Requiere `fn_framework` (para `fn::local_path()` usado en Ask AI export). diff --git a/modules/framework/module.md b/modules/framework/module.md new file mode 100644 index 00000000..23fa0886 --- /dev/null +++ b/modules/framework/module.md @@ -0,0 +1,52 @@ +--- +name: framework +version: 1.0.0 +lang: cpp +description: "Core C++ ImGui app shell: fn::run_app, AppConfig, GLFW + OpenGL + ImGui + ImPlot bootstrap, theming (Mantine dark + indigo), settings/about/menubar/layouts UI, Tabler icons, logging, viewports & AltSnap-safe sizemove, local_files dir, embedded layout storage." +members: + - tokens_cpp_core + - icon_font_cpp_core + - app_settings_cpp_core + - app_about_cpp_core + - fps_overlay_cpp_core + - panel_menu_cpp_core + - layouts_menu_cpp_core + - app_menubar_cpp_core + - logger_cpp_core + - log_window_cpp_core + - gl_loader_cpp_gfx + - layout_storage_cpp_core + - selectable_text_cpp_core +tags: [framework, imgui, cpp, core] +dir_path: cpp/framework +--- + +## Documentation + +Foundational C++ ImGui app shell shared by every desktop app in the registry. Apps opt-in transparently — every C++ app already links `fn_framework` via the `add_imgui_app` macro. The framework provides: + +- `fn::run_app(cfg, render_fn)`: GLFW + OpenGL3 + ImGui + ImPlot setup, multi-viewport, docking, AltSnap-safe sizemove, icon attach, layouts persistence, log window, settings window, about window, menubar. +- `fn::local_path(name)`: scoped writable path under `/local_files/`. +- Design tokens (Mantine dark + indigo accent). +- Tabler icons (TI_* macros). +- `fn::framework_version()` / `fn::framework_description()` (post 1.0.0). + +Apps NEVER list these members in their own `uses_functions` — they come transitively via `fn_framework`. Audited via [[cpp_apps]] rule. + +### Version policy + +Semver. Major = breaking ABI/API of public `fn::run_app` or `AppConfig`. Minor = additive (new optional config field, new helper). Patch = bugfix. + +### Boundaries + +Framework does NOT include modules like `data_table`. Apps that want tables opt-in via `uses_modules: [data_table_cpp]` and `target_link_libraries( PRIVATE fn_module_data_table)`. The framework is intentionally small. + +## Capability growth log + +- v1.0.0 (2026-05-16) — Initial framing as a versioned module. Members above are the bundled units of `fn_framework` static lib. Pre-1.0.0 history lives in git. + +## Notes + +- Static lib target: `fn_framework` (defined in `cpp/CMakeLists.txt`). +- Generated header `cpp/framework/version_generated.h` (gitignored) exposes `FN_FRAMEWORK_VERSION` constant. +- About panel of every app reads `fn::framework_version()` at runtime. diff --git a/python/functions/core/validate_recipe_yaml.py b/python/functions/core/validate_recipe_yaml.py index 1850870d..1d20129b 100644 --- a/python/functions/core/validate_recipe_yaml.py +++ b/python/functions/core/validate_recipe_yaml.py @@ -110,11 +110,23 @@ def validate_recipe_yaml(yaml_text: str) -> dict: ) sink = output.get("sink") - valid_sinks = {"data_factory.runs", "stdout", "json_file"} + # duckdb sink: requires output.duckdb_path (relative or absolute) and + # output.table (table name). Optional output.database_id (default = + # recipe_name + "_db") used to register/lookup in data_factory.databases. + valid_sinks = {"data_factory.runs", "stdout", "json_file", "duckdb"} if sink is not None and sink not in valid_sinks: errors.append( f"Campo 'output.sink' debe ser uno de {sorted(valid_sinks)}, got '{sink}'." ) + if sink == "duckdb": + if not output.get("duckdb_path"): + errors.append( + "Sink 'duckdb' requiere 'output.duckdb_path' (ruta al archivo .duckdb)." + ) + if not output.get("table"): + errors.append( + "Sink 'duckdb' requiere 'output.table' (nombre de la tabla destino)." + ) return { "valid": len(errors) == 0, diff --git a/python/functions/infra/claude_cli_prompt.py b/python/functions/infra/claude_cli_prompt.py index 8fe11f02..c8b69b7c 100644 --- a/python/functions/infra/claude_cli_prompt.py +++ b/python/functions/infra/claude_cli_prompt.py @@ -1,9 +1,28 @@ """Invoca `claude -p` via subprocess y devuelve la respuesta como string.""" +import os import shutil import subprocess +def _resolve_claude_bin() -> str | None: + """Localiza claude CLI: PATH first, luego rutas convencionales.""" + p = shutil.which("claude") + if p: + return p + # Fallback paths comunes (WSL subsession sin .profile cargado, etc). + home = os.path.expanduser("~") + candidates = [ + f"{home}/.local/bin/claude", + "/usr/local/bin/claude", + "/opt/homebrew/bin/claude", + ] + for c in candidates: + if os.path.isfile(c) and os.access(c, os.X_OK): + return c + return None + + def claude_cli_prompt( prompt: str, timeout_s: int = 60, @@ -24,16 +43,18 @@ def claude_cli_prompt( Respuesta de Claude como texto (stdout), truncada a max_chars_response. Raises: - FileNotFoundError: Si `claude` no esta en PATH. + FileNotFoundError: Si `claude` no esta en PATH ni rutas convencionales. RuntimeError: Si exit code != 0 (incluye primeros 500 chars de stderr). subprocess.TimeoutExpired: Si la llamada supera timeout_s segundos. """ - if shutil.which("claude") is None: + claude_bin = _resolve_claude_bin() + if claude_bin is None: raise FileNotFoundError( - "'claude' CLI no encontrado en PATH. Instala Claude Code." + "'claude' CLI no encontrado en PATH ni rutas convencionales " + "(~/.local/bin, /usr/local/bin, /opt/homebrew/bin). Instala Claude Code." ) - cmd = ["claude", "-p", prompt] + cmd = [claude_bin, "-p", prompt] if model: cmd.extend(["--model", model]) if extra_args: diff --git a/python/functions/infra/codegen_app_modules.md b/python/functions/infra/codegen_app_modules.md new file mode 100644 index 00000000..e8ce5fbc --- /dev/null +++ b/python/functions/infra/codegen_app_modules.md @@ -0,0 +1,75 @@ +--- +name: codegen_app_modules +kind: function +lang: py +domain: infra +version: "1.0.0" +purity: impure +signature: "generate(app_md: Path, modules_root: Path, app_name: str, out_path: Path) -> int" +description: "Reads app.md uses_modules + modules//module.md frontmatters, emits _modules_generated.cpp with fn::app_modules_array[] + fn::app_modules_count. CMake hook for add_imgui_app. Pure YAML parsing, no registry.db dep." +tags: [codegen, modules, cmake, cpp, build] +uses_functions: [] +uses_types: [] +returns: [] +returns_optional: false +error_type: "error_go_core" +imports: + - yaml +example: | + python python/functions/infra/codegen_app_modules.py \ + --app-md apps/data_factory/app.md \ + --modules-root modules \ + --app-name data_factory \ + --out cpp/build/apps/data_factory/data_factory_modules_generated.cpp +file_path: "python/functions/infra/codegen_app_modules.py" +params: + - name: app_md + desc: "Path absoluto al app.md de la app consumidora. Lee uses_modules del frontmatter YAML." + - name: modules_root + desc: "Raiz del directorio modules/. Cada modulo es modules//module.md." + - name: app_name + desc: "Nombre de la app (solo para el comment-header del .cpp generado)." + - name: out_path + desc: "Path donde escribir el .cpp generado. Idempotente: skip si contenido coincide." +output: "Exit code: 0 si OK, 2 si OK pero algun modulo declarado no existe (warning), >0 si error." +--- + +## Ejemplo + +Generar el .cpp para `data_factory`: + +```bash +python python/functions/infra/codegen_app_modules.py \ + --app-md apps/data_factory/app.md \ + --modules-root modules \ + --app-name data_factory \ + --out /tmp/data_factory_modules_generated.cpp +``` + +Si `data_factory/app.md` declara `uses_modules: [data_table_cpp]`, el .cpp generado es: + +```cpp +// Auto-generated by codegen_app_modules.py — do not edit. +// App: data_factory +// Source of truth: apps/data_factory/app.md (uses_modules) + +#include "app_modules.h" + +namespace fn { +const ModuleInfo app_modules_array[] = { + { "data_table", "1.4.0", "Reusable C++ ImGui module..." }, +}; +const unsigned long app_modules_count = 1; +} // namespace fn +``` + +## Cuando usarla + +CMake hook automatico — la macro `add_imgui_app` la invoca al configurar el build. Apps no la llaman manualmente. Manual override: solo si quieres regenerar fuera del flujo cmake (debugging). + +## Gotchas + +- Resuelve `_cpp` strippeando el sufijo `_cpp/_py/_ts/_bash/_go`. Mismo patron que `GenerateModuleID`. +- Si un modulo declarado en `uses_modules` no existe, emite warning a stderr y EXIT=2 (no falla el build). +- Idempotente: solo reescribe si el contenido cambia. Evita rebuilds innecesarios cuando los modulos no cambiaron. +- Requiere `pyyaml`. Disponible en `python/.venv` del registry. diff --git a/python/functions/infra/codegen_app_modules.py b/python/functions/infra/codegen_app_modules.py new file mode 100644 index 00000000..23736c29 --- /dev/null +++ b/python/functions/infra/codegen_app_modules.py @@ -0,0 +1,149 @@ +"""Generate _modules_generated.cpp from app.md uses_modules + modules/*/module.md. + +Stand-alone — no dependencies beyond PyYAML. Invoked from CMake at configure time. +Reads YAML frontmatter directly (no registry.db dependency, no Go binary). +""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path +from typing import Optional + +import yaml + + +def _read_frontmatter(md_path: Path) -> dict: + if not md_path.exists(): + return {} + text = md_path.read_text(encoding="utf-8") + if not text.startswith("---\n") and not text.startswith("---\r\n"): + return {} + end = text.find("\n---", 4) + if end < 0: + return {} + raw = text[4:end] + try: + return yaml.safe_load(raw) or {} + except yaml.YAMLError: + return {} + + +def _escape_c_string(s: str) -> str: + out = [] + for ch in s or "": + if ch == "\\": + out.append("\\\\") + elif ch == '"': + out.append('\\"') + elif ch == "\n": + out.append("\\n") + elif ch == "\r": + out.append("\\r") + elif ch == "\t": + out.append("\\t") + elif ord(ch) < 32: + out.append(f"\\x{ord(ch):02x}") + else: + out.append(ch) + return "".join(out) + + +def _resolve_module(modules_root: Path, mod_id: str) -> Optional[dict]: + """mod_id is e.g. `data_table_cpp`. Lookup module.md by name (strip _).""" + name = mod_id + for suffix in ("_cpp", "_py", "_ts", "_bash", "_go"): + if name.endswith(suffix): + name = name[: -len(suffix)] + break + md = modules_root / name / "module.md" + fm = _read_frontmatter(md) + if not fm: + return None + return { + "name": fm.get("name", name), + "version": fm.get("version", "0.0.0"), + "description": fm.get("description", ""), + } + + +def generate(app_md: Path, modules_root: Path, app_name: str, out_path: Path) -> int: + fm = _read_frontmatter(app_md) + uses_modules = fm.get("uses_modules") or [] + if not isinstance(uses_modules, list): + uses_modules = [] + + entries: list[dict] = [] + missing: list[str] = [] + for mid in uses_modules: + info = _resolve_module(modules_root, str(mid)) + if info is None: + missing.append(str(mid)) + continue + entries.append(info) + + lines: list[str] = [] + lines.append(f"// Auto-generated by codegen_app_modules.py — do not edit.") + lines.append(f"// App: {app_name}") + lines.append(f"// Source of truth: {app_md.as_posix()} (uses_modules)") + lines.append("") + lines.append('#include "app_modules.h"') + lines.append("") + lines.append("namespace fn {") + if entries: + lines.append("const ModuleInfo app_modules_array[] = {") + for e in entries: + lines.append( + ' { "%s", "%s", "%s" },' + % ( + _escape_c_string(e["name"]), + _escape_c_string(e["version"]), + _escape_c_string(e["description"]), + ) + ) + lines.append("};") + lines.append(f"const unsigned long app_modules_count = {len(entries)};") + else: + lines.append("const ModuleInfo app_modules_array[1] = { { nullptr, nullptr, nullptr } };") + lines.append("const unsigned long app_modules_count = 0;") + lines.append("} // namespace fn") + lines.append("") + + new_content = "\n".join(lines) + + # Idempotent: skip rewrite when content matches. + if out_path.exists() and out_path.read_text(encoding="utf-8") == new_content: + return 0 if not missing else 2 + + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_text(new_content, encoding="utf-8") + + if missing: + sys.stderr.write( + f"codegen_app_modules: WARNING — module(s) not found: {', '.join(missing)} " + f"(app {app_name})\n" + ) + return 2 + return 0 + + +def main() -> int: + ap = argparse.ArgumentParser(description="Generate _modules_generated.cpp from app.md") + ap.add_argument("--app-md", required=True, help="Path to app.md") + ap.add_argument("--modules-root", required=True, help="Path to modules/ root") + ap.add_argument("--app-name", required=True, help="App name (for comment header)") + ap.add_argument("--out", required=True, help="Output path for generated .cpp") + args = ap.parse_args() + + rc = generate( + app_md=Path(args.app_md), + modules_root=Path(args.modules_root), + app_name=args.app_name, + out_path=Path(args.out), + ) + return 0 if rc in (0, 2) else rc + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/python/functions/infra/export_hub_manifest.md b/python/functions/infra/export_hub_manifest.md new file mode 100644 index 00000000..237645a9 --- /dev/null +++ b/python/functions/infra/export_hub_manifest.md @@ -0,0 +1,72 @@ +--- +name: export_hub_manifest +kind: function +lang: py +domain: infra +version: "1.0.0" +purity: impure +signature: "export_hub_manifest(out_path: str, *, registry_root: str | None = None) -> dict" +description: "Genera el TSV sidecar para app_hub_launcher: consulta registry.db por todas las apps cpp/imgui, lee su app.md para extraer nombre, descripcion y accent_hex, y escribe un archivo TSV con cabecera a out_path. Retorna {ok, count, out_path}." +tags: [hub, launcher, manifest, suite, cpp-windows] +uses_functions: [] +uses_types: [] +returns: [] +returns_optional: false +error_type: "error_go_core" +imports: [sqlite3, yaml, pathlib] +params: + - name: out_path + desc: "Ruta de destino del archivo TSV. Puede ser absoluta o relativa al cwd. El directorio padre se crea si no existe." + - name: registry_root + desc: "Raiz del fn_registry. Si None, usa la variable de entorno FN_REGISTRY_ROOT o /home/lucas/fn_registry como fallback." +output: "Dict {ok: True, count: N, out_path: str} con la ruta absoluta del TSV escrito y el numero de apps incluidas." +tested: false +tests: [] +test_file_path: "" +file_path: "python/functions/infra/export_hub_manifest.py" +--- + +## Ejemplo + +```bash +# Uso directo con fn run (la salida JSON se imprime en stdout) +./fn run export_hub_manifest_py_infra /mnt/c/Users/lucas/Desktop/apps/app_hub_launcher/local_files/hub_manifest.tsv +``` + +```python +# Desde un heredoc o pipeline Python +import sys +sys.path.insert(0, "python/functions") +from infra import export_hub_manifest + +result = export_hub_manifest( + "/mnt/c/Users/lucas/Desktop/apps/app_hub_launcher/local_files/hub_manifest.tsv" +) +print(result) +# {'ok': True, 'count': 12, 'out_path': '/mnt/c/Users/lucas/Desktop/apps/app_hub_launcher/local_files/hub_manifest.tsv'} +``` + +```bash +# Ver el contenido del TSV generado +head -5 /mnt/c/Users/lucas/Desktop/apps/app_hub_launcher/local_files/hub_manifest.tsv +# name display_name description accent_hex +# chart_demo Chart Demo Demo ImGui de primitivos viz... #0ea5e9 +# dag_engine_ui Dag Engine Ui Motor de DAGs con frontend... #f59e0b +``` + +## Cuando usarla + +Antes de desplegar `app_hub_launcher` a Windows: genera el `hub_manifest.tsv` que el hub lee al arrancar para listar y colorear los botones de cada app. El hub en runtime no tiene acceso a `registry.db` ni a los `app.md` del WSL, por lo que necesita este sidecar. Ejecutar tras añadir o modificar una app C++ imgui en el registry. + +## Gotchas + +- **PyYAML en el venv**: requiere `yaml` disponible en `python/.venv`. Ya instalado por defecto. Si falta: `cd python && uv pip install pyyaml`. +- **app.md faltante no aborta**: si un `app.md` no existe o tiene frontmatter malformado, la app sigue apareciendo en el TSV con `description` vacía y accent `#64748b` (slate). Se imprime un WARN a stderr. +- **Filtro estricto `lang='cpp' AND framework='imgui'`**: solo apps C++ con el shell `fn::run_app`. Apps Python, Bash o C++ sin imgui quedan excluidas. Correcto para el hub. +- **La ruta `dir_path` en registry.db es relativa a la raiz del registry**: la funcion la combina con `registry_root` para construir el path absoluto al `app.md`. Si una app tiene `dir_path` incorrecto en su `app.md`, el WARN indicara cual falló. +- **TSV UTF-8**: el hub debe abrir el archivo con encoding UTF-8. Tabs y saltos de linea en los campos se limpian automaticamente (reemplazados por espacio). +- **`display_name` es generado, no leido**: se deriva del `name` de la app convirtiendo snake_case a Title Case. No se puede personalizar desde el `app.md` en esta version. + +## Capability growth log + +*(sin cambios desde v1.0.0)* diff --git a/python/functions/infra/export_hub_manifest.py b/python/functions/infra/export_hub_manifest.py new file mode 100644 index 00000000..dd9b13e4 --- /dev/null +++ b/python/functions/infra/export_hub_manifest.py @@ -0,0 +1,142 @@ +"""export_hub_manifest — genera el TSV sidecar para app_hub_launcher.""" + +from __future__ import annotations + +import os +import sqlite3 +import sys +from pathlib import Path +from typing import Any + + +def _read_frontmatter(md_path: Path) -> dict[str, Any]: + """Parse YAML frontmatter from a .md file. Returns {} on any error.""" + try: + import yaml # PyYAML — available in python/.venv + + text = md_path.read_text(encoding="utf-8") + if not text.startswith("---"): + return {} + # Find the closing --- + end = text.find("\n---", 3) + if end == -1: + return {} + yaml_block = text[3:end].strip() + data = yaml.safe_load(yaml_block) + return data if isinstance(data, dict) else {} + except Exception as exc: + print(f"[export_hub_manifest] WARN: could not parse {md_path}: {exc}", file=sys.stderr) + return {} + + +def _snake_to_display(name: str) -> str: + """Convert snake_case name to Title Case With Spaces. + + Examples: + graph_explorer -> Graph Explorer + dag_engine_ui -> Dag Engine Ui + app_hub_launcher -> App Hub Launcher + """ + return " ".join(part.capitalize() for part in name.split("_")) + + +def export_hub_manifest(out_path: str, *, registry_root: str | None = None) -> dict: + """Generate TSV sidecar manifest for app_hub_launcher. + + Queries registry.db for all cpp/imgui apps, reads their app.md + frontmatter to extract name, description and accent color, then + writes a UTF-8 TSV to out_path. + + Args: + out_path: Destination path for the TSV manifest file. + registry_root: Path to the fn_registry root directory. + Defaults to FN_REGISTRY_ROOT env var or /home/lucas/fn_registry. + + Returns: + {"ok": True, "count": N, "out_path": ""} + """ + root = Path( + registry_root + or os.environ.get("FN_REGISTRY_ROOT", "/home/lucas/fn_registry") + ).resolve() + + db_path = root / "registry.db" + if not db_path.exists(): + raise FileNotFoundError(f"registry.db not found at {db_path}") + + con = sqlite3.connect(str(db_path)) + con.row_factory = sqlite3.Row + try: + rows = con.execute( + "SELECT id, name, dir_path FROM apps WHERE lang='cpp' AND framework='imgui' ORDER BY name" + ).fetchall() + finally: + con.close() + + DEFAULT_ACCENT = "#64748b" + TSV_HEADER = "name\tdisplay_name\tdescription\taccent_hex\n" + + lines: list[str] = [TSV_HEADER] + count = 0 + + for row in rows: + app_name: str = row["name"] + dir_path: str = row["dir_path"] + + # Derive defaults in case app.md is missing / malformed + display_name = _snake_to_display(app_name) + description = "" + accent_hex = DEFAULT_ACCENT + + md_path = root / dir_path / "app.md" + if md_path.exists(): + fm = _read_frontmatter(md_path) + if fm: + description = fm.get("description", "") or "" + icon_block = fm.get("icon") + if isinstance(icon_block, dict): + accent_hex = icon_block.get("accent", DEFAULT_ACCENT) or DEFAULT_ACCENT + else: + print( + f"[export_hub_manifest] WARN: empty/malformed frontmatter in {md_path}", + file=sys.stderr, + ) + else: + print( + f"[export_hub_manifest] WARN: app.md missing for {app_name} at {md_path}", + file=sys.stderr, + ) + + # Sanitize: TSV values must not contain tabs or newlines + def clean(s: str) -> str: + return s.replace("\t", " ").replace("\n", " ").replace("\r", "") + + lines.append( + f"{clean(app_name)}\t{clean(display_name)}\t{clean(description)}\t{clean(accent_hex)}\n" + ) + count += 1 + + out = Path(out_path).resolve() + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text("".join(lines), encoding="utf-8") + + return {"ok": True, "count": count, "out_path": str(out)} + + +if __name__ == "__main__": + import argparse + import json + + parser = argparse.ArgumentParser( + description="Export hub manifest TSV for app_hub_launcher." + ) + parser.add_argument("out_path", help="Destination .tsv file path") + parser.add_argument( + "--registry-root", + default=None, + help="Path to fn_registry root (default: FN_REGISTRY_ROOT env or /home/lucas/fn_registry)", + ) + args = parser.parse_args() + + result = export_hub_manifest(args.out_path, registry_root=args.registry_root) + print(json.dumps(result, indent=2)) diff --git a/python/functions/pipelines/cdp_extract_recipe.md b/python/functions/pipelines/cdp_extract_recipe.md index 10b15169..90d10ea9 100644 --- a/python/functions/pipelines/cdp_extract_recipe.md +++ b/python/functions/pipelines/cdp_extract_recipe.md @@ -3,7 +3,7 @@ name: cdp_extract_recipe kind: pipeline lang: py domain: pipelines -version: "1.0.0" +version: "1.2.0" purity: impure signature: "def cdp_extract_recipe(recipe_path: str, debug_port: int = 9222, tab_id: str | None = None, record_run: bool = True) -> dict" description: "Ejecuta una recipe YAML contra Chrome remoto via CDP. Valida recipe, busca tab por url_pattern, ejecuta steps (wait_selector/js) y envia resultado al sink declarado." @@ -22,7 +22,7 @@ params: - name: tab_id desc: "ID del tab a usar. Si None, busca tab cuyo URL matchee url_pattern de la recipe." - name: record_run - desc: "Si True y output.sink=='data_factory.runs', registra la ejecucion en data_factory." + desc: "Si True, registra la ejecucion en data_factory.runs (para sink 'data_factory.runs' y 'duckdb')." output: "dict {status: ok|error, rows_out: int, kb_out: float, duration_ms: int, error: str, sample_rows: list}" tested: false tests: [] @@ -60,6 +60,10 @@ output: Cuando tienes una recipe YAML validada y Chrome corriendo con remote debugging, y quieres extraer datos en un solo paso sin montar pipeline manualmente. Encadena con `cdp_open_url_and_wait` si necesitas abrir la URL primero. +## Capability growth log + +- v1.2.0 (2026-05-16) — sink `duckdb` writes rows to a DuckDB file + registers run in data_factory.runs with storage_db_id/storage_table for traceability. + ## Gotchas - Chrome debe estar corriendo con `--remote-debugging-port=`. diff --git a/python/functions/pipelines/cdp_extract_recipe.py b/python/functions/pipelines/cdp_extract_recipe.py index e87c9224..e97ec425 100644 --- a/python/functions/pipelines/cdp_extract_recipe.py +++ b/python/functions/pipelines/cdp_extract_recipe.py @@ -41,9 +41,14 @@ def _ws_send_recv(ws, msg_id: int, method: str, params: dict, timeout: float = 1 def _poll_selector(ws, selector: str, timeout_s: float = 10.0) -> bool: - """Polling cada 200ms hasta que document.querySelector(selector) no sea null.""" + """Polling cada 200ms hasta que document.querySelector(selector) no sea null. + + Drena eventos CDP (paginas con Page.enable emiten loads, frames, etc.) y + matchea por `id` para evitar leer respuestas ajenas o eventos del server. + """ deadline = time.time() + timeout_s msg_id = 1000 + ws.settimeout(0.5) while time.time() < deadline: ws.send(json.dumps({ "id": msg_id, @@ -53,19 +58,28 @@ def _poll_selector(ws, selector: str, timeout_s: float = 10.0) -> bool: "returnByValue": True, } })) - time.sleep(0.2) - msg_id += 1 - # Leer respuesta en loop simple (websocket-client sync) - # Para modo sync usamos recv() - try: - raw = ws.sock.recv() - if raw: + # Leer hasta 30 frames buscando uno con nuestro id; ignorar eventos. + got_response = False + for _ in range(30): + try: + raw = ws.recv() + except Exception: + break + if not raw: + break + try: msg = json.loads(raw) + except Exception: + continue + if msg.get("id") == msg_id: + got_response = True val = msg.get("result", {}).get("result", {}).get("value", False) if val: return True - except Exception: - pass + break + msg_id += 1 + if not got_response: + time.sleep(0.2) return False @@ -188,16 +202,114 @@ def cdp_extract_recipe( out_path = output_cfg.get("path", "output.json") with open(out_path, "w", encoding="utf-8") as f: json.dump(rows, f, ensure_ascii=False, indent=2) + elif sink == "duckdb": + duckdb_path = output_cfg.get("duckdb_path", "") + table_name = output_cfg.get("table", "") + if not duckdb_path or not table_name: + # not fatal: rows already returned via sample_rows + pass + else: + import duckdb + import uuid + import datetime + # resolve duckdb_path relative to FN_REGISTRY_ROOT if not absolute + if not os.path.isabs(duckdb_path): + duckdb_path = os.path.join(os.environ.get("FN_REGISTRY_ROOT", ""), duckdb_path) + os.makedirs(os.path.dirname(duckdb_path), exist_ok=True) + conn = duckdb.connect(duckdb_path) + try: + if rows: + # Detect columns from first row keys (assumes list of dicts). + if not isinstance(rows[0], dict): + # Fallback: wrap scalar rows as {"value": v}. + rows = [{"value": r} for r in rows] + cols = list(rows[0].keys()) + # Build CREATE TABLE IF NOT EXISTS with VARCHAR for safety + # plus extracted_at TIMESTAMP and run_id VARCHAR for lineage. + col_defs = ", ".join(f'"{c}" VARCHAR' for c in cols) + ddl = ( + f'CREATE TABLE IF NOT EXISTS "{table_name}" (' + f' run_id VARCHAR, extracted_at TIMESTAMP, {col_defs}' + f')' + ) + conn.execute(ddl) + run_id_str = uuid.uuid4().hex[:16] + now_iso = datetime.datetime.utcnow().isoformat() + "Z" + placeholders = ", ".join(["?"] * (len(cols) + 2)) + insert_sql = ( + f'INSERT INTO "{table_name}" ' + f'(run_id, extracted_at, {", ".join(chr(34) + c + chr(34) for c in cols)}) ' + f'VALUES ({placeholders})' + ) + for r in rows: + vals = [run_id_str, now_iso] + [str(r.get(c, "")) for c in cols] + conn.execute(insert_sql, vals) + # Also record into data_factory.runs with storage info + registry_root = os.environ.get("FN_REGISTRY_ROOT", "") + if registry_root and record_run: + import sqlite3 + df_db = os.path.join(registry_root, "apps", "data_factory", "data_factory.db") + if os.path.exists(df_db): + try: + df_conn = sqlite3.connect(df_db) + df_conn.execute("PRAGMA foreign_keys = ON") + trigger = "dag" if os.environ.get("DAGU_ENV") else "manual" + db_id = output_cfg.get("database_id", recipe.get("name", "unknown") + "_db") + df_run_id = uuid.uuid4().hex[:16] + df_conn.execute( + "INSERT INTO runs(id, node_id, started_at, finished_at, status," + " rows_in, rows_out, kb_in, kb_out, duration_ms, trigger, error, notes," + " storage_db_id, storage_table)" + " VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", + ( + df_run_id, recipe.get("name", "unknown"), + now_iso, now_iso, "success", + 0, rows_out, 0, int(round(kb_out)), duration_ms, + trigger, "", + json.dumps({"sample": sample_rows[:2]}, ensure_ascii=False)[:1000], + db_id, table_name, + ), + ) + df_conn.commit() + df_conn.close() + except Exception: + pass + finally: + conn.close() elif sink == "data_factory.runs" and record_run: + # Escribe DIRECTO a data_factory.db evitando spawn `fn run` (loop infinito + # si data_factory_record_run re-ejecuta esta misma funcion). Confia en que + # el node ya existe en `nodes` con id == recipe.name. try: - from pipelines.data_factory_record_run import data_factory_record_run - data_factory_record_run( - node_id=recipe.get("name", "unknown"), - function_id="cdp_extract_recipe_py_pipelines", - args={"recipe_path": recipe_path, "debug_port": debug_port}, + import sqlite3 + import datetime + import uuid + registry_root = os.environ.get("FN_REGISTRY_ROOT", "").strip() + if not registry_root: + # No fatal — el dato ya fue extraido / impreso por otro sink + raise RuntimeError("FN_REGISTRY_ROOT not set; cannot locate data_factory.db") + db_path = os.path.join(registry_root, "apps", "data_factory", "data_factory.db") + trigger = "dag" if os.environ.get("DAGU_ENV") else "manual" + run_id = uuid.uuid4().hex[:16] + now = datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S.%fZ") + node_id = recipe.get("name", "unknown") + conn = sqlite3.connect(db_path) + conn.execute("PRAGMA foreign_keys = ON") + conn.execute( + "INSERT INTO runs(id, node_id, started_at, finished_at, status," + " rows_in, rows_out, kb_in, kb_out, duration_ms, trigger, error, notes)" + " VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?)", + ( + run_id, node_id, now, now, "success", + 0, rows_out, 0, int(round(kb_out)), duration_ms, + trigger, "", + json.dumps({"sample": sample_rows[:2]}, ensure_ascii=False)[:1000], + ), ) - except Exception as e: - # No fatal — el dato ya fue extraido + conn.commit() + conn.close() + except Exception: + # No fatal — el dato ya fue extraido (sample_rows en retorno) pass return { diff --git a/python/functions/pipelines/dedup_duckdb_table_by_hash.md b/python/functions/pipelines/dedup_duckdb_table_by_hash.md new file mode 100644 index 00000000..ca9ee540 --- /dev/null +++ b/python/functions/pipelines/dedup_duckdb_table_by_hash.md @@ -0,0 +1,60 @@ +--- +name: dedup_duckdb_table_by_hash +kind: pipeline +lang: py +domain: pipelines +purity: impure +version: "1.0.0" +signature: "def dedup_duckdb_table_by_hash(duckdb_path: str, table: str, exclude_cols: list[str] | None = None) -> dict" +description: "Elimina filas duplicadas de una tabla DuckDB calculando un md5 de las columnas de datos. Anade columna row_hash idempotentemente, actualiza hashes nulos y borra duplicados conservando la primera insercion por rowid." +tags: [dedup, duckdb, transformer, pipeline, dataops] +uses_functions: [cdp_extract_recipe_py_pipelines] +uses_types: [] +returns: [] +returns_optional: false +error_type: error_go_core +imports: [duckdb] +tested: true +tests: + - "dedup elimina filas duplicadas y conserva unicas" +test_file_path: "python/functions/pipelines/dedup_duckdb_table_by_hash_test.py" +file_path: "python/functions/pipelines/dedup_duckdb_table_by_hash.py" +params: + - name: duckdb_path + desc: "Ruta DuckDB file (absoluta o relativa a FN_REGISTRY_ROOT)." + - name: table + desc: "Nombre tabla a deduplicar." + - name: exclude_cols + desc: "Cols a excluir del hash (metadata como run_id, extracted_at, row_hash). None usa default [run_id, extracted_at, row_hash]." +output: "dict {status, rows_before, rows_after, dedup_removed, duration_ms, hash_column}" +--- + +## Ejemplo + +```python +from pipelines.dedup_duckdb_table_by_hash import dedup_duckdb_table_by_hash + +r = dedup_duckdb_table_by_hash("apps/data_factory/data/hn_top_stories.duckdb", "hn_stories") +print(r) +# {"status": "ok", "rows_before": 120, "rows_after": 30, "dedup_removed": 90, "duration_ms": 45, "hash_column": "row_hash"} +``` + +CLI directo: + +```bash +/home/lucas/fn_registry/python/.venv/bin/python3 \ + python/functions/pipelines/dedup_duckdb_table_by_hash.py \ + apps/data_factory/data/hn_top_stories.duckdb hn_stories +``` + +## Cuando usarla + +Cuando un extractor periodico re-inserta filas iguales (mismo contenido, distinto `run_id`/`extracted_at`) y quieres deduplicar in-place sin tocar el pipeline upstream. Tipicamente como paso `transformer` despues de `cdp_extract_recipe` en un DAG de scraping. + +## Gotchas + +- **rowid y VACUUM**: DuckDB rowid puede recalcularse tras `VACUUM`. En esta funcion solo se usa dentro de la misma transaccion de DELETE, por lo que no hay inconsistencia practica. +- **Colisiones md5**: md5 no colisiona en practica para tablas de escala HN (miles de filas). Si la tabla crece a millones de filas con datos binarios, cambiar `md5(...)` por `sha256(...)` en el SQL. +- **Tabla inexistente**: si `` no existe en el DuckDB, retorna `status=error` con mensaje descriptivo en lugar de lanzar excepcion. +- **exclude_cols case**: la comparacion de columnas excluidas es case-insensitive (`c.lower()`), pero el nombre en la query se usa tal cual lo devuelve `DESCRIBE`. +- **Primera ejecucion**: si la tabla ya tiene `row_hash` de una ejecucion anterior, solo se actualizan las filas con `row_hash IS NULL` (idempotente). diff --git a/python/functions/pipelines/dedup_duckdb_table_by_hash.py b/python/functions/pipelines/dedup_duckdb_table_by_hash.py new file mode 100644 index 00000000..5aa9e9ce --- /dev/null +++ b/python/functions/pipelines/dedup_duckdb_table_by_hash.py @@ -0,0 +1,141 @@ +"""dedup_duckdb_table_by_hash — Remove duplicate rows from a DuckDB table using md5 hash of data columns.""" + +from __future__ import annotations + +import os +import time +from typing import Any + + +def dedup_duckdb_table_by_hash( + duckdb_path: str, + table: str, + exclude_cols: list[str] | None = None, +) -> dict[str, Any]: + """Remove duplicate rows from a DuckDB table by computing md5 hash of data columns. + + Args: + duckdb_path: Path to DuckDB file. Absolute or relative to FN_REGISTRY_ROOT. + table: Table name to deduplicate. + exclude_cols: Columns to exclude from hash computation (metadata cols). + Defaults to ["run_id", "extracted_at", "row_hash"]. + + Returns: + dict with keys: status, rows_before, rows_after, dedup_removed, + duration_ms, hash_column. + """ + import duckdb # type: ignore + + t0 = time.monotonic() + + # Resolve path against FN_REGISTRY_ROOT if relative + if not os.path.isabs(duckdb_path): + root = os.environ.get("FN_REGISTRY_ROOT", os.getcwd()) + duckdb_path = os.path.join(root, duckdb_path) + + if exclude_cols is None: + exclude_cols = ["run_id", "extracted_at", "row_hash"] + + exclude_set = {c.lower() for c in exclude_cols} + + conn = duckdb.connect(duckdb_path) + try: + # Verify table exists + tables = [r[0] for r in conn.execute("SHOW TABLES").fetchall()] + if table not in tables: + return { + "status": "error", + "error": f"Table '{table}' not found in {duckdb_path}. Available: {tables}", + "rows_before": 0, + "rows_after": 0, + "dedup_removed": 0, + "duration_ms": int((time.monotonic() - t0) * 1000), + "hash_column": "row_hash", + } + + # Introspect columns + desc = conn.execute(f'DESCRIBE "{table}"').fetchall() + all_cols = [r[0] for r in desc] + existing_col_names_lower = {c.lower() for c in all_cols} + + # Add row_hash column if missing (idempotent) + if "row_hash" not in existing_col_names_lower: + conn.execute(f'ALTER TABLE "{table}" ADD COLUMN row_hash VARCHAR') + all_cols.append("row_hash") + existing_col_names_lower.add("row_hash") + + # Data columns = all columns minus excluded + data_cols = [c for c in all_cols if c.lower() not in exclude_set] + + if not data_cols: + return { + "status": "error", + "error": "No data columns remaining after exclusion.", + "rows_before": 0, + "rows_after": 0, + "dedup_removed": 0, + "duration_ms": int((time.monotonic() - t0) * 1000), + "hash_column": "row_hash", + } + + # Build md5 expression: md5(col1 || '\t' || col2 || ...) + # Each col: COALESCE(CAST("colname" AS VARCHAR), '') + parts = " || '\t' || ".join( + f"COALESCE(CAST(\"{c}\" AS VARCHAR), '')" for c in data_cols + ) + hash_expr = f"md5({parts})" + + # Update row_hash where NULL + conn.execute( + f'UPDATE "{table}" SET row_hash = {hash_expr} WHERE row_hash IS NULL' + ) + + # Count rows before dedup + rows_before = conn.execute(f'SELECT count(*) FROM "{table}"').fetchone()[0] + + # Delete duplicates, keeping row with smallest rowid (earliest insert) + conn.execute( + f""" + DELETE FROM "{table}" + WHERE rowid NOT IN ( + SELECT min(rowid) FROM "{table}" GROUP BY row_hash + ) + """ + ) + + # Count rows after dedup + rows_after = conn.execute(f'SELECT count(*) FROM "{table}"').fetchone()[0] + + finally: + conn.close() + + duration_ms = int((time.monotonic() - t0) * 1000) + dedup_removed = rows_before - rows_after + + return { + "status": "ok", + "rows_before": rows_before, + "rows_after": rows_after, + "dedup_removed": dedup_removed, + "duration_ms": duration_ms, + "hash_column": "row_hash", + } + + +if __name__ == "__main__": + import argparse + import json + + parser = argparse.ArgumentParser(description="Dedup a DuckDB table by row hash.") + parser.add_argument("duckdb_path", help="Path to DuckDB file") + parser.add_argument("table", help="Table name to deduplicate") + parser.add_argument( + "--exclude-cols", + nargs="*", + default=None, + help="Columns to exclude from hash (default: run_id extracted_at row_hash)", + ) + args = parser.parse_args() + + result = dedup_duckdb_table_by_hash(args.duckdb_path, args.table, args.exclude_cols) + print(json.dumps(result, indent=2)) diff --git a/python/functions/pipelines/dedup_duckdb_table_by_hash_test.py b/python/functions/pipelines/dedup_duckdb_table_by_hash_test.py new file mode 100644 index 00000000..5629720f --- /dev/null +++ b/python/functions/pipelines/dedup_duckdb_table_by_hash_test.py @@ -0,0 +1,95 @@ +"""Tests para dedup_duckdb_table_by_hash.""" + +from __future__ import annotations + +import os +import tempfile + +import duckdb +import pytest + +from pipelines.dedup_duckdb_table_by_hash import dedup_duckdb_table_by_hash + + +def _make_test_db(path: str) -> None: + """Create a test DuckDB with 5 rows: 3 unique data, 2 duplicates.""" + conn = duckdb.connect(path) + conn.execute( + """ + CREATE TABLE stories ( + run_id VARCHAR, + extracted_at TIMESTAMP, + rank INTEGER, + title VARCHAR, + url VARCHAR, + points INTEGER + ) + """ + ) + conn.execute( + """ + INSERT INTO stories VALUES + ('run-001', '2026-05-16 10:00:00', 1, 'Story A', 'https://a.com', 100), + ('run-001', '2026-05-16 10:00:00', 2, 'Story B', 'https://b.com', 200), + ('run-001', '2026-05-16 10:00:00', 3, 'Story C', 'https://c.com', 300), + ('run-002', '2026-05-16 10:30:00', 1, 'Story A', 'https://a.com', 100), + ('run-002', '2026-05-16 10:30:00', 2, 'Story B', 'https://b.com', 200) + """ + ) + conn.close() + + +def test_dedup_elimina_filas_duplicadas_y_conserva_unicas(): + """dedup elimina filas duplicadas y conserva unicas""" + with tempfile.TemporaryDirectory() as tmpdir: + db_path = os.path.join(tmpdir, "test.duckdb") + _make_test_db(db_path) + + result = dedup_duckdb_table_by_hash(db_path, "stories") + + assert result["status"] == "ok", f"Expected ok, got: {result}" + assert result["rows_before"] == 5 + assert result["rows_after"] == 3, f"Expected 3 unique rows, got {result['rows_after']}" + assert result["dedup_removed"] == 2 + assert result["hash_column"] == "row_hash" + assert result["duration_ms"] >= 0 + + # Verify row_hash column exists and is populated + conn = duckdb.connect(db_path) + hashes = conn.execute("SELECT DISTINCT row_hash FROM stories").fetchall() + conn.close() + assert len(hashes) == 3, f"Expected 3 distinct hashes, got {len(hashes)}" + # All hashes should be non-null + assert all(h[0] is not None for h in hashes), "Some row_hash values are NULL" + + +def test_dedup_idempotente(): + """Running dedup twice leaves rows_after unchanged.""" + with tempfile.TemporaryDirectory() as tmpdir: + db_path = os.path.join(tmpdir, "test.duckdb") + _make_test_db(db_path) + + r1 = dedup_duckdb_table_by_hash(db_path, "stories") + r2 = dedup_duckdb_table_by_hash(db_path, "stories") + + assert r1["status"] == "ok" + assert r2["status"] == "ok" + assert r2["rows_before"] == 3 + assert r2["rows_after"] == 3 + assert r2["dedup_removed"] == 0 + + +def test_dedup_tabla_inexistente(): + """Returns status=error when table does not exist.""" + with tempfile.TemporaryDirectory() as tmpdir: + db_path = os.path.join(tmpdir, "empty.duckdb") + conn = duckdb.connect(db_path) + conn.close() + + result = dedup_duckdb_table_by_hash(db_path, "nonexistent_table") + assert result["status"] == "error" + assert "nonexistent_table" in result["error"] + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/python/functions/pipelines/regenerate_app_icons.md b/python/functions/pipelines/regenerate_app_icons.md new file mode 100644 index 00000000..1e49c111 --- /dev/null +++ b/python/functions/pipelines/regenerate_app_icons.md @@ -0,0 +1,66 @@ +--- +name: regenerate_app_icons +kind: pipeline +lang: py +domain: pipelines +version: "1.0.0" +purity: impure +signature: "def regenerate_app_icons(only: list[str] | None = None) -> dict" +description: "Escanea todas las apps C++ del registry, lee el bloque `icon: {phosphor, accent}` de cada app.md y regenera el appicon.ico via generate_app_icon. Reemplaza el script ad-hoc dev/gen_app_icons.py." +tags: [cpp-windows, icon, phosphor, batch] +uses_functions: [generate_app_icon_py_infra] +uses_types: [] +returns: [] +returns_optional: false +error_type: "error_go_core" +imports: [os, sys, pathlib, typing, yaml] +params: + - name: only + desc: "Lista opcional de nombres de app (campo `name` del frontmatter) a procesar. Si None, regenera todas las apps C++ con icon: declarado." +output: "dict {ok: [name], skipped: [{name, reason}], failed: [{name, error}]}" +tested: false +tests: [] +test_file_path: "" +file_path: "python/functions/pipelines/regenerate_app_icons.py" +--- + +## Ejemplo + +```bash +# Regenerar todas las apps C++ con icon: declarado +./fn run regenerate_app_icons + +# Solo una app +./fn run regenerate_app_icons chart_demo + +# Varias apps +./fn run regenerate_app_icons chart_demo registry_dashboard +``` + +```python +import sys +sys.path.insert(0, "python/functions") +from pipelines.regenerate_app_icons import regenerate_app_icons + +result = regenerate_app_icons() +print(f"OK: {len(result['ok'])}, FAIL: {len(result['failed'])}") +``` + +Bloque `icon:` esperado en `app.md`: +```yaml +icon: + phosphor: "chart-bar" + accent: "#0ea5e9" +``` + +## Cuando usarla + +Cuando anades una app C++ nueva (anades `icon:` a su `app.md` y corres el pipeline), cambias el color/glyph de una app existente, o pulleas cambios de iconos desde otra rama. Antes de `redeploy_cpp_app_windows` para que el `.exe` lleve el icono actualizado. + +## Gotchas + +- **Sobreescribe `appicon.ico` sin warning** — igual que `generate_app_icon`. Hacer backup si necesitas preservar version anterior. +- **Requiere `sources/phosphor-core/`**: clonar con `git clone --depth=1 https://github.com/phosphor-icons/core.git sources/phosphor-core` si no existe. +- **Solo procesa apps con `lang: cpp`** en frontmatter — apps Go/Python se ignoran aunque tengan `icon:`. +- **Apps sin `icon:` se reportan en `skipped`**, no son error. Util para detectar apps C++ a las que falta declarar el icono. +- **No invalida el cache de iconos de Windows** — si Explorer no muestra el icono nuevo tras redeploy: `ie4uinit.exe -show` o reiniciar Explorer. diff --git a/python/functions/pipelines/regenerate_app_icons.py b/python/functions/pipelines/regenerate_app_icons.py new file mode 100644 index 00000000..bd75b4e8 --- /dev/null +++ b/python/functions/pipelines/regenerate_app_icons.py @@ -0,0 +1,97 @@ +"""Regenera el appicon.ico de todas las apps C++ que declaren bloque icon: en su app.md.""" + +import os +import sys +from pathlib import Path +from typing import Optional + +import yaml + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) + +from infra.generate_app_icon import generate_app_icon + + +def _find_registry_root() -> Path: + env_root = os.environ.get("FN_REGISTRY_ROOT") + if env_root: + return Path(env_root).resolve() + current = Path(__file__).resolve() + for parent in current.parents: + if (parent / "registry.db").exists(): + return parent + raise FileNotFoundError("registry.db no encontrado; define FN_REGISTRY_ROOT") + + +def _read_frontmatter(md_path: Path) -> Optional[dict]: + text = md_path.read_text(encoding="utf-8") + if not text.startswith("---"): + return None + end = text.find("\n---", 3) + if end < 0: + return None + try: + return yaml.safe_load(text[3:end]) + except yaml.YAMLError: + return None + + +def _iter_cpp_app_mds(root: Path): + for pattern in ("apps/*/app.md", "projects/*/apps/*/app.md"): + for md in sorted(root.glob(pattern)): + fm = _read_frontmatter(md) + if not fm or fm.get("lang") != "cpp": + continue + yield md, fm + + +def regenerate_app_icons(only: Optional[list[str]] = None) -> dict: + """Recorre apps C++ con bloque icon: en su frontmatter y regenera appicon.ico. + + Args: + only: Lista opcional de nombres de app a filtrar (campo `name`). Si None, + procesa todas las apps C++ con `icon:` declarado. + + Returns: + dict con keys: ok (list[str]), skipped (list[dict]), failed (list[dict]). + """ + root = _find_registry_root() + ok, skipped, failed = [], [], [] + + for md, fm in _iter_cpp_app_mds(root): + name = fm.get("name", md.parent.name) + if only and name not in only: + continue + icon = fm.get("icon") + if not icon or not isinstance(icon, dict): + skipped.append({"name": name, "reason": "no icon: block"}) + continue + phosphor = icon.get("phosphor") + accent = icon.get("accent") + if not phosphor or not accent: + skipped.append({"name": name, "reason": "icon: missing phosphor/accent"}) + continue + out_ico = md.parent / "appicon.ico" + try: + generate_app_icon( + phosphor_icon_name=phosphor, + accent_hex=accent, + out_ico_path=str(out_ico), + ) + ok.append(name) + except Exception as e: + failed.append({"name": name, "error": str(e)}) + + return {"ok": ok, "skipped": skipped, "failed": failed} + + +if __name__ == "__main__": + only = sys.argv[1:] or None + result = regenerate_app_icons(only=only) + for name in result["ok"]: + print(f"OK {name}") + for s in result["skipped"]: + print(f"SKIP {s['name']}: {s['reason']}") + for f in result["failed"]: + print(f"FAIL {f['name']}: {f['error']}") + sys.exit(1 if result["failed"] else 0) diff --git a/registry/hash.go b/registry/hash.go index 0d2afea8..8a451b52 100644 --- a/registry/hash.go +++ b/registry/hash.go @@ -61,6 +61,7 @@ func ComputeAppHash(a *App) string { fmt.Fprintf(h, "|%s", marshalStrings(a.Tags)) fmt.Fprintf(h, "|%s", marshalStrings(a.UsesFunctions)) fmt.Fprintf(h, "|%s", marshalStrings(a.UsesTypes)) + fmt.Fprintf(h, "|%s", marshalStrings(a.UsesModules)) fmt.Fprintf(h, "|%s|%s|%s|%s|%s|%s", a.Framework, a.EntryPoint, a.Documentation, a.Notes, a.DirPath, a.RepoURL) return fmt.Sprintf("%x", h.Sum(nil)) } @@ -73,10 +74,22 @@ func ComputeAnalysisHash(a *Analysis) string { fmt.Fprintf(h, "|%s", marshalStrings(a.Tags)) fmt.Fprintf(h, "|%s", marshalStrings(a.UsesFunctions)) fmt.Fprintf(h, "|%s", marshalStrings(a.UsesTypes)) + fmt.Fprintf(h, "|%s", marshalStrings(a.UsesModules)) fmt.Fprintf(h, "|%s|%s|%s|%s|%s|%s", a.Framework, a.EntryPoint, a.Documentation, a.Notes, a.DirPath, a.RepoURL) return fmt.Sprintf("%x", h.Sum(nil)) } +// ComputeModuleHash computes a deterministic hash of all content fields of a Module. +func ComputeModuleHash(m *Module) string { + h := sha256.New() + fmt.Fprintf(h, "%s|%s|%s|%s|%s", + m.ID, m.Name, m.Version, m.Lang, m.Description) + fmt.Fprintf(h, "|%s", marshalStrings(m.Members)) + fmt.Fprintf(h, "|%s", marshalStrings(m.Tags)) + fmt.Fprintf(h, "|%s|%s|%s|%s", m.DirPath, m.RepoURL, m.Documentation, m.Notes) + return fmt.Sprintf("%x", h.Sum(nil)) +} + // ComputeProjectHash computes a deterministic hash of all content fields of a Project. func ComputeProjectHash(p *Project) string { h := sha256.New() @@ -98,7 +111,7 @@ func ComputeVaultHash(v *Vault) string { // LoadTimestamps reads existing id → {created_at, updated_at, content_hash} from all tables. // Called before Purge so we can preserve dates across reindexing. -func (db *DB) LoadTimestamps() (funcs, types, apps, analysis, projects, vaults map[string]timestampRecord, err error) { +func (db *DB) LoadTimestamps() (funcs, types, apps, analysis, projects, vaults, modules map[string]timestampRecord, err error) { funcs, err = loadTable(db, "functions") if err != nil { return @@ -120,6 +133,10 @@ func (db *DB) LoadTimestamps() (funcs, types, apps, analysis, projects, vaults m return } vaults, err = loadTable(db, "vaults") + if err != nil { + return + } + modules, err = loadTable(db, "modules") return } diff --git a/registry/indexer.go b/registry/indexer.go index 2168f42f..b39822ff 100644 --- a/registry/indexer.go +++ b/registry/indexer.go @@ -16,6 +16,7 @@ type IndexResult struct { Analysis int Projects int Vaults int + Modules int UnitTests int ValidationErrors []string Warnings []string @@ -31,7 +32,7 @@ type IndexResult struct { // directories (e.g. python/functions/, python/types/). func Index(db *DB, root string) (*IndexResult, error) { // Load existing timestamps before purging so we can preserve created_at - oldFuncs, oldTypes, oldApps, oldAnalysis, oldProjects, oldVaults, err := db.LoadTimestamps() + oldFuncs, oldTypes, oldApps, oldAnalysis, oldProjects, oldVaults, oldModules, err := db.LoadTimestamps() if err != nil { return nil, fmt.Errorf("loading timestamps: %w", err) } @@ -62,6 +63,20 @@ func Index(db *DB, root string) (*IndexResult, error) { } } + // Discover module directories (modules//) — each may contain function .md + // files alongside the module.md. Module entrypoint .md files (e.g. data_table.md) + // live in their module dir; types still live in types/ to keep cross-module reuse. + modRoot := filepath.Join(root, "modules") + if fi, err := os.Stat(modRoot); err == nil && fi.IsDir() { + modEntries, _ := os.ReadDir(modRoot) + for _, me := range modEntries { + if !me.IsDir() { + continue + } + funcDirs = append(funcDirs, filepath.Join(modRoot, me.Name())) + } + } + for _, dir := range funcDirs { walkMD(dir, func(path string) { f, err := ParseFunctionMD(path, root) @@ -146,6 +161,31 @@ func Index(db *DB, root string) (*IndexResult, error) { } } + // Parse modules from modules/*/module.md + var modules []*Module + modulesDir := filepath.Join(root, "modules") + if fi, err := os.Stat(modulesDir); err == nil && fi.IsDir() { + modEntries, _ := os.ReadDir(modulesDir) + for _, me := range modEntries { + if !me.IsDir() { + continue + } + modMD := filepath.Join(modulesDir, me.Name(), "module.md") + if _, err := os.Stat(modMD); err != nil { + continue + } + m, err := ParseModuleMD(modMD, root) + if err != nil { + result.Errors = append(result.Errors, fmt.Sprintf("parse %s: %v", modMD, err)) + continue + } + if m.DirPath == "" { + m.DirPath = filepath.Join("modules", me.Name()) + } + modules = append(modules, m) + } + } + // Parse projects from projects/*/project.md var projects []*Project var vaults []*Vault @@ -347,6 +387,19 @@ func Index(db *DB, root string) (*IndexResult, error) { result.Vaults++ } + for _, m := range modules { + m.ContentHash = ComputeModuleHash(m) + applyTimestamps(&m.CreatedAt, &m.UpdatedAt, m.ContentHash, oldModules[m.ID], now) + if err := db.InsertModule(m); err != nil { + result.Errors = append(result.Errors, fmt.Sprintf("insert module %s: %v", m.ID, err)) + continue + } + if err := emitModuleVersionHeader(m, root); err != nil { + result.Warnings = append(result.Warnings, fmt.Sprintf("module %s: codegen version header: %v", m.ID, err)) + } + result.Modules++ + } + // Extract unit tests from test files of tested functions if err := db.PurgeUnitTests(); err != nil { result.Warnings = append(result.Warnings, fmt.Sprintf("purging unit_tests: %v", err)) @@ -437,7 +490,8 @@ func applyTimestamps(createdAt, updatedAt *time.Time, newHash string, old timest } } -// walkMD walks a directory recursively and calls fn for each .md file found. +// walkMD walks a directory recursively and calls fn for each .md file found, +// skipping module.md (which is parsed separately as a Module entry). func walkMD(dir string, fn func(path string)) { if _, err := os.Stat(dir); err != nil { return @@ -446,6 +500,9 @@ func walkMD(dir string, fn func(path string)) { if err != nil || info.IsDir() || !strings.HasSuffix(path, ".md") { return nil } + if filepath.Base(path) == "module.md" { + return nil + } fn(path) return nil }) diff --git a/registry/migrations/013_modules.sql b/registry/migrations/013_modules.sql new file mode 100644 index 00000000..c24489fe --- /dev/null +++ b/registry/migrations/013_modules.sql @@ -0,0 +1,57 @@ +-- Modules: reusable cohesive units (e.g. data_table) versioned with semver. +-- A module groups a set of related registry functions/types under a single +-- versioned artefact that apps opt into via uses_modules in app.md. +-- +-- Modules son datos vivos: fn sync los replica entre PCs igual que apps/proposals. +-- Aunque la fuente es modules/*/module.md (parseable), conservamos created_at / +-- updated_at de forma persistente para mantener historico cross-PC. + +CREATE TABLE IF NOT EXISTS modules ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL, + version TEXT NOT NULL DEFAULT '0.0.0', + lang TEXT NOT NULL, + description TEXT NOT NULL DEFAULT '', + members TEXT NOT NULL DEFAULT '[]', + tags TEXT NOT NULL DEFAULT '[]', + dir_path TEXT NOT NULL DEFAULT '', + repo_url TEXT NOT NULL DEFAULT '', + documentation TEXT NOT NULL DEFAULT '', + notes TEXT NOT NULL DEFAULT '', + content_hash TEXT NOT NULL DEFAULT '', + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL +); + +CREATE VIRTUAL TABLE IF NOT EXISTS modules_fts USING fts5( + id, + name, + description, + tags, + members, + documentation, + notes, + content='modules', + content_rowid='rowid' +); + +CREATE TRIGGER IF NOT EXISTS modules_ai AFTER INSERT ON modules BEGIN + INSERT INTO modules_fts(rowid, id, name, description, tags, members, documentation, notes) + VALUES (new.rowid, new.id, new.name, new.description, new.tags, new.members, new.documentation, new.notes); +END; + +CREATE TRIGGER IF NOT EXISTS modules_ad AFTER DELETE ON modules BEGIN + INSERT INTO modules_fts(modules_fts, rowid, id, name, description, tags, members, documentation, notes) + VALUES ('delete', old.rowid, old.id, old.name, old.description, old.tags, old.members, old.documentation, old.notes); +END; + +CREATE TRIGGER IF NOT EXISTS modules_au AFTER UPDATE ON modules BEGIN + INSERT INTO modules_fts(modules_fts, rowid, id, name, description, tags, members, documentation, notes) + VALUES ('delete', old.rowid, old.id, old.name, old.description, old.tags, old.members, old.documentation, old.notes); + INSERT INTO modules_fts(rowid, id, name, description, tags, members, documentation, notes) + VALUES (new.rowid, new.id, new.name, new.description, new.tags, new.members, new.documentation, new.notes); +END; + +-- uses_modules en apps/analysis: lista declarativa de modulos consumidos. +ALTER TABLE apps ADD COLUMN uses_modules TEXT NOT NULL DEFAULT '[]'; +ALTER TABLE analysis ADD COLUMN uses_modules TEXT NOT NULL DEFAULT '[]'; diff --git a/registry/models.go b/registry/models.go index 3c38f9b0..6a91e568 100644 --- a/registry/models.go +++ b/registry/models.go @@ -113,6 +113,7 @@ type App struct { Tags []string `json:"tags"` UsesFunctions []string `json:"uses_functions"` UsesTypes []string `json:"uses_types"` + UsesModules []string `json:"uses_modules"` Framework string `json:"framework"` EntryPoint string `json:"entry_point"` Documentation string `json:"documentation"` @@ -135,6 +136,7 @@ type Analysis struct { Tags []string `json:"tags"` UsesFunctions []string `json:"uses_functions"` UsesTypes []string `json:"uses_types"` + UsesModules []string `json:"uses_modules"` Framework string `json:"framework"` EntryPoint string `json:"entry_point"` Documentation string `json:"documentation"` @@ -147,6 +149,27 @@ type Analysis struct { UpdatedAt time.Time `json:"updated_at"` } +// Module represents an entry in the modules table. +// A module groups related registry functions/types under a single versioned +// artefact that apps opt into via uses_modules in app.md. Living data: kept +// in sync across PCs via fn sync. +type Module struct { + ID string `json:"id"` + Name string `json:"name"` + Version string `json:"version"` + Lang string `json:"lang"` + Description string `json:"description"` + Members []string `json:"members"` + Tags []string `json:"tags"` + DirPath string `json:"dir_path"` + RepoURL string `json:"repo_url"` + Documentation string `json:"documentation"` + Notes string `json:"notes"` + ContentHash string `json:"content_hash"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} + // ProposalKind classifies a proposal. type ProposalKind string @@ -241,3 +264,9 @@ type PcLocation struct { func GenerateID(name, lang, domain string) string { return name + "_" + lang + "_" + domain } + +// GenerateModuleID builds the module canonical ID: {name}_{lang}. +// Modules are language-scoped but domain-agnostic; they live at modules//. +func GenerateModuleID(name, lang string) string { + return name + "_" + lang +} diff --git a/registry/module_codegen.go b/registry/module_codegen.go new file mode 100644 index 00000000..9ac5efab --- /dev/null +++ b/registry/module_codegen.go @@ -0,0 +1,60 @@ +package registry + +import ( + "fmt" + "os" + "path/filepath" + "strings" +) + +// emitModuleVersionHeader writes /version_generated.h for a module. +// The header exposes constants the C++ side can read: +// +// FN_MODULE__VERSION (const char*) +// FN_MODULE__NAME (const char*) +// FN_MODULE__DESCRIPTION (const char*) +// +// For C++ modules only (lang == "cpp"). For other langs, returns nil silently. +// Idempotent: only rewrites when content differs. +func emitModuleVersionHeader(m *Module, root string) error { + if m.Lang != "cpp" { + return nil + } + if m.DirPath == "" { + return nil + } + + upper := strings.ToUpper(m.Name) + macroPrefix := "FN_MODULE_" + upper + guard := macroPrefix + "_VERSION_GENERATED_H" + + body := fmt.Sprintf(`// Auto-generated by `+"`fn index`"+` — do not edit. +// Module: %s +// Source of truth: modules/%s/module.md +#ifndef %s +#define %s + +#define %s_NAME %q +#define %s_VERSION %q +#define %s_DESCRIPTION %q + +#endif // %s +`, m.Name, m.Name, + guard, guard, + macroPrefix, m.Name, + macroPrefix, m.Version, + macroPrefix, m.Description, + guard) + + headerPath := filepath.Join(root, m.DirPath, "version_generated.h") + + // Idempotent: skip write when content already matches. + if existing, err := os.ReadFile(headerPath); err == nil && string(existing) == body { + return nil + } + + if err := os.MkdirAll(filepath.Dir(headerPath), 0o755); err != nil { + return fmt.Errorf("mkdir %s: %w", filepath.Dir(headerPath), err) + } + return os.WriteFile(headerPath, []byte(body), 0o644) +} diff --git a/registry/parser.go b/registry/parser.go index 5e7afb57..d352f898 100644 --- a/registry/parser.go +++ b/registry/parser.go @@ -82,6 +82,7 @@ type rawApp struct { Tags []string `yaml:"tags"` UsesFunctions []string `yaml:"uses_functions"` UsesTypes []string `yaml:"uses_types"` + UsesModules []string `yaml:"uses_modules"` Framework string `yaml:"framework"` EntryPoint string `yaml:"entry_point"` DirPath string `yaml:"dir_path"` @@ -97,12 +98,25 @@ type rawAnalysis struct { Tags []string `yaml:"tags"` UsesFunctions []string `yaml:"uses_functions"` UsesTypes []string `yaml:"uses_types"` + UsesModules []string `yaml:"uses_modules"` Framework string `yaml:"framework"` EntryPoint string `yaml:"entry_point"` DirPath string `yaml:"dir_path"` RepoURL string `yaml:"repo_url"` } +// rawModule mirrors the YAML frontmatter of a module.md file. +type rawModule struct { + Name string `yaml:"name"` + Version string `yaml:"version"` + Lang string `yaml:"lang"` + Description string `yaml:"description"` + Members []string `yaml:"members"` + Tags []string `yaml:"tags"` + DirPath string `yaml:"dir_path"` + RepoURL string `yaml:"repo_url"` +} + // rawProject mirrors the YAML frontmatter of a project .md file. type rawProject struct { Name string `yaml:"name"` @@ -320,6 +334,7 @@ func ParseAppMD(path string, root string) (*App, error) { Tags: raw.Tags, UsesFunctions: raw.UsesFunctions, UsesTypes: raw.UsesTypes, + UsesModules: raw.UsesModules, Framework: raw.Framework, EntryPoint: raw.EntryPoint, Documentation: sections.documentation, @@ -366,6 +381,7 @@ func ParseAnalysisMD(path string, root string) (*Analysis, error) { Tags: raw.Tags, UsesFunctions: raw.UsesFunctions, UsesTypes: raw.UsesTypes, + UsesModules: raw.UsesModules, Framework: raw.Framework, EntryPoint: raw.EntryPoint, Documentation: sections.documentation, @@ -377,6 +393,55 @@ func ParseAnalysisMD(path string, root string) (*Analysis, error) { return an, nil } +// ParseModuleMD parses a module .md file into a Module. +func ParseModuleMD(path string, root string) (*Module, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("reading %s: %w", path, err) + } + + fm, body, err := extractFrontmatter(data) + if err != nil { + return nil, fmt.Errorf("parsing %s: %w", path, err) + } + + var raw rawModule + if err := yaml.Unmarshal(fm, &raw); err != nil { + return nil, fmt.Errorf("parsing YAML in %s: %w", path, err) + } + + if raw.Name == "" { + return nil, fmt.Errorf("%s: name is required", path) + } + if raw.Lang == "" { + return nil, fmt.Errorf("%s: lang is required", path) + } + if raw.Description == "" { + return nil, fmt.Errorf("%s: description is required", path) + } + if raw.Version == "" { + raw.Version = "0.0.0" + } + + sections := extractSections(body) + + m := &Module{ + ID: GenerateModuleID(raw.Name, raw.Lang), + Name: raw.Name, + Version: raw.Version, + Lang: raw.Lang, + Description: raw.Description, + Members: raw.Members, + Tags: raw.Tags, + DirPath: raw.DirPath, + RepoURL: raw.RepoURL, + Documentation: sections.documentation, + Notes: sections.notes, + } + + return m, nil +} + // ParseProjectMD parses a project .md file into a Project. func ParseProjectMD(path string, root string) (*Project, error) { data, err := os.ReadFile(path) diff --git a/registry/store.go b/registry/store.go index abf60298..a07e6b8b 100644 --- a/registry/store.go +++ b/registry/store.go @@ -307,12 +307,12 @@ func (db *DB) InsertApp(a *App) error { INSERT OR REPLACE INTO apps ( id, name, lang, domain, description, tags, uses_functions, uses_types, framework, entry_point, - documentation, notes, dir_path, content_hash, created_at, updated_at, repo_url, project_id - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + documentation, notes, dir_path, content_hash, created_at, updated_at, repo_url, project_id, uses_modules + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, a.ID, a.Name, a.Lang, a.Domain, a.Description, marshalStrings(a.Tags), marshalStrings(a.UsesFunctions), marshalStrings(a.UsesTypes), a.Framework, a.EntryPoint, a.Documentation, a.Notes, a.DirPath, a.ContentHash, a.CreatedAt.Format(time.RFC3339), a.UpdatedAt.Format(time.RFC3339), - a.RepoURL, a.ProjectID, + a.RepoURL, a.ProjectID, marshalStrings(a.UsesModules), ) return err } @@ -372,14 +372,14 @@ func scanApps(rows interface{ Next() bool; Scan(...any) error }) ([]App, error) var result []App for rows.Next() { var a App - var tagsJSON, usesFnJSON, usesTypJSON string + var tagsJSON, usesFnJSON, usesTypJSON, usesModJSON string var createdAt, updatedAt string err := rows.Scan( &a.ID, &a.Name, &a.Lang, &a.Domain, &a.Description, &tagsJSON, &usesFnJSON, &usesTypJSON, &a.Framework, &a.EntryPoint, &a.Documentation, &a.Notes, &a.DirPath, &createdAt, &updatedAt, &a.ContentHash, - &a.RepoURL, &a.ProjectID, + &a.RepoURL, &a.ProjectID, &usesModJSON, ) if err != nil { return nil, fmt.Errorf("scanning app: %w", err) @@ -388,6 +388,7 @@ func scanApps(rows interface{ Next() bool; Scan(...any) error }) ([]App, error) a.Tags = unmarshalStrings(tagsJSON) a.UsesFunctions = unmarshalStrings(usesFnJSON) a.UsesTypes = unmarshalStrings(usesTypJSON) + a.UsesModules = unmarshalStrings(usesModJSON) a.CreatedAt, _ = time.Parse(time.RFC3339, createdAt) a.UpdatedAt, _ = time.Parse(time.RFC3339, updatedAt) @@ -396,7 +397,7 @@ func scanApps(rows interface{ Next() bool; Scan(...any) error }) ([]App, error) return result, nil } -// Purge deletes all data from functions, types, apps, analysis, projects and vaults. Used before re-indexing. +// Purge deletes all data from functions, types, apps, analysis, projects, vaults and modules. Used before re-indexing. func (db *DB) Purge() error { if _, err := db.conn.Exec("DELETE FROM functions"); err != nil { return err @@ -413,7 +414,10 @@ func (db *DB) Purge() error { if _, err := db.conn.Exec("DELETE FROM projects"); err != nil { return err } - _, err := db.conn.Exec("DELETE FROM vaults") + if _, err := db.conn.Exec("DELETE FROM vaults"); err != nil { + return err + } + _, err := db.conn.Exec("DELETE FROM modules") return err } @@ -458,6 +462,10 @@ func (db *DB) PurgeLocalOnly(localAppIDs, localAnalysisIDs, localProjectIDs map[ if _, err := db.conn.Exec("DELETE FROM vaults"); err != nil { return err } + // Modules: always purge and re-insert from modules/*/module.md + if _, err := db.conn.Exec("DELETE FROM modules"); err != nil { + return err + } return nil } @@ -481,12 +489,12 @@ func (db *DB) InsertAnalysis(a *Analysis) error { INSERT OR REPLACE INTO analysis ( id, name, lang, domain, description, tags, uses_functions, uses_types, framework, entry_point, - documentation, notes, repo_url, dir_path, content_hash, created_at, updated_at, project_id - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + documentation, notes, repo_url, dir_path, content_hash, created_at, updated_at, project_id, uses_modules + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, a.ID, a.Name, a.Lang, a.Domain, a.Description, marshalStrings(a.Tags), marshalStrings(a.UsesFunctions), marshalStrings(a.UsesTypes), a.Framework, a.EntryPoint, a.Documentation, a.Notes, a.RepoURL, a.DirPath, a.ContentHash, - a.CreatedAt.Format(time.RFC3339), a.UpdatedAt.Format(time.RFC3339), a.ProjectID, + a.CreatedAt.Format(time.RFC3339), a.UpdatedAt.Format(time.RFC3339), a.ProjectID, marshalStrings(a.UsesModules), ) return err } @@ -556,14 +564,14 @@ func scanAnalysis(rows interface{ Next() bool; Scan(...any) error }) ([]Analysis var result []Analysis for rows.Next() { var a Analysis - var tagsJSON, usesFnJSON, usesTypJSON string + var tagsJSON, usesFnJSON, usesTypJSON, usesModJSON string var createdAt, updatedAt string err := rows.Scan( &a.ID, &a.Name, &a.Lang, &a.Domain, &a.Description, &tagsJSON, &usesFnJSON, &usesTypJSON, &a.Framework, &a.EntryPoint, &a.Documentation, &a.Notes, &a.RepoURL, &a.DirPath, &a.ContentHash, - &createdAt, &updatedAt, &a.ProjectID, + &createdAt, &updatedAt, &a.ProjectID, &usesModJSON, ) if err != nil { return nil, fmt.Errorf("scanning analysis: %w", err) @@ -572,6 +580,7 @@ func scanAnalysis(rows interface{ Next() bool; Scan(...any) error }) ([]Analysis a.Tags = unmarshalStrings(tagsJSON) a.UsesFunctions = unmarshalStrings(usesFnJSON) a.UsesTypes = unmarshalStrings(usesTypJSON) + a.UsesModules = unmarshalStrings(usesModJSON) a.CreatedAt, _ = time.Parse(time.RFC3339, createdAt) a.UpdatedAt, _ = time.Parse(time.RFC3339, updatedAt) @@ -1223,3 +1232,115 @@ func (db *DB) AllProposals() ([]Proposal, error) { func (db *DB) AllVaults() ([]Vault, error) { return db.SearchVaults("", "") } + +// --- Module CRUD --- + +// InsertModule inserts or replaces a module entry. +func (db *DB) InsertModule(m *Module) error { + now := time.Now().UTC() + if m.CreatedAt.IsZero() { + m.CreatedAt = now + } + if m.UpdatedAt.IsZero() { + m.UpdatedAt = now + } + if m.ID == "" { + m.ID = GenerateModuleID(m.Name, m.Lang) + } + + _, err := db.conn.Exec(` + INSERT OR REPLACE INTO modules ( + id, name, version, lang, description, members, tags, + dir_path, repo_url, documentation, notes, content_hash, created_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + m.ID, m.Name, m.Version, m.Lang, m.Description, + marshalStrings(m.Members), marshalStrings(m.Tags), + m.DirPath, m.RepoURL, m.Documentation, m.Notes, m.ContentHash, + m.CreatedAt.Format(time.RFC3339), m.UpdatedAt.Format(time.RFC3339), + ) + return err +} + +// GetModule returns a single module by ID. +func (db *DB) GetModule(id string) (*Module, error) { + rows, err := db.conn.Query("SELECT * FROM modules WHERE id = ?", id) + if err != nil { + return nil, err + } + defer rows.Close() + + items, err := scanModules(rows) + if err != nil { + return nil, err + } + if len(items) == 0 { + return nil, fmt.Errorf("module %q not found", id) + } + return &items[0], nil +} + +// SearchModules performs FTS search on modules with optional filters. +func (db *DB) SearchModules(query, lang string) ([]Module, error) { + where := []string{} + args := []any{} + + if query != "" { + where = append(where, "m.id IN (SELECT id FROM modules_fts WHERE modules_fts MATCH ?)") + args = append(args, query) + } + if lang != "" { + where = append(where, "m.lang = ?") + args = append(args, lang) + } + + sql := "SELECT * FROM modules m" + if len(where) > 0 { + sql += " WHERE " + strings.Join(where, " AND ") + } + sql += " ORDER BY m.name" + + rows, err := db.conn.Query(sql, args...) + if err != nil { + return nil, fmt.Errorf("search modules: %w", err) + } + defer rows.Close() + + return scanModules(rows) +} + +// ListAllModules returns all module entries. +func (db *DB) ListAllModules() ([]Module, error) { + return db.SearchModules("", "") +} + +// AllModules returns all modules (for sync export). +func (db *DB) AllModules() ([]Module, error) { + return db.SearchModules("", "") +} + +func scanModules(rows interface{ Next() bool; Scan(...any) error }) ([]Module, error) { + var result []Module + for rows.Next() { + var m Module + var membersJSON, tagsJSON string + var createdAt, updatedAt string + + err := rows.Scan( + &m.ID, &m.Name, &m.Version, &m.Lang, &m.Description, + &membersJSON, &tagsJSON, + &m.DirPath, &m.RepoURL, &m.Documentation, &m.Notes, &m.ContentHash, + &createdAt, &updatedAt, + ) + if err != nil { + return nil, fmt.Errorf("scanning module: %w", err) + } + + m.Members = unmarshalStrings(membersJSON) + m.Tags = unmarshalStrings(tagsJSON) + m.CreatedAt, _ = time.Parse(time.RFC3339, createdAt) + m.UpdatedAt, _ = time.Parse(time.RFC3339, updatedAt) + + result = append(result, m) + } + return result, nil +}