diff --git a/.claude/CLAUDE.md b/.claude/CLAUDE.md index fc5c974b..6c837914 100644 --- a/.claude/CLAUDE.md +++ b/.claude/CLAUDE.md @@ -2,6 +2,46 @@ Registry personal de codigo reutilizable con busqueda FTS. Diseñado para composicion funcional y agentes. +El sistema tiene **dos bases de datos SQLite**: +- **registry.db** — conocimiento estatico: que funciones y tipos existen, como se componen, y que mejoras se proponen. Vive en la raiz del repositorio. Regenerable con `fn index` (excepto proposals). +- **operations.db** — conocimiento dinamico por proyecto: que entities existen, como se relacionan, que ejecuciones se han hecho, y que calidad tienen los datos. Vive en cada proyecto que usa el registry. + +--- + +## Arquitectura: dos BDs, un bucle + +``` +┌─────────────────────────────────────────────────────────────────────┐ +│ registry.db (central) │ +│ ┌──────────┐ ┌──────────┐ ┌──────────┐ │ +│ │ functions │ │ types │ │proposals │ │ +│ │ (120+) │ │ (31+) │ │(auto/man)│ │ +│ └──────────┘ └──────────┘ └──────────┘ │ +│ ↑ busca funciones ↑ crea proposals │ +└───────┼────────────────────────────┼────────────────────────────────┘ + │ │ +┌───────┼────────────────────────────┼────────────────────────────────┐ +│ ↓ usa en relaciones │ reactive loop │ +│ operations.db (por proyecto) │ +│ ┌──────────┐ ┌──────────┐ ┌──────────┐ ┌──────────────────┐ │ +│ │ entities │→ │relations │ │executions│→ │assertions │ │ +│ │ │ │ │ │ │ │ → assertion_results│ │ +│ └──────────┘ └──────────┘ └──────────┘ └──────────────────┘ │ +│ ┌──────────────┐ ┌──────────────┐ │ +│ │relation_inputs│ │types_snapshot│ │ +│ └──────────────┘ └──────────────┘ │ +└─────────────────────────────────────────────────────────────────────┘ +``` + +**Bucle autonomo:** EJECUTAR → EVALUAR → REACCIONAR → PROPONER +1. Un pipeline se ejecuta → se registra en `executions` +2. Las `assertions` activas de la entity se evaluan automaticamente +3. Si critical falla → entity pasa a `corrupted` + se crea `proposal` en registry +4. Si warning falla → entity pasa a `stale` +5. El humano revisa proposals y decide si implementar mejoras + +--- + ## Explorar el registry (USAR SIEMPRE) La BD SQLite `registry.db` es tu mapa del repositorio. Antes de escribir codigo, SIEMPRE consultala para saber que existe, evitar duplicados y descubrir funciones reutilizables. @@ -36,33 +76,324 @@ sqlite3 registry.db "SELECT domain, COUNT(*) FROM functions GROUP BY domain;" # Ver todo el schema sqlite3 registry.db ".schema" + +# Ver proposals pendientes +sqlite3 registry.db "SELECT id, kind, status, title FROM proposals WHERE status = 'pending';" ``` La BD usa WAL mode — puedes leerla mientras se escribe sin bloqueos. +--- + +## Tablas: registry.db + +### functions (27 columnas) + +Cada funcion registrada: su firma, purity, dependencias, y metadata. + +| Columna | Tipo | Descripcion | +|---|---|---| +| id | TEXT PK | `{name}_{lang}_{domain}` | +| name | TEXT | snake_case | +| kind | TEXT | `function` / `pipeline` / `component` | +| lang | TEXT | `go` / `python` / `typescript` / `sql` | +| domain | TEXT | `core` / `finance` / `datascience` / `cybersecurity` / ... | +| version | TEXT | semver | +| purity | TEXT | `pure` / `impure` | +| signature | TEXT | firma completa | +| description | TEXT | que hace y cuando usarla | +| tags | JSON[] | etiquetas | +| uses_functions | JSON[] | IDs de funciones que invoca | +| uses_types | JSON[] | IDs de tipos que recibe | +| returns | JSON[] | IDs de tipos que devuelve (no tipos nativos) | +| returns_optional | INT | 0/1 | +| error_type | TEXT | ID del tipo de error (obligatorio si impure) | +| imports | JSON[] | dependencias externas | +| example | TEXT | codigo de ejemplo extraido del .md | +| tested | INT | 0/1 | +| tests | JSON[] | nombres de tests | +| test_file_path | TEXT | ruta al archivo de test | +| file_path | TEXT | ruta relativa al .go/.py/.tsx | +| props | JSON[] | solo components: PropDef[] | +| emits | JSON[] | solo components: eventos emitidos | +| has_state | INT? | solo components: nullable | +| framework | TEXT | solo components: react/vue/... | +| variant | JSON[] | solo components: variantes | +| created_at, updated_at | TEXT | RFC3339 | + +### types (13 columnas) + +Tipos algebraicos: product (struct) y sum (interface/union). + +| Columna | Tipo | Descripcion | +|---|---|---| +| id | TEXT PK | `{name}_{lang}_{domain}` | +| name | TEXT | snake_case | +| lang | TEXT | lenguaje | +| domain | TEXT | dominio | +| version | TEXT | semver | +| algebraic | TEXT | `product` / `sum` | +| definition | TEXT | codigo fuente del tipo | +| description | TEXT | descripcion | +| tags | JSON[] | etiquetas | +| uses_types | JSON[] | IDs de tipos que compone (sin auto-ref) | +| file_path | TEXT | ruta relativa | +| created_at, updated_at | TEXT | RFC3339 | + +### proposals (11 columnas) + +Mejoras propuestas al registry. Las crea el agente (reactive loop) o el humano. + +| Columna | Tipo | Descripcion | +|---|---|---| +| id | TEXT PK | `proposal_{timestamp}` o manual | +| kind | TEXT | `new_function` / `new_type` / `improve_function` / `improve_type` / `new_pipeline` | +| target_id | TEXT | ID de la funcion/tipo a mejorar (obligatorio para improve_*) | +| title | TEXT | titulo corto | +| description | TEXT | detalle | +| evidence | JSON{} | datos que justifican la propuesta (assertion failures, metrics, etc) | +| status | TEXT | `pending` / `approved` / `rejected` / `implemented` | +| created_by | TEXT | quien creo (agente, humano, reactive_loop) | +| reviewed_by | TEXT | quien reviso | +| created_at, updated_at | TEXT | RFC3339 | + +### FTS5 (busqueda full-text) + +- `functions_fts` — indexa: id, name, description, tags, signature, domain +- `types_fts` — indexa: id, name, description, tags, domain +- `proposals_fts` — indexa: id, title, description, evidence + +Sincronizados automaticamente via triggers (INSERT/UPDATE/DELETE). + +--- + +## Tablas: operations.db + +### entities (12 columnas) + +Instancia concreta de un tipo del registry dentro de un proyecto. + +| Columna | Tipo | Descripcion | +|---|---|---| +| id | TEXT PK | identificador unico en el proyecto | +| name | TEXT | nombre descriptivo | +| type_ref | TEXT | ID del tipo en registry (ej: `ohlcv_go_finance`) | +| status | TEXT | `active` / `stale` / `corrupted` / `archived` | +| description | TEXT | que representa esta entity | +| domain | TEXT | dominio | +| tags | JSON[] | etiquetas | +| source | TEXT | origen de los datos (obligatorio) | +| metadata | JSON{} | campos del tipo instanciados con valores reales | +| notes | TEXT | notas libres | +| created_at, updated_at | TEXT | RFC3339 | + +### relations (17 columnas) + +Conexion/transformacion entre entities. Puede ser causal (via funcion) o semantica. + +| Columna | Tipo | Descripcion | +|---|---|---| +| id | TEXT PK | identificador | +| name | TEXT | nombre descriptivo | +| from_entity | TEXT | entity origen | +| to_entity | TEXT | entity destino (obligatorio) | +| via | TEXT | ID de funcion del registry que transforma (si vacio = semantica) | +| description | TEXT | que hace esta relacion | +| purity | TEXT | `pure` / `impure` / `` | +| direction | TEXT | `unidirectional` / `bidirectional` / `inverse` | +| weight | REAL | 0.0-1.0, importancia | +| status | TEXT | `designed` / `implemented` / `tested` / `running` / `deprecated` | +| started_at, ended_at | TEXT | ciclo de vida | +| order | INT | para secuencias | +| tags | JSON[] | etiquetas | +| notes | TEXT | notas | +| created_at, updated_at | TEXT | RFC3339 | + +### relation_inputs (5 columnas) + +Multi-input para relaciones N→1 (joins, merges, agregaciones). + +| Columna | Tipo | Descripcion | +|---|---|---| +| id | TEXT PK | | +| relation_id | TEXT FK | referencia a relations.id (CASCADE delete) | +| entity_id | TEXT FK | referencia a entities.id | +| role | TEXT | rol semantico del input (ej: "left", "right", "config") | +| order | INT | orden de procesamiento | + +### types_snapshot (7 columnas) + +Copia inmutable de un tipo del registry en el momento de uso. Hace operations.db autonoma. + +| Columna | Tipo | Descripcion | +|---|---|---| +| id | TEXT PK | mismo ID que en registry.types | +| version | TEXT | version capturada | +| lang | TEXT | lenguaje | +| algebraic | TEXT | product/sum | +| definition | TEXT | codigo fuente capturado | +| description | TEXT | descripcion capturada | +| snapped_at | TEXT | cuando se hizo el snapshot | + +### executions (12 columnas) + +Cada ejecucion de un pipeline. Memoria de comportamiento del sistema. + +| Columna | Tipo | Descripcion | +|---|---|---| +| id | TEXT PK | `exec_{timestamp}` o manual | +| pipeline_id | TEXT | ID de funcion del registry (ej: `tick_to_ohlcv_go_finance`) | +| relation_id | TEXT | relacion asociada (opcional) | +| status | TEXT | `success` / `failure` / `partial` | +| started_at | TEXT | inicio (obligatorio) | +| ended_at | TEXT | fin (nullable si en progreso) | +| duration_ms | INT | auto-calculado si started_at y ended_at presentes | +| records_in | INT | registros de entrada (nullable) | +| records_out | INT | registros de salida (nullable) | +| error | TEXT | mensaje de error si fallo | +| metrics | JSON{} | metricas custom (ej: `{"mean_close": 42000}`) | +| created_at | TEXT | RFC3339 | + +### assertions (9 columnas) + +Regla de calidad formal sobre una entity. Evaluable automaticamente contra metadata. + +| Columna | Tipo | Descripcion | +|---|---|---| +| id | TEXT PK | | +| entity_id | TEXT FK | referencia a entities.id | +| name | TEXT | nombre descriptivo | +| kind | TEXT | tipo libre: `range`, `null`, `statistical`, `consistency`, `freshness`, o custom | +| rule | TEXT | expresion SQL evaluable (ver motor de evaluacion abajo) | +| severity | TEXT | `critical` / `warning` / `info` | +| description | TEXT | que verifica | +| active | INT | 0/1 — solo las activas se evaluan | +| created_at | TEXT | RFC3339 | + +### assertion_results (7 columnas) + +Historial de evaluaciones de assertions. + +| Columna | Tipo | Descripcion | +|---|---|---| +| id | TEXT PK | | +| assertion_id | TEXT FK | referencia a assertions.id | +| execution_id | TEXT | referencia a executions.id (vacio si eval manual) | +| status | TEXT | `pass` / `fail` / `skip` | +| value | JSON{} | datos capturados en el momento de evaluacion | +| message | TEXT | detalle del resultado | +| evaluated_at | TEXT | RFC3339 | + +### FTS5 operations + +- `entities_fts` — indexa: id, name, description, tags, domain +- `assertions_fts` — indexa: id, name, description, rule + +--- + +## Motor de evaluacion de assertions + +Las rules se escriben como expresiones SQL. Campos sin prefijo se reescriben automaticamente a `json_extract(metadata, '$.campo')`: + +``` +close > 0 → json_extract(metadata, '$.close') > 0 +low <= close AND close <= high → json_extract(metadata, '$.low') <= ... +open IS NOT NULL → json_extract(metadata, '$.open') IS NOT NULL +``` + +Si la rule ya usa `json_extract`, se deja como esta. Palabras SQL (AND, OR, NOT, IS, NULL, BETWEEN, etc) y funciones SQLite (datetime, abs, max, min, etc) no se reescriben. + +**Kinds documentados** (puedes añadir nuevos sin tocar schema): + +| Kind | Descripcion | Ejemplo de rule | +|---|---|---| +| `range` | Valor dentro de rango | `close BETWEEN 0 AND 1000000` | +| `null` | Campo no nulo | `open IS NOT NULL` | +| `consistency` | Relacion entre campos | `low <= close AND close <= high` | +| `freshness` | Datos recientes | `json_extract(metadata, '$.ts') > datetime('now', '-1 hour')` | +| `statistical` | Desviacion estadistica | (evaluar externamente, registrar manual) | + +**Dos modos:** +- **Auto**: `fn ops assertion eval --entity-id X` ejecuta rules SQL contra `entities.metadata` +- **Manual**: `fn ops assertion result add` registra resultados de assertions que el sistema no puede evaluar + +**Bucle reactivo** (con `--react`): +- `fn ops assertion eval --entity-id X --react` evalua Y reacciona: + - Critical fail → entity.status = `corrupted` + auto-crea proposal en registry.db + - Warning fail → entity.status = `stale` (solo si era `active`) + - Info fail → sin cambio + +--- + +## Sistema de migraciones + +Ambas BDs usan un sistema de migraciones con `embed.FS`: + +``` +registry/migrations/ + 001_init.sql # functions + types + FTS + 002_proposals.sql # proposals + FTS + +fn_operations/migrations/ + 001_init.sql # entities + relations + relation_inputs + types_snapshot + FTS + 002_executions_assertions.sql # executions + assertions + assertion_results + FTS +``` + +- Tabla `schema_migrations` en cada BD rastrea versiones aplicadas +- `CREATE TABLE IF NOT EXISTS` + transacciones por migracion = idempotente +- Al hacer `Open()` se aplican automaticamente las migraciones pendientes +- Para añadir una nueva migracion: crear `NNN_nombre.sql` en la carpeta correspondiente + +--- + ## Estructura del repositorio ``` fn-registry/ - functions/ # Codigo y docs de funciones - core/ # Utilidades genericas (filter, map, pipeline, retry...) - finance/ # Indicadores, riesgo, IO de mercado - datascience/ # Estadistica, DSP, IO de datos - cybersecurity/ # Crypto, analisis de red, IO de seguridad - pipelines/ # Composiciones de funciones, siempre impuras - components/ # Componentes React (.tsx) - types/ # Tipos algebraicos (product y sum) - core/ # Result, Option, Pair, Error - finance/ # OHLCV, Tick, BollingerResult, DrawdownResult - datascience/ # OutlierResult - cybersecurity/ # CIDRBlock, ThreatResult, PortResult - registry/ # Libreria Go: modelos, SQLite, parser, indexer, validacion - cmd/fn/ # CLI - docs/ # Schema de documentacion v1.0 (fuente de verdad del diseño) - docs/templates/ # Plantillas de frontmatter para function, pipeline, component, type - registry.db # Indice SQLite FTS5+WAL (regenerable, NO commitear) + functions/ # Codigo y docs de funciones + core/ # Utilidades genericas (filter, map, pipeline, retry...) + finance/ # Indicadores, riesgo, IO de mercado + datascience/ # Estadistica, DSP, IO de datos + cybersecurity/ # Crypto, analisis de red, IO de seguridad + pipelines/ # Composiciones de funciones, siempre impuras + components/ # Componentes React (.tsx) + types/ # Tipos algebraicos (product y sum) + core/ # Result, Option, Pair, Error + finance/ # OHLCV, Tick, BollingerResult, DrawdownResult + datascience/ # OutlierResult + cybersecurity/ # CIDRBlock, ThreatResult, PortResult + registry/ # Paquete Go: modelos, SQLite, parser, indexer, validacion, migraciones + models.go # Function, Type, Proposal structs + enums (Kind, Purity, Algebraic, ProposalKind, ProposalStatus) + db.go # Open/Close/Drop + WAL + migraciones + store.go # CRUD: Insert/Get/Update/Delete/List/Search para functions, types, proposals + validate.go # ValidateFunction, ValidateType, ValidateProposal + parser.go # ParseFunctionMD, ParseTypeMD (YAML frontmatter) + indexer.go # Index() — two-pass: parse → validate refs → insert + migrate.go # Motor de migraciones (embed.FS) + migrations/ # Archivos .sql numerados + fn_operations/ # Paquete Go: operations database (libreria, NO apps) + models.go # Entity, Relation, RelationInput, TypeSnapshot, Execution, Assertion, AssertionResult + enums + db.go # Open/Close/Drop/Conn + store.go # CRUD para todas las tablas + validate.go # ValidateEntity, ValidateRelation, ValidateExecution, ValidateAssertion, DetectCycle + operations.go # Alto nivel: InsertEntityWithSnapshot, InsertRelationSafe, React, ExecuteAndReact + eval.go # Motor de evaluacion: rewriteRule, EvalAssertion, EvalEntityAssertions + migrate.go # Motor de migraciones + migrations/ # Archivos .sql numerados + apps/ # Aplicaciones ejecutables (TUIs, CLIs) — modulos Go independientes + docker_tui/ # TUI fullscreen para gestionar Docker + pipeline_launcher/ # TUI para lanzar pipelines y registrar ejecuciones + cmd/fn/ # CLI + main.go # Subcomandos: index, search, list, show, add, ops, proposal + ops.go # fn ops: entity, relation, graph, snapshot, execution, assertion + proposal.go # fn proposal: add, list, show, update + docs/ # Specs de diseño (fuente de verdad del schema) + docs/templates/ # Plantillas de frontmatter + registry.db # Indice SQLite FTS5+WAL (regenerable con fn index, excepto proposals) ``` +--- + ## Build ```bash @@ -71,6 +402,78 @@ CGO_ENABLED=1 go test -tags fts5 ./... CGO_ENABLED=1 go build -tags fts5 -o fn ./cmd/fn/ ``` +--- + +## CLI completo (cmd/fn) + +### Registry + +```bash +fn index # Regenera registry.db desde los .md +fn search "texto" # Busqueda FTS en functions + types +fn search -k function -p pure -d core "slice" +fn list # Lista todo +fn list -d finance -k function # Lista por dominio y kind +fn show filter_slice_go_core # Muestra entrada completa +fn add -k function # Muestra template para copiar +``` + +### Proposals + +```bash +fn proposal add --kind new_function --title "..." --created-by agent [--target-id ] [--evidence '{}'] [--description "..."] +fn proposal list [-k kind] [-s status] +fn proposal show +fn proposal update --status approved [--reviewed-by lucas] +``` + +### Operations + +```bash +fn ops init [path] # Crea operations.db en el directorio +fn ops help # Ayuda + +# Entities +fn ops entity add --id --name --type-ref --source [--metadata '{}'] [--domain d] [--tags t1,t2] +fn ops entity list [--domain d] [--status s] +fn ops entity show +fn ops entity delete + +# Relations +fn ops relation add --id --name --from --to [--via ] [--direction uni] [--status designed] +fn ops relation list [--from ] +fn ops relation show +fn ops relation delete + +# Graph +fn ops graph # ASCII graph de entities y relations + +# Snapshots +fn ops snapshot list # Lista type snapshots +fn ops snapshot check # Compara snapshots vs registry actual +fn ops snapshot update |--all # Re-snapshot desde registry + +# Executions +fn ops execution add --pipeline-id --status success [--started-at ] [--ended-at ] [--records-in N] [--records-out N] [--metrics '{}'] [--error "msg"] +fn ops execution list [--pipeline-id ] [--relation-id ] [-s status] +fn ops execution show + +# Assertions +fn ops assertion add --entity-id --name "close positivo" --kind range --rule "close > 0" --severity critical [--description "..."] +fn ops assertion list [--entity-id ] [--active] [--inactive] +fn ops assertion show # Incluye ultimos 5 resultados +fn ops assertion delete +fn ops assertion eval --entity-id [--execution-id ] [--react] # Evalua assertions activas + +# Assertion results (registro manual) +fn ops assertion result add --assertion-id --status pass|fail|skip [--execution-id ] [--value '{}'] [--message "..."] +fn ops assertion result list [--assertion-id ] [--execution-id ] +``` + +`FN_REGISTRY_ROOT` env var permite que `fn ops` acceda a registry.db desde cualquier directorio. + +--- + ## Reglas para añadir funciones nuevas ### Antes de crear @@ -129,12 +532,12 @@ Explicacion adicional si es necesario. | Regla | Condicion | |---|---| -| Pipeline siempre impuro | `kind: pipeline` -> `purity: impure` + `uses_functions` no vacio | -| Pura sin side effects | `purity: pure` -> `returns_optional: false` + `error_type: ""` | -| Impura declara errores | `purity: impure` -> `error_type` obligatorio (usar `error_go_core`) | -| Tests coherentes | `tested: true` -> `test_file_path` y `tests` obligatorios | +| Pipeline siempre impuro | `kind: pipeline` → `purity: impure` + `uses_functions` no vacio | +| Pura sin side effects | `purity: pure` → `returns_optional: false` + `error_type: ""` | +| Impura declara errores | `purity: impure` → `error_type` obligatorio (usar `error_go_core`) | +| Tests coherentes | `tested: true` → `test_file_path` y `tests` obligatorios | | Referencias validas | `uses_functions`, `uses_types`, `returns`, `error_type` deben apuntar a IDs existentes | -| Component tiene framework | `kind: component` -> `framework` obligatorio, `returns` vacio (usar `emits`) | +| Component tiene framework | `kind: component` → `framework` obligatorio, `returns` vacio (usar `emits`) | | Rutas relativas | `file_path` siempre relativa a la raiz, nunca absoluta | | IDs unicos | Formato `{name}_{lang}_{domain}`, colisiones rechazadas | @@ -154,6 +557,8 @@ CGO_ENABLED=1 go build -tags fts5 -o fn ./cmd/fn/ && ./fn index ./fn show {name}_{lang}_{domain} ``` +--- + ## Reglas para añadir tipos nuevos Cada tipo requiere dos archivos: `types/{domain}/{name}.go` y `types/{domain}/{name}.md`. @@ -174,6 +579,8 @@ file_path: "types/{domain}/{name}.go" --- ``` +--- + ## Convenciones - **IDs:** `{name}_{lang}_{domain}` (ej: `filter_slice_go_core`) @@ -182,6 +589,10 @@ file_path: "types/{domain}/{name}.go" - **Tipos en firmas:** usar tipos nativos (float64, []float64, string) para evitar imports circulares entre paquetes de funciones. Documentar los tipos del registry en `uses_types`/`returns` del .md - **Purity:** puras en el centro, impuras en los bordes. Una pura NUNCA depende de una impura - **Stubs impuros:** si la implementacion real requiere dependencias externas no disponibles, crear stub con `return ..., fmt.Errorf("not implemented")` y documentar completamente el .md +- **Assertions:** kind es texto libre — puedes inventar nuevos kinds sin tocar schema +- **Proposals:** las crea el bucle reactivo automaticamente (created_by: reactive_loop) o el humano/agente manualmente + +--- ## Fuentes de verdad @@ -191,15 +602,49 @@ file_path: "types/{domain}/{name}.go" | Documentacion | archivos .md junto al codigo | | Diseño del schema | carpeta docs/ | | Indice de busqueda | registry.db (regenerable con `fn index`) | +| Proposals | registry.db tabla proposals (NO regenerable, son datos vivos) | +| Entities, relations | operations.db por proyecto (datos vivos) | +| Executions, assertions | operations.db por proyecto (datos vivos) | -## CLI (cmd/fn) +**Importante:** `fn index` regenera functions y types desde los .md pero NO toca proposals. Las proposals, entities, relations, executions, assertions y assertion_results son datos vivos que persisten. -```bash -fn index # Regenera registry.db desde los .md -fn search "texto" # Busqueda FTS -fn search -k function -p pure -d core "slice" -fn list # Lista todo -fn list -d finance -k function # Lista por dominio y kind -fn show filter_slice_go_core # Muestra entrada completa -fn add -k function # Muestra template para copiar +--- + +## Codigo Go: patrones clave + +### JSON en columnas TEXT + +Arrays y objetos se guardan como JSON serializado en columnas TEXT: + +```go +marshalStrings([]string) string // ["a","b"] → string +unmarshalStrings(string) []string // string → ["a","b"] +marshalJSON(map[string]any) string // {k:v} → string +unmarshalJSON(string) map[string]any ``` + +### Validacion + +Acumula errores, retorna nil o *ValidationError: + +```go +func ValidateX(x *X) *ValidationError { + var errs []string + if x.Name == "" { errs = append(errs, "name required") } + if len(errs) > 0 { return &ValidationError{ID: x.ID, Errors: errs} } + return nil +} +``` + +### Migraciones + +```go +//go:embed migrations/*.sql +var migrationsFS embed.FS + +func migrate(conn *sql.DB) error { ... } // aplica pendientes en transacciones +``` + +### Cycle detection + +Solo relaciones causales (via != "") se verifican. BFS desde to_entity buscando from_entity. diff --git a/fn_operations/docker_tui/.gitignore b/apps/docker_tui/.gitignore similarity index 100% rename from fn_operations/docker_tui/.gitignore rename to apps/docker_tui/.gitignore diff --git a/fn_operations/docker_tui/Makefile b/apps/docker_tui/Makefile similarity index 100% rename from fn_operations/docker_tui/Makefile rename to apps/docker_tui/Makefile diff --git a/fn_operations/docker_tui/app/model.go b/apps/docker_tui/app/model.go similarity index 100% rename from fn_operations/docker_tui/app/model.go rename to apps/docker_tui/app/model.go diff --git a/fn_operations/docker_tui/build.sh b/apps/docker_tui/build.sh similarity index 100% rename from fn_operations/docker_tui/build.sh rename to apps/docker_tui/build.sh diff --git a/fn_operations/docker_tui/config/config.go b/apps/docker_tui/config/config.go similarity index 100% rename from fn_operations/docker_tui/config/config.go rename to apps/docker_tui/config/config.go diff --git a/fn_operations/docker_tui/go.mod b/apps/docker_tui/go.mod similarity index 100% rename from fn_operations/docker_tui/go.mod rename to apps/docker_tui/go.mod diff --git a/fn_operations/docker_tui/go.sum b/apps/docker_tui/go.sum similarity index 100% rename from fn_operations/docker_tui/go.sum rename to apps/docker_tui/go.sum diff --git a/fn_operations/docker_tui/go.work b/apps/docker_tui/go.work similarity index 100% rename from fn_operations/docker_tui/go.work rename to apps/docker_tui/go.work diff --git a/fn_operations/docker_tui/go.work.sum b/apps/docker_tui/go.work.sum similarity index 100% rename from fn_operations/docker_tui/go.work.sum rename to apps/docker_tui/go.work.sum diff --git a/fn_operations/docker_tui/main.go b/apps/docker_tui/main.go similarity index 100% rename from fn_operations/docker_tui/main.go rename to apps/docker_tui/main.go diff --git a/fn_operations/docker_tui/operations.db b/apps/docker_tui/operations.db similarity index 100% rename from fn_operations/docker_tui/operations.db rename to apps/docker_tui/operations.db diff --git a/fn_operations/docker_tui/views/compose.go b/apps/docker_tui/views/compose.go similarity index 100% rename from fn_operations/docker_tui/views/compose.go rename to apps/docker_tui/views/compose.go diff --git a/fn_operations/docker_tui/views/containers.go b/apps/docker_tui/views/containers.go similarity index 100% rename from fn_operations/docker_tui/views/containers.go rename to apps/docker_tui/views/containers.go diff --git a/fn_operations/docker_tui/views/docker.go b/apps/docker_tui/views/docker.go similarity index 100% rename from fn_operations/docker_tui/views/docker.go rename to apps/docker_tui/views/docker.go diff --git a/fn_operations/docker_tui/views/images.go b/apps/docker_tui/views/images.go similarity index 100% rename from fn_operations/docker_tui/views/images.go rename to apps/docker_tui/views/images.go diff --git a/fn_operations/docker_tui/views/keys.go b/apps/docker_tui/views/keys.go similarity index 100% rename from fn_operations/docker_tui/views/keys.go rename to apps/docker_tui/views/keys.go diff --git a/fn_operations/docker_tui/views/networks.go b/apps/docker_tui/views/networks.go similarity index 100% rename from fn_operations/docker_tui/views/networks.go rename to apps/docker_tui/views/networks.go diff --git a/fn_operations/docker_tui/views/types.go b/apps/docker_tui/views/types.go similarity index 100% rename from fn_operations/docker_tui/views/types.go rename to apps/docker_tui/views/types.go diff --git a/fn_operations/docker_tui/views/volumes.go b/apps/docker_tui/views/volumes.go similarity index 100% rename from fn_operations/docker_tui/views/volumes.go rename to apps/docker_tui/views/volumes.go diff --git a/apps/pipeline_launcher/app/model.go b/apps/pipeline_launcher/app/model.go new file mode 100644 index 00000000..96cbb51d --- /dev/null +++ b/apps/pipeline_launcher/app/model.go @@ -0,0 +1,181 @@ +package app + +import ( + "fmt" + + ops "fn-registry/fn_operations" + "fn-registry/registry" + "pipeline-launcher/config" + "pipeline-launcher/views" + + tea "github.com/charmbracelet/bubbletea" + "github.com/charmbracelet/lipgloss" + "github.com/lucasdataproyects/devfactory/tui" +) + +// View identifies which tab is active. +type View int + +const ( + ViewPipelines View = iota + ViewHistory +) + +var tabNames = []string{"Pipelines", "History"} + +// Model is the top-level TUI model with two tabs. +type Model struct { + tui.BaseModel + activeTab int + pipelines views.PipelinesModel + history views.HistoryModel + ready bool + registryDB *registry.DB + opsDB *ops.DB +} + +// New creates the Model, opening both databases. +func New(cfg config.Config) (Model, error) { + regDB, err := registry.Open(cfg.RegistryDB) + if err != nil { + return Model{}, fmt.Errorf("opening registry: %w", err) + } + + opsDB, err := ops.Open(cfg.OperationsDB) + if err != nil { + regDB.Close() + return Model{}, fmt.Errorf("opening operations: %w", err) + } + + // Build pipeline name map for history view + fns, _ := regDB.SearchFunctions("", registry.KindPipeline, "", "", "") + names := make(map[string]string, len(fns)) + for _, f := range fns { + names[f.ID] = f.Name + } + + styles := tui.DarkStyles() + + return Model{ + BaseModel: tui.NewBaseModel().WithStyles(styles), + pipelines: views.NewPipelinesModel(styles, regDB, opsDB, cfg.RegistryRoot), + history: views.NewHistoryModel(styles, opsDB, names), + registryDB: regDB, + opsDB: opsDB, + }, nil +} + +// Close closes both database connections. +func (m Model) Close() { + if m.registryDB != nil { + m.registryDB.Close() + } + if m.opsDB != nil { + m.opsDB.Close() + } +} + +func (m Model) Init() tea.Cmd { + return m.pipelines.Init() +} + +func (m Model) Update(msg tea.Msg) (tea.Model, tea.Cmd) { + switch msg := msg.(type) { + case tea.KeyMsg: + switch msg.String() { + case views.KeyQuit: + return m, tea.Quit + case "q": + updated, atBase := m.handleBack() + if atBase { + return updated, tea.Quit + } + return updated, nil + case views.KeyEsc, views.KeyBack: + updated, atBase := m.handleBack() + if atBase { + return updated, nil + } + return updated, nil + case views.KeyTab: + m.activeTab = (m.activeTab + 1) % len(tabNames) + return m, m.initActiveView() + case "shift+tab": + m.activeTab = (m.activeTab - 1 + len(tabNames)) % len(tabNames) + return m, m.initActiveView() + } + case tea.WindowSizeMsg: + m.HandleWindowSize(msg) + m.ready = true + } + + var cmd tea.Cmd + switch View(m.activeTab) { + case ViewPipelines: + m.pipelines, cmd = m.pipelines.Update(msg) + case ViewHistory: + m.history, cmd = m.history.Update(msg) + } + return m, cmd +} + +func (m Model) View() string { + if !m.ready { + return "Loading..." + } + + tabs := m.renderTabs() + + var content string + switch View(m.activeTab) { + case ViewPipelines: + content = m.pipelines.View() + case ViewHistory: + content = m.history.View() + } + + status := m.Styles.StatusBar.Render(" Tab: switch view │ Ctrl+C: quit │ Enter: action │ r: refresh") + + return lipgloss.JoinVertical(lipgloss.Left, + tabs, + "", + content, + "", + status, + ) +} + +func (m Model) renderTabs() string { + var tabs []string + for i, name := range tabNames { + if i == m.activeTab { + tabs = append(tabs, m.Styles.Selected.Render(" "+name+" ")) + } else { + tabs = append(tabs, m.Styles.Muted.Render(" "+name+" ")) + } + } + row := lipgloss.JoinHorizontal(lipgloss.Top, tabs...) + return m.Styles.Header.Render("Pipeline Launcher") + " " + row +} + +func (m Model) handleBack() (Model, bool) { + switch View(m.activeTab) { + case ViewPipelines: + atBase := m.pipelines.HandleBack() + return m, atBase + case ViewHistory: + atBase := m.history.HandleBack() + return m, atBase + } + return m, true +} + +func (m Model) initActiveView() tea.Cmd { + switch View(m.activeTab) { + case ViewPipelines: + return m.pipelines.Init() + case ViewHistory: + return m.history.Init() + } + return nil +} diff --git a/apps/pipeline_launcher/config/config.go b/apps/pipeline_launcher/config/config.go new file mode 100644 index 00000000..4dcd3622 --- /dev/null +++ b/apps/pipeline_launcher/config/config.go @@ -0,0 +1,27 @@ +package config + +import ( + "os" + "path/filepath" +) + +// Config holds paths to databases. +type Config struct { + RegistryDB string // Path to registry.db + OperationsDB string // Path to operations.db + RegistryRoot string // Root directory of the registry (for resolving file paths) +} + +// Default returns a Config resolved from environment or sensible defaults. +func Default() Config { + root := os.Getenv("FN_REGISTRY_ROOT") + if root == "" { + root = "." + } + + return Config{ + RegistryDB: filepath.Join(root, "registry.db"), + OperationsDB: filepath.Join(root, "operations.db"), + RegistryRoot: root, + } +} diff --git a/apps/pipeline_launcher/go.mod b/apps/pipeline_launcher/go.mod new file mode 100644 index 00000000..0efb6cda --- /dev/null +++ b/apps/pipeline_launcher/go.mod @@ -0,0 +1,37 @@ +module pipeline-launcher + +go 1.22.2 + +require ( + fn-registry v0.0.0 + github.com/charmbracelet/bubbletea v0.25.0 + github.com/charmbracelet/lipgloss v0.9.1 + github.com/lucasdataproyects/devfactory v0.0.0 +) + +require ( + github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect + github.com/charmbracelet/bubbles v0.18.0 // indirect + github.com/charmbracelet/harmonica v0.2.0 // indirect + github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81 // indirect + github.com/lucasb-eyer/go-colorful v1.2.0 // indirect + github.com/mattn/go-isatty v0.0.19 // indirect + github.com/mattn/go-localereader v0.0.1 // indirect + github.com/mattn/go-runewidth v0.0.15 // indirect + github.com/mattn/go-sqlite3 v1.14.37 // indirect + github.com/muesli/ansi v0.0.0-20211018074035-2e021307bc4b // indirect + github.com/muesli/cancelreader v0.2.2 // indirect + github.com/muesli/reflow v0.3.0 // indirect + github.com/muesli/termenv v0.15.2 // indirect + github.com/rivo/uniseg v0.4.6 // indirect + golang.org/x/sync v0.4.0 // indirect + golang.org/x/sys v0.13.0 // indirect + golang.org/x/term v0.6.0 // indirect + golang.org/x/text v0.13.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) + +replace ( + fn-registry => /home/lucas/fn_registry + github.com/lucasdataproyects/devfactory => /home/lucas/.local_agentes/backend +) diff --git a/apps/pipeline_launcher/go.sum b/apps/pipeline_launcher/go.sum new file mode 100644 index 00000000..7278f31d --- /dev/null +++ b/apps/pipeline_launcher/go.sum @@ -0,0 +1,49 @@ +github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= +github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= +github.com/charmbracelet/bubbles v0.18.0 h1:PYv1A036luoBGroX6VWjQIE9Syf2Wby2oOl/39KLfy0= +github.com/charmbracelet/bubbles v0.18.0/go.mod h1:08qhZhtIwzgrtBjAcJnij1t1H0ZRjwHyGsy6AL11PSw= +github.com/charmbracelet/bubbletea v0.25.0 h1:bAfwk7jRz7FKFl9RzlIULPkStffg5k6pNt5dywy4TcM= +github.com/charmbracelet/bubbletea v0.25.0/go.mod h1:EN3QDR1T5ZdWmdfDzYcqOCAps45+QIJbLOBxmVNWNNg= +github.com/charmbracelet/harmonica v0.2.0 h1:8NxJWRWg/bzKqqEaaeFNipOu77YR5t8aSwG4pgaUBiQ= +github.com/charmbracelet/harmonica v0.2.0/go.mod h1:KSri/1RMQOZLbw7AHqgcBycp8pgJnQMYYT8QZRqZ1Ao= +github.com/charmbracelet/lipgloss v0.9.1 h1:PNyd3jvaJbg4jRHKWXnCj1akQm4rh8dbEzN1p/u1KWg= +github.com/charmbracelet/lipgloss v0.9.1/go.mod h1:1mPmG4cxScwUQALAAnacHaigiiHB9Pmr+v1VEawJl6I= +github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81 h1:q2hJAaP1k2wIvVRd/hEHD7lacgqrCPS+k8g1MndzfWY= +github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81/go.mod h1:YynlIjWYF8myEu6sdkwKIvGQq+cOckRm6So2avqoYAk= +github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= +github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= +github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2JC/oIi4= +github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88= +github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= +github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= +github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mattn/go-sqlite3 v1.14.37 h1:3DOZp4cXis1cUIpCfXLtmlGolNLp2VEqhiB/PARNBIg= +github.com/mattn/go-sqlite3 v1.14.37/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/muesli/ansi v0.0.0-20211018074035-2e021307bc4b h1:1XF24mVaiu7u+CFywTdcDo2ie1pzzhwjt6RHqzpMU34= +github.com/muesli/ansi v0.0.0-20211018074035-2e021307bc4b/go.mod h1:fQuZ0gauxyBcmsdE3ZT4NasjaRdxmbCS0jRHsrWu3Ho= +github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELUXHmA= +github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo= +github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s= +github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8= +github.com/muesli/termenv v0.15.2 h1:GohcuySI0QmI3wN8Ok9PtKGkgkFIk7y6Vpb5PvrY+Wo= +github.com/muesli/termenv v0.15.2/go.mod h1:Epx+iuz8sNs7mNKhxzH4fWXGNpZwUaJKRS1noLXviQ8= +github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.4.6 h1:Sovz9sDSwbOz9tgUy8JpT+KgCkPYJEN/oYzlJiYTNLg= +github.com/rivo/uniseg v0.4.6/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ= +golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.6.0 h1:clScbb1cHjoCkyRbWwBEUZ5H/tIFu5TAXIqaZD0Gcjw= +golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/apps/pipeline_launcher/main.go b/apps/pipeline_launcher/main.go new file mode 100644 index 00000000..18cc1890 --- /dev/null +++ b/apps/pipeline_launcher/main.go @@ -0,0 +1,29 @@ +package main + +import ( + "fmt" + "os" + + "pipeline-launcher/app" + "pipeline-launcher/config" + + "github.com/lucasdataproyects/devfactory/tui" +) + +func main() { + cfg := config.Default() + + model, err := app.New(cfg) + if err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + os.Exit(1) + } + defer model.Close() + + result := tui.RunFullscreen(model) + + if result.IsErr() { + fmt.Fprintf(os.Stderr, "error: %v\n", result.Error()) + os.Exit(1) + } +} diff --git a/apps/pipeline_launcher/views/history.go b/apps/pipeline_launcher/views/history.go new file mode 100644 index 00000000..df19bccd --- /dev/null +++ b/apps/pipeline_launcher/views/history.go @@ -0,0 +1,216 @@ +package views + +import ( + "encoding/json" + "fmt" + "strings" + + ops "fn-registry/fn_operations" + + tea "github.com/charmbracelet/bubbletea" + "github.com/charmbracelet/lipgloss" + "github.com/lucasdataproyects/devfactory/tui" +) + +type historyState int + +const ( + historyLoading historyState = iota + historyList + historyDetail +) + +type historyLoadedMsg []ops.Execution + +// HistoryModel shows execution history. +type HistoryModel struct { + state historyState + list tui.FilteredListModel + spinner tui.SpinnerModel + styles tui.Styles + executions []ops.Execution + detail string + scrollOff int + opsDB *ops.DB + pipelineNames map[string]string +} + +// NewHistoryModel creates a new history view. +func NewHistoryModel(styles tui.Styles, opsDB *ops.DB, names map[string]string) HistoryModel { + return HistoryModel{ + state: historyLoading, + list: tui.NewFilteredList(nil, "Filter executions..."), + spinner: tui.NewSpinner("Loading history..."), + styles: styles, + opsDB: opsDB, + pipelineNames: names, + } +} + +func (m HistoryModel) Init() tea.Cmd { + return tea.Batch( + m.spinner.Init(), + m.loadHistory(), + ) +} + +func (m HistoryModel) loadHistory() tea.Cmd { + return func() tea.Msg { + execs, err := m.opsDB.ListExecutions("", "", "") + if err != nil { + return historyLoadedMsg(nil) + } + return historyLoadedMsg(execs) + } +} + +func (m HistoryModel) Update(msg tea.Msg) (HistoryModel, tea.Cmd) { + switch msg := msg.(type) { + case historyLoadedMsg: + m.executions = []ops.Execution(msg) + items := make([]tui.ListItem, len(m.executions)) + for i, e := range m.executions { + icon := "●" + switch e.Status { + case ops.ExecSuccess: + icon = "✓" + case ops.ExecFailure: + icon = "✗" + case ops.ExecPartial: + icon = "~" + } + name := e.PipelineID + if n, ok := m.pipelineNames[e.PipelineID]; ok { + name = n + } + dur := "" + if e.DurationMs != nil { + dur = fmt.Sprintf("%dms", *e.DurationMs) + } + items[i] = tui.ListItem{ + Title: fmt.Sprintf("%s %s", icon, name), + Description: fmt.Sprintf("%s — %s — %s", string(e.Status), dur, e.StartedAt.Format("2006-01-02 15:04:05")), + Value: e, + } + } + m.list.SetItems(items) + m.state = historyList + return m, nil + + case tea.KeyMsg: + switch m.state { + case historyList: + switch msg.String() { + case "r": + m.state = historyLoading + m.spinner = tui.NewSpinner("Loading history...") + return m, tea.Batch(m.spinner.Init(), m.loadHistory()) + case "enter": + if item := m.list.SelectedItem(); item != nil { + e := item.Value.(ops.Execution) + m.detail = formatExecution(e) + m.state = historyDetail + m.scrollOff = 0 + } + return m, nil + } + case historyDetail: + switch msg.String() { + case "j", "down": + m.scrollOff++ + case "k", "up": + if m.scrollOff > 0 { + m.scrollOff-- + } + } + return m, nil + } + } + + // Delegate to sub-components + var cmd tea.Cmd + switch m.state { + case historyLoading: + var spinnerModel tea.Model + spinnerModel, cmd = m.spinner.Update(msg) + m.spinner = spinnerModel.(tui.SpinnerModel) + case historyList: + var listModel tea.Model + listModel, cmd = m.list.Update(msg) + m.list = listModel.(tui.FilteredListModel) + } + return m, cmd +} + +// HandleBack retrocede un nivel. Retorna true si ya en estado base. +func (m *HistoryModel) HandleBack() bool { + switch m.state { + case historyDetail: + m.state = historyList + return false + default: + return true + } +} + +func (m HistoryModel) View() string { + switch m.state { + case historyLoading: + return m.spinner.View() + case historyList: + if len(m.executions) == 0 { + return m.styles.Muted.Render("No executions found. Launch a pipeline first.") + } + help := m.styles.Muted.Render(" Enter: details │ r: refresh │ /: filter") + return m.list.View() + "\n" + help + case historyDetail: + return m.renderDetail() + } + return "" +} + +func (m HistoryModel) renderDetail() string { + lines := splitLines(m.detail) + maxLines := 20 + if m.scrollOff >= len(lines) { + m.scrollOff = max(0, len(lines)-1) + } + end := min(m.scrollOff+maxLines, len(lines)) + visible := lines[m.scrollOff:end] + + header := m.styles.Header.Render("Execution Detail") + content := lipgloss.JoinVertical(lipgloss.Left, visible...) + help := m.styles.Muted.Render(" j/k: scroll │ Esc: back") + + return header + "\n" + content + "\n" + help +} + +func formatExecution(e ops.Execution) string { + var sb strings.Builder + sb.WriteString(fmt.Sprintf("ID: %s\n", e.ID)) + sb.WriteString(fmt.Sprintf("Pipeline: %s\n", e.PipelineID)) + sb.WriteString(fmt.Sprintf("Status: %s\n", e.Status)) + sb.WriteString(fmt.Sprintf("Started: %s\n", e.StartedAt.Format("2006-01-02 15:04:05"))) + if e.EndedAt != nil { + sb.WriteString(fmt.Sprintf("Ended: %s\n", e.EndedAt.Format("2006-01-02 15:04:05"))) + } + if e.DurationMs != nil { + sb.WriteString(fmt.Sprintf("Duration: %dms\n", *e.DurationMs)) + } + if e.RecordsIn != nil { + sb.WriteString(fmt.Sprintf("Records In: %d\n", *e.RecordsIn)) + } + if e.RecordsOut != nil { + sb.WriteString(fmt.Sprintf("Records Out: %d\n", *e.RecordsOut)) + } + if e.Error != "" { + sb.WriteString(fmt.Sprintf("\n--- Error ---\n%s\n", e.Error)) + } + if len(e.Metrics) > 0 { + sb.WriteString("\n--- Metrics ---\n") + b, _ := json.MarshalIndent(e.Metrics, "", " ") + sb.WriteString(string(b)) + sb.WriteString("\n") + } + return sb.String() +} diff --git a/apps/pipeline_launcher/views/keys.go b/apps/pipeline_launcher/views/keys.go new file mode 100644 index 00000000..2ed80d08 --- /dev/null +++ b/apps/pipeline_launcher/views/keys.go @@ -0,0 +1,14 @@ +package views + +// Navigation key constants. +const ( + KeyQuit = "ctrl+c" + KeyEsc = "esc" + KeyBack = "0" + KeyTab = "tab" +) + +// IsBack returns true if the key should trigger back navigation. +func IsBack(key string) bool { + return key == KeyEsc || key == KeyBack +} diff --git a/apps/pipeline_launcher/views/pipelines.go b/apps/pipeline_launcher/views/pipelines.go new file mode 100644 index 00000000..a75b6f74 --- /dev/null +++ b/apps/pipeline_launcher/views/pipelines.go @@ -0,0 +1,249 @@ +package views + +import ( + "fmt" + "strings" + + ops "fn-registry/fn_operations" + "fn-registry/registry" + + tea "github.com/charmbracelet/bubbletea" + "github.com/charmbracelet/lipgloss" + "github.com/lucasdataproyects/devfactory/tui" +) + +type pipelinesState int + +const ( + pipelinesLoading pipelinesState = iota + pipelinesList + pipelinesRunning + pipelinesOutput +) + +type pipelinesLoadedMsg []registry.Function +type pipelineFinishedMsg RunResult + +// PipelinesModel lists and launches pipelines. +type PipelinesModel struct { + state pipelinesState + list tui.FilteredListModel + spinner tui.SpinnerModel + styles tui.Styles + pipelines []registry.Function + output string + lastResult *RunResult + scrollOff int + err error + registryDB *registry.DB + opsDB *ops.DB + registryRoot string +} + +// NewPipelinesModel creates a new pipelines view. +func NewPipelinesModel(styles tui.Styles, regDB *registry.DB, opsDB *ops.DB, root string) PipelinesModel { + return PipelinesModel{ + state: pipelinesLoading, + list: tui.NewFilteredList(nil, "Filter pipelines..."), + spinner: tui.NewSpinner("Loading pipelines..."), + styles: styles, + registryDB: regDB, + opsDB: opsDB, + registryRoot: root, + } +} + +func (m PipelinesModel) Init() tea.Cmd { + return tea.Batch( + m.spinner.Init(), + m.loadPipelines(), + ) +} + +func (m PipelinesModel) loadPipelines() tea.Cmd { + return func() tea.Msg { + fns, err := m.registryDB.SearchFunctions("", registry.KindPipeline, "", "", "") + if err != nil { + return pipelinesLoadedMsg(nil) + } + return pipelinesLoadedMsg(fns) + } +} + +func (m PipelinesModel) Update(msg tea.Msg) (PipelinesModel, tea.Cmd) { + switch msg := msg.(type) { + case pipelinesLoadedMsg: + m.pipelines = []registry.Function(msg) + items := make([]tui.ListItem, len(m.pipelines)) + for i, p := range m.pipelines { + items[i] = tui.ListItem{ + Title: p.Name, + Description: fmt.Sprintf("%s — %s", p.Domain, truncate(p.Description, 60)), + Value: p, + } + } + m.list.SetItems(items) + m.state = pipelinesList + return m, nil + + case pipelineFinishedMsg: + result := RunResult(msg) + m.lastResult = &result + // Build output + var sb strings.Builder + if result.Status == ops.ExecSuccess { + sb.WriteString("[OK] ") + } else { + sb.WriteString("[FAIL] ") + } + sb.WriteString(fmt.Sprintf("Pipeline: %s\n", result.PipelineID)) + sb.WriteString(fmt.Sprintf("Execution: %s\n", result.ExecID)) + sb.WriteString(fmt.Sprintf("Duration: %dms\n", result.DurationMs)) + sb.WriteString("\n--- stdout ---\n") + if result.Stdout != "" { + sb.WriteString(result.Stdout) + } else { + sb.WriteString("(empty)") + } + if result.Stderr != "" { + sb.WriteString("\n--- stderr ---\n") + sb.WriteString(result.Stderr) + } + if result.Err != nil { + sb.WriteString(fmt.Sprintf("\n--- error ---\n%v", result.Err)) + } + m.output = sb.String() + m.state = pipelinesOutput + m.scrollOff = 0 + return m, nil + + case tea.KeyMsg: + switch m.state { + case pipelinesList: + switch msg.String() { + case "r": + m.state = pipelinesLoading + m.spinner = tui.NewSpinner("Loading pipelines...") + return m, tea.Batch(m.spinner.Init(), m.loadPipelines()) + case "enter": + if item := m.list.SelectedItem(); item != nil { + fn := item.Value.(registry.Function) + m.state = pipelinesRunning + m.spinner = tui.NewSpinner(fmt.Sprintf("Running %s...", fn.Name)) + return m, tea.Batch(m.spinner.Init(), m.runPipelineCmd(&fn)) + } + } + case pipelinesOutput: + switch msg.String() { + case "j", "down": + m.scrollOff++ + case "k", "up": + if m.scrollOff > 0 { + m.scrollOff-- + } + } + return m, nil + } + } + + // Delegate to sub-components + var cmd tea.Cmd + switch m.state { + case pipelinesLoading, pipelinesRunning: + var spinnerModel tea.Model + spinnerModel, cmd = m.spinner.Update(msg) + m.spinner = spinnerModel.(tui.SpinnerModel) + case pipelinesList: + var listModel tea.Model + listModel, cmd = m.list.Update(msg) + m.list = listModel.(tui.FilteredListModel) + } + return m, cmd +} + +func (m PipelinesModel) runPipelineCmd(fn *registry.Function) tea.Cmd { + regRoot := m.registryRoot + opsDB := m.opsDB + fnCopy := *fn + return func() tea.Msg { + result := RunPipeline(&fnCopy, regRoot, opsDB) + return pipelineFinishedMsg(result) + } +} + +// HandleBack retrocede un nivel. Retorna true si ya en estado base. +func (m *PipelinesModel) HandleBack() bool { + switch m.state { + case pipelinesOutput: + m.state = pipelinesList + return false + default: + return true + } +} + +func (m PipelinesModel) View() string { + switch m.state { + case pipelinesLoading: + return m.spinner.View() + case pipelinesList: + if len(m.pipelines) == 0 { + return m.styles.Muted.Render("No pipelines found. Press 'r' to refresh.") + } + help := m.styles.Muted.Render(" Enter: launch │ r: refresh │ /: filter") + return m.list.View() + "\n" + help + case pipelinesRunning: + return m.spinner.View() + case pipelinesOutput: + return m.renderOutput() + } + return "" +} + +func (m PipelinesModel) renderOutput() string { + lines := splitLines(m.output) + maxLines := 20 + if m.scrollOff >= len(lines) { + m.scrollOff = max(0, len(lines)-1) + } + end := min(m.scrollOff+maxLines, len(lines)) + visible := lines[m.scrollOff:end] + + header := m.styles.Header.Render("Pipeline Output") + content := lipgloss.JoinVertical(lipgloss.Left, visible...) + help := m.styles.Muted.Render(" j/k: scroll │ Esc: back") + + return header + "\n" + content + "\n" + help +} + +func splitLines(s string) []string { + if s == "" { + return []string{"(empty)"} + } + lines := strings.Split(s, "\n") + if len(lines) == 0 { + return []string{"(empty)"} + } + return lines +} + +func truncate(s string, n int) string { + if len(s) <= n { + return s + } + return s[:n-3] + "..." +} + +func max(a, b int) int { + if a > b { + return a + } + return b +} + +func min(a, b int) int { + if a < b { + return a + } + return b +} diff --git a/apps/pipeline_launcher/views/runner.go b/apps/pipeline_launcher/views/runner.go new file mode 100644 index 00000000..3dfdae22 --- /dev/null +++ b/apps/pipeline_launcher/views/runner.go @@ -0,0 +1,88 @@ +package views + +import ( + "bytes" + "fmt" + "os/exec" + "path/filepath" + "time" + + ops "fn-registry/fn_operations" + "fn-registry/registry" +) + +// RunResult holds the outcome of a pipeline execution. +type RunResult struct { + Stdout string + Stderr string + ExecID string + PipelineID string + Status ops.ExecutionStatus + DurationMs int64 + Err error +} + +// RunPipeline executes a pipeline as a subprocess and records the execution. +func RunPipeline(fn *registry.Function, registryRoot string, opsDB *ops.DB) RunResult { + absPath := filepath.Join(registryRoot, fn.FilePath) + dir := filepath.Dir(absPath) + + startedAt := time.Now().UTC() + + cmd := exec.Command("go", "run", ".") + cmd.Dir = dir + + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + err := cmd.Run() + endedAt := time.Now().UTC() + + status := ops.ExecSuccess + var execErr string + if err != nil { + status = ops.ExecFailure + execErr = err.Error() + if stderr.Len() > 0 { + execErr = stderr.String() + } + } + + execID := fmt.Sprintf("exec_%d", time.Now().UnixNano()) + durationMs := endedAt.Sub(startedAt).Milliseconds() + + execution := &ops.Execution{ + ID: execID, + PipelineID: fn.ID, + Status: status, + StartedAt: startedAt, + EndedAt: &endedAt, + DurationMs: &durationMs, + Error: execErr, + CreatedAt: time.Now().UTC(), + } + + insertErr := ops.InsertExecutionSafe(opsDB, execution) + if insertErr != nil { + return RunResult{ + Stdout: stdout.String(), + Stderr: stderr.String(), + ExecID: execID, + PipelineID: fn.ID, + Status: status, + DurationMs: durationMs, + Err: fmt.Errorf("pipeline ran but failed to record: %w", insertErr), + } + } + + return RunResult{ + Stdout: stdout.String(), + Stderr: stderr.String(), + ExecID: execID, + PipelineID: fn.ID, + Status: status, + DurationMs: durationMs, + Err: err, + } +} diff --git a/cmd/fn/main.go b/cmd/fn/main.go index e8415f00..497ecead 100644 --- a/cmd/fn/main.go +++ b/cmd/fn/main.go @@ -31,6 +31,8 @@ func main() { cmdAdd(os.Args[2:]) case "ops": cmdOps(os.Args[2:]) + case "proposal": + cmdProposal(os.Args[2:]) case "help", "-h", "--help": printUsage() default: @@ -49,7 +51,8 @@ Usage: fn list [-k kind] [-d domain] [-l lang] fn show Muestra entrada completa fn add [-k kind] Abre $EDITOR con template - fn ops Gestiona operations.db (fn ops help)`) + fn ops Gestiona operations.db (fn ops help) + fn proposal Gestiona proposals`) } func root() string { diff --git a/cmd/fn/ops.go b/cmd/fn/ops.go index bb57331e..82761aee 100644 --- a/cmd/fn/ops.go +++ b/cmd/fn/ops.go @@ -8,6 +8,7 @@ import ( "path/filepath" "strings" "text/tabwriter" + "time" ops "fn-registry/fn_operations" "fn-registry/registry" @@ -32,6 +33,10 @@ func cmdOps(args []string) { cmdOpsGraph() case "snapshot": cmdOpsSnapshot(args[1:]) + case "execution": + cmdOpsExecution(args[1:]) + case "assertion": + cmdOpsAssertion(args[1:]) case "help", "-h", "--help": printOpsUsage() default: @@ -68,7 +73,21 @@ Relation flags: --id --name --from --to --via --direction --status --purity --weight <0.0-1.0> --description - --tags --notes `) + --tags --notes + +Execution commands: + fn ops execution add Registra ejecucion + fn ops execution list [--pipeline-id ] [-s status] + fn ops execution show Muestra ejecucion + +Assertion commands: + fn ops assertion add Añade assertion + fn ops assertion list [--entity-id ] Lista assertions + fn ops assertion show Muestra assertion + fn ops assertion delete Elimina assertion + fn ops assertion eval --entity-id Evalua assertions activas + fn ops assertion result add Registra resultado manual + fn ops assertion result list [--assertion-id ]`) } // --- ops init --- @@ -722,6 +741,663 @@ func openOpsDB() *ops.DB { return db } +// --- Execution subcommands --- + +func cmdOpsExecution(args []string) { + if len(args) < 1 { + fmt.Fprintln(os.Stderr, "usage: fn ops execution ") + os.Exit(1) + } + switch args[0] { + case "add": + cmdOpsExecutionAdd(args[1:]) + case "list": + cmdOpsExecutionList(args[1:]) + case "show": + cmdOpsExecutionShow(args[1:]) + default: + fmt.Fprintf(os.Stderr, "unknown execution command: %s\n", args[0]) + os.Exit(1) + } +} + +func cmdOpsExecutionAdd(args []string) { + var id, pipelineID, relationID, status, startedAtStr, endedAtStr, errorMsg, metricsStr string + var recordsIn, recordsOut *int64 + i := 0 + for i < len(args) { + switch args[i] { + case "--id": + i++ + id = args[i] + case "--pipeline-id": + i++ + pipelineID = args[i] + case "--relation-id": + i++ + relationID = args[i] + case "--status", "-s": + i++ + status = args[i] + case "--started-at": + i++ + startedAtStr = args[i] + case "--ended-at": + i++ + endedAtStr = args[i] + case "--records-in": + i++ + v := parseInt64(args[i]) + recordsIn = &v + case "--records-out": + i++ + v := parseInt64(args[i]) + recordsOut = &v + case "--error": + i++ + errorMsg = args[i] + case "--metrics": + i++ + metricsStr = args[i] + } + i++ + } + + if pipelineID == "" || status == "" { + fmt.Fprintln(os.Stderr, "error: --pipeline-id and --status are required") + os.Exit(1) + } + + if id == "" { + id = fmt.Sprintf("exec_%d", timeNow().UnixNano()) + } + + var startedAt time.Time + if startedAtStr != "" { + var err error + startedAt, err = time.Parse(time.RFC3339, startedAtStr) + if err != nil { + fmt.Fprintf(os.Stderr, "error: invalid started-at: %v\n", err) + os.Exit(1) + } + } else { + startedAt = timeNow() + } + + var endedAt *time.Time + if endedAtStr != "" { + t, err := time.Parse(time.RFC3339, endedAtStr) + if err != nil { + fmt.Fprintf(os.Stderr, "error: invalid ended-at: %v\n", err) + os.Exit(1) + } + endedAt = &t + } + + var metrics map[string]any + if metricsStr != "" { + if err := json.Unmarshal([]byte(metricsStr), &metrics); err != nil { + fmt.Fprintf(os.Stderr, "error: invalid metrics JSON: %v\n", err) + os.Exit(1) + } + } + + e := &ops.Execution{ + ID: id, + PipelineID: pipelineID, + RelationID: relationID, + Status: ops.ExecutionStatus(status), + StartedAt: startedAt, + EndedAt: endedAt, + RecordsIn: recordsIn, + RecordsOut: recordsOut, + Error: errorMsg, + Metrics: metrics, + } + + db := openOpsDB() + defer db.Close() + + if err := ops.InsertExecutionSafe(db, e); err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + os.Exit(1) + } + + fmt.Printf("Recorded execution: %s\n", e.ID) +} + +func cmdOpsExecutionList(args []string) { + var pipelineID, relationID, status string + i := 0 + for i < len(args) { + switch args[i] { + case "--pipeline-id": + i++ + pipelineID = args[i] + case "--relation-id": + i++ + relationID = args[i] + case "-s": + i++ + status = args[i] + } + i++ + } + + db := openOpsDB() + defer db.Close() + + execs, err := db.ListExecutions(pipelineID, relationID, ops.ExecutionStatus(status)) + if err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + os.Exit(1) + } + + if len(execs) == 0 { + fmt.Println("No executions.") + return + } + + w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0) + fmt.Fprintln(w, "ID\tPIPELINE\tSTATUS\tDURATION_MS\tRECORDS_IN\tRECORDS_OUT") + for _, e := range execs { + dur := "-" + if e.DurationMs != nil { + dur = fmt.Sprintf("%d", *e.DurationMs) + } + rin := "-" + if e.RecordsIn != nil { + rin = fmt.Sprintf("%d", *e.RecordsIn) + } + rout := "-" + if e.RecordsOut != nil { + rout = fmt.Sprintf("%d", *e.RecordsOut) + } + fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\n", e.ID, e.PipelineID, e.Status, dur, rin, rout) + } + w.Flush() +} + +func cmdOpsExecutionShow(args []string) { + if len(args) < 1 { + fmt.Fprintln(os.Stderr, "usage: fn ops execution show ") + os.Exit(1) + } + + db := openOpsDB() + defer db.Close() + + e, err := db.GetExecution(args[0]) + if err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + os.Exit(1) + } + if e == nil { + fmt.Fprintf(os.Stderr, "execution %q not found\n", args[0]) + os.Exit(1) + } + + fmt.Printf("ID: %s\n", e.ID) + fmt.Printf("Pipeline: %s\n", e.PipelineID) + if e.RelationID != "" { + fmt.Printf("Relation: %s\n", e.RelationID) + } + fmt.Printf("Status: %s\n", e.Status) + fmt.Printf("Started at: %s\n", e.StartedAt.Format(time.RFC3339)) + if e.EndedAt != nil { + fmt.Printf("Ended at: %s\n", e.EndedAt.Format(time.RFC3339)) + } + if e.DurationMs != nil { + fmt.Printf("Duration: %d ms\n", *e.DurationMs) + } + if e.RecordsIn != nil { + fmt.Printf("Records in: %d\n", *e.RecordsIn) + } + if e.RecordsOut != nil { + fmt.Printf("Records out: %d\n", *e.RecordsOut) + } + if e.Error != "" { + fmt.Printf("Error: %s\n", e.Error) + } + if len(e.Metrics) > 0 { + m, _ := json.MarshalIndent(e.Metrics, " ", " ") + fmt.Printf("Metrics: %s\n", string(m)) + } +} + +// --- Assertion subcommands --- + +func cmdOpsAssertion(args []string) { + if len(args) < 1 { + fmt.Fprintln(os.Stderr, "usage: fn ops assertion ") + os.Exit(1) + } + switch args[0] { + case "add": + cmdOpsAssertionAdd(args[1:]) + case "list": + cmdOpsAssertionList(args[1:]) + case "show": + cmdOpsAssertionShow(args[1:]) + case "delete": + cmdOpsAssertionDelete(args[1:]) + case "eval": + cmdOpsAssertionEval(args[1:]) + case "result": + cmdOpsAssertionResult(args[1:]) + default: + fmt.Fprintf(os.Stderr, "unknown assertion command: %s\n", args[0]) + os.Exit(1) + } +} + +func cmdOpsAssertionAdd(args []string) { + var id, entityID, name, kind, rule, severity, description string + i := 0 + for i < len(args) { + switch args[i] { + case "--id": + i++ + id = args[i] + case "--entity-id": + i++ + entityID = args[i] + case "--name": + i++ + name = args[i] + case "--kind": + i++ + kind = args[i] + case "--rule": + i++ + rule = args[i] + case "--severity": + i++ + severity = args[i] + case "--description": + i++ + description = args[i] + } + i++ + } + + if entityID == "" || name == "" || kind == "" || rule == "" { + fmt.Fprintln(os.Stderr, "error: --entity-id, --name, --kind, and --rule are required") + os.Exit(1) + } + + if id == "" { + id = fmt.Sprintf("assert_%s_%d", name, timeNow().UnixNano()) + } + if severity == "" { + severity = "warning" + } + + a := &ops.Assertion{ + ID: id, + EntityID: entityID, + Name: name, + Kind: kind, + Rule: rule, + Severity: ops.Severity(severity), + Description: description, + Active: true, + } + + db := openOpsDB() + defer db.Close() + + if err := ops.InsertAssertionSafe(db, a); err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + os.Exit(1) + } + + fmt.Printf("Created assertion: %s\n", a.ID) +} + +func cmdOpsAssertionList(args []string) { + var entityID string + var activeFilter *bool + i := 0 + for i < len(args) { + switch args[i] { + case "--entity-id": + i++ + entityID = args[i] + case "--active": + v := true + activeFilter = &v + case "--inactive": + v := false + activeFilter = &v + } + i++ + } + + db := openOpsDB() + defer db.Close() + + assertions, err := db.ListAssertions(entityID, activeFilter) + if err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + os.Exit(1) + } + + if len(assertions) == 0 { + fmt.Println("No assertions.") + return + } + + w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0) + fmt.Fprintln(w, "ID\tENTITY\tNAME\tKIND\tSEVERITY\tACTIVE") + for _, a := range assertions { + active := "yes" + if !a.Active { + active = "no" + } + fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\n", a.ID, a.EntityID, a.Name, a.Kind, a.Severity, active) + } + w.Flush() +} + +func cmdOpsAssertionShow(args []string) { + if len(args) < 1 { + fmt.Fprintln(os.Stderr, "usage: fn ops assertion show ") + os.Exit(1) + } + + db := openOpsDB() + defer db.Close() + + a, err := db.GetAssertion(args[0]) + if err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + os.Exit(1) + } + if a == nil { + fmt.Fprintf(os.Stderr, "assertion %q not found\n", args[0]) + os.Exit(1) + } + + active := "yes" + if !a.Active { + active = "no" + } + + fmt.Printf("ID: %s\n", a.ID) + fmt.Printf("Entity: %s\n", a.EntityID) + fmt.Printf("Name: %s\n", a.Name) + fmt.Printf("Kind: %s\n", a.Kind) + fmt.Printf("Rule: %s\n", a.Rule) + fmt.Printf("Severity: %s\n", a.Severity) + fmt.Printf("Description: %s\n", a.Description) + fmt.Printf("Active: %s\n", active) + fmt.Printf("Created: %s\n", a.CreatedAt.Format(time.RFC3339)) + + // Show recent results + results, err := db.ListAssertionResults(a.ID, "") + if err == nil && len(results) > 0 { + fmt.Printf("\nRecent results:\n") + limit := 5 + if len(results) < limit { + limit = len(results) + } + for _, r := range results[:limit] { + fmt.Printf(" [%s] %s — %s %s\n", r.EvaluatedAt.Format(time.RFC3339), r.Status, r.Message, formatResultValue(r.Value)) + } + } +} + +func cmdOpsAssertionDelete(args []string) { + if len(args) < 1 { + fmt.Fprintln(os.Stderr, "usage: fn ops assertion delete ") + os.Exit(1) + } + + db := openOpsDB() + defer db.Close() + + if err := db.DeleteAssertion(args[0]); err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + os.Exit(1) + } + fmt.Printf("Deleted assertion: %s\n", args[0]) +} + +func cmdOpsAssertionEval(args []string) { + var entityID, executionID string + react := false + i := 0 + for i < len(args) { + switch args[i] { + case "--entity-id": + i++ + entityID = args[i] + case "--execution-id": + i++ + executionID = args[i] + case "--react": + react = true + } + i++ + } + + if entityID == "" { + fmt.Fprintln(os.Stderr, "error: --entity-id is required") + os.Exit(1) + } + + db := openOpsDB() + defer db.Close() + + results, err := ops.EvalEntityAssertions(db, entityID, executionID) + if err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + os.Exit(1) + } + + if len(results) == 0 { + fmt.Println("No active assertions for this entity.") + return + } + + w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0) + fmt.Fprintln(w, "ASSERTION\tSTATUS\tMESSAGE") + for _, r := range results { + fmt.Fprintf(w, "%s\t%s\t%s\n", r.AssertionID, r.Status, truncate(r.Message, 60)) + } + w.Flush() + + // Summary + pass, fail, skip := 0, 0, 0 + for _, r := range results { + switch r.Status { + case ops.ResultPass: + pass++ + case ops.ResultFail: + fail++ + case ops.ResultSkip: + skip++ + } + } + fmt.Printf("\nResults: %d pass, %d fail, %d skip\n", pass, fail, skip) + + // Reactive loop: apply severity rules + if react && fail > 0 { + var regDB *registry.DB + regDB = tryOpenRegistryDB() + // regDB can be nil — React handles it (just won't create proposals) + + rr, err := ops.React(db, regDB, results) + if regDB != nil { + defer regDB.Close() + } + if err != nil { + fmt.Fprintf(os.Stderr, "error in reactive loop: %v\n", err) + os.Exit(1) + } + + if len(rr.EntityUpdates) > 0 { + fmt.Println("\nEntity status changes:") + for _, u := range rr.EntityUpdates { + fmt.Printf(" %s: %s -> %s (%s)\n", u.EntityID, u.OldStatus, u.NewStatus, u.Reason) + } + } + if len(rr.Proposals) > 0 { + fmt.Println("\nProposals created:") + for _, pid := range rr.Proposals { + fmt.Printf(" %s\n", pid) + } + } + } else if !react && fail > 0 { + fmt.Println("\nTip: use --react to apply severity rules (update entity status, create proposals)") + } +} + +// --- Assertion Result subcommands --- + +func cmdOpsAssertionResult(args []string) { + if len(args) < 1 { + fmt.Fprintln(os.Stderr, "usage: fn ops assertion result ") + os.Exit(1) + } + switch args[0] { + case "add": + cmdOpsAssertionResultAdd(args[1:]) + case "list": + cmdOpsAssertionResultList(args[1:]) + default: + fmt.Fprintf(os.Stderr, "unknown assertion result command: %s\n", args[0]) + os.Exit(1) + } +} + +func cmdOpsAssertionResultAdd(args []string) { + var id, assertionID, executionID, status, valueStr, message string + i := 0 + for i < len(args) { + switch args[i] { + case "--id": + i++ + id = args[i] + case "--assertion-id": + i++ + assertionID = args[i] + case "--execution-id": + i++ + executionID = args[i] + case "--status": + i++ + status = args[i] + case "--value": + i++ + valueStr = args[i] + case "--message": + i++ + message = args[i] + } + i++ + } + + if assertionID == "" || status == "" { + fmt.Fprintln(os.Stderr, "error: --assertion-id and --status are required") + os.Exit(1) + } + + if id == "" { + id = fmt.Sprintf("ar_%d", timeNow().UnixNano()) + } + + var value map[string]any + if valueStr != "" { + if err := json.Unmarshal([]byte(valueStr), &value); err != nil { + fmt.Fprintf(os.Stderr, "error: invalid value JSON: %v\n", err) + os.Exit(1) + } + } + + ar := &ops.AssertionResult{ + ID: id, + AssertionID: assertionID, + ExecutionID: executionID, + Status: ops.AssertionResultStatus(status), + Value: value, + Message: message, + EvaluatedAt: timeNow(), + } + + db := openOpsDB() + defer db.Close() + + if err := db.InsertAssertionResult(ar); err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + os.Exit(1) + } + + fmt.Printf("Recorded result: %s (%s)\n", ar.ID, ar.Status) +} + +func cmdOpsAssertionResultList(args []string) { + var assertionID, executionID string + i := 0 + for i < len(args) { + switch args[i] { + case "--assertion-id": + i++ + assertionID = args[i] + case "--execution-id": + i++ + executionID = args[i] + } + i++ + } + + db := openOpsDB() + defer db.Close() + + results, err := db.ListAssertionResults(assertionID, executionID) + if err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + os.Exit(1) + } + + if len(results) == 0 { + fmt.Println("No assertion results.") + return + } + + w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0) + fmt.Fprintln(w, "ID\tASSERTION\tEXECUTION\tSTATUS\tEVALUATED_AT") + for _, r := range results { + fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\n", r.ID, r.AssertionID, r.ExecutionID, r.Status, r.EvaluatedAt.Format(time.RFC3339)) + } + w.Flush() +} + +// --- helpers for new commands --- + +func timeNow() time.Time { + return time.Now().UTC() +} + +func parseInt64(s string) int64 { + var v int64 + fmt.Sscanf(s, "%d", &v) + return v +} + +func formatResultValue(v map[string]any) string { + if len(v) == 0 { + return "" + } + b, _ := json.Marshal(v) + s := string(b) + if len(s) > 40 { + return s[:37] + "..." + } + return s +} + func tryOpenRegistryDB() *registry.DB { // Try FN_REGISTRY_ROOT env var first if envRoot := os.Getenv("FN_REGISTRY_ROOT"); envRoot != "" { diff --git a/cmd/fn/proposal.go b/cmd/fn/proposal.go new file mode 100644 index 00000000..da185ff6 --- /dev/null +++ b/cmd/fn/proposal.go @@ -0,0 +1,265 @@ +package main + +import ( + "encoding/json" + "fmt" + "os" + "strings" + "text/tabwriter" + "time" + + "fn-registry/registry" +) + +func cmdProposal(args []string) { + if len(args) < 1 { + printProposalUsage() + os.Exit(1) + } + + switch args[0] { + case "add": + cmdProposalAdd(args[1:]) + case "list": + cmdProposalList(args[1:]) + case "show": + cmdProposalShow(args[1:]) + case "update": + cmdProposalUpdate(args[1:]) + case "help", "-h", "--help": + printProposalUsage() + default: + fmt.Fprintf(os.Stderr, "unknown proposal command: %s\n", args[0]) + printProposalUsage() + os.Exit(1) + } +} + +func printProposalUsage() { + fmt.Println(`fn proposal — gestiona proposals + +Usage: + fn proposal add --kind --title [options] + fn proposal list [-k kind] [-s status] + fn proposal show <id> + fn proposal update <id> --status <status> [--reviewed-by <who>] + +Kinds: new_function, new_type, improve_function, improve_type, new_pipeline +Status: pending, approved, rejected, implemented`) +} + +func cmdProposalAdd(args []string) { + var id, kind, targetID, title, description, evidenceStr, createdBy string + i := 0 + for i < len(args) { + switch args[i] { + case "--id": + i++ + id = args[i] + case "--kind": + i++ + kind = args[i] + case "--target-id": + i++ + targetID = args[i] + case "--title": + i++ + title = args[i] + case "--description": + i++ + description = args[i] + case "--evidence": + i++ + evidenceStr = args[i] + case "--created-by": + i++ + createdBy = args[i] + } + i++ + } + + if kind == "" || title == "" { + fmt.Fprintln(os.Stderr, "error: --kind and --title are required") + os.Exit(1) + } + + if id == "" { + id = fmt.Sprintf("proposal_%d", time.Now().UnixNano()) + } + + var evidence map[string]any + if evidenceStr != "" { + if err := json.Unmarshal([]byte(evidenceStr), &evidence); err != nil { + fmt.Fprintf(os.Stderr, "error: invalid evidence JSON: %v\n", err) + os.Exit(1) + } + } + + p := ®istry.Proposal{ + ID: id, + Kind: registry.ProposalKind(kind), + TargetID: targetID, + Title: title, + Description: description, + Evidence: evidence, + Status: registry.ProposalPending, + CreatedBy: createdBy, + } + + if err := registry.ValidateProposal(p); err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + os.Exit(1) + } + + db := openDB() + defer db.Close() + + if err := db.InsertProposal(p); err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + os.Exit(1) + } + + fmt.Printf("Created proposal: %s\n", p.ID) +} + +func cmdProposalList(args []string) { + var kind, status string + i := 0 + for i < len(args) { + switch args[i] { + case "-k": + i++ + kind = args[i] + case "-s": + i++ + status = args[i] + } + i++ + } + + db := openDB() + defer db.Close() + + proposals, err := db.ListProposals(registry.ProposalKind(kind), registry.ProposalStatus(status)) + if err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + os.Exit(1) + } + + if len(proposals) == 0 { + fmt.Println("No proposals.") + return + } + + w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0) + fmt.Fprintln(w, "ID\tKIND\tSTATUS\tTITLE\tCREATED_BY") + for _, p := range proposals { + fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\n", p.ID, p.Kind, p.Status, truncate(p.Title, 40), p.CreatedBy) + } + w.Flush() +} + +func cmdProposalShow(args []string) { + if len(args) < 1 { + fmt.Fprintln(os.Stderr, "usage: fn proposal show <id>") + os.Exit(1) + } + + db := openDB() + defer db.Close() + + p, err := db.GetProposal(args[0]) + if err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + os.Exit(1) + } + + fmt.Printf("ID: %s\n", p.ID) + fmt.Printf("Kind: %s\n", p.Kind) + fmt.Printf("Status: %s\n", p.Status) + fmt.Printf("Title: %s\n", p.Title) + fmt.Printf("Description: %s\n", p.Description) + if p.TargetID != "" { + fmt.Printf("Target ID: %s\n", p.TargetID) + } + if len(p.Evidence) > 0 { + ev, _ := json.MarshalIndent(p.Evidence, " ", " ") + fmt.Printf("Evidence: %s\n", string(ev)) + } + fmt.Printf("Created by: %s\n", p.CreatedBy) + if p.ReviewedBy != "" { + fmt.Printf("Reviewed by: %s\n", p.ReviewedBy) + } + fmt.Printf("Created: %s\n", p.CreatedAt.Format(time.RFC3339)) + fmt.Printf("Updated: %s\n", p.UpdatedAt.Format(time.RFC3339)) +} + +func cmdProposalUpdate(args []string) { + if len(args) < 1 { + fmt.Fprintln(os.Stderr, "usage: fn proposal update <id> --status <status> [--reviewed-by <who>]") + os.Exit(1) + } + + id := args[0] + var status, reviewedBy string + i := 1 + for i < len(args) { + switch args[i] { + case "--status": + i++ + status = args[i] + case "--reviewed-by": + i++ + reviewedBy = args[i] + } + i++ + } + + db := openDB() + defer db.Close() + + p, err := db.GetProposal(id) + if err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + os.Exit(1) + } + + if status != "" { + p.Status = registry.ProposalStatus(status) + } + if reviewedBy != "" { + p.ReviewedBy = reviewedBy + } + + // Validate updated proposal + validKinds := map[string]bool{ + "pending": true, "approved": true, "rejected": true, "implemented": true, + } + if status != "" && !validKinds[status] { + fmt.Fprintf(os.Stderr, "error: invalid status %q\n", status) + os.Exit(1) + } + + if err := db.UpdateProposal(p); err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + os.Exit(1) + } + + fmt.Printf("Updated proposal: %s (status: %s)\n", p.ID, p.Status) +} + +func formatEvidence(evidence map[string]any) string { + if len(evidence) == 0 { + return "{}" + } + b, _ := json.MarshalIndent(evidence, "", " ") + return string(b) +} + +// formatStrings joins a slice for display, handling nil/empty. +func formatStrings(ss []string) string { + if len(ss) == 0 { + return "" + } + return strings.Join(ss, ", ") +} diff --git a/docs/fn-registry-system-complete.md b/docs/fn-registry-system-complete.md new file mode 100644 index 00000000..41193562 --- /dev/null +++ b/docs/fn-registry-system-complete.md @@ -0,0 +1,437 @@ +# fn-registry — Sistema de conocimiento componible para agentes autónomos + +Schema completo · Bucle autónomo · Optimizaciones para agentes + +--- + +## Visión general del sistema + +| | fn_registry | fn_operations | +|---|---|---| +| **Naturaleza** | Conocimiento estático | Conocimiento dinámico | +| **Pregunta** | ¿Qué existe y qué puedo hacer? | ¿Qué ha ocurrido y cómo se conecta? | +| **Tablas** | `functions` · `types` · `proposals` | `entities` · `relations` · `relation_inputs` · `types_snapshot` · `executions` · `assertions` · `assertion_results` | +| **Scope** | Compartido entre todos los proyectos | Local a cada proyecto — un `.db` por proyecto | +| **Ontología** | Intensional — define conceptos en abstracto | Extensional — instancias concretas con valores reales | + +--- + +## REGISTRY + +### Tabla: functions + +Almacena funciones atómicas, pipelines y componentes React. Un pipeline es `kind: pipeline`, siempre impuro, que orquesta otras funciones. Un componente es `kind: component` con campos adicionales para su API visual. + +| Campo | Tipo | Descripción | Notas / Restricciones | +|---|---|---|---| +| `id` | string | Identificador único. | Formato: `{name}_{lang}_{domain}` ej: `filter_slice_go_core` | +| `name` | string | Nombre de la función o pipeline. | Obligatorio. snake_case. | +| `kind` | enum | Clasificación. | `function` \| `pipeline` \| `component` | +| `lang` | enum | Lenguaje. | `go` \| `python` \| `sql` \| `typescript` \| ... | +| `domain` | string | Namespace de dominio. | `finance`, `dsp`, `core`, `io`... | +| `version` | string | Versión semántica. | Semver: `1.0.0` | +| `purity` | enum | Side effects. | `pure` \| `impure` | +| `signature` | string | Firma completa. | Con parámetros y tipos de retorno. | +| `description` | text | Qué hace, por qué existe, cuándo usarla. | Obligatorio. | +| `tags` | []string | Etiquetas de búsqueda. | | +| `uses_functions` | []string | Funciones del registry que invoca. | IDs validados. | +| `uses_types` | []string | Tipos que recibe como parámetros. | IDs validados. | +| `returns` | []string | Tipos que emite. | IDs validados. | +| `returns_optional` | bool | Si el retorno puede estar ausente. | Siempre `false` en puras. | +| `error_type` | string | Tipo de error que puede emitir. | Obligatorio en impuras. | +| `imports` | []string | Dependencias externas. | | +| `example` | text | Ejemplo de uso. | Preferiblemente compilable. | +| `tested` | bool | Si tiene test. | `true` → `test_file_path` obligatorio. | +| `tests` | []string | Lista de tests aplicados. | | +| `test_file_path` | string | Ruta al archivo de test. | Relativa a raíz del registry. | +| `file_path` | string | Ruta al archivo de implementación. | Relativa a raíz del registry. | +| `created_at` | datetime | Fecha de creación. | ISO 8601. Automático. | +| `updated_at` | datetime | Fecha de última modificación. | ISO 8601. Automático. | + +#### Campos adicionales — kind: component + +| Campo | Tipo | Descripción | Notas / Restricciones | +|---|---|---|---| +| `props` | []PropDef | API de entrada. | `{ name, type, required, description }` | +| `emits` | []string | Eventos hacia el padre. | Ej: `[onClick, onChange]`. No son `returns`. | +| `has_state` | bool | Gestiona estado interno. | `true` → `purity: impure` automáticamente. | +| `framework` | enum | Framework de UI. | `react`. Obligatorio en `kind: component`. | +| `variant` | []string | Variantes disponibles. | Ej: `[primary, ghost, danger]` | + +#### Reglas de integridad — functions + +| Condición | Regla | +|---|---| +| `kind: pipeline` | `purity` siempre `impure`. `uses_functions` no puede estar vacío. | +| `kind: component` | `framework` obligatorio. `returns` vacío — usar `emits`. | +| `purity: pure` | `returns_optional: false`. `error_type` vacío. Una pura que devuelve opcional debe modelarse como tipo suma. | +| `purity: impure` | `error_type` obligatorio. | +| `tested: true` | `test_file_path` y `tests` obligatorios. | +| `uses_functions[]` | Todos los IDs deben existir en `functions`. | +| `uses_types[]` | Todos los IDs deben existir en `types`. | +| `has_state: true` | `purity: impure` automáticamente en componentes. | + +--- + +### Tabla: types + +Los tipos son el contrato entre funciones. Permiten al agente razonar sobre composabilidad sin leer implementaciones. + +**Tipo producto** — todos los campos siempre presentes. Modela datos y entidades. Valores posibles = N₁ × N₂ × ... × Nₙ + +**Tipo suma** — un caso activo a la vez. Modela estados, resultados y alternativas. Valores posibles = N₁ + N₂ + ... + Nₙ + +| Campo | Tipo | Descripción | Notas / Restricciones | +|---|---|---|---| +| `id` | string | Identificador único. | Formato: `{name}_{lang}_{domain}` ej: `ohlcv_go_finance` | +| `name` | string | Nombre del tipo. | PascalCase recomendado. | +| `lang` | enum | Lenguaje. | Mismo enum que `functions`. | +| `domain` | string | Namespace de dominio. | | +| `version` | string | Versión semántica. | Semver: `1.0.0` | +| `algebraic` | enum | Categoría algebraica. | `product` \| `sum` | +| `definition` | text | Definición completa. | Código real. | +| `description` | text | Qué modela y cuándo usarlo. | Obligatorio. | +| `tags` | []string | Etiquetas de búsqueda. | | +| `uses_types` | []string | Tipos que compone internamente. | IDs validados. | +| `file_path` | string | Ruta al archivo. | Relativa a raíz del registry. | +| `created_at` | datetime | Fecha de creación. | ISO 8601. Automático. | +| `updated_at` | datetime | Fecha de última modificación. | ISO 8601. Automático. | + +#### Reglas de integridad — types + +| Condición | Regla | +|---|---| +| `algebraic: product` | Todos los campos siempre presentes. Modela datos. | +| `algebraic: sum` | Un caso activo a la vez. Modela estados y resultados. | +| `uses_types[]` | Solo IDs existentes. Sin auto-referencias. | + +--- + +### Tabla: proposals + +El agente escribe propuestas de mejora al registry cuando assertions fallan o métricas son pobres. El humano revisa y aprueba. El registry crece de forma controlada. + +Las proposals viven en **registry** — no en operations — porque su destino es el conocimiento estático compartido entre todos los proyectos. + +``` +operations → detecta el problema (assertions, assertion_results) +registry → propuesta de solución (proposals) — beneficia a todos los proyectos +``` + +| Campo | Tipo | Descripción | Notas / Restricciones | +|---|---|---|---| +| `id` | string | Identificador único. | | +| `kind` | enum | Tipo de propuesta. | `new_function` \| `new_type` \| `improve_function` \| `improve_type` \| `new_pipeline` | +| `target_id` | string | ID del elemento a mejorar si aplica. | Vacío para propuestas nuevas. Referencia a `functions.id` o `types.id`. | +| `title` | string | Título corto. | Obligatorio. | +| `description` | text | Qué propone y por qué. | El agente explica su razonamiento. | +| `evidence` | json | Datos que justifican la propuesta. | Ej: `assertion_ids` que fallaron, métricas de `executions`. | +| `status` | enum | Estado de revisión. | `pending` \| `approved` \| `rejected` \| `implemented` | +| `created_by` | string | Quién propone. | Ej: `agent`, `human`. | +| `reviewed_by` | string | Quién revisó. | Vacío si `pending`. | +| `created_at` | datetime | Fecha de la propuesta. | ISO 8601. | +| `updated_at` | datetime | Fecha de última modificación. | ISO 8601. | + +--- + +## OPERATIONS + +### Tabla: entities + +Una entity es una instancia concreta de un tipo del registry dentro del contexto de un proyecto. + +``` +type_ref → lo que la entidad ES (heredado del registry) +metadata → valores concretos del tipo (la instancia real) +``` + +| Campo | Tipo | Descripción | Notas / Restricciones | +|---|---|---|---| +| `id` | string | Identificador único en el proyecto. | Formato: `{name}_{context}` ej: `ticks_btcusdt_2024` | +| `name` | string | Nombre semántico. | Obligatorio. snake_case. | +| `type_ref` | string | ID del tipo del registry. | Referencia a `types.id`. Obligatorio. | +| `status` | enum | Estado actual. | `active` \| `stale` \| `corrupted` \| `archived` | +| `description` | text | Rol en el contexto del proyecto. | | +| `domain` | string | Área de dominio. | Ej: `market_data`, `auth`. | +| `tags` | []string | Etiquetas de búsqueda. | | +| `source` | string | Origen del dato. | Obligatorio. Ej: `binance_api`, `pipeline_output`. | +| `metadata` | json | Valores concretos de los campos del tipo. | Ej: si `type_ref` es `person_go_core` con campos `name` y `dni`, aquí va `{"name":"Juan","dni":"32435"}`. Es la instancia real del tipo. | +| `notes` | text | Contexto extra libre. | Para el agente o para ti. | +| `created_at` | datetime | Fecha de registro. | ISO 8601. Automático. | +| `updated_at` | datetime | Fecha de modificación. | ISO 8601. Automático. | + +#### Valores de status — entities + +| Valor | Significado | +|---|---| +| `active` | Dato en uso, actualizado y fiable. | +| `stale` | Puede estar desactualizado. El agente verifica antes de usar. | +| `corrupted` | Problemas de integridad. El agente no lo usa como input en ningún pipeline activo. | +| `archived` | Ya no se usa. Se conserva por trazabilidad histórica. | + +#### Reglas de integridad — entities + +| Condición | Regla | +|---|---| +| `type_ref` | Debe existir en `fn_registry.types.id` o en `types_snapshot.id`. | +| `source` | Obligatorio. Todo dato tiene un origen conocido. | +| `metadata` | JSON válido. Contiene los valores de los campos definidos por el tipo del registry. | +| `status: corrupted` | El agente nunca usa esta entity como input de una relation activa. | + +--- + +### Tabla: types_snapshot + +Copia local de los tipos del registry en el momento de su primer uso. Hace `operations.db` completamente autónomo — no necesita `registry.db` en runtime ni en producción. + +| Campo | Tipo | Descripción | Notas / Restricciones | +|---|---|---|---| +| `id` | string | ID original del tipo en el registry. | | +| `version` | string | Versión en el momento del snapshot. | Semver. Inmutable tras el snapshot. | +| `lang` | string | Lenguaje. | | +| `algebraic` | enum | `product` \| `sum`. | | +| `definition` | text | Definición copiada. | Inmutable. | +| `description` | text | Descripción copiada. | Inmutable. | +| `snapped_at` | datetime | Cuándo se hizo el snapshot. | ISO 8601. Automático. | + +--- + +### Tabla: relations + +Describe cómo una entidad se conecta o transforma en otra. El campo `name` expresa el significado humano de la conexión. `via` es opcional — vacío para relaciones puramente semánticas. + +``` +juan (type: person) → CONOCE A → paula (type: person) +ticks → AGREGA → ohlcv_1h (via: tick_to_ohlcv_go_finance) +``` + +Para relaciones que consumen múltiples entidades (joins, merges) se usa `relation_inputs` en lugar de `from_entity`. + +| Campo | Tipo | Descripción | Notas / Restricciones | +|---|---|---|---| +| `id` | string | Identificador único. | Formato: `{from}__to__{to}__via__{fn}` | +| `name` | string | Etiqueta semántica de la relación. | Ej: `CONOCE A`, `TRANSFORMA`, `PRODUCE`, `DEPENDE DE`. Obligatorio. | +| `from_entity` | string | Entidad origen para relaciones 1-a-1. | Opcional si se usa `relation_inputs`. | +| `to_entity` | string | Entidad destino. | Obligatorio. | +| `via` | string | Función del registry que transforma. | Opcional — vacío si relación semántica. | +| `description` | text | Qué ocurre en este contexto. | | +| `purity` | enum | Naturaleza de la transformación. | `pure` \| `impure`. Consistente con `via.purity` si informado. | +| `direction` | enum | Dirección semántica. | `unidirectional` \| `bidirectional` \| `inverse` | +| `weight` | float | Ponderación opcional. | Rango `0.0 – 1.0`. | +| `status` | enum | Estado del flujo. | `designed` \| `implemented` \| `tested` \| `running` \| `deprecated` | +| `started_at` | datetime | Cuándo empezó el flujo realmente. | Distinto de `created_at`. | +| `ended_at` | datetime | Cuándo terminó. | Vacío si sigue activo. | +| `order` | int | Posición en secuencia. | Opcional. | +| `tags` | []string | Etiquetas. | | +| `notes` | text | Observaciones y contexto histórico. | El agente lee esto. | +| `created_at` | datetime | Fecha de documentación. | ISO 8601. Automático. | +| `updated_at` | datetime | Fecha de modificación. | ISO 8601. Automático. | + +#### Valores de direction + +| Valor | Significado | +|---|---| +| `unidirectional` | A → B. Un solo sentido. Valor por defecto. | +| `bidirectional` | A ↔ B. Simétrica. Ej: `CONOCE A`. | +| `inverse` | Documentada de B a A, lectura natural A a B. Para linaje inverso. | + +#### Valores de status — relations + +| Valor | Significado | +|---|---| +| `designed` | Planificado. En arquitectura pero sin código. | +| `implemented` | Código escrito. No verificado aún. | +| `tested` | Verificado. Los datos fluyen correctamente. | +| `running` | Activo en producción. | +| `deprecated` | Ya no se usa. Se mantiene por trazabilidad. | + +#### Reglas de integridad — relations + +| Condición | Regla | +|---|---| +| `name` | Obligatorio. | +| `from_entity` o `relation_inputs` | Al menos uno presente. Una relation sin origen es inválida. | +| `via` | Si informado, debe existir en `fn_registry.functions.id`. | +| `weight` | Si informado, rango `0.0 – 1.0`. | +| `started_at` / `ended_at` | Si ambos informados, `started_at` anterior a `ended_at`. | +| **Sin ciclos causales** | El CLI valida ciclos solo en relations con `via` informado. Las relaciones semánticas sin `via` pueden ser bidireccionales libremente. | + +--- + +### Tabla: relation_inputs + +Para relaciones que consumen múltiples entidades simultáneamente — joins, merges, agregaciones multi-fuente. Cuando se usa esta tabla, `from_entity` en `relations` se deja vacío. + +``` +join(ticks_btcusdt, metadata_binance) → ENRIQUECE → ohlcv_enriquecido + ↑ role: "base" ↑ role: "lookup" +``` + +| Campo | Tipo | Descripción | Notas / Restricciones | +|---|---|---|---| +| `id` | string | Identificador único. | Automático. | +| `relation_id` | string | Relation a la que pertenece. | Referencia a `relations.id`. Obligatorio. | +| `entity_id` | string | Entidad que actúa como input. | Referencia a `entities.id`. Obligatorio. | +| `role` | string | Rol semántico en la relación. | Ej: `base`, `lookup`, `filter`, `left`, `right`. Obligatorio. | +| `order` | int | Orden si la relación es sensible al orden. | Opcional. | + +#### Reglas de integridad — relation_inputs + +| Condición | Regla | +|---|---| +| `role` | Obligatorio. Sin rol el agente no puede razonar sobre cómo se usa el input. | +| Mínimo 2 inputs | Si se usa esta tabla, al menos 2 entradas. Si hay una sola usar `from_entity`. | + +--- + +### Tabla: executions + +Registra cada ejecución de un pipeline. Es la memoria de comportamiento del sistema — el agente consulta `executions` para detectar degradación, comparar rendimiento histórico y justificar `proposals`. + +| Campo | Tipo | Descripción | Notas / Restricciones | +|---|---|---|---| +| `id` | string | Identificador único. | Automático. | +| `pipeline_id` | string | Función/pipeline ejecutado. | Referencia a `functions.id` en registry. | +| `relation_id` | string | Relation que disparó la ejecución. | Referencia a `relations.id`. | +| `status` | enum | Resultado. | `success` \| `failure` \| `partial` | +| `started_at` | datetime | Inicio real de la ejecución. | ISO 8601. | +| `ended_at` | datetime | Fin de la ejecución. | ISO 8601. | +| `duration_ms` | int | Duración en milisegundos. | | +| `records_in` | int | Registros de entrada procesados. | | +| `records_out` | int | Registros de salida producidos. | | +| `error` | text | Mensaje de error si falló. | Vacío si `success`. | +| `metrics` | json | Métricas clave del output. | Ej: `{"mean_close":42000,"nulls":0}` | +| `created_at` | datetime | Fecha de registro. | ISO 8601. Automático. | + +--- + +### Tabla: assertions + +Reglas de calidad formales sobre entities. Condiciones computables que el agente evalúa automáticamente tras cada ejecución. No texto libre — condiciones que devuelven `pass` o `fail`. + +| Campo | Tipo | Descripción | Notas / Restricciones | +|---|---|---|---| +| `id` | string | Identificador único. | Ej: `ohlcv_close_positive` | +| `entity_id` | string | Entity sobre la que aplica. | Referencia a `entities.id`. | +| `name` | string | Nombre legible. | Ej: `close debe ser positivo` | +| `kind` | enum | Tipo de assertion. | `range` \| `null` \| `statistical` \| `consistency` \| `freshness` | +| `rule` | text | Condición formal evaluable. | SQL o expresión. Ej: `close > 0 AND close < open * 3` | +| `severity` | enum | Qué ocurre si falla. | `critical` \| `warning` \| `info` | +| `description` | text | Por qué existe esta regla. | Conocimiento de dominio que la justifica. | +| `active` | bool | Si se evalúa en cada ejecución. | Permite desactivar sin borrar. | +| `created_at` | datetime | Fecha de creación. | ISO 8601. | + +#### Tipos de assertion + +| Kind | Descripción y ejemplo | +|---|---| +| `range` | El valor debe estar dentro de un rango. Ej: `close BETWEEN 0 AND 1000000` | +| `null` | El campo no puede ser nulo. Ej: `open IS NOT NULL` | +| `statistical` | No debe desviarse más de N desviaciones estándar. Ej: `zscore(close) < 3` | +| `consistency` | Relación entre campos del mismo registro. Ej: `low <= close AND close <= high` | +| `freshness` | Los datos no pueden ser más antiguos de X. Ej: `max(ts) > now() - 1h` | + +#### Lógica de severidad + +| Severidad | Acción automática | +|---|---| +| `critical` | `entity.status = corrupted`. El agente no usa esta entity. Escribe en `proposals`. | +| `warning` | `entity.status = stale`. El agente la usa con precaución y lo documenta. | +| `info` | No cambia `status`. Solo registra en `assertion_results` para análisis histórico. | + +--- + +### Tabla: assertion_results + +Historial de cada evaluación de assertion. Permite al agente detectar tendencias — "esta assertion falla el 20% de las veces en ejecuciones nocturnas". + +| Campo | Tipo | Descripción | Notas / Restricciones | +|---|---|---|---| +| `id` | string | Identificador único. | | +| `assertion_id` | string | Assertion evaluada. | Referencia a `assertions.id`. | +| `execution_id` | string | Ejecución que disparó la evaluación. | Referencia a `executions.id`. | +| `status` | enum | Resultado. | `pass` \| `fail` \| `skip` | +| `value` | json | Valor concreto evaluado. | Lo que tenía el dato cuando falló. | +| `message` | text | Descripción del fallo. | Generado automáticamente. | +| `evaluated_at` | datetime | Cuándo se evaluó. | ISO 8601. | + +--- + +## El bucle autónomo + +El sistema completo habilita un ciclo de mejora continua donde el agente construye, ejecuta, analiza y propone mejoras sin intervención humana salvo en la aprobación final. + +### 1. CONSTRUIR +- Agente consulta registry → recupera funciones testeadas por FTS sobre `name`, `description`, `tags`. +- Razona sobre composabilidad comparando `returns` con `uses_types`. +- Prioriza funciones puras para el núcleo, aísla impuras en los bordes. +- Registra el pipeline en operations como `status: designed → implemented`. + +### 2. EJECUTAR +- Pipeline corre → inserta registro en `executions` con `duration_ms`, `records_in`, `records_out`, `metrics`. +- `operations.relations.status = running`. +- Si falla → `execution.status = failure`, `error` capturado. + +### 3. RECOPILAR +- Entities se pueblan — `metadata` contiene los valores concretos de los campos del tipo. +- `types_snapshot` garantiza que `operations.db` es autónomo sin `registry.db`. +- El agente actualiza `entity.status` según los datos recibidos. + +### 4. ANALIZAR +- Agente evalúa todas las `assertions` activas sobre las entities producidas. +- Compara `metrics` de la ejecución actual con `executions` históricas. +- `critical` falla → `entity.status = corrupted`. +- `warning` falla → `entity.status = stale`. +- Resultados en `assertion_results` con `value` concreto para debugging. + +### 5. MEJORAR +- Si assertions fallan o métricas degradan → agente escribe en `proposals`. +- `proposals.evidence` referencia los `assertion_ids` y `execution_ids` que lo justifican. +- El humano revisa `proposals.status: pending → approved → implemented`. +- El registry crece de forma controlada y trazable. + +### 6. REPEAT +- Con el registry mejorado el agente construye mejor en la siguiente iteración. +- Cada ciclo acumula conocimiento — funciones más robustas, tipos más precisos, assertions más finas. +- El sistema mejora continuamente sin resetear el contexto acumulado. + +--- + +## Optimizaciones para el agente + +| Área | Mejora | Beneficio para el agente | +|---|---|---| +| Búsqueda en registry | Índice FTS5 sobre `name + description + tags + signature` | Recuperación en <5ms sin embeddings | +| `types_snapshot` | Copiar tipo completo al primer uso | `operations.db` completamente autónomo sin `registry.db` | +| Ciclos en grafo | Validar solo relations con `via` informado (causales) | Relaciones semánticas bidireccionales permitidas | +| `assertions` | Evaluar automáticamente tras cada `execution` | Agente detecta degradación sin intervención humana | +| `metadata` en entities | Almacenar instancia real del tipo como JSON | Agente lee valores sin acceder a ficheros externos | +| `proposals.evidence` | Referenciar `assertion_ids` y `execution_ids` concretos | Trazabilidad completa del razonamiento del agente | +| WAL mode SQLite | Activado por defecto en plantilla | Lectura simultánea mientras el agente escribe | +| `version` en `functions` y `types` | Semver explícito | Detección automática de contratos rotos | +| `direction` en relations | `unidirectional / bidirectional / inverse` | Agente recorre linaje hacia adelante y hacia atrás | +| `proposals` en registry | No en operations | Un solo punto de revisión humana para todos los proyectos | + +--- + +## Referencias cruzadas completas + +| Campo origen | → | Destino | +|---|---|---| +| `functions.uses_functions[]` | → | `functions.id` | +| `functions.uses_types[]` | → | `types.id` | +| `functions.returns[]` | → | `types.id` | +| `functions.error_type` | → | `types.id` | +| `types.uses_types[]` | → | `types.id` | +| `proposals.target_id` | → | `functions.id` o `types.id` (opcional) | +| `entities.type_ref` | → | `fn_registry → types.id` / `types_snapshot.id` | +| `relations.from_entity` | → | `entities.id` (opcional) | +| `relations.to_entity` | → | `entities.id` | +| `relations.via` | → | `fn_registry → functions.id` (opcional) | +| `relation_inputs.relation_id` | → | `relations.id` | +| `relation_inputs.entity_id` | → | `entities.id` | +| `executions.pipeline_id` | → | `fn_registry → functions.id` | +| `executions.relation_id` | → | `relations.id` | +| `assertions.entity_id` | → | `entities.id` | +| `assertion_results.assertion_id` | → | `assertions.id` | +| `assertion_results.execution_id` | → | `executions.id` | diff --git a/fn_operations/db.go b/fn_operations/db.go index 0ce57f2a..1aacc124 100644 --- a/fn_operations/db.go +++ b/fn_operations/db.go @@ -9,89 +9,6 @@ import ( _ "github.com/mattn/go-sqlite3" ) -const schemaSQL = ` -CREATE TABLE IF NOT EXISTS types_snapshot ( - id TEXT PRIMARY KEY, - version TEXT NOT NULL DEFAULT '1.0.0', - lang TEXT NOT NULL, - algebraic TEXT NOT NULL CHECK(algebraic IN ('product','sum')), - definition TEXT NOT NULL DEFAULT '', - description TEXT NOT NULL DEFAULT '', - snapped_at TEXT NOT NULL -); - -CREATE TABLE IF NOT EXISTS entities ( - id TEXT PRIMARY KEY, - name TEXT NOT NULL, - type_ref TEXT NOT NULL, - status TEXT NOT NULL DEFAULT 'active' CHECK(status IN ('active','stale','corrupted','archived')), - description TEXT NOT NULL DEFAULT '', - domain TEXT NOT NULL DEFAULT '', - tags TEXT NOT NULL DEFAULT '[]', - source TEXT NOT NULL, - metadata TEXT NOT NULL DEFAULT '{}', - notes TEXT NOT NULL DEFAULT '', - created_at TEXT NOT NULL, - updated_at TEXT NOT NULL -); - -CREATE TABLE IF NOT EXISTS relations ( - id TEXT PRIMARY KEY, - name TEXT NOT NULL, - from_entity TEXT NOT NULL DEFAULT '', - to_entity TEXT NOT NULL, - via TEXT NOT NULL DEFAULT '', - description TEXT NOT NULL DEFAULT '', - purity TEXT NOT NULL DEFAULT '' CHECK(purity IN ('','pure','impure')), - direction TEXT NOT NULL DEFAULT 'unidirectional' CHECK(direction IN ('unidirectional','bidirectional','inverse')), - weight REAL, - status TEXT NOT NULL DEFAULT 'designed' CHECK(status IN ('designed','implemented','tested','running','deprecated')), - started_at TEXT, - ended_at TEXT, - "order" INTEGER, - tags TEXT NOT NULL DEFAULT '[]', - notes TEXT NOT NULL DEFAULT '', - created_at TEXT NOT NULL, - updated_at TEXT NOT NULL -); - -CREATE TABLE IF NOT EXISTS relation_inputs ( - id TEXT PRIMARY KEY, - relation_id TEXT NOT NULL REFERENCES relations(id) ON DELETE CASCADE, - entity_id TEXT NOT NULL REFERENCES entities(id), - role TEXT NOT NULL, - "order" INTEGER -); - -CREATE VIRTUAL TABLE IF NOT EXISTS entities_fts USING fts5( - id, - name, - description, - tags, - domain, - content='entities', - content_rowid='rowid' -); - --- Triggers to keep entities FTS in sync -CREATE TRIGGER IF NOT EXISTS entities_ai AFTER INSERT ON entities BEGIN - INSERT INTO entities_fts(rowid, id, name, description, tags, domain) - VALUES (new.rowid, new.id, new.name, new.description, new.tags, new.domain); -END; - -CREATE TRIGGER IF NOT EXISTS entities_ad AFTER DELETE ON entities BEGIN - INSERT INTO entities_fts(entities_fts, rowid, id, name, description, tags, domain) - VALUES ('delete', old.rowid, old.id, old.name, old.description, old.tags, old.domain); -END; - -CREATE TRIGGER IF NOT EXISTS entities_au AFTER UPDATE ON entities BEGIN - INSERT INTO entities_fts(entities_fts, rowid, id, name, description, tags, domain) - VALUES ('delete', old.rowid, old.id, old.name, old.description, old.tags, old.domain); - INSERT INTO entities_fts(rowid, id, name, description, tags, domain) - VALUES (new.rowid, new.id, new.name, new.description, new.tags, new.domain); -END; -` - // DB wraps a SQLite connection for an operations database. type DB struct { conn *sql.DB @@ -115,9 +32,9 @@ func Open(path string) (*DB, error) { return nil, fmt.Errorf("setting WAL mode: %w", err) } - if _, err := conn.Exec(schemaSQL); err != nil { + if err := migrate(conn); err != nil { conn.Close() - return nil, fmt.Errorf("applying schema: %w", err) + return nil, fmt.Errorf("running migrations: %w", err) } return &DB{conn: conn, path: path}, nil diff --git a/fn_operations/eval.go b/fn_operations/eval.go new file mode 100644 index 00000000..542ec1f8 --- /dev/null +++ b/fn_operations/eval.go @@ -0,0 +1,117 @@ +package fn_operations + +import ( + "fmt" + "regexp" + "strings" + "time" +) + +// fieldPattern matches bare field names (word chars) that are NOT already inside +// a json_extract call, a SQL keyword, a string literal, or a number. +var fieldPattern = regexp.MustCompile(`\b([a-zA-Z_][a-zA-Z0-9_]*)\b`) + +// sqlKeywords that should not be rewritten to json_extract. +var sqlKeywords = map[string]bool{ + "AND": true, "OR": true, "NOT": true, "IS": true, "NULL": true, + "IN": true, "BETWEEN": true, "LIKE": true, "GLOB": true, + "TRUE": true, "FALSE": true, "CASE": true, "WHEN": true, + "THEN": true, "ELSE": true, "END": true, "SELECT": true, + "FROM": true, "WHERE": true, "AS": true, "CAST": true, +} + +// sqlFunctions that should not be rewritten. +var sqlFunctions = map[string]bool{ + "json_extract": true, "datetime": true, "now": true, + "abs": true, "avg": true, "count": true, "max": true, "min": true, + "sum": true, "total": true, "length": true, "typeof": true, + "coalesce": true, "ifnull": true, "nullif": true, + "upper": true, "lower": true, "trim": true, "replace": true, + "substr": true, "instr": true, "round": true, +} + +// rewriteRule transforms a rule expression into SQL that operates on entity metadata. +// Bare field names are rewritten to json_extract(metadata, '$.field'). +// If the rule already uses json_extract, it is left as-is. +func rewriteRule(rule string) string { + if strings.Contains(rule, "json_extract") { + return rule + } + + return fieldPattern.ReplaceAllStringFunc(rule, func(match string) string { + upper := strings.ToUpper(match) + if sqlKeywords[upper] { + return match + } + if sqlFunctions[strings.ToLower(match)] { + return match + } + // Skip numeric-looking tokens (shouldn't match the regex, but be safe) + if match[0] >= '0' && match[0] <= '9' { + return match + } + return fmt.Sprintf("json_extract(metadata, '$.%s')", match) + }) +} + +// EvalAssertion evaluates a single assertion against its entity's metadata. +// Returns a result with pass/fail/skip status. +func EvalAssertion(db *DB, a *Assertion, executionID string) (*AssertionResult, error) { + result := &AssertionResult{ + ID: fmt.Sprintf("ar_%s_%d", a.ID, time.Now().UnixNano()), + AssertionID: a.ID, + ExecutionID: executionID, + EvaluatedAt: time.Now().UTC(), + } + + rewritten := rewriteRule(a.Rule) + + q := fmt.Sprintf(` + SELECT CASE WHEN (%s) THEN 'pass' ELSE 'fail' END, + metadata + FROM entities WHERE id = ?`, rewritten) + + var status, metadataJSON string + err := db.conn.QueryRow(q, a.EntityID).Scan(&status, &metadataJSON) + if err != nil { + result.Status = ResultSkip + result.Message = fmt.Sprintf("evaluation error: %v", err) + return result, nil + } + + if status == "pass" { + result.Status = ResultPass + } else { + result.Status = ResultFail + result.Value = unmarshalJSON(metadataJSON) + result.Message = fmt.Sprintf("rule failed: %s", a.Rule) + } + + return result, nil +} + +// EvalEntityAssertions evaluates all active assertions for an entity. +// Returns results and persists them in the database. +func EvalEntityAssertions(db *DB, entityID, executionID string) ([]AssertionResult, error) { + active := true + assertions, err := db.ListAssertions(entityID, &active) + if err != nil { + return nil, fmt.Errorf("listing assertions for entity %s: %w", entityID, err) + } + + var results []AssertionResult + for _, a := range assertions { + ar, err := EvalAssertion(db, &a, executionID) + if err != nil { + return nil, fmt.Errorf("evaluating assertion %s: %w", a.ID, err) + } + + if err := db.InsertAssertionResult(ar); err != nil { + return nil, fmt.Errorf("storing result for assertion %s: %w", a.ID, err) + } + + results = append(results, *ar) + } + + return results, nil +} diff --git a/fn_operations/migrate.go b/fn_operations/migrate.go new file mode 100644 index 00000000..782b97f4 --- /dev/null +++ b/fn_operations/migrate.go @@ -0,0 +1,117 @@ +package fn_operations + +import ( + "database/sql" + "embed" + "fmt" + "path" + "sort" + "strconv" + "strings" + "time" +) + +//go:embed migrations/*.sql +var migrationsFS embed.FS + +const migrationTableSQL = ` +CREATE TABLE IF NOT EXISTS schema_migrations ( + version INTEGER PRIMARY KEY, + name TEXT NOT NULL, + applied_at TEXT NOT NULL +);` + +// migrate applies pending migrations to the database. +func migrate(conn *sql.DB) error { + if _, err := conn.Exec(migrationTableSQL); err != nil { + return fmt.Errorf("creating schema_migrations table: %w", err) + } + + current, err := currentVersion(conn) + if err != nil { + return err + } + + files, err := listMigrations() + if err != nil { + return err + } + + for _, mf := range files { + if mf.version <= current { + continue + } + + content, err := migrationsFS.ReadFile(path.Join("migrations", mf.filename)) + if err != nil { + return fmt.Errorf("reading migration %s: %w", mf.filename, err) + } + + tx, err := conn.Begin() + if err != nil { + return fmt.Errorf("beginning transaction for migration %d: %w", mf.version, err) + } + + if _, err := tx.Exec(string(content)); err != nil { + tx.Rollback() + return fmt.Errorf("applying migration %s: %w", mf.filename, err) + } + + if _, err := tx.Exec( + "INSERT INTO schema_migrations (version, name, applied_at) VALUES (?, ?, ?)", + mf.version, mf.filename, time.Now().UTC().Format(time.RFC3339), + ); err != nil { + tx.Rollback() + return fmt.Errorf("recording migration %s: %w", mf.filename, err) + } + + if err := tx.Commit(); err != nil { + return fmt.Errorf("committing migration %s: %w", mf.filename, err) + } + } + + return nil +} + +func currentVersion(conn *sql.DB) (int, error) { + var v int + err := conn.QueryRow("SELECT COALESCE(MAX(version), 0) FROM schema_migrations").Scan(&v) + if err != nil { + return 0, fmt.Errorf("reading current schema version: %w", err) + } + return v, nil +} + +type migrationFile struct { + version int + filename string +} + +func listMigrations() ([]migrationFile, error) { + entries, err := migrationsFS.ReadDir("migrations") + if err != nil { + return nil, fmt.Errorf("reading migrations directory: %w", err) + } + + var files []migrationFile + for _, e := range entries { + if e.IsDir() || !strings.HasSuffix(e.Name(), ".sql") { + continue + } + parts := strings.SplitN(e.Name(), "_", 2) + if len(parts) < 2 { + continue + } + v, err := strconv.Atoi(parts[0]) + if err != nil { + continue + } + files = append(files, migrationFile{version: v, filename: e.Name()}) + } + + sort.Slice(files, func(i, j int) bool { + return files[i].version < files[j].version + }) + + return files, nil +} diff --git a/fn_operations/migrations/002_executions_assertions.sql b/fn_operations/migrations/002_executions_assertions.sql new file mode 100644 index 00000000..993c8ce6 --- /dev/null +++ b/fn_operations/migrations/002_executions_assertions.sql @@ -0,0 +1,65 @@ +-- Executions, assertions, and assertion_results tables. +-- Closes the autonomous improvement loop: execute -> assert -> analyze. + +CREATE TABLE IF NOT EXISTS executions ( + id TEXT PRIMARY KEY, + pipeline_id TEXT NOT NULL, + relation_id TEXT NOT NULL DEFAULT '', + status TEXT NOT NULL CHECK(status IN ('success','failure','partial')), + started_at TEXT NOT NULL, + ended_at TEXT, + duration_ms INTEGER, + records_in INTEGER, + records_out INTEGER, + error TEXT NOT NULL DEFAULT '', + metrics TEXT NOT NULL DEFAULT '{}', + created_at TEXT NOT NULL +); + +CREATE TABLE IF NOT EXISTS assertions ( + id TEXT PRIMARY KEY, + entity_id TEXT NOT NULL REFERENCES entities(id), + name TEXT NOT NULL, + kind TEXT NOT NULL, + rule TEXT NOT NULL, + severity TEXT NOT NULL DEFAULT 'warning' CHECK(severity IN ('critical','warning','info')), + description TEXT NOT NULL DEFAULT '', + active INTEGER NOT NULL DEFAULT 1, + created_at TEXT NOT NULL +); + +CREATE TABLE IF NOT EXISTS assertion_results ( + id TEXT PRIMARY KEY, + assertion_id TEXT NOT NULL REFERENCES assertions(id), + execution_id TEXT NOT NULL DEFAULT '', + status TEXT NOT NULL CHECK(status IN ('pass','fail','skip')), + value TEXT NOT NULL DEFAULT '{}', + message TEXT NOT NULL DEFAULT '', + evaluated_at TEXT NOT NULL +); + +CREATE VIRTUAL TABLE IF NOT EXISTS assertions_fts USING fts5( + id, + name, + description, + rule, + content='assertions', + content_rowid='rowid' +); + +CREATE TRIGGER IF NOT EXISTS assertions_ai AFTER INSERT ON assertions BEGIN + INSERT INTO assertions_fts(rowid, id, name, description, rule) + VALUES (new.rowid, new.id, new.name, new.description, new.rule); +END; + +CREATE TRIGGER IF NOT EXISTS assertions_ad AFTER DELETE ON assertions BEGIN + INSERT INTO assertions_fts(assertions_fts, rowid, id, name, description, rule) + VALUES ('delete', old.rowid, old.id, old.name, old.description, old.rule); +END; + +CREATE TRIGGER IF NOT EXISTS assertions_au AFTER UPDATE ON assertions BEGIN + INSERT INTO assertions_fts(assertions_fts, rowid, id, name, description, rule) + VALUES ('delete', old.rowid, old.id, old.name, old.description, old.rule); + INSERT INTO assertions_fts(rowid, id, name, description, rule) + VALUES (new.rowid, new.id, new.name, new.description, new.rule); +END; diff --git a/fn_operations/models.go b/fn_operations/models.go index 2eefeab3..3e1da45f 100644 --- a/fn_operations/models.go +++ b/fn_operations/models.go @@ -78,6 +78,73 @@ type RelationInput struct { Order *int `json:"order"` } +// ExecutionStatus represents the result of a pipeline execution. +type ExecutionStatus string + +const ( + ExecSuccess ExecutionStatus = "success" + ExecFailure ExecutionStatus = "failure" + ExecPartial ExecutionStatus = "partial" +) + +// Severity classifies the impact of an assertion failure. +type Severity string + +const ( + SeverityCritical Severity = "critical" + SeverityWarning Severity = "warning" + SeverityInfo Severity = "info" +) + +// AssertionResultStatus represents the outcome of an assertion evaluation. +type AssertionResultStatus string + +const ( + ResultPass AssertionResultStatus = "pass" + ResultFail AssertionResultStatus = "fail" + ResultSkip AssertionResultStatus = "skip" +) + +// Execution records a pipeline run with its metrics and outcome. +type Execution struct { + ID string `json:"id"` + PipelineID string `json:"pipeline_id"` + RelationID string `json:"relation_id"` + Status ExecutionStatus `json:"status"` + StartedAt time.Time `json:"started_at"` + EndedAt *time.Time `json:"ended_at"` + DurationMs *int64 `json:"duration_ms"` + RecordsIn *int64 `json:"records_in"` + RecordsOut *int64 `json:"records_out"` + Error string `json:"error"` + Metrics map[string]any `json:"metrics"` + CreatedAt time.Time `json:"created_at"` +} + +// Assertion is a formal quality rule evaluated against an entity. +type Assertion struct { + ID string `json:"id"` + EntityID string `json:"entity_id"` + Name string `json:"name"` + Kind string `json:"kind"` // free text: range, null, statistical, consistency, freshness, ... + Rule string `json:"rule"` + Severity Severity `json:"severity"` + Description string `json:"description"` + Active bool `json:"active"` + CreatedAt time.Time `json:"created_at"` +} + +// AssertionResult records one evaluation of an assertion. +type AssertionResult struct { + ID string `json:"id"` + AssertionID string `json:"assertion_id"` + ExecutionID string `json:"execution_id"` + Status AssertionResultStatus `json:"status"` + Value map[string]any `json:"value"` + Message string `json:"message"` + EvaluatedAt time.Time `json:"evaluated_at"` +} + // TypeSnapshot is an immutable copy of a registry type at point of use. type TypeSnapshot struct { ID string `json:"id"` diff --git a/fn_operations/operations.go b/fn_operations/operations.go index 973c3f64..58af8fae 100644 --- a/fn_operations/operations.go +++ b/fn_operations/operations.go @@ -277,6 +277,276 @@ func UpdateSnapshot(opsDB *DB, registryDB *registry.DB, typeID string) (old, new return oldSnap, newSnap, nil } +// InsertExecutionSafe validates and inserts an execution. +// Auto-calculates duration_ms if both started_at and ended_at are set. +func InsertExecutionSafe(db *DB, e *Execution) error { + if err := ValidateExecution(e); err != nil { + return err + } + + // Auto-calculate duration + if e.EndedAt != nil && !e.StartedAt.IsZero() && e.DurationMs == nil { + ms := e.EndedAt.Sub(e.StartedAt).Milliseconds() + e.DurationMs = &ms + } + + return db.InsertExecution(e) +} + +// InsertAssertionSafe validates that the entity exists, then inserts the assertion. +func InsertAssertionSafe(db *DB, a *Assertion) error { + entities, err := buildEntitySet(db) + if err != nil { + return err + } + + if err := ValidateAssertion(a, entities); err != nil { + return err + } + + return db.InsertAssertion(a) +} + +// RecordExecutionWithResults inserts an execution and its assertion results in a transaction. +func RecordExecutionWithResults(db *DB, e *Execution, results []AssertionResult) error { + if err := ValidateExecution(e); err != nil { + return err + } + + // Auto-calculate duration + if e.EndedAt != nil && !e.StartedAt.IsZero() && e.DurationMs == nil { + ms := e.EndedAt.Sub(e.StartedAt).Milliseconds() + e.DurationMs = &ms + } + + tx, err := db.Conn().Begin() + if err != nil { + return fmt.Errorf("beginning transaction: %w", err) + } + defer tx.Rollback() + + // Insert execution + if e.CreatedAt.IsZero() { + e.CreatedAt = time.Now().UTC() + } + var endedAt *string + if e.EndedAt != nil { + s := e.EndedAt.Format(time.RFC3339) + endedAt = &s + } + _, err = tx.Exec(` + INSERT OR REPLACE INTO executions ( + id, pipeline_id, relation_id, status, started_at, ended_at, + duration_ms, records_in, records_out, error, metrics, created_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + e.ID, e.PipelineID, e.RelationID, string(e.Status), + e.StartedAt.Format(time.RFC3339), endedAt, + e.DurationMs, e.RecordsIn, e.RecordsOut, e.Error, + marshalJSON(e.Metrics), e.CreatedAt.Format(time.RFC3339), + ) + if err != nil { + return fmt.Errorf("inserting execution: %w", err) + } + + // Insert assertion results + for _, ar := range results { + if ar.EvaluatedAt.IsZero() { + ar.EvaluatedAt = time.Now().UTC() + } + _, err = tx.Exec(` + INSERT OR REPLACE INTO assertion_results ( + id, assertion_id, execution_id, status, value, message, evaluated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?)`, + ar.ID, ar.AssertionID, ar.ExecutionID, string(ar.Status), + marshalJSON(ar.Value), ar.Message, ar.EvaluatedAt.Format(time.RFC3339), + ) + if err != nil { + return fmt.Errorf("inserting assertion_result: %w", err) + } + } + + return tx.Commit() +} + +// --- Reactive loop --- + +// ReactiveResult holds what the reactive loop did after analyzing assertion results. +type ReactiveResult struct { + EntityUpdates []EntityStatusChange + Proposals []string // IDs of proposals created +} + +// EntityStatusChange records a status transition triggered by assertions. +type EntityStatusChange struct { + EntityID string + OldStatus EntityStatus + NewStatus EntityStatus + Reason string +} + +// React analyzes assertion results and applies the severity rules: +// - critical fail → entity.status = corrupted +// - warning fail → entity.status = stale (only if currently active) +// - info fail → no status change +// +// If registryDB is provided and critical failures exist, a proposal is auto-created. +// Returns what changed so the caller can log or display it. +func React(opsDB *DB, registryDB *registry.DB, results []AssertionResult) (*ReactiveResult, error) { + rr := &ReactiveResult{} + + // Group failures by entity, tracking worst severity per entity. + type entityImpact struct { + worstSeverity Severity + failures []AssertionResult + assertionIDs []string + } + impacts := map[string]*entityImpact{} + + for _, ar := range results { + if ar.Status != ResultFail { + continue + } + + // Look up the assertion to get entity_id and severity + a, err := opsDB.GetAssertion(ar.AssertionID) + if err != nil || a == nil { + continue + } + + imp, ok := impacts[a.EntityID] + if !ok { + imp = &entityImpact{} + impacts[a.EntityID] = imp + } + + imp.failures = append(imp.failures, ar) + imp.assertionIDs = append(imp.assertionIDs, ar.AssertionID) + + // Track worst severity (critical > warning > info) + if severityRank(a.Severity) > severityRank(imp.worstSeverity) { + imp.worstSeverity = a.Severity + } + } + + // Apply status changes per entity + for entityID, imp := range impacts { + entity, err := opsDB.GetEntity(entityID) + if err != nil || entity == nil { + continue + } + + var newStatus EntityStatus + switch imp.worstSeverity { + case SeverityCritical: + newStatus = StatusCorrupted + case SeverityWarning: + // Only degrade active → stale, don't touch corrupted/archived + if entity.Status == StatusActive { + newStatus = StatusStale + } + default: + continue // info: no status change + } + + if newStatus == "" || newStatus == entity.Status { + continue + } + + oldStatus := entity.Status + entity.Status = newStatus + if err := opsDB.UpdateEntity(entity); err != nil { + return nil, fmt.Errorf("updating entity %s status: %w", entityID, err) + } + + reason := fmt.Sprintf("%s assertion(s) failed: %v", imp.worstSeverity, imp.assertionIDs) + rr.EntityUpdates = append(rr.EntityUpdates, EntityStatusChange{ + EntityID: entityID, + OldStatus: oldStatus, + NewStatus: newStatus, + Reason: reason, + }) + } + + // Create proposals in registry for critical failures + if registryDB != nil { + for entityID, imp := range impacts { + if imp.worstSeverity != SeverityCritical { + continue + } + + // Build evidence from failures + failureDetails := make([]map[string]any, 0, len(imp.failures)) + for _, f := range imp.failures { + failureDetails = append(failureDetails, map[string]any{ + "assertion_id": f.AssertionID, + "execution_id": f.ExecutionID, + "message": f.Message, + "value": f.Value, + }) + } + + p := ®istry.Proposal{ + ID: fmt.Sprintf("proposal_react_%s_%d", entityID, time.Now().UnixNano()), + Kind: registry.ProposalImproveFunction, + Title: fmt.Sprintf("Critical assertion failures on entity %s", entityID), + Description: fmt.Sprintf("%d critical assertion(s) failed. Entity marked as corrupted.", len(imp.failures)), + Evidence: map[string]any{ + "entity_id": entityID, + "assertion_ids": imp.assertionIDs, + "failures": failureDetails, + }, + Status: registry.ProposalPending, + CreatedBy: "reactive_loop", + } + + if err := registryDB.InsertProposal(p); err != nil { + return nil, fmt.Errorf("creating proposal for entity %s: %w", entityID, err) + } + + rr.Proposals = append(rr.Proposals, p.ID) + } + } + + return rr, nil +} + +// ExecuteAndReact is the full autonomous loop step: +// 1. Record the execution +// 2. Evaluate all active assertions on affected entities +// 3. React to failures (update entity status, create proposals) +func ExecuteAndReact(opsDB *DB, registryDB *registry.DB, e *Execution, entityIDs []string) (*ReactiveResult, error) { + // 1. Record execution + if err := InsertExecutionSafe(opsDB, e); err != nil { + return nil, fmt.Errorf("recording execution: %w", err) + } + + // 2. Evaluate assertions for each affected entity + var allResults []AssertionResult + for _, entityID := range entityIDs { + results, err := EvalEntityAssertions(opsDB, entityID, e.ID) + if err != nil { + return nil, fmt.Errorf("evaluating assertions for entity %s: %w", entityID, err) + } + allResults = append(allResults, results...) + } + + // 3. React + return React(opsDB, registryDB, allResults) +} + +func severityRank(s Severity) int { + switch s { + case SeverityCritical: + return 3 + case SeverityWarning: + return 2 + case SeverityInfo: + return 1 + default: + return 0 + } +} + func buildEntitySet(db *DB) (map[string]bool, error) { all, err := db.ListEntities("", "") if err != nil { diff --git a/fn_operations/operations_test.go b/fn_operations/operations_test.go index 23b81ec7..3d708984 100644 --- a/fn_operations/operations_test.go +++ b/fn_operations/operations_test.go @@ -4,6 +4,7 @@ import ( "os" "path/filepath" "testing" + "time" ) func tempDB(t *testing.T) *DB { @@ -351,6 +352,479 @@ func TestValidateEntity(t *testing.T) { } } +func TestExecutionCRUD(t *testing.T) { + db := tempDB(t) + + started := time.Date(2026, 3, 28, 10, 0, 0, 0, time.UTC) + ended := time.Date(2026, 3, 28, 10, 5, 0, 0, time.UTC) + rin := int64(1000) + rout := int64(60) + + e := &Execution{ + ID: "exec_1", + PipelineID: "tick_to_ohlcv_go_finance", + RelationID: "rel_1", + Status: ExecSuccess, + StartedAt: started, + EndedAt: &ended, + RecordsIn: &rin, + RecordsOut: &rout, + Metrics: map[string]any{"mean_close": 42000}, + } + + if err := InsertExecutionSafe(db, e); err != nil { + t.Fatalf("insert: %v", err) + } + + // Auto-calculated duration + if e.DurationMs == nil || *e.DurationMs != 300000 { + t.Errorf("duration_ms = %v, want 300000", e.DurationMs) + } + + got, err := db.GetExecution("exec_1") + if err != nil { + t.Fatalf("get: %v", err) + } + if got == nil { + t.Fatal("expected execution, got nil") + } + if got.PipelineID != "tick_to_ohlcv_go_finance" { + t.Errorf("pipeline_id = %q", got.PipelineID) + } + if got.Metrics["mean_close"] != float64(42000) { + t.Errorf("metrics mean_close = %v", got.Metrics["mean_close"]) + } + + // List + all, err := db.ListExecutions("", "", "") + if err != nil { + t.Fatalf("list: %v", err) + } + if len(all) != 1 { + t.Errorf("list = %d, want 1", len(all)) + } + + byPipeline, _ := db.ListExecutions("tick_to_ohlcv_go_finance", "", "") + if len(byPipeline) != 1 { + t.Errorf("list by pipeline = %d, want 1", len(byPipeline)) + } + + byStatus, _ := db.ListExecutions("", "", ExecFailure) + if len(byStatus) != 0 { + t.Errorf("list by failure status = %d, want 0", len(byStatus)) + } +} + +func TestAssertionCRUD(t *testing.T) { + db := tempDB(t) + + db.InsertTypeSnapshot(&TypeSnapshot{ID: "t1", Version: "1.0.0", Lang: "go", Algebraic: "product"}) + db.InsertEntity(&Entity{ID: "e1", Name: "e1", TypeRef: "t1", Status: StatusActive, Source: "test", + Metadata: map[string]any{"close": 120, "high": 150, "low": 90}}) + + a := &Assertion{ + ID: "assert_close_positive", + EntityID: "e1", + Name: "close positivo", + Kind: "range", + Rule: "close > 0", + Severity: SeverityCritical, + Active: true, + } + + if err := InsertAssertionSafe(db, a); err != nil { + t.Fatalf("insert: %v", err) + } + + got, err := db.GetAssertion("assert_close_positive") + if err != nil { + t.Fatalf("get: %v", err) + } + if got == nil { + t.Fatal("expected assertion, got nil") + } + if got.Rule != "close > 0" { + t.Errorf("rule = %q", got.Rule) + } + if !got.Active { + t.Error("expected active = true") + } + + // Update + got.Active = false + if err := db.UpdateAssertion(got); err != nil { + t.Fatalf("update: %v", err) + } + updated, _ := db.GetAssertion("assert_close_positive") + if updated.Active { + t.Error("expected active = false after update") + } + + // List with active filter + active := true + byActive, _ := db.ListAssertions("e1", &active) + if len(byActive) != 0 { + t.Errorf("list active = %d, want 0 (we deactivated it)", len(byActive)) + } + + // Search FTS + updated.Active = true + db.UpdateAssertion(updated) + found, err := db.SearchAssertions("close", "") + if err != nil { + t.Fatalf("search: %v", err) + } + if len(found) != 1 { + t.Errorf("search 'close' = %d, want 1", len(found)) + } + + // Delete + if err := db.DeleteAssertion("assert_close_positive"); err != nil { + t.Fatalf("delete: %v", err) + } + deleted, _ := db.GetAssertion("assert_close_positive") + if deleted != nil { + t.Error("expected nil after delete") + } +} + +func TestAssertionResultCRUD(t *testing.T) { + db := tempDB(t) + + db.InsertTypeSnapshot(&TypeSnapshot{ID: "t1", Version: "1.0.0", Lang: "go", Algebraic: "product"}) + db.InsertEntity(&Entity{ID: "e1", Name: "e1", TypeRef: "t1", Status: StatusActive, Source: "test"}) + db.InsertAssertion(&Assertion{ID: "a1", EntityID: "e1", Name: "test", Kind: "range", Rule: "x > 0", Severity: SeverityWarning, Active: true}) + + ar := &AssertionResult{ + ID: "ar_1", + AssertionID: "a1", + ExecutionID: "exec_1", + Status: ResultFail, + Value: map[string]any{"x": -5}, + Message: "rule failed: x > 0", + EvaluatedAt: time.Now().UTC(), + } + + if err := db.InsertAssertionResult(ar); err != nil { + t.Fatalf("insert: %v", err) + } + + got, err := db.GetAssertionResult("ar_1") + if err != nil { + t.Fatalf("get: %v", err) + } + if got == nil { + t.Fatal("expected result, got nil") + } + if got.Status != ResultFail { + t.Errorf("status = %q, want fail", got.Status) + } + + // List by assertion + byAssertion, _ := db.ListAssertionResults("a1", "") + if len(byAssertion) != 1 { + t.Errorf("list by assertion = %d, want 1", len(byAssertion)) + } + + // List by execution + byExec, _ := db.ListAssertionResults("", "exec_1") + if len(byExec) != 1 { + t.Errorf("list by execution = %d, want 1", len(byExec)) + } +} + +func TestEvalAssertion(t *testing.T) { + db := tempDB(t) + + db.InsertTypeSnapshot(&TypeSnapshot{ID: "t1", Version: "1.0.0", Lang: "go", Algebraic: "product"}) + db.InsertEntity(&Entity{ + ID: "e1", Name: "e1", TypeRef: "t1", Status: StatusActive, Source: "test", + Metadata: map[string]any{"close": 120, "high": 150, "low": 90, "open": 100}, + }) + + tests := []struct { + name string + rule string + kind string + wantStatus AssertionResultStatus + }{ + {"range pass", "close > 0 AND close < 1000000", "range", ResultPass}, + {"range fail", "close > 200", "range", ResultFail}, + {"consistency pass", "low <= close AND close <= high", "consistency", ResultPass}, + {"consistency fail", "low > close", "consistency", ResultFail}, + {"null pass (field exists)", "close IS NOT NULL", "null", ResultPass}, + {"json_extract direct", "json_extract(metadata, '$.close') > 0", "range", ResultPass}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + a := &Assertion{ + ID: "test_" + tt.name, EntityID: "e1", Name: tt.name, + Kind: tt.kind, Rule: tt.rule, Severity: SeverityWarning, Active: true, + } + result, err := EvalAssertion(db, a, "") + if err != nil { + t.Fatalf("eval: %v", err) + } + if result.Status != tt.wantStatus { + t.Errorf("status = %q, want %q (message: %s)", result.Status, tt.wantStatus, result.Message) + } + }) + } +} + +func TestEvalEntityAssertions(t *testing.T) { + db := tempDB(t) + + db.InsertTypeSnapshot(&TypeSnapshot{ID: "t1", Version: "1.0.0", Lang: "go", Algebraic: "product"}) + db.InsertEntity(&Entity{ + ID: "e1", Name: "e1", TypeRef: "t1", Status: StatusActive, Source: "test", + Metadata: map[string]any{"close": 120}, + }) + + db.InsertAssertion(&Assertion{ID: "a1", EntityID: "e1", Name: "pass", Kind: "range", Rule: "close > 0", Severity: SeverityWarning, Active: true}) + db.InsertAssertion(&Assertion{ID: "a2", EntityID: "e1", Name: "fail", Kind: "range", Rule: "close > 200", Severity: SeverityCritical, Active: true}) + db.InsertAssertion(&Assertion{ID: "a3", EntityID: "e1", Name: "inactive", Kind: "range", Rule: "close > 999", Severity: SeverityInfo, Active: false}) + + results, err := EvalEntityAssertions(db, "e1", "exec_test") + if err != nil { + t.Fatalf("eval: %v", err) + } + + // Should only eval active assertions (a1, a2), not a3 + if len(results) != 2 { + t.Fatalf("expected 2 results, got %d", len(results)) + } + + pass, fail := 0, 0 + for _, r := range results { + switch r.Status { + case ResultPass: + pass++ + case ResultFail: + fail++ + } + if r.ExecutionID != "exec_test" { + t.Errorf("execution_id = %q, want exec_test", r.ExecutionID) + } + } + if pass != 1 || fail != 1 { + t.Errorf("pass=%d fail=%d, want 1 and 1", pass, fail) + } + + // Results should be persisted + stored, _ := db.ListAssertionResults("", "exec_test") + if len(stored) != 2 { + t.Errorf("stored results = %d, want 2", len(stored)) + } +} + +func TestReactCriticalCorruptsEntity(t *testing.T) { + db := tempDB(t) + + db.InsertTypeSnapshot(&TypeSnapshot{ID: "t1", Version: "1.0.0", Lang: "go", Algebraic: "product"}) + db.InsertEntity(&Entity{ + ID: "e1", Name: "e1", TypeRef: "t1", Status: StatusActive, Source: "test", + Metadata: map[string]any{"close": 120}, + }) + db.InsertAssertion(&Assertion{ID: "a_crit", EntityID: "e1", Name: "critical_fail", Kind: "range", Rule: "close > 200", Severity: SeverityCritical, Active: true}) + + // Simulate failed assertion result + results := []AssertionResult{ + {ID: "ar1", AssertionID: "a_crit", Status: ResultFail, Message: "close > 200 failed", EvaluatedAt: time.Now().UTC()}, + } + + rr, err := React(db, nil, results) + if err != nil { + t.Fatalf("react: %v", err) + } + + if len(rr.EntityUpdates) != 1 { + t.Fatalf("expected 1 entity update, got %d", len(rr.EntityUpdates)) + } + if rr.EntityUpdates[0].NewStatus != StatusCorrupted { + t.Errorf("new status = %q, want corrupted", rr.EntityUpdates[0].NewStatus) + } + + // Verify entity in DB + entity, _ := db.GetEntity("e1") + if entity.Status != StatusCorrupted { + t.Errorf("entity status = %q, want corrupted", entity.Status) + } +} + +func TestReactWarningStalesEntity(t *testing.T) { + db := tempDB(t) + + db.InsertTypeSnapshot(&TypeSnapshot{ID: "t1", Version: "1.0.0", Lang: "go", Algebraic: "product"}) + db.InsertEntity(&Entity{ + ID: "e1", Name: "e1", TypeRef: "t1", Status: StatusActive, Source: "test", + Metadata: map[string]any{"close": 120}, + }) + db.InsertAssertion(&Assertion{ID: "a_warn", EntityID: "e1", Name: "warn_fail", Kind: "range", Rule: "close > 200", Severity: SeverityWarning, Active: true}) + + results := []AssertionResult{ + {ID: "ar1", AssertionID: "a_warn", Status: ResultFail, Message: "warning", EvaluatedAt: time.Now().UTC()}, + } + + rr, err := React(db, nil, results) + if err != nil { + t.Fatalf("react: %v", err) + } + + if len(rr.EntityUpdates) != 1 { + t.Fatalf("expected 1 entity update, got %d", len(rr.EntityUpdates)) + } + if rr.EntityUpdates[0].NewStatus != StatusStale { + t.Errorf("new status = %q, want stale", rr.EntityUpdates[0].NewStatus) + } +} + +func TestReactWarningDoesNotDowngradeCorrupted(t *testing.T) { + db := tempDB(t) + + db.InsertTypeSnapshot(&TypeSnapshot{ID: "t1", Version: "1.0.0", Lang: "go", Algebraic: "product"}) + db.InsertEntity(&Entity{ + ID: "e1", Name: "e1", TypeRef: "t1", Status: StatusCorrupted, Source: "test", + }) + db.InsertAssertion(&Assertion{ID: "a_warn", EntityID: "e1", Name: "warn", Kind: "range", Rule: "x > 0", Severity: SeverityWarning, Active: true}) + + results := []AssertionResult{ + {ID: "ar1", AssertionID: "a_warn", Status: ResultFail, EvaluatedAt: time.Now().UTC()}, + } + + rr, err := React(db, nil, results) + if err != nil { + t.Fatalf("react: %v", err) + } + + // Warning should NOT change corrupted entity to stale + if len(rr.EntityUpdates) != 0 { + t.Errorf("expected no entity updates, got %d", len(rr.EntityUpdates)) + } +} + +func TestReactInfoNoStatusChange(t *testing.T) { + db := tempDB(t) + + db.InsertTypeSnapshot(&TypeSnapshot{ID: "t1", Version: "1.0.0", Lang: "go", Algebraic: "product"}) + db.InsertEntity(&Entity{ + ID: "e1", Name: "e1", TypeRef: "t1", Status: StatusActive, Source: "test", + }) + db.InsertAssertion(&Assertion{ID: "a_info", EntityID: "e1", Name: "info", Kind: "range", Rule: "x > 0", Severity: SeverityInfo, Active: true}) + + results := []AssertionResult{ + {ID: "ar1", AssertionID: "a_info", Status: ResultFail, EvaluatedAt: time.Now().UTC()}, + } + + rr, err := React(db, nil, results) + if err != nil { + t.Fatalf("react: %v", err) + } + + if len(rr.EntityUpdates) != 0 { + t.Errorf("info fail should not change status, got %d updates", len(rr.EntityUpdates)) + } +} + +func TestValidateExecution(t *testing.T) { + tests := []struct { + name string + e Execution + wantErr bool + }{ + {"valid", Execution{ID: "e1", PipelineID: "p1", Status: ExecSuccess, StartedAt: time.Now()}, false}, + {"missing id", Execution{PipelineID: "p1", Status: ExecSuccess, StartedAt: time.Now()}, true}, + {"missing pipeline", Execution{ID: "e1", Status: ExecSuccess, StartedAt: time.Now()}, true}, + {"missing status", Execution{ID: "e1", PipelineID: "p1", StartedAt: time.Now()}, true}, + {"invalid status", Execution{ID: "e1", PipelineID: "p1", Status: "invalid", StartedAt: time.Now()}, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := ValidateExecution(&tt.e) + if (err != nil) != tt.wantErr { + t.Errorf("error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestValidateAssertion(t *testing.T) { + known := map[string]bool{"e1": true} + tests := []struct { + name string + a Assertion + wantErr bool + }{ + {"valid", Assertion{ID: "a1", EntityID: "e1", Name: "test", Kind: "range", Rule: "x > 0", Severity: SeverityWarning}, false}, + {"missing entity", Assertion{ID: "a1", EntityID: "unknown", Name: "test", Kind: "range", Rule: "x > 0", Severity: SeverityWarning}, true}, + {"missing rule", Assertion{ID: "a1", EntityID: "e1", Name: "test", Kind: "range", Severity: SeverityWarning}, true}, + {"missing kind", Assertion{ID: "a1", EntityID: "e1", Name: "test", Rule: "x > 0", Severity: SeverityWarning}, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := ValidateAssertion(&tt.a, known) + if (err != nil) != tt.wantErr { + t.Errorf("error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestMigrations(t *testing.T) { + db := tempDB(t) + + var count int + err := db.conn.QueryRow("SELECT COUNT(*) FROM schema_migrations").Scan(&count) + if err != nil { + t.Fatalf("query: %v", err) + } + if count < 2 { + t.Errorf("expected at least 2 migrations, got %d", count) + } + + // Verify all new tables exist + for _, table := range []string{"executions", "assertions", "assertion_results"} { + _, err := db.conn.Exec("SELECT 1 FROM " + table + " LIMIT 1") + if err != nil { + t.Errorf("table %s should exist: %v", table, err) + } + } +} + +func TestRecordExecutionWithResults(t *testing.T) { + db := tempDB(t) + + db.InsertTypeSnapshot(&TypeSnapshot{ID: "t1", Version: "1.0.0", Lang: "go", Algebraic: "product"}) + db.InsertEntity(&Entity{ID: "e1", Name: "e1", TypeRef: "t1", Status: StatusActive, Source: "test"}) + db.InsertAssertion(&Assertion{ID: "a1", EntityID: "e1", Name: "test", Kind: "range", Rule: "x > 0", Severity: SeverityWarning, Active: true}) + + started := time.Now().UTC() + ended := started.Add(5 * time.Second) + e := &Execution{ + ID: "exec_tx", PipelineID: "p1", Status: ExecSuccess, + StartedAt: started, EndedAt: &ended, + } + results := []AssertionResult{ + {ID: "ar_tx_1", AssertionID: "a1", ExecutionID: "exec_tx", Status: ResultPass, EvaluatedAt: time.Now().UTC()}, + } + + if err := RecordExecutionWithResults(db, e, results); err != nil { + t.Fatalf("record: %v", err) + } + + // Verify both were persisted + gotExec, _ := db.GetExecution("exec_tx") + if gotExec == nil { + t.Fatal("execution not found") + } + + gotResult, _ := db.GetAssertionResult("ar_tx_1") + if gotResult == nil { + t.Fatal("assertion result not found") + } +} + func TestGetEntityGraph(t *testing.T) { db := tempDB(t) diff --git a/fn_operations/store.go b/fn_operations/store.go index ed1f47c3..592b8e7d 100644 --- a/fn_operations/store.go +++ b/fn_operations/store.go @@ -471,3 +471,346 @@ func (db *DB) DeleteRelationInputs(relationID string) error { _, err := db.conn.Exec("DELETE FROM relation_inputs WHERE relation_id = ?", relationID) return err } + +// --- Execution CRUD --- + +// InsertExecution inserts an execution record. +func (db *DB) InsertExecution(e *Execution) error { + if e.CreatedAt.IsZero() { + e.CreatedAt = time.Now().UTC() + } + + var endedAt *string + if e.EndedAt != nil { + s := e.EndedAt.Format(time.RFC3339) + endedAt = &s + } + + _, err := db.conn.Exec(` + INSERT OR REPLACE INTO executions ( + id, pipeline_id, relation_id, status, started_at, ended_at, + duration_ms, records_in, records_out, error, metrics, created_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + e.ID, e.PipelineID, e.RelationID, string(e.Status), + e.StartedAt.Format(time.RFC3339), endedAt, + e.DurationMs, e.RecordsIn, e.RecordsOut, e.Error, + marshalJSON(e.Metrics), e.CreatedAt.Format(time.RFC3339), + ) + return err +} + +// GetExecution returns an execution by ID. +func (db *DB) GetExecution(id string) (*Execution, error) { + row := db.conn.QueryRow(` + SELECT id, pipeline_id, relation_id, status, started_at, ended_at, + duration_ms, records_in, records_out, error, metrics, created_at + FROM executions WHERE id = ?`, id) + + var e Execution + var metricsJSON, createdAt, startedAt string + var endedAt *string + err := row.Scan(&e.ID, &e.PipelineID, &e.RelationID, &e.Status, + &startedAt, &endedAt, + &e.DurationMs, &e.RecordsIn, &e.RecordsOut, &e.Error, + &metricsJSON, &createdAt) + if err == sql.ErrNoRows { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("scanning execution: %w", err) + } + e.Metrics = unmarshalJSON(metricsJSON) + e.StartedAt, _ = time.Parse(time.RFC3339, startedAt) + e.CreatedAt, _ = time.Parse(time.RFC3339, createdAt) + if endedAt != nil { + t, _ := time.Parse(time.RFC3339, *endedAt) + e.EndedAt = &t + } + return &e, nil +} + +// ListExecutions returns executions filtered by pipeline, relation, and/or status. +func (db *DB) ListExecutions(pipelineID, relationID string, status ExecutionStatus) ([]Execution, error) { + where := []string{} + args := []any{} + if pipelineID != "" { + where = append(where, "pipeline_id = ?") + args = append(args, pipelineID) + } + if relationID != "" { + where = append(where, "relation_id = ?") + args = append(args, relationID) + } + if status != "" { + where = append(where, "status = ?") + args = append(args, string(status)) + } + + q := `SELECT id, pipeline_id, relation_id, status, started_at, ended_at, + duration_ms, records_in, records_out, error, metrics, created_at + FROM executions` + if len(where) > 0 { + q += " WHERE " + strings.Join(where, " AND ") + } + q += " ORDER BY created_at DESC" + + rows, err := db.conn.Query(q, args...) + if err != nil { + return nil, err + } + defer rows.Close() + return scanExecutions(rows) +} + +func scanExecutions(rows *sql.Rows) ([]Execution, error) { + var result []Execution + for rows.Next() { + var e Execution + var metricsJSON, createdAt, startedAt string + var endedAt *string + if err := rows.Scan(&e.ID, &e.PipelineID, &e.RelationID, &e.Status, + &startedAt, &endedAt, + &e.DurationMs, &e.RecordsIn, &e.RecordsOut, &e.Error, + &metricsJSON, &createdAt); err != nil { + return nil, fmt.Errorf("scanning execution: %w", err) + } + e.Metrics = unmarshalJSON(metricsJSON) + e.StartedAt, _ = time.Parse(time.RFC3339, startedAt) + e.CreatedAt, _ = time.Parse(time.RFC3339, createdAt) + if endedAt != nil { + t, _ := time.Parse(time.RFC3339, *endedAt) + e.EndedAt = &t + } + result = append(result, e) + } + return result, nil +} + +// --- Assertion CRUD --- + +// InsertAssertion inserts or replaces an assertion. +func (db *DB) InsertAssertion(a *Assertion) error { + if a.CreatedAt.IsZero() { + a.CreatedAt = time.Now().UTC() + } + + active := 0 + if a.Active { + active = 1 + } + + _, err := db.conn.Exec(` + INSERT OR REPLACE INTO assertions ( + id, entity_id, name, kind, rule, severity, description, active, created_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, + a.ID, a.EntityID, a.Name, a.Kind, a.Rule, + string(a.Severity), a.Description, active, + a.CreatedAt.Format(time.RFC3339), + ) + return err +} + +// GetAssertion returns an assertion by ID. +func (db *DB) GetAssertion(id string) (*Assertion, error) { + row := db.conn.QueryRow(` + SELECT id, entity_id, name, kind, rule, severity, description, active, created_at + FROM assertions WHERE id = ?`, id) + + var a Assertion + var active int + var createdAt string + err := row.Scan(&a.ID, &a.EntityID, &a.Name, &a.Kind, &a.Rule, + &a.Severity, &a.Description, &active, &createdAt) + if err == sql.ErrNoRows { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("scanning assertion: %w", err) + } + a.Active = active == 1 + a.CreatedAt, _ = time.Parse(time.RFC3339, createdAt) + return &a, nil +} + +// UpdateAssertion updates an existing assertion. +func (db *DB) UpdateAssertion(a *Assertion) error { + active := 0 + if a.Active { + active = 1 + } + + _, err := db.conn.Exec(` + UPDATE assertions SET entity_id=?, name=?, kind=?, rule=?, severity=?, + description=?, active=? + WHERE id=?`, + a.EntityID, a.Name, a.Kind, a.Rule, string(a.Severity), + a.Description, active, a.ID, + ) + return err +} + +// DeleteAssertion removes an assertion by ID. +func (db *DB) DeleteAssertion(id string) error { + _, err := db.conn.Exec("DELETE FROM assertions WHERE id = ?", id) + return err +} + +// ListAssertions returns assertions filtered by entity and/or active state. +func (db *DB) ListAssertions(entityID string, active *bool) ([]Assertion, error) { + where := []string{} + args := []any{} + if entityID != "" { + where = append(where, "entity_id = ?") + args = append(args, entityID) + } + if active != nil { + v := 0 + if *active { + v = 1 + } + where = append(where, "active = ?") + args = append(args, v) + } + + q := `SELECT id, entity_id, name, kind, rule, severity, description, active, created_at + FROM assertions` + if len(where) > 0 { + q += " WHERE " + strings.Join(where, " AND ") + } + q += " ORDER BY name" + + rows, err := db.conn.Query(q, args...) + if err != nil { + return nil, err + } + defer rows.Close() + return scanAssertions(rows) +} + +// SearchAssertions performs FTS search on assertions. +func (db *DB) SearchAssertions(query, entityID string) ([]Assertion, error) { + where := []string{} + args := []any{} + if query != "" { + where = append(where, "a.id IN (SELECT id FROM assertions_fts WHERE assertions_fts MATCH ?)") + args = append(args, query) + } + if entityID != "" { + where = append(where, "a.entity_id = ?") + args = append(args, entityID) + } + + q := `SELECT a.id, a.entity_id, a.name, a.kind, a.rule, a.severity, a.description, a.active, a.created_at + FROM assertions a` + if len(where) > 0 { + q += " WHERE " + strings.Join(where, " AND ") + } + q += " ORDER BY a.name" + + rows, err := db.conn.Query(q, args...) + if err != nil { + return nil, err + } + defer rows.Close() + return scanAssertions(rows) +} + +func scanAssertions(rows *sql.Rows) ([]Assertion, error) { + var result []Assertion + for rows.Next() { + var a Assertion + var active int + var createdAt string + if err := rows.Scan(&a.ID, &a.EntityID, &a.Name, &a.Kind, &a.Rule, + &a.Severity, &a.Description, &active, &createdAt); err != nil { + return nil, fmt.Errorf("scanning assertion: %w", err) + } + a.Active = active == 1 + a.CreatedAt, _ = time.Parse(time.RFC3339, createdAt) + result = append(result, a) + } + return result, nil +} + +// --- AssertionResult CRUD --- + +// InsertAssertionResult inserts an assertion result. +func (db *DB) InsertAssertionResult(ar *AssertionResult) error { + if ar.EvaluatedAt.IsZero() { + ar.EvaluatedAt = time.Now().UTC() + } + + _, err := db.conn.Exec(` + INSERT OR REPLACE INTO assertion_results ( + id, assertion_id, execution_id, status, value, message, evaluated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?)`, + ar.ID, ar.AssertionID, ar.ExecutionID, string(ar.Status), + marshalJSON(ar.Value), ar.Message, ar.EvaluatedAt.Format(time.RFC3339), + ) + return err +} + +// GetAssertionResult returns an assertion result by ID. +func (db *DB) GetAssertionResult(id string) (*AssertionResult, error) { + row := db.conn.QueryRow(` + SELECT id, assertion_id, execution_id, status, value, message, evaluated_at + FROM assertion_results WHERE id = ?`, id) + + var ar AssertionResult + var valueJSON, evaluatedAt string + err := row.Scan(&ar.ID, &ar.AssertionID, &ar.ExecutionID, &ar.Status, + &valueJSON, &ar.Message, &evaluatedAt) + if err == sql.ErrNoRows { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("scanning assertion_result: %w", err) + } + ar.Value = unmarshalJSON(valueJSON) + ar.EvaluatedAt, _ = time.Parse(time.RFC3339, evaluatedAt) + return &ar, nil +} + +// ListAssertionResults returns results filtered by assertion and/or execution. +func (db *DB) ListAssertionResults(assertionID, executionID string) ([]AssertionResult, error) { + where := []string{} + args := []any{} + if assertionID != "" { + where = append(where, "assertion_id = ?") + args = append(args, assertionID) + } + if executionID != "" { + where = append(where, "execution_id = ?") + args = append(args, executionID) + } + + q := `SELECT id, assertion_id, execution_id, status, value, message, evaluated_at + FROM assertion_results` + if len(where) > 0 { + q += " WHERE " + strings.Join(where, " AND ") + } + q += " ORDER BY evaluated_at DESC" + + rows, err := db.conn.Query(q, args...) + if err != nil { + return nil, err + } + defer rows.Close() + return scanAssertionResults(rows) +} + +func scanAssertionResults(rows *sql.Rows) ([]AssertionResult, error) { + var result []AssertionResult + for rows.Next() { + var ar AssertionResult + var valueJSON, evaluatedAt string + if err := rows.Scan(&ar.ID, &ar.AssertionID, &ar.ExecutionID, &ar.Status, + &valueJSON, &ar.Message, &evaluatedAt); err != nil { + return nil, fmt.Errorf("scanning assertion_result: %w", err) + } + ar.Value = unmarshalJSON(valueJSON) + ar.EvaluatedAt, _ = time.Parse(time.RFC3339, evaluatedAt) + result = append(result, ar) + } + return result, nil +} diff --git a/fn_operations/validate.go b/fn_operations/validate.go index 5eb759a0..6719c879 100644 --- a/fn_operations/validate.go +++ b/fn_operations/validate.go @@ -130,6 +130,100 @@ func ValidateRelationInputs(inputs []RelationInput, knownEntities map[string]boo return nil } +// ValidateExecution checks execution integrity rules. +func ValidateExecution(e *Execution) *ValidationError { + var errs []string + + if e.ID == "" { + errs = append(errs, "id is required") + } + if e.PipelineID == "" { + errs = append(errs, "pipeline_id is required") + } + + switch e.Status { + case ExecSuccess, ExecFailure, ExecPartial: + case "": + errs = append(errs, "status is required") + default: + errs = append(errs, fmt.Sprintf("invalid status: %s", e.Status)) + } + + if e.StartedAt.IsZero() { + errs = append(errs, "started_at is required") + } + + if len(errs) > 0 { + return &ValidationError{ID: e.ID, Errors: errs} + } + return nil +} + +// ValidateAssertion checks assertion integrity rules. +func ValidateAssertion(a *Assertion, knownEntities map[string]bool) *ValidationError { + var errs []string + + if a.ID == "" { + errs = append(errs, "id is required") + } + if a.EntityID == "" { + errs = append(errs, "entity_id is required") + } else if knownEntities != nil && !knownEntities[a.EntityID] { + errs = append(errs, fmt.Sprintf("entity_id references unknown entity: %s", a.EntityID)) + } + if a.Name == "" { + errs = append(errs, "name is required") + } + if a.Kind == "" { + errs = append(errs, "kind is required") + } + if a.Rule == "" { + errs = append(errs, "rule is required") + } + + switch a.Severity { + case SeverityCritical, SeverityWarning, SeverityInfo: + case "": + errs = append(errs, "severity is required") + default: + errs = append(errs, fmt.Sprintf("invalid severity: %s", a.Severity)) + } + + if len(errs) > 0 { + return &ValidationError{ID: a.ID, Errors: errs} + } + return nil +} + +// ValidateAssertionResult checks assertion result integrity. +func ValidateAssertionResult(ar *AssertionResult) *ValidationError { + var errs []string + + if ar.ID == "" { + errs = append(errs, "id is required") + } + if ar.AssertionID == "" { + errs = append(errs, "assertion_id is required") + } + + switch ar.Status { + case ResultPass, ResultFail, ResultSkip: + case "": + errs = append(errs, "status is required") + default: + errs = append(errs, fmt.Sprintf("invalid status: %s", ar.Status)) + } + + if ar.EvaluatedAt.IsZero() { + errs = append(errs, "evaluated_at is required") + } + + if len(errs) > 0 { + return &ValidationError{ID: ar.ID, Errors: errs} + } + return nil +} + // DetectCycle checks if adding a causal relation (from -> to) creates a cycle. // Only considers relations where via != "" (causal/transformational). // Semantic relations (via == "") are exempt from cycle detection. diff --git a/functions/infra/docker_create_network.go b/functions/infra/docker_create_network.go new file mode 100644 index 00000000..d72b6c63 --- /dev/null +++ b/functions/infra/docker_create_network.go @@ -0,0 +1,26 @@ +package infra + +import ( + "fmt" + "os/exec" + "strings" +) + +// DockerCreateNetwork crea una red Docker con el nombre y driver dados. +// Si el driver está vacío, usa "bridge" por defecto. +// Devuelve el ID de la red creada. +func DockerCreateNetwork(name, driver string) (string, error) { + if name == "" { + return "", fmt.Errorf("network name required") + } + if driver == "" { + driver = "bridge" + } + + out, err := exec.Command("docker", "network", "create", "--driver", driver, name).CombinedOutput() + if err != nil { + return "", fmt.Errorf("docker network create %s: %s", name, strings.TrimSpace(string(out))) + } + + return strings.TrimSpace(string(out)), nil +} diff --git a/functions/infra/docker_create_network.md b/functions/infra/docker_create_network.md new file mode 100644 index 00000000..1ddd2b31 --- /dev/null +++ b/functions/infra/docker_create_network.md @@ -0,0 +1,31 @@ +--- +name: docker_create_network +kind: function +lang: go +domain: infra +version: "1.0.0" +purity: impure +signature: "func DockerCreateNetwork(name, driver string) (string, error)" +description: "Crea una red Docker con el nombre y driver dados. Si driver está vacío usa bridge por defecto. Devuelve el ID de la red creada." +tags: [docker, network, create, infra] +uses_functions: [] +uses_types: [] +returns: [] +returns_optional: false +error_type: "error_go_core" +imports: [fmt, os/exec, strings] +tested: false +tests: [] +test_file_path: "" +file_path: "functions/infra/docker_create_network.go" +--- + +## Ejemplo + +```go +netID, err := DockerCreateNetwork("my-app-net", "bridge") +if err != nil { + log.Fatal(err) +} +fmt.Println("Network ID:", netID) +``` diff --git a/functions/infra/docker_remove_network.go b/functions/infra/docker_remove_network.go new file mode 100644 index 00000000..6941e5eb --- /dev/null +++ b/functions/infra/docker_remove_network.go @@ -0,0 +1,16 @@ +package infra + +import ( + "fmt" + "os/exec" + "strings" +) + +// DockerRemoveNetwork elimina una red Docker por nombre o ID. +func DockerRemoveNetwork(nameOrID string) error { + out, err := exec.Command("docker", "network", "rm", nameOrID).CombinedOutput() + if err != nil { + return fmt.Errorf("docker network rm %s: %s", nameOrID, strings.TrimSpace(string(out))) + } + return nil +} diff --git a/functions/infra/docker_remove_network.md b/functions/infra/docker_remove_network.md new file mode 100644 index 00000000..bf497443 --- /dev/null +++ b/functions/infra/docker_remove_network.md @@ -0,0 +1,30 @@ +--- +name: docker_remove_network +kind: function +lang: go +domain: infra +version: "1.0.0" +purity: impure +signature: "func DockerRemoveNetwork(nameOrID string) error" +description: "Elimina una red Docker por nombre o ID." +tags: [docker, network, remove, infra] +uses_functions: [] +uses_types: [] +returns: [] +returns_optional: false +error_type: "error_go_core" +imports: [fmt, os/exec, strings] +tested: false +tests: [] +test_file_path: "" +file_path: "functions/infra/docker_remove_network.go" +--- + +## Ejemplo + +```go +err := DockerRemoveNetwork("my-app-net") +if err != nil { + log.Fatal(err) +} +``` diff --git a/functions/pipelines/docker_tui.go b/functions/pipelines/docker_tui.go index 8eb5d6a8..d5238c7f 100644 --- a/functions/pipelines/docker_tui.go +++ b/functions/pipelines/docker_tui.go @@ -6,4 +6,4 @@ package pipelines // Vistas: Containers, Images, Volumes, Networks, Compose. // Arquitectura: Elm (Model-Update-View) con tabs y state machines. // -// Implementation: fn_operations/docker_tui/main.go +// Implementation: apps/docker_tui/main.go diff --git a/functions/pipelines/docker_tui.md b/functions/pipelines/docker_tui.md index 04af1bba..bfe5b9ad 100644 --- a/functions/pipelines/docker_tui.md +++ b/functions/pipelines/docker_tui.md @@ -43,13 +43,13 @@ imports: tested: false tests: [] test_file_path: "" -file_path: "fn_operations/docker_tui/main.go" +file_path: "apps/docker_tui/main.go" --- ## Ejemplo ```bash -cd fn_operations/docker_tui && go run . +cd apps/docker_tui && go run . ``` ## Notas diff --git a/functions/pipelines/init_metabase.go b/functions/pipelines/init_metabase.go new file mode 100644 index 00000000..c421e0ae --- /dev/null +++ b/functions/pipelines/init_metabase.go @@ -0,0 +1,12 @@ +package pipelines + +// InitMetabase despliega un stack Metabase + Postgres en Docker. +// +// Pasos: +// 1. Crear red Docker compartida +// 2. Pull de imágenes postgres:16 y metabase/metabase:latest +// 3. Iniciar Postgres con volume persistente +// 4. Esperar a que Postgres acepte conexiones (health check con retry) +// 5. Iniciar Metabase conectado a Postgres +// +// Implementation: functions/pipelines/init_metabase/main.go diff --git a/functions/pipelines/init_metabase.md b/functions/pipelines/init_metabase.md new file mode 100644 index 00000000..76f620d0 --- /dev/null +++ b/functions/pipelines/init_metabase.md @@ -0,0 +1,72 @@ +--- +name: init_metabase +kind: pipeline +lang: go +domain: infra +version: "1.0.0" +purity: impure +signature: "func main() — Despliega stack Metabase + Postgres en Docker" +description: "Pipeline que inicializa un contenedor Metabase con su base de datos Postgres. Crea red Docker, pull de imágenes, inicia Postgres con volume persistente, espera health check y lanza Metabase conectado." +tags: [docker, metabase, postgres, pipeline, infra, analytics] +uses_functions: + - docker_create_network_go_infra + - docker_pull_image_go_infra + - docker_run_container_go_infra + - docker_inspect_container_go_infra + - retry_with_backoff_go_core +uses_types: + - container_info_go_infra + - network_go_docker +returns: [] +returns_optional: false +error_type: "error_go_core" +imports: [os/exec, encoding/json] +tested: false +tests: [] +test_file_path: "" +file_path: "functions/pipelines/init_metabase/main.go" +--- + +## Ejemplo + +```bash +go run functions/pipelines/init_metabase/main.go \ + --project analytics \ + --metabase-port 3000 \ + --pg-port 5432 \ + --pg-user metabase \ + --pg-password metabase \ + --pg-database metabase +``` + +Salida JSON: +```json +{ + "network_id": "abc123...", + "postgres_id": "def456...", + "metabase_id": "ghi789...", + "network_name": "analytics-net", + "postgres_name": "analytics-postgres", + "metabase_name": "analytics-metabase", + "metabase_url": "http://localhost:3000" +} +``` + +## Notas + +El pipeline orquesta 5 pasos secuenciales: + +1. **Red Docker** — crea `{project}-net` con driver bridge +2. **Pull** — descarga `postgres:16` y `metabase/metabase:latest` +3. **Postgres** — inicia con volume persistente (named volume por defecto o bind mount con `--pg-volume`) +4. **Health check** — retry exponencial (hasta ~34 min) con `pg_isready` dentro del contenedor +5. **Metabase** — conecta a Postgres via red interna, expone en puerto configurable + +Reutiliza conceptualmente `docker_create_network`, `docker_pull_image`, `docker_run_container`, `docker_inspect_container` y `retry_with_backoff`, reimplementadas inline por ser un ejecutable independiente. + +Para destruir el stack: +```bash +docker stop analytics-metabase analytics-postgres +docker rm analytics-metabase analytics-postgres +docker network rm analytics-net +``` diff --git a/functions/pipelines/init_metabase/main.go b/functions/pipelines/init_metabase/main.go new file mode 100644 index 00000000..e061cc26 --- /dev/null +++ b/functions/pipelines/init_metabase/main.go @@ -0,0 +1,176 @@ +package main + +import ( + "encoding/json" + "flag" + "fmt" + "os" + "os/exec" + "strings" + "time" +) + +// MetabaseResult contiene los IDs y nombres de los recursos creados. +type MetabaseResult struct { + NetworkID string `json:"network_id"` + PostgresID string `json:"postgres_id"` + MetabaseID string `json:"metabase_id"` + NetworkName string `json:"network_name"` + PostgresName string `json:"postgres_name"` + MetabaseName string `json:"metabase_name"` + MetabaseURL string `json:"metabase_url"` +} + +func main() { + project := flag.String("project", "", "Prefijo para nombres de contenedores y red (requerido)") + mbPort := flag.String("metabase-port", "3000", "Puerto host para Metabase") + pgPort := flag.String("pg-port", "5432", "Puerto host para Postgres") + pgUser := flag.String("pg-user", "metabase", "Usuario Postgres") + pgPass := flag.String("pg-password", "metabase", "Password Postgres") + pgDB := flag.String("pg-database", "metabase", "Base de datos Postgres") + pgVolume := flag.String("pg-volume", "", "Path host para persistencia (default: docker named volume)") + flag.Parse() + + if *project == "" { + fmt.Fprintln(os.Stderr, "error: --project requerido") + flag.Usage() + os.Exit(1) + } + + result, err := initMetabase(*project, *mbPort, *pgPort, *pgUser, *pgPass, *pgDB, *pgVolume) + if err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + os.Exit(1) + } + + enc := json.NewEncoder(os.Stdout) + enc.SetIndent("", " ") + enc.Encode(result) +} + +func initMetabase(project, mbPort, pgPort, pgUser, pgPass, pgDB, pgVolume string) (*MetabaseResult, error) { + networkName := project + "-net" + pgName := project + "-postgres" + mbName := project + "-metabase" + + // 1. Crear red + fmt.Fprintf(os.Stderr, "[1/5] Creando red %s...\n", networkName) + netID, err := dockerCreateNetwork(networkName, "bridge") + if err != nil { + return nil, fmt.Errorf("creating network: %w", err) + } + + // 2. Pull imágenes + for i, img := range []string{"postgres:16", "metabase/metabase:latest"} { + fmt.Fprintf(os.Stderr, "[2/5] Pulling %s (%d/2)...\n", img, i+1) + if err := dockerPull(img); err != nil { + return nil, fmt.Errorf("pulling %s: %w", img, err) + } + } + + // 3. Iniciar Postgres + fmt.Fprintf(os.Stderr, "[3/5] Iniciando Postgres %s...\n", pgName) + vol := pgName + "-data:/var/lib/postgresql/data" + if pgVolume != "" { + vol = pgVolume + ":/var/lib/postgresql/data" + } + + pgArgs := []string{ + "run", "-d", + "--name", pgName, + "--network", networkName, + "-p", pgPort + ":5432", + "-e", "POSTGRES_USER=" + pgUser, + "-e", "POSTGRES_PASSWORD=" + pgPass, + "-e", "POSTGRES_DB=" + pgDB, + "-v", vol, + "postgres:16", + } + pgID, err := dockerCmd(pgArgs...) + if err != nil { + return nil, fmt.Errorf("starting postgres: %w", err) + } + + // 4. Health check con retry exponencial + fmt.Fprintf(os.Stderr, "[4/5] Esperando a que Postgres esté listo...\n") + err = retryWithBackoff(func() error { + out, execErr := dockerCmd("exec", pgName, "pg_isready", "-U", pgUser) + if execErr != nil { + return fmt.Errorf("pg_isready: %s", out) + } + return nil + }, 10, 2*time.Second) + if err != nil { + return nil, fmt.Errorf("waiting for postgres: %w", err) + } + fmt.Fprintf(os.Stderr, " Postgres listo.\n") + + // 5. Iniciar Metabase + fmt.Fprintf(os.Stderr, "[5/5] Iniciando Metabase %s...\n", mbName) + mbArgs := []string{ + "run", "-d", + "--name", mbName, + "--network", networkName, + "-p", mbPort + ":3000", + "-e", "MB_DB_TYPE=postgres", + "-e", "MB_DB_DBNAME=" + pgDB, + "-e", "MB_DB_PORT=5432", + "-e", "MB_DB_USER=" + pgUser, + "-e", "MB_DB_PASS=" + pgPass, + "-e", "MB_DB_HOST=" + pgName, + "metabase/metabase:latest", + } + mbID, err := dockerCmd(mbArgs...) + if err != nil { + return nil, fmt.Errorf("starting metabase: %w", err) + } + + mbURL := fmt.Sprintf("http://localhost:%s", mbPort) + fmt.Fprintf(os.Stderr, "\nStack listo. Metabase disponible en %s\n", mbURL) + + return &MetabaseResult{ + NetworkID: netID, + PostgresID: pgID, + MetabaseID: mbID, + NetworkName: networkName, + PostgresName: pgName, + MetabaseName: mbName, + MetabaseURL: mbURL, + }, nil +} + +// --- Funciones Docker (reimplementan las del registry sin imports internos) --- + +func dockerCreateNetwork(name, driver string) (string, error) { + return dockerCmd("network", "create", "--driver", driver, name) +} + +func dockerPull(image string) error { + _, err := dockerCmd("pull", image) + return err +} + +func dockerCmd(args ...string) (string, error) { + out, err := exec.Command("docker", args...).CombinedOutput() + if err != nil { + return "", fmt.Errorf("docker %s: %s", args[0], strings.TrimSpace(string(out))) + } + return strings.TrimSpace(string(out)), nil +} + +// retryWithBackoff reimplementa RetryWithBackoff de core para funciones () error. +func retryWithBackoff(fn func() error, maxRetries int, baseDelay time.Duration) error { + var lastErr error + for attempt := 0; attempt <= maxRetries; attempt++ { + if err := fn(); err == nil { + return nil + } else { + lastErr = err + } + if attempt < maxRetries { + delay := baseDelay * (1 << uint(attempt)) + time.Sleep(delay) + } + } + return fmt.Errorf("all %d retries exhausted: %w", maxRetries+1, lastErr) +} diff --git a/functions/pipelines/pipeline_launcher.go b/functions/pipelines/pipeline_launcher.go new file mode 100644 index 00000000..6ab02d5a --- /dev/null +++ b/functions/pipelines/pipeline_launcher.go @@ -0,0 +1,8 @@ +package pipelines + +// PipelineLauncher es una TUI fullscreen para listar, lanzar y monitorear pipelines. +// +// Tabs: Pipelines (lista filtrable + lanzamiento) y History (historial de ejecuciones). +// Registra cada ejecución en operations.db usando fn_operations. +// +// Implementation: apps/pipeline_launcher/main.go diff --git a/functions/pipelines/pipeline_launcher.md b/functions/pipelines/pipeline_launcher.md new file mode 100644 index 00000000..96365c69 --- /dev/null +++ b/functions/pipelines/pipeline_launcher.md @@ -0,0 +1,47 @@ +--- +name: pipeline_launcher +kind: pipeline +lang: go +domain: infra +version: "1.0.0" +purity: impure +signature: "func main() — TUI fullscreen para lanzar pipelines y registrar ejecuciones" +description: "TUI interactiva que lista pipelines del registry, permite lanzarlos como subprocesos y registra cada ejecución en operations.db. Dos tabs: Pipelines (filtro + launch) y History (historial)." +tags: [tui, pipeline, launcher, infra, operations, bubbletea, devfactory] +uses_functions: + - docker_tui_go_infra +uses_types: + - base_model_go_tui + - filtered_list_model_go_tui + - spinner_model_go_tui + - styles_go_tui +returns: [] +returns_optional: false +error_type: "error_go_core" +imports: + - github.com/charmbracelet/bubbletea + - github.com/lucasdataproyects/devfactory + - fn-registry/registry + - fn-registry/fn_operations +tested: false +tests: [] +test_file_path: "" +file_path: "apps/pipeline_launcher/main.go" +--- + +## Ejemplo + +```bash +FN_REGISTRY_ROOT=/home/lucas/fn_registry go run apps/pipeline_launcher/main.go +``` + +## Notas + +TUI con dos tabs: + +- **Pipelines**: lista filtrable de todos los pipelines del registry. Enter lanza el pipeline seleccionado como subproceso (`go run .`), muestra spinner durante la ejecución, y al terminar muestra stdout/stderr y registra la ejecución en operations.db. +- **History**: historial de ejecuciones pasadas con status, duración y timestamp. Enter muestra detalle completo. + +Teclas: Tab/Shift+Tab cambia vista, Enter ejecuta acción, r refresca, / filtra, j/k scroll, Esc vuelve. + +Requiere `FN_REGISTRY_ROOT` apuntando a la raíz del fn-registry para encontrar registry.db y los file_path de los pipelines. diff --git a/launcher.sh b/launcher.sh new file mode 100755 index 00000000..8d24ff4e --- /dev/null +++ b/launcher.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +export FN_REGISTRY_ROOT="$(cd "$(dirname "$0")" && pwd)" +cd "$FN_REGISTRY_ROOT/apps/pipeline_launcher" +CGO_ENABLED=1 exec go run -tags fts5 . diff --git a/registry/db.go b/registry/db.go index d902ec72..d7f9586f 100644 --- a/registry/db.go +++ b/registry/db.go @@ -9,112 +9,6 @@ import ( _ "github.com/mattn/go-sqlite3" ) -const schemaSQL = ` -CREATE TABLE IF NOT EXISTS functions ( - id TEXT PRIMARY KEY, - name TEXT NOT NULL, - kind TEXT NOT NULL CHECK(kind IN ('function','pipeline','component')), - lang TEXT NOT NULL, - domain TEXT NOT NULL, - version TEXT NOT NULL DEFAULT '1.0.0', - purity TEXT NOT NULL CHECK(purity IN ('pure','impure')), - signature TEXT NOT NULL DEFAULT '', - description TEXT NOT NULL, - tags TEXT NOT NULL DEFAULT '[]', - uses_functions TEXT NOT NULL DEFAULT '[]', - uses_types TEXT NOT NULL DEFAULT '[]', - returns TEXT NOT NULL DEFAULT '[]', - returns_optional INTEGER NOT NULL DEFAULT 0, - error_type TEXT NOT NULL DEFAULT '', - imports TEXT NOT NULL DEFAULT '[]', - example TEXT NOT NULL DEFAULT '', - tested INTEGER NOT NULL DEFAULT 0, - tests TEXT NOT NULL DEFAULT '[]', - test_file_path TEXT NOT NULL DEFAULT '', - file_path TEXT NOT NULL DEFAULT '', - created_at TEXT NOT NULL, - updated_at TEXT NOT NULL, - -- Component fields - props TEXT NOT NULL DEFAULT '[]', - emits TEXT NOT NULL DEFAULT '[]', - has_state INTEGER, - framework TEXT NOT NULL DEFAULT '', - variant TEXT NOT NULL DEFAULT '[]' -); - -CREATE TABLE IF NOT EXISTS types ( - id TEXT PRIMARY KEY, - name TEXT NOT NULL, - lang TEXT NOT NULL, - domain TEXT NOT NULL, - version TEXT NOT NULL DEFAULT '1.0.0', - algebraic TEXT NOT NULL CHECK(algebraic IN ('product','sum')), - definition TEXT NOT NULL DEFAULT '', - description TEXT NOT NULL, - tags TEXT NOT NULL DEFAULT '[]', - uses_types TEXT NOT NULL DEFAULT '[]', - file_path TEXT NOT NULL DEFAULT '', - created_at TEXT NOT NULL, - updated_at TEXT NOT NULL -); - -CREATE VIRTUAL TABLE IF NOT EXISTS functions_fts USING fts5( - id, - name, - description, - tags, - signature, - domain, - content='functions', - content_rowid='rowid' -); - -CREATE VIRTUAL TABLE IF NOT EXISTS types_fts USING fts5( - id, - name, - description, - tags, - domain, - content='types', - content_rowid='rowid' -); - --- Triggers to keep FTS in sync -CREATE TRIGGER IF NOT EXISTS functions_ai AFTER INSERT ON functions BEGIN - INSERT INTO functions_fts(rowid, id, name, description, tags, signature, domain) - VALUES (new.rowid, new.id, new.name, new.description, new.tags, new.signature, new.domain); -END; - -CREATE TRIGGER IF NOT EXISTS functions_ad AFTER DELETE ON functions BEGIN - INSERT INTO functions_fts(functions_fts, rowid, id, name, description, tags, signature, domain) - VALUES ('delete', old.rowid, old.id, old.name, old.description, old.tags, old.signature, old.domain); -END; - -CREATE TRIGGER IF NOT EXISTS functions_au AFTER UPDATE ON functions BEGIN - INSERT INTO functions_fts(functions_fts, rowid, id, name, description, tags, signature, domain) - VALUES ('delete', old.rowid, old.id, old.name, old.description, old.tags, old.signature, old.domain); - INSERT INTO functions_fts(rowid, id, name, description, tags, signature, domain) - VALUES (new.rowid, new.id, new.name, new.description, new.tags, new.signature, new.domain); -END; - -CREATE TRIGGER IF NOT EXISTS types_ai AFTER INSERT ON types BEGIN - INSERT INTO types_fts(rowid, id, name, description, tags, domain) - VALUES (new.rowid, new.id, new.name, new.description, new.tags, new.domain); -END; - -CREATE TRIGGER IF NOT EXISTS types_ad AFTER DELETE ON types BEGIN - INSERT INTO types_fts(types_fts, rowid, id, name, description, tags, domain) - VALUES ('delete', old.rowid, old.id, old.name, old.description, old.tags, old.domain); -END; - -CREATE TRIGGER IF NOT EXISTS types_au AFTER UPDATE ON types BEGIN - INSERT INTO types_fts(types_fts, rowid, id, name, description, tags, domain) - VALUES ('delete', old.rowid, old.id, old.name, old.description, old.tags, old.domain); - INSERT INTO types_fts(rowid, id, name, description, tags, domain) - VALUES (new.rowid, new.id, new.name, new.description, new.tags, new.domain); -END; -` - // DB wraps a SQLite connection for the registry. type DB struct { conn *sql.DB @@ -134,15 +28,14 @@ func Open(path string) (*DB, error) { } // WAL mode: enables concurrent reads while writing. - // Persists in the file — any client opening the DB inherits it. if _, err := conn.Exec("PRAGMA journal_mode=WAL"); err != nil { conn.Close() return nil, fmt.Errorf("setting WAL mode: %w", err) } - if _, err := conn.Exec(schemaSQL); err != nil { + if err := migrate(conn); err != nil { conn.Close() - return nil, fmt.Errorf("applying schema: %w", err) + return nil, fmt.Errorf("running migrations: %w", err) } return &DB{conn: conn, path: path}, nil diff --git a/registry/migrate.go b/registry/migrate.go new file mode 100644 index 00000000..212aa350 --- /dev/null +++ b/registry/migrate.go @@ -0,0 +1,117 @@ +package registry + +import ( + "database/sql" + "embed" + "fmt" + "path" + "sort" + "strconv" + "strings" + "time" +) + +//go:embed migrations/*.sql +var migrationsFS embed.FS + +const migrationTableSQL = ` +CREATE TABLE IF NOT EXISTS schema_migrations ( + version INTEGER PRIMARY KEY, + name TEXT NOT NULL, + applied_at TEXT NOT NULL +);` + +// migrate applies pending migrations to the database. +func migrate(conn *sql.DB) error { + if _, err := conn.Exec(migrationTableSQL); err != nil { + return fmt.Errorf("creating schema_migrations table: %w", err) + } + + current, err := currentVersion(conn) + if err != nil { + return err + } + + files, err := listMigrations() + if err != nil { + return err + } + + for _, mf := range files { + if mf.version <= current { + continue + } + + content, err := migrationsFS.ReadFile(path.Join("migrations", mf.filename)) + if err != nil { + return fmt.Errorf("reading migration %s: %w", mf.filename, err) + } + + tx, err := conn.Begin() + if err != nil { + return fmt.Errorf("beginning transaction for migration %d: %w", mf.version, err) + } + + if _, err := tx.Exec(string(content)); err != nil { + tx.Rollback() + return fmt.Errorf("applying migration %s: %w", mf.filename, err) + } + + if _, err := tx.Exec( + "INSERT INTO schema_migrations (version, name, applied_at) VALUES (?, ?, ?)", + mf.version, mf.filename, time.Now().UTC().Format(time.RFC3339), + ); err != nil { + tx.Rollback() + return fmt.Errorf("recording migration %s: %w", mf.filename, err) + } + + if err := tx.Commit(); err != nil { + return fmt.Errorf("committing migration %s: %w", mf.filename, err) + } + } + + return nil +} + +func currentVersion(conn *sql.DB) (int, error) { + var v int + err := conn.QueryRow("SELECT COALESCE(MAX(version), 0) FROM schema_migrations").Scan(&v) + if err != nil { + return 0, fmt.Errorf("reading current schema version: %w", err) + } + return v, nil +} + +type migrationFile struct { + version int + filename string +} + +func listMigrations() ([]migrationFile, error) { + entries, err := migrationsFS.ReadDir("migrations") + if err != nil { + return nil, fmt.Errorf("reading migrations directory: %w", err) + } + + var files []migrationFile + for _, e := range entries { + if e.IsDir() || !strings.HasSuffix(e.Name(), ".sql") { + continue + } + parts := strings.SplitN(e.Name(), "_", 2) + if len(parts) < 2 { + continue + } + v, err := strconv.Atoi(parts[0]) + if err != nil { + continue + } + files = append(files, migrationFile{version: v, filename: e.Name()}) + } + + sort.Slice(files, func(i, j int) bool { + return files[i].version < files[j].version + }) + + return files, nil +} diff --git a/registry/migrations/001_init.sql b/registry/migrations/001_init.sql new file mode 100644 index 00000000..e854dced --- /dev/null +++ b/registry/migrations/001_init.sql @@ -0,0 +1,106 @@ +-- registry schema v1.0.0 +-- Functions and types tables with FTS5 search. + +CREATE TABLE IF NOT EXISTS functions ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL, + kind TEXT NOT NULL CHECK(kind IN ('function','pipeline','component')), + lang TEXT NOT NULL, + domain TEXT NOT NULL, + version TEXT NOT NULL DEFAULT '1.0.0', + purity TEXT NOT NULL CHECK(purity IN ('pure','impure')), + signature TEXT NOT NULL DEFAULT '', + description TEXT NOT NULL, + tags TEXT NOT NULL DEFAULT '[]', + uses_functions TEXT NOT NULL DEFAULT '[]', + uses_types TEXT NOT NULL DEFAULT '[]', + returns TEXT NOT NULL DEFAULT '[]', + returns_optional INTEGER NOT NULL DEFAULT 0, + error_type TEXT NOT NULL DEFAULT '', + imports TEXT NOT NULL DEFAULT '[]', + example TEXT NOT NULL DEFAULT '', + tested INTEGER NOT NULL DEFAULT 0, + tests TEXT NOT NULL DEFAULT '[]', + test_file_path TEXT NOT NULL DEFAULT '', + file_path TEXT NOT NULL DEFAULT '', + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + -- Component fields + props TEXT NOT NULL DEFAULT '[]', + emits TEXT NOT NULL DEFAULT '[]', + has_state INTEGER, + framework TEXT NOT NULL DEFAULT '', + variant TEXT NOT NULL DEFAULT '[]' +); + +CREATE TABLE IF NOT EXISTS types ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL, + lang TEXT NOT NULL, + domain TEXT NOT NULL, + version TEXT NOT NULL DEFAULT '1.0.0', + algebraic TEXT NOT NULL CHECK(algebraic IN ('product','sum')), + definition TEXT NOT NULL DEFAULT '', + description TEXT NOT NULL, + tags TEXT NOT NULL DEFAULT '[]', + uses_types TEXT NOT NULL DEFAULT '[]', + file_path TEXT NOT NULL DEFAULT '', + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL +); + +CREATE VIRTUAL TABLE IF NOT EXISTS functions_fts USING fts5( + id, + name, + description, + tags, + signature, + domain, + content='functions', + content_rowid='rowid' +); + +CREATE VIRTUAL TABLE IF NOT EXISTS types_fts USING fts5( + id, + name, + description, + tags, + domain, + content='types', + content_rowid='rowid' +); + +-- Triggers to keep FTS in sync +CREATE TRIGGER IF NOT EXISTS functions_ai AFTER INSERT ON functions BEGIN + INSERT INTO functions_fts(rowid, id, name, description, tags, signature, domain) + VALUES (new.rowid, new.id, new.name, new.description, new.tags, new.signature, new.domain); +END; + +CREATE TRIGGER IF NOT EXISTS functions_ad AFTER DELETE ON functions BEGIN + INSERT INTO functions_fts(functions_fts, rowid, id, name, description, tags, signature, domain) + VALUES ('delete', old.rowid, old.id, old.name, old.description, old.tags, old.signature, old.domain); +END; + +CREATE TRIGGER IF NOT EXISTS functions_au AFTER UPDATE ON functions BEGIN + INSERT INTO functions_fts(functions_fts, rowid, id, name, description, tags, signature, domain) + VALUES ('delete', old.rowid, old.id, old.name, old.description, old.tags, old.signature, old.domain); + INSERT INTO functions_fts(rowid, id, name, description, tags, signature, domain) + VALUES (new.rowid, new.id, new.name, new.description, new.tags, new.signature, new.domain); +END; + +CREATE TRIGGER IF NOT EXISTS types_ai AFTER INSERT ON types BEGIN + INSERT INTO types_fts(rowid, id, name, description, tags, domain) + VALUES (new.rowid, new.id, new.name, new.description, new.tags, new.domain); +END; + +CREATE TRIGGER IF NOT EXISTS types_ad AFTER DELETE ON types BEGIN + INSERT INTO types_fts(types_fts, rowid, id, name, description, tags, domain) + VALUES ('delete', old.rowid, old.id, old.name, old.description, old.tags, old.domain); +END; + +CREATE TRIGGER IF NOT EXISTS types_au AFTER UPDATE ON types BEGIN + INSERT INTO types_fts(types_fts, rowid, id, name, description, tags, domain) + VALUES ('delete', old.rowid, old.id, old.name, old.description, old.tags, old.domain); + INSERT INTO types_fts(rowid, id, name, description, tags, domain) + VALUES (new.rowid, new.id, new.name, new.description, new.tags, new.domain); +END; diff --git a/registry/migrations/002_proposals.sql b/registry/migrations/002_proposals.sql new file mode 100644 index 00000000..7ab642ed --- /dev/null +++ b/registry/migrations/002_proposals.sql @@ -0,0 +1,42 @@ +-- Proposals table: agent-generated improvement suggestions for the registry. +-- Lives in registry (not operations) because proposals benefit all projects. + +CREATE TABLE IF NOT EXISTS proposals ( + id TEXT PRIMARY KEY, + kind TEXT NOT NULL CHECK(kind IN ('new_function','new_type','improve_function','improve_type','new_pipeline')), + target_id TEXT NOT NULL DEFAULT '', + title TEXT NOT NULL, + description TEXT NOT NULL DEFAULT '', + evidence TEXT NOT NULL DEFAULT '{}', + status TEXT NOT NULL DEFAULT 'pending' CHECK(status IN ('pending','approved','rejected','implemented')), + created_by TEXT NOT NULL DEFAULT '', + reviewed_by TEXT NOT NULL DEFAULT '', + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL +); + +CREATE VIRTUAL TABLE IF NOT EXISTS proposals_fts USING fts5( + id, + title, + description, + evidence, + content='proposals', + content_rowid='rowid' +); + +CREATE TRIGGER IF NOT EXISTS proposals_ai AFTER INSERT ON proposals BEGIN + INSERT INTO proposals_fts(rowid, id, title, description, evidence) + VALUES (new.rowid, new.id, new.title, new.description, new.evidence); +END; + +CREATE TRIGGER IF NOT EXISTS proposals_ad AFTER DELETE ON proposals BEGIN + INSERT INTO proposals_fts(proposals_fts, rowid, id, title, description, evidence) + VALUES ('delete', old.rowid, old.id, old.title, old.description, old.evidence); +END; + +CREATE TRIGGER IF NOT EXISTS proposals_au AFTER UPDATE ON proposals BEGIN + INSERT INTO proposals_fts(proposals_fts, rowid, id, title, description, evidence) + VALUES ('delete', old.rowid, old.id, old.title, old.description, old.evidence); + INSERT INTO proposals_fts(rowid, id, title, description, evidence) + VALUES (new.rowid, new.id, new.title, new.description, new.evidence); +END; diff --git a/registry/models.go b/registry/models.go index 06346058..bf3b60a5 100644 --- a/registry/models.go +++ b/registry/models.go @@ -87,6 +87,42 @@ type Type struct { UpdatedAt time.Time `json:"updated_at"` } +// ProposalKind classifies a proposal. +type ProposalKind string + +const ( + ProposalNewFunction ProposalKind = "new_function" + ProposalNewType ProposalKind = "new_type" + ProposalImproveFunction ProposalKind = "improve_function" + ProposalImproveType ProposalKind = "improve_type" + ProposalNewPipeline ProposalKind = "new_pipeline" +) + +// ProposalStatus represents the review state of a proposal. +type ProposalStatus string + +const ( + ProposalPending ProposalStatus = "pending" + ProposalApproved ProposalStatus = "approved" + ProposalRejected ProposalStatus = "rejected" + ProposalImplemented ProposalStatus = "implemented" +) + +// Proposal represents a suggested improvement to the registry. +type Proposal struct { + ID string `json:"id"` + Kind ProposalKind `json:"kind"` + TargetID string `json:"target_id"` + Title string `json:"title"` + Description string `json:"description"` + Evidence map[string]any `json:"evidence"` + Status ProposalStatus `json:"status"` + CreatedBy string `json:"created_by"` + ReviewedBy string `json:"reviewed_by"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} + // GenerateID builds the canonical ID: {name}_{lang}_{domain} func GenerateID(name, lang, domain string) string { return name + "_" + lang + "_" + domain diff --git a/registry/store.go b/registry/store.go index c7cac284..91b75801 100644 --- a/registry/store.go +++ b/registry/store.go @@ -24,6 +24,23 @@ func unmarshalStrings(s string) []string { return out } +func marshalJSON(v map[string]any) string { + if v == nil { + v = map[string]any{} + } + b, _ := json.Marshal(v) + return string(b) +} + +func unmarshalJSON(s string) map[string]any { + var out map[string]any + json.Unmarshal([]byte(s), &out) + if out == nil { + out = map[string]any{} + } + return out +} + func marshalProps(ps []PropDef) string { if ps == nil { ps = []PropDef{} @@ -305,3 +322,153 @@ func scanTypes(rows interface{ Next() bool; Scan(...any) error }) ([]Type, error } return result, nil } + +// --- Proposal CRUD --- + +// InsertProposal inserts or replaces a proposal. +func (db *DB) InsertProposal(p *Proposal) error { + now := time.Now().UTC() + if p.CreatedAt.IsZero() { + p.CreatedAt = now + } + p.UpdatedAt = now + + if p.Status == "" { + p.Status = ProposalPending + } + + _, err := db.conn.Exec(` + INSERT OR REPLACE INTO proposals ( + id, kind, target_id, title, description, evidence, + status, created_by, reviewed_by, created_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + p.ID, string(p.Kind), p.TargetID, p.Title, p.Description, + marshalJSON(p.Evidence), string(p.Status), p.CreatedBy, p.ReviewedBy, + p.CreatedAt.Format(time.RFC3339), p.UpdatedAt.Format(time.RFC3339), + ) + return err +} + +// GetProposal returns a proposal by ID. +func (db *DB) GetProposal(id string) (*Proposal, error) { + rows, err := db.conn.Query(` + SELECT id, kind, target_id, title, description, evidence, + status, created_by, reviewed_by, created_at, updated_at + FROM proposals WHERE id = ?`, id) + if err != nil { + return nil, err + } + defer rows.Close() + + ps, err := scanProposals(rows) + if err != nil { + return nil, err + } + if len(ps) == 0 { + return nil, fmt.Errorf("proposal %q not found", id) + } + return &ps[0], nil +} + +// UpdateProposal updates an existing proposal. +func (db *DB) UpdateProposal(p *Proposal) error { + p.UpdatedAt = time.Now().UTC() + _, err := db.conn.Exec(` + UPDATE proposals SET kind=?, target_id=?, title=?, description=?, evidence=?, + status=?, created_by=?, reviewed_by=?, updated_at=? + WHERE id=?`, + string(p.Kind), p.TargetID, p.Title, p.Description, + marshalJSON(p.Evidence), string(p.Status), p.CreatedBy, p.ReviewedBy, + p.UpdatedAt.Format(time.RFC3339), p.ID, + ) + return err +} + +// DeleteProposal removes a proposal by ID. +func (db *DB) DeleteProposal(id string) error { + _, err := db.conn.Exec("DELETE FROM proposals WHERE id = ?", id) + return err +} + +// ListProposals returns proposals filtered by kind and/or status. +func (db *DB) ListProposals(kind ProposalKind, status ProposalStatus) ([]Proposal, error) { + where := []string{} + args := []any{} + if kind != "" { + where = append(where, "kind = ?") + args = append(args, string(kind)) + } + if status != "" { + where = append(where, "status = ?") + args = append(args, string(status)) + } + + q := `SELECT id, kind, target_id, title, description, evidence, + status, created_by, reviewed_by, created_at, updated_at + FROM proposals` + if len(where) > 0 { + q += " WHERE " + strings.Join(where, " AND ") + } + q += " ORDER BY created_at DESC" + + rows, err := db.conn.Query(q, args...) + if err != nil { + return nil, err + } + defer rows.Close() + return scanProposals(rows) +} + +// SearchProposals performs FTS search on proposals with optional filters. +func (db *DB) SearchProposals(query string, kind ProposalKind, status ProposalStatus) ([]Proposal, error) { + where := []string{} + args := []any{} + + if query != "" { + where = append(where, "p.id IN (SELECT id FROM proposals_fts WHERE proposals_fts MATCH ?)") + args = append(args, query) + } + if kind != "" { + where = append(where, "p.kind = ?") + args = append(args, string(kind)) + } + if status != "" { + where = append(where, "p.status = ?") + args = append(args, string(status)) + } + + q := `SELECT p.id, p.kind, p.target_id, p.title, p.description, p.evidence, + p.status, p.created_by, p.reviewed_by, p.created_at, p.updated_at + FROM proposals p` + if len(where) > 0 { + q += " WHERE " + strings.Join(where, " AND ") + } + q += " ORDER BY p.created_at DESC" + + rows, err := db.conn.Query(q, args...) + if err != nil { + return nil, err + } + defer rows.Close() + return scanProposals(rows) +} + +func scanProposals(rows interface{ Next() bool; Scan(...any) error }) ([]Proposal, error) { + var result []Proposal + for rows.Next() { + var p Proposal + var evidenceJSON, createdAt, updatedAt string + err := rows.Scan( + &p.ID, &p.Kind, &p.TargetID, &p.Title, &p.Description, &evidenceJSON, + &p.Status, &p.CreatedBy, &p.ReviewedBy, &createdAt, &updatedAt, + ) + if err != nil { + return nil, fmt.Errorf("scanning proposal: %w", err) + } + p.Evidence = unmarshalJSON(evidenceJSON) + p.CreatedAt, _ = time.Parse(time.RFC3339, createdAt) + p.UpdatedAt, _ = time.Parse(time.RFC3339, updatedAt) + result = append(result, p) + } + return result, nil +} diff --git a/registry/store_test.go b/registry/store_test.go index c3a4204c..e3831dfb 100644 --- a/registry/store_test.go +++ b/registry/store_test.go @@ -151,6 +151,138 @@ func TestPurge(t *testing.T) { } } +func TestProposalCRUD(t *testing.T) { + db := tempDB(t) + + p := &Proposal{ + ID: "proposal_test_1", + Kind: ProposalNewFunction, + Title: "Add retry with backoff", + Description: "Exponential backoff for HTTP clients", + Evidence: map[string]any{"assertion_ids": []any{"a1", "a2"}}, + CreatedBy: "agent", + } + + if err := db.InsertProposal(p); err != nil { + t.Fatalf("insert: %v", err) + } + + if p.Status != ProposalPending { + t.Errorf("default status = %q, want pending", p.Status) + } + + got, err := db.GetProposal("proposal_test_1") + if err != nil { + t.Fatalf("get: %v", err) + } + if got.Title != "Add retry with backoff" { + t.Errorf("title = %q, want %q", got.Title, "Add retry with backoff") + } + if got.Evidence["assertion_ids"] == nil { + t.Error("evidence should contain assertion_ids") + } + + // Update + got.Status = ProposalApproved + got.ReviewedBy = "lucas" + if err := db.UpdateProposal(got); err != nil { + t.Fatalf("update: %v", err) + } + updated, _ := db.GetProposal("proposal_test_1") + if updated.Status != ProposalApproved { + t.Errorf("status = %q, want approved", updated.Status) + } + if updated.ReviewedBy != "lucas" { + t.Errorf("reviewed_by = %q, want lucas", updated.ReviewedBy) + } + + // List with filter + db.InsertProposal(&Proposal{ + ID: "proposal_test_2", Kind: ProposalImproveType, TargetID: "ohlcv_go_finance", + Title: "Improve OHLCV", CreatedBy: "agent", + }) + + all, err := db.ListProposals("", "") + if err != nil { + t.Fatalf("list all: %v", err) + } + if len(all) != 2 { + t.Errorf("list all = %d, want 2", len(all)) + } + + byKind, _ := db.ListProposals(ProposalNewFunction, "") + if len(byKind) != 1 { + t.Errorf("list by kind = %d, want 1", len(byKind)) + } + + byStatus, _ := db.ListProposals("", ProposalPending) + if len(byStatus) != 1 { + t.Errorf("list by status pending = %d, want 1", len(byStatus)) + } + + // Search FTS + found, err := db.SearchProposals("backoff", "", "") + if err != nil { + t.Fatalf("search: %v", err) + } + if len(found) != 1 { + t.Errorf("search 'backoff' = %d, want 1", len(found)) + } + + // Delete + if err := db.DeleteProposal("proposal_test_1"); err != nil { + t.Fatalf("delete: %v", err) + } + _, err = db.GetProposal("proposal_test_1") + if err == nil { + t.Error("expected error after delete") + } +} + +func TestValidateProposal(t *testing.T) { + tests := []struct { + name string + p Proposal + wantErr bool + }{ + {"valid new_function", Proposal{ID: "p1", Kind: ProposalNewFunction, Title: "test"}, false}, + {"valid improve with target", Proposal{ID: "p2", Kind: ProposalImproveFunction, Title: "test", TargetID: "fn_go_core"}, false}, + {"missing title", Proposal{ID: "p3", Kind: ProposalNewFunction}, true}, + {"missing kind", Proposal{ID: "p4", Title: "test"}, true}, + {"improve without target", Proposal{ID: "p5", Kind: ProposalImproveType, Title: "test"}, true}, + {"invalid kind", Proposal{ID: "p6", Kind: "invalid", Title: "test"}, true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := ValidateProposal(&tt.p) + if (err != nil) != tt.wantErr { + t.Errorf("ValidateProposal() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestMigrations(t *testing.T) { + db := tempDB(t) + + // Verify schema_migrations table exists and has entries + var count int + err := db.conn.QueryRow("SELECT COUNT(*) FROM schema_migrations").Scan(&count) + if err != nil { + t.Fatalf("query schema_migrations: %v", err) + } + if count < 2 { + t.Errorf("expected at least 2 migrations, got %d", count) + } + + // Verify proposals table exists + _, err = db.conn.Exec("SELECT 1 FROM proposals LIMIT 1") + if err != nil { + t.Fatalf("proposals table should exist: %v", err) + } +} + func TestDrop(t *testing.T) { path := filepath.Join(t.TempDir(), "drop.db") db, err := Open(path) diff --git a/registry/validate.go b/registry/validate.go index 4d1b5e57..0ec04c9e 100644 --- a/registry/validate.go +++ b/registry/validate.go @@ -126,6 +126,41 @@ func ValidateFunction(f *Function, knownFunctions, knownTypes map[string]bool) * return nil } +// ValidateProposal checks integrity rules for proposals. +func ValidateProposal(p *Proposal) *ValidationError { + var errs []string + + if p.ID == "" { + errs = append(errs, "id is required") + } + if p.Title == "" { + errs = append(errs, "title is required") + } + + switch p.Kind { + case ProposalNewFunction, ProposalNewType, ProposalImproveFunction, ProposalImproveType, ProposalNewPipeline: + case "": + errs = append(errs, "kind is required") + default: + errs = append(errs, fmt.Sprintf("invalid kind: %s", p.Kind)) + } + + switch p.Status { + case ProposalPending, ProposalApproved, ProposalRejected, ProposalImplemented, "": + default: + errs = append(errs, fmt.Sprintf("invalid status: %s", p.Status)) + } + + if (p.Kind == ProposalImproveFunction || p.Kind == ProposalImproveType) && p.TargetID == "" { + errs = append(errs, "target_id is required for improve_* proposals") + } + + if len(errs) > 0 { + return &ValidationError{ID: p.ID, Errors: errs} + } + return nil +} + // ValidateType checks integrity rules for types. func ValidateType(t *Type, knownTypes map[string]bool) *ValidationError { var errs []string