merge issue/0114: DoD evidence schema + fn doctor dod
This commit is contained in:
@@ -4,6 +4,7 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"text/tabwriter"
|
||||
|
||||
@@ -65,6 +66,8 @@ func cmdDoctor(args []string) {
|
||||
doctorAppLocation(r, jsonOut)
|
||||
case "modules":
|
||||
doctorModules(r, jsonOut)
|
||||
case "dod":
|
||||
doctorDod(r, jsonOut)
|
||||
default:
|
||||
fmt.Fprintf(os.Stderr, "unknown doctor subcommand: %s\n", sub)
|
||||
doctorUsage()
|
||||
@@ -93,6 +96,7 @@ Subcommands:
|
||||
capabilities Drift entre docs/capabilities/INDEX.md, tags de funciones, y paginas <grupo>.md (issue 0086)
|
||||
app-location Detecta artefactos (apps/analysis) en carpetas de lenguaje (cpp/apps/, etc.) - issue 0096
|
||||
modules Drift entre uses_modules (app.md) y fn_module_<x> link calls (CMakeLists.txt) - issue 0097
|
||||
dod Audita bloque dod_evidence_schema en dev/issues/ y dev/flows/ (issue 0114)
|
||||
|
||||
Flags:
|
||||
--json Salida JSON (para scripting/agentes)
|
||||
@@ -643,3 +647,41 @@ func doctorModules(root string, jsonOut bool) {
|
||||
fmt.Println("Fix: align uses_modules in app.md with target_link_libraries(fn_module_*) in CMakeLists.txt.")
|
||||
}
|
||||
}
|
||||
|
||||
func doctorDod(root string, jsonOut bool) {
|
||||
issuesDir := filepath.Join(root, "dev", "issues")
|
||||
flowsDir := filepath.Join(root, "dev", "flows")
|
||||
report, err := infra.AuditDodSchema(issuesDir, flowsDir)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "error: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if jsonOut {
|
||||
emit(report)
|
||||
return
|
||||
}
|
||||
fmt.Println("=== DoD Schema Audit ===")
|
||||
fmt.Printf("files scanned: %d\n", report.TotalFiles)
|
||||
fmt.Printf("with schema: %d\n", report.FilesWithItems)
|
||||
fmt.Printf("total items: %d\n", report.TotalItems)
|
||||
fmt.Printf("invalid items: %d\n", report.InvalidItems)
|
||||
if report.InvalidItems == 0 {
|
||||
fmt.Println("\nAll DoD schemas valid.")
|
||||
return
|
||||
}
|
||||
fmt.Println()
|
||||
rel := func(p string) string {
|
||||
if r, err := filepath.Rel(root, p); err == nil {
|
||||
return r
|
||||
}
|
||||
return p
|
||||
}
|
||||
for _, f := range report.Files {
|
||||
if len(f.Errors) == 0 {
|
||||
continue
|
||||
}
|
||||
for _, e := range f.Errors {
|
||||
fmt.Printf("%s : %s\n", rel(f.Path), e)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -56,6 +56,48 @@ Regla: si la respuesta a "donde lo ves" es "en una BD" o "en un log" -> NO vale.
|
||||
|
||||
`/flow done` rechaza el cierre si falta alguno de los 4 user-facing checks o si `## Notas` no contiene parrafo onboarding.
|
||||
|
||||
### DoD evidence schema (issue 0114, opcional)
|
||||
|
||||
Ademas de los checkboxes humanos del bloque `## Definition of Done`, cada flow puede declarar en su frontmatter un bloque `dod_evidence_schema:` con la version maquinable de la DoD: lista de evidencias con id unico, `kind` cerrado, `expected` libre y `required` bool. Auditable con `fn doctor dod`.
|
||||
|
||||
```yaml
|
||||
dod_evidence_schema:
|
||||
- id: surface_dashboard
|
||||
kind: url
|
||||
expected: "https://metabase.organic-machine.com/dashboard/12 muestra ultimo refresh hoy"
|
||||
required: true
|
||||
- id: matrix_room_msg
|
||||
kind: screenshot
|
||||
expected: "sala matrix #flows recibe mensaje con resumen del run"
|
||||
required: true
|
||||
- id: data_factory_run
|
||||
kind: cmd
|
||||
expected: "sqlite3 data_factory.db 'SELECT count(*) FROM runs WHERE flow=NNNN' > 0"
|
||||
required: true
|
||||
- id: error_path_log
|
||||
kind: log
|
||||
expected: "fallar collector intencional deja entry status=error sin crash"
|
||||
required: false
|
||||
```
|
||||
|
||||
Reglas:
|
||||
- `kind` ∈ {`screenshot`, `log`, `url`, `cmd`}.
|
||||
- `id` unico por flow.
|
||||
- `expected` no vacio.
|
||||
- `required` default `true`.
|
||||
|
||||
Ejemplos por kind:
|
||||
|
||||
| kind | que pones en `expected` |
|
||||
|---|---|
|
||||
| `screenshot` | "frame de la app/sala mostrando estado Y" |
|
||||
| `log` | "fichero <path> contiene linea con texto Z" |
|
||||
| `url` | "GET <url> devuelve 200 con campo W" o "dashboard tal carga ultima fila < 24h" |
|
||||
| `cmd` | comando shell con exit 0 (incluido SQL via sqlite3) |
|
||||
|
||||
Plantilla canonica: `docs/templates/flow.md`. Validador: `fn doctor dod` + `audit_dod_schema_go_infra`.
|
||||
|
||||
|
||||
## Para agentes / LLMs
|
||||
|
||||
Antes de crear o editar un flow, lee `AGENT_GUIDE.md`. Define:
|
||||
|
||||
@@ -2,6 +2,48 @@
|
||||
|
||||
> **Frontmatter YAML** es la fuente de verdad desde 2026-05-17 (issue 0100).
|
||||
> Cada `.md` empieza con bloque `---` con `id`, `title`, `status`, `type`, `domain`, `scope`, `priority`, `depends`, `blocks`, `related`, `created`, `updated`, `tags`.
|
||||
|
||||
## DoD evidence schema (issue 0114, opcional)
|
||||
|
||||
Cualquier issue puede declarar en su frontmatter un bloque `dod_evidence_schema:` que enumera **evidencias concretas** capturables al cerrar. Es la version maquinable del bloque humano `## Definition of Done`: lista de items con id unico, `kind` cerrado, `expected` libre y `required` bool. Auditable con `fn doctor dod` (sale humana y `--json`).
|
||||
|
||||
```yaml
|
||||
dod_evidence_schema:
|
||||
- id: surface_1_board_drag
|
||||
kind: screenshot
|
||||
expected: "kanban_cpp board con card en columna Doing (agent)"
|
||||
required: true
|
||||
- id: backend_health
|
||||
kind: cmd
|
||||
expected: "curl -fsS http://localhost:8403/api/health == 200"
|
||||
required: true
|
||||
- id: timeline_entry
|
||||
kind: url
|
||||
expected: "http://localhost:8486/api/runs?app=kanban_cpp devuelve >=1 run"
|
||||
required: false
|
||||
- id: agent_log
|
||||
kind: log
|
||||
expected: "agent_runs/<run_id>/agent.log contiene 'workflow done'"
|
||||
required: true
|
||||
```
|
||||
|
||||
Reglas:
|
||||
- `kind` ∈ {`screenshot`, `log`, `url`, `cmd`}. Otro valor rechaza el item.
|
||||
- `id` unico por issue (los duplicados se reportan).
|
||||
- `expected` no vacio (descripcion observable de la evidencia).
|
||||
- `required` bool (default `true` si se omite).
|
||||
|
||||
Ejemplos por kind:
|
||||
|
||||
| kind | que pones en `expected` |
|
||||
|---|---|
|
||||
| `screenshot` | "frame de la UI X mostrando estado Y" — capturable manual o por test visual |
|
||||
| `log` | "fichero <path> contiene linea con texto Z" — grep-eable |
|
||||
| `url` | "GET <url> devuelve 200 y body con campo W" — curl-eable |
|
||||
| `cmd` | comando que debe terminar con exit 0 (o assert explicito) |
|
||||
|
||||
Plantilla canonica: `docs/templates/issue.md`. Validador: `fn doctor dod` + `audit_dod_schema_go_infra`.
|
||||
|
||||
> Para listar/filtrar: `/issue list --domain trading --status pendiente` (cuando `dev_console` exista — issue 0101).
|
||||
>
|
||||
> Dominios canonicos: `meta cpp-stack kanban trading gamedev osint data-ingest registry-quality notify imagegen apps-infra dev-ux deploy frontend mcp browser telemetry docs`.
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
---
|
||||
id: "0114"
|
||||
title: "DoD evidence schema canonico: frontmatter + BD + validator"
|
||||
status: pendiente
|
||||
type: feature
|
||||
domain:
|
||||
- taxonomy
|
||||
- dev-loop
|
||||
- registry-quality
|
||||
scope: registry
|
||||
priority: alta
|
||||
depends: []
|
||||
blocks:
|
||||
- "0115"
|
||||
- "0117"
|
||||
related:
|
||||
- "0008"
|
||||
- "0100"
|
||||
- "0102"
|
||||
created: 2026-05-18
|
||||
updated: 2026-05-18
|
||||
tags: [dod, evidence, frontmatter, taxonomy, validator]
|
||||
flow: "0008"
|
||||
---
|
||||
|
||||
# 0114 — DoD evidence schema canonico
|
||||
|
||||
## Problema
|
||||
|
||||
Hoy `## Definition of Done` es una lista markdown libre. `dod_user:` existe en frontmatter (issue 0102) como ratio. Falta una forma **estructurada** de declarar QUE evidencia tiene que aportar el agente por cada item DoD (screenshot, log, url, output cmd).
|
||||
|
||||
Sin schema, el agente no sabe que adjuntar y el validador no puede checkear automaticamente.
|
||||
|
||||
## Decision
|
||||
|
||||
Anadir bloque `dod_evidence_schema:` al frontmatter de **issues** y **flows**. Lista de items con shape canonico:
|
||||
|
||||
```yaml
|
||||
dod_evidence_schema:
|
||||
- id: surface_1_board_drag
|
||||
kind: screenshot
|
||||
expected: "kanban_cpp.exe board con card en columna Doing (agent), barra progreso visible"
|
||||
required: true
|
||||
- id: backend_health
|
||||
kind: cmd
|
||||
expected: "curl -fsS http://localhost:8403/api/health == 200"
|
||||
required: true
|
||||
- id: timeline_entry
|
||||
kind: url
|
||||
expected: "http://localhost:8486/api/runs?app=kanban_cpp devuelve >=1 run"
|
||||
required: false
|
||||
- id: agent_log
|
||||
kind: log
|
||||
expected: "agent_runs/<run_id>/agent.log contiene 'workflow done'"
|
||||
required: true
|
||||
```
|
||||
|
||||
### Kinds
|
||||
|
||||
| `kind` | Que adjunta el agente | Como valida |
|
||||
|---|---|---|
|
||||
| `screenshot` | path PNG en `agent_runs/<run_id>/evidence/<item_id>.png` | check existe + tamaño > 0 + dimensions sensatas |
|
||||
| `log` | path file (txt/log) | check existe + grep pattern de `expected` (opcional) |
|
||||
| `url` | URL string | HEAD request (2xx/3xx) o GET + match pattern |
|
||||
| `cmd` | comando + stdout esperado | exec + compare exit code + grep stdout |
|
||||
|
||||
### Persistencia
|
||||
|
||||
Frontmatter declara el SCHEMA (lo que se espera). `agent_runner_api` (0113) crea un row en `dod_items` por cada entrada al iniciar run. Agente luego adjunta `dod_evidence` rows.
|
||||
|
||||
## Validator: `audit_dod_schema_go_infra`
|
||||
|
||||
Funcion Go nueva en `functions/infra/`. Lee `.md` de `dev/issues/` + `dev/flows/`, parsea frontmatter, valida:
|
||||
|
||||
- `id` unico por archivo.
|
||||
- `kind` in [`screenshot`, `log`, `url`, `cmd`].
|
||||
- `expected` no vacio.
|
||||
- `required` bool (default true).
|
||||
|
||||
Output: tabla caveman con drift / errores.
|
||||
|
||||
Wrapper CLI: `fn doctor dod`.
|
||||
|
||||
## Criterios de aceptacion
|
||||
|
||||
- [ ] Plantilla `docs/templates/issue.md` + `docs/templates/flow.md` actualizadas con bloque opcional `dod_evidence_schema:` y ejemplo.
|
||||
- [ ] `audit_dod_schema_go_infra` registrado (`functions/infra/audit_dod_schema.{go,md}`).
|
||||
- [ ] `fn doctor dod` muestra: items por archivo + drift + errores.
|
||||
- [ ] Indexer (`registry/parser.go`) lee `dod_evidence_schema:` y lo persiste si afecta a la tabla `issues`/`flows` (en `apps/issues_api/`).
|
||||
- [ ] Migracion `apps/agent_runner_api/migrations/004_dod_items.sql` referencia este schema (issue 0113).
|
||||
- [ ] Doc en `dev/issues/README.md` + `dev/flows/README.md`: cuando declarar evidence schema, ejemplos por kind.
|
||||
- [ ] Al menos 2 issues piloto con bloque rellenado (recomendado: 0112 + 0116).
|
||||
- [ ] Tests Go: `audit_dod_schema_test.go` cubre kinds validos/invalidos + frontmatter malformed.
|
||||
|
||||
## Gotchas
|
||||
|
||||
- `dod_user:` (0102) es METRICA (ratio). `dod_evidence_schema:` es DECLARACION. NO renombrar ni fusionar — son cosas distintas.
|
||||
- Frontmatter YAML con array de objects: parser actual debe soportarlo. Verificar con `registry/parser.go` antes.
|
||||
- Schema retroactivo: issues viejos sin bloque siguen validos (`dod_evidence_schema: []` o ausente -> sin validacion automatica).
|
||||
- `cmd` con secretos/credenciales: NUNCA en el `expected`. Si el comando los necesita, env var.
|
||||
|
||||
## Out of scope
|
||||
|
||||
- UI para editar schema (eso vive en kanban_cpp/skill_tree v2).
|
||||
- Validacion en CI / pre-commit (futuro: hook que rechaza issue sin schema si type=feature).
|
||||
- Schema versioning — por ahora v1 implicito.
|
||||
Vendored
+73
@@ -0,0 +1,73 @@
|
||||
---
|
||||
name: <slug-del-flow>
|
||||
id: NNNN
|
||||
status: pending # pending | running | done | failed | deferred
|
||||
created: 2026-05-18
|
||||
updated: 2026-05-18
|
||||
priority: high # low | medium | high
|
||||
risk: low # low | medium | high (sensibilidad de datos)
|
||||
related_issues: []
|
||||
apps: []
|
||||
trigger: manual # manual | cron | webhook
|
||||
schedule: ""
|
||||
expected_runtime_s: 60
|
||||
tags: []
|
||||
|
||||
# OPCIONAL (issue 0114): contrato de evidencia DoD canonico.
|
||||
# Cada item es una superficie/check observable que prueba que el flow funciono.
|
||||
dod_evidence_schema:
|
||||
- id: surface_dashboard
|
||||
kind: url
|
||||
expected: "https://metabase.organic-machine.com/dashboard/12 muestra ultimo refresh hoy"
|
||||
required: true
|
||||
- id: matrix_room_msg
|
||||
kind: screenshot
|
||||
expected: "sala matrix #flows recibe mensaje con resumen del run"
|
||||
required: true
|
||||
- id: data_factory_run
|
||||
kind: cmd
|
||||
expected: "sqlite3 data_factory.db 'SELECT count(*) FROM runs WHERE flow=NNNN AND created_at > date(now,-1 day)' > 0"
|
||||
required: true
|
||||
- id: error_path_log
|
||||
kind: log
|
||||
expected: "fallar collector intencional deja entry status=error en operations.db sin crash"
|
||||
required: false
|
||||
---
|
||||
|
||||
## Goal
|
||||
Una frase: que estamos probando.
|
||||
|
||||
## Pre-requisitos
|
||||
- Lista de requisitos manuales (ej. Chrome con remote-debugging).
|
||||
|
||||
## Flow
|
||||
Pasos numerados. Cada paso puede ser:
|
||||
- texto libre (manual)
|
||||
- `function: <id>` (registry function)
|
||||
- `cmd: <bash>`
|
||||
- `js: <expression>` (en tab Chrome)
|
||||
|
||||
## Acceptance
|
||||
- [ ] Checklist
|
||||
- [ ] ...
|
||||
|
||||
## Definition of Done
|
||||
|
||||
- [ ] **Repetibilidad**: corre N veces consecutivas sin intervencion manual.
|
||||
- [ ] **Observabilidad**: call_monitor.calls + data_factory.runs + dashboard.
|
||||
- [ ] **Error-path**: 1 modo de fallo probado y manejado (no crash silencioso).
|
||||
- [ ] **Idempotencia**: re-ejecutar no duplica datos.
|
||||
- [ ] **Secrets**: cero credenciales fuera de pass/vaults.
|
||||
- [ ] **Docs**: `## Notas` rellenado con hallazgos reales.
|
||||
- [ ] **Registry-first**: todas las piezas existen como funciones del registry.
|
||||
- [ ] **INDEX + status**: status=done + fila INDEX.md + movido a completed/.
|
||||
- [ ] **User-facing**: <accion + lugar exacto>.
|
||||
- [ ] **User-facing repeat**: humano vuelve manana y ve datos frescos.
|
||||
- [ ] **User-facing onboarding**: parrafo en `## Notas` explica "para ver/usar esto: hacer X".
|
||||
- [ ] **User-facing latencia**: humano percibe cambio en <X segundos|minutos>.
|
||||
|
||||
## Telemetria esperada
|
||||
Que cambia en call_monitor / data_factory.runs / dag_engine.
|
||||
|
||||
## Notas
|
||||
Hallazgos tras correr. Incluye el parrafo onboarding.
|
||||
Vendored
+78
@@ -0,0 +1,78 @@
|
||||
---
|
||||
id: "NNNN"
|
||||
title: "<titulo corto del issue>"
|
||||
status: pendiente # pendiente | in-progress | bloqueado | completado | descartado
|
||||
type: feature # feature | bugfix | refactor | docs | chore | research | infra
|
||||
domain:
|
||||
- <dominio> # ver dev/TAXONOMY.md (meta, cpp-stack, kanban, trading, ...)
|
||||
scope: registry-only # registry-only | app:<id> | flow:<NNNN>
|
||||
priority: media # critica | alta | media | baja
|
||||
depends: [] # ["0099", ...] (IDs de issues bloqueantes)
|
||||
blocks: [] # ["0120", ...] (IDs que este issue desbloquea)
|
||||
related: []
|
||||
created: 2026-05-18
|
||||
updated: 2026-05-18
|
||||
tags: []
|
||||
|
||||
# OPCIONAL (issue 0114): contrato de evidencia DoD canonico.
|
||||
# Cada item es una prueba concreta que debe quedar capturada al cerrar el issue.
|
||||
# kind in {screenshot, log, url, cmd}. expected NO vacio. id unico.
|
||||
dod_evidence_schema:
|
||||
- id: surface_1_board_drag
|
||||
kind: screenshot
|
||||
expected: "kanban_cpp board con card en columna Doing (agent)"
|
||||
required: true
|
||||
- id: backend_health
|
||||
kind: cmd
|
||||
expected: "curl -fsS http://localhost:8403/api/health == 200"
|
||||
required: true
|
||||
- id: timeline_entry
|
||||
kind: url
|
||||
expected: "http://localhost:8486/api/runs?app=kanban_cpp devuelve >=1 run"
|
||||
required: false
|
||||
- id: agent_log
|
||||
kind: log
|
||||
expected: "agent_runs/<run_id>/agent.log contiene 'workflow done'"
|
||||
required: true
|
||||
---
|
||||
# NNNN — <titulo corto del issue>
|
||||
|
||||
**Status:** pendiente
|
||||
**Created:** 2026-05-18
|
||||
**Type:** feature
|
||||
**Priority:** media
|
||||
**Domain:** <dominio>
|
||||
**Scope:** registry-only
|
||||
**Depends:** —
|
||||
**Blocks:** —
|
||||
|
||||
## Problema
|
||||
|
||||
<que esta fallando o que falta y por que importa>
|
||||
|
||||
## Objetivo
|
||||
|
||||
<criterio observable de cuando el issue esta hecho>
|
||||
|
||||
## Plan
|
||||
|
||||
1. ...
|
||||
2. ...
|
||||
|
||||
## Acceptance
|
||||
|
||||
- [ ] check 1
|
||||
- [ ] check 2
|
||||
|
||||
## Definition of Done
|
||||
|
||||
- [ ] **Repetibilidad**: pasa N veces consecutivas sin intervencion manual.
|
||||
- [ ] **Observabilidad**: queda trazado en call_monitor + dashboard correspondiente.
|
||||
- [ ] **User-facing**: <accion concreta del humano + lugar exacto donde ve el output>.
|
||||
- [ ] **User-facing repeat**: el humano vuelve manana y ve datos frescos sin conocer el issue.
|
||||
- [ ] **User-facing onboarding**: parrafo en `## Notas` que explica "para ver/usar esto: hacer X".
|
||||
- [ ] **User-facing latencia**: el humano percibe el cambio en <X segundos|minutos>.
|
||||
|
||||
## Notas
|
||||
|
||||
<hallazgos, comandos para reproducir, parrafo onboarding>
|
||||
@@ -0,0 +1,211 @@
|
||||
package infra
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
// DodSchemaItem represents one declared evidence item in a DoD schema block.
|
||||
type DodSchemaItem struct {
|
||||
ID string `yaml:"id" json:"id"`
|
||||
Kind string `yaml:"kind" json:"kind"` // screenshot|log|url|cmd
|
||||
Expected string `yaml:"expected" json:"expected"` // free text
|
||||
Required bool `yaml:"required" json:"required"` // default true if missing
|
||||
}
|
||||
|
||||
// DodSchemaIssue represents one issue/flow file scanned and its parsed schema.
|
||||
type DodSchemaIssue struct {
|
||||
Path string `json:"path"`
|
||||
Type string `json:"type"` // "issue" | "flow"
|
||||
Items []DodSchemaItem `json:"items"` // parsed items (may be empty)
|
||||
Errors []string `json:"errors"` // per-file validation errors
|
||||
}
|
||||
|
||||
// DodSchemaReport aggregates the scan of dev/issues/ and dev/flows/.
|
||||
type DodSchemaReport struct {
|
||||
Files []DodSchemaIssue `json:"files"`
|
||||
TotalFiles int `json:"total_files"`
|
||||
FilesWithItems int `json:"files_with_items"`
|
||||
TotalItems int `json:"total_items"`
|
||||
InvalidItems int `json:"invalid_items"`
|
||||
}
|
||||
|
||||
// dodValidKinds is the closed set of allowed evidence kinds.
|
||||
var dodValidKinds = map[string]struct{}{
|
||||
"screenshot": {},
|
||||
"log": {},
|
||||
"url": {},
|
||||
"cmd": {},
|
||||
}
|
||||
|
||||
// dodRawFrontmatter is used for YAML unmarshal — we keep `required` as a
|
||||
// pointer so we can distinguish "missing" (defaults to true) from "false".
|
||||
type dodRawItem struct {
|
||||
ID string `yaml:"id"`
|
||||
Kind string `yaml:"kind"`
|
||||
Expected string `yaml:"expected"`
|
||||
Required *bool `yaml:"required"`
|
||||
}
|
||||
|
||||
type dodRawFrontmatter struct {
|
||||
DodEvidenceSchema []dodRawItem `yaml:"dod_evidence_schema"`
|
||||
}
|
||||
|
||||
// AuditDodSchema scans dev/issues/ (recursively, incl. completed/) and
|
||||
// dev/flows/ (recursively, incl. completed/) under `issuesDir` and `flowsDir`,
|
||||
// parses the `dod_evidence_schema:` block from each `.md` frontmatter, and
|
||||
// returns a structured report. Read-only — does not write anything.
|
||||
//
|
||||
// Validations per item:
|
||||
// - id non-empty and unique within the file
|
||||
// - kind in {screenshot, log, url, cmd}
|
||||
// - expected non-empty
|
||||
// - required defaults to true when missing
|
||||
//
|
||||
// Files with malformed frontmatter are reported with errors but do not abort
|
||||
// the scan.
|
||||
func AuditDodSchema(issuesDir, flowsDir string) (DodSchemaReport, error) {
|
||||
var report DodSchemaReport
|
||||
|
||||
collect := func(root, typ string) error {
|
||||
if root == "" {
|
||||
return nil
|
||||
}
|
||||
info, err := os.Stat(root)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
if !info.IsDir() {
|
||||
return nil
|
||||
}
|
||||
return filepath.WalkDir(root, func(p string, d os.DirEntry, walkErr error) error {
|
||||
if walkErr != nil {
|
||||
return nil
|
||||
}
|
||||
if d.IsDir() {
|
||||
return nil
|
||||
}
|
||||
if !strings.HasSuffix(p, ".md") {
|
||||
return nil
|
||||
}
|
||||
// Skip README/INDEX/template/AGENT_GUIDE — convention files, not
|
||||
// real issues/flows.
|
||||
base := strings.ToLower(filepath.Base(p))
|
||||
if base == "readme.md" || base == "index.md" || base == "template.md" || base == "agent_guide.md" || base == "taxonomy.md" {
|
||||
return nil
|
||||
}
|
||||
entry := parseDodFile(p, typ)
|
||||
report.Files = append(report.Files, entry)
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
if err := collect(issuesDir, "issue"); err != nil {
|
||||
return report, fmt.Errorf("audit_dod_schema: scan issues: %w", err)
|
||||
}
|
||||
if err := collect(flowsDir, "flow"); err != nil {
|
||||
return report, fmt.Errorf("audit_dod_schema: scan flows: %w", err)
|
||||
}
|
||||
|
||||
sort.Slice(report.Files, func(i, j int) bool {
|
||||
return report.Files[i].Path < report.Files[j].Path
|
||||
})
|
||||
|
||||
report.TotalFiles = len(report.Files)
|
||||
for _, f := range report.Files {
|
||||
if len(f.Items) > 0 {
|
||||
report.FilesWithItems++
|
||||
}
|
||||
report.TotalItems += len(f.Items)
|
||||
for _, e := range f.Errors {
|
||||
// Each item-level validation error counts as one invalid item.
|
||||
// Frontmatter-level errors (e.g. malformed YAML) also count.
|
||||
if strings.HasPrefix(e, "item ") || strings.Contains(e, "duplicate id") || strings.Contains(e, "malformed") {
|
||||
report.InvalidItems++
|
||||
}
|
||||
}
|
||||
}
|
||||
return report, nil
|
||||
}
|
||||
|
||||
// parseDodFile reads a single .md, extracts the YAML frontmatter, parses the
|
||||
// dod_evidence_schema block (if any), and validates each item.
|
||||
func parseDodFile(path, typ string) DodSchemaIssue {
|
||||
entry := DodSchemaIssue{Path: path, Type: typ}
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
entry.Errors = append(entry.Errors, fmt.Sprintf("read error: %v", err))
|
||||
return entry
|
||||
}
|
||||
s := string(data)
|
||||
if !strings.HasPrefix(s, "---") {
|
||||
// No frontmatter — silently skip (not every .md must have one).
|
||||
return entry
|
||||
}
|
||||
// Skip leading "---\n" (4 bytes when LF, 5 when CRLF).
|
||||
rest := s[3:]
|
||||
if strings.HasPrefix(rest, "\r\n") {
|
||||
rest = rest[2:]
|
||||
} else if strings.HasPrefix(rest, "\n") {
|
||||
rest = rest[1:]
|
||||
}
|
||||
end := strings.Index(rest, "\n---")
|
||||
if end < 0 {
|
||||
// No closing --- — treat as malformed but do not crash.
|
||||
entry.Errors = append(entry.Errors, "malformed frontmatter: missing closing ---")
|
||||
return entry
|
||||
}
|
||||
fm := rest[:end]
|
||||
|
||||
var raw dodRawFrontmatter
|
||||
if err := yaml.Unmarshal([]byte(fm), &raw); err != nil {
|
||||
entry.Errors = append(entry.Errors, fmt.Sprintf("malformed frontmatter yaml: %v", err))
|
||||
return entry
|
||||
}
|
||||
if len(raw.DodEvidenceSchema) == 0 {
|
||||
return entry
|
||||
}
|
||||
|
||||
seen := map[string]struct{}{}
|
||||
for i, it := range raw.DodEvidenceSchema {
|
||||
item := DodSchemaItem{
|
||||
ID: strings.TrimSpace(it.ID),
|
||||
Kind: strings.TrimSpace(it.Kind),
|
||||
Expected: strings.TrimSpace(it.Expected),
|
||||
Required: true, // default
|
||||
}
|
||||
if it.Required != nil {
|
||||
item.Required = *it.Required
|
||||
}
|
||||
|
||||
// Validation — errors are reported but the item is still appended so
|
||||
// the caller sees the (partial) data.
|
||||
label := item.ID
|
||||
if label == "" {
|
||||
label = fmt.Sprintf("#%d", i)
|
||||
entry.Errors = append(entry.Errors, fmt.Sprintf("item %s missing id", label))
|
||||
} else if _, dup := seen[item.ID]; dup {
|
||||
entry.Errors = append(entry.Errors, fmt.Sprintf("item '%s' duplicate id", item.ID))
|
||||
} else {
|
||||
seen[item.ID] = struct{}{}
|
||||
}
|
||||
if item.Kind == "" {
|
||||
entry.Errors = append(entry.Errors, fmt.Sprintf("item '%s' missing kind (valid: screenshot|log|url|cmd)", label))
|
||||
} else if _, ok := dodValidKinds[item.Kind]; !ok {
|
||||
entry.Errors = append(entry.Errors, fmt.Sprintf("item '%s' invalid kind '%s' (valid: screenshot|log|url|cmd)", label, item.Kind))
|
||||
}
|
||||
if item.Expected == "" {
|
||||
entry.Errors = append(entry.Errors, fmt.Sprintf("item '%s' empty expected", label))
|
||||
}
|
||||
entry.Items = append(entry.Items, item)
|
||||
}
|
||||
return entry
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
---
|
||||
name: audit_dod_schema
|
||||
kind: function
|
||||
lang: go
|
||||
domain: infra
|
||||
version: "1.0.0"
|
||||
purity: impure
|
||||
signature: "func AuditDodSchema(issuesDir, flowsDir string) (DodSchemaReport, error)"
|
||||
description: "Escanea dev/issues/ y dev/flows/ (incluidos subdirectorios completed/) y para cada .md parsea el bloque dod_evidence_schema del frontmatter YAML. Valida que cada item tenga id unico, kind in {screenshot,log,url,cmd}, expected no vacio y required bool (default true). Read-only: no modifica nada. Devuelve un DodSchemaReport con files (uno por archivo con items o errores), totales y conteo de items invalidos. Tolerante a frontmatter ausente o malformed — registra el error en el archivo afectado y continua."
|
||||
tags: [doctor, dod, evidence, frontmatter, taxonomy, validator]
|
||||
uses_functions: []
|
||||
uses_types: []
|
||||
returns: []
|
||||
returns_optional: false
|
||||
error_type: "error_go_core"
|
||||
imports: ["fmt", "os", "path/filepath", "sort", "strings", "gopkg.in/yaml.v3"]
|
||||
params:
|
||||
- name: issuesDir
|
||||
desc: "ruta absoluta al directorio dev/issues/ del registry. Vacio = skip."
|
||||
- name: flowsDir
|
||||
desc: "ruta absoluta al directorio dev/flows/ del registry. Vacio = skip."
|
||||
output: "DodSchemaReport con Files (slice de DodSchemaIssue por archivo escaneado), TotalFiles, FilesWithItems, TotalItems, InvalidItems. Cada DodSchemaIssue contiene Path, Type (issue|flow), Items parseados y Errors validados (item-level y frontmatter-level). Error retornado solo si el WalkDir falla; archivos individuales con errores se incluyen en Files."
|
||||
tested: true
|
||||
tests:
|
||||
- "valid dod_evidence_schema block parsed"
|
||||
- "invalid kind detected"
|
||||
- "duplicate id detected"
|
||||
- "empty expected detected"
|
||||
- "malformed yaml frontmatter does not crash"
|
||||
- "file without block returns empty Items"
|
||||
test_file_path: "functions/infra/audit_dod_schema_test.go"
|
||||
file_path: "functions/infra/audit_dod_schema.go"
|
||||
---
|
||||
|
||||
## Ejemplo
|
||||
|
||||
```go
|
||||
report, err := infra.AuditDodSchema(
|
||||
"/home/lucas/fn_registry/dev/issues",
|
||||
"/home/lucas/fn_registry/dev/flows",
|
||||
)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
fmt.Printf("scanned %d files, %d items, %d invalid\n",
|
||||
report.TotalFiles, report.TotalItems, report.InvalidItems)
|
||||
for _, f := range report.Files {
|
||||
for _, e := range f.Errors {
|
||||
fmt.Printf("%s: %s\n", f.Path, e)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Cuando usarla
|
||||
|
||||
Antes de cerrar un issue o flow para confirmar que el bloque `dod_evidence_schema:` del frontmatter cumple el contrato canonico (issue 0114). La usa `fn doctor dod` para auditar todo el repo de un vistazo. Tambien util desde `/issue done` y `/flow done` para bloquear cierre si la DoD declarada tiene items invalidos.
|
||||
|
||||
## Gotchas
|
||||
|
||||
- Lee disco — clasificada `impure` aunque sea read-only. No escribe nada.
|
||||
- Archivos sin frontmatter (no empiezan con `---`) se incluyen en Files con Items vacios y Errors vacios. No es error.
|
||||
- Frontmatter sin bloque `dod_evidence_schema:` -> Items vacios, sin error. El bloque es opcional.
|
||||
- `required:` ausente se trata como `true` (default conservador).
|
||||
- README.md, INDEX.md, template.md, AGENT_GUIDE.md, TAXONOMY.md se ignoran (convencion de carpeta, no son issues/flows reales).
|
||||
- Subdirectorios `completed/` se escanean igual que la raiz — un issue cerrado con DoD invalida sigue apareciendo en el reporte.
|
||||
- YAML malformed no crashea — se registra como `malformed frontmatter yaml: <err>` en `Errors` del archivo y se continua.
|
||||
@@ -0,0 +1,229 @@
|
||||
package infra
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// helper: write a file under tmp/dev/issues or tmp/dev/flows with given content.
|
||||
func writeMD(t *testing.T, dir, name, body string) string {
|
||||
t.Helper()
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
t.Fatalf("mkdir %s: %v", dir, err)
|
||||
}
|
||||
p := filepath.Join(dir, name)
|
||||
if err := os.WriteFile(p, []byte(body), 0o644); err != nil {
|
||||
t.Fatalf("write %s: %v", p, err)
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
func TestDodSchema_Valid(t *testing.T) {
|
||||
tmp := t.TempDir()
|
||||
issues := filepath.Join(tmp, "dev", "issues")
|
||||
flows := filepath.Join(tmp, "dev", "flows")
|
||||
|
||||
writeMD(t, issues, "0200-ok.md", `---
|
||||
id: "0200"
|
||||
title: ok
|
||||
dod_evidence_schema:
|
||||
- id: surface_ui
|
||||
kind: screenshot
|
||||
expected: "kanban shows new card"
|
||||
required: true
|
||||
- id: backend_health
|
||||
kind: cmd
|
||||
expected: "curl -fsS http://localhost:8000/api/health"
|
||||
required: false
|
||||
---
|
||||
body
|
||||
`)
|
||||
|
||||
report, err := AuditDodSchema(issues, flows)
|
||||
if err != nil {
|
||||
t.Fatalf("AuditDodSchema: %v", err)
|
||||
}
|
||||
if report.TotalFiles != 1 {
|
||||
t.Fatalf("TotalFiles = %d, want 1", report.TotalFiles)
|
||||
}
|
||||
if report.FilesWithItems != 1 {
|
||||
t.Fatalf("FilesWithItems = %d, want 1", report.FilesWithItems)
|
||||
}
|
||||
if report.TotalItems != 2 {
|
||||
t.Fatalf("TotalItems = %d, want 2", report.TotalItems)
|
||||
}
|
||||
if report.InvalidItems != 0 {
|
||||
t.Fatalf("InvalidItems = %d, want 0; errors=%v", report.InvalidItems, report.Files[0].Errors)
|
||||
}
|
||||
f := report.Files[0]
|
||||
if f.Type != "issue" {
|
||||
t.Errorf("Type = %q, want issue", f.Type)
|
||||
}
|
||||
if f.Items[0].ID != "surface_ui" || f.Items[0].Kind != "screenshot" {
|
||||
t.Errorf("item[0] = %+v", f.Items[0])
|
||||
}
|
||||
if !f.Items[0].Required {
|
||||
t.Errorf("item[0].Required = false, want true")
|
||||
}
|
||||
if f.Items[1].Required {
|
||||
t.Errorf("item[1].Required = true, want false")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDodSchema_InvalidKind(t *testing.T) {
|
||||
tmp := t.TempDir()
|
||||
issues := filepath.Join(tmp, "dev", "issues")
|
||||
writeMD(t, issues, "0201-badkind.md", `---
|
||||
id: "0201"
|
||||
dod_evidence_schema:
|
||||
- id: bad
|
||||
kind: png
|
||||
expected: "nope"
|
||||
---
|
||||
`)
|
||||
report, _ := AuditDodSchema(issues, "")
|
||||
if len(report.Files) != 1 {
|
||||
t.Fatalf("want 1 file, got %d", len(report.Files))
|
||||
}
|
||||
errs := strings.Join(report.Files[0].Errors, "|")
|
||||
if !strings.Contains(errs, "invalid kind 'png'") {
|
||||
t.Errorf("expected invalid kind error, got %q", errs)
|
||||
}
|
||||
if report.InvalidItems != 1 {
|
||||
t.Errorf("InvalidItems = %d, want 1", report.InvalidItems)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDodSchema_DuplicateID(t *testing.T) {
|
||||
tmp := t.TempDir()
|
||||
flows := filepath.Join(tmp, "dev", "flows")
|
||||
writeMD(t, flows, "0001-dup.md", `---
|
||||
name: dup
|
||||
dod_evidence_schema:
|
||||
- id: surface_1
|
||||
kind: url
|
||||
expected: "open dashboard"
|
||||
- id: surface_1
|
||||
kind: cmd
|
||||
expected: "other"
|
||||
---
|
||||
`)
|
||||
report, _ := AuditDodSchema("", flows)
|
||||
errs := strings.Join(report.Files[0].Errors, "|")
|
||||
if !strings.Contains(errs, "duplicate id") {
|
||||
t.Errorf("expected duplicate id error, got %q", errs)
|
||||
}
|
||||
if report.Files[0].Type != "flow" {
|
||||
t.Errorf("Type = %q, want flow", report.Files[0].Type)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDodSchema_EmptyExpected(t *testing.T) {
|
||||
tmp := t.TempDir()
|
||||
issues := filepath.Join(tmp, "dev", "issues")
|
||||
writeMD(t, issues, "0202-noexpect.md", `---
|
||||
dod_evidence_schema:
|
||||
- id: empty
|
||||
kind: log
|
||||
expected: ""
|
||||
---
|
||||
`)
|
||||
report, _ := AuditDodSchema(issues, "")
|
||||
errs := strings.Join(report.Files[0].Errors, "|")
|
||||
if !strings.Contains(errs, "empty expected") {
|
||||
t.Errorf("expected empty expected error, got %q", errs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDodSchema_MalformedYAML(t *testing.T) {
|
||||
tmp := t.TempDir()
|
||||
issues := filepath.Join(tmp, "dev", "issues")
|
||||
writeMD(t, issues, "0203-bad.md", `---
|
||||
id: "0203
|
||||
dod_evidence_schema:
|
||||
- id: x
|
||||
kind: cmd
|
||||
expected: nope
|
||||
---
|
||||
`)
|
||||
report, err := AuditDodSchema(issues, "")
|
||||
if err != nil {
|
||||
t.Fatalf("AuditDodSchema returned err on malformed: %v", err)
|
||||
}
|
||||
if len(report.Files) != 1 {
|
||||
t.Fatalf("want 1 file, got %d", len(report.Files))
|
||||
}
|
||||
errs := strings.Join(report.Files[0].Errors, "|")
|
||||
if !strings.Contains(errs, "malformed") {
|
||||
t.Errorf("expected malformed error, got %q", errs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDodSchema_NoBlock(t *testing.T) {
|
||||
tmp := t.TempDir()
|
||||
issues := filepath.Join(tmp, "dev", "issues")
|
||||
writeMD(t, issues, "0204-noblock.md", `---
|
||||
id: "0204"
|
||||
title: no schema here
|
||||
---
|
||||
body
|
||||
`)
|
||||
report, _ := AuditDodSchema(issues, "")
|
||||
if len(report.Files) != 1 {
|
||||
t.Fatalf("want 1 file, got %d", len(report.Files))
|
||||
}
|
||||
if len(report.Files[0].Items) != 0 {
|
||||
t.Errorf("Items=%d, want 0", len(report.Files[0].Items))
|
||||
}
|
||||
if len(report.Files[0].Errors) != 0 {
|
||||
t.Errorf("Errors=%v, want none", report.Files[0].Errors)
|
||||
}
|
||||
if report.FilesWithItems != 0 {
|
||||
t.Errorf("FilesWithItems = %d, want 0", report.FilesWithItems)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDodSchema_SkipsConventionFiles(t *testing.T) {
|
||||
tmp := t.TempDir()
|
||||
issues := filepath.Join(tmp, "dev", "issues")
|
||||
// README/INDEX/template/AGENT_GUIDE should be skipped even if they have a dod_evidence_schema block.
|
||||
writeMD(t, issues, "README.md", `---
|
||||
dod_evidence_schema:
|
||||
- id: x
|
||||
kind: png
|
||||
expected: ""
|
||||
---
|
||||
`)
|
||||
writeMD(t, issues, "0205-real.md", `---
|
||||
id: "0205"
|
||||
---
|
||||
`)
|
||||
report, _ := AuditDodSchema(issues, "")
|
||||
if report.TotalFiles != 1 {
|
||||
t.Fatalf("TotalFiles=%d, want 1 (README must be skipped)", report.TotalFiles)
|
||||
}
|
||||
if !strings.HasSuffix(report.Files[0].Path, "0205-real.md") {
|
||||
t.Errorf("unexpected file scanned: %s", report.Files[0].Path)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDodSchema_RecurseCompleted(t *testing.T) {
|
||||
tmp := t.TempDir()
|
||||
issues := filepath.Join(tmp, "dev", "issues")
|
||||
writeMD(t, filepath.Join(issues, "completed"), "0206-done.md", `---
|
||||
dod_evidence_schema:
|
||||
- id: a
|
||||
kind: url
|
||||
expected: "http://localhost"
|
||||
---
|
||||
`)
|
||||
report, _ := AuditDodSchema(issues, "")
|
||||
if report.TotalFiles != 1 {
|
||||
t.Fatalf("TotalFiles=%d, want 1 (completed/ must be walked)", report.TotalFiles)
|
||||
}
|
||||
if report.TotalItems != 1 {
|
||||
t.Errorf("TotalItems=%d, want 1", report.TotalItems)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user