merge issue/0114: DoD evidence schema + fn doctor dod

2026-05-18 18:17:20 +02:00
parent b9716a7cd6 df1eb701d3
commit 7ee351a505
9 changed files with 889 additions and 0 deletions
@@ -0,0 +1,211 @@
+package infra
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+
+	"gopkg.in/yaml.v3"
+)
+
+// DodSchemaItem represents one declared evidence item in a DoD schema block.
+type DodSchemaItem struct {
+	ID       string `yaml:"id" json:"id"`
+	Kind     string `yaml:"kind" json:"kind"`         // screenshot|log|url|cmd
+	Expected string `yaml:"expected" json:"expected"` // free text
+	Required bool   `yaml:"required" json:"required"` // default true if missing
+}
+
+// DodSchemaIssue represents one issue/flow file scanned and its parsed schema.
+type DodSchemaIssue struct {
+	Path   string          `json:"path"`
+	Type   string          `json:"type"`   // "issue" | "flow"
+	Items  []DodSchemaItem `json:"items"`  // parsed items (may be empty)
+	Errors []string        `json:"errors"` // per-file validation errors
+}
+
+// DodSchemaReport aggregates the scan of dev/issues/ and dev/flows/.
+type DodSchemaReport struct {
+	Files          []DodSchemaIssue `json:"files"`
+	TotalFiles     int              `json:"total_files"`
+	FilesWithItems int              `json:"files_with_items"`
+	TotalItems     int              `json:"total_items"`
+	InvalidItems   int              `json:"invalid_items"`
+}
+
+// dodValidKinds is the closed set of allowed evidence kinds.
+var dodValidKinds = map[string]struct{}{
+	"screenshot": {},
+	"log":        {},
+	"url":        {},
+	"cmd":        {},
+}
+
+// dodRawFrontmatter is used for YAML unmarshal — we keep `required` as a
+// pointer so we can distinguish "missing" (defaults to true) from "false".
+type dodRawItem struct {
+	ID       string `yaml:"id"`
+	Kind     string `yaml:"kind"`
+	Expected string `yaml:"expected"`
+	Required *bool  `yaml:"required"`
+}
+
+type dodRawFrontmatter struct {
+	DodEvidenceSchema []dodRawItem `yaml:"dod_evidence_schema"`
+}
+
+// AuditDodSchema scans dev/issues/ (recursively, incl. completed/) and
+// dev/flows/ (recursively, incl. completed/) under `issuesDir` and `flowsDir`,
+// parses the `dod_evidence_schema:` block from each `.md` frontmatter, and
+// returns a structured report. Read-only — does not write anything.
+//
+// Validations per item:
+//   - id non-empty and unique within the file
+//   - kind in {screenshot, log, url, cmd}
+//   - expected non-empty
+//   - required defaults to true when missing
+//
+// Files with malformed frontmatter are reported with errors but do not abort
+// the scan.
+func AuditDodSchema(issuesDir, flowsDir string) (DodSchemaReport, error) {
+	var report DodSchemaReport
+
+	collect := func(root, typ string) error {
+		if root == "" {
+			return nil
+		}
+		info, err := os.Stat(root)
+		if err != nil {
+			if os.IsNotExist(err) {
+				return nil
+			}
+			return err
+		}
+		if !info.IsDir() {
+			return nil
+		}
+		return filepath.WalkDir(root, func(p string, d os.DirEntry, walkErr error) error {
+			if walkErr != nil {
+				return nil
+			}
+			if d.IsDir() {
+				return nil
+			}
+			if !strings.HasSuffix(p, ".md") {
+				return nil
+			}
+			// Skip README/INDEX/template/AGENT_GUIDE — convention files, not
+			// real issues/flows.
+			base := strings.ToLower(filepath.Base(p))
+			if base == "readme.md" || base == "index.md" || base == "template.md" || base == "agent_guide.md" || base == "taxonomy.md" {
+				return nil
+			}
+			entry := parseDodFile(p, typ)
+			report.Files = append(report.Files, entry)
+			return nil
+		})
+	}
+
+	if err := collect(issuesDir, "issue"); err != nil {
+		return report, fmt.Errorf("audit_dod_schema: scan issues: %w", err)
+	}
+	if err := collect(flowsDir, "flow"); err != nil {
+		return report, fmt.Errorf("audit_dod_schema: scan flows: %w", err)
+	}
+
+	sort.Slice(report.Files, func(i, j int) bool {
+		return report.Files[i].Path < report.Files[j].Path
+	})
+
+	report.TotalFiles = len(report.Files)
+	for _, f := range report.Files {
+		if len(f.Items) > 0 {
+			report.FilesWithItems++
+		}
+		report.TotalItems += len(f.Items)
+		for _, e := range f.Errors {
+			// Each item-level validation error counts as one invalid item.
+			// Frontmatter-level errors (e.g. malformed YAML) also count.
+			if strings.HasPrefix(e, "item ") || strings.Contains(e, "duplicate id") || strings.Contains(e, "malformed") {
+				report.InvalidItems++
+			}
+		}
+	}
+	return report, nil
+}
+
+// parseDodFile reads a single .md, extracts the YAML frontmatter, parses the
+// dod_evidence_schema block (if any), and validates each item.
+func parseDodFile(path, typ string) DodSchemaIssue {
+	entry := DodSchemaIssue{Path: path, Type: typ}
+	data, err := os.ReadFile(path)
+	if err != nil {
+		entry.Errors = append(entry.Errors, fmt.Sprintf("read error: %v", err))
+		return entry
+	}
+	s := string(data)
+	if !strings.HasPrefix(s, "---") {
+		// No frontmatter — silently skip (not every .md must have one).
+		return entry
+	}
+	// Skip leading "---\n" (4 bytes when LF, 5 when CRLF).
+	rest := s[3:]
+	if strings.HasPrefix(rest, "\r\n") {
+		rest = rest[2:]
+	} else if strings.HasPrefix(rest, "\n") {
+		rest = rest[1:]
+	}
+	end := strings.Index(rest, "\n---")
+	if end < 0 {
+		// No closing --- — treat as malformed but do not crash.
+		entry.Errors = append(entry.Errors, "malformed frontmatter: missing closing ---")
+		return entry
+	}
+	fm := rest[:end]
+
+	var raw dodRawFrontmatter
+	if err := yaml.Unmarshal([]byte(fm), &raw); err != nil {
+		entry.Errors = append(entry.Errors, fmt.Sprintf("malformed frontmatter yaml: %v", err))
+		return entry
+	}
+	if len(raw.DodEvidenceSchema) == 0 {
+		return entry
+	}
+
+	seen := map[string]struct{}{}
+	for i, it := range raw.DodEvidenceSchema {
+		item := DodSchemaItem{
+			ID:       strings.TrimSpace(it.ID),
+			Kind:     strings.TrimSpace(it.Kind),
+			Expected: strings.TrimSpace(it.Expected),
+			Required: true, // default
+		}
+		if it.Required != nil {
+			item.Required = *it.Required
+		}
+
+		// Validation — errors are reported but the item is still appended so
+		// the caller sees the (partial) data.
+		label := item.ID
+		if label == "" {
+			label = fmt.Sprintf("#%d", i)
+			entry.Errors = append(entry.Errors, fmt.Sprintf("item %s missing id", label))
+		} else if _, dup := seen[item.ID]; dup {
+			entry.Errors = append(entry.Errors, fmt.Sprintf("item '%s' duplicate id", item.ID))
+		} else {
+			seen[item.ID] = struct{}{}
+		}
+		if item.Kind == "" {
+			entry.Errors = append(entry.Errors, fmt.Sprintf("item '%s' missing kind (valid: screenshot|log|url|cmd)", label))
+		} else if _, ok := dodValidKinds[item.Kind]; !ok {
+			entry.Errors = append(entry.Errors, fmt.Sprintf("item '%s' invalid kind '%s' (valid: screenshot|log|url|cmd)", label, item.Kind))
+		}
+		if item.Expected == "" {
+			entry.Errors = append(entry.Errors, fmt.Sprintf("item '%s' empty expected", label))
+		}
+		entry.Items = append(entry.Items, item)
+	}
+	return entry
+}
@@ -0,0 +1,66 @@
+---
+name: audit_dod_schema
+kind: function
+lang: go
+domain: infra
+version: "1.0.0"
+purity: impure
+signature: "func AuditDodSchema(issuesDir, flowsDir string) (DodSchemaReport, error)"
+description: "Escanea dev/issues/ y dev/flows/ (incluidos subdirectorios completed/) y para cada .md parsea el bloque dod_evidence_schema del frontmatter YAML. Valida que cada item tenga id unico, kind in {screenshot,log,url,cmd}, expected no vacio y required bool (default true). Read-only: no modifica nada. Devuelve un DodSchemaReport con files (uno por archivo con items o errores), totales y conteo de items invalidos. Tolerante a frontmatter ausente o malformed — registra el error en el archivo afectado y continua."
+tags: [doctor, dod, evidence, frontmatter, taxonomy, validator]
+uses_functions: []
+uses_types: []
+returns: []
+returns_optional: false
+error_type: "error_go_core"
+imports: ["fmt", "os", "path/filepath", "sort", "strings", "gopkg.in/yaml.v3"]
+params:
+  - name: issuesDir
+    desc: "ruta absoluta al directorio dev/issues/ del registry. Vacio = skip."
+  - name: flowsDir
+    desc: "ruta absoluta al directorio dev/flows/ del registry. Vacio = skip."
+output: "DodSchemaReport con Files (slice de DodSchemaIssue por archivo escaneado), TotalFiles, FilesWithItems, TotalItems, InvalidItems. Cada DodSchemaIssue contiene Path, Type (issue|flow), Items parseados y Errors validados (item-level y frontmatter-level). Error retornado solo si el WalkDir falla; archivos individuales con errores se incluyen en Files."
+tested: true
+tests:
+  - "valid dod_evidence_schema block parsed"
+  - "invalid kind detected"
+  - "duplicate id detected"
+  - "empty expected detected"
+  - "malformed yaml frontmatter does not crash"
+  - "file without block returns empty Items"
+test_file_path: "functions/infra/audit_dod_schema_test.go"
+file_path: "functions/infra/audit_dod_schema.go"
+---
+
+## Ejemplo
+
+```go
+report, err := infra.AuditDodSchema(
+    "/home/lucas/fn_registry/dev/issues",
+    "/home/lucas/fn_registry/dev/flows",
+)
+if err != nil {
+    log.Fatal(err)
+}
+fmt.Printf("scanned %d files, %d items, %d invalid\n",
+    report.TotalFiles, report.TotalItems, report.InvalidItems)
+for _, f := range report.Files {
+    for _, e := range f.Errors {
+        fmt.Printf("%s: %s\n", f.Path, e)
+    }
+}
+```
+
+## Cuando usarla
+
+Antes de cerrar un issue o flow para confirmar que el bloque `dod_evidence_schema:` del frontmatter cumple el contrato canonico (issue 0114). La usa `fn doctor dod` para auditar todo el repo de un vistazo. Tambien util desde `/issue done` y `/flow done` para bloquear cierre si la DoD declarada tiene items invalidos.
+
+## Gotchas
+
+- Lee disco — clasificada `impure` aunque sea read-only. No escribe nada.
+- Archivos sin frontmatter (no empiezan con `---`) se incluyen en Files con Items vacios y Errors vacios. No es error.
+- Frontmatter sin bloque `dod_evidence_schema:` -> Items vacios, sin error. El bloque es opcional.
+- `required:` ausente se trata como `true` (default conservador).
+- README.md, INDEX.md, template.md, AGENT_GUIDE.md, TAXONOMY.md se ignoran (convencion de carpeta, no son issues/flows reales).
+- Subdirectorios `completed/` se escanean igual que la raiz — un issue cerrado con DoD invalida sigue apareciendo en el reporte.
+- YAML malformed no crashea — se registra como `malformed frontmatter yaml: <err>` en `Errors` del archivo y se continua.
@@ -0,0 +1,229 @@
+package infra
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+// helper: write a file under tmp/dev/issues or tmp/dev/flows with given content.
+func writeMD(t *testing.T, dir, name, body string) string {
+	t.Helper()
+	if err := os.MkdirAll(dir, 0o755); err != nil {
+		t.Fatalf("mkdir %s: %v", dir, err)
+	}
+	p := filepath.Join(dir, name)
+	if err := os.WriteFile(p, []byte(body), 0o644); err != nil {
+		t.Fatalf("write %s: %v", p, err)
+	}
+	return p
+}
+
+func TestDodSchema_Valid(t *testing.T) {
+	tmp := t.TempDir()
+	issues := filepath.Join(tmp, "dev", "issues")
+	flows := filepath.Join(tmp, "dev", "flows")
+
+	writeMD(t, issues, "0200-ok.md", `---
+id: "0200"
+title: ok
+dod_evidence_schema:
+  - id: surface_ui
+    kind: screenshot
+    expected: "kanban shows new card"
+    required: true
+  - id: backend_health
+    kind: cmd
+    expected: "curl -fsS http://localhost:8000/api/health"
+    required: false
+---
+body
+`)
+
+	report, err := AuditDodSchema(issues, flows)
+	if err != nil {
+		t.Fatalf("AuditDodSchema: %v", err)
+	}
+	if report.TotalFiles != 1 {
+		t.Fatalf("TotalFiles = %d, want 1", report.TotalFiles)
+	}
+	if report.FilesWithItems != 1 {
+		t.Fatalf("FilesWithItems = %d, want 1", report.FilesWithItems)
+	}
+	if report.TotalItems != 2 {
+		t.Fatalf("TotalItems = %d, want 2", report.TotalItems)
+	}
+	if report.InvalidItems != 0 {
+		t.Fatalf("InvalidItems = %d, want 0; errors=%v", report.InvalidItems, report.Files[0].Errors)
+	}
+	f := report.Files[0]
+	if f.Type != "issue" {
+		t.Errorf("Type = %q, want issue", f.Type)
+	}
+	if f.Items[0].ID != "surface_ui" || f.Items[0].Kind != "screenshot" {
+		t.Errorf("item[0] = %+v", f.Items[0])
+	}
+	if !f.Items[0].Required {
+		t.Errorf("item[0].Required = false, want true")
+	}
+	if f.Items[1].Required {
+		t.Errorf("item[1].Required = true, want false")
+	}
+}
+
+func TestDodSchema_InvalidKind(t *testing.T) {
+	tmp := t.TempDir()
+	issues := filepath.Join(tmp, "dev", "issues")
+	writeMD(t, issues, "0201-badkind.md", `---
+id: "0201"
+dod_evidence_schema:
+  - id: bad
+    kind: png
+    expected: "nope"
+---
+`)
+	report, _ := AuditDodSchema(issues, "")
+	if len(report.Files) != 1 {
+		t.Fatalf("want 1 file, got %d", len(report.Files))
+	}
+	errs := strings.Join(report.Files[0].Errors, "|")
+	if !strings.Contains(errs, "invalid kind 'png'") {
+		t.Errorf("expected invalid kind error, got %q", errs)
+	}
+	if report.InvalidItems != 1 {
+		t.Errorf("InvalidItems = %d, want 1", report.InvalidItems)
+	}
+}
+
+func TestDodSchema_DuplicateID(t *testing.T) {
+	tmp := t.TempDir()
+	flows := filepath.Join(tmp, "dev", "flows")
+	writeMD(t, flows, "0001-dup.md", `---
+name: dup
+dod_evidence_schema:
+  - id: surface_1
+    kind: url
+    expected: "open dashboard"
+  - id: surface_1
+    kind: cmd
+    expected: "other"
+---
+`)
+	report, _ := AuditDodSchema("", flows)
+	errs := strings.Join(report.Files[0].Errors, "|")
+	if !strings.Contains(errs, "duplicate id") {
+		t.Errorf("expected duplicate id error, got %q", errs)
+	}
+	if report.Files[0].Type != "flow" {
+		t.Errorf("Type = %q, want flow", report.Files[0].Type)
+	}
+}
+
+func TestDodSchema_EmptyExpected(t *testing.T) {
+	tmp := t.TempDir()
+	issues := filepath.Join(tmp, "dev", "issues")
+	writeMD(t, issues, "0202-noexpect.md", `---
+dod_evidence_schema:
+  - id: empty
+    kind: log
+    expected: ""
+---
+`)
+	report, _ := AuditDodSchema(issues, "")
+	errs := strings.Join(report.Files[0].Errors, "|")
+	if !strings.Contains(errs, "empty expected") {
+		t.Errorf("expected empty expected error, got %q", errs)
+	}
+}
+
+func TestDodSchema_MalformedYAML(t *testing.T) {
+	tmp := t.TempDir()
+	issues := filepath.Join(tmp, "dev", "issues")
+	writeMD(t, issues, "0203-bad.md", `---
+id: "0203
+dod_evidence_schema:
+  - id: x
+    kind: cmd
+    expected: nope
+---
+`)
+	report, err := AuditDodSchema(issues, "")
+	if err != nil {
+		t.Fatalf("AuditDodSchema returned err on malformed: %v", err)
+	}
+	if len(report.Files) != 1 {
+		t.Fatalf("want 1 file, got %d", len(report.Files))
+	}
+	errs := strings.Join(report.Files[0].Errors, "|")
+	if !strings.Contains(errs, "malformed") {
+		t.Errorf("expected malformed error, got %q", errs)
+	}
+}
+
+func TestDodSchema_NoBlock(t *testing.T) {
+	tmp := t.TempDir()
+	issues := filepath.Join(tmp, "dev", "issues")
+	writeMD(t, issues, "0204-noblock.md", `---
+id: "0204"
+title: no schema here
+---
+body
+`)
+	report, _ := AuditDodSchema(issues, "")
+	if len(report.Files) != 1 {
+		t.Fatalf("want 1 file, got %d", len(report.Files))
+	}
+	if len(report.Files[0].Items) != 0 {
+		t.Errorf("Items=%d, want 0", len(report.Files[0].Items))
+	}
+	if len(report.Files[0].Errors) != 0 {
+		t.Errorf("Errors=%v, want none", report.Files[0].Errors)
+	}
+	if report.FilesWithItems != 0 {
+		t.Errorf("FilesWithItems = %d, want 0", report.FilesWithItems)
+	}
+}
+
+func TestDodSchema_SkipsConventionFiles(t *testing.T) {
+	tmp := t.TempDir()
+	issues := filepath.Join(tmp, "dev", "issues")
+	// README/INDEX/template/AGENT_GUIDE should be skipped even if they have a dod_evidence_schema block.
+	writeMD(t, issues, "README.md", `---
+dod_evidence_schema:
+  - id: x
+    kind: png
+    expected: ""
+---
+`)
+	writeMD(t, issues, "0205-real.md", `---
+id: "0205"
+---
+`)
+	report, _ := AuditDodSchema(issues, "")
+	if report.TotalFiles != 1 {
+		t.Fatalf("TotalFiles=%d, want 1 (README must be skipped)", report.TotalFiles)
+	}
+	if !strings.HasSuffix(report.Files[0].Path, "0205-real.md") {
+		t.Errorf("unexpected file scanned: %s", report.Files[0].Path)
+	}
+}
+
+func TestDodSchema_RecurseCompleted(t *testing.T) {
+	tmp := t.TempDir()
+	issues := filepath.Join(tmp, "dev", "issues")
+	writeMD(t, filepath.Join(issues, "completed"), "0206-done.md", `---
+dod_evidence_schema:
+  - id: a
+    kind: url
+    expected: "http://localhost"
+---
+`)
+	report, _ := AuditDodSchema(issues, "")
+	if report.TotalFiles != 1 {
+		t.Fatalf("TotalFiles=%d, want 1 (completed/ must be walked)", report.TotalFiles)
+	}
+	if report.TotalItems != 1 {
+		t.Errorf("TotalItems=%d, want 1", report.TotalItems)
+	}
+}