From 49eecd0c87ace89a81c2f89f9dd72812e5549992 Mon Sep 17 00:00:00 2001 From: Egutierrez Date: Sat, 28 Mar 2026 20:32:15 +0100 Subject: [PATCH] feat: campos documentation, notes y code en registry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Añade campos documentation, notes y code a functions y types. El parser extrae el contenido del .md y el código fuente del archivo referenciado en file_path. El indexer los almacena en SQLite y los incluye en FTS5 para búsqueda sobre código y documentación. Nueva migración 003_documentation.sql para añadir las columnas. --- cmd/fn/main.go | 21 ++++ registry/indexer.go | 66 +++++++---- registry/migrations/003_documentation.sql | 103 +++++++++++++++++ registry/models.go | 13 ++- registry/parser.go | 130 ++++++++++++++++------ registry/parser_test.go | 10 +- registry/store.go | 15 ++- 7 files changed, 290 insertions(+), 68 deletions(-) create mode 100644 registry/migrations/003_documentation.sql diff --git a/cmd/fn/main.go b/cmd/fn/main.go index 497ecead..cfa6a068 100644 --- a/cmd/fn/main.go +++ b/cmd/fn/main.go @@ -289,6 +289,15 @@ func printFunction(f *registry.Function) { if f.Example != "" { fmt.Printf("\nExample:\n%s\n", f.Example) } + if f.Notes != "" { + fmt.Printf("\nNotes:\n%s\n", f.Notes) + } + if f.Documentation != "" { + fmt.Printf("\nDocumentation:\n%s\n", f.Documentation) + } + if f.Code != "" { + fmt.Printf("\nCode:\n%s\n", f.Code) + } if f.Kind == registry.KindComponent { fmt.Printf("Framework: %s\n", f.Framework) if f.HasState != nil { @@ -316,6 +325,18 @@ func printType(t *registry.Type) { if t.Definition != "" { fmt.Printf("\nDefinition:\n%s\n", t.Definition) } + if t.Examples != "" { + fmt.Printf("\nExamples:\n%s\n", t.Examples) + } + if t.Notes != "" { + fmt.Printf("\nNotes:\n%s\n", t.Notes) + } + if t.Documentation != "" { + fmt.Printf("\nDocumentation:\n%s\n", t.Documentation) + } + if t.Code != "" { + fmt.Printf("\nCode:\n%s\n", t.Code) + } } // --- add --- diff --git a/registry/indexer.go b/registry/indexer.go index 6aee63bf..9a85f2ea 100644 --- a/registry/indexer.go +++ b/registry/indexer.go @@ -19,6 +19,9 @@ type IndexResult struct { // and populates the database. It uses two passes: // 1. Parse all entries and collect known IDs // 2. Validate references against known IDs, then insert valid entries +// +// Scans functions/ and types/ at the root level, plus any language-specific +// directories (e.g. python/functions/, python/types/). func Index(db *DB, root string) (*IndexResult, error) { if err := db.Purge(); err != nil { return nil, fmt.Errorf("purging database: %w", err) @@ -26,39 +29,50 @@ func Index(db *DB, root string) (*IndexResult, error) { result := &IndexResult{} - // Pass 1: parse everything + // Pass 1: parse everything from all source directories var functions []*Function var types []*Type - functionsDir := filepath.Join(root, "functions") - if _, err := os.Stat(functionsDir); err == nil { - filepath.Walk(functionsDir, func(path string, info os.FileInfo, err error) error { - if err != nil || info.IsDir() || !strings.HasSuffix(path, ".md") { - return nil - } - f, err := ParseFunctionMD(path) + // Directories to scan for functions and types. + // Base dirs + language-specific dirs discovered automatically. + funcDirs := []string{filepath.Join(root, "functions")} + typeDirs := []string{filepath.Join(root, "types")} + + // Discover language-specific directories (e.g. python/functions/, python/types/) + entries, _ := os.ReadDir(root) + for _, e := range entries { + if !e.IsDir() { + continue + } + langFuncs := filepath.Join(root, e.Name(), "functions") + if fi, err := os.Stat(langFuncs); err == nil && fi.IsDir() { + funcDirs = append(funcDirs, langFuncs) + } + langTypes := filepath.Join(root, e.Name(), "types") + if fi, err := os.Stat(langTypes); err == nil && fi.IsDir() { + typeDirs = append(typeDirs, langTypes) + } + } + + for _, dir := range funcDirs { + walkMD(dir, func(path string) { + f, err := ParseFunctionMD(path, root) if err != nil { result.Errors = append(result.Errors, fmt.Sprintf("parse %s: %v", path, err)) - return nil + return } functions = append(functions, f) - return nil }) } - typesDir := filepath.Join(root, "types") - if _, err := os.Stat(typesDir); err == nil { - filepath.Walk(typesDir, func(path string, info os.FileInfo, err error) error { - if err != nil || info.IsDir() || !strings.HasSuffix(path, ".md") { - return nil - } - t, err := ParseTypeMD(path) + for _, dir := range typeDirs { + walkMD(dir, func(path string) { + t, err := ParseTypeMD(path, root) if err != nil { result.Errors = append(result.Errors, fmt.Sprintf("parse %s: %v", path, err)) - return nil + return } types = append(types, t) - return nil }) } @@ -99,3 +113,17 @@ func Index(db *DB, root string) (*IndexResult, error) { return result, nil } + +// walkMD walks a directory recursively and calls fn for each .md file found. +func walkMD(dir string, fn func(path string)) { + if _, err := os.Stat(dir); err != nil { + return + } + filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + if err != nil || info.IsDir() || !strings.HasSuffix(path, ".md") { + return nil + } + fn(path) + return nil + }) +} diff --git a/registry/migrations/003_documentation.sql b/registry/migrations/003_documentation.sql new file mode 100644 index 00000000..d7f4be6b --- /dev/null +++ b/registry/migrations/003_documentation.sql @@ -0,0 +1,103 @@ +-- Add documentation fields to functions and types. +-- examples: extracted code blocks from ## Ejemplo +-- notes: extracted text from ## Notas +-- documentation: remaining body text from .md +-- code: source code from the referenced .go/.py/.tsx file + +ALTER TABLE functions ADD COLUMN notes TEXT NOT NULL DEFAULT ''; +ALTER TABLE functions ADD COLUMN documentation TEXT NOT NULL DEFAULT ''; +ALTER TABLE functions ADD COLUMN code TEXT NOT NULL DEFAULT ''; + +ALTER TABLE types ADD COLUMN examples TEXT NOT NULL DEFAULT ''; +ALTER TABLE types ADD COLUMN notes TEXT NOT NULL DEFAULT ''; +ALTER TABLE types ADD COLUMN documentation TEXT NOT NULL DEFAULT ''; +ALTER TABLE types ADD COLUMN code TEXT NOT NULL DEFAULT ''; + +-- Rebuild FTS for functions: add examples, notes, documentation, code +DROP TRIGGER IF EXISTS functions_ai; +DROP TRIGGER IF EXISTS functions_ad; +DROP TRIGGER IF EXISTS functions_au; + +INSERT INTO functions_fts(functions_fts) VALUES('rebuild'); +DROP TABLE IF EXISTS functions_fts; + +CREATE VIRTUAL TABLE functions_fts USING fts5( + id, + name, + description, + tags, + signature, + domain, + example, + notes, + documentation, + code, + content='functions', + content_rowid='rowid' +); + +-- Populate FTS from existing data +INSERT INTO functions_fts(rowid, id, name, description, tags, signature, domain, example, notes, documentation, code) +SELECT rowid, id, name, description, tags, signature, domain, example, notes, documentation, code +FROM functions; + +CREATE TRIGGER functions_ai AFTER INSERT ON functions BEGIN + INSERT INTO functions_fts(rowid, id, name, description, tags, signature, domain, example, notes, documentation, code) + VALUES (new.rowid, new.id, new.name, new.description, new.tags, new.signature, new.domain, new.example, new.notes, new.documentation, new.code); +END; + +CREATE TRIGGER functions_ad AFTER DELETE ON functions BEGIN + INSERT INTO functions_fts(functions_fts, rowid, id, name, description, tags, signature, domain, example, notes, documentation, code) + VALUES ('delete', old.rowid, old.id, old.name, old.description, old.tags, old.signature, old.domain, old.example, old.notes, old.documentation, old.code); +END; + +CREATE TRIGGER functions_au AFTER UPDATE ON functions BEGIN + INSERT INTO functions_fts(functions_fts, rowid, id, name, description, tags, signature, domain, example, notes, documentation, code) + VALUES ('delete', old.rowid, old.id, old.name, old.description, old.tags, old.signature, old.domain, old.example, old.notes, old.documentation, old.code); + INSERT INTO functions_fts(rowid, id, name, description, tags, signature, domain, example, notes, documentation, code) + VALUES (new.rowid, new.id, new.name, new.description, new.tags, new.signature, new.domain, new.example, new.notes, new.documentation, new.code); +END; + +-- Rebuild FTS for types: add examples, notes, documentation, code +DROP TRIGGER IF EXISTS types_ai; +DROP TRIGGER IF EXISTS types_ad; +DROP TRIGGER IF EXISTS types_au; + +INSERT INTO types_fts(types_fts) VALUES('rebuild'); +DROP TABLE IF EXISTS types_fts; + +CREATE VIRTUAL TABLE types_fts USING fts5( + id, + name, + description, + tags, + domain, + examples, + notes, + documentation, + code, + content='types', + content_rowid='rowid' +); + +-- Populate FTS from existing data +INSERT INTO types_fts(rowid, id, name, description, tags, domain, examples, notes, documentation, code) +SELECT rowid, id, name, description, tags, domain, examples, notes, documentation, code +FROM types; + +CREATE TRIGGER types_ai AFTER INSERT ON types BEGIN + INSERT INTO types_fts(rowid, id, name, description, tags, domain, examples, notes, documentation, code) + VALUES (new.rowid, new.id, new.name, new.description, new.tags, new.domain, new.examples, new.notes, new.documentation, new.code); +END; + +CREATE TRIGGER types_ad AFTER DELETE ON types BEGIN + INSERT INTO types_fts(types_fts, rowid, id, name, description, tags, domain, examples, notes, documentation, code) + VALUES ('delete', old.rowid, old.id, old.name, old.description, old.tags, old.domain, old.examples, old.notes, old.documentation, old.code); +END; + +CREATE TRIGGER types_au AFTER UPDATE ON types BEGIN + INSERT INTO types_fts(types_fts, rowid, id, name, description, tags, domain, examples, notes, documentation, code) + VALUES ('delete', old.rowid, old.id, old.name, old.description, old.tags, old.domain, old.examples, old.notes, old.documentation, old.code); + INSERT INTO types_fts(rowid, id, name, description, tags, domain, examples, notes, documentation, code) + VALUES (new.rowid, new.id, new.name, new.description, new.tags, new.domain, new.examples, new.notes, new.documentation, new.code); +END; diff --git a/registry/models.go b/registry/models.go index bf3b60a5..53dafa70 100644 --- a/registry/models.go +++ b/registry/models.go @@ -47,6 +47,9 @@ type Function struct { ErrorType string `json:"error_type"` Imports []string `json:"imports"` Example string `json:"example"` + Notes string `json:"notes"` + Documentation string `json:"documentation"` + Code string `json:"code"` Tested bool `json:"tested"` Tests []string `json:"tests"` TestFilePath string `json:"test_file_path"` @@ -82,9 +85,13 @@ type Type struct { Description string `json:"description"` Tags []string `json:"tags"` UsesTypes []string `json:"uses_types"` - FilePath string `json:"file_path"` - CreatedAt time.Time `json:"created_at"` - UpdatedAt time.Time `json:"updated_at"` + Examples string `json:"examples"` + Notes string `json:"notes"` + Documentation string `json:"documentation"` + Code string `json:"code"` + FilePath string `json:"file_path"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` } // ProposalKind classifies a proposal. diff --git a/registry/parser.go b/registry/parser.go index e4212667..a5875173 100644 --- a/registry/parser.go +++ b/registry/parser.go @@ -4,6 +4,7 @@ import ( "bytes" "fmt" "os" + "path/filepath" "strings" "gopkg.in/yaml.v3" @@ -73,7 +74,8 @@ func extractFrontmatter(data []byte) ([]byte, []byte, error) { } // ParseFunctionMD parses a function .md file into a Function. -func ParseFunctionMD(path string) (*Function, error) { +// root is the registry root directory, used to resolve file_path for code reading. +func ParseFunctionMD(path string, root string) (*Function, error) { data, err := os.ReadFile(path) if err != nil { return nil, fmt.Errorf("reading %s: %w", path, err) @@ -99,7 +101,7 @@ func ParseFunctionMD(path string) (*Function, error) { return nil, fmt.Errorf("%s: description is required", path) } - example := extractExample(body) + sections := extractSections(body) f := &Function{ ID: GenerateID(raw.Name, raw.Lang, raw.Domain), @@ -118,6 +120,9 @@ func ParseFunctionMD(path string) (*Function, error) { ReturnsOptional: raw.ReturnsOptional, ErrorType: raw.ErrorType, Imports: raw.Imports, + Example: sections.example, + Notes: sections.notes, + Documentation: sections.documentation, Tested: raw.Tested, Tests: raw.Tests, TestFilePath: raw.TestFilePath, @@ -129,21 +134,25 @@ func ParseFunctionMD(path string) (*Function, error) { Variant: raw.Variant, } - if example != "" && f.Example == "" { - f.Example = example + if root != "" && raw.FilePath != "" { + codePath := filepath.Join(root, raw.FilePath) + if codeData, err := os.ReadFile(codePath); err == nil { + f.Code = string(codeData) + } } return f, nil } // ParseTypeMD parses a type .md file into a Type. -func ParseTypeMD(path string) (*Type, error) { +// root is the registry root directory, used to resolve file_path for code reading. +func ParseTypeMD(path string, root string) (*Type, error) { data, err := os.ReadFile(path) if err != nil { return nil, fmt.Errorf("reading %s: %w", path, err) } - fm, _, err := extractFrontmatter(data) + fm, body, err := extractFrontmatter(data) if err != nil { return nil, fmt.Errorf("parsing %s: %w", path, err) } @@ -160,49 +169,96 @@ func ParseTypeMD(path string) (*Type, error) { return nil, fmt.Errorf("%s: description is required", path) } + sections := extractSections(body) + t := &Type{ - ID: GenerateID(raw.Name, raw.Lang, raw.Domain), - Name: raw.Name, - Lang: raw.Lang, - Domain: raw.Domain, - Version: raw.Version, - Algebraic: Algebraic(raw.Algebraic), - Definition: strings.TrimSpace(raw.Definition), - Description: raw.Description, - Tags: raw.Tags, - UsesTypes: raw.UsesTypes, - FilePath: raw.FilePath, + ID: GenerateID(raw.Name, raw.Lang, raw.Domain), + Name: raw.Name, + Lang: raw.Lang, + Domain: raw.Domain, + Version: raw.Version, + Algebraic: Algebraic(raw.Algebraic), + Definition: strings.TrimSpace(raw.Definition), + Description: raw.Description, + Tags: raw.Tags, + UsesTypes: raw.UsesTypes, + Examples: sections.example, + Notes: sections.notes, + Documentation: sections.documentation, + FilePath: raw.FilePath, + } + + if root != "" && raw.FilePath != "" { + codePath := filepath.Join(root, raw.FilePath) + if codeData, err := os.ReadFile(codePath); err == nil { + t.Code = string(codeData) + } } return t, nil } -// extractExample pulls the first code block after an "## Ejemplo" heading. -func extractExample(body []byte) string { +// bodySections holds the extracted sections from a .md body. +type bodySections struct { + example string // content under ## Ejemplo + notes string // content under ## Notas + documentation string // everything else +} + +// extractSections splits the markdown body into named sections. +// Known sections (## Ejemplo, ## Notas) are extracted separately. +// All other content (including unknown ## headings) goes into documentation. +func extractSections(body []byte) bodySections { lines := strings.Split(string(body), "\n") - inExample := false - inCode := false - var code []string + var s bodySections + + type section struct { + name string + lines []string + } + + var current *section + var sections []section for _, line := range lines { trimmed := strings.TrimSpace(line) - if strings.HasPrefix(trimmed, "## Ejemplo") { - inExample = true - continue - } - if inExample && !inCode && strings.HasPrefix(trimmed, "```") { - inCode = true - continue - } - if inCode { - if strings.HasPrefix(trimmed, "```") { - return strings.Join(code, "\n") + if strings.HasPrefix(trimmed, "## ") { + if current != nil { + sections = append(sections, *current) } - code = append(code, line) + current = §ion{name: trimmed} + continue } - if inExample && !inCode && strings.HasPrefix(trimmed, "##") { - break + if current != nil { + current.lines = append(current.lines, line) + } else { + // Content before any ## heading goes to documentation + sections = append(sections, section{name: "_preamble", lines: []string{line}}) } } - return "" + if current != nil { + sections = append(sections, *current) + } + + var docParts []string + for _, sec := range sections { + content := strings.TrimSpace(strings.Join(sec.lines, "\n")) + if content == "" && sec.name == "_preamble" { + continue + } + switch { + case strings.HasPrefix(sec.name, "## Ejemplo"): + s.example = content + case strings.HasPrefix(sec.name, "## Notas"): + s.notes = content + case sec.name == "_preamble": + docParts = append(docParts, content) + default: + // Unknown sections go to documentation with their heading + docParts = append(docParts, sec.name+"\n\n"+content) + } + } + s.documentation = strings.TrimSpace(strings.Join(docParts, "\n\n")) + + return s } diff --git a/registry/parser_test.go b/registry/parser_test.go index f393c13f..f44c05bd 100644 --- a/registry/parser_test.go +++ b/registry/parser_test.go @@ -79,7 +79,7 @@ imports: [react] tested: false tests: [] test_file_path: "" -file_path: "functions/components/DataTable.tsx" +file_path: "frontend/functions/ui/data_table.tsx" props: - name: data type: "T[]" @@ -105,7 +105,7 @@ func writeTempFile(t *testing.T, dir, name, content string) string { func TestParseFunctionMD(t *testing.T) { path := writeTempFile(t, t.TempDir(), "filter_slice.md", functionMD) - f, err := ParseFunctionMD(path) + f, err := ParseFunctionMD(path, "") if err != nil { t.Fatal(err) } @@ -130,7 +130,7 @@ func TestParseFunctionMD(t *testing.T) { func TestParseTypeMD(t *testing.T) { path := writeTempFile(t, t.TempDir(), "ohlcv.md", typeMD) - typ, err := ParseTypeMD(path) + typ, err := ParseTypeMD(path, "") if err != nil { t.Fatal(err) } @@ -149,7 +149,7 @@ func TestParseTypeMD(t *testing.T) { func TestParseComponentMD(t *testing.T) { path := writeTempFile(t, t.TempDir(), "DataTable.md", componentMD) - f, err := ParseFunctionMD(path) + f, err := ParseFunctionMD(path, "") if err != nil { t.Fatal(err) } @@ -174,7 +174,7 @@ func TestParseComponentMD(t *testing.T) { func TestParseMissingFrontmatter(t *testing.T) { path := writeTempFile(t, t.TempDir(), "bad.md", "# No frontmatter here\n") - _, err := ParseFunctionMD(path) + _, err := ParseFunctionMD(path, "") if err == nil { t.Error("expected error for missing frontmatter") } diff --git a/registry/store.go b/registry/store.go index 91b75801..dd063fc2 100644 --- a/registry/store.go +++ b/registry/store.go @@ -82,19 +82,22 @@ func (db *DB) InsertFunction(f *Function) error { description, tags, uses_functions, uses_types, returns, returns_optional, error_type, imports, example, tested, tests, test_file_path, file_path, created_at, updated_at, - props, emits, has_state, framework, variant + props, emits, has_state, framework, variant, + notes, documentation, code ) VALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, - ?, ?, ?, ?, ? + ?, ?, ?, ?, ?, + ?, ?, ? )`, f.ID, f.Name, string(f.Kind), f.Lang, f.Domain, f.Version, string(f.Purity), f.Signature, f.Description, marshalStrings(f.Tags), marshalStrings(f.UsesFunctions), marshalStrings(f.UsesTypes), marshalStrings(f.Returns), f.ReturnsOptional, f.ErrorType, marshalStrings(f.Imports), f.Example, f.Tested, marshalStrings(f.Tests), f.TestFilePath, f.FilePath, f.CreatedAt.Format(time.RFC3339), now, marshalProps(f.Props), marshalStrings(f.Emits), hasState, f.Framework, marshalStrings(f.Variant), + f.Notes, f.Documentation, f.Code, ) return err } @@ -115,11 +118,13 @@ func (db *DB) InsertType(t *Type) error { INSERT OR REPLACE INTO types ( id, name, lang, domain, version, algebraic, definition, description, tags, uses_types, - file_path, created_at, updated_at - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + file_path, created_at, updated_at, + examples, notes, documentation, code + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, t.ID, t.Name, t.Lang, t.Domain, t.Version, string(t.Algebraic), t.Definition, t.Description, marshalStrings(t.Tags), marshalStrings(t.UsesTypes), t.FilePath, t.CreatedAt.Format(time.RFC3339), now, + t.Examples, t.Notes, t.Documentation, t.Code, ) return err } @@ -270,6 +275,7 @@ func scanFunctions(rows interface{ Next() bool; Scan(...any) error }) ([]Functio &f.ReturnsOptional, &f.ErrorType, &importsJSON, &f.Example, &f.Tested, &testsJSON, &f.TestFilePath, &f.FilePath, &createdAt, &updatedAt, &propsJSON, &emitsJSON, &hasState, &f.Framework, &variantJSON, + &f.Notes, &f.Documentation, &f.Code, ) if err != nil { return nil, fmt.Errorf("scanning function: %w", err) @@ -308,6 +314,7 @@ func scanTypes(rows interface{ Next() bool; Scan(...any) error }) ([]Type, error &t.ID, &t.Name, &t.Lang, &t.Domain, &t.Version, &t.Algebraic, &t.Definition, &t.Description, &tagsJSON, &usesTypJSON, &t.FilePath, &createdAt, &updatedAt, + &t.Examples, &t.Notes, &t.Documentation, &t.Code, ) if err != nil { return nil, fmt.Errorf("scanning type: %w", err)