feat: campos documentation, notes y code en registry

Añade campos documentation, notes y code a functions y types.
El parser extrae el contenido del .md y el código fuente del archivo
referenciado en file_path. El indexer los almacena en SQLite y los
incluye en FTS5 para búsqueda sobre código y documentación.
Nueva migración 003_documentation.sql para añadir las columnas.
This commit is contained in:
2026-03-28 20:32:15 +01:00
parent 476696dd10
commit 49eecd0c87
7 changed files with 290 additions and 68 deletions
+21
View File
@@ -289,6 +289,15 @@ func printFunction(f *registry.Function) {
if f.Example != "" {
fmt.Printf("\nExample:\n%s\n", f.Example)
}
if f.Notes != "" {
fmt.Printf("\nNotes:\n%s\n", f.Notes)
}
if f.Documentation != "" {
fmt.Printf("\nDocumentation:\n%s\n", f.Documentation)
}
if f.Code != "" {
fmt.Printf("\nCode:\n%s\n", f.Code)
}
if f.Kind == registry.KindComponent {
fmt.Printf("Framework: %s\n", f.Framework)
if f.HasState != nil {
@@ -316,6 +325,18 @@ func printType(t *registry.Type) {
if t.Definition != "" {
fmt.Printf("\nDefinition:\n%s\n", t.Definition)
}
if t.Examples != "" {
fmt.Printf("\nExamples:\n%s\n", t.Examples)
}
if t.Notes != "" {
fmt.Printf("\nNotes:\n%s\n", t.Notes)
}
if t.Documentation != "" {
fmt.Printf("\nDocumentation:\n%s\n", t.Documentation)
}
if t.Code != "" {
fmt.Printf("\nCode:\n%s\n", t.Code)
}
}
// --- add ---
+46 -18
View File
@@ -19,6 +19,9 @@ type IndexResult struct {
// and populates the database. It uses two passes:
// 1. Parse all entries and collect known IDs
// 2. Validate references against known IDs, then insert valid entries
//
// Scans functions/ and types/ at the root level, plus any language-specific
// directories (e.g. python/functions/, python/types/).
func Index(db *DB, root string) (*IndexResult, error) {
if err := db.Purge(); err != nil {
return nil, fmt.Errorf("purging database: %w", err)
@@ -26,39 +29,50 @@ func Index(db *DB, root string) (*IndexResult, error) {
result := &IndexResult{}
// Pass 1: parse everything
// Pass 1: parse everything from all source directories
var functions []*Function
var types []*Type
functionsDir := filepath.Join(root, "functions")
if _, err := os.Stat(functionsDir); err == nil {
filepath.Walk(functionsDir, func(path string, info os.FileInfo, err error) error {
if err != nil || info.IsDir() || !strings.HasSuffix(path, ".md") {
return nil
// Directories to scan for functions and types.
// Base dirs + language-specific dirs discovered automatically.
funcDirs := []string{filepath.Join(root, "functions")}
typeDirs := []string{filepath.Join(root, "types")}
// Discover language-specific directories (e.g. python/functions/, python/types/)
entries, _ := os.ReadDir(root)
for _, e := range entries {
if !e.IsDir() {
continue
}
f, err := ParseFunctionMD(path)
langFuncs := filepath.Join(root, e.Name(), "functions")
if fi, err := os.Stat(langFuncs); err == nil && fi.IsDir() {
funcDirs = append(funcDirs, langFuncs)
}
langTypes := filepath.Join(root, e.Name(), "types")
if fi, err := os.Stat(langTypes); err == nil && fi.IsDir() {
typeDirs = append(typeDirs, langTypes)
}
}
for _, dir := range funcDirs {
walkMD(dir, func(path string) {
f, err := ParseFunctionMD(path, root)
if err != nil {
result.Errors = append(result.Errors, fmt.Sprintf("parse %s: %v", path, err))
return nil
return
}
functions = append(functions, f)
return nil
})
}
typesDir := filepath.Join(root, "types")
if _, err := os.Stat(typesDir); err == nil {
filepath.Walk(typesDir, func(path string, info os.FileInfo, err error) error {
if err != nil || info.IsDir() || !strings.HasSuffix(path, ".md") {
return nil
}
t, err := ParseTypeMD(path)
for _, dir := range typeDirs {
walkMD(dir, func(path string) {
t, err := ParseTypeMD(path, root)
if err != nil {
result.Errors = append(result.Errors, fmt.Sprintf("parse %s: %v", path, err))
return nil
return
}
types = append(types, t)
return nil
})
}
@@ -99,3 +113,17 @@ func Index(db *DB, root string) (*IndexResult, error) {
return result, nil
}
// walkMD walks a directory recursively and calls fn for each .md file found.
func walkMD(dir string, fn func(path string)) {
if _, err := os.Stat(dir); err != nil {
return
}
filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
if err != nil || info.IsDir() || !strings.HasSuffix(path, ".md") {
return nil
}
fn(path)
return nil
})
}
+103
View File
@@ -0,0 +1,103 @@
-- Add documentation fields to functions and types.
-- examples: extracted code blocks from ## Ejemplo
-- notes: extracted text from ## Notas
-- documentation: remaining body text from .md
-- code: source code from the referenced .go/.py/.tsx file
ALTER TABLE functions ADD COLUMN notes TEXT NOT NULL DEFAULT '';
ALTER TABLE functions ADD COLUMN documentation TEXT NOT NULL DEFAULT '';
ALTER TABLE functions ADD COLUMN code TEXT NOT NULL DEFAULT '';
ALTER TABLE types ADD COLUMN examples TEXT NOT NULL DEFAULT '';
ALTER TABLE types ADD COLUMN notes TEXT NOT NULL DEFAULT '';
ALTER TABLE types ADD COLUMN documentation TEXT NOT NULL DEFAULT '';
ALTER TABLE types ADD COLUMN code TEXT NOT NULL DEFAULT '';
-- Rebuild FTS for functions: add examples, notes, documentation, code
DROP TRIGGER IF EXISTS functions_ai;
DROP TRIGGER IF EXISTS functions_ad;
DROP TRIGGER IF EXISTS functions_au;
INSERT INTO functions_fts(functions_fts) VALUES('rebuild');
DROP TABLE IF EXISTS functions_fts;
CREATE VIRTUAL TABLE functions_fts USING fts5(
id,
name,
description,
tags,
signature,
domain,
example,
notes,
documentation,
code,
content='functions',
content_rowid='rowid'
);
-- Populate FTS from existing data
INSERT INTO functions_fts(rowid, id, name, description, tags, signature, domain, example, notes, documentation, code)
SELECT rowid, id, name, description, tags, signature, domain, example, notes, documentation, code
FROM functions;
CREATE TRIGGER functions_ai AFTER INSERT ON functions BEGIN
INSERT INTO functions_fts(rowid, id, name, description, tags, signature, domain, example, notes, documentation, code)
VALUES (new.rowid, new.id, new.name, new.description, new.tags, new.signature, new.domain, new.example, new.notes, new.documentation, new.code);
END;
CREATE TRIGGER functions_ad AFTER DELETE ON functions BEGIN
INSERT INTO functions_fts(functions_fts, rowid, id, name, description, tags, signature, domain, example, notes, documentation, code)
VALUES ('delete', old.rowid, old.id, old.name, old.description, old.tags, old.signature, old.domain, old.example, old.notes, old.documentation, old.code);
END;
CREATE TRIGGER functions_au AFTER UPDATE ON functions BEGIN
INSERT INTO functions_fts(functions_fts, rowid, id, name, description, tags, signature, domain, example, notes, documentation, code)
VALUES ('delete', old.rowid, old.id, old.name, old.description, old.tags, old.signature, old.domain, old.example, old.notes, old.documentation, old.code);
INSERT INTO functions_fts(rowid, id, name, description, tags, signature, domain, example, notes, documentation, code)
VALUES (new.rowid, new.id, new.name, new.description, new.tags, new.signature, new.domain, new.example, new.notes, new.documentation, new.code);
END;
-- Rebuild FTS for types: add examples, notes, documentation, code
DROP TRIGGER IF EXISTS types_ai;
DROP TRIGGER IF EXISTS types_ad;
DROP TRIGGER IF EXISTS types_au;
INSERT INTO types_fts(types_fts) VALUES('rebuild');
DROP TABLE IF EXISTS types_fts;
CREATE VIRTUAL TABLE types_fts USING fts5(
id,
name,
description,
tags,
domain,
examples,
notes,
documentation,
code,
content='types',
content_rowid='rowid'
);
-- Populate FTS from existing data
INSERT INTO types_fts(rowid, id, name, description, tags, domain, examples, notes, documentation, code)
SELECT rowid, id, name, description, tags, domain, examples, notes, documentation, code
FROM types;
CREATE TRIGGER types_ai AFTER INSERT ON types BEGIN
INSERT INTO types_fts(rowid, id, name, description, tags, domain, examples, notes, documentation, code)
VALUES (new.rowid, new.id, new.name, new.description, new.tags, new.domain, new.examples, new.notes, new.documentation, new.code);
END;
CREATE TRIGGER types_ad AFTER DELETE ON types BEGIN
INSERT INTO types_fts(types_fts, rowid, id, name, description, tags, domain, examples, notes, documentation, code)
VALUES ('delete', old.rowid, old.id, old.name, old.description, old.tags, old.domain, old.examples, old.notes, old.documentation, old.code);
END;
CREATE TRIGGER types_au AFTER UPDATE ON types BEGIN
INSERT INTO types_fts(types_fts, rowid, id, name, description, tags, domain, examples, notes, documentation, code)
VALUES ('delete', old.rowid, old.id, old.name, old.description, old.tags, old.domain, old.examples, old.notes, old.documentation, old.code);
INSERT INTO types_fts(rowid, id, name, description, tags, domain, examples, notes, documentation, code)
VALUES (new.rowid, new.id, new.name, new.description, new.tags, new.domain, new.examples, new.notes, new.documentation, new.code);
END;
+7
View File
@@ -47,6 +47,9 @@ type Function struct {
ErrorType string `json:"error_type"`
Imports []string `json:"imports"`
Example string `json:"example"`
Notes string `json:"notes"`
Documentation string `json:"documentation"`
Code string `json:"code"`
Tested bool `json:"tested"`
Tests []string `json:"tests"`
TestFilePath string `json:"test_file_path"`
@@ -82,6 +85,10 @@ type Type struct {
Description string `json:"description"`
Tags []string `json:"tags"`
UsesTypes []string `json:"uses_types"`
Examples string `json:"examples"`
Notes string `json:"notes"`
Documentation string `json:"documentation"`
Code string `json:"code"`
FilePath string `json:"file_path"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
+80 -24
View File
@@ -4,6 +4,7 @@ import (
"bytes"
"fmt"
"os"
"path/filepath"
"strings"
"gopkg.in/yaml.v3"
@@ -73,7 +74,8 @@ func extractFrontmatter(data []byte) ([]byte, []byte, error) {
}
// ParseFunctionMD parses a function .md file into a Function.
func ParseFunctionMD(path string) (*Function, error) {
// root is the registry root directory, used to resolve file_path for code reading.
func ParseFunctionMD(path string, root string) (*Function, error) {
data, err := os.ReadFile(path)
if err != nil {
return nil, fmt.Errorf("reading %s: %w", path, err)
@@ -99,7 +101,7 @@ func ParseFunctionMD(path string) (*Function, error) {
return nil, fmt.Errorf("%s: description is required", path)
}
example := extractExample(body)
sections := extractSections(body)
f := &Function{
ID: GenerateID(raw.Name, raw.Lang, raw.Domain),
@@ -118,6 +120,9 @@ func ParseFunctionMD(path string) (*Function, error) {
ReturnsOptional: raw.ReturnsOptional,
ErrorType: raw.ErrorType,
Imports: raw.Imports,
Example: sections.example,
Notes: sections.notes,
Documentation: sections.documentation,
Tested: raw.Tested,
Tests: raw.Tests,
TestFilePath: raw.TestFilePath,
@@ -129,21 +134,25 @@ func ParseFunctionMD(path string) (*Function, error) {
Variant: raw.Variant,
}
if example != "" && f.Example == "" {
f.Example = example
if root != "" && raw.FilePath != "" {
codePath := filepath.Join(root, raw.FilePath)
if codeData, err := os.ReadFile(codePath); err == nil {
f.Code = string(codeData)
}
}
return f, nil
}
// ParseTypeMD parses a type .md file into a Type.
func ParseTypeMD(path string) (*Type, error) {
// root is the registry root directory, used to resolve file_path for code reading.
func ParseTypeMD(path string, root string) (*Type, error) {
data, err := os.ReadFile(path)
if err != nil {
return nil, fmt.Errorf("reading %s: %w", path, err)
}
fm, _, err := extractFrontmatter(data)
fm, body, err := extractFrontmatter(data)
if err != nil {
return nil, fmt.Errorf("parsing %s: %w", path, err)
}
@@ -160,6 +169,8 @@ func ParseTypeMD(path string) (*Type, error) {
return nil, fmt.Errorf("%s: description is required", path)
}
sections := extractSections(body)
t := &Type{
ID: GenerateID(raw.Name, raw.Lang, raw.Domain),
Name: raw.Name,
@@ -171,38 +182,83 @@ func ParseTypeMD(path string) (*Type, error) {
Description: raw.Description,
Tags: raw.Tags,
UsesTypes: raw.UsesTypes,
Examples: sections.example,
Notes: sections.notes,
Documentation: sections.documentation,
FilePath: raw.FilePath,
}
if root != "" && raw.FilePath != "" {
codePath := filepath.Join(root, raw.FilePath)
if codeData, err := os.ReadFile(codePath); err == nil {
t.Code = string(codeData)
}
}
return t, nil
}
// extractExample pulls the first code block after an "## Ejemplo" heading.
func extractExample(body []byte) string {
// bodySections holds the extracted sections from a .md body.
type bodySections struct {
example string // content under ## Ejemplo
notes string // content under ## Notas
documentation string // everything else
}
// extractSections splits the markdown body into named sections.
// Known sections (## Ejemplo, ## Notas) are extracted separately.
// All other content (including unknown ## headings) goes into documentation.
func extractSections(body []byte) bodySections {
lines := strings.Split(string(body), "\n")
inExample := false
inCode := false
var code []string
var s bodySections
type section struct {
name string
lines []string
}
var current *section
var sections []section
for _, line := range lines {
trimmed := strings.TrimSpace(line)
if strings.HasPrefix(trimmed, "## Ejemplo") {
inExample = true
if strings.HasPrefix(trimmed, "## ") {
if current != nil {
sections = append(sections, *current)
}
current = &section{name: trimmed}
continue
}
if inExample && !inCode && strings.HasPrefix(trimmed, "```") {
inCode = true
if current != nil {
current.lines = append(current.lines, line)
} else {
// Content before any ## heading goes to documentation
sections = append(sections, section{name: "_preamble", lines: []string{line}})
}
}
if current != nil {
sections = append(sections, *current)
}
var docParts []string
for _, sec := range sections {
content := strings.TrimSpace(strings.Join(sec.lines, "\n"))
if content == "" && sec.name == "_preamble" {
continue
}
if inCode {
if strings.HasPrefix(trimmed, "```") {
return strings.Join(code, "\n")
}
code = append(code, line)
}
if inExample && !inCode && strings.HasPrefix(trimmed, "##") {
break
switch {
case strings.HasPrefix(sec.name, "## Ejemplo"):
s.example = content
case strings.HasPrefix(sec.name, "## Notas"):
s.notes = content
case sec.name == "_preamble":
docParts = append(docParts, content)
default:
// Unknown sections go to documentation with their heading
docParts = append(docParts, sec.name+"\n\n"+content)
}
}
return ""
s.documentation = strings.TrimSpace(strings.Join(docParts, "\n\n"))
return s
}
+5 -5
View File
@@ -79,7 +79,7 @@ imports: [react]
tested: false
tests: []
test_file_path: ""
file_path: "functions/components/DataTable.tsx"
file_path: "frontend/functions/ui/data_table.tsx"
props:
- name: data
type: "T[]"
@@ -105,7 +105,7 @@ func writeTempFile(t *testing.T, dir, name, content string) string {
func TestParseFunctionMD(t *testing.T) {
path := writeTempFile(t, t.TempDir(), "filter_slice.md", functionMD)
f, err := ParseFunctionMD(path)
f, err := ParseFunctionMD(path, "")
if err != nil {
t.Fatal(err)
}
@@ -130,7 +130,7 @@ func TestParseFunctionMD(t *testing.T) {
func TestParseTypeMD(t *testing.T) {
path := writeTempFile(t, t.TempDir(), "ohlcv.md", typeMD)
typ, err := ParseTypeMD(path)
typ, err := ParseTypeMD(path, "")
if err != nil {
t.Fatal(err)
}
@@ -149,7 +149,7 @@ func TestParseTypeMD(t *testing.T) {
func TestParseComponentMD(t *testing.T) {
path := writeTempFile(t, t.TempDir(), "DataTable.md", componentMD)
f, err := ParseFunctionMD(path)
f, err := ParseFunctionMD(path, "")
if err != nil {
t.Fatal(err)
}
@@ -174,7 +174,7 @@ func TestParseComponentMD(t *testing.T) {
func TestParseMissingFrontmatter(t *testing.T) {
path := writeTempFile(t, t.TempDir(), "bad.md", "# No frontmatter here\n")
_, err := ParseFunctionMD(path)
_, err := ParseFunctionMD(path, "")
if err == nil {
t.Error("expected error for missing frontmatter")
}
+11 -4
View File
@@ -82,19 +82,22 @@ func (db *DB) InsertFunction(f *Function) error {
description, tags, uses_functions, uses_types, returns,
returns_optional, error_type, imports, example, tested,
tests, test_file_path, file_path, created_at, updated_at,
props, emits, has_state, framework, variant
props, emits, has_state, framework, variant,
notes, documentation, code
) VALUES (
?, ?, ?, ?, ?, ?, ?, ?,
?, ?, ?, ?, ?,
?, ?, ?, ?, ?,
?, ?, ?, ?, ?,
?, ?, ?, ?, ?
?, ?, ?, ?, ?,
?, ?, ?
)`,
f.ID, f.Name, string(f.Kind), f.Lang, f.Domain, f.Version, string(f.Purity), f.Signature,
f.Description, marshalStrings(f.Tags), marshalStrings(f.UsesFunctions), marshalStrings(f.UsesTypes), marshalStrings(f.Returns),
f.ReturnsOptional, f.ErrorType, marshalStrings(f.Imports), f.Example, f.Tested,
marshalStrings(f.Tests), f.TestFilePath, f.FilePath, f.CreatedAt.Format(time.RFC3339), now,
marshalProps(f.Props), marshalStrings(f.Emits), hasState, f.Framework, marshalStrings(f.Variant),
f.Notes, f.Documentation, f.Code,
)
return err
}
@@ -115,11 +118,13 @@ func (db *DB) InsertType(t *Type) error {
INSERT OR REPLACE INTO types (
id, name, lang, domain, version, algebraic,
definition, description, tags, uses_types,
file_path, created_at, updated_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
file_path, created_at, updated_at,
examples, notes, documentation, code
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
t.ID, t.Name, t.Lang, t.Domain, t.Version, string(t.Algebraic),
t.Definition, t.Description, marshalStrings(t.Tags), marshalStrings(t.UsesTypes),
t.FilePath, t.CreatedAt.Format(time.RFC3339), now,
t.Examples, t.Notes, t.Documentation, t.Code,
)
return err
}
@@ -270,6 +275,7 @@ func scanFunctions(rows interface{ Next() bool; Scan(...any) error }) ([]Functio
&f.ReturnsOptional, &f.ErrorType, &importsJSON, &f.Example, &f.Tested,
&testsJSON, &f.TestFilePath, &f.FilePath, &createdAt, &updatedAt,
&propsJSON, &emitsJSON, &hasState, &f.Framework, &variantJSON,
&f.Notes, &f.Documentation, &f.Code,
)
if err != nil {
return nil, fmt.Errorf("scanning function: %w", err)
@@ -308,6 +314,7 @@ func scanTypes(rows interface{ Next() bool; Scan(...any) error }) ([]Type, error
&t.ID, &t.Name, &t.Lang, &t.Domain, &t.Version, &t.Algebraic,
&t.Definition, &t.Description, &tagsJSON, &usesTypJSON,
&t.FilePath, &createdAt, &updatedAt,
&t.Examples, &t.Notes, &t.Documentation, &t.Code,
)
if err != nil {
return nil, fmt.Errorf("scanning type: %w", err)