fn_registry/registry/indexer.go

package registry

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"
)

// IndexResult holds stats from an indexing run.
type IndexResult struct {
	Functions        int
	Types            int
	Apps             int
	ValidationErrors []string
	Errors           []string
}

// Index walks the registry root, parses all .md files, validates integrity,
// and populates the database. It uses two passes:
// 1. Parse all entries and collect known IDs
// 2. Validate references against known IDs, then insert valid entries
//
// Scans functions/ and types/ at the root level, plus any language-specific
// directories (e.g. python/functions/, python/types/).
func Index(db *DB, root string) (*IndexResult, error) {
	if err := db.Purge(); err != nil {
		return nil, fmt.Errorf("purging database: %w", err)
	}

	result := &IndexResult{}

	// Pass 1: parse everything from all source directories
	var functions []*Function
	var types []*Type

	// Directories to scan for functions and types.
	// Base dirs + language-specific dirs discovered automatically.
	funcDirs := []string{filepath.Join(root, "functions")}
	typeDirs := []string{filepath.Join(root, "types")}

	// Discover language-specific directories (e.g. python/functions/, python/types/)
	entries, _ := os.ReadDir(root)
	for _, e := range entries {
		if !e.IsDir() {
			continue
		}
		langFuncs := filepath.Join(root, e.Name(), "functions")
		if fi, err := os.Stat(langFuncs); err == nil && fi.IsDir() {
			funcDirs = append(funcDirs, langFuncs)
		}
		langTypes := filepath.Join(root, e.Name(), "types")
		if fi, err := os.Stat(langTypes); err == nil && fi.IsDir() {
			typeDirs = append(typeDirs, langTypes)
		}
	}

	for _, dir := range funcDirs {
		walkMD(dir, func(path string) {
			f, err := ParseFunctionMD(path, root)
			if err != nil {
				result.Errors = append(result.Errors, fmt.Sprintf("parse %s: %v", path, err))
				return
			}
			functions = append(functions, f)
		})
	}

	for _, dir := range typeDirs {
		walkMD(dir, func(path string) {
			t, err := ParseTypeMD(path, root)
			if err != nil {
				result.Errors = append(result.Errors, fmt.Sprintf("parse %s: %v", path, err))
				return
			}
			types = append(types, t)
		})
	}

	// Parse apps from apps/*/app.md
	var apps []*App
	appsDir := filepath.Join(root, "apps")
	if fi, err := os.Stat(appsDir); err == nil && fi.IsDir() {
		entries, _ := os.ReadDir(appsDir)
		for _, e := range entries {
			if !e.IsDir() {
				continue
			}
			appMD := filepath.Join(appsDir, e.Name(), "app.md")
			if _, err := os.Stat(appMD); err != nil {
				continue
			}
			a, err := ParseAppMD(appMD, root)
			if err != nil {
				result.Errors = append(result.Errors, fmt.Sprintf("parse %s: %v", appMD, err))
				continue
			}
			apps = append(apps, a)
		}
	}

	// Build known ID sets
	knownFunctions := make(map[string]bool, len(functions))
	for _, f := range functions {
		knownFunctions[f.ID] = true
	}
	knownTypes := make(map[string]bool, len(types))
	for _, t := range types {
		knownTypes[t.ID] = true
	}

	// Pass 2: validate and insert
	for _, t := range types {
		if verr := ValidateType(t, knownTypes); verr != nil {
			result.ValidationErrors = append(result.ValidationErrors, verr.Error())
			continue
		}
		if err := db.InsertType(t); err != nil {
			result.Errors = append(result.Errors, fmt.Sprintf("insert %s: %v", t.ID, err))
			continue
		}
		result.Types++
	}

	for _, f := range functions {
		if verr := ValidateFunction(f, knownFunctions, knownTypes); verr != nil {
			result.ValidationErrors = append(result.ValidationErrors, verr.Error())
			continue
		}
		if err := db.InsertFunction(f); err != nil {
			result.Errors = append(result.Errors, fmt.Sprintf("insert %s: %v", f.ID, err))
			continue
		}
		result.Functions++
	}

	for _, a := range apps {
		if verr := ValidateApp(a, knownFunctions, knownTypes); verr != nil {
			result.ValidationErrors = append(result.ValidationErrors, verr.Error())
			continue
		}
		if err := db.InsertApp(a); err != nil {
			result.Errors = append(result.Errors, fmt.Sprintf("insert %s: %v", a.ID, err))
			continue
		}
		result.Apps++
	}

	return result, nil
}

// walkMD walks a directory recursively and calls fn for each .md file found.
func walkMD(dir string, fn func(path string)) {
	if _, err := os.Stat(dir); err != nil {
		return
	}
	filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
		if err != nil || info.IsDir() || !strings.HasSuffix(path, ".md") {
			return nil
		}
		fn(path)
		return nil
	})
}