package infra import ( "bufio" "database/sql" "encoding/json" "fmt" "os" "path/filepath" "regexp" "strings" "unicode" _ "github.com/mattn/go-sqlite3" ) // UsesFunctionsAudit holds the drift report for a single app. type UsesFunctionsAudit struct { AppID string // registry id, e.g. "kanban_go_tools" Lang string // "go" or "py" DirPath string // dir_path as stored in registry.db Missing []string // function IDs found in imports but absent from app.md uses_functions Unused []string // function IDs declared in app.md but not detected in code } // auditFnMeta holds registry metadata for a single function. type auditFnMeta struct { id string name string domain string lang string } // AuditUsesFunctions checks every Go and Python app registered in registry.db // and compares the uses_functions declared in the app.md manifest against the // functions actually imported by the app's source code. // // For Go apps it greps for "fn-registry/functions/" import paths, then // searches the source for the exported symbol derived from each function name // (snake_case → PascalCase) to achieve per-function granularity within a package. // // For Python apps it scans for "from import X" patterns where matches // a known registry domain, then resolves X to a function ID by matching the name // field in registry.db. // // Returns an error only if registry.db cannot be opened. Apps where dir_path // does not exist on disk are reported with Missing/Unused = nil (cannot inspect). func AuditUsesFunctions(registryRoot string) ([]UsesFunctionsAudit, error) { dbPath := filepath.Join(registryRoot, "registry.db") dsn := fmt.Sprintf("file:%s?mode=ro&_foreign_keys=on", dbPath) db, err := sql.Open("sqlite3", dsn) if err != nil { return nil, fmt.Errorf("audit_uses_functions: open db: %w", err) } defer db.Close() if err := db.Ping(); err != nil { return nil, fmt.Errorf("audit_uses_functions: ping db: %w", err) } // Load all Go/Python functions from registry: id → name, domain, lang. rows, err := db.Query(`SELECT id, name, domain, lang FROM functions WHERE lang IN ('go','py')`) if err != nil { return nil, fmt.Errorf("audit_uses_functions: query functions: %w", err) } allFunctions := make(map[string]auditFnMeta) // id → meta for rows.Next() { var m auditFnMeta if err := rows.Scan(&m.id, &m.name, &m.domain, &m.lang); err != nil { continue } allFunctions[m.id] = m } rows.Close() // Load apps with lang go or py. type appRow struct { id string lang string dirPath string usesFunctions []string } rows2, err := db.Query(`SELECT id, lang, dir_path, uses_functions FROM apps WHERE lang IN ('go','py')`) if err != nil { return nil, fmt.Errorf("audit_uses_functions: query apps: %w", err) } var apps []appRow for rows2.Next() { var a appRow var ufJSON string if err := rows2.Scan(&a.id, &a.lang, &a.dirPath, &ufJSON); err != nil { continue } _ = json.Unmarshal([]byte(ufJSON), &a.usesFunctions) apps = append(apps, a) } rows2.Close() var results []UsesFunctionsAudit for _, app := range apps { absDir := app.dirPath if !filepath.IsAbs(absDir) { absDir = filepath.Join(registryRoot, app.dirPath) } audit := UsesFunctionsAudit{ AppID: app.id, Lang: app.lang, DirPath: app.dirPath, } if _, err := os.Stat(absDir); os.IsNotExist(err) { // Cannot inspect — skip diff, leave Missing/Unused nil. results = append(results, audit) continue } var importedIDs []string switch app.lang { case "go": importedIDs = auditGoApp(absDir, allFunctions) case "py": importedIDs = auditPyApp(absDir, allFunctions) } declaredSet := make(map[string]bool) for _, id := range app.usesFunctions { declaredSet[id] = true } importedSet := make(map[string]bool) for _, id := range importedIDs { importedSet[id] = true } for id := range importedSet { if !declaredSet[id] { audit.Missing = append(audit.Missing, id) } } for id := range declaredSet { if !importedSet[id] { audit.Unused = append(audit.Unused, id) } } results = append(results, audit) } return results, nil } // auditGoApp returns function IDs detected in the Go source files of appDir. // Strategy: // 1. Find all "fn-registry/functions/" import paths. // 2. For each domain, collect registry functions in that domain. // 3. Grep source files for the exported symbol (PascalCase of name). // If any source file contains the token, the function is considered used. func auditGoApp(appDir string, all map[string]auditFnMeta) []string { // Step 1: collect imported domains. importedDomains := collectGoImportedDomains(appDir) if len(importedDomains) == 0 { return nil } // Step 2: for each function in those domains, grep for its exported name. var used []string // Read all .go source once into a single blob for fast search. blob := readGoSourceBlob(appDir) if blob == "" { return nil } for _, m := range all { if m.lang != "go" { continue } if !importedDomains[m.domain] { continue } exported := snakeToPascal(m.name) // Use word-boundary-like check: look for the token as a standalone identifier. // We check the domain qualifier pattern e.g. "infra.SQLiteOpen" or bare "SQLiteOpen(". if containsToken(blob, exported) { used = append(used, m.id) } } return used } // collectGoImportedDomains returns the set of registry domains imported by .go files. var goImportRe = regexp.MustCompile(`"fn-registry/functions/([a-z]+)"`) func collectGoImportedDomains(appDir string) map[string]bool { domains := make(map[string]bool) _ = filepath.Walk(appDir, func(path string, info os.FileInfo, err error) error { if err != nil || info.IsDir() || !strings.HasSuffix(path, ".go") { return nil } f, err := os.Open(path) if err != nil { return nil } defer f.Close() sc := bufio.NewScanner(f) for sc.Scan() { line := sc.Text() if m := goImportRe.FindStringSubmatch(line); m != nil { domains[m[1]] = true } } return nil }) return domains } // readGoSourceBlob concatenates all .go file contents in appDir. func readGoSourceBlob(appDir string) string { var sb strings.Builder _ = filepath.Walk(appDir, func(path string, info os.FileInfo, err error) error { if err != nil || info.IsDir() || !strings.HasSuffix(path, ".go") { return nil } data, err := os.ReadFile(path) if err != nil { return nil } sb.Write(data) sb.WriteByte('\n') return nil }) return sb.String() } // containsToken reports whether the exported symbol appears as an identifier // in src (preceded and followed by non-letter/non-digit/non-underscore runes, // or at string boundaries). This avoids matching substrings inside longer names. func containsToken(src, token string) bool { idx := 0 for { pos := strings.Index(src[idx:], token) if pos < 0 { return false } abs := idx + pos before := abs == 0 || !isIdentRune(rune(src[abs-1])) after := abs+len(token) >= len(src) || !isIdentRune(rune(src[abs+len(token)])) if before && after { return true } idx = abs + 1 } } func isIdentRune(r rune) bool { return r == '_' || unicode.IsLetter(r) || unicode.IsDigit(r) } // auditPyApp returns function IDs detected in the Python source of appDir. // Looks for: "from import X, Y" patterns and resolves X, Y to function IDs. var pyFromImportRe = regexp.MustCompile(`from\s+(\w+)\s+import\s+(.+)`) func auditPyApp(appDir string, all map[string]auditFnMeta) []string { // Build name→id map for py functions. nameToID := make(map[string]string) // "metabase_auth" → "metabase_auth_py_infra" for _, m := range all { if m.lang == "py" { nameToID[m.name] = m.id } } usedSet := make(map[string]bool) _ = filepath.Walk(appDir, func(path string, info os.FileInfo, err error) error { if err != nil || info.IsDir() || !strings.HasSuffix(path, ".py") { return nil } f, err := os.Open(path) if err != nil { return nil } defer f.Close() sc := bufio.NewScanner(f) for sc.Scan() { line := strings.TrimSpace(sc.Text()) if m := pyFromImportRe.FindStringSubmatch(line); m != nil { // m[2] = "X, Y, Z" or "X" names := strings.Split(m[2], ",") for _, nm := range names { nm = strings.TrimSpace(nm) nm = strings.Fields(nm)[0] // strip "as alias" if id, ok := nameToID[nm]; ok { usedSet[id] = true } } } } return nil }) var used []string for id := range usedSet { used = append(used, id) } return used } // snakeToPascal converts snake_case to PascalCase (Go exported name). // E.g. "sqlite_open" → "SQLiteOpen", "http_json_response" → "HTTPJSONResponse". // Common abbreviations are uppercased in full. var commonAbbrevs = map[string]string{ "http": "HTTP", "https": "HTTPS", "sql": "SQL", "sqlite": "SQLite", "url": "URL", "api": "API", "id": "ID", "db": "DB", "tls": "TLS", "tcp": "TCP", "udp": "UDP", "ip": "IP", "json": "JSON", "yaml": "YAML", "xml": "XML", "html": "HTML", "css": "CSS", "csv": "CSV", "ssh": "SSH", "jwt": "JWT", "oauth": "OAuth", "oauth2": "OAuth2", "spa": "SPA", "cors": "CORS", "rbac": "RBAC", "crud": "CRUD", "cli": "CLI", "cpu": "CPU", "gpu": "GPU", "os": "OS", "s3": "S3", "gcs": "GCS", "bq": "BQ", "ttl": "TTL", "rgb": "RGB", "rgba": "RGBA", "sse": "SSE", "ws": "WS", "smtp": "SMTP", "imap": "IMAP", "pop3": "POP3", "dns": "DNS", "vpn": "VPN", "cmd": "Cmd", "ctx": "Ctx", "cfg": "Cfg", "env": "Env", "io": "IO", "ok": "OK", "ui": "UI", } func snakeToPascal(s string) string { parts := strings.Split(s, "_") var sb strings.Builder for _, p := range parts { if p == "" { continue } if abbr, ok := commonAbbrevs[strings.ToLower(p)]; ok { sb.WriteString(abbr) } else { sb.WriteString(strings.ToUpper(p[:1]) + p[1:]) } } return sb.String() }