diff --git a/functions/infra/audit_uses_functions.go b/functions/infra/audit_uses_functions.go index 79715702..ea525da1 100644 --- a/functions/infra/audit_uses_functions.go +++ b/functions/infra/audit_uses_functions.go @@ -30,6 +30,7 @@ type auditFnMeta struct { domain string lang string signature string + filePath string // registry-relative path to the .go source (Go funcs only) } // skipDirs are directory names ignored when walking source for audits. @@ -80,15 +81,16 @@ func AuditUsesFunctions(registryRoot string) ([]UsesFunctionsAudit, error) { return nil, fmt.Errorf("audit_uses_functions: ping db: %w", err) } - // Load all Go/Python/TS functions from registry: id → name, domain, lang, signature. - rows, err := db.Query(`SELECT id, name, domain, lang, COALESCE(signature, '') FROM functions WHERE lang IN ('go','py','ts')`) + // Load all Go/Python/TS functions from registry: id → name, domain, lang, + // signature, file_path. file_path feeds the Go .go fallback (see auditGoApp). + rows, err := db.Query(`SELECT id, name, domain, lang, COALESCE(signature, ''), COALESCE(file_path, '') FROM functions WHERE lang IN ('go','py','ts')`) if err != nil { return nil, fmt.Errorf("audit_uses_functions: query functions: %w", err) } allFunctions := make(map[string]auditFnMeta) // id → meta for rows.Next() { var m auditFnMeta - if err := rows.Scan(&m.id, &m.name, &m.domain, &m.lang, &m.signature); err != nil { + if err := rows.Scan(&m.id, &m.name, &m.domain, &m.lang, &m.signature, &m.filePath); err != nil { continue } allFunctions[m.id] = m @@ -144,7 +146,7 @@ func AuditUsesFunctions(registryRoot string) ([]UsesFunctionsAudit, error) { switch app.lang { case "go": - importedIDs = append(importedIDs, auditGoApp(absDir, allFunctions)...) + importedIDs = append(importedIDs, auditGoApp(absDir, allFunctions, registryRoot)...) scannedLangs["go"] = true case "py": importedIDs = append(importedIDs, auditPyApp(absDir, allFunctions)...) @@ -197,11 +199,18 @@ func AuditUsesFunctions(registryRoot string) ([]UsesFunctionsAudit, error) { // Strategy: // 1. Find all "fn-registry/functions/" import paths (production code only). // 2. For each domain, collect registry functions in that domain. -// 3. Grep source files for the exported symbol. The token tried first is the -// real Go func identifier parsed from the registry signature; fallback is -// PascalCase(name). Many functions deviate (e.g. sqlite_column_exists has -// `func ColumnExists`), so signature is the source of truth. -func auditGoApp(appDir string, all map[string]auditFnMeta) []string { +// 3. Grep source files for the exported symbol. Tokens tried, in order: +// a) the real Go func identifier parsed from the registry signature; +// b) PascalCase(name) (with commonAbbrevs); +// c) the real exported func read straight from the function's .go file. +// +// Many functions deviate from snake_case→PascalCase (e.g. sqlite_column_exists +// has `func ColumnExists`, wails_bind_crud has `func GenerateWailsCRUD`). The +// signature is usually the source of truth, but some signatures omit the `func` +// keyword or list a different primary symbol; step (c) reads the .go file as a +// last-resort fallback so those cases stop being false positives ("unused"). +// The .go read is cached per execution to avoid reopening the same file. +func auditGoApp(appDir string, all map[string]auditFnMeta, registryRoot string) []string { // Step 1: collect imported domains. importedDomains := collectGoImportedDomains(appDir) if len(importedDomains) == 0 { @@ -216,6 +225,10 @@ func auditGoApp(appDir string, all map[string]auditFnMeta) []string { return nil } + // Cache for the .go fallback: registry file_path → real exported func name. + // Populated lazily, only when the cheaper tokens fail to match. + goFileSymbolCache := make(map[string]string) + for _, m := range all { if m.lang != "go" { continue @@ -223,17 +236,76 @@ func auditGoApp(appDir string, all map[string]auditFnMeta) []string { if !importedDomains[m.domain] { continue } - tokens := goCandidateTokens(m) - for _, tok := range tokens { + matched := false + for _, tok := range goCandidateTokens(m) { if containsToken(blob, tok) { - used = append(used, m.id) + matched = true break } } + if !matched && goSignatureSymbol(m) == "" { + // Fallback (c): read the registry .go file and look for the real + // exported func name. Gated on an EMPTY signature symbol on purpose: + // when the signature already yields a concrete `func ` it is the + // authoritative symbol, so reading the .go (which can only guess the + // file's first exported func) must not override it. Several registry + // functions share one .go file via the "TU adicional" pattern (e.g. + // cdp_new_tab lives in cdp_list_tabs.go); without this gate the first + // func would be mis-attributed to every sibling and suppress real + // "unused" findings. The file read therefore only happens for the rare + // functions whose stored signature omits the `func` keyword. + if sym := goRealExportedName(registryRoot, m.filePath, goFileSymbolCache); sym != "" { + if containsToken(blob, sym) { + matched = true + } + } + } + if matched { + used = append(used, m.id) + } } return used } +// goRealExportedFnRe matches a top-level exported func declaration in a .go +// source file: `func Name(` or the generic form `func Name[T any](`. It captures +// the func identifier. Method declarations (`func (r *T) Name(`) are skipped on +// purpose — a registry function's primary symbol is a top-level func, and method +// names would risk spurious matches. Used by the .go fallback to recover the real +// symbol name when the registry signature/name heuristics fail. +var goRealExportedFnRe = regexp.MustCompile(`^func\s+([A-Z][A-Za-z0-9_]*)\s*[\(\[]`) + +// goRealExportedName reads the registry .go file at filePath (relative to +// registryRoot) and returns the first exported func identifier found. Results +// are memoised in cache (filePath → symbol, "" when the file is unreadable or +// has no exported func) so a file is opened at most once per audit run. +func goRealExportedName(registryRoot, filePath string, cache map[string]string) string { + if filePath == "" { + return "" + } + if sym, ok := cache[filePath]; ok { + return sym + } + cache[filePath] = "" // pre-seed so an unreadable file is not retried + abs := filePath + if !filepath.IsAbs(abs) { + abs = filepath.Join(registryRoot, filePath) + } + f, err := os.Open(abs) + if err != nil { + return "" + } + defer f.Close() + sc := bufio.NewScanner(f) + for sc.Scan() { + if m := goRealExportedFnRe.FindStringSubmatch(sc.Text()); m != nil { + cache[filePath] = m[1] + return m[1] + } + } + return "" +} + // goCandidateTokens returns the identifiers we try when looking for usages // of a Go function in source. Real exported name from signature first, // PascalCase(name) as fallback. @@ -241,10 +313,8 @@ var goSignatureFnRe = regexp.MustCompile(`^\s*func\s+(?:\([^)]*\)\s+)?([A-Z][A-Z func goCandidateTokens(m auditFnMeta) []string { out := []string{} - if m.signature != "" { - if match := goSignatureFnRe.FindStringSubmatch(m.signature); match != nil { - out = append(out, match[1]) - } + if sym := goSignatureSymbol(m); sym != "" { + out = append(out, sym) } pascal := snakeToPascal(m.name) if pascal != "" && (len(out) == 0 || out[0] != pascal) { @@ -253,6 +323,21 @@ func goCandidateTokens(m auditFnMeta) []string { return out } +// goSignatureSymbol returns the exported Go identifier parsed from the registry +// signature (`func Name(...)` or `func (r *T) Name(...)`), or "" when the +// signature is empty or does not start with a `func` declaration. A non-empty +// result is the authoritative symbol for the function and gates off the .go +// fallback in auditGoApp. +func goSignatureSymbol(m auditFnMeta) string { + if m.signature == "" { + return "" + } + if match := goSignatureFnRe.FindStringSubmatch(m.signature); match != nil { + return match[1] + } + return "" +} + // collectGoImportedDomains returns the set of registry domains imported by .go files. var goImportRe = regexp.MustCompile(`"fn-registry/functions/([a-z]+)"`) @@ -452,6 +537,34 @@ var commonAbbrevs = map[string]string{ "io": "IO", "ok": "OK", "ui": "UI", + // Issue 0057 — abbreviations verified consistent across the registry's own + // Go func names (each entry maps a real `func ` deviation). These only + // improve the PascalCase fallback; the signature and the .go fallback remain + // the primary sources of truth. Deliberately NOT added because the registry + // itself is inconsistent for them (mapping would create more mismatches than + // it fixes): "cdp" (uses Cdp: CdpGetHTML, CdpNavigate — not CDP) and + // "pdf" (CdpPrintPDF vs PdfSimpleReport). + "ohlcv": "OHLCV", + "duckdb": "DuckDB", + "clickhouse": "ClickHouse", + "nordvpn": "NordVPN", + "sha256": "SHA256", + "md5": "MD5", + "ansi": "ANSI", + "cidr": "CIDR", + "aead": "AEAD", + "pty": "PTY", + "vps": "VPS", + "wg": "WG", + "vt": "VT", + "fft": "FFT", + "ema": "EMA", + "rsi": "RSI", + "sma": "SMA", + "vwap": "VWAP", + "ax": "AX", + "e2e": "E2E", + "urls": "URLs", } // hasTSSources reports whether appDir contains any production .ts/.tsx files diff --git a/functions/infra/audit_uses_functions_test.go b/functions/infra/audit_uses_functions_test.go index 4a7132a4..c27fdbfc 100644 --- a/functions/infra/audit_uses_functions_test.go +++ b/functions/infra/audit_uses_functions_test.go @@ -148,6 +148,273 @@ func main() { fmt.Println("hello") } }) } +// TestSnakeToPascal_HandlesAbbreviations verifies the commonAbbrevs expansion +// (issue 0057, Fase 1). Each "want" is the exported Go symbol the registry +// actually uses for that snake_case name. It also pins the deliberate +// non-mappings (cdp, pdf): the registry's own convention is mixed-case there, +// so the abbreviation must NOT fire. +func TestSnakeToPascal_HandlesAbbreviations(t *testing.T) { + cases := []struct{ in, want string }{ + // New abbreviations added by issue 0057 (verified against real func names). + {"fetch_ohlcv", "FetchOHLCV"}, + {"normalize_ohlcv", "NormalizeOHLCV"}, + {"duckdb_open", "DuckDBOpen"}, + {"load_ohlcv_from_duckdb", "LoadOHLCVFromDuckDB"}, + {"clickhouse_open", "ClickHouseOpen"}, + {"nordvpn_container_run", "NordVPNContainerRun"}, + {"parse_nordvpn_status", "ParseNordVPNStatus"}, + {"hash_sha256", "HashSHA256"}, + {"hash_md5", "HashMD5"}, + {"strip_ansi", "StripANSI"}, + {"parse_ip_cidr", "ParseIPCIDR"}, + {"open_aead", "OpenAEAD"}, + {"seal_aead", "SealAEAD"}, + {"pty_capture_stream", "PTYCaptureStream"}, + {"setup_vps_app", "SetupVPSApp"}, + {"vps_setup_app", "VPSSetupApp"}, + {"wg_keygen", "WGKeygen"}, + {"wg_peer_add", "WGPeerAdd"}, + {"vt_render", "VTRender"}, + {"fft", "FFT"}, + {"ema", "EMA"}, + {"rsi", "RSI"}, + {"sma", "SMA"}, + {"vwap", "VWAP"}, + {"cdp_get_ax_outline", "CdpGetAXOutline"}, + {"audit_e2e_coverage", "AuditE2ECoverage"}, + {"e2e_run_checks", "E2ERunChecks"}, + {"extract_urls", "ExtractURLs"}, + // Pre-existing abbreviations (regression guard — must keep working). + {"http_json_response", "HTTPJSONResponse"}, + {"sqlite_open", "SQLiteOpen"}, + {"random_hex_id", "RandomHexID"}, + // Deliberate non-mappings: registry uses mixed-case (Cdp, Pdf) here, so + // the snake_case→Pascal conversion must leave them mixed-case. These are + // the cases the .go fallback (Fase 2) and the signature path cover. + {"cdp_get_html", "CdpGetHTML"}, + {"cdp_navigate", "CdpNavigate"}, + {"pdf_simple_report", "PdfSimpleReport"}, + } + for _, c := range cases { + if got := snakeToPascal(c.in); got != c.want { + t.Errorf("snakeToPascal(%q) = %q, want %q", c.in, got, c.want) + } + } +} + +// goFallbackEnv builds a minimal registry.db + app on disk for the .go fallback +// test. The registry function gen_wails_crud_go_infra mimics wails_bind_crud: +// its signature omits the `func` keyword (so the signature regex misses) and its +// PascalCase("gen_wails_crud")="GenWailsCRUD" differs from the real exported +// symbol "GenerateWailsCRUD". The app calls the real symbol. When writeFnFile is +// true, the registry .go file exists and the fallback can recover the symbol. +func goFallbackEnv(t *testing.T, fnFilePath string, writeFnFile bool) UsesFunctionsAudit { + t.Helper() + root := t.TempDir() + dbPath := filepath.Join(root, "registry.db") + db, err := sql.Open("sqlite3", dbPath) + if err != nil { + t.Fatal(err) + } + _, err = db.Exec(` +CREATE TABLE functions (id TEXT PRIMARY KEY, name TEXT, domain TEXT, lang TEXT, signature TEXT, file_path TEXT); +CREATE TABLE apps (id TEXT PRIMARY KEY, lang TEXT, dir_path TEXT, uses_functions TEXT DEFAULT '[]'); +`) + if err != nil { + t.Fatal(err) + } + _, err = db.Exec( + `INSERT INTO functions (id,name,domain,lang,signature,file_path) VALUES (?,?,?,?,?,?)`, + "gen_wails_crud_go_infra", "gen_wails_crud", "infra", "go", + "GenerateWailsCRUD(spec WailsCRUDSpec) string", fnFilePath, + ) + if err != nil { + t.Fatal(err) + } + _, err = db.Exec( + `INSERT INTO apps (id,lang,dir_path,uses_functions) VALUES (?,?,?,?)`, + "myapp_go_infra", "go", "apps/myapp", `["gen_wails_crud_go_infra"]`, + ) + if err != nil { + t.Fatal(err) + } + db.Close() + + if writeFnFile { + fnAbsDir := filepath.Join(root, filepath.Dir(fnFilePath)) + if err := os.MkdirAll(fnAbsDir, 0755); err != nil { + t.Fatal(err) + } + src := "package infra\n\ntype WailsCRUDSpec struct{}\n\nfunc GenerateWailsCRUD(spec WailsCRUDSpec) string { return \"\" }\n" + if err := os.WriteFile(filepath.Join(root, fnFilePath), []byte(src), 0644); err != nil { + t.Fatal(err) + } + } + + appDir := filepath.Join(root, "apps", "myapp") + if err := os.MkdirAll(appDir, 0755); err != nil { + t.Fatal(err) + } + appSrc := "package main\n\nimport (\n\t\"fmt\"\n\t\"fn-registry/functions/infra\"\n)\n\nfunc main() {\n\tfmt.Println(infra.GenerateWailsCRUD(infra.WailsCRUDSpec{}))\n}\n" + if err := os.WriteFile(filepath.Join(appDir, "main.go"), []byte(appSrc), 0644); err != nil { + t.Fatal(err) + } + + results, err := AuditUsesFunctions(root) + if err != nil { + t.Fatalf("AuditUsesFunctions: %v", err) + } + if len(results) != 1 { + t.Fatalf("expected 1 result, got %d", len(results)) + } + return results[0] +} + +// TestAuditUsesFunctions_GoFileFallback verifies the .go fallback (issue 0057, +// Fase 2): when neither the registry signature nor PascalCase(name) yields the +// real exported symbol, the auditor reads the function's .go file to recover it, +// so a genuinely-used function is not a false "unused". The error sub-case (file +// absent) shows the fallback degrades gracefully and the function is then +// correctly reported unused — proving the fallback is load-bearing. +func TestAuditUsesFunctions_GoFileFallback(t *testing.T) { + t.Run("golden: .go fallback recovers real symbol -> not unused", func(t *testing.T) { + got := goFallbackEnv(t, "functions/infra/gen_wails_crud.go", true) + if len(got.Unused) != 0 { + t.Errorf("Unused = %v, want [] (fallback should find GenerateWailsCRUD)", got.Unused) + } + if len(got.Missing) != 0 { + t.Errorf("Missing = %v, want []", got.Missing) + } + }) + + t.Run("error: missing .go file -> flagged unused, no crash", func(t *testing.T) { + got := goFallbackEnv(t, "functions/infra/gen_wails_crud.go", false) + if len(got.Unused) != 1 || got.Unused[0] != "gen_wails_crud_go_infra" { + t.Errorf("Unused = %v, want [gen_wails_crud_go_infra] (no fallback file to read)", got.Unused) + } + }) +} + +// TestAuditUsesFunctions_SharedGoFileNotMisattributed pins the regression caught +// during issue 0057 verification: several registry functions can share one .go +// file (the "TU adicional" pattern, e.g. cdp_new_tab living in cdp_list_tabs.go). +// Because they have valid signatures, the .go fallback must stay GATED OFF for +// them — otherwise the file's first exported func (here ListTabs) would be +// mis-attributed to a sibling (NewTab) and suppress a genuine "unused" finding. +// The app below uses only ListTabs; NewTab must remain flagged unused. +func TestAuditUsesFunctions_SharedGoFileNotMisattributed(t *testing.T) { + root := t.TempDir() + dbPath := filepath.Join(root, "registry.db") + db, err := sql.Open("sqlite3", dbPath) + if err != nil { + t.Fatal(err) + } + _, err = db.Exec(` +CREATE TABLE functions (id TEXT PRIMARY KEY, name TEXT, domain TEXT, lang TEXT, signature TEXT, file_path TEXT); +CREATE TABLE apps (id TEXT PRIMARY KEY, lang TEXT, dir_path TEXT, uses_functions TEXT DEFAULT '[]'); +INSERT INTO functions (id,name,domain,lang,signature,file_path) VALUES + ('list_tabs_go_browser','list_tabs','browser','go','func ListTabs() error','functions/browser/tabs.go'), + ('new_tab_go_browser','new_tab','browser','go','func NewTab() error','functions/browser/tabs.go'); +INSERT INTO apps (id,lang,dir_path,uses_functions) VALUES + ('tabsapp_go_browser','go','apps/tabsapp','["list_tabs_go_browser","new_tab_go_browser"]'); +`) + if err != nil { + t.Fatal(err) + } + db.Close() + + // Shared registry .go file: ListTabs is the FIRST exported func. + fnDir := filepath.Join(root, "functions", "browser") + if err := os.MkdirAll(fnDir, 0755); err != nil { + t.Fatal(err) + } + tabsSrc := "package browser\n\nfunc ListTabs() error { return nil }\n\nfunc NewTab() error { return nil }\n" + if err := os.WriteFile(filepath.Join(fnDir, "tabs.go"), []byte(tabsSrc), 0644); err != nil { + t.Fatal(err) + } + + // App calls only ListTabs, but declares both. + appDir := filepath.Join(root, "apps", "tabsapp") + if err := os.MkdirAll(appDir, 0755); err != nil { + t.Fatal(err) + } + appSrc := "package main\n\nimport (\n\t\"fmt\"\n\t\"fn-registry/functions/browser\"\n)\n\nfunc main() {\n\tfmt.Println(browser.ListTabs())\n}\n" + if err := os.WriteFile(filepath.Join(appDir, "main.go"), []byte(appSrc), 0644); err != nil { + t.Fatal(err) + } + + results, err := AuditUsesFunctions(root) + if err != nil { + t.Fatalf("AuditUsesFunctions: %v", err) + } + if len(results) != 1 { + t.Fatalf("expected 1 result, got %d", len(results)) + } + got := results[0] + if len(got.Unused) != 1 || got.Unused[0] != "new_tab_go_browser" { + t.Errorf("Unused = %v, want [new_tab_go_browser] (sibling must NOT rescue via shared file)", got.Unused) + } +} + +// TestGoRealExportedName verifies the .go symbol extractor: top-level exported +// funcs (plain and generic) are recovered, method receivers are skipped, the +// result is cached, and unreadable/empty paths return "" without error. +func TestGoRealExportedName(t *testing.T) { + root := t.TempDir() + if err := os.MkdirAll(filepath.Join(root, "functions", "infra"), 0755); err != nil { + t.Fatal(err) + } + // File whose first exported func is preceded by an unexported func + a method. + src := "package infra\n\n" + + "import \"fmt\"\n\n" + + "func helper() {}\n\n" + + "type T struct{}\n\n" + + "func (t *T) Save() {}\n\n" + + "func GenerateWailsCRUD(spec int) string { fmt.Println(spec); return \"\" }\n\n" + + "func WailsStreamData[X any](xs []X) {}\n" + rel := "functions/infra/sample.go" + if err := os.WriteFile(filepath.Join(root, rel), []byte(src), 0644); err != nil { + t.Fatal(err) + } + cache := map[string]string{} + + t.Run("golden: first top-level exported func (skips helper + method)", func(t *testing.T) { + if got := goRealExportedName(root, rel, cache); got != "GenerateWailsCRUD" { + t.Errorf("got %q, want GenerateWailsCRUD", got) + } + if cache[rel] != "GenerateWailsCRUD" { + t.Errorf("cache[%q] = %q, want GenerateWailsCRUD", rel, cache[rel]) + } + }) + + t.Run("edge: generic func form func Name[T any](", func(t *testing.T) { + genRel := "functions/infra/gen.go" + genSrc := "package infra\n\nfunc WailsStreamData[X any](xs []X) {}\n" + if err := os.WriteFile(filepath.Join(root, genRel), []byte(genSrc), 0644); err != nil { + t.Fatal(err) + } + if got := goRealExportedName(root, genRel, cache); got != "WailsStreamData" { + t.Errorf("got %q, want WailsStreamData", got) + } + }) + + t.Run("error: missing file -> empty string, cached", func(t *testing.T) { + missRel := "functions/infra/does_not_exist.go" + if got := goRealExportedName(root, missRel, cache); got != "" { + t.Errorf("got %q, want empty for missing file", got) + } + if v, ok := cache[missRel]; !ok || v != "" { + t.Errorf("missing file should be cached as empty, got ok=%v v=%q", ok, v) + } + }) + + t.Run("error: empty file_path -> empty string", func(t *testing.T) { + if got := goRealExportedName(root, "", cache); got != "" { + t.Errorf("got %q, want empty for empty path", got) + } + }) +} + // TestAuditUsesFunctions_MissingDir verifies that apps whose dir_path does not // exist on disk get an entry with nil Missing/Unused slices (cannot inspect). func TestAuditUsesFunctions_MissingDir(t *testing.T) {