fn_registry/functions/infra/generate_proposals_from_telemetry.go

package infra

import (
	"crypto/sha1"
	"database/sql"
	"encoding/json"
	"fmt"
	"path/filepath"
	"strings"
	"time"

	_ "github.com/mattn/go-sqlite3"
)

// ProposalDraft is a candidate proposal computed from call_monitor telemetry
// + registry static data. Persistence step writes it into registry.db.proposals
// with deterministic id (so re-runs are idempotent).
type ProposalDraft struct {
	ID          string         `json:"id"`
	Kind        string         `json:"kind"`         // new_function | improve_function | deprecate_function | new_pipeline | ...
	TargetID    string         `json:"target_id"`    // existing function id (empty for new_function)
	Title       string         `json:"title"`
	Description string         `json:"description"`
	RuleID      string         `json:"rule_id"`      // copy_detected | orphan | bug | wrapper_skipped | ...
	Evidence    map[string]any `json:"evidence"`
}

// GenerateProposalsFromTelemetry inspects call_monitor.operations.db and
// registry.db, applies the rules documented in issue 0085, and returns the
// proposed drafts. It does NOT write to any DB. Call PersistProposalDrafts
// to upsert into registry.db.proposals.
//
// Implemented rules (MVP):
//   - copy_detected → improve_function (target = matched registry_id)
//   - orphan        → deprecate_function (calls_90d=0 AND writes_count=0
//                     AND no upstream consumer in registry.db.functions.uses_functions)
//   - bug           → improve_function (error_rate > 0.1 AND calls_total > 3)
//   - wrapper_skip  → improve_function (violations.function_id grouped, count > 3)
//
// Skipped rules (no data yet): perf_regression, test_flaky, e2e_blast_radius,
// pattern_inline (requires patterns table populated).
func GenerateProposalsFromTelemetry(registryRoot string) ([]ProposalDraft, error) {
	regPath := filepath.Join(registryRoot, "registry.db")
	monPath := filepath.Join(registryRoot, "projects", "fn_monitoring", "apps", "call_monitor", "operations.db")

	regDB, err := sql.Open("sqlite3", regPath+"?_journal_mode=WAL")
	if err != nil {
		return nil, fmt.Errorf("open registry: %w", err)
	}
	defer regDB.Close()

	monDB, err := sql.Open("sqlite3", monPath+"?_journal_mode=WAL")
	if err != nil {
		return nil, fmt.Errorf("open call_monitor: %w", err)
	}
	defer monDB.Close()

	// Build set of "function_ids consumed by anything upstream" from registry
	// (functions/apps/analysis.uses_functions JSON array). Used to filter orphan
	// candidates.
	consumers, err := loadConsumerSet(regDB)
	if err != nil {
		return nil, err
	}

	var drafts []ProposalDraft

	// --- Rule 1: copy_detected ---
	rows, err := monDB.Query(`SELECT app_file, app_function, registry_id, kind, similarity
		FROM copied_code ORDER BY detected_at DESC`)
	if err == nil {
		for rows.Next() {
			var appFile, appFunc, regID, kind string
			var sim float64
			if err := rows.Scan(&appFile, &appFunc, &regID, &kind, &sim); err != nil {
				continue
			}
			drafts = append(drafts, ProposalDraft{
				ID:       deterministicID("copy_detected", regID, appFile+":"+appFunc),
				Kind:     "improve_function",
				TargetID: regID,
				Title:    fmt.Sprintf("Replace inline copy with import: %s in %s::%s", regID, shortPath(appFile), appFunc),
				Description: fmt.Sprintf("App `%s` declares `%s` with a body matching `%s` (%s, similarity %.2f). "+
					"Replace the inline body with a registry import to centralize maintenance.",
					appFile, appFunc, regID, kind, sim),
				RuleID: "copy_detected",
				Evidence: map[string]any{
					"rule_id":      "copy_detected",
					"app_file":     appFile,
					"app_function": appFunc,
					"registry_id":  regID,
					"kind":         kind,
					"similarity":   sim,
				},
			})
		}
		rows.Close()
	}

	// --- Rule 2: orphan (deprecate_function) ---
	// function_stats rows with no calls in 90d AND no writes.
	// Cross-check against consumers set: if anybody imports it, NOT orphan.
	rows, err = monDB.Query(`
		SELECT function_id, calls_90d, writes_count
		FROM function_stats
		WHERE calls_90d = 0 AND writes_count = 0`)
	if err == nil {
		for rows.Next() {
			var fnID string
			var calls90d, writes int
			if err := rows.Scan(&fnID, &calls90d, &writes); err != nil {
				continue
			}
			if _, isConsumed := consumers[fnID]; isConsumed {
				continue
			}
			drafts = append(drafts, ProposalDraft{
				ID:       deterministicID("orphan", fnID, ""),
				Kind:     "deprecate_function",
				TargetID: fnID,
				Title:    "Deprecate orphan function: " + fnID,
				Description: fmt.Sprintf("`%s` has no calls in 90d, no recent writes, and no upstream consumer "+
					"declared in registry. Candidate for deprecation. Verify with `fn doctor unused` and manual review.", fnID),
				RuleID: "orphan",
				Evidence: map[string]any{
					"rule_id":      "orphan",
					"calls_90d":    calls90d,
					"writes_count": writes,
				},
			})
		}
		rows.Close()
	}

	// --- Rule 3: bug (improve_function on error_rate) ---
	rows, err = monDB.Query(`
		SELECT function_id, calls_total, errors_total, error_rate
		FROM function_stats
		WHERE error_rate > 0.1 AND calls_total > 3`)
	if err == nil {
		for rows.Next() {
			var fnID string
			var calls, errs int
			var rate float64
			if err := rows.Scan(&fnID, &calls, &errs, &rate); err != nil {
				continue
			}
			drafts = append(drafts, ProposalDraft{
				ID:       deterministicID("bug", fnID, ""),
				Kind:     "improve_function",
				TargetID: fnID,
				Title:    fmt.Sprintf("Investigate high error rate: %s (%.0f%% over %d calls)", fnID, rate*100, calls),
				Description: fmt.Sprintf("`%s` fails %.1f%% of the time across %d calls (%d errors). "+
					"Check call_monitor.operations.db for recent error_class samples.",
					fnID, rate*100, calls, errs),
				RuleID: "bug",
				Evidence: map[string]any{
					"rule_id":      "bug",
					"calls_total":  calls,
					"errors_total": errs,
					"error_rate":   rate,
				},
			})
		}
		rows.Close()
	}

	// --- Rule 4: wrapper_skip (violations grouped) ---
	rows, err = monDB.Query(`
		SELECT function_id, COUNT(*) AS hits, MAX(severity) AS sev
		FROM violations
		WHERE function_id != ''
		GROUP BY function_id
		HAVING hits > 3`)
	if err == nil {
		for rows.Next() {
			var fnID, sev string
			var hits int
			if err := rows.Scan(&fnID, &hits, &sev); err != nil {
				continue
			}
			drafts = append(drafts, ProposalDraft{
				ID:       deterministicID("wrapper_skip", fnID, ""),
				Kind:     "improve_function",
				TargetID: fnID,
				Title:    fmt.Sprintf("Wrapper skipped %d times: %s", hits, fnID),
				Description: fmt.Sprintf("Agent bypassed the registry wrapper for `%s` %d times. "+
					"API surface may be incomplete — check what callers needed and extend.", fnID, hits),
				RuleID: "wrapper_skip",
				Evidence: map[string]any{
					"rule_id":  "wrapper_skip",
					"hits":     hits,
					"severity": sev,
				},
			})
		}
		rows.Close()
	}

	return drafts, nil
}

// PersistProposalDrafts upserts each draft into registry.db.proposals via
// INSERT OR IGNORE so previously-reviewed proposals (approved/rejected) are
// preserved untouched. Returns (newly_inserted, total_seen, error).
func PersistProposalDrafts(registryRoot string, drafts []ProposalDraft) (int, int, error) {
	if len(drafts) == 0 {
		return 0, 0, nil
	}
	regPath := filepath.Join(registryRoot, "registry.db")
	conn, err := sql.Open("sqlite3", regPath+"?_journal_mode=WAL")
	if err != nil {
		return 0, 0, fmt.Errorf("open registry: %w", err)
	}
	defer conn.Close()

	tx, err := conn.Begin()
	if err != nil {
		return 0, 0, err
	}
	stmt, err := tx.Prepare(`INSERT OR IGNORE INTO proposals
		(id, kind, target_id, title, description, evidence, status, created_by, reviewed_by, created_at, updated_at)
		VALUES (?, ?, ?, ?, ?, ?, 'pending', 'reactive_loop', '', ?, ?)`)
	if err != nil {
		_ = tx.Rollback()
		return 0, 0, err
	}
	defer stmt.Close()

	now := time.Now().UTC().Format(time.RFC3339)
	inserted := 0
	for _, d := range drafts {
		evidence, _ := json.Marshal(d.Evidence)
		res, err := stmt.Exec(d.ID, d.Kind, d.TargetID, d.Title, d.Description, string(evidence), now, now)
		if err != nil {
			_ = tx.Rollback()
			return 0, 0, err
		}
		if n, _ := res.RowsAffected(); n > 0 {
			inserted++
		}
	}
	if err := tx.Commit(); err != nil {
		return 0, 0, err
	}
	return inserted, len(drafts), nil
}

// ---- Helpers ----

// loadConsumerSet returns every function id that appears in any uses_functions
// JSON array across functions, apps, and analysis tables. Used to filter orphan
// candidates.
func loadConsumerSet(db *sql.DB) (map[string]struct{}, error) {
	consumers := make(map[string]struct{})

	queries := []string{
		"SELECT uses_functions FROM functions WHERE uses_functions != '[]'",
		"SELECT uses_functions FROM apps WHERE uses_functions != '[]'",
		"SELECT uses_functions FROM analysis WHERE uses_functions != '[]'",
	}
	for _, q := range queries {
		rows, err := db.Query(q)
		if err != nil {
			// Table may not exist (e.g. analysis); ignore.
			continue
		}
		for rows.Next() {
			var raw string
			if err := rows.Scan(&raw); err != nil {
				continue
			}
			var ids []string
			if err := json.Unmarshal([]byte(raw), &ids); err != nil {
				continue
			}
			for _, id := range ids {
				consumers[id] = struct{}{}
			}
		}
		rows.Close()
	}
	return consumers, nil
}

// deterministicID returns a stable proposal id derived from (rule, target, extra).
// Format: "auto_<rule>_<sha1-10>" so re-running the generator never inserts
// duplicates of the same logical proposal.
func deterministicID(rule, target, extra string) string {
	key := rule + "|" + target + "|" + extra
	h := sha1.Sum([]byte(key))
	short := fmt.Sprintf("%x", h)[:10]
	return "auto_" + rule + "_" + short
}

func shortPath(p string) string {
	idx := strings.LastIndex(p, "/")
	if idx == -1 || idx == len(p)-1 {
		return p
	}
	parent := p[:idx]
	pidx := strings.LastIndex(parent, "/")
	if pidx == -1 {
		return p
	}
	return p[pidx+1:]
}