From 5546ce645379d6cef638032e289a8600aa09f4fa Mon Sep 17 00:00:00 2001 From: Egutierrez Date: Fri, 24 Apr 2026 20:22:33 +0200 Subject: [PATCH] chore: untrack sqlite_api + registry.db, expand fn_monitoring docs - sqlite_api se extrae a su propio repo Gitea (dataforge/sqlite_api), siguiendo la convencion de apps/*/ (cada app = su repo). - registry.db ya estaba en .gitignore (regenerable con fn index + fn sync), pero seguia tracked por historia. Destracked. - project.md de fn_monitoring ampliado con operacion completa: arranque del service (dev / start.sh / systemd user), flujo de datos dashboard, troubleshooting, como extender. Co-Authored-By: Claude Opus 4.7 (1M context) --- projects/fn_monitoring/apps/sqlite_api/app.md | 59 ---- .../fn_monitoring/apps/sqlite_api/config.go | 190 ---------- .../fn_monitoring/apps/sqlite_api/handlers.go | 324 ------------------ .../apps/sqlite_api/handlers_test.go | 276 --------------- .../fn_monitoring/apps/sqlite_api/main.go | 68 ---- .../fn_monitoring/apps/sqlite_api/start.sh | 40 --- projects/fn_monitoring/project.md | 222 +++++++++++- 7 files changed, 212 insertions(+), 967 deletions(-) delete mode 100644 projects/fn_monitoring/apps/sqlite_api/app.md delete mode 100644 projects/fn_monitoring/apps/sqlite_api/config.go delete mode 100644 projects/fn_monitoring/apps/sqlite_api/handlers.go delete mode 100644 projects/fn_monitoring/apps/sqlite_api/handlers_test.go delete mode 100644 projects/fn_monitoring/apps/sqlite_api/main.go delete mode 100755 projects/fn_monitoring/apps/sqlite_api/start.sh diff --git a/projects/fn_monitoring/apps/sqlite_api/app.md b/projects/fn_monitoring/apps/sqlite_api/app.md deleted file mode 100644 index 66f4d446..00000000 --- a/projects/fn_monitoring/apps/sqlite_api/app.md +++ /dev/null @@ -1,59 +0,0 @@ ---- -name: sqlite_api -lang: go -domain: infra -description: "API REST HTTP read-only sobre registry.db y operations.db de cada app. Permite consultas SQL (solo SELECT/PRAGMA), busqueda FTS5, exploracion de tablas y schema. Bind por defecto a localhost:8484." -tags: [service, api, sqlite, http, registry, fts5] -uses_functions: [] -uses_types: [] -framework: "net/http" -entry_point: "main.go" -dir_path: "projects/fn_monitoring/apps/sqlite_api" ---- - -## Uso - -```bash -# Arrancar (default: 127.0.0.1:8484) -cd apps/sqlite_api && go run -tags fts5 . - -# Bind personalizado -go run -tags fts5 . --bind 0.0.0.0:8484 -``` - -## Endpoints - -| Metodo | Path | Descripcion | -|--------|------|-------------| -| GET | `/health` | Health check | -| GET | `/api/databases` | Lista DBs disponibles (registry + ops:*) | -| GET | `/api/databases/:db/tables` | Tablas y vistas de una DB | -| GET | `/api/databases/:db/schema` | Schema SQL completo | -| POST | `/api/databases/:db/query` | Ejecuta query SQL read-only | -| GET | `/api/databases/:db/fts?q=...&table=...` | Busqueda FTS5 directa | - -## Seguridad - -- Solo queries SELECT, PRAGMA, WITH y EXPLAIN -- SQLite abierto con `?mode=ro` (read-only a nivel driver) -- Timeout de 5 segundos por query -- Bind a localhost por defecto -- CORS habilitado para acceso desde frontends - -## Bases de datos - -- `registry` — registry.db de la raiz -- `ops:{app}` — operations.db de apps/{app}/ y projects/*/apps/{app}/ - -Auto-descubre operations.db al arrancar escaneando apps/ y projects/*/apps/. - -## Health check - -```bash -curl http://localhost:8484/health -# {"status":"ok"} -``` - -## Puerto - -8484 (no colisiona con Metabase 3000, Jupyter 8888, deploy_server 9090). diff --git a/projects/fn_monitoring/apps/sqlite_api/config.go b/projects/fn_monitoring/apps/sqlite_api/config.go deleted file mode 100644 index a5622f4c..00000000 --- a/projects/fn_monitoring/apps/sqlite_api/config.go +++ /dev/null @@ -1,190 +0,0 @@ -package main - -import ( - "database/sql" - "fmt" - "os" - "path/filepath" - "strings" - "sync" - - _ "github.com/mattn/go-sqlite3" -) - -// DBEntry represents a registered database. -type DBEntry struct { - Alias string // "registry" or "ops:app_name" - Path string // absolute path to .db file - Kind string // "registry" or "operations" -} - -// DBPool manages read-only connections to registered databases. -type DBPool struct { - mu sync.RWMutex - entries map[string]DBEntry - conns map[string]*sql.DB -} - -func NewDBPool() *DBPool { - return &DBPool{ - entries: make(map[string]DBEntry), - conns: make(map[string]*sql.DB), - } -} - -// Register adds a database entry. Does not open the connection until first use. -func (p *DBPool) Register(entry DBEntry) { - p.mu.Lock() - defer p.mu.Unlock() - p.entries[entry.Alias] = entry -} - -// Get returns a read-only connection to the named database. -func (p *DBPool) Get(alias string) (*sql.DB, error) { - p.mu.RLock() - if db, ok := p.conns[alias]; ok { - p.mu.RUnlock() - return db, nil - } - p.mu.RUnlock() - - p.mu.Lock() - defer p.mu.Unlock() - - // Double-check after acquiring write lock. - if db, ok := p.conns[alias]; ok { - return db, nil - } - - entry, ok := p.entries[alias] - if !ok { - return nil, fmt.Errorf("database %q not found", alias) - } - - dsn := fmt.Sprintf("file:%s?mode=ro", entry.Path) - db, err := sql.Open("sqlite3", dsn) - if err != nil { - return nil, fmt.Errorf("opening %s: %w", alias, err) - } - if err := db.Ping(); err != nil { - db.Close() - return nil, fmt.Errorf("pinging %s: %w", alias, err) - } - db.SetMaxOpenConns(4) - p.conns[alias] = db - return db, nil -} - -// List returns all registered database entries. -func (p *DBPool) List() []DBEntry { - p.mu.RLock() - defer p.mu.RUnlock() - out := make([]DBEntry, 0, len(p.entries)) - for _, e := range p.entries { - out = append(out, e) - } - return out -} - -// Close closes all open connections. -func (p *DBPool) Close() { - p.mu.Lock() - defer p.mu.Unlock() - for _, db := range p.conns { - db.Close() - } -} - -// Config holds the server configuration. -type Config struct { - Bind string // address to bind (default "127.0.0.1:8484") - RegistryDir string // root of fn_registry (where registry.db lives) -} - -// DiscoverDatabases finds registry.db and all operations.db files. -func DiscoverDatabases(root string) []DBEntry { - var entries []DBEntry - - // registry.db - regPath := filepath.Join(root, "registry.db") - if _, err := os.Stat(regPath); err == nil { - entries = append(entries, DBEntry{ - Alias: "registry", - Path: regPath, - Kind: "registry", - }) - } - - // apps/*/operations.db - appsDir := filepath.Join(root, "apps") - dirEntries, err := os.ReadDir(appsDir) - if err != nil { - return entries - } - for _, d := range dirEntries { - if !d.IsDir() { - continue - } - opsPath := filepath.Join(appsDir, d.Name(), "operations.db") - if _, err := os.Stat(opsPath); err == nil { - entries = append(entries, DBEntry{ - Alias: "ops:" + d.Name(), - Path: opsPath, - Kind: "operations", - }) - } - } - - // projects/*/apps/*/operations.db - projectsDir := filepath.Join(root, "projects") - projEntries, err := os.ReadDir(projectsDir) - if err != nil { - return entries - } - for _, p := range projEntries { - if !p.IsDir() { - continue - } - projAppsDir := filepath.Join(projectsDir, p.Name(), "apps") - projAppEntries, err := os.ReadDir(projAppsDir) - if err != nil { - continue - } - for _, a := range projAppEntries { - if !a.IsDir() { - continue - } - opsPath := filepath.Join(projAppsDir, a.Name(), "operations.db") - if _, err := os.Stat(opsPath); err == nil { - entries = append(entries, DBEntry{ - Alias: "ops:" + a.Name(), - Path: opsPath, - Kind: "operations", - }) - } - } - } - - return entries -} - -// ValidateQuery checks that the SQL is a read-only statement (SELECT or PRAGMA). -func ValidateQuery(sql string) error { - trimmed := strings.TrimSpace(sql) - upper := strings.ToUpper(trimmed) - - // Remove leading comments - for strings.HasPrefix(upper, "--") { - idx := strings.Index(upper, "\n") - if idx < 0 { - return fmt.Errorf("query contains only comments") - } - upper = strings.TrimSpace(upper[idx+1:]) - } - - if strings.HasPrefix(upper, "SELECT") || strings.HasPrefix(upper, "PRAGMA") || - strings.HasPrefix(upper, "WITH") || strings.HasPrefix(upper, "EXPLAIN") { - return nil - } - return fmt.Errorf("only SELECT, PRAGMA, WITH, and EXPLAIN queries are allowed") -} diff --git a/projects/fn_monitoring/apps/sqlite_api/handlers.go b/projects/fn_monitoring/apps/sqlite_api/handlers.go deleted file mode 100644 index da4c054e..00000000 --- a/projects/fn_monitoring/apps/sqlite_api/handlers.go +++ /dev/null @@ -1,324 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "fmt" - "net/http" - "strings" - "time" -) - -const queryTimeout = 5 * time.Second - -// Server holds the HTTP handlers and DB pool. -type Server struct { - pool *DBPool -} - -func NewServer(pool *DBPool) *Server { - return &Server{pool: pool} -} - -// Routes registers all API routes on the given mux. -func (s *Server) Routes(mux *http.ServeMux) { - mux.HandleFunc("GET /health", s.handleHealth) - mux.HandleFunc("GET /api/databases", s.handleDatabases) - mux.HandleFunc("GET /api/databases/{db}/tables", s.handleTables) - mux.HandleFunc("GET /api/databases/{db}/schema", s.handleSchema) - mux.HandleFunc("POST /api/databases/{db}/query", s.handleQuery) - mux.HandleFunc("GET /api/databases/{db}/fts", s.handleFTS) -} - -func (s *Server) handleHealth(w http.ResponseWriter, r *http.Request) { - writeJSON(w, http.StatusOK, map[string]string{"status": "ok"}) -} - -func (s *Server) handleDatabases(w http.ResponseWriter, r *http.Request) { - entries := s.pool.List() - type dbInfo struct { - Alias string `json:"alias"` - Kind string `json:"kind"` - } - out := make([]dbInfo, len(entries)) - for i, e := range entries { - out[i] = dbInfo{Alias: e.Alias, Kind: e.Kind} - } - writeJSON(w, http.StatusOK, out) -} - -func (s *Server) handleTables(w http.ResponseWriter, r *http.Request) { - dbAlias := resolveDBAlias(r.PathValue("db")) - db, err := s.pool.Get(dbAlias) - if err != nil { - writeError(w, http.StatusNotFound, err.Error()) - return - } - - ctx, cancel := context.WithTimeout(r.Context(), queryTimeout) - defer cancel() - - rows, err := db.QueryContext(ctx, "SELECT type, name FROM sqlite_master WHERE type IN ('table','view') ORDER BY type, name") - if err != nil { - writeError(w, http.StatusInternalServerError, err.Error()) - return - } - defer rows.Close() - - type tableInfo struct { - Type string `json:"type"` - Name string `json:"name"` - } - var tables []tableInfo - for rows.Next() { - var t tableInfo - if err := rows.Scan(&t.Type, &t.Name); err != nil { - writeError(w, http.StatusInternalServerError, err.Error()) - return - } - tables = append(tables, t) - } - if tables == nil { - tables = []tableInfo{} - } - writeJSON(w, http.StatusOK, tables) -} - -func (s *Server) handleSchema(w http.ResponseWriter, r *http.Request) { - dbAlias := resolveDBAlias(r.PathValue("db")) - db, err := s.pool.Get(dbAlias) - if err != nil { - writeError(w, http.StatusNotFound, err.Error()) - return - } - - ctx, cancel := context.WithTimeout(r.Context(), queryTimeout) - defer cancel() - - rows, err := db.QueryContext(ctx, "SELECT sql FROM sqlite_master WHERE sql IS NOT NULL ORDER BY type, name") - if err != nil { - writeError(w, http.StatusInternalServerError, err.Error()) - return - } - defer rows.Close() - - var statements []string - for rows.Next() { - var stmt string - if err := rows.Scan(&stmt); err != nil { - writeError(w, http.StatusInternalServerError, err.Error()) - return - } - statements = append(statements, stmt) - } - writeJSON(w, http.StatusOK, map[string]any{ - "statements": statements, - "count": len(statements), - }) -} - -func (s *Server) handleQuery(w http.ResponseWriter, r *http.Request) { - dbAlias := resolveDBAlias(r.PathValue("db")) - db, err := s.pool.Get(dbAlias) - if err != nil { - writeError(w, http.StatusNotFound, err.Error()) - return - } - - var req struct { - SQL string `json:"sql"` - } - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - writeError(w, http.StatusBadRequest, "invalid JSON body") - return - } - if req.SQL == "" { - writeError(w, http.StatusBadRequest, "sql field is required") - return - } - if err := ValidateQuery(req.SQL); err != nil { - writeError(w, http.StatusBadRequest, err.Error()) - return - } - - ctx, cancel := context.WithTimeout(r.Context(), queryTimeout) - defer cancel() - - start := time.Now() - rows, err := db.QueryContext(ctx, req.SQL) - if err != nil { - writeError(w, http.StatusBadRequest, err.Error()) - return - } - defer rows.Close() - - columns, err := rows.Columns() - if err != nil { - writeError(w, http.StatusInternalServerError, err.Error()) - return - } - - var resultRows [][]any - for rows.Next() { - vals := make([]any, len(columns)) - ptrs := make([]any, len(columns)) - for i := range vals { - ptrs[i] = &vals[i] - } - if err := rows.Scan(ptrs...); err != nil { - writeError(w, http.StatusInternalServerError, err.Error()) - return - } - // Convert []byte to string for JSON serialization. - for i, v := range vals { - if b, ok := v.([]byte); ok { - vals[i] = string(b) - } - } - resultRows = append(resultRows, vals) - } - if resultRows == nil { - resultRows = [][]any{} - } - - durationMs := time.Since(start).Milliseconds() - writeJSON(w, http.StatusOK, map[string]any{ - "columns": columns, - "rows": resultRows, - "count": len(resultRows), - "duration_ms": durationMs, - }) -} - -func (s *Server) handleFTS(w http.ResponseWriter, r *http.Request) { - dbAlias := resolveDBAlias(r.PathValue("db")) - db, err := s.pool.Get(dbAlias) - if err != nil { - writeError(w, http.StatusNotFound, err.Error()) - return - } - - q := r.URL.Query().Get("q") - if q == "" { - writeError(w, http.StatusBadRequest, "q parameter is required") - return - } - - table := r.URL.Query().Get("table") - if table == "" { - table = "functions" - } - - // Map table to its FTS table and columns to return. - ftsTable := table + "_fts" - var selectCols string - switch table { - case "functions": - selectCols = "id, name, kind, purity, domain, description" - case "types": - selectCols = "id, name, algebraic, domain, description" - case "unit_tests": - selectCols = "id, name, function_id, lang" - default: - writeError(w, http.StatusBadRequest, fmt.Sprintf("unsupported FTS table: %s", table)) - return - } - - limit := r.URL.Query().Get("limit") - if limit == "" { - limit = "20" - } - - query := fmt.Sprintf( - "SELECT %s FROM %s WHERE id IN (SELECT id FROM %s WHERE %s MATCH ?) LIMIT %s", - selectCols, table, ftsTable, ftsTable, limit, - ) - - ctx, cancel := context.WithTimeout(r.Context(), queryTimeout) - defer cancel() - - start := time.Now() - rows, err := db.QueryContext(ctx, query, q) - if err != nil { - writeError(w, http.StatusBadRequest, err.Error()) - return - } - defer rows.Close() - - columns, err := rows.Columns() - if err != nil { - writeError(w, http.StatusInternalServerError, err.Error()) - return - } - - var resultRows [][]any - for rows.Next() { - vals := make([]any, len(columns)) - ptrs := make([]any, len(columns)) - for i := range vals { - ptrs[i] = &vals[i] - } - if err := rows.Scan(ptrs...); err != nil { - writeError(w, http.StatusInternalServerError, err.Error()) - return - } - for i, v := range vals { - if b, ok := v.([]byte); ok { - vals[i] = string(b) - } - } - resultRows = append(resultRows, vals) - } - if resultRows == nil { - resultRows = [][]any{} - } - - durationMs := time.Since(start).Milliseconds() - writeJSON(w, http.StatusOK, map[string]any{ - "columns": columns, - "rows": resultRows, - "count": len(resultRows), - "duration_ms": durationMs, - }) -} - -// resolveDBAlias converts URL path segment to pool alias. -// "registry" → "registry", "ops:app_name" → "ops:app_name" -// URL-encoded colons: "ops%3Aapp_name" → already decoded by net/http. -func resolveDBAlias(raw string) string { - return raw -} - -// CORS middleware for browser access. -func corsMiddleware(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Access-Control-Allow-Origin", "*") - w.Header().Set("Access-Control-Allow-Methods", "GET, POST, OPTIONS") - w.Header().Set("Access-Control-Allow-Headers", "Content-Type") - if r.Method == http.MethodOptions { - w.WriteHeader(http.StatusNoContent) - return - } - next.ServeHTTP(w, r) - }) -} - -func writeJSON(w http.ResponseWriter, status int, data any) { - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(status) - json.NewEncoder(w).Encode(data) -} - -func writeError(w http.ResponseWriter, status int, msg string) { - writeJSON(w, status, map[string]string{"error": msg}) -} - -// sanitizeFTS escapes special FTS5 characters in user input. -func sanitizeFTS(input string) string { - // Wrap each term in double quotes to treat as literal. - terms := strings.Fields(input) - for i, t := range terms { - terms[i] = `"` + strings.ReplaceAll(t, `"`, `""`) + `"` - } - return strings.Join(terms, " ") -} diff --git a/projects/fn_monitoring/apps/sqlite_api/handlers_test.go b/projects/fn_monitoring/apps/sqlite_api/handlers_test.go deleted file mode 100644 index 9f89c26d..00000000 --- a/projects/fn_monitoring/apps/sqlite_api/handlers_test.go +++ /dev/null @@ -1,276 +0,0 @@ -package main - -import ( - "database/sql" - "encoding/json" - "net/http" - "net/http/httptest" - "os" - "path/filepath" - "strings" - "testing" - - _ "github.com/mattn/go-sqlite3" -) - -func setupTestDB(t *testing.T) (*DBPool, string) { - t.Helper() - dir := t.TempDir() - dbPath := filepath.Join(dir, "test.db") - - // Create a small test database with FTS5. - db, err := sql.Open("sqlite3", dbPath) - if err != nil { - t.Fatal(err) - } - - stmts := []string{ - `CREATE TABLE items (id TEXT PRIMARY KEY, name TEXT, kind TEXT)`, - `INSERT INTO items VALUES ('a', 'alpha', 'first')`, - `INSERT INTO items VALUES ('b', 'beta', 'second')`, - `CREATE VIRTUAL TABLE items_fts USING fts5(id, name, kind, content=items, content_rowid=rowid)`, - `INSERT INTO items_fts(items_fts) VALUES('rebuild')`, - } - for _, s := range stmts { - if _, err := db.Exec(s); err != nil { - t.Fatalf("setup sql %q: %v", s, err) - } - } - db.Close() - - pool := NewDBPool() - pool.Register(DBEntry{Alias: "testdb", Path: dbPath, Kind: "test"}) - return pool, dir -} - -func TestHealthEndpoint(t *testing.T) { - pool := NewDBPool() - srv := NewServer(pool) - mux := http.NewServeMux() - srv.Routes(mux) - - req := httptest.NewRequest("GET", "/health", nil) - w := httptest.NewRecorder() - mux.ServeHTTP(w, req) - - if w.Code != http.StatusOK { - t.Fatalf("expected 200, got %d", w.Code) - } - var resp map[string]string - json.NewDecoder(w.Body).Decode(&resp) - if resp["status"] != "ok" { - t.Fatalf("expected status ok, got %s", resp["status"]) - } -} - -func TestDatabasesEndpoint(t *testing.T) { - pool := NewDBPool() - pool.Register(DBEntry{Alias: "registry", Path: "/fake/path", Kind: "registry"}) - pool.Register(DBEntry{Alias: "ops:myapp", Path: "/fake/path2", Kind: "operations"}) - - srv := NewServer(pool) - mux := http.NewServeMux() - srv.Routes(mux) - - req := httptest.NewRequest("GET", "/api/databases", nil) - w := httptest.NewRecorder() - mux.ServeHTTP(w, req) - - if w.Code != http.StatusOK { - t.Fatalf("expected 200, got %d", w.Code) - } - var resp []map[string]string - json.NewDecoder(w.Body).Decode(&resp) - if len(resp) != 2 { - t.Fatalf("expected 2 databases, got %d", len(resp)) - } -} - -func TestQueryEndpoint(t *testing.T) { - pool, _ := setupTestDB(t) - defer pool.Close() - - srv := NewServer(pool) - mux := http.NewServeMux() - srv.Routes(mux) - - body := `{"sql": "SELECT id, name FROM items ORDER BY id"}` - req := httptest.NewRequest("POST", "/api/databases/testdb/query", strings.NewReader(body)) - req.Header.Set("Content-Type", "application/json") - w := httptest.NewRecorder() - mux.ServeHTTP(w, req) - - if w.Code != http.StatusOK { - t.Fatalf("expected 200, got %d: %s", w.Code, w.Body.String()) - } - - var resp map[string]any - json.NewDecoder(w.Body).Decode(&resp) - if int(resp["count"].(float64)) != 2 { - t.Fatalf("expected 2 rows, got %v", resp["count"]) - } - cols := resp["columns"].([]any) - if len(cols) != 2 || cols[0] != "id" || cols[1] != "name" { - t.Fatalf("unexpected columns: %v", cols) - } -} - -func TestQueryRejectsWrite(t *testing.T) { - pool, _ := setupTestDB(t) - defer pool.Close() - - srv := NewServer(pool) - mux := http.NewServeMux() - srv.Routes(mux) - - cases := []string{ - `{"sql": "INSERT INTO items VALUES ('c', 'gamma', 'third')"}`, - `{"sql": "UPDATE items SET name = 'x' WHERE id = 'a'"}`, - `{"sql": "DELETE FROM items WHERE id = 'a'"}`, - `{"sql": "DROP TABLE items"}`, - } - - for _, body := range cases { - req := httptest.NewRequest("POST", "/api/databases/testdb/query", strings.NewReader(body)) - req.Header.Set("Content-Type", "application/json") - w := httptest.NewRecorder() - mux.ServeHTTP(w, req) - - if w.Code != http.StatusBadRequest { - t.Errorf("expected 400 for %s, got %d", body, w.Code) - } - } -} - -func TestTablesEndpoint(t *testing.T) { - pool, _ := setupTestDB(t) - defer pool.Close() - - srv := NewServer(pool) - mux := http.NewServeMux() - srv.Routes(mux) - - req := httptest.NewRequest("GET", "/api/databases/testdb/tables", nil) - w := httptest.NewRecorder() - mux.ServeHTTP(w, req) - - if w.Code != http.StatusOK { - t.Fatalf("expected 200, got %d: %s", w.Code, w.Body.String()) - } - - var resp []map[string]string - json.NewDecoder(w.Body).Decode(&resp) - if len(resp) == 0 { - t.Fatal("expected at least one table") - } - found := false - for _, tbl := range resp { - if tbl["name"] == "items" { - found = true - } - } - if !found { - t.Fatal("expected 'items' table in response") - } -} - -func TestSchemaEndpoint(t *testing.T) { - pool, _ := setupTestDB(t) - defer pool.Close() - - srv := NewServer(pool) - mux := http.NewServeMux() - srv.Routes(mux) - - req := httptest.NewRequest("GET", "/api/databases/testdb/schema", nil) - w := httptest.NewRecorder() - mux.ServeHTTP(w, req) - - if w.Code != http.StatusOK { - t.Fatalf("expected 200, got %d: %s", w.Code, w.Body.String()) - } - - var resp map[string]any - json.NewDecoder(w.Body).Decode(&resp) - count := int(resp["count"].(float64)) - if count == 0 { - t.Fatal("expected at least one schema statement") - } -} - -func TestNotFoundDB(t *testing.T) { - pool := NewDBPool() - srv := NewServer(pool) - mux := http.NewServeMux() - srv.Routes(mux) - - body := `{"sql": "SELECT 1"}` - req := httptest.NewRequest("POST", "/api/databases/nonexistent/query", strings.NewReader(body)) - req.Header.Set("Content-Type", "application/json") - w := httptest.NewRecorder() - mux.ServeHTTP(w, req) - - if w.Code != http.StatusNotFound { - t.Fatalf("expected 404, got %d", w.Code) - } -} - -func TestValidateQuery(t *testing.T) { - valid := []string{ - "SELECT * FROM t", - " select id from t", - "PRAGMA table_info(t)", - "WITH cte AS (SELECT 1) SELECT * FROM cte", - "EXPLAIN SELECT * FROM t", - "-- comment\nSELECT 1", - } - for _, q := range valid { - if err := ValidateQuery(q); err != nil { - t.Errorf("expected valid: %q, got error: %v", q, err) - } - } - - invalid := []string{ - "INSERT INTO t VALUES (1)", - "UPDATE t SET x = 1", - "DELETE FROM t", - "DROP TABLE t", - "CREATE TABLE t (id INT)", - "ALTER TABLE t ADD COLUMN x INT", - } - for _, q := range invalid { - if err := ValidateQuery(q); err == nil { - t.Errorf("expected invalid: %q, got nil error", q) - } - } -} - -func TestDiscoverDatabases(t *testing.T) { - dir := t.TempDir() - - // Create registry.db - os.WriteFile(filepath.Join(dir, "registry.db"), []byte{}, 0644) - - // Create apps/myapp/operations.db - os.MkdirAll(filepath.Join(dir, "apps", "myapp"), 0755) - os.WriteFile(filepath.Join(dir, "apps", "myapp", "operations.db"), []byte{}, 0644) - - // Create projects/proj1/apps/papp/operations.db - os.MkdirAll(filepath.Join(dir, "projects", "proj1", "apps", "papp"), 0755) - os.WriteFile(filepath.Join(dir, "projects", "proj1", "apps", "papp", "operations.db"), []byte{}, 0644) - - entries := DiscoverDatabases(dir) - if len(entries) != 3 { - t.Fatalf("expected 3 entries, got %d: %+v", len(entries), entries) - } - - aliases := map[string]bool{} - for _, e := range entries { - aliases[e.Alias] = true - } - for _, want := range []string{"registry", "ops:myapp", "ops:papp"} { - if !aliases[want] { - t.Errorf("missing alias %q", want) - } - } -} diff --git a/projects/fn_monitoring/apps/sqlite_api/main.go b/projects/fn_monitoring/apps/sqlite_api/main.go deleted file mode 100644 index 231392c6..00000000 --- a/projects/fn_monitoring/apps/sqlite_api/main.go +++ /dev/null @@ -1,68 +0,0 @@ -package main - -import ( - "flag" - "fmt" - "log" - "net/http" - "os" - "path/filepath" -) - -func main() { - bind := flag.String("bind", "127.0.0.1:8484", "address to bind") - flag.Parse() - - root := findRegistryRoot() - if root == "" { - log.Fatal("cannot find fn_registry root (no registry.db found). Set FN_REGISTRY_ROOT or run from the registry directory.") - } - - pool := NewDBPool() - for _, entry := range DiscoverDatabases(root) { - pool.Register(entry) - log.Printf("registered database: %s (%s)", entry.Alias, entry.Path) - } - - srv := NewServer(pool) - mux := http.NewServeMux() - srv.Routes(mux) - - handler := corsMiddleware(mux) - - log.Printf("sqlite_api listening on %s (registry root: %s)", *bind, root) - if err := http.ListenAndServe(*bind, handler); err != nil { - log.Fatalf("server error: %v", err) - } -} - -// findRegistryRoot walks up from cwd (or uses FN_REGISTRY_ROOT) to find registry.db. -func findRegistryRoot() string { - if env := os.Getenv("FN_REGISTRY_ROOT"); env != "" { - if _, err := os.Stat(filepath.Join(env, "registry.db")); err == nil { - return env - } - } - - dir, err := os.Getwd() - if err != nil { - return "" - } - for { - if _, err := os.Stat(filepath.Join(dir, "registry.db")); err == nil { - return dir - } - parent := filepath.Dir(dir) - if parent == dir { - break - } - dir = parent - } - return "" -} - -func init() { - log.SetFlags(log.Ltime) - log.SetPrefix("[sqlite_api] ") - fmt.Fprintln(os.Stderr, "sqlite_api — HTTP API for fn_registry databases") -} diff --git a/projects/fn_monitoring/apps/sqlite_api/start.sh b/projects/fn_monitoring/apps/sqlite_api/start.sh deleted file mode 100755 index f8087772..00000000 --- a/projects/fn_monitoring/apps/sqlite_api/start.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env bash -# Start sqlite_api in the background. Logs to sqlite_api.log. -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" -REGISTRY_ROOT="$(cd "$SCRIPT_DIR/../../../.." && pwd)" -LOG="$SCRIPT_DIR/sqlite_api.log" -PID_FILE="$SCRIPT_DIR/sqlite_api.pid" - -# Kill previous instance if running -if [ -f "$PID_FILE" ]; then - old_pid=$(cat "$PID_FILE") - if kill -0 "$old_pid" 2>/dev/null; then - echo "Stopping previous instance (PID $old_pid)" - kill "$old_pid" - sleep 0.5 - fi - rm -f "$PID_FILE" -fi - -export FN_REGISTRY_ROOT="$REGISTRY_ROOT" - -cd "$REGISTRY_ROOT" -CGO_ENABLED=1 go run -tags fts5 ./projects/fn_monitoring/apps/sqlite_api/ "$@" \ - >"$LOG" 2>&1 & - -echo $! > "$PID_FILE" -echo "sqlite_api started (PID $!, log: $LOG)" -sleep 0.5 -if curl -sf http://127.0.0.1:8484/health >/dev/null 2>&1; then - echo "Health check OK" -else - echo "Waiting for startup..." - sleep 1.5 - if curl -sf http://127.0.0.1:8484/health >/dev/null 2>&1; then - echo "Health check OK" - else - echo "Warning: health check failed — check $LOG" - fi -fi diff --git a/projects/fn_monitoring/project.md b/projects/fn_monitoring/project.md index 35a4c201..bb2a8776 100644 --- a/projects/fn_monitoring/project.md +++ b/projects/fn_monitoring/project.md @@ -7,18 +7,220 @@ repo_url: "" ## Apps -| App | Descripcion | -|-----|-------------| -| sqlite_api | API REST HTTP read-only para registry.db y operations.db (Go, net/http, puerto 8484) | -| registry_dashboard | Dashboard ImGui con KPIs, charts y tablas del registry (C++, consume sqlite_api) | +| App | Lang | Descripcion | +|-----|------|-------------| +| [sqlite_api](apps/sqlite_api/app.md) | Go | API REST HTTP read-only sobre `registry.db` y todas las `operations.db`. Puerto `8484`. | +| [registry_dashboard](apps/registry_dashboard/app.md) | C++ / ImGui | Dashboard con KPIs, charts y tablas del registry. Consume `sqlite_api` (HTTP) con fallback a SQLite directo. | + +Cada `app.md` es la referencia canonica del binario — endpoints completos, flags, dependencias. Este documento cubre **como operar el proyecto como un todo**: arranque, service, flujo de datos, troubleshooting. + +--- ## Arquitectura ``` -sqlite_api (Go, :8484) - └── registry.db, apps/*/operations.db (read-only) - ↑ -registry_dashboard (C++/ImGui) - └── HTTP GET/POST → sqlite_api - └── Fallback: SQLite directo si API no disponible + registry.db (raiz) + apps/*/operations.db + projects/*/apps/*/operations.db + │ (read-only, mode=ro) + ▼ + ┌──────────────────────────────────────────────┐ + │ sqlite_api (Go net/http, :8484) │ + │ /health │ + │ /api/databases │ + │ /api/databases/:db/tables │ + │ /api/databases/:db/schema │ + │ /api/databases/:db/query (POST, SELECT) │ + │ /api/databases/:db/fts │ + └──────────────────────────────────────────────┘ + ▲ + │ HTTP GET/POST + │ (cpp-httplib + nlohmann/json) + │ + ┌──────────────────────────────────────────────┐ + │ registry_dashboard (C++ / ImGui + ImPlot) │ + │ main.cpp → reload_data() │ + │ data_http.cpp (primario, HTTP) │ + │ data.cpp (fallback, SQLite C API) │ + │ views.cpp → KPI row, charts, tables │ + └──────────────────────────────────────────────┘ ``` + +**Separacion de responsabilidades:** + +- `sqlite_api` **no conoce el dashboard**. Es una API generica: expone cualquier DB SQLite de `fn_registry/` read-only con FTS5. +- `registry_dashboard` **no conoce la estructura de registry.db directamente**, solo a traves del JSON que devuelve la API. El modo SQLite directo es fallback para entornos sin red. + +**Puerto `8484`** — elegido para no colisionar con Metabase (3000), Jupyter (8888) ni deploy_server (9090). + +--- + +## Servicio sqlite_api + +### Modos de arranque + +| Modo | Comando | Cuando usarlo | +|------|---------|---------------| +| Dev (foreground, `go run`) | `cd projects/fn_monitoring/apps/sqlite_api && go run -tags fts5 .` | Iteracion rapida, ver logs en la terminal | +| Dev (background) | `./start.sh` (dentro de `apps/sqlite_api/`) | Probar el dashboard rapido sin systemd. Escribe PID en `sqlite_api.pid` y log en `sqlite_api.log` | +| Production (systemd) | `sudo systemctl start sqlite_api` | Arranque en boot, restart on failure, logs en journal | + +### Variables de entorno + +| Var | Valor | Proposito | +|-----|-------|-----------| +| `FN_REGISTRY_ROOT` | ruta absoluta a la raiz del registry | Evita que el binario busque `registry.db` subiendo por el cwd. Obligatoria bajo systemd. | + +### Instalar como servicio systemd (local) + +Usar el pipeline del registry `install_systemd_service_bash_pipelines`: + +```bash +cd /home/lucas/fn_registry + +# 1. Build del binario +CGO_ENABLED=1 go build -tags fts5 \ + -o projects/fn_monitoring/apps/sqlite_api/sqlite_api \ + ./projects/fn_monitoring/apps/sqlite_api/ + +# 2. Instalar unit + enable + start (requiere sudo sin password para systemctl) +source bash/functions/pipelines/install_systemd_service.sh +install_systemd_service \ + --name sqlite_api \ + --exec "$(pwd)/projects/fn_monitoring/apps/sqlite_api/sqlite_api" \ + --workdir "$(pwd)" \ + --env "FN_REGISTRY_ROOT=$(pwd)" \ + --description "fn_registry SQLite HTTP API" \ + --after network.target \ + --restart on-failure +``` + +### Operacion + +```bash +sudo systemctl status sqlite_api # estado + ultimas lineas del journal +sudo systemctl restart sqlite_api # tras rebuild del binario +sudo systemctl stop sqlite_api # parar +journalctl -u sqlite_api -f # logs en vivo +curl http://127.0.0.1:8484/health # health check +``` + +### Redeploy tras cambios en el codigo Go + +```bash +cd /home/lucas/fn_registry +CGO_ENABLED=1 go build -tags fts5 \ + -o projects/fn_monitoring/apps/sqlite_api/sqlite_api \ + ./projects/fn_monitoring/apps/sqlite_api/ +sudo systemctl restart sqlite_api +``` + +No hace falta reinstalar el unit — solo recompilar y reiniciar. + +--- + +## Dashboard registry_dashboard + +### Build + +```bash +cd cpp +cmake -B build/linux -S . +cmake --build build/linux --target registry_dashboard -j$(nproc) +``` + +El binario queda en `cpp/build/linux/registry_dashboard` (o `projects/fn_monitoring/apps/registry_dashboard/registry_dashboard.exe` en Windows). + +### Ejecucion + +```bash +# Modo API (por defecto, intenta localhost:8484) +./registry_dashboard + +# API remoto +./registry_dashboard --api http://192.168.1.10:8484 + +# API + fallback SQLite +./registry_dashboard --api http://127.0.0.1:8484 /home/lucas/fn_registry/registry.db + +# Solo SQLite (sin API) +./registry_dashboard /home/lucas/fn_registry/registry.db +``` + +La UI muestra en la cabecera de donde vienen los datos (HTTP vs SQLite). `F5` recarga. + +### Flujo de datos + +1. `main.cpp::reload_data()` intenta HTTP primero via `load_registry_data_http()`. +2. Si la API responde `200` y el JSON parsea, los datos pueblan `RegistryData`. +3. Si falla la API (timeout, 5xx, JSON invalido) y hay `--db`, cae a `load_registry_data()` (SQLite directo). +4. Si ninguno funciona, la UI muestra un mensaje de error y no hay reintento automatico — hay que pulsar reload. + +### Vistas + +| Seccion | Datos | Query subyacente | +|---------|-------|------------------| +| KPI row (8 cards) | totales y porcentajes | `SELECT COUNT(*)` sobre functions, types, apps, analysis, unit_tests, proposals + agregados tested/pure | +| Charts (bar + pie) | funciones por lang/domain, reparto pure/impure, kind | `GROUP BY lang`, `GROUP BY domain`, `GROUP BY purity`, `GROUP BY kind` | +| Tablas | ultimas 20 functions, apps, analysis, types | `ORDER BY updated_at DESC LIMIT 20` | + +Detalle de composicion de componentes viz en `apps/registry_dashboard/app.md`. + +--- + +## Troubleshooting + +| Sintoma | Causa probable | Verificacion / Fix | +|---------|----------------|--------------------| +| Dashboard dice "HTTP API failed, falling back to SQLite" | `sqlite_api` no esta corriendo | `curl http://127.0.0.1:8484/health` — si falla, `systemctl status sqlite_api` o `./start.sh` | +| `sqlite_api` arranca y muere inmediatamente | No encuentra `registry.db` | Exportar `FN_REGISTRY_ROOT=/home/lucas/fn_registry` o correr desde la raiz del registry | +| `systemctl start sqlite_api` pide password | Falta sudoers para systemctl | Ver `.claude/rules/deploy.md` — el usuario necesita `NOPASSWD` para `systemctl`, `mv` a `/etc/systemd/system/` | +| Dashboard abre pero todas las cifras son 0 | API conecta pero devuelve DB vacia | `curl -X POST http://127.0.0.1:8484/api/databases/registry/query -d '{"sql":"SELECT COUNT(*) FROM functions"}'` | +| API responde lento / timeout | Query pesada sobre FTS5 | Timeout hardcoded a 5s en `handlers.go`. Revisar la query en journal. | +| Bind rechazado (`address already in use`) | Otro proceso en `8484` | `ss -tlnp | grep 8484` — matar el huerfano o cambiar `--bind` | + +### Logs + +```bash +# systemd +journalctl -u sqlite_api -n 100 --no-pager +journalctl -u sqlite_api -f + +# start.sh +tail -f projects/fn_monitoring/apps/sqlite_api/sqlite_api.log +``` + +--- + +## Como extender + +### Anadir un endpoint a sqlite_api + +1. Registrar la ruta en `Server.Routes()` (`handlers.go`). +2. Handler lee `r.URL.Path` / `r.Body`, delega en `DBPool` para resolver la DB, ejecuta SQL read-only. +3. Test en `handlers_test.go` (patron: tabla de casos HTTP). +4. Rebuild + `systemctl restart sqlite_api`. +5. Documentar en `apps/sqlite_api/app.md` (tabla de endpoints). + +### Anadir una vista al dashboard + +1. Nuevo campo en `RegistryData` (`data.h`) + su equivalente en la respuesta JSON. +2. Parseo en `data_http.cpp` y carga SQL en `data.cpp` (ambos paths, para mantener el fallback). +3. Renderizado en `views.cpp` usando componentes del dominio `viz` (`kpi_card`, `bar_chart`, etc.) — ver regla `frontend_theming` analoga para C++: usar primitivos del registry antes que ImGui crudo. +4. Rebuild con CMake. + +### Anadir una DB nueva + +La descubre automaticamente `DiscoverDatabases()` escaneando `apps/*/operations.db` y `projects/*/apps/*/operations.db`. No hay que registrar nada — al reiniciar `sqlite_api` aparecen con alias `ops:{app_name}`. + +--- + +## Deploy en otros PCs + +Este proyecto se instala identico en cualquier maquina con el registry clonado: + +1. `fn sync` para traer los metadatos del proyecto. +2. Build + systemd install (seccion "Instalar como servicio systemd" arriba). +3. Build del dashboard. + +Los datos son los `.db` locales — cada PC ve su propio estado del registry y sus propias `operations.db`. No hay sincronizacion remota de datos en este servicio: para eso existe `fn sync` contra `registry_api` (proyecto diferente, ver memoria `project_registry_api`).