agent
272 lines
9.6 KiB
Python
272 lines
9.6 KiB
Python
#!/usr/bin/env python3
|
|
"""Service FastAPI osint_db: dueño único de la base DuckDB data/osint.duckdb.
|
|
|
|
La base es la fuente de verdad estructurada del ecosistema OSINT: índice del
|
|
vault de Obsidian (notes + entidades con note_path), maestras DAV importadas de
|
|
Xandikos (contacts, events) y derivadas computadas (schema derived, sin
|
|
referencias a notas). Solo este proceso escribe la base; las lecturas de
|
|
/api/query usan una conexión read_only separada (duckdb_query_readonly).
|
|
|
|
Registry-first: el service NO reimplementa parseo de Markdown, protocolo DAV,
|
|
acceso a pass, ejecución read-only de DuckDB, render de tablas Markdown ni
|
|
bloques sentinel — importa esas funciones del registry (server/registry_bridge.py)
|
|
y solo aporta la lógica propia del dominio (mapeo vault→tablas, matching
|
|
contacto→ficha, derivadas y la API).
|
|
|
|
Seguridad: el vault contiene datos personales sensibles, así que el server
|
|
escucha SOLO en 127.0.0.1 y /api/query es estrictamente de solo lectura.
|
|
|
|
Uso:
|
|
.venv/bin/python server/main.py --vault ~/Obsidian/osint --port 8771
|
|
|
|
Contrato (el plugin de Obsidian parsea el body, no el código HTTP: los
|
|
endpoints de datos responden SIEMPRE 200 con status ok|error en el body):
|
|
GET /api/health estado + db_path + número de tablas
|
|
GET /api/tables inventario de tablas (schema, kind, filas, columnas)
|
|
POST /api/query SELECT arbitrario read-only {sql, params, max_rows}
|
|
GET /api/queries catálogo de queries con nombre
|
|
POST /api/query/named ejecuta una query del catálogo {name, max_rows}
|
|
POST /api/ingest/vault re-escanea el vault y reconstruye maestras+derivadas
|
|
POST /api/ingest/dav baja Xandikos y reconstruye contacts/events+derivadas
|
|
POST /api/render/note ejecuta query y la upserta como bloque sentinel en una nota
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import os
|
|
import sys
|
|
|
|
# Permite ejecutar tanto `python server/main.py` como importar `server.main`.
|
|
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|
|
|
from fastapi import FastAPI # noqa: E402
|
|
from pydantic import BaseModel, Field # noqa: E402
|
|
|
|
from server.config import SENTINEL_MARKER, Config # noqa: E402
|
|
from server.db import apply_migrations, list_tables # noqa: E402
|
|
from server.ingest import ingest_dav, ingest_vault # noqa: E402
|
|
from server.named_queries import NAMED_QUERIES # noqa: E402
|
|
from server.registry_bridge import ( # noqa: E402
|
|
create_obsidian_note,
|
|
duckdb_query_readonly,
|
|
read_obsidian_note,
|
|
render_markdown_table,
|
|
update_obsidian_note,
|
|
upsert_sentinel_block,
|
|
)
|
|
|
|
# Tope de filas que un render vuelca en una nota (las notas no son un export).
|
|
RENDER_MAX_ROWS = 200
|
|
|
|
|
|
class QueryBody(BaseModel):
|
|
"""Body de POST /api/query."""
|
|
|
|
sql: str
|
|
params: list = Field(default_factory=list)
|
|
max_rows: int = 500
|
|
|
|
|
|
class NamedQueryBody(BaseModel):
|
|
"""Body de POST /api/query/named."""
|
|
|
|
name: str
|
|
max_rows: int = 500
|
|
|
|
|
|
class RenderNoteBody(BaseModel):
|
|
"""Body de POST /api/render/note. Exactamente uno de sql|query."""
|
|
|
|
note_path: str
|
|
block_id: str
|
|
sql: str | None = None
|
|
query: str | None = None
|
|
title: str | None = None
|
|
|
|
|
|
def create_app(cfg: Config) -> FastAPI:
|
|
"""Construye la app FastAPI con la configuración dada (inyectable en tests)."""
|
|
app = FastAPI(title="osint_db", docs_url=None, redoc_url=None)
|
|
|
|
def run_readonly(sql: str, params: list, max_rows: int) -> dict:
|
|
"""Ejecuta un SELECT con la conexión read_only del registry, acotado."""
|
|
max_rows = max(1, min(int(max_rows), 10000))
|
|
return duckdb_query_readonly(cfg.db_path, sql, params or [], max_rows)
|
|
|
|
@app.get("/api/health")
|
|
def health() -> dict:
|
|
try:
|
|
tables = list_tables(cfg.db_path)
|
|
except Exception as e: # noqa: BLE001
|
|
return {"status": "error", "db_path": cfg.db_path, "error": str(e)}
|
|
return {"status": "ok", "db_path": cfg.db_path, "tables": len(tables)}
|
|
|
|
@app.get("/api/tables")
|
|
def tables() -> dict:
|
|
try:
|
|
return {"status": "ok", "tables": list_tables(cfg.db_path)}
|
|
except Exception as e: # noqa: BLE001
|
|
return {"status": "error", "error": str(e)}
|
|
|
|
@app.post("/api/query")
|
|
def query(body: QueryBody) -> dict:
|
|
return run_readonly(body.sql, body.params, body.max_rows)
|
|
|
|
@app.get("/api/queries")
|
|
def queries() -> dict:
|
|
return {
|
|
"status": "ok",
|
|
"queries": [
|
|
{"name": name, "description": q["description"], "sql": q["sql"]}
|
|
for name, q in NAMED_QUERIES.items()
|
|
],
|
|
}
|
|
|
|
@app.post("/api/query/named")
|
|
def query_named(body: NamedQueryBody) -> dict:
|
|
entry = NAMED_QUERIES.get(body.name)
|
|
if entry is None:
|
|
return {
|
|
"status": "error",
|
|
"error": f"query con nombre desconocida: {body.name!r} "
|
|
f"(disponibles: {', '.join(sorted(NAMED_QUERIES))})",
|
|
}
|
|
return run_readonly(entry["sql"], [], body.max_rows)
|
|
|
|
@app.post("/api/ingest/vault")
|
|
def api_ingest_vault() -> dict:
|
|
try:
|
|
return ingest_vault(cfg)
|
|
except Exception as e: # noqa: BLE001
|
|
return {"status": "error", "error": str(e)}
|
|
|
|
@app.post("/api/ingest/dav")
|
|
def api_ingest_dav() -> dict:
|
|
try:
|
|
return ingest_dav(cfg)
|
|
except Exception as e: # noqa: BLE001
|
|
return {"status": "error", "error": str(e)}
|
|
|
|
@app.post("/api/render/note")
|
|
def render_note(body: RenderNoteBody) -> dict:
|
|
# Resolver el SQL: o viene literal, o por nombre del catálogo.
|
|
if bool(body.sql) == bool(body.query):
|
|
return {
|
|
"status": "error",
|
|
"error": "indica exactamente uno de los campos 'sql' o 'query' (named)",
|
|
}
|
|
if body.query:
|
|
entry = NAMED_QUERIES.get(body.query)
|
|
if entry is None:
|
|
return {
|
|
"status": "error",
|
|
"error": f"query con nombre desconocida: {body.query!r}",
|
|
}
|
|
sql = entry["sql"]
|
|
else:
|
|
sql = body.sql
|
|
|
|
result = run_readonly(sql, [], RENDER_MAX_ROWS)
|
|
if result.get("status") != "ok":
|
|
return result
|
|
|
|
table_md = render_markdown_table(
|
|
result["rows"], columns=result["columns"], max_rows=RENDER_MAX_ROWS
|
|
)
|
|
content = table_md if table_md else "_(sin filas)_"
|
|
if body.title:
|
|
content = f"### {body.title}\n\n{content}"
|
|
|
|
# La nota se referencia por path relativo al vault; el path no puede
|
|
# escapar del vault (mismo criterio anti-traversal que osint_web).
|
|
rel = body.note_path if body.note_path.endswith(".md") else body.note_path + ".md"
|
|
abs_path = os.path.abspath(os.path.join(cfg.vault_dir, rel))
|
|
vault_real = os.path.realpath(cfg.vault_dir)
|
|
if not os.path.realpath(abs_path).startswith(vault_real + os.sep):
|
|
return {"status": "error", "error": f"note_path fuera del vault: {body.note_path!r}"}
|
|
|
|
try:
|
|
if os.path.exists(abs_path):
|
|
note = read_obsidian_note(abs_path)
|
|
new_body = upsert_sentinel_block(
|
|
note.get("body", "") or "",
|
|
body.block_id,
|
|
content,
|
|
marker=SENTINEL_MARKER,
|
|
)
|
|
update_obsidian_note(abs_path, body=new_body)
|
|
else:
|
|
new_body = upsert_sentinel_block(
|
|
"", body.block_id, content, marker=SENTINEL_MARKER
|
|
)
|
|
create_obsidian_note(
|
|
cfg.vault_dir,
|
|
rel,
|
|
body=new_body,
|
|
frontmatter={"tipo": "tablero", "tags": ["osintdb"]},
|
|
)
|
|
except Exception as e: # noqa: BLE001
|
|
return {"status": "error", "error": str(e)}
|
|
|
|
return {
|
|
"status": "ok",
|
|
"note_path": rel,
|
|
"rows_rendered": result["row_count"],
|
|
}
|
|
|
|
return app
|
|
|
|
|
|
def main() -> int:
|
|
"""Punto de entrada CLI: valida el vault, migra la base y arranca uvicorn."""
|
|
parser = argparse.ArgumentParser(
|
|
description="Service osint_db: DuckDB fuente de verdad del ecosistema OSINT."
|
|
)
|
|
parser.add_argument("--vault", default=None, help="directorio del vault Obsidian")
|
|
parser.add_argument("--db", default=None, help="ruta del archivo osint.duckdb")
|
|
parser.add_argument("--port", type=int, default=None, help="puerto de escucha")
|
|
parser.add_argument(
|
|
"--host",
|
|
default="127.0.0.1",
|
|
help="host de escucha (datos sensibles: déjalo en 127.0.0.1)",
|
|
)
|
|
args = parser.parse_args()
|
|
|
|
cfg = Config()
|
|
if args.vault:
|
|
cfg.vault_dir = os.path.expanduser(args.vault)
|
|
if args.db:
|
|
cfg.db_path = os.path.expanduser(args.db)
|
|
if args.port is not None:
|
|
cfg.port = args.port
|
|
cfg.host = args.host
|
|
|
|
if not os.path.isdir(cfg.vault_dir):
|
|
print(
|
|
f"ERROR: el vault no existe o no es un directorio: {cfg.vault_dir}",
|
|
file=sys.stderr,
|
|
)
|
|
return 2
|
|
|
|
if cfg.host != "127.0.0.1":
|
|
print(
|
|
"AVISO: la base contiene datos personales sensibles; escuchar fuera "
|
|
"de 127.0.0.1 no está soportado.",
|
|
file=sys.stderr,
|
|
)
|
|
|
|
applied = apply_migrations(cfg.db_path)
|
|
if applied:
|
|
print(f"migraciones aplicadas: {', '.join(applied)}")
|
|
|
|
import uvicorn
|
|
|
|
app = create_app(cfg)
|
|
uvicorn.run(app, host=cfg.host, port=cfg.port, log_level="warning")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|