dataforge/fn_registry
1
0
Fork 0
Code Issues Pull Requests 3 Actions Packages Projects Releases Wiki Activity

feat(infra): auto-commit con 56 cambios

Browse Source 
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Egutierrez
2026-06-21 14:22:55 +02:00
parent c1071a82b3
commit 32c7336bf6
56 changed files with 5307 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/functions/pipelines/profile_table.py
+46 -20
View File
@@ -38,8 +38,9 @@ from datascience import (
run_eda_models,
summarize_categorical,
summarize_table_duckdb,
summarize_table_pg,
)
from infra import duckdb_query_readonly
from infra import duckdb_query_readonly, pg_query
# semantic_types que justifican promocionar inferred_type -> "numeric".
_NUMERIC_SEMANTIC = ("integer", "decimal", "currency")
@@ -82,10 +83,13 @@ def _to_float(value):
return None
def _sample_values(db_path: str, table: str, name: str, sample: int) -> list:
"""Trae hasta `sample` valores no nulos de una columna (read-only)."""
q = duckdb_query_readonly(
db_path,
def _sample_values(query_fn, table: str, name: str, sample: int) -> list:
"""Trae hasta `sample` valores no nulos de una columna (read-only).
query_fn(sql) -> dict es el lector read-only del backend activo
(duckdb_query_readonly o pg_query), inyectado por profile_table.
"""
q = query_fn(
f'SELECT "{name}" AS v FROM "{table}" WHERE "{name}" IS NOT NULL '
f"LIMIT {int(sample)}",
)
@@ -94,19 +98,18 @@ def _sample_values(db_path: str, table: str, name: str, sample: int) -> list:
return [row.get("v") for row in q.get("rows", [])]
def _sample_rows(db_path: str, table: str, names: list, sample: int) -> list:
def _sample_rows(query_fn, table: str, names: list, sample: int) -> list:
"""Trae hasta `sample` filas completas con las columnas alineadas por fila.
A diferencia de _sample_values (una columna, solo no nulos), esto preserva la
alineacion por fila entre columnas, requisito de la matriz de asociacion
(los pares (a_i, b_i) deben venir de la misma fila).
(los pares (a_i, b_i) deben venir de la misma fila). query_fn es el lector
read-only del backend activo, inyectado por profile_table.
"""
if not names:
return []
cols_sql = ", ".join(f'"{n}"' for n in names)
q = duckdb_query_readonly(
db_path, f'SELECT {cols_sql} FROM "{table}" LIMIT {int(sample)}'
)
q = query_fn(f'SELECT {cols_sql} FROM "{table}" LIMIT {int(sample)}')
if q.get("status") != "ok":
return []
return q.get("rows", [])
@@ -115,17 +118,20 @@ def _sample_rows(db_path: str, table: str, names: list, sample: int) -> list:
def profile_table(
db_path: str,
table: str,
backend: str = "duckdb",
sample: int = 5000,
run_models: bool = False,
run_llm: bool = False,
report_dir: str = "reports",
write_report: bool = True,
) -> dict:
"""Perfila una tabla DuckDB end-to-end y emite el TableProfile completo.
"""Perfila una tabla (DuckDB o PostgreSQL) end-to-end y emite el TableProfile.
Args:
db_path: ruta al archivo DuckDB (read-only, debe existir).
db_path: ruta al archivo DuckDB, o DSN PostgreSQL si backend="postgres".
table: nombre de la tabla a perfilar.
backend: "duckdb" (default) o "postgres". Selecciona el motor de
perfilado base (summarize) y de muestreo read-only.
sample: maximo de valores no nulos muestreados por columna para el
enriquecimiento (describe_numeric / summarize_categorical /
infer_semantic_type). Default 5000.
@@ -141,8 +147,22 @@ def profile_table(
lanzar): {status:'error', error:str}.
"""
try:
# 1) Perfil base por columna (push-down SQL).
r = summarize_table_duckdb(db_path, table)
# 1) Perfil base por columna (push-down SQL) + lector read-only del
# backend activo, inyectado en el muestreo (_sample_values/_sample_rows).
if backend == "postgres":
r = summarize_table_pg(db_path, table)
def _q(sql):
return pg_query(db_path, sql)
elif backend == "duckdb":
r = summarize_table_duckdb(db_path, table)
def _q(sql):
return duckdb_query_readonly(db_path, sql)
else:
return {"status": "error", "error": f"backend desconocido: {backend}"}
if r.get("status") != "ok":
return {"status": "error", "error": r.get("error", "summarize failed")}
prof = r["profile"]
@@ -153,7 +173,7 @@ def profile_table(
inferred = col.get("inferred_type")
# 2) Muestra de valores no nulos.
vals = _sample_values(db_path, table, name, sample)
vals = _sample_values(_q, table, name, sample)
# 3) Promocion de tipo sobre columnas textuales.
if inferred in ("categorical", "text"):
@@ -239,7 +259,7 @@ def profile_table(
assoc_cols = [c for c in cols if not _skip_for_assoc(c)]
rows = _sample_rows(
db_path, table, [c["name"] for c in assoc_cols], corr_sample
_q, table, [c["name"] for c in assoc_cols], corr_sample
)
assoc_input = {}
for c in assoc_cols:
@@ -256,12 +276,18 @@ def profile_table(
prof["correlations"] = (
association_matrix(assoc_input) if len(assoc_input) >= 2 else None
)
# Modelos baratos opt-in (PCA/KMeans/IsolationForest/normalidad).
if run_models:
prof["models"] = run_eda_models(assoc_input)
except Exception: # noqa: BLE001
prof["correlations"] = None
prof["models"] = None
assoc_input = {}
# Modelos baratos opt-in en su PROPIO try: un fallo de los modelos NUNCA
# debe tumbar las correlaciones ya calculadas (bug detectado en EDAs PG
# reales: un try/except compartido ponia ambos campos a None).
if run_models:
try:
prof["models"] = run_eda_models(assoc_input)
except Exception: # noqa: BLE001
prof["models"] = None
# 8.6) Capa LLM opcional: interpreta el perfil ya calculado en UNA
# llamada (data dictionary, resumen, granularidad de fila, PII, limpieza,