297 lines
12 KiB
Python
297 lines
12 KiB
Python
"""summarize_table_duckdb — perfil base de una tabla DuckDB en una sola pasada SQL.
|
|
|
|
Funcion impura: lee de disco a traves de DuckDB (via la primitiva read-only del
|
|
grupo `duckdb`, `duckdb_query_readonly`). Es el CORAZON del grupo de capacidad
|
|
`eda` (exploratory data analysis): construye el esqueleto de un TableProfile con
|
|
el perfil base por columna usando exclusivamente `SUMMARIZE`, que hace push-down
|
|
en el motor de DuckDB y NO trae filas a RAM.
|
|
|
|
Lo que NO calcula aqui (a proposito, para ser barata): skew, kurtosis, histograma,
|
|
percentiles finos (p1/p5/p95/p99), moda, outliers, correlaciones, key_candidates,
|
|
quality_score ni el semantic_type. Esas claves quedan en None / [] para que las
|
|
rellenen luego otras funciones del grupo `eda` (p.ej. describe_numeric) sobre una
|
|
muestra. El contrato de claves (TableProfile / ColumnProfile) es compartido por
|
|
todo el grupo `eda` y debe mantenerse estable.
|
|
|
|
Estilo dict-no-throw del grupo duckdb: nunca lanza; captura cualquier error y
|
|
devuelve {status:'error', error:str}.
|
|
"""
|
|
|
|
import re
|
|
from datetime import datetime, timezone
|
|
|
|
from infra import duckdb_query_readonly
|
|
|
|
# Identificador SQL valido. DuckDB SUMMARIZE no admite parametros posicionales
|
|
# para el nombre de la tabla, asi que hay que validar e interpolar citado.
|
|
_IDENT_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
|
|
|
|
# Umbral de filas por debajo del cual calculamos COUNT(DISTINCT) EXACTO en una
|
|
# sola query combinada (barato). Por encima usamos el approx_unique de SUMMARIZE
|
|
# (HyperLogLog), capado a n_rows para que distinct_count nunca exceda las filas.
|
|
_EXACT_DISTINCT_MAX_ROWS = 200_000
|
|
|
|
# Tipos fisicos DuckDB que mapean a "numeric".
|
|
_NUMERIC_TYPES = {
|
|
"TINYINT", "SMALLINT", "INTEGER", "BIGINT", "HUGEINT",
|
|
"UTINYINT", "USMALLINT", "UINTEGER", "UBIGINT", "UHUGEINT",
|
|
"FLOAT", "REAL", "DOUBLE", "DECIMAL", "NUMERIC",
|
|
}
|
|
# Tipos fisicos DuckDB que mapean a "datetime".
|
|
_DATETIME_TYPES = {
|
|
"DATE", "TIME", "TIMESTAMP", "DATETIME",
|
|
"TIMESTAMP_S", "TIMESTAMP_MS", "TIMESTAMP_NS", "TIMESTAMP_US",
|
|
"TIMESTAMP WITH TIME ZONE", "TIMESTAMPTZ", "TIMETZ",
|
|
}
|
|
|
|
# Claves del sub-dict numeric. summarize solo rellena unas pocas; el resto
|
|
# quedan en None hasta que una funcion de muestreo (describe_numeric) las complete.
|
|
_NUMERIC_SUB_KEYS = (
|
|
"min", "max", "mean", "median", "mode", "std", "variance", "cv",
|
|
"p1", "p5", "p25", "p50", "p75", "p95", "p99", "iqr",
|
|
"skew", "kurtosis", "n_outliers", "outlier_pct", "zero_pct",
|
|
"negative_pct", "distribution_type", "histogram",
|
|
)
|
|
|
|
|
|
def _base_physical_type(column_type: str) -> str:
|
|
"""Normaliza un column_type fisico de DuckDB a su forma base en mayusculas.
|
|
|
|
Quita los parametros (DECIMAL(10,2) -> DECIMAL) y los modificadores de array
|
|
(INTEGER[] -> INTEGER) para poder compararlo contra los conjuntos de tipos.
|
|
"""
|
|
t = (column_type or "").strip().upper()
|
|
# Quitar sufijo de array/lista (INTEGER[], VARCHAR[3], etc.).
|
|
t = re.sub(r"\[.*\]$", "", t).strip()
|
|
# Quitar parametros: DECIMAL(10,2) -> DECIMAL, VARCHAR(50) -> VARCHAR.
|
|
t = re.sub(r"\(.*\)$", "", t).strip()
|
|
return t
|
|
|
|
|
|
def _infer_type(column_type: str, distinct_count, n_rows: int) -> str:
|
|
"""Mapea el tipo fisico DuckDB al inferred_type del contrato.
|
|
|
|
numeric / datetime / boolean salen directos del tipo fisico. Para VARCHAR/TEXT
|
|
se decide entre categorical y text con una heuristica de cardinalidad:
|
|
categorical si distinct_count <= 50 o distinct_count/n_rows < 0.5; si no text.
|
|
"""
|
|
base = _base_physical_type(column_type)
|
|
if base in _NUMERIC_TYPES:
|
|
return "numeric"
|
|
if base in _DATETIME_TYPES:
|
|
return "datetime"
|
|
if base in ("BOOLEAN", "BOOL"):
|
|
return "boolean"
|
|
if base in ("VARCHAR", "TEXT", "STRING", "CHAR", "BPCHAR"):
|
|
au = distinct_count if distinct_count is not None else 0
|
|
if n_rows <= 0:
|
|
return "categorical"
|
|
if au <= 50 or (au / n_rows) < 0.5:
|
|
return "categorical"
|
|
return "text"
|
|
# Tipos complejos (STRUCT, MAP, LIST, BLOB, UUID, ...): tratamos como text.
|
|
return "text"
|
|
|
|
|
|
def _to_float(value):
|
|
"""Convierte a float un valor que SUMMARIZE devuelve como string/Decimal.
|
|
|
|
SUMMARIZE entrega min/max/avg/std/q25/q50/q75 como cadenas (o None). Para
|
|
columnas no numericas (o fechas) la conversion fallara y devolvemos None.
|
|
"""
|
|
if value is None:
|
|
return None
|
|
try:
|
|
return float(value)
|
|
except (TypeError, ValueError):
|
|
return None
|
|
|
|
|
|
def summarize_table_duckdb(
|
|
db_path: str, table: str, high_card_ratio: float = 0.9
|
|
) -> dict:
|
|
"""Perfila una tabla DuckDB en una sola pasada SQL (push-down, sin traer filas).
|
|
|
|
Args:
|
|
db_path: ruta al archivo DuckDB. Debe existir (lectura read-only, no se crea).
|
|
table: nombre de la tabla a perfilar. Se valida contra
|
|
^[A-Za-z_][A-Za-z0-9_]*$ y se cita en el SQL (SUMMARIZE no admite
|
|
parametros posicionales para el identificador).
|
|
high_card_ratio: umbral de unicidad (unique_pct) a partir del cual una
|
|
columna categorical se marca con el flag "high_cardinality". Default 0.9.
|
|
|
|
Returns:
|
|
dict. En exito: {status:'ok', profile: <TableProfile>}. En error (sin
|
|
lanzar): {status:'error', error:str}.
|
|
"""
|
|
try:
|
|
if not _IDENT_RE.match(table or ""):
|
|
return {
|
|
"status": "error",
|
|
"error": (
|
|
f"nombre de tabla invalido: {table!r} "
|
|
"(debe casar con ^[A-Za-z_][A-Za-z0-9_]*$)"
|
|
),
|
|
}
|
|
|
|
quoted = f'"{table}"'
|
|
|
|
# 1) Numero total de filas.
|
|
count_res = duckdb_query_readonly(db_path, f"SELECT count(*) AS n FROM {quoted}")
|
|
if count_res["status"] != "ok":
|
|
return {"status": "error", "error": count_res["error"]}
|
|
n_rows = int(count_res["rows"][0]["n"]) if count_res["rows"] else 0
|
|
|
|
# 2) SUMMARIZE: perfil base por columna en el motor.
|
|
summ_res = duckdb_query_readonly(db_path, f"SUMMARIZE {quoted}")
|
|
if summ_res["status"] != "ok":
|
|
return {"status": "error", "error": summ_res["error"]}
|
|
|
|
# 3) distinct_count EXACTO para tablas pequenas/medianas. SUMMARIZE usa
|
|
# approx_unique (HyperLogLog), que SOBREESTIMA: en tablas pequenas puede
|
|
# reportar mas distintos que filas, inflando unique_pct por encima de 1.0
|
|
# y disparando flags possible_id falsos. Para n_rows <= umbral calculamos
|
|
# COUNT(DISTINCT) EXACTO en UNA sola query combinada (barato). Por encima
|
|
# del umbral nos quedamos con approx_unique, pero capado a n_rows en
|
|
# _build_column_profile. Mapea column_name -> distinct exacto.
|
|
exact_distinct = {}
|
|
col_names = [r.get("column_name") for r in summ_res["rows"]]
|
|
if n_rows > 0 and n_rows <= _EXACT_DISTINCT_MAX_ROWS and col_names:
|
|
select_parts = [
|
|
f'count(DISTINCT "{name}") AS c{i}'
|
|
for i, name in enumerate(col_names)
|
|
]
|
|
distinct_sql = f"SELECT {', '.join(select_parts)} FROM {quoted}"
|
|
distinct_res = duckdb_query_readonly(db_path, distinct_sql)
|
|
if distinct_res["status"] != "ok":
|
|
return {"status": "error", "error": distinct_res["error"]}
|
|
if distinct_res["rows"]:
|
|
drow = distinct_res["rows"][0]
|
|
for i, name in enumerate(col_names):
|
|
val = drow.get(f"c{i}")
|
|
if val is not None:
|
|
exact_distinct[name] = int(val)
|
|
|
|
columns = []
|
|
for row in summ_res["rows"]:
|
|
columns.append(
|
|
_build_column_profile(row, n_rows, high_card_ratio, exact_distinct)
|
|
)
|
|
|
|
type_breakdown = {
|
|
"numeric": 0,
|
|
"categorical": 0,
|
|
"datetime": 0,
|
|
"text": 0,
|
|
"boolean": 0,
|
|
}
|
|
for col in columns:
|
|
it = col["inferred_type"]
|
|
if it in type_breakdown:
|
|
type_breakdown[it] += 1
|
|
|
|
constant_cols = [c["name"] for c in columns if "constant" in c["flags"]]
|
|
all_null_cols = [c["name"] for c in columns if c["null_pct"] == 1.0]
|
|
null_cell_pct = (
|
|
sum(c["null_pct"] for c in columns) / len(columns) if columns else 0.0
|
|
)
|
|
|
|
profile = {
|
|
"table": table,
|
|
"source": "duckdb",
|
|
"profiled_at": datetime.now(timezone.utc).isoformat(),
|
|
"n_rows": n_rows,
|
|
"n_cols": len(columns),
|
|
"size_bytes": None,
|
|
"duplicate_rows": None,
|
|
"duplicate_pct": None,
|
|
"constant_cols": constant_cols,
|
|
"all_null_cols": all_null_cols,
|
|
"null_cell_pct": null_cell_pct,
|
|
"type_breakdown": type_breakdown,
|
|
"columns": columns,
|
|
"correlations": None,
|
|
"key_candidates": [],
|
|
"quality_score": None,
|
|
"llm": None,
|
|
"models": None,
|
|
}
|
|
return {"status": "ok", "profile": profile}
|
|
except Exception as e: # noqa: BLE001
|
|
return {"status": "error", "error": str(e)}
|
|
|
|
|
|
def _build_column_profile(
|
|
row: dict, n_rows: int, high_card_ratio: float, exact_distinct: dict = None
|
|
) -> dict:
|
|
"""Convierte una fila de SUMMARIZE en un ColumnProfile del contrato eda.
|
|
|
|
distinct_count: si la columna tiene un valor en `exact_distinct` (tablas
|
|
pequenas/medianas perfiladas con COUNT(DISTINCT) exacto), se usa ese valor.
|
|
Si no (tablas grandes), se usa approx_unique de SUMMARIZE CAPADO a n_rows
|
|
para que nunca supere el numero de filas. unique_pct queda limitado a 1.0.
|
|
"""
|
|
name = row.get("column_name")
|
|
physical_type = row.get("column_type")
|
|
approx_unique = row.get("approx_unique")
|
|
# null_percentage viene en escala 0-100 (Decimal). Lo pasamos a fraccion 0-1.
|
|
null_pct_raw = row.get("null_percentage")
|
|
null_pct = float(null_pct_raw) / 100.0 if null_pct_raw is not None else 0.0
|
|
|
|
# distinct_count corregido (exacto si disponible; si no approx capado a n_rows).
|
|
exact_distinct = exact_distinct or {}
|
|
if name in exact_distinct:
|
|
distinct_count = exact_distinct[name]
|
|
else:
|
|
approx = int(approx_unique) if approx_unique is not None else 0
|
|
distinct_count = min(approx, n_rows) if n_rows > 0 else approx
|
|
|
|
# Inferencia categorical/text con la cardinalidad ya corregida.
|
|
inferred_type = _infer_type(physical_type, distinct_count, n_rows)
|
|
|
|
null_count = round(null_pct * n_rows)
|
|
non_null_count = n_rows - null_count # SUMMARIZE.count es el total, no el no-nulo.
|
|
|
|
unique_pct = min(distinct_count / n_rows, 1.0) if n_rows > 0 else 0.0
|
|
|
|
numeric = None
|
|
if inferred_type == "numeric":
|
|
numeric = {k: None for k in _NUMERIC_SUB_KEYS}
|
|
numeric["min"] = _to_float(row.get("min"))
|
|
numeric["max"] = _to_float(row.get("max"))
|
|
numeric["mean"] = _to_float(row.get("avg"))
|
|
numeric["std"] = _to_float(row.get("std"))
|
|
numeric["p25"] = _to_float(row.get("q25"))
|
|
numeric["p50"] = _to_float(row.get("q50"))
|
|
numeric["p75"] = _to_float(row.get("q75"))
|
|
|
|
flags = []
|
|
if distinct_count <= 1:
|
|
flags.append("constant")
|
|
if unique_pct >= 0.99 and null_pct == 0:
|
|
flags.append("possible_id")
|
|
if inferred_type == "categorical" and unique_pct >= high_card_ratio:
|
|
flags.append("high_cardinality")
|
|
if null_pct > 0.5:
|
|
flags.append("mostly_null")
|
|
|
|
return {
|
|
"name": name,
|
|
"physical_type": physical_type,
|
|
"inferred_type": inferred_type,
|
|
"semantic_type": "",
|
|
"count": non_null_count,
|
|
"n_rows": n_rows,
|
|
"null_count": null_count,
|
|
"null_pct": null_pct,
|
|
"empty_count": None,
|
|
"empty_pct": None,
|
|
"distinct_count": distinct_count,
|
|
"unique_pct": unique_pct,
|
|
"flags": flags,
|
|
"quality_score": None,
|
|
"numeric": numeric,
|
|
"categorical": None,
|
|
"datetime": None,
|
|
}
|