297 lines
12 KiB
Python
297 lines
12 KiB
Python
"""infer_fk_containment_duckdb — infiere FOREIGN KEYs candidatas por containment.
|
|
|
|
Funcion impura: lee de disco a traves de DuckDB (via las primitivas read-only del
|
|
grupo `duckdb`: duckdb_list_tables, duckdb_table_schema, duckdb_query_readonly).
|
|
Pertenece al grupo de capacidad `eda` (relaciones inter-tabla): descubre que
|
|
columnas de una tabla son una clave foranea probable hacia la clave de otra,
|
|
SIN que la base la haya declarado.
|
|
|
|
Idea: para un par (columna A de T1, columna B de T2), la inclusion (o containment)
|
|
de A en B es:
|
|
|
|
inclusion(A subseteq B) = |distinct(A) interseccion distinct(B)| / |distinct(A)|
|
|
|
|
Si inclusion >= min_inclusion y B "parece clave" (alta unicidad en T2, distinct(B)
|
|
/ count(T2) >= 0.95), entonces A -> B es una FK candidata. Todo se calcula con
|
|
push-down en el motor de DuckDB (COUNT DISTINCT / INTERSECT); nunca se traen filas
|
|
a RAM.
|
|
|
|
PODA por tipo: solo se evaluan pares cuyas columnas comparten tipo base (ambos
|
|
enteros, ambos varchar, ambos fecha, ...). Esto evita el O(n^2) de calcular
|
|
containment para todos los pares de columnas, y descarta pares incompatibles que
|
|
nunca podrian ser una FK real.
|
|
|
|
Estilo dict-no-throw del grupo duckdb: nunca lanza; captura cualquier error y
|
|
devuelve {status:'error', error:str}.
|
|
"""
|
|
|
|
import re
|
|
|
|
from infra import (
|
|
duckdb_list_tables,
|
|
duckdb_query_readonly,
|
|
duckdb_table_schema,
|
|
)
|
|
|
|
# Identificador SQL valido. Los nombres de tabla/columna se interpolan citados en
|
|
# el SQL (COUNT DISTINCT / INTERSECT no admiten parametros posicionales para
|
|
# identificadores), asi que se validan antes de tocar la base.
|
|
_IDENT_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
|
|
|
|
# Clases de tipo base. Dos columnas solo se comparan si caen en la misma clase.
|
|
# Agrupar por clase (no por tipo exacto) permite emparejar INTEGER con BIGINT,
|
|
# DECIMAL con DOUBLE, etc. — combinaciones legitimas de FK numerica.
|
|
_INTEGER_TYPES = {
|
|
"TINYINT", "SMALLINT", "INTEGER", "BIGINT", "HUGEINT",
|
|
"UTINYINT", "USMALLINT", "UINTEGER", "UBIGINT", "UHUGEINT",
|
|
}
|
|
_FLOAT_TYPES = {"FLOAT", "REAL", "DOUBLE", "DECIMAL", "NUMERIC"}
|
|
_TEXT_TYPES = {"VARCHAR", "TEXT", "STRING", "CHAR", "BPCHAR", "UUID"}
|
|
_DATETIME_TYPES = {
|
|
"DATE", "TIME", "TIMESTAMP", "DATETIME",
|
|
"TIMESTAMP_S", "TIMESTAMP_MS", "TIMESTAMP_NS", "TIMESTAMP_US",
|
|
"TIMESTAMP WITH TIME ZONE", "TIMESTAMPTZ", "TIMETZ",
|
|
}
|
|
_BOOL_TYPES = {"BOOLEAN", "BOOL"}
|
|
|
|
|
|
def _base_physical_type(column_type: str) -> str:
|
|
"""Normaliza un tipo fisico DuckDB a su forma base en mayusculas.
|
|
|
|
Quita parametros (DECIMAL(10,2) -> DECIMAL) y modificadores de array
|
|
(INTEGER[] -> INTEGER) para poder mapearlo a una clase de tipo.
|
|
"""
|
|
t = (column_type or "").strip().upper()
|
|
t = re.sub(r"\[.*\]$", "", t).strip() # INTEGER[] -> INTEGER
|
|
t = re.sub(r"\(.*\)$", "", t).strip() # VARCHAR(50) -> VARCHAR
|
|
return t
|
|
|
|
|
|
def _type_class(column_type: str) -> str:
|
|
"""Mapea un tipo fisico DuckDB a una clase comparable.
|
|
|
|
Devuelve 'integer' | 'float' | 'text' | 'datetime' | 'boolean' | 'other'.
|
|
Dos columnas solo se consideran emparejables para FK si comparten clase y la
|
|
clase no es 'other'. Entero y float NO se mezclan: una FK entera contra una
|
|
columna float es semanticamente sospechosa y casi nunca una FK real.
|
|
"""
|
|
base = _base_physical_type(column_type)
|
|
if base in _INTEGER_TYPES:
|
|
return "integer"
|
|
if base in _FLOAT_TYPES:
|
|
return "float"
|
|
if base in _TEXT_TYPES:
|
|
return "text"
|
|
if base in _DATETIME_TYPES:
|
|
return "datetime"
|
|
if base in _BOOL_TYPES:
|
|
return "boolean"
|
|
return "other"
|
|
|
|
|
|
def _valid_idents(*names) -> bool:
|
|
"""True si todos los identificadores casan con ^[A-Za-z_][A-Za-z0-9_]*$."""
|
|
return all(isinstance(n, str) and _IDENT_RE.match(n) for n in names)
|
|
|
|
|
|
def _scalar(res: dict):
|
|
"""Extrae el unico valor escalar de un resultado duckdb_query_readonly.
|
|
|
|
Devuelve None si el resultado no es ok o no trae filas.
|
|
"""
|
|
if res["status"] != "ok" or not res["rows"]:
|
|
return None
|
|
row = res["rows"][0]
|
|
# La query siempre alias-a la unica columna; devolvemos su valor.
|
|
return next(iter(row.values()))
|
|
|
|
|
|
def infer_fk_containment_duckdb(
|
|
db_path: str,
|
|
tables: list = None,
|
|
min_inclusion: float = 0.9,
|
|
max_card: int = 200000,
|
|
) -> dict:
|
|
"""Infiere FOREIGN KEYs candidatas entre tablas DuckDB por containment de valores.
|
|
|
|
Args:
|
|
db_path: ruta al archivo DuckDB. Debe existir (lectura read-only via las
|
|
primitivas del grupo duckdb; no se crea).
|
|
tables: lista de nombres de tabla a considerar. None (default) usa todas
|
|
las del esquema main (duckdb_list_tables).
|
|
min_inclusion: umbral minimo de inclusion (0-1) para emitir una FK
|
|
candidata. inclusion(A subseteq B) = |distinct(A) interseccion
|
|
distinct(B)| / |distinct(A)|. Default 0.9.
|
|
max_card: tope de filas en la tabla destino (lado B, el caro del INTERSECT).
|
|
Si count(T2) > max_card, el par se salta para no disparar un INTERSECT
|
|
gigante; se acumula una nota en skipped[]. Default 200000.
|
|
|
|
Returns:
|
|
dict dict-no-throw. En exito:
|
|
{status:'ok',
|
|
fk_candidates:[{from_table, from_col, to_table, to_col, inclusion,
|
|
cardinality, to_is_key}, ...], # ordenado por inclusion desc
|
|
tables:[str], skipped:[str]}
|
|
En error (sin lanzar): {status:'error', error:str}.
|
|
"""
|
|
try:
|
|
# 1) Lista de tablas a considerar.
|
|
if tables is None:
|
|
list_res = duckdb_list_tables(db_path)
|
|
if list_res["status"] != "ok":
|
|
return {"status": "error", "error": list_res["error"]}
|
|
tables = list_res["tables"]
|
|
|
|
if not isinstance(tables, list):
|
|
return {"status": "error", "error": "tables debe ser una lista o None"}
|
|
|
|
tables = [t for t in tables if isinstance(t, str)]
|
|
if not _valid_idents(*tables):
|
|
return {
|
|
"status": "error",
|
|
"error": "algun nombre de tabla no casa con ^[A-Za-z_][A-Za-z0-9_]*$",
|
|
}
|
|
|
|
skipped = []
|
|
|
|
# 2) Schema + count + cache de columnas por tabla.
|
|
# cols_by_table[t] = [{name, type, type_class}, ...]
|
|
cols_by_table = {}
|
|
count_by_table = {}
|
|
for t in tables:
|
|
sch = duckdb_table_schema(db_path, t)
|
|
if sch["status"] != "ok":
|
|
return {"status": "error", "error": sch["error"]}
|
|
cols = []
|
|
for c in sch["columns"]:
|
|
if not _valid_idents(c["name"]):
|
|
# Columna con nombre no interpolable: la ignoramos sin abortar.
|
|
continue
|
|
cols.append(
|
|
{
|
|
"name": c["name"],
|
|
"type": c["type"],
|
|
"type_class": _type_class(c["type"]),
|
|
}
|
|
)
|
|
cols_by_table[t] = cols
|
|
|
|
cnt = _scalar(
|
|
duckdb_query_readonly(db_path, f'SELECT count(*) AS n FROM "{t}"')
|
|
)
|
|
count_by_table[t] = int(cnt) if cnt is not None else 0
|
|
|
|
# 3) Cache de distinct(col) por (tabla, columna) para no recomputarlo.
|
|
distinct_cache = {}
|
|
|
|
def distinct_count(table: str, col: str):
|
|
key = (table, col)
|
|
if key in distinct_cache:
|
|
return distinct_cache[key]
|
|
val = _scalar(
|
|
duckdb_query_readonly(
|
|
db_path, f'SELECT count(DISTINCT "{col}") AS d FROM "{table}"'
|
|
)
|
|
)
|
|
val = int(val) if val is not None else 0
|
|
distinct_cache[key] = val
|
|
return val
|
|
|
|
# 4) Cache de "B es key-ish" por (tabla destino, columna). distinct/count
|
|
# >= 0.95. Solo se evalua para columnas que aparecen como lado B.
|
|
key_cache = {}
|
|
|
|
def to_is_key(table: str, col: str):
|
|
cache_key = (table, col)
|
|
if cache_key in key_cache:
|
|
return key_cache[cache_key]
|
|
n = count_by_table[table]
|
|
if n <= 0:
|
|
key_cache[cache_key] = (False, 0.0)
|
|
return key_cache[cache_key]
|
|
d = distinct_count(table, col)
|
|
ratio = d / n
|
|
key_cache[cache_key] = (ratio >= 0.95, ratio)
|
|
return key_cache[cache_key]
|
|
|
|
candidates = []
|
|
|
|
# 5) Pares (A en T1, B en T2) con T1 != T2 y misma clase de tipo (PODA).
|
|
for t1 in tables:
|
|
for t2 in tables:
|
|
if t1 == t2:
|
|
continue
|
|
# Lado caro: el INTERSECT lee distinct de T2. Si T2 es enorme,
|
|
# saltamos todos los pares hacia el (B en T2) y dejamos nota.
|
|
if count_by_table[t2] > max_card:
|
|
note = (
|
|
f"skip pares -> '{t2}': count {count_by_table[t2]} "
|
|
f"> max_card {max_card}"
|
|
)
|
|
if note not in skipped:
|
|
skipped.append(note)
|
|
continue
|
|
|
|
for a in cols_by_table[t1]:
|
|
if a["type_class"] == "other":
|
|
continue
|
|
for b in cols_by_table[t2]:
|
|
# PODA: solo pares con la misma clase de tipo base.
|
|
if a["type_class"] != b["type_class"]:
|
|
continue
|
|
|
|
# distinct(A); si es 0, no hay containment que medir.
|
|
d_a = distinct_count(t1, a["name"])
|
|
if d_a == 0:
|
|
continue
|
|
|
|
# B debe parecer key (alta unicidad en T2).
|
|
b_is_key, _b_ratio = to_is_key(t2, b["name"])
|
|
if not b_is_key:
|
|
continue
|
|
|
|
# interseccion de distintos via INTERSECT (push-down).
|
|
inter_sql = (
|
|
"SELECT count(*) AS c FROM ("
|
|
f'SELECT DISTINCT "{a["name"]}" FROM "{t1}" '
|
|
"INTERSECT "
|
|
f'SELECT DISTINCT "{b["name"]}" FROM "{t2}"'
|
|
")"
|
|
)
|
|
inter = _scalar(duckdb_query_readonly(db_path, inter_sql))
|
|
if inter is None:
|
|
continue
|
|
inter = int(inter)
|
|
|
|
inclusion = inter / d_a
|
|
if inclusion < min_inclusion:
|
|
continue
|
|
|
|
# Cardinalidad: si A es (casi) unica en T1 -> 1:1; si no N:1.
|
|
n_t1 = count_by_table[t1]
|
|
a_unique = n_t1 > 0 and (d_a / n_t1) >= 0.95
|
|
cardinality = "1:1" if a_unique else "N:1"
|
|
|
|
candidates.append(
|
|
{
|
|
"from_table": t1,
|
|
"from_col": a["name"],
|
|
"to_table": t2,
|
|
"to_col": b["name"],
|
|
"inclusion": inclusion,
|
|
"cardinality": cardinality,
|
|
"to_is_key": True,
|
|
}
|
|
)
|
|
|
|
candidates.sort(key=lambda c: c["inclusion"], reverse=True)
|
|
|
|
return {
|
|
"status": "ok",
|
|
"fk_candidates": candidates,
|
|
"tables": tables,
|
|
"skipped": skipped,
|
|
}
|
|
except Exception as e: # noqa: BLE001
|
|
return {"status": "error", "error": str(e)}
|