536 lines
22 KiB
Python
536 lines
22 KiB
Python
"""NuExtract 2.0-2B GPU — version 'production' con todas las mejoras:
|
|
- repetition_penalty=1.1 (evita bucles)
|
|
- chunking forzado (max 800 chars / ~250 tokens) en TODO texto
|
|
- 97 chunks completos del PDF (no muestra)
|
|
- 25 frases ES troceadas adecuadamente
|
|
- agregacion deduplicada con conteo
|
|
- coreferencia simple (normalize + substring)
|
|
|
|
Vuelca a nuextract_full.json — listo para notebook 08.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import gc
|
|
import json
|
|
import os
|
|
import re
|
|
import sys
|
|
import time
|
|
import warnings
|
|
from collections import Counter, defaultdict
|
|
from pathlib import Path
|
|
|
|
warnings.filterwarnings("ignore")
|
|
os.environ.setdefault("HF_HUB_DISABLE_PROGRESS_BARS", "1")
|
|
|
|
HERE = Path(__file__).resolve().parent
|
|
_pf = "/home/lucas/fn_registry/python/functions"
|
|
sys.path = [p for p in sys.path if not p.startswith(_pf + "/")]
|
|
if _pf not in sys.path:
|
|
sys.path.insert(0, _pf)
|
|
|
|
from core.extract_pdf_text import extract_pdf_text
|
|
|
|
|
|
VAULT = Path("/home/lucas/vaults/osint_nlp_models")
|
|
PDF_PATH = VAULT / "test_documents" / "politica_proteccion_datos.pdf"
|
|
|
|
|
|
def clean_pdf_text(text: str) -> str:
|
|
text = re.sub(r"\b\d{1,2}/\d{1,2}\b", " ", text)
|
|
text = text.replace("\t", " ")
|
|
text = re.sub(r"-\s*\n\s*", "", text)
|
|
text = re.sub(r"(?<![\.!?])\n+", " ", text)
|
|
text = re.sub(r" {2,}", " ", text)
|
|
text = "\n".join(line.strip() for line in text.split("\n") if line.strip())
|
|
return text.strip()
|
|
|
|
|
|
def chunk_with_overlap(text: str, max_chars: int = 800, overlap_sentences: int = 1):
|
|
sentences = re.split(r"(?<=[\.!?])\s+", text)
|
|
sentences = [s.strip() for s in sentences if s.strip()]
|
|
chunks = []
|
|
i = 0
|
|
while i < len(sentences):
|
|
current_sents: list[str] = []
|
|
current_len = 0
|
|
if chunks and overlap_sentences > 0:
|
|
prev_sents = chunks[-1]["sentences"][-overlap_sentences:]
|
|
overlap_len = sum(len(s) + 1 for s in prev_sents)
|
|
next_sentence_len = len(sentences[i]) + 1
|
|
if overlap_len + next_sentence_len <= max_chars:
|
|
current_sents = list(prev_sents)
|
|
current_len = overlap_len
|
|
if i < len(sentences):
|
|
current_sents.append(sentences[i])
|
|
current_len += len(sentences[i]) + 1
|
|
i += 1
|
|
while i < len(sentences) and current_len + len(sentences[i]) + 1 <= max_chars:
|
|
current_sents.append(sentences[i])
|
|
current_len += len(sentences[i]) + 1
|
|
i += 1
|
|
chunks.append({"text": " ".join(current_sents), "sentences": current_sents})
|
|
return chunks
|
|
|
|
|
|
LONG_TEXT_ES = (
|
|
"BBVA, presidido por Carlos Torres, completo en 2024 la integracion operativa de Banco Sabadell tras la fusion. "
|
|
"Onur Genc, consejero delegado del banco desde 2018, lidero el proceso desde la sede central en Bilbao. "
|
|
"El banco mantiene oficinas en Plaza San Nicolas 4 y opera en mas de 25 paises. "
|
|
"Banco Santander, dirigido por Ana Botin, sigue siendo el primer banco espanol por capitalizacion bursatil. "
|
|
"Hector Grisi asumio el cargo de CEO global de Santander en enero de 2023, reemplazando a Jose Antonio Alvarez. "
|
|
"CaixaBank, presidida por Jose Ignacio Goirigolzarri y con sede en Valencia desde 2017, completo la fusion con Bankia. "
|
|
"Gonzalo Gortazar es el consejero delegado de CaixaBank y reporta al consejo formado en parte por La Caixa. "
|
|
"El Banco de Espana, gobernado por Pablo Hernandez de Cos hasta 2024 y por Margarita Delgado en 2025, supervisa el sector. "
|
|
"Luis de Guindos, vicepresidente del Banco Central Europeo, fue ministro de Economia en el gobierno de Mariano Rajoy. "
|
|
"La Comision Nacional del Mercado de Valores, presidida por Rodrigo Buenaventura, regula los mercados financieros. "
|
|
"BBVA anuncio en mayo de 2024 una OPA hostil sobre Banco Sabadell que el consejo del banco rechazo inicialmente. "
|
|
"Cesar Gonzalez-Bueno, CEO de Sabadell, defendio la independencia del banco junto con su presidente Josep Oliu. "
|
|
"Repsol, presidida por Antonio Brufau y con CEO Josu Jon Imaz, vendio su filial mexicana a Macquarie. "
|
|
"Iberdrola, liderada por Ignacio Galan, opera Avangrid en EEUU y firmo un acuerdo PPA con Amazon. "
|
|
"Andy Jassy, CEO de Amazon desde Seattle, agradecio el contrato a Iberdrola en una nota publica. "
|
|
"Endesa, filial de la italiana Enel, tiene como CEO a Marina Serrano y opera en Espana, Portugal y Marruecos. "
|
|
"Ferrovial, presidida por Rafael del Pino, traslado su sede social a Holanda en 2022 generando polemica politica. "
|
|
"ACS, presidida por Florentino Perez, sigue siendo lider mundial en concesiones de infraestructura. "
|
|
"Inditex, fundada por Amancio Ortega y presidida por Marta Ortega desde 2022, tiene su sede en Arteixo, A Coruna. "
|
|
"Pablo Isla, expresidente de Inditex y actual consejero de Telefonica, se incorporo al consejo en 2024. "
|
|
"Telefonica, presidida por Jose Maria Alvarez-Pallete, sufrio la entrada del estado en su capital con SEPI. "
|
|
"Saudi Telecom Company adquirio un 9.9% de Telefonica en 2023, lo que motivo la respuesta del gobierno espanol. "
|
|
"Cristina Aldamiz-Echevarria fue nombrada directora de Recursos Humanos del Grupo Mapfre, dirigido por Antonio Huertas. "
|
|
"Naturgy, presidida por Francisco Reynes, recibio una OPA parcial del fondo emirati IFM en 2021 que se cancelo. "
|
|
"Indra, con Marc Murtra como presidente, se ha posicionado como contratista clave de Defensa para el ministerio de Margarita Robles."
|
|
)
|
|
|
|
|
|
SCHEMA_RICH_CORPORATE = """{
|
|
"organizations": [
|
|
{
|
|
"name": "string",
|
|
"ceo": "string",
|
|
"chairman_president": "string",
|
|
"headquartered_in": "string",
|
|
"subsidiaries": ["string"],
|
|
"parent_company": "string"
|
|
}
|
|
],
|
|
"people": [
|
|
{
|
|
"name": "string",
|
|
"role": "string",
|
|
"organization": "string"
|
|
}
|
|
],
|
|
"agreements": [
|
|
{
|
|
"between": ["string"],
|
|
"topic": "string",
|
|
"amount": "string"
|
|
}
|
|
]
|
|
}"""
|
|
|
|
SCHEMA_RICH_GDPR = """{
|
|
"data_controller": {
|
|
"name": "string",
|
|
"address": "string",
|
|
"registration": "string"
|
|
},
|
|
"dpo_contact": {
|
|
"email": "string",
|
|
"address": "string"
|
|
},
|
|
"data_categories": ["string"],
|
|
"rights_listed": ["string"],
|
|
"authorities_mentioned": [
|
|
{
|
|
"name": "string",
|
|
"url_or_contact": "string"
|
|
}
|
|
],
|
|
"laws_mentioned": ["string"]
|
|
}"""
|
|
|
|
|
|
def parse_json_safe(text: str):
|
|
"""Parser robusto: busca el PRIMER `{` y trunca progresivamente."""
|
|
if not text: return None
|
|
s = text.find("{")
|
|
if s < 0: return None
|
|
for end in range(len(text), s, -1):
|
|
try:
|
|
return json.loads(text[s:end])
|
|
except Exception:
|
|
continue
|
|
return None
|
|
|
|
|
|
def run_extract(model, tokenizer, device, document, template, max_new_tokens=1024):
|
|
messages = [{"role": "user", "content": document}]
|
|
text = tokenizer.apply_chat_template(
|
|
messages, template=template, tokenize=False, add_generation_prompt=True,
|
|
)
|
|
inputs = tokenizer([text], padding=True, return_tensors="pt").to(device)
|
|
t0 = time.time()
|
|
generated = model.generate(
|
|
**inputs,
|
|
do_sample=False,
|
|
num_beams=1,
|
|
max_new_tokens=max_new_tokens,
|
|
repetition_penalty=1.15, # ⭐ EVITA BUCLES
|
|
pad_token_id=tokenizer.eos_token_id,
|
|
)
|
|
elapsed = time.time() - t0
|
|
n_input = inputs["input_ids"].shape[1]
|
|
n_output = generated.shape[1] - n_input
|
|
out_text = tokenizer.decode(generated[0][n_input:], skip_special_tokens=True)
|
|
parsed = parse_json_safe(out_text)
|
|
return {
|
|
"elapsed_s": round(elapsed, 2),
|
|
"n_input_tokens": int(n_input),
|
|
"n_output_tokens": int(n_output),
|
|
"raw_text": out_text,
|
|
"parsed": parsed,
|
|
}
|
|
|
|
|
|
# ── agregadores y coreferencia ──
|
|
|
|
def aggregate_corporate(results: list[dict]) -> dict:
|
|
"""Acumula organizations / people / agreements de N chunks."""
|
|
orgs = {} # name_lower -> dict (con counts y mejores valores)
|
|
people = {} # name_lower -> dict
|
|
agreements = []
|
|
|
|
for r in results:
|
|
parsed = r.get("parsed") or {}
|
|
for o in parsed.get("organizations", []) or []:
|
|
if not isinstance(o, dict): continue
|
|
name = (o.get("name") or "").strip()
|
|
if not name: continue
|
|
key = name.lower()
|
|
if key not in orgs:
|
|
orgs[key] = {"name": name, "count": 0, "ceo": [], "chairman_president": [],
|
|
"headquartered_in": [], "subsidiaries": set(), "parent_company": []}
|
|
orgs[key]["count"] += 1
|
|
for f in ("ceo", "chairman_president", "headquartered_in", "parent_company"):
|
|
v = o.get(f)
|
|
if v and isinstance(v, str) and v.strip():
|
|
orgs[key][f].append(v.strip())
|
|
for sub in (o.get("subsidiaries") or []):
|
|
if isinstance(sub, str) and sub.strip():
|
|
orgs[key]["subsidiaries"].add(sub.strip())
|
|
|
|
for p in parsed.get("people", []) or []:
|
|
if not isinstance(p, dict): continue
|
|
name = (p.get("name") or "").strip()
|
|
if not name: continue
|
|
key = name.lower()
|
|
if key not in people:
|
|
people[key] = {"name": name, "count": 0, "roles": [], "organizations": []}
|
|
people[key]["count"] += 1
|
|
r_ = p.get("role")
|
|
if r_ and isinstance(r_, str) and r_.strip():
|
|
people[key]["roles"].append(r_.strip())
|
|
o_ = p.get("organization")
|
|
if o_ and isinstance(o_, str) and o_.strip():
|
|
people[key]["organizations"].append(o_.strip())
|
|
|
|
for ag in parsed.get("agreements", []) or []:
|
|
if not isinstance(ag, dict): continue
|
|
parties = [p.strip() for p in (ag.get("between") or []) if isinstance(p, str) and p.strip()]
|
|
if len(parties) >= 2:
|
|
agreements.append({"between": parties, "topic": ag.get("topic"), "amount": ag.get("amount")})
|
|
|
|
# Convertir sets a listas
|
|
for o in orgs.values():
|
|
o["subsidiaries"] = sorted(o["subsidiaries"])
|
|
return {"organizations": list(orgs.values()), "people": list(people.values()), "agreements": agreements}
|
|
|
|
|
|
def aggregate_gdpr(results: list[dict]) -> dict:
|
|
out = {
|
|
"data_controllers": [], # multiple by chunk
|
|
"dpo_contacts": [],
|
|
"data_categories": Counter(),
|
|
"rights_listed": Counter(),
|
|
"authorities": {}, # name_lower -> {name, contact_options[], count}
|
|
"laws": Counter(),
|
|
}
|
|
for r in results:
|
|
parsed = r.get("parsed") or {}
|
|
dc = parsed.get("data_controller") or {}
|
|
if isinstance(dc, dict) and dc.get("name"):
|
|
out["data_controllers"].append(dc)
|
|
dpo = parsed.get("dpo_contact") or {}
|
|
if isinstance(dpo, dict) and (dpo.get("email") or dpo.get("address")):
|
|
out["dpo_contacts"].append(dpo)
|
|
for c in parsed.get("data_categories", []) or []:
|
|
if isinstance(c, str) and c.strip():
|
|
out["data_categories"][c.strip()] += 1
|
|
for rt in parsed.get("rights_listed", []) or []:
|
|
if isinstance(rt, str) and rt.strip():
|
|
out["rights_listed"][rt.strip()] += 1
|
|
for a in parsed.get("authorities_mentioned", []) or []:
|
|
if not isinstance(a, dict): continue
|
|
name = (a.get("name") or "").strip()
|
|
if not name: continue
|
|
key = name.lower()
|
|
if key not in out["authorities"]:
|
|
out["authorities"][key] = {"name": name, "contact_options": [], "count": 0}
|
|
out["authorities"][key]["count"] += 1
|
|
c = a.get("url_or_contact")
|
|
if c and isinstance(c, str) and c.strip():
|
|
out["authorities"][key]["contact_options"].append(c.strip())
|
|
for l in parsed.get("laws_mentioned", []) or []:
|
|
if isinstance(l, str) and l.strip():
|
|
out["laws"][l.strip()] += 1
|
|
out["data_categories"] = dict(out["data_categories"])
|
|
out["rights_listed"] = dict(out["rights_listed"])
|
|
out["laws"] = dict(out["laws"])
|
|
out["authorities"] = list(out["authorities"].values())
|
|
return out
|
|
|
|
|
|
def normalize_name(s: str) -> str:
|
|
s = s.strip()
|
|
s = re.sub(r"[\.,;:\"'`()\[\]]", "", s)
|
|
s = re.sub(r"\s+", " ", s)
|
|
return s.strip().lower()
|
|
|
|
|
|
def merge_aliases(entity_names: list[str]) -> dict:
|
|
"""Devuelve un dict {nombre_original → nombre_canonico}."""
|
|
norm_groups = defaultdict(list)
|
|
for n in entity_names:
|
|
norm_groups[normalize_name(n)].append(n)
|
|
canonical: dict = {}
|
|
canonical_data: dict = {}
|
|
for nrm, group in norm_groups.items():
|
|
winner = max(group, key=lambda x: (len(x), x))
|
|
for n in group:
|
|
canonical[n] = winner
|
|
canonical_data[winner] = group
|
|
canon_names = sorted(canonical_data.keys(), key=len, reverse=True)
|
|
absorbed = {}
|
|
for long_n in canon_names:
|
|
long_norm = normalize_name(long_n)
|
|
for short_n in canon_names:
|
|
if short_n == long_n or short_n in absorbed: continue
|
|
short_norm = normalize_name(short_n)
|
|
if len(short_norm) < 4: continue
|
|
if re.search(r"\b" + re.escape(short_norm) + r"\b", long_norm):
|
|
absorbed[short_n] = long_n
|
|
final = {}
|
|
for orig, canon in canonical.items():
|
|
final[orig] = absorbed.get(canon, canon)
|
|
return final
|
|
|
|
|
|
def build_corporate_graph(agg: dict, alias_map: dict | None = None) -> dict:
|
|
"""Construye nodos y aristas del grafo corporate."""
|
|
if alias_map is None: alias_map = {}
|
|
def resolve(n): return alias_map.get(n, n)
|
|
|
|
nodes = {} # name -> type
|
|
edges = set() # (h, kind, t)
|
|
|
|
for org in agg["organizations"]:
|
|
name = resolve(org["name"])
|
|
nodes[name] = "organization"
|
|
for ceo in org["ceo"]:
|
|
ceo_r = resolve(ceo)
|
|
nodes.setdefault(ceo_r, "person")
|
|
edges.add((ceo_r, "ceo_of", name))
|
|
for pres in org["chairman_president"]:
|
|
pres_r = resolve(pres)
|
|
nodes.setdefault(pres_r, "person")
|
|
edges.add((pres_r, "president_of", name))
|
|
for hq in org["headquartered_in"]:
|
|
hq_r = resolve(hq)
|
|
nodes.setdefault(hq_r, "location")
|
|
edges.add((name, "headquartered_in", hq_r))
|
|
for parent in org["parent_company"]:
|
|
parent_r = resolve(parent)
|
|
nodes.setdefault(parent_r, "organization")
|
|
edges.add((name, "subsidiary_of", parent_r))
|
|
for sub in org["subsidiaries"]:
|
|
sub_r = resolve(sub)
|
|
nodes.setdefault(sub_r, "organization")
|
|
edges.add((sub_r, "subsidiary_of", name))
|
|
|
|
for p in agg["people"]:
|
|
name = resolve(p["name"])
|
|
nodes.setdefault(name, "person")
|
|
for org in p["organizations"]:
|
|
org_r = resolve(org)
|
|
nodes.setdefault(org_r, "organization")
|
|
edges.add((name, "works_at", org_r))
|
|
|
|
for ag in agg["agreements"]:
|
|
parties = [resolve(p) for p in ag["between"]]
|
|
for p in parties:
|
|
nodes.setdefault(p, "organization")
|
|
for i, a in enumerate(parties):
|
|
for b in parties[i+1:]:
|
|
edges.add((a, "agreement_with", b))
|
|
|
|
return {"nodes": nodes, "edges": list(edges)}
|
|
|
|
|
|
def build_gdpr_graph(agg: dict, alias_map: dict | None = None) -> dict:
|
|
if alias_map is None: alias_map = {}
|
|
def resolve(n): return alias_map.get(n, n)
|
|
nodes = {}
|
|
edges = set()
|
|
|
|
# data_controller — pick the first non-empty
|
|
for dc in agg["data_controllers"]:
|
|
if dc.get("name"):
|
|
name = resolve(dc["name"].strip())
|
|
nodes[name] = "data_controller"
|
|
if dc.get("address"):
|
|
addr = resolve(dc["address"].strip())
|
|
nodes.setdefault(addr, "location")
|
|
edges.add((name, "located_in", addr))
|
|
break # solo el primero
|
|
|
|
for dpo in agg["dpo_contacts"]:
|
|
if dpo.get("email"):
|
|
email = dpo["email"].strip()
|
|
nodes.setdefault(email, "email")
|
|
if dpo.get("address"):
|
|
addr = dpo["address"].strip()
|
|
nodes.setdefault(addr, "location")
|
|
|
|
for c in agg["data_categories"]:
|
|
nodes.setdefault(c, "data_category")
|
|
for r in agg["rights_listed"]:
|
|
nodes.setdefault(r, "right")
|
|
for a in agg["authorities"]:
|
|
name = resolve(a["name"].strip())
|
|
nodes.setdefault(name, "authority")
|
|
for c in a["contact_options"][:1]: # 1 contact por authority
|
|
nodes.setdefault(c, "url")
|
|
edges.add((name, "contact", c))
|
|
for l in agg["laws"]:
|
|
nodes.setdefault(l, "law")
|
|
|
|
return {"nodes": nodes, "edges": list(edges)}
|
|
|
|
|
|
# ── main ──
|
|
|
|
def main():
|
|
print("[load] loading model + tokenizer...", flush=True)
|
|
t0 = time.time()
|
|
import torch
|
|
from transformers import AutoTokenizer, AutoModelForImageTextToText
|
|
|
|
if not torch.cuda.is_available():
|
|
print("CUDA not available — exiting", flush=True)
|
|
return
|
|
device = "cuda"
|
|
dtype = torch.bfloat16
|
|
print(f"[device] {device} dtype={dtype}", flush=True)
|
|
|
|
tokenizer = AutoTokenizer.from_pretrained(
|
|
"numind/NuExtract-2.0-2B", trust_remote_code=True, padding_side="left",
|
|
)
|
|
model = AutoModelForImageTextToText.from_pretrained(
|
|
"numind/NuExtract-2.0-2B",
|
|
trust_remote_code=True,
|
|
torch_dtype=dtype,
|
|
attn_implementation="sdpa",
|
|
).to(device)
|
|
model.eval()
|
|
print(f"[load] done in {time.time()-t0:.1f}s", flush=True)
|
|
|
|
out: dict = {"meta": {"device": device, "dtype": str(dtype),
|
|
"model": "numind/NuExtract-2.0-2B",
|
|
"repetition_penalty": 1.15, "max_chars_chunk": 800}}
|
|
|
|
# ── A. LONG_TEXT_ES con chunking
|
|
print("\n[A] LONG_TEXT_ES — chunking + run...", flush=True)
|
|
long_chunks = chunk_with_overlap(LONG_TEXT_ES, max_chars=800, overlap_sentences=1)
|
|
print(f" {len(LONG_TEXT_ES)} chars → {len(long_chunks)} chunks", flush=True)
|
|
long_results = []
|
|
t_start = time.time()
|
|
for i, c in enumerate(long_chunks):
|
|
r = run_extract(model, tokenizer, device, c["text"], SCHEMA_RICH_CORPORATE)
|
|
ok = "OK" if r["parsed"] else "FAIL"
|
|
print(f" [chunk {i+1}/{len(long_chunks)}] {len(c['text'])}c {r['elapsed_s']}s out={r['n_output_tokens']} {ok}", flush=True)
|
|
long_results.append(r)
|
|
long_elapsed = time.time() - t_start
|
|
long_agg = aggregate_corporate(long_results)
|
|
# alias map sobre todos los nombres mencionados
|
|
all_names_long = ([o["name"] for o in long_agg["organizations"]]
|
|
+ [p["name"] for p in long_agg["people"]]
|
|
+ [hq for o in long_agg["organizations"] for hq in o["headquartered_in"]]
|
|
+ [s for o in long_agg["organizations"] for s in o["subsidiaries"]])
|
|
alias_long = merge_aliases(list(set(all_names_long)))
|
|
long_graph = build_corporate_graph(long_agg, alias_long)
|
|
print(f" total {long_elapsed:.1f}s agregado: orgs={len(long_agg['organizations'])} people={len(long_agg['people'])} agreements={len(long_agg['agreements'])}", flush=True)
|
|
print(f" grafo: nodos={len(long_graph['nodes'])} aristas={len(long_graph['edges'])}", flush=True)
|
|
out["long_text"] = {
|
|
"elapsed_s": round(long_elapsed, 1),
|
|
"n_chunks": len(long_chunks),
|
|
"n_chunks_parsed_ok": sum(1 for r in long_results if r["parsed"] is not None),
|
|
"agg": long_agg,
|
|
"graph": {"nodes": long_graph["nodes"], "edges": long_graph["edges"]},
|
|
"n_nodes": len(long_graph["nodes"]),
|
|
"n_edges": len(long_graph["edges"]),
|
|
"n_isolates": sum(1 for n in long_graph["nodes"] if n not in {a for a, _, _ in long_graph["edges"]} | {b for _, _, b in long_graph["edges"]}),
|
|
}
|
|
del long_results
|
|
gc.collect()
|
|
|
|
# ── B. PDF entero
|
|
print("\n[B] PDF — extract + clean + chunk + run all chunks...", flush=True)
|
|
raw = extract_pdf_text(str(PDF_PATH))
|
|
clean = clean_pdf_text(raw)
|
|
pdf_chunks = chunk_with_overlap(clean, max_chars=800, overlap_sentences=1)
|
|
print(f" PDF: {len(raw):,} → {len(clean):,} chars → {len(pdf_chunks)} chunks", flush=True)
|
|
pdf_results = []
|
|
t_start = time.time()
|
|
for i, c in enumerate(pdf_chunks):
|
|
r = run_extract(model, tokenizer, device, c["text"], SCHEMA_RICH_GDPR)
|
|
if (i+1) % 10 == 0:
|
|
ok_count = sum(1 for r in pdf_results if r["parsed"] is not None)
|
|
print(f" [chunk {i+1}/{len(pdf_chunks)}] {ok_count}/{i+1} parsed OK ({time.time()-t_start:.0f}s acumulado)", flush=True)
|
|
pdf_results.append(r)
|
|
pdf_elapsed = time.time() - t_start
|
|
pdf_agg = aggregate_gdpr(pdf_results)
|
|
# alias map para autoridades + data controllers
|
|
all_names_pdf = ([dc["name"] for dc in pdf_agg["data_controllers"] if dc.get("name")]
|
|
+ [a["name"] for a in pdf_agg["authorities"]])
|
|
alias_pdf = merge_aliases(list(set(all_names_pdf)))
|
|
pdf_graph = build_gdpr_graph(pdf_agg, alias_pdf)
|
|
print(f" total {pdf_elapsed:.1f}s = {pdf_elapsed/60:.1f} min", flush=True)
|
|
print(f" parsed OK: {sum(1 for r in pdf_results if r['parsed'] is not None)}/{len(pdf_chunks)}", flush=True)
|
|
print(f" grafo: nodos={len(pdf_graph['nodes'])} aristas={len(pdf_graph['edges'])}", flush=True)
|
|
out["pdf"] = {
|
|
"elapsed_s": round(pdf_elapsed, 1),
|
|
"n_chunks": len(pdf_chunks),
|
|
"n_chunks_parsed_ok": sum(1 for r in pdf_results if r["parsed"] is not None),
|
|
"agg_summary": {
|
|
"n_data_controllers": len(pdf_agg["data_controllers"]),
|
|
"n_dpo_contacts": len(pdf_agg["dpo_contacts"]),
|
|
"n_data_categories": len(pdf_agg["data_categories"]),
|
|
"n_rights": len(pdf_agg["rights_listed"]),
|
|
"n_authorities": len(pdf_agg["authorities"]),
|
|
"n_laws": len(pdf_agg["laws"]),
|
|
},
|
|
"agg_full": pdf_agg,
|
|
"graph": {"nodes": pdf_graph["nodes"], "edges": pdf_graph["edges"]},
|
|
"n_nodes": len(pdf_graph["nodes"]),
|
|
"n_edges": len(pdf_graph["edges"]),
|
|
}
|
|
|
|
out_path = HERE / "nuextract_full.json"
|
|
out_path.write_text(json.dumps(out, indent=2, ensure_ascii=False))
|
|
print(f"\n[saved] {out_path}", flush=True)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|