egutierrez
dabc945eda
Extrae al registry funciones del proyecto interno footprint_aurgi: - core (6): slugify_ascii, normalize_for_join, cp_provincia_es, infer_provincia_from_cp, safe_read_csv_fallback, csv_to_parquet_duckdb - geo puras (7): haversine_km, point_in_ring, point_in_polygon, point_in_polygons_bbox, polygon_bbox, extent_with_padding, distance_bucket - geo I/O (4): load_geojson_polygons, load_boundary_gdf, add_basemap_osm, add_basemap_with_timeout - valhalla client (4): valhalla_route, valhalla_isochrone, valhalla_isochrones_async, valhalla_matrix_1_to_n - datascience stats (7): trimmed_mean, geometric_mean, detect_distribution_type, best_central_tendency, summary_stats, kde_density_levels, alpha_shape_concave_hull - datascience fuzzy (3): fuzzy_merge_adaptive (rapidfuzz), words_to_dataset, remove_words_from_column - datascience viz (2): plot_kde_2d, plot_heatmap_log - infra (4): compress_pdf_ghostscript, render_table_page_pdfpages, add_header_logo, osm2pgsql_ingest - pipelines (4): setup_geo_stack_docker, compute_centers_reachability, generate_isochrones_by_zone, count_points_per_zone - types geo (4): LonLat, BBox, IsochroneRequest, Centro Incluye: - apps/footprint_geo_stack/ (PostGIS + Martin + Valhalla via docker-compose) - 131/132 tests pasan (1 skip esperado: osm2pgsql en PATH) - Issue tracker dev/issues/0052-footprint-aurgi-extraction.md - Atribucion uniforme: source_repo internal:footprint_aurgi, source_license internal-aurgi - Build con 9 agentes en paralelo (8 wave 1 + 1 wave 2 pipelines) Tambien commitea trabajo previo no commiteado: aggregate_extraction_results, chunk_with_overlap, clean_pdf_text, merge_entity_aliases, extract_graph_gliner2, extract_relations_mrebel, extract_triples_spacy_es, gliner2/mrebel/marianmt/rebel/spacy_es load_model, parse_rebel_output, translate_es_to_en, issue 0050/0051. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
106 lines
3.4 KiB
Python
106 lines
3.4 KiB
Python
"""Parser puro del wire format de REBEL / mREBEL."""
|
|
|
|
from __future__ import annotations
|
|
|
|
|
|
def parse_rebel_output(decoded_text: str) -> list[dict]:
|
|
"""Parse REBEL / mREBEL decoded output into typed triplets.
|
|
|
|
The input is the string produced by the HuggingFace tokenizer with
|
|
``skip_special_tokens=False``, e.g.::
|
|
|
|
tp_XX<triplet> Pablo Isla <per> Inditex <org> employer<triplet> ...
|
|
|
|
Args:
|
|
decoded_text: Raw decoded string from the seq2seq model, including
|
|
special tokens like ``<triplet>``, ``<relation>``, ``<per>``,
|
|
``<org>``, ``<loc>``, etc.
|
|
|
|
Returns:
|
|
List of dicts with keys:
|
|
``head`` (str), ``head_type`` (str),
|
|
``type`` (str), ``tail`` (str), ``tail_type`` (str).
|
|
Returns an empty list on empty input or if no complete triplet is
|
|
found. Never raises.
|
|
"""
|
|
if not decoded_text or not decoded_text.strip():
|
|
return []
|
|
|
|
triplets: list[dict] = []
|
|
|
|
# Strip language / padding tokens common to mREBEL.
|
|
text = (
|
|
decoded_text
|
|
.replace("<s>", "")
|
|
.replace("<pad>", "")
|
|
.replace("</s>", "")
|
|
.replace("tp_XX", "")
|
|
.replace("__en__", "")
|
|
.strip()
|
|
)
|
|
|
|
current = "x" # x=init, t=head span, s=tail span, o=relation span
|
|
subject = ""
|
|
relation = ""
|
|
object_ = ""
|
|
object_type = ""
|
|
subject_type = ""
|
|
|
|
for token in text.split():
|
|
if token in ("<triplet>", "<relation>"):
|
|
current = "t"
|
|
if relation:
|
|
triplets.append(
|
|
{
|
|
"head": subject.strip(),
|
|
"head_type": subject_type,
|
|
"type": relation.strip(),
|
|
"tail": object_.strip(),
|
|
"tail_type": object_type,
|
|
}
|
|
)
|
|
relation = ""
|
|
subject = ""
|
|
elif token.startswith("<") and token.endswith(">"):
|
|
if current in ("t", "o"):
|
|
# Closing the head span — now reading tail.
|
|
current = "s"
|
|
if relation:
|
|
triplets.append(
|
|
{
|
|
"head": subject.strip(),
|
|
"head_type": subject_type,
|
|
"type": relation.strip(),
|
|
"tail": object_.strip(),
|
|
"tail_type": object_type,
|
|
}
|
|
)
|
|
object_ = ""
|
|
subject_type = token[1:-1]
|
|
else:
|
|
# Closing the tail span — now reading relation.
|
|
current = "o"
|
|
object_type = token[1:-1]
|
|
relation = ""
|
|
else:
|
|
if current == "t":
|
|
subject += " " + token
|
|
elif current == "s":
|
|
object_ += " " + token
|
|
elif current == "o":
|
|
relation += " " + token
|
|
|
|
# Flush the last triplet if all fields are present.
|
|
if subject and relation and object_ and object_type and subject_type:
|
|
triplets.append(
|
|
{
|
|
"head": subject.strip(),
|
|
"head_type": subject_type,
|
|
"type": relation.strip(),
|
|
"tail": object_.strip(),
|
|
"tail_type": object_type,
|
|
}
|
|
)
|
|
|
|
return triplets
|