egutierrez
dabc945eda
Extrae al registry funciones del proyecto interno footprint_aurgi: - core (6): slugify_ascii, normalize_for_join, cp_provincia_es, infer_provincia_from_cp, safe_read_csv_fallback, csv_to_parquet_duckdb - geo puras (7): haversine_km, point_in_ring, point_in_polygon, point_in_polygons_bbox, polygon_bbox, extent_with_padding, distance_bucket - geo I/O (4): load_geojson_polygons, load_boundary_gdf, add_basemap_osm, add_basemap_with_timeout - valhalla client (4): valhalla_route, valhalla_isochrone, valhalla_isochrones_async, valhalla_matrix_1_to_n - datascience stats (7): trimmed_mean, geometric_mean, detect_distribution_type, best_central_tendency, summary_stats, kde_density_levels, alpha_shape_concave_hull - datascience fuzzy (3): fuzzy_merge_adaptive (rapidfuzz), words_to_dataset, remove_words_from_column - datascience viz (2): plot_kde_2d, plot_heatmap_log - infra (4): compress_pdf_ghostscript, render_table_page_pdfpages, add_header_logo, osm2pgsql_ingest - pipelines (4): setup_geo_stack_docker, compute_centers_reachability, generate_isochrones_by_zone, count_points_per_zone - types geo (4): LonLat, BBox, IsochroneRequest, Centro Incluye: - apps/footprint_geo_stack/ (PostGIS + Martin + Valhalla via docker-compose) - 131/132 tests pasan (1 skip esperado: osm2pgsql en PATH) - Issue tracker dev/issues/0052-footprint-aurgi-extraction.md - Atribucion uniforme: source_repo internal:footprint_aurgi, source_license internal-aurgi - Build con 9 agentes en paralelo (8 wave 1 + 1 wave 2 pipelines) Tambien commitea trabajo previo no commiteado: aggregate_extraction_results, chunk_with_overlap, clean_pdf_text, merge_entity_aliases, extract_graph_gliner2, extract_relations_mrebel, extract_triples_spacy_es, gliner2/mrebel/marianmt/rebel/spacy_es load_model, parse_rebel_output, translate_es_to_en, issue 0050/0051. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
55 lines
1.8 KiB
Python
55 lines
1.8 KiB
Python
"""Extrae palabras y sus ocurrencias de textos en bruto."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
from collections import Counter
|
|
from typing import Iterable
|
|
|
|
|
|
_STOPWORDS_ES: frozenset[str] = frozenset({
|
|
"DE", "LA", "EL", "EN", "Y", "A", "LOS", "DEL", "SE", "LAS",
|
|
"UN", "POR", "CON", "NO", "UNA", "SU", "PARA", "ES", "AL", "LO",
|
|
"COMO", "MAS", "O", "PERO", "SUS", "LE", "YA", "ESTE",
|
|
"SI", "PORQUE", "ESTA", "ENTRE", "CUANDO", "MUY", "SIN", "SOBRE",
|
|
"TAMBIEN", "ME", "HASTA", "HAY", "DONDE", "QUIEN", "DESDE", "TODO",
|
|
"NOS", "DURANTE", "TODOS", "UNO", "LES", "NI", "CONTRA", "OTROS",
|
|
})
|
|
|
|
|
|
def words_to_dataset(
|
|
texts: Iterable[str | None],
|
|
min_ocurrencias: int = 1,
|
|
eliminar_stopwords: bool = False,
|
|
) -> list[dict]:
|
|
"""Extrae palabras y ocurrencias de una coleccion de textos.
|
|
|
|
Sin dependencias externas. Tokeniza cada texto con regex \\b\\w+\\b,
|
|
convierte a mayusculas, cuenta ocurrencias y filtra por minimo.
|
|
|
|
Args:
|
|
texts: Iterable de strings (o None). Los None se ignoran.
|
|
min_ocurrencias: Numero minimo de ocurrencias para incluir una
|
|
palabra. Default 1.
|
|
eliminar_stopwords: Si True, filtra palabras comunes en espanol.
|
|
|
|
Returns:
|
|
Lista de dicts {"palabra": str, "ocurrencias": int} ordenada
|
|
por ocurrencias descendente.
|
|
"""
|
|
all_words: list[str] = []
|
|
for text in texts:
|
|
if text is None:
|
|
continue
|
|
words = re.findall(r"\b\w+\b", str(text).upper())
|
|
if eliminar_stopwords:
|
|
words = [w for w in words if w not in _STOPWORDS_ES]
|
|
all_words.extend(words)
|
|
|
|
counter = Counter(all_words)
|
|
return [
|
|
{"palabra": word, "ocurrencias": count}
|
|
for word, count in counter.most_common()
|
|
if count >= min_ocurrencias
|
|
]
|