egutierrez
7a94160fd2
Bloque de cambios revisados y validados con el usuario en sesiones previas que no habian aterrizado en commits propios. Lista por tema: * enrichers: web_search ahora usa lite.duckduckgo.com como endpoint primario (mas tolerante con bot detection desde IP residencial), con fallback al endpoint html. Detecta pagina captcha y emite error claro si ambos fallan. Anyade _DDGLiteParser para el formato lite + auto-pick de parser por contenido. * enrichers: tipo Webpage unificado en Url (campos de cuerpo cacheado viven en metadata del Url). Manifests actualizados (applies_to: [Url]). fetch_webpage ya no convierte Url->Webpage. * enrichers/manifest: campo `params` parseado a EnricherSpec.params (name, type, default_value, description). UI puede renderizar dialog de configuracion. * jobs: fix de path conversion para Python embebido nativo Windows (no convertir a /mnt/c/... cuando el subproceso es Windows-native; solo cuando es bash o python via WSL). * main.cpp: ventana ImGui (no modal) "Run enricher" con layout 2-col (label izq, input der). Inserta job con JSON tipado. Layout clustering apretado: hijos del mismo anchor en un solo anillo alrededor del padre, sin desperdigar por anillos crecientes. * views: inspector con layout 2-col via BeginTable (Identity, Schema fields, Extras). Description full-width debajo de su label. * tests: portable conftest (auto-detecta REGISTRY_ROOT, PYTHON_BIN, ENRICHERS_DIR para WSL y Windows portable). _runner.py trampoline inyecta stub via sys.path porque embedded Python ignora PYTHONPATH. Tests bash-only (vendor_script, freeze, dispatcher bash, resolver Linux-binary) skipean en Windows. Tests existentes adaptados a Webpage->Url. Resultado actual: 32 passed WSL, 21 passed + 11 skipped Windows.
78 lines
2.8 KiB
Python
78 lines
2.8 KiB
Python
"""Tests del enricher fetch_webpage con red mockeada via stub de requests."""
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
from pathlib import Path
|
|
|
|
from conftest import (
|
|
base_ctx, get_entity, list_entities, list_relations,
|
|
make_node, run_enricher, stub_requests,
|
|
)
|
|
|
|
|
|
SAMPLE_HTML = """<!DOCTYPE html>
|
|
<html><head><title>Acme Demo</title></head>
|
|
<body>
|
|
<h1>Hola</h1>
|
|
<p>Esta es la pagina de prueba con un <a href="/x">enlace</a>.</p>
|
|
<p>Email de contacto: ops@acme.example</p>
|
|
</body></html>
|
|
"""
|
|
|
|
|
|
def test_fetch_webpage_creates_domain_and_caches(ops_db, app_dir, registry_root,
|
|
tmp_path):
|
|
make_node(ops_db, node_id="u1", name="acme",
|
|
type_ref="Url", metadata={"url": "https://www.acme.example/"})
|
|
plan = {
|
|
"default": {"text": SAMPLE_HTML, "status": 200,
|
|
"headers": {"Content-Type": "text/html; charset=utf-8"}},
|
|
}
|
|
env = stub_requests(tmp_path, plan)
|
|
|
|
ctx = base_ctx(ops_db=ops_db, app_dir=app_dir, registry_root=registry_root,
|
|
node_id="u1", node_name="acme", node_type="Url",
|
|
metadata={"url": "https://www.acme.example/"})
|
|
|
|
rc, out, err = run_enricher("fetch_webpage", ctx, env=env)
|
|
assert rc == 0, f"stderr={err}"
|
|
assert out is not None, err
|
|
assert out["status_code"] == 200
|
|
assert out["title"] == "Acme Demo"
|
|
assert out["entities_added"] == 1 # Domain
|
|
assert out["relations_added"] == 1 # BELONGS_TO
|
|
|
|
# El nodo Url permanece como Url (Webpage se unifico en Url).
|
|
e = get_entity(ops_db, "u1")
|
|
assert e["type_ref"] == "Url", e
|
|
assert e["metadata"]["title"] == "Acme Demo"
|
|
assert e["metadata"]["status_code"] == 200
|
|
|
|
# Cache existe.
|
|
html_path = Path(app_dir) / e["metadata"]["html_path"]
|
|
assert html_path.exists()
|
|
assert "Acme Demo" in html_path.read_text(encoding="utf-8")
|
|
|
|
# Domain creado con relacion.
|
|
domains = list_entities(ops_db, type_ref="Domain")
|
|
assert any(d["name"] == "www.acme.example" for d in domains)
|
|
rels = list_relations(ops_db, name="BELONGS_TO")
|
|
assert len(rels) == 1
|
|
|
|
|
|
def test_fetch_webpage_handles_http_error(ops_db, app_dir, registry_root,
|
|
tmp_path):
|
|
make_node(ops_db, node_id="u1", name="bad",
|
|
type_ref="Url", metadata={"url": "https://no.example/"})
|
|
plan = {"default": {"text": "<html></html>", "status": 404}}
|
|
env = stub_requests(tmp_path, plan)
|
|
|
|
ctx = base_ctx(ops_db=ops_db, app_dir=app_dir, registry_root=registry_root,
|
|
node_id="u1", node_name="bad", node_type="Url",
|
|
metadata={"url": "https://no.example/"})
|
|
|
|
rc, out, err = run_enricher("fetch_webpage", ctx, env=env)
|
|
# 404 es respuesta valida — exit 0 con status_code en el resumen.
|
|
assert rc == 0, err
|
|
assert out["status_code"] == 404
|