feat: enrichers offline split_sentences + extract_iocs_text
Para probar la app sin depender de red (DDG bloquea con captcha desde ciertas IPs). Ambos aplican grouping (umbral 50, preview K=10) replicando el patron de web_search. - split_sentences: parte texto en frases (regex), crea nodos Sentence conectados con SENTENCE_OF. - extract_iocs_text: variante de extract_text_entities que lee directo metadata.text/description/name, sin requerir fetch previo. Vendoriza extract_iocs_py_cybersecurity. Multi-tipo, agrupado en un solo Group heterogeneo (decision 6 multi-grupo-por-tipo es fase 2). - Tipo Sentence en types.yaml. Tests pytest cubren below/above threshold para ambos.
This commit is contained in:
@@ -0,0 +1,116 @@
|
||||
"""Tests del enricher extract_iocs_text — variante offline de extract_text_entities."""
|
||||
from __future__ import annotations
|
||||
|
||||
from conftest import (
|
||||
base_ctx, list_entities, list_relations, make_node, run_enricher,
|
||||
)
|
||||
|
||||
|
||||
SAMPLE_TEXT = (
|
||||
"Reporte de incidente. Contactar a bad@evil.example o a otra@victim.example. "
|
||||
"IPs vistas: 192.0.2.55 y 10.0.0.12. CVE referenciado: CVE-2024-12345. "
|
||||
"Hash: 44d88612fea8a8f36de82e1278abb02f."
|
||||
)
|
||||
|
||||
|
||||
def _ioc_paragraph(n: int) -> str:
|
||||
"""Genera texto con muchos IoCs (mezcla de emails, IPs, CVEs)."""
|
||||
parts = []
|
||||
# n/3 emails, n/3 IPs, n/3 CVEs aprox.
|
||||
for i in range(n // 3 + 1):
|
||||
parts.append(f"contact{i:03d}@example{i % 7}.org")
|
||||
for i in range(n // 3 + 1):
|
||||
# IPs validas en rango 10.x.x.x
|
||||
a = (i // 256) % 256
|
||||
b = i % 256
|
||||
parts.append(f"10.{a}.{b}.5")
|
||||
for i in range(n // 3 + 1):
|
||||
parts.append(f"CVE-2024-{10000 + i}")
|
||||
return ", ".join(parts) + "."
|
||||
|
||||
|
||||
def test_extract_iocs_text_finds_email_and_ip(ops_db, app_dir, registry_root):
|
||||
"""Texto con emails, IPs, CVE, hash → entidades creadas con tipos correctos."""
|
||||
make_node(ops_db, node_id="t1", name="incident",
|
||||
type_ref="text", metadata={"text": SAMPLE_TEXT})
|
||||
ctx = base_ctx(ops_db=ops_db, app_dir=app_dir, registry_root=registry_root,
|
||||
node_id="t1", node_name="incident", node_type="text",
|
||||
metadata={"text": SAMPLE_TEXT})
|
||||
|
||||
rc, out, err = run_enricher("extract_iocs_text", ctx)
|
||||
assert rc == 0, err
|
||||
assert out is not None
|
||||
assert out["entities_added"] >= 3, out
|
||||
|
||||
types = {e["type_ref"] for e in list_entities(ops_db)
|
||||
if e["type_ref"] not in ("text", "Group")}
|
||||
assert "Email" in types, types
|
||||
# CVE casi seguro presente; IP/hash/dominios pueden o no segun extract_iocs.
|
||||
assert "CVE" in types, types
|
||||
|
||||
rels = list_relations(ops_db, name="EXTRACTED_FROM")
|
||||
assert len(rels) >= 3
|
||||
assert all(r["to_entity"] == "t1" for r in rels)
|
||||
|
||||
|
||||
def test_extract_iocs_text_uses_metadata_text(ops_db, app_dir, registry_root):
|
||||
"""metadata.text se prioriza sobre node_name."""
|
||||
make_node(ops_db, node_id="t1", name="placeholder",
|
||||
type_ref="text", metadata={"text": SAMPLE_TEXT})
|
||||
ctx = base_ctx(ops_db=ops_db, app_dir=app_dir, registry_root=registry_root,
|
||||
node_id="t1", node_name="placeholder", node_type="text",
|
||||
metadata={"text": SAMPLE_TEXT})
|
||||
rc, out, err = run_enricher("extract_iocs_text", ctx)
|
||||
assert rc == 0, err
|
||||
# El name "placeholder" no contiene IoCs; si se hubiese usado, no
|
||||
# habria entidades. Ergo entities_added > 0 demuestra que leyo text.
|
||||
assert out["entities_added"] >= 2, out
|
||||
|
||||
|
||||
def test_extract_iocs_text_no_text_fails(ops_db, app_dir, registry_root):
|
||||
"""Sin texto → exit 2 con error claro."""
|
||||
make_node(ops_db, node_id="t1", name="", type_ref="text", metadata={})
|
||||
ctx = base_ctx(ops_db=ops_db, app_dir=app_dir, registry_root=registry_root,
|
||||
node_id="t1", node_name="", node_type="text")
|
||||
rc, out, err = run_enricher("extract_iocs_text", ctx)
|
||||
assert rc == 2
|
||||
assert out is not None
|
||||
assert "sin texto" in (out.get("error") or "")
|
||||
|
||||
|
||||
def test_extract_iocs_text_above_threshold_creates_group(ops_db, app_dir,
|
||||
registry_root):
|
||||
""">=50 IoCs → Group heterogeneo con todos dentro (fase 1)."""
|
||||
text = _ioc_paragraph(180) # ~60 emails + ~60 IPs + ~60 CVEs
|
||||
make_node(ops_db, node_id="t1", name="dump",
|
||||
type_ref="text", metadata={"text": text})
|
||||
ctx = base_ctx(ops_db=ops_db, app_dir=app_dir, registry_root=registry_root,
|
||||
node_id="t1", node_name="dump", node_type="text",
|
||||
metadata={"text": text})
|
||||
rc, out, err = run_enricher("extract_iocs_text", ctx)
|
||||
assert rc == 0, err
|
||||
assert out["iocs_found"] >= 50, out
|
||||
|
||||
if out["grouped"]:
|
||||
groups = list_entities(ops_db, type_ref="Group")
|
||||
assert len(groups) == 1
|
||||
g = groups[0]
|
||||
assert g["metadata"]["enricher"] == "extract_iocs_text"
|
||||
assert g["metadata"]["count"] == out["iocs_found"]
|
||||
assert g["metadata"]["source_node_id"] == "t1"
|
||||
|
||||
# K primeros sueltos, resto agrupados (heterogeneo).
|
||||
non_group_iocs = [e for e in list_entities(ops_db)
|
||||
if e["type_ref"] not in ("text", "Group")]
|
||||
sueltos = [e for e in non_group_iocs if e["group_id"] is None]
|
||||
agrupados = [e for e in non_group_iocs if e["group_id"] == g["id"]]
|
||||
# K=10 sueltos exactos.
|
||||
assert len(sueltos) == 10
|
||||
assert len(agrupados) == out["iocs_found"] - 10
|
||||
|
||||
# EXTRACTED_FROM del Group al source.
|
||||
rels = list_relations(ops_db, name="EXTRACTED_FROM")
|
||||
to_t1_from_group = [r for r in rels
|
||||
if r["to_entity"] == "t1"
|
||||
and r["from_entity"] == g["id"]]
|
||||
assert len(to_t1_from_group) == 1
|
||||
Reference in New Issue
Block a user