egutierrez
fed245a738
Capacidades nuevas y cambios (40 -> 42 tools): - page_perceive ahora se genera de forma NATIVA en Go sobre la conexion CDP viva del pool (cdp_get_ax_outline_go_browser). Elimina el subprocess `fn run cdp_perceive_outline` (Python), el venv y la dependencia del binario `fn` en runtime (se borra resolveRoot/exec.Command). Respeta tab_select. - page_perceive acepta frame_id para percibir DENTRO de un iframe. El campo tab_id queda obsoleto (se ignora; usar tab_select) pero se conserva por compatibilidad. - frame_get_text (nueva, lectura): innerText de un iframe via cdp_get_text_in_frame_go_browser. Activa tambien bajo --read-only. - dom_click_xy (nueva, MUTA): click humanizado por coordenadas absolutas via cdp_click_xy_human_go_browser, con mode human/fast/instant y auto-observe. Fallback para actuar sobre lo que el LLM ve en page_screenshot. - page_screenshot devuelve la imagen como image content (cdp_screenshot_bytes_go_browser + mcp.NewToolResultImage) para que el LLM vea los pixeles; path pasa a ser opcional (si se da, ademas guarda a disco). - Auto-observe de las tools *_ref sube su truncado de 4000 a 8000 chars. - Fix de seguridad documental: todas las descripciones del parametro port que decian "Default 9222" (navegador diario del usuario) corregidas a "Default 9333" (Chrome aislado del MCP). El codigo ya usaba 9333; la doc era falsa y podia inducir al modelo a tocar pestanas de banca/correo. uses_functions del app.md: +cdp_get_ax_outline, +cdp_get_text_in_frame, +cdp_screenshot_bytes; -cdp_perceive_outline_py_pipelines. Verificacion: go build OK, go test OK (4 unit pass, 3 e2e skip gated BMCP_E2E=1), go vet OK, gofmt limpio, sin "Default 9222" en el codigo. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
373 lines
15 KiB
Go
373 lines
15 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"time"
|
|
|
|
"github.com/mark3labs/mcp-go/mcp"
|
|
"github.com/mark3labs/mcp-go/server"
|
|
|
|
"fn-registry/functions/browser"
|
|
)
|
|
|
|
// registerDomTools wires DOM interaction tools. find/wait stay on under --read-only.
|
|
func registerDomTools(s *server.MCPServer, d *deps) {
|
|
s.AddTool(domFindByTextTool(), mcp.NewTypedToolHandler(d.handleDomFindByText))
|
|
s.AddTool(domFindRefByTextTool(), mcp.NewTypedToolHandler(d.handleDomFindRefByText))
|
|
s.AddTool(domWaitElementTool(), mcp.NewTypedToolHandler(d.handleDomWaitElement))
|
|
|
|
if !d.readOnly {
|
|
s.AddTool(domClickTool(), mcp.NewTypedToolHandler(d.handleDomClick))
|
|
s.AddTool(domClickHumanTool(), mcp.NewTypedToolHandler(d.handleDomClickHuman))
|
|
s.AddTool(domClickTextTool(), mcp.NewTypedToolHandler(d.handleDomClickText))
|
|
s.AddTool(domTypeTool(), mcp.NewTypedToolHandler(d.handleDomType))
|
|
s.AddTool(domClickRefTool(), mcp.NewTypedToolHandler(d.handleDomClickRef))
|
|
s.AddTool(domTypeRefTool(), mcp.NewTypedToolHandler(d.handleDomTypeRef))
|
|
s.AddTool(domHoverRefTool(), mcp.NewTypedToolHandler(d.handleDomHoverRef))
|
|
s.AddTool(domClickXYTool(), mcp.NewTypedToolHandler(d.handleDomClickXY))
|
|
}
|
|
}
|
|
|
|
// settleDelay es la espera breve tras una acción mutante antes de re-percibir,
|
|
// dando tiempo a que el DOM se asiente (navegación, focus, repaint).
|
|
const settleDelay = 400 * time.Millisecond
|
|
|
|
// ---- dom_click_ref (MUTA) — bucle percibir→actuar ----
|
|
|
|
type domClickRefArgs struct {
|
|
Port int `json:"port"`
|
|
Ref int `json:"ref"`
|
|
Mode string `json:"mode"`
|
|
}
|
|
|
|
func domClickRefTool() mcp.Tool {
|
|
return mcp.NewTool("dom_click_ref",
|
|
mcp.WithDescription("Click sobre el elemento por su #ref del outline de page_perceive (backendDOMNodeId estable). Devuelve el outline actualizado tras la acción (auto-observe)."),
|
|
mcp.WithNumber("port", mcp.Description("CDP port. Default 9333 (Chrome isolated del MCP); usa 9222 explícito solo para adjuntarte al navegador diario.")),
|
|
mcp.WithNumber("ref", mcp.Required(), mcp.Description("#ref del elemento (backendDOMNodeId) leído del outline de page_perceive.")),
|
|
mcp.WithString("mode", mcp.Description("Velocidad: 'human' (default, Bézier+jitter anti-bot), 'fast' (movimiento reducido, scraping masivo), 'instant' (element.click() JS, sin eventos de ratón; también fallback si el elemento no tiene geometría).")),
|
|
)
|
|
}
|
|
|
|
func (d *deps) handleDomClickRef(_ context.Context, _ mcp.CallToolRequest, a domClickRefArgs) (*mcp.CallToolResult, error) {
|
|
port := portOr(a.Port)
|
|
err := d.withConn(port, func(c *browser.CDPConn) error {
|
|
return browser.CdpClickRef(c, a.Ref, browser.MouseProfileForMode(a.Mode))
|
|
})
|
|
if err != nil {
|
|
return mcp.NewToolResultError(err.Error()), nil
|
|
}
|
|
time.Sleep(settleDelay)
|
|
outline, _ := d.perceiveOutline(port, 8000)
|
|
return mcp.NewToolResultText("clicked ref " + fmt.Sprint(a.Ref) + "\n\n" + outline), nil
|
|
}
|
|
|
|
// ---- dom_type_ref (MUTA) — bucle percibir→actuar ----
|
|
|
|
type domTypeRefArgs struct {
|
|
Port int `json:"port"`
|
|
Ref int `json:"ref"`
|
|
Text string `json:"text"`
|
|
}
|
|
|
|
func domTypeRefTool() mcp.Tool {
|
|
return mcp.NewTool("dom_type_ref",
|
|
mcp.WithDescription("Enfoca el elemento por su #ref del outline de page_perceive (backendDOMNodeId estable) y escribe el texto. Devuelve el outline actualizado tras la acción (auto-observe)."),
|
|
mcp.WithNumber("port", mcp.Description("CDP port. Default 9333 (Chrome isolated del MCP); usa 9222 explícito solo para adjuntarte al navegador diario.")),
|
|
mcp.WithNumber("ref", mcp.Required(), mcp.Description("#ref del elemento (backendDOMNodeId) leído del outline de page_perceive.")),
|
|
mcp.WithString("text", mcp.Required(), mcp.Description("Texto a escribir en el elemento.")),
|
|
)
|
|
}
|
|
|
|
func (d *deps) handleDomTypeRef(_ context.Context, _ mcp.CallToolRequest, a domTypeRefArgs) (*mcp.CallToolResult, error) {
|
|
if a.Text == "" {
|
|
return mcp.NewToolResultError("text is required"), nil
|
|
}
|
|
port := portOr(a.Port)
|
|
// TODO: preset de humanización por sesión (human/fast/instant)
|
|
err := d.withConn(port, func(c *browser.CDPConn) error {
|
|
return browser.CdpTypeRef(c, a.Ref, a.Text)
|
|
})
|
|
if err != nil {
|
|
return mcp.NewToolResultError(err.Error()), nil
|
|
}
|
|
time.Sleep(settleDelay)
|
|
outline, _ := d.perceiveOutline(port, 8000)
|
|
return mcp.NewToolResultText("typed into ref " + fmt.Sprint(a.Ref) + "\n\n" + outline), nil
|
|
}
|
|
|
|
// ---- dom_hover_ref (MUTA) — bucle percibir→actuar ----
|
|
|
|
type domHoverRefArgs struct {
|
|
Port int `json:"port"`
|
|
Ref int `json:"ref"`
|
|
Mode string `json:"mode"`
|
|
}
|
|
|
|
func domHoverRefTool() mcp.Tool {
|
|
return mcp.NewTool("dom_hover_ref",
|
|
mcp.WithDescription("Hover sobre el elemento por su #ref del outline de page_perceive (backendDOMNodeId estable). Devuelve el outline actualizado tras la acción (auto-observe)."),
|
|
mcp.WithNumber("port", mcp.Description("CDP port. Default 9333 (Chrome isolated del MCP); usa 9222 explícito solo para adjuntarte al navegador diario.")),
|
|
mcp.WithNumber("ref", mcp.Required(), mcp.Description("#ref del elemento (backendDOMNodeId) leído del outline de page_perceive.")),
|
|
mcp.WithString("mode", mcp.Description("Velocidad: 'human' (default, Bézier+jitter), 'fast' (movimiento reducido), 'instant' (sin movimiento de ratón).")),
|
|
)
|
|
}
|
|
|
|
func (d *deps) handleDomHoverRef(_ context.Context, _ mcp.CallToolRequest, a domHoverRefArgs) (*mcp.CallToolResult, error) {
|
|
port := portOr(a.Port)
|
|
err := d.withConn(port, func(c *browser.CDPConn) error {
|
|
return browser.CdpHoverRef(c, a.Ref, browser.MouseProfileForMode(a.Mode))
|
|
})
|
|
if err != nil {
|
|
return mcp.NewToolResultError(err.Error()), nil
|
|
}
|
|
time.Sleep(settleDelay)
|
|
outline, _ := d.perceiveOutline(port, 8000)
|
|
return mcp.NewToolResultText("hovered ref " + fmt.Sprint(a.Ref) + "\n\n" + outline), nil
|
|
}
|
|
|
|
// ---- dom_click_xy (MUTA) — click humanizado por coordenadas absolutas ----
|
|
|
|
type domClickXYArgs struct {
|
|
Port int `json:"port"`
|
|
X float64 `json:"x"`
|
|
Y float64 `json:"y"`
|
|
Mode string `json:"mode"`
|
|
}
|
|
|
|
func domClickXYTool() mcp.Tool {
|
|
return mcp.NewTool("dom_click_xy",
|
|
mcp.WithDescription("Fallback de click por coordenadas absolutas (x, y) en CSS pixels del viewport, con movimiento de ratón humanizado por defecto. Pensado para usarse sobre lo que el agente VE en page_screenshot cuando el outline de page_perceive no basta (canvas, mapas, layouts visuales). Prefiere dom_click_ref cuando el elemento aparece en el outline. Devuelve el outline actualizado tras la acción (auto-observe)."),
|
|
mcp.WithNumber("port", mcp.Description("CDP port. Default 9333 (Chrome isolated del MCP); usa 9222 explícito solo para adjuntarte al navegador diario.")),
|
|
mcp.WithNumber("x", mcp.Required(), mcp.Description("Coordenada X absoluta en CSS pixels del viewport.")),
|
|
mcp.WithNumber("y", mcp.Required(), mcp.Description("Coordenada Y absoluta en CSS pixels del viewport.")),
|
|
mcp.WithString("mode", mcp.Description("Velocidad: 'human' (default, Bézier+jitter anti-bot), 'fast' (movimiento reducido, scraping masivo), 'instant' (sin movimiento de ratón).")),
|
|
)
|
|
}
|
|
|
|
func (d *deps) handleDomClickXY(_ context.Context, _ mcp.CallToolRequest, a domClickXYArgs) (*mcp.CallToolResult, error) {
|
|
port := portOr(a.Port)
|
|
err := d.withConn(port, func(c *browser.CDPConn) error {
|
|
return browser.CdpClickXYHuman(c, a.X, a.Y, browser.MouseProfileForMode(a.Mode))
|
|
})
|
|
if err != nil {
|
|
return mcp.NewToolResultError(err.Error()), nil
|
|
}
|
|
time.Sleep(settleDelay)
|
|
outline, _ := d.perceiveOutline(port, 8000)
|
|
return mcp.NewToolResultText(fmt.Sprintf("clicked at (%g, %g)\n\n%s", a.X, a.Y, outline)), nil
|
|
}
|
|
|
|
// ---- dom_click (MUTA) ----
|
|
|
|
type domClickArgs struct {
|
|
Port int `json:"port"`
|
|
Selector string `json:"selector"`
|
|
}
|
|
|
|
func domClickTool() mcp.Tool {
|
|
return mcp.NewTool("dom_click",
|
|
mcp.WithDescription("Click the element matching the CSS selector (synthetic CDP click)."),
|
|
mcp.WithNumber("port", mcp.Description("CDP port. Default 9333 (Chrome isolated del MCP); usa 9222 explícito solo para adjuntarte al navegador diario.")),
|
|
mcp.WithString("selector", mcp.Required(), mcp.Description("CSS selector of the element to click.")),
|
|
)
|
|
}
|
|
|
|
func (d *deps) handleDomClick(_ context.Context, _ mcp.CallToolRequest, a domClickArgs) (*mcp.CallToolResult, error) {
|
|
if a.Selector == "" {
|
|
return mcp.NewToolResultError("selector is required"), nil
|
|
}
|
|
err := d.withConn(portOr(a.Port), func(c *browser.CDPConn) error {
|
|
return browser.CdpClick(c, a.Selector)
|
|
})
|
|
if err != nil {
|
|
return mcp.NewToolResultError(err.Error()), nil
|
|
}
|
|
return mcp.NewToolResultText("clicked " + a.Selector), nil
|
|
}
|
|
|
|
// ---- dom_click_human (MUTA) ----
|
|
|
|
type domClickHumanArgs struct {
|
|
Port int `json:"port"`
|
|
Selector string `json:"selector"`
|
|
}
|
|
|
|
func domClickHumanTool() mcp.Tool {
|
|
return mcp.NewTool("dom_click_human",
|
|
mcp.WithDescription("Click the element matching the CSS selector with human-like mouse movement (Bézier path + jitter + press/release pause)."),
|
|
mcp.WithNumber("port", mcp.Description("CDP port. Default 9333 (Chrome isolated del MCP); usa 9222 explícito solo para adjuntarte al navegador diario.")),
|
|
mcp.WithString("selector", mcp.Required(), mcp.Description("CSS selector of the element to click.")),
|
|
)
|
|
}
|
|
|
|
func (d *deps) handleDomClickHuman(_ context.Context, _ mcp.CallToolRequest, a domClickHumanArgs) (*mcp.CallToolResult, error) {
|
|
if a.Selector == "" {
|
|
return mcp.NewToolResultError("selector is required"), nil
|
|
}
|
|
err := d.withConn(portOr(a.Port), func(c *browser.CDPConn) error {
|
|
return browser.CdpClickHuman(c, a.Selector, browser.MouseHumanOpts{})
|
|
})
|
|
if err != nil {
|
|
return mcp.NewToolResultError(err.Error()), nil
|
|
}
|
|
return mcp.NewToolResultText("clicked (human) " + a.Selector), nil
|
|
}
|
|
|
|
// ---- dom_click_text (MUTA) ----
|
|
|
|
type domClickTextArgs struct {
|
|
Port int `json:"port"`
|
|
Text string `json:"text"`
|
|
}
|
|
|
|
func domClickTextTool() mcp.Tool {
|
|
return mcp.NewTool("dom_click_text",
|
|
mcp.WithDescription("Find the first element whose visible text matches and click it."),
|
|
mcp.WithNumber("port", mcp.Description("CDP port. Default 9333 (Chrome isolated del MCP); usa 9222 explícito solo para adjuntarte al navegador diario.")),
|
|
mcp.WithString("text", mcp.Required(), mcp.Description("Visible text to match (substring).")),
|
|
)
|
|
}
|
|
|
|
func (d *deps) handleDomClickText(_ context.Context, _ mcp.CallToolRequest, a domClickTextArgs) (*mcp.CallToolResult, error) {
|
|
if a.Text == "" {
|
|
return mcp.NewToolResultError("text is required"), nil
|
|
}
|
|
err := d.withConn(portOr(a.Port), func(c *browser.CDPConn) error {
|
|
return browser.CdpClickText(c, a.Text, browser.FindByTextOpts{})
|
|
})
|
|
if err != nil {
|
|
return mcp.NewToolResultError(err.Error()), nil
|
|
}
|
|
return mcp.NewToolResultText("clicked text " + a.Text), nil
|
|
}
|
|
|
|
// ---- dom_type (MUTA) ----
|
|
|
|
type domTypeArgs struct {
|
|
Port int `json:"port"`
|
|
Text string `json:"text"`
|
|
}
|
|
|
|
func domTypeTool() mcp.Tool {
|
|
return mcp.NewTool("dom_type",
|
|
mcp.WithDescription("Type text into the currently focused element (dispatches key events char by char)."),
|
|
mcp.WithNumber("port", mcp.Description("CDP port. Default 9333 (Chrome isolated del MCP); usa 9222 explícito solo para adjuntarte al navegador diario.")),
|
|
mcp.WithString("text", mcp.Required(), mcp.Description("Text to type.")),
|
|
)
|
|
}
|
|
|
|
func (d *deps) handleDomType(_ context.Context, _ mcp.CallToolRequest, a domTypeArgs) (*mcp.CallToolResult, error) {
|
|
if a.Text == "" {
|
|
return mcp.NewToolResultError("text is required"), nil
|
|
}
|
|
err := d.withConn(portOr(a.Port), func(c *browser.CDPConn) error {
|
|
return browser.CdpTypeText(c, a.Text)
|
|
})
|
|
if err != nil {
|
|
return mcp.NewToolResultError(err.Error()), nil
|
|
}
|
|
return mcp.NewToolResultText("typed text"), nil
|
|
}
|
|
|
|
// ---- dom_find_by_text ----
|
|
|
|
type domFindByTextArgs struct {
|
|
Port int `json:"port"`
|
|
Text string `json:"text"`
|
|
}
|
|
|
|
func domFindByTextTool() mcp.Tool {
|
|
return mcp.NewTool("dom_find_by_text",
|
|
mcp.WithDescription("Find the first element whose visible text matches and return a unique CSS selector for it (empty string if none)."),
|
|
mcp.WithNumber("port", mcp.Description("CDP port. Default 9333 (Chrome isolated del MCP); usa 9222 explícito solo para adjuntarte al navegador diario.")),
|
|
mcp.WithString("text", mcp.Required(), mcp.Description("Visible text to match (substring).")),
|
|
)
|
|
}
|
|
|
|
func (d *deps) handleDomFindByText(_ context.Context, _ mcp.CallToolRequest, a domFindByTextArgs) (*mcp.CallToolResult, error) {
|
|
if a.Text == "" {
|
|
return mcp.NewToolResultError("text is required"), nil
|
|
}
|
|
var sel string
|
|
err := d.withConn(portOr(a.Port), func(c *browser.CDPConn) error {
|
|
var e error
|
|
sel, e = browser.CdpFindByText(c, a.Text, browser.FindByTextOpts{})
|
|
return e
|
|
})
|
|
if err != nil {
|
|
return mcp.NewToolResultError(err.Error()), nil
|
|
}
|
|
return mcp.NewToolResultText(sel), nil
|
|
}
|
|
|
|
// ---- dom_find_ref_by_text ----
|
|
|
|
type domFindRefByTextArgs struct {
|
|
Port int `json:"port"`
|
|
Text string `json:"text"`
|
|
}
|
|
|
|
func domFindRefByTextTool() mcp.Tool {
|
|
return mcp.NewTool("dom_find_ref_by_text",
|
|
mcp.WithDescription("Find the first element whose visible text matches and return its #ref (backendDOMNodeId) ready for dom_click_ref/dom_hover_ref — no fragile CSS selector. Also reports how many elements match (count>1 = ambiguous)."),
|
|
mcp.WithNumber("port", mcp.Description("CDP port. Default 9333 (Chrome isolated del MCP); usa 9222 explícito solo para adjuntarte al navegador diario.")),
|
|
mcp.WithString("text", mcp.Required(), mcp.Description("Visible text to match (substring).")),
|
|
)
|
|
}
|
|
|
|
func (d *deps) handleDomFindRefByText(_ context.Context, _ mcp.CallToolRequest, a domFindRefByTextArgs) (*mcp.CallToolResult, error) {
|
|
if a.Text == "" {
|
|
return mcp.NewToolResultError("text is required"), nil
|
|
}
|
|
var ref, count int
|
|
err := d.withConn(portOr(a.Port), func(c *browser.CDPConn) error {
|
|
var e error
|
|
ref, count, e = browser.CdpFindRefByText(c, a.Text, browser.FindByTextOpts{})
|
|
return e
|
|
})
|
|
if err != nil {
|
|
return mcp.NewToolResultError(err.Error()), nil
|
|
}
|
|
msg := fmt.Sprintf("ref=%d count=%d", ref, count)
|
|
if count > 1 {
|
|
msg += " (ambiguous: returning the first match; refine the text to disambiguate)"
|
|
}
|
|
return mcp.NewToolResultText(msg), nil
|
|
}
|
|
|
|
// ---- dom_wait_element ----
|
|
|
|
type domWaitElementArgs struct {
|
|
Port int `json:"port"`
|
|
Selector string `json:"selector"`
|
|
TimeoutMs int `json:"timeout_ms"`
|
|
}
|
|
|
|
func domWaitElementTool() mcp.Tool {
|
|
return mcp.NewTool("dom_wait_element",
|
|
mcp.WithDescription("Block until an element matching the CSS selector appears in the DOM (or timeout)."),
|
|
mcp.WithNumber("port", mcp.Description("CDP port. Default 9333 (Chrome isolated del MCP); usa 9222 explícito solo para adjuntarte al navegador diario.")),
|
|
mcp.WithString("selector", mcp.Required(), mcp.Description("CSS selector to wait for.")),
|
|
mcp.WithNumber("timeout_ms", mcp.Description("Max wait in ms. Default 10000.")),
|
|
)
|
|
}
|
|
|
|
func (d *deps) handleDomWaitElement(_ context.Context, _ mcp.CallToolRequest, a domWaitElementArgs) (*mcp.CallToolResult, error) {
|
|
if a.Selector == "" {
|
|
return mcp.NewToolResultError("selector is required"), nil
|
|
}
|
|
timeout := a.TimeoutMs
|
|
if timeout <= 0 {
|
|
timeout = 10000
|
|
}
|
|
err := d.withConn(portOr(a.Port), func(c *browser.CDPConn) error {
|
|
return browser.CdpWaitElement(c, a.Selector, time.Duration(timeout)*time.Millisecond)
|
|
})
|
|
if err != nil {
|
|
return mcp.NewToolResultError(err.Error()), nil
|
|
}
|
|
return mcp.NewToolResultText("element appeared: " + a.Selector), nil
|
|
}
|