How to Build an AI Browser Agent That Navigates Websites and Completes Tasks

ai-browser-agent/
├── app/
│   ├── __init__.py
│   ├── dom_indexer.py
│   ├── schemas.py
│   └── navigator.py
├── tests/
│   └── test_browser_agent.py
└── requirements.txt

from typing import List, Dict
from bs4 import BeautifulSoup
from pydantic import BaseModel

INTERACTIVE_TAGS = {"a", "button", "input", "select", "textarea"}

class IndexedElement(BaseModel):
    index: int
    tag: str
    element_type: str
    text_or_label: str
    selector: str
    is_sensitive_submit: bool = False

SENSITIVE_KEYWORDS = ("pay", "purchase", "buy now", "checkout", "delete", "transfer", "confirm order", "wire")

def index_interactive_dom_from_html(html: str) -> List[IndexedElement]:
    """Parses HTML and assigns sequential 1-based indices to interactive elements."""
    soup = BeautifulSoup(html, "html.parser")
    elements: List[IndexedElement] = []
    idx = 1

    for tag in soup.find_all(lambda t: t.name in INTERACTIVE_TAGS or t.get("role") == "button"):
        if tag.get("type") == "hidden" or "display:none" in str(tag.get("style", "")).replace(" ", "").lower():
            continue

        el_type = str(tag.get("type", tag.name))
        label = (
            tag.get_text(strip=True)
            or str(tag.get("placeholder", ""))
            or str(tag.get("aria-label", ""))
            or str(tag.get("name", ""))
            or str(tag.get("value", ""))
        )[:80]

        el_id = tag.get("id")
        el_name = tag.get("name")
        if el_id:
            selector = f"#{el_id}"
        elif el_name:
            selector = f"{tag.name}[name='{el_name}']"
        else:
            selector = f"{tag.name}:nth-of-type({idx})"

        combined_text = f"{label} {el_id or ''} {el_name or ''}".lower()
        is_sensitive = any(k in combined_text for k in SENSITIVE_KEYWORDS)

        elements.append(
            IndexedElement(
                index=idx,
                tag=tag.name,
                element_type=el_type,
                text_or_label=label,
                selector=selector,
                is_sensitive_submit=is_sensitive,
            )
        )
        idx += 1

    return elements

def format_indexed_dom_for_llm(elements: List[IndexedElement]) -> str:
    """Formats the interactive element map into a compact token representation for the LLM."""
    lines = []
    for el in elements:
        flag = " [SENSITIVE_ACTION_REQUIRES_APPROVAL]" if el.is_sensitive_submit else ""
        lines.append(f"[{el.index}] <{el.tag} type='{el.element_type}'> \"{el.text_or_label}\"{flag}")
    return "\n".join(lines)

from enum import Enum
from typing import Optional, List
from pydantic import BaseModel, Field
from app.dom_indexer import IndexedElement

class ActionType(str, Enum):
    CLICK = "CLICK"
    FILL_TEXT = "FILL_TEXT"
    GOTO_URL = "GOTO_URL"
    ASK_HUMAN_APPROVAL = "ASK_HUMAN_APPROVAL"
    COMPLETE = "COMPLETE"

class BrowserStepAction(BaseModel):
    action_type: ActionType
    target_element_index: Optional[int] = Field(None, ge=1)
    text_input: Optional[str] = None
    reasoning: str

def verify_action_safety(
    action: BrowserStepAction,
    indexed_elements: List[IndexedElement],
    human_approved: bool = False,
) -> BrowserStepAction:
    """Intercepts clicks on sensitive checkout/delete/payment buttons if human_approved is False."""
    if action.action_type == ActionType.CLICK and action.target_element_index is not None:
        matched = next((e for e in indexed_elements if e.index == action.target_element_index), None)
        if matched is None:
            raise ValueError(f"Element index [{action.target_element_index}] does not exist on page.")
        if matched.is_sensitive_submit and not human_approved:
            return BrowserStepAction(
                action_type=ActionType.ASK_HUMAN_APPROVAL,
                target_element_index=matched.index,
                reasoning=f"Intercepted sensitive action on [{matched.index}] '{matched.text_or_label}'. Waiting for human confirmation.",
            )
    return action

from app.dom_indexer import index_interactive_dom_from_html, format_indexed_dom_for_llm
from app.schemas import BrowserStepAction, ActionType, verify_action_safety

SAMPLE_CHECKOUT_HTML = """
<html>
  <body>
    <input id="search-box" type="text" placeholder="Search products" />
    <input id="csrf-token" type="hidden" value="secret123" />
    <button id="filter-btn">Apply Filter</button>
    <button id="pay-btn">Pay Now $499</button>
  </body>
</html>
"""

def test_dom_indexer_skips_hidden_inputs_and_flags_sensitive_buttons():
    elements = index_interactive_dom_from_html(SAMPLE_CHECKOUT_HTML)
    assert len(elements) == 3  # hidden input skipped
    assert elements[0].selector == "#search-box"
    assert elements[1].is_sensitive_submit is False
    assert elements[2].text_or_label == "Pay Now $499"
    assert elements[2].is_sensitive_submit is True

    prompt_view = format_indexed_dom_for_llm(elements)
    assert "[3] <button type='button'> \"Pay Now $499\" [SENSITIVE_ACTION_REQUIRES_APPROVAL]" in prompt_view

def test_safety_gate_intercepts_unapproved_payment_click():
    elements = index_interactive_dom_from_html(SAMPLE_CHECKOUT_HTML)
    proposed_click = BrowserStepAction(
        action_type=ActionType.CLICK,
        target_element_index=3,
        reasoning="Click the Pay Now button to complete purchase.",
    )
    gated = verify_action_safety(proposed_click, elements, human_approved=False)
    assert gated.action_type == ActionType.ASK_HUMAN_APPROVAL

    approved = verify_action_safety(proposed_click, elements, human_approved=True)
    assert approved.action_type == ActionType.CLICK