Building Agentic Web Browsing Tools: Integrating LLMs with Headless Browser Automation for Real-Time Data Grounding

{
  "url": "https://example.com/products",
  "visible_text": "Showing 1‑24 of 89 results… Load more",
  "elements": [
    {"tag":"button","id":"load-more","aria-label":"Load more results","innerText":"Load more"},
    {"tag":"span","class":"price","innerText":"$29.99"}
  ]
}

You are a web‑browsing agent. Given the current page observation and the user goal, return the next action as a JSON object.
Allowed actions:
- { "type": "click", "selector": "<css selector>" }
- { "type": "type", "selector": "<css selector>", "text": "<string>" }
- { "type": "extract", "selector": "<css selector>", "attribute": "innerText|href|value" }
- { "type": "wait", "milliseconds": <number> }
- { "type": "done" }
If unsure, return { "type": "wait", "milliseconds": 1000 }.

import alterlab
import json
import openai
import os
import time

# Initialize clients
browser = alterlab.Client(os.getenv("ALTERLAB_API_KEY"))
openai.api_key = os.getenv("OPENAI_API_KEY")

GOAL = "Extract the titles of all items on the page"
MAX_STEPS = 20

def observe(page):
    """Return a compact JSON observation for the LLM."""
    obs = page.evaluate("""() => {
        const inner = document.body.innerText.slice(0, 1500);
        const buttons = Array.from(document.querySelectorAll('button, [role="button"]'))
            .map(b => ({tag:b.tagName.toLowerCase(), id:b.id||'', 'aria-label':b.getAttribute('aria-label')||'', innerText:b.innerText.slice(0,50)}));
        const inputs = Array.from(document.querySelectorAll('input, textarea, select'))
            .map(i => ({tag:i.tagName.toLowerCase(), id:i.id||'', 'aria-label':i.getAttribute('aria-label')||'', innerText:i.value||''}));
        return {url: location.href, visible_text: inner, elements: [...buttons, ...inputs]};
    }""")
    return obs

def llm_decide(obs, goal):
    prompt = f"""You are a web‑browsing agent.
Goal: {goal}
Observation: {json.dumps(obs)}
Return ONLY a JSON action as described."""
    response = openai.chat.completions.create(
        model="gpt-4o",
        messages=[{"role":"system","content":"You output JSON actions only."},
                  {"role":"user","content":prompt}],
        temperature=0,
        max_tokens=200,
    )
    content = response.choices[0].message.content.strip()
    # Expect pure JSON; fallback to wait if malformed
    try:
        return json.loads(content)
    except json.JSONDecodeError:
        return {"type":"wait","milliseconds":1000}

def run():
    # Start a headless session with AlterLab (smart rendering enabled)
    session = browser.sessions.create({
        "url": "https://example.com/listings",
        "render": True,
        "wait_for": "networkidle"
    })
    page = session.page

    for step in range(MAX_STEPS):
        obs = observe(page)
        action = llm_decide(obs, GOAL)
        print(f"Step {step}: {action}")

        if action.get("type") == "click":
            selector = action["selector"]
            page.click(selector)
            page.wait_for_load_state("networkidle")
        elif action.get("type") == "type":
            selector = action["selector"]
            text = action["text"]
            page.fill(selector, text)
            page.keyboard.press("Enter")
            page.wait_for_load_state("networkidle")
        elif action.get("type") == "extract":
            selector = action["selector"]
            attr = action.get("attribute","innerText")
            value = page.evaluate(f"""() => {{
                const el = document.querySelector("{selector}");
                return el ? el.{attr} : null;
            }}""")
            print(f"Extracted: {value}")
        elif action.get("type") == "wait":
            time.sleep(action.get("milliseconds",1000)/1000)
        elif action.get("type") == "done":
            break
        else:
            # unknown action → wait
            time.sleep(1)

    browser.sessions.delete(session.id)

if __name__ == "__main__":
    run()

#!/usr/bin/env bash
API_KEY="${ALTERLAB_API_KEY}"
LLM_URL="http://localhost:11434/api/generate"  # Ollama endpoint
GOAL="Extract all product prices"
MAX_STEPS=15

# 1. Create session
SESSION_ID=$(curl -s -X POST "https://api.alterlab.io/v1/sessions" \
  -H "X-API-Key: $API_KEY" \
  -d '{"url":"https://example.com/