7 AutoGPT Web Browsing Capabilities (Selenium, Playwright)

# Selenium
pip install selenium webdriver-manager

# Playwright
pip install playwright
playwright install chromium  # Downloads browser binaries

# Common dependencies
pip install beautifulsoup4 openai pillow

from abc import ABC, abstractmethod
from typing import Optional
import time

class BaseBrowserAgent(ABC):
    """Abstract base for browser-based agent tools."""
    
    @abstractmethod
    def get(self, url: str) -> str:
        """Navigate to URL and return page content."""
        pass
    
    @abstractmethod
    def click(self, selector: str) -> bool:
        """Click element matching selector."""
        pass
    
    @abstractmethod
    def fill(self, selector: str, value: str) -> bool:
        """Fill input field."""
        pass
    
    @abstractmethod
    def screenshot(self, save_path: str) -> str:
        """Take screenshot and return path."""
        pass
    
    @abstractmethod
    def close(self):
        """Clean up browser resources."""
        pass

from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.chrome.service import Service
from bs4 import BeautifulSoup

class SeleniumBrowser(BaseBrowserAgent):
    def __init__(self, headless: bool = True):
        options = Options()
        if headless:
            options.add_argument("--headless=new")
        options.add_argument("--no-sandbox")
        options.add_argument("--disable-dev-shm-usage")
        options.add_argument("--window-size=1920,1080")
        # Reduce automation fingerprint
        options.add_experimental_option("excludeSwitches", ["enable-automation"])
        options.add_experimental_option("useAutomationExtension", False)
        
        service = Service(ChromeDriverManager().install())
        self.driver = webdriver.Chrome(service=service, options=options)
        self.wait = WebDriverWait(self.driver, timeout=15)
    
    def get(self, url: str) -> str:
        self.driver.get(url)
        # Wait for page to be interactive
        self.wait.until(
            EC.presence_of_element_located((By.TAG_NAME, "body"))
        )
        time.sleep(1)  # Brief pause for JS to settle
        
        # Extract readable text
        soup = BeautifulSoup(self.driver.page_source, "html.parser")
        
        # Remove noise elements
        for tag in soup(["script", "style", "nav", "footer", "aside"]):
            tag.decompose()
        
        text = soup.get_text(separator=" ", strip=True)
        # Collapse whitespace
        import re
        text = re.sub(r'\s+', ' ', text).strip()
        return text[:5000]  # Limit for LLM context
    
    def get_structured(self, url: str) -> dict:
        """Extract both text and structure."""
        self.driver.get(url)
        soup = BeautifulSoup(self.driver.page_source, "html.parser")
        
        return {
            "title": soup.title.string if soup.title else "",
            "headings": [h.get_text() for h in soup.find_all(["h1", "h2", "h3"])],
            "paragraphs": [p.get_text() for p in soup.find_all("p")][:10],
            "links": [(a.get_text(), a.get("href")) for a in soup.find_all("a", href=True)][:20],
            "url": url
        }
    
    def click(self, selector: str) -> bool:
        try:
            element = self.wait.until(
                EC.element_to_be_clickable((By.CSS_SELECTOR, selector))
            )
            element.click()
            return True
        except Exception as e:
            print(f"Click failed: {e}")
            return False
    
    def fill(self, selector: str, value: str) -> bool:
        try:
            element = self.wait.until(
                EC.visibility_of_element_located((By.CSS_SELECTOR, selector))
            )
            element.clear()
            element.send_keys(value)
            return True
        except Exception as e:
            print(f"Fill failed: {e}")
            return False
    
    def screenshot(self, save_path: str = "screenshot.png") -> str:
        self.driver.save_screenshot(save_path)
        return save_path
    
    def execute_js(self, script: str):
        return self.driver.execute_script(script)
    
    def close(self):
        self.driver.quit()

from playwright.sync_api import sync_playwright, Page
from bs4 import BeautifulSoup
import re

class PlaywrightBrowser(BaseBrowserAgent):
    def __init__(self, headless: bool = True):
        self._playwright = sync_playwright().start()
        self.browser = self._playwright.chromium.launch(
            headless=headless,
            args=["--no-sandbox", "--disable-dev-shm-usage"]
        )
        self.context = self.browser.new_context(
            viewport={"width": 1920, "height": 1080},
            user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
        )
        self.page: Page = self.context.new_page()
    
    def get(self, url: str) -> str:
        # Playwright's wait_until is cleaner than Selenium's explicit waits
        self.page.goto(url, wait_until="domcontentloaded", timeout=30000)
        self.page.wait_for_load_state("networkidle", timeout=10000)
        
        content = self.page.content()
        soup = BeautifulSoup(content, "html.parser")
        
        for tag in soup(["script", "style", "nav", "footer"]):
            tag.decompose()
        
        text = soup.get_text(separator=" ", strip=True)
        return re.sub(r'\s+', ' ', text).strip()[:5000]
    
    def click(self, selector: str) -> bool:
        try:
            # Playwright auto-waits for element to be actionable
            self.page.click(selector, timeout=10000)
            return True
        except Exception as e:
            print(f"Click failed: {e}")
            return False
    
    def fill(self, selector: str, value: str) -> bool:
        try:
            self.page.fill(selector, value)
            return True
        except Exception as e:
            print(f"Fill failed: {e}")
            return False
    
    def screenshot(self, save_path: str = "screenshot.png") -> str:
        self.page.screenshot(path=save_path, full_page=True)
        return save_path
    
    def execute_js(self, script: str):
        return self.page.evaluate(script)
    
    def close(self):
        self.browser.close()
        self._playwright.stop()

def browse_with_interactions(browser: BaseBrowserAgent, url: str, actions: list) -> str:
    """Execute a sequence of interactions on a page."""
    results = []
    
    # Load initial page
    content = browser.get(url)
    results.append(f"Loaded: {url}")
    
    for action in actions:
        action_type = action.get("type")
        selector = action.get("selector", "")
        value = action.get("value", "")
        
        if action_type == "click":
            success = browser.click(selector)
            results.append(f"Clicked {selector}: {'OK' if success else 'FAILED'}")
            time.sleep(0.5)  # Allow page to respond
        
        elif action_type == "fill":
            success = browser.fill(selector, value)
            results.append(f"Filled {selector} with '{value}': {'OK' if success else 'FAILED'}")
        
        elif action_type == "read":
            # Re-read page after interactions
            if isinstance(browser, SeleniumBrowser):
                soup = BeautifulSoup(browser.driver.page_source, "html.parser")
                content = soup.get_text()[:2000]
            elif isinstance(browser, PlaywrightBrowser):
                soup = BeautifulSoup(browser.page.content(), "html.parser")
                content = soup.get_text()[:2000]
            results.append(f"Page content after actions: {content[:500]}")
    
    return "\n".join(results)

# Example: Search on a website
search_actions = [
    {"type": "click", "selector": "#search-button"},
    {"type": "fill", "selector": "#search-input", "value": "AI agents 2025"},
    {"type": "click", "selector": "button[type='submit']"},
    {"type": "read"}
]

def fill_search_form(browser: BaseBrowserAgent) -> list:
    """Demonstrate form filling capabilities."""
    
    # Navigate to a search engine
    browser.get("https://duckduckgo.com")
    
    # Fill search box and submit
    browser.fill("#searchbox_input", "AutoGPT agent capabilities 2025")
    browser.click("#searchbox_input ~ button[type='submit']")
    
    time.sleep(2)
    
    # Extract search results
    if isinstance(browser, SeleniumBrowser):
        soup = BeautifulSoup(browser.driver.page_source, "html.parser")
    else:
        soup = BeautifulSoup(browser.page.content(), "html.parser")
    
    results = []
    for result in soup.select(".result__title a")[:5]:
        results.append({
            "title": result.get_text(),
            "url": result.get("href", "")
        })
    
    return results

def extract_table_data(browser: BaseBrowserAgent, url: str) -> list:
    """Extract structured data from HTML tables."""
    browser.get(url)
    
    if isinstance(browser, SeleniumBrowser):
        source = browser.driver.page_source
    else:
        source = browser.page.content()
    
    soup = BeautifulSoup(source, "html.parser")
    tables = []
    
    for table in soup.find_all("table"):
        headers = [th.get_text(strip=True) for th in table.find_all("th")]
        rows = []
        
        for tr in table.find_all("tr"):
            cells = [td.get_text(strip=True) for td in tr.find_all("td")]
            if cells:
                if headers:
                    rows.append(dict(zip(headers, cells)))
                else:
                    rows.append(cells)
        
        if rows:
            tables.append({"headers": headers, "rows": rows[:20]})
    
    return tables

def scrape_paginated_results(
    browser: BaseBrowserAgent,
    start_url: str,
    next_selector: str,
    content_selector: str,
    max_pages: int = 5
) -> list:
    """Navigate through paginated content."""
    all_content = []
    current_url = start_url
    
    for page_num in range(max_pages):
        print(f"Scraping page {page_num + 1}: {current_url}")
        browser.get(current_url)
        
        if isinstance(browser, SeleniumBrowser):
            source = browser.driver.page_source
        else:
            source = browser.page.content()
        
        soup = BeautifulSoup(source, "html.parser")
        
        # Extract content
        content_elements = soup.select(content_selector)
        for elem in content_elements:
            all_content.append({
                "text": elem.get_text(strip=True),
                "page": page_num + 1,
                "url": current_url
            })
        
        # Find next page
        next_link = soup.select_one(next_selector)
        if not next_link or not next_link.get("href"):
            print("No next page found — stopping.")
            break
        
        next_href = next_link["href"]
        if next_href.startswith("http"):
            current_url = next_href
        else:
            from urllib.parse import urljoin
            current_url = urljoin(current_url, next_href)
        
        time.sleep(1)  # Respectful rate limiting
    
    return all_content

def visual_verification_agent(browser: BaseBrowserAgent, url: str) -> dict:
    """Take screenshot and use vision model to verify page state."""
    from openai import OpenAI
    import base64
    from pathlib import Path
    
    client = OpenAI()
    
    # Take screenshot
    screenshot_path = "verification_screenshot.png"
    browser.get(url)
    browser.screenshot(screenshot_path)
    
    # Encode for vision API
    with open(screenshot_path, "rb") as f:
        image_data = base64.b64encode(f.read()).decode("utf-8")
    
    # Ask vision model to verify
    response = client.chat.completions.create(
        model="gpt-4o",
        messages=[{
            "role": "user",
            "content": [
                {"type": "text", "text": "What is on this webpage? Is it loading correctly? Any error messages?"},
                {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{image_data}"}}
            ]
        }]
    )
    
    return {
        "screenshot": screenshot_path,
        "analysis": response.choices[0].message.content
    }

def execute_js_on_page(browser: BaseBrowserAgent, url: str) -> dict:
    """Use JavaScript for interactions that CSS selectors can't reach."""
    browser.get(url)
    
    # Get page metadata via JS
    metadata = browser.execute_js("""
        return {
            title: document.title,
            url: window.location.href,
            links: Array.from(document.querySelectorAll('a')).length,
            images: Array.from(document.querySelectorAll('img')).length,
            hasLogin: document.querySelector('input[type="password"]') !== null,
            scrollHeight: document.documentElement.scrollHeight
        }
    """)
    
    return metadata

class SafeAutonomousBrowser:
    """Browser wrapper with safety controls for autonomous agents."""
    
    BLOCKED_PATTERNS = [
        r"checkout", r"purchase", r"buy-now", r"confirm-order",
        r"delete", r"remove-account", r"unsubscribe",
        r"transfer", r"payment", r"billing"
    ]
    
    def __init__(self, browser: BaseBrowserAgent, allowed_domains: list = None):
        self.browser = browser
        self.allowed_domains = allowed_domains or []
        self.visited_urls = []
        self.blocked_count = 0
    
    def safe_get(self, url: str) -> str:
        """Navigate with domain and content checks."""
        import re
        from urllib.parse import urlparse
        
        # Domain restriction
        if self.allowed_domains:
            domain = urlparse(url).netloc
            if not any(allowed in domain for allowed in self.allowed_domains):
                return f"BLOCKED: Domain {domain} not in allowed list"
        
        # URL pattern check
        for pattern in self.BLOCKED_PATTERNS:
            if re.search(pattern, url, re.IGNORECASE):
                self.blocked_count += 1
                return f"BLOCKED: URL matches restricted pattern '{pattern}'"
        
        self.visited_urls.append(url)
        return self.browser.get(url)
    
    def safe_click(self, selector: str, page_content: str = "") -> bool:
        """Click with content safety check."""
        danger_phrases = ["buy", "purchase", "confirm", "delete", "submit payment"]
        
        if any(phrase in page_content.lower() for phrase in danger_phrases):
            print(f"WARNING: Skipping click on {selector} — dangerous page context detected")
            return False
        
        return self.browser.click(selector)
    
    def get_activity_report(self) -> dict:
        return {
            "urls_visited": len(self.visited_urls),
            "urls_blocked": self.blocked_count,
            "visited": self.visited_urls[-5:]  # Last 5
        }