feat: Implement ASCII browser using w3m (stitionai#440)

- Add W3MBrowser class for lightweight web browsing
- Implement navigation and content extraction
- Add comprehensive test suite
- Add documentation for ASCII browser integration
- Update requirements.txt with test dependencies

Co-Authored-By: Erkin Alp Güney <[email protected]>

docs/browser/ASCII_BROWSER.md

@@ -0,0 +1,55 @@
+# ASCII Browser Integration
+
+This document describes the ASCII browser integration in Devika using w3m.
+
+## Overview
+
+The ASCII browser implementation provides a lightweight alternative to API-based web access and screenshot-based browsing. It uses w3m to render web pages in ASCII format, which is both efficient and suitable for LLM processing.
+
+## Requirements
+
+- w3m must be installed on the system (`apt-get install w3m`)
+- Python 3.6+ with subprocess module
+
+## Usage
+
+```python
+from src.browser.w3m_browser import W3MBrowser
+
+# Initialize browser
+browser = W3MBrowser()
+
+# Navigate to a URL
+success, content = browser.navigate("http://example.com")
+
+# Get current content
+current_content = browser.get_current_content()
+```
+
+## Benefits
+
+1. No API costs or rate limits
+2. No third-party dependencies for basic web access
+3. Efficient text-based content suitable for LLM processing
+4. Reduced bandwidth usage compared to full browser rendering
+
+## Limitations
+
+1. No JavaScript support
+2. Limited rendering of complex layouts
+3. No image support in ASCII mode
+
+## Error Handling
+
+The browser implementation includes robust error handling for:
+- Missing w3m installation
+- Navigation failures
+- Invalid URLs
+
+## Installation
+
+To install w3m on Ubuntu/Debian:
+```bash
+sudo apt-get update
+sudo apt-get install w3m
+```

requirements.txt

@@ -2,7 +2,7 @@ flask
 flask-cors
 toml
 urllib3
-requests
+requests>=2.31.0
 colorama
 fastlogging
 Jinja2
@@ -12,7 +12,7 @@ pdfminer.six
 playwright
 pytest-playwright
 tiktoken
-ollama
+ollama>=0.1.6
 openai
 anthropic
 google-generativeai
@@ -31,3 +31,5 @@ orjson
 gevent
 gevent-websocket
 curl_cffi
+pytest>=7.4.0
+pytest-mock>=3.12.0

src/browser/w3m_browser.py

@@ -0,0 +1,55 @@
+"""
+ASCII-based browser implementation using w3m.
+
+This module provides a lightweight terminal-based browser implementation
+using w3m for web access without requiring expensive API calls or screenshots.
+"""
+import subprocess
+import logging
+from typing import Optional, Tuple
+
+logger = logging.getLogger(__name__)
+
+class W3MBrowser:
+    """Terminal-based browser using w3m."""
+
+    def __init__(self):
+        """Initialize the W3M browser wrapper."""
+        self._verify_w3m_installation()
+        self.current_url: Optional[str] = None
+        self.current_content: Optional[str] = None
+
+    def _verify_w3m_installation(self) -> None:
+        """Verify w3m is installed and accessible."""
+        try:
+            subprocess.run(['w3m', '-version'],
+                         check=True,
+                         capture_output=True,
+                         text=True)
+        except (subprocess.CalledProcessError, FileNotFoundError) as e:
+            raise RuntimeError("w3m is not installed or not accessible") from e
+
+    def navigate(self, url: str) -> Tuple[bool, str]:
+        """Navigate to a URL and return the page content in ASCII format."""
+        self.current_url = url
+        try:
+            result = subprocess.run(
+                ['w3m', '-dump', url],
+                check=True,
+                capture_output=True,
+                text=True
+            )
+            self.current_content = result.stdout
+            return True, self.current_content
+        except subprocess.CalledProcessError as e:
+            error_msg = f"Failed to load URL {url}: {str(e)}"
+            logger.error(error_msg)
+            return False, error_msg
+
+    def get_current_content(self) -> Optional[str]:
+        """Get the current page content."""
+        return self.current_content
+
+    def get_current_url(self) -> Optional[str]:
+        """Get the current URL."""
+        return self.current_url

tests/test_w3m_browser.py

@@ -0,0 +1,54 @@
+"""Tests for the W3M browser implementation."""
+import pytest
+from unittest.mock import patch, MagicMock
+import subprocess
+from src.browser.w3m_browser import W3MBrowser
+
+def test_w3m_browser_init():
+    """Test W3MBrowser initialization."""
+    with patch('subprocess.run') as mock_run:
+        mock_run.return_value = MagicMock(returncode=0)
+        browser = W3MBrowser()
+        assert browser.get_current_url() is None
+        assert browser.get_current_content() is None
+        mock_run.assert_called_once()
+
+def test_w3m_browser_init_failure():
+    """Test W3MBrowser initialization failure."""
+    with patch('subprocess.run') as mock_run:
+        mock_run.side_effect = FileNotFoundError()
+        with pytest.raises(RuntimeError, match="w3m is not installed"):
+            W3MBrowser()
+
+def test_navigate_success():
+    """Test successful navigation."""
+    with patch('subprocess.run') as mock_run:
+        # Mock successful initialization
+        mock_run.return_value = MagicMock(returncode=0)
+        browser = W3MBrowser()
+
+        # Mock successful navigation
+        mock_run.return_value = MagicMock(
+            returncode=0,
+            stdout="Test Content"
+        )
+
+        success, content = browser.navigate("http://example.com")
+        assert success is True
+        assert content == "Test Content"
+        assert browser.get_current_url() == "http://example.com"
+        assert browser.get_current_content() == "Test Content"
+
+def test_navigate_failure():
+    """Test navigation failure."""
+    with patch('subprocess.run') as mock_run:
+        # Mock successful initialization
+        mock_run.return_value = MagicMock(returncode=0)
+        browser = W3MBrowser()
+
+        # Mock failed navigation
+        mock_run.side_effect = subprocess.CalledProcessError(1, 'w3m')
+
+        success, content = browser.navigate("http://invalid.example")
+        assert success is False
+        assert "Failed to load URL" in content