engineering/arborist
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

crawler: verbatim lift from agents.ai.unturf.com (off by default)

Browse source 
Lifts the async web fetcher into aborist as an opt-in source. The
implementation comes directly from ~/git/agents.ai.unturf.com/core
(rev 2026-04-28); aborist's adaptations are minimal and documented in
aborist/sources/crawler/__init__.py:

  core/async_web_fetcher.py  ->  aborist/sources/crawler/async_web_fetcher.py
  core/web_fetch.py          ->  aborist/sources/crawler/web_fetch.py

Two source-side changes during the lift:

1. Heavy deps (aiohttp, bs4, miniuri) wrapped in try/except so a bare
   `import aborist.sources.crawler` raises ImportError with the install
   hint instead of leaking AttributeErrors deep in user code.

2. Chat-bot fetch triggers (`has_fresh_fetch_trigger`,
   `has_web_fetch_trigger` from agents.ai.unturf.com/core/keywords)
   replaced with NotImplementedError stubs. Aborist has no chat
   surface — fetch intent is detected at the application layer. The
   two test classes that exercised these triggers are
   `@pytest.mark.skip`'d with the same rationale.

Not lifted: web_cache_manager.py — it backs page caching with
SQLAlchemy. Aborist has its own content-addressed cache via
providence_cache; no need to carry SQLAlchemy as a dep just for
crawled-page memoization.

Off by default:

- `[crawler]` extras section in pyproject.toml carries the heavy
  deps. `[dev]` pulls them in so the crawler tests can run.
- `make test` ignores tests/crawler/ entirely.
- `make bootstrap-crawler` installs the extras into the venv.
- `make test-crawler` runs only the lifted tests after extras land.

Tests: 74 passed, 9 skipped (the chat-bot trigger tests deliberately
dropped). Default `make test` stays at 273 passed, 1 skipped.
This commit is contained in:
russell@unturf.com 2026-04-28 20:23:36 -04:00
parent c92ebad4ed
commit a5e709a470
No known key found for this signature in database
7 changed files with 4206 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

470
tests/crawler/test_async_web_fetcher.py Normal file
View file

@ -0,0 +1,470 @@
"""
Tests for async_web_fetcher module.
Tests URL handling, media extraction, robots.txt compliance,
and async fetching functionality.
"""
import pytest
from unittest.mock import Mock, AsyncMock, patch, MagicMock
import asyncio
import aiohttp
from aborist.sources.crawler.async_web_fetcher import (
CrawlMode,
MediaItem,
strip_uri_fragment,
normalize_link,
get_media_type_from_extension,
get_media_type_from_mime,
extract_media_from_html,
AsyncWebFetcher,
IMAGE_EXTENSIONS,
VIDEO_EXTENSIONS,
AUDIO_EXTENSIONS,
DEFAULT_CRAWL_DELAY,
get_last_fetch_error,
clear_last_fetch_error,
)
class TestCrawlMode:
"""Test CrawlMode enum."""
def test_crawl_mode_values(self):
"""Test all crawl mode values exist."""
assert CrawlMode.TEXT.value == "text"
assert CrawlMode.IMAGES.value == "images"
assert CrawlMode.VIDEOS.value == "videos"
assert CrawlMode.MEDIA.value == "media"
assert CrawlMode.ALL.value == "all"
def test_crawl_mode_from_string(self):
"""Test creating CrawlMode from string."""
assert CrawlMode("text") == CrawlMode.TEXT
assert CrawlMode("images") == CrawlMode.IMAGES
assert CrawlMode("media") == CrawlMode.MEDIA
class TestMediaItem:
"""Test MediaItem dataclass."""
def test_media_item_creation(self):
"""Test creating a MediaItem."""
item = MediaItem(
url="https://example.com/image.jpg",
source_page="https://example.com/",
media_type="image",
mime_type="image/jpeg",
)
assert item.url == "https://example.com/image.jpg"
assert item.source_page == "https://example.com/"
assert item.media_type == "image"
assert item.mime_type == "image/jpeg"
assert item.discovered_at is not None
def test_media_item_optional_fields(self):
"""Test MediaItem with optional fields."""
item = MediaItem(
url="https://example.com/video.mp4",
source_page="https://example.com/",
media_type="video",
alt_text="A video",
width=1920,
height=1080,
)
assert item.alt_text == "A video"
assert item.width == 1920
assert item.height == 1080
class TestURIHandling:
"""Test URI handling functions."""
def test_strip_uri_fragment_basic(self):
"""Test stripping fragment from URI."""
uri = "https://example.com/page#section"
result = strip_uri_fragment(uri)
assert result == "https://example.com/page"
def test_strip_uri_fragment_no_fragment(self):
"""Test URI without fragment."""
uri = "https://example.com/page"
result = strip_uri_fragment(uri)
assert result == "https://example.com/page"
def test_strip_uri_fragment_with_query(self):
"""Test URI with query and fragment."""
uri = "https://example.com/page?q=test#section"
result = strip_uri_fragment(uri)
assert result == "https://example.com/page?q=test"
def test_strip_uri_fragment_empty(self):
"""Test empty URI."""
assert strip_uri_fragment("") == ""
assert strip_uri_fragment(None) is None
def test_normalize_link_dict(self):
"""Test normalizing link from dict."""
link = {"url": "https://example.com/page#section", "anchor_text": "Click here"}
uri, anchor = normalize_link(link)
assert uri == "https://example.com/page"
assert anchor == "Click here"
def test_normalize_link_string(self):
"""Test normalizing link from string."""
uri, anchor = normalize_link("https://example.com/page#section")
assert uri == "https://example.com/page"
assert anchor == ""
class TestMediaTypeDetection:
"""Test media type detection functions."""
def test_get_media_type_from_extension_image(self):
"""Test detecting image from extension."""
assert get_media_type_from_extension("https://example.com/photo.jpg") == "image"
assert get_media_type_from_extension("https://example.com/photo.png") == "image"
assert get_media_type_from_extension("https://example.com/photo.webp") == "image"
assert get_media_type_from_extension("https://example.com/photo.gif") == "image"
def test_get_media_type_from_extension_video(self):
"""Test detecting video from extension."""
assert get_media_type_from_extension("https://example.com/video.mp4") == "video"
assert get_media_type_from_extension("https://example.com/video.webm") == "video"
assert get_media_type_from_extension("https://example.com/video.mov") == "video"
def test_get_media_type_from_extension_audio(self):
"""Test detecting audio from extension."""
assert get_media_type_from_extension("https://example.com/audio.mp3") == "audio"
assert get_media_type_from_extension("https://example.com/audio.wav") == "audio"
assert get_media_type_from_extension("https://example.com/audio.ogg") == "audio"
def test_get_media_type_from_extension_code(self):
"""Test detecting code from extension."""
assert get_media_type_from_extension("https://example.com/file.txt") == "code"
assert get_media_type_from_extension("https://example.com/script.py") == "code"
assert get_media_type_from_extension("https://example.com/app.js") == "code"
def test_get_media_type_from_extension_unknown(self):
"""Test unknown extension."""
assert get_media_type_from_extension("https://example.com/page.html") is None
assert get_media_type_from_extension("https://example.com/file.xyz") is None
def test_get_media_type_from_mime_image(self):
"""Test detecting image from MIME type."""
assert get_media_type_from_mime("image/jpeg") == "image"
assert get_media_type_from_mime("image/png") == "image"
assert get_media_type_from_mime("image/webp") == "image"
def test_get_media_type_from_mime_video(self):
"""Test detecting video from MIME type."""
assert get_media_type_from_mime("video/mp4") == "video"
assert get_media_type_from_mime("video/webm") == "video"
def test_get_media_type_from_mime_audio(self):
"""Test detecting audio from MIME type."""
assert get_media_type_from_mime("audio/mpeg") == "audio"
assert get_media_type_from_mime("audio/wav") == "audio"
def test_get_media_type_from_mime_code(self):
"""Test detecting code from MIME type."""
assert get_media_type_from_mime("application/json") == "code"
assert get_media_type_from_mime("text/plain") == "code"
assert get_media_type_from_mime("application/javascript") == "code"
def test_get_media_type_from_mime_unknown(self):
"""Test unknown MIME type."""
assert get_media_type_from_mime("text/html") is None
assert get_media_type_from_mime("application/octet-stream") is None
class TestMediaExtraction:
"""Test media extraction from HTML."""
def test_extract_images_from_html(self):
"""Test extracting images from HTML."""
html = '''
<html>
<body>
<img src="/images/photo.jpg" alt="A photo">
<img src="https://cdn.example.com/logo.png" alt="Logo">
</body>
</html>
'''
media = extract_media_from_html(html, "https://example.com/", CrawlMode.IMAGES)
assert len(media) >= 1
urls = [m['url'] for m in media]
# Should have resolved relative URL
assert any('photo.jpg' in url for url in urls)
def test_extract_videos_from_html(self):
"""Test extracting videos from HTML."""
html = '''
<html>
<body>
<video src="/videos/movie.mp4"></video>
<source src="/videos/clip.webm" type="video/webm">
</body>
</html>
'''
media = extract_media_from_html(html, "https://example.com/", CrawlMode.VIDEOS)
urls = [m['url'] for m in media]
assert any('movie.mp4' in url or 'clip.webm' in url for url in urls)
def test_extract_media_filters_by_mode(self):
"""Test that extraction respects crawl mode."""
html = '''
<html>
<body>
<img src="/photo.jpg">
<video src="/video.mp4"></video>
<audio src="/audio.mp3"></audio>
</body>
</html>
'''
# Images only
images = extract_media_from_html(html, "https://example.com/", CrawlMode.IMAGES)
image_urls = [m['url'] for m in images]
assert any('photo.jpg' in url for url in image_urls)
assert not any('video.mp4' in url for url in image_urls)
# Videos only
videos = extract_media_from_html(html, "https://example.com/", CrawlMode.VIDEOS)
video_urls = [m['url'] for m in videos]
assert any('video.mp4' in url for url in video_urls)
class TestAsyncWebFetcher:
"""Test AsyncWebFetcher class."""
def test_fetcher_initialization(self):
"""Test AsyncWebFetcher initialization."""
fetcher = AsyncWebFetcher(
user_agent="TestBot/1.0",
default_crawl_delay=1.0,
)
assert fetcher.user_agent == "TestBot/1.0"
assert fetcher.default_crawl_delay == 1.0
def test_fetcher_default_values(self):
"""Test AsyncWebFetcher default values."""
fetcher = AsyncWebFetcher()
assert fetcher.default_crawl_delay == DEFAULT_CRAWL_DELAY
def test_get_domain(self):
"""Test domain extraction."""
fetcher = AsyncWebFetcher()
assert fetcher._get_domain("https://example.com/page") == "example.com"
assert fetcher._get_domain("https://sub.example.com/page") == "sub.example.com"
def test_simple_stem(self):
"""Test simple word stemming."""
fetcher = AsyncWebFetcher()
# Test stemming - actual behavior may vary
stem = fetcher._simple_stem("running")
assert "run" in stem # Should contain "run"
# Short words should not be stemmed
assert fetcher._simple_stem("go") == "go"
def test_extract_text_content(self):
"""Test text extraction from HTML."""
fetcher = AsyncWebFetcher()
html = '''
<html>
<head><title>Test Page</title></head>
<body>
<h1>Hello World</h1>
<p>This is a test paragraph.</p>
<script>alert('ignored');</script>
<style>.hidden { display: none; }</style>
</body>
</html>
'''
text = fetcher.extract_text_content(html)
assert "Hello World" in text
assert "test paragraph" in text
assert "alert" not in text # Script content should be removed
assert ".hidden" not in text # Style content should be removed
def test_extract_title(self):
"""Test title extraction from HTML."""
fetcher = AsyncWebFetcher()
html = '<html><head><title>Page Title</title></head><body></body></html>'
title = fetcher.extract_title(html)
assert title == "Page Title"
def test_extract_title_missing(self):
"""Test title extraction when missing."""
fetcher = AsyncWebFetcher()
html = '<html><head></head><body></body></html>'
title = fetcher.extract_title(html)
# Returns "Untitled" when no title present
assert title == "Untitled" or title == ""
class TestAsyncWebFetcherScoring:
"""Test link and page scoring functionality."""
def test_score_link_keyword_match(self):
"""Test link scoring with keyword matches."""
fetcher = AsyncWebFetcher()
# Link with keyword in URL
score = fetcher._score_link(
"https://example.com/pricing",
"View Pricing",
query_keywords=["pricing", "cost"],
)
assert score > 0 # Should have positive score
# Link without keywords
score_none = fetcher._score_link(
"https://example.com/about",
"About Us",
query_keywords=["pricing", "cost"],
)
assert score > score_none # Keyword match should score higher
def test_score_link_anchor_text(self):
"""Test link scoring with anchor text."""
fetcher = AsyncWebFetcher()
# Anchor text with keyword
score = fetcher._score_link(
"https://example.com/page",
"Our Pricing Plans",
query_keywords=["pricing"],
)
assert score > 0
def test_score_page_keyword_match(self):
"""Test page scoring with keyword matches."""
fetcher = AsyncWebFetcher()
page_data = {
"url": "https://example.com/pricing",
"title": "Pricing Plans",
"extracted_text": "Our pricing is competitive. See our cost breakdown.",
}
score = fetcher._score_page(
page_data,
query_keywords=["pricing", "cost"],
)
assert score > 0
class TestAsyncWebFetcherAsync:
"""Test async methods of AsyncWebFetcher."""
@pytest.mark.asyncio
async def test_can_fetch_respects_robots(self):
"""Test that _can_fetch respects robots.txt."""
fetcher = AsyncWebFetcher()
# Mock the robots.txt fetch to return a parser that disallows
with patch.object(fetcher, '_fetch_robots_txt', new_callable=AsyncMock) as mock_robots:
mock_parser = MagicMock()
mock_parser.can_fetch.return_value = False
mock_robots.return_value = mock_parser
result = await fetcher._can_fetch("https://example.com/blocked")
assert result is False
@pytest.mark.asyncio
async def test_can_fetch_allows_when_parser_allows(self):
"""Test _can_fetch when robots parser allows."""
fetcher = AsyncWebFetcher()
with patch.object(fetcher, '_fetch_robots_txt', new_callable=AsyncMock) as mock_robots:
mock_parser = MagicMock()
mock_parser.can_fetch.return_value = True
mock_robots.return_value = mock_parser
result = await fetcher._can_fetch("https://example.com/allowed")
assert result is True
@pytest.mark.asyncio
async def test_enforce_crawl_delay(self):
"""Test crawl delay enforcement."""
fetcher = AsyncWebFetcher(default_crawl_delay=0.1)
domain = "example.com"
# First request should not delay
start = asyncio.get_event_loop().time()
await fetcher._enforce_crawl_delay(domain)
first_time = asyncio.get_event_loop().time() - start
# Second request should delay
start = asyncio.get_event_loop().time()
await fetcher._enforce_crawl_delay(domain)
second_time = asyncio.get_event_loop().time() - start
# Second call should have some delay (at least part of 0.1s)
assert second_time >= 0.05 # Allow some tolerance
@pytest.mark.asyncio
async def test_check_media_url_signature(self):
"""Test check_media_url function signature."""
fetcher = AsyncWebFetcher()
# Verify the method exists and has correct signature
import inspect
sig = inspect.signature(fetcher.check_media_url)
params = list(sig.parameters.keys())
# Should have url parameter
assert 'url' in params or len(params) >= 1
class TestErrorHandling:
"""Test error handling functionality."""
def test_get_last_fetch_error_initially_none(self):
"""Test that last fetch error is None initially."""
clear_last_fetch_error()
error = get_last_fetch_error()
assert error is None
def test_clear_last_fetch_error(self):
"""Test clearing the last fetch error."""
clear_last_fetch_error()
assert get_last_fetch_error() is None
class TestExtensionSets:
"""Test extension and MIME type sets."""
def test_image_extensions(self):
"""Test image extension set."""
assert '.jpg' in IMAGE_EXTENSIONS
assert '.jpeg' in IMAGE_EXTENSIONS
assert '.png' in IMAGE_EXTENSIONS
assert '.gif' in IMAGE_EXTENSIONS
assert '.webp' in IMAGE_EXTENSIONS
assert '.svg' in IMAGE_EXTENSIONS
def test_video_extensions(self):
"""Test video extension set."""
assert '.mp4' in VIDEO_EXTENSIONS
assert '.webm' in VIDEO_EXTENSIONS
assert '.mov' in VIDEO_EXTENSIONS
assert '.avi' in VIDEO_EXTENSIONS
def test_audio_extensions(self):
"""Test audio extension set."""
assert '.mp3' in AUDIO_EXTENSIONS
assert '.wav' in AUDIO_EXTENSIONS
assert '.ogg' in AUDIO_EXTENSIONS
assert '.flac' in AUDIO_EXTENSIONS
if __name__ == '__main__':
pytest.main([__file__, '-v'])