russell@unturf.com
a5e709a470
Lifts the async web fetcher into aborist as an opt-in source. The implementation comes directly from ~/git/agents.ai.unturf.com/core (rev 2026-04-28); aborist's adaptations are minimal and documented in aborist/sources/crawler/__init__.py: core/async_web_fetcher.py -> aborist/sources/crawler/async_web_fetcher.py core/web_fetch.py -> aborist/sources/crawler/web_fetch.py Two source-side changes during the lift: 1. Heavy deps (aiohttp, bs4, miniuri) wrapped in try/except so a bare `import aborist.sources.crawler` raises ImportError with the install hint instead of leaking AttributeErrors deep in user code. 2. Chat-bot fetch triggers (`has_fresh_fetch_trigger`, `has_web_fetch_trigger` from agents.ai.unturf.com/core/keywords) replaced with NotImplementedError stubs. Aborist has no chat surface — fetch intent is detected at the application layer. The two test classes that exercised these triggers are `@pytest.mark.skip`'d with the same rationale. Not lifted: web_cache_manager.py — it backs page caching with SQLAlchemy. Aborist has its own content-addressed cache via providence_cache; no need to carry SQLAlchemy as a dep just for crawled-page memoization. Off by default: - `[crawler]` extras section in pyproject.toml carries the heavy deps. `[dev]` pulls them in so the crawler tests can run. - `make test` ignores tests/crawler/ entirely. - `make bootstrap-crawler` installs the extras into the venv. - `make test-crawler` runs only the lifted tests after extras land. Tests: 74 passed, 9 skipped (the chat-bot trigger tests deliberately dropped). Default `make test` stays at 273 passed, 1 skipped.
343 lines
13 KiB
Python
343 lines
13 KiB
Python
"""
|
|
Tests for core.web_fetch module.
|
|
|
|
Tests URL extraction, sources footer building, and web fetch helpers.
|
|
"""
|
|
|
|
import pytest
|
|
from unittest.mock import Mock, AsyncMock, patch
|
|
from aborist.sources.crawler.web_fetch import (
|
|
extract_url_from_content,
|
|
is_web_fetch_request,
|
|
build_sources_footer,
|
|
detect_fresh_fetch_intent,
|
|
extract_domain,
|
|
format_progress_message,
|
|
format_answering_message,
|
|
generate_keyword_stems,
|
|
)
|
|
|
|
|
|
class TestURLExtraction:
|
|
"""Test URL extraction from message content."""
|
|
|
|
def test_extract_url_with_protocol(self):
|
|
"""Test extracting URL with http:// or https://"""
|
|
content = "Check out https://example.com for more info"
|
|
url = extract_url_from_content(content)
|
|
assert url == "https://example.com"
|
|
|
|
def test_extract_url_http_protocol(self):
|
|
"""Test extracting URL with http:// protocol"""
|
|
content = "Visit http://insecure.example.com"
|
|
url = extract_url_from_content(content)
|
|
assert url == "http://insecure.example.com"
|
|
|
|
def test_extract_url_without_protocol(self):
|
|
"""Test extracting URL without protocol (adds https://)"""
|
|
content = "Go to example.com"
|
|
url = extract_url_from_content(content)
|
|
assert url == "https://example.com"
|
|
|
|
def test_extract_subdomain_url(self):
|
|
"""Test extracting URL with subdomain"""
|
|
content = "Check media.unturf.com/test"
|
|
url = extract_url_from_content(content)
|
|
assert url == "https://media.unturf.com/test"
|
|
|
|
def test_extract_url_with_path(self):
|
|
"""Test extracting URL with path"""
|
|
content = "Read https://docs.example.com/api/v1/reference"
|
|
url = extract_url_from_content(content)
|
|
assert url == "https://docs.example.com/api/v1/reference"
|
|
|
|
def test_extract_url_with_query(self):
|
|
"""Test extracting URL with query parameters"""
|
|
content = "See https://example.com/search?q=test&lang=en"
|
|
url = extract_url_from_content(content)
|
|
assert url == "https://example.com/search?q=test&lang=en"
|
|
|
|
def test_no_url_in_content(self):
|
|
"""Test when no URL is present"""
|
|
content = "This is just a regular message without any URLs"
|
|
url = extract_url_from_content(content)
|
|
assert url is None
|
|
|
|
def test_extract_first_url_when_multiple(self):
|
|
"""Test that first URL is extracted when multiple are present"""
|
|
content = "Check https://first.com and https://second.com"
|
|
url = extract_url_from_content(content)
|
|
assert url == "https://first.com"
|
|
|
|
def test_extract_url_with_www(self):
|
|
"""Test extracting URL with www prefix"""
|
|
content = "Visit www.example.com"
|
|
url = extract_url_from_content(content)
|
|
assert url == "https://www.example.com"
|
|
|
|
def test_extract_url_ignores_discord_embeds(self):
|
|
"""Test that angle brackets don't break URL extraction"""
|
|
content = "Check out <https://example.com>"
|
|
url = extract_url_from_content(content)
|
|
# Angle brackets should be part of the URL match
|
|
assert "example.com" in url
|
|
|
|
|
|
@pytest.mark.skip(
|
|
reason="chat-bot fetch triggers (has_web_fetch_trigger) were "
|
|
"dropped during the aborist lift — aborist has no chat surface; "
|
|
"see aborist/sources/crawler/__init__.py for rationale."
|
|
)
|
|
class TestWebFetchDetection:
|
|
"""Test web fetch request detection."""
|
|
|
|
def test_is_web_fetch_request_with_url(self):
|
|
"""Test detection when URL is present"""
|
|
assert is_web_fetch_request("Fetch https://example.com") is True
|
|
|
|
def test_is_web_fetch_request_without_url(self):
|
|
"""Test detection when no URL is present"""
|
|
assert is_web_fetch_request("Tell me about Python") is False
|
|
|
|
def test_is_web_fetch_request_domain_only(self):
|
|
"""Test detection with domain-only URL"""
|
|
assert is_web_fetch_request("What's on example.com?") is True
|
|
|
|
|
|
@pytest.mark.skip(
|
|
reason="chat-bot fresh-fetch triggers (has_fresh_fetch_trigger) were "
|
|
"dropped during the aborist lift — aborist has no chat surface; "
|
|
"see aborist/sources/crawler/__init__.py for rationale."
|
|
)
|
|
class TestFreshFetchDetection:
|
|
"""Test fresh fetch intent detection."""
|
|
|
|
def test_detect_fresh_with_keyword(self):
|
|
"""Test detection with 'fresh' keyword"""
|
|
assert detect_fresh_fetch_intent("Get fresh data from example.com") is True
|
|
|
|
def test_detect_refresh_with_keyword(self):
|
|
"""Test detection with 'refresh' keyword"""
|
|
assert detect_fresh_fetch_intent("Refresh the content") is True
|
|
|
|
def test_detect_reload_with_keyword(self):
|
|
"""Test detection with 'reload' keyword"""
|
|
assert detect_fresh_fetch_intent("Reload example.com") is True
|
|
|
|
def test_detect_latest_with_keyword(self):
|
|
"""Test detection with 'latest' keyword"""
|
|
assert detect_fresh_fetch_intent("Get the latest from example.com") is True
|
|
|
|
def test_no_fresh_intent(self):
|
|
"""Test when no fresh fetch keywords present"""
|
|
assert detect_fresh_fetch_intent("Fetch example.com") is False
|
|
|
|
def test_fresh_detection_case_insensitive(self):
|
|
"""Test that fresh detection is case insensitive"""
|
|
assert detect_fresh_fetch_intent("FRESH data please") is True
|
|
|
|
|
|
class TestSourcesFooter:
|
|
"""Test sources footer building."""
|
|
|
|
def test_build_sources_footer_single_source(self):
|
|
"""Test building footer with single source"""
|
|
sources = [{'url': 'https://example.com', 'title': 'Example Page'}]
|
|
footer = build_sources_footer(sources)
|
|
assert 'Sources:' in footer
|
|
assert 'Example Page' in footer
|
|
assert '<https://example.com>' in footer
|
|
|
|
def test_build_sources_footer_multiple_sources(self):
|
|
"""Test building footer with multiple sources"""
|
|
sources = [
|
|
{'url': 'https://example.com', 'title': 'Example Page'},
|
|
{'url': 'https://test.com', 'title': 'Test Page'}
|
|
]
|
|
footer = build_sources_footer(sources)
|
|
assert 'Example Page' in footer
|
|
assert 'Test Page' in footer
|
|
assert footer.count('<') == 2 # Two URLs with angle brackets
|
|
|
|
def test_build_sources_footer_empty_list(self):
|
|
"""Test building footer with empty source list"""
|
|
footer = build_sources_footer([])
|
|
assert footer == ""
|
|
|
|
def test_build_sources_footer_missing_title(self):
|
|
"""Test building footer when title is missing"""
|
|
sources = [{'url': 'https://example.com'}]
|
|
footer = build_sources_footer(sources)
|
|
assert 'Untitled' in footer
|
|
assert '<https://example.com>' in footer
|
|
|
|
def test_build_sources_footer_missing_url(self):
|
|
"""Test building footer when URL is missing"""
|
|
sources = [{'title': 'Example Page'}]
|
|
footer = build_sources_footer(sources)
|
|
# Should not include entry if URL is missing
|
|
assert footer == '**Sources:**\n'
|
|
|
|
def test_build_sources_footer_angle_brackets_suppress_embeds(self):
|
|
"""Test that angle brackets are used to suppress Discord embeds"""
|
|
sources = [{'url': 'https://example.com', 'title': 'Test'}]
|
|
footer = build_sources_footer(sources)
|
|
# URLs should be in angle brackets
|
|
assert '<https://example.com>' in footer
|
|
assert 'https://example.com>' not in footer.replace('<https://example.com>', '')
|
|
|
|
|
|
class TestDomainExtraction:
|
|
"""Test domain extraction from URLs."""
|
|
|
|
def test_extract_domain_basic(self):
|
|
"""Test extracting domain from basic URL"""
|
|
domain = extract_domain("https://example.com/path")
|
|
assert domain == "example.com"
|
|
|
|
def test_extract_domain_with_subdomain(self):
|
|
"""Test extracting domain with subdomain"""
|
|
domain = extract_domain("https://api.example.com")
|
|
assert domain == "api.example.com"
|
|
|
|
def test_extract_domain_with_port(self):
|
|
"""Test extracting domain with port"""
|
|
domain = extract_domain("https://example.com:8080/path")
|
|
assert domain == "example.com:8080"
|
|
|
|
def test_extract_domain_invalid_url(self):
|
|
"""Test extracting domain from invalid URL"""
|
|
domain = extract_domain("not a url")
|
|
# Should return None or handle gracefully
|
|
assert domain is None or isinstance(domain, str)
|
|
|
|
|
|
class TestProgressFormatting:
|
|
"""Test progress message formatting."""
|
|
|
|
def test_format_progress_message_basic(self):
|
|
"""Test basic progress message formatting"""
|
|
msg = format_progress_message(
|
|
url="https://example.com",
|
|
task="Find pricing",
|
|
keywords=["pricing", "cost"],
|
|
depth=2
|
|
)
|
|
assert "example.com" in msg
|
|
assert "pricing" in msg
|
|
assert "cost" in msg
|
|
assert "**Depth:** 2" in msg
|
|
|
|
def test_format_progress_message_with_crawl_delay(self):
|
|
"""Test progress message with crawl delay"""
|
|
msg = format_progress_message(
|
|
url="https://example.com",
|
|
crawl_delay=5.0
|
|
)
|
|
assert "**Crawl delay:** 5.0s" in msg
|
|
|
|
def test_format_progress_message_no_crawl_delay_if_default(self):
|
|
"""Test that default crawl delay is not shown"""
|
|
msg = format_progress_message(
|
|
url="https://example.com",
|
|
crawl_delay=2.0
|
|
)
|
|
assert "Crawl delay" not in msg
|
|
|
|
def test_format_progress_message_fresh_mode(self):
|
|
"""Test progress message with fresh mode"""
|
|
msg = format_progress_message(
|
|
url="https://example.com",
|
|
cache_mode="fresh"
|
|
)
|
|
assert "**Mode:** fresh" in msg
|
|
|
|
def test_format_answering_message(self):
|
|
"""Test answering phase message formatting"""
|
|
msg = format_answering_message(5)
|
|
assert "5 page(s)" in msg
|
|
assert "Analyzing" in msg
|
|
|
|
|
|
class TestKeywordStemming:
|
|
"""Test keyword stemming functionality."""
|
|
|
|
def test_generate_stems_basic(self):
|
|
"""Test generating stems from keywords"""
|
|
stems = generate_keyword_stems(["running", "tests"], [])
|
|
assert "run" in stems or "test" in stems
|
|
|
|
def test_generate_stems_no_duplicates(self):
|
|
"""Test that stems don't duplicate keywords"""
|
|
stems = generate_keyword_stems(["run"], ["running"])
|
|
# "run" is already in keywords, so stem shouldn't be added
|
|
assert "run" not in stems
|
|
|
|
def test_generate_stems_variations(self):
|
|
"""Test stem generation from variations"""
|
|
stems = generate_keyword_stems([], ["pricing", "prices"])
|
|
# Should generate stems if they're different from variations
|
|
assert isinstance(stems, list)
|
|
|
|
def test_generate_stems_no_stemming_needed(self):
|
|
"""Test when no stemming is needed"""
|
|
stems = generate_keyword_stems(["api", "cli"], [])
|
|
# Short words shouldn't be stemmed
|
|
assert len(stems) == 0
|
|
|
|
def test_generate_stems_phrase(self):
|
|
"""Test stem generation from phrase"""
|
|
stems = generate_keyword_stems(["pull requests"], [])
|
|
# Should split phrase and stem individual words
|
|
assert isinstance(stems, list)
|
|
|
|
|
|
class TestWebFetchIntegration:
|
|
"""Test web fetch integration with mock web cache manager."""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_fetch_and_cache_success(self):
|
|
"""Test successful fetch and cache operation"""
|
|
# This would require mocking web_cache_manager
|
|
# For now, just verify the function exists and has correct signature
|
|
from aborist.sources.crawler.web_fetch import fetch_and_cache
|
|
import inspect
|
|
sig = inspect.signature(fetch_and_cache)
|
|
assert 'url' in sig.parameters
|
|
assert 'web_cache_manager' in sig.parameters
|
|
assert 'keywords' in sig.parameters
|
|
assert 'depth' in sig.parameters
|
|
assert 'fresh' in sig.parameters
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_fetch_and_cache_with_progress_callback(self):
|
|
"""Test fetch with progress callback"""
|
|
from aborist.sources.crawler.web_fetch import fetch_and_cache
|
|
|
|
# Mock web cache manager
|
|
mock_manager = Mock()
|
|
mock_manager.get_or_fetch = AsyncMock(return_value={
|
|
'url': 'https://example.com',
|
|
'title': 'Test',
|
|
'extracted_text': 'Content',
|
|
'source_urls': []
|
|
})
|
|
|
|
# Mock progress callback
|
|
progress_calls = []
|
|
async def progress_callback(msg):
|
|
progress_calls.append(msg)
|
|
|
|
result = await fetch_and_cache(
|
|
url='https://example.com',
|
|
web_cache_manager=mock_manager,
|
|
progress_callback=progress_callback
|
|
)
|
|
|
|
assert result is not None
|
|
assert result['url'] == 'https://example.com'
|
|
mock_manager.get_or_fetch.assert_called_once()
|
|
|
|
|
|
if __name__ == '__main__':
|
|
pytest.main([__file__, '-v'])
|