arborist/tests/crawler/test_web_fetch.py

"""
Tests for core.web_fetch module.

Tests URL extraction, sources footer building, and web fetch helpers.
"""

import pytest
from unittest.mock import Mock, AsyncMock, patch
from arborist.sources.crawler.web_fetch import (
    extract_url_from_content,
    is_web_fetch_request,
    build_sources_footer,
    detect_fresh_fetch_intent,
    extract_domain,
    format_progress_message,
    format_answering_message,
    generate_keyword_stems,
)


class TestURLExtraction:
    """Test URL extraction from message content."""

    def test_extract_url_with_protocol(self):
        """Test extracting URL with http:// or https://"""
        content = "Check out https://example.com for more info"
        url = extract_url_from_content(content)
        assert url == "https://example.com"

    def test_extract_url_http_protocol(self):
        """Test extracting URL with http:// protocol"""
        content = "Visit http://insecure.example.com"
        url = extract_url_from_content(content)
        assert url == "http://insecure.example.com"

    def test_extract_url_without_protocol(self):
        """Test extracting URL without protocol (adds https://)"""
        content = "Go to example.com"
        url = extract_url_from_content(content)
        assert url == "https://example.com"

    def test_extract_subdomain_url(self):
        """Test extracting URL with subdomain"""
        content = "Check media.unturf.com/test"
        url = extract_url_from_content(content)
        assert url == "https://media.unturf.com/test"

    def test_extract_url_with_path(self):
        """Test extracting URL with path"""
        content = "Read https://docs.example.com/api/v1/reference"
        url = extract_url_from_content(content)
        assert url == "https://docs.example.com/api/v1/reference"

    def test_extract_url_with_query(self):
        """Test extracting URL with query parameters"""
        content = "See https://example.com/search?q=test&lang=en"
        url = extract_url_from_content(content)
        assert url == "https://example.com/search?q=test&lang=en"

    def test_no_url_in_content(self):
        """Test when no URL is present"""
        content = "This is just a regular message without any URLs"
        url = extract_url_from_content(content)
        assert url is None

    def test_extract_first_url_when_multiple(self):
        """Test that first URL is extracted when multiple are present"""
        content = "Check https://first.com and https://second.com"
        url = extract_url_from_content(content)
        assert url == "https://first.com"

    def test_extract_url_with_www(self):
        """Test extracting URL with www prefix"""
        content = "Visit www.example.com"
        url = extract_url_from_content(content)
        assert url == "https://www.example.com"

    def test_extract_url_ignores_discord_embeds(self):
        """Test that angle brackets don't break URL extraction"""
        content = "Check out <https://example.com>"
        url = extract_url_from_content(content)
        # Angle brackets should be part of the URL match
        assert "example.com" in url


@pytest.mark.skip(
    reason="chat-bot fetch triggers (has_web_fetch_trigger) were "
    "dropped during the arborist lift — arborist has no chat surface; "
    "see arborist/sources/crawler/__init__.py for rationale."
)
class TestWebFetchDetection:
    """Test web fetch request detection."""

    def test_is_web_fetch_request_with_url(self):
        """Test detection when URL is present"""
        assert is_web_fetch_request("Fetch https://example.com") is True

    def test_is_web_fetch_request_without_url(self):
        """Test detection when no URL is present"""
        assert is_web_fetch_request("Tell me about Python") is False

    def test_is_web_fetch_request_domain_only(self):
        """Test detection with domain-only URL"""
        assert is_web_fetch_request("What's on example.com?") is True


@pytest.mark.skip(
    reason="chat-bot fresh-fetch triggers (has_fresh_fetch_trigger) were "
    "dropped during the arborist lift — arborist has no chat surface; "
    "see arborist/sources/crawler/__init__.py for rationale."
)
class TestFreshFetchDetection:
    """Test fresh fetch intent detection."""

    def test_detect_fresh_with_keyword(self):
        """Test detection with 'fresh' keyword"""
        assert detect_fresh_fetch_intent("Get fresh data from example.com") is True

    def test_detect_refresh_with_keyword(self):
        """Test detection with 'refresh' keyword"""
        assert detect_fresh_fetch_intent("Refresh the content") is True

    def test_detect_reload_with_keyword(self):
        """Test detection with 'reload' keyword"""
        assert detect_fresh_fetch_intent("Reload example.com") is True

    def test_detect_latest_with_keyword(self):
        """Test detection with 'latest' keyword"""
        assert detect_fresh_fetch_intent("Get the latest from example.com") is True

    def test_no_fresh_intent(self):
        """Test when no fresh fetch keywords present"""
        assert detect_fresh_fetch_intent("Fetch example.com") is False

    def test_fresh_detection_case_insensitive(self):
        """Test that fresh detection is case insensitive"""
        assert detect_fresh_fetch_intent("FRESH data please") is True


class TestSourcesFooter:
    """Test sources footer building."""

    def test_build_sources_footer_single_source(self):
        """Test building footer with single source"""
        sources = [{'url': 'https://example.com', 'title': 'Example Page'}]
        footer = build_sources_footer(sources)
        assert 'Sources:' in footer
        assert 'Example Page' in footer
        assert '<https://example.com>' in footer

    def test_build_sources_footer_multiple_sources(self):
        """Test building footer with multiple sources"""
        sources = [
            {'url': 'https://example.com', 'title': 'Example Page'},
            {'url': 'https://test.com', 'title': 'Test Page'}
        ]
        footer = build_sources_footer(sources)
        assert 'Example Page' in footer
        assert 'Test Page' in footer
        assert footer.count('<') == 2  # Two URLs with angle brackets

    def test_build_sources_footer_empty_list(self):
        """Test building footer with empty source list"""
        footer = build_sources_footer([])
        assert footer == ""

    def test_build_sources_footer_missing_title(self):
        """Test building footer when title is missing"""
        sources = [{'url': 'https://example.com'}]
        footer = build_sources_footer(sources)
        assert 'Untitled' in footer
        assert '<https://example.com>' in footer

    def test_build_sources_footer_missing_url(self):
        """Test building footer when URL is missing"""
        sources = [{'title': 'Example Page'}]
        footer = build_sources_footer(sources)
        # Should not include entry if URL is missing
        assert footer == '**Sources:**\n'

    def test_build_sources_footer_angle_brackets_suppress_embeds(self):
        """Test that angle brackets are used to suppress Discord embeds"""
        sources = [{'url': 'https://example.com', 'title': 'Test'}]
        footer = build_sources_footer(sources)
        # URLs should be in angle brackets
        assert '<https://example.com>' in footer
        assert 'https://example.com>' not in footer.replace('<https://example.com>', '')


class TestDomainExtraction:
    """Test domain extraction from URLs."""

    def test_extract_domain_basic(self):
        """Test extracting domain from basic URL"""
        domain = extract_domain("https://example.com/path")
        assert domain == "example.com"

    def test_extract_domain_with_subdomain(self):
        """Test extracting domain with subdomain"""
        domain = extract_domain("https://api.example.com")
        assert domain == "api.example.com"

    def test_extract_domain_with_port(self):
        """Test extracting domain with port"""
        domain = extract_domain("https://example.com:8080/path")
        assert domain == "example.com:8080"

    def test_extract_domain_invalid_url(self):
        """Test extracting domain from invalid URL"""
        domain = extract_domain("not a url")
        # Should return None or handle gracefully
        assert domain is None or isinstance(domain, str)


class TestProgressFormatting:
    """Test progress message formatting."""

    def test_format_progress_message_basic(self):
        """Test basic progress message formatting"""
        msg = format_progress_message(
            url="https://example.com",
            task="Find pricing",
            keywords=["pricing", "cost"],
            depth=2
        )
        assert "example.com" in msg
        assert "pricing" in msg
        assert "cost" in msg
        assert "**Depth:** 2" in msg

    def test_format_progress_message_with_crawl_delay(self):
        """Test progress message with crawl delay"""
        msg = format_progress_message(
            url="https://example.com",
            crawl_delay=5.0
        )
        assert "**Crawl delay:** 5.0s" in msg

    def test_format_progress_message_no_crawl_delay_if_default(self):
        """Test that default crawl delay is not shown"""
        msg = format_progress_message(
            url="https://example.com",
            crawl_delay=2.0
        )
        assert "Crawl delay" not in msg

    def test_format_progress_message_fresh_mode(self):
        """Test progress message with fresh mode"""
        msg = format_progress_message(
            url="https://example.com",
            cache_mode="fresh"
        )
        assert "**Mode:** fresh" in msg

    def test_format_answering_message(self):
        """Test answering phase message formatting"""
        msg = format_answering_message(5)
        assert "5 page(s)" in msg
        assert "Analyzing" in msg


class TestKeywordStemming:
    """Test keyword stemming functionality."""

    def test_generate_stems_basic(self):
        """Test generating stems from keywords"""
        stems = generate_keyword_stems(["running", "tests"], [])
        assert "run" in stems or "test" in stems

    def test_generate_stems_no_duplicates(self):
        """Test that stems don't duplicate keywords"""
        stems = generate_keyword_stems(["run"], ["running"])
        # "run" is already in keywords, so stem shouldn't be added
        assert "run" not in stems

    def test_generate_stems_variations(self):
        """Test stem generation from variations"""
        stems = generate_keyword_stems([], ["pricing", "prices"])
        # Should generate stems if they're different from variations
        assert isinstance(stems, list)

    def test_generate_stems_no_stemming_needed(self):
        """Test when no stemming is needed"""
        stems = generate_keyword_stems(["api", "cli"], [])
        # Short words shouldn't be stemmed
        assert len(stems) == 0

    def test_generate_stems_phrase(self):
        """Test stem generation from phrase"""
        stems = generate_keyword_stems(["pull requests"], [])
        # Should split phrase and stem individual words
        assert isinstance(stems, list)


class TestWebFetchIntegration:
    """Test web fetch integration with mock web cache manager."""

    @pytest.mark.asyncio
    async def test_fetch_and_cache_success(self):
        """Test successful fetch and cache operation"""
        # This would require mocking web_cache_manager
        # For now, just verify the function exists and has correct signature
        from arborist.sources.crawler.web_fetch import fetch_and_cache
        import inspect
        sig = inspect.signature(fetch_and_cache)
        assert 'url' in sig.parameters
        assert 'web_cache_manager' in sig.parameters
        assert 'keywords' in sig.parameters
        assert 'depth' in sig.parameters
        assert 'fresh' in sig.parameters

    @pytest.mark.asyncio
    async def test_fetch_and_cache_with_progress_callback(self):
        """Test fetch with progress callback"""
        from arborist.sources.crawler.web_fetch import fetch_and_cache

        # Mock web cache manager
        mock_manager = Mock()
        mock_manager.get_or_fetch = AsyncMock(return_value={
            'url': 'https://example.com',
            'title': 'Test',
            'extracted_text': 'Content',
            'source_urls': []
        })

        # Mock progress callback
        progress_calls = []
        async def progress_callback(msg):
            progress_calls.append(msg)

        result = await fetch_and_cache(
            url='https://example.com',
            web_cache_manager=mock_manager,
            progress_callback=progress_callback
        )

        assert result is not None
        assert result['url'] == 'https://example.com'
        mock_manager.get_or_fetch.assert_called_once()


if __name__ == '__main__':
    pytest.main([__file__, '-v'])