Russell Ballestrini
7883ae0eba
Phase 1 of module restructuring as outlined in docs/REFACTOR.md: New neopig/ package modules: - live.py: Live media queue (get_live_queue, emit_live_media) - state.py: AppendOnlyStateLog, state file helpers - html_utils.py: trim_html_wrapper, extract_meta_from_html - logging.py: TqdmLoggingHandler, job logging functions - backfill/: markdown and screenshot backfill operations Package features: - Lazy import of NeoPig/main from neopig.py via __getattr__ - Full backwards compatibility with existing imports - 42 new unit tests for extracted modules Total: 458 tests passing
120 lines
4.5 KiB
Python
120 lines
4.5 KiB
Python
"""
|
|
Tests for neopig.html_utils module.
|
|
|
|
Tests HTML processing utilities.
|
|
"""
|
|
|
|
import pytest
|
|
|
|
from neopig.html_utils import trim_html_wrapper, extract_meta_from_html
|
|
|
|
|
|
class TestTrimHtmlWrapper:
|
|
"""Test trim_html_wrapper function."""
|
|
|
|
def test_removes_nav(self):
|
|
"""Test removing nav elements."""
|
|
html = '<html><nav>Menu</nav><main>Content</main></html>'
|
|
result = trim_html_wrapper(html)
|
|
assert '<nav>' not in result
|
|
assert 'Content' in result
|
|
|
|
def test_removes_header(self):
|
|
"""Test removing header elements."""
|
|
html = '<html><header>Header</header><main>Content</main></html>'
|
|
result = trim_html_wrapper(html)
|
|
assert '<header>' not in result
|
|
assert 'Content' in result
|
|
|
|
def test_removes_footer(self):
|
|
"""Test removing footer elements."""
|
|
html = '<html><main>Content</main><footer>Footer</footer></html>'
|
|
result = trim_html_wrapper(html)
|
|
assert '<footer>' not in result
|
|
assert 'Content' in result
|
|
|
|
def test_removes_sidebar_class(self):
|
|
"""Test removing elements with sidebar class."""
|
|
html = '<html><div class="sidebar">Side</div><main>Content</main></html>'
|
|
result = trim_html_wrapper(html)
|
|
assert 'sidebar' not in result
|
|
assert 'Content' in result
|
|
|
|
def test_removes_logo_images(self):
|
|
"""Test removing logo images."""
|
|
html = '<html><img class="logo" src="logo.png"><img src="content.jpg"></html>'
|
|
result = trim_html_wrapper(html)
|
|
assert 'logo.png' not in result
|
|
assert 'content.jpg' in result
|
|
|
|
def test_preserves_content(self):
|
|
"""Test that main content is preserved."""
|
|
html = '<html><body><article><h1>Title</h1><p>Content</p></article></body></html>'
|
|
result = trim_html_wrapper(html)
|
|
assert 'Title' in result
|
|
assert 'Content' in result
|
|
|
|
|
|
class TestExtractMetaFromHtml:
|
|
"""Test extract_meta_from_html function."""
|
|
|
|
def test_extracts_description(self):
|
|
"""Test extracting meta description."""
|
|
html = '<html><head><meta name="description" content="Test description"></head></html>'
|
|
description, keywords = extract_meta_from_html(html)
|
|
assert description == "Test description"
|
|
|
|
def test_extracts_keywords(self):
|
|
"""Test extracting meta keywords."""
|
|
html = '<html><head><meta name="keywords" content="python, crawler, media"></head></html>'
|
|
description, keywords = extract_meta_from_html(html)
|
|
assert "python" in keywords
|
|
assert "crawler" in keywords
|
|
assert "media" in keywords
|
|
|
|
def test_extracts_og_description_fallback(self):
|
|
"""Test falling back to og:description."""
|
|
html = '<html><head><meta property="og:description" content="OG description"></head></html>'
|
|
description, keywords = extract_meta_from_html(html)
|
|
assert description == "OG description"
|
|
|
|
def test_extracts_article_tags(self):
|
|
"""Test extracting article:tag meta tags."""
|
|
html = '''<html><head>
|
|
<meta property="article:tag" content="python">
|
|
<meta property="article:tag" content="web">
|
|
</head></html>'''
|
|
description, keywords = extract_meta_from_html(html)
|
|
assert "python" in keywords
|
|
assert "web" in keywords
|
|
|
|
def test_deduplicates_keywords(self):
|
|
"""Test that keywords are deduplicated."""
|
|
html = '<html><head><meta name="keywords" content="python, Python, PYTHON"></head></html>'
|
|
description, keywords = extract_meta_from_html(html)
|
|
# All should be lowercase and deduplicated
|
|
assert keywords.count("python") == 1
|
|
|
|
def test_limits_keywords(self):
|
|
"""Test that keywords are limited to 20."""
|
|
kw_list = ", ".join([f"keyword{i}" for i in range(30)])
|
|
html = f'<html><head><meta name="keywords" content="{kw_list}"></head></html>'
|
|
description, keywords = extract_meta_from_html(html)
|
|
assert len(keywords) <= 20
|
|
|
|
def test_truncates_long_description(self):
|
|
"""Test that description is truncated to 500 chars."""
|
|
long_desc = "x" * 600
|
|
html = f'<html><head><meta name="description" content="{long_desc}"></head></html>'
|
|
description, keywords = extract_meta_from_html(html)
|
|
assert len(description) == 500
|
|
|
|
def test_empty_html(self):
|
|
"""Test handling empty HTML."""
|
|
description, keywords = extract_meta_from_html("")
|
|
assert description == ""
|
|
assert keywords == []
|
|
|
|
|
|
if __name__ == '__main__':
|
|
pytest.main([__file__, '-v'])
|