136 lines
5.3 KiB
Python
136 lines
5.3 KiB
Python
# This is free software for the public good of a permacomputer hosted at
|
|
# permacomputer.com, an always-on computer by the people, for the people.
|
|
# One which is durable, easy to repair, & distributed like tap water
|
|
# for machine learning intelligence.
|
|
#
|
|
# The permacomputer is community-owned infrastructure optimized around
|
|
# four values:
|
|
#
|
|
# TRUTH First principles, math & science, open source code freely distributed
|
|
# FREEDOM Voluntary partnerships, freedom from tyranny & corporate control
|
|
# HARMONY Minimal waste, self-renewing systems with diverse thriving connections
|
|
# LOVE Be yourself without hurting others, cooperation through natural law
|
|
#
|
|
# This software contributes to that vision by archiving the web, preserving digital knowledge before it disappears.
|
|
# Code is seeds to sprout on any abandoned technology.
|
|
|
|
"""
|
|
Tests for neopig.html_utils module.
|
|
|
|
Tests HTML processing utilities.
|
|
"""
|
|
|
|
import pytest
|
|
|
|
from neopig.html_utils import trim_html_wrapper, extract_meta_from_html
|
|
|
|
|
|
class TestTrimHtmlWrapper:
|
|
"""Test trim_html_wrapper function."""
|
|
|
|
def test_removes_nav(self):
|
|
"""Test removing nav elements."""
|
|
html = '<html><nav>Menu</nav><main>Content</main></html>'
|
|
result = trim_html_wrapper(html)
|
|
assert '<nav>' not in result
|
|
assert 'Content' in result
|
|
|
|
def test_removes_header(self):
|
|
"""Test removing header elements."""
|
|
html = '<html><header>Header</header><main>Content</main></html>'
|
|
result = trim_html_wrapper(html)
|
|
assert '<header>' not in result
|
|
assert 'Content' in result
|
|
|
|
def test_removes_footer(self):
|
|
"""Test removing footer elements."""
|
|
html = '<html><main>Content</main><footer>Footer</footer></html>'
|
|
result = trim_html_wrapper(html)
|
|
assert '<footer>' not in result
|
|
assert 'Content' in result
|
|
|
|
def test_removes_sidebar_class(self):
|
|
"""Test removing elements with sidebar class."""
|
|
html = '<html><div class="sidebar">Side</div><main>Content</main></html>'
|
|
result = trim_html_wrapper(html)
|
|
assert 'sidebar' not in result
|
|
assert 'Content' in result
|
|
|
|
def test_removes_logo_images(self):
|
|
"""Test removing logo images."""
|
|
html = '<html><img class="logo" src="logo.png"><img src="content.jpg"></html>'
|
|
result = trim_html_wrapper(html)
|
|
assert 'logo.png' not in result
|
|
assert 'content.jpg' in result
|
|
|
|
def test_preserves_content(self):
|
|
"""Test that main content is preserved."""
|
|
html = '<html><body><article><h1>Title</h1><p>Content</p></article></body></html>'
|
|
result = trim_html_wrapper(html)
|
|
assert 'Title' in result
|
|
assert 'Content' in result
|
|
|
|
|
|
class TestExtractMetaFromHtml:
|
|
"""Test extract_meta_from_html function."""
|
|
|
|
def test_extracts_description(self):
|
|
"""Test extracting meta description."""
|
|
html = '<html><head><meta name="description" content="Test description"></head></html>'
|
|
description, keywords = extract_meta_from_html(html)
|
|
assert description == "Test description"
|
|
|
|
def test_extracts_keywords(self):
|
|
"""Test extracting meta keywords."""
|
|
html = '<html><head><meta name="keywords" content="python, crawler, media"></head></html>'
|
|
description, keywords = extract_meta_from_html(html)
|
|
assert "python" in keywords
|
|
assert "crawler" in keywords
|
|
assert "media" in keywords
|
|
|
|
def test_extracts_og_description_fallback(self):
|
|
"""Test falling back to og:description."""
|
|
html = '<html><head><meta property="og:description" content="OG description"></head></html>'
|
|
description, keywords = extract_meta_from_html(html)
|
|
assert description == "OG description"
|
|
|
|
def test_extracts_article_tags(self):
|
|
"""Test extracting article:tag meta tags."""
|
|
html = '''<html><head>
|
|
<meta property="article:tag" content="python">
|
|
<meta property="article:tag" content="web">
|
|
</head></html>'''
|
|
description, keywords = extract_meta_from_html(html)
|
|
assert "python" in keywords
|
|
assert "web" in keywords
|
|
|
|
def test_deduplicates_keywords(self):
|
|
"""Test that keywords are deduplicated."""
|
|
html = '<html><head><meta name="keywords" content="python, Python, PYTHON"></head></html>'
|
|
description, keywords = extract_meta_from_html(html)
|
|
# All should be lowercase and deduplicated
|
|
assert keywords.count("python") == 1
|
|
|
|
def test_limits_keywords(self):
|
|
"""Test that keywords are limited to 20."""
|
|
kw_list = ", ".join([f"keyword{i}" for i in range(30)])
|
|
html = f'<html><head><meta name="keywords" content="{kw_list}"></head></html>'
|
|
description, keywords = extract_meta_from_html(html)
|
|
assert len(keywords) <= 20
|
|
|
|
def test_truncates_long_description(self):
|
|
"""Test that description is truncated to 500 chars."""
|
|
long_desc = "x" * 600
|
|
html = f'<html><head><meta name="description" content="{long_desc}"></head></html>'
|
|
description, keywords = extract_meta_from_html(html)
|
|
assert len(description) == 500
|
|
|
|
def test_empty_html(self):
|
|
"""Test handling empty HTML."""
|
|
description, keywords = extract_meta_from_html("")
|
|
assert description == ""
|
|
assert keywords == []
|
|
|
|
|
|
if __name__ == '__main__':
|
|
pytest.main([__file__, '-v'])
|