Russell Ballestrini
7883ae0eba
Phase 1 of module restructuring as outlined in docs/REFACTOR.md: New neopig/ package modules: - live.py: Live media queue (get_live_queue, emit_live_media) - state.py: AppendOnlyStateLog, state file helpers - html_utils.py: trim_html_wrapper, extract_meta_from_html - logging.py: TqdmLoggingHandler, job logging functions - backfill/: markdown and screenshot backfill operations Package features: - Lazy import of NeoPig/main from neopig.py via __getattr__ - Full backwards compatibility with existing imports - 42 new unit tests for extracted modules Total: 458 tests passing
83 lines
2.8 KiB
Python
83 lines
2.8 KiB
Python
"""
|
|
HTML processing utilities for neopig.
|
|
|
|
Functions for cleaning and extracting metadata from HTML content.
|
|
"""
|
|
|
|
|
|
def trim_html_wrapper(html: str) -> str:
|
|
"""Strip nav, header, footer, sidebar, and logo elements from HTML.
|
|
|
|
Useful for cleaning up Discourse and similar sites before markdown conversion.
|
|
"""
|
|
from bs4 import BeautifulSoup
|
|
soup = BeautifulSoup(html, 'html.parser')
|
|
|
|
# Remove common wrapper elements
|
|
selectors_to_remove = [
|
|
'nav', 'header', 'footer', 'aside',
|
|
'.sidebar', '.nav', '.navigation', '.menu',
|
|
'.header', '.footer', '.logo', '.site-logo',
|
|
'#header', '#footer', '#nav', '#sidebar',
|
|
'.d-header', '.d-footer', # Discourse specific
|
|
'.header-wrapper', '.footer-wrapper',
|
|
'[role="banner"]', '[role="navigation"]', '[role="contentinfo"]',
|
|
]
|
|
|
|
for selector in selectors_to_remove:
|
|
for tag in soup.select(selector):
|
|
tag.decompose()
|
|
|
|
# Remove site logo images (be specific to avoid removing content images)
|
|
for img in soup.find_all('img'):
|
|
src = img.get('src', '').lower()
|
|
alt = img.get('alt', '').lower()
|
|
cls = ' '.join(img.get('class', [])).lower()
|
|
# Only remove if it's clearly a site logo, not general icons
|
|
is_logo = 'logo' in cls or 'brand' in cls or 'site-logo' in src
|
|
is_logo = is_logo or (alt and ('logo' in alt or 'brand' in alt))
|
|
if is_logo:
|
|
img.decompose()
|
|
|
|
return str(soup)
|
|
|
|
|
|
def extract_meta_from_html(html: str) -> tuple[str, list[str]]:
|
|
"""Extract meta description and keywords from HTML.
|
|
|
|
Returns:
|
|
Tuple of (description, keywords_list)
|
|
"""
|
|
from bs4 import BeautifulSoup
|
|
import re
|
|
|
|
soup = BeautifulSoup(html, 'html.parser')
|
|
description = ""
|
|
keywords = []
|
|
|
|
# Extract meta description
|
|
meta_desc = soup.find('meta', attrs={'name': re.compile(r'^description$', re.I)})
|
|
if meta_desc and meta_desc.get('content'):
|
|
description = meta_desc['content'].strip()[:500]
|
|
|
|
# Extract meta keywords
|
|
meta_kw = soup.find('meta', attrs={'name': re.compile(r'^keywords$', re.I)})
|
|
if meta_kw and meta_kw.get('content'):
|
|
raw_kw = meta_kw['content']
|
|
keywords = [k.strip().lower() for k in raw_kw.split(',') if k.strip()]
|
|
|
|
# Also check og:description as fallback
|
|
if not description:
|
|
og_desc = soup.find('meta', attrs={'property': 'og:description'})
|
|
if og_desc and og_desc.get('content'):
|
|
description = og_desc['content'].strip()[:500]
|
|
|
|
# Extract from article:tag meta tags (common in blogs)
|
|
for tag_meta in soup.find_all('meta', attrs={'property': 'article:tag'}):
|
|
if tag_meta.get('content'):
|
|
keywords.append(tag_meta['content'].strip().lower())
|
|
|
|
# Dedupe keywords
|
|
keywords = list(dict.fromkeys(keywords))[:20]
|
|
|
|
return description, keywords
|