99 lines
3.7 KiB
Python
99 lines
3.7 KiB
Python
# This is free software for the public good of a permacomputer hosted at
|
|
# permacomputer.com, an always-on computer by the people, for the people.
|
|
# One which is durable, easy to repair, & distributed like tap water
|
|
# for machine learning intelligence.
|
|
#
|
|
# The permacomputer is community-owned infrastructure optimized around
|
|
# four values:
|
|
#
|
|
# TRUTH First principles, math & science, open source code freely distributed
|
|
# FREEDOM Voluntary partnerships, freedom from tyranny & corporate control
|
|
# HARMONY Minimal waste, self-renewing systems with diverse thriving connections
|
|
# LOVE Be yourself without hurting others, cooperation through natural law
|
|
#
|
|
# This software contributes to that vision by archiving the web, preserving digital knowledge before it disappears.
|
|
# Code is seeds to sprout on any abandoned technology.
|
|
|
|
"""
|
|
HTML processing utilities for neopig.
|
|
|
|
Functions for cleaning and extracting metadata from HTML content.
|
|
"""
|
|
|
|
|
|
def trim_html_wrapper(html: str) -> str:
|
|
"""Strip nav, header, footer, sidebar, and logo elements from HTML.
|
|
|
|
Useful for cleaning up Discourse and similar sites before markdown conversion.
|
|
"""
|
|
from bs4 import BeautifulSoup
|
|
soup = BeautifulSoup(html, 'html.parser')
|
|
|
|
# Remove common wrapper elements
|
|
selectors_to_remove = [
|
|
'nav', 'header', 'footer', 'aside',
|
|
'.sidebar', '.nav', '.navigation', '.menu',
|
|
'.header', '.footer', '.logo', '.site-logo',
|
|
'#header', '#footer', '#nav', '#sidebar',
|
|
'.d-header', '.d-footer', # Discourse specific
|
|
'.header-wrapper', '.footer-wrapper',
|
|
'[role="banner"]', '[role="navigation"]', '[role="contentinfo"]',
|
|
]
|
|
|
|
for selector in selectors_to_remove:
|
|
for tag in soup.select(selector):
|
|
tag.decompose()
|
|
|
|
# Remove site logo images (be specific to avoid removing content images)
|
|
for img in soup.find_all('img'):
|
|
src = img.get('src', '').lower()
|
|
alt = img.get('alt', '').lower()
|
|
cls = ' '.join(img.get('class', [])).lower()
|
|
# Only remove if it's clearly a site logo, not general icons
|
|
is_logo = 'logo' in cls or 'brand' in cls or 'site-logo' in src
|
|
is_logo = is_logo or (alt and ('logo' in alt or 'brand' in alt))
|
|
if is_logo:
|
|
img.decompose()
|
|
|
|
return str(soup)
|
|
|
|
|
|
def extract_meta_from_html(html: str) -> tuple[str, list[str]]:
|
|
"""Extract meta description and keywords from HTML.
|
|
|
|
Returns:
|
|
Tuple of (description, keywords_list)
|
|
"""
|
|
from bs4 import BeautifulSoup
|
|
import re
|
|
|
|
soup = BeautifulSoup(html, 'html.parser')
|
|
description = ""
|
|
keywords = []
|
|
|
|
# Extract meta description
|
|
meta_desc = soup.find('meta', attrs={'name': re.compile(r'^description$', re.I)})
|
|
if meta_desc and meta_desc.get('content'):
|
|
description = meta_desc['content'].strip()[:500]
|
|
|
|
# Extract meta keywords
|
|
meta_kw = soup.find('meta', attrs={'name': re.compile(r'^keywords$', re.I)})
|
|
if meta_kw and meta_kw.get('content'):
|
|
raw_kw = meta_kw['content']
|
|
keywords = [k.strip().lower() for k in raw_kw.split(',') if k.strip()]
|
|
|
|
# Also check og:description as fallback
|
|
if not description:
|
|
og_desc = soup.find('meta', attrs={'property': 'og:description'})
|
|
if og_desc and og_desc.get('content'):
|
|
description = og_desc['content'].strip()[:500]
|
|
|
|
# Extract from article:tag meta tags (common in blogs)
|
|
for tag_meta in soup.find_all('meta', attrs={'property': 'article:tag'}):
|
|
if tag_meta.get('content'):
|
|
keywords.append(tag_meta['content'].strip().lower())
|
|
|
|
# Dedupe keywords
|
|
keywords = list(dict.fromkeys(keywords))[:20]
|
|
|
|
return description, keywords
|