Russell Ballestrini
027f5afc57
- Add LICENSE file (public domain, permacomputer values)
- Add license headers to all 27 Python source files
- Restore full about_ch9_p2 with "terminated 🟣" content
- Translate about_ch9_p2 to all 26 languages
The permacomputer is community-owned infrastructure optimized around:
TRUTH, FREEDOM, HARMONY, LOVE
106 lines
3.9 KiB
Python
106 lines
3.9 KiB
Python
# PUBLIC DOMAIN - NO LICENSE, NO WARRANTY
|
|
#
|
|
# This is free public domain software for the public good of a permacomputer hosted
|
|
# at permacomputer.com - an always-on computer by the people, for the people. One
|
|
# which is durable, easy to repair, and distributed like tap water for machine
|
|
# learning intelligence.
|
|
#
|
|
# The permacomputer is community-owned infrastructure optimized around four values:
|
|
#
|
|
# TRUTH - First principles, math & science, open source code freely distributed
|
|
# FREEDOM - Voluntary partnerships, freedom from tyranny & corporate control
|
|
# HARMONY - Minimal waste, self-renewing systems with diverse thriving connections
|
|
# LOVE - Be yourself without hurting others, cooperation through natural law
|
|
#
|
|
# Anyone is free to copy, modify, publish, use, compile, sell, or distribute this
|
|
# software, either in source code form or as a compiled binary, for any purpose,
|
|
# commercial or non-commercial, and by any means.
|
|
#
|
|
# NO WARRANTY. THE SOFTWARE IS PROVIDED "AS IS" WITHOUT WARRANTY OF ANY KIND.
|
|
#
|
|
# Copyright 2025 TimeHexOn & foxhop & russell@unturf
|
|
# https://git.unturf.com/engineering/unturf/pig.py
|
|
|
|
"""
|
|
HTML processing utilities for neopig.
|
|
|
|
Functions for cleaning and extracting metadata from HTML content.
|
|
"""
|
|
|
|
|
|
def trim_html_wrapper(html: str) -> str:
|
|
"""Strip nav, header, footer, sidebar, and logo elements from HTML.
|
|
|
|
Useful for cleaning up Discourse and similar sites before markdown conversion.
|
|
"""
|
|
from bs4 import BeautifulSoup
|
|
soup = BeautifulSoup(html, 'html.parser')
|
|
|
|
# Remove common wrapper elements
|
|
selectors_to_remove = [
|
|
'nav', 'header', 'footer', 'aside',
|
|
'.sidebar', '.nav', '.navigation', '.menu',
|
|
'.header', '.footer', '.logo', '.site-logo',
|
|
'#header', '#footer', '#nav', '#sidebar',
|
|
'.d-header', '.d-footer', # Discourse specific
|
|
'.header-wrapper', '.footer-wrapper',
|
|
'[role="banner"]', '[role="navigation"]', '[role="contentinfo"]',
|
|
]
|
|
|
|
for selector in selectors_to_remove:
|
|
for tag in soup.select(selector):
|
|
tag.decompose()
|
|
|
|
# Remove site logo images (be specific to avoid removing content images)
|
|
for img in soup.find_all('img'):
|
|
src = img.get('src', '').lower()
|
|
alt = img.get('alt', '').lower()
|
|
cls = ' '.join(img.get('class', [])).lower()
|
|
# Only remove if it's clearly a site logo, not general icons
|
|
is_logo = 'logo' in cls or 'brand' in cls or 'site-logo' in src
|
|
is_logo = is_logo or (alt and ('logo' in alt or 'brand' in alt))
|
|
if is_logo:
|
|
img.decompose()
|
|
|
|
return str(soup)
|
|
|
|
|
|
def extract_meta_from_html(html: str) -> tuple[str, list[str]]:
|
|
"""Extract meta description and keywords from HTML.
|
|
|
|
Returns:
|
|
Tuple of (description, keywords_list)
|
|
"""
|
|
from bs4 import BeautifulSoup
|
|
import re
|
|
|
|
soup = BeautifulSoup(html, 'html.parser')
|
|
description = ""
|
|
keywords = []
|
|
|
|
# Extract meta description
|
|
meta_desc = soup.find('meta', attrs={'name': re.compile(r'^description$', re.I)})
|
|
if meta_desc and meta_desc.get('content'):
|
|
description = meta_desc['content'].strip()[:500]
|
|
|
|
# Extract meta keywords
|
|
meta_kw = soup.find('meta', attrs={'name': re.compile(r'^keywords$', re.I)})
|
|
if meta_kw and meta_kw.get('content'):
|
|
raw_kw = meta_kw['content']
|
|
keywords = [k.strip().lower() for k in raw_kw.split(',') if k.strip()]
|
|
|
|
# Also check og:description as fallback
|
|
if not description:
|
|
og_desc = soup.find('meta', attrs={'property': 'og:description'})
|
|
if og_desc and og_desc.get('content'):
|
|
description = og_desc['content'].strip()[:500]
|
|
|
|
# Extract from article:tag meta tags (common in blogs)
|
|
for tag_meta in soup.find_all('meta', attrs={'property': 'article:tag'}):
|
|
if tag_meta.get('content'):
|
|
keywords.append(tag_meta['content'].strip().lower())
|
|
|
|
# Dedupe keywords
|
|
keywords = list(dict.fromkeys(keywords))[:20]
|
|
|
|
return description, keywords
|