engineering/pig.py
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add Reddit and Remarkbox forum detection selectors

Browse source
This commit is contained in:
Russell Ballestrini 2026-01-02 18:24:44 -05:00
parent d3264d80d4
commit 18a22020b1
1 changed files with 18 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

22
html2md.py
View file

@ -82,7 +82,7 @@ class SmartMarkdownConverter:
def _detect_and_extract(self, soup: BeautifulSoup) -> Tuple[str, any]:
"""Detect content type and extract structured content."""
# Forum detection (Discourse, phpBB, vBulletin, etc.)
# Forum detection (Discourse, phpBB, vBulletin, Reddit, etc.)
forum_indicators = [
'.post_container', # Discourse archived (has avatar_container + post)
'.topic-post', # Discourse live
@ -90,6 +90,11 @@ class SmartMarkdownConverter:
'.message', # XenForo
'.postcontainer', # vBulletin
'article.boxed', # Generic
'.thing.comment', # Reddit old
'shreddit-comment', # Reddit new
'.Comment', # Reddit redesign
'.remarkbox-comment', # Remarkbox
'.comment', # Generic comments
]
for post_sel in forum_indicators:
posts = soup.select(post_sel)
@ -153,7 +158,10 @@ class SmartMarkdownConverter:
post.select_one('.avatar-flair img') or
post.select_one('.user-avatar img') or
post.select_one('.postprofile img') or
post.select_one('.author img')
post.select_one('.author img') or
post.select_one('img[alt*="avatar"]') or # Reddit
post.select_one('.flair img') or # Reddit flair
post.select_one('.remarkbox-avatar img') # Remarkbox
)
if avatar:
src = avatar.get('src', '')
@ -171,7 +179,10 @@ class SmartMarkdownConverter:
post.select_one('a[data-user-card]') or
post.select_one('.names .name') or
post.select_one('.postprofile dt') or
post.select_one('strong.username')
post.select_one('strong.username') or
post.select_one('a[href*="/user/"]') or # Reddit
post.select_one('a[href*="/u/"]') or # Reddit shorthand
post.select_one('.remarkbox-author') # Remarkbox
)
if username_el:
post_data['username'] = username_el.get_text(strip=True)
@ -197,7 +208,10 @@ class SmartMarkdownConverter:
post.select_one('.post_body') or
post.select_one('.entry-content') or
post.select_one('.content') or
post.select_one('.post') # Fallback to inner .post div
post.select_one('.post') or # Fallback to inner .post div
post.select_one('.md') or # Reddit markdown content
post.select_one('.usertext-body') or # Reddit old
post.select_one('.remarkbox-content') # Remarkbox
)
if content_el:
post_data['content'] = self._element_to_markdown(content_el)