modified: CLAUDE.md
modified: archive.py modified: database.py modified: neopig.py modified: screenshot.py modified: serp.py
This commit is contained in:
parent
36edc3b8dc
commit
222c23bb14
6 changed files with 710 additions and 59 deletions
|
|
@ -4,6 +4,10 @@
|
|||
|
||||
**NEVER attribute Claude in commit messages.** No co-author tags, no "Generated with Claude" footers, no AI mentions. Keep commits clean and human-authored in appearance.
|
||||
|
||||
## Naming Conventions
|
||||
|
||||
**Always use `uri`, never `url`.** This applies to variable names, function names, column names, and comments. URI is the correct term (Uniform Resource Identifier).
|
||||
|
||||
## Quick Start
|
||||
|
||||
```bash
|
||||
|
|
|
|||
381
archive.py
381
archive.py
|
|
@ -708,7 +708,7 @@ async function search() {{
|
|||
}}
|
||||
container.innerHTML = data.map(r => `
|
||||
<div class="result">
|
||||
<h3><a href="/${{r.path}}">${{r.title}}</a></h3>
|
||||
<h3><a href="/view/${{r.path.replace('html/', '')}}">${{r.title}}</a></h3>
|
||||
<p class="snippet">${{r.snippet}}</p>
|
||||
<p class="path">${{r.path}}</p>
|
||||
</div>
|
||||
|
|
@ -779,27 +779,31 @@ MEDIA_HTML = """<!DOCTYPE html>
|
|||
margin: 0; padding: 20px;
|
||||
background: #0a0a0a; color: #e0e0e0;
|
||||
}}
|
||||
.container {{ max-width: 1400px; margin: 0 auto; }}
|
||||
.container {{ max-width: 1600px; margin: 0 auto; }}
|
||||
h1 {{ color: #ff6b6b; }}
|
||||
nav {{ margin-bottom: 20px; }}
|
||||
nav a {{ color: #ff6b6b; margin-right: 15px; text-decoration: none; }}
|
||||
nav a:hover {{ text-decoration: underline; }}
|
||||
.grid {{
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fill, minmax(200px, 1fr));
|
||||
gap: 15px;
|
||||
grid-template-columns: repeat(auto-fill, minmax(220px, 1fr));
|
||||
grid-auto-rows: min-content;
|
||||
gap: 12px;
|
||||
}}
|
||||
.card {{
|
||||
background: #1a1a1a; border-radius: 8px;
|
||||
overflow: hidden; transition: transform 0.2s;
|
||||
break-inside: avoid;
|
||||
}}
|
||||
.card:hover {{ transform: scale(1.02); }}
|
||||
.card a {{ display: block; }}
|
||||
.card img, .card video {{
|
||||
width: 100%; height: 150px;
|
||||
object-fit: contain; background: #222;
|
||||
width: 100%;
|
||||
height: auto;
|
||||
display: block;
|
||||
background: #222;
|
||||
}}
|
||||
.card-info {{ padding: 10px; }}
|
||||
.card-info {{ padding: 8px 10px; }}
|
||||
.card-title {{
|
||||
font-size: 11px; color: #888;
|
||||
overflow: hidden; text-overflow: ellipsis; white-space: nowrap;
|
||||
|
|
@ -823,6 +827,249 @@ MEDIA_HTML = """<!DOCTYPE html>
|
|||
</body>
|
||||
</html>"""
|
||||
|
||||
MEDIA_DETAIL_HTML = """<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>{caption} - neopig Archive</title>
|
||||
<style>
|
||||
* {{ box-sizing: border-box; }}
|
||||
body {{
|
||||
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
|
||||
margin: 0; padding: 20px;
|
||||
background: #0a0a0a; color: #e0e0e0;
|
||||
line-height: 1.6;
|
||||
}}
|
||||
.container {{ max-width: 1000px; margin: 0 auto; }}
|
||||
nav {{ margin-bottom: 20px; padding-bottom: 15px; border-bottom: 1px solid #333; }}
|
||||
nav a {{ color: #ff6b6b; margin-right: 15px; text-decoration: none; }}
|
||||
nav a:hover {{ text-decoration: underline; }}
|
||||
.hero {{
|
||||
background: #111; border-radius: 12px; padding: 20px;
|
||||
margin-bottom: 25px; text-align: center;
|
||||
}}
|
||||
.hero img, .hero video {{
|
||||
max-width: 100%; max-height: 70vh;
|
||||
border-radius: 8px; margin-bottom: 15px;
|
||||
}}
|
||||
.caption {{
|
||||
font-size: 1.2em; color: #fff; margin: 15px 0 10px;
|
||||
}}
|
||||
.meta {{
|
||||
font-size: 12px; color: #666; margin: 10px 0;
|
||||
}}
|
||||
.meta a {{ color: #6bb3ff; word-break: break-all; }}
|
||||
.meta-row {{ margin: 5px 0; }}
|
||||
.meta-label {{ color: #888; }}
|
||||
.divider {{
|
||||
border: none; border-top: 1px solid #333;
|
||||
margin: 25px 0;
|
||||
}}
|
||||
.source-heading {{
|
||||
color: #ff6b6b; font-size: 1.1em; margin-bottom: 15px;
|
||||
}}
|
||||
.content {{
|
||||
background: #111; border-radius: 8px; padding: 25px;
|
||||
font-size: 15px;
|
||||
}}
|
||||
.content h1, .content h2, .content h3 {{ color: #ff6b6b; margin-top: 1.5em; }}
|
||||
.content h1:first-child, .content h2:first-child {{ margin-top: 0; }}
|
||||
.content a {{ color: #6bb3ff; }}
|
||||
.content code {{
|
||||
background: #1a1a1a; padding: 2px 6px; border-radius: 3px;
|
||||
font-family: monospace; font-size: 0.9em;
|
||||
}}
|
||||
.content pre {{
|
||||
background: #1a1a1a; padding: 15px; border-radius: 6px;
|
||||
overflow-x: auto; font-size: 0.85em;
|
||||
}}
|
||||
.content pre code {{ background: none; padding: 0; }}
|
||||
.content blockquote {{
|
||||
border-left: 3px solid #ff6b6b; margin: 1em 0;
|
||||
padding-left: 15px; color: #aaa;
|
||||
}}
|
||||
.content img {{ max-width: 100%; height: auto; border-radius: 4px; }}
|
||||
.content ul, .content ol {{ padding-left: 25px; }}
|
||||
.content li {{ margin: 0.3em 0; }}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="container">
|
||||
<nav>
|
||||
<a href="/">Search</a>
|
||||
<a href="/browse">Browse Pages</a>
|
||||
<a href="/media">Media</a>
|
||||
<a href="/screenshots">Screenshots</a>
|
||||
</nav>
|
||||
|
||||
<div class="hero">
|
||||
{media_element}
|
||||
<div class="caption">{caption}</div>
|
||||
<div class="meta">
|
||||
<div class="meta-row"><span class="meta-label">Image:</span> <a href="{media_uri}" target="_blank">{media_uri}</a></div>
|
||||
<div class="meta-row"><span class="meta-label">Source:</span> <a href="/view/{page_path}">{page_uri}</a></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<hr class="divider">
|
||||
|
||||
<div class="source-heading">Source Page Content</div>
|
||||
<div class="content">
|
||||
{page_content}
|
||||
</div>
|
||||
</div>
|
||||
</body>
|
||||
</html>"""
|
||||
|
||||
|
||||
PAGE_VIEW_HTML = """<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>{title} - neopig Archive</title>
|
||||
<style>
|
||||
* {{ box-sizing: border-box; }}
|
||||
body {{
|
||||
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
|
||||
margin: 0; padding: 20px;
|
||||
background: #0a0a0a; color: #e0e0e0;
|
||||
line-height: 1.6;
|
||||
}}
|
||||
.container {{ max-width: 900px; margin: 0 auto; }}
|
||||
h1 {{ color: #ff6b6b; margin-bottom: 5px; font-size: 1.5em; }}
|
||||
nav {{ margin-bottom: 20px; padding-bottom: 15px; border-bottom: 1px solid #333; }}
|
||||
nav a {{ color: #ff6b6b; margin-right: 15px; text-decoration: none; }}
|
||||
nav a:hover {{ text-decoration: underline; }}
|
||||
.meta {{ color: #666; font-size: 12px; margin-bottom: 20px; }}
|
||||
.meta a {{ color: #888; }}
|
||||
.content {{
|
||||
background: #111; border-radius: 8px; padding: 25px;
|
||||
font-size: 15px;
|
||||
}}
|
||||
.content h1, .content h2, .content h3 {{ color: #ff6b6b; margin-top: 1.5em; }}
|
||||
.content h1:first-child, .content h2:first-child {{ margin-top: 0; }}
|
||||
.content a {{ color: #6bb3ff; }}
|
||||
.content code {{
|
||||
background: #1a1a1a; padding: 2px 6px; border-radius: 3px;
|
||||
font-family: monospace; font-size: 0.9em;
|
||||
}}
|
||||
.content pre {{
|
||||
background: #1a1a1a; padding: 15px; border-radius: 6px;
|
||||
overflow-x: auto; font-size: 0.85em;
|
||||
}}
|
||||
.content pre code {{ background: none; padding: 0; }}
|
||||
.content blockquote {{
|
||||
border-left: 3px solid #ff6b6b; margin: 1em 0;
|
||||
padding-left: 15px; color: #aaa;
|
||||
}}
|
||||
.content img {{ max-width: 100%; height: auto; border-radius: 4px; }}
|
||||
.content ul, .content ol {{ padding-left: 25px; }}
|
||||
.content li {{ margin: 0.3em 0; }}
|
||||
.content hr {{ border: none; border-top: 1px solid #333; margin: 2em 0; }}
|
||||
.screenshot {{ margin-top: 20px; }}
|
||||
.screenshot img {{ max-width: 100%; border: 1px solid #333; border-radius: 4px; }}
|
||||
.screenshot-label {{ color: #666; font-size: 12px; margin-bottom: 5px; }}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="container">
|
||||
<nav>
|
||||
<a href="/">Search</a>
|
||||
<a href="/browse">Browse Pages</a>
|
||||
<a href="/media">Media</a>
|
||||
<a href="/screenshots">Screenshots</a>
|
||||
</nav>
|
||||
<h1>{title}</h1>
|
||||
<div class="meta">
|
||||
<a href="/html/{path}">[View Original HTML]</a>
|
||||
{screenshot_link}
|
||||
</div>
|
||||
<div class="content">
|
||||
{content}
|
||||
</div>
|
||||
{screenshot_embed}
|
||||
</div>
|
||||
</body>
|
||||
</html>"""
|
||||
|
||||
|
||||
def simple_markdown_to_html(md: str) -> str:
|
||||
"""Convert markdown to HTML (simple stdlib-only implementation)."""
|
||||
import re
|
||||
lines = md.split('\n')
|
||||
html_lines = []
|
||||
in_code_block = False
|
||||
in_list = False
|
||||
|
||||
for line in lines:
|
||||
# Code blocks
|
||||
if line.startswith('```'):
|
||||
if in_code_block:
|
||||
html_lines.append('</code></pre>')
|
||||
in_code_block = False
|
||||
else:
|
||||
html_lines.append('<pre><code>')
|
||||
in_code_block = True
|
||||
continue
|
||||
|
||||
if in_code_block:
|
||||
html_lines.append(html.escape(line))
|
||||
continue
|
||||
|
||||
# Close list if needed
|
||||
if in_list and not line.strip().startswith(('- ', '* ', '1. ')):
|
||||
html_lines.append('</ul>')
|
||||
in_list = False
|
||||
|
||||
# Headers
|
||||
if line.startswith('### '):
|
||||
html_lines.append(f'<h3>{html.escape(line[4:])}</h3>')
|
||||
elif line.startswith('## '):
|
||||
html_lines.append(f'<h2>{html.escape(line[3:])}</h2>')
|
||||
elif line.startswith('# '):
|
||||
html_lines.append(f'<h1>{html.escape(line[2:])}</h1>')
|
||||
# Blockquotes
|
||||
elif line.startswith('> '):
|
||||
html_lines.append(f'<blockquote>{html.escape(line[2:])}</blockquote>')
|
||||
# Horizontal rule
|
||||
elif line.strip() in ('---', '***', '___'):
|
||||
html_lines.append('<hr>')
|
||||
# Lists
|
||||
elif line.strip().startswith(('- ', '* ')):
|
||||
if not in_list:
|
||||
html_lines.append('<ul>')
|
||||
in_list = True
|
||||
content = line.strip()[2:]
|
||||
html_lines.append(f'<li>{html.escape(content)}</li>')
|
||||
# Empty line
|
||||
elif not line.strip():
|
||||
html_lines.append('<br>')
|
||||
# Regular paragraph
|
||||
else:
|
||||
escaped = html.escape(line)
|
||||
# Inline code
|
||||
escaped = re.sub(r'`([^`]+)`', r'<code>\1</code>', escaped)
|
||||
# Bold
|
||||
escaped = re.sub(r'[*][*]([^*]+)[*][*]', r'<strong>\1</strong>', escaped)
|
||||
# Italic
|
||||
escaped = re.sub(r'[*]([^*]+)[*]', r'<em>\1</em>', escaped)
|
||||
# Links [text](url)
|
||||
link_re = re.compile(r'\[([^]]+)\]\(([^)]+)\)')
|
||||
escaped = link_re.sub(r'<a href="\2">\1</a>', escaped)
|
||||
# Images 
|
||||
img_re = re.compile(r'!\[([^]]*)\]\(([^)]+)\)')
|
||||
escaped = img_re.sub(r'<img src="\2" alt="\1">', escaped)
|
||||
html_lines.append(f'<p>{escaped}</p>')
|
||||
|
||||
if in_list:
|
||||
html_lines.append('</ul>')
|
||||
if in_code_block:
|
||||
html_lines.append('</code></pre>')
|
||||
|
||||
return '\n'.join(html_lines)
|
||||
|
||||
|
||||
class ArchiveHandler(BaseHTTPRequestHandler):
|
||||
"""HTTP request handler for the archive."""
|
||||
|
|
@ -882,7 +1129,7 @@ class ArchiveHandler(BaseHTTPRequestHandler):
|
|||
if path == '/browse':
|
||||
pages = list_files('html', '*.html')
|
||||
items = ''.join([
|
||||
f'<div class="list-item"><a href="/html/{html.escape(p)}">{html.escape(p)}</a></div>'
|
||||
f'<div class="list-item"><a href="/view/{html.escape(p)}">{html.escape(p)}</a></div>'
|
||||
for p in pages[:500]
|
||||
])
|
||||
if not items:
|
||||
|
|
@ -891,6 +1138,55 @@ class ArchiveHandler(BaseHTTPRequestHandler):
|
|||
self.send_html(content)
|
||||
return
|
||||
|
||||
# Page view - render markdown with neopig styling
|
||||
if path.startswith('/view/'):
|
||||
page_path = path[6:] # Remove '/view/'
|
||||
# Try markdown first, fall back to HTML
|
||||
md_path = 'markdown/' + page_path.replace('.html', '.md')
|
||||
html_path = 'html/' + page_path
|
||||
|
||||
md_content, _ = read_file(md_path)
|
||||
html_content_raw, _ = read_file(html_path)
|
||||
|
||||
if md_content:
|
||||
# Render markdown
|
||||
rendered = simple_markdown_to_html(md_content.decode('utf-8', errors='replace'))
|
||||
elif html_content_raw:
|
||||
# Extract body from HTML and show as-is
|
||||
html_str = html_content_raw.decode('utf-8', errors='replace')
|
||||
# Simple body extraction
|
||||
import re as re_mod
|
||||
body_match = re_mod.search(r'<body[^>]*>(.*?)</body>', html_str, re_mod.DOTALL | re_mod.IGNORECASE)
|
||||
rendered = body_match.group(1) if body_match else html_str
|
||||
else:
|
||||
self.send_404()
|
||||
return
|
||||
|
||||
# Extract title
|
||||
title = page_path.replace('.html', '').replace('/', ' > ')
|
||||
|
||||
# Check for screenshot
|
||||
ss_path = page_path.replace('.html', '.png').replace('/', '_')
|
||||
screenshot_exists = f'screenshots/{ss_path}' in [f'screenshots/{f}' for f in list_files('screenshots', '*.png')]
|
||||
|
||||
screenshot_link = f' | <a href="/screenshots/{html.escape(ss_path)}">[View Screenshot]</a>' if screenshot_exists else ''
|
||||
screenshot_embed = (
|
||||
'<div class="screenshot">'
|
||||
'<div class="screenshot-label">Page Screenshot:</div>'
|
||||
f'<img src="/screenshots/{html.escape(ss_path)}" alt="Screenshot">'
|
||||
'</div>'
|
||||
) if screenshot_exists else ''
|
||||
|
||||
content = PAGE_VIEW_HTML.format(
|
||||
title=html.escape(title),
|
||||
path=html.escape(page_path),
|
||||
content=rendered,
|
||||
screenshot_link=screenshot_link,
|
||||
screenshot_embed=screenshot_embed,
|
||||
)
|
||||
self.send_html(content)
|
||||
return
|
||||
|
||||
if path == '/media':
|
||||
files = list_files('media')
|
||||
items = []
|
||||
|
|
@ -900,9 +1196,10 @@ class ArchiveHandler(BaseHTTPRequestHandler):
|
|||
media_el = f'<video src="/media/{html.escape(f)}" preload="metadata"></video>'
|
||||
else:
|
||||
media_el = f'<img src="/media/{html.escape(f)}" loading="lazy">'
|
||||
# Link to detail view instead of raw file
|
||||
items.append(
|
||||
'<div class="card">'
|
||||
f'<a href="/media/{html.escape(f)}" target="_blank">{media_el}</a>'
|
||||
f'<a href="/media/detail/{html.escape(f)}">{media_el}</a>'
|
||||
f'<div class="card-info"><div class="card-title">{html.escape(f)}</div></div>'
|
||||
'</div>'
|
||||
)
|
||||
|
|
@ -914,6 +1211,72 @@ class ArchiveHandler(BaseHTTPRequestHandler):
|
|||
self.send_html(content)
|
||||
return
|
||||
|
||||
# Media detail view - image at top with source page content below
|
||||
if path.startswith('/media/detail/'):
|
||||
media_path = path[14:] # Remove '/media/detail/'
|
||||
media_file = 'media/' + media_path
|
||||
|
||||
# Check media exists
|
||||
media_content, mime = read_file(media_file)
|
||||
if not media_content:
|
||||
self.send_404()
|
||||
return
|
||||
|
||||
# Determine media element type
|
||||
ext = Path(media_path).suffix.lower()
|
||||
if ext in ('.mp4', '.webm', '.mov'):
|
||||
media_el = f'<video src="/media/{html.escape(media_path)}" controls autoplay muted></video>'
|
||||
else:
|
||||
media_el = f'<img src="/media/{html.escape(media_path)}">'
|
||||
|
||||
# Try to find the source page - media path mirrors URL structure
|
||||
# e.g., media/images/foo.jpg might come from t/topic-name/123.html
|
||||
# For now, use filename as caption
|
||||
caption = Path(media_path).stem.replace('-', ' ').replace('_', ' ')
|
||||
media_uri = f'/media/{media_path}'
|
||||
page_uri = METADATA.get('target_url', METADATA.get('domain', 'unknown'))
|
||||
page_path = 'index.html'
|
||||
|
||||
# Try to find associated page content from the database
|
||||
page_content = '<p class="empty">Source page content not available in archive database.</p>'
|
||||
if DB_PATH and DB_PATH.exists():
|
||||
conn = sqlite3.connect(DB_PATH)
|
||||
c = conn.cursor()
|
||||
try:
|
||||
# Search for pages that might contain this media
|
||||
media_name = Path(media_path).name
|
||||
c.execute("""
|
||||
SELECT path, title, content FROM pages
|
||||
WHERE content LIKE ? OR path LIKE ?
|
||||
LIMIT 1
|
||||
""", (f'%{media_name}%', f'%{media_name}%'))
|
||||
row = c.fetchone()
|
||||
if row:
|
||||
page_path = row[0].replace('html/', '')
|
||||
caption = row[1] or caption
|
||||
# Render the content
|
||||
md_file = 'markdown/' + page_path.replace('.html', '.md')
|
||||
md_content, _ = read_file(md_file)
|
||||
if md_content:
|
||||
page_content = simple_markdown_to_html(md_content.decode('utf-8', errors='replace'))
|
||||
else:
|
||||
page_content = f'<p>{html.escape(row[2][:2000] if row[2] else "")}...</p>'
|
||||
except Exception:
|
||||
pass
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
content = MEDIA_DETAIL_HTML.format(
|
||||
media_element=media_el,
|
||||
caption=html.escape(caption),
|
||||
media_uri=html.escape(media_uri),
|
||||
page_uri=html.escape(page_uri),
|
||||
page_path=html.escape(page_path),
|
||||
page_content=page_content,
|
||||
)
|
||||
self.send_html(content)
|
||||
return
|
||||
|
||||
if path == '/screenshots':
|
||||
files = list_files('screenshots', '*.png')
|
||||
items = []
|
||||
|
|
|
|||
106
database.py
106
database.py
|
|
@ -88,6 +88,53 @@ class Database:
|
|||
)
|
||||
""")
|
||||
|
||||
# Pages table for full-text search
|
||||
await db.execute("""
|
||||
CREATE TABLE IF NOT EXISTS pages (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
uri TEXT NOT NULL UNIQUE,
|
||||
path TEXT,
|
||||
title TEXT,
|
||||
content TEXT,
|
||||
crawl_job_id INTEGER,
|
||||
crawled_at TEXT NOT NULL,
|
||||
FOREIGN KEY (crawl_job_id) REFERENCES crawl_jobs(id)
|
||||
)
|
||||
""")
|
||||
|
||||
# FTS5 virtual table for page search
|
||||
await db.execute("""
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS pages_fts USING fts5(
|
||||
title, content, uri, path,
|
||||
content='pages',
|
||||
content_rowid='id'
|
||||
)
|
||||
""")
|
||||
|
||||
# Triggers to keep FTS in sync
|
||||
await db.execute("""
|
||||
CREATE TRIGGER IF NOT EXISTS pages_ai AFTER INSERT ON pages BEGIN
|
||||
INSERT INTO pages_fts(rowid, title, content, uri, path)
|
||||
VALUES (new.id, new.title, new.content, new.uri, new.path);
|
||||
END
|
||||
""")
|
||||
|
||||
await db.execute("""
|
||||
CREATE TRIGGER IF NOT EXISTS pages_ad AFTER DELETE ON pages BEGIN
|
||||
INSERT INTO pages_fts(pages_fts, rowid, title, content, uri, path)
|
||||
VALUES ('delete', old.id, old.title, old.content, old.uri, old.path);
|
||||
END
|
||||
""")
|
||||
|
||||
await db.execute("""
|
||||
CREATE TRIGGER IF NOT EXISTS pages_au AFTER UPDATE ON pages BEGIN
|
||||
INSERT INTO pages_fts(pages_fts, rowid, title, content, uri, path)
|
||||
VALUES ('delete', old.id, old.title, old.content, old.uri, old.path);
|
||||
INSERT INTO pages_fts(rowid, title, content, uri, path)
|
||||
VALUES (new.id, new.title, new.content, new.uri, new.path);
|
||||
END
|
||||
""")
|
||||
|
||||
# Indexes
|
||||
await db.execute("CREATE INDEX IF NOT EXISTS idx_media_type ON media(media_type)")
|
||||
await db.execute("CREATE INDEX IF NOT EXISTS idx_media_analysis ON media(analysis_status)")
|
||||
|
|
@ -95,6 +142,7 @@ class Database:
|
|||
await db.execute("CREATE INDEX IF NOT EXISTS idx_sources_job ON media_sources(crawl_job_id)")
|
||||
await db.execute("CREATE INDEX IF NOT EXISTS idx_sources_media_uri ON media_sources(media_uri)")
|
||||
await db.execute("CREATE INDEX IF NOT EXISTS idx_sources_page_uri ON media_sources(page_uri)")
|
||||
await db.execute("CREATE INDEX IF NOT EXISTS idx_pages_uri ON pages(uri)")
|
||||
|
||||
await db.commit()
|
||||
|
||||
|
|
@ -309,6 +357,64 @@ class Database:
|
|||
rows = await cursor.fetchall()
|
||||
return {row[0] for row in rows}
|
||||
|
||||
async def store_page(
|
||||
self,
|
||||
uri: str,
|
||||
title: str,
|
||||
content: str,
|
||||
path: str = "",
|
||||
crawl_job_id: int = None
|
||||
) -> None:
|
||||
"""Store a page for full-text search."""
|
||||
async with aiosqlite.connect(self.db_path) as db:
|
||||
await db.execute(
|
||||
"""
|
||||
INSERT OR REPLACE INTO pages (uri, path, title, content, crawl_job_id, crawled_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(uri, path, title, content[:100000], crawl_job_id, datetime.now(timezone.utc).isoformat())
|
||||
)
|
||||
await db.commit()
|
||||
|
||||
async def search_pages(self, query: str, limit: int = 50) -> List[Dict[str, Any]]:
|
||||
"""Search pages using FTS5 with LIKE fallback."""
|
||||
async with aiosqlite.connect(self.db_path) as db:
|
||||
db.row_factory = aiosqlite.Row
|
||||
results = []
|
||||
|
||||
# Try FTS5 with prefix matching
|
||||
try:
|
||||
fts_query = ' '.join(f'"{word}"*' for word in query.split())
|
||||
cursor = await db.execute("""
|
||||
SELECT p.uri, p.path, p.title,
|
||||
snippet(pages_fts, 1, '<mark>', '</mark>', '...', 40) as snippet
|
||||
FROM pages_fts
|
||||
JOIN pages p ON pages_fts.rowid = p.id
|
||||
WHERE pages_fts MATCH ?
|
||||
ORDER BY rank
|
||||
LIMIT ?
|
||||
""", (fts_query, limit))
|
||||
results = [dict(row) for row in await cursor.fetchall()]
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Fallback to LIKE
|
||||
if not results:
|
||||
try:
|
||||
like_q = f'%{query}%'
|
||||
cursor = await db.execute("""
|
||||
SELECT uri, path, title, substr(content, 1, 200) as snippet
|
||||
FROM pages
|
||||
WHERE title LIKE ? COLLATE NOCASE
|
||||
OR content LIKE ? COLLATE NOCASE
|
||||
LIMIT ?
|
||||
""", (like_q, like_q, limit))
|
||||
results = [dict(row) for row in await cursor.fetchall()]
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return results
|
||||
|
||||
async def get_stats(self) -> Dict[str, Any]:
|
||||
"""Get database statistics."""
|
||||
async with aiosqlite.connect(self.db_path) as db:
|
||||
|
|
|
|||
58
neopig.py
58
neopig.py
|
|
@ -212,21 +212,60 @@ class NeoPig:
|
|||
self._domain_stats[domain] = {'pages_changed': 0, 'media_new': 0, 'screenshots_new': 0}
|
||||
self._domain_stats[domain][stat] += increment
|
||||
|
||||
def _extract_text_from_html(self, html: str) -> tuple:
|
||||
"""Extract title and text content from HTML for search indexing."""
|
||||
try:
|
||||
from bs4 import BeautifulSoup
|
||||
soup = BeautifulSoup(html, 'html.parser')
|
||||
|
||||
# Extract title
|
||||
title = ''
|
||||
title_tag = soup.find('title')
|
||||
if title_tag:
|
||||
title = title_tag.get_text(strip=True)
|
||||
|
||||
# Remove script and style elements
|
||||
for tag in soup(['script', 'style', 'nav', 'header', 'footer']):
|
||||
tag.decompose()
|
||||
|
||||
# Get text content
|
||||
text = soup.get_text(separator=' ', strip=True)
|
||||
# Clean up whitespace
|
||||
import re
|
||||
text = re.sub(r'\s+', ' ', text)
|
||||
|
||||
return title, text[:100000]
|
||||
except Exception:
|
||||
return '', ''
|
||||
|
||||
async def _archive_page_to_vault(
|
||||
self,
|
||||
url: str,
|
||||
uri: str,
|
||||
html: str,
|
||||
media_mappings: Dict[str, str] = None,
|
||||
crawl_job_id: int = None,
|
||||
):
|
||||
"""Archive a page to the HTML vault."""
|
||||
domain = self._get_domain(url)
|
||||
"""Archive a page to the HTML vault and store for search."""
|
||||
domain = self._get_domain(uri)
|
||||
html_vault = self.domain_vaults.get_html_vault(domain)
|
||||
is_changed, _ = await html_vault.archive_page(url, html, media_mappings)
|
||||
is_changed, _ = await html_vault.archive_page(uri, html, media_mappings)
|
||||
self.stats['bytes_downloaded'] += len(html.encode('utf-8'))
|
||||
if is_changed:
|
||||
self._track_domain_stat(domain, 'pages_changed')
|
||||
self.stats['pages_changed'] += 1
|
||||
|
||||
# Store page content for full-text search
|
||||
title, content = self._extract_text_from_html(html)
|
||||
parsed = urlparse(uri)
|
||||
path = parsed.path or '/'
|
||||
await self.db.store_page(
|
||||
uri=uri,
|
||||
title=title,
|
||||
content=content,
|
||||
path=path,
|
||||
crawl_job_id=crawl_job_id,
|
||||
)
|
||||
|
||||
async def _archive_media_to_vault(
|
||||
self,
|
||||
url: str,
|
||||
|
|
@ -424,17 +463,16 @@ class NeoPig:
|
|||
|
||||
# Page callback - archive raw HTML to vault and capture screenshot
|
||||
# Screenshot happens here (same crawl delay window as page fetch)
|
||||
async def on_page_fetched(url: str, html: str):
|
||||
async def on_page_fetched(uri: str, html: str):
|
||||
# Track this page as crawled for resume support
|
||||
self.seen_pages.add(url)
|
||||
self.seen_pages.add(uri)
|
||||
|
||||
# For now, archive without media URL rewriting (we'd need to download media first)
|
||||
# TODO: Build media_mappings after media is downloaded
|
||||
await self._archive_page_to_vault(url, html, media_mappings=None)
|
||||
# Archive to vault and store for search (with job_id for tracking)
|
||||
await self._archive_page_to_vault(uri, html, media_mappings=None, crawl_job_id=job_id)
|
||||
|
||||
# Capture screenshot for every page (honors crawl delay as a UNIT with page fetch)
|
||||
if self.screenshot_config.enabled:
|
||||
await self._capture_page_screenshot(url, job_id, page_title='')
|
||||
await self._capture_page_screenshot(uri, job_id, page_title='')
|
||||
|
||||
# Save state periodically for resume support
|
||||
self._save_state(target_uri)
|
||||
|
|
|
|||
|
|
@ -30,8 +30,8 @@ logger = logging.getLogger(__name__)
|
|||
|
||||
# Engine preference order - lightest/fastest first
|
||||
ENGINE_PREFERENCE = [
|
||||
'wkhtmltoimage', # Native Qt WebKit - very fast, no browser download
|
||||
'cutycapt', # Native Qt WebKit - very fast, no browser download
|
||||
'cutycapt', # Native Qt WebKit - fast, supports full-page screenshots
|
||||
'wkhtmltoimage', # Native Qt WebKit - fast, viewport-only
|
||||
'playwright-webkit', # WebKit via Playwright - lighter than Chromium
|
||||
'playwright-firefox',
|
||||
'playwright-chromium',
|
||||
|
|
|
|||
216
serp.py
216
serp.py
|
|
@ -210,6 +210,13 @@ SEARCH_HTML = """
|
|||
font-size: 14px;
|
||||
color: #888;
|
||||
}
|
||||
.section-title {
|
||||
color: #ff6b6b;
|
||||
font-size: 18px;
|
||||
margin: 25px 0 15px 0;
|
||||
border-bottom: 1px solid #333;
|
||||
padding-bottom: 8px;
|
||||
}
|
||||
.results {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fill, minmax(250px, 1fr));
|
||||
|
|
@ -276,14 +283,56 @@ SEARCH_HTML = """
|
|||
.result-hash:hover {
|
||||
text-decoration: underline;
|
||||
}
|
||||
/* Page results */
|
||||
.page-results {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 10px;
|
||||
}
|
||||
.page-result {
|
||||
background: #1a1a1a;
|
||||
border-radius: 8px;
|
||||
padding: 15px;
|
||||
transition: background 0.2s;
|
||||
}
|
||||
.page-result:hover {
|
||||
background: #252525;
|
||||
}
|
||||
.page-result a {
|
||||
color: #ff6b6b;
|
||||
text-decoration: none;
|
||||
font-size: 16px;
|
||||
font-weight: 500;
|
||||
}
|
||||
.page-result a:hover {
|
||||
text-decoration: underline;
|
||||
}
|
||||
.page-path {
|
||||
font-size: 12px;
|
||||
color: #4ade80;
|
||||
margin-top: 4px;
|
||||
font-family: monospace;
|
||||
}
|
||||
.page-snippet {
|
||||
font-size: 13px;
|
||||
color: #999;
|
||||
margin-top: 8px;
|
||||
line-height: 1.5;
|
||||
}
|
||||
.page-snippet mark {
|
||||
background: #ff6b6b33;
|
||||
color: #ff9999;
|
||||
padding: 1px 3px;
|
||||
border-radius: 2px;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<h1>🐷 neopig</h1>
|
||||
<p class="subtitle">Search hydrated media</p>
|
||||
<h1>neopig</h1>
|
||||
<p class="subtitle">Search hydrated media and pages</p>
|
||||
|
||||
<div class="search-box">
|
||||
<input type="text" id="query" placeholder="Search keywords, alt text, titles..." autofocus>
|
||||
<input type="text" id="query" placeholder="Search keywords, alt text, page content..." autofocus>
|
||||
<select id="type">
|
||||
<option value="">All types</option>
|
||||
<option value="image">Images</option>
|
||||
|
|
@ -294,61 +343,96 @@ SEARCH_HTML = """
|
|||
</div>
|
||||
|
||||
<div class="stats" id="stats">Loading stats...</div>
|
||||
<div class="results" id="results"></div>
|
||||
|
||||
<div id="page-section" style="display:none;">
|
||||
<h2 class="section-title">Pages</h2>
|
||||
<div class="page-results" id="page-results"></div>
|
||||
</div>
|
||||
|
||||
<div id="media-section">
|
||||
<h2 class="section-title">Media</h2>
|
||||
<div class="results" id="results"></div>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
async function loadStats() {
|
||||
const res = await fetch('/api/stats');
|
||||
const stats = await res.json();
|
||||
document.getElementById('stats').innerHTML =
|
||||
`<strong>${stats.total_media}</strong> media indexed | ` +
|
||||
`<strong>${stats.total_media}</strong> media | ` +
|
||||
`<strong>${stats.by_type?.image || 0}</strong> images | ` +
|
||||
`<strong>${stats.by_type?.video || 0}</strong> videos | ` +
|
||||
`<strong>${stats.total_sources}</strong> source URLs`;
|
||||
`<strong>${stats.total_sources}</strong> sources | ` +
|
||||
`<strong>${stats.total_pages || 0}</strong> pages`;
|
||||
}
|
||||
|
||||
async function search() {
|
||||
const query = document.getElementById('query').value;
|
||||
const type = document.getElementById('type').value;
|
||||
|
||||
let url = `/api/search?q=${encodeURIComponent(query)}&limit=100`;
|
||||
if (type) url += `&type=${type}`;
|
||||
// Search media
|
||||
let mediaUrl = `/api/search?q=${encodeURIComponent(query)}&limit=100`;
|
||||
if (type) mediaUrl += `&type=${type}`;
|
||||
|
||||
const res = await fetch(url);
|
||||
const results = await res.json();
|
||||
const mediaRes = await fetch(mediaUrl);
|
||||
const mediaResults = await mediaRes.json();
|
||||
|
||||
const container = document.getElementById('results');
|
||||
const mediaContainer = document.getElementById('results');
|
||||
const mediaSection = document.getElementById('media-section');
|
||||
|
||||
if (results.length === 0) {
|
||||
container.innerHTML = '<div class="no-results">No results found</div>';
|
||||
return;
|
||||
if (mediaResults.length === 0) {
|
||||
mediaContainer.innerHTML = '<div class="no-results">No media found</div>';
|
||||
} else {
|
||||
mediaContainer.innerHTML = mediaResults.map(r => {
|
||||
const isVideo = r.media_type === 'video';
|
||||
const mediaEl = isVideo
|
||||
? `<video src="/media/${r.md5_hash}" controls preload="metadata"></video>`
|
||||
: `<img src="/media/${r.md5_hash}" alt="${r.alt_text || ''}" loading="lazy">`;
|
||||
|
||||
const keywords = JSON.parse(r.keywords || '[]');
|
||||
const tagsHtml = keywords.map(k => `<span class="tag">${k}</span>`).join('');
|
||||
|
||||
return `
|
||||
<div class="result">
|
||||
<a href="/media/${r.md5_hash}" target="_blank" class="media-link">
|
||||
${mediaEl}
|
||||
</a>
|
||||
<div class="result-info">
|
||||
<a href="/view/${r.md5_hash}" class="result-hash">${r.md5_hash}</a>
|
||||
<div class="result-meta">
|
||||
${r.media_type} · ${formatBytes(r.file_size)}
|
||||
${r.alt_text ? ` · ${r.alt_text.substring(0, 50)}` : ''}
|
||||
</div>
|
||||
<div class="result-keywords">${tagsHtml}</div>
|
||||
</div>
|
||||
</div>
|
||||
`;
|
||||
}).join('');
|
||||
}
|
||||
|
||||
container.innerHTML = results.map(r => {
|
||||
const isVideo = r.media_type === 'video';
|
||||
const mediaEl = isVideo
|
||||
? `<video src="/media/${r.md5_hash}" controls preload="metadata"></video>`
|
||||
: `<img src="/media/${r.md5_hash}" alt="${r.alt_text || ''}" loading="lazy">`;
|
||||
// Search pages (only if there's a query)
|
||||
const pageSection = document.getElementById('page-section');
|
||||
const pageContainer = document.getElementById('page-results');
|
||||
|
||||
const keywords = JSON.parse(r.keywords || '[]');
|
||||
const tagsHtml = keywords.map(k => `<span class="tag">${k}</span>`).join('');
|
||||
if (query.trim()) {
|
||||
const pageRes = await fetch(`/api/search/pages?q=${encodeURIComponent(query)}&limit=20`);
|
||||
const pageResults = await pageRes.json();
|
||||
|
||||
return `
|
||||
<div class="result">
|
||||
<a href="/media/${r.md5_hash}" target="_blank" class="media-link">
|
||||
${mediaEl}
|
||||
</a>
|
||||
<div class="result-info">
|
||||
<a href="/view/${r.md5_hash}" class="result-hash">${r.md5_hash}</a>
|
||||
<div class="result-meta">
|
||||
${r.media_type} · ${formatBytes(r.file_size)}
|
||||
${r.alt_text ? ` · ${r.alt_text.substring(0, 50)}` : ''}
|
||||
</div>
|
||||
<div class="result-keywords">${tagsHtml}</div>
|
||||
if (pageResults.length > 0) {
|
||||
pageSection.style.display = 'block';
|
||||
pageContainer.innerHTML = pageResults.map(p => `
|
||||
<div class="page-result">
|
||||
<a href="${p.uri}" target="_blank">${p.title || p.uri}</a>
|
||||
<div class="page-path">${p.path || p.uri}</div>
|
||||
<div class="page-snippet">${p.snippet || ''}</div>
|
||||
</div>
|
||||
</div>
|
||||
`;
|
||||
}).join('');
|
||||
`).join('');
|
||||
} else {
|
||||
pageSection.style.display = 'none';
|
||||
}
|
||||
} else {
|
||||
pageSection.style.display = 'none';
|
||||
}
|
||||
}
|
||||
|
||||
function formatBytes(bytes) {
|
||||
|
|
@ -366,7 +450,7 @@ SEARCH_HTML = """
|
|||
// Load stats on page load
|
||||
loadStats();
|
||||
|
||||
// Initial search (show all)
|
||||
// Initial search (show all media)
|
||||
search();
|
||||
</script>
|
||||
</body>
|
||||
|
|
@ -1030,7 +1114,7 @@ async def view_media_page(md5_hash: str):
|
|||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<nav><a href="/">← Back to Search</a></nav>
|
||||
<nav><a href="javascript:history.back()">← Back</a></nav>
|
||||
<h1>{media.get('alt_text') or media.get('title') or 'Untitled'}</h1>
|
||||
|
||||
<div class="media-container">
|
||||
|
|
@ -1108,6 +1192,13 @@ async def get_stats():
|
|||
cursor = await db.execute("SELECT COUNT(*) FROM crawl_jobs")
|
||||
stats['total_jobs'] = (await cursor.fetchone())[0]
|
||||
|
||||
# Page count (table may not exist in older databases)
|
||||
try:
|
||||
cursor = await db.execute("SELECT COUNT(*) FROM pages")
|
||||
stats['total_pages'] = (await cursor.fetchone())[0]
|
||||
except Exception:
|
||||
stats['total_pages'] = 0
|
||||
|
||||
return stats
|
||||
|
||||
|
||||
|
|
@ -1167,6 +1258,55 @@ async def search(
|
|||
return [dict(row) for row in rows]
|
||||
|
||||
|
||||
@app.get("/api/search/pages")
|
||||
async def search_pages(
|
||||
q: str = Query("", description="Search query"),
|
||||
limit: int = Query(50, le=500),
|
||||
):
|
||||
"""
|
||||
Search pages by text query using FTS5.
|
||||
"""
|
||||
if not q:
|
||||
return []
|
||||
|
||||
async with aiosqlite.connect(DB_PATH) as db:
|
||||
db.row_factory = aiosqlite.Row
|
||||
results = []
|
||||
|
||||
# Try FTS5 with prefix matching
|
||||
try:
|
||||
fts_query = ' '.join(f'"{word}"*' for word in q.split())
|
||||
cursor = await db.execute("""
|
||||
SELECT p.uri, p.path, p.title,
|
||||
snippet(pages_fts, 1, '<mark>', '</mark>', '...', 40) as snippet
|
||||
FROM pages_fts
|
||||
JOIN pages p ON pages_fts.rowid = p.id
|
||||
WHERE pages_fts MATCH ?
|
||||
ORDER BY rank
|
||||
LIMIT ?
|
||||
""", (fts_query, limit))
|
||||
results = [dict(row) for row in await cursor.fetchall()]
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Fallback to LIKE
|
||||
if not results:
|
||||
try:
|
||||
like_q = f'%{q}%'
|
||||
cursor = await db.execute("""
|
||||
SELECT uri, path, title, substr(content, 1, 200) as snippet
|
||||
FROM pages
|
||||
WHERE title LIKE ? COLLATE NOCASE
|
||||
OR content LIKE ? COLLATE NOCASE
|
||||
LIMIT ?
|
||||
""", (like_q, like_q, limit))
|
||||
results = [dict(row) for row in await cursor.fetchall()]
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return results
|
||||
|
||||
|
||||
@app.get("/api/media/{md5_hash}")
|
||||
async def get_media_info(md5_hash: str):
|
||||
"""Get full media info including all source URLs."""
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue