diff --git a/CLAUDE.md b/CLAUDE.md index 9a31844..c54fb5a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -96,7 +96,7 @@ c.delete_node(node_id) # permanent, moderator only - **Identity**: Authenticated as `timehexon@unturf.com` (display name: `timehexon`) - **Journey thread**: `9f970183-ffaf-11f0-b565-040140774501` on `meta.remarkbox.com` -- update this after finishing work - **Content limit**: 500,000 characters (~128k tokens) -- **Rate limits**: 120 reads/min, 30 writes/min (wait if you hit 429) +- **Rate limits**: 120 reads/min, 30 writes/min, 1 thread creation per 7 min (wait if you hit 429) ### Endpoints @@ -105,7 +105,7 @@ c.delete_node(node_id) # permanent, moderator only | GET | `/api/v1/version` | Deployed git commit hash | | GET | `/api/v1/threads?namespace=X` | List threads | | GET | `/api/v1/threads/{id}` | Thread with replies | -| POST | `/api/v1/threads` | Create thread | +| POST | `/api/v1/threads` | Create thread (1 per 7 min limit) | | POST | `/api/v1/threads/{id}/replies` | Reply to thread | | GET | `/api/v1/nodes/{id}` | Single node | | PATCH | `/api/v1/nodes/{id}` | Edit node (data, title, disabled, approved, locked) | @@ -115,6 +115,8 @@ c.delete_node(node_id) # permanent, moderator only | GET | `/api/v1/user/profile` | Get profile | | PATCH | `/api/v1/user/profile` | Update display name | | GET | `/api/v1/clients/python` | Download Python client | +| GET | `/api/v1/admin/namespaces` | List all namespaces (superuser only) | +| GET | `/api/v1/admin/recent-nodes?days=7` | Recent nodes network-wide (superuser only) | ### Authentication @@ -136,6 +138,56 @@ env/bin/python remarkbox/api/functional_test.py https://my.remarkbox.com meta.re This exercises all endpoints and updates the journey thread with results. +## Superuser (Global Moderator) + +Users with `is_superuser=True` can moderate across all namespaces. This bypasses the +normal namespace-scoped `is_moderator()` check. The admin UI is at `/topsecret/users` +where you can promote/demote users by email. + +Bootstrap the first superuser via the database script: +```bash +env/bin/python scripts/promote_superuser.py --ini development.ini --email timehexon@unturf.com +``` + +After that, use the web UI at `/topsecret/users` or the existing topsecret admin pages +(all guarded by `@super_fly_required` which checks `is_superuser`). + +Admin client methods: +```python +c.admin_list_namespaces() # list all namespaces +c.admin_recent_nodes(days=7) # recent nodes network-wide +``` + +## Spam Prevention + +Spam detection runs automatically on `POST /api/v1/threads` and `POST /api/v1/threads/{id}/replies`. +Superusers bypass all spam checks. + +**Scoring signals**: link density, known spam patterns, duplicate content, new account velocity, +IP reputation (disabled post count), content length anomalies. + +**Thresholds** (configurable in `.ini`): +- `spam.hard_threshold = 0.8` -- reject with 403 +- `spam.soft_threshold = 0.5` -- allow but set `approved=False` (held for moderation) + +**Thread creation rate limit**: 1 new thread per 7 minutes per user/IP via the API. +This does not affect browser users or replies. + +**Spam hunting scripts** (require superuser cookie): +```bash +# Scan recent posts for spam +python scripts/spam/scan.py --days=7 --threshold=0.3 + +# Scan and output JSON +python scripts/spam/scan.py --json --threshold=0.5 + +# Bulk disable flagged posts +python scripts/spam/scan.py --json --threshold=0.8 | python scripts/spam/disable_spam.py --from-json + +# Disable specific nodes +python scripts/spam/disable_spam.py node-uuid-1 node-uuid-2 +``` + ## Production Rules **NEVER run direct SQL or raw database commands on production.** No `sqlite3`, no `UPDATE`, no `DELETE`, no direct file edits on the production database. Ever. If the API doesn't support what you need, add the endpoint first, push it, then use the client. diff --git a/development.ini b/development.ini index c39d85b..55e682c 100644 --- a/development.ini +++ b/development.ini @@ -46,6 +46,19 @@ api.enabled = true api.rate_limit.read_requests = 120 api.rate_limit.write_requests = 30 api.rate_limit.window = 60 +api.rate_limit.create_thread_requests = 1 +api.rate_limit.create_thread_window = 420 + +### +# spam detection. +### +spam.enabled = true +spam.hard_threshold = 0.8 +spam.soft_threshold = 0.5 +spam.llm.enabled = true +spam.llm.endpoint = https://hermes.ai.unturf.com/v1/chat/completions +spam.llm.model = adamo1139/Hermes-3-Llama-3.1-8B-FP8-Dynamic +spam.llm.timeout = 5 ### # custom app configuration. diff --git a/remarkbox/api/__init__.py b/remarkbox/api/__init__.py index 11c7737..cb132db 100644 --- a/remarkbox/api/__init__.py +++ b/remarkbox/api/__init__.py @@ -10,4 +10,6 @@ def includeme(config): config.add_route("api-user-profile", "/api/v1/user/profile") config.add_route("api-client-python", "/api/v1/clients/python") config.add_route("api-webmention", "/api/v1/webmention") + config.add_route("api-admin-namespaces", "/api/v1/admin/namespaces") + config.add_route("api-admin-recent-nodes", "/api/v1/admin/recent-nodes") config.scan("remarkbox.api.views") diff --git a/remarkbox/api/rate_limit.py b/remarkbox/api/rate_limit.py index 5d8bb16..1ce8900 100644 --- a/remarkbox/api/rate_limit.py +++ b/remarkbox/api/rate_limit.py @@ -14,15 +14,20 @@ def rate_limit_tween_factory(handler, registry): api.rate_limit.read_requests = 120 api.rate_limit.write_requests = 30 api.rate_limit.window = 60 + api.rate_limit.create_thread_requests = 1 + api.rate_limit.create_thread_window = 420 """ settings = registry.settings api_enabled = settings.get("api.enabled", "true").strip().lower() in ("true", "1", "yes") read_limit = int(settings.get("api.rate_limit.read_requests", 120)) write_limit = int(settings.get("api.rate_limit.write_requests", 30)) window = int(settings.get("api.rate_limit.window", 60)) + create_thread_limit = int(settings.get("api.rate_limit.create_thread_requests", 5)) + create_thread_window = int(settings.get("api.rate_limit.create_thread_window", 3600)) # In-memory storage: {key: [timestamp, ...]} request_log = defaultdict(list) + create_thread_log = defaultdict(list) def rate_limit_tween(request): if not request.path.startswith("/api/v1/"): @@ -58,6 +63,29 @@ def rate_limit_tween_factory(handler, registry): content_type="application/json", ) + # Stricter limit for thread creation to prevent spam floods + is_create_thread = ( + request.method == "POST" and request.path == "/api/v1/threads" + ) + if is_create_thread: + ct_cutoff = now - create_thread_window + create_thread_log[key] = [ + t for t in create_thread_log[key] if t > ct_cutoff + ] + if len(create_thread_log[key]) >= create_thread_limit: + retry_after = int( + create_thread_log[key][0] + create_thread_window - now + ) + 1 + return Response( + json_body={ + "error": "Thread creation rate limit exceeded", + "retry_after": retry_after, + }, + status=429, + content_type="application/json", + ) + create_thread_log[key].append(now) + request_log[key].append(now) return handler(request) diff --git a/remarkbox/api/remarkbox_client.py b/remarkbox/api/remarkbox_client.py index ccf25f0..2ea31d8 100644 --- a/remarkbox/api/remarkbox_client.py +++ b/remarkbox/api/remarkbox_client.py @@ -421,6 +421,29 @@ class RemarkboxClient: """ return self._request("PATCH", "/api/v1/user/profile", {"name": name}) + # ----- Admin (superuser only) ----- + + def admin_list_namespaces(self): + """List all namespaces (requires superuser). + + Returns: + dict with key: namespaces (list of namespace dicts) + """ + return self._request("GET", "/api/v1/admin/namespaces") + + def admin_recent_nodes(self, days=7, limit=100): + """List recent nodes across all namespaces (requires superuser). + + Args: + days: Number of days to look back (default 7, max 90) + limit: Max results (default 100, max 500) + + Returns: + dict with keys: days, count, nodes + """ + params = urllib.parse.urlencode({"days": days, "limit": limit}) + return self._request("GET", "/api/v1/admin/recent-nodes?" + params) + # ----- CLI ----- diff --git a/remarkbox/api/views.py b/remarkbox/api/views.py index e660081..dfa973d 100644 --- a/remarkbox/api/views.py +++ b/remarkbox/api/views.py @@ -17,11 +17,15 @@ from remarkbox.models.user import ( is_user_name_valid, is_user_name_available, ) -from remarkbox.models.namespace import get_or_create_namespace +from remarkbox.models.namespace import get_or_create_namespace, get_topsecret_namespaces +from remarkbox.models.node import Node from remarkbox.lib.mail import send_verification_digits_to_email from remarkbox.lib.notify import schedule_notifications from remarkbox.views import verify_pending_nodes_in_session +from remarkbox.models.spam import score_content +from remarkbox.models.spam_llm import check_thread_relevance, check_reply_relevance + from .serializers import serialize_node, serialize_namespace_brief MAX_CONTENT_LENGTH = 500000 @@ -53,6 +57,95 @@ def get_param(request, key, default=None): return request.params.get(key, default) +def check_spam(request, data, user=None, namespace=None, title=None, + parent_node=None): + """Check content for spam. Returns error dict or None. + + - Hard threshold: reject with 403. + - Soft threshold: returns {"spam_held": True} so caller can set approved=False. + - LLM relevance check: if enabled and basic score is suspicious, ask the LLM + whether the content is relevant to its context. + - Superusers bypass all checks. + """ + settings = request.registry.settings + if settings.get("spam.enabled", "true").strip().lower() not in ("true", "1", "yes"): + return None + + if user and getattr(user, "is_superuser", False): + return None + + hard_threshold = float(settings.get("spam.hard_threshold", 0.8)) + soft_threshold = float(settings.get("spam.soft_threshold", 0.5)) + + spam_score, signals = score_content( + data, + user=user, + ip_address=str(request.client_addr), + dbsession=request.dbsession, + settings=settings, + ) + + # LLM relevance check: runs on every message when spam.llm.enabled is set + if settings.get("spam.llm.enabled", "false").strip().lower() in ("true", "1", "yes"): + relevant, explanation = _llm_relevance_check( + settings, data, title=title, namespace=namespace, + parent_node=parent_node, + ) + if relevant is False: + spam_score = min(spam_score + 0.4, 1.0) + signals.append("llm_irrelevant") + if explanation: + signals.append("llm_reason:{}".format(explanation[:80])) + + if spam_score >= hard_threshold: + request.response.status_code = 403 + return {"error": "Content flagged as spam", "spam_score": spam_score} + + if spam_score >= soft_threshold: + return {"spam_held": True, "spam_score": spam_score, "signals": signals} + + return None + + +def _llm_relevance_check(settings, data, title=None, namespace=None, + parent_node=None): + """Run LLM relevance check if enabled. Returns (relevant, explanation). + + For replies, includes the parent page URL (embed mode) so the LLM knows + what the parent site is about. + """ + if parent_node is not None: + # Reply: check relevance to thread + root = parent_node.root if parent_node else None + # In embed mode, root.uri.data has the parent page URL + page_url = None + if root and getattr(root, "has_uri", False) and root.uri: + page_url = root.uri.data + ns_name = namespace.name if namespace else None + return check_reply_relevance( + thread_title=root.title if root else None, + thread_content=root.data if root else None, + parent_content=parent_node.data if parent_node else None, + reply_content=data, + page_url=page_url, + namespace_name=ns_name, + settings=settings, + ) + elif namespace is not None: + # New thread: check relevance to namespace + ns_desc = None + if hasattr(namespace, "description"): + ns_desc = namespace.description + return check_thread_relevance( + namespace_name=namespace.name, + namespace_description=ns_desc, + title=title, + content=data, + settings=settings, + ) + return None, None + + def _get_git_commit(): """Read the current git commit hash, once at import time. @@ -384,6 +477,13 @@ def api_create_thread(request): if not user and email: user = get_or_create_user_by_email(request.dbsession, email) + # Spam check (before creating anything) + spam_result = check_spam(request, data, user=user, namespace=namespace, + title=title) + if spam_result and "error" in spam_result: + return spam_result + spam_held = spam_result and spam_result.get("spam_held") + if namespace.allow_anonymous and not user: if not anonymous_name: anonymous_name = "Anonymous" @@ -412,6 +512,9 @@ def api_create_thread(request): node_event = node.new_event(user, "created") request.dbsession.add(user) + if spam_held: + node.approved = False + request.dbsession.add(node) if node_event: request.dbsession.add(node_event) @@ -496,6 +599,13 @@ def api_reply(request): if not user and email: user = get_or_create_user_by_email(request.dbsession, email) + # Spam check (before creating anything) + spam_result = check_spam(request, data, user=user, namespace=namespace, + parent_node=parent) + if spam_result and "error" in spam_result: + return spam_result + spam_held = spam_result and spam_result.get("spam_held") + if namespace.allow_anonymous and not user: if not anonymous_name: anonymous_name = "Anonymous" @@ -521,7 +631,9 @@ def api_reply(request): child.verified = user.authenticated child_event = child.new_event(user, "commented") - if namespace.hide_unless_approved: + if spam_held: + child.approved = False + elif namespace.hide_unless_approved: if user: child.approved = namespace.is_moderator(user) else: @@ -864,3 +976,110 @@ def api_client_python(request): content_type="text/plain", charset="utf-8", ) + + +# --------------------------------------------------------------------------- +# Admin (superuser only) +# --------------------------------------------------------------------------- + + +def _require_superuser(request): + """Return error dict if user is not a superuser, else None.""" + if not request.user or not request.user.authenticated: + request.response.status_code = 401 + return {"error": "Authentication required"} + if not getattr(request.user, "is_superuser", False): + request.response.status_code = 403 + return {"error": "Superuser access required"} + return None + + +@view_config( + route_name="api-admin-namespaces", + request_method="GET", + renderer="json", + require_csrf=False, +) +def api_admin_namespaces(request): + """List all namespaces (superuser only).""" + denied = _require_superuser(request) + if denied: + return denied + + namespaces = get_topsecret_namespaces(request.dbsession) + return { + "namespaces": [ + { + "id": str(ns.id), + "name": ns.name, + "subscription_type": ns.subscription_type, + "owner_count": len(ns.owners), + "root_count": ns.roots.count(), + } + for ns in namespaces + ], + } + + +@view_config( + route_name="api-admin-recent-nodes", + request_method="GET", + renderer="json", + require_csrf=False, +) +def api_admin_recent_nodes(request): + """List recent nodes across all namespaces (superuser only). + + Query parameters: + days: Number of days to look back (default 7, max 90). + limit: Max results (default 100, max 500). + """ + denied = _require_superuser(request) + if denied: + return denied + + import time + try: + days = int(get_param(request, "days", 7)) + except (TypeError, ValueError): + days = 7 + days = max(1, min(days, 90)) + + try: + limit = int(get_param(request, "limit", 100)) + except (TypeError, ValueError): + limit = 100 + limit = max(1, min(limit, 500)) + + cutoff_ms = int((time.time() - days * 86400) * 1000) + + nodes = ( + request.dbsession.query(Node) + .filter(Node.created > cutoff_ms) + .order_by(Node.created.desc()) + .limit(limit) + .all() + ) + + return { + "days": days, + "count": len(nodes), + "nodes": [ + { + "id": str(n.id), + "root_id": str(n.root_id) if n.root_id else None, + "namespace": n.root.namespace.name if n.root and n.root.namespace else None, + "title": n.title, + "data": n.data[:200] if n.data else None, + "ip_address": n.ip_address, + "created_ago": n.human_created_timestamp, + "disabled": n.disabled, + "verified": n.verified, + "approved": n.approved, + "author": n.user.name if n.user else ( + n.user_surrogate.name if n.user_surrogate else None + ), + } + for n in nodes + ], + } diff --git a/remarkbox/models/namespace.py b/remarkbox/models/namespace.py index c4dff4a..5a2dd23 100644 --- a/remarkbox/models/namespace.py +++ b/remarkbox/models/namespace.py @@ -371,9 +371,15 @@ class Namespace(RBase, Base): return False def is_moderator(self, user): - """Return True if given user moderates this namespace, else False.""" - if user and user.authenticated and user in self.moderators: - return True + """Return True if given user moderates this namespace, else False. + + Superusers are treated as moderators on every namespace. + """ + if user and user.authenticated: + if getattr(user, "is_superuser", False): + return True + if user in self.moderators: + return True return False def can_alter_node(self, node, user): diff --git a/remarkbox/models/spam.py b/remarkbox/models/spam.py new file mode 100644 index 0000000..6965748 --- /dev/null +++ b/remarkbox/models/spam.py @@ -0,0 +1,221 @@ +""" +Spam scoring for Remarkbox. + +Scores content on a 0.0-1.0 scale where higher = more likely spam. +Used by API views to reject or hold posts for moderation. +""" + +import hashlib +import os +import re +import time +from collections import defaultdict + +from .node import Node + +# In-memory caches (reset on process restart) +_recent_hashes = defaultdict(list) # {ip_or_user_key: [(hash, timestamp), ...]} +_disabled_ip_counts = {} # {ip: count} -- refreshed periodically +_disabled_ip_cache_time = 0 + +# Default spam patterns (common in comment spam) +DEFAULT_PATTERNS = [ + r"buy\s+now", + r"click\s+here\s+to", + r"free\s+trial", + r"limited\s+time\s+offer", + r"act\s+now", + r"order\s+today", + r"100%\s+free", + r"make\s+money\s+fast", + r"work\s+from\s+home", + r"casino\s+online", + r"viagra|cialis", + r"payday\s+loan", + r"seo\s+service", + r"followers?\s+for\s+(free|sale|\$)", + r"crypto\s+invest", +] + +_compiled_patterns = None +_custom_patterns = None + + +def _get_patterns(settings=None): + """Load and compile spam patterns. Cached after first call.""" + global _compiled_patterns, _custom_patterns + + patterns_file = None + if settings: + patterns_file = settings.get("spam.patterns_file") + + if _compiled_patterns is not None and _custom_patterns == patterns_file: + return _compiled_patterns + + patterns = list(DEFAULT_PATTERNS) + + if patterns_file and os.path.exists(patterns_file): + with open(patterns_file) as f: + for line in f: + line = line.strip() + if line and not line.startswith("#"): + patterns.append(line) + + _compiled_patterns = [re.compile(p, re.IGNORECASE) for p in patterns] + _custom_patterns = patterns_file + return _compiled_patterns + + +def _content_hash(text): + """Return a short hash of the text for duplicate detection.""" + normalized = re.sub(r"\s+", " ", text.strip().lower()) + return hashlib.md5(normalized.encode("utf-8")).hexdigest()[:16] + + +def _link_density(text): + """Return the ratio of URL characters to total text length.""" + if not text: + return 0.0 + urls = re.findall(r"https?://\S+", text) + url_chars = sum(len(u) for u in urls) + return url_chars / len(text) if text else 0.0 + + +def _link_count(text): + """Return number of URLs in text.""" + if not text: + return 0 + return len(re.findall(r"https?://\S+", text)) + + +def score_content(text, user=None, ip_address=None, dbsession=None, settings=None): + """Score content for spam likelihood. + + Args: + text: The post content. + user: User object (may be None for anonymous). + ip_address: Client IP address string. + dbsession: SQLAlchemy session (for IP reputation checks). + settings: Pyramid registry settings dict. + + Returns: + (score, signals) where score is 0.0-1.0 and signals is a list + of strings describing what triggered. + """ + if not text: + return 0.0, [] + + signals = [] + score = 0.0 + + # 1. Link density + density = _link_density(text) + links = _link_count(text) + if density > 0.5: + score += 0.4 + signals.append("link_density:{:.0%}".format(density)) + elif density > 0.3: + score += 0.2 + signals.append("link_density:{:.0%}".format(density)) + if links > 5: + score += 0.2 + signals.append("link_count:{}".format(links)) + + # 2. Known spam patterns + patterns = _get_patterns(settings) + pattern_hits = 0 + for pattern in patterns: + if pattern.search(text): + pattern_hits += 1 + if pattern_hits >= 3: + score += 0.5 + signals.append("spam_patterns:{}".format(pattern_hits)) + elif pattern_hits >= 1: + score += 0.2 + signals.append("spam_patterns:{}".format(pattern_hits)) + + # 3. Duplicate content (in-memory, recent posts by same IP/user) + key = None + if user and hasattr(user, "id"): + key = "user:{}".format(user.id) + elif ip_address: + key = "ip:{}".format(ip_address) + + if key: + content_hash = _content_hash(text) + now = time.time() + cutoff = now - 3600 # Look back 1 hour + + # Clean old entries + _recent_hashes[key] = [ + (h, t) for h, t in _recent_hashes[key] if t > cutoff + ] + + # Check for duplicates + existing_hashes = [h for h, t in _recent_hashes[key]] + if content_hash in existing_hashes: + score += 0.4 + signals.append("duplicate_content") + + # Record this hash + _recent_hashes[key].append((content_hash, now)) + + # 4. New account velocity + if user and hasattr(user, "created"): + now_ms = int(time.time() * 1000) + account_age_ms = now_ms - user.created + one_hour_ms = 3600000 + + if account_age_ms < one_hour_ms: + # Account less than 1 hour old + node_count = user.nodes.count() if hasattr(user.nodes, "count") else 0 + if node_count > 5: + score += 0.3 + signals.append("new_account_velocity:{}posts_in_{}min".format( + node_count, account_age_ms // 60000 + )) + elif node_count > 2: + score += 0.1 + signals.append("new_account_velocity:{}posts".format(node_count)) + + # 5. IP reputation (disabled posts from same IP) + if ip_address and dbsession: + global _disabled_ip_counts, _disabled_ip_cache_time + now = time.time() + + # Refresh IP reputation cache every 5 minutes + if now - _disabled_ip_cache_time > 300: + _disabled_ip_counts = {} + _disabled_ip_cache_time = now + + if ip_address not in _disabled_ip_counts: + count = ( + dbsession.query(Node) + .filter(Node.ip_address == ip_address, Node.disabled == True) + .count() + ) + _disabled_ip_counts[ip_address] = count + + disabled_count = _disabled_ip_counts[ip_address] + if disabled_count > 3: + score += 0.3 + signals.append("ip_reputation:{}disabled".format(disabled_count)) + + # 6. Content length anomalies (from new users) + is_new_user = False + if user and hasattr(user, "created"): + now_ms = int(time.time() * 1000) + is_new_user = (now_ms - user.created) < 86400000 # < 1 day + + if is_new_user: + if len(text) < 10: + score += 0.1 + signals.append("very_short_content") + elif len(text) > 50000: + score += 0.2 + signals.append("very_long_content_new_user") + + # Cap at 1.0 + score = min(score, 1.0) + + return score, signals diff --git a/remarkbox/models/spam_llm.py b/remarkbox/models/spam_llm.py new file mode 100644 index 0000000..7aa2d4f --- /dev/null +++ b/remarkbox/models/spam_llm.py @@ -0,0 +1,202 @@ +""" +LLM-based relevance checking for spam detection. + +Uses an OpenAI-compatible inference endpoint (e.g. hermes.ai.unturf.com) +to check whether a post is relevant to its context (namespace or thread). + +Configuration (from .ini): + spam.llm.enabled = true + spam.llm.endpoint = https://hermes.ai.unturf.com/v1/chat/completions + spam.llm.model = adamo1139/Hermes-3-Llama-3.1-8B-FP8-Dynamic + spam.llm.timeout = 5 +""" + +import json +import logging +import urllib.request +import urllib.error + +log = logging.getLogger(__name__) + +# Defaults +DEFAULT_ENDPOINT = "https://hermes.ai.unturf.com/v1/chat/completions" +DEFAULT_MODEL = "adamo1139/Hermes-3-Llama-3.1-8B-FP8-Dynamic" +DEFAULT_TIMEOUT = 5 # seconds + + +def _llm_request(endpoint, model, messages, timeout): + """Make a chat completion request to an OpenAI-compatible endpoint.""" + body = json.dumps({ + "model": model, + "messages": messages, + "max_tokens": 150, + "temperature": 0.1, + }).encode("utf-8") + + req = urllib.request.Request( + endpoint, + data=body, + headers={"Content-Type": "application/json"}, + method="POST", + ) + + try: + resp = urllib.request.urlopen(req, timeout=timeout) + data = json.loads(resp.read().decode("utf-8")) + return data["choices"][0]["message"]["content"].strip() + except (urllib.error.URLError, urllib.error.HTTPError, KeyError, Exception) as e: + log.warning("LLM relevance check failed: %s", e) + return None + + +def check_thread_relevance(namespace_name, namespace_description, title, content, settings=None): + """Check if a new thread is relevant to the namespace. + + Args: + namespace_name: The namespace (e.g. "meta.remarkbox.com") + namespace_description: The namespace description (may be None) + title: The thread title + content: The thread body text + settings: Pyramid registry settings dict + + Returns: + (relevant, explanation) where relevant is True/False/None (None = LLM unavailable) + """ + if not _is_enabled(settings): + return None, None + + endpoint, model, timeout = _get_config(settings) + + ns_context = namespace_name + if namespace_description: + ns_context = "{} ({})".format(namespace_name, namespace_description) + + messages = [ + { + "role": "system", + "content": ( + "You are a content moderation assistant. Your job is to determine if " + "a new discussion thread is relevant to the site it is being posted on. " + "Respond with exactly 'RELEVANT' or 'IRRELEVANT' on the first line, " + "followed by a brief one-sentence explanation." + ), + }, + { + "role": "user", + "content": ( + "Site: {ns}\n\n" + "New thread title: {title}\n\n" + "Thread content (first 500 chars):\n{content}\n\n" + "Is this thread relevant to this site?" + ).format( + ns=ns_context, + title=title or "(no title)", + content=(content or "")[:500], + ), + }, + ] + + response = _llm_request(endpoint, model, messages, timeout) + return _parse_verdict(response) + + +def check_reply_relevance(thread_title, thread_content, parent_content, + reply_content, page_url=None, namespace_name=None, + settings=None): + """Check if a reply is relevant to the thread. + + Args: + thread_title: Root thread title + thread_content: Root thread body (first 300 chars) + parent_content: Direct parent node body (first 300 chars) + reply_content: The reply being checked + page_url: Parent page URL if this is an embed-mode thread (may be None) + namespace_name: Namespace/site name (may be None) + settings: Pyramid registry settings dict + + Returns: + (relevant, explanation) where relevant is True/False/None (None = LLM unavailable) + """ + if not _is_enabled(settings): + return None, None + + endpoint, model, timeout = _get_config(settings) + + # Build context block -- include parent page info when available (embed mode) + context_parts = [] + if namespace_name: + context_parts.append("Site: {}".format(namespace_name)) + if page_url: + context_parts.append("Parent page URL: {}".format(page_url)) + if thread_title: + context_parts.append("Thread title: {}".format(thread_title)) + if thread_content: + context_parts.append( + "Thread content (first 300 chars):\n{}".format( + (thread_content or "")[:300] + ) + ) + if parent_content: + context_parts.append( + "Parent comment (first 300 chars):\n{}".format( + (parent_content or "")[:300] + ) + ) + + user_msg = "{context}\n\nNew reply (first 500 chars):\n{reply}\n\n" \ + "Is this reply relevant to the discussion?".format( + context="\n\n".join(context_parts), + reply=(reply_content or "")[:500], + ) + + messages = [ + { + "role": "system", + "content": ( + "You are a content moderation assistant. Your job is to determine if " + "a reply is relevant to the discussion thread it is being posted in. " + "Off-topic spam, promotional content, and gibberish should be marked irrelevant. " + "Respond with exactly 'RELEVANT' or 'IRRELEVANT' on the first line, " + "followed by a brief one-sentence explanation." + ), + }, + { + "role": "user", + "content": user_msg, + }, + ] + + response = _llm_request(endpoint, model, messages, timeout) + return _parse_verdict(response) + + +def _is_enabled(settings): + """Check if LLM relevance checking is enabled.""" + if not settings: + return False + return settings.get("spam.llm.enabled", "false").strip().lower() in ("true", "1", "yes") + + +def _get_config(settings): + """Extract LLM config from settings.""" + endpoint = settings.get("spam.llm.endpoint", DEFAULT_ENDPOINT) + model = settings.get("spam.llm.model", DEFAULT_MODEL) + timeout = int(settings.get("spam.llm.timeout", DEFAULT_TIMEOUT)) + return endpoint, model, timeout + + +def _parse_verdict(response): + """Parse a RELEVANT/IRRELEVANT response from the LLM.""" + if not response: + return None, None + + first_line = response.split("\n")[0].strip().upper() + explanation = response.split("\n", 1)[1].strip() if "\n" in response else "" + + if "IRRELEVANT" in first_line: + return False, explanation + elif "RELEVANT" in first_line: + return True, explanation + + # Ambiguous response -- treat as inconclusive + return None, response diff --git a/remarkbox/models/user.py b/remarkbox/models/user.py index 38a319b..650fc5c 100644 --- a/remarkbox/models/user.py +++ b/remarkbox/models/user.py @@ -104,6 +104,8 @@ class User(RBase, Base): gravatar = Column(Boolean, default=False) verified = Column(Boolean, default=False) disabled = Column(Boolean, default=False) + # Global moderator: can moderate across all namespaces. + is_superuser = Column(Boolean, default=False) # automatically watch any threads I create. auto_watch_threads_i_create = Column(Boolean, default=True, nullable=False) # automatically watch any threads I participate in. diff --git a/remarkbox/routes.py b/remarkbox/routes.py index af932a1..69c3db7 100644 --- a/remarkbox/routes.py +++ b/remarkbox/routes.py @@ -66,6 +66,9 @@ def includeme(config): config.add_route("topsecret-namespaces", "/topsecret/namespaces") config.add_route("topsecret-notifications", "/topsecret/notifications") config.add_route("topsecret-nodes", "/topsecret/nodes") + config.add_route("topsecret-users", "/topsecret/users") + config.add_route("topsecret-user-promote", "/topsecret/users/promote") + config.add_route("topsecret-user-demote", "/topsecret/users/demote") config.add_route("topsecret", "/topsecret") # embed routes: diff --git a/remarkbox/scripts/alembic/versions/03061161fc3d_add_is_superuser_column_to_user.py b/remarkbox/scripts/alembic/versions/03061161fc3d_add_is_superuser_column_to_user.py new file mode 100644 index 0000000..e2bb7f2 --- /dev/null +++ b/remarkbox/scripts/alembic/versions/03061161fc3d_add_is_superuser_column_to_user.py @@ -0,0 +1,24 @@ +"""add is_superuser column to user + +Revision ID: 03061161fc3d +Revises: b7f3a2d1e8c9 +Create Date: 2026-02-02 08:01:12.446084 + +""" + +# revision identifiers, used by Alembic. +revision = '03061161fc3d' +down_revision = 'b7f3a2d1e8c9' +branch_labels = None +depends_on = None + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + op.add_column('rb_user', sa.Column('is_superuser', sa.Boolean(), nullable=True)) + + +def downgrade(): + op.drop_column('rb_user', 'is_superuser') diff --git a/remarkbox/templates/list-users.j2 b/remarkbox/templates/list-users.j2 new file mode 100644 index 0000000..18e7323 --- /dev/null +++ b/remarkbox/templates/list-users.j2 @@ -0,0 +1,40 @@ +{% extends request.base_template -%} + +{% block title %}{{ the_title }} | {{ request.domain }}{%- endblock -%} +{% block content -%} + +

{{ the_title }}

+ +

Current Superusers

+ +{% if superusers %} + + +{% for user in superusers %} + + + + + +{% endfor %} +
NameEmailAction
{{ user.name }}{{ user.email }} +
+ + + +
+
+{% else %} +

No superusers configured.

+{% endif %} + +

Promote User to Superuser

+ +
+ +
+ + +
+ +{%- endblock -%} diff --git a/remarkbox/views/__init__.py b/remarkbox/views/__init__.py index c2ae93d..311ee54 100644 --- a/remarkbox/views/__init__.py +++ b/remarkbox/views/__init__.py @@ -35,12 +35,11 @@ def user_required( # view decorator. def super_fly_required(fn): """This view requires that the request has a super fly admin.""" - # TODO: don't rely on this hack for super fly admin. def wrapped(request): if ( request.user and request.user.authenticated - and (request.user.name == "Remarkbox" or request.user.name == "russell") + and getattr(request.user, "is_superuser", False) ): return fn(request) request.session.flash( diff --git a/remarkbox/views/authenticated/topsecret.py b/remarkbox/views/authenticated/topsecret.py index 680d58d..fc0a52a 100644 --- a/remarkbox/views/authenticated/topsecret.py +++ b/remarkbox/views/authenticated/topsecret.py @@ -1,3 +1,4 @@ +from pyramid.httpexceptions import HTTPFound from pyramid.view import view_config from remarkbox.models import ( @@ -7,6 +8,7 @@ from remarkbox.models import ( get_topsecret_roots, get_topsecret_nodes, ) +from remarkbox.models.user import get_user_by_email, User from remarkbox.views import super_fly_required @@ -78,3 +80,55 @@ def topsecret_nodes(request): ), "the_title": "topsecret activity on everything!", } + + +@view_config(route_name="topsecret-users", renderer="list-users.j2") +@super_fly_required +def topsecret_users(request): + superusers = request.dbsession.query(User).filter( + User.is_superuser == True + ).all() + return { + "superusers": superusers, + "the_title": "topsecret superusers!", + } + + +@view_config(route_name="topsecret-user-promote", request_method="POST") +@super_fly_required +def topsecret_user_promote(request): + email = request.params.get("email", "").strip().lower() + if email: + user = get_user_by_email(request.dbsession, email) + if user: + user.is_superuser = True + request.dbsession.add(user) + request.dbsession.flush() + request.session.flash( + ("Promoted {} ({}) to superuser.".format(user.name, user.email), "success") + ) + else: + request.session.flash( + ("No user found with email: {}".format(email), "error") + ) + return HTTPFound(request.route_url("topsecret-users")) + + +@view_config(route_name="topsecret-user-demote", request_method="POST") +@super_fly_required +def topsecret_user_demote(request): + email = request.params.get("email", "").strip().lower() + if email: + user = get_user_by_email(request.dbsession, email) + if user: + user.is_superuser = False + request.dbsession.add(user) + request.dbsession.flush() + request.session.flash( + ("Demoted {} ({}) from superuser.".format(user.name, user.email), "success") + ) + else: + request.session.flash( + ("No user found with email: {}".format(email), "error") + ) + return HTTPFound(request.route_url("topsecret-users")) diff --git a/remarkbox_client.py b/remarkbox_client.py index ccf25f0..2ea31d8 100644 --- a/remarkbox_client.py +++ b/remarkbox_client.py @@ -421,6 +421,29 @@ class RemarkboxClient: """ return self._request("PATCH", "/api/v1/user/profile", {"name": name}) + # ----- Admin (superuser only) ----- + + def admin_list_namespaces(self): + """List all namespaces (requires superuser). + + Returns: + dict with key: namespaces (list of namespace dicts) + """ + return self._request("GET", "/api/v1/admin/namespaces") + + def admin_recent_nodes(self, days=7, limit=100): + """List recent nodes across all namespaces (requires superuser). + + Args: + days: Number of days to look back (default 7, max 90) + limit: Max results (default 100, max 500) + + Returns: + dict with keys: days, count, nodes + """ + params = urllib.parse.urlencode({"days": days, "limit": limit}) + return self._request("GET", "/api/v1/admin/recent-nodes?" + params) + # ----- CLI ----- diff --git a/scripts/promote_superuser.py b/scripts/promote_superuser.py new file mode 100644 index 0000000..8eaf9a2 --- /dev/null +++ b/scripts/promote_superuser.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +""" +Promote or demote a user to/from superuser status. + +This is a direct database script for bootstrapping the first superuser. +After that, use the /topsecret/users web UI to manage superusers. + +Usage: + python scripts/promote_superuser.py --ini development.ini --email user@example.com + python scripts/promote_superuser.py --ini development.ini --email user@example.com --demote +""" + +import argparse +import sys + +from pyramid.paster import bootstrap + + +def main(): + parser = argparse.ArgumentParser(description="Promote/demote superuser") + parser.add_argument("--ini", required=True, help="Path to .ini config file") + parser.add_argument("--email", required=True, help="User email address") + parser.add_argument("--demote", action="store_true", help="Demote instead of promote") + args = parser.parse_args() + + env = bootstrap(args.ini) + request = env["request"] + dbsession = request.dbsession + + from remarkbox.models.user import get_user_by_email + + user = get_user_by_email(dbsession, args.email) + if user is None: + print("No user found with email: {}".format(args.email), file=sys.stderr) + sys.exit(1) + + if args.demote: + user.is_superuser = False + dbsession.add(user) + import transaction + transaction.commit() + print("Demoted {} ({}) from superuser.".format(user.name, user.email)) + else: + user.is_superuser = True + dbsession.add(user) + import transaction + transaction.commit() + print("Promoted {} ({}) to superuser.".format(user.name, user.email)) + + env["closer"]() + + +if __name__ == "__main__": + main() diff --git a/scripts/spam/disable_spam.py b/scripts/spam/disable_spam.py new file mode 100644 index 0000000..ab96661 --- /dev/null +++ b/scripts/spam/disable_spam.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +""" +Bulk disable spam nodes on Remarkbox. + +Requires a superuser session cookie. + +Usage: + python scripts/spam/disable_spam.py node_id [node_id ...] + python scripts/spam/scan.py --json | python scripts/spam/disable_spam.py --from-json + +Examples: + python scripts/spam/disable_spam.py abc123 def456 + python scripts/spam/scan.py --json --threshold=0.8 | python scripts/spam/disable_spam.py --from-json +""" + +import argparse +import json +import os +import sys +import time + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "remarkbox", "api")) +from remarkbox_client import RemarkboxClient + + +def main(): + parser = argparse.ArgumentParser(description="Bulk disable spam nodes") + parser.add_argument("node_ids", nargs="*", help="Node IDs to disable") + parser.add_argument("--from-json", action="store_true", + help="Read node IDs from JSON on stdin (from scan.py --json)") + parser.add_argument("--url", default="https://my.remarkbox.com", help="Remarkbox URL") + parser.add_argument("--cookie", default=os.path.expanduser("~/.config/remarkbox/cookies.txt"), + help="Cookie file path") + parser.add_argument("--dry-run", action="store_true", help="Just print what would be disabled") + args = parser.parse_args() + + node_ids = list(args.node_ids) + + if args.from_json: + data = json.load(sys.stdin) + if isinstance(data, list): + node_ids.extend(item["id"] for item in data if "id" in item) + elif isinstance(data, dict) and "nodes" in data: + node_ids.extend(item["id"] for item in data["nodes"] if "id" in item) + + if not node_ids: + print("No node IDs provided. Use positional args or --from-json.", file=sys.stderr) + sys.exit(1) + + # Deduplicate + node_ids = list(dict.fromkeys(node_ids)) + + client = RemarkboxClient(args.url, cookie_file=args.cookie) + + disabled = 0 + failed = 0 + skipped = 0 + + for node_id in node_ids: + if args.dry_run: + print("[dry-run] would disable {}".format(node_id)) + continue + + try: + client.disable_node(node_id) + print("[ok] disabled {}".format(node_id)) + disabled += 1 + time.sleep(0.5) + except Exception as e: + print("[FAIL] {}: {}".format(node_id, e), file=sys.stderr) + failed += 1 + + if args.dry_run: + print("\nDry run: {} nodes would be disabled.".format(len(node_ids))) + else: + print("\nDone. {} disabled, {} failed.".format(disabled, failed)) + + +if __name__ == "__main__": + main() diff --git a/scripts/spam/scan.py b/scripts/spam/scan.py new file mode 100644 index 0000000..46086b6 --- /dev/null +++ b/scripts/spam/scan.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python3 +""" +Scan recent Remarkbox posts for spam. + +Requires a superuser session cookie. + +Usage: + python scripts/spam/scan.py [--days=7] [--threshold=0.3] [--namespace=X] [--limit=100] + +Examples: + python scripts/spam/scan.py + python scripts/spam/scan.py --days=30 --threshold=0.1 + python scripts/spam/scan.py --namespace=meta.remarkbox.com +""" + +import argparse +import json +import os +import re +import sys + +# Add the remarkbox api dir to path for the client +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "remarkbox", "api")) +from remarkbox_client import RemarkboxClient + + +# Client-side spam scoring (mirrors server-side logic in remarkbox/models/spam.py) +DEFAULT_PATTERNS = [ + r"buy\s+now", r"click\s+here\s+to", r"free\s+trial", + r"limited\s+time\s+offer", r"act\s+now", r"order\s+today", + r"100%\s+free", r"make\s+money\s+fast", r"work\s+from\s+home", + r"casino\s+online", r"viagra|cialis", r"payday\s+loan", + r"seo\s+service", r"followers?\s+for\s+(free|sale|\$)", + r"crypto\s+invest", +] + +COMPILED_PATTERNS = [re.compile(p, re.IGNORECASE) for p in DEFAULT_PATTERNS] + + +def score_content_client(text): + """Score content for spam (client-side, no DB access).""" + if not text: + return 0.0, [] + + signals = [] + score = 0.0 + + # Link density + urls = re.findall(r"https?://\S+", text) + url_chars = sum(len(u) for u in urls) + density = url_chars / len(text) if text else 0.0 + if density > 0.5: + score += 0.4 + signals.append("link_density:{:.0%}".format(density)) + elif density > 0.3: + score += 0.2 + signals.append("link_density:{:.0%}".format(density)) + if len(urls) > 5: + score += 0.2 + signals.append("link_count:{}".format(len(urls))) + + # Spam patterns + hits = sum(1 for p in COMPILED_PATTERNS if p.search(text)) + if hits >= 3: + score += 0.5 + signals.append("spam_patterns:{}".format(hits)) + elif hits >= 1: + score += 0.2 + signals.append("spam_patterns:{}".format(hits)) + + # Very short + if len(text) < 10: + score += 0.1 + signals.append("very_short") + + return min(score, 1.0), signals + + +def main(): + parser = argparse.ArgumentParser(description="Scan Remarkbox for spam") + parser.add_argument("--days", type=int, default=7, help="Days to look back (default 7)") + parser.add_argument("--threshold", type=float, default=0.3, help="Minimum score to report (default 0.3)") + parser.add_argument("--namespace", help="Limit to a specific namespace") + parser.add_argument("--limit", type=int, default=100, help="Max nodes to scan (default 100)") + parser.add_argument("--url", default="https://my.remarkbox.com", help="Remarkbox URL") + parser.add_argument("--cookie", default=os.path.expanduser("~/.config/remarkbox/cookies.txt"), + help="Cookie file path") + parser.add_argument("--json", action="store_true", help="Output as JSON") + args = parser.parse_args() + + client = RemarkboxClient(args.url, cookie_file=args.cookie) + + # Fetch recent nodes via admin endpoint + try: + result = client._request("GET", "/api/v1/admin/recent-nodes?days={}&limit={}".format( + args.days, args.limit + )) + except Exception as e: + print("Error: {}".format(e), file=sys.stderr) + print("Make sure you are authenticated as a superuser.", file=sys.stderr) + sys.exit(1) + + flagged = [] + for node in result.get("nodes", []): + # Filter by namespace if specified + if args.namespace and node.get("namespace") != args.namespace: + continue + + text = node.get("data", "") or "" + spam_score, signals = score_content_client(text) + + if spam_score >= args.threshold: + flagged.append({ + "id": node["id"], + "namespace": node.get("namespace"), + "author": node.get("author"), + "ip": node.get("ip_address"), + "score": round(spam_score, 2), + "signals": signals, + "disabled": node.get("disabled"), + "preview": text[:100].replace("\n", " "), + "created_ago": node.get("created_ago"), + }) + + if args.json: + print(json.dumps(flagged, indent=2)) + else: + if not flagged: + print("No spam found above threshold {}.".format(args.threshold)) + return + + print("Found {} suspicious posts (threshold {}):".format(len(flagged), args.threshold)) + print() + for item in flagged: + status = "DISABLED" if item["disabled"] else "active" + print(" [{:.1f}] {} [{}]".format(item["score"], item["id"], status)) + print(" namespace: {} author: {} ip: {}".format( + item["namespace"], item["author"], item["ip"] + )) + print(" signals: {}".format(", ".join(item["signals"]))) + print(" preview: {}".format(item["preview"])) + print() + + +if __name__ == "__main__": + main() diff --git a/test.ini b/test.ini index 77b18e3..d60e684 100644 --- a/test.ini +++ b/test.ini @@ -42,6 +42,14 @@ api.enabled = true api.rate_limit.read_requests = 1000 api.rate_limit.write_requests = 1000 api.rate_limit.window = 60 +api.rate_limit.create_thread_requests = 1000 +api.rate_limit.create_thread_window = 3600 + +### +# spam detection (disabled for tests). +### +spam.enabled = false +spam.llm.enabled = false ### # custom app configuration.