diff --git a/remarkbox/api/views.py b/remarkbox/api/views.py index 184e859..6ede2e1 100644 --- a/remarkbox/api/views.py +++ b/remarkbox/api/views.py @@ -64,20 +64,28 @@ def get_param(request, key, default=None): def check_spam(request, data, user=None, namespace=None, title=None, parent_node=None): - """Check content for spam. Returns error dict or None. + """Check content for spam. Always returns a result dict. - - Hard threshold: reject with 403. - - Soft threshold: returns {"spam_held": True} so caller can set approved=False. - - LLM relevance check: if enabled and basic score is suspicious, ask the LLM - whether the content is relevant to its context. - - Superusers bypass all checks. + Returns dict with keys: + action: None (clean), "held", or "rejected" + spam_score: float 0.0-1.0 + signals: list of signal strings + spam_reason: human-readable LLM explanation or None + error: present only when action == "rejected" + + - Hard threshold: action="rejected", sets 403 on response. + - Soft threshold: action="held", caller should set approved=False. + - Below thresholds: action=None, spam data still returned for storage. + - Superusers bypass all checks (returns score 0.0). """ settings = request.registry.settings + result = {"action": None, "spam_score": 0.0, "signals": [], "spam_reason": None} + if settings.get("spam.enabled", "true").strip().lower() not in ("true", "1", "yes"): - return None + return result if user and getattr(user, "is_superuser", False): - return None + return result hard_threshold = float(settings.get("spam.hard_threshold", 0.8)) soft_threshold = float(settings.get("spam.soft_threshold", 0.5)) @@ -90,26 +98,41 @@ def check_spam(request, data, user=None, namespace=None, title=None, settings=settings, ) - # LLM relevance check: runs on every message when spam.llm.enabled is set - if settings.get("spam.llm.enabled", "false").strip().lower() in ("true", "1", "yes"): + spam_reason = None + + # LLM relevance check: runs when globally enabled AND namespace allows it + llm_globally_enabled = settings.get( + "spam.llm.enabled", "false" + ).strip().lower() in ("true", "1", "yes") + ns_filter_enabled = True + if namespace and hasattr(namespace, "spam_filter_enabled"): + ns_filter_enabled = namespace.spam_filter_enabled is not False + + if llm_globally_enabled and ns_filter_enabled: relevant, explanation = _llm_relevance_check( settings, data, title=title, namespace=namespace, parent_node=parent_node, ) + if explanation: + spam_reason = explanation if relevant is False: spam_score = min(spam_score + 0.4, 1.0) signals.append("llm_irrelevant") if explanation: signals.append("llm_reason:{}".format(explanation[:80])) + result["spam_score"] = spam_score + result["signals"] = signals + result["spam_reason"] = spam_reason + if spam_score >= hard_threshold: request.response.status_code = 403 - return {"error": "Content flagged as spam", "spam_score": spam_score} + result["action"] = "rejected" + result["error"] = "Content flagged as spam" + elif spam_score >= soft_threshold: + result["action"] = "held" - if spam_score >= soft_threshold: - return {"spam_held": True, "spam_score": spam_score, "signals": signals} - - return None + return result def _llm_relevance_check(settings, data, title=None, namespace=None, @@ -489,9 +512,8 @@ def api_create_thread(request): # Spam check (before creating anything) spam_result = check_spam(request, data, user=user, namespace=namespace, title=title) - if spam_result and "error" in spam_result: - return spam_result - spam_held = spam_result and spam_result.get("spam_held") + if spam_result.get("action") == "rejected": + return {"error": spam_result["error"], "spam_score": spam_result["spam_score"]} if namespace.allow_anonymous and not user: if not anonymous_name: @@ -569,9 +591,13 @@ def api_create_thread(request): node_event = node.new_event(user, "created") request.dbsession.add(user) - if spam_held: + if spam_result.get("action") == "held": node.approved = False + # Store spam data on node for moderation UI + node.spam_score = spam_result.get("spam_score") + node.spam_reason = spam_result.get("spam_reason") + request.dbsession.add(node) if node_event: request.dbsession.add(node_event) @@ -659,9 +685,8 @@ def api_reply(request): # Spam check (before creating anything) spam_result = check_spam(request, data, user=user, namespace=namespace, parent_node=parent) - if spam_result and "error" in spam_result: - return spam_result - spam_held = spam_result and spam_result.get("spam_held") + if spam_result.get("action") == "rejected": + return {"error": spam_result["error"], "spam_score": spam_result["spam_score"]} if namespace.allow_anonymous and not user: if not anonymous_name: @@ -688,7 +713,7 @@ def api_reply(request): child.verified = user.authenticated child_event = child.new_event(user, "commented") - if spam_held: + if spam_result.get("action") == "held": child.approved = False elif namespace.hide_unless_approved: if user: @@ -696,6 +721,10 @@ def api_reply(request): else: child.approved = False + # Store spam data on node for moderation UI + child.spam_score = spam_result.get("spam_score") + child.spam_reason = spam_result.get("spam_reason") + # Bump thread parent.root.changed = child.changed parent._invalidate_cache() diff --git a/remarkbox/models/namespace.py b/remarkbox/models/namespace.py index 5a2dd23..2714600 100644 --- a/remarkbox/models/namespace.py +++ b/remarkbox/models/namespace.py @@ -130,6 +130,8 @@ class Namespace(RBase, Base): # T8: nesting depth settings max_nesting_depth = Column(Integer, default=None, nullable=True) collapse_depth = Column(Integer, default=None, nullable=True) + # Spam filter: namespace owners can disable Hermes LLM checks. + spam_filter_enabled = Column(Boolean, default=True) # the type of subscription of this Namespace. subscription_type = Column( Enum(*SUBSCRIPTION_TYPES, name="subscription_type"), @@ -330,6 +332,12 @@ class Namespace(RBase, Base): or_(Node.approved == False, Node.approved.is_(None)), Node.disabled == False ) + @property + def spam_nodes(self): + return self.nodes.filter( + Node.spam_score >= 0.5, Node.disabled == False + ) + @property def visible_nodes(self): return get_nodes_who_share_roots_query(self.dbsession, self.visible_roots) diff --git a/remarkbox/models/node.py b/remarkbox/models/node.py index 8b33053..992ec76 100644 --- a/remarkbox/models/node.py +++ b/remarkbox/models/node.py @@ -4,7 +4,7 @@ from collections import ( ) from sqlalchemy import ( - BigInteger, Boolean, Column, Integer, Unicode, UnicodeText, or_, func + BigInteger, Boolean, Column, Float, Integer, Unicode, UnicodeText, or_, func ) from sqlalchemy.orm import relationship, backref @@ -84,6 +84,9 @@ class Node(RBase, Base): locked = Column(Boolean, default=False) # by default comments are approved. unless Namespace hide_unless_approved. approved = Column(Boolean, default=True) + # Spam detection: score (0.0-1.0) and human-readable reason from Hermes LLM. + spam_score = Column(Float, default=None) + spam_reason = Column(UnicodeText, default=None) # is there a related Uri model to this node? has_uri = Column(Boolean, default=False, nullable=False) ip_address = Column(Unicode(45), default=None) diff --git a/remarkbox/routes.py b/remarkbox/routes.py index 69c3db7..08ad2f3 100644 --- a/remarkbox/routes.py +++ b/remarkbox/routes.py @@ -113,6 +113,7 @@ def includeme(config): config.add_route("basic-verification-challenge", "/verification-challenge") config.add_route("basic-namespace-nodes", "/ns/{namespace}/nodes") + config.add_route("basic-namespace-spam-bulk", "/ns/{namespace}/spam-bulk") config.add_route("basic-namespace-delete", "/ns/{namespace}/delete") config.add_route("basic-namespace-settings", "/ns/{namespace}/settings") config.add_route("basic-namespace-import-comments", "/ns/{namespace}/import-comments") diff --git a/remarkbox/scripts/alembic/versions/e99b1524710c_add_spam_score_spam_reason_to_node_and_.py b/remarkbox/scripts/alembic/versions/e99b1524710c_add_spam_score_spam_reason_to_node_and_.py new file mode 100644 index 0000000..685c93e --- /dev/null +++ b/remarkbox/scripts/alembic/versions/e99b1524710c_add_spam_score_spam_reason_to_node_and_.py @@ -0,0 +1,28 @@ +"""Add spam_score spam_reason to Node and spam_filter_enabled to Namespace + +Revision ID: e99b1524710c +Revises: 03061161fc3d +Create Date: 2026-02-02 13:04:45.514654 + +""" + +# revision identifiers, used by Alembic. +revision = 'e99b1524710c' +down_revision = '03061161fc3d' +branch_labels = None +depends_on = None + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + op.add_column('rb_node', sa.Column('spam_score', sa.Float(), nullable=True)) + op.add_column('rb_node', sa.Column('spam_reason', sa.UnicodeText(), nullable=True)) + op.add_column('rb_namespace', sa.Column('spam_filter_enabled', sa.Boolean(), nullable=True)) + + +def downgrade(): + op.drop_column('rb_namespace', 'spam_filter_enabled') + op.drop_column('rb_node', 'spam_reason') + op.drop_column('rb_node', 'spam_score') diff --git a/remarkbox/templates/list-nodes.j2 b/remarkbox/templates/list-nodes.j2 index 483a547..4c495a9 100644 --- a/remarkbox/templates/list-nodes.j2 +++ b/remarkbox/templates/list-nodes.j2 @@ -28,6 +28,57 @@ page: {{ request.page_number }}

{{ the_title }}

+{% if 'spam' in request.params and nodes %} +
+ + +
+ +   + + +
+ +{%- for node in nodes %} + +
+ + +
+ + {{ snippets.node_avatar(node) }} + {{ snippets.author_and_date(node) }} + {{ snippets.status(node) }} + + {% if node.spam_reason %} +
+ Hermes: {{ node.spam_reason }} +
+ {% elif node.spam_score is not none %} +
+ Spam score: {{ "%.1f"|format(node.spam_score) }} +
+ {% endif %} + +
+ {{ node.data_html | safe }} +
+ + {{ snippets.permalinks(node) }} + {{ snippets.actions(node) }} +
+ +{% endfor -%} + +
+
+ + +
+
+ +{% else %} + {%- for node in nodes %}
@@ -56,4 +107,6 @@ page: {{ request.page_number }} {% endfor -%} +{% endif %} + {%- endblock -%} diff --git a/remarkbox/templates/namespace-settings.j2 b/remarkbox/templates/namespace-settings.j2 index ef29b09..48ee286 100644 --- a/remarkbox/templates/namespace-settings.j2 +++ b/remarkbox/templates/namespace-settings.j2 @@ -79,6 +79,11 @@ If checked, hide the Remarkbox Logo in footer.{% if request.namespace.hide_power If checked, allow JSON API access to this namespace for agents and integrations. Uncheck to block all /api/v1/ requests targeting this namespace. +
+
+ + +If checked, use AI to check whether comments are relevant to this namespace. Irrelevant or spammy posts will be labeled.

diff --git a/remarkbox/templates/snippets/snippets.j2 b/remarkbox/templates/snippets/snippets.j2 index e8c9e90..88fb240 100644 --- a/remarkbox/templates/snippets/snippets.j2 +++ b/remarkbox/templates/snippets/snippets.j2 @@ -76,6 +76,8 @@ {% if node.disabled %} (waiting for deletion) +{% elif node.spam_score is not none and node.spam_score >= 0.5 %} +(spam) {% elif request.namespace.hide_unless_approved and not node.approved %} (hidden: waiting for approval) {% endif %} @@ -245,6 +247,7 @@ {% set namespace_unapproved_node_count = namespace.unapproved_nodes.count() %} {% set namespace_approved_node_count = namespace.approved_nodes.count() %} {% set namespace_disabled_node_count = namespace.disabled_nodes.count() %} + {% set namespace_spam_node_count = namespace.spam_nodes.count() %}
@@ -260,6 +263,9 @@ {% else %} nodes ({{ namespace_all_node_count }}) | {% endif %} + {% if namespace_spam_node_count > 0 %} + spam ({{ namespace_spam_node_count }}) | + {% endif %} disabled nodes ({{ namespace_disabled_node_count }}) {% if request.user in namespace.owners %} diff --git a/remarkbox/tests/test_spam.py b/remarkbox/tests/test_spam.py new file mode 100644 index 0000000..bc11d8d --- /dev/null +++ b/remarkbox/tests/test_spam.py @@ -0,0 +1,132 @@ +"""Unit tests for spam scoring module.""" + +import unittest +from unittest.mock import MagicMock, patch + +from remarkbox.models.spam import ( + score_content, + _link_density, + _link_count, + _content_hash, +) + + +class TestLinkDensity(unittest.TestCase): + + def test_no_links(self): + self.assertEqual(_link_density("Hello world"), 0.0) + + def test_all_links(self): + text = "https://example.com" + density = _link_density(text) + self.assertAlmostEqual(density, 1.0) + + def test_mixed_content(self): + text = "Check out https://example.com for more info" + density = _link_density(text) + self.assertGreater(density, 0.0) + self.assertLess(density, 1.0) + + def test_empty_string(self): + self.assertEqual(_link_density(""), 0.0) + + def test_none(self): + self.assertEqual(_link_density(None), 0.0) + + +class TestLinkCount(unittest.TestCase): + + def test_no_links(self): + self.assertEqual(_link_count("Hello world"), 0) + + def test_multiple_links(self): + text = "Visit https://a.com and https://b.com and https://c.com" + self.assertEqual(_link_count(text), 3) + + def test_empty(self): + self.assertEqual(_link_count(""), 0) + + +class TestContentHash(unittest.TestCase): + + def test_same_content(self): + self.assertEqual(_content_hash("hello"), _content_hash("hello")) + + def test_different_content(self): + self.assertNotEqual(_content_hash("hello"), _content_hash("world")) + + def test_whitespace_normalization(self): + self.assertEqual( + _content_hash("hello world"), + _content_hash("hello world"), + ) + + def test_case_normalization(self): + self.assertEqual(_content_hash("HELLO"), _content_hash("hello")) + + +class TestScoreContent(unittest.TestCase): + + def test_clean_content_scores_zero(self): + score, signals = score_content("This is a normal discussion post.") + self.assertEqual(score, 0.0) + self.assertEqual(signals, []) + + def test_empty_content(self): + score, signals = score_content("") + self.assertEqual(score, 0.0) + + def test_high_link_density(self): + text = "https://spam1.com https://spam2.com https://spam3.com" + score, signals = score_content(text) + self.assertGreater(score, 0.0) + self.assertTrue(any("link_density" in s for s in signals)) + + def test_many_links(self): + text = "Visit " + " ".join( + "https://site{}.com".format(i) for i in range(10) + ) + score, signals = score_content(text) + self.assertTrue(any("link_count" in s for s in signals)) + + def test_spam_pattern_single(self): + text = "You should buy now before the sale ends!" + score, signals = score_content(text) + self.assertGreater(score, 0.0) + self.assertTrue(any("spam_patterns" in s for s in signals)) + + def test_spam_pattern_multiple(self): + text = "Buy now! Click here to get a free trial. Act now! Limited time offer!" + score, signals = score_content(text) + self.assertGreaterEqual(score, 0.5) + + def test_duplicate_content(self): + # First post is fine + score1, signals1 = score_content( + "Duplicate test post xyz123", + ip_address="10.99.99.99", + ) + # Second identical post from same IP triggers duplicate + score2, signals2 = score_content( + "Duplicate test post xyz123", + ip_address="10.99.99.99", + ) + self.assertTrue(any("duplicate_content" in s for s in signals2)) + + def test_score_capped_at_one(self): + # Content with every signal firing should not exceed 1.0 + text = ( + "Buy now! Click here to free trial! Act now! " + + " ".join("https://spam{}.com".format(i) for i in range(20)) + ) + score, signals = score_content(text) + self.assertLessEqual(score, 1.0) + + def test_short_content_from_new_user(self): + import time + user = MagicMock() + user.id = "test-short" + user.created = int(time.time() * 1000) - 1000 # 1 second old + user.nodes.count.return_value = 0 + score, signals = score_content("Hi", user=user) + self.assertTrue(any("very_short_content" in s for s in signals)) diff --git a/remarkbox/tests/test_spam_llm.py b/remarkbox/tests/test_spam_llm.py new file mode 100644 index 0000000..8bb6aed --- /dev/null +++ b/remarkbox/tests/test_spam_llm.py @@ -0,0 +1,571 @@ +"""Unit and integration tests for LLM-based spam relevance checking. + +Unit tests mock the HTTP call. Integration tests hit the real Hermes endpoint. +""" + +import json +import unittest +from unittest.mock import patch, MagicMock + +from remarkbox.models.spam_llm import ( + check_thread_relevance, + check_reply_relevance, + _parse_verdict, + _is_enabled, + _llm_request, +) + + +# --------------------------------------------------------------------------- +# Unit tests (mocked, no network) +# --------------------------------------------------------------------------- + + +class TestParseVerdict(unittest.TestCase): + + def test_relevant(self): + relevant, explanation = _parse_verdict("RELEVANT\nThis is on topic.") + self.assertTrue(relevant) + self.assertEqual(explanation, "This is on topic.") + + def test_irrelevant(self): + relevant, explanation = _parse_verdict("IRRELEVANT\nThis is spam.") + self.assertFalse(relevant) + self.assertEqual(explanation, "This is spam.") + + def test_relevant_no_explanation(self): + relevant, explanation = _parse_verdict("RELEVANT") + self.assertTrue(relevant) + self.assertEqual(explanation, "") + + def test_irrelevant_no_explanation(self): + relevant, explanation = _parse_verdict("IRRELEVANT") + self.assertFalse(relevant) + self.assertEqual(explanation, "") + + def test_none_response(self): + relevant, explanation = _parse_verdict(None) + self.assertIsNone(relevant) + self.assertIsNone(explanation) + + def test_empty_response(self): + relevant, explanation = _parse_verdict("") + self.assertIsNone(relevant) + self.assertIsNone(explanation) + + def test_ambiguous_response(self): + relevant, explanation = _parse_verdict("I'm not sure about this.") + self.assertIsNone(relevant) + self.assertEqual(explanation, "I'm not sure about this.") + + def test_case_insensitive(self): + relevant, _ = _parse_verdict("relevant\nYes it is.") + self.assertTrue(relevant) + + def test_irrelevant_in_sentence(self): + relevant, _ = _parse_verdict("IRRELEVANT - clearly spam\nPromo content.") + self.assertFalse(relevant) + + +class TestIsEnabled(unittest.TestCase): + + def test_none_settings(self): + self.assertFalse(_is_enabled(None)) + + def test_missing_key(self): + self.assertFalse(_is_enabled({})) + + def test_enabled_true(self): + self.assertTrue(_is_enabled({"spam.llm.enabled": "true"})) + + def test_enabled_yes(self): + self.assertTrue(_is_enabled({"spam.llm.enabled": "yes"})) + + def test_enabled_one(self): + self.assertTrue(_is_enabled({"spam.llm.enabled": "1"})) + + def test_disabled_false(self): + self.assertFalse(_is_enabled({"spam.llm.enabled": "false"})) + + def test_disabled_empty(self): + self.assertFalse(_is_enabled({"spam.llm.enabled": ""})) + + +class TestCheckThreadRelevanceMocked(unittest.TestCase): + + def test_disabled_returns_none(self): + relevant, explanation = check_thread_relevance( + "example.com", None, "Hello", "World", + settings={"spam.llm.enabled": "false"}, + ) + self.assertIsNone(relevant) + self.assertIsNone(explanation) + + @patch("remarkbox.models.spam_llm._llm_request") + def test_relevant_thread(self, mock_req): + mock_req.return_value = "RELEVANT\nThe thread is about the site topic." + relevant, explanation = check_thread_relevance( + "cooking.example.com", "A cooking forum", + "Best pasta recipe", "I love making pasta with fresh ingredients.", + settings={"spam.llm.enabled": "true"}, + ) + self.assertTrue(relevant) + mock_req.assert_called_once() + + @patch("remarkbox.models.spam_llm._llm_request") + def test_irrelevant_thread(self, mock_req): + mock_req.return_value = "IRRELEVANT\nThis is unrelated spam." + relevant, explanation = check_thread_relevance( + "cooking.example.com", "A cooking forum", + "Buy cheap watches", "Best prices on luxury watches online!", + settings={"spam.llm.enabled": "true"}, + ) + self.assertFalse(relevant) + + @patch("remarkbox.models.spam_llm._llm_request") + def test_llm_failure_returns_none(self, mock_req): + mock_req.return_value = None + relevant, explanation = check_thread_relevance( + "example.com", None, "Test", "Test", + settings={"spam.llm.enabled": "true"}, + ) + self.assertIsNone(relevant) + + +class TestCheckReplyRelevanceMocked(unittest.TestCase): + + @patch("remarkbox.models.spam_llm._llm_request") + def test_relevant_reply(self, mock_req): + mock_req.return_value = "RELEVANT\nThe reply contributes to the discussion." + relevant, explanation = check_reply_relevance( + thread_title="Best pasta recipe", + thread_content="I love making pasta.", + parent_content="What kind of pasta do you use?", + reply_content="I prefer fresh fettuccine.", + settings={"spam.llm.enabled": "true"}, + ) + self.assertTrue(relevant) + + @patch("remarkbox.models.spam_llm._llm_request") + def test_irrelevant_reply(self, mock_req): + mock_req.return_value = "IRRELEVANT\nThis is promotional spam." + relevant, explanation = check_reply_relevance( + thread_title="Best pasta recipe", + thread_content="I love making pasta.", + parent_content="What kind of pasta do you use?", + reply_content="Buy cheap watches at watches.com!", + settings={"spam.llm.enabled": "true"}, + ) + self.assertFalse(relevant) + + @patch("remarkbox.models.spam_llm._llm_request") + def test_reply_with_page_url(self, mock_req): + """Embed mode: page_url and namespace_name are included in context.""" + mock_req.return_value = "RELEVANT\nOn topic for the page." + relevant, explanation = check_reply_relevance( + thread_title="How to bake bread", + thread_content="A guide to baking.", + parent_content=None, + reply_content="Great guide, thanks!", + page_url="https://cooking-blog.com/bread-guide", + namespace_name="cooking-blog.com", + settings={"spam.llm.enabled": "true"}, + ) + self.assertTrue(relevant) + # Verify the LLM was called with context containing the page URL + call_args = mock_req.call_args + messages = call_args[0][2] # third positional arg + user_msg = messages[1]["content"] + self.assertIn("cooking-blog.com", user_msg) + self.assertIn("https://cooking-blog.com/bread-guide", user_msg) + + +# --------------------------------------------------------------------------- +# Integration tests (real Hermes endpoint) +# --------------------------------------------------------------------------- + + +HERMES_SETTINGS = { + "spam.llm.enabled": "true", + "spam.llm.endpoint": "https://hermes.ai.unturf.com/v1/chat/completions", + "spam.llm.model": "adamo1139/Hermes-3-Llama-3.1-8B-FP8-Dynamic", + "spam.llm.timeout": "10", +} + + +class TestHermesIntegration(unittest.TestCase): + """Integration tests that hit the real Hermes LLM endpoint. + + These are not mocked -- they make real HTTP requests to hermes.ai.unturf.com. + Skipped if the endpoint is unreachable. + """ + + @classmethod + def setUpClass(cls): + import urllib.request + import urllib.error + try: + req = urllib.request.Request( + "https://hermes.ai.unturf.com/v1/models", + method="GET", + ) + urllib.request.urlopen(req, timeout=5) + cls.hermes_available = True + except Exception: + cls.hermes_available = False + + def setUp(self): + if not self.hermes_available: + self.skipTest("Hermes endpoint not reachable") + + def test_thread_relevant_to_namespace(self): + relevant, explanation = check_thread_relevance( + namespace_name="meta.remarkbox.com", + namespace_description="Discussion about the Remarkbox commenting platform", + title="Feature request: dark mode", + content="It would be great if Remarkbox had a dark mode option for embedded comments.", + settings=HERMES_SETTINGS, + ) + self.assertTrue(relevant, "Expected relevant, got: {}".format(explanation)) + + def test_thread_irrelevant_to_namespace(self): + relevant, explanation = check_thread_relevance( + namespace_name="meta.remarkbox.com", + namespace_description="Discussion about the Remarkbox commenting platform", + title="Best pizza in New York", + content="Looking for recommendations for pizza places in Manhattan. I love thin crust.", + settings=HERMES_SETTINGS, + ) + self.assertFalse(relevant, "Expected irrelevant, got: {}".format(explanation)) + + def test_reply_relevant_to_thread(self): + relevant, explanation = check_reply_relevance( + thread_title="How to install Remarkbox", + thread_content="I'm trying to install Remarkbox on my server and need help with the configuration.", + parent_content="Have you tried checking the documentation?", + reply_content="Yes, I followed the docs but got stuck on the database setup step.", + settings=HERMES_SETTINGS, + ) + self.assertTrue(relevant, "Expected relevant, got: {}".format(explanation)) + + def test_reply_spam_irrelevant(self): + relevant, explanation = check_reply_relevance( + thread_title="How to install Remarkbox", + thread_content="I'm trying to install Remarkbox on my server.", + parent_content="Have you tried checking the documentation?", + reply_content="Buy cheap designer watches at www.fake-watches-sale.com! Best prices guaranteed!", + settings=HERMES_SETTINGS, + ) + self.assertFalse(relevant, "Expected irrelevant, got: {}".format(explanation)) + + def test_reply_with_embed_page_context(self): + """Test that embed mode page URL helps the LLM understand context.""" + relevant, explanation = check_reply_relevance( + thread_title="Getting Started with Python", + thread_content="", + parent_content=None, + reply_content="This tutorial helped me understand list comprehensions, thanks!", + page_url="https://python-tutorial.example.com/getting-started", + namespace_name="python-tutorial.example.com", + settings=HERMES_SETTINGS, + ) + self.assertTrue(relevant, "Expected relevant, got: {}".format(explanation)) + + def test_raw_llm_request(self): + """Verify we can make a raw request to Hermes and get a response.""" + response = _llm_request( + endpoint="https://hermes.ai.unturf.com/v1/chat/completions", + model="adamo1139/Hermes-3-Llama-3.1-8B-FP8-Dynamic", + messages=[ + {"role": "system", "content": "Reply with exactly the word PONG."}, + {"role": "user", "content": "PING"}, + ], + timeout=10, + ) + self.assertIsNotNone(response) + self.assertIn("PONG", response.upper()) + + +class TestHermesEmbedMode(unittest.TestCase): + """Integration tests for embed mode with parent page context. + + In embed mode, threads represent comment sections on external pages. + The LLM receives the parent page URL and namespace to understand what + the page is about, even when the thread has minimal content. + """ + + @classmethod + def setUpClass(cls): + import urllib.request + try: + req = urllib.request.Request( + "https://hermes.ai.unturf.com/v1/models", method="GET", + ) + urllib.request.urlopen(req, timeout=5) + cls.hermes_available = True + except Exception: + cls.hermes_available = False + + def setUp(self): + if not self.hermes_available: + self.skipTest("Hermes endpoint not reachable") + + def test_embed_relevant_comment_on_blog(self): + """Comment that's relevant to a blog post page.""" + relevant, explanation = check_reply_relevance( + thread_title="10 Tips for Growing Tomatoes in Small Spaces", + thread_content="", # embed threads often have no body + parent_content=None, + reply_content="Tip #3 about container size really helped me. " + "I switched to 5-gallon buckets and my cherry " + "tomatoes are thriving now.", + page_url="https://gardening-blog.example.com/tomatoes-small-spaces", + namespace_name="gardening-blog.example.com", + settings=HERMES_SETTINGS, + ) + self.assertTrue(relevant, "Expected relevant: {}".format(explanation)) + + def test_embed_spam_comment_on_blog(self): + """Obvious spam comment on a gardening blog post.""" + relevant, explanation = check_reply_relevance( + thread_title="10 Tips for Growing Tomatoes in Small Spaces", + thread_content="", + parent_content=None, + reply_content="Amazing deals on designer handbags! Visit " + "cheapbags.example.com for 90% off Louis Vuitton!", + page_url="https://gardening-blog.example.com/tomatoes-small-spaces", + namespace_name="gardening-blog.example.com", + settings=HERMES_SETTINGS, + ) + self.assertFalse(relevant, "Expected irrelevant: {}".format(explanation)) + + def test_embed_crypto_spam_on_tech_blog(self): + """Crypto spam on an unrelated tech article.""" + relevant, explanation = check_reply_relevance( + thread_title="Understanding CSS Grid Layout", + thread_content="A comprehensive guide to CSS Grid for modern web layouts.", + parent_content=None, + reply_content="I made $5000 in one week with this crypto trading bot! " + "Sign up at crypto-scam.example.com and start earning now!", + page_url="https://webdev-tutorials.example.com/css-grid-guide", + namespace_name="webdev-tutorials.example.com", + settings=HERMES_SETTINGS, + ) + self.assertFalse(relevant, "Expected irrelevant: {}".format(explanation)) + + def test_embed_tangential_but_relevant_comment(self): + """A comment that's loosely related to the page topic.""" + relevant, explanation = check_reply_relevance( + thread_title="Python asyncio Tutorial", + thread_content="", + parent_content="I found the event loop explanation very clear.", + reply_content="If you liked asyncio, you should also check out " + "trio which has a simpler API for structured concurrency.", + page_url="https://python-guides.example.com/asyncio-tutorial", + namespace_name="python-guides.example.com", + settings=HERMES_SETTINGS, + ) + self.assertTrue(relevant, "Expected relevant: {}".format(explanation)) + + def test_embed_no_thread_content_only_page_url(self): + """Embed thread with only a page URL, no title or content.""" + relevant, explanation = check_reply_relevance( + thread_title=None, + thread_content=None, + parent_content=None, + reply_content="This recipe is delicious! I added extra garlic.", + page_url="https://recipes.example.com/garlic-bread", + namespace_name="recipes.example.com", + settings=HERMES_SETTINGS, + ) + self.assertTrue(relevant, "Expected relevant: {}".format(explanation)) + + +class TestHermesSiteMode(unittest.TestCase): + """Integration tests for site mode (standalone threads with full context). + + In site mode, threads have titles and content. The LLM checks whether + new threads are relevant to the namespace and replies to the discussion. + """ + + @classmethod + def setUpClass(cls): + import urllib.request + try: + req = urllib.request.Request( + "https://hermes.ai.unturf.com/v1/models", method="GET", + ) + urllib.request.urlopen(req, timeout=5) + cls.hermes_available = True + except Exception: + cls.hermes_available = False + + def setUp(self): + if not self.hermes_available: + self.skipTest("Hermes endpoint not reachable") + + def test_site_relevant_thread_on_forum(self): + """Thread about Remarkbox features on meta.remarkbox.com.""" + relevant, explanation = check_thread_relevance( + namespace_name="meta.remarkbox.com", + namespace_description="Discussion about Remarkbox, the privacy-first comment system", + title="Email notifications are delayed", + content="I noticed that email notifications for new replies are taking " + "about 30 minutes to arrive. Is this expected or is something " + "wrong with the mail server? I'm using the hosted version at " + "my.remarkbox.com with the default settings.", + settings=HERMES_SETTINGS, + ) + self.assertTrue(relevant, "Expected relevant: {}".format(explanation)) + + def test_site_irrelevant_thread_on_forum(self): + """Completely unrelated thread on meta.remarkbox.com.""" + relevant, explanation = check_thread_relevance( + namespace_name="meta.remarkbox.com", + namespace_description="Discussion about Remarkbox, the privacy-first comment system", + title="How to train for a marathon", + content="I'm starting my marathon training plan and wondering about " + "the best shoes for long distance running. I currently run " + "about 20 miles per week and want to increase to 40.", + settings=HERMES_SETTINGS, + ) + self.assertFalse(relevant, "Expected irrelevant: {}".format(explanation)) + + def test_site_reply_to_deep_thread(self): + """Reply deep in a conversation thread, relevant to the discussion.""" + relevant, explanation = check_reply_relevance( + thread_title="Custom CSS for embedded comments", + thread_content="I want to customize the look of Remarkbox comments " + "on my website. The default styling clashes with my " + "dark theme. How can I override the default CSS?", + parent_content="You can use the remarkbox-theme-meta approach. " + "Add a tag with name='remarkbox-css' and " + "point it to your custom stylesheet URL.", + reply_content="Thanks! I added the meta tag and it works. One issue " + "though - the reply form textarea still uses the default " + "white background. Do I need a more specific selector?", + namespace_name="meta.remarkbox.com", + settings=HERMES_SETTINGS, + ) + self.assertTrue(relevant, "Expected relevant: {}".format(explanation)) + + def test_site_gibberish_reply(self): + """Gibberish/nonsense reply to a legitimate thread.""" + relevant, explanation = check_reply_relevance( + thread_title="Custom CSS for embedded comments", + thread_content="I want to customize the look of Remarkbox comments " + "on my website.", + parent_content="You can use remarkbox-theme-meta.", + reply_content="asdf jkl; qwerty uiop zxcv bnm 12345 " + "hgfdsa poiuytrewq mnbvcxz", + namespace_name="meta.remarkbox.com", + settings=HERMES_SETTINGS, + ) + self.assertFalse(relevant, "Expected irrelevant: {}".format(explanation)) + + def test_site_thread_with_rich_namespace_description(self): + """Namespace with a detailed description gives the LLM more context.""" + relevant, explanation = check_thread_relevance( + namespace_name="recipes.cooking-community.example.com", + namespace_description=( + "A community for sharing and discussing recipes. " + "Topics include baking, grilling, meal prep, international cuisine, " + "dietary restrictions, and kitchen equipment reviews." + ), + title="Best cast iron skillet for beginners", + content="I want to get my first cast iron skillet. I've been using " + "non-stick pans but want to try cast iron for better searing. " + "Budget is around $40. Any recommendations?", + settings=HERMES_SETTINGS, + ) + self.assertTrue(relevant, "Expected relevant: {}".format(explanation)) + + def test_site_seo_spam_thread(self): + """SEO spam disguised as a question.""" + relevant, explanation = check_thread_relevance( + namespace_name="recipes.cooking-community.example.com", + namespace_description="A community for sharing and discussing recipes.", + title="Best SEO services for your website", + content="Are you looking to rank #1 on Google? Our professional SEO " + "team can help you get more traffic and leads. Visit " + "seo-services.example.com for a free audit. We guarantee " + "first page rankings within 30 days or your money back!", + settings=HERMES_SETTINGS, + ) + self.assertFalse(relevant, "Expected irrelevant: {}".format(explanation)) + + +class TestHermesEdgeCases(unittest.TestCase): + """Integration tests for edge cases and tricky content. + + Tests that exercise Hermes with content that might be ambiguous, + minimal, or challenging to classify. + """ + + @classmethod + def setUpClass(cls): + import urllib.request + try: + req = urllib.request.Request( + "https://hermes.ai.unturf.com/v1/models", method="GET", + ) + urllib.request.urlopen(req, timeout=5) + cls.hermes_available = True + except Exception: + cls.hermes_available = False + + def setUp(self): + if not self.hermes_available: + self.skipTest("Hermes endpoint not reachable") + + def test_short_agreement_reply(self): + """Very short reply that just agrees -- should be relevant.""" + relevant, explanation = check_reply_relevance( + thread_title="Should we add dark mode?", + thread_content="I think Remarkbox needs a dark mode option.", + parent_content="I agree, dark mode would be great.", + reply_content="+1", + namespace_name="meta.remarkbox.com", + settings=HERMES_SETTINGS, + ) + self.assertTrue(relevant, "Expected relevant: {}".format(explanation)) + + def test_link_that_is_relevant(self): + """A reply that's mostly a link but is relevant to the discussion.""" + relevant, explanation = check_reply_relevance( + thread_title="Resources for learning Python", + thread_content="Let's collect good resources for learning Python.", + parent_content="I found the official tutorial helpful.", + reply_content="Here's another great resource: " + "https://docs.python.org/3/tutorial/index.html", + namespace_name="programming.example.com", + settings=HERMES_SETTINGS, + ) + self.assertTrue(relevant, "Expected relevant: {}".format(explanation)) + + def test_promotional_but_relevant(self): + """Self-promotion that is still on-topic for the namespace.""" + relevant, explanation = check_thread_relevance( + namespace_name="meta.remarkbox.com", + namespace_description="Discussion about the Remarkbox comment system", + title="I built a Remarkbox plugin for WordPress", + content="Hey everyone, I created a WordPress plugin that makes it " + "easy to embed Remarkbox comments. It handles the JavaScript " + "snippet injection and lets you configure settings from the " + "WP admin panel. Free and open source on GitHub.", + settings=HERMES_SETTINGS, + ) + self.assertTrue(relevant, "Expected relevant: {}".format(explanation)) + + def test_different_language_spam(self): + """Spam in a different language on an English namespace.""" + relevant, explanation = check_reply_relevance( + thread_title="How to configure email notifications", + thread_content="I want to set up email notifications for Remarkbox.", + parent_content="Check the namespace settings panel.", + reply_content="Compre relogios baratos! Melhor preco garantido! " + "Visite nossa loja online: relogios-baratos.example.com", + namespace_name="meta.remarkbox.com", + settings=HERMES_SETTINGS, + ) + self.assertFalse(relevant, "Expected irrelevant: {}".format(explanation)) diff --git a/remarkbox/views/authenticated/authenticated.py b/remarkbox/views/authenticated/authenticated.py index d7e1979..23d356d 100644 --- a/remarkbox/views/authenticated/authenticated.py +++ b/remarkbox/views/authenticated/authenticated.py @@ -84,6 +84,7 @@ def namespace_settings(request): ignore_query_string_checkbox = p.get("ignore-query-string-checkbox", "off") hide_powered_by_checkbox = p.get("hide-powered-by-checkbox", "off") api_access_checkbox = p.get("api-access-checkbox", "off") + spam_filter_enabled_checkbox = p.get("spam-filter-enabled-checkbox", "off") hide_unless_approved = checkbox_to_bool(hide_unless_approved_checkbox) allow_anonymous = checkbox_to_bool(allow_anonymous_checkbox) @@ -94,6 +95,7 @@ def namespace_settings(request): ignore_query_string = checkbox_to_bool(ignore_query_string_checkbox) hide_powered_by = checkbox_to_bool(hide_powered_by_checkbox) api_access = checkbox_to_bool(api_access_checkbox) + spam_filter_enabled = checkbox_to_bool(spam_filter_enabled_checkbox) if stylesheet_embed != request.namespace.stylesheet_embed and ( stylesheet_embed or request.namespace.stylesheet_embed @@ -252,6 +254,17 @@ def namespace_settings(request): ) ) + if spam_filter_enabled != (request.namespace.spam_filter_enabled is not False): + request.namespace.spam_filter_enabled = spam_filter_enabled + request.session.flash( + ( + "You turned {} spam_filter (Hermes AI)".format( + spam_filter_enabled_checkbox + ), + "success", + ) + ) + # T4: customizable button text and comment labels if submit_button_text != request.namespace.submit_button_text and ( submit_button_text or request.namespace.submit_button_text diff --git a/remarkbox/views/list_nodes.py b/remarkbox/views/list_nodes.py index cca1861..15504bd 100644 --- a/remarkbox/views/list_nodes.py +++ b/remarkbox/views/list_nodes.py @@ -68,43 +68,39 @@ def namespace(request): def namespace_nodes(request): # TODO: the way I'm currently protecting views (or in this case certain urls) # is causing a lot of copy and paste and even confusing conditional logic. - if request.namespace.hide_unless_approved: - if "disabled" in request.params or "pending" in request.params: - # TODO: maybe dry this out and turn it into a decorator? - if not request.user or not request.user.authenticated: - request.session.flash(("You must log in to access that area.", "error")) - return HTTPFound(get_join_or_log_in_route_uri(request)) - # TODO: maybe dry this out and turn it into a decorator? - if request.user not in request.namespace.moderators: - request.session.flash( - ("You must be a moderator in to access that area.", "error") - ) - return HTTPFound(get_join_or_log_in_route_uri(request)) + # Spam, disabled, and pending views require moderator access. + mod_views = ("disabled", "pending", "spam") + if any(v in request.params for v in mod_views): + if not request.user or not request.user.authenticated: + request.session.flash(("You must log in to access that area.", "error")) + return HTTPFound(get_join_or_log_in_route_uri(request)) - if "disabled" in request.params: - # TODO: pagination. - state = "disabled" - nodes = request.namespace.disabled_nodes - elif "pending" in request.params: - # TODO: pagination. - state = "pending" - nodes = request.namespace.unapproved_nodes - else: - state = "active" - nodes = request.namespace.page_nodes( - limit=request.page_size, offset=request.page_offset + is_mod = (request.user in request.namespace.moderators + or getattr(request.user, "is_superuser", False)) + if not is_mod: + request.session.flash( + ("You must be a moderator to access that area.", "error") ) + return HTTPFound(get_join_or_log_in_route_uri(request)) + + if "spam" in request.params: + state = "spam" + nodes = request.namespace.spam_nodes + elif "disabled" in request.params: + state = "disabled" + nodes = request.namespace.disabled_nodes + elif "pending" in request.params and request.namespace.hide_unless_approved: + state = "pending" + nodes = request.namespace.unapproved_nodes + elif "approved" in request.params and request.namespace.hide_unless_approved: + state = "active" + nodes = request.namespace.approved_nodes else: - if "disabled" in request.params: - # TODO: pagination. - state = "disabled" - nodes = request.namespace.disabled_nodes - else: - state = "active" - nodes = request.namespace.page_nodes( - limit=request.page_size, offset=request.page_offset - ) + state = "active" + nodes = request.namespace.page_nodes( + limit=request.page_size, offset=request.page_offset + ) return { "nodes": nodes, "the_title": "{} - {} nodes".format(request.namespace.name, state), diff --git a/remarkbox/views/modify_node.py b/remarkbox/views/modify_node.py index c5bd80a..a0fb3e4 100644 --- a/remarkbox/views/modify_node.py +++ b/remarkbox/views/modify_node.py @@ -2,6 +2,8 @@ from pyramid.view import view_config from pyramid.httpexceptions import HTTPFound +from remarkbox.models.node import Node + from . import get_referer_or_home, get_node_route_uri, get_embed_route_uri @@ -98,3 +100,48 @@ def deny_node(request): request.dbsession.add(request.node) request.dbsession.flush() return HTTPFound(_node_route_uri(request)) + + +@view_config(route_name="basic-namespace-spam-bulk", request_method="POST") +def spam_bulk_action(request): + """Bulk approve or disable spam-flagged nodes.""" + if not request.user or not request.user.authenticated: + request.session.flash(("You must log in to access that.", "error")) + return HTTPFound(get_referer_or_home(request)) + + is_mod = (request.user in request.namespace.moderators + or getattr(request.user, "is_superuser", False)) + if not is_mod: + request.session.flash(("You must be a moderator.", "error")) + return HTTPFound(get_referer_or_home(request)) + + action = request.params.get("bulk_action", "") + node_ids = request.params.getall("node_ids") + count = 0 + + for node_id in node_ids: + node = request.dbsession.query(Node).filter( + Node.id == node_id, + Node.namespace_id == request.namespace.id, + ).first() + if node is None: + continue + if action == "approve": + node.approved = True + node.spam_score = 0.0 + node.spam_reason = None + request.dbsession.add(node) + count += 1 + elif action == "disable": + node.disable() + request.dbsession.add(node) + count += 1 + + request.dbsession.flush() + request.session.flash( + ("{} node(s) {}d.".format(count, action), "success") + ) + return HTTPFound( + request.route_url("basic-namespace-nodes", namespace=request.namespace.name) + + "?spam" + )