+Import comments and threads from WordPress, Disqus, or pre-formatted JSON exports using the +blog-to-json tool. +
+ +pip install blog-to-jsonNote: The import process will: +
Get the tool: https://github.com/russellballestrini/blog-to-json
+Some content here
", + "comments": [ + { + "id": "wp-1", + "date": "2011-04-03 10:33:07", + "timestamp": 1301841187, + "content": "Hi, this was just what I needed", + "email": "kristian@example.com", + "author": "Kristian", + "author_ip": "192.168.1.5", + "parent_id": None, + }, + { + "id": "wp-2", + "date": "2011-04-03 14:19:46", + "timestamp": 1301854786, + "content": "I'm interested in the modifications", + "email": "russell@example.com", + "author": "Russell Ballestrini", + "author_ip": "192.168.1.6", + "parent_id": None, + } + ], + } + } + + json_content = json.dumps(wordpress_data).encode("utf-8") + + res = self.testapp.post( + "/ns/test.example.com/import-comments", + { + "csrf_token": self.csrf, + }, + upload_files=[("json-file", "wordpress.json", json_content)], + status=200, + ) + + self.assertIn(b"Successfully imported", res.body) + self.assertIn(b"1 threads", res.body) + self.assertIn(b"2 comments", res.body) + + # Verify users were created + kristian = get_user_by_email(self.dbsession, "kristian@example.com") + self.assertIsNotNone(kristian) + russell = get_user_by_email(self.dbsession, "russell@example.com") + self.assertIsNotNone(russell) + + def test_import_graphcomment_format(self): + """Test import from Graphcomment WordPress XML export (blog-to-json format)""" + # Mock Graphcomment export data converted by graphcomment-xml-to-json + graphcomment_data = { + "posts_jupyter-orgmode": { + "content": None, + "link": "https://abc.xyz/posts/jupyter-orgmode/", + "name": "posts_jupyter-orgmode", + "title": "Reflections on Jupyter", + "date": "2020-09-22 15:57:34", + "timestamp": 1600790254, + "id": "5f6a1eee2f57815d17188de2", + "metadata": {}, + "comments": [ + { + "id": "60750d2613ebd3704ec85f6f", + "content": "Thanks!!!!! a lot!!!", + "parent_id": None, + "author": "SamTux", + "date": "2021-04-13 03:16:54", + "timestamp": 1618283814, + "author_ip": "190.25.34.217", + "email": "samtux@example.com" + }, + { + "id": "60750d2613ebd3704ec85f70", + "content": "You're welcome!", + "parent_id": "60750d2613ebd3704ec85f6f", + "author": "BlogAuthor", + "date": "2021-04-14 10:20:00", + "timestamp": 1618395600, + "author_ip": "192.168.1.1", + "email": "author@example.com" + } + ], + } + } + + json_content = json.dumps(graphcomment_data).encode("utf-8") + + res = self.testapp.post( + "/ns/test.example.com/import-comments", + { + "csrf_token": self.csrf, + }, + upload_files=[("json-file", "graphcomment.json", json_content)], + status=200, + ) + + self.assertIn(b"Successfully imported", res.body) + self.assertIn(b"1 threads", res.body) + self.assertIn(b"2 comments", res.body) + + def test_import_with_deep_nesting(self): + """Test import with deeply nested comment hierarchy""" + nested_data = { + "nested-discussion": { + "link": "https://example.com/nested-discussion", + "title": "Nested Discussion", + "timestamp": 1234567890, + "comments": [ + { + "id": "1", + "author": "User1", + "email": "user1@example.com", + "content": "Top level comment", + "timestamp": 1234567890, + "parent_id": None, + "author_ip": "127.0.0.1", + }, + { + "id": "2", + "author": "User2", + "email": "user2@example.com", + "content": "Reply to comment 1", + "timestamp": 1234567900, + "parent_id": "1", + "author_ip": "127.0.0.2", + }, + { + "id": "3", + "author": "User3", + "email": "user3@example.com", + "content": "Reply to comment 2", + "timestamp": 1234567910, + "parent_id": "2", + "author_ip": "127.0.0.3", + }, + { + "id": "4", + "author": "User4", + "email": "user4@example.com", + "content": "Reply to comment 3", + "timestamp": 1234567920, + "parent_id": "3", + "author_ip": "127.0.0.4", + }, + ], + } + } + + json_content = json.dumps(nested_data).encode("utf-8") + + res = self.testapp.post( + "/ns/test.example.com/import-comments", + { + "csrf_token": self.csrf, + }, + upload_files=[("json-file", "nested.json", json_content)], + status=200, + ) + + self.assertIn(b"Successfully imported", res.body) + self.assertIn(b"1 threads", res.body) + self.assertIn(b"4 comments", res.body) + + def test_import_duplicate_prevention(self): + """Test that re-importing the same data reuses existing users""" + test_data = { + "test-duplicate": { + "link": "https://example.com/test-duplicate", + "comments": [ + { + "id": "dup-1", + "author": "DupUser", + "email": "dup@example.com", + "content": "First import", + "timestamp": 1234567890, + "parent_id": None, + "author_ip": "127.0.0.1", + }, + ], + } + } + + json_content = json.dumps(test_data).encode("utf-8") + + # First import + res1 = self.testapp.post( + "/ns/test.example.com/import-comments", + { + "csrf_token": self.csrf, + }, + upload_files=[("json-file", "dup.json", json_content)], + status=200, + ) + + self.assertIn(b"Successfully imported", res1.body) + self.assertIn(b"1 threads", res1.body) + self.assertIn(b"1 comments", res1.body) + + # Second import - should succeed and reuse existing user + res2 = self.testapp.post( + "/ns/test.example.com/import-comments", + { + "csrf_token": self.csrf, + }, + upload_files=[("json-file", "dup.json", json_content)], + status=200, + ) + + self.assertIn(b"Successfully imported", res2.body) + self.assertIn(b"1 threads", res2.body) + self.assertIn(b"1 comments", res2.body) + + # Verify only one user was created (this check works across transactions) + from remarkbox.models import User + user_count = self.dbsession.query(User).filter( + User.email == "dup@example.com" + ).count() + self.assertEqual(user_count, 1) + + def test_import_with_locked_group_postfix(self): + """Test that group postfix gets locked after first import""" + # Create a unique namespace for this test + from remarkbox.models import get_namespace_by_name + lock_test_ns = get_or_create_namespace(self.dbsession, "lock-test.example.com") + lock_test_ns.set_role_for_user(self.test_user, role="owner") + self.dbsession.add(lock_test_ns) + self.dbsession.flush() + self.tm.commit() + + test_data = { + "test-lock": { + "link": "https://example.com/test-lock", + "comments": [ + { + "id": "lock-1", + "author": "LockUser", + "email": "lock@example.com", + "content": "Test locking", + "timestamp": 1234567890, + "parent_id": None, + "author_ip": "127.0.0.1", + }, + ], + } + } + + json_content = json.dumps(test_data).encode("utf-8") + + # First import - postfix should be set + res1 = self.testapp.post( + "/ns/lock-test.example.com/import-comments", + { + "csrf_token": self.csrf, + "group-prefix": "mygrp", + }, + upload_files=[("json-file", "lock.json", json_content)], + status=200, + ) + + self.assertIn(b"Successfully imported", res1.body) + + # Reload namespace to check postfix was locked + namespace = get_namespace_by_name(self.dbsession, "lock-test.example.com") + self.assertEqual(namespace.import_group_postfix, "mygrp") + + # Second import should use locked postfix regardless of input + res2 = self.testapp.post( + "/ns/lock-test.example.com/import-comments", + { + "csrf_token": self.csrf, + "group-prefix": "different", # This should be ignored + }, + upload_files=[("json-file", "lock2.json", json_content)], + status=200, + ) + + # Verify postfix didn't change + namespace = get_namespace_by_name(self.dbsession, "lock-test.example.com") + self.assertEqual(namespace.import_group_postfix, "mygrp") + + def test_import_with_missing_email_creates_surrogates(self): + """Test that comments without email create unique surrogates""" + test_data = { + "test-surrogates": { + "link": "https://example.com/test-surrogates", + "comments": [ + { + "id": "s1", + "author": "Guest1", + "email": "", + "content": "First guest comment", + "timestamp": 1234567890, + "parent_id": None, + "author_ip": "127.0.0.1", + }, + { + "id": "s2", + "author": "Guest2", + "email": "", + "content": "Second guest comment", + "timestamp": 1234567900, + "parent_id": None, + "author_ip": "127.0.0.2", + }, + { + "id": "s3", + "author": "Guest1", # Same name as first + "email": "", + "content": "Another comment from Guest1", + "timestamp": 1234567910, + "parent_id": None, + "author_ip": "127.0.0.3", + }, + ], + } + } + + json_content = json.dumps(test_data).encode("utf-8") + + res = self.testapp.post( + "/ns/test.example.com/import-comments", + { + "csrf_token": self.csrf, + }, + upload_files=[("json-file", "surrogates.json", json_content)], + status=200, + ) + + self.assertIn(b"Successfully imported", res.body) + self.assertIn(b"3 comments", res.body) + + # Verify surrogates were created + from remarkbox.models import UserSurrogate + namespace = self.get_test_namespace() + surrogates = self.dbsession.query(UserSurrogate).filter( + UserSurrogate.namespace_id == namespace.id + ).all() + + # Should have 2 unique surrogates (Guest1 and Guest2) + # Guest1 appearing twice should reuse the same surrogate + self.assertEqual(len(surrogates), 2) + + # Verify surrogate names include group postfix + surrogate_names = [s.name for s in surrogates] + for name in surrogate_names: + self.assertIn("-", name) # Should have postfix + + def test_import_with_missing_fields(self): + """Test import handles missing optional fields gracefully""" + test_data = { + "test-missing-fields": { + "link": "https://example.com/test-missing", + "comments": [ + { + "id": "m1", + "author": "MinimalUser", + "email": "minimal@example.com", + "content": "Minimal comment", + "timestamp": 1234567890, + # Missing parent_id + # Missing author_ip + }, + ], + } + } + + json_content = json.dumps(test_data).encode("utf-8") + + res = self.testapp.post( + "/ns/test.example.com/import-comments", + { + "csrf_token": self.csrf, + }, + upload_files=[("json-file", "missing.json", json_content)], + status=200, + ) + + self.assertIn(b"Successfully imported", res.body) + self.assertIn(b"1 comments", res.body) + + def test_import_multiple_threads(self): + """Test importing multiple threads in one file""" + multi_thread_data = { + "thread-1": { + "link": "https://example.com/thread-1", + "comments": [ + { + "id": "t1-c1", + "author": "User1", + "email": "user1@example.com", + "content": "Comment on thread 1", + "timestamp": 1234567890, + "parent_id": None, + "author_ip": "127.0.0.1", + }, + ], + }, + "thread-2": { + "link": "https://example.com/thread-2", + "comments": [ + { + "id": "t2-c1", + "author": "User2", + "email": "user2@example.com", + "content": "Comment on thread 2", + "timestamp": 1234567900, + "parent_id": None, + "author_ip": "127.0.0.2", + }, + ], + }, + "thread-3": { + "link": "https://example.com/thread-3", + "comments": [ + { + "id": "t3-c1", + "author": "User3", + "email": "user3@example.com", + "content": "Comment on thread 3", + "timestamp": 1234567910, + "parent_id": None, + "author_ip": "127.0.0.3", + }, + ], + }, + } + + json_content = json.dumps(multi_thread_data).encode("utf-8") + + res = self.testapp.post( + "/ns/test.example.com/import-comments", + { + "csrf_token": self.csrf, + }, + upload_files=[("json-file", "multi.json", json_content)], + status=200, + ) + + self.assertIn(b"Successfully imported", res.body) + self.assertIn(b"3 threads", res.body) + self.assertIn(b"3 comments", res.body) + + def test_import_with_invalid_group_postfix(self): + """Test that invalid group postfix shows error""" + # Create a unique namespace for this test + invalid_test_ns = get_or_create_namespace(self.dbsession, "invalid-test.example.com") + invalid_test_ns.set_role_for_user(self.test_user, role="owner") + self.dbsession.add(invalid_test_ns) + self.dbsession.flush() + self.tm.commit() + + test_data = { + "test": { + "link": "https://example.com/test", + "comments": [ + { + "id": "1", + "author": "User", + "email": "user@example.com", + "content": "Test", + "timestamp": 1234567890, + "parent_id": None, + "author_ip": "127.0.0.1", + }, + ], + } + } + + json_content = json.dumps(test_data).encode("utf-8") + + # Try with too short postfix + res = self.testapp.post( + "/ns/invalid-test.example.com/import-comments", + { + "csrf_token": self.csrf, + "group-prefix": "x", # Only 1 char, minimum is 2 + }, + upload_files=[("json-file", "invalid.json", json_content)], + status=200, + ) + + self.assertIn(b"Group postfix is required and must be at least 2 alphanumeric characters", res.body) + + +class ImportCommentsUnitTests(unittest.TestCase): + """Unit tests for import_comments view functions""" + + def test_generate_password(self): + """Test password generation utility""" + from remarkbox.views.authenticated.import_comments import generate_password + + password = generate_password(32) + self.assertEqual(len(password), 32) + self.assertTrue(all(c.isalnum() for c in password)) + + def test_generate_password_custom_size(self): + """Test password generation with custom size""" + from remarkbox.views.authenticated.import_comments import generate_password + + password = generate_password(16) + self.assertEqual(len(password), 16) + + def test_generate_group_prefix_from_namespace(self): + """Test group prefix generation from namespace domain""" + from remarkbox.views.authenticated.import_comments import generate_group_prefix_from_namespace + + # Test multi-part domain + self.assertEqual(generate_group_prefix_from_namespace("russell.ballestrini.net"), "rb") + + # Test with www prefix + self.assertEqual(generate_group_prefix_from_namespace("www.example.com"), "exampl") + + # Test single-part domain + self.assertEqual(generate_group_prefix_from_namespace("example.com"), "exampl") + + # Test my.remarkbox.com style + prefix = generate_group_prefix_from_namespace("my.remarkbox.com") + self.assertTrue(len(prefix) <= 6) + self.assertTrue(prefix.isalnum()) diff --git a/remarkbox/views/authenticated/import_comments.py b/remarkbox/views/authenticated/import_comments.py new file mode 100644 index 0000000..a420d2f --- /dev/null +++ b/remarkbox/views/authenticated/import_comments.py @@ -0,0 +1,276 @@ +from pyramid.view import view_config +from pyramid.httpexceptions import HTTPFound +import json +import tempfile +import os +from datetime import datetime + +from remarkbox.models import ( + User, + UserSurrogate, + generate_user_name, + get_user_by_email, + get_user_surrogate_by_name, + get_or_create_node_by_uri, + is_user_name_valid, + is_user_name_available, +) + +from remarkbox.views import get_referer_or_home, user_required + +try: + unicode("") +except: + from six import u as unicode + + +def generate_password(size=32): + """Return a system generated password""" + from random import choice + letters = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" + digits = "0123456789" + pool = letters + digits + return "".join([choice(pool) for i in range(size)]) + + +def generate_group_prefix_from_namespace(namespace_name): + """ + Generate a short, safe group prefix from namespace domain. + Takes first letters of domain parts, max 6 chars. + Example: russell.ballestrini.net -> rb, my.remarkbox.com -> mrc + """ + # Remove common TLDs and split on dots + domain = namespace_name.lower() + # Remove www prefix if present + if domain.startswith('www.'): + domain = domain[4:] + + # Split on dots and filter out common TLDs + parts = domain.split('.') + # Remove TLDs like com, org, net, io, etc + tlds = {'com', 'org', 'net', 'io', 'co', 'edu', 'gov', 'mil', 'int'} + parts = [p for p in parts if p not in tlds] + + if not parts: + # Use first 6 chars of full domain if no valid parts + prefix = ''.join(c for c in namespace_name if c.isalnum())[:6] + if len(prefix) < 2: + # Absolutely no valid characters, use hash of namespace + import hashlib + prefix = hashlib.md5(namespace_name.encode()).hexdigest()[:6] + return prefix + + # Strategy 1: Take first letter of each part (e.g., russell.ballestrini -> rb) + prefix = ''.join(p[0] for p in parts if p) + + # Strategy 2: If too short or just one part, take first few chars + if len(prefix) < 2 or len(parts) == 1: + prefix = parts[0][:6] + + # Ensure max 6 chars and only alphanumeric + prefix = ''.join(c for c in prefix if c.isalnum())[:6] + + # Final safety check - ensure at least 2 chars + if len(prefix) < 2: + # Use hash of namespace as last resort + import hashlib + prefix = hashlib.md5(namespace_name.encode()).hexdigest()[:6] + + return prefix + + + + +@view_config(route_name="basic-namespace-import-comments", renderer="import-comments.j2") +@view_config(route_name="embed-namespace-import-comments", renderer="import-comments.j2") +@user_required() +def import_comments(request): + """ + Import comments and threads from JSON dumps created by the blog-to-json tool. + Supports WordPress, Disqus, and pre-formatted JSON exports. + This view handles file upload and processes the JSON data. + """ + + if not request.user in request.namespace.owners: + request.session.flash(("You do not own that Namespace.", "error")) + return HTTPFound(get_referer_or_home(request)) + + # Check if namespace already has a locked group postfix + if request.namespace.import_group_postfix: + # Use the locked postfix + default_postfix = request.namespace.import_group_postfix + postfix_locked = True + else: + # Generate default group postfix from namespace + default_postfix = generate_group_prefix_from_namespace(request.namespace.name) + postfix_locked = False + + if request.method == "POST": + # Get the group postfix, allow user to override if not locked + if postfix_locked: + group = request.namespace.import_group_postfix + else: + group_postfix = request.params.get("group-prefix", "").strip() + if not group_postfix: + group_postfix = default_postfix + + # Sanitize: max 6 chars, alphanumeric only + group = ''.join(c for c in group_postfix if c.isalnum())[:6] + if not group or len(group) < 2: + request.session.flash(("Group postfix is required and must be at least 2 alphanumeric characters.", "error")) + return { + "the_title": "Import Comments", + "default_prefix": default_postfix, + "postfix_locked": postfix_locked, + } + + # Lock the group postfix in the namespace on first use + request.namespace.import_group_postfix = group + request.dbsession.add(request.namespace) + request.dbsession.flush() + + # Get the uploaded file + upload_file = request.params.get("json-file", None) + + if upload_file is None or upload_file == b'': + request.session.flash(("Please select a JSON file to upload.", "error")) + return {"the_title": "Import Comments"} + + try: + # Read the uploaded file + json_content = upload_file.file.read() + + # Parse JSON + try: + pages = json.loads(json_content) + except json.JSONDecodeError as e: + request.session.flash((f"Invalid JSON file: {str(e)}", "error")) + return {"the_title": "Import Comments"} + + # Process the import + users = {} + user_surrogates = {} + nodes = {} + imported_threads = 0 + imported_comments = 0 + + for slug, page_data in pages.items(): + comments = page_data.get("comments", []) + + if len(comments) == 0: + continue + + uri = page_data.get("link", "") + if not uri: + continue + + # Create the root node for this thread + root = get_or_create_node_by_uri(request.dbsession, uri) + root.namespace = request.namespace + request.dbsession.add(root) + request.dbsession.flush() + imported_threads += 1 + + for comment in comments: + email = comment.get("email", "") + + user = None + user_surrogate = None + + if not email: + # Always create surrogates for comments without email + # Use group postfix to make them identifiable + author_name = comment.get("author", "Anonymous") + # Create full name with group postfix (e.g., "Anonymous-rb-20231122-143045") + surrogate_name_with_group = f"{author_name}-{group}" + + # Check in-memory cache first + if surrogate_name_with_group not in user_surrogates: + # Check if this surrogate already exists in the database + existing_surrogate = get_user_surrogate_by_name( + request.dbsession, + surrogate_name_with_group, + root.namespace + ) + if existing_surrogate: + user_surrogates[surrogate_name_with_group] = existing_surrogate + else: + # Create new surrogate + user_surrogates[surrogate_name_with_group] = UserSurrogate( + surrogate_name_with_group, + root.namespace, + ) + request.dbsession.add(user_surrogates[surrogate_name_with_group]) + request.dbsession.flush() + + user_surrogate = user_surrogates[surrogate_name_with_group] + + elif email in users: + user = users[email] + else: + user = get_user_by_email(request.dbsession, email) + if user: + users[email] = user + else: + desired_name = ( + comment.get("author", "Anonymous").replace(" ", "-").replace("_", "-") + ) + # Always append group name to identify imported users + desired_name = desired_name + "-" + group + + if not is_user_name_valid( + desired_name + ) or not is_user_name_available( + request.dbsession, desired_name + ): + desired_name = generate_user_name( + request.dbsession, desired_name + ) + user = User(email) + user.name = unicode(desired_name) + users[email] = user + + request.dbsession.add(user) + request.dbsession.flush() + + # Create the comment node + node = root.new_child() + node.set_data(unicode(comment.get("content", ""))) + node.created = int(comment.get("timestamp", 0)) * 1000 + node.changed = int(comment.get("timestamp", 0)) * 1000 + + if user is not None: + node.user = user + elif user_surrogate is not None: + node.user_surrogate = user_surrogate + + node.verified = True + node.ip_address = comment.get("author_ip", None) + nodes[comment.get("id", "")] = node + + request.dbsession.add(node) + request.dbsession.flush() + imported_comments += 1 + + # Set up parent-child relationships + for comment in comments: + parent_id = comment.get("parent_id") + comment_id = comment.get("id", "") + if parent_id and nodes.get(comment_id) and nodes.get(parent_id): + nodes[comment_id].parent_id = nodes[parent_id].id + request.dbsession.add(nodes[comment_id]) + request.dbsession.flush() + + request.session.flash(( + f"Successfully imported {imported_threads} threads and {imported_comments} comments.", + "success" + )) + + except Exception as e: + request.session.flash((f"Error during import: {str(e)}", "error")) + + return { + "the_title": "Import Comments", + "default_prefix": default_postfix, + "postfix_locked": postfix_locked, + }