From 50b9d1ee8eb2fa891819a21fefd0b4d4fee292e6 Mon Sep 17 00:00:00 2001
From: "russell@unturf.com" <russell@unturf.com>
Date: Fri, 5 Jun 2026 21:03:59 -0400
Subject: [PATCH] feat: source_format auto-detect with expandable detector
 registry
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The create-form selector forced authors to remember to switch from
markdown to rst before typing. Auto-detect runs on every preview
keystroke + at form submit time, picking the right format from
content shape so authors just write.

SOURCE_FORMAT_DETECTORS in custom.js carries the registry — first
match wins, ordered most-specific first. Today: html, latex,
mediawiki, rst, with markdown as the catch-all default.

Adding a new format (e.g. asciidoc, org, ipynb):
  1. Push one entry to SOURCE_FORMAT_DETECTORS in custom.js
  2. Add one <option> to source_format_select in create.j2
  3. Confirm pandoc accepts the format name

No Python changes needed for new formats — the existing
preview_post + new_thread paths pass source_format through to
pandoc untouched.

UX:
  - Default selector value: 'auto' (was 'markdown')
  - "detected: rst" label appears next to selector when auto picks
  - Manual override options stay for ambiguous content
  - Form onsubmit resolves 'auto' to a concrete format before POST
    so node.source_format never persists as the literal 'auto'

Defense in depth: new_thread.py + preview_post both treat 'auto'
as a no-op (falls to markdown default) for no-JS clients or racy
submits.
---
 remarkbox/static/js/custom.js          | 97 +++++++++++++++++++++++++-
 remarkbox/templates/snippets/create.j2 | 11 ++-
 remarkbox/views/misc.py                |  3 +
 remarkbox/views/new_thread.py          |  5 ++
 4 files changed, 111 insertions(+), 5 deletions(-)

diff --git a/remarkbox/static/js/custom.js b/remarkbox/static/js/custom.js
index d125e59..88bbf03 100644
--- a/remarkbox/static/js/custom.js
+++ b/remarkbox/static/js/custom.js
@@ -78,15 +78,108 @@ function previewAjax(textarea, div, show_raw, mathjax) {
     );
 }
 
+// ===== source_format detection — expandable registry ========================
+// First match wins. To add a new format:
+//   1. Push a new entry into SOURCE_FORMAT_DETECTORS below.
+//   2. Add a matching <option value="..."> to source_format_select in any
+//      template that exposes the selector (create.j2 today).
+//   3. Confirm remarkbox/lib/pandoc.py + the preview_post view accept the
+//      format name (pandoc subprocess uses whatever string you pass).
+// Detection runs on every preview keystroke + on form submit when the
+// selector is set to "auto". For "auto" cases we resolve to a concrete
+// format before saving so the persisted node.source_format is honest.
+// Order detectors most-specific → least-specific; markdown is the catch-all
+// fallback when nothing matches.
+var SOURCE_FORMAT_DETECTORS = [
+    {
+        format: 'html',
+        // Unambiguous root markers — full HTML documents.
+        test: function(text) {
+            return /<!doctype\s+html|<html\b/i.test(text);
+        }
+    },
+    {
+        format: 'latex',
+        test: function(text) {
+            return /\\documentclass\b|\\begin\{document\}|\\section\{|\\subsection\{/.test(text);
+        }
+    },
+    {
+        format: 'mediawiki',
+        // Need both heading + bold pattern to distinguish from rst/md.
+        test: function(text) {
+            return /^==+\s+.+\s+==+\s*$/m.test(text) && /'''[^']/.test(text);
+        }
+    },
+    {
+        format: 'rst',
+        test: function(text) {
+            if (/^\.\.\s+[a-z][\w-]*::/m.test(text)) return true;   // directives
+            if (/^\.\.\s+_\S+:/m.test(text)) return true;            // link targets
+            // Section underline: line made of one repeated char of length
+            // >= the previous non-empty line's length.
+            var lines = text.split('\n');
+            for (var i = 1; i < lines.length; i++) {
+                var prev = lines[i-1].trim();
+                var curr = lines[i].trim();
+                if (prev.length >= 3 &&
+                    /^([=\-~^"'`*+#])\1+$/.test(curr) &&
+                    curr.length >= prev.length) {
+                    return true;
+                }
+            }
+            return false;
+        }
+    }
+];
+
+function detectSourceFormat(text) {
+    if (!text || text.length < 3) return null;
+    for (var i = 0; i < SOURCE_FORMAT_DETECTORS.length; i++) {
+        try {
+            if (SOURCE_FORMAT_DETECTORS[i].test(text)) {
+                return SOURCE_FORMAT_DETECTORS[i].format;
+            }
+        } catch (e) { /* malformed regex / bad text — skip this detector */ }
+    }
+    return null;
+}
+
+// Hook called from create.j2 form onsubmit. When source_format_select is on
+// 'auto' we replace its value with the detected format so the POST carries
+// a concrete name. Falls to 'markdown' when detection finds nothing.
+function resolveAutoSourceFormat(form) {
+    var sel = form.querySelector('#source_format_select');
+    var ta = form.querySelector('#thread_data_textarea');
+    if (sel && ta && sel.value === 'auto') {
+        sel.value = detectSourceFormat(ta.value) || 'markdown';
+    }
+    return true;
+}
+
+// Show "detected: rst" next to the selector so authors see what auto picked.
+function updateDetectionLabel(ta, detected) {
+    var span = document.querySelector('#detected_format_label');
+    if (!span) return;
+    span.textContent = detected ? ('detected: ' + detected) : '';
+}
+
 function sendPreview(textarea, div, mathjax) {
     var url = '/preview-post';
     var ta = document.getElementById(textarea);
     var data = new FormData();
     data.append('data', ta.value);
     data.append('csrf_token', csrf_token);
-    // textarea declares its source_format via data-source-format; the
-    // endpoint dispatches to pandoc for rst/html/mediawiki/latex when set.
+    // textarea declares its source_format via data-source-format. When the
+    // value is 'auto' (default on create form), run client-side detection
+    // and substitute the concrete format before posting. The endpoint
+    // dispatches to pandoc for rst/html/mediawiki/latex/etc.
     var fmt = ta.dataset.sourceFormat;
+    if (fmt === 'auto') {
+        var detected = detectSourceFormat(ta.value);
+        updateDetectionLabel(ta, detected);
+        fmt = detected || 'markdown';
+    }
     if (fmt) data.append('source_format', fmt);
 
     fetch(url, {
diff --git a/remarkbox/templates/snippets/create.j2 b/remarkbox/templates/snippets/create.j2
index 5ba1b5a..e341379 100644
--- a/remarkbox/templates/snippets/create.j2
+++ b/remarkbox/templates/snippets/create.j2
@@ -1,4 +1,4 @@
- <form method="post" action="" onsubmit="submit.disabled = true; return true;">
+ <form method="post" action="" onsubmit="resolveAutoSourceFormat(this); submit.disabled = true; return true;">
     <input
         name        = "thread_title"
         id          = "thread_title_input"
@@ -11,15 +11,19 @@
         name        = "thread_data"
         id          = "thread_data_textarea"
         class       = "common-textarea"
-        data-source-format = "markdown"
+        data-source-format = "auto"
         onkeyup     = "previewAjax( 'thread_data_textarea', 'preview', show_raw=true, mathjax={{ request.mathjax }} )"
         placeholder = "What do you want to say? markdown"
         required></textarea>
 
   <label for="source_format_select" style="display:inline-block;margin-right:0.5em">format:</label>
+  {# Source-format selector. 'auto' defers to detectSourceFormat() in custom.js,
+     which runs an expandable detector registry. To add a new format here, push
+     an entry to SOURCE_FORMAT_DETECTORS in custom.js + add a <option> below. #}
   <select id="source_format_select" name="source_format"
           onchange="document.getElementById('thread_data_textarea').dataset.sourceFormat = this.value; previewAjax('thread_data_textarea', 'preview', show_raw=true, mathjax={{ request.mathjax }})">
-    <option value="markdown" selected>markdown</option>
+    <option value="auto" selected>auto-detect</option>
+    <option value="markdown">markdown</option>
     <option value="rst">reStructuredText</option>
     <option value="html">html</option>
     <option value="mediawiki">mediawiki</option>
@@ -28,6 +32,7 @@
     <option value="asciidoc">asciidoc</option>
     <option value="textile">textile</option>
   </select>
+  <span id="detected_format_label" style="margin-left:0.5em;font-size:0.85em;color:#888"></span>
 
   <div id='preview' name='preview' class="preview"></div>
 
diff --git a/remarkbox/views/misc.py b/remarkbox/views/misc.py
index ace245f..5c0f49f 100644
--- a/remarkbox/views/misc.py
+++ b/remarkbox/views/misc.py
@@ -64,6 +64,9 @@ def preview_post(request):
     """
     data = request.params.get("data", "")
     source_format = (request.params.get("source_format") or "markdown").strip().lower()
+    # JS auto-resolves to a concrete format, but defend against stragglers.
+    if source_format == "auto":
+        source_format = "markdown"
     try:
         if source_format in ("", "markdown"):
             return markdown_to_html(data, request.namespace, dbsession=request.dbsession)
diff --git a/remarkbox/views/new_thread.py b/remarkbox/views/new_thread.py
index 2e0f914..ecd6767 100644
--- a/remarkbox/views/new_thread.py
+++ b/remarkbox/views/new_thread.py
@@ -23,6 +23,11 @@ def new_thread(request):
     thread_data = request.params.get("thread_data", "")
     anonymous_name = request.params.get("anonymous_name", "").strip()
     source_format = request.params.get("source_format", "").strip() or None
+    # JS resolves 'auto' to a concrete format before submit, but defend against
+    # no-JS clients / racy submits — never persist the literal string 'auto'
+    # as a node.source_format. None falls back to markdown via set_data.
+    if source_format == "auto":
+        source_format = None
 
     if request.spam:
         return request.spam