Add self-extracting archives with bundled neopig

- Bootstrap C program extracts serve.py and runs from tarball
- Archives now include neopig source files for self-contained crawling
- --upgrade-neopig flag to update neopig in existing archives
- html2md.py: smart HTML-to-markdown converter for forums/blogs/Q&A
- Fix vault path defaults (data/vault instead of vault)
- Streaming tar.gz creation without temp copies
- URL rewriting for local media references in archives

make-executable.sh

@@ -0,0 +1,75 @@
+#!/bin/bash
+#
+# make-executable.sh - Create self-extracting searchable archive
+#
+# Usage:
+#   ./make-executable.sh archive.tar.gz
+#   ./archive.run
+#
+# The resulting .run file is a single executable that:
+#   1. Extracts the Python serve.py to /tmp
+#   2. Runs it pointing at itself as the tarball source
+#   3. Serves the archive on http://localhost:6543
+#
+# Requirements:
+#   - gcc with zlib (-lz)
+#   - python3 with pyramid installed (on target system)
+
+set -e
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+BOOTSTRAP_C="$SCRIPT_DIR/bootstrap.c"
+
+if [ $# -lt 1 ]; then
+    echo "Usage: $0 <archive.tar.gz> [output.run]"
+    echo ""
+    echo "Creates a self-extracting searchable archive."
+    echo "The output file can be run directly to start the search server."
+    exit 1
+fi
+
+TARBALL="$1"
+OUTPUT="${2:-${TARBALL%.tar.gz}.run}"
+
+if [ ! -f "$TARBALL" ]; then
+    echo "Error: Tarball not found: $TARBALL"
+    exit 1
+fi
+
+if [ ! -f "$BOOTSTRAP_C" ]; then
+    echo "Error: bootstrap.c not found: $BOOTSTRAP_C"
+    exit 1
+fi
+
+# Create temp directory
+TMPDIR=$(mktemp -d)
+trap "rm -rf $TMPDIR" EXIT
+
+echo "Compiling bootstrap..."
+gcc -O2 -o "$TMPDIR/bootstrap" "$BOOTSTRAP_C" -lz
+
+BOOTSTRAP_SIZE=$(stat -c%s "$TMPDIR/bootstrap" 2>/dev/null || stat -f%z "$TMPDIR/bootstrap")
+echo "Bootstrap size: $BOOTSTRAP_SIZE bytes"
+
+echo "Creating self-extracting archive..."
+
+# Concatenate: bootstrap + tarball + magic + offset
+cat "$TMPDIR/bootstrap" "$TARBALL" > "$TMPDIR/combined"
+
+# Append magic marker
+echo -n "NEOPIG" >> "$TMPDIR/combined"
+
+# Append offset as 16-char hex string
+printf '%016x' "$BOOTSTRAP_SIZE" >> "$TMPDIR/combined"
+
+# Make executable and move to output
+chmod +x "$TMPDIR/combined"
+mv "$TMPDIR/combined" "$OUTPUT"
+
+FINAL_SIZE=$(stat -c%s "$OUTPUT" 2>/dev/null || stat -f%z "$OUTPUT")
+echo ""
+echo "Created: $OUTPUT ($FINAL_SIZE bytes)"
+echo ""
+echo "To run:"
+echo "  ./$OUTPUT"
+echo "  # Opens http://localhost:6543"