Reorganize project structure: move modules to neopig/, scripts to scripts/, data to data/

scripts/bootstrap-neopig.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+set -e
+
+# Clone or update neopig from git (idempotent)
+if [ -d "neopig" ]; then
+    cd neopig
+    git fetch origin
+    git reset --hard origin/main
+else
+    git clone https://git.unturf.com/engineering/unturf/pig.py.git neopig
+    cd neopig
+fi
+
+# Create virtualenv and install dependencies
+python3 -m venv .venv
+source .venv/bin/activate
+pip install -r requirements.txt
+
+# Create data directory for database
+mkdir -p data
+
+# Kill any existing neopig server before starting (for redeployments)
+pkill -f "serp.py" || true
+sleep 1
+
+# Run neopig.py --serve on port 80 with import mode (tar.gz uploads)
+NEOPIG_IMPORT=1 NEOPIG_DISABLE_CRAWL=1 python neopig.py --serve --host 0.0.0.0 --port 80

scripts/bootstrap.c

@@ -0,0 +1,397 @@
+/*
+ * bootstrap.c - Self-extracting archive bootstrap
+ *
+ * A small C program that boots neopig from an embedded tar.gz archive.
+ * When compiled and concatenated with a tar.gz archive, it:
+ *   1. Extracts neopig/*.py from the embedded tarball to /tmp
+ *   2. Runs: python3 /tmp/neopig/serp.py <self>
+ *   3. The Python script serves directly from the tarball portion
+ *
+ * Build:
+ *   gcc -O2 -o bootstrap bootstrap.c -lz
+ *
+ * Create self-extracting archive:
+ *   cat bootstrap archive.tar.gz > archive.run
+ *   chmod +x archive.run
+ *   echo -n "NEOPIG" >> archive.run           # Magic marker
+ *   printf '%016x' $(stat -c%s bootstrap) >> archive.run  # Offset (hex)
+ *
+ * Or use: make run TARBALL=archive.tar.gz
+ */
+
+#define _GNU_SOURCE
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <sys/stat.h>
+#include <sys/wait.h>
+#include <fcntl.h>
+#include <errno.h>
+#include <zlib.h>
+
+#define MAGIC "NEOPIG"
+#define MAGIC_LEN 6
+#define OFFSET_LEN 16
+#define TRAILER_LEN (MAGIC_LEN + OFFSET_LEN)
+
+#define CHUNK_SIZE 16384
+
+/* Find python3 interpreter */
+static const char *find_python(void) {
+    static const char *pythons[] = {
+        "/usr/bin/python3",
+        "/usr/local/bin/python3",
+        "/opt/homebrew/bin/python3",
+        "python3",
+        NULL
+    };
+
+    for (int i = 0; pythons[i]; i++) {
+        if (access(pythons[i], X_OK) == 0) {
+            return pythons[i];
+        }
+    }
+
+    /* Try PATH lookup */
+    return "python3";
+}
+
+/* Read trailer to get tarball offset */
+static long read_trailer(const char *self_path) {
+    FILE *f = fopen(self_path, "rb");
+    if (!f) {
+        perror("fopen self");
+        return -1;
+    }
+
+    /* Seek to trailer */
+    if (fseek(f, -TRAILER_LEN, SEEK_END) != 0) {
+        perror("fseek trailer");
+        fclose(f);
+        return -1;
+    }
+
+    char trailer[TRAILER_LEN + 1];
+    if (fread(trailer, 1, TRAILER_LEN, f) != TRAILER_LEN) {
+        perror("fread trailer");
+        fclose(f);
+        return -1;
+    }
+    trailer[TRAILER_LEN] = '\0';
+    fclose(f);
+
+    /* Check magic */
+    if (memcmp(trailer, MAGIC, MAGIC_LEN) != 0) {
+        fprintf(stderr, "Error: Invalid archive (missing NEOPIG magic)\n");
+        fprintf(stderr, "This executable must be created with make-executable.sh\n");
+        return -1;
+    }
+
+    /* Parse hex offset */
+    long offset = strtol(trailer + MAGIC_LEN, NULL, 16);
+    return offset;
+}
+
+/* Create directory recursively */
+static int mkdir_p(const char *path) {
+    char tmp[4096];
+    char *p = NULL;
+    size_t len;
+
+    snprintf(tmp, sizeof(tmp), "%s", path);
+    len = strlen(tmp);
+    if (tmp[len - 1] == '/') tmp[len - 1] = 0;
+
+    for (p = tmp + 1; *p; p++) {
+        if (*p == '/') {
+            *p = 0;
+            mkdir(tmp, 0755);
+            *p = '/';
+        }
+    }
+    return mkdir(tmp, 0755);
+}
+
+/* Extract neopig/*.py from gzipped tarball */
+static char *extract_neopig(const char *self_path, long tar_offset) {
+    /* Create temp directory */
+    char *temp_dir = strdup("/tmp/neopig_XXXXXX");
+    if (!temp_dir) return NULL;
+
+    if (!mkdtemp(temp_dir)) {
+        perror("mkdtemp");
+        free(temp_dir);
+        return NULL;
+    }
+
+    /* Open self and seek to tarball */
+    FILE *self = fopen(self_path, "rb");
+    if (!self) {
+        perror("fopen self for tar");
+        free(temp_dir);
+        return NULL;
+    }
+
+    if (fseek(self, tar_offset, SEEK_SET) != 0) {
+        perror("fseek to tar");
+        fclose(self);
+        free(temp_dir);
+        return NULL;
+    }
+
+    /* Open gzip stream */
+    gzFile gz = gzdopen(dup(fileno(self)), "rb");
+    if (!gz) {
+        fprintf(stderr, "gzdopen failed\n");
+        fclose(self);
+        free(temp_dir);
+        return NULL;
+    }
+    fclose(self);
+
+    /* Read tar headers looking for neopig/*.py */
+    unsigned char header[512];
+    int found_any = 0;
+
+    while (gzread(gz, header, 512) == 512) {
+        /* Check for end of archive (all zeros) */
+        int all_zero = 1;
+        for (int i = 0; i < 512 && all_zero; i++) {
+            if (header[i] != 0) all_zero = 0;
+        }
+        if (all_zero) break;
+
+        /* Get filename (first 100 bytes) */
+        char filename[101];
+        memcpy(filename, header, 100);
+        filename[100] = '\0';
+
+        /* Get file size (octal, bytes 124-135) */
+        char size_str[13];
+        memcpy(size_str, header + 124, 12);
+        size_str[12] = '\0';
+        long filesize = strtol(size_str, NULL, 8);
+
+        /* Check if this is a neopig file we want to extract */
+        char *neopig_pos = strstr(filename, "/neopig/");
+        char *py_ext = strstr(filename, ".py");
+        char *req_file = strstr(filename, "/requirements.txt");
+        char *static_vendor = strstr(filename, "/neopig/static/vendor/");
+        char *css_ext = strstr(filename, ".css");
+        char *js_ext = strstr(filename, ".js");
+
+        int is_neopig_py = (neopig_pos && py_ext && py_ext > neopig_pos);
+        int is_requirements = (req_file != NULL);
+        int is_static_asset = (static_vendor && (css_ext || js_ext));
+
+        if (is_neopig_py || is_requirements || is_static_asset) {
+            /* Extract this file */
+            char out_path[4096];
+            if (is_static_asset) {
+                /* Create static/vendor/ subdirectory */
+                char vendor_dir[4096];
+                snprintf(vendor_dir, sizeof(vendor_dir), "%s/static/vendor", temp_dir);
+                mkdir(vendor_dir, 0755);
+                char static_dir[4096];
+                snprintf(static_dir, sizeof(static_dir), "%s/static", temp_dir);
+                mkdir(static_dir, 0755);
+                mkdir(vendor_dir, 0755);
+                char *basename = static_vendor + 22;  /* Skip "/neopig/static/vendor/" */
+                snprintf(out_path, sizeof(out_path), "%s/static/vendor/%s", temp_dir, basename);
+            } else if (is_neopig_py) {
+                char *basename = neopig_pos + 8;  /* Skip "/neopig/" */
+                snprintf(out_path, sizeof(out_path), "%s/%s", temp_dir, basename);
+            } else {
+                snprintf(out_path, sizeof(out_path), "%s/requirements.txt", temp_dir);
+            }
+
+            int fd = open(out_path, O_WRONLY | O_CREAT | O_TRUNC, 0644);
+            if (fd >= 0) {
+                unsigned char buf[CHUNK_SIZE];
+                long remaining = filesize;
+
+                while (remaining > 0) {
+                    int to_read = remaining > CHUNK_SIZE ? CHUNK_SIZE : remaining;
+                    int got = gzread(gz, buf, to_read);
+                    if (got <= 0) break;
+                    write(fd, buf, got);
+                    remaining -= got;
+                }
+                close(fd);
+                found_any = 1;
+                if (is_static_asset) {
+                    char *basename = static_vendor + 22;
+                    printf("  Extracted: static/vendor/%s\n", basename);
+                } else if (is_neopig_py) {
+                    char *basename = neopig_pos + 8;
+                    printf("  Extracted: %s\n", basename);
+                } else {
+                    printf("  Extracted: requirements.txt\n");
+                }
+
+                /* Skip padding to 512 boundary */
+                int pad = (512 - (filesize % 512)) % 512;
+                if (pad > 0) {
+                    gzread(gz, buf, pad);
+                }
+            } else {
+                /* Skip this file's content */
+                long blocks = (filesize + 511) / 512;
+                for (long i = 0; i < blocks; i++) {
+                    gzread(gz, header, 512);
+                }
+            }
+        } else {
+            /* Skip this file's content (padded to 512 bytes) */
+            long blocks = (filesize + 511) / 512;
+            for (long i = 0; i < blocks; i++) {
+                if (gzread(gz, header, 512) != 512) break;
+            }
+        }
+    }
+
+    gzclose(gz);
+
+    if (!found_any) {
+        fprintf(stderr, "Error: No neopig/*.py files found in archive\n");
+        free(temp_dir);
+        return NULL;
+    }
+
+    return temp_dir;
+}
+
+int main(int argc, char *argv[]) {
+    /* Get path to self */
+    char self_path[4096];
+    ssize_t len = readlink("/proc/self/exe", self_path, sizeof(self_path) - 1);
+    if (len < 0) {
+        /* Fallback to argv[0] */
+        if (argv[0][0] == '/') {
+            strncpy(self_path, argv[0], sizeof(self_path) - 1);
+        } else {
+            char *cwd = getcwd(NULL, 0);
+            snprintf(self_path, sizeof(self_path), "%s/%s", cwd, argv[0]);
+            free(cwd);
+        }
+    } else {
+        self_path[len] = '\0';
+    }
+
+    printf("neopig self-extracting archive\n");
+    printf("==============================\n");
+
+    /* Read trailer to get tarball offset */
+    long tar_offset = read_trailer(self_path);
+    if (tar_offset < 0) {
+        return 1;
+    }
+
+    printf("Archive offset: %ld bytes\n", tar_offset);
+
+    /* Extract neopig directory */
+    printf("Extracting neopig...\n");
+    char *neopig_dir = extract_neopig(self_path, tar_offset);
+    if (!neopig_dir) {
+        return 1;
+    }
+
+    /* Build path to serp.py */
+    char serp_path[4096];
+    snprintf(serp_path, sizeof(serp_path), "%s/serp.py", neopig_dir);
+
+    /* Find python */
+    const char *python = find_python();
+
+    /* Create virtualenv and install deps */
+    char venv_path[4096];
+    snprintf(venv_path, sizeof(venv_path), "%s/.venv", neopig_dir);
+
+    char req_path[4096];
+    snprintf(req_path, sizeof(req_path), "%s/requirements.txt", neopig_dir);
+
+    /* Check if requirements.txt exists */
+    if (access(req_path, F_OK) == 0) {
+        printf("\nCreating virtualenv...\n");
+        char venv_cmd[4096];
+        snprintf(venv_cmd, sizeof(venv_cmd), "%s -m venv %s", python, venv_path);
+        if (system(venv_cmd) != 0) {
+            fprintf(stderr, "Failed to create virtualenv\n");
+            free(neopig_dir);
+            return 1;
+        }
+
+        printf("Installing dependencies...\n");
+        char pip_cmd[8192];
+        snprintf(pip_cmd, sizeof(pip_cmd),
+            "%s/.venv/bin/pip install -r %s", neopig_dir, req_path);
+        if (system(pip_cmd) != 0) {
+            fprintf(stderr, "Failed to install dependencies\n");
+            free(neopig_dir);
+            return 1;
+        }
+
+        /* Use venv python */
+        char venv_python[4096];
+        snprintf(venv_python, sizeof(venv_python), "%s/.venv/bin/python", neopig_dir);
+
+        printf("\nStarting server...\n");
+        printf("Command: %s %s %s\n\n", venv_python, serp_path, self_path);
+
+        /* Fork and exec with venv python
+         * IMPORTANT: Use full path as argv[0] for proper venv detection */
+        pid_t pid = fork();
+        if (pid < 0) {
+            perror("fork");
+            free(neopig_dir);
+            return 1;
+        }
+        if (pid == 0) {
+            execl(venv_python, venv_python, serp_path, self_path, NULL);
+            perror("execl venv python");
+            _exit(1);
+        }
+        int status;
+        waitpid(pid, &status, 0);
+
+        printf("\nCleaning up %s...\n", neopig_dir);
+        char rm_cmd[4096];
+        snprintf(rm_cmd, sizeof(rm_cmd), "rm -rf %s", neopig_dir);
+        system(rm_cmd);
+        free(neopig_dir);
+        return WIFEXITED(status) ? WEXITSTATUS(status) : 1;
+    }
+
+    /* No requirements.txt - use system python */
+    printf("\nStarting server (no venv)...\n");
+    printf("Command: %s %s %s\n\n", python, serp_path, self_path);
+
+    /* Fork and exec */
+    pid_t pid = fork();
+    if (pid < 0) {
+        perror("fork");
+        free(neopig_dir);
+        return 1;
+    }
+
+    if (pid == 0) {
+        /* Child - exec python (use full path as argv[0]) */
+        execl(python, python, serp_path, self_path, NULL);
+        perror("execl python");
+        _exit(1);
+    }
+
+    /* Parent - wait for child */
+    int status;
+    waitpid(pid, &status, 0);
+
+    /* Cleanup temp directory */
+    printf("\nCleaning up %s...\n", neopig_dir);
+    char rm_cmd[4096];
+    snprintf(rm_cmd, sizeof(rm_cmd), "rm -rf %s", neopig_dir);
+    system(rm_cmd);
+    free(neopig_dir);
+
+    return WIFEXITED(status) ? WEXITSTATUS(status) : 1;
+}

scripts/inject_rules.py

@@ -0,0 +1,70 @@
+#!/usr/bin/env python3
+"""Inject Benedict rules from rules_benedict.py into i18n.py."""
+
+import re
+
+def inject_rules():
+    # Import rules
+    from rules_benedict import RULES
+
+    # Read current i18n.py
+    with open("i18n.py", "r") as f:
+        content = f.read()
+
+    # For each language in RULES, add the rules to the corresponding language in TRANSLATIONS
+    for lang, rules in RULES.items():
+        if lang == "en":
+            continue  # Already added
+
+        print(f"Processing {lang}...")
+
+        # Find the language dict in i18n.py
+        lang_pattern = f'"{lang}": {{'
+        lang_start = content.find(lang_pattern)
+        if lang_start == -1:
+            print(f"  Could not find '{lang}' in i18n.py")
+            continue
+
+        # Find the closing of this language dict
+        brace_count = 0
+        pos = lang_start
+        in_lang = False
+        insert_pos = None
+        for i, ch in enumerate(content[lang_start:]):
+            if ch == '{':
+                brace_count += 1
+                in_lang = True
+            elif ch == '}':
+                brace_count -= 1
+                if brace_count == 0 and in_lang:
+                    insert_pos = lang_start + i
+                    break
+
+        if insert_pos is None:
+            print(f"  Could not find end of '{lang}' dict")
+            continue
+
+        # Check if rules already exist
+        if f'"rule_1":' in content[lang_start:insert_pos]:
+            print(f"  Rules already exist for {lang}, skipping")
+            continue
+
+        # Generate rules string
+        rules_str = ""
+        for key, value in rules.items():
+            # Escape quotes in value
+            escaped = value.replace('\\', '\\\\').replace('"', '\\"')
+            rules_str += f'        "{key}": "{escaped}",\n'
+
+        # Insert before the closing }
+        content = content[:insert_pos] + rules_str + content[insert_pos:]
+        print(f"  Added {len(rules)} rules")
+
+    # Write updated content
+    with open("i18n.py", "w") as f:
+        f.write(content)
+
+    print("Done!")
+
+if __name__ == "__main__":
+    inject_rules()

scripts/inject_translations.py

@@ -0,0 +1,39 @@
+#!/usr/bin/env python3
+"""Inject body translations into i18n.py"""
+import re
+from translations_body import BODY
+
+def main():
+    with open('i18n.py', 'r', encoding='utf-8') as f:
+        content = f.read()
+
+    for lang, translations in BODY.items():
+        print(f"Processing {lang}...")
+
+        # Build the replacement text
+        lines = ['        # About page body translations']
+        for key, value in translations.items():
+            escaped = value.replace('\\', '\\\\').replace('"', '\\"')
+            lines.append(f'        "{key}": "{escaped}",')
+        new_block = '\n'.join(lines)
+
+        # Find the existing body translations for this language
+        # Pattern: from "# About page body translations" to "about_footer_remember": "...",
+        pattern = rf'(    "{lang}": \{{[^}}]+?"about_footer_p5": "[^"]*",)\n        # About page body translations\n.*?"about_footer_remember": "[^"]*",'
+
+        match = re.search(pattern, content, re.DOTALL)
+        if match:
+            old_block = match.group(0)
+            new_full = match.group(1) + '\n' + new_block
+            content = content.replace(old_block, new_full)
+            print(f"  Updated {lang} with {len(translations)} translations")
+        else:
+            print(f"  Could not find pattern for {lang}")
+
+    with open('i18n.py', 'w', encoding='utf-8') as f:
+        f.write(content)
+
+    print("Done!")
+
+if __name__ == "__main__":
+    main()

scripts/make-executable.sh

@@ -0,0 +1,75 @@
+#!/bin/bash
+#
+# make-executable.sh - Create self-extracting searchable archive
+#
+# Usage:
+#   ./make-executable.sh archive.tar.gz
+#   ./archive.run
+#
+# The resulting .run file is a single executable that:
+#   1. Extracts the Python serve.py to /tmp
+#   2. Runs it pointing at itself as the tarball source
+#   3. Serves the archive on http://localhost:6543
+#
+# Requirements:
+#   - gcc with zlib (-lz)
+#   - python3 with pyramid installed (on target system)
+
+set -e
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+BOOTSTRAP_C="$SCRIPT_DIR/bootstrap.c"
+
+if [ $# -lt 1 ]; then
+    echo "Usage: $0 <archive.tar.gz> [output.run]"
+    echo ""
+    echo "Creates a self-extracting searchable archive."
+    echo "The output file can be run directly to start the search server."
+    exit 1
+fi
+
+TARBALL="$1"
+OUTPUT="${2:-${TARBALL%.tar.gz}.run}"
+
+if [ ! -f "$TARBALL" ]; then
+    echo "Error: Tarball not found: $TARBALL"
+    exit 1
+fi
+
+if [ ! -f "$BOOTSTRAP_C" ]; then
+    echo "Error: bootstrap.c not found: $BOOTSTRAP_C"
+    exit 1
+fi
+
+# Create temp directory
+TMPDIR=$(mktemp -d)
+trap "rm -rf $TMPDIR" EXIT
+
+echo "Compiling bootstrap..."
+gcc -O2 -o "$TMPDIR/bootstrap" "$BOOTSTRAP_C" -lz
+
+BOOTSTRAP_SIZE=$(stat -c%s "$TMPDIR/bootstrap" 2>/dev/null || stat -f%z "$TMPDIR/bootstrap")
+echo "Bootstrap size: $BOOTSTRAP_SIZE bytes"
+
+echo "Creating self-extracting archive..."
+
+# Concatenate: bootstrap + tarball + magic + offset
+cat "$TMPDIR/bootstrap" "$TARBALL" > "$TMPDIR/combined"
+
+# Append magic marker
+echo -n "NEOPIG" >> "$TMPDIR/combined"
+
+# Append offset as 16-char hex string
+printf '%016x' "$BOOTSTRAP_SIZE" >> "$TMPDIR/combined"
+
+# Make executable and move to output
+chmod +x "$TMPDIR/combined"
+mv "$TMPDIR/combined" "$OUTPUT"
+
+FINAL_SIZE=$(stat -c%s "$OUTPUT" 2>/dev/null || stat -f%z "$OUTPUT")
+echo ""
+echo "Created: $OUTPUT ($FINAL_SIZE bytes)"
+echo ""
+echo "To run:"
+echo "  ./$OUTPUT"
+echo "  # Opens http://localhost:6543"