- Bootstrap C program extracts serve.py and runs from tarball - Archives now include neopig source files for self-contained crawling - --upgrade-neopig flag to update neopig in existing archives - html2md.py: smart HTML-to-markdown converter for forums/blogs/Q&A - Fix vault path defaults (data/vault instead of vault) - Streaming tar.gz creation without temp copies - URL rewriting for local media references in archives
269 lines
6.9 KiB
C
269 lines
6.9 KiB
C
/*
|
|
* bootstrap.c - Self-extracting archive bootstrap
|
|
*
|
|
* A small C program that boots an embedded Python archive server.
|
|
* When compiled and concatenated with a tar.gz archive, it:
|
|
* 1. Extracts serve.py from the embedded tarball
|
|
* 2. Runs: python3 serve.py <self>
|
|
* 3. The Python script serves directly from the tarball portion
|
|
*
|
|
* Build:
|
|
* gcc -O2 -o bootstrap bootstrap.c -lz
|
|
*
|
|
* Create self-extracting archive:
|
|
* cat bootstrap archive.tar.gz > archive.run
|
|
* chmod +x archive.run
|
|
* echo -n "NEOPIG" >> archive.run # Magic marker
|
|
* printf '%016x' $(stat -c%s bootstrap) >> archive.run # Offset (hex)
|
|
*
|
|
* Or use the provided make-executable.sh script.
|
|
*/
|
|
|
|
#define _GNU_SOURCE
|
|
#include <stdio.h>
|
|
#include <stdlib.h>
|
|
#include <string.h>
|
|
#include <unistd.h>
|
|
#include <sys/stat.h>
|
|
#include <sys/wait.h>
|
|
#include <fcntl.h>
|
|
#include <errno.h>
|
|
#include <zlib.h>
|
|
|
|
#define MAGIC "NEOPIG"
|
|
#define MAGIC_LEN 6
|
|
#define OFFSET_LEN 16
|
|
#define TRAILER_LEN (MAGIC_LEN + OFFSET_LEN)
|
|
|
|
#define SERVE_PY_NAME "serve.py"
|
|
#define CHUNK_SIZE 16384
|
|
|
|
/* Find python3 interpreter */
|
|
static const char *find_python(void) {
|
|
static const char *pythons[] = {
|
|
"/usr/bin/python3",
|
|
"/usr/local/bin/python3",
|
|
"/opt/homebrew/bin/python3",
|
|
"python3",
|
|
NULL
|
|
};
|
|
|
|
for (int i = 0; pythons[i]; i++) {
|
|
if (access(pythons[i], X_OK) == 0) {
|
|
return pythons[i];
|
|
}
|
|
}
|
|
|
|
/* Try PATH lookup */
|
|
return "python3";
|
|
}
|
|
|
|
/* Read trailer to get tarball offset */
|
|
static long read_trailer(const char *self_path) {
|
|
FILE *f = fopen(self_path, "rb");
|
|
if (!f) {
|
|
perror("fopen self");
|
|
return -1;
|
|
}
|
|
|
|
/* Seek to trailer */
|
|
if (fseek(f, -TRAILER_LEN, SEEK_END) != 0) {
|
|
perror("fseek trailer");
|
|
fclose(f);
|
|
return -1;
|
|
}
|
|
|
|
char trailer[TRAILER_LEN + 1];
|
|
if (fread(trailer, 1, TRAILER_LEN, f) != TRAILER_LEN) {
|
|
perror("fread trailer");
|
|
fclose(f);
|
|
return -1;
|
|
}
|
|
trailer[TRAILER_LEN] = '\0';
|
|
fclose(f);
|
|
|
|
/* Check magic */
|
|
if (memcmp(trailer, MAGIC, MAGIC_LEN) != 0) {
|
|
fprintf(stderr, "Error: Invalid archive (missing NEOPIG magic)\n");
|
|
fprintf(stderr, "This executable must be created with make-executable.sh\n");
|
|
return -1;
|
|
}
|
|
|
|
/* Parse hex offset */
|
|
long offset = strtol(trailer + MAGIC_LEN, NULL, 16);
|
|
return offset;
|
|
}
|
|
|
|
/* Extract serve.py from gzipped tarball */
|
|
static char *extract_serve_py(const char *self_path, long tar_offset) {
|
|
/* Create temp file for serve.py */
|
|
char *temp_path = strdup("/tmp/neopig_serve_XXXXXX.py");
|
|
if (!temp_path) return NULL;
|
|
|
|
/* mkstemps for .py suffix */
|
|
int fd = mkstemps(temp_path, 3);
|
|
if (fd < 0) {
|
|
perror("mkstemps");
|
|
free(temp_path);
|
|
return NULL;
|
|
}
|
|
|
|
/* Open self and seek to tarball */
|
|
FILE *self = fopen(self_path, "rb");
|
|
if (!self) {
|
|
perror("fopen self for tar");
|
|
close(fd);
|
|
unlink(temp_path);
|
|
free(temp_path);
|
|
return NULL;
|
|
}
|
|
|
|
if (fseek(self, tar_offset, SEEK_SET) != 0) {
|
|
perror("fseek to tar");
|
|
fclose(self);
|
|
close(fd);
|
|
unlink(temp_path);
|
|
free(temp_path);
|
|
return NULL;
|
|
}
|
|
|
|
/* Open gzip stream */
|
|
gzFile gz = gzdopen(fileno(self), "rb");
|
|
if (!gz) {
|
|
fprintf(stderr, "gzdopen failed\n");
|
|
fclose(self);
|
|
close(fd);
|
|
unlink(temp_path);
|
|
free(temp_path);
|
|
return NULL;
|
|
}
|
|
|
|
/* Read tar headers looking for serve.py */
|
|
unsigned char header[512];
|
|
int found = 0;
|
|
|
|
while (gzread(gz, header, 512) == 512) {
|
|
/* Check for end of archive (all zeros) */
|
|
int all_zero = 1;
|
|
for (int i = 0; i < 512 && all_zero; i++) {
|
|
if (header[i] != 0) all_zero = 0;
|
|
}
|
|
if (all_zero) break;
|
|
|
|
/* Get filename (first 100 bytes) */
|
|
char filename[101];
|
|
memcpy(filename, header, 100);
|
|
filename[100] = '\0';
|
|
|
|
/* Get file size (octal, bytes 124-135) */
|
|
char size_str[13];
|
|
memcpy(size_str, header + 124, 12);
|
|
size_str[12] = '\0';
|
|
long filesize = strtol(size_str, NULL, 8);
|
|
|
|
/* Check if this is serve.py */
|
|
char *basename = strrchr(filename, '/');
|
|
basename = basename ? basename + 1 : filename;
|
|
|
|
if (strcmp(basename, SERVE_PY_NAME) == 0) {
|
|
/* Extract this file */
|
|
unsigned char buf[CHUNK_SIZE];
|
|
long remaining = filesize;
|
|
|
|
while (remaining > 0) {
|
|
int to_read = remaining > CHUNK_SIZE ? CHUNK_SIZE : remaining;
|
|
int got = gzread(gz, buf, to_read);
|
|
if (got <= 0) break;
|
|
write(fd, buf, got);
|
|
remaining -= got;
|
|
}
|
|
|
|
found = 1;
|
|
break;
|
|
} else {
|
|
/* Skip this file's content (padded to 512 bytes) */
|
|
long blocks = (filesize + 511) / 512;
|
|
for (long i = 0; i < blocks; i++) {
|
|
if (gzread(gz, header, 512) != 512) break;
|
|
}
|
|
}
|
|
}
|
|
|
|
gzclose(gz);
|
|
close(fd);
|
|
|
|
if (!found) {
|
|
fprintf(stderr, "Error: serve.py not found in archive\n");
|
|
unlink(temp_path);
|
|
free(temp_path);
|
|
return NULL;
|
|
}
|
|
|
|
return temp_path;
|
|
}
|
|
|
|
int main(int argc, char *argv[]) {
|
|
/* Get path to self */
|
|
char self_path[4096];
|
|
ssize_t len = readlink("/proc/self/exe", self_path, sizeof(self_path) - 1);
|
|
if (len < 0) {
|
|
/* Fallback to argv[0] */
|
|
if (argv[0][0] == '/') {
|
|
strncpy(self_path, argv[0], sizeof(self_path) - 1);
|
|
} else {
|
|
char *cwd = getcwd(NULL, 0);
|
|
snprintf(self_path, sizeof(self_path), "%s/%s", cwd, argv[0]);
|
|
free(cwd);
|
|
}
|
|
} else {
|
|
self_path[len] = '\0';
|
|
}
|
|
|
|
/* Read trailer to get tarball offset */
|
|
long tar_offset = read_trailer(self_path);
|
|
if (tar_offset < 0) {
|
|
return 1;
|
|
}
|
|
|
|
printf("Archive offset: %ld bytes\n", tar_offset);
|
|
|
|
/* Extract serve.py */
|
|
char *serve_py_path = extract_serve_py(self_path, tar_offset);
|
|
if (!serve_py_path) {
|
|
return 1;
|
|
}
|
|
|
|
printf("Extracted: %s\n", serve_py_path);
|
|
|
|
/* Find python */
|
|
const char *python = find_python();
|
|
|
|
/* Build command: python3 serve.py <self_path> */
|
|
printf("Starting: %s %s %s\n", python, serve_py_path, self_path);
|
|
|
|
/* Fork and exec */
|
|
pid_t pid = fork();
|
|
if (pid < 0) {
|
|
perror("fork");
|
|
unlink(serve_py_path);
|
|
free(serve_py_path);
|
|
return 1;
|
|
}
|
|
|
|
if (pid == 0) {
|
|
/* Child - exec python */
|
|
execl(python, "python3", serve_py_path, self_path, NULL);
|
|
perror("execl python3");
|
|
_exit(1);
|
|
}
|
|
|
|
/* Parent - wait for child and cleanup */
|
|
int status;
|
|
waitpid(pid, &status, 0);
|
|
|
|
/* Cleanup temp file */
|
|
unlink(serve_py_path);
|
|
free(serve_py_path);
|
|
|
|
return WIFEXITED(status) ? WEXITSTATUS(status) : 1;
|
|
}
|