pig.py/bootstrap.c
Russell Ballestrini 416b3cb760 Add self-extracting archives with bundled neopig
- Bootstrap C program extracts serve.py and runs from tarball
- Archives now include neopig source files for self-contained crawling
- --upgrade-neopig flag to update neopig in existing archives
- html2md.py: smart HTML-to-markdown converter for forums/blogs/Q&A
- Fix vault path defaults (data/vault instead of vault)
- Streaming tar.gz creation without temp copies
- URL rewriting for local media references in archives
2025-12-30 06:55:20 -05:00

269 lines
6.9 KiB
C

/*
* bootstrap.c - Self-extracting archive bootstrap
*
* A small C program that boots an embedded Python archive server.
* When compiled and concatenated with a tar.gz archive, it:
* 1. Extracts serve.py from the embedded tarball
* 2. Runs: python3 serve.py <self>
* 3. The Python script serves directly from the tarball portion
*
* Build:
* gcc -O2 -o bootstrap bootstrap.c -lz
*
* Create self-extracting archive:
* cat bootstrap archive.tar.gz > archive.run
* chmod +x archive.run
* echo -n "NEOPIG" >> archive.run # Magic marker
* printf '%016x' $(stat -c%s bootstrap) >> archive.run # Offset (hex)
*
* Or use the provided make-executable.sh script.
*/
#define _GNU_SOURCE
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <sys/stat.h>
#include <sys/wait.h>
#include <fcntl.h>
#include <errno.h>
#include <zlib.h>
#define MAGIC "NEOPIG"
#define MAGIC_LEN 6
#define OFFSET_LEN 16
#define TRAILER_LEN (MAGIC_LEN + OFFSET_LEN)
#define SERVE_PY_NAME "serve.py"
#define CHUNK_SIZE 16384
/* Find python3 interpreter */
static const char *find_python(void) {
static const char *pythons[] = {
"/usr/bin/python3",
"/usr/local/bin/python3",
"/opt/homebrew/bin/python3",
"python3",
NULL
};
for (int i = 0; pythons[i]; i++) {
if (access(pythons[i], X_OK) == 0) {
return pythons[i];
}
}
/* Try PATH lookup */
return "python3";
}
/* Read trailer to get tarball offset */
static long read_trailer(const char *self_path) {
FILE *f = fopen(self_path, "rb");
if (!f) {
perror("fopen self");
return -1;
}
/* Seek to trailer */
if (fseek(f, -TRAILER_LEN, SEEK_END) != 0) {
perror("fseek trailer");
fclose(f);
return -1;
}
char trailer[TRAILER_LEN + 1];
if (fread(trailer, 1, TRAILER_LEN, f) != TRAILER_LEN) {
perror("fread trailer");
fclose(f);
return -1;
}
trailer[TRAILER_LEN] = '\0';
fclose(f);
/* Check magic */
if (memcmp(trailer, MAGIC, MAGIC_LEN) != 0) {
fprintf(stderr, "Error: Invalid archive (missing NEOPIG magic)\n");
fprintf(stderr, "This executable must be created with make-executable.sh\n");
return -1;
}
/* Parse hex offset */
long offset = strtol(trailer + MAGIC_LEN, NULL, 16);
return offset;
}
/* Extract serve.py from gzipped tarball */
static char *extract_serve_py(const char *self_path, long tar_offset) {
/* Create temp file for serve.py */
char *temp_path = strdup("/tmp/neopig_serve_XXXXXX.py");
if (!temp_path) return NULL;
/* mkstemps for .py suffix */
int fd = mkstemps(temp_path, 3);
if (fd < 0) {
perror("mkstemps");
free(temp_path);
return NULL;
}
/* Open self and seek to tarball */
FILE *self = fopen(self_path, "rb");
if (!self) {
perror("fopen self for tar");
close(fd);
unlink(temp_path);
free(temp_path);
return NULL;
}
if (fseek(self, tar_offset, SEEK_SET) != 0) {
perror("fseek to tar");
fclose(self);
close(fd);
unlink(temp_path);
free(temp_path);
return NULL;
}
/* Open gzip stream */
gzFile gz = gzdopen(fileno(self), "rb");
if (!gz) {
fprintf(stderr, "gzdopen failed\n");
fclose(self);
close(fd);
unlink(temp_path);
free(temp_path);
return NULL;
}
/* Read tar headers looking for serve.py */
unsigned char header[512];
int found = 0;
while (gzread(gz, header, 512) == 512) {
/* Check for end of archive (all zeros) */
int all_zero = 1;
for (int i = 0; i < 512 && all_zero; i++) {
if (header[i] != 0) all_zero = 0;
}
if (all_zero) break;
/* Get filename (first 100 bytes) */
char filename[101];
memcpy(filename, header, 100);
filename[100] = '\0';
/* Get file size (octal, bytes 124-135) */
char size_str[13];
memcpy(size_str, header + 124, 12);
size_str[12] = '\0';
long filesize = strtol(size_str, NULL, 8);
/* Check if this is serve.py */
char *basename = strrchr(filename, '/');
basename = basename ? basename + 1 : filename;
if (strcmp(basename, SERVE_PY_NAME) == 0) {
/* Extract this file */
unsigned char buf[CHUNK_SIZE];
long remaining = filesize;
while (remaining > 0) {
int to_read = remaining > CHUNK_SIZE ? CHUNK_SIZE : remaining;
int got = gzread(gz, buf, to_read);
if (got <= 0) break;
write(fd, buf, got);
remaining -= got;
}
found = 1;
break;
} else {
/* Skip this file's content (padded to 512 bytes) */
long blocks = (filesize + 511) / 512;
for (long i = 0; i < blocks; i++) {
if (gzread(gz, header, 512) != 512) break;
}
}
}
gzclose(gz);
close(fd);
if (!found) {
fprintf(stderr, "Error: serve.py not found in archive\n");
unlink(temp_path);
free(temp_path);
return NULL;
}
return temp_path;
}
int main(int argc, char *argv[]) {
/* Get path to self */
char self_path[4096];
ssize_t len = readlink("/proc/self/exe", self_path, sizeof(self_path) - 1);
if (len < 0) {
/* Fallback to argv[0] */
if (argv[0][0] == '/') {
strncpy(self_path, argv[0], sizeof(self_path) - 1);
} else {
char *cwd = getcwd(NULL, 0);
snprintf(self_path, sizeof(self_path), "%s/%s", cwd, argv[0]);
free(cwd);
}
} else {
self_path[len] = '\0';
}
/* Read trailer to get tarball offset */
long tar_offset = read_trailer(self_path);
if (tar_offset < 0) {
return 1;
}
printf("Archive offset: %ld bytes\n", tar_offset);
/* Extract serve.py */
char *serve_py_path = extract_serve_py(self_path, tar_offset);
if (!serve_py_path) {
return 1;
}
printf("Extracted: %s\n", serve_py_path);
/* Find python */
const char *python = find_python();
/* Build command: python3 serve.py <self_path> */
printf("Starting: %s %s %s\n", python, serve_py_path, self_path);
/* Fork and exec */
pid_t pid = fork();
if (pid < 0) {
perror("fork");
unlink(serve_py_path);
free(serve_py_path);
return 1;
}
if (pid == 0) {
/* Child - exec python */
execl(python, "python3", serve_py_path, self_path, NULL);
perror("execl python3");
_exit(1);
}
/* Parent - wait for child and cleanup */
int status;
waitpid(pid, &status, 0);
/* Cleanup temp file */
unlink(serve_py_path);
free(serve_py_path);
return WIFEXITED(status) ? WEXITSTATUS(status) : 1;
}