russell@unturf.com
649aeec79a
aborist/progress.py — stdlib-only rate-limited stderr reporter.
Periodic lines (default every 2s) showing:
elapsed | seen | inserted | docs/s now | docs/s avg | percent | ETA
Wired into ingest_source via a `progress` parameter; CLI default-on
with --quiet to suppress and --total-estimate N to enable percent/ETA.
Demo on a 5k-doc ingest with --progress-interval 0.8:
[ 1s] | 200 seen | 200 new | 193 now | ( 193 avg) docs/s | 4.0% | ETA 24s
[ 7s] | 3,200 seen | 3,200 new | 458 now | ( 424 avg) docs/s | 64.0% | ETA 3s
[ 12s] | 5,001 seen | 5,000 new | 392 now | ( 413 avg) docs/s | 100.0% | ETA 0s
bench/run.sh + `make bench` — reproducible 5000-doc workload through
three configs:
serial single process, single SQLite
parallel-shared N shards, one shared SQLite (WAL serialized)
attached N shards, per-shard SQLite (true parallel writes)
Output is a one-shot table plus CSV at /tmp/aborist-bench/results.csv
so the ratchet is visible as we keep optimizing. Override via
BENCH_DOCS=N and SHARDS=N.
Latest baseline (this commit, on this machine):
config wall_s docs docs/s
serial 11.74 5000 425.7
parallel-shared 10.39 5000 481.1
attached 7.99 5000 625.5
53 tests still passing.
107 lines
3.2 KiB
Bash
Executable file
107 lines
3.2 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
# Reproducible ETL throughput benchmark.
|
|
#
|
|
# Three configurations against a fixed BENCH_DOCS slice of the Wikipedia
|
|
# 2003-05-16 cur dump:
|
|
#
|
|
# serial single process, single SQLite (Phase 1a)
|
|
# parallel-shared N shards, one shared SQLite (Phase 1b — WAL serialized)
|
|
# attached N shards, per-shard SQLite (Phase 2 — true parallel writes)
|
|
#
|
|
# Output: per-config wall, throughput (docs/s), and a one-line summary table.
|
|
set -uo pipefail
|
|
|
|
BENCH_DOCS=${1:-5000}
|
|
BENCH_DIR=${BENCH_DIR:-/tmp/aborist-bench}
|
|
SHARDS=${SHARDS:-4}
|
|
DUMP=${DUMP:-data/20030516_cur_tablesql.bz2}
|
|
ABORIST=${ABORIST:-.venv/bin/aborist}
|
|
|
|
mkdir -p "$BENCH_DIR"
|
|
|
|
if [[ ! -f "$DUMP" ]]; then
|
|
echo "missing dump $DUMP — run 'make fetch-cur' first" >&2
|
|
exit 2
|
|
fi
|
|
|
|
run_silent() {
|
|
# $1 = label (writes wall time in seconds to stdout)
|
|
local start end
|
|
start=$(date +%s.%N)
|
|
"$@" >/dev/null 2>&1
|
|
end=$(date +%s.%N)
|
|
echo "$end - $start" | bc -l
|
|
}
|
|
|
|
# Each row will be: label,wall_s,docs/s
|
|
declare -a results
|
|
|
|
bench_serial() {
|
|
local db=$BENCH_DIR/serial.db
|
|
rm -f "$db" "$db-"*
|
|
local wall
|
|
wall=$(run_silent "$ABORIST" --db "$db" ingest --quiet \
|
|
--source wikipedia_cur --path "$DUMP" --limit "$BENCH_DOCS")
|
|
local docs
|
|
docs=$(sqlite3 "$db" "SELECT COUNT(*) FROM documents")
|
|
printf "serial,%s,%s\n" "$wall" "$docs"
|
|
}
|
|
|
|
bench_parallel_shared() {
|
|
local db=$BENCH_DIR/shared.db
|
|
rm -f "$db" "$db-"*
|
|
local per=$((BENCH_DOCS / SHARDS))
|
|
local start end wall
|
|
start=$(date +%s.%N)
|
|
for i in $(seq 0 $((SHARDS - 1))); do
|
|
"$ABORIST" --db "$db" ingest --quiet \
|
|
--source wikipedia_cur --path "$DUMP" \
|
|
--shard "$i/$SHARDS" --limit "$per" >/dev/null 2>&1 &
|
|
done
|
|
wait
|
|
end=$(date +%s.%N)
|
|
wall=$(echo "$end - $start" | bc -l)
|
|
local docs
|
|
docs=$(sqlite3 "$db" "SELECT COUNT(*) FROM documents")
|
|
printf "parallel-shared,%s,%s\n" "$wall" "$docs"
|
|
}
|
|
|
|
bench_attached() {
|
|
local dir=$BENCH_DIR/attached
|
|
rm -rf "$dir"
|
|
mkdir -p "$dir"
|
|
local per=$((BENCH_DOCS / SHARDS))
|
|
local start end wall
|
|
start=$(date +%s.%N)
|
|
for i in $(seq 0 $((SHARDS - 1))); do
|
|
"$ABORIST" ingest --quiet \
|
|
--source wikipedia_cur --path "$DUMP" \
|
|
--shards-dir "$dir" --shard "$i/$SHARDS" --limit "$per" >/dev/null 2>&1 &
|
|
done
|
|
wait
|
|
end=$(date +%s.%N)
|
|
wall=$(echo "$end - $start" | bc -l)
|
|
local docs
|
|
docs=$("$ABORIST" --shards-dir "$dir" stats 2>/dev/null \
|
|
| python3 -c "import json,sys; print(json.load(sys.stdin)['documents_total'])")
|
|
printf "attached,%s,%s\n" "$wall" "$docs"
|
|
}
|
|
|
|
echo "=== aborist ETL benchmark — $BENCH_DOCS docs target, $SHARDS shards ==="
|
|
echo
|
|
|
|
results+=("$(bench_serial)")
|
|
results+=("$(bench_parallel_shared)")
|
|
results+=("$(bench_attached)")
|
|
|
|
# Pretty table.
|
|
printf " %-18s %10s %10s %12s\n" "config" "wall_s" "docs" "docs/s"
|
|
printf " %-18s %10s %10s %12s\n" "------" "------" "----" "------"
|
|
for row in "${results[@]}"; do
|
|
IFS=',' read -r label wall docs <<<"$row"
|
|
rate=$(echo "scale=1; $docs / $wall" | bc -l)
|
|
printf " %-18s %10.2f %10s %12s\n" "$label" "$wall" "$docs" "$rate"
|
|
done
|
|
echo
|
|
echo " (raw CSV: $BENCH_DIR/results.csv)"
|
|
{ printf "config,wall_s,docs\n"; printf "%s\n" "${results[@]}"; } > "$BENCH_DIR/results.csv"
|