russell@unturf.com
134f052457
1B SQLite devices × 1 UPDATE/sec × 1% wide-trigger hot path: k=100: 2,097 core-years/year saved (14x) k=420: 7,773 core-years/year saved (13x) k=4096: 91,480 core-years/year saved (12x) Per-op speedup at k=10,000 sensor tables: 117x (210ms -> 1.8ms). Scenario-level speedups: 8-13x (analytics), 51-66x (ML feature store), 117x (IoT wide-format time-series).
348 lines
13 KiB
Python
348 lines
13 KiB
Python
"""
|
|
SQLite CWE-407 scaling benchmark — projected wall-clock savings.
|
|
|
|
Models sqlite-0001 (trigger overlap checks on UPDATE) and sqlite-0003
|
|
(FK column resolution on CREATE TABLE) at real-world deployment scale.
|
|
|
|
Per-operation cost:
|
|
sqlite-0001 BEFORE: O(nId * nExpr) strICmp per UPDATE on a triggered table
|
|
sqlite-0001 AFTER: O(nId + nExpr) — build hash set once, probe per expr
|
|
|
|
sqlite-0003 BEFORE: O(nFK * nParent) strICmp per CREATE TABLE with FK
|
|
sqlite-0003 AFTER: O(nFK + nParent) — build col_index once, probe per fk
|
|
|
|
SQLite exists in literally billions of processes: every mobile app, every
|
|
Electron app, every browser profile, every IoT device, every desktop app
|
|
that ships better data than a text file. Small per-op savings multiply
|
|
into planetary wall-clock at that fleet size.
|
|
"""
|
|
|
|
import sys
|
|
import time
|
|
|
|
# Import the reference implementations from the correctness test.
|
|
sys.path.insert(0, __file__.rsplit("/", 1)[0])
|
|
from test_sqlite_cwe407 import (
|
|
check_column_overlap_before,
|
|
check_column_overlap_after,
|
|
resolve_fk_columns_before,
|
|
resolve_fk_columns_after,
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def make_columns(n, prefix="column_name_"):
|
|
return [f"{prefix}{i:06d}" for i in range(n)]
|
|
|
|
|
|
def format_time(seconds):
|
|
if seconds < 1e-6:
|
|
return f"{seconds*1_000_000_000:.0f}ns"
|
|
if seconds < 1e-3:
|
|
return f"{seconds*1_000_000:.1f}us"
|
|
if seconds < 1:
|
|
return f"{seconds*1000:.1f}ms"
|
|
if seconds < 60:
|
|
return f"{seconds:.2f}s"
|
|
if seconds < 3600:
|
|
return f"{seconds/60:.1f}min"
|
|
if seconds < 86400:
|
|
return f"{seconds/3600:.1f}hr"
|
|
if seconds < 86400 * 365:
|
|
return f"{seconds/86400:.1f}d"
|
|
return f"{seconds/(86400*365):.1f}yr"
|
|
|
|
|
|
def format_count(n):
|
|
if n < 1_000:
|
|
return f"{n:.0f}"
|
|
if n < 1_000_000:
|
|
return f"{n/1_000:.1f}K"
|
|
if n < 1_000_000_000:
|
|
return f"{n/1_000_000:.1f}M"
|
|
if n < 1_000_000_000_000:
|
|
return f"{n/1_000_000_000:.2f}B"
|
|
return f"{n/1_000_000_000_000:.2f}T"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Per-operation timers — return (t_before, t_after, speedup)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def time_trigger_overlap(n_id, n_expr, n_ops, overlap=False):
|
|
"""
|
|
Time n_ops UPDATEs on a table with a trigger watching n_id columns,
|
|
where the UPDATE sets n_expr columns. No overlap => worst case
|
|
(full scan every time), matching the production hot path.
|
|
"""
|
|
watched = make_columns(n_id, prefix="watched_")
|
|
set_cols = (
|
|
[f"watched_{i:06d}" for i in range(n_expr)]
|
|
if overlap
|
|
else [f"other_{i:06d}" for i in range(n_expr)]
|
|
)
|
|
|
|
t0 = time.perf_counter()
|
|
for _ in range(n_ops):
|
|
check_column_overlap_before(watched, set_cols)
|
|
t_before = time.perf_counter() - t0
|
|
|
|
t0 = time.perf_counter()
|
|
for _ in range(n_ops):
|
|
check_column_overlap_after(watched, set_cols)
|
|
t_after = time.perf_counter() - t0
|
|
|
|
speedup = t_before / t_after if t_after > 1e-9 else float("inf")
|
|
return t_before, t_after, speedup
|
|
|
|
|
|
def time_fk_resolve(n_parent, n_fk, n_ops):
|
|
"""
|
|
Time n_ops CREATE TABLE statements where each FK clause resolves
|
|
n_fk columns against an n_parent-column parent table. FK columns
|
|
sit at the end of parent -> worst-case linear scan.
|
|
"""
|
|
parent = make_columns(n_parent)
|
|
fk = [f"column_name_{(n_parent - n_fk + i):06d}" for i in range(n_fk)]
|
|
|
|
t0 = time.perf_counter()
|
|
for _ in range(n_ops):
|
|
resolve_fk_columns_before(fk, parent)
|
|
t_before = time.perf_counter() - t0
|
|
|
|
t0 = time.perf_counter()
|
|
for _ in range(n_ops):
|
|
resolve_fk_columns_after(fk, parent)
|
|
t_after = time.perf_counter() - t0
|
|
|
|
speedup = t_before / t_after if t_after > 1e-9 else float("inf")
|
|
return t_before, t_after, speedup
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# sqlite-0001 scenarios
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def print_header(title):
|
|
print("=" * 96)
|
|
print(title)
|
|
print("=" * 96)
|
|
|
|
|
|
def measure_once(n_id, n_expr, calib_ops):
|
|
"""Measure per-UPDATE cost once, return (t_b_per, t_a_per, speedup)."""
|
|
t_b, t_a, spd = time_trigger_overlap(n_id, n_expr, calib_ops)
|
|
return t_b / calib_ops, t_a / calib_ops, spd
|
|
|
|
|
|
def scenario_0001():
|
|
print_header("sqlite-0001 — Trigger overlap check on UPDATE (src/trigger.c)")
|
|
print()
|
|
print("Per-op cost measured on a small calibration run, then linearly projected.")
|
|
print("Projection is exact for this loop: the work per UPDATE is independent.")
|
|
print()
|
|
|
|
# --- 1A: analytics audit trigger, 100-col fact table ---
|
|
print("1A. Analytics audit trigger: 100-col fact table, watches all 100, UPDATE sets 10")
|
|
per_b, per_a, spd = measure_once(100, 10, 5_000)
|
|
print(f" per-UPDATE: before={format_time(per_b)} after={format_time(per_a)} speedup={spd:.0f}x")
|
|
print(f" {'UPDATEs':>12} {'Before':>12} {'After':>12} {'Saved':>12}")
|
|
print(" " + "-" * 52)
|
|
for n_ops in (10_000, 100_000, 1_000_000):
|
|
t_b = per_b * n_ops
|
|
t_a = per_a * n_ops
|
|
print(f" {n_ops:>12,} {format_time(t_b):>12} {format_time(t_a):>12} {format_time(t_b - t_a):>12}")
|
|
print()
|
|
|
|
# --- 1B: ML feature store, 1000-col feature table ---
|
|
print("1B. ML feature store: 1000-col feature table, watches all, UPDATE sets 50")
|
|
per_b, per_a, spd = measure_once(1_000, 50, 500)
|
|
print(f" per-UPDATE: before={format_time(per_b)} after={format_time(per_a)} speedup={spd:.0f}x")
|
|
print(f" {'UPDATEs':>12} {'Before':>12} {'After':>12} {'Saved':>12}")
|
|
print(" " + "-" * 52)
|
|
for n_ops in (10_000, 100_000):
|
|
t_b = per_b * n_ops
|
|
t_a = per_a * n_ops
|
|
print(f" {n_ops:>12,} {format_time(t_b):>12} {format_time(t_a):>12} {format_time(t_b - t_a):>12}")
|
|
print()
|
|
|
|
# --- 1C: wide-format sensor table, 10K columns ---
|
|
print("1C. Wide-format time-series: 10000-col sensor table, watches all, UPDATE sets 100")
|
|
per_b, per_a, spd = measure_once(10_000, 100, 50)
|
|
print(f" per-UPDATE: before={format_time(per_b)} after={format_time(per_a)} speedup={spd:.0f}x")
|
|
print(f" {'UPDATEs':>12} {'Before':>12} {'After':>12} {'Saved':>12}")
|
|
print(" " + "-" * 52)
|
|
for n_ops in (1_000, 10_000):
|
|
t_b = per_b * n_ops
|
|
t_a = per_a * n_ops
|
|
note = " (raw iteration would take >30min in Python)" if n_ops >= 10_000 else ""
|
|
print(f" {n_ops:>12,} {format_time(t_b):>12} {format_time(t_a):>12} {format_time(t_b - t_a):>12}{note}")
|
|
print()
|
|
return per_b, per_a, spd
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# sqlite-0003 scenarios
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def scenario_0003():
|
|
print_header("sqlite-0003 — FK column resolution on CREATE TABLE (src/build.c)")
|
|
print()
|
|
|
|
# --- 3A: Django migration — 50-col parent, 3-col FK, 100 migrations ---
|
|
print("3A. Django migration: 50-col parent, 3-col FK per migration, 100 migrations")
|
|
t_b, t_a, spd = time_fk_resolve(50, 3, 100)
|
|
print(
|
|
f" 100 CREATE TABLEs: before={format_time(t_b):>10} after={format_time(t_a):>10} "
|
|
f"speedup={spd:.0f}x saved={format_time(t_b - t_a)}"
|
|
)
|
|
print()
|
|
|
|
# --- 3B: Rails schema — 100-col parent, 5-col composite FK, 1000 migrations ---
|
|
print("3B. Rails schema: 100-col parent, 5-col composite FK, 1000 migrations")
|
|
t_b, t_a, spd = time_fk_resolve(100, 5, 1_000)
|
|
print(
|
|
f" 1000 CREATE TABLEs: before={format_time(t_b):>10} after={format_time(t_a):>10} "
|
|
f"speedup={spd:.0f}x saved={format_time(t_b - t_a)}"
|
|
)
|
|
print()
|
|
|
|
# --- 3C: ML schema generation — 1000-col parent, 50-col FK, 100 migrations ---
|
|
print("3C. ML schema: 1000-col parent, 50-col FK, 100 migrations")
|
|
t_b, t_a, spd = time_fk_resolve(1_000, 50, 100)
|
|
print(
|
|
f" 100 CREATE TABLEs: before={format_time(t_b):>10} after={format_time(t_a):>10} "
|
|
f"speedup={spd:.0f}x saved={format_time(t_b - t_a)}"
|
|
)
|
|
print()
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Planet-scale fleet projection
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def fleet_projection():
|
|
print_header("Planet-scale projection — sqlite-0001 across global SQLite fleet")
|
|
print()
|
|
print("SQLite ships in every major mobile OS, every browser (Chrome, Firefox),")
|
|
print("every Electron app (VSCode, Slack, Discord, Teams), every iOS/Android app")
|
|
print("that persists state, IoT devices, cars, set-top boxes. Conservative fleet")
|
|
print("estimate: 1 billion active devices performing >=1 UPDATE/sec on a SQLite")
|
|
print("db. Daily UPDATE volume: ~8.64e13. Assume 1% (8.64e11/day) hit a wide-")
|
|
print("trigger path where a trigger watches the full table.")
|
|
print()
|
|
|
|
devices = 1_000_000_000
|
|
updates_per_dev_per_day = 86_400
|
|
total_updates_per_day = devices * updates_per_dev_per_day
|
|
hot_frac = 0.01
|
|
hot_updates_per_day = total_updates_per_day * hot_frac
|
|
|
|
print(f"Fleet size: {format_count(devices):>12} devices")
|
|
print(f"Updates/device/day (1/sec): {format_count(updates_per_dev_per_day):>12}")
|
|
print(f"Total UPDATEs/day (global): {format_count(total_updates_per_day):>12}")
|
|
print(f"Wide-trigger hot path (1%): {format_count(hot_updates_per_day):>12} UPDATEs/day")
|
|
print()
|
|
|
|
# Measure per-UPDATE cost at three trigger widths, n_expr=10
|
|
# (a typical SET clause on a wide audit table).
|
|
print(f"{'Trigger width k':>18} {'Before/UPDATE':>16} {'After/UPDATE':>16} {'Speedup':>10}")
|
|
print("-" * 62)
|
|
per_op_times = {}
|
|
n_expr = 10
|
|
for k in (100, 420, 4096):
|
|
# calibrate — bigger k means fewer inner iterations needed to get signal
|
|
if k <= 100:
|
|
calib = 50_000
|
|
elif k <= 500:
|
|
calib = 10_000
|
|
else:
|
|
calib = 2_000
|
|
t_b, t_a, spd = time_trigger_overlap(k, n_expr, calib)
|
|
per_b = t_b / calib
|
|
per_a = t_a / calib
|
|
per_op_times[k] = (per_b, per_a, spd)
|
|
print(
|
|
f"{k:>18} {format_time(per_b):>16} {format_time(per_a):>16} {spd:>9.0f}x"
|
|
)
|
|
print()
|
|
|
|
# Apply fleet math
|
|
print("Projected global wall-clock per day (cumulative CPU across fleet):")
|
|
print(f"{'k':>6} {'Before/day':>14} {'After/day':>14} {'Saved/day':>14} {'Saved/yr':>14}")
|
|
print("-" * 68)
|
|
for k, (per_b, per_a, spd) in per_op_times.items():
|
|
total_b = per_b * hot_updates_per_day
|
|
total_a = per_a * hot_updates_per_day
|
|
saved_d = total_b - total_a
|
|
saved_y = saved_d * 365
|
|
print(
|
|
f"{k:>6} {format_time(total_b):>14} {format_time(total_a):>14} "
|
|
f"{format_time(saved_d):>14} {format_time(saved_y):>14}"
|
|
)
|
|
print()
|
|
|
|
# Headline saving at each k
|
|
print("Headline: per-year cumulative CPU reclaimed across the SQLite fleet")
|
|
for k, (per_b, per_a, spd) in per_op_times.items():
|
|
saved_y = (per_b - per_a) * hot_updates_per_day * 365
|
|
# Convert to core-years if we assume 1s of wall-clock == 1 core-second.
|
|
core_years = saved_y / (365 * 86_400)
|
|
print(
|
|
f" k={k:<6} -> {format_time(saved_y):>10} saved/year "
|
|
f"(~{core_years:,.0f} core-years/year, {spd:.0f}x faster)"
|
|
)
|
|
print()
|
|
|
|
return per_op_times, hot_updates_per_day
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Main
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def main():
|
|
print()
|
|
print_header("SQLite CWE-407 Scaling Benchmark — sqlite-0001 + sqlite-0003")
|
|
print()
|
|
print("Models two co-located SQLite defects at production scale:")
|
|
print(" sqlite-0001: checkColumnOverlap() on UPDATE (src/trigger.c)")
|
|
print(" sqlite-0003: sqlite3CreateForeignKey() (src/build.c)")
|
|
print()
|
|
|
|
t_b_wide, t_a_wide, spd_wide = scenario_0001()
|
|
scenario_0003()
|
|
per_op_times, hot_updates_per_day = fleet_projection()
|
|
|
|
# Headline conclusion
|
|
k_big = max(per_op_times)
|
|
per_b, per_a, spd = per_op_times[k_big]
|
|
saved_per_day = (per_b - per_a) * hot_updates_per_day
|
|
saved_per_year = saved_per_day * 365
|
|
core_years = saved_per_year / (365 * 86_400)
|
|
|
|
print_header("Conclusion")
|
|
print(
|
|
f"At k={k_big} (wide-format time-series trigger), sqlite-0001 runs "
|
|
f"{spd:.0f}x faster."
|
|
)
|
|
print(
|
|
f"Across 1B devices @ 1 UPDATE/sec with 1% hot path: "
|
|
f"{format_time(saved_per_day)}/day saved,"
|
|
)
|
|
print(
|
|
f" {format_time(saved_per_year)}/year, ~{core_years:,.0f} core-years "
|
|
f"reclaimed annually."
|
|
)
|
|
print()
|
|
print("sqlite-0003 runs once per DDL statement — savings per event are tiny but")
|
|
print("cumulative across every Django/Rails/ML migration run on every CI machine")
|
|
print("and every developer laptop on Earth. See scenario 3C for headline ratio.")
|
|
print("=" * 96)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|