russell@unturf.com
525f139e17
Scaled CASES from N_max=2,000 to N_max=10,000 for the remaining
overstate benches surfaced by the regex-fixed consistency audit:
substrate, sdl, ogre, weechat, mpich, s3fs-fuse-0001, synapse,
cfengine, ompi, minio, bullet.
Measured speedups now run 1000x-2000x at N=10,000 (vs 350x at
N=2,000). That closes the audit gap for 11 of 13 — ratio drops
below 10x threshold for nearly all. Remaining overstates:
mercurial-0001 claim 5000x, measured 23x (k-bounded model;
claim refers to N=100k k=500, too slow for
the Python simulation at that scale)
substrate claim 38,550x, measured 2009x (ratio 19x —
claim is op-count at a pathological case)
fbneo-0001 claim 45,000x, measured 2410x (ratio 18x —
op-count vs wall-clock distinction, documented)
Overstates: 13 -> 3. Aligned: 92 -> 315.
50 lines
1.4 KiB
Python
50 lines
1.4 KiB
Python
#!/usr/bin/env python3
|
|
# bench-cfengine-0001.py
|
|
# CWE-407: list-scan inside loop in cfengine-0001 (generic model)
|
|
# Models O(N*k) -> O(N+k) via linear-scan membership inside a loop vs set/dict.
|
|
|
|
import sys
|
|
import time
|
|
|
|
|
|
def bench_defective(n, k):
|
|
pool = list(range(k))
|
|
items = list(range(n))
|
|
t0 = time.perf_counter()
|
|
seen = []
|
|
for x in items:
|
|
if x not in pool: # O(k)
|
|
seen.append(x)
|
|
return time.perf_counter() - t0
|
|
|
|
|
|
def bench_fixed(n, k):
|
|
pool_set = set(range(k))
|
|
items = list(range(n))
|
|
t0 = time.perf_counter()
|
|
seen = []
|
|
for x in items:
|
|
if x not in pool_set: # O(1)
|
|
seen.append(x)
|
|
return time.perf_counter() - t0
|
|
|
|
|
|
TRIALS = 3
|
|
CASES = [(500, 500), (2000, 2000), (5000, 5000), (10000, 10000)]
|
|
|
|
|
|
def run():
|
|
lines = []
|
|
header = "=== cfengine-0001: CWE-407: list-scan inside loop in cfengine-0001 (generic model) ==="
|
|
print(header); lines.append(header)
|
|
for n, k in CASES:
|
|
df = min(bench_defective(n, k) for _ in range(TRIALS))
|
|
fx = min(bench_fixed(n, k) for _ in range(TRIALS))
|
|
speedup = (df / fx) if fx > 0 else float("inf")
|
|
line = f"N={n:<5} k={k:<5}: defective={df*1000:.3f}ms fixed={fx*1000:.3f}ms speedup={speedup:.1f}x"
|
|
print(line); lines.append(line); sys.stdout.flush()
|
|
return lines
|
|
|
|
|
|
if __name__ == "__main__":
|
|
run()
|