diff --git a/defects/0ad-0001/Makefile b/defects/0ad-0001/Makefile new file mode 100644 index 000000000..fbb083eb5 --- /dev/null +++ b/defects/0ad-0001/Makefile @@ -0,0 +1,7 @@ +# 0ad-0001 bench runner +.PHONY: all bench clean +all: bench +bench: + python3 bench/run_all.py +clean: + rm -rf bench/__pycache__ __pycache__ diff --git a/defects/0ad-0001/bench/__pycache__/bench-0ad-0001-0001.cpython-312.pyc b/defects/0ad-0001/bench/__pycache__/bench-0ad-0001-0001.cpython-312.pyc new file mode 100644 index 000000000..adcca251f Binary files /dev/null and b/defects/0ad-0001/bench/__pycache__/bench-0ad-0001-0001.cpython-312.pyc differ diff --git a/defects/0ad-0001/bench/bench-0ad-0001-0001.py b/defects/0ad-0001/bench/bench-0ad-0001-0001.py new file mode 100644 index 000000000..bb834ebfe --- /dev/null +++ b/defects/0ad-0001/bench/bench-0ad-0001-0001.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +# bench-0ad-0001-0001.py +# CCmpObstructionManager dirty shape tracking: std::find on std::vector for +# dedup, called per nearby shape per frame. With N nearby shapes × D dirty +# list size, cost is O(N·D) = O(N²) in large battles (200v200, hundreds of +# shapes moving per frame). Fix: std::unordered_set for O(1) membership. + +import sys +import time + + +def bench_defective(n_shapes, dirty_before): + """std::find over a growing vector, called per shape per frame.""" + dirty = list(range(dirty_before)) # starts with D entries + updates = [i for i in range(n_shapes)] # N shapes to dedup-add + + t0 = time.perf_counter() + for s in updates: + if s not in dirty: # list.__contains__: O(len(dirty)) + dirty.append(s) + return time.perf_counter() - t0 + + +def bench_fixed(n_shapes, dirty_before): + """unordered_set membership, O(1) per check.""" + dirty = set(range(dirty_before)) + updates = [i for i in range(n_shapes)] + + t0 = time.perf_counter() + for s in updates: + if s not in dirty: + dirty.add(s) + return time.perf_counter() - t0 + + +TRIALS = 3 +CASES = [(100, 100), (300, 300), (500, 500), (1000, 1000)] + + +def run(): + lines = [] + header = "=== 0ad-0001-0001: CCmpObstructionManager dirty shapes std::find vs unordered_set ===" + print(header); lines.append(header) + for n, d in CASES: + df = min(bench_defective(n, d) for _ in range(TRIALS)) + fx = min(bench_fixed(n, d) for _ in range(TRIALS)) + speedup = (df / fx) if fx > 0 else float("inf") + line = f"N={n:<4} D={d:<4}: defective={df*1000:.3f}ms fixed={fx*1000:.3f}ms speedup={speedup:.1f}x" + print(line); lines.append(line); sys.stdout.flush() + return lines + + +if __name__ == "__main__": + run() diff --git a/defects/0ad-0001/bench/results.txt b/defects/0ad-0001/bench/results.txt new file mode 100644 index 000000000..66514e621 --- /dev/null +++ b/defects/0ad-0001/bench/results.txt @@ -0,0 +1,6 @@ +=== 0ad-0001-0001: CCmpObstructionManager dirty shapes std::find vs unordered_set === +N=100 D=100 : defective=0.108ms fixed=0.004ms speedup=26.0x +N=300 D=300 : defective=0.963ms fixed=0.014ms speedup=70.0x +N=500 D=500 : defective=2.580ms fixed=0.025ms speedup=104.1x +N=1000 D=1000: defective=11.218ms fixed=0.187ms speedup=59.9x + diff --git a/defects/0ad-0001/bench/run_all.py b/defects/0ad-0001/bench/run_all.py new file mode 100644 index 000000000..453fba431 --- /dev/null +++ b/defects/0ad-0001/bench/run_all.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 +import importlib.util, os, sys +BENCH_DIR = os.path.dirname(os.path.abspath(__file__)) + +def load_module(filename): + path = os.path.join(BENCH_DIR, filename) + spec = importlib.util.spec_from_file_location("mod", path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + +all_lines = [] +for fname in ["bench-0ad-0001-0001.py"]: + mod = load_module(fname) + lines = mod.run() + all_lines.extend(lines); all_lines.append("") + print(); sys.stdout.flush() + +out_path = os.path.join(BENCH_DIR, "results.txt") +with open(out_path, "w") as f: + f.write("\n".join(all_lines) + "\n") +print(f"results written to {out_path}"); sys.stdout.flush() diff --git a/defects/0ad-0002/Makefile b/defects/0ad-0002/Makefile new file mode 100644 index 000000000..1e1800c91 --- /dev/null +++ b/defects/0ad-0002/Makefile @@ -0,0 +1,7 @@ +# 0ad-0002 bench runner +.PHONY: all bench clean +all: bench +bench: + python3 bench/run_all.py +clean: + rm -rf bench/__pycache__ __pycache__ diff --git a/defects/0ad-0002/bench/__pycache__/bench-0ad-0002-0002.cpython-312.pyc b/defects/0ad-0002/bench/__pycache__/bench-0ad-0002-0002.cpython-312.pyc new file mode 100644 index 000000000..18257055a Binary files /dev/null and b/defects/0ad-0002/bench/__pycache__/bench-0ad-0002-0002.cpython-312.pyc differ diff --git a/defects/0ad-0002/bench/bench-0ad-0002-0002.py b/defects/0ad-0002/bench/bench-0ad-0002-0002.py new file mode 100644 index 000000000..f3a3271fc --- /dev/null +++ b/defects/0ad-0002/bench/bench-0ad-0002-0002.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +# bench-0ad-0002-0002.py +# CCmpSelectable modified-entity tracking: std::find on m_ModifiedEntities +# vector inside a per-entity loop. O(N²) as the set of modified entities +# grows over a frame. Fix: unordered_set for O(1) membership. + +import sys +import time + + +def bench_defective(n): + modified = [] + entities = list(range(n)) + t0 = time.perf_counter() + for ent in entities: + if ent not in modified: # O(|modified|) + modified.append(ent) + return time.perf_counter() - t0 + + +def bench_fixed(n): + modified = set() + entities = list(range(n)) + t0 = time.perf_counter() + for ent in entities: + if ent not in modified: + modified.add(ent) + return time.perf_counter() - t0 + + +TRIALS = 3 +SIZES = [100, 500, 1000, 2000] + + +def run(): + lines = [] + header = "=== 0ad-0002-0002: CCmpSelectable modified-entities std::find vs unordered_set ===" + print(header); lines.append(header) + for n in SIZES: + df = min(bench_defective(n) for _ in range(TRIALS)) + fx = min(bench_fixed(n) for _ in range(TRIALS)) + speedup = (df / fx) if fx > 0 else float("inf") + line = f"N={n:<5}: defective={df*1000:.3f}ms fixed={fx*1000:.3f}ms speedup={speedup:.1f}x" + print(line); lines.append(line); sys.stdout.flush() + return lines + + +if __name__ == "__main__": + run() diff --git a/defects/0ad-0002/bench/results.txt b/defects/0ad-0002/bench/results.txt new file mode 100644 index 000000000..328c059d3 --- /dev/null +++ b/defects/0ad-0002/bench/results.txt @@ -0,0 +1,6 @@ +=== 0ad-0002-0002: CCmpSelectable modified-entities std::find vs unordered_set === +N=100 : defective=0.221ms fixed=0.020ms speedup=10.9x +N=500 : defective=4.980ms fixed=0.078ms speedup=63.9x +N=1000 : defective=19.368ms fixed=0.176ms speedup=110.1x +N=2000 : defective=72.550ms fixed=0.212ms speedup=341.5x + diff --git a/defects/0ad-0002/bench/run_all.py b/defects/0ad-0002/bench/run_all.py new file mode 100644 index 000000000..4a5ebb2a4 --- /dev/null +++ b/defects/0ad-0002/bench/run_all.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 +import importlib.util, os, sys +BENCH_DIR = os.path.dirname(os.path.abspath(__file__)) + +def load_module(filename): + path = os.path.join(BENCH_DIR, filename) + spec = importlib.util.spec_from_file_location("mod", path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + +all_lines = [] +for fname in ["bench-0ad-0002-0002.py"]: + mod = load_module(fname) + lines = mod.run() + all_lines.extend(lines); all_lines.append("") + print(); sys.stdout.flush() + +out_path = os.path.join(BENCH_DIR, "results.txt") +with open(out_path, "w") as f: + f.write("\n".join(all_lines) + "\n") +print(f"results written to {out_path}"); sys.stdout.flush() diff --git a/defects/0ad-0003/Makefile b/defects/0ad-0003/Makefile new file mode 100644 index 000000000..10ca1b3e4 --- /dev/null +++ b/defects/0ad-0003/Makefile @@ -0,0 +1,7 @@ +# 0ad-0003 bench runner +.PHONY: all bench clean +all: bench +bench: + python3 bench/run_all.py +clean: + rm -rf bench/__pycache__ __pycache__ diff --git a/defects/0ad-0003/bench/__pycache__/bench-0ad-0003-0003.cpython-312.pyc b/defects/0ad-0003/bench/__pycache__/bench-0ad-0003-0003.cpython-312.pyc new file mode 100644 index 000000000..05111ef35 Binary files /dev/null and b/defects/0ad-0003/bench/__pycache__/bench-0ad-0003-0003.cpython-312.pyc differ diff --git a/defects/0ad-0003/bench/bench-0ad-0003-0003.py b/defects/0ad-0003/bench/bench-0ad-0003-0003.py new file mode 100644 index 000000000..f4bb3d19e --- /dev/null +++ b/defects/0ad-0003/bench/bench-0ad-0003-0003.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +# bench-0ad-0003-0003.py +# Template-cache usedTemplates tracking: std::find on usedTemplates vector +# inside a per-template-instance loop. O(N²) across instances × template names. +# Fix: unordered_set of template ids for O(1) dedup. + +import sys +import time + + +def bench_defective(n, unique_ratio=0.5): + used = [] + distinct = max(1, int(n * unique_ratio)) + candidates = [f"tpl_{i % distinct}" for i in range(n)] + t0 = time.perf_counter() + for t in candidates: + if t not in used: # O(|used|) + used.append(t) + return time.perf_counter() - t0 + + +def bench_fixed(n, unique_ratio=0.5): + used = set() + distinct = max(1, int(n * unique_ratio)) + candidates = [f"tpl_{i % distinct}" for i in range(n)] + t0 = time.perf_counter() + for t in candidates: + if t not in used: + used.add(t) + return time.perf_counter() - t0 + + +TRIALS = 3 +SIZES = [100, 500, 1000, 2000] + + +def run(): + lines = [] + header = "=== 0ad-0003-0003: usedTemplates std::find vs unordered_set ===" + print(header); lines.append(header) + for n in SIZES: + df = min(bench_defective(n) for _ in range(TRIALS)) + fx = min(bench_fixed(n) for _ in range(TRIALS)) + speedup = (df / fx) if fx > 0 else float("inf") + line = f"N={n:<5}: defective={df*1000:.3f}ms fixed={fx*1000:.3f}ms speedup={speedup:.1f}x" + print(line); lines.append(line); sys.stdout.flush() + return lines + + +if __name__ == "__main__": + run() diff --git a/defects/0ad-0003/bench/results.txt b/defects/0ad-0003/bench/results.txt new file mode 100644 index 000000000..aefc1f4bb --- /dev/null +++ b/defects/0ad-0003/bench/results.txt @@ -0,0 +1,6 @@ +=== 0ad-0003-0003: usedTemplates std::find vs unordered_set === +N=100 : defective=0.120ms fixed=0.018ms speedup=6.6x +N=500 : defective=2.615ms fixed=0.118ms speedup=22.1x +N=1000 : defective=8.730ms fixed=0.111ms speedup=78.4x +N=2000 : defective=37.381ms fixed=0.372ms speedup=100.6x + diff --git a/defects/0ad-0003/bench/run_all.py b/defects/0ad-0003/bench/run_all.py new file mode 100644 index 000000000..517a954f0 --- /dev/null +++ b/defects/0ad-0003/bench/run_all.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 +import importlib.util, os, sys +BENCH_DIR = os.path.dirname(os.path.abspath(__file__)) + +def load_module(filename): + path = os.path.join(BENCH_DIR, filename) + spec = importlib.util.spec_from_file_location("mod", path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + +all_lines = [] +for fname in ["bench-0ad-0003-0003.py"]: + mod = load_module(fname) + lines = mod.run() + all_lines.extend(lines); all_lines.append("") + print(); sys.stdout.flush() + +out_path = os.path.join(BENCH_DIR, "results.txt") +with open(out_path, "w") as f: + f.write("\n".join(all_lines) + "\n") +print(f"results written to {out_path}"); sys.stdout.flush() diff --git a/defects/0ad-0004/Makefile b/defects/0ad-0004/Makefile new file mode 100644 index 000000000..e0673caa8 --- /dev/null +++ b/defects/0ad-0004/Makefile @@ -0,0 +1,7 @@ +# 0ad-0004 bench runner +.PHONY: all bench clean +all: bench +bench: + python3 bench/run_all.py +clean: + rm -rf bench/__pycache__ __pycache__ diff --git a/defects/0ad-0004/bench/__pycache__/bench-0ad-0004-0004.cpython-312.pyc b/defects/0ad-0004/bench/__pycache__/bench-0ad-0004-0004.cpython-312.pyc new file mode 100644 index 000000000..33df4549c Binary files /dev/null and b/defects/0ad-0004/bench/__pycache__/bench-0ad-0004-0004.cpython-312.pyc differ diff --git a/defects/0ad-0004/bench/bench-0ad-0004-0004.py b/defects/0ad-0004/bench/bench-0ad-0004-0004.py new file mode 100644 index 000000000..eabd2724f --- /dev/null +++ b/defects/0ad-0004/bench/bench-0ad-0004-0004.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +# bench-0ad-0004-0004.py +# XmppClient + NetServer: lobby auth tokens logged verbatim. CWE-312 / MOAD-0004 +# A Logged Secret. Correctness metric: does log output contain the raw token? +# Fixed path replaces token with "[REDACTED]" before logging. + +import re +import sys + + +def emit_defective(username, token, logsink): + # Verbatim log of the token — the defect. + logsink.append(f"XmppClient: Received lobby auth: {token} from {username}") + + +def emit_fixed(username, token, logsink): + # Redacted log — the fix. + logsink.append(f"XmppClient: Received lobby auth: [REDACTED] from {username}") + + +def count_leaks(logsink, tokens): + """Return the number of log lines that contain a raw token value.""" + leaks = 0 + for line in logsink: + for tok in tokens: + if tok in line: + leaks += 1 + break + return leaks + + +def run(): + lines = [] + header = "=== 0ad-0004-0004: lobby auth token log redaction (correctness) ===" + print(header); lines.append(header) + + # Simulate N auth events with random-looking tokens + cases = [100, 1000, 10000] + for n in cases: + tokens = [f"tok_{i:08x}" for i in range(n)] + users = [f"user{i}" for i in range(n)] + + sink_def = [] + sink_fix = [] + for u, t in zip(users, tokens): + emit_defective(u, t, sink_def) + emit_fixed(u, t, sink_fix) + + leaks_def = count_leaks(sink_def, tokens) + leaks_fix = count_leaks(sink_fix, tokens) + line = (f"N={n:<5}: defective_leaks={leaks_def:>5} fixed_leaks={leaks_fix:>5}" + f" redaction_rate={(n - leaks_fix) / n * 100:.1f}%") + print(line); lines.append(line); sys.stdout.flush() + return lines + + +if __name__ == "__main__": + run() diff --git a/defects/0ad-0004/bench/results.txt b/defects/0ad-0004/bench/results.txt new file mode 100644 index 000000000..3c2521f43 --- /dev/null +++ b/defects/0ad-0004/bench/results.txt @@ -0,0 +1,5 @@ +=== 0ad-0004-0004: lobby auth token log redaction (correctness) === +N=100 : defective_leaks= 100 fixed_leaks= 0 redaction_rate=100.0% +N=1000 : defective_leaks= 1000 fixed_leaks= 0 redaction_rate=100.0% +N=10000: defective_leaks=10000 fixed_leaks= 0 redaction_rate=100.0% + diff --git a/defects/0ad-0004/bench/run_all.py b/defects/0ad-0004/bench/run_all.py new file mode 100644 index 000000000..2233799f7 --- /dev/null +++ b/defects/0ad-0004/bench/run_all.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 +import importlib.util, os, sys +BENCH_DIR = os.path.dirname(os.path.abspath(__file__)) + +def load_module(filename): + path = os.path.join(BENCH_DIR, filename) + spec = importlib.util.spec_from_file_location("mod", path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + +all_lines = [] +for fname in ["bench-0ad-0004-0004.py"]: + mod = load_module(fname) + lines = mod.run() + all_lines.extend(lines); all_lines.append("") + print(); sys.stdout.flush() + +out_path = os.path.join(BENCH_DIR, "results.txt") +with open(out_path, "w") as f: + f.write("\n".join(all_lines) + "\n") +print(f"results written to {out_path}"); sys.stdout.flush() diff --git a/defects/activemq/Makefile b/defects/activemq/Makefile new file mode 100644 index 000000000..470c53582 --- /dev/null +++ b/defects/activemq/Makefile @@ -0,0 +1,7 @@ +# activemq bench runner +.PHONY: all bench clean +all: bench +bench: + python3 bench/run_all.py +clean: + rm -rf bench/__pycache__ __pycache__ diff --git a/defects/activemq/bench/__pycache__/bench-activemq-0001.cpython-312.pyc b/defects/activemq/bench/__pycache__/bench-activemq-0001.cpython-312.pyc new file mode 100644 index 000000000..6aff4b085 Binary files /dev/null and b/defects/activemq/bench/__pycache__/bench-activemq-0001.cpython-312.pyc differ diff --git a/defects/activemq/bench/__pycache__/bench-activemq-0002.cpython-312.pyc b/defects/activemq/bench/__pycache__/bench-activemq-0002.cpython-312.pyc new file mode 100644 index 000000000..44961a8e8 Binary files /dev/null and b/defects/activemq/bench/__pycache__/bench-activemq-0002.cpython-312.pyc differ diff --git a/defects/activemq/bench/__pycache__/bench-activemq-0003.cpython-312.pyc b/defects/activemq/bench/__pycache__/bench-activemq-0003.cpython-312.pyc new file mode 100644 index 000000000..bcf1c04d5 Binary files /dev/null and b/defects/activemq/bench/__pycache__/bench-activemq-0003.cpython-312.pyc differ diff --git a/defects/activemq/bench/bench-activemq-0001.py b/defects/activemq/bench/bench-activemq-0001.py new file mode 100644 index 000000000..9569403d0 --- /dev/null +++ b/defects/activemq/bench/bench-activemq-0001.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +# bench-activemq-0001.py +# ActiveMQ Queue/Topic consumer list: per-message round-robin rotation via +# remove+add on the consumers List. For C consumers and M messages, cost is +# O(M·C) per dispatch. Topic path also does linear contains() for dedup. +# Fix: rotation-index pointer + parallel Set for O(1) checks. + +import sys +import time + + +def bench_defective(n_consumers, n_messages): + """Remove + append to rotate; O(C) per message.""" + consumers = list(range(n_consumers)) + t0 = time.perf_counter() + for m in range(n_messages): + target = consumers[0] + # remove target from list and re-append — O(C) + consumers.pop(0) + consumers.append(target) + return time.perf_counter() - t0 + + +def bench_fixed(n_consumers, n_messages): + """Rotation index cursor; O(1) per message.""" + consumers = list(range(n_consumers)) + rot = 0 + t0 = time.perf_counter() + for m in range(n_messages): + target = consumers[rot] + rot = (rot + 1) % len(consumers) + return time.perf_counter() - t0 + + +TRIALS = 3 +CASES = [(50, 1000), (200, 5000), (500, 10000), (1000, 20000)] + + +def run(): + lines = [] + header = "=== activemq-0001: Queue/Topic consumer rotation list.remove+add vs index cursor ===" + print(header); lines.append(header) + for c, m in CASES: + df = min(bench_defective(c, m) for _ in range(TRIALS)) + fx = min(bench_fixed(c, m) for _ in range(TRIALS)) + speedup = (df / fx) if fx > 0 else float("inf") + line = f"C={c:<5} M={m:<6}: defective={df*1000:.3f}ms fixed={fx*1000:.3f}ms speedup={speedup:.1f}x" + print(line); lines.append(line); sys.stdout.flush() + return lines + + +if __name__ == "__main__": + run() diff --git a/defects/activemq/bench/bench-activemq-0002.py b/defects/activemq/bench/bench-activemq-0002.py new file mode 100644 index 000000000..cc7926c31 --- /dev/null +++ b/defects/activemq/bench/bench-activemq-0002.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# bench-activemq-0002.py +# DemandBridge candidateConsumers.contains() inside a per-consumer loop: +# O(N²) across N candidates. Fix: parallel HashSet for O(1) membership. + +import sys +import time + + +def bench_defective(n): + candidates = [] + incoming = list(range(n)) + t0 = time.perf_counter() + for c in incoming: + if c not in candidates: # O(len(candidates)) + candidates.append(c) + return time.perf_counter() - t0 + + +def bench_fixed(n): + candidates_set = set() + candidates = [] + incoming = list(range(n)) + t0 = time.perf_counter() + for c in incoming: + if c not in candidates_set: + candidates_set.add(c) + candidates.append(c) + return time.perf_counter() - t0 + + +TRIALS = 3 +SIZES = [100, 500, 1000, 2000] + + +def run(): + lines = [] + header = "=== activemq-0002: DemandBridge candidateConsumers List.contains vs HashSet ===" + print(header); lines.append(header) + for n in SIZES: + df = min(bench_defective(n) for _ in range(TRIALS)) + fx = min(bench_fixed(n) for _ in range(TRIALS)) + speedup = (df / fx) if fx > 0 else float("inf") + line = f"N={n:<5}: defective={df*1000:.3f}ms fixed={fx*1000:.3f}ms speedup={speedup:.1f}x" + print(line); lines.append(line); sys.stdout.flush() + return lines + + +if __name__ == "__main__": + run() diff --git a/defects/activemq/bench/bench-activemq-0003.py b/defects/activemq/bench/bench-activemq-0003.py new file mode 100644 index 000000000..e2326e2dd --- /dev/null +++ b/defects/activemq/bench/bench-activemq-0003.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +# bench-activemq-0003.py +# TransactionContext endedXATransactions List: .contains check before add on +# a growing ended-XA-txn list. Per-transaction O(N) check over all prior +# endings; fix is a parallel HashSet for O(1) membership. + +import sys +import time + + +def bench_defective(n): + ended = [] + incoming = [f"xid_{i:06d}" for i in range(n)] + t0 = time.perf_counter() + for xid in incoming: + if xid not in ended: # O(|ended|) + ended.append(xid) + return time.perf_counter() - t0 + + +def bench_fixed(n): + ended_set = set() + ended = [] + incoming = [f"xid_{i:06d}" for i in range(n)] + t0 = time.perf_counter() + for xid in incoming: + if xid not in ended_set: + ended_set.add(xid) + ended.append(xid) + return time.perf_counter() - t0 + + +TRIALS = 3 +SIZES = [100, 500, 1000, 2000] + + +def run(): + lines = [] + header = "=== activemq-0003: TransactionContext endedXATransactions List.contains vs HashSet ===" + print(header); lines.append(header) + for n in SIZES: + df = min(bench_defective(n) for _ in range(TRIALS)) + fx = min(bench_fixed(n) for _ in range(TRIALS)) + speedup = (df / fx) if fx > 0 else float("inf") + line = f"N={n:<5}: defective={df*1000:.3f}ms fixed={fx*1000:.3f}ms speedup={speedup:.1f}x" + print(line); lines.append(line); sys.stdout.flush() + return lines + + +if __name__ == "__main__": + run() diff --git a/defects/activemq/bench/results.txt b/defects/activemq/bench/results.txt new file mode 100644 index 000000000..e0166d010 --- /dev/null +++ b/defects/activemq/bench/results.txt @@ -0,0 +1,18 @@ +=== activemq-0001: Queue/Topic consumer rotation list.remove+add vs index cursor === +C=50 M=1000 : defective=0.307ms fixed=0.104ms speedup=3.0x +C=200 M=5000 : defective=0.735ms fixed=0.537ms speedup=1.4x +C=500 M=10000 : defective=1.630ms fixed=1.489ms speedup=1.1x +C=1000 M=20000 : defective=4.415ms fixed=3.119ms speedup=1.4x + +=== activemq-0002: DemandBridge candidateConsumers List.contains vs HashSet === +N=100 : defective=0.089ms fixed=0.010ms speedup=8.7x +N=500 : defective=2.128ms fixed=0.047ms speedup=45.4x +N=1000 : defective=8.872ms fixed=0.090ms speedup=98.4x +N=2000 : defective=35.776ms fixed=0.218ms speedup=163.8x + +=== activemq-0003: TransactionContext endedXATransactions List.contains vs HashSet === +N=100 : defective=0.119ms fixed=0.014ms speedup=8.8x +N=500 : defective=2.891ms fixed=0.058ms speedup=49.4x +N=1000 : defective=11.783ms fixed=0.124ms speedup=95.3x +N=2000 : defective=51.216ms fixed=0.288ms speedup=178.0x + diff --git a/defects/activemq/bench/run_all.py b/defects/activemq/bench/run_all.py new file mode 100644 index 000000000..e09de9693 --- /dev/null +++ b/defects/activemq/bench/run_all.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 +import importlib.util, os, sys +BENCH_DIR = os.path.dirname(os.path.abspath(__file__)) + +def load_module(filename): + path = os.path.join(BENCH_DIR, filename) + spec = importlib.util.spec_from_file_location("mod", path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + +all_lines = [] +for fname in ["bench-activemq-0001.py", "bench-activemq-0002.py", "bench-activemq-0003.py"]: + mod = load_module(fname) + lines = mod.run() + all_lines.extend(lines); all_lines.append("") + print(); sys.stdout.flush() + +out_path = os.path.join(BENCH_DIR, "results.txt") +with open(out_path, "w") as f: + f.write("\n".join(all_lines) + "\n") +print(f"results written to {out_path}"); sys.stdout.flush() diff --git a/defects/lean4/bench/__pycache__/bench-lean4-0004.cpython-312.pyc b/defects/lean4/bench/__pycache__/bench-lean4-0004.cpython-312.pyc new file mode 100644 index 000000000..805c7a868 Binary files /dev/null and b/defects/lean4/bench/__pycache__/bench-lean4-0004.cpython-312.pyc differ diff --git a/defects/lean4/bench/__pycache__/bench-lean4-0005.cpython-312.pyc b/defects/lean4/bench/__pycache__/bench-lean4-0005.cpython-312.pyc new file mode 100644 index 000000000..3c96b6029 Binary files /dev/null and b/defects/lean4/bench/__pycache__/bench-lean4-0005.cpython-312.pyc differ diff --git a/defects/lean4/bench/__pycache__/bench-lean4-0006.cpython-312.pyc b/defects/lean4/bench/__pycache__/bench-lean4-0006.cpython-312.pyc new file mode 100644 index 000000000..94d86bf75 Binary files /dev/null and b/defects/lean4/bench/__pycache__/bench-lean4-0006.cpython-312.pyc differ diff --git a/defects/lean4/bench/__pycache__/bench-lean4-0007.cpython-312.pyc b/defects/lean4/bench/__pycache__/bench-lean4-0007.cpython-312.pyc new file mode 100644 index 000000000..1327dc058 Binary files /dev/null and b/defects/lean4/bench/__pycache__/bench-lean4-0007.cpython-312.pyc differ diff --git a/defects/lean4/bench/bench-lean4-0004.py b/defects/lean4/bench/bench-lean4-0004.py new file mode 100644 index 000000000..5bc90203f --- /dev/null +++ b/defects/lean4/bench/bench-lean4-0004.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +# bench-lean4-0004.py +# ir_interpreter lookup_symbol: shared_lock read -> unlock -> unique_lock insert +# (defective) leaves a window where another thread can insert/mutate the cache. +# Fixed path: single unique_lock acquisition. Metric is correctness (lost +# inserts) not wall-clock — race windows are per-operation, microseconds wide. + +import sys +import threading +import time + + +def bench_defective(n_threads, ops_per_thread): + """Model: read-without-lock → shared_lock → unlock → unique_lock → insert. + We simulate the unlock-then-relock race by splitting the critical section + with a yield, giving concurrent writers a chance to observe stale state. + """ + cache = {} + lock = threading.Lock() + lost_updates = [0] + + def worker(tid): + for i in range(ops_per_thread): + key = f"fn_{(tid * 1000 + i) % (ops_per_thread * 2)}" + # defective: check presence without lock + if key in cache: + continue + # yield: another thread may insert the same key now + time.sleep(0) + with lock: + if key in cache: + # We raced: another thread inserted. Lost update if we + # were about to compute and store a value. + lost_updates[0] += 1 + else: + cache[key] = tid + + t0 = time.perf_counter() + threads = [threading.Thread(target=worker, args=(i,)) for i in range(n_threads)] + for t in threads: t.start() + for t in threads: t.join() + elapsed = time.perf_counter() - t0 + return elapsed, lost_updates[0] + + +def bench_fixed(n_threads, ops_per_thread): + """Single unique_lock acquisition — no unlock-relock window.""" + cache = {} + lock = threading.Lock() + lost_updates = [0] + + def worker(tid): + for i in range(ops_per_thread): + key = f"fn_{(tid * 1000 + i) % (ops_per_thread * 2)}" + with lock: + if key in cache: + continue + cache[key] = tid + + t0 = time.perf_counter() + threads = [threading.Thread(target=worker, args=(i,)) for i in range(n_threads)] + for t in threads: t.start() + for t in threads: t.join() + elapsed = time.perf_counter() - t0 + return elapsed, lost_updates[0] + + +TRIALS = 3 + + +def run(): + lines = [] + header = "=== lean4-0004: ir_interpreter double-checked lock race (correctness) ===" + print(header); lines.append(header) + for n_threads, ops in [(2, 500), (4, 500), (8, 500)]: + # Run multiple trials and report worst case (most races observed) + def_races = [] + fix_races = [] + for _ in range(TRIALS): + _, dr = bench_defective(n_threads, ops) + _, fr = bench_fixed(n_threads, ops) + def_races.append(dr); fix_races.append(fr) + line = (f"threads={n_threads:<2} ops={ops:<4}: " + f"defective_lost_updates(max)={max(def_races):>4} " + f"fixed_lost_updates(max)={max(fix_races):>4}") + print(line); lines.append(line); sys.stdout.flush() + return lines + + +if __name__ == "__main__": + run() diff --git a/defects/lean4/bench/bench-lean4-0005.py b/defects/lean4/bench/bench-lean4-0005.py new file mode 100644 index 000000000..a3f1204b0 --- /dev/null +++ b/defects/lean4/bench/bench-lean4-0005.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +# bench-lean4-0005.py +# registerJob IO.Ref.modify: concurrent tasks .push their OpaqueJob into a +# shared IO.Ref (Array OpaqueJob). .modify is not atomic across tasks; pushes +# race and get lost. Fixed path: IO.Mutex (Array OpaqueJob) with .atomically. +# Correctness metric: registered count vs expected count. + +import sys +import threading +import time + + +def bench_defective(n_workers, pushes_per_worker): + """Model: read-modify-write without lock. Observable on CPython via the + GIL is weak; we emulate a non-atomic read-then-write by reading the + current list, appending locally, then assigning — this is the same + semantics as IO.Ref.modify without a mutex.""" + registered = [[]] + + def worker(tid): + for i in range(pushes_per_worker): + job = f"job_{tid}_{i}" + # Non-atomic: read current, append, write back + current = registered[0] + # Yield to widen the race window + time.sleep(0) + registered[0] = current + [job] + + t0 = time.perf_counter() + threads = [threading.Thread(target=worker, args=(i,)) for i in range(n_workers)] + for t in threads: t.start() + for t in threads: t.join() + elapsed = time.perf_counter() - t0 + expected = n_workers * pushes_per_worker + actual = len(registered[0]) + return elapsed, expected, actual + + +def bench_fixed(n_workers, pushes_per_worker): + """IO.Mutex equivalent: guarded append under lock.""" + registered = [] + lock = threading.Lock() + + def worker(tid): + for i in range(pushes_per_worker): + job = f"job_{tid}_{i}" + with lock: + registered.append(job) + + t0 = time.perf_counter() + threads = [threading.Thread(target=worker, args=(i,)) for i in range(n_workers)] + for t in threads: t.start() + for t in threads: t.join() + elapsed = time.perf_counter() - t0 + expected = n_workers * pushes_per_worker + actual = len(registered) + return elapsed, expected, actual + + +TRIALS = 3 + + +def run(): + lines = [] + header = "=== lean4-0005: Lake jobreg IO.Ref.modify race (correctness) ===" + print(header); lines.append(header) + for n_workers, pushes in [(4, 200), (8, 200), (16, 200)]: + def_losses = [] + fix_losses = [] + for _ in range(TRIALS): + _, exp, act_d = bench_defective(n_workers, pushes) + _, _, act_f = bench_fixed(n_workers, pushes) + def_losses.append(exp - act_d) + fix_losses.append(exp - act_f) + line = (f"workers={n_workers:<2} pushes={pushes:<4}: " + f"expected={n_workers*pushes:<5} " + f"defective_lost(max)={max(def_losses):>4} " + f"fixed_lost(max)={max(fix_losses):>4}") + print(line); lines.append(line); sys.stdout.flush() + return lines + + +if __name__ == "__main__": + run() diff --git a/defects/lean4/bench/bench-lean4-0006.py b/defects/lean4/bench/bench-lean4-0006.py new file mode 100644 index 000000000..41c18093e --- /dev/null +++ b/defects/lean4/bench/bench-lean4-0006.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +# bench-lean4-0006.py +# LEAN_THREAD_PTR(g_opts) not reset at task boundaries: pooled thread inherits +# previous task's trace options. Fixed path: reset g_opts to nullptr on task +# finalize. Correctness metric: observed leakage count (task B sees task A's +# options). + +import sys +import threading +import queue +import time + + +def make_pool(n_threads, task_queue, context_fn): + """Minimal thread-pool. context_fn receives (tid, task) and runs under + whatever thread-local setup context_fn itself establishes.""" + def loop(tid): + while True: + task = task_queue.get() + if task is None: + task_queue.task_done() + break + context_fn(tid, task) + task_queue.task_done() + + threads = [threading.Thread(target=loop, args=(i,)) for i in range(n_threads)] + for t in threads: + t.start() + return threads + + +def bench_defective(n_threads, n_tasks): + """g_opts modelled as a thread-local dict; defective path does NOT clear + g_opts between tasks, so a reused thread keeps the prior task's options. + Count leakages: task T_new observes g_opts_from T_prev.""" + g_opts = threading.local() + leakages = [0] + leak_lock = threading.Lock() + tq = queue.Queue() + + def run_task(tid, task): + prior = getattr(g_opts, 'payload', None) + if prior is not None and prior != task['tid']: + with leak_lock: + leakages[0] += 1 + g_opts.payload = task['tid'] + # simulate work + _ = sum(range(50)) + # defective: no reset + + threads = make_pool(n_threads, tq, run_task) + t0 = time.perf_counter() + for i in range(n_tasks): + tq.put({'tid': f't{i}'}) + for _ in threads: + tq.put(None) + for t in threads: + t.join() + elapsed = time.perf_counter() - t0 + return elapsed, leakages[0] + + +def bench_fixed(n_threads, n_tasks): + """Fixed: task finalizer resets g_opts to None before returning.""" + g_opts = threading.local() + leakages = [0] + leak_lock = threading.Lock() + tq = queue.Queue() + + def run_task(tid, task): + prior = getattr(g_opts, 'payload', None) + if prior is not None and prior != task['tid']: + with leak_lock: + leakages[0] += 1 + g_opts.payload = task['tid'] + _ = sum(range(50)) + g_opts.payload = None # task finalizer: reset + + threads = make_pool(n_threads, tq, run_task) + t0 = time.perf_counter() + for i in range(n_tasks): + tq.put({'tid': f't{i}'}) + for _ in threads: + tq.put(None) + for t in threads: + t.join() + elapsed = time.perf_counter() - t0 + return elapsed, leakages[0] + + +TRIALS = 3 + + +def run(): + lines = [] + header = "=== lean4-0006: kernel/trace g_opts thread-local leakage (correctness) ===" + print(header); lines.append(header) + for n_threads, n_tasks in [(2, 200), (4, 400), (8, 800)]: + def_leaks = [] + fix_leaks = [] + for _ in range(TRIALS): + _, dl = bench_defective(n_threads, n_tasks) + _, fl = bench_fixed(n_threads, n_tasks) + def_leaks.append(dl); fix_leaks.append(fl) + line = (f"threads={n_threads:<2} tasks={n_tasks:<4}: " + f"defective_leakages(max)={max(def_leaks):>5} " + f"fixed_leakages(max)={max(fix_leaks):>5}") + print(line); lines.append(line); sys.stdout.flush() + return lines + + +if __name__ == "__main__": + run() diff --git a/defects/lean4/bench/bench-lean4-0007.py b/defects/lean4/bench/bench-lean4-0007.py new file mode 100644 index 000000000..e71205123 --- /dev/null +++ b/defects/lean4/bench/bench-lean4-0007.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +# bench-lean4-0007.py +# Windows env var inheritance: new_env_vars.count({key_begin, key_end}) constructs +# a std::string per iteration over N inherited env vars vs a pre-built +# std::unordered_set of override keys for O(1) amortized lookup. + +import sys +import time + + +def bench_defective(n): + """Model: list-backed key set with per-iteration allocation.""" + override_keys = [f"KEY_{i}" for i in range(max(1, n // 4))] + env_keys = [f"ENV_{i}" for i in range(n)] + [f"KEY_{i}" for i in range(n // 4)] + + t0 = time.perf_counter() + kept = [] + for k in env_keys: + # Emulate per-iteration std::string construction + linear find. + tmp = "".join(list(k)) + if tmp not in override_keys: + kept.append(k) + return time.perf_counter() - t0 + + +def bench_fixed(n): + """Pre-built unordered_set of override keys; O(1) amortized lookup.""" + override_keys = {f"KEY_{i}" for i in range(max(1, n // 4))} + env_keys = [f"ENV_{i}" for i in range(n)] + [f"KEY_{i}" for i in range(n // 4)] + + t0 = time.perf_counter() + kept = [] + for k in env_keys: + if k not in override_keys: + kept.append(k) + return time.perf_counter() - t0 + + +TRIALS = 3 +SIZES = [100, 500, 1000, 2000] + + +def run(): + lines = [] + header = "=== lean4-0007: process.cpp Windows env vars list-count vs unordered_set ===" + print(header); lines.append(header) + for n in SIZES: + d = min(bench_defective(n) for _ in range(TRIALS)) + f = min(bench_fixed(n) for _ in range(TRIALS)) + speedup = (d / f) if f > 0 else float("inf") + line = f"N={n:<5}: defective={d*1000:.3f}ms fixed={f*1000:.3f}ms speedup={speedup:.1f}x" + print(line); lines.append(line); sys.stdout.flush() + return lines + + +if __name__ == "__main__": + run() diff --git a/defects/lean4/bench/results.txt b/defects/lean4/bench/results.txt index d4ef233e2..ecab869ef 100644 --- a/defects/lean4/bench/results.txt +++ b/defects/lean4/bench/results.txt @@ -1,16 +1,37 @@ === lean4-0001: guardCycle List vs HashSet === -N=100 : defective=0.206ms fixed=0.027ms speedup=7.7x -N=500 : defective=3.366ms fixed=0.086ms speedup=39.1x -N=1000 : defective=7.688ms fixed=0.292ms speedup=26.3x -N=2000 : defective=38.064ms fixed=1.101ms speedup=34.6x +N=100 : defective=0.125ms fixed=0.021ms speedup=6.0x +N=500 : defective=3.103ms fixed=0.139ms speedup=22.3x +N=1000 : defective=12.449ms fixed=0.300ms speedup=41.4x +N=2000 : defective=40.612ms fixed=1.025ms speedup=39.6x === lean4-0002: inductive type check std::find vs set === -K=N=100 : defective=0.319ms fixed=0.002ms speedup=146.7x -K=N=500 : defective=2.772ms fixed=0.011ms speedup=259.5x -K=N=1000: defective=14.088ms fixed=0.021ms speedup=678.3x +K=N=100 : defective=0.206ms fixed=0.004ms speedup=57.6x +K=N=500 : defective=5.312ms fixed=0.016ms speedup=325.6x +K=N=1000: defective=18.706ms fixed=0.030ms speedup=629.7x === lean4-0003: fresh name generation === -N=100 : defective=0.105ms fixed=0.004ms speedup=29.7x -N=500 : defective=3.317ms fixed=0.016ms speedup=205.6x -N=1000 : defective=19.888ms fixed=0.095ms speedup=209.7x +N=100 : defective=0.167ms fixed=0.004ms speedup=44.0x +N=500 : defective=4.501ms fixed=0.022ms speedup=206.1x +N=1000 : defective=17.938ms fixed=0.049ms speedup=369.1x + +=== lean4-0004: ir_interpreter double-checked lock race (correctness) === +threads=2 ops=500 : defective_lost_updates(max)= 490 fixed_lost_updates(max)= 0 +threads=4 ops=500 : defective_lost_updates(max)=1084 fixed_lost_updates(max)= 0 +threads=8 ops=500 : defective_lost_updates(max)= 994 fixed_lost_updates(max)= 0 + +=== lean4-0005: Lake jobreg IO.Ref.modify race (correctness) === +workers=4 pushes=200 : expected=800 defective_lost(max)= 599 fixed_lost(max)= 0 +workers=8 pushes=200 : expected=1600 defective_lost(max)=1400 fixed_lost(max)= 0 +workers=16 pushes=200 : expected=3200 defective_lost(max)=2993 fixed_lost(max)= 0 + +=== lean4-0006: kernel/trace g_opts thread-local leakage (correctness) === +threads=2 tasks=200 : defective_leakages(max)= 199 fixed_leakages(max)= 0 +threads=4 tasks=400 : defective_leakages(max)= 399 fixed_leakages(max)= 0 +threads=8 tasks=800 : defective_leakages(max)= 798 fixed_leakages(max)= 0 + +=== lean4-0007: process.cpp Windows env vars list-count vs unordered_set === +N=100 : defective=0.315ms fixed=0.028ms speedup=11.2x +N=500 : defective=4.737ms fixed=0.127ms speedup=37.4x +N=1000 : defective=19.896ms fixed=0.281ms speedup=70.9x +N=2000 : defective=23.665ms fixed=0.171ms speedup=138.0x diff --git a/defects/lean4/bench/run_all.py b/defects/lean4/bench/run_all.py index 1dfbd1401..78d08d0bb 100644 --- a/defects/lean4/bench/run_all.py +++ b/defects/lean4/bench/run_all.py @@ -4,7 +4,6 @@ import sys import os import importlib.util -import time BENCH_DIR = os.path.dirname(os.path.abspath(__file__)) @@ -17,7 +16,15 @@ def load_module(filename): all_lines = [] -for fname in ["bench-lean4-0001.py", "bench-lean4-0002.py", "bench-lean4-0003.py"]: +for fname in [ + "bench-lean4-0001.py", + "bench-lean4-0002.py", + "bench-lean4-0003.py", + "bench-lean4-0004.py", + "bench-lean4-0005.py", + "bench-lean4-0006.py", + "bench-lean4-0007.py", +]: mod = load_module(fname) lines = mod.run() all_lines.extend(lines) diff --git a/defects/linux/Makefile b/defects/linux/Makefile new file mode 100644 index 000000000..12575577e --- /dev/null +++ b/defects/linux/Makefile @@ -0,0 +1,8 @@ +.PHONY: all bench bench-kernel clean +all: bench +bench: + python3 bench/run_all.py +bench-kernel: + bash bench/build-and-bench.sh +clean: + rm -rf bench/__pycache__ __pycache__ diff --git a/defects/linux/bench/bench-linux-0001.py b/defects/linux/bench/bench-linux-0001.py new file mode 100644 index 000000000..3d6968426 --- /dev/null +++ b/defects/linux/bench/bench-linux-0001.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +# bench-linux-0001.py +# scripts/headerdep.pl detect_cycles: grep{} membership inside BFS over header chains. +# Models O(D*K) → O(D+K) via list-scan inside a loop vs set/dict lookup. + +import sys +import time + + +def bench_defective(n, k): + """Outer loop over N, inner linear list scan over K — O(N*K).""" + pool = list(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + accepted = [] + for x in items: + if x in pool: # list __contains__ = O(K) + accepted.append(x) + # simulate nested work: scan pool to find position + try: + _ = pool.index(x) + except ValueError: + pass + return time.perf_counter() - t0 + + +def bench_fixed(n, k): + """Outer loop over N, inner O(1) set/dict lookup.""" + pool = list(range(k)) + pool_set = set(pool) + pool_pos = {v: i for i, v in enumerate(pool)} + items = list(range(n)) + t0 = time.perf_counter() + accepted = [] + for x in items: + if x in pool_set: # O(1) + accepted.append(x) + _ = pool_pos.get(x) # O(1) + return time.perf_counter() - t0 + + +TRIALS = 3 +CASES = [(100, 10), (500, 20), (1000, 30), (2000, 50)] + + +def run(): + lines = [] + header = "=== linux-0001: headerdep detect_cycles BFS list-grep ===" + print(header); lines.append(header) + for n, k in CASES: + df = min(bench_defective(n, k) for _ in range(TRIALS)) + fx = min(bench_fixed(n, k) for _ in range(TRIALS)) + speedup = (df / fx) if fx > 0 else float("inf") + line = f"D={n:<5} K={k:<5}: defective={df*1000:.3f}ms fixed={fx*1000:.3f}ms speedup={speedup:.1f}x" + print(line); lines.append(line); sys.stdout.flush() + return lines + + +if __name__ == "__main__": + run() diff --git a/defects/linux/bench/bench-linux-0002.py b/defects/linux/bench/bench-linux-0002.py new file mode 100644 index 000000000..e7361c4c5 --- /dev/null +++ b/defects/linux/bench/bench-linux-0002.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +# bench-linux-0002.py +# kernel/auditsc.c audit_filter_rules: per-name linear scan of inode list. +# Models O(F*R) → O(F+R) via list-scan inside a loop vs set/dict lookup. + +import sys +import time + + +def bench_defective(n, k): + """Outer loop over N, inner linear list scan over K — O(N*K).""" + pool = list(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + accepted = [] + for x in items: + if x in pool: # list __contains__ = O(K) + accepted.append(x) + # simulate nested work: scan pool to find position + try: + _ = pool.index(x) + except ValueError: + pass + return time.perf_counter() - t0 + + +def bench_fixed(n, k): + """Outer loop over N, inner O(1) set/dict lookup.""" + pool = list(range(k)) + pool_set = set(pool) + pool_pos = {v: i for i, v in enumerate(pool)} + items = list(range(n)) + t0 = time.perf_counter() + accepted = [] + for x in items: + if x in pool_set: # O(1) + accepted.append(x) + _ = pool_pos.get(x) # O(1) + return time.perf_counter() - t0 + + +TRIALS = 3 +CASES = [(100, 20), (500, 50), (1000, 100), (2000, 200)] + + +def run(): + lines = [] + header = "=== linux-0002: auditsc audit_filter_rules inode-list scan ===" + print(header); lines.append(header) + for n, k in CASES: + df = min(bench_defective(n, k) for _ in range(TRIALS)) + fx = min(bench_fixed(n, k) for _ in range(TRIALS)) + speedup = (df / fx) if fx > 0 else float("inf") + line = f"F={n:<5} R={k:<5}: defective={df*1000:.3f}ms fixed={fx*1000:.3f}ms speedup={speedup:.1f}x" + print(line); lines.append(line); sys.stdout.flush() + return lines + + +if __name__ == "__main__": + run() diff --git a/defects/linux/bench/bench-linux-0003.py b/defects/linux/bench/bench-linux-0003.py new file mode 100644 index 000000000..123cc68f2 --- /dev/null +++ b/defects/linux/bench/bench-linux-0003.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +# bench-linux-0003.py +# net/core/dev.c __dev_alloc_name: nested netdev_for_each_altname per device. +# Models O(D*A) → O(D+A) via list-scan inside a loop vs set/dict lookup. + +import sys +import time + + +def bench_defective(n, k): + """Outer loop over N, inner linear list scan over K — O(N*K).""" + pool = list(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + accepted = [] + for x in items: + if x in pool: # list __contains__ = O(K) + accepted.append(x) + # simulate nested work: scan pool to find position + try: + _ = pool.index(x) + except ValueError: + pass + return time.perf_counter() - t0 + + +def bench_fixed(n, k): + """Outer loop over N, inner O(1) set/dict lookup.""" + pool = list(range(k)) + pool_set = set(pool) + pool_pos = {v: i for i, v in enumerate(pool)} + items = list(range(n)) + t0 = time.perf_counter() + accepted = [] + for x in items: + if x in pool_set: # O(1) + accepted.append(x) + _ = pool_pos.get(x) # O(1) + return time.perf_counter() - t0 + + +TRIALS = 3 +CASES = [(100, 10), (500, 20), (1000, 30), (2000, 50)] + + +def run(): + lines = [] + header = "=== linux-0003: dev.c __dev_alloc_name altname linear scan ===" + print(header); lines.append(header) + for n, k in CASES: + df = min(bench_defective(n, k) for _ in range(TRIALS)) + fx = min(bench_fixed(n, k) for _ in range(TRIALS)) + speedup = (df / fx) if fx > 0 else float("inf") + line = f"D={n:<5} A={k:<5}: defective={df*1000:.3f}ms fixed={fx*1000:.3f}ms speedup={speedup:.1f}x" + print(line); lines.append(line); sys.stdout.flush() + return lines + + +if __name__ == "__main__": + run() diff --git a/defects/linux/bench/bench-linux-0004.py b/defects/linux/bench/bench-linux-0004.py new file mode 100644 index 000000000..893050aa9 --- /dev/null +++ b/defects/linux/bench/bench-linux-0004.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +# bench-linux-0004.py +# net/core/neighbour.c lookup_neigh_parms: tbl->parms_list linear scan per op. +# Models O(N) lookups -> O(1) via hash/dict membership. + +import sys +import time + + +def bench_defective(n): + """Per-op linear list scan — O(N) per op.""" + items = list(range(n)) + ops = list(range(n)) + t0 = time.perf_counter() + for op in ops: + _ = op in items # O(N) per op + return time.perf_counter() - t0 + + +def bench_fixed(n): + """Per-op O(1) dict/set lookup.""" + items = set(range(n)) + ops = list(range(n)) + t0 = time.perf_counter() + for op in ops: + _ = op in items # O(1) per op + return time.perf_counter() - t0 + + +TRIALS = 3 +SIZES = [100, 500, 1000, 2000] + + +def run(): + lines = [] + header = "=== linux-0004: neighbour.c lookup_neigh_parms list walk ===" + print(header); lines.append(header) + for n in SIZES: + df = min(bench_defective(n) for _ in range(TRIALS)) + fx = min(bench_fixed(n) for _ in range(TRIALS)) + speedup = (df / fx) if fx > 0 else float("inf") + line = f"P={n:<5}: defective={df*1000:.3f}ms fixed={fx*1000:.3f}ms speedup={speedup:.1f}x" + print(line); lines.append(line); sys.stdout.flush() + return lines + + +if __name__ == "__main__": + run() diff --git a/defects/linux/bench/bench-linux-0005.py b/defects/linux/bench/bench-linux-0005.py new file mode 100644 index 000000000..8ae672d40 --- /dev/null +++ b/defects/linux/bench/bench-linux-0005.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +# bench-linux-0005.py +# drivers/base/component.c find_component: nested list walk per match entry. +# Models O(A*M) → O(A+M) via list-scan inside a loop vs set/dict lookup. + +import sys +import time + + +def bench_defective(n, k): + """Outer loop over N, inner linear list scan over K — O(N*K).""" + pool = list(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + accepted = [] + for x in items: + if x in pool: # list __contains__ = O(K) + accepted.append(x) + # simulate nested work: scan pool to find position + try: + _ = pool.index(x) + except ValueError: + pass + return time.perf_counter() - t0 + + +def bench_fixed(n, k): + """Outer loop over N, inner O(1) set/dict lookup.""" + pool = list(range(k)) + pool_set = set(pool) + pool_pos = {v: i for i, v in enumerate(pool)} + items = list(range(n)) + t0 = time.perf_counter() + accepted = [] + for x in items: + if x in pool_set: # O(1) + accepted.append(x) + _ = pool_pos.get(x) # O(1) + return time.perf_counter() - t0 + + +TRIALS = 3 +CASES = [(50, 50), (100, 100), (200, 200), (500, 500)] + + +def run(): + lines = [] + header = "=== linux-0005: component.c find_component nested matches ===" + print(header); lines.append(header) + for n, k in CASES: + df = min(bench_defective(n, k) for _ in range(TRIALS)) + fx = min(bench_fixed(n, k) for _ in range(TRIALS)) + speedup = (df / fx) if fx > 0 else float("inf") + line = f"A={n:<5} M={k:<5}: defective={df*1000:.3f}ms fixed={fx*1000:.3f}ms speedup={speedup:.1f}x" + print(line); lines.append(line); sys.stdout.flush() + return lines + + +if __name__ == "__main__": + run() diff --git a/defects/linux/bench/bench-linux-0006.py b/defects/linux/bench/bench-linux-0006.py new file mode 100644 index 000000000..7ed3e0532 --- /dev/null +++ b/defects/linux/bench/bench-linux-0006.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +# bench-linux-0006.py +# kernel/bpf/btf.c bpf_find_btf_id: idr_for_each_entry per kptr field. +# Models O(F*M) → O(F+M) via list-scan inside a loop vs set/dict lookup. + +import sys +import time + + +def bench_defective(n, k): + """Outer loop over N, inner linear list scan over K — O(N*K).""" + pool = list(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + accepted = [] + for x in items: + if x in pool: # list __contains__ = O(K) + accepted.append(x) + # simulate nested work: scan pool to find position + try: + _ = pool.index(x) + except ValueError: + pass + return time.perf_counter() - t0 + + +def bench_fixed(n, k): + """Outer loop over N, inner O(1) set/dict lookup.""" + pool = list(range(k)) + pool_set = set(pool) + pool_pos = {v: i for i, v in enumerate(pool)} + items = list(range(n)) + t0 = time.perf_counter() + accepted = [] + for x in items: + if x in pool_set: # O(1) + accepted.append(x) + _ = pool_pos.get(x) # O(1) + return time.perf_counter() - t0 + + +TRIALS = 3 +CASES = [(50, 100), (100, 500), (200, 1000), (500, 2000)] + + +def run(): + lines = [] + header = "=== linux-0006: btf.c bpf_find_btf_id IDR scan per field ===" + print(header); lines.append(header) + for n, k in CASES: + df = min(bench_defective(n, k) for _ in range(TRIALS)) + fx = min(bench_fixed(n, k) for _ in range(TRIALS)) + speedup = (df / fx) if fx > 0 else float("inf") + line = f"F={n:<5} M={k:<5}: defective={df*1000:.3f}ms fixed={fx*1000:.3f}ms speedup={speedup:.1f}x" + print(line); lines.append(line); sys.stdout.flush() + return lines + + +if __name__ == "__main__": + run() diff --git a/defects/linux/bench/bench-linux-0007.py b/defects/linux/bench/bench-linux-0007.py new file mode 100644 index 000000000..5e26bb122 --- /dev/null +++ b/defects/linux/bench/bench-linux-0007.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +# bench-linux-0007.py +# net/core/pktgen.c __pktgen_NN_threads: threads list + per-thread if_list walk. +# Models O(T*D) → O(T+D) via list-scan inside a loop vs set/dict lookup. + +import sys +import time + + +def bench_defective(n, k): + """Outer loop over N, inner linear list scan over K — O(N*K).""" + pool = list(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + accepted = [] + for x in items: + if x in pool: # list __contains__ = O(K) + accepted.append(x) + # simulate nested work: scan pool to find position + try: + _ = pool.index(x) + except ValueError: + pass + return time.perf_counter() - t0 + + +def bench_fixed(n, k): + """Outer loop over N, inner O(1) set/dict lookup.""" + pool = list(range(k)) + pool_set = set(pool) + pool_pos = {v: i for i, v in enumerate(pool)} + items = list(range(n)) + t0 = time.perf_counter() + accepted = [] + for x in items: + if x in pool_set: # O(1) + accepted.append(x) + _ = pool_pos.get(x) # O(1) + return time.perf_counter() - t0 + + +TRIALS = 3 +CASES = [(50, 100), (100, 200), (200, 500), (500, 1000)] + + +def run(): + lines = [] + header = "=== linux-0007: pktgen.c thread-dev list walk ===" + print(header); lines.append(header) + for n, k in CASES: + df = min(bench_defective(n, k) for _ in range(TRIALS)) + fx = min(bench_fixed(n, k) for _ in range(TRIALS)) + speedup = (df / fx) if fx > 0 else float("inf") + line = f"T={n:<5} D={k:<5}: defective={df*1000:.3f}ms fixed={fx*1000:.3f}ms speedup={speedup:.1f}x" + print(line); lines.append(line); sys.stdout.flush() + return lines + + +if __name__ == "__main__": + run() diff --git a/defects/linux/bench/bench-linux-0008.py b/defects/linux/bench/bench-linux-0008.py new file mode 100644 index 000000000..c605b5133 --- /dev/null +++ b/defects/linux/bench/bench-linux-0008.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +# bench-linux-0008.py +# kernel/taskstats.c add_del_listener: per-CPU listener list walk per CPU. +# Models O(CPUs*L) → O(CPUs+L) via list-scan inside a loop vs set/dict lookup. + +import sys +import time + + +def bench_defective(n, k): + """Outer loop over N, inner linear list scan over K — O(N*K).""" + pool = list(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + accepted = [] + for x in items: + if x in pool: # list __contains__ = O(K) + accepted.append(x) + # simulate nested work: scan pool to find position + try: + _ = pool.index(x) + except ValueError: + pass + return time.perf_counter() - t0 + + +def bench_fixed(n, k): + """Outer loop over N, inner O(1) set/dict lookup.""" + pool = list(range(k)) + pool_set = set(pool) + pool_pos = {v: i for i, v in enumerate(pool)} + items = list(range(n)) + t0 = time.perf_counter() + accepted = [] + for x in items: + if x in pool_set: # O(1) + accepted.append(x) + _ = pool_pos.get(x) # O(1) + return time.perf_counter() - t0 + + +TRIALS = 3 +CASES = [(32, 50), (64, 100), (128, 200), (256, 500)] + + +def run(): + lines = [] + header = "=== linux-0008: taskstats.c add_del_listener per-CPU list ===" + print(header); lines.append(header) + for n, k in CASES: + df = min(bench_defective(n, k) for _ in range(TRIALS)) + fx = min(bench_fixed(n, k) for _ in range(TRIALS)) + speedup = (df / fx) if fx > 0 else float("inf") + line = f"CPUs={n:<5} L={k:<5}: defective={df*1000:.3f}ms fixed={fx*1000:.3f}ms speedup={speedup:.1f}x" + print(line); lines.append(line); sys.stdout.flush() + return lines + + +if __name__ == "__main__": + run() diff --git a/defects/linux/bench/results.txt b/defects/linux/bench/results.txt new file mode 100644 index 000000000..5db3bc416 --- /dev/null +++ b/defects/linux/bench/results.txt @@ -0,0 +1,48 @@ +=== linux-0001: headerdep detect_cycles BFS list-grep === +D=100 K=10 : defective=0.123ms fixed=0.013ms speedup=9.7x +D=500 K=20 : defective=0.786ms fixed=0.051ms speedup=15.4x +D=1000 K=30 : defective=1.919ms fixed=0.091ms speedup=21.0x +D=2000 K=50 : defective=5.459ms fixed=0.312ms speedup=17.5x + +=== linux-0002: auditsc audit_filter_rules inode-list scan === +F=100 R=20 : defective=0.252ms fixed=0.015ms speedup=17.0x +F=500 R=50 : defective=1.407ms fixed=0.045ms speedup=31.0x +F=1000 R=100 : defective=4.392ms fixed=0.084ms speedup=52.2x +F=2000 R=200 : defective=17.351ms fixed=0.163ms speedup=106.7x + +=== linux-0003: dev.c __dev_alloc_name altname linear scan === +D=100 A=10 : defective=0.086ms fixed=0.009ms speedup=9.7x +D=500 A=20 : defective=0.637ms fixed=0.041ms speedup=15.4x +D=1000 A=30 : defective=1.661ms fixed=0.079ms speedup=21.0x +D=2000 A=50 : defective=4.854ms fixed=0.188ms speedup=25.8x + +=== linux-0004: neighbour.c lookup_neigh_parms list walk === +P=100 : defective=0.097ms fixed=0.004ms speedup=25.4x +P=500 : defective=2.472ms fixed=0.023ms speedup=108.6x +P=1000 : defective=10.423ms fixed=0.051ms speedup=205.4x +P=2000 : defective=36.584ms fixed=0.094ms speedup=389.2x + +=== linux-0005: component.c find_component nested matches === +A=50 M=50 : defective=0.049ms fixed=0.005ms speedup=9.9x +A=100 M=100 : defective=0.185ms fixed=0.010ms speedup=19.4x +A=200 M=200 : defective=0.716ms fixed=0.018ms speedup=38.8x +A=500 M=500 : defective=4.691ms fixed=0.056ms speedup=84.0x + +=== linux-0006: btf.c bpf_find_btf_id IDR scan per field === +F=50 M=100 : defective=0.049ms fixed=0.005ms speedup=9.4x +F=100 M=500 : defective=0.184ms fixed=0.010ms speedup=18.7x +F=200 M=1000 : defective=0.724ms fixed=0.019ms speedup=37.6x +F=500 M=2000 : defective=4.906ms fixed=0.057ms speedup=86.4x + +=== linux-0007: pktgen.c thread-dev list walk === +T=50 D=100 : defective=0.049ms fixed=0.005ms speedup=9.7x +T=100 D=200 : defective=0.184ms fixed=0.010ms speedup=19.3x +T=200 D=500 : defective=0.730ms fixed=0.038ms speedup=19.3x +T=500 D=1000 : defective=4.909ms fixed=0.055ms speedup=88.9x + +=== linux-0008: taskstats.c add_del_listener per-CPU list === +CPUs=32 L=50 : defective=0.021ms fixed=0.003ms speedup=6.3x +CPUs=64 L=100 : defective=0.078ms fixed=0.006ms speedup=12.3x +CPUs=128 L=200 : defective=0.297ms fixed=0.013ms speedup=23.5x +CPUs=256 L=500 : defective=1.226ms fixed=0.024ms speedup=50.8x + diff --git a/defects/linux/bench/run_all.py b/defects/linux/bench/run_all.py new file mode 100644 index 000000000..685bf47c3 --- /dev/null +++ b/defects/linux/bench/run_all.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +import importlib.util, os, sys +BENCH_DIR = os.path.dirname(os.path.abspath(__file__)) + +def load_module(filename): + path = os.path.join(BENCH_DIR, filename) + spec = importlib.util.spec_from_file_location("mod", path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + +all_lines = [] +for fname in ["bench-linux-0001.py", "bench-linux-0002.py", "bench-linux-0003.py", + "bench-linux-0004.py", "bench-linux-0005.py", "bench-linux-0006.py", + "bench-linux-0007.py", "bench-linux-0008.py"]: + mod = load_module(fname) + lines = mod.run() + all_lines.extend(lines); all_lines.append("") + print(); sys.stdout.flush() + +out_path = os.path.join(BENCH_DIR, "results.txt") +with open(out_path, "w") as f: + f.write("\n".join(all_lines) + "\n") +print(f"results written to {out_path}"); sys.stdout.flush() diff --git a/defects/mercurial-0001/Makefile b/defects/mercurial-0001/Makefile new file mode 100644 index 000000000..125b5b8f6 --- /dev/null +++ b/defects/mercurial-0001/Makefile @@ -0,0 +1,6 @@ +.PHONY: all bench clean +all: bench +bench: + python3 bench/run_all.py +clean: + rm -rf bench/__pycache__ __pycache__ diff --git a/defects/mercurial-0001/bench/bench-mercurial-0001-0001.py b/defects/mercurial-0001/bench/bench-mercurial-0001-0001.py new file mode 100644 index 000000000..a633daf56 --- /dev/null +++ b/defects/mercurial-0001/bench/bench-mercurial-0001-0001.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 +# bench-mercurial-0001-0001.py +# graphmod.colored(): list.index() + list-membership lookups inside an +# O(k) inner loop per commit, where k is the number of parallel branches in +# the frontier. Total cost O(N·k²). Fix: dict-based O(1) position lookup +# brings it to O(N·k). Standalone Python model — a fuller benchmark that +# imports the real graphmod lives alongside as bench_google_scale.py. + +import sys +import time + + +def _step_defective(seen, cur, parents): + if cur not in seen: # O(k) + seen.append(cur) + col = seen.index(cur) # O(k) + next_ = seen[:] + addparents = [] + for p in parents: + if p not in next_: # O(k) + addparents.append(p) + next_[col: col + 1] = addparents + # inner enumerate loop: O(k) * O(k) membership/index = O(k^2) + for ecol, eid in enumerate(seen): + if eid in next_: # O(k) + _ = next_.index(eid) # O(k) + elif eid == cur: + for p in parents: + _ = next_.index(p) # O(k) + return next_ + + +def _step_fixed(seen, seen_pos, cur, parents): + if cur not in seen_pos: # O(1) + seen_pos[cur] = len(seen) + seen.append(cur) + col = seen_pos[cur] # O(1) + next_ = seen[:] + addparents = [] + for p in parents: + if p not in seen_pos: # O(1) + addparents.append(p) + next_[col: col + 1] = addparents + next_pos = {n: i for i, n in enumerate(next_)} # O(k) + for ecol, eid in enumerate(seen): + if eid in next_pos: # O(1) + _ = next_pos[eid] # O(1) + elif eid == cur: + for p in parents: + _ = next_pos[p] # O(1) + return next_, next_pos + + +def bench_defective(n, k): + seen = [] + t0 = time.perf_counter() + for rev in range(n - 1, -1, -1): + parent = rev - k + parents = [parent] if parent >= 0 else [] + seen = _step_defective(seen, rev, parents) + # bound seen growth to ~k so we measure the inner-loop work at k-frontier + if len(seen) > k: + seen = seen[:k] + return time.perf_counter() - t0 + + +def bench_fixed(n, k): + seen = [] + seen_pos = {} + t0 = time.perf_counter() + for rev in range(n - 1, -1, -1): + parent = rev - k + parents = [parent] if parent >= 0 else [] + seen, seen_pos = _step_fixed(seen, seen_pos, rev, parents) + if len(seen) > k: + seen = seen[:k] + seen_pos = {n_: i for i, n_ in enumerate(seen)} + return time.perf_counter() - t0 + + +TRIALS = 2 +CASES = [(1000, 10), (1000, 50), (1000, 100), (2000, 100), (2000, 200)] + + +def run(): + lines = [] + header = "=== mercurial-0001-0001: graphmod.colored list.index vs dict O(k^2)->O(k) ===" + print(header); lines.append(header) + for n, k in CASES: + df = min(bench_defective(n, k) for _ in range(TRIALS)) + fx = min(bench_fixed(n, k) for _ in range(TRIALS)) + speedup = (df / fx) if fx > 0 else float("inf") + line = f"N={n:<5} k={k:<4}: defective={df*1000:.3f}ms fixed={fx*1000:.3f}ms speedup={speedup:.1f}x" + print(line); lines.append(line); sys.stdout.flush() + return lines + + +if __name__ == "__main__": + run() diff --git a/defects/mercurial-0001/bench/results.txt b/defects/mercurial-0001/bench/results.txt new file mode 100644 index 000000000..8129bfad0 --- /dev/null +++ b/defects/mercurial-0001/bench/results.txt @@ -0,0 +1,7 @@ +=== mercurial-0001-0001: graphmod.colored list.index vs dict O(k^2)->O(k) === +N=1000 k=10 : defective=13.609ms fixed=4.922ms speedup=2.8x +N=1000 k=50 : defective=56.922ms fixed=9.799ms speedup=5.8x +N=1000 k=100 : defective=182.979ms fixed=16.737ms speedup=10.9x +N=2000 k=100 : defective=390.266ms fixed=34.759ms speedup=11.2x +N=2000 k=200 : defective=1382.872ms fixed=68.164ms speedup=20.3x + diff --git a/defects/mercurial-0001/bench/run_all.py b/defects/mercurial-0001/bench/run_all.py new file mode 100644 index 000000000..faeb19321 --- /dev/null +++ b/defects/mercurial-0001/bench/run_all.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 +import importlib.util, os, sys +BENCH_DIR = os.path.dirname(os.path.abspath(__file__)) + +def load_module(filename): + path = os.path.join(BENCH_DIR, filename) + spec = importlib.util.spec_from_file_location("mod", path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + +all_lines = [] +for fname in ["bench-mercurial-0001-0001.py"]: + mod = load_module(fname) + lines = mod.run() + all_lines.extend(lines); all_lines.append("") + print(); sys.stdout.flush() + +out_path = os.path.join(BENCH_DIR, "results.txt") +with open(out_path, "w") as f: + f.write("\n".join(all_lines) + "\n") +print(f"results written to {out_path}"); sys.stdout.flush()