diff --git a/defects/ans/Makefile b/defects/ans/Makefile new file mode 100644 index 000000000..125b5b8f6 --- /dev/null +++ b/defects/ans/Makefile @@ -0,0 +1,6 @@ +.PHONY: all bench clean +all: bench +bench: + python3 bench/run_all.py +clean: + rm -rf bench/__pycache__ __pycache__ diff --git a/defects/ans/bench/bench-ans-0001.py b/defects/ans/bench/bench-ans-0001.py new file mode 100644 index 000000000..ac2e437ea --- /dev/null +++ b/defects/ans/bench/bench-ans-0001.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# bench-ans-0001.py +# CWE-407: list-scan inside loop in ans-0001 (generic model) +# Models O(N*k) -> O(N+k) via linear-scan membership inside a loop vs set/dict. + +import sys +import time + + +def bench_defective(n, k): + pool = list(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool: # O(k) + seen.append(x) + return time.perf_counter() - t0 + + +def bench_fixed(n, k): + pool_set = set(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool_set: # O(1) + seen.append(x) + return time.perf_counter() - t0 + + +TRIALS = 3 +CASES = [(100, 100), (500, 500), (1000, 1000), (2000, 2000)] + + +def run(): + lines = [] + header = "=== ans-0001: CWE-407: list-scan inside loop in ans-0001 (generic model) ===" + print(header); lines.append(header) + for n, k in CASES: + df = min(bench_defective(n, k) for _ in range(TRIALS)) + fx = min(bench_fixed(n, k) for _ in range(TRIALS)) + speedup = (df / fx) if fx > 0 else float("inf") + line = f"N={n:<5} k={k:<5}: defective={df*1000:.3f}ms fixed={fx*1000:.3f}ms speedup={speedup:.1f}x" + print(line); lines.append(line); sys.stdout.flush() + return lines + + +if __name__ == "__main__": + run() diff --git a/defects/ans/bench/bench-ans-0002.py b/defects/ans/bench/bench-ans-0002.py new file mode 100644 index 000000000..615b543cf --- /dev/null +++ b/defects/ans/bench/bench-ans-0002.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# bench-ans-0002.py +# CWE-407: list-scan inside loop in ans-0002 (generic model) +# Models O(N*k) -> O(N+k) via linear-scan membership inside a loop vs set/dict. + +import sys +import time + + +def bench_defective(n, k): + pool = list(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool: # O(k) + seen.append(x) + return time.perf_counter() - t0 + + +def bench_fixed(n, k): + pool_set = set(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool_set: # O(1) + seen.append(x) + return time.perf_counter() - t0 + + +TRIALS = 3 +CASES = [(100, 100), (500, 500), (1000, 1000), (2000, 2000)] + + +def run(): + lines = [] + header = "=== ans-0002: CWE-407: list-scan inside loop in ans-0002 (generic model) ===" + print(header); lines.append(header) + for n, k in CASES: + df = min(bench_defective(n, k) for _ in range(TRIALS)) + fx = min(bench_fixed(n, k) for _ in range(TRIALS)) + speedup = (df / fx) if fx > 0 else float("inf") + line = f"N={n:<5} k={k:<5}: defective={df*1000:.3f}ms fixed={fx*1000:.3f}ms speedup={speedup:.1f}x" + print(line); lines.append(line); sys.stdout.flush() + return lines + + +if __name__ == "__main__": + run() diff --git a/defects/ans/bench/results.txt b/defects/ans/bench/results.txt new file mode 100644 index 000000000..f072c052a --- /dev/null +++ b/defects/ans/bench/results.txt @@ -0,0 +1,12 @@ +=== ans-0001: CWE-407: list-scan inside loop in ans-0001 (generic model) === +N=100 k=100 : defective=0.102ms fixed=0.004ms speedup=25.1x +N=500 k=500 : defective=2.660ms fixed=0.025ms speedup=104.9x +N=1000 k=1000 : defective=12.643ms fixed=0.127ms speedup=99.3x +N=2000 k=2000 : defective=51.882ms fixed=0.106ms speedup=487.9x + +=== ans-0002: CWE-407: list-scan inside loop in ans-0002 (generic model) === +N=100 k=100 : defective=0.116ms fixed=0.004ms speedup=31.5x +N=500 k=500 : defective=2.785ms fixed=0.023ms speedup=119.2x +N=1000 k=1000 : defective=13.994ms fixed=0.114ms speedup=123.1x +N=2000 k=2000 : defective=54.807ms fixed=0.185ms speedup=296.1x + diff --git a/defects/ans/bench/run_all.py b/defects/ans/bench/run_all.py new file mode 100644 index 000000000..14667ce4e --- /dev/null +++ b/defects/ans/bench/run_all.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 +import importlib.util, os, sys +BENCH_DIR = os.path.dirname(os.path.abspath(__file__)) + +def load_module(filename): + path = os.path.join(BENCH_DIR, filename) + spec = importlib.util.spec_from_file_location("mod", path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + +all_lines = [] +for fname in ["bench-ans-0001.py", "bench-ans-0002.py"]: + mod = load_module(fname) + lines = mod.run() + all_lines.extend(lines); all_lines.append("") + print(); sys.stdout.flush() + +out_path = os.path.join(BENCH_DIR, "results.txt") +with open(out_path, "w") as f: + f.write("\n".join(all_lines) + "\n") +print(f"results written to {out_path}"); sys.stdout.flush() diff --git a/defects/argo/Makefile b/defects/argo/Makefile new file mode 100644 index 000000000..125b5b8f6 --- /dev/null +++ b/defects/argo/Makefile @@ -0,0 +1,6 @@ +.PHONY: all bench clean +all: bench +bench: + python3 bench/run_all.py +clean: + rm -rf bench/__pycache__ __pycache__ diff --git a/defects/argo/bench/bench-argo-0001.py b/defects/argo/bench/bench-argo-0001.py new file mode 100644 index 000000000..1f6efc720 --- /dev/null +++ b/defects/argo/bench/bench-argo-0001.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# bench-argo-0001.py +# CWE-407: list-scan inside loop in argo-0001 (generic model) +# Models O(N*k) -> O(N+k) via linear-scan membership inside a loop vs set/dict. + +import sys +import time + + +def bench_defective(n, k): + pool = list(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool: # O(k) + seen.append(x) + return time.perf_counter() - t0 + + +def bench_fixed(n, k): + pool_set = set(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool_set: # O(1) + seen.append(x) + return time.perf_counter() - t0 + + +TRIALS = 3 +CASES = [(100, 100), (500, 500), (1000, 1000), (2000, 2000)] + + +def run(): + lines = [] + header = "=== argo-0001: CWE-407: list-scan inside loop in argo-0001 (generic model) ===" + print(header); lines.append(header) + for n, k in CASES: + df = min(bench_defective(n, k) for _ in range(TRIALS)) + fx = min(bench_fixed(n, k) for _ in range(TRIALS)) + speedup = (df / fx) if fx > 0 else float("inf") + line = f"N={n:<5} k={k:<5}: defective={df*1000:.3f}ms fixed={fx*1000:.3f}ms speedup={speedup:.1f}x" + print(line); lines.append(line); sys.stdout.flush() + return lines + + +if __name__ == "__main__": + run() diff --git a/defects/argo/bench/bench-argo-cd-0001.py b/defects/argo/bench/bench-argo-cd-0001.py new file mode 100644 index 000000000..a301ca305 --- /dev/null +++ b/defects/argo/bench/bench-argo-cd-0001.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# bench-argo-cd-0001.py +# CWE-407: list-scan inside loop in argo-cd-0001 (generic model) +# Models O(N*k) -> O(N+k) via linear-scan membership inside a loop vs set/dict. + +import sys +import time + + +def bench_defective(n, k): + pool = list(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool: # O(k) + seen.append(x) + return time.perf_counter() - t0 + + +def bench_fixed(n, k): + pool_set = set(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool_set: # O(1) + seen.append(x) + return time.perf_counter() - t0 + + +TRIALS = 3 +CASES = [(100, 100), (500, 500), (1000, 1000), (2000, 2000)] + + +def run(): + lines = [] + header = "=== argo-cd-0001: CWE-407: list-scan inside loop in argo-cd-0001 (generic model) ===" + print(header); lines.append(header) + for n, k in CASES: + df = min(bench_defective(n, k) for _ in range(TRIALS)) + fx = min(bench_fixed(n, k) for _ in range(TRIALS)) + speedup = (df / fx) if fx > 0 else float("inf") + line = f"N={n:<5} k={k:<5}: defective={df*1000:.3f}ms fixed={fx*1000:.3f}ms speedup={speedup:.1f}x" + print(line); lines.append(line); sys.stdout.flush() + return lines + + +if __name__ == "__main__": + run() diff --git a/defects/argo/bench/bench-argo-workflows-0001.py b/defects/argo/bench/bench-argo-workflows-0001.py new file mode 100644 index 000000000..e9df9c75d --- /dev/null +++ b/defects/argo/bench/bench-argo-workflows-0001.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# bench-argo-workflows-0001.py +# CWE-407: list-scan inside loop in argo-workflows-0001 (generic model) +# Models O(N*k) -> O(N+k) via linear-scan membership inside a loop vs set/dict. + +import sys +import time + + +def bench_defective(n, k): + pool = list(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool: # O(k) + seen.append(x) + return time.perf_counter() - t0 + + +def bench_fixed(n, k): + pool_set = set(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool_set: # O(1) + seen.append(x) + return time.perf_counter() - t0 + + +TRIALS = 3 +CASES = [(100, 100), (500, 500), (1000, 1000), (2000, 2000)] + + +def run(): + lines = [] + header = "=== argo-workflows-0001: CWE-407: list-scan inside loop in argo-workflows-0001 (generic model) ===" + print(header); lines.append(header) + for n, k in CASES: + df = min(bench_defective(n, k) for _ in range(TRIALS)) + fx = min(bench_fixed(n, k) for _ in range(TRIALS)) + speedup = (df / fx) if fx > 0 else float("inf") + line = f"N={n:<5} k={k:<5}: defective={df*1000:.3f}ms fixed={fx*1000:.3f}ms speedup={speedup:.1f}x" + print(line); lines.append(line); sys.stdout.flush() + return lines + + +if __name__ == "__main__": + run() diff --git a/defects/argo/bench/results.txt b/defects/argo/bench/results.txt new file mode 100644 index 000000000..e5229961e --- /dev/null +++ b/defects/argo/bench/results.txt @@ -0,0 +1,18 @@ +=== argo-0001: CWE-407: list-scan inside loop in argo-0001 (generic model) === +N=100 k=100 : defective=0.102ms fixed=0.004ms speedup=24.8x +N=500 k=500 : defective=2.588ms fixed=0.024ms speedup=108.0x +N=1000 k=1000 : defective=12.395ms fixed=0.104ms speedup=118.6x +N=2000 k=2000 : defective=44.671ms fixed=0.096ms speedup=464.6x + +=== argo-cd-0001: CWE-407: list-scan inside loop in argo-cd-0001 (generic model) === +N=100 k=100 : defective=0.085ms fixed=0.003ms speedup=25.0x +N=500 k=500 : defective=2.117ms fixed=0.021ms speedup=99.5x +N=1000 k=1000 : defective=11.472ms fixed=0.046ms speedup=247.7x +N=2000 k=2000 : defective=47.905ms fixed=0.101ms speedup=474.2x + +=== argo-workflows-0001: CWE-407: list-scan inside loop in argo-workflows-0001 (generic model) === +N=100 k=100 : defective=0.088ms fixed=0.004ms speedup=24.5x +N=500 k=500 : defective=2.836ms fixed=0.088ms speedup=32.4x +N=1000 k=1000 : defective=9.246ms fixed=0.045ms speedup=207.3x +N=2000 k=2000 : defective=38.788ms fixed=0.096ms speedup=405.6x + diff --git a/defects/argo/bench/run_all.py b/defects/argo/bench/run_all.py new file mode 100644 index 000000000..19fe409c9 --- /dev/null +++ b/defects/argo/bench/run_all.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 +import importlib.util, os, sys +BENCH_DIR = os.path.dirname(os.path.abspath(__file__)) + +def load_module(filename): + path = os.path.join(BENCH_DIR, filename) + spec = importlib.util.spec_from_file_location("mod", path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + +all_lines = [] +for fname in ["bench-argo-0001.py", "bench-argo-cd-0001.py", "bench-argo-workflows-0001.py"]: + mod = load_module(fname) + lines = mod.run() + all_lines.extend(lines); all_lines.append("") + print(); sys.stdout.flush() + +out_path = os.path.join(BENCH_DIR, "results.txt") +with open(out_path, "w") as f: + f.write("\n".join(all_lines) + "\n") +print(f"results written to {out_path}"); sys.stdout.flush() diff --git a/defects/cel/Makefile b/defects/cel/Makefile new file mode 100644 index 000000000..125b5b8f6 --- /dev/null +++ b/defects/cel/Makefile @@ -0,0 +1,6 @@ +.PHONY: all bench clean +all: bench +bench: + python3 bench/run_all.py +clean: + rm -rf bench/__pycache__ __pycache__ diff --git a/defects/cel/bench/bench-cel-0001.py b/defects/cel/bench/bench-cel-0001.py new file mode 100644 index 000000000..54f0e9d27 --- /dev/null +++ b/defects/cel/bench/bench-cel-0001.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# bench-cel-0001.py +# CWE-407: list-scan inside loop in cel-0001 (generic model) +# Models O(N*k) -> O(N+k) via linear-scan membership inside a loop vs set/dict. + +import sys +import time + + +def bench_defective(n, k): + pool = list(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool: # O(k) + seen.append(x) + return time.perf_counter() - t0 + + +def bench_fixed(n, k): + pool_set = set(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool_set: # O(1) + seen.append(x) + return time.perf_counter() - t0 + + +TRIALS = 3 +CASES = [(100, 100), (500, 500), (1000, 1000), (2000, 2000)] + + +def run(): + lines = [] + header = "=== cel-0001: CWE-407: list-scan inside loop in cel-0001 (generic model) ===" + print(header); lines.append(header) + for n, k in CASES: + df = min(bench_defective(n, k) for _ in range(TRIALS)) + fx = min(bench_fixed(n, k) for _ in range(TRIALS)) + speedup = (df / fx) if fx > 0 else float("inf") + line = f"N={n:<5} k={k:<5}: defective={df*1000:.3f}ms fixed={fx*1000:.3f}ms speedup={speedup:.1f}x" + print(line); lines.append(line); sys.stdout.flush() + return lines + + +if __name__ == "__main__": + run() diff --git a/defects/cel/bench/results.txt b/defects/cel/bench/results.txt new file mode 100644 index 000000000..ed61d1be8 --- /dev/null +++ b/defects/cel/bench/results.txt @@ -0,0 +1,6 @@ +=== cel-0001: CWE-407: list-scan inside loop in cel-0001 (generic model) === +N=100 k=100 : defective=0.093ms fixed=0.004ms speedup=24.9x +N=500 k=500 : defective=2.669ms fixed=0.035ms speedup=76.8x +N=1000 k=1000 : defective=11.148ms fixed=0.049ms speedup=227.9x +N=2000 k=2000 : defective=41.315ms fixed=0.096ms speedup=429.0x + diff --git a/defects/cel/bench/run_all.py b/defects/cel/bench/run_all.py new file mode 100644 index 000000000..31fee2566 --- /dev/null +++ b/defects/cel/bench/run_all.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 +import importlib.util, os, sys +BENCH_DIR = os.path.dirname(os.path.abspath(__file__)) + +def load_module(filename): + path = os.path.join(BENCH_DIR, filename) + spec = importlib.util.spec_from_file_location("mod", path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + +all_lines = [] +for fname in ["bench-cel-0001.py"]: + mod = load_module(fname) + lines = mod.run() + all_lines.extend(lines); all_lines.append("") + print(); sys.stdout.flush() + +out_path = os.path.join(BENCH_DIR, "results.txt") +with open(out_path, "w") as f: + f.write("\n".join(all_lines) + "\n") +print(f"results written to {out_path}"); sys.stdout.flush() diff --git a/defects/cfe/Makefile b/defects/cfe/Makefile new file mode 100644 index 000000000..125b5b8f6 --- /dev/null +++ b/defects/cfe/Makefile @@ -0,0 +1,6 @@ +.PHONY: all bench clean +all: bench +bench: + python3 bench/run_all.py +clean: + rm -rf bench/__pycache__ __pycache__ diff --git a/defects/cfe/bench/bench-cfe-0001.py b/defects/cfe/bench/bench-cfe-0001.py new file mode 100644 index 000000000..cac430ac9 --- /dev/null +++ b/defects/cfe/bench/bench-cfe-0001.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# bench-cfe-0001.py +# CWE-407: list-scan inside loop in cfe-0001 (generic model) +# Models O(N*k) -> O(N+k) via linear-scan membership inside a loop vs set/dict. + +import sys +import time + + +def bench_defective(n, k): + pool = list(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool: # O(k) + seen.append(x) + return time.perf_counter() - t0 + + +def bench_fixed(n, k): + pool_set = set(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool_set: # O(1) + seen.append(x) + return time.perf_counter() - t0 + + +TRIALS = 3 +CASES = [(100, 100), (500, 500), (1000, 1000), (2000, 2000)] + + +def run(): + lines = [] + header = "=== cfe-0001: CWE-407: list-scan inside loop in cfe-0001 (generic model) ===" + print(header); lines.append(header) + for n, k in CASES: + df = min(bench_defective(n, k) for _ in range(TRIALS)) + fx = min(bench_fixed(n, k) for _ in range(TRIALS)) + speedup = (df / fx) if fx > 0 else float("inf") + line = f"N={n:<5} k={k:<5}: defective={df*1000:.3f}ms fixed={fx*1000:.3f}ms speedup={speedup:.1f}x" + print(line); lines.append(line); sys.stdout.flush() + return lines + + +if __name__ == "__main__": + run() diff --git a/defects/cfe/bench/bench-cfe-0002.py b/defects/cfe/bench/bench-cfe-0002.py new file mode 100644 index 000000000..a3dcbb2c0 --- /dev/null +++ b/defects/cfe/bench/bench-cfe-0002.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# bench-cfe-0002.py +# CWE-407: list-scan inside loop in cfe-0002 (generic model) +# Models O(N*k) -> O(N+k) via linear-scan membership inside a loop vs set/dict. + +import sys +import time + + +def bench_defective(n, k): + pool = list(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool: # O(k) + seen.append(x) + return time.perf_counter() - t0 + + +def bench_fixed(n, k): + pool_set = set(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool_set: # O(1) + seen.append(x) + return time.perf_counter() - t0 + + +TRIALS = 3 +CASES = [(100, 100), (500, 500), (1000, 1000), (2000, 2000)] + + +def run(): + lines = [] + header = "=== cfe-0002: CWE-407: list-scan inside loop in cfe-0002 (generic model) ===" + print(header); lines.append(header) + for n, k in CASES: + df = min(bench_defective(n, k) for _ in range(TRIALS)) + fx = min(bench_fixed(n, k) for _ in range(TRIALS)) + speedup = (df / fx) if fx > 0 else float("inf") + line = f"N={n:<5} k={k:<5}: defective={df*1000:.3f}ms fixed={fx*1000:.3f}ms speedup={speedup:.1f}x" + print(line); lines.append(line); sys.stdout.flush() + return lines + + +if __name__ == "__main__": + run() diff --git a/defects/cfe/bench/bench-cfe-0003.py b/defects/cfe/bench/bench-cfe-0003.py new file mode 100644 index 000000000..984e992c6 --- /dev/null +++ b/defects/cfe/bench/bench-cfe-0003.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# bench-cfe-0003.py +# CWE-407: list-scan inside loop in cfe-0003 (generic model) +# Models O(N*k) -> O(N+k) via linear-scan membership inside a loop vs set/dict. + +import sys +import time + + +def bench_defective(n, k): + pool = list(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool: # O(k) + seen.append(x) + return time.perf_counter() - t0 + + +def bench_fixed(n, k): + pool_set = set(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool_set: # O(1) + seen.append(x) + return time.perf_counter() - t0 + + +TRIALS = 3 +CASES = [(100, 100), (500, 500), (1000, 1000), (2000, 2000)] + + +def run(): + lines = [] + header = "=== cfe-0003: CWE-407: list-scan inside loop in cfe-0003 (generic model) ===" + print(header); lines.append(header) + for n, k in CASES: + df = min(bench_defective(n, k) for _ in range(TRIALS)) + fx = min(bench_fixed(n, k) for _ in range(TRIALS)) + speedup = (df / fx) if fx > 0 else float("inf") + line = f"N={n:<5} k={k:<5}: defective={df*1000:.3f}ms fixed={fx*1000:.3f}ms speedup={speedup:.1f}x" + print(line); lines.append(line); sys.stdout.flush() + return lines + + +if __name__ == "__main__": + run() diff --git a/defects/cfe/bench/results.txt b/defects/cfe/bench/results.txt new file mode 100644 index 000000000..5a8e5e42d --- /dev/null +++ b/defects/cfe/bench/results.txt @@ -0,0 +1,18 @@ +=== cfe-0001: CWE-407: list-scan inside loop in cfe-0001 (generic model) === +N=100 k=100 : defective=0.084ms fixed=0.003ms speedup=24.4x +N=500 k=500 : defective=2.319ms fixed=0.021ms speedup=111.0x +N=1000 k=1000 : defective=9.082ms fixed=0.046ms speedup=198.9x +N=2000 k=2000 : defective=43.871ms fixed=0.097ms speedup=450.4x + +=== cfe-0002: CWE-407: list-scan inside loop in cfe-0002 (generic model) === +N=100 k=100 : defective=0.085ms fixed=0.003ms speedup=25.0x +N=500 k=500 : defective=2.338ms fixed=0.021ms speedup=113.4x +N=1000 k=1000 : defective=9.300ms fixed=0.046ms speedup=202.4x +N=2000 k=2000 : defective=41.926ms fixed=0.097ms speedup=430.5x + +=== cfe-0003: CWE-407: list-scan inside loop in cfe-0003 (generic model) === +N=100 k=100 : defective=0.085ms fixed=0.004ms speedup=24.0x +N=500 k=500 : defective=2.117ms fixed=0.020ms speedup=104.5x +N=1000 k=1000 : defective=10.777ms fixed=0.093ms speedup=115.3x +N=2000 k=2000 : defective=43.851ms fixed=0.101ms speedup=432.5x + diff --git a/defects/cfe/bench/run_all.py b/defects/cfe/bench/run_all.py new file mode 100644 index 000000000..c070f6aee --- /dev/null +++ b/defects/cfe/bench/run_all.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 +import importlib.util, os, sys +BENCH_DIR = os.path.dirname(os.path.abspath(__file__)) + +def load_module(filename): + path = os.path.join(BENCH_DIR, filename) + spec = importlib.util.spec_from_file_location("mod", path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + +all_lines = [] +for fname in ["bench-cfe-0001.py", "bench-cfe-0002.py", "bench-cfe-0003.py"]: + mod = load_module(fname) + lines = mod.run() + all_lines.extend(lines); all_lines.append("") + print(); sys.stdout.flush() + +out_path = os.path.join(BENCH_DIR, "results.txt") +with open(out_path, "w") as f: + f.write("\n".join(all_lines) + "\n") +print(f"results written to {out_path}"); sys.stdout.flush() diff --git a/defects/element/Makefile b/defects/element/Makefile new file mode 100644 index 000000000..125b5b8f6 --- /dev/null +++ b/defects/element/Makefile @@ -0,0 +1,6 @@ +.PHONY: all bench clean +all: bench +bench: + python3 bench/run_all.py +clean: + rm -rf bench/__pycache__ __pycache__ diff --git a/defects/element/bench/bench-element-web-0001.py b/defects/element/bench/bench-element-web-0001.py new file mode 100644 index 000000000..6b4a82c73 --- /dev/null +++ b/defects/element/bench/bench-element-web-0001.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# bench-element-web-0001.py +# CWE-407: list-scan inside loop in element-web-0001 (generic model) +# Models O(N*k) -> O(N+k) via linear-scan membership inside a loop vs set/dict. + +import sys +import time + + +def bench_defective(n, k): + pool = list(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool: # O(k) + seen.append(x) + return time.perf_counter() - t0 + + +def bench_fixed(n, k): + pool_set = set(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool_set: # O(1) + seen.append(x) + return time.perf_counter() - t0 + + +TRIALS = 3 +CASES = [(100, 100), (500, 500), (1000, 1000), (2000, 2000)] + + +def run(): + lines = [] + header = "=== element-web-0001: CWE-407: list-scan inside loop in element-web-0001 (generic model) ===" + print(header); lines.append(header) + for n, k in CASES: + df = min(bench_defective(n, k) for _ in range(TRIALS)) + fx = min(bench_fixed(n, k) for _ in range(TRIALS)) + speedup = (df / fx) if fx > 0 else float("inf") + line = f"N={n:<5} k={k:<5}: defective={df*1000:.3f}ms fixed={fx*1000:.3f}ms speedup={speedup:.1f}x" + print(line); lines.append(line); sys.stdout.flush() + return lines + + +if __name__ == "__main__": + run() diff --git a/defects/element/bench/bench-element-web-0002.py b/defects/element/bench/bench-element-web-0002.py new file mode 100644 index 000000000..db9d83dd7 --- /dev/null +++ b/defects/element/bench/bench-element-web-0002.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# bench-element-web-0002.py +# CWE-407: list-scan inside loop in element-web-0002 (generic model) +# Models O(N*k) -> O(N+k) via linear-scan membership inside a loop vs set/dict. + +import sys +import time + + +def bench_defective(n, k): + pool = list(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool: # O(k) + seen.append(x) + return time.perf_counter() - t0 + + +def bench_fixed(n, k): + pool_set = set(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool_set: # O(1) + seen.append(x) + return time.perf_counter() - t0 + + +TRIALS = 3 +CASES = [(100, 100), (500, 500), (1000, 1000), (2000, 2000)] + + +def run(): + lines = [] + header = "=== element-web-0002: CWE-407: list-scan inside loop in element-web-0002 (generic model) ===" + print(header); lines.append(header) + for n, k in CASES: + df = min(bench_defective(n, k) for _ in range(TRIALS)) + fx = min(bench_fixed(n, k) for _ in range(TRIALS)) + speedup = (df / fx) if fx > 0 else float("inf") + line = f"N={n:<5} k={k:<5}: defective={df*1000:.3f}ms fixed={fx*1000:.3f}ms speedup={speedup:.1f}x" + print(line); lines.append(line); sys.stdout.flush() + return lines + + +if __name__ == "__main__": + run() diff --git a/defects/element/bench/bench-element-web-0003.py b/defects/element/bench/bench-element-web-0003.py new file mode 100644 index 000000000..d461bc290 --- /dev/null +++ b/defects/element/bench/bench-element-web-0003.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# bench-element-web-0003.py +# CWE-407: list-scan inside loop in element-web-0003 (generic model) +# Models O(N*k) -> O(N+k) via linear-scan membership inside a loop vs set/dict. + +import sys +import time + + +def bench_defective(n, k): + pool = list(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool: # O(k) + seen.append(x) + return time.perf_counter() - t0 + + +def bench_fixed(n, k): + pool_set = set(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool_set: # O(1) + seen.append(x) + return time.perf_counter() - t0 + + +TRIALS = 3 +CASES = [(100, 100), (500, 500), (1000, 1000), (2000, 2000)] + + +def run(): + lines = [] + header = "=== element-web-0003: CWE-407: list-scan inside loop in element-web-0003 (generic model) ===" + print(header); lines.append(header) + for n, k in CASES: + df = min(bench_defective(n, k) for _ in range(TRIALS)) + fx = min(bench_fixed(n, k) for _ in range(TRIALS)) + speedup = (df / fx) if fx > 0 else float("inf") + line = f"N={n:<5} k={k:<5}: defective={df*1000:.3f}ms fixed={fx*1000:.3f}ms speedup={speedup:.1f}x" + print(line); lines.append(line); sys.stdout.flush() + return lines + + +if __name__ == "__main__": + run() diff --git a/defects/element/bench/bench-element-web-0004.py b/defects/element/bench/bench-element-web-0004.py new file mode 100644 index 000000000..c0239f4cc --- /dev/null +++ b/defects/element/bench/bench-element-web-0004.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# bench-element-web-0004.py +# CWE-407: list-scan inside loop in element-web-0004 (generic model) +# Models O(N*k) -> O(N+k) via linear-scan membership inside a loop vs set/dict. + +import sys +import time + + +def bench_defective(n, k): + pool = list(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool: # O(k) + seen.append(x) + return time.perf_counter() - t0 + + +def bench_fixed(n, k): + pool_set = set(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool_set: # O(1) + seen.append(x) + return time.perf_counter() - t0 + + +TRIALS = 3 +CASES = [(100, 100), (500, 500), (1000, 1000), (2000, 2000)] + + +def run(): + lines = [] + header = "=== element-web-0004: CWE-407: list-scan inside loop in element-web-0004 (generic model) ===" + print(header); lines.append(header) + for n, k in CASES: + df = min(bench_defective(n, k) for _ in range(TRIALS)) + fx = min(bench_fixed(n, k) for _ in range(TRIALS)) + speedup = (df / fx) if fx > 0 else float("inf") + line = f"N={n:<5} k={k:<5}: defective={df*1000:.3f}ms fixed={fx*1000:.3f}ms speedup={speedup:.1f}x" + print(line); lines.append(line); sys.stdout.flush() + return lines + + +if __name__ == "__main__": + run() diff --git a/defects/element/bench/bench-element-web.py b/defects/element/bench/bench-element-web.py new file mode 100644 index 000000000..0461ac6a0 --- /dev/null +++ b/defects/element/bench/bench-element-web.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# bench-element-web.py +# CWE-407: list-scan inside loop in element-web (generic model) +# Models O(N*k) -> O(N+k) via linear-scan membership inside a loop vs set/dict. + +import sys +import time + + +def bench_defective(n, k): + pool = list(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool: # O(k) + seen.append(x) + return time.perf_counter() - t0 + + +def bench_fixed(n, k): + pool_set = set(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool_set: # O(1) + seen.append(x) + return time.perf_counter() - t0 + + +TRIALS = 3 +CASES = [(100, 100), (500, 500), (1000, 1000), (2000, 2000)] + + +def run(): + lines = [] + header = "=== element-web: CWE-407: list-scan inside loop in element-web (generic model) ===" + print(header); lines.append(header) + for n, k in CASES: + df = min(bench_defective(n, k) for _ in range(TRIALS)) + fx = min(bench_fixed(n, k) for _ in range(TRIALS)) + speedup = (df / fx) if fx > 0 else float("inf") + line = f"N={n:<5} k={k:<5}: defective={df*1000:.3f}ms fixed={fx*1000:.3f}ms speedup={speedup:.1f}x" + print(line); lines.append(line); sys.stdout.flush() + return lines + + +if __name__ == "__main__": + run() diff --git a/defects/element/bench/results.txt b/defects/element/bench/results.txt new file mode 100644 index 000000000..20e6be89a --- /dev/null +++ b/defects/element/bench/results.txt @@ -0,0 +1,30 @@ +=== element-web-0001: CWE-407: list-scan inside loop in element-web-0001 (generic model) === +N=100 k=100 : defective=0.097ms fixed=0.004ms speedup=25.1x +N=500 k=500 : defective=3.221ms fixed=0.037ms speedup=87.1x +N=1000 k=1000 : defective=17.058ms fixed=0.057ms speedup=299.5x +N=2000 k=2000 : defective=50.789ms fixed=0.097ms speedup=521.6x + +=== element-web-0002: CWE-407: list-scan inside loop in element-web-0002 (generic model) === +N=100 k=100 : defective=0.085ms fixed=0.004ms speedup=24.2x +N=500 k=500 : defective=2.153ms fixed=0.021ms speedup=104.9x +N=1000 k=1000 : defective=10.996ms fixed=0.048ms speedup=230.3x +N=2000 k=2000 : defective=47.956ms fixed=0.158ms speedup=304.0x + +=== element-web-0003: CWE-407: list-scan inside loop in element-web-0003 (generic model) === +N=100 k=100 : defective=0.153ms fixed=0.005ms speedup=30.2x +N=500 k=500 : defective=2.253ms fixed=0.021ms speedup=106.9x +N=1000 k=1000 : defective=13.024ms fixed=0.104ms speedup=125.0x +N=2000 k=2000 : defective=41.579ms fixed=0.155ms speedup=268.3x + +=== element-web-0004: CWE-407: list-scan inside loop in element-web-0004 (generic model) === +N=100 k=100 : defective=0.085ms fixed=0.003ms speedup=24.9x +N=500 k=500 : defective=2.244ms fixed=0.032ms speedup=69.1x +N=1000 k=1000 : defective=9.021ms fixed=0.047ms speedup=193.8x +N=2000 k=2000 : defective=42.478ms fixed=0.179ms speedup=237.6x + +=== element-web: CWE-407: list-scan inside loop in element-web (generic model) === +N=100 k=100 : defective=0.146ms fixed=0.007ms speedup=21.9x +N=500 k=500 : defective=2.167ms fixed=0.021ms speedup=102.8x +N=1000 k=1000 : defective=8.907ms fixed=0.047ms speedup=191.3x +N=2000 k=2000 : defective=51.391ms fixed=0.105ms speedup=488.6x + diff --git a/defects/element/bench/run_all.py b/defects/element/bench/run_all.py new file mode 100644 index 000000000..e73ea4205 --- /dev/null +++ b/defects/element/bench/run_all.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 +import importlib.util, os, sys +BENCH_DIR = os.path.dirname(os.path.abspath(__file__)) + +def load_module(filename): + path = os.path.join(BENCH_DIR, filename) + spec = importlib.util.spec_from_file_location("mod", path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + +all_lines = [] +for fname in ["bench-element-web-0001.py", "bench-element-web-0002.py", "bench-element-web-0003.py", "bench-element-web-0004.py", "bench-element-web.py"]: + mod = load_module(fname) + lines = mod.run() + all_lines.extend(lines); all_lines.append("") + print(); sys.stdout.flush() + +out_path = os.path.join(BENCH_DIR, "results.txt") +with open(out_path, "w") as f: + f.write("\n".join(all_lines) + "\n") +print(f"results written to {out_path}"); sys.stdout.flush() diff --git a/defects/geth/Makefile b/defects/geth/Makefile new file mode 100644 index 000000000..125b5b8f6 --- /dev/null +++ b/defects/geth/Makefile @@ -0,0 +1,6 @@ +.PHONY: all bench clean +all: bench +bench: + python3 bench/run_all.py +clean: + rm -rf bench/__pycache__ __pycache__ diff --git a/defects/geth/bench/bench-geth-0001.py b/defects/geth/bench/bench-geth-0001.py new file mode 100644 index 000000000..16bf7ae52 --- /dev/null +++ b/defects/geth/bench/bench-geth-0001.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# bench-geth-0001.py +# CWE-407: list-scan inside loop in geth-0001 (generic model) +# Models O(N*k) -> O(N+k) via linear-scan membership inside a loop vs set/dict. + +import sys +import time + + +def bench_defective(n, k): + pool = list(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool: # O(k) + seen.append(x) + return time.perf_counter() - t0 + + +def bench_fixed(n, k): + pool_set = set(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool_set: # O(1) + seen.append(x) + return time.perf_counter() - t0 + + +TRIALS = 3 +CASES = [(100, 100), (500, 500), (1000, 1000), (2000, 2000)] + + +def run(): + lines = [] + header = "=== geth-0001: CWE-407: list-scan inside loop in geth-0001 (generic model) ===" + print(header); lines.append(header) + for n, k in CASES: + df = min(bench_defective(n, k) for _ in range(TRIALS)) + fx = min(bench_fixed(n, k) for _ in range(TRIALS)) + speedup = (df / fx) if fx > 0 else float("inf") + line = f"N={n:<5} k={k:<5}: defective={df*1000:.3f}ms fixed={fx*1000:.3f}ms speedup={speedup:.1f}x" + print(line); lines.append(line); sys.stdout.flush() + return lines + + +if __name__ == "__main__": + run() diff --git a/defects/geth/bench/results.txt b/defects/geth/bench/results.txt new file mode 100644 index 000000000..01499dc21 --- /dev/null +++ b/defects/geth/bench/results.txt @@ -0,0 +1,6 @@ +=== geth-0001: CWE-407: list-scan inside loop in geth-0001 (generic model) === +N=100 k=100 : defective=0.092ms fixed=0.004ms speedup=25.0x +N=500 k=500 : defective=2.360ms fixed=0.023ms speedup=103.4x +N=1000 k=1000 : defective=12.443ms fixed=0.074ms speedup=168.1x +N=2000 k=2000 : defective=54.010ms fixed=0.189ms speedup=285.1x + diff --git a/defects/geth/bench/run_all.py b/defects/geth/bench/run_all.py new file mode 100644 index 000000000..b5d45296e --- /dev/null +++ b/defects/geth/bench/run_all.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 +import importlib.util, os, sys +BENCH_DIR = os.path.dirname(os.path.abspath(__file__)) + +def load_module(filename): + path = os.path.join(BENCH_DIR, filename) + spec = importlib.util.spec_from_file_location("mod", path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + +all_lines = [] +for fname in ["bench-geth-0001.py"]: + mod = load_module(fname) + lines = mod.run() + all_lines.extend(lines); all_lines.append("") + print(); sys.stdout.flush() + +out_path = os.path.join(BENCH_DIR, "results.txt") +with open(out_path, "w") as f: + f.write("\n".join(all_lines) + "\n") +print(f"results written to {out_path}"); sys.stdout.flush() diff --git a/defects/go-stdlib/Makefile b/defects/go-stdlib/Makefile new file mode 100644 index 000000000..125b5b8f6 --- /dev/null +++ b/defects/go-stdlib/Makefile @@ -0,0 +1,6 @@ +.PHONY: all bench clean +all: bench +bench: + python3 bench/run_all.py +clean: + rm -rf bench/__pycache__ __pycache__ diff --git a/defects/go-stdlib/bench/bench-go-stdlib-0001.py b/defects/go-stdlib/bench/bench-go-stdlib-0001.py new file mode 100644 index 000000000..1e9fe8988 --- /dev/null +++ b/defects/go-stdlib/bench/bench-go-stdlib-0001.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# bench-go-stdlib-0001.py +# CWE-407: list-scan inside loop in go-stdlib-0001 (generic model) +# Models O(N*k) -> O(N+k) via linear-scan membership inside a loop vs set/dict. + +import sys +import time + + +def bench_defective(n, k): + pool = list(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool: # O(k) + seen.append(x) + return time.perf_counter() - t0 + + +def bench_fixed(n, k): + pool_set = set(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool_set: # O(1) + seen.append(x) + return time.perf_counter() - t0 + + +TRIALS = 3 +CASES = [(100, 100), (500, 500), (1000, 1000), (2000, 2000)] + + +def run(): + lines = [] + header = "=== go-stdlib-0001: CWE-407: list-scan inside loop in go-stdlib-0001 (generic model) ===" + print(header); lines.append(header) + for n, k in CASES: + df = min(bench_defective(n, k) for _ in range(TRIALS)) + fx = min(bench_fixed(n, k) for _ in range(TRIALS)) + speedup = (df / fx) if fx > 0 else float("inf") + line = f"N={n:<5} k={k:<5}: defective={df*1000:.3f}ms fixed={fx*1000:.3f}ms speedup={speedup:.1f}x" + print(line); lines.append(line); sys.stdout.flush() + return lines + + +if __name__ == "__main__": + run() diff --git a/defects/go-stdlib/bench/results.txt b/defects/go-stdlib/bench/results.txt new file mode 100644 index 000000000..2f16d4b35 --- /dev/null +++ b/defects/go-stdlib/bench/results.txt @@ -0,0 +1,6 @@ +=== go-stdlib-0001: CWE-407: list-scan inside loop in go-stdlib-0001 (generic model) === +N=100 k=100 : defective=0.282ms fixed=0.018ms speedup=15.7x +N=500 k=500 : defective=2.615ms fixed=0.024ms speedup=108.1x +N=1000 k=1000 : defective=10.622ms fixed=0.051ms speedup=206.5x +N=2000 k=2000 : defective=53.730ms fixed=0.169ms speedup=317.3x + diff --git a/defects/go-stdlib/bench/run_all.py b/defects/go-stdlib/bench/run_all.py new file mode 100644 index 000000000..2c4a4d6db --- /dev/null +++ b/defects/go-stdlib/bench/run_all.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 +import importlib.util, os, sys +BENCH_DIR = os.path.dirname(os.path.abspath(__file__)) + +def load_module(filename): + path = os.path.join(BENCH_DIR, filename) + spec = importlib.util.spec_from_file_location("mod", path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + +all_lines = [] +for fname in ["bench-go-stdlib-0001.py"]: + mod = load_module(fname) + lines = mod.run() + all_lines.extend(lines); all_lines.append("") + print(); sys.stdout.flush() + +out_path = os.path.join(BENCH_DIR, "results.txt") +with open(out_path, "w") as f: + f.write("\n".join(all_lines) + "\n") +print(f"results written to {out_path}"); sys.stdout.flush() diff --git a/defects/hv/Makefile b/defects/hv/Makefile new file mode 100644 index 000000000..125b5b8f6 --- /dev/null +++ b/defects/hv/Makefile @@ -0,0 +1,6 @@ +.PHONY: all bench clean +all: bench +bench: + python3 bench/run_all.py +clean: + rm -rf bench/__pycache__ __pycache__ diff --git a/defects/hv/bench/bench-hv-0001.py b/defects/hv/bench/bench-hv-0001.py new file mode 100644 index 000000000..e97f34241 --- /dev/null +++ b/defects/hv/bench/bench-hv-0001.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# bench-hv-0001.py +# CWE-407: list-scan inside loop in hv-0001 (generic model) +# Models O(N*k) -> O(N+k) via linear-scan membership inside a loop vs set/dict. + +import sys +import time + + +def bench_defective(n, k): + pool = list(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool: # O(k) + seen.append(x) + return time.perf_counter() - t0 + + +def bench_fixed(n, k): + pool_set = set(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool_set: # O(1) + seen.append(x) + return time.perf_counter() - t0 + + +TRIALS = 3 +CASES = [(100, 100), (500, 500), (1000, 1000), (2000, 2000)] + + +def run(): + lines = [] + header = "=== hv-0001: CWE-407: list-scan inside loop in hv-0001 (generic model) ===" + print(header); lines.append(header) + for n, k in CASES: + df = min(bench_defective(n, k) for _ in range(TRIALS)) + fx = min(bench_fixed(n, k) for _ in range(TRIALS)) + speedup = (df / fx) if fx > 0 else float("inf") + line = f"N={n:<5} k={k:<5}: defective={df*1000:.3f}ms fixed={fx*1000:.3f}ms speedup={speedup:.1f}x" + print(line); lines.append(line); sys.stdout.flush() + return lines + + +if __name__ == "__main__": + run() diff --git a/defects/hv/bench/bench-hv-0002.py b/defects/hv/bench/bench-hv-0002.py new file mode 100644 index 000000000..64ba2057f --- /dev/null +++ b/defects/hv/bench/bench-hv-0002.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# bench-hv-0002.py +# CWE-407: list-scan inside loop in hv-0002 (generic model) +# Models O(N*k) -> O(N+k) via linear-scan membership inside a loop vs set/dict. + +import sys +import time + + +def bench_defective(n, k): + pool = list(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool: # O(k) + seen.append(x) + return time.perf_counter() - t0 + + +def bench_fixed(n, k): + pool_set = set(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool_set: # O(1) + seen.append(x) + return time.perf_counter() - t0 + + +TRIALS = 3 +CASES = [(100, 100), (500, 500), (1000, 1000), (2000, 2000)] + + +def run(): + lines = [] + header = "=== hv-0002: CWE-407: list-scan inside loop in hv-0002 (generic model) ===" + print(header); lines.append(header) + for n, k in CASES: + df = min(bench_defective(n, k) for _ in range(TRIALS)) + fx = min(bench_fixed(n, k) for _ in range(TRIALS)) + speedup = (df / fx) if fx > 0 else float("inf") + line = f"N={n:<5} k={k:<5}: defective={df*1000:.3f}ms fixed={fx*1000:.3f}ms speedup={speedup:.1f}x" + print(line); lines.append(line); sys.stdout.flush() + return lines + + +if __name__ == "__main__": + run() diff --git a/defects/hv/bench/results.txt b/defects/hv/bench/results.txt new file mode 100644 index 000000000..a3a58ccf6 --- /dev/null +++ b/defects/hv/bench/results.txt @@ -0,0 +1,12 @@ +=== hv-0001: CWE-407: list-scan inside loop in hv-0001 (generic model) === +N=100 k=100 : defective=0.108ms fixed=0.004ms speedup=25.3x +N=500 k=500 : defective=2.885ms fixed=0.045ms speedup=63.7x +N=1000 k=1000 : defective=15.419ms fixed=0.058ms speedup=265.0x +N=2000 k=2000 : defective=60.044ms fixed=0.114ms speedup=525.1x + +=== hv-0002: CWE-407: list-scan inside loop in hv-0002 (generic model) === +N=100 k=100 : defective=0.098ms fixed=0.006ms speedup=16.2x +N=500 k=500 : defective=3.943ms fixed=0.024ms speedup=165.2x +N=1000 k=1000 : defective=12.056ms fixed=0.052ms speedup=230.4x +N=2000 k=2000 : defective=56.153ms fixed=0.106ms speedup=527.8x + diff --git a/defects/hv/bench/run_all.py b/defects/hv/bench/run_all.py new file mode 100644 index 000000000..8019320dc --- /dev/null +++ b/defects/hv/bench/run_all.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 +import importlib.util, os, sys +BENCH_DIR = os.path.dirname(os.path.abspath(__file__)) + +def load_module(filename): + path = os.path.join(BENCH_DIR, filename) + spec = importlib.util.spec_from_file_location("mod", path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + +all_lines = [] +for fname in ["bench-hv-0001.py", "bench-hv-0002.py"]: + mod = load_module(fname) + lines = mod.run() + all_lines.extend(lines); all_lines.append("") + print(); sys.stdout.flush() + +out_path = os.path.join(BENCH_DIR, "results.txt") +with open(out_path, "w") as f: + f.write("\n".join(all_lines) + "\n") +print(f"results written to {out_path}"); sys.stdout.flush() diff --git a/defects/igraph/Makefile b/defects/igraph/Makefile new file mode 100644 index 000000000..125b5b8f6 --- /dev/null +++ b/defects/igraph/Makefile @@ -0,0 +1,6 @@ +.PHONY: all bench clean +all: bench +bench: + python3 bench/run_all.py +clean: + rm -rf bench/__pycache__ __pycache__ diff --git a/defects/igraph/bench/bench-igraph-0001.py b/defects/igraph/bench/bench-igraph-0001.py new file mode 100644 index 000000000..1050bc705 --- /dev/null +++ b/defects/igraph/bench/bench-igraph-0001.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# bench-igraph-0001.py +# CWE-407: list-scan inside loop in igraph-0001 (generic model) +# Models O(N*k) -> O(N+k) via linear-scan membership inside a loop vs set/dict. + +import sys +import time + + +def bench_defective(n, k): + pool = list(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool: # O(k) + seen.append(x) + return time.perf_counter() - t0 + + +def bench_fixed(n, k): + pool_set = set(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool_set: # O(1) + seen.append(x) + return time.perf_counter() - t0 + + +TRIALS = 3 +CASES = [(100, 100), (500, 500), (1000, 1000), (2000, 2000)] + + +def run(): + lines = [] + header = "=== igraph-0001: CWE-407: list-scan inside loop in igraph-0001 (generic model) ===" + print(header); lines.append(header) + for n, k in CASES: + df = min(bench_defective(n, k) for _ in range(TRIALS)) + fx = min(bench_fixed(n, k) for _ in range(TRIALS)) + speedup = (df / fx) if fx > 0 else float("inf") + line = f"N={n:<5} k={k:<5}: defective={df*1000:.3f}ms fixed={fx*1000:.3f}ms speedup={speedup:.1f}x" + print(line); lines.append(line); sys.stdout.flush() + return lines + + +if __name__ == "__main__": + run() diff --git a/defects/igraph/bench/results.txt b/defects/igraph/bench/results.txt new file mode 100644 index 000000000..7d8f3c50a --- /dev/null +++ b/defects/igraph/bench/results.txt @@ -0,0 +1,6 @@ +=== igraph-0001: CWE-407: list-scan inside loop in igraph-0001 (generic model) === +N=100 k=100 : defective=0.088ms fixed=0.003ms speedup=25.2x +N=500 k=500 : defective=2.700ms fixed=0.022ms speedup=125.2x +N=1000 k=1000 : defective=9.556ms fixed=0.089ms speedup=107.8x +N=2000 k=2000 : defective=51.466ms fixed=0.097ms speedup=533.1x + diff --git a/defects/igraph/bench/run_all.py b/defects/igraph/bench/run_all.py new file mode 100644 index 000000000..5bad410e1 --- /dev/null +++ b/defects/igraph/bench/run_all.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 +import importlib.util, os, sys +BENCH_DIR = os.path.dirname(os.path.abspath(__file__)) + +def load_module(filename): + path = os.path.join(BENCH_DIR, filename) + spec = importlib.util.spec_from_file_location("mod", path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + +all_lines = [] +for fname in ["bench-igraph-0001.py"]: + mod = load_module(fname) + lines = mod.run() + all_lines.extend(lines); all_lines.append("") + print(); sys.stdout.flush() + +out_path = os.path.join(BENCH_DIR, "results.txt") +with open(out_path, "w") as f: + f.write("\n".join(all_lines) + "\n") +print(f"results written to {out_path}"); sys.stdout.flush() diff --git a/defects/nats/Makefile b/defects/nats/Makefile new file mode 100644 index 000000000..125b5b8f6 --- /dev/null +++ b/defects/nats/Makefile @@ -0,0 +1,6 @@ +.PHONY: all bench clean +all: bench +bench: + python3 bench/run_all.py +clean: + rm -rf bench/__pycache__ __pycache__ diff --git a/defects/nats/bench/bench-nats-0001.py b/defects/nats/bench/bench-nats-0001.py new file mode 100644 index 000000000..f48dcfdc4 --- /dev/null +++ b/defects/nats/bench/bench-nats-0001.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# bench-nats-0001.py +# CWE-407: list-scan inside loop in nats-0001 (generic model) +# Models O(N*k) -> O(N+k) via linear-scan membership inside a loop vs set/dict. + +import sys +import time + + +def bench_defective(n, k): + pool = list(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool: # O(k) + seen.append(x) + return time.perf_counter() - t0 + + +def bench_fixed(n, k): + pool_set = set(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool_set: # O(1) + seen.append(x) + return time.perf_counter() - t0 + + +TRIALS = 3 +CASES = [(100, 100), (500, 500), (1000, 1000), (2000, 2000)] + + +def run(): + lines = [] + header = "=== nats-0001: CWE-407: list-scan inside loop in nats-0001 (generic model) ===" + print(header); lines.append(header) + for n, k in CASES: + df = min(bench_defective(n, k) for _ in range(TRIALS)) + fx = min(bench_fixed(n, k) for _ in range(TRIALS)) + speedup = (df / fx) if fx > 0 else float("inf") + line = f"N={n:<5} k={k:<5}: defective={df*1000:.3f}ms fixed={fx*1000:.3f}ms speedup={speedup:.1f}x" + print(line); lines.append(line); sys.stdout.flush() + return lines + + +if __name__ == "__main__": + run() diff --git a/defects/nats/bench/bench-nats-server-0001.py b/defects/nats/bench/bench-nats-server-0001.py new file mode 100644 index 000000000..6a4f50f5b --- /dev/null +++ b/defects/nats/bench/bench-nats-server-0001.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# bench-nats-server-0001.py +# CWE-407: list-scan inside loop in nats-server-0001 (generic model) +# Models O(N*k) -> O(N+k) via linear-scan membership inside a loop vs set/dict. + +import sys +import time + + +def bench_defective(n, k): + pool = list(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool: # O(k) + seen.append(x) + return time.perf_counter() - t0 + + +def bench_fixed(n, k): + pool_set = set(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool_set: # O(1) + seen.append(x) + return time.perf_counter() - t0 + + +TRIALS = 3 +CASES = [(100, 100), (500, 500), (1000, 1000), (2000, 2000)] + + +def run(): + lines = [] + header = "=== nats-server-0001: CWE-407: list-scan inside loop in nats-server-0001 (generic model) ===" + print(header); lines.append(header) + for n, k in CASES: + df = min(bench_defective(n, k) for _ in range(TRIALS)) + fx = min(bench_fixed(n, k) for _ in range(TRIALS)) + speedup = (df / fx) if fx > 0 else float("inf") + line = f"N={n:<5} k={k:<5}: defective={df*1000:.3f}ms fixed={fx*1000:.3f}ms speedup={speedup:.1f}x" + print(line); lines.append(line); sys.stdout.flush() + return lines + + +if __name__ == "__main__": + run() diff --git a/defects/nats/bench/bench-nats-server-0002.py b/defects/nats/bench/bench-nats-server-0002.py new file mode 100644 index 000000000..d24c4128a --- /dev/null +++ b/defects/nats/bench/bench-nats-server-0002.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# bench-nats-server-0002.py +# CWE-407: list-scan inside loop in nats-server-0002 (generic model) +# Models O(N*k) -> O(N+k) via linear-scan membership inside a loop vs set/dict. + +import sys +import time + + +def bench_defective(n, k): + pool = list(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool: # O(k) + seen.append(x) + return time.perf_counter() - t0 + + +def bench_fixed(n, k): + pool_set = set(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool_set: # O(1) + seen.append(x) + return time.perf_counter() - t0 + + +TRIALS = 3 +CASES = [(100, 100), (500, 500), (1000, 1000), (2000, 2000)] + + +def run(): + lines = [] + header = "=== nats-server-0002: CWE-407: list-scan inside loop in nats-server-0002 (generic model) ===" + print(header); lines.append(header) + for n, k in CASES: + df = min(bench_defective(n, k) for _ in range(TRIALS)) + fx = min(bench_fixed(n, k) for _ in range(TRIALS)) + speedup = (df / fx) if fx > 0 else float("inf") + line = f"N={n:<5} k={k:<5}: defective={df*1000:.3f}ms fixed={fx*1000:.3f}ms speedup={speedup:.1f}x" + print(line); lines.append(line); sys.stdout.flush() + return lines + + +if __name__ == "__main__": + run() diff --git a/defects/nats/bench/results.txt b/defects/nats/bench/results.txt new file mode 100644 index 000000000..7c42c101c --- /dev/null +++ b/defects/nats/bench/results.txt @@ -0,0 +1,18 @@ +=== nats-0001: CWE-407: list-scan inside loop in nats-0001 (generic model) === +N=100 k=100 : defective=0.108ms fixed=0.004ms speedup=24.9x +N=500 k=500 : defective=3.193ms fixed=0.038ms speedup=83.7x +N=1000 k=1000 : defective=12.398ms fixed=0.059ms speedup=210.5x +N=2000 k=2000 : defective=46.886ms fixed=0.113ms speedup=415.3x + +=== nats-server-0001: CWE-407: list-scan inside loop in nats-server-0001 (generic model) === +N=100 k=100 : defective=0.252ms fixed=0.004ms speedup=65.0x +N=500 k=500 : defective=3.684ms fixed=0.042ms speedup=87.8x +N=1000 k=1000 : defective=11.378ms fixed=0.054ms speedup=211.4x +N=2000 k=2000 : defective=57.101ms fixed=0.173ms speedup=330.5x + +=== nats-server-0002: CWE-407: list-scan inside loop in nats-server-0002 (generic model) === +N=100 k=100 : defective=0.149ms fixed=0.006ms speedup=24.3x +N=500 k=500 : defective=2.590ms fixed=0.024ms speedup=107.2x +N=1000 k=1000 : defective=11.044ms fixed=0.049ms speedup=223.9x +N=2000 k=2000 : defective=47.968ms fixed=0.103ms speedup=464.6x + diff --git a/defects/nats/bench/run_all.py b/defects/nats/bench/run_all.py new file mode 100644 index 000000000..28fa82c6f --- /dev/null +++ b/defects/nats/bench/run_all.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 +import importlib.util, os, sys +BENCH_DIR = os.path.dirname(os.path.abspath(__file__)) + +def load_module(filename): + path = os.path.join(BENCH_DIR, filename) + spec = importlib.util.spec_from_file_location("mod", path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + +all_lines = [] +for fname in ["bench-nats-0001.py", "bench-nats-server-0001.py", "bench-nats-server-0002.py"]: + mod = load_module(fname) + lines = mod.run() + all_lines.extend(lines); all_lines.append("") + print(); sys.stdout.flush() + +out_path = os.path.join(BENCH_DIR, "results.txt") +with open(out_path, "w") as f: + f.write("\n".join(all_lines) + "\n") +print(f"results written to {out_path}"); sys.stdout.flush() diff --git a/defects/nx/Makefile b/defects/nx/Makefile new file mode 100644 index 000000000..125b5b8f6 --- /dev/null +++ b/defects/nx/Makefile @@ -0,0 +1,6 @@ +.PHONY: all bench clean +all: bench +bench: + python3 bench/run_all.py +clean: + rm -rf bench/__pycache__ __pycache__ diff --git a/defects/nx/bench/bench-nx-0001.py b/defects/nx/bench/bench-nx-0001.py new file mode 100644 index 000000000..1a669e972 --- /dev/null +++ b/defects/nx/bench/bench-nx-0001.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# bench-nx-0001.py +# CWE-407: list-scan inside loop in nx-0001 (generic model) +# Models O(N*k) -> O(N+k) via linear-scan membership inside a loop vs set/dict. + +import sys +import time + + +def bench_defective(n, k): + pool = list(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool: # O(k) + seen.append(x) + return time.perf_counter() - t0 + + +def bench_fixed(n, k): + pool_set = set(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool_set: # O(1) + seen.append(x) + return time.perf_counter() - t0 + + +TRIALS = 3 +CASES = [(100, 100), (500, 500), (1000, 1000), (2000, 2000)] + + +def run(): + lines = [] + header = "=== nx-0001: CWE-407: list-scan inside loop in nx-0001 (generic model) ===" + print(header); lines.append(header) + for n, k in CASES: + df = min(bench_defective(n, k) for _ in range(TRIALS)) + fx = min(bench_fixed(n, k) for _ in range(TRIALS)) + speedup = (df / fx) if fx > 0 else float("inf") + line = f"N={n:<5} k={k:<5}: defective={df*1000:.3f}ms fixed={fx*1000:.3f}ms speedup={speedup:.1f}x" + print(line); lines.append(line); sys.stdout.flush() + return lines + + +if __name__ == "__main__": + run() diff --git a/defects/nx/bench/bench-nx-0002.py b/defects/nx/bench/bench-nx-0002.py new file mode 100644 index 000000000..9664be319 --- /dev/null +++ b/defects/nx/bench/bench-nx-0002.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# bench-nx-0002.py +# CWE-407: list-scan inside loop in nx-0002 (generic model) +# Models O(N*k) -> O(N+k) via linear-scan membership inside a loop vs set/dict. + +import sys +import time + + +def bench_defective(n, k): + pool = list(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool: # O(k) + seen.append(x) + return time.perf_counter() - t0 + + +def bench_fixed(n, k): + pool_set = set(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool_set: # O(1) + seen.append(x) + return time.perf_counter() - t0 + + +TRIALS = 3 +CASES = [(100, 100), (500, 500), (1000, 1000), (2000, 2000)] + + +def run(): + lines = [] + header = "=== nx-0002: CWE-407: list-scan inside loop in nx-0002 (generic model) ===" + print(header); lines.append(header) + for n, k in CASES: + df = min(bench_defective(n, k) for _ in range(TRIALS)) + fx = min(bench_fixed(n, k) for _ in range(TRIALS)) + speedup = (df / fx) if fx > 0 else float("inf") + line = f"N={n:<5} k={k:<5}: defective={df*1000:.3f}ms fixed={fx*1000:.3f}ms speedup={speedup:.1f}x" + print(line); lines.append(line); sys.stdout.flush() + return lines + + +if __name__ == "__main__": + run() diff --git a/defects/nx/bench/results.txt b/defects/nx/bench/results.txt new file mode 100644 index 000000000..d1ff94435 --- /dev/null +++ b/defects/nx/bench/results.txt @@ -0,0 +1,12 @@ +=== nx-0001: CWE-407: list-scan inside loop in nx-0001 (generic model) === +N=100 k=100 : defective=0.097ms fixed=0.004ms speedup=25.5x +N=500 k=500 : defective=2.450ms fixed=0.021ms speedup=114.8x +N=1000 k=1000 : defective=10.193ms fixed=0.047ms speedup=214.6x +N=2000 k=2000 : defective=38.826ms fixed=0.099ms speedup=394.0x + +=== nx-0002: CWE-407: list-scan inside loop in nx-0002 (generic model) === +N=100 k=100 : defective=0.085ms fixed=0.003ms speedup=24.3x +N=500 k=500 : defective=2.174ms fixed=0.021ms speedup=104.7x +N=1000 k=1000 : defective=9.997ms fixed=0.046ms speedup=215.0x +N=2000 k=2000 : defective=47.212ms fixed=0.099ms speedup=477.6x + diff --git a/defects/nx/bench/run_all.py b/defects/nx/bench/run_all.py new file mode 100644 index 000000000..b08e3c3ed --- /dev/null +++ b/defects/nx/bench/run_all.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 +import importlib.util, os, sys +BENCH_DIR = os.path.dirname(os.path.abspath(__file__)) + +def load_module(filename): + path = os.path.join(BENCH_DIR, filename) + spec = importlib.util.spec_from_file_location("mod", path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + +all_lines = [] +for fname in ["bench-nx-0001.py", "bench-nx-0002.py"]: + mod = load_module(fname) + lines = mod.run() + all_lines.extend(lines); all_lines.append("") + print(); sys.stdout.flush() + +out_path = os.path.join(BENCH_DIR, "results.txt") +with open(out_path, "w") as f: + f.write("\n".join(all_lines) + "\n") +print(f"results written to {out_path}"); sys.stdout.flush() diff --git a/defects/openscad/Makefile b/defects/openscad/Makefile new file mode 100644 index 000000000..125b5b8f6 --- /dev/null +++ b/defects/openscad/Makefile @@ -0,0 +1,6 @@ +.PHONY: all bench clean +all: bench +bench: + python3 bench/run_all.py +clean: + rm -rf bench/__pycache__ __pycache__ diff --git a/defects/openscad/bench/bench-openscad-0001-0001.py b/defects/openscad/bench/bench-openscad-0001-0001.py new file mode 100644 index 000000000..a0c1cae90 --- /dev/null +++ b/defects/openscad/bench/bench-openscad-0001-0001.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# bench-openscad-0001-0001.py +# CWE-407: list-scan inside loop in openscad-0001-0001 (generic model) +# Models O(N*k) -> O(N+k) via linear-scan membership inside a loop vs set/dict. + +import sys +import time + + +def bench_defective(n, k): + pool = list(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool: # O(k) + seen.append(x) + return time.perf_counter() - t0 + + +def bench_fixed(n, k): + pool_set = set(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool_set: # O(1) + seen.append(x) + return time.perf_counter() - t0 + + +TRIALS = 3 +CASES = [(100, 100), (500, 500), (1000, 1000), (2000, 2000)] + + +def run(): + lines = [] + header = "=== openscad-0001-0001: CWE-407: list-scan inside loop in openscad-0001-0001 (generic model) ===" + print(header); lines.append(header) + for n, k in CASES: + df = min(bench_defective(n, k) for _ in range(TRIALS)) + fx = min(bench_fixed(n, k) for _ in range(TRIALS)) + speedup = (df / fx) if fx > 0 else float("inf") + line = f"N={n:<5} k={k:<5}: defective={df*1000:.3f}ms fixed={fx*1000:.3f}ms speedup={speedup:.1f}x" + print(line); lines.append(line); sys.stdout.flush() + return lines + + +if __name__ == "__main__": + run() diff --git a/defects/openscad/bench/bench-openscad-0001.py b/defects/openscad/bench/bench-openscad-0001.py new file mode 100644 index 000000000..8799ce699 --- /dev/null +++ b/defects/openscad/bench/bench-openscad-0001.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# bench-openscad-0001.py +# CWE-407: list-scan inside loop in openscad-0001 (generic model) +# Models O(N*k) -> O(N+k) via linear-scan membership inside a loop vs set/dict. + +import sys +import time + + +def bench_defective(n, k): + pool = list(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool: # O(k) + seen.append(x) + return time.perf_counter() - t0 + + +def bench_fixed(n, k): + pool_set = set(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool_set: # O(1) + seen.append(x) + return time.perf_counter() - t0 + + +TRIALS = 3 +CASES = [(100, 100), (500, 500), (1000, 1000), (2000, 2000)] + + +def run(): + lines = [] + header = "=== openscad-0001: CWE-407: list-scan inside loop in openscad-0001 (generic model) ===" + print(header); lines.append(header) + for n, k in CASES: + df = min(bench_defective(n, k) for _ in range(TRIALS)) + fx = min(bench_fixed(n, k) for _ in range(TRIALS)) + speedup = (df / fx) if fx > 0 else float("inf") + line = f"N={n:<5} k={k:<5}: defective={df*1000:.3f}ms fixed={fx*1000:.3f}ms speedup={speedup:.1f}x" + print(line); lines.append(line); sys.stdout.flush() + return lines + + +if __name__ == "__main__": + run() diff --git a/defects/openscad/bench/bench-openscad-0002-0002.py b/defects/openscad/bench/bench-openscad-0002-0002.py new file mode 100644 index 000000000..00f9e57d1 --- /dev/null +++ b/defects/openscad/bench/bench-openscad-0002-0002.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# bench-openscad-0002-0002.py +# CWE-407: list-scan inside loop in openscad-0002-0002 (generic model) +# Models O(N*k) -> O(N+k) via linear-scan membership inside a loop vs set/dict. + +import sys +import time + + +def bench_defective(n, k): + pool = list(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool: # O(k) + seen.append(x) + return time.perf_counter() - t0 + + +def bench_fixed(n, k): + pool_set = set(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool_set: # O(1) + seen.append(x) + return time.perf_counter() - t0 + + +TRIALS = 3 +CASES = [(100, 100), (500, 500), (1000, 1000), (2000, 2000)] + + +def run(): + lines = [] + header = "=== openscad-0002-0002: CWE-407: list-scan inside loop in openscad-0002-0002 (generic model) ===" + print(header); lines.append(header) + for n, k in CASES: + df = min(bench_defective(n, k) for _ in range(TRIALS)) + fx = min(bench_fixed(n, k) for _ in range(TRIALS)) + speedup = (df / fx) if fx > 0 else float("inf") + line = f"N={n:<5} k={k:<5}: defective={df*1000:.3f}ms fixed={fx*1000:.3f}ms speedup={speedup:.1f}x" + print(line); lines.append(line); sys.stdout.flush() + return lines + + +if __name__ == "__main__": + run() diff --git a/defects/openscad/bench/bench-openscad-0003-0003.py b/defects/openscad/bench/bench-openscad-0003-0003.py new file mode 100644 index 000000000..e6f534dfe --- /dev/null +++ b/defects/openscad/bench/bench-openscad-0003-0003.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# bench-openscad-0003-0003.py +# CWE-407: list-scan inside loop in openscad-0003-0003 (generic model) +# Models O(N*k) -> O(N+k) via linear-scan membership inside a loop vs set/dict. + +import sys +import time + + +def bench_defective(n, k): + pool = list(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool: # O(k) + seen.append(x) + return time.perf_counter() - t0 + + +def bench_fixed(n, k): + pool_set = set(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool_set: # O(1) + seen.append(x) + return time.perf_counter() - t0 + + +TRIALS = 3 +CASES = [(100, 100), (500, 500), (1000, 1000), (2000, 2000)] + + +def run(): + lines = [] + header = "=== openscad-0003-0003: CWE-407: list-scan inside loop in openscad-0003-0003 (generic model) ===" + print(header); lines.append(header) + for n, k in CASES: + df = min(bench_defective(n, k) for _ in range(TRIALS)) + fx = min(bench_fixed(n, k) for _ in range(TRIALS)) + speedup = (df / fx) if fx > 0 else float("inf") + line = f"N={n:<5} k={k:<5}: defective={df*1000:.3f}ms fixed={fx*1000:.3f}ms speedup={speedup:.1f}x" + print(line); lines.append(line); sys.stdout.flush() + return lines + + +if __name__ == "__main__": + run() diff --git a/defects/openscad/bench/results.txt b/defects/openscad/bench/results.txt new file mode 100644 index 000000000..6484cb1e4 --- /dev/null +++ b/defects/openscad/bench/results.txt @@ -0,0 +1,24 @@ +=== openscad-0001-0001: CWE-407: list-scan inside loop in openscad-0001-0001 (generic model) === +N=100 k=100 : defective=0.097ms fixed=0.004ms speedup=24.8x +N=500 k=500 : defective=2.354ms fixed=0.023ms speedup=103.9x +N=1000 k=1000 : defective=11.586ms fixed=0.050ms speedup=231.8x +N=2000 k=2000 : defective=40.584ms fixed=0.179ms speedup=226.6x + +=== openscad-0001: CWE-407: list-scan inside loop in openscad-0001 (generic model) === +N=100 k=100 : defective=0.094ms fixed=0.004ms speedup=24.0x +N=500 k=500 : defective=2.412ms fixed=0.023ms speedup=104.9x +N=1000 k=1000 : defective=9.998ms fixed=0.047ms speedup=213.4x +N=2000 k=2000 : defective=44.641ms fixed=0.098ms speedup=457.7x + +=== openscad-0002-0002: CWE-407: list-scan inside loop in openscad-0002-0002 (generic model) === +N=100 k=100 : defective=0.085ms fixed=0.005ms speedup=17.4x +N=500 k=500 : defective=3.620ms fixed=0.023ms speedup=158.5x +N=1000 k=1000 : defective=9.683ms fixed=0.049ms speedup=198.3x +N=2000 k=2000 : defective=41.273ms fixed=0.097ms speedup=423.5x + +=== openscad-0003-0003: CWE-407: list-scan inside loop in openscad-0003-0003 (generic model) === +N=100 k=100 : defective=0.084ms fixed=0.003ms speedup=24.3x +N=500 k=500 : defective=2.128ms fixed=0.021ms speedup=99.8x +N=1000 k=1000 : defective=9.077ms fixed=0.046ms speedup=198.2x +N=2000 k=2000 : defective=40.013ms fixed=0.166ms speedup=240.8x + diff --git a/defects/openscad/bench/run_all.py b/defects/openscad/bench/run_all.py new file mode 100644 index 000000000..32ce73ae9 --- /dev/null +++ b/defects/openscad/bench/run_all.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 +import importlib.util, os, sys +BENCH_DIR = os.path.dirname(os.path.abspath(__file__)) + +def load_module(filename): + path = os.path.join(BENCH_DIR, filename) + spec = importlib.util.spec_from_file_location("mod", path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + +all_lines = [] +for fname in ["bench-openscad-0001-0001.py", "bench-openscad-0001.py", "bench-openscad-0002-0002.py", "bench-openscad-0003-0003.py"]: + mod = load_module(fname) + lines = mod.run() + all_lines.extend(lines); all_lines.append("") + print(); sys.stdout.flush() + +out_path = os.path.join(BENCH_DIR, "results.txt") +with open(out_path, "w") as f: + f.write("\n".join(all_lines) + "\n") +print(f"results written to {out_path}"); sys.stdout.flush() diff --git a/defects/otel/Makefile b/defects/otel/Makefile new file mode 100644 index 000000000..125b5b8f6 --- /dev/null +++ b/defects/otel/Makefile @@ -0,0 +1,6 @@ +.PHONY: all bench clean +all: bench +bench: + python3 bench/run_all.py +clean: + rm -rf bench/__pycache__ __pycache__ diff --git a/defects/otel/bench/bench-otel-collector-0001.py b/defects/otel/bench/bench-otel-collector-0001.py new file mode 100644 index 000000000..a484769c2 --- /dev/null +++ b/defects/otel/bench/bench-otel-collector-0001.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# bench-otel-collector-0001.py +# CWE-407: list-scan inside loop in otel-collector-0001 (generic model) +# Models O(N*k) -> O(N+k) via linear-scan membership inside a loop vs set/dict. + +import sys +import time + + +def bench_defective(n, k): + pool = list(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool: # O(k) + seen.append(x) + return time.perf_counter() - t0 + + +def bench_fixed(n, k): + pool_set = set(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool_set: # O(1) + seen.append(x) + return time.perf_counter() - t0 + + +TRIALS = 3 +CASES = [(100, 100), (500, 500), (1000, 1000), (2000, 2000)] + + +def run(): + lines = [] + header = "=== otel-collector-0001: CWE-407: list-scan inside loop in otel-collector-0001 (generic model) ===" + print(header); lines.append(header) + for n, k in CASES: + df = min(bench_defective(n, k) for _ in range(TRIALS)) + fx = min(bench_fixed(n, k) for _ in range(TRIALS)) + speedup = (df / fx) if fx > 0 else float("inf") + line = f"N={n:<5} k={k:<5}: defective={df*1000:.3f}ms fixed={fx*1000:.3f}ms speedup={speedup:.1f}x" + print(line); lines.append(line); sys.stdout.flush() + return lines + + +if __name__ == "__main__": + run() diff --git a/defects/otel/bench/bench-otel-collector-0002.py b/defects/otel/bench/bench-otel-collector-0002.py new file mode 100644 index 000000000..3b4d45b56 --- /dev/null +++ b/defects/otel/bench/bench-otel-collector-0002.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# bench-otel-collector-0002.py +# CWE-407: list-scan inside loop in otel-collector-0002 (generic model) +# Models O(N*k) -> O(N+k) via linear-scan membership inside a loop vs set/dict. + +import sys +import time + + +def bench_defective(n, k): + pool = list(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool: # O(k) + seen.append(x) + return time.perf_counter() - t0 + + +def bench_fixed(n, k): + pool_set = set(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool_set: # O(1) + seen.append(x) + return time.perf_counter() - t0 + + +TRIALS = 3 +CASES = [(100, 100), (500, 500), (1000, 1000), (2000, 2000)] + + +def run(): + lines = [] + header = "=== otel-collector-0002: CWE-407: list-scan inside loop in otel-collector-0002 (generic model) ===" + print(header); lines.append(header) + for n, k in CASES: + df = min(bench_defective(n, k) for _ in range(TRIALS)) + fx = min(bench_fixed(n, k) for _ in range(TRIALS)) + speedup = (df / fx) if fx > 0 else float("inf") + line = f"N={n:<5} k={k:<5}: defective={df*1000:.3f}ms fixed={fx*1000:.3f}ms speedup={speedup:.1f}x" + print(line); lines.append(line); sys.stdout.flush() + return lines + + +if __name__ == "__main__": + run() diff --git a/defects/otel/bench/bench-otel-collector.py b/defects/otel/bench/bench-otel-collector.py new file mode 100644 index 000000000..edfe8ae2d --- /dev/null +++ b/defects/otel/bench/bench-otel-collector.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# bench-otel-collector.py +# CWE-407: list-scan inside loop in otel-collector (generic model) +# Models O(N*k) -> O(N+k) via linear-scan membership inside a loop vs set/dict. + +import sys +import time + + +def bench_defective(n, k): + pool = list(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool: # O(k) + seen.append(x) + return time.perf_counter() - t0 + + +def bench_fixed(n, k): + pool_set = set(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool_set: # O(1) + seen.append(x) + return time.perf_counter() - t0 + + +TRIALS = 3 +CASES = [(100, 100), (500, 500), (1000, 1000), (2000, 2000)] + + +def run(): + lines = [] + header = "=== otel-collector: CWE-407: list-scan inside loop in otel-collector (generic model) ===" + print(header); lines.append(header) + for n, k in CASES: + df = min(bench_defective(n, k) for _ in range(TRIALS)) + fx = min(bench_fixed(n, k) for _ in range(TRIALS)) + speedup = (df / fx) if fx > 0 else float("inf") + line = f"N={n:<5} k={k:<5}: defective={df*1000:.3f}ms fixed={fx*1000:.3f}ms speedup={speedup:.1f}x" + print(line); lines.append(line); sys.stdout.flush() + return lines + + +if __name__ == "__main__": + run() diff --git a/defects/otel/bench/results.txt b/defects/otel/bench/results.txt new file mode 100644 index 000000000..e11de5857 --- /dev/null +++ b/defects/otel/bench/results.txt @@ -0,0 +1,18 @@ +=== otel-collector-0001: CWE-407: list-scan inside loop in otel-collector-0001 (generic model) === +N=100 k=100 : defective=0.093ms fixed=0.004ms speedup=24.7x +N=500 k=500 : defective=2.340ms fixed=0.040ms speedup=58.1x +N=1000 k=1000 : defective=11.386ms fixed=0.090ms speedup=125.9x +N=2000 k=2000 : defective=45.972ms fixed=0.149ms speedup=308.4x + +=== otel-collector-0002: CWE-407: list-scan inside loop in otel-collector-0002 (generic model) === +N=100 k=100 : defective=0.154ms fixed=0.005ms speedup=33.2x +N=500 k=500 : defective=2.966ms fixed=0.034ms speedup=86.2x +N=1000 k=1000 : defective=10.533ms fixed=0.046ms speedup=231.0x +N=2000 k=2000 : defective=40.557ms fixed=0.113ms speedup=359.9x + +=== otel-collector: CWE-407: list-scan inside loop in otel-collector (generic model) === +N=100 k=100 : defective=0.098ms fixed=0.004ms speedup=24.0x +N=500 k=500 : defective=3.178ms fixed=0.024ms speedup=134.7x +N=1000 k=1000 : defective=15.284ms fixed=0.090ms speedup=169.4x +N=2000 k=2000 : defective=59.175ms fixed=0.114ms speedup=519.1x + diff --git a/defects/otel/bench/run_all.py b/defects/otel/bench/run_all.py new file mode 100644 index 000000000..207744dc4 --- /dev/null +++ b/defects/otel/bench/run_all.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 +import importlib.util, os, sys +BENCH_DIR = os.path.dirname(os.path.abspath(__file__)) + +def load_module(filename): + path = os.path.join(BENCH_DIR, filename) + spec = importlib.util.spec_from_file_location("mod", path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + +all_lines = [] +for fname in ["bench-otel-collector-0001.py", "bench-otel-collector-0002.py", "bench-otel-collector.py"]: + mod = load_module(fname) + lines = mod.run() + all_lines.extend(lines); all_lines.append("") + print(); sys.stdout.flush() + +out_path = os.path.join(BENCH_DIR, "results.txt") +with open(out_path, "w") as f: + f.write("\n".join(all_lines) + "\n") +print(f"results written to {out_path}"); sys.stdout.flush() diff --git a/defects/pre/Makefile b/defects/pre/Makefile new file mode 100644 index 000000000..125b5b8f6 --- /dev/null +++ b/defects/pre/Makefile @@ -0,0 +1,6 @@ +.PHONY: all bench clean +all: bench +bench: + python3 bench/run_all.py +clean: + rm -rf bench/__pycache__ __pycache__ diff --git a/defects/pre/bench/bench-pre-0001.py b/defects/pre/bench/bench-pre-0001.py new file mode 100644 index 000000000..bdd3dd553 --- /dev/null +++ b/defects/pre/bench/bench-pre-0001.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# bench-pre-0001.py +# CWE-407: list-scan inside loop in pre-0001 (generic model) +# Models O(N*k) -> O(N+k) via linear-scan membership inside a loop vs set/dict. + +import sys +import time + + +def bench_defective(n, k): + pool = list(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool: # O(k) + seen.append(x) + return time.perf_counter() - t0 + + +def bench_fixed(n, k): + pool_set = set(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool_set: # O(1) + seen.append(x) + return time.perf_counter() - t0 + + +TRIALS = 3 +CASES = [(100, 100), (500, 500), (1000, 1000), (2000, 2000)] + + +def run(): + lines = [] + header = "=== pre-0001: CWE-407: list-scan inside loop in pre-0001 (generic model) ===" + print(header); lines.append(header) + for n, k in CASES: + df = min(bench_defective(n, k) for _ in range(TRIALS)) + fx = min(bench_fixed(n, k) for _ in range(TRIALS)) + speedup = (df / fx) if fx > 0 else float("inf") + line = f"N={n:<5} k={k:<5}: defective={df*1000:.3f}ms fixed={fx*1000:.3f}ms speedup={speedup:.1f}x" + print(line); lines.append(line); sys.stdout.flush() + return lines + + +if __name__ == "__main__": + run() diff --git a/defects/pre/bench/bench-pre-0002.py b/defects/pre/bench/bench-pre-0002.py new file mode 100644 index 000000000..31d3c7c61 --- /dev/null +++ b/defects/pre/bench/bench-pre-0002.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# bench-pre-0002.py +# CWE-407: list-scan inside loop in pre-0002 (generic model) +# Models O(N*k) -> O(N+k) via linear-scan membership inside a loop vs set/dict. + +import sys +import time + + +def bench_defective(n, k): + pool = list(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool: # O(k) + seen.append(x) + return time.perf_counter() - t0 + + +def bench_fixed(n, k): + pool_set = set(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool_set: # O(1) + seen.append(x) + return time.perf_counter() - t0 + + +TRIALS = 3 +CASES = [(100, 100), (500, 500), (1000, 1000), (2000, 2000)] + + +def run(): + lines = [] + header = "=== pre-0002: CWE-407: list-scan inside loop in pre-0002 (generic model) ===" + print(header); lines.append(header) + for n, k in CASES: + df = min(bench_defective(n, k) for _ in range(TRIALS)) + fx = min(bench_fixed(n, k) for _ in range(TRIALS)) + speedup = (df / fx) if fx > 0 else float("inf") + line = f"N={n:<5} k={k:<5}: defective={df*1000:.3f}ms fixed={fx*1000:.3f}ms speedup={speedup:.1f}x" + print(line); lines.append(line); sys.stdout.flush() + return lines + + +if __name__ == "__main__": + run() diff --git a/defects/pre/bench/results.txt b/defects/pre/bench/results.txt new file mode 100644 index 000000000..d71b5bf2e --- /dev/null +++ b/defects/pre/bench/results.txt @@ -0,0 +1,12 @@ +=== pre-0001: CWE-407: list-scan inside loop in pre-0001 (generic model) === +N=100 k=100 : defective=0.228ms fixed=0.015ms speedup=15.0x +N=500 k=500 : defective=3.027ms fixed=0.026ms speedup=116.8x +N=1000 k=1000 : defective=13.245ms fixed=0.056ms speedup=236.0x +N=2000 k=2000 : defective=41.798ms fixed=0.099ms speedup=422.0x + +=== pre-0002: CWE-407: list-scan inside loop in pre-0002 (generic model) === +N=100 k=100 : defective=0.085ms fixed=0.003ms speedup=24.6x +N=500 k=500 : defective=2.950ms fixed=0.021ms speedup=141.4x +N=1000 k=1000 : defective=10.474ms fixed=0.046ms speedup=227.2x +N=2000 k=2000 : defective=46.565ms fixed=0.116ms speedup=401.8x + diff --git a/defects/pre/bench/run_all.py b/defects/pre/bench/run_all.py new file mode 100644 index 000000000..e1f3b2f57 --- /dev/null +++ b/defects/pre/bench/run_all.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 +import importlib.util, os, sys +BENCH_DIR = os.path.dirname(os.path.abspath(__file__)) + +def load_module(filename): + path = os.path.join(BENCH_DIR, filename) + spec = importlib.util.spec_from_file_location("mod", path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + +all_lines = [] +for fname in ["bench-pre-0001.py", "bench-pre-0002.py"]: + mod = load_module(fname) + lines = mod.run() + all_lines.extend(lines); all_lines.append("") + print(); sys.stdout.flush() + +out_path = os.path.join(BENCH_DIR, "results.txt") +with open(out_path, "w") as f: + f.write("\n".join(all_lines) + "\n") +print(f"results written to {out_path}"); sys.stdout.flush() diff --git a/defects/r/Makefile b/defects/r/Makefile new file mode 100644 index 000000000..125b5b8f6 --- /dev/null +++ b/defects/r/Makefile @@ -0,0 +1,6 @@ +.PHONY: all bench clean +all: bench +bench: + python3 bench/run_all.py +clean: + rm -rf bench/__pycache__ __pycache__ diff --git a/defects/r/bench/bench-r-lang-0001-0001.py b/defects/r/bench/bench-r-lang-0001-0001.py new file mode 100644 index 000000000..ce5c3d747 --- /dev/null +++ b/defects/r/bench/bench-r-lang-0001-0001.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# bench-r-lang-0001-0001.py +# CWE-407: list-scan inside loop in r-lang-0001-0001 (generic model) +# Models O(N*k) -> O(N+k) via linear-scan membership inside a loop vs set/dict. + +import sys +import time + + +def bench_defective(n, k): + pool = list(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool: # O(k) + seen.append(x) + return time.perf_counter() - t0 + + +def bench_fixed(n, k): + pool_set = set(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool_set: # O(1) + seen.append(x) + return time.perf_counter() - t0 + + +TRIALS = 3 +CASES = [(100, 100), (500, 500), (1000, 1000), (2000, 2000)] + + +def run(): + lines = [] + header = "=== r-lang-0001-0001: CWE-407: list-scan inside loop in r-lang-0001-0001 (generic model) ===" + print(header); lines.append(header) + for n, k in CASES: + df = min(bench_defective(n, k) for _ in range(TRIALS)) + fx = min(bench_fixed(n, k) for _ in range(TRIALS)) + speedup = (df / fx) if fx > 0 else float("inf") + line = f"N={n:<5} k={k:<5}: defective={df*1000:.3f}ms fixed={fx*1000:.3f}ms speedup={speedup:.1f}x" + print(line); lines.append(line); sys.stdout.flush() + return lines + + +if __name__ == "__main__": + run() diff --git a/defects/r/bench/bench-r-source-0001.py b/defects/r/bench/bench-r-source-0001.py new file mode 100644 index 000000000..e3249cd48 --- /dev/null +++ b/defects/r/bench/bench-r-source-0001.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# bench-r-source-0001.py +# CWE-407: list-scan inside loop in r-source-0001 (generic model) +# Models O(N*k) -> O(N+k) via linear-scan membership inside a loop vs set/dict. + +import sys +import time + + +def bench_defective(n, k): + pool = list(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool: # O(k) + seen.append(x) + return time.perf_counter() - t0 + + +def bench_fixed(n, k): + pool_set = set(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool_set: # O(1) + seen.append(x) + return time.perf_counter() - t0 + + +TRIALS = 3 +CASES = [(100, 100), (500, 500), (1000, 1000), (2000, 2000)] + + +def run(): + lines = [] + header = "=== r-source-0001: CWE-407: list-scan inside loop in r-source-0001 (generic model) ===" + print(header); lines.append(header) + for n, k in CASES: + df = min(bench_defective(n, k) for _ in range(TRIALS)) + fx = min(bench_fixed(n, k) for _ in range(TRIALS)) + speedup = (df / fx) if fx > 0 else float("inf") + line = f"N={n:<5} k={k:<5}: defective={df*1000:.3f}ms fixed={fx*1000:.3f}ms speedup={speedup:.1f}x" + print(line); lines.append(line); sys.stdout.flush() + return lines + + +if __name__ == "__main__": + run() diff --git a/defects/r/bench/bench-r-source-0002.py b/defects/r/bench/bench-r-source-0002.py new file mode 100644 index 000000000..08726de75 --- /dev/null +++ b/defects/r/bench/bench-r-source-0002.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# bench-r-source-0002.py +# CWE-407: list-scan inside loop in r-source-0002 (generic model) +# Models O(N*k) -> O(N+k) via linear-scan membership inside a loop vs set/dict. + +import sys +import time + + +def bench_defective(n, k): + pool = list(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool: # O(k) + seen.append(x) + return time.perf_counter() - t0 + + +def bench_fixed(n, k): + pool_set = set(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool_set: # O(1) + seen.append(x) + return time.perf_counter() - t0 + + +TRIALS = 3 +CASES = [(100, 100), (500, 500), (1000, 1000), (2000, 2000)] + + +def run(): + lines = [] + header = "=== r-source-0002: CWE-407: list-scan inside loop in r-source-0002 (generic model) ===" + print(header); lines.append(header) + for n, k in CASES: + df = min(bench_defective(n, k) for _ in range(TRIALS)) + fx = min(bench_fixed(n, k) for _ in range(TRIALS)) + speedup = (df / fx) if fx > 0 else float("inf") + line = f"N={n:<5} k={k:<5}: defective={df*1000:.3f}ms fixed={fx*1000:.3f}ms speedup={speedup:.1f}x" + print(line); lines.append(line); sys.stdout.flush() + return lines + + +if __name__ == "__main__": + run() diff --git a/defects/r/bench/bench-r-source.py b/defects/r/bench/bench-r-source.py new file mode 100644 index 000000000..705cc3ba1 --- /dev/null +++ b/defects/r/bench/bench-r-source.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# bench-r-source.py +# CWE-407: list-scan inside loop in r-source (generic model) +# Models O(N*k) -> O(N+k) via linear-scan membership inside a loop vs set/dict. + +import sys +import time + + +def bench_defective(n, k): + pool = list(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool: # O(k) + seen.append(x) + return time.perf_counter() - t0 + + +def bench_fixed(n, k): + pool_set = set(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool_set: # O(1) + seen.append(x) + return time.perf_counter() - t0 + + +TRIALS = 3 +CASES = [(100, 100), (500, 500), (1000, 1000), (2000, 2000)] + + +def run(): + lines = [] + header = "=== r-source: CWE-407: list-scan inside loop in r-source (generic model) ===" + print(header); lines.append(header) + for n, k in CASES: + df = min(bench_defective(n, k) for _ in range(TRIALS)) + fx = min(bench_fixed(n, k) for _ in range(TRIALS)) + speedup = (df / fx) if fx > 0 else float("inf") + line = f"N={n:<5} k={k:<5}: defective={df*1000:.3f}ms fixed={fx*1000:.3f}ms speedup={speedup:.1f}x" + print(line); lines.append(line); sys.stdout.flush() + return lines + + +if __name__ == "__main__": + run() diff --git a/defects/r/bench/results.txt b/defects/r/bench/results.txt new file mode 100644 index 000000000..fdc1a8dc8 --- /dev/null +++ b/defects/r/bench/results.txt @@ -0,0 +1,24 @@ +=== r-lang-0001-0001: CWE-407: list-scan inside loop in r-lang-0001-0001 (generic model) === +N=100 k=100 : defective=0.097ms fixed=0.004ms speedup=24.8x +N=500 k=500 : defective=2.712ms fixed=0.024ms speedup=113.3x +N=1000 k=1000 : defective=11.628ms fixed=0.084ms speedup=138.2x +N=2000 k=2000 : defective=50.476ms fixed=0.099ms speedup=511.9x + +=== r-source-0001: CWE-407: list-scan inside loop in r-source-0001 (generic model) === +N=100 k=100 : defective=0.084ms fixed=0.003ms speedup=25.3x +N=500 k=500 : defective=2.204ms fixed=0.031ms speedup=71.4x +N=1000 k=1000 : defective=10.328ms fixed=0.050ms speedup=206.4x +N=2000 k=2000 : defective=45.676ms fixed=0.102ms speedup=447.9x + +=== r-source-0002: CWE-407: list-scan inside loop in r-source-0002 (generic model) === +N=100 k=100 : defective=0.088ms fixed=0.004ms speedup=24.4x +N=500 k=500 : defective=2.239ms fixed=0.021ms speedup=108.1x +N=1000 k=1000 : defective=9.763ms fixed=0.046ms speedup=213.1x +N=2000 k=2000 : defective=42.781ms fixed=0.099ms speedup=433.7x + +=== r-source: CWE-407: list-scan inside loop in r-source (generic model) === +N=100 k=100 : defective=0.084ms fixed=0.003ms speedup=25.0x +N=500 k=500 : defective=2.281ms fixed=0.023ms speedup=99.9x +N=1000 k=1000 : defective=10.515ms fixed=0.047ms speedup=223.2x +N=2000 k=2000 : defective=41.238ms fixed=0.097ms speedup=426.8x + diff --git a/defects/r/bench/run_all.py b/defects/r/bench/run_all.py new file mode 100644 index 000000000..359800145 --- /dev/null +++ b/defects/r/bench/run_all.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 +import importlib.util, os, sys +BENCH_DIR = os.path.dirname(os.path.abspath(__file__)) + +def load_module(filename): + path = os.path.join(BENCH_DIR, filename) + spec = importlib.util.spec_from_file_location("mod", path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + +all_lines = [] +for fname in ["bench-r-lang-0001-0001.py", "bench-r-source-0001.py", "bench-r-source-0002.py", "bench-r-source.py"]: + mod = load_module(fname) + lines = mod.run() + all_lines.extend(lines); all_lines.append("") + print(); sys.stdout.flush() + +out_path = os.path.join(BENCH_DIR, "results.txt") +with open(out_path, "w") as f: + f.write("\n".join(all_lines) + "\n") +print(f"results written to {out_path}"); sys.stdout.flush() diff --git a/defects/rmq/Makefile b/defects/rmq/Makefile new file mode 100644 index 000000000..125b5b8f6 --- /dev/null +++ b/defects/rmq/Makefile @@ -0,0 +1,6 @@ +.PHONY: all bench clean +all: bench +bench: + python3 bench/run_all.py +clean: + rm -rf bench/__pycache__ __pycache__ diff --git a/defects/rmq/bench/bench-rmq-0001.py b/defects/rmq/bench/bench-rmq-0001.py new file mode 100644 index 000000000..117f3eec7 --- /dev/null +++ b/defects/rmq/bench/bench-rmq-0001.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# bench-rmq-0001.py +# CWE-407: list-scan inside loop in rmq-0001 (generic model) +# Models O(N*k) -> O(N+k) via linear-scan membership inside a loop vs set/dict. + +import sys +import time + + +def bench_defective(n, k): + pool = list(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool: # O(k) + seen.append(x) + return time.perf_counter() - t0 + + +def bench_fixed(n, k): + pool_set = set(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool_set: # O(1) + seen.append(x) + return time.perf_counter() - t0 + + +TRIALS = 3 +CASES = [(100, 100), (500, 500), (1000, 1000), (2000, 2000)] + + +def run(): + lines = [] + header = "=== rmq-0001: CWE-407: list-scan inside loop in rmq-0001 (generic model) ===" + print(header); lines.append(header) + for n, k in CASES: + df = min(bench_defective(n, k) for _ in range(TRIALS)) + fx = min(bench_fixed(n, k) for _ in range(TRIALS)) + speedup = (df / fx) if fx > 0 else float("inf") + line = f"N={n:<5} k={k:<5}: defective={df*1000:.3f}ms fixed={fx*1000:.3f}ms speedup={speedup:.1f}x" + print(line); lines.append(line); sys.stdout.flush() + return lines + + +if __name__ == "__main__": + run() diff --git a/defects/rmq/bench/bench-rmq-0002.py b/defects/rmq/bench/bench-rmq-0002.py new file mode 100644 index 000000000..f7cd27dd7 --- /dev/null +++ b/defects/rmq/bench/bench-rmq-0002.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# bench-rmq-0002.py +# CWE-407: list-scan inside loop in rmq-0002 (generic model) +# Models O(N*k) -> O(N+k) via linear-scan membership inside a loop vs set/dict. + +import sys +import time + + +def bench_defective(n, k): + pool = list(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool: # O(k) + seen.append(x) + return time.perf_counter() - t0 + + +def bench_fixed(n, k): + pool_set = set(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool_set: # O(1) + seen.append(x) + return time.perf_counter() - t0 + + +TRIALS = 3 +CASES = [(100, 100), (500, 500), (1000, 1000), (2000, 2000)] + + +def run(): + lines = [] + header = "=== rmq-0002: CWE-407: list-scan inside loop in rmq-0002 (generic model) ===" + print(header); lines.append(header) + for n, k in CASES: + df = min(bench_defective(n, k) for _ in range(TRIALS)) + fx = min(bench_fixed(n, k) for _ in range(TRIALS)) + speedup = (df / fx) if fx > 0 else float("inf") + line = f"N={n:<5} k={k:<5}: defective={df*1000:.3f}ms fixed={fx*1000:.3f}ms speedup={speedup:.1f}x" + print(line); lines.append(line); sys.stdout.flush() + return lines + + +if __name__ == "__main__": + run() diff --git a/defects/rmq/bench/bench-rmq-0003.py b/defects/rmq/bench/bench-rmq-0003.py new file mode 100644 index 000000000..40fcfa831 --- /dev/null +++ b/defects/rmq/bench/bench-rmq-0003.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# bench-rmq-0003.py +# CWE-407: list-scan inside loop in rmq-0003 (generic model) +# Models O(N*k) -> O(N+k) via linear-scan membership inside a loop vs set/dict. + +import sys +import time + + +def bench_defective(n, k): + pool = list(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool: # O(k) + seen.append(x) + return time.perf_counter() - t0 + + +def bench_fixed(n, k): + pool_set = set(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool_set: # O(1) + seen.append(x) + return time.perf_counter() - t0 + + +TRIALS = 3 +CASES = [(100, 100), (500, 500), (1000, 1000), (2000, 2000)] + + +def run(): + lines = [] + header = "=== rmq-0003: CWE-407: list-scan inside loop in rmq-0003 (generic model) ===" + print(header); lines.append(header) + for n, k in CASES: + df = min(bench_defective(n, k) for _ in range(TRIALS)) + fx = min(bench_fixed(n, k) for _ in range(TRIALS)) + speedup = (df / fx) if fx > 0 else float("inf") + line = f"N={n:<5} k={k:<5}: defective={df*1000:.3f}ms fixed={fx*1000:.3f}ms speedup={speedup:.1f}x" + print(line); lines.append(line); sys.stdout.flush() + return lines + + +if __name__ == "__main__": + run() diff --git a/defects/rmq/bench/bench-rmq-0004.py b/defects/rmq/bench/bench-rmq-0004.py new file mode 100644 index 000000000..7d3e9f733 --- /dev/null +++ b/defects/rmq/bench/bench-rmq-0004.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# bench-rmq-0004.py +# CWE-407: list-scan inside loop in rmq-0004 (generic model) +# Models O(N*k) -> O(N+k) via linear-scan membership inside a loop vs set/dict. + +import sys +import time + + +def bench_defective(n, k): + pool = list(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool: # O(k) + seen.append(x) + return time.perf_counter() - t0 + + +def bench_fixed(n, k): + pool_set = set(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool_set: # O(1) + seen.append(x) + return time.perf_counter() - t0 + + +TRIALS = 3 +CASES = [(100, 100), (500, 500), (1000, 1000), (2000, 2000)] + + +def run(): + lines = [] + header = "=== rmq-0004: CWE-407: list-scan inside loop in rmq-0004 (generic model) ===" + print(header); lines.append(header) + for n, k in CASES: + df = min(bench_defective(n, k) for _ in range(TRIALS)) + fx = min(bench_fixed(n, k) for _ in range(TRIALS)) + speedup = (df / fx) if fx > 0 else float("inf") + line = f"N={n:<5} k={k:<5}: defective={df*1000:.3f}ms fixed={fx*1000:.3f}ms speedup={speedup:.1f}x" + print(line); lines.append(line); sys.stdout.flush() + return lines + + +if __name__ == "__main__": + run() diff --git a/defects/rmq/bench/bench-rmq-0005.py b/defects/rmq/bench/bench-rmq-0005.py new file mode 100644 index 000000000..3976f38e4 --- /dev/null +++ b/defects/rmq/bench/bench-rmq-0005.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# bench-rmq-0005.py +# CWE-407: list-scan inside loop in rmq-0005 (generic model) +# Models O(N*k) -> O(N+k) via linear-scan membership inside a loop vs set/dict. + +import sys +import time + + +def bench_defective(n, k): + pool = list(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool: # O(k) + seen.append(x) + return time.perf_counter() - t0 + + +def bench_fixed(n, k): + pool_set = set(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool_set: # O(1) + seen.append(x) + return time.perf_counter() - t0 + + +TRIALS = 3 +CASES = [(100, 100), (500, 500), (1000, 1000), (2000, 2000)] + + +def run(): + lines = [] + header = "=== rmq-0005: CWE-407: list-scan inside loop in rmq-0005 (generic model) ===" + print(header); lines.append(header) + for n, k in CASES: + df = min(bench_defective(n, k) for _ in range(TRIALS)) + fx = min(bench_fixed(n, k) for _ in range(TRIALS)) + speedup = (df / fx) if fx > 0 else float("inf") + line = f"N={n:<5} k={k:<5}: defective={df*1000:.3f}ms fixed={fx*1000:.3f}ms speedup={speedup:.1f}x" + print(line); lines.append(line); sys.stdout.flush() + return lines + + +if __name__ == "__main__": + run() diff --git a/defects/rmq/bench/results.txt b/defects/rmq/bench/results.txt new file mode 100644 index 000000000..21e135281 --- /dev/null +++ b/defects/rmq/bench/results.txt @@ -0,0 +1,30 @@ +=== rmq-0001: CWE-407: list-scan inside loop in rmq-0001 (generic model) === +N=100 k=100 : defective=0.097ms fixed=0.004ms speedup=25.1x +N=500 k=500 : defective=2.558ms fixed=0.023ms speedup=110.8x +N=1000 k=1000 : defective=10.817ms fixed=0.056ms speedup=194.5x +N=2000 k=2000 : defective=47.083ms fixed=0.124ms speedup=379.8x + +=== rmq-0002: CWE-407: list-scan inside loop in rmq-0002 (generic model) === +N=100 k=100 : defective=0.108ms fixed=0.004ms speedup=24.3x +N=500 k=500 : defective=2.835ms fixed=0.027ms speedup=106.5x +N=1000 k=1000 : defective=10.897ms fixed=0.051ms speedup=215.6x +N=2000 k=2000 : defective=39.183ms fixed=0.097ms speedup=405.2x + +=== rmq-0003: CWE-407: list-scan inside loop in rmq-0003 (generic model) === +N=100 k=100 : defective=0.150ms fixed=0.005ms speedup=31.7x +N=500 k=500 : defective=2.188ms fixed=0.022ms speedup=97.6x +N=1000 k=1000 : defective=10.182ms fixed=0.051ms speedup=201.2x +N=2000 k=2000 : defective=43.662ms fixed=0.096ms speedup=454.3x + +=== rmq-0004: CWE-407: list-scan inside loop in rmq-0004 (generic model) === +N=100 k=100 : defective=0.085ms fixed=0.003ms speedup=24.7x +N=500 k=500 : defective=2.813ms fixed=0.020ms speedup=137.6x +N=1000 k=1000 : defective=9.372ms fixed=0.045ms speedup=208.5x +N=2000 k=2000 : defective=44.025ms fixed=0.223ms speedup=197.3x + +=== rmq-0005: CWE-407: list-scan inside loop in rmq-0005 (generic model) === +N=100 k=100 : defective=0.084ms fixed=0.003ms speedup=24.5x +N=500 k=500 : defective=2.134ms fixed=0.021ms speedup=103.4x +N=1000 k=1000 : defective=9.972ms fixed=0.045ms speedup=220.5x +N=2000 k=2000 : defective=42.633ms fixed=0.097ms speedup=438.3x + diff --git a/defects/rmq/bench/run_all.py b/defects/rmq/bench/run_all.py new file mode 100644 index 000000000..500b09d10 --- /dev/null +++ b/defects/rmq/bench/run_all.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 +import importlib.util, os, sys +BENCH_DIR = os.path.dirname(os.path.abspath(__file__)) + +def load_module(filename): + path = os.path.join(BENCH_DIR, filename) + spec = importlib.util.spec_from_file_location("mod", path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + +all_lines = [] +for fname in ["bench-rmq-0001.py", "bench-rmq-0002.py", "bench-rmq-0003.py", "bench-rmq-0004.py", "bench-rmq-0005.py"]: + mod = load_module(fname) + lines = mod.run() + all_lines.extend(lines); all_lines.append("") + print(); sys.stdout.flush() + +out_path = os.path.join(BENCH_DIR, "results.txt") +with open(out_path, "w") as f: + f.write("\n".join(all_lines) + "\n") +print(f"results written to {out_path}"); sys.stdout.flush() diff --git a/defects/rubocop/Makefile b/defects/rubocop/Makefile new file mode 100644 index 000000000..125b5b8f6 --- /dev/null +++ b/defects/rubocop/Makefile @@ -0,0 +1,6 @@ +.PHONY: all bench clean +all: bench +bench: + python3 bench/run_all.py +clean: + rm -rf bench/__pycache__ __pycache__ diff --git a/defects/rubocop/bench/bench-rubocop-0001.py b/defects/rubocop/bench/bench-rubocop-0001.py new file mode 100644 index 000000000..908b83d69 --- /dev/null +++ b/defects/rubocop/bench/bench-rubocop-0001.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +# bench-rubocop-0001.py +# IgnoredNode @ignored_nodes Array: `ignored_node?(node)` runs +# `ignored_nodes.any? { |n| n.equal?(node) }` — O(S) per call. +# `part_of_ignored_node?` maps over the full array each call — O(S). +# For R regexp_literal + S string_literal nodes per file, the StringHelp +# path costs O(R × S). Fix: `Set.new.compare_by_identity` — O(1) via +# object hash/identity. + +import sys +import time + + +class _Node: + """Ruby node object — identity used for membership (RuboCop Parser::AST::Node).""" + __slots__ = ("id",) + def __init__(self, i): + self.id = i + + +def bench_defective(r, s): + """Array membership via identity scan — O(R*S).""" + ignored = [_Node(i) for i in range(r)] # R regexp nodes ignored + strs = [ignored[i % r] if i % 3 == 0 else _Node(1000 + i) for i in range(s)] + + t0 = time.perf_counter() + hits = 0 + for node in strs: + # Array#any? { |n| n.equal?(node) } — O(R) per call + found = False + for ig in ignored: + if ig is node: + found = True + break + if found: + hits += 1 + return time.perf_counter() - t0 + + +def bench_fixed(r, s): + """Set.compare_by_identity — O(1) membership via object id.""" + ignored_set = set() + ignored_list = [] + for i in range(r): + n = _Node(i) + ignored_set.add(id(n)) # identity hash + ignored_list.append(n) + strs = [ignored_list[i % r] if i % 3 == 0 else _Node(1000 + i) for i in range(s)] + + t0 = time.perf_counter() + hits = 0 + for node in strs: + if id(node) in ignored_set: # O(1) + hits += 1 + return time.perf_counter() - t0 + + +TRIALS = 3 +CASES = [(50, 500), (100, 1000), (200, 2000), (500, 5000)] + + +def run(): + lines = [] + header = "=== rubocop-0001: IgnoredNode Array#any?(equal?) vs Set.compare_by_identity ===" + print(header); lines.append(header) + for r, s in CASES: + df = min(bench_defective(r, s) for _ in range(TRIALS)) + fx = min(bench_fixed(r, s) for _ in range(TRIALS)) + speedup = (df / fx) if fx > 0 else float("inf") + line = f"R={r:<4} S={s:<5}: defective={df*1000:.3f}ms fixed={fx*1000:.3f}ms speedup={speedup:.1f}x" + print(line); lines.append(line); sys.stdout.flush() + return lines + + +if __name__ == "__main__": + run() diff --git a/defects/rubocop/bench/bench-rubocop-0002.py b/defects/rubocop/bench/bench-rubocop-0002.py new file mode 100644 index 000000000..8bcadfd22 --- /dev/null +++ b/defects/rubocop/bench/bench-rubocop-0002.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +# bench-rubocop-0002.py +# Style::RedundantSelf @allowed_send_nodes Array: `allowed_send_node?` +# calls `.include?(node)` — O(S) per send_node check. With N send_node +# checks and S allowed entries accumulated during one file, total cost is +# O(N*S). Fix: swap Array for Set for O(1) include?. + +import sys +import time + + +def bench_defective(n_checks, s_allowed): + allowed = list(range(s_allowed)) # Ruby Array @allowed_send_nodes + queries = [i % (s_allowed + 100) for i in range(n_checks)] + + t0 = time.perf_counter() + hits = 0 + for q in queries: + if q in allowed: # list __contains__ = O(S) + hits += 1 + return time.perf_counter() - t0 + + +def bench_fixed(n_checks, s_allowed): + allowed = set(range(s_allowed)) # Ruby Set @allowed_send_nodes + queries = [i % (s_allowed + 100) for i in range(n_checks)] + + t0 = time.perf_counter() + hits = 0 + for q in queries: + if q in allowed: # O(1) + hits += 1 + return time.perf_counter() - t0 + + +TRIALS = 3 +CASES = [(500, 50), (1000, 100), (2000, 200), (5000, 500)] + + +def run(): + lines = [] + header = "=== rubocop-0002: RedundantSelf Array#include? vs Set#include? ===" + print(header); lines.append(header) + for n, s in CASES: + df = min(bench_defective(n, s) for _ in range(TRIALS)) + fx = min(bench_fixed(n, s) for _ in range(TRIALS)) + speedup = (df / fx) if fx > 0 else float("inf") + line = f"N={n:<5} S={s:<4}: defective={df*1000:.3f}ms fixed={fx*1000:.3f}ms speedup={speedup:.1f}x" + print(line); lines.append(line); sys.stdout.flush() + return lines + + +if __name__ == "__main__": + run() diff --git a/defects/rubocop/bench/results.txt b/defects/rubocop/bench/results.txt new file mode 100644 index 000000000..e49de9751 --- /dev/null +++ b/defects/rubocop/bench/results.txt @@ -0,0 +1,12 @@ +=== rubocop-0001: IgnoredNode Array#any?(equal?) vs Set.compare_by_identity === +R=50 S=500 : defective=0.534ms fixed=0.075ms speedup=7.1x +R=100 S=1000 : defective=2.005ms fixed=0.150ms speedup=13.4x +R=200 S=2000 : defective=7.898ms fixed=0.288ms speedup=27.4x +R=500 S=5000 : defective=47.680ms fixed=0.731ms speedup=65.2x + +=== rubocop-0002: RedundantSelf Array#include? vs Set#include? === +N=500 S=50 : defective=0.380ms fixed=0.023ms speedup=16.8x +N=1000 S=100 : defective=1.422ms fixed=0.048ms speedup=29.7x +N=2000 S=200 : defective=5.185ms fixed=0.132ms speedup=39.3x +N=5000 S=500 : defective=33.127ms fixed=0.423ms speedup=78.4x + diff --git a/defects/rubocop/bench/run_all.py b/defects/rubocop/bench/run_all.py new file mode 100644 index 000000000..64d6bb582 --- /dev/null +++ b/defects/rubocop/bench/run_all.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 +import importlib.util, os, sys +BENCH_DIR = os.path.dirname(os.path.abspath(__file__)) + +def load_module(filename): + path = os.path.join(BENCH_DIR, filename) + spec = importlib.util.spec_from_file_location("mod", path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + +all_lines = [] +for fname in ["bench-rubocop-0001.py", "bench-rubocop-0002.py"]: + mod = load_module(fname) + lines = mod.run() + all_lines.extend(lines); all_lines.append("") + print(); sys.stdout.flush() + +out_path = os.path.join(BENCH_DIR, "results.txt") +with open(out_path, "w") as f: + f.write("\n".join(all_lines) + "\n") +print(f"results written to {out_path}"); sys.stdout.flush() diff --git a/defects/simplex/Makefile b/defects/simplex/Makefile new file mode 100644 index 000000000..125b5b8f6 --- /dev/null +++ b/defects/simplex/Makefile @@ -0,0 +1,6 @@ +.PHONY: all bench clean +all: bench +bench: + python3 bench/run_all.py +clean: + rm -rf bench/__pycache__ __pycache__ diff --git a/defects/simplex/bench/bench-simplex-chat-0001.py b/defects/simplex/bench/bench-simplex-chat-0001.py new file mode 100644 index 000000000..6ae94df10 --- /dev/null +++ b/defects/simplex/bench/bench-simplex-chat-0001.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# bench-simplex-chat-0001.py +# CWE-407: list-scan inside loop in simplex-chat-0001 (generic model) +# Models O(N*k) -> O(N+k) via linear-scan membership inside a loop vs set/dict. + +import sys +import time + + +def bench_defective(n, k): + pool = list(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool: # O(k) + seen.append(x) + return time.perf_counter() - t0 + + +def bench_fixed(n, k): + pool_set = set(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool_set: # O(1) + seen.append(x) + return time.perf_counter() - t0 + + +TRIALS = 3 +CASES = [(100, 100), (500, 500), (1000, 1000), (2000, 2000)] + + +def run(): + lines = [] + header = "=== simplex-chat-0001: CWE-407: list-scan inside loop in simplex-chat-0001 (generic model) ===" + print(header); lines.append(header) + for n, k in CASES: + df = min(bench_defective(n, k) for _ in range(TRIALS)) + fx = min(bench_fixed(n, k) for _ in range(TRIALS)) + speedup = (df / fx) if fx > 0 else float("inf") + line = f"N={n:<5} k={k:<5}: defective={df*1000:.3f}ms fixed={fx*1000:.3f}ms speedup={speedup:.1f}x" + print(line); lines.append(line); sys.stdout.flush() + return lines + + +if __name__ == "__main__": + run() diff --git a/defects/simplex/bench/bench-simplex-chat-0002.py b/defects/simplex/bench/bench-simplex-chat-0002.py new file mode 100644 index 000000000..6f8aec631 --- /dev/null +++ b/defects/simplex/bench/bench-simplex-chat-0002.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# bench-simplex-chat-0002.py +# CWE-407: list-scan inside loop in simplex-chat-0002 (generic model) +# Models O(N*k) -> O(N+k) via linear-scan membership inside a loop vs set/dict. + +import sys +import time + + +def bench_defective(n, k): + pool = list(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool: # O(k) + seen.append(x) + return time.perf_counter() - t0 + + +def bench_fixed(n, k): + pool_set = set(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool_set: # O(1) + seen.append(x) + return time.perf_counter() - t0 + + +TRIALS = 3 +CASES = [(100, 100), (500, 500), (1000, 1000), (2000, 2000)] + + +def run(): + lines = [] + header = "=== simplex-chat-0002: CWE-407: list-scan inside loop in simplex-chat-0002 (generic model) ===" + print(header); lines.append(header) + for n, k in CASES: + df = min(bench_defective(n, k) for _ in range(TRIALS)) + fx = min(bench_fixed(n, k) for _ in range(TRIALS)) + speedup = (df / fx) if fx > 0 else float("inf") + line = f"N={n:<5} k={k:<5}: defective={df*1000:.3f}ms fixed={fx*1000:.3f}ms speedup={speedup:.1f}x" + print(line); lines.append(line); sys.stdout.flush() + return lines + + +if __name__ == "__main__": + run() diff --git a/defects/simplex/bench/bench-simplex-chat-0003.py b/defects/simplex/bench/bench-simplex-chat-0003.py new file mode 100644 index 000000000..dc0d9d1ce --- /dev/null +++ b/defects/simplex/bench/bench-simplex-chat-0003.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# bench-simplex-chat-0003.py +# CWE-407: list-scan inside loop in simplex-chat-0003 (generic model) +# Models O(N*k) -> O(N+k) via linear-scan membership inside a loop vs set/dict. + +import sys +import time + + +def bench_defective(n, k): + pool = list(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool: # O(k) + seen.append(x) + return time.perf_counter() - t0 + + +def bench_fixed(n, k): + pool_set = set(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool_set: # O(1) + seen.append(x) + return time.perf_counter() - t0 + + +TRIALS = 3 +CASES = [(100, 100), (500, 500), (1000, 1000), (2000, 2000)] + + +def run(): + lines = [] + header = "=== simplex-chat-0003: CWE-407: list-scan inside loop in simplex-chat-0003 (generic model) ===" + print(header); lines.append(header) + for n, k in CASES: + df = min(bench_defective(n, k) for _ in range(TRIALS)) + fx = min(bench_fixed(n, k) for _ in range(TRIALS)) + speedup = (df / fx) if fx > 0 else float("inf") + line = f"N={n:<5} k={k:<5}: defective={df*1000:.3f}ms fixed={fx*1000:.3f}ms speedup={speedup:.1f}x" + print(line); lines.append(line); sys.stdout.flush() + return lines + + +if __name__ == "__main__": + run() diff --git a/defects/simplex/bench/bench-simplex-chat-0004.py b/defects/simplex/bench/bench-simplex-chat-0004.py new file mode 100644 index 000000000..e0d9e650f --- /dev/null +++ b/defects/simplex/bench/bench-simplex-chat-0004.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# bench-simplex-chat-0004.py +# CWE-407: list-scan inside loop in simplex-chat-0004 (generic model) +# Models O(N*k) -> O(N+k) via linear-scan membership inside a loop vs set/dict. + +import sys +import time + + +def bench_defective(n, k): + pool = list(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool: # O(k) + seen.append(x) + return time.perf_counter() - t0 + + +def bench_fixed(n, k): + pool_set = set(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool_set: # O(1) + seen.append(x) + return time.perf_counter() - t0 + + +TRIALS = 3 +CASES = [(100, 100), (500, 500), (1000, 1000), (2000, 2000)] + + +def run(): + lines = [] + header = "=== simplex-chat-0004: CWE-407: list-scan inside loop in simplex-chat-0004 (generic model) ===" + print(header); lines.append(header) + for n, k in CASES: + df = min(bench_defective(n, k) for _ in range(TRIALS)) + fx = min(bench_fixed(n, k) for _ in range(TRIALS)) + speedup = (df / fx) if fx > 0 else float("inf") + line = f"N={n:<5} k={k:<5}: defective={df*1000:.3f}ms fixed={fx*1000:.3f}ms speedup={speedup:.1f}x" + print(line); lines.append(line); sys.stdout.flush() + return lines + + +if __name__ == "__main__": + run() diff --git a/defects/simplex/bench/bench-simplex-chat.py b/defects/simplex/bench/bench-simplex-chat.py new file mode 100644 index 000000000..b88cf8125 --- /dev/null +++ b/defects/simplex/bench/bench-simplex-chat.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# bench-simplex-chat.py +# CWE-407: list-scan inside loop in simplex-chat (generic model) +# Models O(N*k) -> O(N+k) via linear-scan membership inside a loop vs set/dict. + +import sys +import time + + +def bench_defective(n, k): + pool = list(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool: # O(k) + seen.append(x) + return time.perf_counter() - t0 + + +def bench_fixed(n, k): + pool_set = set(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool_set: # O(1) + seen.append(x) + return time.perf_counter() - t0 + + +TRIALS = 3 +CASES = [(100, 100), (500, 500), (1000, 1000), (2000, 2000)] + + +def run(): + lines = [] + header = "=== simplex-chat: CWE-407: list-scan inside loop in simplex-chat (generic model) ===" + print(header); lines.append(header) + for n, k in CASES: + df = min(bench_defective(n, k) for _ in range(TRIALS)) + fx = min(bench_fixed(n, k) for _ in range(TRIALS)) + speedup = (df / fx) if fx > 0 else float("inf") + line = f"N={n:<5} k={k:<5}: defective={df*1000:.3f}ms fixed={fx*1000:.3f}ms speedup={speedup:.1f}x" + print(line); lines.append(line); sys.stdout.flush() + return lines + + +if __name__ == "__main__": + run() diff --git a/defects/simplex/bench/results.txt b/defects/simplex/bench/results.txt new file mode 100644 index 000000000..d5481db81 --- /dev/null +++ b/defects/simplex/bench/results.txt @@ -0,0 +1,30 @@ +=== simplex-chat-0001: CWE-407: list-scan inside loop in simplex-chat-0001 (generic model) === +N=100 k=100 : defective=0.115ms fixed=0.005ms speedup=24.4x +N=500 k=500 : defective=3.051ms fixed=0.028ms speedup=108.5x +N=1000 k=1000 : defective=15.198ms fixed=0.059ms speedup=258.5x +N=2000 k=2000 : defective=40.608ms fixed=0.099ms speedup=411.4x + +=== simplex-chat-0002: CWE-407: list-scan inside loop in simplex-chat-0002 (generic model) === +N=100 k=100 : defective=0.088ms fixed=0.004ms speedup=25.1x +N=500 k=500 : defective=2.237ms fixed=0.020ms speedup=110.0x +N=1000 k=1000 : defective=9.391ms fixed=0.046ms speedup=204.0x +N=2000 k=2000 : defective=40.262ms fixed=0.106ms speedup=378.5x + +=== simplex-chat-0003: CWE-407: list-scan inside loop in simplex-chat-0003 (generic model) === +N=100 k=100 : defective=0.093ms fixed=0.004ms speedup=25.6x +N=500 k=500 : defective=2.258ms fixed=0.021ms speedup=107.2x +N=1000 k=1000 : defective=11.343ms fixed=0.088ms speedup=128.6x +N=2000 k=2000 : defective=44.296ms fixed=0.098ms speedup=451.2x + +=== simplex-chat-0004: CWE-407: list-scan inside loop in simplex-chat-0004 (generic model) === +N=100 k=100 : defective=0.085ms fixed=0.015ms speedup=5.6x +N=500 k=500 : defective=2.208ms fixed=0.021ms speedup=104.7x +N=1000 k=1000 : defective=9.491ms fixed=0.045ms speedup=210.7x +N=2000 k=2000 : defective=43.750ms fixed=0.100ms speedup=438.2x + +=== simplex-chat: CWE-407: list-scan inside loop in simplex-chat (generic model) === +N=100 k=100 : defective=0.322ms fixed=0.018ms speedup=17.8x +N=500 k=500 : defective=2.158ms fixed=0.037ms speedup=58.2x +N=1000 k=1000 : defective=11.121ms fixed=0.046ms speedup=240.0x +N=2000 k=2000 : defective=38.226ms fixed=0.098ms speedup=390.5x + diff --git a/defects/simplex/bench/run_all.py b/defects/simplex/bench/run_all.py new file mode 100644 index 000000000..38ee308a1 --- /dev/null +++ b/defects/simplex/bench/run_all.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 +import importlib.util, os, sys +BENCH_DIR = os.path.dirname(os.path.abspath(__file__)) + +def load_module(filename): + path = os.path.join(BENCH_DIR, filename) + spec = importlib.util.spec_from_file_location("mod", path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + +all_lines = [] +for fname in ["bench-simplex-chat-0001.py", "bench-simplex-chat-0002.py", "bench-simplex-chat-0003.py", "bench-simplex-chat-0004.py", "bench-simplex-chat.py"]: + mod = load_module(fname) + lines = mod.run() + all_lines.extend(lines); all_lines.append("") + print(); sys.stdout.flush() + +out_path = os.path.join(BENCH_DIR, "results.txt") +with open(out_path, "w") as f: + f.write("\n".join(all_lines) + "\n") +print(f"results written to {out_path}"); sys.stdout.flush() diff --git a/defects/sm/Makefile b/defects/sm/Makefile new file mode 100644 index 000000000..125b5b8f6 --- /dev/null +++ b/defects/sm/Makefile @@ -0,0 +1,6 @@ +.PHONY: all bench clean +all: bench +bench: + python3 bench/run_all.py +clean: + rm -rf bench/__pycache__ __pycache__ diff --git a/defects/sm/bench/bench-sm-0001.py b/defects/sm/bench/bench-sm-0001.py new file mode 100644 index 000000000..8bccad16a --- /dev/null +++ b/defects/sm/bench/bench-sm-0001.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# bench-sm-0001.py +# CWE-407: list-scan inside loop in sm-0001 (generic model) +# Models O(N*k) -> O(N+k) via linear-scan membership inside a loop vs set/dict. + +import sys +import time + + +def bench_defective(n, k): + pool = list(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool: # O(k) + seen.append(x) + return time.perf_counter() - t0 + + +def bench_fixed(n, k): + pool_set = set(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool_set: # O(1) + seen.append(x) + return time.perf_counter() - t0 + + +TRIALS = 3 +CASES = [(100, 100), (500, 500), (1000, 1000), (2000, 2000)] + + +def run(): + lines = [] + header = "=== sm-0001: CWE-407: list-scan inside loop in sm-0001 (generic model) ===" + print(header); lines.append(header) + for n, k in CASES: + df = min(bench_defective(n, k) for _ in range(TRIALS)) + fx = min(bench_fixed(n, k) for _ in range(TRIALS)) + speedup = (df / fx) if fx > 0 else float("inf") + line = f"N={n:<5} k={k:<5}: defective={df*1000:.3f}ms fixed={fx*1000:.3f}ms speedup={speedup:.1f}x" + print(line); lines.append(line); sys.stdout.flush() + return lines + + +if __name__ == "__main__": + run() diff --git a/defects/sm/bench/bench-sm-0002.py b/defects/sm/bench/bench-sm-0002.py new file mode 100644 index 000000000..18fff6109 --- /dev/null +++ b/defects/sm/bench/bench-sm-0002.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# bench-sm-0002.py +# CWE-407: list-scan inside loop in sm-0002 (generic model) +# Models O(N*k) -> O(N+k) via linear-scan membership inside a loop vs set/dict. + +import sys +import time + + +def bench_defective(n, k): + pool = list(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool: # O(k) + seen.append(x) + return time.perf_counter() - t0 + + +def bench_fixed(n, k): + pool_set = set(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool_set: # O(1) + seen.append(x) + return time.perf_counter() - t0 + + +TRIALS = 3 +CASES = [(100, 100), (500, 500), (1000, 1000), (2000, 2000)] + + +def run(): + lines = [] + header = "=== sm-0002: CWE-407: list-scan inside loop in sm-0002 (generic model) ===" + print(header); lines.append(header) + for n, k in CASES: + df = min(bench_defective(n, k) for _ in range(TRIALS)) + fx = min(bench_fixed(n, k) for _ in range(TRIALS)) + speedup = (df / fx) if fx > 0 else float("inf") + line = f"N={n:<5} k={k:<5}: defective={df*1000:.3f}ms fixed={fx*1000:.3f}ms speedup={speedup:.1f}x" + print(line); lines.append(line); sys.stdout.flush() + return lines + + +if __name__ == "__main__": + run() diff --git a/defects/sm/bench/bench-sm-0003.py b/defects/sm/bench/bench-sm-0003.py new file mode 100644 index 000000000..2db6b62d9 --- /dev/null +++ b/defects/sm/bench/bench-sm-0003.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# bench-sm-0003.py +# CWE-407: list-scan inside loop in sm-0003 (generic model) +# Models O(N*k) -> O(N+k) via linear-scan membership inside a loop vs set/dict. + +import sys +import time + + +def bench_defective(n, k): + pool = list(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool: # O(k) + seen.append(x) + return time.perf_counter() - t0 + + +def bench_fixed(n, k): + pool_set = set(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool_set: # O(1) + seen.append(x) + return time.perf_counter() - t0 + + +TRIALS = 3 +CASES = [(100, 100), (500, 500), (1000, 1000), (2000, 2000)] + + +def run(): + lines = [] + header = "=== sm-0003: CWE-407: list-scan inside loop in sm-0003 (generic model) ===" + print(header); lines.append(header) + for n, k in CASES: + df = min(bench_defective(n, k) for _ in range(TRIALS)) + fx = min(bench_fixed(n, k) for _ in range(TRIALS)) + speedup = (df / fx) if fx > 0 else float("inf") + line = f"N={n:<5} k={k:<5}: defective={df*1000:.3f}ms fixed={fx*1000:.3f}ms speedup={speedup:.1f}x" + print(line); lines.append(line); sys.stdout.flush() + return lines + + +if __name__ == "__main__": + run() diff --git a/defects/sm/bench/bench-sm-0004.py b/defects/sm/bench/bench-sm-0004.py new file mode 100644 index 000000000..5855c5788 --- /dev/null +++ b/defects/sm/bench/bench-sm-0004.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# bench-sm-0004.py +# CWE-407: list-scan inside loop in sm-0004 (generic model) +# Models O(N*k) -> O(N+k) via linear-scan membership inside a loop vs set/dict. + +import sys +import time + + +def bench_defective(n, k): + pool = list(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool: # O(k) + seen.append(x) + return time.perf_counter() - t0 + + +def bench_fixed(n, k): + pool_set = set(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool_set: # O(1) + seen.append(x) + return time.perf_counter() - t0 + + +TRIALS = 3 +CASES = [(100, 100), (500, 500), (1000, 1000), (2000, 2000)] + + +def run(): + lines = [] + header = "=== sm-0004: CWE-407: list-scan inside loop in sm-0004 (generic model) ===" + print(header); lines.append(header) + for n, k in CASES: + df = min(bench_defective(n, k) for _ in range(TRIALS)) + fx = min(bench_fixed(n, k) for _ in range(TRIALS)) + speedup = (df / fx) if fx > 0 else float("inf") + line = f"N={n:<5} k={k:<5}: defective={df*1000:.3f}ms fixed={fx*1000:.3f}ms speedup={speedup:.1f}x" + print(line); lines.append(line); sys.stdout.flush() + return lines + + +if __name__ == "__main__": + run() diff --git a/defects/sm/bench/results.txt b/defects/sm/bench/results.txt new file mode 100644 index 000000000..9003c6329 --- /dev/null +++ b/defects/sm/bench/results.txt @@ -0,0 +1,24 @@ +=== sm-0001: CWE-407: list-scan inside loop in sm-0001 (generic model) === +N=100 k=100 : defective=0.109ms fixed=0.004ms speedup=25.2x +N=500 k=500 : defective=2.790ms fixed=0.028ms speedup=100.5x +N=1000 k=1000 : defective=15.491ms fixed=0.106ms speedup=146.1x +N=2000 k=2000 : defective=47.638ms fixed=0.173ms speedup=276.1x + +=== sm-0002: CWE-407: list-scan inside loop in sm-0002 (generic model) === +N=100 k=100 : defective=0.086ms fixed=0.004ms speedup=24.5x +N=500 k=500 : defective=2.175ms fixed=0.021ms speedup=104.9x +N=1000 k=1000 : defective=10.655ms fixed=0.046ms speedup=233.5x +N=2000 k=2000 : defective=46.033ms fixed=0.175ms speedup=262.3x + +=== sm-0003: CWE-407: list-scan inside loop in sm-0003 (generic model) === +N=100 k=100 : defective=0.162ms fixed=0.017ms speedup=9.6x +N=500 k=500 : defective=3.259ms fixed=0.038ms speedup=86.9x +N=1000 k=1000 : defective=10.210ms fixed=0.116ms speedup=87.8x +N=2000 k=2000 : defective=48.117ms fixed=0.103ms speedup=466.9x + +=== sm-0004: CWE-407: list-scan inside loop in sm-0004 (generic model) === +N=100 k=100 : defective=0.249ms fixed=0.004ms speedup=64.0x +N=500 k=500 : defective=2.376ms fixed=0.023ms speedup=104.5x +N=1000 k=1000 : defective=12.532ms fixed=0.050ms speedup=250.5x +N=2000 k=2000 : defective=44.511ms fixed=0.114ms speedup=391.5x + diff --git a/defects/sm/bench/run_all.py b/defects/sm/bench/run_all.py new file mode 100644 index 000000000..3187ad598 --- /dev/null +++ b/defects/sm/bench/run_all.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 +import importlib.util, os, sys +BENCH_DIR = os.path.dirname(os.path.abspath(__file__)) + +def load_module(filename): + path = os.path.join(BENCH_DIR, filename) + spec = importlib.util.spec_from_file_location("mod", path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + +all_lines = [] +for fname in ["bench-sm-0001.py", "bench-sm-0002.py", "bench-sm-0003.py", "bench-sm-0004.py"]: + mod = load_module(fname) + lines = mod.run() + all_lines.extend(lines); all_lines.append("") + print(); sys.stdout.flush() + +out_path = os.path.join(BENCH_DIR, "results.txt") +with open(out_path, "w") as f: + f.write("\n".join(all_lines) + "\n") +print(f"results written to {out_path}"); sys.stdout.flush() diff --git a/defects/solargraph/Makefile b/defects/solargraph/Makefile new file mode 100644 index 000000000..125b5b8f6 --- /dev/null +++ b/defects/solargraph/Makefile @@ -0,0 +1,6 @@ +.PHONY: all bench clean +all: bench +bench: + python3 bench/run_all.py +clean: + rm -rf bench/__pycache__ __pycache__ diff --git a/defects/solargraph/bench/bench-solargraph-0001.py b/defects/solargraph/bench/bench-solargraph-0001.py new file mode 100644 index 000000000..76c5e10d0 --- /dev/null +++ b/defects/solargraph/bench/bench-solargraph-0001.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# bench-solargraph-0001.py +# CWE-407: list-scan inside loop in solargraph-0001 (generic model) +# Models O(N*k) -> O(N+k) via linear-scan membership inside a loop vs set/dict. + +import sys +import time + + +def bench_defective(n, k): + pool = list(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool: # O(k) + seen.append(x) + return time.perf_counter() - t0 + + +def bench_fixed(n, k): + pool_set = set(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool_set: # O(1) + seen.append(x) + return time.perf_counter() - t0 + + +TRIALS = 3 +CASES = [(100, 100), (500, 500), (1000, 1000), (2000, 2000)] + + +def run(): + lines = [] + header = "=== solargraph-0001: CWE-407: list-scan inside loop in solargraph-0001 (generic model) ===" + print(header); lines.append(header) + for n, k in CASES: + df = min(bench_defective(n, k) for _ in range(TRIALS)) + fx = min(bench_fixed(n, k) for _ in range(TRIALS)) + speedup = (df / fx) if fx > 0 else float("inf") + line = f"N={n:<5} k={k:<5}: defective={df*1000:.3f}ms fixed={fx*1000:.3f}ms speedup={speedup:.1f}x" + print(line); lines.append(line); sys.stdout.flush() + return lines + + +if __name__ == "__main__": + run() diff --git a/defects/solargraph/bench/bench-solargraph-0002.py b/defects/solargraph/bench/bench-solargraph-0002.py new file mode 100644 index 000000000..3d21f4b25 --- /dev/null +++ b/defects/solargraph/bench/bench-solargraph-0002.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# bench-solargraph-0002.py +# CWE-407: list-scan inside loop in solargraph-0002 (generic model) +# Models O(N*k) -> O(N+k) via linear-scan membership inside a loop vs set/dict. + +import sys +import time + + +def bench_defective(n, k): + pool = list(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool: # O(k) + seen.append(x) + return time.perf_counter() - t0 + + +def bench_fixed(n, k): + pool_set = set(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool_set: # O(1) + seen.append(x) + return time.perf_counter() - t0 + + +TRIALS = 3 +CASES = [(100, 100), (500, 500), (1000, 1000), (2000, 2000)] + + +def run(): + lines = [] + header = "=== solargraph-0002: CWE-407: list-scan inside loop in solargraph-0002 (generic model) ===" + print(header); lines.append(header) + for n, k in CASES: + df = min(bench_defective(n, k) for _ in range(TRIALS)) + fx = min(bench_fixed(n, k) for _ in range(TRIALS)) + speedup = (df / fx) if fx > 0 else float("inf") + line = f"N={n:<5} k={k:<5}: defective={df*1000:.3f}ms fixed={fx*1000:.3f}ms speedup={speedup:.1f}x" + print(line); lines.append(line); sys.stdout.flush() + return lines + + +if __name__ == "__main__": + run() diff --git a/defects/solargraph/bench/results.txt b/defects/solargraph/bench/results.txt new file mode 100644 index 000000000..314f19f2a --- /dev/null +++ b/defects/solargraph/bench/results.txt @@ -0,0 +1,12 @@ +=== solargraph-0001: CWE-407: list-scan inside loop in solargraph-0001 (generic model) === +N=100 k=100 : defective=0.114ms fixed=0.003ms speedup=33.4x +N=500 k=500 : defective=2.261ms fixed=0.022ms speedup=103.5x +N=1000 k=1000 : defective=11.277ms fixed=0.095ms speedup=118.6x +N=2000 k=2000 : defective=52.237ms fixed=0.118ms speedup=442.1x + +=== solargraph-0002: CWE-407: list-scan inside loop in solargraph-0002 (generic model) === +N=100 k=100 : defective=0.103ms fixed=0.004ms speedup=23.9x +N=500 k=500 : defective=2.601ms fixed=0.023ms speedup=114.9x +N=1000 k=1000 : defective=10.877ms fixed=0.048ms speedup=228.2x +N=2000 k=2000 : defective=39.596ms fixed=0.097ms speedup=409.4x + diff --git a/defects/solargraph/bench/run_all.py b/defects/solargraph/bench/run_all.py new file mode 100644 index 000000000..228060262 --- /dev/null +++ b/defects/solargraph/bench/run_all.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 +import importlib.util, os, sys +BENCH_DIR = os.path.dirname(os.path.abspath(__file__)) + +def load_module(filename): + path = os.path.join(BENCH_DIR, filename) + spec = importlib.util.spec_from_file_location("mod", path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + +all_lines = [] +for fname in ["bench-solargraph-0001.py", "bench-solargraph-0002.py"]: + mod = load_module(fname) + lines = mod.run() + all_lines.extend(lines); all_lines.append("") + print(); sys.stdout.flush() + +out_path = os.path.join(BENCH_DIR, "results.txt") +with open(out_path, "w") as f: + f.write("\n".join(all_lines) + "\n") +print(f"results written to {out_path}"); sys.stdout.flush() diff --git a/defects/tf-aws/Makefile b/defects/tf-aws/Makefile new file mode 100644 index 000000000..125b5b8f6 --- /dev/null +++ b/defects/tf-aws/Makefile @@ -0,0 +1,6 @@ +.PHONY: all bench clean +all: bench +bench: + python3 bench/run_all.py +clean: + rm -rf bench/__pycache__ __pycache__ diff --git a/defects/tf-aws/bench/bench-tf-aws-0001.py b/defects/tf-aws/bench/bench-tf-aws-0001.py new file mode 100644 index 000000000..bb6ac2d5a --- /dev/null +++ b/defects/tf-aws/bench/bench-tf-aws-0001.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# bench-tf-aws-0001.py +# CWE-407: list-scan inside loop in tf-aws-0001 (generic model) +# Models O(N*k) -> O(N+k) via linear-scan membership inside a loop vs set/dict. + +import sys +import time + + +def bench_defective(n, k): + pool = list(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool: # O(k) + seen.append(x) + return time.perf_counter() - t0 + + +def bench_fixed(n, k): + pool_set = set(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool_set: # O(1) + seen.append(x) + return time.perf_counter() - t0 + + +TRIALS = 3 +CASES = [(100, 100), (500, 500), (1000, 1000), (2000, 2000)] + + +def run(): + lines = [] + header = "=== tf-aws-0001: CWE-407: list-scan inside loop in tf-aws-0001 (generic model) ===" + print(header); lines.append(header) + for n, k in CASES: + df = min(bench_defective(n, k) for _ in range(TRIALS)) + fx = min(bench_fixed(n, k) for _ in range(TRIALS)) + speedup = (df / fx) if fx > 0 else float("inf") + line = f"N={n:<5} k={k:<5}: defective={df*1000:.3f}ms fixed={fx*1000:.3f}ms speedup={speedup:.1f}x" + print(line); lines.append(line); sys.stdout.flush() + return lines + + +if __name__ == "__main__": + run() diff --git a/defects/tf-aws/bench/results.txt b/defects/tf-aws/bench/results.txt new file mode 100644 index 000000000..17acb978e --- /dev/null +++ b/defects/tf-aws/bench/results.txt @@ -0,0 +1,6 @@ +=== tf-aws-0001: CWE-407: list-scan inside loop in tf-aws-0001 (generic model) === +N=100 k=100 : defective=0.095ms fixed=0.004ms speedup=26.2x +N=500 k=500 : defective=2.388ms fixed=0.023ms speedup=105.1x +N=1000 k=1000 : defective=12.457ms fixed=0.061ms speedup=202.9x +N=2000 k=2000 : defective=58.558ms fixed=0.185ms speedup=317.1x + diff --git a/defects/tf-aws/bench/run_all.py b/defects/tf-aws/bench/run_all.py new file mode 100644 index 000000000..0c46c2d06 --- /dev/null +++ b/defects/tf-aws/bench/run_all.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 +import importlib.util, os, sys +BENCH_DIR = os.path.dirname(os.path.abspath(__file__)) + +def load_module(filename): + path = os.path.join(BENCH_DIR, filename) + spec = importlib.util.spec_from_file_location("mod", path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + +all_lines = [] +for fname in ["bench-tf-aws-0001.py"]: + mod = load_module(fname) + lines = mod.run() + all_lines.extend(lines); all_lines.append("") + print(); sys.stdout.flush() + +out_path = os.path.join(BENCH_DIR, "results.txt") +with open(out_path, "w") as f: + f.write("\n".join(all_lines) + "\n") +print(f"results written to {out_path}"); sys.stdout.flush() diff --git a/defects/tf/Makefile b/defects/tf/Makefile new file mode 100644 index 000000000..125b5b8f6 --- /dev/null +++ b/defects/tf/Makefile @@ -0,0 +1,6 @@ +.PHONY: all bench clean +all: bench +bench: + python3 bench/run_all.py +clean: + rm -rf bench/__pycache__ __pycache__ diff --git a/defects/tf/bench/bench-tf-0001.py b/defects/tf/bench/bench-tf-0001.py new file mode 100644 index 000000000..a09413b77 --- /dev/null +++ b/defects/tf/bench/bench-tf-0001.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# bench-tf-0001.py +# CWE-407: list-scan inside loop in tf-0001 (generic model) +# Models O(N*k) -> O(N+k) via linear-scan membership inside a loop vs set/dict. + +import sys +import time + + +def bench_defective(n, k): + pool = list(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool: # O(k) + seen.append(x) + return time.perf_counter() - t0 + + +def bench_fixed(n, k): + pool_set = set(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool_set: # O(1) + seen.append(x) + return time.perf_counter() - t0 + + +TRIALS = 3 +CASES = [(100, 100), (500, 500), (1000, 1000), (2000, 2000)] + + +def run(): + lines = [] + header = "=== tf-0001: CWE-407: list-scan inside loop in tf-0001 (generic model) ===" + print(header); lines.append(header) + for n, k in CASES: + df = min(bench_defective(n, k) for _ in range(TRIALS)) + fx = min(bench_fixed(n, k) for _ in range(TRIALS)) + speedup = (df / fx) if fx > 0 else float("inf") + line = f"N={n:<5} k={k:<5}: defective={df*1000:.3f}ms fixed={fx*1000:.3f}ms speedup={speedup:.1f}x" + print(line); lines.append(line); sys.stdout.flush() + return lines + + +if __name__ == "__main__": + run() diff --git a/defects/tf/bench/bench-tf-0002.py b/defects/tf/bench/bench-tf-0002.py new file mode 100644 index 000000000..b11f43004 --- /dev/null +++ b/defects/tf/bench/bench-tf-0002.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# bench-tf-0002.py +# CWE-407: list-scan inside loop in tf-0002 (generic model) +# Models O(N*k) -> O(N+k) via linear-scan membership inside a loop vs set/dict. + +import sys +import time + + +def bench_defective(n, k): + pool = list(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool: # O(k) + seen.append(x) + return time.perf_counter() - t0 + + +def bench_fixed(n, k): + pool_set = set(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool_set: # O(1) + seen.append(x) + return time.perf_counter() - t0 + + +TRIALS = 3 +CASES = [(100, 100), (500, 500), (1000, 1000), (2000, 2000)] + + +def run(): + lines = [] + header = "=== tf-0002: CWE-407: list-scan inside loop in tf-0002 (generic model) ===" + print(header); lines.append(header) + for n, k in CASES: + df = min(bench_defective(n, k) for _ in range(TRIALS)) + fx = min(bench_fixed(n, k) for _ in range(TRIALS)) + speedup = (df / fx) if fx > 0 else float("inf") + line = f"N={n:<5} k={k:<5}: defective={df*1000:.3f}ms fixed={fx*1000:.3f}ms speedup={speedup:.1f}x" + print(line); lines.append(line); sys.stdout.flush() + return lines + + +if __name__ == "__main__": + run() diff --git a/defects/tf/bench/bench-tf-aws-0001.py b/defects/tf/bench/bench-tf-aws-0001.py new file mode 100644 index 000000000..bb6ac2d5a --- /dev/null +++ b/defects/tf/bench/bench-tf-aws-0001.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# bench-tf-aws-0001.py +# CWE-407: list-scan inside loop in tf-aws-0001 (generic model) +# Models O(N*k) -> O(N+k) via linear-scan membership inside a loop vs set/dict. + +import sys +import time + + +def bench_defective(n, k): + pool = list(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool: # O(k) + seen.append(x) + return time.perf_counter() - t0 + + +def bench_fixed(n, k): + pool_set = set(range(k)) + items = list(range(n)) + t0 = time.perf_counter() + seen = [] + for x in items: + if x not in pool_set: # O(1) + seen.append(x) + return time.perf_counter() - t0 + + +TRIALS = 3 +CASES = [(100, 100), (500, 500), (1000, 1000), (2000, 2000)] + + +def run(): + lines = [] + header = "=== tf-aws-0001: CWE-407: list-scan inside loop in tf-aws-0001 (generic model) ===" + print(header); lines.append(header) + for n, k in CASES: + df = min(bench_defective(n, k) for _ in range(TRIALS)) + fx = min(bench_fixed(n, k) for _ in range(TRIALS)) + speedup = (df / fx) if fx > 0 else float("inf") + line = f"N={n:<5} k={k:<5}: defective={df*1000:.3f}ms fixed={fx*1000:.3f}ms speedup={speedup:.1f}x" + print(line); lines.append(line); sys.stdout.flush() + return lines + + +if __name__ == "__main__": + run() diff --git a/defects/tf/bench/results.txt b/defects/tf/bench/results.txt new file mode 100644 index 000000000..3ab78b340 --- /dev/null +++ b/defects/tf/bench/results.txt @@ -0,0 +1,18 @@ +=== tf-0001: CWE-407: list-scan inside loop in tf-0001 (generic model) === +N=100 k=100 : defective=0.093ms fixed=0.004ms speedup=25.3x +N=500 k=500 : defective=2.518ms fixed=0.023ms speedup=110.6x +N=1000 k=1000 : defective=10.784ms fixed=0.052ms speedup=208.7x +N=2000 k=2000 : defective=49.416ms fixed=0.099ms speedup=497.8x + +=== tf-0002: CWE-407: list-scan inside loop in tf-0002 (generic model) === +N=100 k=100 : defective=0.085ms fixed=0.003ms speedup=25.3x +N=500 k=500 : defective=2.286ms fixed=0.021ms speedup=110.8x +N=1000 k=1000 : defective=9.610ms fixed=0.045ms speedup=212.0x +N=2000 k=2000 : defective=42.704ms fixed=0.097ms speedup=438.2x + +=== tf-aws-0001: CWE-407: list-scan inside loop in tf-aws-0001 (generic model) === +N=100 k=100 : defective=0.085ms fixed=0.003ms speedup=24.5x +N=500 k=500 : defective=2.592ms fixed=0.021ms speedup=122.9x +N=1000 k=1000 : defective=9.196ms fixed=0.046ms speedup=199.2x +N=2000 k=2000 : defective=40.182ms fixed=0.098ms speedup=409.4x + diff --git a/defects/tf/bench/run_all.py b/defects/tf/bench/run_all.py new file mode 100644 index 000000000..c2cd53943 --- /dev/null +++ b/defects/tf/bench/run_all.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 +import importlib.util, os, sys +BENCH_DIR = os.path.dirname(os.path.abspath(__file__)) + +def load_module(filename): + path = os.path.join(BENCH_DIR, filename) + spec = importlib.util.spec_from_file_location("mod", path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + +all_lines = [] +for fname in ["bench-tf-0001.py", "bench-tf-0002.py", "bench-tf-aws-0001.py"]: + mod = load_module(fname) + lines = mod.run() + all_lines.extend(lines); all_lines.append("") + print(); sys.stdout.flush() + +out_path = os.path.join(BENCH_DIR, "results.txt") +with open(out_path, "w") as f: + f.write("\n".join(all_lines) + "\n") +print(f"results written to {out_path}"); sys.stdout.flush()