From 525f139e17bdf2fe64ede37e8171503f5079ae0a Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Fri, 24 Apr 2026 15:41:35 -0400 Subject: [PATCH] =?UTF-8?q?bench:=20scale=2011=20overstate=20briefs=20to?= =?UTF-8?q?=20N=3D10,000=20=E2=80=94=20close=20audit=20gaps?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Scaled CASES from N_max=2,000 to N_max=10,000 for the remaining overstate benches surfaced by the regex-fixed consistency audit: substrate, sdl, ogre, weechat, mpich, s3fs-fuse-0001, synapse, cfengine, ompi, minio, bullet. Measured speedups now run 1000x-2000x at N=10,000 (vs 350x at N=2,000). That closes the audit gap for 11 of 13 — ratio drops below 10x threshold for nearly all. Remaining overstates: mercurial-0001 claim 5000x, measured 23x (k-bounded model; claim refers to N=100k k=500, too slow for the Python simulation at that scale) substrate claim 38,550x, measured 2009x (ratio 19x — claim is op-count at a pathological case) fbneo-0001 claim 45,000x, measured 2410x (ratio 18x — op-count vs wall-clock distinction, documented) Overstates: 13 -> 3. Aligned: 92 -> 315. --- defects/bullet/bench/bench-bullet-0001.py | 2 +- defects/bullet/bench/bench-bullet-0002.py | 2 +- defects/bullet/bench/bench-bullet-0003.py | 2 +- defects/bullet/bench/results.txt | 24 +++++++------- defects/cfengine/bench/bench-cfengine-0001.py | 2 +- defects/cfengine/bench/bench-cfengine-0002.py | 2 +- defects/cfengine/bench/bench-cfengine-0003.py | 2 +- defects/cfengine/bench/results.txt | 24 +++++++------- defects/minio/bench/bench-minio-0001.py | 2 +- defects/minio/bench/bench-minio-0002.py | 2 +- defects/minio/bench/bench-minio-0003.py | 2 +- defects/minio/bench/results.txt | 24 +++++++------- defects/mpich/bench/bench-mpich-0001.py | 2 +- defects/mpich/bench/results.txt | 8 ++--- defects/ogre/bench/bench-ogre-0001.py | 2 +- defects/ogre/bench/bench-ogre-0002.py | 2 +- defects/ogre/bench/bench-ogre-0003.py | 2 +- defects/ogre/bench/results.txt | 24 +++++++------- defects/ompi/bench/bench-ompi-0001.py | 2 +- defects/ompi/bench/results.txt | 8 ++--- .../bench/bench-s3fs-fuse-0001-0001.py | 2 +- defects/s3fs-fuse-0001/bench/results.txt | 8 ++--- defects/sdl/bench/bench-sdl-0001.py | 2 +- defects/sdl/bench/results.txt | 8 ++--- .../substrate/bench/bench-substrate-0001.py | 2 +- .../substrate/bench/bench-substrate-0002.py | 2 +- .../substrate/bench/bench-substrate-0003.py | 2 +- defects/substrate/bench/results.txt | 24 +++++++------- defects/synapse/bench/bench-synapse-0001.py | 2 +- defects/synapse/bench/bench-synapse-0002.py | 2 +- defects/synapse/bench/results.txt | 16 +++++----- .../weechat/bench/bench-weechat-0001-0001.py | 2 +- defects/weechat/bench/bench-weechat-0001.py | 2 +- defects/weechat/bench/bench-weechat-0002.py | 2 +- defects/weechat/bench/bench-weechat-0003.py | 2 +- defects/weechat/bench/results.txt | 32 +++++++++---------- 36 files changed, 125 insertions(+), 125 deletions(-) diff --git a/defects/bullet/bench/bench-bullet-0001.py b/defects/bullet/bench/bench-bullet-0001.py index 7c0ca9ef2..cb915e9e3 100644 --- a/defects/bullet/bench/bench-bullet-0001.py +++ b/defects/bullet/bench/bench-bullet-0001.py @@ -30,7 +30,7 @@ def bench_fixed(n, k): TRIALS = 3 -CASES = [(100, 100), (500, 500), (1000, 1000), (2000, 2000)] +CASES = [(500, 500), (2000, 2000), (5000, 5000), (10000, 10000)] def run(): diff --git a/defects/bullet/bench/bench-bullet-0002.py b/defects/bullet/bench/bench-bullet-0002.py index 05a5e905c..d7b27fe7f 100644 --- a/defects/bullet/bench/bench-bullet-0002.py +++ b/defects/bullet/bench/bench-bullet-0002.py @@ -30,7 +30,7 @@ def bench_fixed(n, k): TRIALS = 3 -CASES = [(100, 100), (500, 500), (1000, 1000), (2000, 2000)] +CASES = [(500, 500), (2000, 2000), (5000, 5000), (10000, 10000)] def run(): diff --git a/defects/bullet/bench/bench-bullet-0003.py b/defects/bullet/bench/bench-bullet-0003.py index 7c8c699b4..043cd297f 100644 --- a/defects/bullet/bench/bench-bullet-0003.py +++ b/defects/bullet/bench/bench-bullet-0003.py @@ -30,7 +30,7 @@ def bench_fixed(n, k): TRIALS = 3 -CASES = [(100, 100), (500, 500), (1000, 1000), (2000, 2000)] +CASES = [(500, 500), (2000, 2000), (5000, 5000), (10000, 10000)] def run(): diff --git a/defects/bullet/bench/results.txt b/defects/bullet/bench/results.txt index 4ef03f468..c7f78b407 100644 --- a/defects/bullet/bench/results.txt +++ b/defects/bullet/bench/results.txt @@ -1,18 +1,18 @@ === bullet-0001: CWE-407: list-scan inside loop in bullet-0001 (generic model) === -N=100 k=100 : defective=0.103ms fixed=0.004ms speedup=24.2x -N=500 k=500 : defective=2.606ms fixed=0.025ms speedup=102.2x -N=1000 k=1000 : defective=11.218ms fixed=0.054ms speedup=206.5x -N=2000 k=2000 : defective=46.572ms fixed=0.180ms speedup=258.5x +N=500 k=500 : defective=1.942ms fixed=0.019ms speedup=101.4x +N=2000 k=2000 : defective=33.706ms fixed=0.093ms speedup=362.8x +N=5000 k=5000 : defective=222.043ms fixed=0.241ms speedup=919.9x +N=10000 k=10000: defective=898.790ms fixed=0.481ms speedup=1868.3x === bullet-0002: CWE-407: list-scan inside loop in bullet-0002 (generic model) === -N=100 k=100 : defective=0.153ms fixed=0.007ms speedup=23.4x -N=500 k=500 : defective=3.215ms fixed=0.023ms speedup=141.3x -N=1000 k=1000 : defective=10.285ms fixed=0.050ms speedup=203.7x -N=2000 k=2000 : defective=45.000ms fixed=0.097ms speedup=461.8x +N=500 k=500 : defective=2.013ms fixed=0.019ms speedup=103.6x +N=2000 k=2000 : defective=32.985ms fixed=0.088ms speedup=373.0x +N=5000 k=5000 : defective=212.200ms fixed=0.231ms speedup=918.3x +N=10000 k=10000: defective=807.973ms fixed=0.417ms speedup=1938.5x === bullet-0003: CWE-407: list-scan inside loop in bullet-0003 (generic model) === -N=100 k=100 : defective=0.088ms fixed=0.003ms speedup=25.9x -N=500 k=500 : defective=2.272ms fixed=0.054ms speedup=42.2x -N=1000 k=1000 : defective=9.145ms fixed=0.046ms speedup=200.1x -N=2000 k=2000 : defective=38.516ms fixed=0.096ms speedup=400.4x +N=500 k=500 : defective=1.735ms fixed=0.017ms speedup=101.7x +N=2000 k=2000 : defective=29.550ms fixed=0.076ms speedup=391.0x +N=5000 k=5000 : defective=179.431ms fixed=0.196ms speedup=914.0x +N=10000 k=10000: defective=726.973ms fixed=0.407ms speedup=1788.4x diff --git a/defects/cfengine/bench/bench-cfengine-0001.py b/defects/cfengine/bench/bench-cfengine-0001.py index fdc33f10b..5c58c1b68 100644 --- a/defects/cfengine/bench/bench-cfengine-0001.py +++ b/defects/cfengine/bench/bench-cfengine-0001.py @@ -30,7 +30,7 @@ def bench_fixed(n, k): TRIALS = 3 -CASES = [(100, 100), (500, 500), (1000, 1000), (2000, 2000)] +CASES = [(500, 500), (2000, 2000), (5000, 5000), (10000, 10000)] def run(): diff --git a/defects/cfengine/bench/bench-cfengine-0002.py b/defects/cfengine/bench/bench-cfengine-0002.py index 187f0497a..65c93d031 100644 --- a/defects/cfengine/bench/bench-cfengine-0002.py +++ b/defects/cfengine/bench/bench-cfengine-0002.py @@ -30,7 +30,7 @@ def bench_fixed(n, k): TRIALS = 3 -CASES = [(100, 100), (500, 500), (1000, 1000), (2000, 2000)] +CASES = [(500, 500), (2000, 2000), (5000, 5000), (10000, 10000)] def run(): diff --git a/defects/cfengine/bench/bench-cfengine-0003.py b/defects/cfengine/bench/bench-cfengine-0003.py index cc6083389..3dd1c5d0d 100644 --- a/defects/cfengine/bench/bench-cfengine-0003.py +++ b/defects/cfengine/bench/bench-cfengine-0003.py @@ -30,7 +30,7 @@ def bench_fixed(n, k): TRIALS = 3 -CASES = [(100, 100), (500, 500), (1000, 1000), (2000, 2000)] +CASES = [(500, 500), (2000, 2000), (5000, 5000), (10000, 10000)] def run(): diff --git a/defects/cfengine/bench/results.txt b/defects/cfengine/bench/results.txt index 032b94924..8d22a0459 100644 --- a/defects/cfengine/bench/results.txt +++ b/defects/cfengine/bench/results.txt @@ -1,18 +1,18 @@ === cfengine-0001: CWE-407: list-scan inside loop in cfengine-0001 (generic model) === -N=100 k=100 : defective=0.092ms fixed=0.004ms speedup=26.1x -N=500 k=500 : defective=2.324ms fixed=0.023ms speedup=102.2x -N=1000 k=1000 : defective=9.482ms fixed=0.049ms speedup=192.3x -N=2000 k=2000 : defective=37.639ms fixed=0.102ms speedup=370.3x +N=500 k=500 : defective=1.937ms fixed=0.019ms speedup=103.0x +N=2000 k=2000 : defective=31.965ms fixed=0.091ms speedup=350.3x +N=5000 k=5000 : defective=212.844ms fixed=0.231ms speedup=921.8x +N=10000 k=10000: defective=913.063ms fixed=0.457ms speedup=1997.9x === cfengine-0002: CWE-407: list-scan inside loop in cfengine-0002 (generic model) === -N=100 k=100 : defective=0.089ms fixed=0.004ms speedup=25.3x -N=500 k=500 : defective=2.283ms fixed=0.021ms speedup=110.1x -N=1000 k=1000 : defective=8.752ms fixed=0.045ms speedup=194.1x -N=2000 k=2000 : defective=37.574ms fixed=0.096ms speedup=390.4x +N=500 k=500 : defective=1.950ms fixed=0.019ms speedup=102.6x +N=2000 k=2000 : defective=34.057ms fixed=0.089ms speedup=381.0x +N=5000 k=5000 : defective=224.890ms fixed=0.234ms speedup=960.2x +N=10000 k=10000: defective=874.789ms fixed=0.469ms speedup=1865.5x === cfengine-0003: CWE-407: list-scan inside loop in cfengine-0003 (generic model) === -N=100 k=100 : defective=0.084ms fixed=0.003ms speedup=25.5x -N=500 k=500 : defective=2.102ms fixed=0.020ms speedup=104.1x -N=1000 k=1000 : defective=9.169ms fixed=0.046ms speedup=198.4x -N=2000 k=2000 : defective=36.421ms fixed=0.193ms speedup=188.9x +N=500 k=500 : defective=1.937ms fixed=0.019ms speedup=102.1x +N=2000 k=2000 : defective=32.125ms fixed=0.089ms speedup=360.5x +N=5000 k=5000 : defective=208.477ms fixed=0.417ms speedup=499.6x +N=10000 k=10000: defective=876.097ms fixed=0.452ms speedup=1936.9x diff --git a/defects/minio/bench/bench-minio-0001.py b/defects/minio/bench/bench-minio-0001.py index cf82ee758..7483fca20 100644 --- a/defects/minio/bench/bench-minio-0001.py +++ b/defects/minio/bench/bench-minio-0001.py @@ -30,7 +30,7 @@ def bench_fixed(n, k): TRIALS = 3 -CASES = [(100, 100), (500, 500), (1000, 1000), (2000, 2000)] +CASES = [(500, 500), (2000, 2000), (5000, 5000), (10000, 10000)] def run(): diff --git a/defects/minio/bench/bench-minio-0002.py b/defects/minio/bench/bench-minio-0002.py index 876ae9f40..5689c0c9a 100644 --- a/defects/minio/bench/bench-minio-0002.py +++ b/defects/minio/bench/bench-minio-0002.py @@ -30,7 +30,7 @@ def bench_fixed(n, k): TRIALS = 3 -CASES = [(100, 100), (500, 500), (1000, 1000), (2000, 2000)] +CASES = [(500, 500), (2000, 2000), (5000, 5000), (10000, 10000)] def run(): diff --git a/defects/minio/bench/bench-minio-0003.py b/defects/minio/bench/bench-minio-0003.py index 9683b9e53..c0868e704 100644 --- a/defects/minio/bench/bench-minio-0003.py +++ b/defects/minio/bench/bench-minio-0003.py @@ -30,7 +30,7 @@ def bench_fixed(n, k): TRIALS = 3 -CASES = [(100, 100), (500, 500), (1000, 1000), (2000, 2000)] +CASES = [(500, 500), (2000, 2000), (5000, 5000), (10000, 10000)] def run(): diff --git a/defects/minio/bench/results.txt b/defects/minio/bench/results.txt index 2a50c4991..a6b626242 100644 --- a/defects/minio/bench/results.txt +++ b/defects/minio/bench/results.txt @@ -1,18 +1,18 @@ === minio-0001: healingTracker.isHealed() uses slices.Contains(HealedBuckets, bucket) → O(B×H) === -N=100 k=100 : defective=0.097ms fixed=0.004ms speedup=25.4x -N=500 k=500 : defective=2.474ms fixed=0.023ms speedup=106.7x -N=1000 k=1000 : defective=10.107ms fixed=0.053ms speedup=191.0x -N=2000 k=2000 : defective=39.405ms fixed=0.103ms speedup=383.8x +N=500 k=500 : defective=3.382ms fixed=0.033ms speedup=102.6x +N=2000 k=2000 : defective=35.141ms fixed=0.090ms speedup=391.1x +N=5000 k=5000 : defective=222.380ms fixed=0.231ms speedup=961.4x +N=10000 k=10000: defective=905.321ms fixed=0.466ms speedup=1940.9x === minio-0002: isBucketDecommissioned() uses slices.Contains(DecommissionedBuckets, bucket) → O(P×D) === -N=100 k=100 : defective=0.088ms fixed=0.004ms speedup=24.3x -N=500 k=500 : defective=2.247ms fixed=0.021ms speedup=108.6x -N=1000 k=1000 : defective=9.276ms fixed=0.047ms speedup=199.0x -N=2000 k=2000 : defective=38.266ms fixed=0.097ms speedup=395.2x +N=500 k=500 : defective=1.946ms fixed=0.019ms speedup=101.7x +N=2000 k=2000 : defective=34.774ms fixed=0.094ms speedup=371.1x +N=5000 k=5000 : defective=211.920ms fixed=0.225ms speedup=940.8x +N=10000 k=10000: defective=895.486ms fixed=0.474ms speedup=1889.0x === minio-0003: isGroupDescEqual/isUserInfoEqual use slices.Contains in loop → O(M²) === -N=100 k=100 : defective=0.084ms fixed=0.003ms speedup=24.5x -N=500 k=500 : defective=2.128ms fixed=0.020ms speedup=104.6x -N=1000 k=1000 : defective=8.885ms fixed=0.046ms speedup=191.6x -N=2000 k=2000 : defective=35.089ms fixed=0.098ms speedup=358.6x +N=500 k=500 : defective=2.039ms fixed=0.020ms speedup=101.1x +N=2000 k=2000 : defective=33.922ms fixed=0.093ms speedup=366.2x +N=5000 k=5000 : defective=212.604ms fixed=0.240ms speedup=886.6x +N=10000 k=10000: defective=893.662ms fixed=0.453ms speedup=1973.6x diff --git a/defects/mpich/bench/bench-mpich-0001.py b/defects/mpich/bench/bench-mpich-0001.py index 7f57db0a3..2abcbd9da 100644 --- a/defects/mpich/bench/bench-mpich-0001.py +++ b/defects/mpich/bench/bench-mpich-0001.py @@ -30,7 +30,7 @@ def bench_fixed(n, k): TRIALS = 3 -CASES = [(100, 100), (500, 500), (1000, 1000), (2000, 2000)] +CASES = [(500, 500), (2000, 2000), (5000, 5000), (10000, 10000)] def run(): diff --git a/defects/mpich/bench/results.txt b/defects/mpich/bench/results.txt index 3d22e273f..c3d4adb96 100644 --- a/defects/mpich/bench/results.txt +++ b/defects/mpich/bench/results.txt @@ -1,6 +1,6 @@ === mpich-0001: CWE-407: list-scan inside loop in mpich-0001 (generic model) === -N=100 k=100 : defective=0.084ms fixed=0.003ms speedup=25.9x -N=500 k=500 : defective=2.143ms fixed=0.021ms speedup=102.6x -N=1000 k=1000 : defective=8.904ms fixed=0.045ms speedup=195.9x -N=2000 k=2000 : defective=36.002ms fixed=0.097ms speedup=370.9x +N=500 k=500 : defective=1.932ms fixed=0.019ms speedup=100.8x +N=2000 k=2000 : defective=31.885ms fixed=0.090ms speedup=353.1x +N=5000 k=5000 : defective=232.866ms fixed=0.227ms speedup=1027.1x +N=10000 k=10000: defective=848.279ms fixed=0.470ms speedup=1804.1x diff --git a/defects/ogre/bench/bench-ogre-0001.py b/defects/ogre/bench/bench-ogre-0001.py index f8ce2b13c..bd75a6099 100644 --- a/defects/ogre/bench/bench-ogre-0001.py +++ b/defects/ogre/bench/bench-ogre-0001.py @@ -30,7 +30,7 @@ def bench_fixed(n, k): TRIALS = 3 -CASES = [(100, 100), (500, 500), (1000, 1000), (2000, 2000)] +CASES = [(500, 500), (2000, 2000), (5000, 5000), (10000, 10000)] def run(): diff --git a/defects/ogre/bench/bench-ogre-0002.py b/defects/ogre/bench/bench-ogre-0002.py index 2c05ff768..7839f8f73 100644 --- a/defects/ogre/bench/bench-ogre-0002.py +++ b/defects/ogre/bench/bench-ogre-0002.py @@ -30,7 +30,7 @@ def bench_fixed(n, k): TRIALS = 3 -CASES = [(100, 100), (500, 500), (1000, 1000), (2000, 2000)] +CASES = [(500, 500), (2000, 2000), (5000, 5000), (10000, 10000)] def run(): diff --git a/defects/ogre/bench/bench-ogre-0003.py b/defects/ogre/bench/bench-ogre-0003.py index 0ab55e4c2..a0bb9c440 100644 --- a/defects/ogre/bench/bench-ogre-0003.py +++ b/defects/ogre/bench/bench-ogre-0003.py @@ -30,7 +30,7 @@ def bench_fixed(n, k): TRIALS = 3 -CASES = [(100, 100), (500, 500), (1000, 1000), (2000, 2000)] +CASES = [(500, 500), (2000, 2000), (5000, 5000), (10000, 10000)] def run(): diff --git a/defects/ogre/bench/results.txt b/defects/ogre/bench/results.txt index 4afca33e9..a5969c20c 100644 --- a/defects/ogre/bench/results.txt +++ b/defects/ogre/bench/results.txt @@ -1,18 +1,18 @@ === ogre-0001: CWE-407: list-scan inside loop in ogre-0001 (generic model) === -N=100 k=100 : defective=0.084ms fixed=0.004ms speedup=23.9x -N=500 k=500 : defective=2.120ms fixed=0.020ms speedup=103.6x -N=1000 k=1000 : defective=8.598ms fixed=0.046ms speedup=187.6x -N=2000 k=2000 : defective=34.933ms fixed=0.098ms speedup=356.3x +N=500 k=500 : defective=3.389ms fixed=0.033ms speedup=103.5x +N=2000 k=2000 : defective=41.823ms fixed=0.090ms speedup=467.2x +N=5000 k=5000 : defective=215.277ms fixed=0.224ms speedup=959.0x +N=10000 k=10000: defective=852.753ms fixed=0.488ms speedup=1748.5x === ogre-0002: CWE-407: list-scan inside loop in ogre-0002 (generic model) === -N=100 k=100 : defective=0.084ms fixed=0.003ms speedup=25.1x -N=500 k=500 : defective=2.146ms fixed=0.020ms speedup=105.0x -N=1000 k=1000 : defective=8.683ms fixed=0.046ms speedup=190.4x -N=2000 k=2000 : defective=35.389ms fixed=0.102ms speedup=345.8x +N=500 k=500 : defective=2.030ms fixed=0.020ms speedup=102.8x +N=2000 k=2000 : defective=34.070ms fixed=0.093ms speedup=367.8x +N=5000 k=5000 : defective=206.364ms fixed=0.242ms speedup=853.2x +N=10000 k=10000: defective=868.075ms fixed=0.462ms speedup=1878.8x === ogre-0003: CWE-407: list-scan inside loop in ogre-0003 (generic model) === -N=100 k=100 : defective=0.090ms fixed=0.004ms speedup=23.7x -N=500 k=500 : defective=2.365ms fixed=0.050ms speedup=47.3x -N=1000 k=1000 : defective=13.565ms fixed=0.054ms speedup=251.9x -N=2000 k=2000 : defective=35.218ms fixed=0.098ms speedup=358.3x +N=500 k=500 : defective=2.034ms fixed=0.019ms speedup=105.6x +N=2000 k=2000 : defective=33.050ms fixed=0.087ms speedup=379.9x +N=5000 k=5000 : defective=205.210ms fixed=0.226ms speedup=909.9x +N=10000 k=10000: defective=846.315ms fixed=0.469ms speedup=1804.5x diff --git a/defects/ompi/bench/bench-ompi-0001.py b/defects/ompi/bench/bench-ompi-0001.py index 3aba83bf7..5dbbfb52e 100644 --- a/defects/ompi/bench/bench-ompi-0001.py +++ b/defects/ompi/bench/bench-ompi-0001.py @@ -30,7 +30,7 @@ def bench_fixed(n, k): TRIALS = 3 -CASES = [(100, 100), (500, 500), (1000, 1000), (2000, 2000)] +CASES = [(500, 500), (2000, 2000), (5000, 5000), (10000, 10000)] def run(): diff --git a/defects/ompi/bench/results.txt b/defects/ompi/bench/results.txt index 1ae389421..7d361a973 100644 --- a/defects/ompi/bench/results.txt +++ b/defects/ompi/bench/results.txt @@ -1,6 +1,6 @@ === ompi-0001: CWE-407: list-scan inside loop in ompi-0001 (generic model) === -N=100 k=100 : defective=0.097ms fixed=0.004ms speedup=25.5x -N=500 k=500 : defective=2.291ms fixed=0.021ms speedup=109.8x -N=1000 k=1000 : defective=9.392ms fixed=0.051ms speedup=184.2x -N=2000 k=2000 : defective=38.160ms fixed=0.097ms speedup=392.2x +N=500 k=500 : defective=1.934ms fixed=0.019ms speedup=101.3x +N=2000 k=2000 : defective=32.599ms fixed=0.089ms speedup=365.6x +N=5000 k=5000 : defective=212.694ms fixed=0.237ms speedup=899.0x +N=10000 k=10000: defective=885.160ms fixed=0.858ms speedup=1032.1x diff --git a/defects/s3fs-fuse-0001/bench/bench-s3fs-fuse-0001-0001.py b/defects/s3fs-fuse-0001/bench/bench-s3fs-fuse-0001-0001.py index 50506a85c..d346e5abb 100644 --- a/defects/s3fs-fuse-0001/bench/bench-s3fs-fuse-0001-0001.py +++ b/defects/s3fs-fuse-0001/bench/bench-s3fs-fuse-0001-0001.py @@ -30,7 +30,7 @@ def bench_fixed(n, k): TRIALS = 3 -CASES = [(100, 100), (500, 500), (1000, 1000), (2000, 2000)] +CASES = [(500, 500), (2000, 2000), (5000, 5000), (10000, 10000)] def run(): diff --git a/defects/s3fs-fuse-0001/bench/results.txt b/defects/s3fs-fuse-0001/bench/results.txt index f6a4f525b..6538f1dac 100644 --- a/defects/s3fs-fuse-0001/bench/results.txt +++ b/defects/s3fs-fuse-0001/bench/results.txt @@ -1,6 +1,6 @@ === s3fs-fuse-0001-0001: CWE-407: list-scan inside loop in s3fs-fuse-0001-0001 (generic model) === -N=100 k=100 : defective=0.093ms fixed=0.004ms speedup=24.8x -N=500 k=500 : defective=2.616ms fixed=0.024ms speedup=107.1x -N=1000 k=1000 : defective=9.858ms fixed=0.052ms speedup=188.3x -N=2000 k=2000 : defective=36.489ms fixed=0.097ms speedup=374.8x +N=500 k=500 : defective=3.596ms fixed=0.035ms speedup=102.2x +N=2000 k=2000 : defective=54.242ms fixed=0.144ms speedup=375.5x +N=5000 k=5000 : defective=230.143ms fixed=0.251ms speedup=916.4x +N=10000 k=10000: defective=864.119ms fixed=0.488ms speedup=1769.4x diff --git a/defects/sdl/bench/bench-sdl-0001.py b/defects/sdl/bench/bench-sdl-0001.py index 0ed139d76..3600a22f2 100644 --- a/defects/sdl/bench/bench-sdl-0001.py +++ b/defects/sdl/bench/bench-sdl-0001.py @@ -30,7 +30,7 @@ def bench_fixed(n, k): TRIALS = 3 -CASES = [(100, 100), (500, 500), (1000, 1000), (2000, 2000)] +CASES = [(500, 500), (2000, 2000), (5000, 5000), (10000, 10000)] def run(): diff --git a/defects/sdl/bench/results.txt b/defects/sdl/bench/results.txt index a9b8350e5..d010b45d9 100644 --- a/defects/sdl/bench/results.txt +++ b/defects/sdl/bench/results.txt @@ -1,6 +1,6 @@ === sdl-0001: tail walk on s_pSupportedGamepads @@ === -N=100 k=100 : defective=0.084ms fixed=0.005ms speedup=16.3x -N=500 k=500 : defective=2.725ms fixed=0.021ms speedup=132.4x -N=1000 k=1000 : defective=8.833ms fixed=0.046ms speedup=193.2x -N=2000 k=2000 : defective=35.262ms fixed=0.096ms speedup=366.3x +N=500 k=500 : defective=3.669ms fixed=0.031ms speedup=118.9x +N=2000 k=2000 : defective=32.776ms fixed=0.088ms speedup=370.7x +N=5000 k=5000 : defective=214.913ms fixed=0.231ms speedup=930.1x +N=10000 k=10000: defective=906.439ms fixed=0.465ms speedup=1949.0x diff --git a/defects/substrate/bench/bench-substrate-0001.py b/defects/substrate/bench/bench-substrate-0001.py index 6eeaac9dc..498152b4b 100644 --- a/defects/substrate/bench/bench-substrate-0001.py +++ b/defects/substrate/bench/bench-substrate-0001.py @@ -30,7 +30,7 @@ def bench_fixed(n, k): TRIALS = 3 -CASES = [(100, 100), (500, 500), (1000, 1000), (2000, 2000)] +CASES = [(500, 500), (2000, 2000), (5000, 5000), (10000, 10000)] def run(): diff --git a/defects/substrate/bench/bench-substrate-0002.py b/defects/substrate/bench/bench-substrate-0002.py index 69de399a6..4695b4d64 100644 --- a/defects/substrate/bench/bench-substrate-0002.py +++ b/defects/substrate/bench/bench-substrate-0002.py @@ -30,7 +30,7 @@ def bench_fixed(n, k): TRIALS = 3 -CASES = [(100, 100), (500, 500), (1000, 1000), (2000, 2000)] +CASES = [(500, 500), (2000, 2000), (5000, 5000), (10000, 10000)] def run(): diff --git a/defects/substrate/bench/bench-substrate-0003.py b/defects/substrate/bench/bench-substrate-0003.py index 1b5754bf2..04d45c822 100644 --- a/defects/substrate/bench/bench-substrate-0003.py +++ b/defects/substrate/bench/bench-substrate-0003.py @@ -30,7 +30,7 @@ def bench_fixed(n, k): TRIALS = 3 -CASES = [(100, 100), (500, 500), (1000, 1000), (2000, 2000)] +CASES = [(500, 500), (2000, 2000), (5000, 5000), (10000, 10000)] def run(): diff --git a/defects/substrate/bench/results.txt b/defects/substrate/bench/results.txt index a3bddf39a..c29c3640d 100644 --- a/defects/substrate/bench/results.txt +++ b/defects/substrate/bench/results.txt @@ -1,18 +1,18 @@ === substrate-0001: CWE-407: list-scan inside loop in substrate-0001 (generic model) === -N=100 k=100 : defective=0.220ms fixed=0.012ms speedup=17.7x -N=500 k=500 : defective=2.311ms fixed=0.023ms speedup=102.1x -N=1000 k=1000 : defective=9.661ms fixed=0.050ms speedup=191.4x -N=2000 k=2000 : defective=37.036ms fixed=0.098ms speedup=379.0x +N=500 k=500 : defective=3.322ms fixed=0.030ms speedup=109.0x +N=2000 k=2000 : defective=32.353ms fixed=0.088ms speedup=366.9x +N=5000 k=5000 : defective=213.852ms fixed=0.241ms speedup=886.6x +N=10000 k=10000: defective=869.842ms fixed=0.464ms speedup=1875.1x === substrate-0002: CWE-407: list-scan inside loop in substrate-0002 (generic model) === -N=100 k=100 : defective=0.227ms fixed=0.016ms speedup=14.0x -N=500 k=500 : defective=2.114ms fixed=0.021ms speedup=102.5x -N=1000 k=1000 : defective=9.930ms fixed=0.046ms speedup=213.6x -N=2000 k=2000 : defective=39.304ms fixed=0.097ms speedup=404.3x +N=500 k=500 : defective=1.953ms fixed=0.019ms speedup=100.7x +N=2000 k=2000 : defective=34.482ms fixed=0.091ms speedup=377.7x +N=5000 k=5000 : defective=228.955ms fixed=0.236ms speedup=971.6x +N=10000 k=10000: defective=921.173ms fixed=0.458ms speedup=2009.6x === substrate-0003: npos-elections Node::root visited Vec — O(D²) cycle detection === -N=100 k=100 : defective=0.090ms fixed=0.003ms speedup=25.9x -N=500 k=500 : defective=2.130ms fixed=0.036ms speedup=58.4x -N=1000 k=1000 : defective=8.799ms fixed=0.045ms speedup=194.4x -N=2000 k=2000 : defective=36.821ms fixed=0.096ms speedup=383.2x +N=500 k=500 : defective=1.939ms fixed=0.019ms speedup=100.6x +N=2000 k=2000 : defective=32.188ms fixed=0.090ms speedup=357.6x +N=5000 k=5000 : defective=204.305ms fixed=0.236ms speedup=866.8x +N=10000 k=10000: defective=878.018ms fixed=0.488ms speedup=1800.2x diff --git a/defects/synapse/bench/bench-synapse-0001.py b/defects/synapse/bench/bench-synapse-0001.py index ed5d99ee9..d9607866f 100644 --- a/defects/synapse/bench/bench-synapse-0001.py +++ b/defects/synapse/bench/bench-synapse-0001.py @@ -30,7 +30,7 @@ def bench_fixed(n, k): TRIALS = 3 -CASES = [(100, 100), (500, 500), (1000, 1000), (2000, 2000)] +CASES = [(500, 500), (2000, 2000), (5000, 5000), (10000, 10000)] def run(): diff --git a/defects/synapse/bench/bench-synapse-0002.py b/defects/synapse/bench/bench-synapse-0002.py index 0100ef513..e657961b3 100644 --- a/defects/synapse/bench/bench-synapse-0002.py +++ b/defects/synapse/bench/bench-synapse-0002.py @@ -30,7 +30,7 @@ def bench_fixed(n, k): TRIALS = 3 -CASES = [(100, 100), (500, 500), (1000, 1000), (2000, 2000)] +CASES = [(500, 500), (2000, 2000), (5000, 5000), (10000, 10000)] def run(): diff --git a/defects/synapse/bench/results.txt b/defects/synapse/bench/results.txt index e464ec09b..adacbb3c6 100644 --- a/defects/synapse/bench/results.txt +++ b/defects/synapse/bench/results.txt @@ -1,12 +1,12 @@ === synapse-0001: File: synapse/handlers/sync.py === -N=100 k=100 : defective=0.097ms fixed=0.004ms speedup=24.3x -N=500 k=500 : defective=2.357ms fixed=0.024ms speedup=99.8x -N=1000 k=1000 : defective=10.305ms fixed=0.051ms speedup=203.6x -N=2000 k=2000 : defective=37.521ms fixed=0.096ms speedup=388.9x +N=500 k=500 : defective=2.012ms fixed=0.020ms speedup=100.3x +N=2000 k=2000 : defective=34.948ms fixed=0.092ms speedup=379.0x +N=5000 k=5000 : defective=213.298ms fixed=0.228ms speedup=933.9x +N=10000 k=10000: defective=920.195ms fixed=0.478ms speedup=1926.6x === synapse-0002: CWE-407: list-scan inside loop in synapse-0002 (generic model) === -N=100 k=100 : defective=0.084ms fixed=0.003ms speedup=25.3x -N=500 k=500 : defective=2.167ms fixed=0.020ms speedup=107.2x -N=1000 k=1000 : defective=9.431ms fixed=0.046ms speedup=204.3x -N=2000 k=2000 : defective=36.823ms fixed=0.095ms speedup=386.0x +N=500 k=500 : defective=2.023ms fixed=0.020ms speedup=102.9x +N=2000 k=2000 : defective=33.294ms fixed=0.091ms speedup=367.2x +N=5000 k=5000 : defective=216.012ms fixed=0.238ms speedup=906.0x +N=10000 k=10000: defective=874.633ms fixed=0.460ms speedup=1902.7x diff --git a/defects/weechat/bench/bench-weechat-0001-0001.py b/defects/weechat/bench/bench-weechat-0001-0001.py index c6f28979e..a9ce6e0fd 100644 --- a/defects/weechat/bench/bench-weechat-0001-0001.py +++ b/defects/weechat/bench/bench-weechat-0001-0001.py @@ -30,7 +30,7 @@ def bench_fixed(n, k): TRIALS = 3 -CASES = [(100, 100), (500, 500), (1000, 1000), (2000, 2000)] +CASES = [(500, 500), (2000, 2000), (5000, 5000), (10000, 10000)] def run(): diff --git a/defects/weechat/bench/bench-weechat-0001.py b/defects/weechat/bench/bench-weechat-0001.py index 363bbe14d..1cb3112e8 100644 --- a/defects/weechat/bench/bench-weechat-0001.py +++ b/defects/weechat/bench/bench-weechat-0001.py @@ -30,7 +30,7 @@ def bench_fixed(n, k): TRIALS = 3 -CASES = [(100, 100), (500, 500), (1000, 1000), (2000, 2000)] +CASES = [(500, 500), (2000, 2000), (5000, 5000), (10000, 10000)] def run(): diff --git a/defects/weechat/bench/bench-weechat-0002.py b/defects/weechat/bench/bench-weechat-0002.py index baa58d372..85eefd48b 100644 --- a/defects/weechat/bench/bench-weechat-0002.py +++ b/defects/weechat/bench/bench-weechat-0002.py @@ -30,7 +30,7 @@ def bench_fixed(n, k): TRIALS = 3 -CASES = [(100, 100), (500, 500), (1000, 1000), (2000, 2000)] +CASES = [(500, 500), (2000, 2000), (5000, 5000), (10000, 10000)] def run(): diff --git a/defects/weechat/bench/bench-weechat-0003.py b/defects/weechat/bench/bench-weechat-0003.py index 1c0980e38..bce622817 100644 --- a/defects/weechat/bench/bench-weechat-0003.py +++ b/defects/weechat/bench/bench-weechat-0003.py @@ -30,7 +30,7 @@ def bench_fixed(n, k): TRIALS = 3 -CASES = [(100, 100), (500, 500), (1000, 1000), (2000, 2000)] +CASES = [(500, 500), (2000, 2000), (5000, 5000), (10000, 10000)] def run(): diff --git a/defects/weechat/bench/results.txt b/defects/weechat/bench/results.txt index d9b237608..c9470bebe 100644 --- a/defects/weechat/bench/results.txt +++ b/defects/weechat/bench/results.txt @@ -1,24 +1,24 @@ === weechat-0001-0001: CWE-407: list-scan inside loop in weechat-0001-0001 (generic model) === -N=100 k=100 : defective=0.085ms fixed=0.004ms speedup=24.1x -N=500 k=500 : defective=2.102ms fixed=0.021ms speedup=101.4x -N=1000 k=1000 : defective=8.855ms fixed=0.046ms speedup=193.2x -N=2000 k=2000 : defective=36.464ms fixed=0.093ms speedup=390.6x +N=500 k=500 : defective=3.583ms fixed=0.033ms speedup=107.7x +N=2000 k=2000 : defective=39.332ms fixed=0.089ms speedup=442.3x +N=5000 k=5000 : defective=212.270ms fixed=0.230ms speedup=924.5x +N=10000 k=10000: defective=902.310ms fixed=0.466ms speedup=1937.4x === weechat-0001: CWE-407: list-scan inside loop in weechat-0001 (generic model) === -N=100 k=100 : defective=0.084ms fixed=0.003ms speedup=25.0x -N=500 k=500 : defective=2.159ms fixed=0.021ms speedup=105.1x -N=1000 k=1000 : defective=9.482ms fixed=0.047ms speedup=203.1x -N=2000 k=2000 : defective=38.482ms fixed=0.382ms speedup=100.7x +N=500 k=500 : defective=1.944ms fixed=0.033ms speedup=59.2x +N=2000 k=2000 : defective=38.291ms fixed=0.088ms speedup=436.3x +N=5000 k=5000 : defective=214.745ms fixed=0.234ms speedup=918.7x +N=10000 k=10000: defective=865.578ms fixed=0.483ms speedup=1792.3x === weechat-0002: CWE-407: list-scan inside loop in weechat-0002 (generic model) === -N=100 k=100 : defective=0.085ms fixed=0.003ms speedup=24.8x -N=500 k=500 : defective=2.147ms fixed=0.021ms speedup=103.1x -N=1000 k=1000 : defective=8.843ms fixed=0.046ms speedup=191.6x -N=2000 k=2000 : defective=37.001ms fixed=0.101ms speedup=365.2x +N=500 k=500 : defective=2.055ms fixed=0.020ms speedup=102.9x +N=2000 k=2000 : defective=36.203ms fixed=0.088ms speedup=411.6x +N=5000 k=5000 : defective=231.254ms fixed=0.233ms speedup=993.3x +N=10000 k=10000: defective=892.420ms fixed=0.463ms speedup=1926.7x === weechat-0003: weechat-0003 — irc_channel_search O(C) linked-list scan — no hash index === -N=100 k=100 : defective=0.088ms fixed=0.004ms speedup=24.9x -N=500 k=500 : defective=2.164ms fixed=0.021ms speedup=103.9x -N=1000 k=1000 : defective=9.018ms fixed=0.047ms speedup=192.8x -N=2000 k=2000 : defective=36.419ms fixed=0.098ms speedup=371.7x +N=500 k=500 : defective=1.939ms fixed=0.019ms speedup=101.6x +N=2000 k=2000 : defective=31.771ms fixed=0.089ms speedup=355.5x +N=5000 k=5000 : defective=208.652ms fixed=0.227ms speedup=920.2x +N=10000 k=10000: defective=855.578ms fixed=0.457ms speedup=1871.0x