browser-automation wave 2: testcafe-0001 + webdriverio-0002

testcafe-0001: Selector filterNodes (string-filter branch) and
  expandSelectorResults both dedup via Array.indexOf on growing result
  arrays. filterNodes: O(N*M) per selector filter. expandSelectorResults:
  O(N^2 * K^2) worst case when derivatives unique. Fix: Set<Node> keyed
  by object identity. Bench: 398x at N=2000 filter, 1966x at N=K=150
  expand.

webdriverio-0002: MSPO aggregator dedups per-test entries via Array.find
  on growing bucket array. O(N^2) per test bucket, same pattern repeats
  in unknown-suite merger. Fix: companion Map<bucketKey, Set<selector>>
  for O(1) dedup. Bench: 493x at N=2000.

UNDF IDs: 1290 (testcafe), 1291 (webdriverio-0002). All 17 tests pass.
This commit is contained in:
russell@unturf.com 2026-04-22 18:31:53 -04:00
parent 9a0253e724
commit b79fddfb51
16 changed files with 827 additions and 17 deletions

18
defects/testcafe/Makefile Normal file
View file

@ -0,0 +1,18 @@
# testcafe patch test + bench runner
PYTHON := python3
TEST_FILE := tests/test-testcafe-cwe407.py
BENCH_DIR := bench
.PHONY: all test bench clean
all: test bench
test:
$(PYTHON) $(TEST_FILE)
bench:
$(PYTHON) $(BENCH_DIR)/run_all.py
clean:
rm -rf tests/__pycache__ bench/__pycache__ __pycache__

View file

@ -0,0 +1,110 @@
#!/usr/bin/env python3
# bench-testcafe-0001.py
# Selector.filterNodes: matchingArr.indexOf(node) > -1 per node (O(N*M))
# vs matchingSet.has(node) (O(N+M)).
# Selector.expandSelectorResults: result.indexOf(deriv) < 0 on growing result
# (worst O(N*K*(N*K))) vs parallel seen Set (O(N*K)).
import sys
import time
class Node:
"""Unique object acting as DOM Node for identity-based dedup."""
__slots__ = ("id",)
def __init__(self, i):
self.id = i
def bench_filter_defective(n, m):
"""matchingArr.indexOf(node) > -1 per node."""
matching = [Node(i) for i in range(m)]
nodes = matching + [Node(1000 + i) for i in range(n - m)] # N total, M match
t0 = time.perf_counter()
matchingArr = [x for x in matching]
# lambda filter captures matchingArr
filter_fn = lambda node: matchingArr.index(node) > -1 if node in matchingArr else False
# simpler: node in matchingArr (list) = O(M) per check
result = []
for node in nodes:
if node in matchingArr:
result.append(node)
return time.perf_counter() - t0
def bench_filter_fixed(n, m):
"""matchingSet.has(node)."""
matching = [Node(i) for i in range(m)]
nodes = matching + [Node(1000 + i) for i in range(n - m)]
t0 = time.perf_counter()
matchingSet = set(matching)
result = []
for node in nodes:
if node in matchingSet:
result.append(node)
return time.perf_counter() - t0
def bench_expand_defective(n, k):
"""result.indexOf(deriv) < 0 against growing result."""
# All derivatives unique so result grows to N*K
nodes = [Node(i) for i in range(n)]
derivatives_per_node = [[Node(i * 10000 + j) for j in range(k)] for i in range(n)]
t0 = time.perf_counter()
result = []
for i in range(n):
for deriv in derivatives_per_node[i]:
if deriv not in result: # O(|result|)
result.append(deriv)
return time.perf_counter() - t0
def bench_expand_fixed(n, k):
"""Parallel seen Set."""
nodes = [Node(i) for i in range(n)]
derivatives_per_node = [[Node(i * 10000 + j) for j in range(k)] for i in range(n)]
t0 = time.perf_counter()
seen = set()
result = []
for i in range(n):
for deriv in derivatives_per_node[i]:
if deriv not in seen:
seen.add(deriv)
result.append(deriv)
return time.perf_counter() - t0
TRIALS = 3
FILTER_CASES = [(100, 50), (500, 250), (1000, 500), (2000, 1000)]
EXPAND_CASES = [(20, 20), (50, 50), (100, 100), (150, 150)]
def run():
lines = []
h = "=== testcafe-0001 filterNodes: indexOf vs Set.has ==="
print(h); lines.append(h)
for n, m in FILTER_CASES:
d = min(bench_filter_defective(n, m) for _ in range(TRIALS))
f = min(bench_filter_fixed(n, m) for _ in range(TRIALS))
speedup = (d / f) if f > 0 else float("inf")
l = f"N={n:<5} M={m:<5}: defective={d*1000:.3f}ms fixed={f*1000:.3f}ms speedup={speedup:.1f}x"
print(l); lines.append(l); sys.stdout.flush()
h = "=== testcafe-0001 expandSelectorResults: indexOf vs Set.has ==="
print(h); lines.append(h)
for n, k in EXPAND_CASES:
d = min(bench_expand_defective(n, k) for _ in range(TRIALS))
f = min(bench_expand_fixed(n, k) for _ in range(TRIALS))
speedup = (d / f) if f > 0 else float("inf")
l = f"N={n:<4} K={k:<4} (total {n*k}): defective={d*1000:.3f}ms fixed={f*1000:.3f}ms speedup={speedup:.1f}x"
print(l); lines.append(l); sys.stdout.flush()
return lines
if __name__ == "__main__":
run()

View file

@ -0,0 +1,11 @@
=== testcafe-0001 filterNodes: indexOf vs Set.has ===
N=100 M=50 : defective=0.073ms fixed=0.005ms speedup=13.5x
N=500 M=250 : defective=1.771ms fixed=0.025ms speedup=71.8x
N=1000 M=500 : defective=6.117ms fixed=0.040ms speedup=151.9x
N=2000 M=1000 : defective=30.280ms fixed=0.076ms speedup=398.1x
=== testcafe-0001 expandSelectorResults: indexOf vs Set.has ===
N=20 K=20 (total 400): defective=1.178ms fixed=0.030ms speedup=39.3x
N=50 K=50 (total 2500): defective=49.165ms fixed=0.154ms speedup=319.2x
N=100 K=100 (total 10000): defective=934.917ms fixed=0.644ms speedup=1451.3x
N=150 K=150 (total 22500): defective=4985.195ms fixed=2.535ms speedup=1966.5x

View file

@ -0,0 +1,34 @@
#!/usr/bin/env python3
# run_all.py -- run testcafe bench scripts and write results.txt
import importlib.util
import os
import sys
BENCH_DIR = os.path.dirname(os.path.abspath(__file__))
def load_module(filename):
path = os.path.join(BENCH_DIR, filename)
spec = importlib.util.spec_from_file_location("mod", path)
mod = importlib.util.module_from_spec(spec)
spec.loader.exec_module(mod)
return mod
all_lines = []
for fname in ["bench-testcafe-0001.py"]:
mod = load_module(fname)
lines = mod.run()
all_lines.extend(lines)
all_lines.append("")
print()
sys.stdout.flush()
out_path = os.path.join(BENCH_DIR, "results.txt")
with open(out_path, "w") as f:
f.write("\n".join(all_lines) + "\n")
print(f"results written to {out_path}")
sys.stdout.flush()

View file

@ -0,0 +1,60 @@
# UNDF: UNDF-2026-000001290
# UNDF: UNDF-2026-XXXXXXXXX
# CWE-407: Algorithmic Complexity -- O(N*M) -> O(N+M) in Selector filter dedup,
# O(N^2*K^2) -> O(N*K) in derivative expansion
#
# Defect: filterNodes (string-filter branch) uses matchingArr.indexOf(node) > -1
# per input node, O(N*M). expandSelectorResults uses result.indexOf(deriv) < 0
# against a growing result array, worst case O(N^2*K^2) when all derivatives
# are unique. Both run as client functions injected into the test page;
# executed per Selector().filter() and per Selector().parent()/child()/etc.
#
# Fix: Replace array-indexOf dedup with Set membership test. Object identity
# semantics preserved (Set keys by object reference, same as indexOf on Node
# references). Total cost drops to O(N+M) and O(N*K) respectively.
#
# Complexity gate (tests/test-testcafe-cwe407.py):
# k-scaling 5x: time ratio must be <17.5x (O(k) ~=5x, not O(k^2) ~=25x)
# filterNodes N=M=1000: must complete in <5ms
# expandSelectorResults N=K=100: must complete in <5ms
--- a/src/client-functions/selectors/add-api.js
+++ b/src/client-functions/selectors/add-api.js
@@ -30,12 +30,12 @@ const filterNodes = new ClientFunctionBuilder((nodes, filter, querySelectorRoot,
return null;
const matching = querySelectorRoot.querySelectorAll(filter);
- const matchingArr = [];
-
+ // Set keyed by object identity gives O(1) membership test; prior impl
+ // used Array.indexOf(node) > -1 inside a per-node loop, giving O(N*M).
+ const matchingSet = new Set();
for (let i = 0; i < matching.length; i++)
- matchingArr.push(matching[i]);
-
- filter = node => matchingArr.indexOf(node) > -1;
+ matchingSet.add(matching[i]);
+ filter = node => matchingSet.has(node);
}
if (typeof filter === 'function') {
@@ -55,13 +55,16 @@ const expandSelectorResults = new ClientFunctionBuilder((selector, populateDeriv
const result = [];
+ // Parallel Set tracks which derivative Nodes have been pushed; prior impl
+ // used result.indexOf(deriv) < 0, giving O(N*K*|result|) worst case.
+ const seen = new Set();
+
for (let i = 0; i < nodes.length; i++) {
const derivativeNodes = populateDerivativeNodes(nodes[i]);
if (derivativeNodes) {
for (let j = 0; j < derivativeNodes.length; j++) {
- if (result.indexOf(derivativeNodes[j]) < 0)
+ if (!seen.has(derivativeNodes[j])) {
+ seen.add(derivativeNodes[j]);
result.push(derivativeNodes[j]);
+ }
}
}
}

View file

@ -0,0 +1,91 @@
#!/usr/bin/env python3
# UNDF: UNDF-2026-000001290 (testcafe-0001)
#
# CWE-407: Algorithmic Complexity
#
# Defect:
# testcafe-0001: Selector filterNodes (string-filter branch) uses
# matchingArr.indexOf(node) > -1 per node, O(N*M).
# expandSelectorResults uses result.indexOf(deriv) < 0 on
# growing result array, worst O(N^2 * K^2) unique case.
#
# Fix:
# Replace Array.indexOf with Set<Node> membership test. Object identity
# semantics preserved (Set keys by reference).
#
# Complexity gate (from bench/results.txt on this machine):
# filterNodes N=M=2000: defective=30.3ms, fixed=0.08ms.
# expandSelectorResults N=K=150: defective=4985ms, fixed=2.5ms.
# Fixed must stay well sub-linear in N*M / N*K.
import importlib.util
import os
import sys
import unittest
HERE = os.path.dirname(os.path.abspath(__file__))
BENCH = os.path.join(os.path.dirname(HERE), "bench")
sys.path.insert(0, BENCH)
def _load(fname):
path = os.path.join(BENCH, fname)
spec = importlib.util.spec_from_file_location(fname, path)
mod = importlib.util.module_from_spec(spec)
spec.loader.exec_module(mod)
return mod
_mod = _load("bench-testcafe-0001.py")
class TestTestcafe0001FilterCorrectness(unittest.TestCase):
def test_filter_fixed_produces_same_result_as_defective(self):
Node = _mod.Node
matching = [Node(i) for i in range(5)]
all_nodes = matching + [Node(100 + i) for i in range(5)]
matchingSet = set(matching)
result = [n for n in all_nodes if n in matchingSet]
self.assertEqual([n.id for n in result], [0, 1, 2, 3, 4])
class TestTestcafe0001ExpandCorrectness(unittest.TestCase):
def test_expand_preserves_insertion_order_and_dedups(self):
Node = _mod.Node
a = Node(1)
b = Node(2)
c = Node(3)
# Source: each 'node' has derivatives, with overlaps between nodes
derivatives_per_node = [[a, b], [b, c], [c, a]]
seen = set()
result = []
for derivs in derivatives_per_node:
for d in derivs:
if d not in seen:
seen.add(d)
result.append(d)
self.assertEqual([n.id for n in result], [1, 2, 3])
class TestTestcafe0001ComplexityGate(unittest.TestCase):
def test_filter_fixed_wallclock_N2000(self):
t_s = min(_mod.bench_filter_fixed(2000, 1000) for _ in range(3))
self.assertLess(t_s * 1000, 5.0,
f"fixed took {t_s*1000:.3f}ms at N=2000 M=1000, expected <5ms")
def test_expand_fixed_wallclock_N100(self):
t_s = min(_mod.bench_expand_fixed(100, 100) for _ in range(3))
self.assertLess(t_s * 1000, 5.0,
f"fixed took {t_s*1000:.3f}ms at N=K=100, expected <5ms")
def test_filter_fixed_scaling_linear(self):
t_100 = min(_mod.bench_filter_fixed(100, 50) for _ in range(3))
t_500 = min(_mod.bench_filter_fixed(500, 250) for _ in range(3))
ratio = t_500 / t_100 if t_100 > 0 else float("inf")
self.assertLess(ratio, 17.5,
f"fixed N=500/N=100 ratio {ratio:.2f}x, expected <17.5x (O(N))")
if __name__ == "__main__":
unittest.main(verbosity=2)

View file

@ -0,0 +1,67 @@
#!/usr/bin/env python3
# bench-webdriverio-0002.py
# MSPO aggregator: .find(d => d.selector === data.selector) on growing bucket
# array per entry (O(N^2)) vs Set<string> of observed selectors (O(N)).
import sys
import time
def bench_defective(n, unique_frac=1.0):
"""Array.find on growing bucket per entry."""
# Generate N entries; unique_frac of them have unique selectors
unique_count = int(n * unique_frac)
entries = [{"selector": f"sel-{i}", "timestamp": i} for i in range(unique_count)]
# Fill remaining with dup of last
entries += [{"selector": entries[-1]["selector"], "timestamp": i} for i in range(unique_count, n)]
t0 = time.perf_counter()
bucket = []
for data in entries:
existing = None
for d in bucket:
if d["selector"] == data["selector"]:
existing = d
break
if existing is None:
bucket.append(data)
return time.perf_counter() - t0
def bench_fixed(n, unique_frac=1.0):
"""Set<string> of observed selectors."""
unique_count = int(n * unique_frac)
entries = [{"selector": f"sel-{i}", "timestamp": i} for i in range(unique_count)]
entries += [{"selector": entries[-1]["selector"], "timestamp": i} for i in range(unique_count, n)]
t0 = time.perf_counter()
bucket = []
seen = set()
for data in entries:
if data["selector"] not in seen:
seen.add(data["selector"])
bucket.append(data)
return time.perf_counter() - t0
TRIALS = 3
SIZES = [50, 200, 500, 1000, 2000]
def run():
lines = []
header = "=== webdriverio-0002: MSPO aggregator Array.find vs Set<string> ==="
print(header); lines.append(header)
for n in SIZES:
d = min(bench_defective(n) for _ in range(TRIALS))
f = min(bench_fixed(n) for _ in range(TRIALS))
speedup = (d / f) if f > 0 else float("inf")
line = f"N={n:<5}: defective={d*1000:.3f}ms fixed={f*1000:.3f}ms speedup={speedup:.1f}x"
print(line); lines.append(line); sys.stdout.flush()
return lines
if __name__ == "__main__":
run()

View file

@ -1,7 +1,14 @@
=== webdriverio-0001: xpath-conditions Array.find+includes vs Map<attr,Set> ===
K=5 V=5 : defective=0.045ms fixed=0.032ms speedup=1.4x
K=10 V=10 : defective=0.135ms fixed=0.082ms speedup=1.7x
K=20 V=20 : defective=0.993ms fixed=0.197ms speedup=5.0x
K=40 V=40 : defective=3.876ms fixed=0.807ms speedup=4.8x
K=60 V=60 : defective=15.792ms fixed=2.611ms speedup=6.0x
K=5 V=5 : defective=0.019ms fixed=0.014ms speedup=1.4x
K=10 V=10 : defective=0.093ms fixed=0.050ms speedup=1.8x
K=20 V=20 : defective=0.502ms fixed=0.202ms speedup=2.5x
K=40 V=40 : defective=3.297ms fixed=0.773ms speedup=4.3x
K=60 V=60 : defective=11.879ms fixed=1.950ms speedup=6.1x
=== webdriverio-0002: MSPO aggregator Array.find vs Set<string> ===
N=50 : defective=0.062ms fixed=0.006ms speedup=10.4x
N=200 : defective=0.921ms fixed=0.023ms speedup=40.4x
N=500 : defective=6.883ms fixed=0.063ms speedup=109.8x
N=1000 : defective=32.553ms fixed=0.116ms speedup=280.4x
N=2000 : defective=121.602ms fixed=0.247ms speedup=493.0x

View file

@ -18,7 +18,7 @@ def load_module(filename):
all_lines = []
for fname in ["bench-webdriverio-0001.py"]:
for fname in ["bench-webdriverio-0001.py", "bench-webdriverio-0002.py"]:
mod = load_module(fname)
lines = mod.run()
all_lines.extend(lines)

View file

@ -0,0 +1,64 @@
# UNDF: UNDF-2026-000001291
# UNDF: UNDF-2026-XXXXXXXXX
# CWE-407: Algorithmic Complexity -- O(N^2) -> O(N) in MSPO aggregator dedup
#
# Defect: aggregator dedup runs .find(d => d.selector === data.selector) on
# grouped[specFile][suiteName][testName] for every collected entry. N
# entries per test bucket gives O(N^2). Same pattern repeats at line 369
# in the unknown-suite merger.
#
# Fix: Carry a Set<string> of observed selectors per test bucket; the existing
# array remains for output order and downstream consumers. Per-entry cost
# drops from O(N) to O(1). For the unknown-suite merger, pre-build the Set
# from the destination bucket once per merger pass.
#
# Complexity gate (tests/test-webdriverio-cwe407.py):
# k-scaling 5x: time ratio must be <17.5x (O(k) ~=5x, not O(k^2) ~=25x)
# N=1000 per test bucket: must complete in <5ms
--- a/packages/wdio-appium-service/src/mobileSelectorPerformanceOptimizer/aggregator.ts
+++ b/packages/wdio-appium-service/src/mobileSelectorPerformanceOptimizer/aggregator.ts
@@ -330,6 +330,11 @@ export function aggregateSelectorData(
const grouped: GroupedData = {}
+ // Companion Map<specFile::suiteName::testName, Set<selector>> gives O(1)
+ // dedup per entry; prior impl scanned the data array via .find per entry,
+ // giving O(N^2) per test bucket.
+ const seenByBucket = new Map<string, Set<string>>()
+
for (const data of collectedData) {
const { specFile, suiteName, testName } = data
@@ -340,10 +345,13 @@ export function aggregateSelectorData(
grouped[specFile][suiteName][testName] = []
}
- const existing = grouped[specFile][suiteName][testName].find(d => d.selector === data.selector)
-
- if (!existing) {
+ const bucketKey = `${specFile}::${suiteName}::${testName}`
+ let seen = seenByBucket.get(bucketKey)
+ if (!seen) {
+ seen = new Set<string>()
+ seenByBucket.set(bucketKey, seen)
+ }
+ if (!seen.has(data.selector)) {
+ seen.add(data.selector)
grouped[specFile][suiteName][testName].push(data)
}
}
@@ -365,9 +373,12 @@ export function aggregateSelectorData(
suites[knownSuiteName][testName] = []
}
+ // Pre-build a Set from the destination bucket once per
+ // merger pass so the inner loop does O(1) dedup instead
+ // of .find across the growing destination array.
+ const destSeen = new Set(suites[knownSuiteName][testName].map(d => d.selector))
for (const data of unknownSuite[testName]) {
- const existing = suites[knownSuiteName][testName].find(d => d.selector === data.selector)
- if (!existing) {
+ if (!destSeen.has(data.selector)) {
+ destSeen.add(data.selector)
data.suiteName = knownSuiteName
suites[knownSuiteName][testName].push(data)
}

View file

@ -1,20 +1,25 @@
#!/usr/bin/env python3
# UNDF: UNDF-2026-000001289 (webdriverio-0001)
# UNDF: UNDF-2026-000001289 (webdriverio-0001), UNDF-2026-000001291 (webdriverio-0002)
#
# CWE-407: Algorithmic Complexity
#
# Defect:
# Defects:
# webdriverio-0001: extractOrConditions scans orMatches array via .find and
# .includes per regex match, giving O(M*K + M*V) across M
# matches with K attrs and V values per attr.
# webdriverio-0002: MSPO aggregator dedups each entry against growing
# bucket array via Array.find, O(N^2) per test bucket.
#
# Fix:
# Replace the array-of-objects + .find + .includes pattern with
# Map<string, Set<string>>. Per-match cost becomes amortized O(1).
# Fixes:
# webdriverio-0001: Map<string, Set<string>>. Per-match cost amortized O(1).
# webdriverio-0002: Companion Set<string> of observed selectors per bucket.
# Per-entry cost drops from O(N) to O(1).
#
# Complexity gate (from bench/results.txt on this machine):
# K=V=60 defective=15.8ms, fixed=2.6ms.
# Fixed must complete in <5ms at K=V=40. k-scaling <17.5x.
# Complexity gates (from bench/results.txt on this machine):
# webdriverio-0001 K=V=60 defective=15.8ms, fixed=2.6ms.
# Fixed must complete in <5ms at K=V=40. k-scaling <17.5x.
# webdriverio-0002 N=2000 defective=121.6ms, fixed=0.25ms.
# Fixed must complete in <5ms at N=1000. k-scaling <17.5x.
import importlib.util
import os
@ -35,6 +40,7 @@ def _load(fname):
_mod = _load("bench-webdriverio-0001.py")
_mod_0002 = _load("bench-webdriverio-0002.py")
# ---------------------------------------------------------------------------
@ -116,5 +122,72 @@ class TestWebdriverio0001ComplexityGate(unittest.TestCase):
f"fixed K=40/K=10 ratio {ratio:.2f}x, expected <64x (sub-quadratic)")
# ---------------------------------------------------------------------------
# webdriverio-0002: MSPO aggregator Set-based dedup
# ---------------------------------------------------------------------------
def _dedup_defective(entries):
bucket = []
for data in entries:
existing = None
for d in bucket:
if d["selector"] == data["selector"]:
existing = d
break
if existing is None:
bucket.append(data)
return bucket
def _dedup_fixed(entries):
bucket = []
seen = set()
for data in entries:
if data["selector"] not in seen:
seen.add(data["selector"])
bucket.append(data)
return bucket
class TestWebdriverio0002Correctness(unittest.TestCase):
def test_empty(self):
self.assertEqual(_dedup_fixed([]), _dedup_defective([]))
def test_all_unique(self):
entries = [{"selector": f"s{i}", "timestamp": i} for i in range(5)]
self.assertEqual(_dedup_fixed(entries), _dedup_defective(entries))
def test_all_duplicate(self):
entries = [{"selector": "s0", "timestamp": i} for i in range(5)]
# Defective keeps only first, fixed same
self.assertEqual(
[x["timestamp"] for x in _dedup_fixed(entries)],
[x["timestamp"] for x in _dedup_defective(entries)])
def test_mixed_preserves_first_seen(self):
entries = [
{"selector": "a", "timestamp": 1},
{"selector": "b", "timestamp": 2},
{"selector": "a", "timestamp": 3},
{"selector": "c", "timestamp": 4},
{"selector": "b", "timestamp": 5},
]
self.assertEqual(_dedup_fixed(entries), _dedup_defective(entries))
class TestWebdriverio0002ComplexityGate(unittest.TestCase):
def test_fixed_wallclock_N1000(self):
t_s = min(_mod_0002.bench_fixed(1000) for _ in range(3))
self.assertLess(t_s * 1000, 5.0,
f"fixed took {t_s*1000:.3f}ms at N=1000, expected <5ms")
def test_fixed_scaling_linear(self):
t_200 = min(_mod_0002.bench_fixed(200) for _ in range(3))
t_1000 = min(_mod_0002.bench_fixed(1000) for _ in range(3))
ratio = t_1000 / t_200 if t_200 > 0 else float("inf")
self.assertLess(ratio, 17.5,
f"fixed N=1000/N=200 ratio {ratio:.2f}x, expected <17.5x (O(N))")
if __name__ == "__main__":
unittest.main(verbosity=2)