undefect. CWE-407 — 63 sites patched across 27 ecosystems

Authors: russell@unturf.com · brackishbert@gmail.com · foxhop.net · TimeHexOn.com

Patches, unit tests, benchmarks, whitepaper, and outreach briefs.
Public domain — no copyright claimed. Use freely.

defects/spark/unit/SparkAnalyzerWindowAggTest.java

@@ -0,0 +1,173 @@
+package unit;
+
+import java.util.*;
+
+/**
+ * Unit test for spark-0001: seenWindowAggregates ArrayBuffer CWE-407.
+ *
+ * Defect:  Analyzer.scala uses ArrayBuffer[AggregateExpression] for
+ *          seenWindowAggregates and calls .contains(agg) on it during
+ *          window function extraction. O(W) per check, O(A×W) total
+ *          where A = aggregate expressions, W = window aggregates in query.
+ *
+ * Fix:     Replace ArrayBuffer with mutable.LinkedHashSet — O(1) contains.
+ *
+ * This test uses a self-contained Java model of the two strategies:
+ *   DefectiveWindowExtractor  — ArrayList.contains()
+ *   FixedWindowExtractor      — LinkedHashSet.contains()
+ *
+ * Measurement: count element-level comparisons in the membership check.
+ */
+public class SparkAnalyzerWindowAggTest {
+
+    // ── Minimal AggregateExpression stub ────────────────────────────────────
+
+    public static class AggExpr {
+        final String id;
+        AggExpr(String id) { this.id = id; }
+        @Override public boolean equals(Object o) {
+            return o instanceof AggExpr && ((AggExpr) o).id.equals(id);
+        }
+        @Override public int hashCode() { return id.hashCode(); }
+        @Override public String toString() { return "Agg(" + id + ")"; }
+    }
+
+    // ── Result ──────────────────────────────────────────────────────────────
+
+    public static class Result {
+        final List<AggExpr> extracted;
+        public final long comparisons;        // element-level equality checks
+        Result(List<AggExpr> extracted, long comparisons) {
+            this.extracted   = extracted;
+            this.comparisons = comparisons;
+        }
+    }
+
+    // ── DEFECTIVE: ArrayList.contains() ─────────────────────────────────────
+
+    public static Result extractDefective(List<AggExpr> windowAggs, List<AggExpr> otherAggs) {
+        List<AggExpr> seen = new ArrayList<>();
+        long comparisons = 0;
+
+        // Window aggregates are added first (equivalent to WindowExpression case)
+        for (AggExpr agg : windowAggs) {
+            seen.add(agg);
+        }
+
+        // Other aggregates check seen list — O(W) each
+        List<AggExpr> extracted = new ArrayList<>();
+        for (AggExpr agg : otherAggs) {
+            // Simulate ArrayList.contains() — count each comparison
+            boolean found = false;
+            for (AggExpr s : seen) {
+                comparisons++;
+                if (s.equals(agg)) { found = true; break; }
+            }
+            if (!found) {
+                extracted.add(agg);
+                seen.add(agg);
+            }
+        }
+        return new Result(extracted, comparisons);
+    }
+
+    // ── FIXED: LinkedHashSet.contains() ─────────────────────────────────────
+
+    public static Result extractFixed(List<AggExpr> windowAggs, List<AggExpr> otherAggs) {
+        LinkedHashSet<AggExpr> seen = new LinkedHashSet<>();
+        long comparisons = 0;
+
+        for (AggExpr agg : windowAggs) {
+            seen.add(agg);
+        }
+
+        List<AggExpr> extracted = new ArrayList<>();
+        for (AggExpr agg : otherAggs) {
+            comparisons++;   // O(1) hash lookup — counts as 1
+            if (!seen.contains(agg)) {
+                extracted.add(agg);
+                seen.add(agg);
+            }
+        }
+        return new Result(extracted, comparisons);
+    }
+
+    // ── Build test fixtures ──────────────────────────────────────────────────
+
+    public static List<AggExpr> windowAggs(int w) {
+        List<AggExpr> list = new ArrayList<>();
+        for (int i = 0; i < w; i++) list.add(new AggExpr("win_" + i));
+        return list;
+    }
+
+    public static List<AggExpr> otherAggs(int a, List<AggExpr> existing) {
+        // Mix of novel aggregates and ones already in existing (the common case)
+        List<AggExpr> list = new ArrayList<>();
+        for (int i = 0; i < a; i++) {
+            list.add(i % 3 == 0 && i / 3 < existing.size()
+                ? existing.get(i / 3)          // already seen
+                : new AggExpr("agg_" + i));    // novel
+        }
+        return list;
+    }
+
+    // ── Tests ────────────────────────────────────────────────────────────────
+
+    static void testCorrectnessMatch() {
+        List<AggExpr> wins = windowAggs(5);
+        List<AggExpr> aggs = otherAggs(10, wins);
+        Result def = extractDefective(wins, aggs);
+        Result fix = extractFixed(wins, aggs);
+        assert new HashSet<>(def.extracted).equals(new HashSet<>(fix.extracted))
+            : "defective and fixed must extract identical aggregates";
+        System.out.println("PASS testCorrectnessMatch");
+    }
+
+    static void testDefectiveGrowsQuadratically() {
+        // W=A, grow together: comparisons should scale ~quadratically
+        long prev = -1;
+        double prevRatio = -1;
+        for (int n : new int[]{10, 20, 40}) {
+            List<AggExpr> wins = windowAggs(n);
+            List<AggExpr> aggs = otherAggs(n, wins);
+            long c = extractDefective(wins, aggs).comparisons;
+            if (prev > 0) {
+                double ratio = (double) c / prev;
+                assert ratio > 2.5 : "defective comparisons should grow >2.5x when n doubles; got " + ratio;
+            }
+            prev = c;
+        }
+        System.out.println("PASS testDefectiveGrowsQuadratically");
+    }
+
+    static void testFixedGrowsLinearly() {
+        // Fixed: exactly 1 comparison per otherAgg regardless of W
+        for (int n : new int[]{10, 20, 40}) {
+            List<AggExpr> wins = windowAggs(n);
+            List<AggExpr> aggs = otherAggs(n, wins);
+            long c = extractFixed(wins, aggs).comparisons;
+            assert c == n : "fixed must make exactly A comparisons (one per otherAgg); got " + c + " for A=" + n;
+        }
+        System.out.println("PASS testFixedGrowsLinearly");
+    }
+
+    static void testRatioAtScaleIsLarge() {
+        int w = 100, a = 100;
+        List<AggExpr> wins = windowAggs(w);
+        List<AggExpr> aggs = otherAggs(a, wins);
+        long def_c = extractDefective(wins, aggs).comparisons;
+        long fix_c = extractFixed(wins, aggs).comparisons;
+        double ratio = (double) def_c / fix_c;
+        assert ratio > 10 : "at W=A=100, defective should be >10x worse; ratio=" + ratio;
+        System.out.printf("PASS testRatioAtScaleIsLarge  (defective=%d, fixed=%d, ratio=%.1fx)%n",
+            def_c, fix_c, ratio);
+    }
+
+    public static void main(String[] args) {
+        testCorrectnessMatch();
+        testDefectiveGrowsQuadratically();
+        testFixedGrowsLinearly();
+        testRatioAtScaleIsLarge();
+        System.out.println("All spark-0001 tests passed.");
+    }
+}