wave13: 514/239 — flink/nifi/artemis + K8s/cilium/linkerd2 + ES/OpenSearch/Solr + hadoop/hbase/spark

2026-03-27 17:50:40 -04:00 · 2026-03-27 17:50:40 -04:00 · 8f0bc73afa
commit 8f0bc73afa
parent 424a2a7787
32 changed files with 3977 additions and 5 deletions
--- a/defects/spark/unit/MasterCompletedAppsAlgorithm.java
+++ b/defects/spark/unit/MasterCompletedAppsAlgorithm.java
@ -0,0 +1,128 @@
+package unit;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Unit test for spark-0003: Master.completedApps ArrayBuffer.contains() O(n²) on worker failure.
+ *
+ * Standalone — no JUnit. Run: javac -d . MasterCompletedAppsAlgorithm.java && java -ea unit.MasterCompletedAppsAlgorithm
+ */
+public class MasterCompletedAppsAlgorithm {
+
+    // ---------- Slow: ArrayList.contains inside filterNot ----------
+    static long slowFilterNot(Set<Integer> apps, List<Integer> completedApps) {
+        long ops = 0;
+        List<Integer> runningApps = new ArrayList<>();
+        for (Integer app : apps) {
+            ops += completedApps.size(); // O(size) per contains
+            if (!completedApps.contains(app)) {
+                runningApps.add(app);
+            }
+        }
+        return ops;
+    }
+
+    // ---------- Fast: HashSet.contains O(1) ----------
+    static long fastFilterNot(Set<Integer> apps, Set<Integer> completedAppsSet) {
+        long ops = 0;
+        List<Integer> runningApps = new ArrayList<>();
+        for (Integer app : apps) {
+            ops += 1; // O(1) per contains
+            if (!completedAppsSet.contains(app)) {
+                runningApps.add(app);
+            }
+        }
+        return ops;
+    }
+
+    static void testCorrectness() {
+        int A = 50;  // running apps
+        int C = 20;  // completed apps
+
+        Set<Integer> apps = new HashSet<>();
+        for (int i = 0; i < A + C; i++) apps.add(i);
+
+        List<Integer> completedList = new ArrayList<>();
+        for (int i = A; i < A + C; i++) completedList.add(i);
+        Set<Integer> completedSet = new HashSet<>(completedList);
+
+        // Compute running apps both ways
+        Set<Integer> slowRunning = new HashSet<>();
+        for (Integer app : apps) {
+            if (!completedList.contains(app)) slowRunning.add(app);
+        }
+        Set<Integer> fastRunning = new HashSet<>();
+        for (Integer app : apps) {
+            if (!completedSet.contains(app)) fastRunning.add(app);
+        }
+
+        assert slowRunning.equals(fastRunning) :
+            "Result mismatch: slow=" + slowRunning.size() + " fast=" + fastRunning.size();
+        assert slowRunning.size() == A :
+            "Expected " + A + " running apps, got " + slowRunning.size();
+        System.out.println("PASS: correctness — filterNot produces same " + A + " running apps");
+    }
+
+    static void testPerformance() {
+        int A = 100;  // running apps
+        int C = 200;  // completed apps (default retainedApplications)
+
+        Set<Integer> apps = new HashSet<>();
+        for (int i = 0; i < A + C; i++) apps.add(i);
+
+        List<Integer> completedList = new ArrayList<>();
+        Set<Integer> completedSet = new HashSet<>();
+        for (int i = A; i < A + C; i++) {
+            completedList.add(i);
+            completedSet.add(i);
+        }
+
+        long slowOps = slowFilterNot(apps, completedList);
+        long fastOps = fastFilterNot(apps, completedSet);
+        double ratio = (double) slowOps / fastOps;
+
+        System.out.printf("filterNot A=%d C=%d: slow_ops=%,d fast_ops=%,d ratio=%.1fx%n",
+            A, C, slowOps, fastOps, ratio);
+        assert ratio >= 10.0 :
+            "Expected >=10x speedup, got ratio=" + ratio;
+        System.out.println("PASS: performance — ratio >= 10x");
+    }
+
+    static void testLargeRetainedApps() {
+        // Stress test: many completed apps retained
+        int A = 500;
+        int C = 1000;
+
+        Set<Integer> apps = new HashSet<>();
+        for (int i = 0; i < A + C; i++) apps.add(i);
+
+        List<Integer> completedList = new ArrayList<>();
+        Set<Integer> completedSet = new HashSet<>();
+        for (int i = A; i < A + C; i++) {
+            completedList.add(i);
+            completedSet.add(i);
+        }
+
+        long slowOps = slowFilterNot(apps, completedList);
+        long fastOps = fastFilterNot(apps, completedSet);
+        double ratio = (double) slowOps / fastOps;
+
+        System.out.printf("filterNot A=%d C=%d: slow_ops=%,d fast_ops=%,d ratio=%.1fx%n",
+            A, C, slowOps, fastOps, ratio);
+        assert ratio >= 100.0 :
+            "Expected >=100x speedup, got ratio=" + ratio;
+        System.out.println("PASS: large retained — ratio >= 100x");
+    }
+
+    public static void main(String[] args) {
+        testCorrectness();
+        testPerformance();
+        testLargeRetainedApps();
+        System.out.println("ALL PASS (3/3)");
+    }
+}