wave13: 514/239 — flink/nifi/artemis + K8s/cilium/linkerd2 + ES/OpenSearch/Solr + hadoop/hbase/spark

defects/hadoop/unit/DispatcherAlgorithm.java

@@ -0,0 +1,160 @@
+package unit;
+
+import java.util.ArrayList;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * Unit test for hadoop-0004: Dispatcher.srcBlocks ArrayList.contains() O(n²) in block receive loop.
+ *
+ * Standalone — no JUnit. Run: javac -d . DispatcherAlgorithm.java && java -ea unit.DispatcherAlgorithm
+ */
+public class DispatcherAlgorithm {
+
+    // ---------- Slow: ArrayList.contains inside a loop ----------
+    static class SlowSrcBlocks {
+        private final List<Integer> srcBlocks = new ArrayList<>();
+
+        /** Simulates Dispatcher.Source.getBlockIterator() — adds block if not already present. */
+        long receiveBlocks(int[] blocks) {
+            long ops = 0;
+            for (int block : blocks) {
+                ops += srcBlocks.size(); // O(size) per contains
+                if (!srcBlocks.contains(block)) {
+                    srcBlocks.add(block);
+                }
+            }
+            return ops;
+        }
+    }
+
+    // ---------- Fast: LinkedHashSet.contains O(1) ----------
+    static class FastSrcBlocks {
+        private final Set<Integer> srcBlocks = new LinkedHashSet<>();
+
+        long receiveBlocks(int[] blocks) {
+            long ops = 0;
+            for (int block : blocks) {
+                ops += 1; // O(1) per contains
+                srcBlocks.add(block); // LinkedHashSet.add handles dedup
+            }
+            return ops;
+        }
+    }
+
+    // ---------- Slow: locations ArrayList.contains in addLocation ----------
+    static class SlowMovedBlocks {
+        private final List<Integer> locations = new ArrayList<>(3);
+
+        long addLocation(int[] datanodes) {
+            long ops = 0;
+            for (int dn : datanodes) {
+                ops += locations.size(); // O(size) per contains
+                if (!locations.contains(dn)) {
+                    locations.add(dn);
+                }
+            }
+            return ops;
+        }
+    }
+
+    // ---------- Fast: LinkedHashSet ----------
+    static class FastMovedBlocks {
+        private final Set<Integer> locations = new LinkedHashSet<>(3);
+
+        long addLocation(int[] datanodes) {
+            long ops = 0;
+            for (int dn : datanodes) {
+                ops += 1; // O(1)
+                locations.add(dn);
+            }
+            return ops;
+        }
+    }
+
+    // ---------- Correctness check ----------
+    static void testCorrectness() {
+        int N = 200;
+        int[] blocks = new int[N + 50]; // some duplicates
+        for (int i = 0; i < N; i++) blocks[i] = i;
+        for (int i = N; i < blocks.length; i++) blocks[i] = i - N; // duplicates of first 50
+
+        SlowSrcBlocks slow = new SlowSrcBlocks();
+        slow.receiveBlocks(blocks);
+        FastSrcBlocks fast = new FastSrcBlocks();
+        fast.receiveBlocks(blocks);
+
+        assert slow.srcBlocks.size() == N : "slow size wrong: " + slow.srcBlocks.size();
+        assert fast.srcBlocks.size() == N : "fast size wrong: " + fast.srcBlocks.size();
+
+        // Order preserved
+        int idx = 0;
+        for (int v : fast.srcBlocks) {
+            assert v == idx : "fast order wrong at " + idx;
+            idx++;
+        }
+
+        System.out.println("PASS: correctness — srcBlocks dedup preserves N=" + N + " unique blocks");
+    }
+
+    static void testMovedBlocksCorrectness() {
+        int D = 10;
+        int[] datanodes = new int[D + 5];
+        for (int i = 0; i < D; i++) datanodes[i] = i;
+        for (int i = D; i < datanodes.length; i++) datanodes[i] = i - D; // duplicates
+
+        SlowMovedBlocks slow = new SlowMovedBlocks();
+        slow.addLocation(datanodes);
+        FastMovedBlocks fast = new FastMovedBlocks();
+        fast.addLocation(datanodes);
+
+        assert slow.locations.size() == D : "slow size: " + slow.locations.size();
+        assert fast.locations.size() == D : "fast size: " + fast.locations.size();
+        System.out.println("PASS: correctness — locations dedup preserves D=" + D + " unique datanodes");
+    }
+
+    // ---------- Performance ----------
+    static void testPerformance() {
+        int N = 2000; // simulate 2000 blocks per source node
+        int[] blocks = new int[N];
+        for (int i = 0; i < N; i++) blocks[i] = i;
+
+        long slowOps = new SlowSrcBlocks().receiveBlocks(blocks);
+        long fastOps = new FastSrcBlocks().receiveBlocks(blocks);
+
+        System.out.printf("srcBlocks N=%d: slow_ops=%,d fast_ops=%,d ratio=%.1fx%n",
+            N, slowOps, fastOps, (double) slowOps / fastOps);
+        assert slowOps >= fastOps * 10 :
+            "Expected >=10x slower for slow path, got ratio=" + (slowOps / fastOps);
+        System.out.println("PASS: performance — srcBlocks ratio >= 10x");
+    }
+
+    static void testMovedBlocksPerformance() {
+        // Simulate a block with many replicas (e.g., 100 datanodes, called many times)
+        int BLOCKS = 500;
+        int D = 20; // datanodes per block (with some duplicates)
+
+        long slowTotal = 0, fastTotal = 0;
+        for (int b = 0; b < BLOCKS; b++) {
+            int[] datanodes = new int[D];
+            for (int i = 0; i < D; i++) datanodes[i] = i % 10; // lots of duplicates
+            slowTotal += new SlowMovedBlocks().addLocation(datanodes);
+            fastTotal += new FastMovedBlocks().addLocation(datanodes);
+        }
+
+        System.out.printf("locations BLOCKS=%d D=%d: slow_ops=%,d fast_ops=%,d ratio=%.1fx%n",
+            BLOCKS, D, slowTotal, fastTotal, (double) slowTotal / fastTotal);
+        assert slowTotal >= fastTotal * 5 :
+            "Expected >=5x slower for slow path, got ratio=" + (slowTotal / fastTotal);
+        System.out.println("PASS: performance — locations ratio >= 5x");
+    }
+
+    public static void main(String[] args) {
+        testCorrectness();
+        testMovedBlocksCorrectness();
+        testPerformance();
+        testMovedBlocksPerformance();
+        System.out.println("ALL PASS (4/4)");
+    }
+}