java-topology/defects/hadoop/unit/DispatcherAlgorithm.java

package unit;

import java.util.ArrayList;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;

/**
 * Unit test for hadoop-0004: Dispatcher.srcBlocks ArrayList.contains() O(n²) in block receive loop.
 *
 * Standalone — no JUnit. Run: javac -d . DispatcherAlgorithm.java && java -ea unit.DispatcherAlgorithm
 */
public class DispatcherAlgorithm {

    // ---------- Slow: ArrayList.contains inside a loop ----------
    static class SlowSrcBlocks {
        private final List<Integer> srcBlocks = new ArrayList<>();

        /** Simulates Dispatcher.Source.getBlockIterator() — adds block if not already present. */
        long receiveBlocks(int[] blocks) {
            long ops = 0;
            for (int block : blocks) {
                ops += srcBlocks.size(); // O(size) per contains
                if (!srcBlocks.contains(block)) {
                    srcBlocks.add(block);
                }
            }
            return ops;
        }
    }

    // ---------- Fast: LinkedHashSet.contains O(1) ----------
    static class FastSrcBlocks {
        private final Set<Integer> srcBlocks = new LinkedHashSet<>();

        long receiveBlocks(int[] blocks) {
            long ops = 0;
            for (int block : blocks) {
                ops += 1; // O(1) per contains
                srcBlocks.add(block); // LinkedHashSet.add handles dedup
            }
            return ops;
        }
    }

    // ---------- Slow: locations ArrayList.contains in addLocation ----------
    static class SlowMovedBlocks {
        private final List<Integer> locations = new ArrayList<>(3);

        long addLocation(int[] datanodes) {
            long ops = 0;
            for (int dn : datanodes) {
                ops += locations.size(); // O(size) per contains
                if (!locations.contains(dn)) {
                    locations.add(dn);
                }
            }
            return ops;
        }
    }

    // ---------- Fast: LinkedHashSet ----------
    static class FastMovedBlocks {
        private final Set<Integer> locations = new LinkedHashSet<>(3);

        long addLocation(int[] datanodes) {
            long ops = 0;
            for (int dn : datanodes) {
                ops += 1; // O(1)
                locations.add(dn);
            }
            return ops;
        }
    }

    // ---------- Correctness check ----------
    static void testCorrectness() {
        int N = 200;
        int[] blocks = new int[N + 50]; // some duplicates
        for (int i = 0; i < N; i++) blocks[i] = i;
        for (int i = N; i < blocks.length; i++) blocks[i] = i - N; // duplicates of first 50

        SlowSrcBlocks slow = new SlowSrcBlocks();
        slow.receiveBlocks(blocks);
        FastSrcBlocks fast = new FastSrcBlocks();
        fast.receiveBlocks(blocks);

        assert slow.srcBlocks.size() == N : "slow size wrong: " + slow.srcBlocks.size();
        assert fast.srcBlocks.size() == N : "fast size wrong: " + fast.srcBlocks.size();

        // Order preserved
        int idx = 0;
        for (int v : fast.srcBlocks) {
            assert v == idx : "fast order wrong at " + idx;
            idx++;
        }

        System.out.println("PASS: correctness — srcBlocks dedup preserves N=" + N + " unique blocks");
    }

    static void testMovedBlocksCorrectness() {
        int D = 10;
        int[] datanodes = new int[D + 5];
        for (int i = 0; i < D; i++) datanodes[i] = i;
        for (int i = D; i < datanodes.length; i++) datanodes[i] = i - D; // duplicates

        SlowMovedBlocks slow = new SlowMovedBlocks();
        slow.addLocation(datanodes);
        FastMovedBlocks fast = new FastMovedBlocks();
        fast.addLocation(datanodes);

        assert slow.locations.size() == D : "slow size: " + slow.locations.size();
        assert fast.locations.size() == D : "fast size: " + fast.locations.size();
        System.out.println("PASS: correctness — locations dedup preserves D=" + D + " unique datanodes");
    }

    // ---------- Performance ----------
    static void testPerformance() {
        int N = 2000; // simulate 2000 blocks per source node
        int[] blocks = new int[N];
        for (int i = 0; i < N; i++) blocks[i] = i;

        long slowOps = new SlowSrcBlocks().receiveBlocks(blocks);
        long fastOps = new FastSrcBlocks().receiveBlocks(blocks);

        System.out.printf("srcBlocks N=%d: slow_ops=%,d fast_ops=%,d ratio=%.1fx%n",
            N, slowOps, fastOps, (double) slowOps / fastOps);
        assert slowOps >= fastOps * 10 :
            "Expected >=10x slower for slow path, got ratio=" + (slowOps / fastOps);
        System.out.println("PASS: performance — srcBlocks ratio >= 10x");
    }

    static void testMovedBlocksPerformance() {
        // Simulate a block with many replicas (e.g., 100 datanodes, called many times)
        int BLOCKS = 500;
        int D = 20; // datanodes per block (with some duplicates)

        long slowTotal = 0, fastTotal = 0;
        for (int b = 0; b < BLOCKS; b++) {
            int[] datanodes = new int[D];
            for (int i = 0; i < D; i++) datanodes[i] = i % 10; // lots of duplicates
            slowTotal += new SlowMovedBlocks().addLocation(datanodes);
            fastTotal += new FastMovedBlocks().addLocation(datanodes);
        }

        System.out.printf("locations BLOCKS=%d D=%d: slow_ops=%,d fast_ops=%,d ratio=%.1fx%n",
            BLOCKS, D, slowTotal, fastTotal, (double) slowTotal / fastTotal);
        assert slowTotal >= fastTotal * 5 :
            "Expected >=5x slower for slow path, got ratio=" + (slowTotal / fastTotal);
        System.out.println("PASS: performance — locations ratio >= 5x");
    }

    public static void main(String[] args) {
        testCorrectness();
        testMovedBlocksCorrectness();
        testPerformance();
        testMovedBlocksPerformance();
        System.out.println("ALL PASS (4/4)");
    }
}