package unit; import java.util.ArrayList; import java.util.HashSet; /** * PrefectTest * * Models CWE-407 defects in PrefectHQ/prefect: * * PRE-001 (HIGH) — cache_policies.py Inputs.compute_key(): * `for key in inputs: if key not in exclude` where exclude is a list[str]. * O(N×M) total: N task inputs × M excluded keys per cached task invocation. * Fix: frozenset(exclude) for O(1) average membership. * * PRE-002 (LOW) — deployments/steps/core.py run_steps(): * `for warning in w: if message not in printed_messages` where printed_messages * is a list. O(W²) warning deduplication. * Fix: set for O(1) membership. * * All measurements are instrumented operation counts, not wall-clock timing. */ public class PrefectTest { // ----------------------------------------------------------------------- // PRE-001 modelling helpers // // Defective: ArrayList (exclude list) membership — O(M) scan per input key // Fixed: HashSet (frozenset(exclude)) membership — O(1) average // // Returns total membership-test cost for compute_key() with N inputs, M excludes. // ----------------------------------------------------------------------- /** * Models Inputs.compute_key() — defective path. * * exclude is a List; `key not in exclude` inside for-key-in-inputs * costs O(M) per key → O(N×M) total per compute_key() call. */ static long pre001Defective(int numInputs, int numExcludes) { // Build exclude list ArrayList exclude = new ArrayList<>(); for (int i = 0; i < numExcludes; i++) { exclude.add("exclude_key_" + i); } // Build inputs (all keys are unique, none match excludes for worst case) ArrayList inputKeys = new ArrayList<>(); for (int i = 0; i < numInputs; i++) { inputKeys.add("input_key_" + i); } long comparisons = 0; for (String key : inputKeys) { // model `if key not in exclude` — O(M) linear scan comparisons += exclude.size(); // worst-case scan cost // actual check (correctness) if (!exclude.contains(key)) { // would add to hashed_inputs } } return comparisons; } /** * Models Inputs.compute_key() — fixed path. * * exclude is a HashSet (frozenset equivalent); `key not in exclude` is O(1). */ static long pre001Fixed(int numInputs, int numExcludes) { HashSet excludeSet = new HashSet<>(); for (int i = 0; i < numExcludes; i++) { excludeSet.add("exclude_key_" + i); } ArrayList inputKeys = new ArrayList<>(); for (int i = 0; i < numInputs; i++) { inputKeys.add("input_key_" + i); } long lookups = 0; for (String key : inputKeys) { // model `if key not in exclude_set` — O(1) lookups++; // one hash lookup per input key if (!excludeSet.contains(key)) { // would add to hashed_inputs } } return lookups; } // ----------------------------------------------------------------------- // PRE-002 modelling helpers // // Defective: ArrayList (printed_messages) membership — O(W) scan per warning // Fixed: HashSet membership — O(1) // // Returns total deduplication cost for W warnings. // ----------------------------------------------------------------------- static long pre002Defective(int numWarnings) { ArrayList printedMessages = new ArrayList<>(); long scans = 0; for (int i = 0; i < numWarnings; i++) { String message = "DeprecationWarning: deprecated_function_" + (i % (numWarnings / 2)); // model `if message not in printed_messages` — O(W) scan scans += printedMessages.size(); // linear scan cost if (!printedMessages.contains(message)) { printedMessages.add(message); } } return scans; } static long pre002Fixed(int numWarnings) { HashSet printedSet = new HashSet<>(); long lookups = 0; for (int i = 0; i < numWarnings; i++) { String message = "DeprecationWarning: deprecated_function_" + (i % (numWarnings / 2)); // model `if message not in printed_set` — O(1) lookups++; if (!printedSet.contains(message)) { printedSet.add(message); } } return lookups; } // ----------------------------------------------------------------------- // Test 1 — PRE-001: defective O(N×M) vs fixed O(N) at N=100, M=50 // ----------------------------------------------------------------------- static void test1_pre001_excludeListScan() { int N = 100; // task inputs int M = 50; // exclude keys long defectCost = pre001Defective(N, M); long fixedCost = pre001Fixed(N, M); System.out.printf("test1 PRE-001: N=%d inputs M=%d excludes defect=%d fixed=%d%n", N, M, defectCost, fixedCost); assert defectCost > fixedCost : "defect must be more expensive than fix"; // defective: N * M long expectedDefect = (long) N * M; assert defectCost == expectedDefect : "expected defect cost=" + expectedDefect + " got=" + defectCost; // fixed: N (one lookup per input) assert fixedCost == N : "expected fixed cost=" + N + " got=" + fixedCost; double ratio = (double) defectCost / Math.max(1, fixedCost); assert ratio > 20.0 : "expected ratio>20x at N=100 M=50, got " + ratio; } // ----------------------------------------------------------------------- // Test 2 — PRE-001: scaling — doubling M grows defect linearly in M // but ratio vs fixed grows proportionally // ----------------------------------------------------------------------- static void test2_pre001_excludeScaling() { int N = 100; int M1 = 25; int M2 = 50; // double M long d1 = pre001Defective(N, M1); long d2 = pre001Defective(N, M2); long f1 = pre001Fixed(N, M1); long f2 = pre001Fixed(N, M2); double defectGrowth = (double) d2 / Math.max(1, d1); double fixedGrowth = (double) f2 / Math.max(1, f1); System.out.printf("test2 PRE-001: N=%d M1=%d M2=%d defect_growth=%.2fx fixed_growth=%.2fx%n", N, M1, M2, defectGrowth, fixedGrowth); // defect grows 2x when M doubles (O(N×M)) assert defectGrowth > 1.8 && defectGrowth < 2.2 : "defect should grow ~2x when M doubles (O(N×M)), got " + defectGrowth; // fixed is constant in M (always O(N)) assert fixedGrowth >= 0.9 && fixedGrowth <= 1.1 : "fixed should be constant in M (O(N)), got " + fixedGrowth; assert defectGrowth > fixedGrowth : "defect growth must exceed fixed growth"; } // ----------------------------------------------------------------------- // Test 3 — PRE-001: high-frequency scenario — per-invocation cost matters // N=200 inputs, M=100 excludes — typical Prefect ML workflow // ----------------------------------------------------------------------- static void test3_pre001_highFrequency() { int N = 200; int M = 100; long defectCost = pre001Defective(N, M); long fixedCost = pre001Fixed(N, M); double ratio = (double) defectCost / Math.max(1, fixedCost); System.out.printf("test3 PRE-001: N=%d M=%d defect=%d fixed=%d ratio=%.0fx%n", N, M, defectCost, fixedCost, ratio); assert defectCost > fixedCost : "defect must be more expensive at N=" + N + " M=" + M; assert ratio >= 50.0 : "expected ratio>=50x at N=200 M=100, got " + ratio; } // ----------------------------------------------------------------------- // Test 4 — PRE-002: warning deduplication O(W²) vs O(W) // ----------------------------------------------------------------------- static void test4_pre002_warningDedup() { int W = 100; // warnings per step (stress test — normally <10) long defectCost = pre002Defective(W); long fixedCost = pre002Fixed(W); System.out.printf("test4 PRE-002: W=%d warnings defect=%d fixed=%d%n", W, defectCost, fixedCost); assert defectCost > fixedCost : "defect must be more expensive than fix"; double ratio = (double) defectCost / Math.max(1, fixedCost); assert ratio > 5.0 : "expected ratio>5x at W=" + W + ", got " + ratio; } // ----------------------------------------------------------------------- // Main // ----------------------------------------------------------------------- public static void main(String[] args) { System.out.println("=== PrefectTest ==="); System.out.println("Modelling CWE-407: PRE-001 cache_policies exclude list O(N×M), PRE-002 printed_messages O(W²)"); System.out.println(); test1_pre001_excludeListScan(); System.out.println(" PASS test1_pre001_excludeListScan"); test2_pre001_excludeScaling(); System.out.println(" PASS test2_pre001_excludeScaling"); test3_pre001_highFrequency(); System.out.println(" PASS test3_pre001_highFrequency"); test4_pre002_warningDedup(); System.out.println(" PASS test4_pre002_warningDedup"); System.out.println(); System.out.println("4/4 PASS"); } }