260 lines
9.7 KiB
Java
260 lines
9.7 KiB
Java
package unit;
|
||
|
||
import java.util.ArrayList;
|
||
import java.util.HashSet;
|
||
|
||
/**
|
||
* PrefectTest
|
||
*
|
||
* Models CWE-407 defects in PrefectHQ/prefect:
|
||
*
|
||
* PRE-001 (HIGH) — cache_policies.py Inputs.compute_key():
|
||
* `for key in inputs: if key not in exclude` where exclude is a list[str].
|
||
* O(N×M) total: N task inputs × M excluded keys per cached task invocation.
|
||
* Fix: frozenset(exclude) for O(1) average membership.
|
||
*
|
||
* PRE-002 (LOW) — deployments/steps/core.py run_steps():
|
||
* `for warning in w: if message not in printed_messages` where printed_messages
|
||
* is a list. O(W²) warning deduplication.
|
||
* Fix: set for O(1) membership.
|
||
*
|
||
* All measurements are instrumented operation counts, not wall-clock timing.
|
||
*/
|
||
public class PrefectTest {
|
||
|
||
// -----------------------------------------------------------------------
|
||
// PRE-001 modelling helpers
|
||
//
|
||
// Defective: ArrayList (exclude list) membership — O(M) scan per input key
|
||
// Fixed: HashSet (frozenset(exclude)) membership — O(1) average
|
||
//
|
||
// Returns total membership-test cost for compute_key() with N inputs, M excludes.
|
||
// -----------------------------------------------------------------------
|
||
|
||
/**
|
||
* Models Inputs.compute_key() — defective path.
|
||
*
|
||
* exclude is a List<String>; `key not in exclude` inside for-key-in-inputs
|
||
* costs O(M) per key → O(N×M) total per compute_key() call.
|
||
*/
|
||
static long pre001Defective(int numInputs, int numExcludes) {
|
||
// Build exclude list
|
||
ArrayList<String> exclude = new ArrayList<>();
|
||
for (int i = 0; i < numExcludes; i++) {
|
||
exclude.add("exclude_key_" + i);
|
||
}
|
||
|
||
// Build inputs (all keys are unique, none match excludes for worst case)
|
||
ArrayList<String> inputKeys = new ArrayList<>();
|
||
for (int i = 0; i < numInputs; i++) {
|
||
inputKeys.add("input_key_" + i);
|
||
}
|
||
|
||
long comparisons = 0;
|
||
for (String key : inputKeys) {
|
||
// model `if key not in exclude` — O(M) linear scan
|
||
comparisons += exclude.size(); // worst-case scan cost
|
||
// actual check (correctness)
|
||
if (!exclude.contains(key)) {
|
||
// would add to hashed_inputs
|
||
}
|
||
}
|
||
return comparisons;
|
||
}
|
||
|
||
/**
|
||
* Models Inputs.compute_key() — fixed path.
|
||
*
|
||
* exclude is a HashSet (frozenset equivalent); `key not in exclude` is O(1).
|
||
*/
|
||
static long pre001Fixed(int numInputs, int numExcludes) {
|
||
HashSet<String> excludeSet = new HashSet<>();
|
||
for (int i = 0; i < numExcludes; i++) {
|
||
excludeSet.add("exclude_key_" + i);
|
||
}
|
||
|
||
ArrayList<String> inputKeys = new ArrayList<>();
|
||
for (int i = 0; i < numInputs; i++) {
|
||
inputKeys.add("input_key_" + i);
|
||
}
|
||
|
||
long lookups = 0;
|
||
for (String key : inputKeys) {
|
||
// model `if key not in exclude_set` — O(1)
|
||
lookups++; // one hash lookup per input key
|
||
if (!excludeSet.contains(key)) {
|
||
// would add to hashed_inputs
|
||
}
|
||
}
|
||
return lookups;
|
||
}
|
||
|
||
// -----------------------------------------------------------------------
|
||
// PRE-002 modelling helpers
|
||
//
|
||
// Defective: ArrayList (printed_messages) membership — O(W) scan per warning
|
||
// Fixed: HashSet membership — O(1)
|
||
//
|
||
// Returns total deduplication cost for W warnings.
|
||
// -----------------------------------------------------------------------
|
||
|
||
static long pre002Defective(int numWarnings) {
|
||
ArrayList<String> printedMessages = new ArrayList<>();
|
||
long scans = 0;
|
||
|
||
for (int i = 0; i < numWarnings; i++) {
|
||
String message = "DeprecationWarning: deprecated_function_" + (i % (numWarnings / 2));
|
||
// model `if message not in printed_messages` — O(W) scan
|
||
scans += printedMessages.size(); // linear scan cost
|
||
if (!printedMessages.contains(message)) {
|
||
printedMessages.add(message);
|
||
}
|
||
}
|
||
return scans;
|
||
}
|
||
|
||
static long pre002Fixed(int numWarnings) {
|
||
HashSet<String> printedSet = new HashSet<>();
|
||
long lookups = 0;
|
||
|
||
for (int i = 0; i < numWarnings; i++) {
|
||
String message = "DeprecationWarning: deprecated_function_" + (i % (numWarnings / 2));
|
||
// model `if message not in printed_set` — O(1)
|
||
lookups++;
|
||
if (!printedSet.contains(message)) {
|
||
printedSet.add(message);
|
||
}
|
||
}
|
||
return lookups;
|
||
}
|
||
|
||
// -----------------------------------------------------------------------
|
||
// Test 1 — PRE-001: defective O(N×M) vs fixed O(N) at N=100, M=50
|
||
// -----------------------------------------------------------------------
|
||
|
||
static void test1_pre001_excludeListScan() {
|
||
int N = 100; // task inputs
|
||
int M = 50; // exclude keys
|
||
|
||
long defectCost = pre001Defective(N, M);
|
||
long fixedCost = pre001Fixed(N, M);
|
||
|
||
System.out.printf("test1 PRE-001: N=%d inputs M=%d excludes defect=%d fixed=%d%n",
|
||
N, M, defectCost, fixedCost);
|
||
|
||
assert defectCost > fixedCost
|
||
: "defect must be more expensive than fix";
|
||
|
||
// defective: N * M
|
||
long expectedDefect = (long) N * M;
|
||
assert defectCost == expectedDefect
|
||
: "expected defect cost=" + expectedDefect + " got=" + defectCost;
|
||
|
||
// fixed: N (one lookup per input)
|
||
assert fixedCost == N
|
||
: "expected fixed cost=" + N + " got=" + fixedCost;
|
||
|
||
double ratio = (double) defectCost / Math.max(1, fixedCost);
|
||
assert ratio > 20.0
|
||
: "expected ratio>20x at N=100 M=50, got " + ratio;
|
||
}
|
||
|
||
// -----------------------------------------------------------------------
|
||
// Test 2 — PRE-001: scaling — doubling M grows defect linearly in M
|
||
// but ratio vs fixed grows proportionally
|
||
// -----------------------------------------------------------------------
|
||
|
||
static void test2_pre001_excludeScaling() {
|
||
int N = 100;
|
||
int M1 = 25;
|
||
int M2 = 50; // double M
|
||
|
||
long d1 = pre001Defective(N, M1);
|
||
long d2 = pre001Defective(N, M2);
|
||
long f1 = pre001Fixed(N, M1);
|
||
long f2 = pre001Fixed(N, M2);
|
||
|
||
double defectGrowth = (double) d2 / Math.max(1, d1);
|
||
double fixedGrowth = (double) f2 / Math.max(1, f1);
|
||
|
||
System.out.printf("test2 PRE-001: N=%d M1=%d M2=%d defect_growth=%.2fx fixed_growth=%.2fx%n",
|
||
N, M1, M2, defectGrowth, fixedGrowth);
|
||
|
||
// defect grows 2x when M doubles (O(N×M))
|
||
assert defectGrowth > 1.8 && defectGrowth < 2.2
|
||
: "defect should grow ~2x when M doubles (O(N×M)), got " + defectGrowth;
|
||
// fixed is constant in M (always O(N))
|
||
assert fixedGrowth >= 0.9 && fixedGrowth <= 1.1
|
||
: "fixed should be constant in M (O(N)), got " + fixedGrowth;
|
||
assert defectGrowth > fixedGrowth
|
||
: "defect growth must exceed fixed growth";
|
||
}
|
||
|
||
// -----------------------------------------------------------------------
|
||
// Test 3 — PRE-001: high-frequency scenario — per-invocation cost matters
|
||
// N=200 inputs, M=100 excludes — typical Prefect ML workflow
|
||
// -----------------------------------------------------------------------
|
||
|
||
static void test3_pre001_highFrequency() {
|
||
int N = 200;
|
||
int M = 100;
|
||
|
||
long defectCost = pre001Defective(N, M);
|
||
long fixedCost = pre001Fixed(N, M);
|
||
|
||
double ratio = (double) defectCost / Math.max(1, fixedCost);
|
||
System.out.printf("test3 PRE-001: N=%d M=%d defect=%d fixed=%d ratio=%.0fx%n",
|
||
N, M, defectCost, fixedCost, ratio);
|
||
|
||
assert defectCost > fixedCost
|
||
: "defect must be more expensive at N=" + N + " M=" + M;
|
||
assert ratio >= 50.0
|
||
: "expected ratio>=50x at N=200 M=100, got " + ratio;
|
||
}
|
||
|
||
// -----------------------------------------------------------------------
|
||
// Test 4 — PRE-002: warning deduplication O(W²) vs O(W)
|
||
// -----------------------------------------------------------------------
|
||
|
||
static void test4_pre002_warningDedup() {
|
||
int W = 100; // warnings per step (stress test — normally <10)
|
||
|
||
long defectCost = pre002Defective(W);
|
||
long fixedCost = pre002Fixed(W);
|
||
|
||
System.out.printf("test4 PRE-002: W=%d warnings defect=%d fixed=%d%n",
|
||
W, defectCost, fixedCost);
|
||
|
||
assert defectCost > fixedCost
|
||
: "defect must be more expensive than fix";
|
||
|
||
double ratio = (double) defectCost / Math.max(1, fixedCost);
|
||
assert ratio > 5.0
|
||
: "expected ratio>5x at W=" + W + ", got " + ratio;
|
||
}
|
||
|
||
// -----------------------------------------------------------------------
|
||
// Main
|
||
// -----------------------------------------------------------------------
|
||
|
||
public static void main(String[] args) {
|
||
System.out.println("=== PrefectTest ===");
|
||
System.out.println("Modelling CWE-407: PRE-001 cache_policies exclude list O(N×M), PRE-002 printed_messages O(W²)");
|
||
System.out.println();
|
||
|
||
test1_pre001_excludeListScan();
|
||
System.out.println(" PASS test1_pre001_excludeListScan");
|
||
|
||
test2_pre001_excludeScaling();
|
||
System.out.println(" PASS test2_pre001_excludeScaling");
|
||
|
||
test3_pre001_highFrequency();
|
||
System.out.println(" PASS test3_pre001_highFrequency");
|
||
|
||
test4_pre002_warningDedup();
|
||
System.out.println(" PASS test4_pre002_warningDedup");
|
||
|
||
System.out.println();
|
||
System.out.println("4/4 PASS");
|
||
}
|
||
}
|