226 lines
8.7 KiB
Java
226 lines
8.7 KiB
Java
package unit;
|
||
|
||
// NO JUnit. Compile: javac -d . *.java Run: java -ea unit.SegmentProcessorUtilsAlgorithm
|
||
//
|
||
// Pinot pinot-0001: SegmentProcessorUtils sortOrder List.contains per field O(F×S) → O(F)
|
||
//
|
||
// Simulates: for each field in schema, check if it is in the sort order list
|
||
//
|
||
// SLOW: List<String>.contains(fieldName) per field — O(F × S)
|
||
// FAST: HashSet<String> built once from sortOrder, .contains() — O(F)
|
||
|
||
import java.util.ArrayList;
|
||
import java.util.HashSet;
|
||
import java.util.List;
|
||
import java.util.Set;
|
||
|
||
public class SegmentProcessorUtilsAlgorithm {
|
||
|
||
// Simulates the defective getFieldSpecs() classification loop:
|
||
// for each schema field, check if it is in the sort order using List.contains
|
||
static class SlowImpl {
|
||
long opCount = 0;
|
||
|
||
// Returns pair of lists: [metricFields, nonMetricFields] excluding sort fields
|
||
// O(F × S) where F=schema fields, S=sort order length
|
||
List<List<String>> classifyFields(List<String> allFields, List<String> sortOrder,
|
||
Set<String> metricFields) {
|
||
List<String> metrics = new ArrayList<>();
|
||
List<String> nonMetrics = new ArrayList<>();
|
||
|
||
for (String field : allFields) {
|
||
// Simulate List<String>.contains(): linear scan over sortOrder
|
||
boolean inSortOrder = false;
|
||
for (int i = 0; i < sortOrder.size(); i++) {
|
||
opCount++;
|
||
if (sortOrder.get(i).equals(field)) {
|
||
inSortOrder = true;
|
||
break;
|
||
}
|
||
}
|
||
if (!inSortOrder) {
|
||
if (metricFields.contains(field)) {
|
||
metrics.add(field);
|
||
} else {
|
||
nonMetrics.add(field);
|
||
}
|
||
}
|
||
}
|
||
|
||
List<List<String>> result = new ArrayList<>();
|
||
result.add(metrics);
|
||
result.add(nonMetrics);
|
||
return result;
|
||
}
|
||
}
|
||
|
||
// Simulates the fixed HashSet pattern
|
||
static class FastImpl {
|
||
long opCount = 0;
|
||
|
||
// Returns pair of lists: [metricFields, nonMetricFields] excluding sort fields
|
||
// O(F) after O(S) set construction
|
||
List<List<String>> classifyFields(List<String> allFields, List<String> sortOrder,
|
||
Set<String> metricFields) {
|
||
// Build O(1) lookup set once
|
||
Set<String> sortOrderSet = new HashSet<>(sortOrder);
|
||
opCount += sortOrder.size(); // count set-build cost
|
||
|
||
List<String> metrics = new ArrayList<>();
|
||
List<String> nonMetrics = new ArrayList<>();
|
||
|
||
for (String field : allFields) {
|
||
opCount++;
|
||
if (!sortOrderSet.contains(field)) {
|
||
if (metricFields.contains(field)) {
|
||
metrics.add(field);
|
||
} else {
|
||
nonMetrics.add(field);
|
||
}
|
||
}
|
||
}
|
||
|
||
List<List<String>> result = new ArrayList<>();
|
||
result.add(metrics);
|
||
result.add(nonMetrics);
|
||
return result;
|
||
}
|
||
}
|
||
|
||
static List<String> buildSchema(int F) {
|
||
List<String> fields = new ArrayList<>();
|
||
for (int i = 0; i < F; i++) {
|
||
fields.add("field_" + i);
|
||
}
|
||
return fields;
|
||
}
|
||
|
||
static List<String> buildSortOrder(List<String> schema, int S) {
|
||
List<String> sortOrder = new ArrayList<>();
|
||
// sort columns are at the END of the schema — worst case for linear scan
|
||
// (every field must scan to the end to determine non-membership)
|
||
for (int i = schema.size() - S; i < schema.size(); i++) {
|
||
sortOrder.add(schema.get(i));
|
||
}
|
||
return sortOrder;
|
||
}
|
||
|
||
static Set<String> buildMetricFields(List<String> schema, int metricCount) {
|
||
Set<String> metrics = new HashSet<>();
|
||
// metrics are in the middle
|
||
int start = schema.size() / 3;
|
||
for (int i = start; i < start + metricCount && i < schema.size(); i++) {
|
||
metrics.add(schema.get(i));
|
||
}
|
||
return metrics;
|
||
}
|
||
|
||
public static void main(String[] args) {
|
||
int passed = 0;
|
||
int total = 0;
|
||
|
||
// Test 1: correctness — both impls produce same classification
|
||
{
|
||
total++;
|
||
List<String> schema = buildSchema(30);
|
||
List<String> sortOrder = buildSortOrder(schema, 5);
|
||
Set<String> metricFields = buildMetricFields(schema, 8);
|
||
|
||
SlowImpl slow = new SlowImpl();
|
||
FastImpl fast = new FastImpl();
|
||
|
||
List<List<String>> slowResult = slow.classifyFields(schema, sortOrder, metricFields);
|
||
List<List<String>> fastResult = fast.classifyFields(schema, sortOrder, metricFields);
|
||
|
||
assert slowResult.get(0).equals(fastResult.get(0)) :
|
||
"Metric field mismatch: slow=" + slowResult.get(0) + " fast=" + fastResult.get(0);
|
||
assert slowResult.get(1).equals(fastResult.get(1)) :
|
||
"NonMetric field mismatch: slow=" + slowResult.get(1) + " fast=" + fastResult.get(1);
|
||
|
||
// Total classified = F - S
|
||
int expectedCount = schema.size() - sortOrder.size();
|
||
int slowTotal = slowResult.get(0).size() + slowResult.get(1).size();
|
||
assert slowTotal == expectedCount :
|
||
"Expected " + expectedCount + " classified fields, got " + slowTotal;
|
||
|
||
System.out.println("Test 1 PASS: correctness verified, classified=" + slowTotal + " fields");
|
||
passed++;
|
||
}
|
||
|
||
// Test 2: op ratio at F=500 fields, S=20 sort columns
|
||
// Typical high-cardinality OLAP table with event dimensions
|
||
{
|
||
total++;
|
||
int F = 500;
|
||
int S = 20;
|
||
List<String> schema = buildSchema(F);
|
||
List<String> sortOrder = buildSortOrder(schema, S);
|
||
Set<String> metricFields = buildMetricFields(schema, 50);
|
||
|
||
SlowImpl slow = new SlowImpl();
|
||
slow.classifyFields(schema, sortOrder, metricFields);
|
||
long slowOps = slow.opCount;
|
||
|
||
FastImpl fast = new FastImpl();
|
||
fast.classifyFields(schema, sortOrder, metricFields);
|
||
long fastOps = fast.opCount;
|
||
|
||
System.out.println("Test 2: F=" + F + ", S=" + S + ", slow_ops=" + slowOps + ", fast_ops=" + fastOps);
|
||
|
||
// Slow: worst case — sort columns at END, so non-sorted fields scan all S
|
||
// (F-S) fields each scan full S = (F-S)*S ops for non-sorted
|
||
// Sorted fields (S) scan to find themselves = S*(avg S/2) = S*S/2
|
||
// Total ≈ (F-S)*S + S*S/2 ≈ F*S - S*S/2
|
||
long minQuadratic = (long) (F - S) * S;
|
||
assert slowOps >= minQuadratic :
|
||
"slow_ops=" + slowOps + " should be >= " + minQuadratic;
|
||
|
||
// Fast: S (set build) + F (lookups)
|
||
long maxLinear = S + F + 10;
|
||
assert fastOps <= maxLinear :
|
||
"fast_ops=" + fastOps + " should be <= " + maxLinear;
|
||
|
||
assert slowOps >= fastOps * 10 :
|
||
"Expected >= 10x op ratio, got slow=" + slowOps + " fast=" + fastOps;
|
||
|
||
System.out.println("Test 2 PASS: slow_ops=" + slowOps + " >= fast_ops*10 (fast=" + fastOps + ")");
|
||
passed++;
|
||
}
|
||
|
||
// Test 3: large schema F=1000, S=50 (very wide fact table)
|
||
{
|
||
total++;
|
||
int F = 1000;
|
||
int S = 50;
|
||
List<String> schema = buildSchema(F);
|
||
List<String> sortOrder = buildSortOrder(schema, S);
|
||
Set<String> metricFields = buildMetricFields(schema, 100);
|
||
|
||
SlowImpl slow = new SlowImpl();
|
||
slow.classifyFields(schema, sortOrder, metricFields);
|
||
long slowOps = slow.opCount;
|
||
|
||
FastImpl fast = new FastImpl();
|
||
fast.classifyFields(schema, sortOrder, metricFields);
|
||
long fastOps = fast.opCount;
|
||
|
||
System.out.println("Test 3: F=" + F + ", S=" + S + ", slow_ops=" + slowOps + ", fast_ops=" + fastOps);
|
||
|
||
long minQuadratic = (long) (F - S) * S;
|
||
assert slowOps >= minQuadratic :
|
||
"slow_ops=" + slowOps + " should be >= " + minQuadratic;
|
||
|
||
long maxLinear = S + F + 10;
|
||
assert fastOps <= maxLinear :
|
||
"fast_ops=" + fastOps + " should be <= " + maxLinear;
|
||
|
||
assert slowOps >= fastOps * 10 :
|
||
"Expected >= 10x op ratio, got slow=" + slowOps + " fast=" + fastOps;
|
||
|
||
System.out.println("Test 3 PASS: " + (slowOps / fastOps) + "x speedup at F=" + F + ", S=" + S);
|
||
passed++;
|
||
}
|
||
|
||
System.out.println(passed + "/" + total + " PASS");
|
||
}
|
||
}
|