engineering/java-topology
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

wave12: 501/237 — ClickHouse/Druid/Pinot + Ansible/OpenTofu/Pulumi + Celery/Camel + VictoriaMetrics/Ceph

Browse source
This commit is contained in:
russell@unturf.com 2026-03-27 17:34:59 -04:00
parent 19333b378e
commit 424a2a7787
31 changed files with 2994 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

226
defects/pinot/unit/SegmentProcessorUtilsAlgorithm.java Normal file
View file

@ -0,0 +1,226 @@
package unit;
// NO JUnit. Compile: javac -d . *.java Run: java -ea unit.SegmentProcessorUtilsAlgorithm
//
// Pinot pinot-0001: SegmentProcessorUtils sortOrder List.contains per field O(F×S) → O(F)
//
// Simulates: for each field in schema, check if it is in the sort order list
//
// SLOW: List<String>.contains(fieldName) per field — O(F × S)
// FAST: HashSet<String> built once from sortOrder, .contains() — O(F)
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
public class SegmentProcessorUtilsAlgorithm {
// Simulates the defective getFieldSpecs() classification loop:
// for each schema field, check if it is in the sort order using List.contains
static class SlowImpl {
long opCount = 0;
// Returns pair of lists: [metricFields, nonMetricFields] excluding sort fields
// O(F × S) where F=schema fields, S=sort order length
List<List<String>> classifyFields(List<String> allFields, List<String> sortOrder,
Set<String> metricFields) {
List<String> metrics = new ArrayList<>();
List<String> nonMetrics = new ArrayList<>();
for (String field : allFields) {
// Simulate List<String>.contains(): linear scan over sortOrder
boolean inSortOrder = false;
for (int i = 0; i < sortOrder.size(); i++) {
opCount++;
if (sortOrder.get(i).equals(field)) {
inSortOrder = true;
break;
}
}
if (!inSortOrder) {
if (metricFields.contains(field)) {
metrics.add(field);
} else {
nonMetrics.add(field);
}
}
}
List<List<String>> result = new ArrayList<>();
result.add(metrics);
result.add(nonMetrics);
return result;
}
}
// Simulates the fixed HashSet pattern
static class FastImpl {
long opCount = 0;
// Returns pair of lists: [metricFields, nonMetricFields] excluding sort fields
// O(F) after O(S) set construction
List<List<String>> classifyFields(List<String> allFields, List<String> sortOrder,
Set<String> metricFields) {
// Build O(1) lookup set once
Set<String> sortOrderSet = new HashSet<>(sortOrder);
opCount += sortOrder.size(); // count set-build cost
List<String> metrics = new ArrayList<>();
List<String> nonMetrics = new ArrayList<>();
for (String field : allFields) {
opCount++;
if (!sortOrderSet.contains(field)) {
if (metricFields.contains(field)) {
metrics.add(field);
} else {
nonMetrics.add(field);
}
}
}
List<List<String>> result = new ArrayList<>();
result.add(metrics);
result.add(nonMetrics);
return result;
}
}
static List<String> buildSchema(int F) {
List<String> fields = new ArrayList<>();
for (int i = 0; i < F; i++) {
fields.add("field_" + i);
}
return fields;
}
static List<String> buildSortOrder(List<String> schema, int S) {
List<String> sortOrder = new ArrayList<>();
// sort columns are at the END of the schema — worst case for linear scan
// (every field must scan to the end to determine non-membership)
for (int i = schema.size() - S; i < schema.size(); i++) {
sortOrder.add(schema.get(i));
}
return sortOrder;
}
static Set<String> buildMetricFields(List<String> schema, int metricCount) {
Set<String> metrics = new HashSet<>();
// metrics are in the middle
int start = schema.size() / 3;
for (int i = start; i < start + metricCount && i < schema.size(); i++) {
metrics.add(schema.get(i));
}
return metrics;
}
public static void main(String[] args) {
int passed = 0;
int total = 0;
// Test 1: correctness — both impls produce same classification
{
total++;
List<String> schema = buildSchema(30);
List<String> sortOrder = buildSortOrder(schema, 5);
Set<String> metricFields = buildMetricFields(schema, 8);
SlowImpl slow = new SlowImpl();
FastImpl fast = new FastImpl();
List<List<String>> slowResult = slow.classifyFields(schema, sortOrder, metricFields);
List<List<String>> fastResult = fast.classifyFields(schema, sortOrder, metricFields);
assert slowResult.get(0).equals(fastResult.get(0)) :
"Metric field mismatch: slow=" + slowResult.get(0) + " fast=" + fastResult.get(0);
assert slowResult.get(1).equals(fastResult.get(1)) :
"NonMetric field mismatch: slow=" + slowResult.get(1) + " fast=" + fastResult.get(1);
// Total classified = F - S
int expectedCount = schema.size() - sortOrder.size();
int slowTotal = slowResult.get(0).size() + slowResult.get(1).size();
assert slowTotal == expectedCount :
"Expected " + expectedCount + " classified fields, got " + slowTotal;
System.out.println("Test 1 PASS: correctness verified, classified=" + slowTotal + " fields");
passed++;
}
// Test 2: op ratio at F=500 fields, S=20 sort columns
// Typical high-cardinality OLAP table with event dimensions
{
total++;
int F = 500;
int S = 20;
List<String> schema = buildSchema(F);
List<String> sortOrder = buildSortOrder(schema, S);
Set<String> metricFields = buildMetricFields(schema, 50);
SlowImpl slow = new SlowImpl();
slow.classifyFields(schema, sortOrder, metricFields);
long slowOps = slow.opCount;
FastImpl fast = new FastImpl();
fast.classifyFields(schema, sortOrder, metricFields);
long fastOps = fast.opCount;
System.out.println("Test 2: F=" + F + ", S=" + S + ", slow_ops=" + slowOps + ", fast_ops=" + fastOps);
// Slow: worst case — sort columns at END, so non-sorted fields scan all S
// (F-S) fields each scan full S = (F-S)*S ops for non-sorted
// Sorted fields (S) scan to find themselves = S*(avg S/2) = S*S/2
// Total ≈ (F-S)*S + S*S/2 ≈ F*S - S*S/2
long minQuadratic = (long) (F - S) * S;
assert slowOps >= minQuadratic :
"slow_ops=" + slowOps + " should be >= " + minQuadratic;
// Fast: S (set build) + F (lookups)
long maxLinear = S + F + 10;
assert fastOps <= maxLinear :
"fast_ops=" + fastOps + " should be <= " + maxLinear;
assert slowOps >= fastOps * 10 :
"Expected >= 10x op ratio, got slow=" + slowOps + " fast=" + fastOps;
System.out.println("Test 2 PASS: slow_ops=" + slowOps + " >= fast_ops*10 (fast=" + fastOps + ")");
passed++;
}
// Test 3: large schema F=1000, S=50 (very wide fact table)
{
total++;
int F = 1000;
int S = 50;
List<String> schema = buildSchema(F);
List<String> sortOrder = buildSortOrder(schema, S);
Set<String> metricFields = buildMetricFields(schema, 100);
SlowImpl slow = new SlowImpl();
slow.classifyFields(schema, sortOrder, metricFields);
long slowOps = slow.opCount;
FastImpl fast = new FastImpl();
fast.classifyFields(schema, sortOrder, metricFields);
long fastOps = fast.opCount;
System.out.println("Test 3: F=" + F + ", S=" + S + ", slow_ops=" + slowOps + ", fast_ops=" + fastOps);
long minQuadratic = (long) (F - S) * S;
assert slowOps >= minQuadratic :
"slow_ops=" + slowOps + " should be >= " + minQuadratic;
long maxLinear = S + F + 10;
assert fastOps <= maxLinear :
"fast_ops=" + fastOps + " should be <= " + maxLinear;
assert slowOps >= fastOps * 10 :
"Expected >= 10x op ratio, got slow=" + slowOps + " fast=" + fastOps;
System.out.println("Test 3 PASS: " + (slowOps / fastOps) + "x speedup at F=" + F + ", S=" + S);
passed++;
}
System.out.println(passed + "/" + total + " PASS");
}
}