undf: stamp duckdb/arrow patches; update registry to 767
This commit is contained in:
parent
6b975a3b9e
commit
e45af3961b
9 changed files with 135 additions and 160 deletions
|
|
@ -1,188 +1,113 @@
|
|||
import java.util.*;
|
||||
|
||||
/**
|
||||
* Java simulation of TiDB CWE-407 defects.
|
||||
* CWE-407 simulation: TiDB updateDroppingPartitionInfo O(P*D) partition name lookup.
|
||||
*
|
||||
* tidb-0001: mergeInAndNotEQLists – removeValues []int slice + slices.Contains O(P²)
|
||||
* pkg/planner/core/rule/rule_predicate_simplification.go
|
||||
*
|
||||
* tidb-0002: ListPartitionGroup.intersect – findGroupIdx slices.Contains O(G²)
|
||||
* pkg/table/tables/partition.go
|
||||
* tidb-0001: slices.Contains(partLowerNames, oldDefs[i].Name.L) inside
|
||||
* for i := range oldDefs => O(P*D).
|
||||
* Fix: map[string]struct{} for O(1) lookup => O(P+D).
|
||||
*/
|
||||
public class TidbTest {
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// tidb-0001: predicate removeValues dedup
|
||||
// ---------------------------------------------------------------
|
||||
|
||||
/** Unpatched: accumulate remove indices in a list, then filter with list.contains – O(P²) */
|
||||
static List<Integer> mergeFilterUnpatched(List<Integer> predicates) {
|
||||
List<Integer> removeValues = new ArrayList<>();
|
||||
for (int i = 0; i < predicates.size(); i++) {
|
||||
for (int j = i + 1; j < predicates.size(); j++) {
|
||||
// Simulate: if ith is NE predicate and jth is IN predicate
|
||||
if (predicates.get(i) < 0 && predicates.get(j) >= 0) {
|
||||
removeValues.add(i); // O(1) append
|
||||
}
|
||||
// --- Defective: linear scan per partition definition ---
|
||||
static List<String> updateDroppingPartitionDefective(
|
||||
List<String> oldDefNames, List<String> partLowerNames) {
|
||||
List<String> newDefs = new ArrayList<>();
|
||||
List<String> droppingDefs = new ArrayList<>();
|
||||
// "consider using a map to probe partLowerNames if too many partLowerNames"
|
||||
for (String defName : oldDefNames) {
|
||||
boolean found = partLowerNames.contains(defName); // O(D) per call
|
||||
if (found) {
|
||||
droppingDefs.add(defName);
|
||||
} else {
|
||||
newDefs.add(defName);
|
||||
}
|
||||
}
|
||||
List<Integer> result = new ArrayList<>();
|
||||
for (int i = 0; i < predicates.size(); i++) {
|
||||
if (!removeValues.contains(i)) { // O(R) linear scan — the defect
|
||||
result.add(predicates.get(i));
|
||||
return newDefs;
|
||||
}
|
||||
|
||||
// --- Fixed: hash set for O(1) lookup ---
|
||||
static List<String> updateDroppingPartitionFixed(
|
||||
List<String> oldDefNames, List<String> partLowerNames) {
|
||||
Set<String> nameSet = new HashSet<>(partLowerNames); // O(D)
|
||||
List<String> newDefs = new ArrayList<>();
|
||||
List<String> droppingDefs = new ArrayList<>();
|
||||
for (String defName : oldDefNames) {
|
||||
if (nameSet.contains(defName)) { // O(1)
|
||||
droppingDefs.add(defName);
|
||||
} else {
|
||||
newDefs.add(defName);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
return newDefs;
|
||||
}
|
||||
|
||||
/** Patched: use HashSet for O(1) lookup */
|
||||
static List<Integer> mergeFilterPatched(List<Integer> predicates) {
|
||||
Set<Integer> removeSet = new HashSet<>();
|
||||
for (int i = 0; i < predicates.size(); i++) {
|
||||
for (int j = i + 1; j < predicates.size(); j++) {
|
||||
if (predicates.get(i) < 0 && predicates.get(j) >= 0) {
|
||||
removeSet.add(i);
|
||||
}
|
||||
}
|
||||
// --- Correctness ---
|
||||
static void testCorrectness() {
|
||||
List<String> oldDefs = Arrays.asList("p0", "p1", "p2", "p3", "p4");
|
||||
List<String> dropping = Arrays.asList("p1", "p3");
|
||||
|
||||
List<String> resultDefective = updateDroppingPartitionDefective(oldDefs, dropping);
|
||||
List<String> resultFixed = updateDroppingPartitionFixed(oldDefs, dropping);
|
||||
|
||||
assert resultDefective.equals(Arrays.asList("p0", "p2", "p4"))
|
||||
: "Defective correctness failed: " + resultDefective;
|
||||
assert resultFixed.equals(Arrays.asList("p0", "p2", "p4"))
|
||||
: "Fixed correctness failed: " + resultFixed;
|
||||
assert resultDefective.equals(resultFixed)
|
||||
: "Results differ";
|
||||
System.out.println("PASS correctness");
|
||||
}
|
||||
|
||||
// --- Performance ---
|
||||
static void testPerformance() {
|
||||
int P = 8192; // max partitions in TiDB
|
||||
int D = 500; // dropping half
|
||||
|
||||
List<String> oldDefs = new ArrayList<>(P);
|
||||
for (int i = 0; i < P; i++) {
|
||||
oldDefs.add("partition_" + i);
|
||||
}
|
||||
List<Integer> result = new ArrayList<>();
|
||||
for (int i = 0; i < predicates.size(); i++) {
|
||||
if (!removeSet.contains(i)) { // O(1) hash lookup — the fix
|
||||
result.add(predicates.get(i));
|
||||
}
|
||||
List<String> dropping = new ArrayList<>(D);
|
||||
for (int i = 0; i < D; i++) {
|
||||
dropping.add("partition_" + (i * (P / D)));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// tidb-0002: ListPartitionGroup.intersect
|
||||
// ---------------------------------------------------------------
|
||||
|
||||
/** Unpatched: for each gidx in other, call slices.Contains(pg.GroupIdxs) – O(G²) */
|
||||
static List<Integer> intersectUnpatched(List<Integer> pgIdxs, List<Integer> otherIdxs) {
|
||||
List<Integer> result = new ArrayList<>();
|
||||
for (int gidx : otherIdxs) {
|
||||
if (pgIdxs.contains(gidx)) { // O(G) linear scan — the defect
|
||||
result.add(gidx);
|
||||
}
|
||||
// Warm up
|
||||
for (int w = 0; w < 3; w++) {
|
||||
updateDroppingPartitionDefective(oldDefs, dropping);
|
||||
updateDroppingPartitionFixed(oldDefs, dropping);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/** Patched: build HashSet from pg.GroupIdxs first, then O(1) per lookup */
|
||||
static List<Integer> intersectPatched(List<Integer> pgIdxs, List<Integer> otherIdxs) {
|
||||
Set<Integer> existing = new HashSet<>(pgIdxs);
|
||||
List<Integer> result = new ArrayList<>();
|
||||
for (int gidx : otherIdxs) {
|
||||
if (existing.contains(gidx)) { // O(1) — the fix
|
||||
result.add(gidx);
|
||||
}
|
||||
int iterations = 200;
|
||||
|
||||
long startDefective = System.nanoTime();
|
||||
for (int i = 0; i < iterations; i++) {
|
||||
updateDroppingPartitionDefective(oldDefs, dropping);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
long defectiveNs = System.nanoTime() - startDefective;
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// Correctness assertions
|
||||
// ---------------------------------------------------------------
|
||||
|
||||
static void assertEquals(Object a, Object b, String msg) {
|
||||
if (!a.equals(b)) throw new AssertionError(msg + ": expected " + a + " got " + b);
|
||||
System.out.println("PASS " + msg);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// Benchmark helpers
|
||||
// ---------------------------------------------------------------
|
||||
|
||||
static long benchMergeUnpatched(int p) {
|
||||
List<Integer> predicates = new ArrayList<>();
|
||||
for (int i = 0; i < p; i++) {
|
||||
predicates.add(i % 3 == 0 ? -(i + 1) : i + 1);
|
||||
long startFixed = System.nanoTime();
|
||||
for (int i = 0; i < iterations; i++) {
|
||||
updateDroppingPartitionFixed(oldDefs, dropping);
|
||||
}
|
||||
long t0 = System.nanoTime();
|
||||
mergeFilterUnpatched(predicates);
|
||||
return System.nanoTime() - t0;
|
||||
}
|
||||
long fixedNs = System.nanoTime() - startFixed;
|
||||
|
||||
static long benchMergePatched(int p) {
|
||||
List<Integer> predicates = new ArrayList<>();
|
||||
for (int i = 0; i < p; i++) {
|
||||
predicates.add(i % 3 == 0 ? -(i + 1) : i + 1);
|
||||
}
|
||||
long t0 = System.nanoTime();
|
||||
mergeFilterPatched(predicates);
|
||||
return System.nanoTime() - t0;
|
||||
}
|
||||
double ratio = (double) defectiveNs / fixedNs;
|
||||
|
||||
static long benchIntersectUnpatched(int g) {
|
||||
List<Integer> pg = new ArrayList<>();
|
||||
List<Integer> other = new ArrayList<>();
|
||||
for (int i = 0; i < g; i++) { pg.add(i); other.add(g - 1 - i); }
|
||||
long t0 = System.nanoTime();
|
||||
intersectUnpatched(pg, other);
|
||||
return System.nanoTime() - t0;
|
||||
}
|
||||
System.out.printf("tidb-0001 updateDroppingPartitionInfo P=%d D=%d%n", P, D);
|
||||
System.out.printf(" defective: %,d ns%n", defectiveNs);
|
||||
System.out.printf(" fixed: %,d ns%n", fixedNs);
|
||||
System.out.printf(" ratio: %.1fx%n", ratio);
|
||||
|
||||
static long benchIntersectPatched(int g) {
|
||||
List<Integer> pg = new ArrayList<>();
|
||||
List<Integer> other = new ArrayList<>();
|
||||
for (int i = 0; i < g; i++) { pg.add(i); other.add(g - 1 - i); }
|
||||
long t0 = System.nanoTime();
|
||||
intersectPatched(pg, other);
|
||||
return System.nanoTime() - t0;
|
||||
assert ratio > 5.0
|
||||
: "Expected significant speedup, got only " + ratio + "x";
|
||||
System.out.println("PASS performance (ratio=" + String.format("%.1f", ratio) + "x)");
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// Main
|
||||
// ---------------------------------------------------------------
|
||||
|
||||
public static void main(String[] args) {
|
||||
System.out.println("=== tidb-0001: mergeInAndNotEQLists removeValues ===");
|
||||
|
||||
// Correctness
|
||||
List<Integer> preds = Arrays.asList(-1, 2, -3, 4, -5, 6);
|
||||
List<Integer> r1 = mergeFilterUnpatched(preds);
|
||||
List<Integer> r2 = mergeFilterPatched(preds);
|
||||
assertEquals(r1, r2, "tidb-0001 correctness (unpatched==patched output)");
|
||||
|
||||
// Warmup
|
||||
for (int i = 0; i < 3; i++) { benchMergeUnpatched(200); benchMergePatched(200); }
|
||||
|
||||
// Benchmark P=500 predicates
|
||||
int P = 500;
|
||||
long u1 = 0, p1 = 0;
|
||||
int rounds = 5;
|
||||
for (int i = 0; i < rounds; i++) { u1 += benchMergeUnpatched(P); p1 += benchMergePatched(P); }
|
||||
u1 /= rounds; p1 /= rounds;
|
||||
double ratio1 = (double) u1 / Math.max(p1, 1);
|
||||
System.out.printf(" P=%d unpatched=%,d ns patched=%,d ns ratio=%.1fx%n", P, u1, p1, ratio1);
|
||||
if (ratio1 < 2.0) System.out.println(" WARN: ratio below 2x (small N may not show O(N²) effect)");
|
||||
System.out.println("PASS tidb-0001 benchmark");
|
||||
|
||||
System.out.println();
|
||||
System.out.println("=== tidb-0002: ListPartitionGroup.intersect ===");
|
||||
|
||||
// Correctness
|
||||
List<Integer> pg = Arrays.asList(0, 1, 2, 3, 4);
|
||||
List<Integer> other = Arrays.asList(2, 3, 5, 6);
|
||||
List<Integer> r3 = intersectUnpatched(pg, other);
|
||||
List<Integer> r4 = intersectPatched(pg, other);
|
||||
assertEquals(r3, r4, "tidb-0002 correctness (unpatched==patched output)");
|
||||
|
||||
// Warmup
|
||||
for (int i = 0; i < 3; i++) { benchIntersectUnpatched(200); benchIntersectPatched(200); }
|
||||
|
||||
// Benchmark G=1000 group indices
|
||||
int G = 1000;
|
||||
long u2 = 0, p2 = 0;
|
||||
for (int i = 0; i < rounds; i++) { u2 += benchIntersectUnpatched(G); p2 += benchIntersectPatched(G); }
|
||||
u2 /= rounds; p2 /= rounds;
|
||||
double ratio2 = (double) u2 / Math.max(p2, 1);
|
||||
System.out.printf(" G=%d unpatched=%,d ns patched=%,d ns ratio=%.1fx%n", G, u2, p2, ratio2);
|
||||
if (ratio2 < 2.0) System.out.println(" WARN: ratio below 2x");
|
||||
System.out.println("PASS tidb-0002 benchmark");
|
||||
|
||||
System.out.println();
|
||||
System.out.println("ALL PASS");
|
||||
testCorrectness();
|
||||
testPerformance();
|
||||
System.out.println("ALL TESTS PASSED");
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue