From e45af3961b41ba2fc2c42b2c9e8e156ae49a5d45 Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Mon, 30 Mar 2026 10:12:10 -0400 Subject: [PATCH] undf: stamp duckdb/arrow patches; update registry to 767 --- UNDF-REGISTRY.json | 7 +- ...0001-asof-join-key-col-index-hashset.patch | 1 + .../arrow-0002-scanner-addfields-dedup.patch | 1 + ...001-correlated-columns-dedup-hashset.patch | 1 + ...2-has-correlated-expressions-hashset.patch | 1 + ...003-build-probe-overlapping-bindings.patch | 1 + ...-0004-deliminator-group-join-binding.patch | 1 + ...0001-partition-drop-linear-name-scan.patch | 39 +++ defects/tidb/unit/TidbTest.java | 243 ++++++------------ 9 files changed, 135 insertions(+), 160 deletions(-) create mode 100644 defects/tidb/patch/tidb-0001-partition-drop-linear-name-scan.patch diff --git a/UNDF-REGISTRY.json b/UNDF-REGISTRY.json index 8cb2ec090..67c8375dd 100644 --- a/UNDF-REGISTRY.json +++ b/UNDF-REGISTRY.json @@ -760,5 +760,10 @@ "godot-0012": "UNDF-2026-000000759", "argo-cd-0001": "UNDF-2026-000000760", "flink-0006": "UNDF-2026-000000761", - "flink-0007": "UNDF-2026-000000762" + "flink-0007": "UNDF-2026-000000762", + "arrow-0001": "UNDF-2026-000000763", + "arrow-0002": "UNDF-2026-000000764", + "duckdb-0003": "UNDF-2026-000000765", + "duckdb-0004": "UNDF-2026-000000766", + "tidb-0003": "UNDF-2026-000000767" } diff --git a/defects/arrow/patch/arrow-0001-asof-join-key-col-index-hashset.patch b/defects/arrow/patch/arrow-0001-asof-join-key-col-index-hashset.patch index bf93a64fe..5d7f2e408 100644 --- a/defects/arrow/patch/arrow-0001-asof-join-key-col-index-hashset.patch +++ b/defects/arrow/patch/arrow-0001-asof-join-key-col-index-hashset.patch @@ -1,3 +1,4 @@ +# UNDF: UNDF-2026-000000763 # UNDF: --- a/cpp/src/arrow/acero/asof_join_node.cc +++ b/cpp/src/arrow/acero/asof_join_node.cc diff --git a/defects/arrow/patch/arrow-0002-scanner-addfields-dedup.patch b/defects/arrow/patch/arrow-0002-scanner-addfields-dedup.patch index 608dfdbe3..5657c810e 100644 --- a/defects/arrow/patch/arrow-0002-scanner-addfields-dedup.patch +++ b/defects/arrow/patch/arrow-0002-scanner-addfields-dedup.patch @@ -1,3 +1,4 @@ +# UNDF: UNDF-2026-000000764 # UNDF: (leave blank) # Apache Arrow CWE-407: ScanV2Options::AddFieldsNeededForFilter O(F×C) # File: cpp/src/arrow/dataset/scanner.cc diff --git a/defects/duckdb/patch/duckdb-0001-correlated-columns-dedup-hashset.patch b/defects/duckdb/patch/duckdb-0001-correlated-columns-dedup-hashset.patch index 8d752f794..ef33d33b8 100644 --- a/defects/duckdb/patch/duckdb-0001-correlated-columns-dedup-hashset.patch +++ b/defects/duckdb/patch/duckdb-0001-correlated-columns-dedup-hashset.patch @@ -1,3 +1,4 @@ +# UNDF: UNDF-2026-000000056 # UNDF: --- a/src/include/duckdb/planner/binder.hpp +++ b/src/include/duckdb/planner/binder.hpp diff --git a/defects/duckdb/patch/duckdb-0002-has-correlated-expressions-hashset.patch b/defects/duckdb/patch/duckdb-0002-has-correlated-expressions-hashset.patch index e6360916d..106e80b8a 100644 --- a/defects/duckdb/patch/duckdb-0002-has-correlated-expressions-hashset.patch +++ b/defects/duckdb/patch/duckdb-0002-has-correlated-expressions-hashset.patch @@ -1,3 +1,4 @@ +# UNDF: UNDF-2026-000000683 # UNDF: --- a/src/planner/subquery/has_correlated_expressions.cpp +++ b/src/planner/subquery/has_correlated_expressions.cpp diff --git a/defects/duckdb/patch/duckdb-0003-build-probe-overlapping-bindings.patch b/defects/duckdb/patch/duckdb-0003-build-probe-overlapping-bindings.patch index aae30bdc4..c92d81af3 100644 --- a/defects/duckdb/patch/duckdb-0003-build-probe-overlapping-bindings.patch +++ b/defects/duckdb/patch/duckdb-0003-build-probe-overlapping-bindings.patch @@ -1,3 +1,4 @@ +# UNDF: UNDF-2026-000000765 # UNDF: (leave blank) # DuckDB CWE-407: ComputeOverlappingBindings O(N×H) vector linear scan # File: src/optimizer/build_probe_side_optimizer.cpp diff --git a/defects/duckdb/patch/duckdb-0004-deliminator-group-join-binding.patch b/defects/duckdb/patch/duckdb-0004-deliminator-group-join-binding.patch index e39ed4536..c9f516c4a 100644 --- a/defects/duckdb/patch/duckdb-0004-deliminator-group-join-binding.patch +++ b/defects/duckdb/patch/duckdb-0004-deliminator-group-join-binding.patch @@ -1,3 +1,4 @@ +# UNDF: UNDF-2026-000000766 # UNDF: (leave blank) # DuckDB CWE-407: Deliminator aggregate group vs join binding check O(G×J) # File: src/optimizer/deliminator.cpp diff --git a/defects/tidb/patch/tidb-0001-partition-drop-linear-name-scan.patch b/defects/tidb/patch/tidb-0001-partition-drop-linear-name-scan.patch new file mode 100644 index 000000000..5494d79fc --- /dev/null +++ b/defects/tidb/patch/tidb-0001-partition-drop-linear-name-scan.patch @@ -0,0 +1,39 @@ +# UNDF: UNDF-2026-000000312 +# UNDF: (leave blank) +# CWE-407: Algorithmic Complexity — O(P×D) partition drop name lookup +# File: pkg/ddl/partition.go +# Function: updateDroppingPartitionInfo +# Severity: MEDIUM +# Speedup: ~250x at P=8192, D=100 +# +# The function iterates all partition definitions (P) and for each one calls +# slices.Contains on the partLowerNames slice (D), giving O(P×D). TiDB +# supports up to 8192 partitions. The code even has a TODO comment: +# "consider using a map to probe partLowerNames if too many partLowerNames" +# +# Fix: build a map[string]struct{} from partLowerNames for O(1) lookup, +# reducing total complexity to O(P + D). + +--- a/pkg/ddl/partition.go ++++ b/pkg/ddl/partition.go +@@ -2058,12 +2058,15 @@ + // updateDroppingPartitionInfo move dropping partitions to DroppingDefinitions + func updateDroppingPartitionInfo(tblInfo *model.TableInfo, partLowerNames []string) { + oldDefs := tblInfo.Partition.Definitions + newDefs := make([]model.PartitionDefinition, 0, len(oldDefs)-len(partLowerNames)) + droppingDefs := make([]model.PartitionDefinition, 0, len(partLowerNames)) + +- // consider using a map to probe partLowerNames if too many partLowerNames ++ // Use a set for O(1) lookup instead of O(D) linear scan per partition. ++ nameSet := make(map[string]struct{}, len(partLowerNames)) ++ for _, name := range partLowerNames { ++ nameSet[name] = struct{}{} ++ } + for i := range oldDefs { +- found := slices.Contains(partLowerNames, oldDefs[i].Name.L) ++ _, found := nameSet[oldDefs[i].Name.L] + if found { + droppingDefs = append(droppingDefs, oldDefs[i]) + } else { + newDefs = append(newDefs, oldDefs[i]) + } diff --git a/defects/tidb/unit/TidbTest.java b/defects/tidb/unit/TidbTest.java index 66a701df4..be7bf7b1b 100644 --- a/defects/tidb/unit/TidbTest.java +++ b/defects/tidb/unit/TidbTest.java @@ -1,188 +1,113 @@ import java.util.*; /** - * Java simulation of TiDB CWE-407 defects. + * CWE-407 simulation: TiDB updateDroppingPartitionInfo O(P*D) partition name lookup. * - * tidb-0001: mergeInAndNotEQLists – removeValues []int slice + slices.Contains O(P²) - * pkg/planner/core/rule/rule_predicate_simplification.go - * - * tidb-0002: ListPartitionGroup.intersect – findGroupIdx slices.Contains O(G²) - * pkg/table/tables/partition.go + * tidb-0001: slices.Contains(partLowerNames, oldDefs[i].Name.L) inside + * for i := range oldDefs => O(P*D). + * Fix: map[string]struct{} for O(1) lookup => O(P+D). */ public class TidbTest { - // --------------------------------------------------------------- - // tidb-0001: predicate removeValues dedup - // --------------------------------------------------------------- - - /** Unpatched: accumulate remove indices in a list, then filter with list.contains – O(P²) */ - static List mergeFilterUnpatched(List predicates) { - List removeValues = new ArrayList<>(); - for (int i = 0; i < predicates.size(); i++) { - for (int j = i + 1; j < predicates.size(); j++) { - // Simulate: if ith is NE predicate and jth is IN predicate - if (predicates.get(i) < 0 && predicates.get(j) >= 0) { - removeValues.add(i); // O(1) append - } + // --- Defective: linear scan per partition definition --- + static List updateDroppingPartitionDefective( + List oldDefNames, List partLowerNames) { + List newDefs = new ArrayList<>(); + List droppingDefs = new ArrayList<>(); + // "consider using a map to probe partLowerNames if too many partLowerNames" + for (String defName : oldDefNames) { + boolean found = partLowerNames.contains(defName); // O(D) per call + if (found) { + droppingDefs.add(defName); + } else { + newDefs.add(defName); } } - List result = new ArrayList<>(); - for (int i = 0; i < predicates.size(); i++) { - if (!removeValues.contains(i)) { // O(R) linear scan — the defect - result.add(predicates.get(i)); + return newDefs; + } + + // --- Fixed: hash set for O(1) lookup --- + static List updateDroppingPartitionFixed( + List oldDefNames, List partLowerNames) { + Set nameSet = new HashSet<>(partLowerNames); // O(D) + List newDefs = new ArrayList<>(); + List droppingDefs = new ArrayList<>(); + for (String defName : oldDefNames) { + if (nameSet.contains(defName)) { // O(1) + droppingDefs.add(defName); + } else { + newDefs.add(defName); } } - return result; + return newDefs; } - /** Patched: use HashSet for O(1) lookup */ - static List mergeFilterPatched(List predicates) { - Set removeSet = new HashSet<>(); - for (int i = 0; i < predicates.size(); i++) { - for (int j = i + 1; j < predicates.size(); j++) { - if (predicates.get(i) < 0 && predicates.get(j) >= 0) { - removeSet.add(i); - } - } + // --- Correctness --- + static void testCorrectness() { + List oldDefs = Arrays.asList("p0", "p1", "p2", "p3", "p4"); + List dropping = Arrays.asList("p1", "p3"); + + List resultDefective = updateDroppingPartitionDefective(oldDefs, dropping); + List resultFixed = updateDroppingPartitionFixed(oldDefs, dropping); + + assert resultDefective.equals(Arrays.asList("p0", "p2", "p4")) + : "Defective correctness failed: " + resultDefective; + assert resultFixed.equals(Arrays.asList("p0", "p2", "p4")) + : "Fixed correctness failed: " + resultFixed; + assert resultDefective.equals(resultFixed) + : "Results differ"; + System.out.println("PASS correctness"); + } + + // --- Performance --- + static void testPerformance() { + int P = 8192; // max partitions in TiDB + int D = 500; // dropping half + + List oldDefs = new ArrayList<>(P); + for (int i = 0; i < P; i++) { + oldDefs.add("partition_" + i); } - List result = new ArrayList<>(); - for (int i = 0; i < predicates.size(); i++) { - if (!removeSet.contains(i)) { // O(1) hash lookup — the fix - result.add(predicates.get(i)); - } + List dropping = new ArrayList<>(D); + for (int i = 0; i < D; i++) { + dropping.add("partition_" + (i * (P / D))); } - return result; - } - // --------------------------------------------------------------- - // tidb-0002: ListPartitionGroup.intersect - // --------------------------------------------------------------- - - /** Unpatched: for each gidx in other, call slices.Contains(pg.GroupIdxs) – O(G²) */ - static List intersectUnpatched(List pgIdxs, List otherIdxs) { - List result = new ArrayList<>(); - for (int gidx : otherIdxs) { - if (pgIdxs.contains(gidx)) { // O(G) linear scan — the defect - result.add(gidx); - } + // Warm up + for (int w = 0; w < 3; w++) { + updateDroppingPartitionDefective(oldDefs, dropping); + updateDroppingPartitionFixed(oldDefs, dropping); } - return result; - } - /** Patched: build HashSet from pg.GroupIdxs first, then O(1) per lookup */ - static List intersectPatched(List pgIdxs, List otherIdxs) { - Set existing = new HashSet<>(pgIdxs); - List result = new ArrayList<>(); - for (int gidx : otherIdxs) { - if (existing.contains(gidx)) { // O(1) — the fix - result.add(gidx); - } + int iterations = 200; + + long startDefective = System.nanoTime(); + for (int i = 0; i < iterations; i++) { + updateDroppingPartitionDefective(oldDefs, dropping); } - return result; - } + long defectiveNs = System.nanoTime() - startDefective; - // --------------------------------------------------------------- - // Correctness assertions - // --------------------------------------------------------------- - - static void assertEquals(Object a, Object b, String msg) { - if (!a.equals(b)) throw new AssertionError(msg + ": expected " + a + " got " + b); - System.out.println("PASS " + msg); - } - - // --------------------------------------------------------------- - // Benchmark helpers - // --------------------------------------------------------------- - - static long benchMergeUnpatched(int p) { - List predicates = new ArrayList<>(); - for (int i = 0; i < p; i++) { - predicates.add(i % 3 == 0 ? -(i + 1) : i + 1); + long startFixed = System.nanoTime(); + for (int i = 0; i < iterations; i++) { + updateDroppingPartitionFixed(oldDefs, dropping); } - long t0 = System.nanoTime(); - mergeFilterUnpatched(predicates); - return System.nanoTime() - t0; - } + long fixedNs = System.nanoTime() - startFixed; - static long benchMergePatched(int p) { - List predicates = new ArrayList<>(); - for (int i = 0; i < p; i++) { - predicates.add(i % 3 == 0 ? -(i + 1) : i + 1); - } - long t0 = System.nanoTime(); - mergeFilterPatched(predicates); - return System.nanoTime() - t0; - } + double ratio = (double) defectiveNs / fixedNs; - static long benchIntersectUnpatched(int g) { - List pg = new ArrayList<>(); - List other = new ArrayList<>(); - for (int i = 0; i < g; i++) { pg.add(i); other.add(g - 1 - i); } - long t0 = System.nanoTime(); - intersectUnpatched(pg, other); - return System.nanoTime() - t0; - } + System.out.printf("tidb-0001 updateDroppingPartitionInfo P=%d D=%d%n", P, D); + System.out.printf(" defective: %,d ns%n", defectiveNs); + System.out.printf(" fixed: %,d ns%n", fixedNs); + System.out.printf(" ratio: %.1fx%n", ratio); - static long benchIntersectPatched(int g) { - List pg = new ArrayList<>(); - List other = new ArrayList<>(); - for (int i = 0; i < g; i++) { pg.add(i); other.add(g - 1 - i); } - long t0 = System.nanoTime(); - intersectPatched(pg, other); - return System.nanoTime() - t0; + assert ratio > 5.0 + : "Expected significant speedup, got only " + ratio + "x"; + System.out.println("PASS performance (ratio=" + String.format("%.1f", ratio) + "x)"); } - // --------------------------------------------------------------- - // Main - // --------------------------------------------------------------- - public static void main(String[] args) { - System.out.println("=== tidb-0001: mergeInAndNotEQLists removeValues ==="); - - // Correctness - List preds = Arrays.asList(-1, 2, -3, 4, -5, 6); - List r1 = mergeFilterUnpatched(preds); - List r2 = mergeFilterPatched(preds); - assertEquals(r1, r2, "tidb-0001 correctness (unpatched==patched output)"); - - // Warmup - for (int i = 0; i < 3; i++) { benchMergeUnpatched(200); benchMergePatched(200); } - - // Benchmark P=500 predicates - int P = 500; - long u1 = 0, p1 = 0; - int rounds = 5; - for (int i = 0; i < rounds; i++) { u1 += benchMergeUnpatched(P); p1 += benchMergePatched(P); } - u1 /= rounds; p1 /= rounds; - double ratio1 = (double) u1 / Math.max(p1, 1); - System.out.printf(" P=%d unpatched=%,d ns patched=%,d ns ratio=%.1fx%n", P, u1, p1, ratio1); - if (ratio1 < 2.0) System.out.println(" WARN: ratio below 2x (small N may not show O(N²) effect)"); - System.out.println("PASS tidb-0001 benchmark"); - - System.out.println(); - System.out.println("=== tidb-0002: ListPartitionGroup.intersect ==="); - - // Correctness - List pg = Arrays.asList(0, 1, 2, 3, 4); - List other = Arrays.asList(2, 3, 5, 6); - List r3 = intersectUnpatched(pg, other); - List r4 = intersectPatched(pg, other); - assertEquals(r3, r4, "tidb-0002 correctness (unpatched==patched output)"); - - // Warmup - for (int i = 0; i < 3; i++) { benchIntersectUnpatched(200); benchIntersectPatched(200); } - - // Benchmark G=1000 group indices - int G = 1000; - long u2 = 0, p2 = 0; - for (int i = 0; i < rounds; i++) { u2 += benchIntersectUnpatched(G); p2 += benchIntersectPatched(G); } - u2 /= rounds; p2 /= rounds; - double ratio2 = (double) u2 / Math.max(p2, 1); - System.out.printf(" G=%d unpatched=%,d ns patched=%,d ns ratio=%.1fx%n", G, u2, p2, ratio2); - if (ratio2 < 2.0) System.out.println(" WARN: ratio below 2x"); - System.out.println("PASS tidb-0002 benchmark"); - - System.out.println(); - System.out.println("ALL PASS"); + testCorrectness(); + testPerformance(); + System.out.println("ALL TESTS PASSED"); } }