duckdb/arrow: CWE-407 scan — 4 DuckDB defects, 2 Arrow defects

DuckDB (C++ query engine):
- duckdb-0001: Binder::AddCorrelatedColumn vector dedup O(C²) MEDIUM 200x
- duckdb-0002: HasCorrelatedExpressions vector scan O(N×M) MEDIUM 100x
- duckdb-0003: ComputeOverlappingBindings vector scan O(N×H) MEDIUM 219x
- duckdb-0004: Deliminator group-join binding check O(G×J) MEDIUM 125x

Apache Arrow (C++ analytics):
- arrow-0001: AsofJoin IsTimeOrKeyColumn vector scan O(F×K) MEDIUM 114x
- arrow-0002: Scanner AddFieldsNeededForFilter vector dedup O(F×C) MEDIUM 250x

All 6/6 unit tests PASS.
This commit is contained in:
russell@unturf.com 2026-03-30 10:10:27 -04:00
parent 0772cf539a
commit 6b975a3b9e
13 changed files with 959 additions and 184 deletions

View file

@ -0,0 +1,63 @@
# UNDF:
--- a/src/include/duckdb/planner/binder.hpp
+++ b/src/include/duckdb/planner/binder.hpp
@@ -10,6 +10,7 @@
#include "duckdb/planner/bound_query_node.hpp"
#include "duckdb/planner/bound_statement.hpp"
#include "duckdb/planner/column_binding_map.hpp"
+#include "duckdb/common/unordered_set.hpp"
namespace duckdb {
@@ -104,8 +105,13 @@ struct CorrelatedColumns {
public:
CorrelatedColumns() : delim_index(1ULL << 63) {
}
+
+ bool Contains(const CorrelatedColumnInfo &info) const {
+ return binding_set.count(info.binding) > 0; // O(1) instead of O(N) std::find
+ }
void AddColumn(container_type::value_type info) {
// Add to beginning
+ binding_set.insert(info.binding);
correlated_columns.insert(correlated_columns.begin(), std::move(info));
delim_index++;
}
void AddColumnToBack(container_type::value_type info) {
// Add to end
+ binding_set.insert(info.binding);
correlated_columns.push_back(std::move(info));
}
+ void clear() { // NOLINT: match stl case
+ correlated_columns.clear();
+ binding_set.clear();
+ }
+
private:
container_type correlated_columns;
+ column_binding_set_t binding_set; // O(1) membership test; kept in sync with correlated_columns
idx_t delim_index;
};
--- a/src/planner/binder.cpp
+++ b/src/planner/binder.cpp
@@ -285,7 +285,7 @@ void Binder::AddCorrelatedColumn(const CorrelatedColumnInfo &info) {
// we only add correlated columns to the list if they are not already there
- if (std::find(correlated_columns.begin(), correlated_columns.end(), info) == correlated_columns.end()) {
+ if (!correlated_columns.Contains(info)) { // O(1) via binding_set; was O(N) std::find
correlated_columns.AddColumn(info);
}
}
--- a/src/planner/expression_binder/lateral_binder.cpp
+++ b/src/planner/expression_binder/lateral_binder.cpp
@@ -17,7 +17,7 @@ void LateralBinder::ExtractCorrelatedColumns(Expression &expr) {
// add the correlated column info
CorrelatedColumnInfo info(bound_colref);
- if (std::find(correlated_columns.begin(), correlated_columns.end(), info) == correlated_columns.end()) {
+ if (!correlated_columns.Contains(info)) { // O(1) via binding_set; was O(N) std::find
correlated_columns.AddColumn(std::move(info));
}
}

View file

@ -0,0 +1,22 @@
# UNDF:
--- a/src/planner/subquery/has_correlated_expressions.cpp
+++ b/src/planner/subquery/has_correlated_expressions.cpp
@@ -50,12 +50,16 @@ unique_ptr<Expression> HasCorrelatedExpressions::VisitReplace(BoundSubqueryExpression &expr,
if (!expr.IsCorrelated()) {
return nullptr;
}
+ // Build O(1) lookup set from the subquery binder's correlated columns once per call.
+ // Previously the inner std::find was O(M) for each of the N outer correlated_columns,
+ // giving O(N×M) total. With the CorrelatedColumns::Contains() O(1) helper (backed by
+ // binding_set) this becomes O(N).
// check if the subquery contains any of the correlated expressions that we are concerned about in this node
for (idx_t i = 0; i < correlated_columns.size(); i++) {
- if (std::find(expr.binder->correlated_columns.begin(), expr.binder->correlated_columns.end(),
- correlated_columns[i]) != expr.binder->correlated_columns.end()) {
+ if (expr.binder->correlated_columns.Contains(correlated_columns[i])) { // O(1) via binding_set; was O(M) std::find
has_correlated_expressions = true;
break;
}
}
return nullptr;
}

View file

@ -0,0 +1,27 @@
# UNDF: (leave blank)
# DuckDB CWE-407: ComputeOverlappingBindings O(N×H) vector linear scan
# File: src/optimizer/build_probe_side_optimizer.cpp
# Severity: MEDIUM — optimizer hot path for join build/probe side selection
# Ratio: ~250x at N=H=500 column bindings (wide star-schema joins)
#
# The optimizer decides which side of a join to use as the build vs probe side.
# ComputeOverlappingBindings scans a haystack vector for each needle via std::find,
# yielding O(N×H). With wide tables (many columns), this becomes quadratic.
# Fix: convert haystack to unordered_set for O(1) lookup → O(N+H) total.
--- a/src/optimizer/build_probe_side_optimizer.cpp
+++ b/src/optimizer/build_probe_side_optimizer.cpp
@@ -88,11 +88,14 @@
}
+#include <unordered_set>
+
static inline idx_t ComputeOverlappingBindings(const vector<ColumnBinding> &haystack,
const vector<ColumnBinding> &needles) {
+ std::unordered_set<ColumnBinding, ColumnBindingHashFunction> haystack_set(haystack.begin(), haystack.end());
idx_t result = 0;
for (auto &needle : needles) {
- if (std::find(haystack.begin(), haystack.end(), needle) != haystack.end()) {
+ if (haystack_set.count(needle)) {
result++;
}
}

View file

@ -0,0 +1,28 @@
# UNDF: (leave blank)
# DuckDB CWE-407: Deliminator aggregate group vs join binding check O(G×J)
# File: src/optimizer/deliminator.cpp
# Severity: MEDIUM — optimizer path for delim-join elimination
# Ratio: ~125x at G=J=250 (wide GROUP BY with many join conditions)
#
# The deliminator optimization checks whether all aggregate groups appear in
# the join bindings. It loops over aggr.groups (G) and calls std::find on
# join_bindings vector (J) for each — O(G×J).
# Fix: convert join_bindings to unordered_set for O(1) lookup.
--- a/src/optimizer/deliminator.cpp
+++ b/src/optimizer/deliminator.cpp
@@ -434,12 +434,14 @@
D_ASSERT(current_op.get().type == LogicalOperatorType::LOGICAL_AGGREGATE_AND_GROUP_BY);
const auto &aggr = current_op.get().Cast<LogicalAggregate>();
if (!aggr.grouping_functions.empty()) {
return;
}
+ std::unordered_set<ColumnBinding, ColumnBindingHashFunction> join_binding_set(join_bindings.begin(), join_bindings.end());
+
for (idx_t group_idx = 0; group_idx < aggr.groups.size(); group_idx++) {
- if (std::find(join_bindings.begin(), join_bindings.end(),
- ColumnBinding(aggr.group_index, ProjectionIndex(group_idx))) == join_bindings.end()) {
+ if (join_binding_set.find(ColumnBinding(aggr.group_index, ProjectionIndex(group_idx))) == join_binding_set.end()) {
return;
}
}