45 lines
2.5 KiB
Diff
45 lines
2.5 KiB
Diff
# UNDF: UNDF-2026-000000819
|
|
# CWE-407: SharedWorkOptimizer.mergeSchema() uses List.contains() in loops
|
|
# Severity: MEDIUM
|
|
# Speedup: ~250x at C=500 columns
|
|
# File: ql/src/java/org/apache/hadoop/hive/ql/optimizer/SharedWorkOptimizer.java
|
|
#
|
|
# The mergeSchema() method merges needed columns from a discarded TableScanOperator
|
|
# into a retained one. It iterates over the discardable's neededColumnIDs (List<Integer>),
|
|
# neededColumns (List<String>), and virtualCols (List<VirtualColumn>), calling
|
|
# .contains() on the retainable's corresponding Lists. Each .contains() is O(N),
|
|
# making the total O(D*R) per list where D=discardable columns, R=retainable columns.
|
|
# For wide tables (500-2000 columns), this is quadratic.
|
|
#
|
|
# Fix: wrap each retainable list in a HashSet for O(1) membership testing, then
|
|
# add only elements not already present.
|
|
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SharedWorkOptimizer.java
|
|
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SharedWorkOptimizer.java
|
|
@@ -671,18 +671,24 @@
|
|
protected void mergeSchema(TableScanOperator discardableTsOp, TableScanOperator retainableTsOp) {
|
|
- for (int colId : discardableTsOp.getConf().getNeededColumnIDs()) {
|
|
- if (!retainableTsOp.getConf().getNeededColumnIDs().contains(colId)) {
|
|
+ // Use HashSets for O(1) membership testing instead of O(N) List.contains()
|
|
+ Set<Integer> retainedColumnIDs = new HashSet<>(retainableTsOp.getConf().getNeededColumnIDs());
|
|
+ for (int colId : discardableTsOp.getConf().getNeededColumnIDs()) {
|
|
+ if (retainedColumnIDs.add(colId)) {
|
|
retainableTsOp.getConf().getNeededColumnIDs().add(colId);
|
|
}
|
|
}
|
|
- for (String col : discardableTsOp.getConf().getNeededColumns()) {
|
|
- if (!retainableTsOp.getConf().getNeededColumns().contains(col)) {
|
|
+ Set<String> retainedColumns = new HashSet<>(retainableTsOp.getConf().getNeededColumns());
|
|
+ for (String col : discardableTsOp.getConf().getNeededColumns()) {
|
|
+ if (retainedColumns.add(col)) {
|
|
retainableTsOp.getConf().getNeededColumns().add(col);
|
|
}
|
|
}
|
|
- for (VirtualColumn col : discardableTsOp.getConf().getVirtualCols()) {
|
|
- if (!retainableTsOp.getConf().getVirtualCols().contains(col)) {
|
|
+ Set<VirtualColumn> retainedVCols = new HashSet<>(retainableTsOp.getConf().getVirtualCols());
|
|
+ for (VirtualColumn col : discardableTsOp.getConf().getVirtualCols()) {
|
|
+ if (retainedVCols.add(col)) {
|
|
retainableTsOp.getConf().getVirtualCols().add(col);
|
|
}
|
|
}
|
|
}
|