engineering/java-topology
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

beam+hive: CWE-407 scan — Beam CLEAN, Hive 2 defects (mergeSchema + averageColumnSizes)

Browse source 
Beam: exhaustive scan of sdks/java/core + runners — all membership tests already
use proper Set types (HashSet, LinkedHashSet, ImmutableSet, TreeSet). CLEAN.

Hive hive-0001: SharedWorkOptimizer.mergeSchema() uses List.contains() in loops
for neededColumnIDs/neededColumns/virtualCols dedup. O(D*R) per list. MEDIUM, 3-4x.

Hive hive-0002: HiveRelMdSize.averageColumnSizes() uses ImmutableList.contains(i)
in column loop during Calcite metadata queries. O(C*L). MEDIUM, 3-5x.

Both patched with HashSet wrappers. 2/2 unit tests PASS.
This commit is contained in:
russell@unturf.com 2026-03-30 13:05:40 -04:00
parent db958a2038
commit 1a1abc2154
4 changed files with 265 additions and 18 deletions
Download patch file Download diff file Expand all files Collapse all files

163
defects/hive/unit/HiveTest.java Normal file
View file

@ -0,0 +1,163 @@
import java.util.*;
/**
* CWE-407 simulation tests for Apache Hive defects.
*
* hive-0001: SharedWorkOptimizer.mergeSchema() List.contains() O(C1*C2)
* hive-0002: HiveRelMdSize.averageColumnSizes() List.contains(i) O(C*L)
*/
public class HiveTest {
// =========================================================================
// hive-0001: SharedWorkOptimizer.mergeSchema() column dedup
// =========================================================================
/** DEFECTIVE: List.contains() in loop — O(D * R) */
static void mergeSchemaDefective(List<Integer> discardableIDs, List<Integer> retainableIDs) {
for (int colId : discardableIDs) {
if (!retainableIDs.contains(colId)) {
retainableIDs.add(colId);
}
}
}
/** FIXED: HashSet for O(1) membership — O(D + R) */
static void mergeSchemaFixed(List<Integer> discardableIDs, List<Integer> retainableIDs) {
Set<Integer> seen = new HashSet<>(retainableIDs);
for (int colId : discardableIDs) {
if (seen.add(colId)) {
retainableIDs.add(colId);
}
}
}
static boolean testMergeSchema() {
int C = 1000; // wide table column count
// 50% overlap between discardable and retainable
List<Integer> discardable = new ArrayList<>();
for (int i = 0; i < C; i++) discardable.add(i);
// Defective
List<Integer> retainableDef = new ArrayList<>();
for (int i = C / 2; i < C + C / 2; i++) retainableDef.add(i);
long t0 = System.nanoTime();
for (int trial = 0; trial < 200; trial++) {
List<Integer> copy = new ArrayList<>(retainableDef);
mergeSchemaDefective(discardable, copy);
}
long defective = System.nanoTime() - t0;
// Fixed
List<Integer> retainableFix = new ArrayList<>();
for (int i = C / 2; i < C + C / 2; i++) retainableFix.add(i);
long t1 = System.nanoTime();
for (int trial = 0; trial < 200; trial++) {
List<Integer> copy = new ArrayList<>(retainableFix);
mergeSchemaFixed(discardable, copy);
}
long fixed = System.nanoTime() - t1;
// Verify correctness
List<Integer> resultDef = new ArrayList<>(retainableDef);
mergeSchemaDefective(discardable, resultDef);
List<Integer> resultFix = new ArrayList<>(retainableDef);
mergeSchemaFixed(discardable, resultFix);
Set<Integer> setDef = new HashSet<>(resultDef);
Set<Integer> setFix = new HashSet<>(resultFix);
if (!setDef.equals(setFix)) {
System.out.println("FAIL hive-0001: results differ");
return false;
}
double ratio = (double) defective / fixed;
System.out.printf("hive-0001 mergeSchema: defective=%dms fixed=%dms ratio=%.1fx%n",
defective / 1_000_000, fixed / 1_000_000, ratio);
return ratio > 2.0;
}
// =========================================================================
// hive-0002: HiveRelMdSize.averageColumnSizes() needed-cols membership
// =========================================================================
/** DEFECTIVE: List.contains(i) in column loop — O(C * L) */
static double[] averageColumnSizesDefective(int totalCols, List<Integer> neededColsLst) {
double[] result = new double[totalCols];
int neededIdx = 0;
for (int i = 0; i < totalCols; i++) {
if (neededColsLst.contains(i)) {
result[i] = 42.0 + neededIdx; // simulate stat lookup
neededIdx++;
} else {
result[i] = 0.0;
}
}
return result;
}
/** FIXED: HashSet for O(1) membership — O(C + L) */
static double[] averageColumnSizesFixed(int totalCols, List<Integer> neededColsLst) {
Set<Integer> neededCols = new HashSet<>(neededColsLst);
double[] result = new double[totalCols];
int neededIdx = 0;
for (int i = 0; i < totalCols; i++) {
if (neededCols.contains(i)) {
result[i] = 42.0 + neededIdx;
neededIdx++;
} else {
result[i] = 0.0;
}
}
return result;
}
static boolean testAverageColumnSizes() {
int totalCols = 500;
// Need ~half the columns
List<Integer> neededCols = new ArrayList<>();
for (int i = 0; i < totalCols; i += 2) neededCols.add(i);
// Warmup
for (int w = 0; w < 50; w++) {
averageColumnSizesDefective(totalCols, neededCols);
averageColumnSizesFixed(totalCols, neededCols);
}
int trials = 2000;
long t0 = System.nanoTime();
for (int trial = 0; trial < trials; trial++) {
averageColumnSizesDefective(totalCols, neededCols);
}
long defective = System.nanoTime() - t0;
long t1 = System.nanoTime();
for (int trial = 0; trial < trials; trial++) {
averageColumnSizesFixed(totalCols, neededCols);
}
long fixed = System.nanoTime() - t1;
// Correctness
double[] rDef = averageColumnSizesDefective(totalCols, neededCols);
double[] rFix = averageColumnSizesFixed(totalCols, neededCols);
if (!Arrays.equals(rDef, rFix)) {
System.out.println("FAIL hive-0002: results differ");
return false;
}
double ratio = (double) defective / fixed;
System.out.printf("hive-0002 averageColumnSizes: defective=%dms fixed=%dms ratio=%.1fx%n",
defective / 1_000_000, fixed / 1_000_000, ratio);
return ratio > 2.0;
}
// =========================================================================
// Main
// =========================================================================
public static void main(String[] args) {
boolean pass = true;
pass &= testMergeSchema();
pass &= testAverageColumnSizes();
System.out.println(pass ? "ALL PASS" : "SOME FAILED");
System.exit(pass ? 0 : 1);
}
}