engineering/java-topology
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

flink-deeper/storm: CWE-407 findings

Browse source 
flink-0006: MultipleInputNodeCreationProcessor.createMultipleInputNode() group.members ArrayList.contains inside nested loop O(M2*I); fix: HashSet membersSet before loop
flink-0007: MLPredictTypeStrategy.validateTableAndDescriptorArguments() tableFieldNames List<String>.contains in for loop O(D*T); fix: HashSet<String> conversion before loop
storm-0001: WorkerState.refreshConnections() localTaskIds ArrayList<Integer>.contains in per-task loop O(T_out*L); fix: change to HashSet<Integer>
This commit is contained in:
russell@unturf.com 2026-03-30 09:52:27 -04:00
parent cb15f957b5
commit b31ac644bb
5 changed files with 364 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

27
defects/flink/patch/flink-0006-multiple-input-node-members-set.patch Normal file
View file

@ -0,0 +1,27 @@
# UNDF: (leave blank)
--- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/processor/MultipleInputNodeCreationProcessor.java
+++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/processor/MultipleInputNodeCreationProcessor.java
@@ -534,7 +534,7 @@ public class MultipleInputNodeCreationProcessor implements DAGProcessor {
// calculate the inputs of the multiple input node
List<Tuple3<ExecNode<?>, InputProperty, ExecEdge>> inputs = new ArrayList<>();
+ Set<ExecNodeWrapper> membersSet = new HashSet<>(group.members);
for (ExecNodeWrapper member : group.members) {
for (int i = 0; i < member.inputs.size(); i++) {
ExecNodeWrapper memberInput = member.inputs.get(i);
- if (group.members.contains(memberInput)) {
+ if (membersSet.contains(memberInput)) {
continue;
}
@@ -705,10 +705,12 @@ public class MultipleInputNodeCreationProcessor implements DAGProcessor {
Preconditions.checkNotNull(
root, "Multiple input group does not have a root. This is a bug.");
- Set<ExecNodeWrapper> sameGroupInputWrappers = new HashSet<>();
+ Set<ExecNodeWrapper> membersSet = new HashSet<>(members);
+ Set<ExecNodeWrapper> sameGroupInputWrappers = new HashSet<>();
for (ExecNodeWrapper inputWrapper : root.inputs) {
- if (members.contains(inputWrapper)) {
+ if (membersSet.contains(inputWrapper)) {
sameGroupInputWrappers.add(inputWrapper);
}
}

11
defects/flink/patch/flink-0007-ml-predict-tablefieldnames-hashset.patch Normal file
View file

@ -0,0 +1,11 @@
# UNDF: (leave blank)
--- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/MLPredictTypeStrategy.java
+++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/MLPredictTypeStrategy.java
@@ -165,7 +165,7 @@ class MLPredictTypeStrategy implements TypeStrategy {
// Check that descriptor column names exist in table columns
- List<String> tableFieldNames = DataType.getFieldNames(tableSemantics.dataType());
+ Set<String> tableFieldNames = new HashSet<>(DataType.getFieldNames(tableSemantics.dataType()));
List<String> descriptorColumnNames = descriptorColumns.getNames();
for (String descriptorColumnName : descriptorColumnNames) {
if (!tableFieldNames.contains(descriptorColumnName)) {

176
defects/flink/unit/FlinkTest.java Normal file
View file

@ -0,0 +1,176 @@
import java.util.*;
/**
* CWE-407 unit tests for Apache Flink deeper scan (flink-0006, flink-0007).
*
* flink-0006: MultipleInputNodeCreationProcessor.createMultipleInputNode()
* flink-table/flink-table-planner/.../exec/processor/MultipleInputNodeCreationProcessor.java
* In createMultipleInputNode(), the inner loop iterates over group.members (ArrayList) and
* calls group.members.contains(memberInput) for each input of each member:
* for (ExecNodeWrapper member : group.members) { // O(M) outer
* for (int i = 0; i < member.inputs.size(); i++) { // O(I) inner
* if (group.members.contains(memberInput)) { ... } // O(M) linear scan
* Total: O(M * I * M) = O(M²×I).
* Fix: build a HashSet<ExecNodeWrapper> membersSet = new HashSet<>(group.members) before the
* outer loop. Each .contains() becomes O(1), reducing total to O(M*I).
*
* flink-0007: MLPredictTypeStrategy.validateTableAndDescriptorArguments()
* flink-table/flink-table-common/.../inference/strategies/MLPredictTypeStrategy.java
* At planning time for ML_PREDICT(), the validator iterates over descriptor column names and
* calls tableFieldNames.contains() where tableFieldNames is List<String>:
* for (String descriptorColumnName : descriptorColumnNames) { // O(D)
* if (!tableFieldNames.contains(descriptorColumnName)) { // O(T) linear scan
* Total: O(D * T) where D=descriptor columns, T=table fields.
* Fix: convert tableFieldNames to a HashSet<String> before the loop → O(D) total.
*/
public class FlinkTest {
// --- flink-0006 simulation ---
/** Simulate defect: ArrayList.contains in nested loop → O(M² * I) */
static int simulateCreateMultipleInputNode_defect(
List<Integer> members, Map<Integer, List<Integer>> memberInputs) {
int ops = 0;
for (int member : members) {
List<Integer> inputs = memberInputs.getOrDefault(member, Collections.emptyList());
for (int input : inputs) {
ops++;
if (members.contains(input)) { // O(M) per call — defect
continue;
}
// would add to result list here
}
}
return ops;
}
/** Simulate fix: HashSet.contains is O(1) */
static int simulateCreateMultipleInputNode_fixed(
List<Integer> members, Map<Integer, List<Integer>> memberInputs) {
int ops = 0;
Set<Integer> membersSet = new HashSet<>(members); // O(M) once
for (int member : members) {
List<Integer> inputs = memberInputs.getOrDefault(member, Collections.emptyList());
for (int input : inputs) {
ops++;
if (membersSet.contains(input)) { // O(1) per call — fix
continue;
}
}
}
return ops;
}
static void testFlink0006() throws Exception {
// M = members in group, I = inputs per member
int M = 300;
int I = 10;
List<Integer> members = new ArrayList<>();
for (int i = 0; i < M; i++) members.add(i);
// Each member has I inputs: half internal (in members), half external
Map<Integer, List<Integer>> memberInputs = new HashMap<>();
for (int m = 0; m < M; m++) {
List<Integer> inputs = new ArrayList<>();
for (int i = 0; i < I; i++) {
// alternating internal/external inputs
inputs.add(i % 2 == 0 ? m : M + i);
}
memberInputs.put(m, inputs);
}
// Warm up
simulateCreateMultipleInputNode_defect(members, memberInputs);
simulateCreateMultipleInputNode_fixed(members, memberInputs);
int RUNS = 200;
long t0 = System.nanoTime();
for (int r = 0; r < RUNS; r++) {
simulateCreateMultipleInputNode_defect(members, memberInputs);
}
long defectNs = System.nanoTime() - t0;
long t1 = System.nanoTime();
for (int r = 0; r < RUNS; r++) {
simulateCreateMultipleInputNode_fixed(members, memberInputs);
}
long fixedNs = System.nanoTime() - t1;
double ratio = (double) defectNs / Math.max(fixedNs, 1);
System.out.printf("flink-0006 [M=%d I=%d RUNS=%d]: defect=%.1fms fixed=%.1fms ratio=%.1fx%n",
M, I, RUNS, defectNs / 1e6, fixedNs / 1e6, ratio);
if (ratio < 1.5) {
throw new AssertionError("Expected defect to be slower; ratio=" + ratio);
}
System.out.println("flink-0006 PASS");
}
// --- flink-0007 simulation ---
/** Simulate defect: List<String>.contains in loop → O(D*T) */
static boolean validateDescriptors_defect(List<String> tableFieldNames, List<String> descriptorColumnNames) {
for (String col : descriptorColumnNames) { // O(D)
if (!tableFieldNames.contains(col)) { // O(T) — defect
return false;
}
}
return true;
}
/** Simulate fix: HashSet<String>.contains is O(1) */
static boolean validateDescriptors_fixed(List<String> tableFieldNames, List<String> descriptorColumnNames) {
Set<String> tableFieldSet = new HashSet<>(tableFieldNames); // O(T) once
for (String col : descriptorColumnNames) { // O(D)
if (!tableFieldSet.contains(col)) { // O(1) — fix
return false;
}
}
return true;
}
static void testFlink0007() throws Exception {
int T = 500; // table columns
int D = 200; // descriptor columns (subset of table columns)
List<String> tableFields = new ArrayList<>();
for (int i = 0; i < T; i++) tableFields.add("col_" + i);
List<String> descriptorCols = new ArrayList<>();
for (int i = 0; i < D; i++) descriptorCols.add("col_" + (i * 2 % T));
// Warm up
validateDescriptors_defect(tableFields, descriptorCols);
validateDescriptors_fixed(tableFields, descriptorCols);
int RUNS = 2000;
long t0 = System.nanoTime();
for (int r = 0; r < RUNS; r++) {
validateDescriptors_defect(tableFields, descriptorCols);
}
long defectNs = System.nanoTime() - t0;
long t1 = System.nanoTime();
for (int r = 0; r < RUNS; r++) {
validateDescriptors_fixed(tableFields, descriptorCols);
}
long fixedNs = System.nanoTime() - t1;
double ratio = (double) defectNs / Math.max(fixedNs, 1);
System.out.printf("flink-0007 [T=%d D=%d RUNS=%d]: defect=%.1fms fixed=%.1fms ratio=%.1fx%n",
T, D, RUNS, defectNs / 1e6, fixedNs / 1e6, ratio);
if (ratio < 1.5) {
throw new AssertionError("Expected defect to be slower; ratio=" + ratio);
}
System.out.println("flink-0007 PASS");
}
public static void main(String[] args) throws Exception {
testFlink0006();
testFlink0007();
System.out.println("ALL PASS");
}
}

27
defects/storm/patch/storm-0001-worker-localtaskids-hashset.patch Normal file
View file

@ -0,0 +1,27 @@
# UNDF: (leave blank)
--- a/storm-client/src/jvm/org/apache/storm/daemon/worker/WorkerState.java
+++ b/storm-client/src/jvm/org/apache/storm/daemon/worker/WorkerState.java
@@ -109,7 +109,7 @@ public class WorkerState {
// local executors and localTaskIds running in this worker
- final ArrayList<Integer> localTaskIds;
+ final Set<Integer> localTaskIds;
@@ -190,7 +190,7 @@ public class WorkerState {
- this.localTaskIds = new ArrayList<>();
+ this.localTaskIds = new HashSet<>();
for (List<Long> executor : executors) {
List<Integer> taskIds = StormCommon.executorIdToTasks(executor);
this.localTaskIds.addAll(taskIds);
}
- Collections.sort(localTaskIds);
@@ -424,7 +424,7 @@ public class WorkerState {
for (Map.Entry<Integer, NodeInfo> taskToNodePortEntry : taskToNodePort.entrySet()) {
Integer task = taskToNodePortEntry.getKey();
if (outboundTasks.contains(task)) {
newTaskToNodePort.put(task, taskToNodePortEntry.getValue());
- if (!localTaskIds.contains(task)) {
+ if (!localTaskIds.contains(task)) { /* O(1) now that localTaskIds is HashSet */
neededConnections.add(taskToNodePortEntry.getValue());
}
}

123
defects/storm/unit/StormTest.java Normal file
View file

@ -0,0 +1,123 @@
import java.util.*;
/**
* CWE-407 unit test for Apache Storm deeper scan (storm-0001).
*
* storm-0001: WorkerState.refreshConnections()
* storm-client/src/jvm/org/apache/storm/daemon/worker/WorkerState.java
*
* localTaskIds is declared as ArrayList<Integer> and populated from the worker's executors.
* In refreshConnections() (called on a recurring timer in every Storm worker process),
* the code iterates over all outbound tasks and calls localTaskIds.contains(task):
*
* for (Map.Entry<Integer, NodeInfo> entry : taskToNodePort.entrySet()) { // O(T_out)
* Integer task = entry.getKey();
* if (outboundTasks.contains(task)) { // O(1) — Set
* ...
* if (!localTaskIds.contains(task)) { // O(L) — ArrayList linear scan — defect
* neededConnections.add(...);
* }
*
* Total per refresh: O(T_out * L) where T_out = outbound task count, L = local task count.
* Since refreshConnections() is called every second (TOPOLOGY_WORKER_RECEIVER_THREAD_COUNT
* timer), this is a sustained hot path in every running Storm worker.
*
* Fix: change localTaskIds from ArrayList<Integer> to HashSet<Integer>.
* The Collections.sort(localTaskIds) call (used for deterministic ordering) is removed;
* callers that require ordered output (e.g. getLocalTaskIds()) can return a sorted copy
* on demand when needed.
*/
public class StormTest {
/** Simulate defect: ArrayList.contains in refreshConnections loop */
static int simulateRefreshConnections_defect(
List<Integer> localTaskIds, // ArrayList — defect
Set<Integer> outboundTasks,
Map<Integer, String> taskToNodePort) {
Set<String> neededConnections = new HashSet<>();
for (Map.Entry<Integer, String> entry : taskToNodePort.entrySet()) {
Integer task = entry.getKey();
if (outboundTasks.contains(task)) { // O(1) — HashSet
if (!localTaskIds.contains(task)) { // O(L) — defect
neededConnections.add(entry.getValue());
}
}
}
return neededConnections.size();
}
/** Simulate fix: HashSet.contains is O(1) */
static int simulateRefreshConnections_fixed(
Set<Integer> localTaskIds, // HashSet — fix
Set<Integer> outboundTasks,
Map<Integer, String> taskToNodePort) {
Set<String> neededConnections = new HashSet<>();
for (Map.Entry<Integer, String> entry : taskToNodePort.entrySet()) {
Integer task = entry.getKey();
if (outboundTasks.contains(task)) { // O(1)
if (!localTaskIds.contains(task)) { // O(1) — fix
neededConnections.add(entry.getValue());
}
}
}
return neededConnections.size();
}
static void testStorm0001() throws Exception {
// T_out = outbound tasks (topology-wide tasks visible to worker)
// L = local tasks per worker
int T_out = 1000;
int L = 20;
// Populate local task IDs
List<Integer> localTaskIdsList = new ArrayList<>();
Set<Integer> localTaskIdsSet = new HashSet<>();
for (int i = 0; i < L; i++) {
localTaskIdsList.add(i);
localTaskIdsSet.add(i);
}
// outboundTasks is a HashSet of all tasks this worker routes to
Set<Integer> outboundTasks = new HashSet<>();
for (int i = 0; i < T_out; i++) outboundTasks.add(i);
// taskToNodePort: topology assignment map (T_out entries)
Map<Integer, String> taskToNodePort = new LinkedHashMap<>();
for (int i = 0; i < T_out; i++) {
taskToNodePort.put(i, "node" + (i % 50) + ":6700");
}
// Warm up
simulateRefreshConnections_defect(localTaskIdsList, outboundTasks, taskToNodePort);
simulateRefreshConnections_fixed(localTaskIdsSet, outboundTasks, taskToNodePort);
// Simulate timer-based refresh: many iterations
int RUNS = 5000;
long t0 = System.nanoTime();
for (int r = 0; r < RUNS; r++) {
simulateRefreshConnections_defect(localTaskIdsList, outboundTasks, taskToNodePort);
}
long defectNs = System.nanoTime() - t0;
long t1 = System.nanoTime();
for (int r = 0; r < RUNS; r++) {
simulateRefreshConnections_fixed(localTaskIdsSet, outboundTasks, taskToNodePort);
}
long fixedNs = System.nanoTime() - t1;
double ratio = (double) defectNs / Math.max(fixedNs, 1);
System.out.printf("storm-0001 [T_out=%d L=%d RUNS=%d]: defect=%.1fms fixed=%.1fms ratio=%.1fx%n",
T_out, L, RUNS, defectNs / 1e6, fixedNs / 1e6, ratio);
if (ratio < 1.5) {
throw new AssertionError("Expected defect to be slower; ratio=" + ratio);
}
System.out.println("storm-0001 PASS");
}
public static void main(String[] args) throws Exception {
testStorm0001();
System.out.println("ALL PASS");
}
}