engineering/java-topology
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

hive-0003

Browse source
This commit is contained in:
russell@unturf.com 2026-03-29 22:19:47 -04:00
parent 7e7cd2945a
commit 0fb709cb72
11 changed files with 1163 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

22
defects/hive/patch/hive-0001-tasktracker-visited-arraylist-hashset.md → defects/hive/patch/hive-0003-tasktracker-visited-arraylist-hashset.md
View file

@ -1,5 +1,4 @@
# UNDF: UNDF-2026-000000112
# hive-0001: TaskTracker.updateTaskCount ArrayList visited O(T²) in REPL DAG traversal
# hive-0003: TaskTracker.updateTaskCount ArrayList visited O(T²) in REPL DAG traversal
## Classification
- **Severity**: MEDIUM
@ -99,10 +98,8 @@ private void updateTaskCount(Task<?> task, Set<Task<?>> visited) {
```diff
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/util/TaskTracker.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/util/TaskTracker.java
@@ -1,5 +1,6 @@
import java.util.ArrayList;
@@ imports @@
+import java.util.HashSet;
import java.util.List;
+import java.util.Set;
public void addTask(Task<?> task) {
@ -118,7 +115,12 @@ private void updateTaskCount(Task<?> task, Set<Task<?>> visited) {
for (Task<?> task : taskList) {
if (!visited.contains(task)) {
tasks.add(task);
@@ -87,12 +87,12 @@
updateTaskCount(task, visited);
}
}
}
public void addDependentTask(Task<?> dependent) {
if (tasks.isEmpty()) {
addTask(dependent);
} else {
@ -135,4 +137,12 @@ private void updateTaskCount(Task<?> task, Set<Task<?>> visited) {
numberOfTasks += 1;
visited.add(task);
if (task.getChildTasks() != null) {
for (Task<?> childTask : task.getChildTasks()) {
if (visited.contains(childTask)) {
continue;
}
updateTaskCount(childTask, visited);
}
}
}
```

196
defects/hive/unit/TestTaskTrackerVisited.java Normal file
View file

@ -0,0 +1,196 @@
import java.util.*;
/**
* Unit test for hive-0001: TaskTracker.updateTaskCount ArrayList visited O(T²)
*
* Simulates the defective and fixed versions side by side.
* The fix: change ArrayList to HashSet for the visited accumulator.
*/
public class TestTaskTrackerVisited {
// --- Simulate defective version (ArrayList visited) ---
static int countOpsDefective;
static void updateTaskCountDefective(MockTask task, List<MockTask> visited) {
visited.add(task);
for (MockTask child : task.children) {
countOpsDefective++; // count the contains() check
if (visited.contains(child)) { // O(T) linear scan
continue;
}
updateTaskCountDefective(child, visited);
}
}
static int runDefective(MockTask root) {
countOpsDefective = 0;
List<MockTask> visited = new ArrayList<>();
updateTaskCountDefective(root, visited);
return countOpsDefective;
}
// --- Simulate fixed version (HashSet visited) ---
static int countOpsFixed;
static void updateTaskCountFixed(MockTask task, Set<MockTask> visited) {
visited.add(task);
for (MockTask child : task.children) {
countOpsFixed++; // count the contains() check
if (visited.contains(child)) { // O(1) hash lookup
continue;
}
updateTaskCountFixed(child, visited);
}
}
static int runFixed(MockTask root) {
countOpsFixed = 0;
Set<MockTask> visited = new HashSet<>();
updateTaskCountFixed(root, visited);
return countOpsFixed;
}
// --- MockTask for test ---
static class MockTask {
final String name;
final List<MockTask> children = new ArrayList<>();
MockTask(String name) { this.name = name; }
void addChild(MockTask child) { children.add(child); }
@Override
public int hashCode() { return name.hashCode(); }
@Override
public boolean equals(Object o) {
return o instanceof MockTask && ((MockTask) o).name.equals(name);
}
}
// Build a diamond DAG: root -> [A, B, C, ...] -> shared_sink
static MockTask buildDiamondDAG(int branches) {
MockTask root = new MockTask("root");
MockTask sink = new MockTask("sink");
for (int i = 0; i < branches; i++) {
MockTask branch = new MockTask("branch-" + i);
branch.addChild(sink);
root.addChild(branch);
}
return root;
}
// Build a chain: t0 -> t1 -> ... -> tN
static MockTask buildChain(int n) {
MockTask[] tasks = new MockTask[n];
for (int i = 0; i < n; i++) {
tasks[i] = new MockTask("task-" + i);
}
for (int i = 0; i < n - 1; i++) {
tasks[i].addChild(tasks[i + 1]);
}
return tasks[0];
}
public static void main(String[] args) {
int pass = 0, fail = 0;
// Test 1: chain of 50 tasks — both should produce same traversal count
{
MockTask root = buildChain(50);
int defOps = runDefective(root);
int fixOps = runFixed(root);
boolean ok = (defOps == fixOps); // chain: no revisits, same count
System.out.printf("[%s] Chain-50: defective=%d ops, fixed=%d ops%n",
ok ? "PASS" : "FAIL", defOps, fixOps);
if (ok) pass++; else fail++;
}
// Test 2: diamond DAG with 10 branches — defective pays O(T) per revisit check
{
MockTask root = buildDiamondDAG(10);
int defOps = runDefective(root);
int fixOps = runFixed(root);
// Fixed ops == defective ops here (both iterate same children)
// The difference is the *cost* of contains() - we verify both count same checks
boolean ok = (defOps == fixOps);
System.out.printf("[%s] Diamond-10: defective=%d ops, fixed=%d ops%n",
ok ? "PASS" : "FAIL", defOps, fixOps);
if (ok) pass++; else fail++;
}
// Test 3: large diamond DAG — measure relative op counts are equal
{
MockTask root = buildDiamondDAG(100);
int defOps = runDefective(root);
int fixOps = runFixed(root);
boolean ok = (defOps == fixOps);
System.out.printf("[%s] Diamond-100: defective=%d ops, fixed=%d ops%n",
ok ? "PASS" : "FAIL", defOps, fixOps);
if (ok) pass++; else fail++;
}
// Test 4: correctness — both versions visit same number of unique nodes
{
MockTask root = buildDiamondDAG(20);
// Count unique nodes in defective traversal
List<MockTask> visitedDef = new ArrayList<>();
countOpsDefective = 0;
updateTaskCountDefective(root, visitedDef);
int uniqueDef = new HashSet<>(visitedDef).size();
Set<MockTask> visitedFix = new HashSet<>();
countOpsFixed = 0;
updateTaskCountFixed(root, visitedFix);
int uniqueFix = visitedFix.size();
boolean ok = (uniqueDef == uniqueFix);
System.out.printf("[%s] Correctness Diamond-20: defective=%d unique, fixed=%d unique%n",
ok ? "PASS" : "FAIL", uniqueDef, uniqueFix);
if (ok) pass++; else fail++;
}
// Test 5: performance ratio — defective contains() is O(N) vs O(1) for fixed
// We simulate by counting the cost of each contains() call
{
int N = 200;
MockTask root = buildDiamondDAG(N);
// Defective: every contains() call scans the whole list - cost = list.size() at call time
int[] defCost = {0};
simulateDefectiveCost(root, new ArrayList<>(), defCost);
// Fixed: every contains() is O(1) - cost = 1
int[] fixCost = {0};
simulateFixedCost(root, new HashSet<>(), fixCost);
double ratio = (double) defCost[0] / fixCost[0];
boolean ok = ratio >= 50.0; // expect significant speedup at N=200
System.out.printf("[%s] Cost ratio Diamond-%d: defective=%d, fixed=%d, ratio=%.1fx%n",
ok ? "PASS" : "FAIL", N, defCost[0], fixCost[0], ratio);
if (ok) pass++; else fail++;
}
System.out.printf("%nResult: %d PASS, %d FAIL%n", pass, fail);
System.exit(fail > 0 ? 1 : 0);
}
static void simulateDefectiveCost(MockTask task, List<MockTask> visited, int[] cost) {
visited.add(task);
for (MockTask child : task.children) {
cost[0] += visited.size(); // O(N) for ArrayList.contains
if (visited.contains(child)) continue;
simulateDefectiveCost(child, visited, cost);
}
}
static void simulateFixedCost(MockTask task, Set<MockTask> visited, int[] cost) {
visited.add(task);
for (MockTask child : task.children) {
cost[0] += 1; // O(1) for HashSet.contains
if (visited.contains(child)) continue;
simulateFixedCost(child, visited, cost);
}
}
}