netty-0001 + dubbo-0002 + doris-0004: DNS dedup O(R²); MethodWalker O(2^D); NormalizeRepeat O(S×G); count 621→624
- netty-0001: DnsResolveContext.finalResult ArrayList.contains O(R²) dedup on DNS records File: resolver-dns/.../dns/DnsResolveContext.java line ~914 Fix: LinkedHashSet gives O(1) dedup with preserved insertion order Ratio: 24.5x at R=50 records - dubbo-0002: MethodWalker.walkHierarchy no visited guard — O(2^D) diamond recursion File: dubbo-rpc-triple/.../rest/util/MethodWalker.java walkHierarchy() Fix: add visited HashSet, return early if already visited Ratio: 8x at D=3 (common Spring proxy depth) - doris-0004: NormalizeRepeat.buildContextWithAlias ImmutableList.contains O(S×G) for GROUPING SETS File: fe-core/.../nereids/rules/analysis/NormalizeRepeat.java buildContextWithAlias() Fix: convert groupingSetExpressions to HashSet before loop — O(1) lookup Ratio: 49x for CUBE(c1..c8), 1000x+ for CUBE(c1..c10)
This commit is contained in:
parent
4c52d9ee51
commit
72c98d9af6
6 changed files with 898 additions and 0 deletions
|
|
@ -0,0 +1,117 @@
|
|||
# netty-0001: DnsResolveContext.finalResult ArrayList dedup O(R²)
|
||||
|
||||
## Classification
|
||||
- **CWE**: CWE-407 (Inefficient Algorithmic Complexity)
|
||||
- **Severity**: MEDIUM
|
||||
- **Component**: Netty — `resolver-dns`
|
||||
- **File**: `resolver-dns/src/main/java/io/netty/resolver/dns/DnsResolveContext.java`
|
||||
- **Method**: `onResponseAddRecord` (inner, around line 914)
|
||||
- **Complexity**: O(R²) where R = total DNS records accumulated across all nameserver responses
|
||||
|
||||
## Description
|
||||
|
||||
`DnsResolveContext` accumulates resolved records into `finalResult`, an `ArrayList<T>`.
|
||||
Before adding each new record, it checks for duplicates via `finalResult.contains(converted)`.
|
||||
`ArrayList.contains` is an O(N) linear scan, making the dedup loop O(R²) over all R records
|
||||
collected during a hostname resolution.
|
||||
|
||||
The code contains a self-aware comment that admits a `LinkedHashSet` would be better, but
|
||||
incorrectly dismisses it:
|
||||
|
||||
```java
|
||||
// While using a LinkedHashSet or HashSet may sound like the perfect fit for this we will use an
|
||||
// ArrayList here as duplicates should be found quite unfrequently in the wild and we dont want to pay
|
||||
// for the extra memory copy and allocations in this cases later on.
|
||||
```
|
||||
|
||||
This reasoning is flawed: a `LinkedHashSet` avoids the copy because `toArray()` can be
|
||||
called once at the end; duplicates are the normal case for multi-server failover (each
|
||||
server returns the same A records), not rare. With 5 nameservers each returning 10 A
|
||||
records, `contains()` is called 40+ times scanning a list growing from 1 to 10 elements —
|
||||
O(40) vs O(10) for a set. With CNAME chains and search domain retries that accumulate
|
||||
records from multiple passes, the list can reach 20-50 entries, making O(R²) more visible.
|
||||
|
||||
The `DnsAddressResolveContext` subclass sets `isDuplicateAllowed()` to `false`, meaning
|
||||
this dedup path is always executed for address resolution.
|
||||
|
||||
## Defect Code
|
||||
|
||||
```java
|
||||
// resolver-dns/src/main/java/io/netty/resolver/dns/DnsResolveContext.java lines ~906-916
|
||||
if (finalResult == null) {
|
||||
finalResult = new ArrayList<T>(8);
|
||||
finalResult.add(converted);
|
||||
} else if (isDuplicateAllowed() || !finalResult.contains(converted)) { // O(N) scan
|
||||
finalResult.add(converted);
|
||||
} else {
|
||||
shouldRelease = true;
|
||||
}
|
||||
```
|
||||
|
||||
## Fix
|
||||
|
||||
Replace `ArrayList` with `LinkedHashSet` for dedup tracking, then convert to list at
|
||||
`finishResolve()` when results are returned. This maintains insertion order while giving
|
||||
O(1) dedup.
|
||||
|
||||
```java
|
||||
// Change field declaration
|
||||
- private List<T> finalResult;
|
||||
+ private Set<T> finalResult; // LinkedHashSet maintains insertion order, O(1) contains
|
||||
|
||||
// Change allocation site
|
||||
- finalResult = new ArrayList<T>(8);
|
||||
+ finalResult = new LinkedHashSet<T>(8);
|
||||
|
||||
// finishResolve already calls filterResults which can consume any Collection;
|
||||
// the List<T> result = filterResults(finalResult) call works with a Set input.
|
||||
```
|
||||
|
||||
## Patch
|
||||
|
||||
```diff
|
||||
--- a/resolver-dns/src/main/java/io/netty/resolver/dns/DnsResolveContext.java
|
||||
+++ b/resolver-dns/src/main/java/io/netty/resolver/dns/DnsResolveContext.java
|
||||
@@ -116,7 +116,8 @@ abstract class DnsResolveContext<T> {
|
||||
private int allowedQueries;
|
||||
private boolean triedCNAME;
|
||||
private boolean completeEarly;
|
||||
- private List<T> finalResult;
|
||||
+ // LinkedHashSet gives O(1) dedup while preserving insertion order
|
||||
+ private Set<T> finalResult;
|
||||
|
||||
@@ -909,11 +909,8 @@ abstract class DnsResolveContext<T> {
|
||||
if (!promise.isDone()) {
|
||||
- // We want to ensure we do not have duplicates in finalResult as this may be unexpected.
|
||||
- //
|
||||
- // While using a LinkedHashSet or HashSet may sound like the perfect fit for this we will use an
|
||||
- // ArrayList here as duplicates should be found quite unfrequently in the wild and we dont want to pay
|
||||
- // for the extra memory copy and allocations in this cases later on.
|
||||
if (finalResult == null) {
|
||||
- finalResult = new ArrayList<T>(8);
|
||||
- finalResult.add(converted);
|
||||
- } else if (isDuplicateAllowed() || !finalResult.contains(converted)) {
|
||||
+ finalResult = new LinkedHashSet<T>(8);
|
||||
+ }
|
||||
+ if (isDuplicateAllowed() || finalResult.add(converted)) {
|
||||
finalResult.add(converted);
|
||||
} else {
|
||||
shouldRelease = true;
|
||||
```
|
||||
|
||||
## Complexity Comparison
|
||||
|
||||
| N records (total across all responses) | Old (ArrayList.contains) | New (LinkedHashSet.add) |
|
||||
|----------------------------------------|--------------------------|-------------------------|
|
||||
| 10 | 45 ops | 10 ops |
|
||||
| 20 | 190 ops | 20 ops |
|
||||
| 50 | 1,225 ops | 50 ops |
|
||||
|
||||
Ratio at N=50: **24.5x**
|
||||
|
||||
## Hot Path
|
||||
|
||||
Called on every DNS record during `DnsResolveContext` record accumulation — which happens
|
||||
once per DNS address resolution. For services doing frequent hostlookups (microservice
|
||||
discovery, gRPC name resolution, load-balancer refresh), this executes thousands of times
|
||||
per second.
|
||||
163
defects/netty/unit/DnsResolveContextDedupAlgorithm.java
Normal file
163
defects/netty/unit/DnsResolveContextDedupAlgorithm.java
Normal file
|
|
@ -0,0 +1,163 @@
|
|||
/**
|
||||
* netty-0001: DnsResolveContext.finalResult ArrayList dedup O(R²)
|
||||
*
|
||||
* Demonstrates the defect: O(R²) dedup via ArrayList.contains vs O(R) with LinkedHashSet.
|
||||
*
|
||||
* Compile: javac DnsResolveContextDedupAlgorithm.java
|
||||
* Run: java DnsResolveContextDedupAlgorithm
|
||||
*/
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
public class DnsResolveContextDedupAlgorithm {
|
||||
|
||||
// ---- DEFECT: O(R²) ArrayList dedup (mirrors DnsResolveContext lines ~906-916) ----
|
||||
static List<String> dedupArrayList(List<String> responses) {
|
||||
List<String> finalResult = null;
|
||||
int ops = 0;
|
||||
for (String converted : responses) {
|
||||
if (finalResult == null) {
|
||||
finalResult = new ArrayList<>(8);
|
||||
finalResult.add(converted);
|
||||
} else {
|
||||
// O(N) scan — this is the defect
|
||||
ops += finalResult.size();
|
||||
if (!finalResult.contains(converted)) {
|
||||
finalResult.add(converted);
|
||||
}
|
||||
}
|
||||
}
|
||||
return finalResult;
|
||||
}
|
||||
|
||||
// ---- FIX: O(R) LinkedHashSet dedup with preserved insertion order ----
|
||||
static List<String> dedupLinkedHashSet(List<String> responses) {
|
||||
Set<String> finalResult = new LinkedHashSet<>(8);
|
||||
for (String converted : responses) {
|
||||
finalResult.add(converted); // O(1) - returns false on duplicate, no scan needed
|
||||
}
|
||||
return new ArrayList<>(finalResult);
|
||||
}
|
||||
|
||||
static long benchmarkArrayList(List<String> input, int iterations) {
|
||||
long start = System.nanoTime();
|
||||
for (int i = 0; i < iterations; i++) {
|
||||
List<String> result = null;
|
||||
for (String s : input) {
|
||||
if (result == null) {
|
||||
result = new ArrayList<>(8);
|
||||
result.add(s);
|
||||
} else if (!result.contains(s)) {
|
||||
result.add(s);
|
||||
}
|
||||
}
|
||||
}
|
||||
return System.nanoTime() - start;
|
||||
}
|
||||
|
||||
static long benchmarkLinkedHashSet(List<String> input, int iterations) {
|
||||
long start = System.nanoTime();
|
||||
for (int i = 0; i < iterations; i++) {
|
||||
Set<String> result = new LinkedHashSet<>(8);
|
||||
for (String s : input) {
|
||||
result.add(s);
|
||||
}
|
||||
}
|
||||
return System.nanoTime() - start;
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
System.out.println("=== netty-0001: DnsResolveContext finalResult dedup ===");
|
||||
System.out.println();
|
||||
|
||||
// Simulate DNS resolution: 5 nameservers each returning 10 A records (some duplicates)
|
||||
// This mirrors what happens with multi-server failover or CNAME chain following
|
||||
List<String> dnsResponses = new ArrayList<>();
|
||||
// 10 unique IPs repeated across 5 server responses = 50 total records
|
||||
for (int server = 0; server < 5; server++) {
|
||||
for (int ip = 0; ip < 10; ip++) {
|
||||
dnsResponses.add("192.168." + server + "." + ip); // unique per server
|
||||
}
|
||||
}
|
||||
// Add duplicates (second server repeats some from first)
|
||||
for (int ip = 0; ip < 10; ip++) {
|
||||
dnsResponses.add("192.168.0." + ip); // duplicates of server-0 entries
|
||||
}
|
||||
|
||||
System.out.printf("Input: %d DNS records (simulating 5 servers × 10 IPs + 10 duplicates)%n",
|
||||
dnsResponses.size());
|
||||
System.out.println();
|
||||
|
||||
// Verify correctness
|
||||
List<String> resultArrayList = dedupArrayList(dnsResponses);
|
||||
List<String> resultLinkedHashSet = dedupLinkedHashSet(dnsResponses);
|
||||
|
||||
System.out.printf("ArrayList result size: %d%n", resultArrayList.size());
|
||||
System.out.printf("LinkedHashSet result size: %d%n", resultLinkedHashSet.size());
|
||||
|
||||
boolean correctSize = resultArrayList.size() == resultLinkedHashSet.size();
|
||||
System.out.printf("Same result size: %s%n", correctSize ? "PASS" : "FAIL");
|
||||
|
||||
// Check order is preserved (LinkedHashSet maintains insertion order)
|
||||
boolean orderMatch = resultArrayList.equals(resultLinkedHashSet);
|
||||
System.out.printf("Same result order: %s%n", orderMatch ? "PASS" : "FAIL");
|
||||
|
||||
System.out.println();
|
||||
|
||||
// Benchmark with larger input to show O(R²) vs O(R)
|
||||
int iterations = 100_000;
|
||||
|
||||
// Small case: 10 unique records (typical DNS response)
|
||||
List<String> small = new ArrayList<>();
|
||||
for (int i = 0; i < 10; i++) small.add("10.0.0." + i);
|
||||
|
||||
// Medium case: 30 records (multi-server with some overlap)
|
||||
List<String> medium = new ArrayList<>();
|
||||
for (int i = 0; i < 30; i++) medium.add("10.0." + (i / 10) + "." + (i % 10));
|
||||
|
||||
// Large case: 50 records (CNAME chains + search domain retries)
|
||||
List<String> large = new ArrayList<>();
|
||||
for (int i = 0; i < 50; i++) large.add("10." + (i / 20) + "." + (i / 10 % 10) + "." + (i % 10));
|
||||
|
||||
System.out.println("--- Benchmarks (ns per iteration, " + iterations + " iterations) ---");
|
||||
System.out.printf("%-12s %15s %15s %10s%n", "R (records)", "ArrayList O(R²)", "LinkedHashSet O(R)", "Speedup");
|
||||
System.out.printf("%-12s %15s %15s %10s%n", "-----------", "---------------", "-----------------", "-------");
|
||||
|
||||
for (List<String> input : new List[]{small, medium, large}) {
|
||||
// Warmup
|
||||
for (int w = 0; w < 1000; w++) {
|
||||
benchmarkArrayList(input, 1);
|
||||
benchmarkLinkedHashSet(input, 1);
|
||||
}
|
||||
long alTime = benchmarkArrayList(input, iterations);
|
||||
long lhsTime = benchmarkLinkedHashSet(input, iterations);
|
||||
double speedup = (double) alTime / lhsTime;
|
||||
System.out.printf("%-12d %15.0f %15.0f %9.1fx%n",
|
||||
input.size(),
|
||||
(double) alTime / iterations,
|
||||
(double) lhsTime / iterations,
|
||||
speedup);
|
||||
}
|
||||
|
||||
System.out.println();
|
||||
System.out.println("--- Algorithmic operation count ---");
|
||||
System.out.printf("%-12s %15s %15s %10s%n", "R (records)", "ArrayList ops", "LinkedHashSet ops", "Ratio");
|
||||
System.out.printf("%-12s %15s %15s %10s%n", "-----------", "-------------", "-----------------", "-----");
|
||||
for (int r : new int[]{10, 20, 30, 50}) {
|
||||
int alOps = 0;
|
||||
for (int i = 1; i < r; i++) alOps += i; // sum of 0..r-1 = r*(r-1)/2
|
||||
int lhsOps = r;
|
||||
System.out.printf("%-12d %15d %15d %9.1fx%n", r, alOps, lhsOps, (double) alOps / lhsOps);
|
||||
}
|
||||
|
||||
System.out.println();
|
||||
if (correctSize && orderMatch) {
|
||||
System.out.println("RESULT: PASS — fix is correct and faster");
|
||||
} else {
|
||||
System.out.println("RESULT: FAIL");
|
||||
System.exit(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue