netdata CLEAN; telegraf-0001 dedup GetField O(F²) — 1 defect, 1/1 PASS
Netdata: well-engineered with Judy arrays, dictionaries, hash tables throughout. No CWE-407 defects found. Telegraf: dedup processor Apply() calls m.GetField(f.Key) O(F) inside field comparison loop → O(F²). Fix: build field map for O(1) lookup. 125x overhead at F=500.
This commit is contained in:
parent
38634f0b25
commit
27382ae791
4 changed files with 189 additions and 0 deletions
32
defects/netdata/patch/CLEAN.md
Normal file
32
defects/netdata/patch/CLEAN.md
Normal file
|
|
@ -0,0 +1,32 @@
|
|||
# Netdata — CWE-407 Scan Result: CLEAN
|
||||
|
||||
**Date:** 2026-03-30
|
||||
**Scanner:** agent blackops
|
||||
**Target:** https://github.com/netdata/netdata (depth=1)
|
||||
**Language:** C + Go
|
||||
|
||||
## Scan Coverage
|
||||
|
||||
- `src/health/` — alarm templates, silencers, prototypes
|
||||
- `src/streaming/` — stream path, replication, capabilities
|
||||
- `src/database/` — rrdlabels, rrdhost, rrdset, rrddim, contexts, query_target, engine
|
||||
- `src/libnetdata/` — adaptive_resortable_list, dictionary, facets, string dedup, user-auth
|
||||
- `src/web/` — websocket JSONRPC, MCP, API v1
|
||||
- `src/exporting/` — Prometheus exporter server list
|
||||
- `src/registry/` — person/machine URL tracking
|
||||
- `src/collectors/` — ebpf, log2journal
|
||||
- `src/go/plugin/` — job manager, SNMP profile loader, multipath, weblog collector
|
||||
|
||||
## Findings
|
||||
|
||||
No CWE-407 defects found. Netdata's core data structures are well-engineered:
|
||||
|
||||
- **Judy arrays** (JudyL, JudyHS) for labels, metrics registry
|
||||
- **Dictionary** (hash-table based) for rrdhost, rrdset, rrddim, contexts
|
||||
- **SIMPLE_HASHTABLE** for facets value indexing
|
||||
- **Bitmask dedup** for HTTP access flags, RRDR options, stream capabilities
|
||||
- **Adaptive Resortable List** (self-sorting linked list) for /proc parsing — amortized fast path
|
||||
- **STRING dedup** via global hash table for string interning
|
||||
|
||||
The few linked-list scans found (registry person URLs, Prometheus server list, health silencer list)
|
||||
are per-request lookups, not inside nested loops. No O(N²) membership patterns detected.
|
||||
|
|
@ -0,0 +1,44 @@
|
|||
# UNDF: (leave blank)
|
||||
# CWE-407: Dedup processor Apply() uses GetField O(F) inside field loop → O(F²)
|
||||
#
|
||||
# In plugins/processors/dedup/dedup.go, the Apply() method compares each field
|
||||
# of an incoming metric against the cached metric by calling m.GetField(f.Key)
|
||||
# which performs a linear scan of the cached metric's field list. For each
|
||||
# incoming metric, this is O(F_new × F_cached) ≈ O(F²).
|
||||
#
|
||||
# Fix: Build a map from the cached metric's fields for O(1) lookup, reducing
|
||||
# the overall comparison to O(F).
|
||||
#
|
||||
# Severity: MEDIUM — Telegraf metrics typically have 10-50 fields, but
|
||||
# system/CPU/disk metrics can exceed 100 fields. The dedup processor is a
|
||||
# common pipeline component.
|
||||
#
|
||||
--- a/plugins/processors/dedup/dedup.go
|
||||
+++ b/plugins/processors/dedup/dedup.go
|
||||
@@ -31,6 +31,14 @@ func (d *Dedup) Apply(metrics ...telegraf.Metric) []telegraf.Metric {
|
||||
idx := 0
|
||||
for _, metric := range metrics {
|
||||
id := metric.HashID()
|
||||
m, ok := d.cache[id]
|
||||
|
||||
// If not in cache then just save it
|
||||
@@ -53,8 +61,15 @@ func (d *Dedup) Apply(metrics ...telegraf.Metric) []telegraf.Metric {
|
||||
|
||||
// For each field compare value with the cached one
|
||||
changed := false
|
||||
added := false
|
||||
sametime := metric.Time() == m.Time()
|
||||
+
|
||||
+ // Build a map of cached fields for O(1) lookup instead of
|
||||
+ // calling m.GetField() which is O(F) per call
|
||||
+ cachedFields := make(map[string]interface{}, len(m.FieldList()))
|
||||
+ for _, cf := range m.FieldList() {
|
||||
+ cachedFields[cf.Key] = cf.Value
|
||||
+ }
|
||||
+
|
||||
for _, f := range metric.FieldList() {
|
||||
- if value, ok := m.GetField(f.Key); ok {
|
||||
+ if value, ok := cachedFields[f.Key]; ok {
|
||||
if value != f.Value {
|
||||
changed = true
|
||||
break
|
||||
BIN
defects/telegraf/unit/TelegrafTest.class
Normal file
BIN
defects/telegraf/unit/TelegrafTest.class
Normal file
Binary file not shown.
113
defects/telegraf/unit/TelegrafTest.java
Normal file
113
defects/telegraf/unit/TelegrafTest.java
Normal file
|
|
@ -0,0 +1,113 @@
|
|||
import java.util.*;
|
||||
|
||||
/**
|
||||
* CWE-407 unit test for Telegraf dedup processor defect.
|
||||
*
|
||||
* telegraf-0001: Dedup Apply() calls GetField(key) O(F) inside field loop → O(F²)
|
||||
*
|
||||
* Simulates the defect (linear field scan per field) vs the fix (hash map lookup).
|
||||
*/
|
||||
public class TelegrafTest {
|
||||
|
||||
// --- telegraf-0001: Dedup GetField quadratic ---
|
||||
|
||||
/** Simulate metric fields as a list (Go slice of *Field). */
|
||||
static List<String[]> makeFields(int n) {
|
||||
List<String[]> fields = new ArrayList<>(n);
|
||||
for (int i = 0; i < n; i++) {
|
||||
fields.add(new String[]{"field_" + i, "value_" + i});
|
||||
}
|
||||
return fields;
|
||||
}
|
||||
|
||||
/** DEFECTIVE: linear scan for each field lookup, O(F) per call. */
|
||||
static Object getFieldLinear(List<String[]> fields, String key) {
|
||||
for (String[] f : fields) {
|
||||
if (f[0].equals(key)) {
|
||||
return f[1];
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** DEFECTIVE: dedup comparison using linear GetField — O(F²). */
|
||||
static long dedupCompareDefective(List<String[]> incoming, List<String[]> cached) {
|
||||
long ops = 0;
|
||||
for (String[] f : incoming) {
|
||||
// Linear scan of cached fields
|
||||
for (String[] cf : cached) {
|
||||
ops++;
|
||||
if (cf[0].equals(f[0])) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
return ops;
|
||||
}
|
||||
|
||||
/** FIXED: build hash map first, then O(1) lookup — O(F). */
|
||||
static long dedupCompareFixed(List<String[]> incoming, List<String[]> cached) {
|
||||
long ops = 0;
|
||||
Map<String, String> cachedMap = new HashMap<>(cached.size());
|
||||
for (String[] cf : cached) {
|
||||
cachedMap.put(cf[0], cf[1]);
|
||||
ops++;
|
||||
}
|
||||
for (String[] f : incoming) {
|
||||
cachedMap.get(f[0]); // O(1) amortized
|
||||
ops++;
|
||||
}
|
||||
return ops;
|
||||
}
|
||||
|
||||
static boolean testDedupGetField() {
|
||||
System.out.println("=== telegraf-0001: Dedup GetField O(F²) → O(F) ===");
|
||||
boolean pass = true;
|
||||
|
||||
int[] sizes = {10, 50, 100, 500};
|
||||
for (int n : sizes) {
|
||||
List<String[]> incoming = makeFields(n);
|
||||
List<String[]> cached = makeFields(n);
|
||||
|
||||
long defectOps = dedupCompareDefective(incoming, cached);
|
||||
long fixedOps = dedupCompareFixed(incoming, cached);
|
||||
double ratio = (double) defectOps / fixedOps;
|
||||
|
||||
System.out.printf(" F=%d: defect=%d ops, fixed=%d ops, ratio=%.1fx%n",
|
||||
n, defectOps, fixedOps, ratio);
|
||||
|
||||
// At F=100: defect ~5050 ops (sum 1..100), fixed ~200 ops, ratio ~25x
|
||||
// At F=500: defect ~125250 ops, fixed ~1000 ops, ratio ~125x
|
||||
if (n >= 50 && ratio < 2.0) {
|
||||
System.out.printf(" FAIL: expected ratio >= 2.0 at F=%d, got %.1f%n", n, ratio);
|
||||
pass = false;
|
||||
}
|
||||
}
|
||||
|
||||
// Verify correctness: both should find the same matches
|
||||
List<String[]> a = makeFields(20);
|
||||
List<String[]> b = makeFields(20);
|
||||
for (String[] f : a) {
|
||||
Object linearResult = getFieldLinear(b, f[0]);
|
||||
if (linearResult == null || !linearResult.equals(f[1])) {
|
||||
System.out.println(" FAIL: linear lookup returned wrong result for " + f[0]);
|
||||
pass = false;
|
||||
}
|
||||
}
|
||||
|
||||
System.out.println(" " + (pass ? "PASS" : "FAIL"));
|
||||
return pass;
|
||||
}
|
||||
|
||||
// --- Main ---
|
||||
|
||||
public static void main(String[] args) {
|
||||
boolean allPass = true;
|
||||
|
||||
allPass &= testDedupGetField();
|
||||
|
||||
System.out.println();
|
||||
System.out.println(allPass ? "ALL TESTS PASSED" : "SOME TESTS FAILED");
|
||||
System.exit(allPass ? 0 : 1);
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue