undefect. CWE-407 — 92 sites, 42 ecosystems

B&W print-friendly diagrams + tinkerpop-0001 + wave-3 proof sections.
Squash of 94 local commits onto remote master.
This commit is contained in:
russell@unturf.com 2026-03-26 19:48:18 -04:00
parent 0a580b313d
commit db29a08762
1311 changed files with 371202 additions and 1188 deletions

View file

@ -0,0 +1,46 @@
diff --git a/libpromises/evalfunction.c b/libpromises/evalfunction.c
index a1b2c3d..e4f5a6b 100644
--- a/libpromises/evalfunction.c
+++ b/libpromises/evalfunction.c
@@ -3649,15 +3649,24 @@ static FnCallResult FnCallGetIndicesClassic(EvalContext *ctx, ARG_UNUSED const P
}
}
- Rlist *keys = NULL;
+ /* CWE-407 fix: replace Rlist accumulator with StringSet for O(1) dedup.
+ * The defect: RlistAppendScalarIdemp() calls RlistKeyIn() — an O(K)
+ * linked-list walk — on every insertion, giving O(K²) total cost when
+ * the variable table has K matching indices.
+ * Fix: collect unique indices into a StringSet (hash table, O(1) insert
+ * and membership), then convert to Rlist once at return. O(K) total. */
+ StringSet *keys_set = StringSetNew();
VariableTableIterator *iter = EvalContextVariableTableFromRefIteratorNew(ctx, ref);
const Variable *itervar;
while ((itervar = VariableTableIteratorNext(iter)) != NULL)
{
const VarRef *itervar_ref = VariableGetRef(itervar);
if (itervar_ref->num_indices > ref->num_indices)
{
- RlistAppendScalarIdemp(&keys, itervar_ref->indices[ref->num_indices]);
+ /* O(1) hash insert; StringSet silently ignores duplicates. */
+ StringSetAdd(keys_set, xstrdup(itervar_ref->indices[ref->num_indices]));
}
}
VariableTableIteratorDestroy(iter);
VarRefDestroy(ref);
- return (FnCallResult) { FNCALL_SUCCESS, { keys, RVAL_TYPE_LIST } };
+ /* Convert StringSet → Rlist for the caller. */
+ Rlist *keys = NULL;
+ StringSetIterator set_iter = StringSetIteratorInit(keys_set);
+ const char *key;
+ while ((key = StringSetIteratorNext(&set_iter)) != NULL)
+ {
+ RlistAppendScalar(&keys, key);
+ }
+ StringSetDestroy(keys_set);
+
+ return (FnCallResult) { FNCALL_SUCCESS, { keys, RVAL_TYPE_LIST } };
}

View file

@ -0,0 +1,79 @@
diff --git a/libpromises/evalfunction.c b/libpromises/evalfunction.c
index a1b2c3d..f7c8d9e 100644
--- a/libpromises/evalfunction.c
+++ b/libpromises/evalfunction.c
@@ -5768,7 +5768,10 @@ static FnCallResult FnCallSetop(EvalContext *ctx,
StringSet *set_b = StringSetNew();
if (!unique_mode)
{
JsonIterator iter = JsonIteratorInit(json_b);
const JsonElement *e;
while ((e = JsonIteratorNextValueByType(&iter, JSON_ELEMENT_TYPE_PRIMITIVE, true)))
{
StringSetAdd(set_b, xstrdup(JsonPrimitiveGetAsString(e)));
}
}
+ /* CWE-407 fix for unique() mode: build a StringSet from the input first
+ * so membership checks are O(1), then emit one Rlist per unique value.
+ * The defect: when unique_mode is true, set_b is always empty, so
+ * RlistAppendScalarIdemp() falls back to walking the growing returnlist
+ * on every call — O(N) per element, O(N²) total for N input values.
+ * Fix: use a dedicated StringSet to track seen values. O(N) total. */
+ StringSet *seen = unique_mode ? StringSetNew() : NULL;
+
Rlist *returnlist = NULL;
JsonIterator iter = JsonIteratorInit(json);
const JsonElement *e;
while ((e = JsonIteratorNextValueByType(&iter, JSON_ELEMENT_TYPE_PRIMITIVE, true)))
{
const char *value = JsonPrimitiveGetAsString(e);
// Yes, this is an XOR. But it's more legible this way.
if (!unique_mode && difference_mode && StringSetContains(set_b, value))
{
continue;
}
if (!unique_mode && !difference_mode && !StringSetContains(set_b, value))
{
continue;
}
- RlistAppendScalarIdemp(&returnlist, value);
+ if (unique_mode)
+ {
+ /* O(1) hash lookup replaces the O(N) Rlist walk. */
+ if (!StringSetContains(seen, value))
+ {
+ StringSetAdd(seen, xstrdup(value));
+ RlistAppendScalar(&returnlist, value);
+ }
+ }
+ else
+ {
+ /* intersection / difference: set_b already deduplicates by
+ * construction; a given value can appear multiple times in
+ * json_a but set_b membership already filters correctly.
+ * Use Idemp here to preserve the previous dedup behaviour for
+ * the non-unique paths (they are not the hot path). */
+ RlistAppendScalarIdemp(&returnlist, value);
+ }
}
JsonDestroyMaybe(json, allocated);
if (json_b != NULL)
{
JsonDestroyMaybe(json_b, allocated_b);
}
+ if (seen != NULL)
+ {
+ StringSetDestroy(seen);
+ }
+
StringSetDestroy(set_b);
return (FnCallResult) { FNCALL_SUCCESS, (Rval) { returnlist, RVAL_TYPE_LIST } };
}

View file

@ -0,0 +1,62 @@
diff --git a/libpromises/evalfunction.c b/libpromises/evalfunction.c
index a1b2c3d..c2e1f7b 100644
--- a/libpromises/evalfunction.c
+++ b/libpromises/evalfunction.c
@@ -4221,6 +4221,12 @@ static FnCallResult FnCallMapData(EvalContext *ctx, ARG_UNUSED const Policy *pol
bool mapdatamode = (strcmp(fp->name, "mapdata") == 0);
Rlist *returnlist = NULL;
+ /* CWE-407 fix: track already-appended values in a StringSet so the
+ * nested-container branch can dedup in O(1) per element instead of the
+ * O(R) linked-list walk performed by RlistAppendScalarIdemp.
+ * With N total sub-elements, that was O(N²); with StringSet it is O(N).
+ * Initialised here; destroyed at every exit path below. */
+ StringSet *seen = StringSetNew();
+
// This is a delayed evaluation function, so we have to resolve arguments ourselves
// We resolve them once now, to get the second or third argument with the iteration data
Rlist *expargs = NewExpArgs(ctx, policy, fp, NULL);
@@ -4335,6 +4341,7 @@ static FnCallResult FnCallMapData(EvalContext *ctx, ARG_UNUSED const Policy *pol
if (strstr(BufferData(expbuf), "$(this.k)") || strstr(BufferData(expbuf), "${this.k}") ||
strstr(BufferData(expbuf), "$(this.v)") || strstr(BufferData(expbuf), "${this.v}"))
{
+ StringSetDestroy(seen);
RlistDestroy(returnlist);
EvalContextVariableRemoveSpecial(ctx, SPECIAL_SCOPE_THIS, "k");
EvalContextVariableRemoveSpecial(ctx, SPECIAL_SCOPE_THIS, "v");
@@ -4385,6 +4392,7 @@ static FnCallResult FnCallMapData(EvalContext *ctx, ARG_UNUSED const Policy *pol
if (strstr(BufferData(expbuf), "$(this.k)") || strstr(BufferData(expbuf), "${this.k}") ||
(havekey && (strstr(BufferData(expbuf), "$(this.k[1])") || strstr(BufferData(expbuf), "${this.k[1]}"))) ||
strstr(BufferData(expbuf), "$(this.v)") || strstr(BufferData(expbuf), "${this.v}"))
{
+ StringSetDestroy(seen);
RlistDestroy(returnlist);
EvalContextVariableRemoveSpecial(ctx, SPECIAL_SCOPE_THIS, "k");
if (havekey)
@@ -4404,9 +4413,18 @@ static FnCallResult FnCallMapData(EvalContext *ctx, ARG_UNUSED const Policy *pol
if (canonifymode)
{
BufferCanonify(expbuf);
}
- RlistAppendScalarIdemp(&returnlist, BufferData(expbuf));
+ /* CWE-407 fix: O(1) hash membership test replaces O(R) list
+ * walk. RlistAppendScalarIdemp called RlistKeyIn() which
+ * scanned the full returnlist on every iteration. */
+ const char *expanded = BufferData(expbuf);
+ if (!StringSetContains(seen, expanded))
+ {
+ StringSetAdd(seen, xstrdup(expanded));
+ RlistAppendScalar(&returnlist, expanded);
+ }
if (havekey)
{
EvalContextVariableRemoveSpecial(ctx, SPECIAL_SCOPE_THIS, "k[1]");
@@ -4421,6 +4439,8 @@ static FnCallResult FnCallMapData(EvalContext *ctx, ARG_UNUSED const Policy *pol
}
BufferDestroy(expbuf);
+ StringSetDestroy(seen);
+
JsonDestroyMaybe(container, allocated);
RlistDestroy(expargs);

View file

@ -0,0 +1,400 @@
package unit;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.LinkedHashSet;
/**
* CFEngineRlistTest
*
* Models three CWE-407 defects in libpromises/evalfunction.c:
*
* CFE-001 (MEDIUM) — FnCallGetIndicesClassic / getindices():
* RlistAppendScalarIdemp(&keys, ...) calls RlistKeyIn(keys, ...) —
* O(K) linked-list walk per insertion. O(K²) total over K indices.
* Fix: collect indices into a StringSet (O(1) insert), convert once.
*
* CFE-002 (HIGH) — FnCallSetop / unique():
* In unique_mode, set_b is always empty. RlistAppendScalarIdemp
* falls back to walking the growing returnlist on every element —
* O(N) per element, O(N²) total for N input values.
* Fix: maintain a separate StringSet of seen values. O(N) total.
*
* CFE-003 (MEDIUM) — FnCallMapData / maparray() nested-container branch:
* Outer while over JSON object, inner while over sub-container, then
* RlistAppendScalarIdemp(&returnlist, ...) — O(R) scan of growing
* returnlist per sub-element. O(N²) total for N sub-elements.
* Fix: StringSet for dedup tracking, RlistAppendScalar for appends.
*
* Rlist is modelled as ArrayList<String>.
* StringSet is modelled as HashSet<String>.
* Operation counts are instrumented explicitly — not wall-clock timing.
*/
public class CFEngineRlistTest {
// -----------------------------------------------------------------------
// CFE-001 models
// -----------------------------------------------------------------------
/**
* Defective: RlistAppendScalarIdemp — walks the list to check membership
* before each append. Returns total comparison count.
*/
static long cfe001Defective(String[] indices) {
ArrayList<String> keys = new ArrayList<>();
long comparisons = 0;
for (String idx : indices) {
boolean found = false;
// RlistKeyIn: O(K) linear scan of existing keys
for (String existing : keys) {
comparisons++;
if (existing.equals(idx)) {
found = true;
break;
}
}
if (!found) {
keys.add(idx);
}
}
return comparisons;
}
/**
* Fixed: StringSet for dedup, convert to list once at end.
* Returns total hash-lookup count (one per insertion attempt).
*/
static long cfe001Fixed(String[] indices) {
HashSet<String> keysSet = new HashSet<>();
long lookups = 0;
for (String idx : indices) {
lookups++; // one O(1) contains() per element
keysSet.add(idx); // silently ignores duplicates
}
// Convert to list — O(K) one-time cost, not charged here
ArrayList<String> keys = new ArrayList<>(keysSet);
return lookups;
}
// -----------------------------------------------------------------------
// CFE-002 models
// -----------------------------------------------------------------------
/**
* Defective: unique() with empty set_b — dedup falls on returnlist.
* Models the unique_mode path where set_b is always empty.
*/
static long cfe002Defective(String[] values) {
ArrayList<String> returnlist = new ArrayList<>();
long comparisons = 0;
for (String value : values) {
// set_b empty → RlistAppendScalarIdemp walks returnlist
boolean found = false;
for (String existing : returnlist) {
comparisons++;
if (existing.equals(value)) {
found = true;
break;
}
}
if (!found) {
returnlist.add(value);
}
}
return comparisons;
}
/**
* Fixed: separate StringSet tracks seen values; returnlist gets plain
* appends with no membership scan.
*/
static long cfe002Fixed(String[] values) {
HashSet<String> seen = new HashSet<>();
ArrayList<String> returnlist = new ArrayList<>();
long lookups = 0;
for (String value : values) {
lookups++; // one O(1) contains() per element
if (!seen.contains(value)) {
seen.add(value);
returnlist.add(value); // plain append — no scan
}
}
return lookups;
}
// -----------------------------------------------------------------------
// CFE-003 models
// -----------------------------------------------------------------------
/**
* Defective: nested-container iteration with RlistAppendScalarIdemp.
* outerCount outer keys, each with innerCount sub-elements.
*/
static long cfe003Defective(int outerCount, int innerCount) {
ArrayList<String> returnlist = new ArrayList<>();
long comparisons = 0;
for (int i = 0; i < outerCount; i++) {
for (int j = 0; j < innerCount; j++) {
// expanded string: same value across outer keys → many dupes
String expanded = "value_" + j;
// RlistAppendScalarIdemp: O(R) scan of returnlist
boolean found = false;
for (String existing : returnlist) {
comparisons++;
if (existing.equals(expanded)) {
found = true;
break;
}
}
if (!found) {
returnlist.add(expanded);
}
}
}
return comparisons;
}
/**
* Fixed: StringSet tracks seen; plain appends to returnlist.
*/
static long cfe003Fixed(int outerCount, int innerCount) {
HashSet<String> seen = new HashSet<>();
ArrayList<String> returnlist = new ArrayList<>();
long lookups = 0;
for (int i = 0; i < outerCount; i++) {
for (int j = 0; j < innerCount; j++) {
String expanded = "value_" + j;
lookups++; // O(1) contains()
if (!seen.contains(expanded)) {
seen.add(expanded);
returnlist.add(expanded);
}
}
}
return lookups;
}
// -----------------------------------------------------------------------
// Helpers
// -----------------------------------------------------------------------
/** Build K indices where the first K/2 are unique, rest are duplicates. */
static String[] makeIndices(int k) {
int distinct = Math.max(1, k / 2);
String[] out = new String[k];
for (int i = 0; i < k; i++) {
out[i] = "idx_" + (i % distinct);
}
return out;
}
// -----------------------------------------------------------------------
// Test 1 — CFE-001: getindices() dedup cost at K=60
// Defect does O(K²) comparisons; fix does O(K) lookups.
// -----------------------------------------------------------------------
static void test1_cfe001_getindices() {
int k = 60;
String[] indices = makeIndices(k);
long defectOps = cfe001Defective(indices);
long fixedOps = cfe001Fixed(indices);
System.out.printf("test1 CFE-001: k=%d defect_comparisons=%d fixed_lookups=%d%n",
k, defectOps, fixedOps);
assert defectOps > fixedOps
: "defect must do more work than fix at k=" + k;
// With k/2 distinct values, defect compares at minimum triangular(k/2)
long expectedMinDefect = (long)(k / 2) * (k / 2 - 1) / 2;
assert defectOps >= expectedMinDefect
: "defect comparisons=" + defectOps + " expected >= " + expectedMinDefect;
}
// -----------------------------------------------------------------------
// Test 2 — CFE-002: unique() dedup cost at N=80 all-distinct input
// Defect: O(N²); fix: O(N). All-distinct maximises list scan length.
// -----------------------------------------------------------------------
static void test2_cfe002_unique() {
int n = 80;
// All distinct → worst case: every element is a cache miss on returnlist
String[] values = new String[n];
for (int i = 0; i < n; i++) values[i] = "val_" + i;
long defectOps = cfe002Defective(values);
long fixedOps = cfe002Fixed(values);
// Defect: input 0 → list empty (0 comparisons); input i → list.size()=i scans
// Total: 0 + 1 + 2 + ... + (n-1) = n*(n-1)/2
long expectedDefect = (long) n * (n - 1) / 2;
double ratio = (double) defectOps / Math.max(1, fixedOps);
System.out.printf("test2 CFE-002: n=%d distinct defect=%d (expect=%d) fixed=%d ratio=%.1fx%n",
n, defectOps, expectedDefect, fixedOps, ratio);
assert defectOps == expectedDefect
: "defect comparisons=" + defectOps + " expected=" + expectedDefect;
assert ratio > 20.0
: "expected ratio > 20x for all-distinct unique(), got " + ratio;
}
// -----------------------------------------------------------------------
// Test 3 — CFE-003: maparray() nested-container cost
// 10 outer keys × 20 sub-elements, all sub-elements produce same
// values across outer iterations → heavy dedup pressure on returnlist.
// -----------------------------------------------------------------------
static void test3_cfe003_maparray() {
int outer = 10;
int inner = 20;
long defectOps = cfe003Defective(outer, inner);
long fixedOps = cfe003Fixed(outer, inner);
double ratio = (double) defectOps / Math.max(1, fixedOps);
System.out.printf("test3 CFE-003: outer=%d inner=%d defect=%d fixed=%d ratio=%.1fx%n",
outer, inner, defectOps, fixedOps, ratio);
assert defectOps > fixedOps
: "defect must do more work than fix";
// After the first outer iteration, all inner values are known.
// From iteration 2 onward every inner lookup hits immediately at pos 0..inner-1.
// Defect comparisons are > inner * outer (always at least 1 per dup hit).
assert defectOps > inner
: "defect should do more comparisons than a single pass";
assert ratio > 2.0
: "expected ratio > 2x, got " + ratio;
}
// -----------------------------------------------------------------------
// Test 4 — Scaling: doubling N roughly quadruples defect ops (O(N²))
// but only doubles fix ops (O(N)). Covers all three defects.
// -----------------------------------------------------------------------
static void test4_quadraticScaling() {
int n1 = 50;
int n2 = 100; // 2x
// CFE-001 scaling (all-distinct indices → worst case)
String[] idx1 = new String[n1]; for (int i=0;i<n1;i++) idx1[i]="i"+i;
String[] idx2 = new String[n2]; for (int i=0;i<n2;i++) idx2[i]="i"+i;
double d1_growth = (double) cfe001Defective(idx2) / Math.max(1, cfe001Defective(idx1));
double f1_growth = (double) cfe001Fixed(idx2) / Math.max(1, cfe001Fixed(idx1));
// CFE-002 scaling (all-distinct values)
String[] val1 = new String[n1]; for (int i=0;i<n1;i++) val1[i]="v"+i;
String[] val2 = new String[n2]; for (int i=0;i<n2;i++) val2[i]="v"+i;
double d2_growth = (double) cfe002Defective(val2) / Math.max(1, cfe002Defective(val1));
double f2_growth = (double) cfe002Fixed(val2) / Math.max(1, cfe002Fixed(val1));
// CFE-003 scaling (outer fixed=5, double inner from n1/5 to n2/5)
int outer = 5;
double d3_growth = (double) cfe003Defective(outer, n2/outer) / Math.max(1, cfe003Defective(outer, n1/outer));
double f3_growth = (double) cfe003Fixed(outer, n2/outer) / Math.max(1, cfe003Fixed(outer, n1/outer));
System.out.printf("test4 scaling (2x N): CFE-001 defect=%.2fx fix=%.2fx " +
"CFE-002 defect=%.2fx fix=%.2fx CFE-003 defect=%.2fx fix=%.2fx%n",
d1_growth, f1_growth, d2_growth, f2_growth, d3_growth, f3_growth);
// Defect must grow super-linearly (> 2x when N doubles → quadratic)
assert d1_growth > 2.0 : "CFE-001 defect growth should be super-linear, got " + d1_growth;
assert d2_growth > 2.0 : "CFE-002 defect growth should be super-linear, got " + d2_growth;
assert d3_growth > 1.5 : "CFE-003 defect growth should be super-linear, got " + d3_growth;
// Fix must grow at most linearly (≤ 2.5x for 2x N, allowing hash overhead)
assert f1_growth <= 2.5 : "CFE-001 fix growth should be at most linear, got " + f1_growth;
assert f2_growth <= 2.5 : "CFE-002 fix growth should be at most linear, got " + f2_growth;
assert f3_growth <= 2.5 : "CFE-003 fix growth should be at most linear, got " + f3_growth;
// Defect must grow faster than fix for each
assert d1_growth > f1_growth : "CFE-001 defect growth should exceed fix growth";
assert d2_growth > f2_growth : "CFE-002 defect growth should exceed fix growth";
assert d3_growth > f3_growth : "CFE-003 defect growth should exceed fix growth";
}
// -----------------------------------------------------------------------
// Test 5 — Correctness: defect and fix produce identical output sets
// for all three defects.
// -----------------------------------------------------------------------
static void test5_correctness() {
// CFE-001 correctness
String[] indices = makeIndices(40);
HashSet<String> defectKeys = new HashSet<>();
{
ArrayList<String> keys = new ArrayList<>();
for (String idx : indices) {
if (!keys.contains(idx)) keys.add(idx);
}
defectKeys.addAll(keys);
}
HashSet<String> fixedKeys = new HashSet<>();
{
// Fixed just uses a HashSet directly
fixedKeys.addAll(java.util.Arrays.asList(indices));
}
assert defectKeys.equals(fixedKeys)
: "CFE-001: defect and fix must produce same key set";
// CFE-002 correctness
String[] values = makeIndices(40);
ArrayList<String> defectUniq = new ArrayList<>();
for (String v : values) {
if (!defectUniq.contains(v)) defectUniq.add(v);
}
HashSet<String> fixedUniq = new LinkedHashSet<>(java.util.Arrays.asList(values))
.stream().collect(java.util.stream.Collectors.toCollection(HashSet::new));
assert new HashSet<>(defectUniq).equals(fixedUniq)
: "CFE-002: defect and fix must produce same unique set";
// CFE-003 correctness: same distinct expanded strings regardless of strategy
int outer = 4, inner = 8;
ArrayList<String> defectResult = new ArrayList<>();
ArrayList<String> fixedResult = new ArrayList<>();
HashSet<String> seenFixed = new HashSet<>();
for (int i = 0; i < outer; i++) {
for (int j = 0; j < inner; j++) {
String exp = "value_" + j;
if (!defectResult.contains(exp)) defectResult.add(exp);
if (!seenFixed.contains(exp)) { seenFixed.add(exp); fixedResult.add(exp); }
}
}
assert new HashSet<>(defectResult).equals(new HashSet<>(fixedResult))
: "CFE-003: defect and fix must produce same result set";
assert defectResult.equals(fixedResult)
: "CFE-003: insertion order must also match (both first-seen)";
System.out.printf("test5 correctness: CFE-001 keys=%d CFE-002 uniq=%d CFE-003 result=%d%n",
defectKeys.size(), defectUniq.size(), defectResult.size());
}
// -----------------------------------------------------------------------
// Main
// -----------------------------------------------------------------------
public static void main(String[] args) {
System.out.println("=== CFEngineRlistTest ===");
System.out.println("Modelling CWE-407: CFE-001 getindices / CFE-002 unique / CFE-003 maparray");
System.out.println();
test1_cfe001_getindices();
System.out.println(" PASS test1_cfe001_getindices");
test2_cfe002_unique();
System.out.println(" PASS test2_cfe002_unique");
test3_cfe003_maparray();
System.out.println(" PASS test3_cfe003_maparray");
test4_quadraticScaling();
System.out.println(" PASS test4_quadraticScaling");
test5_correctness();
System.out.println(" PASS test5_correctness");
System.out.println();
System.out.println("All 5 tests PASSED.");
}
}