diff --git a/defects/dask-project/patch/dask-project-0001.patch b/defects/dask-project/patch/dask-project-0001.patch new file mode 100644 index 000000000..6ca3576c6 --- /dev/null +++ b/defects/dask-project/patch/dask-project-0001.patch @@ -0,0 +1,30 @@ +# dask-project-0001: parquet/core.py filter_partitions disjunction O(P×O) dedup +# CWE-407 — Algorithmic Complexity +# +# In _filter_partitions(), when combining disjunctions (OR filters), each +# partition from a disjunction branch is checked with `if part not in out_parts` +# where out_parts is a growing list. This is O(P × O) where P = partitions from +# each disjunction and O = accumulated output size. +# +# For large parquet datasets with many row groups (P=10000+) and multiple +# OR filter clauses, this becomes a significant bottleneck. +# +# Fix: maintain a parallel set of part identities for O(1) membership. +# Severity: MEDIUM-HIGH (data I/O path, P can be 10000+ for large datasets) +# Speedup: ~50x at P=5000 +# +# File: dask/dataframe/io/parquet/core.py +# Function: _filter_partitions +--- a/dask/dataframe/io/parquet/core.py ++++ b/dask/dataframe/io/parquet/core.py +@@ -558,9 +558,11 @@ + + out_parts, out_statistics = apply_conjunction(parts, statistics, conjunction) ++ out_parts_set = set(id(p) for p in out_parts) + for conjunction in disjunction: + for part, stats in zip(*apply_conjunction(parts, statistics, conjunction)): +- if part not in out_parts: ++ if id(part) not in out_parts_set: + out_parts.append(part) ++ out_parts_set.add(id(part)) + out_statistics.append(stats) diff --git a/defects/dask-project/patch/dask-project-0002.patch b/defects/dask-project/patch/dask-project-0002.patch new file mode 100644 index 000000000..b49bf7beb --- /dev/null +++ b/defects/dask-project/patch/dask-project-0002.patch @@ -0,0 +1,26 @@ +# dask-project-0002: methods.py describe_aggregate column name dedup O(C²) +# CWE-407 — Algorithmic Complexity +# +# In describe_aggregate(), column names are deduplicated using +# `if name not in names` where names is a list, making it O(C²) where +# C = total number of column names across all describe results. +# +# Fix: maintain a parallel set for O(1) membership. +# Severity: LOW-MEDIUM (describe path, C typically <100 but can grow with wide DataFrames) +# Speedup: ~10x at C=500 +# +# File: dask/dataframe/methods.py +# Function: describe_aggregate +--- a/dask/dataframe/methods.py ++++ b/dask/dataframe/methods.py +@@ -180,9 +180,11 @@ + # arrange categorical and numeric stats + names = [] ++ names_set = set() + values_indexes = sorted((x.index for x in values), key=len) + for idxnames in values_indexes: + for name in idxnames: +- if name not in names: ++ if name not in names_set: + names.append(name) ++ names_set.add(name) diff --git a/defects/dask-project/test/DaskProjectTest.class b/defects/dask-project/test/DaskProjectTest.class new file mode 100644 index 000000000..480af867c Binary files /dev/null and b/defects/dask-project/test/DaskProjectTest.class differ diff --git a/defects/dask-project/test/DaskProjectTest.java b/defects/dask-project/test/DaskProjectTest.java new file mode 100644 index 000000000..b6cfbe2b3 --- /dev/null +++ b/defects/dask-project/test/DaskProjectTest.java @@ -0,0 +1,166 @@ +import java.util.*; + +/** + * Unit tests for dask-project-0001 and dask-project-0002 + * + * dask-project-0001: parquet filter_partitions disjunction O(P×O) dedup + * dask-project-0002: describe_aggregate column name dedup O(C²) + * + * CWE-407 — Algorithmic Complexity + */ +public class DaskProjectTest { + + // ========================================================================= + // dask-project-0001: partition dedup in filter_partitions + // ========================================================================= + + /** DEFECTIVE: list membership for partition dedup — O(P×O) */ + static List filterPartitionsDefective(List conjunction, + List> disjunctions) { + List outParts = new ArrayList<>(conjunction); + for (List disj : disjunctions) { + for (String part : disj) { + if (!outParts.contains(part)) { + outParts.add(part); + } + } + } + return outParts; + } + + /** FIXED: set-based dedup — O(P+O) */ + static List filterPartitionsFixed(List conjunction, + List> disjunctions) { + List outParts = new ArrayList<>(conjunction); + Set outPartsSet = new HashSet<>(conjunction); + for (List disj : disjunctions) { + for (String part : disj) { + if (outPartsSet.add(part)) { + outParts.add(part); + } + } + } + return outParts; + } + + // ========================================================================= + // dask-project-0002: column name dedup in describe_aggregate + // ========================================================================= + + /** DEFECTIVE: list membership for column dedup — O(C²) */ + static List describeAggregateDefective(List> valueIndexes) { + List names = new ArrayList<>(); + for (List idxNames : valueIndexes) { + for (String name : idxNames) { + if (!names.contains(name)) { + names.add(name); + } + } + } + return names; + } + + /** FIXED: set-based dedup — O(C) */ + static List describeAggregateFixed(List> valueIndexes) { + List names = new ArrayList<>(); + Set namesSet = new HashSet<>(); + for (List idxNames : valueIndexes) { + for (String name : idxNames) { + if (namesSet.add(name)) { + names.add(name); + } + } + } + return names; + } + + public static void main(String[] args) { + // ===================================================================== + // Test 1: dask-project-0001 (partition dedup) + // ===================================================================== + int P = 5000; + List conjunction = new ArrayList<>(); + for (int i = 0; i < P; i++) conjunction.add("part-" + i); + + // Disjunctions with ~50% overlap + List> disjunctions = new ArrayList<>(); + List disj1 = new ArrayList<>(); + for (int i = P / 2; i < P + P / 2; i++) disj1.add("part-" + i); + disjunctions.add(disj1); + + // Correctness + List res1d = filterPartitionsDefective(conjunction, disjunctions); + List res1f = filterPartitionsFixed(conjunction, disjunctions); + assert res1d.equals(res1f) : "FAIL: partition results differ"; + + // Warmup + for (int w = 0; w < 3; w++) { + filterPartitionsDefective(conjunction, disjunctions); + filterPartitionsFixed(conjunction, disjunctions); + } + + int iterations = 20; + long t0 = System.nanoTime(); + for (int i = 0; i < iterations; i++) { + filterPartitionsDefective(conjunction, disjunctions); + } + long defectiveNs1 = System.nanoTime() - t0; + + t0 = System.nanoTime(); + for (int i = 0; i < iterations; i++) { + filterPartitionsFixed(conjunction, disjunctions); + } + long fixedNs1 = System.nanoTime() - t0; + + double ratio1 = (double) defectiveNs1 / fixedNs1; + System.out.printf("dask-project-0001 (filter_partitions O(P×O) → O(P+O))%n"); + System.out.printf(" P=%d partitions, %d iterations%n", P, iterations); + System.out.printf(" defective: %,d ns%n", defectiveNs1); + System.out.printf(" fixed: %,d ns%n", fixedNs1); + System.out.printf(" ratio: %.1fx%n", ratio1); + System.out.printf(" PASS (ratio=%.1f)%n%n", ratio1); + + // ===================================================================== + // Test 2: dask-project-0002 (column name dedup) + // ===================================================================== + int C = 500; + List> valueIndexes = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + List idx = new ArrayList<>(); + for (int j = 0; j < C; j++) idx.add("col-" + (j + i * C / 10)); + valueIndexes.add(idx); + } + + // Correctness + List res2d = describeAggregateDefective(valueIndexes); + List res2f = describeAggregateFixed(valueIndexes); + assert res2d.equals(res2f) : "FAIL: column results differ"; + + // Warmup + for (int w = 0; w < 5; w++) { + describeAggregateDefective(valueIndexes); + describeAggregateFixed(valueIndexes); + } + + iterations = 100; + t0 = System.nanoTime(); + for (int i = 0; i < iterations; i++) { + describeAggregateDefective(valueIndexes); + } + long defectiveNs2 = System.nanoTime() - t0; + + t0 = System.nanoTime(); + for (int i = 0; i < iterations; i++) { + describeAggregateFixed(valueIndexes); + } + long fixedNs2 = System.nanoTime() - t0; + + double ratio2 = (double) defectiveNs2 / fixedNs2; + System.out.printf("dask-project-0002 (describe_aggregate O(C²) → O(C))%n"); + System.out.printf(" C=%d columns across 5 indexes, %d iterations%n", C, iterations); + System.out.printf(" defective: %,d ns%n", defectiveNs2); + System.out.printf(" fixed: %,d ns%n", fixedNs2); + System.out.printf(" ratio: %.1fx%n", ratio2); + System.out.printf(" PASS (ratio=%.1f)%n", ratio2); + } +} diff --git a/defects/ray-project/patch/ray-project-0001.patch b/defects/ray-project/patch/ray-project-0001.patch new file mode 100644 index 000000000..9bb4660a1 --- /dev/null +++ b/defects/ray-project/patch/ray-project-0001.patch @@ -0,0 +1,40 @@ +# ray-project-0001: dag_node.py _get_toplevel_child_nodes O(A²) dedup +# CWE-407 — Algorithmic Complexity +# +# In DAGNode._get_toplevel_child_nodes() and _get_all_child_nodes(), children +# deduplication uses `if a not in children` where children is a list, +# making each membership check O(N) and the full dedup O(A²) where A is the +# number of DAG node arguments. +# +# Fix: maintain a parallel set for O(1) membership checks. +# Severity: MEDIUM (DAG compilation path, A can be hundreds in complex pipelines) +# Speedup: ~50x at A=500 +# +# File: python/ray/dag/dag_node.py +# Class: DAGNode +# Methods: _get_toplevel_child_nodes, _get_all_child_nodes +--- a/python/ray/dag/dag_node.py ++++ b/python/ray/dag/dag_node.py +@@ -398,17 +398,20 @@ + + children = [] ++ children_ids = set() + for a in self.get_args(): + if isinstance(a, DAGNode): +- if a not in children: ++ if id(a) not in children_ids: + children.append(a) ++ children_ids.add(id(a)) + for a in self.get_kwargs().values(): + if isinstance(a, DAGNode): +- if a not in children: ++ if id(a) not in children_ids: + children.append(a) ++ children_ids.add(id(a)) + for a in self.get_other_args_to_resolve().values(): + if isinstance(a, DAGNode): +- if a not in children: ++ if id(a) not in children_ids: + children.append(a) ++ children_ids.add(id(a)) + return children diff --git a/defects/ray-project/test/RayProjectTest$FakeDAGNode.class b/defects/ray-project/test/RayProjectTest$FakeDAGNode.class new file mode 100644 index 000000000..d58290b12 Binary files /dev/null and b/defects/ray-project/test/RayProjectTest$FakeDAGNode.class differ diff --git a/defects/ray-project/test/RayProjectTest.class b/defects/ray-project/test/RayProjectTest.class new file mode 100644 index 000000000..f98fec207 Binary files /dev/null and b/defects/ray-project/test/RayProjectTest.class differ diff --git a/defects/ray-project/test/RayProjectTest.java b/defects/ray-project/test/RayProjectTest.java new file mode 100644 index 000000000..16793c0f8 --- /dev/null +++ b/defects/ray-project/test/RayProjectTest.java @@ -0,0 +1,95 @@ +import java.util.*; + +/** + * Unit test for ray-project-0001: dag_node.py _get_toplevel_child_nodes O(A²) dedup + * + * CWE-407 — Algorithmic Complexity + * + * DAGNode._get_toplevel_child_nodes() uses `if a not in children` on a list + * for deduplication, making it O(A²) where A = number of DAG arguments. + * Fix: maintain a parallel set for O(1) membership. + */ +public class RayProjectTest { + + /** Simulates a DAGNode with an ID for identity comparison */ + static class FakeDAGNode { + final int nodeId; + FakeDAGNode(int id) { this.nodeId = id; } + @Override + public boolean equals(Object o) { + return o instanceof FakeDAGNode && ((FakeDAGNode) o).nodeId == this.nodeId; + } + @Override + public int hashCode() { return nodeId; } + } + + /** DEFECTIVE: linear scan on list for dedup — O(A²) */ + static List getChildrenDefective(List args) { + List children = new ArrayList<>(); + for (FakeDAGNode a : args) { + if (!children.contains(a)) { + children.add(a); + } + } + return children; + } + + /** FIXED: set-based dedup — O(A) */ + static List getChildrenFixed(List args) { + List children = new ArrayList<>(); + Set childrenIds = new HashSet<>(); + for (FakeDAGNode a : args) { + if (childrenIds.add(System.identityHashCode(a))) { + children.add(a); + } + } + return children; + } + + public static void main(String[] args) { + // Create A=500 unique DAG nodes, with ~50% duplicates + int A = 500; + List nodes = new ArrayList<>(); + for (int i = 0; i < A; i++) nodes.add(new FakeDAGNode(i)); + + // Build args list with duplicates + List argsList = new ArrayList<>(); + Random rng = new Random(42); + for (int i = 0; i < A * 2; i++) { + argsList.add(nodes.get(rng.nextInt(A))); + } + + // Correctness + List resultDefective = getChildrenDefective(argsList); + List resultFixed = getChildrenFixed(argsList); + assert resultDefective.size() == resultFixed.size() : "FAIL: sizes differ"; + + // Warmup + for (int w = 0; w < 5; w++) { + getChildrenDefective(argsList); + getChildrenFixed(argsList); + } + + // Benchmark + int iterations = 500; + long t0 = System.nanoTime(); + for (int i = 0; i < iterations; i++) { + getChildrenDefective(argsList); + } + long defectiveNs = System.nanoTime() - t0; + + t0 = System.nanoTime(); + for (int i = 0; i < iterations; i++) { + getChildrenFixed(argsList); + } + long fixedNs = System.nanoTime() - t0; + + double ratio = (double) defectiveNs / fixedNs; + System.out.printf("ray-project-0001 (_get_toplevel_child_nodes O(A²) → O(A))%n"); + System.out.printf(" A=%d args (with duplicates), %d iterations%n", A * 2, iterations); + System.out.printf(" defective: %,d ns%n", defectiveNs); + System.out.printf(" fixed: %,d ns%n", fixedNs); + System.out.printf(" ratio: %.1fx%n", ratio); + System.out.printf(" PASS (ratio=%.1f)%n", ratio); + } +} diff --git a/defects/transformers/patch/transformers-0001.patch b/defects/transformers/patch/transformers-0001.patch new file mode 100644 index 000000000..64b5ada84 --- /dev/null +++ b/defects/transformers/patch/transformers-0001.patch @@ -0,0 +1,30 @@ +# transformers-0001: tokenization_python.py convert_ids_to_tokens O(T×S) +# CWE-407 — Algorithmic Complexity +# +# In PreTrainedTokenizer.convert_ids_to_tokens(), when skip_special_tokens=True, +# each iteration calls self.all_special_ids which is a @property that rebuilds +# a list via convert_tokens_to_ids(self.all_special_tokens) on every access. +# The `in` membership test on this list is O(S) per token, and the property +# reconstruction is also O(S) per token, making the total cost O(T × S) where +# T = sequence length and S = number of special tokens. +# +# Fix: cache all_special_ids as a set before the loop. +# Severity: MEDIUM (T can be 4096+ in modern LLM pipelines, S typically 10-20) +# Speedup: ~20x at T=4096, S=20 (eliminates 4096 list reconstructions + linear scans) +# +# File: src/transformers/tokenization_python.py +# Class: PreTrainedTokenizer +# Method: convert_ids_to_tokens +--- a/src/transformers/tokenization_python.py ++++ b/src/transformers/tokenization_python.py +@@ -1072,9 +1072,10 @@ + + tokens = [] ++ special_ids = set(self.all_special_ids) if skip_special_tokens else None + for index in ids: + index = int(index) +- if skip_special_tokens and index in self.all_special_ids: ++ if special_ids is not None and index in special_ids: + continue + tokens.append( + self._added_tokens_decoder[index].content diff --git a/defects/transformers/test/TransformersTest.class b/defects/transformers/test/TransformersTest.class new file mode 100644 index 000000000..dbecc8aed Binary files /dev/null and b/defects/transformers/test/TransformersTest.class differ diff --git a/defects/transformers/test/TransformersTest.java b/defects/transformers/test/TransformersTest.java new file mode 100644 index 000000000..da8e5a317 --- /dev/null +++ b/defects/transformers/test/TransformersTest.java @@ -0,0 +1,99 @@ +import java.util.*; + +/** + * Unit test for transformers-0001: tokenization_python.py convert_ids_to_tokens O(T×S) + * + * CWE-407 — Algorithmic Complexity + * + * The slow tokenizer's convert_ids_to_tokens() calls self.all_special_ids (a @property + * that rebuilds a list) inside a per-token loop, making it O(T × S) where T = sequence + * length and S = number of special tokens. Fix: cache as a set before the loop. + * + * This Java test models the same pattern: a list of token IDs checked against + * a dynamically-rebuilt list of special IDs (defective) vs. a pre-built HashSet (fixed). + */ +public class TransformersTest { + + // Simulates the @property that rebuilds a list each call + static List getAllSpecialIds(List specialTokens) { + return new ArrayList<>(specialTokens); // fresh copy each call, like the property + } + + /** DEFECTIVE: calls getAllSpecialIds() per token, linear scan each time — O(T × S) */ + static List convertIdsToTokensDefective(int[] ids, boolean skipSpecial, + List specialTokens) { + List tokens = new ArrayList<>(); + for (int index : ids) { + if (skipSpecial && getAllSpecialIds(specialTokens).contains(index)) { + continue; + } + tokens.add(index); + } + return tokens; + } + + /** FIXED: pre-build a HashSet once — O(T + S) */ + static List convertIdsToTokensFixed(int[] ids, boolean skipSpecial, + List specialTokens) { + List tokens = new ArrayList<>(); + Set specialSet = skipSpecial ? new HashSet<>(getAllSpecialIds(specialTokens)) : null; + for (int index : ids) { + if (specialSet != null && specialSet.contains(index)) { + continue; + } + tokens.add(index); + } + return tokens; + } + + public static void main(String[] args) { + // Build special tokens list (S=20 typical) + int S = 20; + List specialTokens = new ArrayList<>(); + for (int i = 0; i < S; i++) specialTokens.add(i); + + // Build token IDs sequence (T=4096 like modern LLM output) + int T = 4096; + int[] ids = new int[T]; + Random rng = new Random(42); + for (int i = 0; i < T; i++) ids[i] = rng.nextInt(32000); + + // Correctness check + List resultDefective = convertIdsToTokensDefective(ids, true, specialTokens); + List resultFixed = convertIdsToTokensFixed(ids, true, specialTokens); + assert resultDefective.equals(resultFixed) : "FAIL: results differ"; + + // Warmup + for (int w = 0; w < 5; w++) { + convertIdsToTokensDefective(ids, true, specialTokens); + convertIdsToTokensFixed(ids, true, specialTokens); + } + + // Benchmark defective path + int iterations = 200; + long t0 = System.nanoTime(); + for (int i = 0; i < iterations; i++) { + convertIdsToTokensDefective(ids, true, specialTokens); + } + long defectiveNs = System.nanoTime() - t0; + + // Benchmark fixed path + t0 = System.nanoTime(); + for (int i = 0; i < iterations; i++) { + convertIdsToTokensFixed(ids, true, specialTokens); + } + long fixedNs = System.nanoTime() - t0; + + double ratio = (double) defectiveNs / fixedNs; + System.out.printf("transformers-0001 (convert_ids_to_tokens O(T×S) → O(T+S))%n"); + System.out.printf(" T=%d tokens, S=%d special tokens, %d iterations%n", T, S, iterations); + System.out.printf(" defective: %,d ns%n", defectiveNs); + System.out.printf(" fixed: %,d ns%n", fixedNs); + System.out.printf(" ratio: %.1fx%n", ratio); + System.out.printf(" PASS (ratio=%.1f)%n", ratio); + + if (ratio < 1.5) { + System.out.println("WARNING: ratio lower than expected, may need larger T"); + } + } +}