diff --git a/defects/numpy/unit/NumpyTest.java b/defects/numpy/unit/NumpyTest.java index 2f1643f8a..31f62c621 100644 --- a/defects/numpy/unit/NumpyTest.java +++ b/defects/numpy/unit/NumpyTest.java @@ -10,6 +10,18 @@ import java.util.*; * slow(): counts ops using list membership (words.contains(w)) — O(W) per insert * fast(): counts ops using a parallel HashSet for O(1) membership * Assert: slowOps >= fastOps * 10 for W=500 deps per variable + * + * numpy-0002: recfunctions.stack_arrays — seen=[] list dedup O(A×F²) field-name tracking + * Real code (numpy/lib/recfunctions.py:1396-1405): + * seen = [] + * if name not in seen: seen.append(name) # O(F) per insert + * Fix: seen = set() + * + * numpy-0003: recfunctions.join_by — names list rebuilt inside loop + .index() O(F²) + * Real code (numpy/lib/recfunctions.py:1610-1612): + * names = [name for name, dtype in ndtype] # O(F) rebuild per iteration + * nameidx = names.index(fname) # O(F) linear search + * Fix: name_to_idx dict for O(1) lookup */ public class NumpyTest { @@ -97,12 +109,142 @@ public class NumpyTest { System.out.println("numpy-0001 PASS"); } + // ── numpy-0002: stack_arrays seen=[] list dedup ─────────────────────────── + + /** Simulates stack_arrays seen=[] field dedup — returns total list comparisons */ + static long slowStackArrays(int numArrays, int fieldsPerArray) { + List seen = new ArrayList<>(); + long ops = 0; + for (int a = 0; a < numArrays; a++) { + for (int f = 0; f < fieldsPerArray; f++) { + String name = "field_" + f; // shared field names across arrays + boolean found = false; + for (int k = 0; k < seen.size(); k++) { + ops++; + if (seen.get(k).equals(name)) { found = true; break; } + } + if (!found) seen.add(name); + } + } + return ops; + } + + /** Simulates fixed stack_arrays with seen=set() — returns total ops */ + static long fastStackArrays(int numArrays, int fieldsPerArray) { + Set seen = new HashSet<>(); + long ops = 0; + for (int a = 0; a < numArrays; a++) { + for (int f = 0; f < fieldsPerArray; f++) { + String name = "field_" + f; + ops++; // O(1) set probe + seen.add(name); + } + } + return ops; + } + + static void testStackArrays() { + int A = 100, F = 200; // 100 arrays, 200 fields each + + long slowOps = slowStackArrays(A, F); + long fastOps = fastStackArrays(A, F); + + System.out.printf( + "numpy-0002 A=%-4d F=%-4d slowOps=%-10d fastOps=%-8d ratio=%.1fx%n", + A, F, slowOps, fastOps, (double) slowOps / fastOps + ); + + assert slowOps > fastOps * 30 : + "numpy-0002 FAIL: expected >30x ratio; slow=" + slowOps + " fast=" + fastOps; + System.out.println("numpy-0002 PASS"); + } + + // ── numpy-0003: join_by names-list rebuild + .index() ───────────────────── + + /** Simulates join_by ndtype names-list rebuild + .index() — returns total comparisons */ + static long slowJoinBy(int f1, int f2) { + List ndtype = new ArrayList<>(); + long ops = 0; + for (int i = 0; i < f1; i++) ndtype.add("r1_field_" + i); + for (int i = 0; i < f2; i++) { + String fname = "r2_field_" + i; + // Rebuild list (simulates list comprehension O(F) per iteration) + List names = new ArrayList<>(ndtype); + ops += names.size(); + // .index() linear search + int idx = -1; + for (int k = 0; k < names.size(); k++) { + ops++; + if (names.get(k).equals(fname)) { idx = k; break; } + } + if (idx == -1) ndtype.add(fname); + } + // Site 2: output dtype names tuple membership check + List outputNames = new ArrayList<>(ndtype); + for (int i = 0; i < f1; i++) { + String f = "r1_field_" + i; + for (int k = 0; k < outputNames.size(); k++) { + ops++; + if (outputNames.get(k).equals(f)) break; + } + } + for (int i = 0; i < f2; i++) { + String f = "r2_field_" + i; + for (int k = 0; k < outputNames.size(); k++) { + ops++; + if (outputNames.get(k).equals(f)) break; + } + } + return ops; + } + + /** Simulates fixed join_by with name_to_idx dict — returns total ops */ + static long fastJoinBy(int f1, int f2) { + Map nameToIdx = new HashMap<>(); + List ndtype = new ArrayList<>(); + long ops = 0; + for (int i = 0; i < f1; i++) { + nameToIdx.put("r1_field_" + i, i); + ndtype.add("r1_field_" + i); + } + for (int i = 0; i < f2; i++) { + String fname = "r2_field_" + i; + ops++; // O(1) dict probe + if (!nameToIdx.containsKey(fname)) { + nameToIdx.put(fname, ndtype.size()); + ndtype.add(fname); + } + } + // Site 2: set membership + Set outputNamesSet = new HashSet<>(ndtype); + for (int i = 0; i < f1 + f2; i++) ops++; + return ops; + } + + static void testJoinBy() { + int F1 = 200, F2 = 200; + + long slowOps = slowJoinBy(F1, F2); + long fastOps = fastJoinBy(F1, F2); + + System.out.printf( + "numpy-0003 F1=%-4d F2=%-4d slowOps=%-10d fastOps=%-8d ratio=%.1fx%n", + F1, F2, slowOps, fastOps, (double) slowOps / fastOps + ); + + assert slowOps > fastOps * 50 : + "numpy-0003 FAIL: expected >50x ratio; slow=" + slowOps + " fast=" + fastOps; + System.out.println("numpy-0003 PASS"); + } + // ── main ────────────────────────────────────────────────────────────────── public static void main(String[] args) { - int pass = 0, total = 1; + int pass = 0, total = 3; try { testDependDict(); pass++; } catch (AssertionError e) { System.err.println(e.getMessage()); } + try { testStackArrays(); pass++; } catch (AssertionError e) { System.err.println(e.getMessage()); } + try { testJoinBy(); pass++; } catch (AssertionError e) { System.err.println(e.getMessage()); } System.out.printf("%n%d/%d PASS%n", pass, total); if (pass != total) System.exit(1); diff --git a/defects/pandas/patch/pandas-0002-style-render-hidden-elements-list-scan.md b/defects/pandas/patch/pandas-0002-style-render-hidden-elements-list-scan.md deleted file mode 100644 index cd0bfc79c..000000000 --- a/defects/pandas/patch/pandas-0002-style-render-hidden-elements-list-scan.md +++ /dev/null @@ -1,107 +0,0 @@ -# UNDF: UNDF-2026-000000691 -# UNDF: (pending) -# pandas-0001: _get_level_lengths — hidden_elements list scan O(R×L×H) - -## CWE-407 — Algorithmic Complexity: O(R×L×H) list-contains in DataFrame styler render - -| Field | Value | -|-------|-------| -| ID | pandas-0001 | -| Severity | MEDIUM | -| Ecosystem | pandas | -| Package | `pandas` | -| File | `pandas/io/formats/style_render.py` | -| Lines | 1840–1870 | -| Complexity | O(R×L×H) — rows × index levels × hidden elements | -| Hot path | `_get_level_lengths()` called on every `Styler.render()` / `to_html()` | - -## Background - -`_get_level_lengths(index, sparsify, max_index, hidden_elements)` computes span -lengths for rendering a (Multi)Index in HTML/LaTeX output. It is called twice -per render: once for row index, once for column index. `hidden_elements` is the -list of integer positions that should be omitted from the rendered output — set -by `Styler.hide(rows)` or `Styler.hide(columns)`. - -## Defect - -```python -# pandas/io/formats/style_render.py lines 1840–1870 -if hidden_elements is None: - hidden_elements = [] # list — default type - -# ... -for i, value in enumerate(levels): - if i not in hidden_elements: # DEFECT: O(H) list scan - lengths[(0, i)] = 1 -# ... - -for i, lvl in enumerate(levels): - for j, row in enumerate(lvl): - if not sparsify: - if j not in hidden_elements: # O(H) list scan - lengths[(i, j)] = 1 - elif (row is not lib.no_default) and (j not in hidden_elements): # O(H) - ... - elif j not in hidden_elements: # O(H) - ... -``` - -`hidden_elements` is declared as `Sequence[int]` and stored as a plain `list`: - -```python -# pandas/io/formats/style_render.py line 131 -self.hidden_rows: Sequence[int] = [] -self.hidden_columns: Sequence[int] = [] -``` - -Each `j not in hidden_elements` performs a linear scan. The outer loops run -R×L times (rows × MultiIndex levels), so total work is O(R×L×H). - -### When does this matter? - -Users calling `styler.hide(subset=large_slice)` on wide DataFrames or tall -DataFrames with MultiIndex before rendering: e.g. hiding 80% of 10,000 rows -in a 3-level MultiIndex generates ~24,000 list scans of length ~8,000 = 192M -comparisons per render. - -## Complexity table - -| Rows (R) | Hidden (H) | Levels (L) | list ops | set ops | Speedup | -|---------|----------|-----------|---------|---------|---------| -| 1,000 | 500 | 3 | 1,500,000 | 3,000 | 500× | -| 5,000 | 2,500 | 3 | 37,500,000 | 15,000 | 2,500× | -| 10,000 | 8,000 | 3 | 240,000,000 | 30,000 | 8,000× | - -## Fix - -Convert `hidden_elements` to a `set` at the point of use in -`_get_level_lengths`, or store it as a `frozenset` in `StylerRenderer`: - -```python -def _get_level_lengths( - index: Index, - sparsify: bool, - max_index: int, - hidden_elements: Sequence[int] | None = None, -): - if hidden_elements is None: - hidden_elements_set: frozenset[int] = frozenset() - else: - hidden_elements_set = frozenset(hidden_elements) # FIX: O(1) lookup - - # ... - for i, value in enumerate(levels): - if i not in hidden_elements_set: # O(1) - lengths[(0, i)] = 1 - # ... - for i, lvl in enumerate(levels): - for j, row in enumerate(lvl): - if j not in hidden_elements_set: # O(1) - ... -``` - -Alternatively, store `self.hidden_rows` and `self.hidden_columns` as -`set[int]` rather than `list[int]` throughout `StylerRenderer`, since -membership testing (not ordering) is the only operation performed on them -in the render path. diff --git a/defects/pandas/unit/PandasTest.java b/defects/pandas/unit/PandasTest.java index 9a3504656..21eb94517 100644 --- a/defects/pandas/unit/PandasTest.java +++ b/defects/pandas/unit/PandasTest.java @@ -11,6 +11,14 @@ import java.util.*; * slow(): list.contains(r) per row iteration — O(R×H) * fast(): HashSet.contains(r) per row iteration — O(R) * Assert: slowOps >= fastOps * 10 for R=2000 rows, H=500 hidden rows + * + * pandas-0002: _get_level_lengths — hidden_elements list scan O(R×L×H) + * Real code (pandas/io/formats/style_render.py:1840-1870): + * hidden_elements: Sequence[int] = [] # list + * for j, row in enumerate(lvl): + * if j not in hidden_elements: # O(H) list scan per cell + * Called on every Styler.render()/to_html()/to_latex() with MultiIndex. + * Fix: frozenset(hidden_elements) for O(1) membership. */ public class PandasTest { @@ -95,12 +103,79 @@ public class PandasTest { System.out.println("pandas-0001 PASS"); } + // ── pandas-0002: _get_level_lengths hidden_elements list scan ───────────── + + /** + * Simulates _get_level_lengths with list hidden_elements. + * For each (level, row) cell in a MultiIndex render, checks j not in hidden_elements. + * O(R×L×H) total — R rows, L levels, H hidden elements. + */ + static long slowGetLevelLengths(int rows, int levels, int hidden) { + List hiddenElements = new ArrayList<>(); + for (int i = 0; i < hidden; i++) hiddenElements.add(i * 2); // spread out evenly + + long ops = 0; + for (int lev = 0; lev < levels; lev++) { + for (int j = 0; j < rows; j++) { + // Main check: if j not in hidden_elements + boolean found = false; + for (int k = 0; k < hiddenElements.size(); k++) { + ops++; + if (hiddenElements.get(k) == j) { found = true; break; } + } + // Else-branch check (sparsify path also checks j not in hidden_elements) + if (!found) { + for (int k = 0; k < hiddenElements.size(); k++) { + ops++; + if (hiddenElements.get(k) == j) { found = true; break; } + } + } + } + } + return ops; + } + + /** + * Simulates fixed _get_level_lengths with frozenset(hidden_elements). + * O(R×L) total — O(1) set probe per cell. + */ + static long fastGetLevelLengths(int rows, int levels, int hidden) { + Set hiddenSet = new HashSet<>(); + for (int i = 0; i < hidden; i++) hiddenSet.add(i * 2); + + long ops = 0; + for (int lev = 0; lev < levels; lev++) { + for (int j = 0; j < rows; j++) { + ops++; // O(1) set probe (main check) + if (!hiddenSet.contains(j)) ops++; // O(1) set probe (else-branch) + } + } + return ops; + } + + static void testGetLevelLengths() { + int R = 5000, L = 3, H = 2500; + + long slowOps = slowGetLevelLengths(R, L, H); + long fastOps = fastGetLevelLengths(R, L, H); + + System.out.printf( + "pandas-0002 R=%-5d L=%d H=%-5d slowOps=%-12d fastOps=%-8d ratio=%.1fx%n", + R, L, H, slowOps, fastOps, (double) slowOps / fastOps + ); + + assert slowOps > fastOps * 100 : + "pandas-0002 FAIL: expected >100x ratio; slow=" + slowOps + " fast=" + fastOps; + System.out.println("pandas-0002 PASS"); + } + // ── main ────────────────────────────────────────────────────────────────── public static void main(String[] args) { - int pass = 0, total = 1; + int pass = 0, total = 2; try { testRenderRows(); pass++; } catch (AssertionError e) { System.err.println(e.getMessage()); } + try { testGetLevelLengths(); pass++; } catch (AssertionError e) { System.err.println(e.getMessage()); } System.out.printf("%n%d/%d PASS%n", pass, total); if (pass != total) System.exit(1); diff --git a/defects/sklearn/patch/CLEAN.md b/defects/sklearn/patch/CLEAN.md index 283e581ad..dc10c3086 100644 --- a/defects/sklearn/patch/CLEAN.md +++ b/defects/sklearn/patch/CLEAN.md @@ -1,15 +1,18 @@ -# sklearn — CWE-407 scan result: CLEAN +# sklearn — CWE-407 deeper scan: no NEW defects (sklearn-0001 pre-existing) Scanned: 2026-03-30 -## Scope +Pre-existing: sklearn-0001 (HistGradientBoosting `_check_categories` feature_names.index O(C×F)) -- `sklearn/utils/graph.py` — `single_source_shortest_path_length`: uses `seen = {}` (dict), O(1) membership. CLEAN. -- `sklearn/pipeline.py` — `transformer_names = set(...)`, O(1) membership. CLEAN. +## Deeper scan scope + +- `sklearn/utils/graph.py` — `single_source_shortest_path_length`: uses `seen = {}` (dict), O(1). CLEAN. +- `sklearn/pipeline.py` — `transformer_names = set(...)`, O(1). CLEAN. - `sklearn/feature_extraction/text.py` — `indices = set(vocabulary.values())`, O(1). CLEAN. - `sklearn/feature_extraction/_dict_vectorizer.py` — `vocab` is a dict, O(1). CLEAN. - `sklearn/externals/_arff.py` — `NominalConversor.values = set(values)`, O(1). CLEAN. - `sklearn/compose/_column_transformer.py` — `transformer_names` is a set. CLEAN. -- `sklearn/metrics/_classification.py` — `present_labels` is numpy array; `in` on numpy is expected O(N) scan for small label sets, not a hot-path quadratic. CLEAN. +- `sklearn/metrics/_classification.py` — `present_labels` is numpy array; O(N) but bounded label set, not hot-path quadratic. CLEAN. +- `sklearn/feature_selection/` — no list-dedup patterns in hot paths. CLEAN. -No CWE-407 defects found in scikit-learn 1.x. +No additional CWE-407 defects found beyond sklearn-0001.