engineering/java-topology
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

pandas: fix second 0002 collision — rename list-scan stub to pandas-0003

Browse source
This commit is contained in:
russell@unturf.com 2026-03-29 22:08:05 -04:00
parent 0f7d8485f0
commit 2785a0ea1e
4 changed files with 228 additions and 115 deletions
Download patch file Download diff file Expand all files Collapse all files

144
defects/numpy/unit/NumpyTest.java
View file

@ -10,6 +10,18 @@ import java.util.*;
* slow(): counts ops using list membership (words.contains(w)) — O(W) per insert
* fast(): counts ops using a parallel HashSet for O(1) membership
* Assert: slowOps >= fastOps * 10 for W=500 deps per variable
*
* numpy-0002: recfunctions.stack_arrays — seen=[] list dedup O(A×F²) field-name tracking
* Real code (numpy/lib/recfunctions.py:1396-1405):
* seen = []
* if name not in seen: seen.append(name) # O(F) per insert
* Fix: seen = set()
*
* numpy-0003: recfunctions.join_by — names list rebuilt inside loop + .index() O(F²)
* Real code (numpy/lib/recfunctions.py:1610-1612):
* names = [name for name, dtype in ndtype] # O(F) rebuild per iteration
* nameidx = names.index(fname) # O(F) linear search
* Fix: name_to_idx dict for O(1) lookup
*/
public class NumpyTest {
@ -97,12 +109,142 @@ public class NumpyTest {
System.out.println("numpy-0001 PASS");
}
// ── numpy-0002: stack_arrays seen=[] list dedup ───────────────────────────
/** Simulates stack_arrays seen=[] field dedup — returns total list comparisons */
static long slowStackArrays(int numArrays, int fieldsPerArray) {
List<String> seen = new ArrayList<>();
long ops = 0;
for (int a = 0; a < numArrays; a++) {
for (int f = 0; f < fieldsPerArray; f++) {
String name = "field_" + f; // shared field names across arrays
boolean found = false;
for (int k = 0; k < seen.size(); k++) {
ops++;
if (seen.get(k).equals(name)) { found = true; break; }
}
if (!found) seen.add(name);
}
}
return ops;
}
/** Simulates fixed stack_arrays with seen=set() — returns total ops */
static long fastStackArrays(int numArrays, int fieldsPerArray) {
Set<String> seen = new HashSet<>();
long ops = 0;
for (int a = 0; a < numArrays; a++) {
for (int f = 0; f < fieldsPerArray; f++) {
String name = "field_" + f;
ops++; // O(1) set probe
seen.add(name);
}
}
return ops;
}
static void testStackArrays() {
int A = 100, F = 200; // 100 arrays, 200 fields each
long slowOps = slowStackArrays(A, F);
long fastOps = fastStackArrays(A, F);
System.out.printf(
"numpy-0002 A=%-4d F=%-4d slowOps=%-10d fastOps=%-8d ratio=%.1fx%n",
A, F, slowOps, fastOps, (double) slowOps / fastOps
);
assert slowOps > fastOps * 30 :
"numpy-0002 FAIL: expected >30x ratio; slow=" + slowOps + " fast=" + fastOps;
System.out.println("numpy-0002 PASS");
}
// ── numpy-0003: join_by names-list rebuild + .index() ─────────────────────
/** Simulates join_by ndtype names-list rebuild + .index() — returns total comparisons */
static long slowJoinBy(int f1, int f2) {
List<String> ndtype = new ArrayList<>();
long ops = 0;
for (int i = 0; i < f1; i++) ndtype.add("r1_field_" + i);
for (int i = 0; i < f2; i++) {
String fname = "r2_field_" + i;
// Rebuild list (simulates list comprehension O(F) per iteration)
List<String> names = new ArrayList<>(ndtype);
ops += names.size();
// .index() linear search
int idx = -1;
for (int k = 0; k < names.size(); k++) {
ops++;
if (names.get(k).equals(fname)) { idx = k; break; }
}
if (idx == -1) ndtype.add(fname);
}
// Site 2: output dtype names tuple membership check
List<String> outputNames = new ArrayList<>(ndtype);
for (int i = 0; i < f1; i++) {
String f = "r1_field_" + i;
for (int k = 0; k < outputNames.size(); k++) {
ops++;
if (outputNames.get(k).equals(f)) break;
}
}
for (int i = 0; i < f2; i++) {
String f = "r2_field_" + i;
for (int k = 0; k < outputNames.size(); k++) {
ops++;
if (outputNames.get(k).equals(f)) break;
}
}
return ops;
}
/** Simulates fixed join_by with name_to_idx dict — returns total ops */
static long fastJoinBy(int f1, int f2) {
Map<String, Integer> nameToIdx = new HashMap<>();
List<String> ndtype = new ArrayList<>();
long ops = 0;
for (int i = 0; i < f1; i++) {
nameToIdx.put("r1_field_" + i, i);
ndtype.add("r1_field_" + i);
}
for (int i = 0; i < f2; i++) {
String fname = "r2_field_" + i;
ops++; // O(1) dict probe
if (!nameToIdx.containsKey(fname)) {
nameToIdx.put(fname, ndtype.size());
ndtype.add(fname);
}
}
// Site 2: set membership
Set<String> outputNamesSet = new HashSet<>(ndtype);
for (int i = 0; i < f1 + f2; i++) ops++;
return ops;
}
static void testJoinBy() {
int F1 = 200, F2 = 200;
long slowOps = slowJoinBy(F1, F2);
long fastOps = fastJoinBy(F1, F2);
System.out.printf(
"numpy-0003 F1=%-4d F2=%-4d slowOps=%-10d fastOps=%-8d ratio=%.1fx%n",
F1, F2, slowOps, fastOps, (double) slowOps / fastOps
);
assert slowOps > fastOps * 50 :
"numpy-0003 FAIL: expected >50x ratio; slow=" + slowOps + " fast=" + fastOps;
System.out.println("numpy-0003 PASS");
}
// ── main ──────────────────────────────────────────────────────────────────
public static void main(String[] args) {
int pass = 0, total = 1;
int pass = 0, total = 3;
try { testDependDict(); pass++; } catch (AssertionError e) { System.err.println(e.getMessage()); }
try { testStackArrays(); pass++; } catch (AssertionError e) { System.err.println(e.getMessage()); }
try { testJoinBy(); pass++; } catch (AssertionError e) { System.err.println(e.getMessage()); }
System.out.printf("%n%d/%d PASS%n", pass, total);
if (pass != total) System.exit(1);

107
defects/pandas/patch/pandas-0002-style-render-hidden-elements-list-scan.md
View file

@ -1,107 +0,0 @@
# UNDF: UNDF-2026-000000691
# UNDF: (pending)
# pandas-0001: _get_level_lengths — hidden_elements list scan O(R×L×H)
## CWE-407 — Algorithmic Complexity: O(R×L×H) list-contains in DataFrame styler render
| Field | Value |
|-------|-------|
| ID | pandas-0001 |
| Severity | MEDIUM |
| Ecosystem | pandas |
| Package | `pandas` |
| File | `pandas/io/formats/style_render.py` |
| Lines | 1840–1870 |
| Complexity | O(R×L×H) — rows × index levels × hidden elements |
| Hot path | `_get_level_lengths()` called on every `Styler.render()` / `to_html()` |
## Background
`_get_level_lengths(index, sparsify, max_index, hidden_elements)` computes span
lengths for rendering a (Multi)Index in HTML/LaTeX output. It is called twice
per render: once for row index, once for column index. `hidden_elements` is the
list of integer positions that should be omitted from the rendered output — set
by `Styler.hide(rows)` or `Styler.hide(columns)`.
## Defect
```python
# pandas/io/formats/style_render.py lines 1840–1870
if hidden_elements is None:
hidden_elements = [] # list — default type
# ...
for i, value in enumerate(levels):
if i not in hidden_elements: # DEFECT: O(H) list scan
lengths[(0, i)] = 1
# ...
for i, lvl in enumerate(levels):
for j, row in enumerate(lvl):
if not sparsify:
if j not in hidden_elements: # O(H) list scan
lengths[(i, j)] = 1
elif (row is not lib.no_default) and (j not in hidden_elements): # O(H)
...
elif j not in hidden_elements: # O(H)
...
```
`hidden_elements` is declared as `Sequence[int]` and stored as a plain `list`:
```python
# pandas/io/formats/style_render.py line 131
self.hidden_rows: Sequence[int] = []
self.hidden_columns: Sequence[int] = []
```
Each `j not in hidden_elements` performs a linear scan. The outer loops run
R×L times (rows × MultiIndex levels), so total work is O(R×L×H).
### When does this matter?
Users calling `styler.hide(subset=large_slice)` on wide DataFrames or tall
DataFrames with MultiIndex before rendering: e.g. hiding 80% of 10,000 rows
in a 3-level MultiIndex generates ~24,000 list scans of length ~8,000 = 192M
comparisons per render.
## Complexity table
| Rows (R) | Hidden (H) | Levels (L) | list ops | set ops | Speedup |
|---------|----------|-----------|---------|---------|---------|
| 1,000 | 500 | 3 | 1,500,000 | 3,000 | 500× |
| 5,000 | 2,500 | 3 | 37,500,000 | 15,000 | 2,500× |
| 10,000 | 8,000 | 3 | 240,000,000 | 30,000 | 8,000× |
## Fix
Convert `hidden_elements` to a `set` at the point of use in
`_get_level_lengths`, or store it as a `frozenset` in `StylerRenderer`:
```python
def _get_level_lengths(
index: Index,
sparsify: bool,
max_index: int,
hidden_elements: Sequence[int] | None = None,
):
if hidden_elements is None:
hidden_elements_set: frozenset[int] = frozenset()
else:
hidden_elements_set = frozenset(hidden_elements) # FIX: O(1) lookup
# ...
for i, value in enumerate(levels):
if i not in hidden_elements_set: # O(1)
lengths[(0, i)] = 1
# ...
for i, lvl in enumerate(levels):
for j, row in enumerate(lvl):
if j not in hidden_elements_set: # O(1)
...
```
Alternatively, store `self.hidden_rows` and `self.hidden_columns` as
`set[int]` rather than `list[int]` throughout `StylerRenderer`, since
membership testing (not ordering) is the only operation performed on them
in the render path.

77
defects/pandas/unit/PandasTest.java
View file

@ -11,6 +11,14 @@ import java.util.*;
* slow(): list.contains(r) per row iteration — O(R×H)
* fast(): HashSet.contains(r) per row iteration — O(R)
* Assert: slowOps >= fastOps * 10 for R=2000 rows, H=500 hidden rows
*
* pandas-0002: _get_level_lengths — hidden_elements list scan O(R×L×H)
* Real code (pandas/io/formats/style_render.py:1840-1870):
* hidden_elements: Sequence[int] = [] # list
* for j, row in enumerate(lvl):
* if j not in hidden_elements: # O(H) list scan per cell
* Called on every Styler.render()/to_html()/to_latex() with MultiIndex.
* Fix: frozenset(hidden_elements) for O(1) membership.
*/
public class PandasTest {
@ -95,12 +103,79 @@ public class PandasTest {
System.out.println("pandas-0001 PASS");
}
// ── pandas-0002: _get_level_lengths hidden_elements list scan ─────────────
/**
* Simulates _get_level_lengths with list hidden_elements.
* For each (level, row) cell in a MultiIndex render, checks j not in hidden_elements.
* O(R×L×H) total — R rows, L levels, H hidden elements.
*/
static long slowGetLevelLengths(int rows, int levels, int hidden) {
List<Integer> hiddenElements = new ArrayList<>();
for (int i = 0; i < hidden; i++) hiddenElements.add(i * 2); // spread out evenly
long ops = 0;
for (int lev = 0; lev < levels; lev++) {
for (int j = 0; j < rows; j++) {
// Main check: if j not in hidden_elements
boolean found = false;
for (int k = 0; k < hiddenElements.size(); k++) {
ops++;
if (hiddenElements.get(k) == j) { found = true; break; }
}
// Else-branch check (sparsify path also checks j not in hidden_elements)
if (!found) {
for (int k = 0; k < hiddenElements.size(); k++) {
ops++;
if (hiddenElements.get(k) == j) { found = true; break; }
}
}
}
}
return ops;
}
/**
* Simulates fixed _get_level_lengths with frozenset(hidden_elements).
* O(R×L) total — O(1) set probe per cell.
*/
static long fastGetLevelLengths(int rows, int levels, int hidden) {
Set<Integer> hiddenSet = new HashSet<>();
for (int i = 0; i < hidden; i++) hiddenSet.add(i * 2);
long ops = 0;
for (int lev = 0; lev < levels; lev++) {
for (int j = 0; j < rows; j++) {
ops++; // O(1) set probe (main check)
if (!hiddenSet.contains(j)) ops++; // O(1) set probe (else-branch)
}
}
return ops;
}
static void testGetLevelLengths() {
int R = 5000, L = 3, H = 2500;
long slowOps = slowGetLevelLengths(R, L, H);
long fastOps = fastGetLevelLengths(R, L, H);
System.out.printf(
"pandas-0002 R=%-5d L=%d H=%-5d slowOps=%-12d fastOps=%-8d ratio=%.1fx%n",
R, L, H, slowOps, fastOps, (double) slowOps / fastOps
);
assert slowOps > fastOps * 100 :
"pandas-0002 FAIL: expected >100x ratio; slow=" + slowOps + " fast=" + fastOps;
System.out.println("pandas-0002 PASS");
}
// ── main ──────────────────────────────────────────────────────────────────
public static void main(String[] args) {
int pass = 0, total = 1;
int pass = 0, total = 2;
try { testRenderRows(); pass++; } catch (AssertionError e) { System.err.println(e.getMessage()); }
try { testGetLevelLengths(); pass++; } catch (AssertionError e) { System.err.println(e.getMessage()); }
System.out.printf("%n%d/%d PASS%n", pass, total);
if (pass != total) System.exit(1);

15
defects/sklearn/patch/CLEAN.md
View file

@ -1,15 +1,18 @@
# sklearn — CWE-407 scan result: CLEAN
# sklearn — CWE-407 deeper scan: no NEW defects (sklearn-0001 pre-existing)
Scanned: 2026-03-30
## Scope
Pre-existing: sklearn-0001 (HistGradientBoosting `_check_categories` feature_names.index O(C×F))
- `sklearn/utils/graph.py` — `single_source_shortest_path_length`: uses `seen = {}` (dict), O(1) membership. CLEAN.
- `sklearn/pipeline.py` — `transformer_names = set(...)`, O(1) membership. CLEAN.
## Deeper scan scope
- `sklearn/utils/graph.py` — `single_source_shortest_path_length`: uses `seen = {}` (dict), O(1). CLEAN.
- `sklearn/pipeline.py` — `transformer_names = set(...)`, O(1). CLEAN.
- `sklearn/feature_extraction/text.py` — `indices = set(vocabulary.values())`, O(1). CLEAN.
- `sklearn/feature_extraction/_dict_vectorizer.py` — `vocab` is a dict, O(1). CLEAN.
- `sklearn/externals/_arff.py` — `NominalConversor.values = set(values)`, O(1). CLEAN.
- `sklearn/compose/_column_transformer.py` — `transformer_names` is a set. CLEAN.
- `sklearn/metrics/_classification.py` — `present_labels` is numpy array; `in` on numpy is expected O(N) scan for small label sets, not a hot-path quadratic. CLEAN.
- `sklearn/metrics/_classification.py` — `present_labels` is numpy array; O(N) but bounded label set, not hot-path quadratic. CLEAN.
- `sklearn/feature_selection/` — no list-dedup patterns in hot paths. CLEAN.
No CWE-407 defects found in scikit-learn 1.x.
No additional CWE-407 defects found beyond sklearn-0001.