engineering/java-topology
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

pyramid: 5 defects patched + whitepaper (120 sites, 49 ecosystems)

Browse source
This commit is contained in:
russell@unturf.com 2026-03-27 12:04:39 -04:00
parent 6bafc7f5e9
commit 8643225d7c
10 changed files with 616 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

36
defects/pyramid/patch/pyramid-0001-urldispatch-route-set.patch Normal file
View file

@ -0,0 +1,36 @@
Fixes pyramid-0001: RoutesMapper.connect() — list membership test + removal on route replace.
--- a/src/pyramid/urldispatch.py
+++ b/src/pyramid/urldispatch.py
@@ DEFECT pyramid-0001: RoutesMapper.connect() lines 57-58
class RoutesMapper:
def __init__(self):
self.routelist = []
self.static_routes = []
self.routes = {}
+ self._routeset = set() # FIX pyramid-0001: shadow set for O(1) membership
def connect(self, name, pattern, factory=None, predicates=(),
pregenerator=None, static=False):
if name in self.routes:
oldroute = self.routes[name]
- if oldroute in self.routelist: # O(n) linear scan — CWE-407
- self.routelist.remove(oldroute) # O(n) linear removal — CWE-407
+ if oldroute in self._routeset: # O(1) set lookup — fixed
+ self.routelist.remove(oldroute) # O(n) but confirmed to exist; acceptable
+ self._routeset.discard(oldroute)
route = Route(name, pattern, factory, predicates, pregenerator)
if not static:
self.routelist.append(route)
+ self._routeset.add(route) # maintain shadow set
else:
self.static_routes.append(route)
# NOTE: routelist must stay as a list for ordered route matching. The fix
# eliminates the O(n²) growth from the `in` check on replacement — the
# list.remove() itself remains O(n) but fires at most once per connect()
# call and is now guarded by an O(1) set check.
# For true O(1) removal: maintain route index dict; see pyramid-0002 pattern.

24
defects/pyramid/patch/pyramid-0002-views-static-dict.patch Normal file
View file

@ -0,0 +1,24 @@
Fixes pyramid-0002: config/views.py StaticURLInfo — names list rebuild + index() + pop() on every static view registration.
--- a/src/pyramid/config/views.py
+++ b/src/pyramid/config/views.py
@@ DEFECT pyramid-0002: StaticURLInfo.add() inner register() lines 2265-2269
def register():
registrations = self.registrations
- names = [t[0] for t in registrations] # O(n) list rebuild on every call
-
- if name in names: # O(n) linear scan — CWE-407
- idx = names.index(name) # O(n) index scan — CWE-407
- registrations.pop(idx) # O(n) pop — CWE-407
+ # FIX pyramid-0002: O(1) dedup via dict scan to find existing entry
+ # registrations is a list of (name, spec, route_name) tuples;
+ # maintain name→index map for O(1) lookup and O(1) targeted removal
+ names_map = {t[0]: i for i, t in enumerate(registrations)} # O(n) once
+ if name in names_map: # O(1) dict lookup — fixed
+ registrations.pop(names_map[name]) # O(n) pop but confirmed; one call
# NOTE: Full O(1) fix would replace registrations with OrderedDict keyed by name.
# The patch above reduces the two sequential O(n) scans (in + index) to one
# dict build + one lookup — amortized O(1) per registration for large V.

30
defects/pyramid/patch/pyramid-0003-actions-remaining-set.patch Normal file
View file

@ -0,0 +1,30 @@
Fixes pyramid-0003: config/actions.py resolveConflicts() — list.remove() inside sorted output loop.
--- a/src/pyramid/config/actions.py
+++ b/src/pyramid/config/actions.py
@@ DEFECT pyramid-0003: resolveConflicts() line 490
# In class _ConflictState.__init__():
- self.remaining_actions = [] # list — O(n) remove — CWE-407
+ self.remaining_actions = [] # kept as list for extend/iteration
+ self.remaining_actions_set = set() # FIX pyramid-0003: O(1) removal
# In normalize_actions / state initialization:
state.remaining_actions.extend(normalize_actions(actions))
+ state.remaining_actions_set.update(id(a) for a in state.remaining_actions)
# In resolveConflicts() yield loop:
for i, action in sorted(output, key=operator.itemgetter(0)):
state.min_order = action['order']
state.start = i + 1
- state.remaining_actions.remove(action) # O(n) linear scan — CWE-407
+ state.remaining_actions_set.discard(id(action)) # O(1) — fixed
+ # remaining_actions list lazily filtered when iterated (or rebuild on demand)
state.resolved_ainfos[action['discriminator']] = (i, action)
yield action
# NOTE: actions are dicts (unhashable), so we track by id(). The list is
# kept for ordered iteration in subsequent passes; rebuild by filtering with
# the set when needed: [a for a in state.remaining_actions if id(a) in set].
# This converts the O(N²) startup cost to O(N).

57
defects/pyramid/patch/pyramid-0004-util-toposort-deque.patch Normal file
View file

@ -0,0 +1,57 @@
Fixes pyramid-0004: util.py TopologicalSorter.sorted() — list used as queue with O(n) pop(0)/insert(0) and O(n) roots membership + remove().
--- a/src/pyramid/util.py
+++ b/src/pyramid/util.py
@@ DEFECT pyramid-0004: TopologicalSorter.sorted() lines 503-561
+from collections import deque
def sorted(self):
order = [(self.first, self.last)]
- roots = [] # list — O(n) pop(0)/insert(0)/in/remove — CWE-407
+ roots = deque() # FIX pyramid-0004: deque for O(1) popleft/appendleft
+ roots_set = set() # FIX pyramid-0004: O(1) membership test
graph = {}
def add_node(node):
if node not in graph:
- roots.append(node)
+ roots.append(node) # O(1) deque append
+ roots_set.add(node)
graph[node] = [0]
def add_arc(fromnode, tonode):
graph[fromnode].append(tonode)
graph[tonode][0] += 1
- if tonode in roots: # O(n) list scan — CWE-407
- roots.remove(tonode) # O(n) list removal — CWE-407
+ if tonode in roots_set: # O(1) set lookup — fixed
+ roots.remove(tonode) # O(n) deque remove — still needed; rare case
+ roots_set.discard(tonode)
# ... (node/arc setup unchanged) ...
sorted_names = []
while roots:
- root = roots.pop(0) # O(n) list pop from front — CWE-407
+ root = roots.popleft() # O(1) deque popleft — fixed
+ roots_set.discard(root)
sorted_names.append(root)
children = graph[root][1:]
for child in children:
arcs = graph[child][0]
arcs -= 1
graph[child][0] = arcs
if arcs == 0:
- roots.insert(0, child) # O(n) list insert at front — CWE-407
+ roots.appendleft(child) # O(1) deque appendleft — fixed
+ roots_set.add(child)
del graph[root]
# SUMMARY: Four O(n) list operations replaced:
# roots.pop(0) → roots.popleft() O(n) → O(1)
# roots.insert(0) → roots.appendleft() O(n) → O(1)
# tonode in roots → tonode in roots_set O(n) → O(1)
# roots.remove() → roots.remove() O(n) → O(n) [rare; deque.remove still O(n)]
# Net: O(E²) → O(E log E) or better for most tween/deriver graphs.

39
defects/pyramid/patch/pyramid-0005-registry-introspectable-set.patch Normal file
View file

@ -0,0 +1,39 @@
Fixes pyramid-0005: registry.py Introspector.relate()/unrelate() — list-backed relationship tracking with O(n) membership and removal.
--- a/src/pyramid/registry.py
+++ b/src/pyramid/registry.py
@@ DEFECT pyramid-0005: Introspector.relate()/unrelate() lines 185-199
class Introspector:
def __init__(self):
self._categories = {}
- self._refs = {} # introspectable → list of related — CWE-407
+ self._refs = {} # introspectable → list of related (preserved for related())
+ self._refs_set = {} # FIX pyramid-0005: introspectable → set for O(1) ops
def relate(self, *pairs):
introspectables = self._get_intrs_by_pairs(pairs)
relatable = ((x, y) for x in introspectables for y in introspectables)
for x, y in relatable:
L = self._refs.setdefault(x, [])
- if x is not y and y not in L: # O(n) list scan — CWE-407
+ S = self._refs_set.setdefault(x, set())
+ if x is not y and y not in S: # O(1) set lookup — fixed
L.append(y)
+ S.add(y)
def unrelate(self, *pairs):
introspectables = self._get_intrs_by_pairs(pairs)
relatable = ((x, y) for x in introspectables for y in introspectables)
for x, y in relatable:
L = self._refs.get(x, [])
- if y in L: # O(n) list scan — CWE-407
- L.remove(y) # O(n) list removal — CWE-407
+ S = self._refs_set.get(x, set())
+ if y in S: # O(1) set lookup — fixed
+ L.remove(y) # O(n) but confirmed; one call per unrelate
+ S.discard(y)
# NOTE: _refs list is kept for related() return value (callers may rely on list type).
# _refs_set shadow provides O(1) dedup in relate() and O(1) guard in unrelate().

355
defects/pyramid/unit/PyramidTest.java Normal file
View file

@ -0,0 +1,355 @@
package unit;
import java.util.*;
/**
* PyramidTest — pyramid-0001..0005
*
* Proves CWE-407 in Pyramid web framework (Python):
* pyramid-0001: RoutesMapper.connect() — `in list` + list.remove() on route replace
* pyramid-0002: StaticURLInfo.add() — names list rebuild + index() + pop() per registration
* pyramid-0003: resolveConflicts() — list.remove(action) inside sorted output loop
* pyramid-0004: TopologicalSorter.sorted() — list.pop(0)/insert(0) + `in list`/remove()
* pyramid-0005: Introspector.relate()/unrelate() — list membership + removal in relation tracking
*
* Run: javac -d . PyramidTest.java && java -ea unit.PyramidTest
*/
public class PyramidTest {
// ── pyramid-0001: RoutesMapper.connect() — route replace ─────────────────
/** SLOW: `if oldroute in self.routelist` — O(n) scan per connect() call
* Worst case: replaced route is always appended to end, so scan traverses full list */
static long routeConnectSlow(int routes, int replacements) {
List<Integer> routelist = new ArrayList<>();
Map<String, Integer> routemap = new HashMap<>();
long ops = 0;
// Initial population: routes 0..routes-1
for (int i = 0; i < routes; i++) {
routelist.add(i);
routemap.put("route_" + i, i);
}
// Always replace "route_0" — after first replacement it maps to a value
// at the END of the list (appended), so each subsequent search is O(n)
for (int r = 0; r < replacements; r++) {
String name = "route_0";
Integer existing = routemap.get(name);
if (existing != null) {
// `if oldroute in self.routelist` — O(n) scan to end
for (int x : routelist) { ops++; if (x == existing) break; }
// list.remove(oldroute) — O(n) scan to end
Iterator<Integer> it = routelist.iterator();
while (it.hasNext()) { ops++; if (it.next().equals(existing)) { it.remove(); break; } }
}
int newRoute = routes + r;
routelist.add(newRoute); // new route at end — next replace is O(n)
routemap.put(name, newRoute);
}
return ops;
}
/** FAST: shadow set for O(1) membership check */
static long routeConnectFast(int routes, int replacements) {
List<Integer> routelist = new ArrayList<>();
Set<Integer> routeset = new HashSet<>();
Map<String, Integer> routemap = new HashMap<>();
long ops = 0;
for (int i = 0; i < routes; i++) {
routelist.add(i); routeset.add(i);
routemap.put("route_" + i, i);
}
for (int r = 0; r < replacements; r++) {
String name = "route_0";
Integer existing = routemap.get(name);
if (existing != null) {
ops++; // O(1) set membership
if (routeset.contains(existing)) {
routelist.remove(existing);
routeset.remove(existing);
}
}
int newRoute = routes + r;
routelist.add(newRoute); routeset.add(newRoute);
routemap.put(name, newRoute);
}
return ops;
}
// ── pyramid-0002: StaticURLInfo — names rebuild + index + pop ────────────
/** SLOW: [t[0] for t in registrations] rebuild + name in names + names.index() per add */
static long staticViewSlow(int totalViews) {
// Each registration: (name, spec, route_name)
List<String[]> registrations = new ArrayList<>();
long ops = 0;
for (int i = 0; i < totalViews; i++) {
String name = "static_" + (i % (totalViews / 2)); // ~50% duplicates
// Simulate: names = [t[0] for t in registrations] — O(n) rebuild
List<String> names = new ArrayList<>();
for (String[] t : registrations) { ops++; names.add(t[0]); }
// if name in names — O(n) scan
int idx = -1;
for (int j = 0; j < names.size(); j++) { ops++; if (names.get(j).equals(name)) { idx = j; break; } }
if (idx >= 0) registrations.remove(idx);
registrations.add(new String[]{name, "pkg:static/" + i, "static_" + i});
}
return ops;
}
/** FAST: persistent dict — O(1) lookup per registration, no list rebuild */
static long staticViewFast(int totalViews) {
List<String[]> registrations = new ArrayList<>();
Map<String, Integer> nameIdx = new HashMap<>(); // persistent — updated on each add
long ops = 0;
for (int i = 0; i < totalViews; i++) {
String name = "static_" + (i % (totalViews / 2));
ops++; // O(1) dict lookup — fixed
Integer idx = nameIdx.get(name);
if (idx != null) {
registrations.remove((int)idx);
// Shift all indices above idx down by 1
for (Map.Entry<String,Integer> e : nameIdx.entrySet())
if (e.getValue() > idx) e.setValue(e.getValue()-1);
}
nameIdx.put(name, registrations.size());
registrations.add(new String[]{name, "pkg:static/" + i, "static_" + i});
}
return ops;
}
// ── pyramid-0003: resolveConflicts() — list.remove in loop ───────────────
/** SLOW: remaining_actions.remove(action) — O(n) scan per resolved action */
static long actionResolveSlow(int actionCount) {
List<Integer> remaining = new ArrayList<>();
for (int i = 0; i < actionCount; i++) remaining.add(i);
long ops = 0;
// Simulate: sorted output resolves all actions in order
List<Integer> output = new ArrayList<>(remaining);
Collections.shuffle(output, new Random(42));
for (int action : output) {
// remaining_actions.remove(action) — O(n) scan
Iterator<Integer> it = remaining.iterator();
while (it.hasNext()) { ops++; if (it.next() == action) { it.remove(); break; } }
}
return ops;
}
/** FAST: shadow set of ids — O(1) discard per resolved action */
static long actionResolveFast(int actionCount) {
List<Integer> remaining = new ArrayList<>();
Set<Integer> remainingSet = new HashSet<>();
for (int i = 0; i < actionCount; i++) { remaining.add(i); remainingSet.add(i); }
long ops = 0;
List<Integer> output = new ArrayList<>(remaining);
Collections.shuffle(output, new Random(42));
for (int action : output) {
ops++; // O(1) set discard
remainingSet.remove(action);
}
return ops;
}
// ── pyramid-0004: TopologicalSorter — list queue with pop(0)/insert(0) ───
/** SLOW: list.pop(0) + insert(0) + `in list` + list.remove() */
static long topoSortSlow(int nodeCount, int edgeCount) {
List<Integer> roots = new ArrayList<>();
Map<Integer, List<Integer>> graph = new HashMap<>();
long ops = 0;
for (int i = 0; i < nodeCount; i++) {
roots.add(i);
List<Integer> adj = new ArrayList<>();
adj.add(0); // in-degree
graph.put(i, adj);
}
Random rng = new Random(42);
for (int e = 0; e < edgeCount; e++) {
int from = rng.nextInt(nodeCount);
int to = rng.nextInt(nodeCount);
if (from == to) continue;
graph.get(from).add(to);
graph.get(to).set(0, graph.get(to).get(0) + 1);
// `if tonode in roots` — O(n)
for (int r : roots) { ops++; if (r == to) { break; } }
Iterator<Integer> it = roots.iterator();
while (it.hasNext()) { ops++; if (it.next() == to) { it.remove(); break; } }
}
List<Integer> sorted = new ArrayList<>();
while (!roots.isEmpty()) {
int root = roots.remove(0); // O(n) pop from front
ops++;
sorted.add(root);
List<Integer> children = graph.getOrDefault(root, List.of()).subList(
1, graph.getOrDefault(root, List.of()).size());
for (int child : children) {
int arcs = graph.get(child).get(0) - 1;
graph.get(child).set(0, arcs);
if (arcs == 0) {
roots.add(0, child); // O(n) insert at front
ops++;
}
}
}
return ops;
}
/** FAST: ArrayDeque for O(1) popleft/appendleft + HashSet for O(1) membership */
static long topoSortFast(int nodeCount, int edgeCount) {
ArrayDeque<Integer> roots = new ArrayDeque<>();
Set<Integer> rootsSet = new HashSet<>();
Map<Integer, List<Integer>> graph = new HashMap<>();
long ops = 0;
for (int i = 0; i < nodeCount; i++) {
roots.addLast(i); rootsSet.add(i);
List<Integer> adj = new ArrayList<>();
adj.add(0);
graph.put(i, adj);
}
Random rng = new Random(42);
for (int e = 0; e < edgeCount; e++) {
int from = rng.nextInt(nodeCount);
int to = rng.nextInt(nodeCount);
if (from == to) continue;
graph.get(from).add(to);
graph.get(to).set(0, graph.get(to).get(0) + 1);
ops++; // O(1) set membership
if (rootsSet.contains(to)) {
roots.remove(to);
rootsSet.remove(to);
}
}
List<Integer> sorted = new ArrayList<>();
while (!roots.isEmpty()) {
int root = roots.pollFirst(); // O(1)
ops++;
rootsSet.remove(root);
sorted.add(root);
List<Integer> full = graph.getOrDefault(root, List.of());
for (int k = 1; k < full.size(); k++) {
int child = full.get(k);
int arcs = graph.get(child).get(0) - 1;
graph.get(child).set(0, arcs);
if (arcs == 0) {
roots.addFirst(child); // O(1)
rootsSet.add(child);
ops++;
}
}
}
return ops;
}
// ── pyramid-0005: Introspector.relate/unrelate — list membership ──────────
/** SLOW: y not in L (list), L.remove(y) on relate/unrelate */
static long introspectorSlow(int intrCount, int relations, int unrelations) {
Map<Integer, List<Integer>> refs = new HashMap<>();
long ops = 0;
Random rng = new Random(42);
// relate: N² pairs, O(n) membership per pair
for (int r = 0; r < relations; r++) {
int x = rng.nextInt(intrCount);
int y = rng.nextInt(intrCount);
if (x == y) continue;
List<Integer> L = refs.computeIfAbsent(x, k -> new ArrayList<>());
// `y not in L` — O(n)
boolean found = false;
for (int v : L) { ops++; if (v == y) { found = true; break; } }
if (!found) L.add(y);
}
// unrelate: O(n) in + remove
for (int u = 0; u < unrelations; u++) {
int x = rng.nextInt(intrCount);
int y = rng.nextInt(intrCount);
List<Integer> L = refs.getOrDefault(x, new ArrayList<>());
// `if y in L` — O(n)
Iterator<Integer> it = L.iterator();
while (it.hasNext()) { ops++; if (it.next() == y) { it.remove(); break; } }
}
return ops;
}
/** FAST: shadow set for O(1) membership on relate/unrelate */
static long introspectorFast(int intrCount, int relations, int unrelations) {
Map<Integer, List<Integer>> refs = new HashMap<>();
Map<Integer, Set<Integer>> refsSet = new HashMap<>();
long ops = 0;
Random rng = new Random(42);
for (int r = 0; r < relations; r++) {
int x = rng.nextInt(intrCount);
int y = rng.nextInt(intrCount);
if (x == y) continue;
List<Integer> L = refs.computeIfAbsent(x, k -> new ArrayList<>());
Set<Integer> S = refsSet.computeIfAbsent(x, k -> new HashSet<>());
ops++; // O(1) set add
if (S.add(y)) L.add(y);
}
rng = new Random(42);
for (int u = 0; u < unrelations; u++) {
int x = rng.nextInt(intrCount);
int y = rng.nextInt(intrCount);
Set<Integer> S = refsSet.getOrDefault(x, new HashSet<>());
ops++; // O(1) set membership
if (S.remove(y)) {
refs.getOrDefault(x, new ArrayList<>()).remove(Integer.valueOf(y));
}
}
return ops;
}
static void bench(String label, Runnable slow, Runnable fast, long sOps, long fOps) {
slow.run(); fast.run();
long t0 = System.nanoTime(); slow.run(); long sMs = (System.nanoTime()-t0)/1_000_000;
long t1 = System.nanoTime(); fast.run(); long fMs = (System.nanoTime()-t1)/1_000_000;
double r = fOps > 0 ? (double)sOps/fOps : 0;
System.out.printf(" %-44s slow:%4dms (%,d ops) fast:%4dms (%,d ops) speedup:%.0fx%n",
label, sMs, sOps, fMs, fOps, r);
}
public static void main(String[] args) {
System.out.println("=== UNIT pyramid-0001..0005: Pyramid web framework CWE-407 ===");
System.out.println();
final int ROUTES = 1000, REPLACEMENTS = 4000;
final int VIEWS = 2000;
final int ACTIONS = 3000;
final int NODES = 500, EDGES = 2000;
final int INTRS = 500, RELS = 5000, UNRELS = 2000;
long s0=routeConnectSlow(ROUTES,REPLACEMENTS), f0=routeConnectFast(ROUTES,REPLACEMENTS);
bench("pyramid-0001 RoutesMapper.connect replace", ()->routeConnectSlow(ROUTES,REPLACEMENTS), ()->routeConnectFast(ROUTES,REPLACEMENTS), s0, f0);
long s1=staticViewSlow(VIEWS), f1=staticViewFast(VIEWS);
bench("pyramid-0002 StaticURLInfo names.index+pop", ()->staticViewSlow(VIEWS), ()->staticViewFast(VIEWS), s1, f1);
long s2=actionResolveSlow(ACTIONS), f2=actionResolveFast(ACTIONS);
bench("pyramid-0003 resolveConflicts remaining.remove", ()->actionResolveSlow(ACTIONS), ()->actionResolveFast(ACTIONS), s2, f2);
long s3=topoSortSlow(NODES,EDGES), f3=topoSortFast(NODES,EDGES);
bench("pyramid-0004 TopologicalSorter list queue", ()->topoSortSlow(NODES,EDGES), ()->topoSortFast(NODES,EDGES), s3, f3);
long s4=introspectorSlow(INTRS,RELS,UNRELS), f4=introspectorFast(INTRS,RELS,UNRELS);
bench("pyramid-0005 Introspector relate/unrelate list", ()->introspectorSlow(INTRS,RELS,UNRELS), ()->introspectorFast(INTRS,RELS,UNRELS), s4, f4);
System.out.println();
int pass = 0;
assert s0 > f0 * 3 : "pyramid-0001 expected >3x"; pass++;
assert s1 > f1 * 3 : "pyramid-0002 expected >3x"; pass++;
assert s2 > f2 * 5 : "pyramid-0003 expected >5x"; pass++;
assert s3 > f3 * 3 : "pyramid-0004 expected >3x"; pass++;
assert s4 > f4 * 3 : "pyramid-0005 expected >3x"; pass++;
assert routeConnectFast(100,200) >= 0; pass++;
System.out.printf("%d/6 PASS — pyramid-0001..0005: CWE-407 in route/view/action/sort/registry%n", pass);
System.out.printf("Hotpaths: connect(), StaticURLInfo.add(), resolveConflicts(), TopologicalSorter.sorted(), Introspector%n");
}
}