31 lines
1.4 KiB
Diff
31 lines
1.4 KiB
Diff
# UNDF: UNDF-2026-000000914
|
||
# transformers-0001: tokenization_python.py convert_ids_to_tokens O(T×S)
|
||
# CWE-407 — Algorithmic Complexity
|
||
#
|
||
# In PreTrainedTokenizer.convert_ids_to_tokens(), when skip_special_tokens=True,
|
||
# each iteration calls self.all_special_ids which is a @property that rebuilds
|
||
# a list via convert_tokens_to_ids(self.all_special_tokens) on every access.
|
||
# The `in` membership test on this list is O(S) per token, and the property
|
||
# reconstruction is also O(S) per token, making the total cost O(T × S) where
|
||
# T = sequence length and S = number of special tokens.
|
||
#
|
||
# Fix: cache all_special_ids as a set before the loop.
|
||
# Severity: MEDIUM (T can be 4096+ in modern LLM pipelines, S typically 10-20)
|
||
# Speedup: ~20x at T=4096, S=20 (eliminates 4096 list reconstructions + linear scans)
|
||
#
|
||
# File: src/transformers/tokenization_python.py
|
||
# Class: PreTrainedTokenizer
|
||
# Method: convert_ids_to_tokens
|
||
--- a/src/transformers/tokenization_python.py
|
||
+++ b/src/transformers/tokenization_python.py
|
||
@@ -1072,9 +1072,10 @@
|
||
|
||
tokens = []
|
||
+ special_ids = set(self.all_special_ids) if skip_special_tokens else None
|
||
for index in ids:
|
||
index = int(index)
|
||
- if skip_special_tokens and index in self.all_special_ids:
|
||
+ if special_ids is not None and index in special_ids:
|
||
continue
|
||
tokens.append(
|
||
self._added_tokens_decoder[index].content
|