# UNDF: UNDF-2026-000000914
# transformers-0001: tokenization_python.py convert_ids_to_tokens O(T×S)
# CWE-407 — Algorithmic Complexity
#
# In PreTrainedTokenizer.convert_ids_to_tokens(), when skip_special_tokens=True,
# each iteration calls self.all_special_ids which is a @property that rebuilds
# a list via convert_tokens_to_ids(self.all_special_tokens) on every access.
# The `in` membership test on this list is O(S) per token, and the property
# reconstruction is also O(S) per token, making the total cost O(T × S) where
# T = sequence length and S = number of special tokens.
#
# Fix: cache all_special_ids as a set before the loop.
# Severity: MEDIUM (T can be 4096+ in modern LLM pipelines, S typically 10-20)
# Speedup: ~20x at T=4096, S=20 (eliminates 4096 list reconstructions + linear scans)
#
# File: src/transformers/tokenization_python.py
# Class: PreTrainedTokenizer
# Method: convert_ids_to_tokens
--- a/src/transformers/tokenization_python.py
+++ b/src/transformers/tokenization_python.py
@@ -1072,9 +1072,10 @@

         tokens = []
+        special_ids = set(self.all_special_ids) if skip_special_tokens else None
         for index in ids:
             index = int(index)
-            if skip_special_tokens and index in self.all_special_ids:
+            if special_ids is not None and index in special_ids:
                 continue
             tokens.append(
                 self._added_tokens_decoder[index].content