73 lines
3.9 KiB
Diff
73 lines
3.9 KiB
Diff
# UNDF: UNDF-2026-000001171
|
||
# transformers-0003: convert_tokens_to_string O(T×S) list scan for special tokens
|
||
# CWE-407 — Algorithmic Complexity
|
||
# MOAD-0001 — The Sedimentary Defect
|
||
#
|
||
# Multiple tokenizers (marian, m2m_100, speech_to_text, siglip, gpt_sw3, and others)
|
||
# call `if token in self.all_special_tokens` inside `convert_tokens_to_string()`.
|
||
# `all_special_tokens` is a @property returning list[str] — an O(S) linear scan
|
||
# per token. For a sequence of T decoded tokens the total cost is O(T × S).
|
||
#
|
||
# For M2M-100 translation models the special-token list includes 100+ language-code
|
||
# tokens (e.g. `__af__`, `__am__`, ...) making S ≈ 108. For a translation output of
|
||
# T=512 tokens that is 55,296 unnecessary string comparisons per decode call.
|
||
#
|
||
# The fix is the same in all affected tokenizers: cache `set(self.all_special_tokens)`
|
||
# before the loop and use the local set for the O(1) membership check.
|
||
#
|
||
# Severity: MEDIUM — decode is called once per batch result, S is bounded ~100-110,
|
||
# but this runs on every translate/transcribe output across all API requests.
|
||
# Speedup: ~100x at T=512, S=108 (eliminates 55K comparisons per call)
|
||
#
|
||
# Affected files (same pattern, same fix):
|
||
# src/transformers/models/marian/tokenization_marian.py line ~283
|
||
# src/transformers/models/m2m_100/tokenization_m2m_100.py line ~216
|
||
# src/transformers/models/speech_to_text/tokenization_speech_to_text.py line ~193
|
||
# src/transformers/models/siglip/tokenization_siglip.py line ~320
|
||
# src/transformers/models/gpt_sw3/tokenization_gpt_sw3.py line ~174
|
||
|
||
--- a/src/transformers/models/marian/tokenization_marian.py
|
||
+++ b/src/transformers/models/marian/tokenization_marian.py
|
||
@@ -279,10 +279,11 @@ class MarianTokenizer(PreTrainedTokenizer):
|
||
sp_model = self.spm_source if self._decode_use_source_tokenizer else self.spm_target
|
||
current_sub_tokens = []
|
||
out_string = ""
|
||
+ special_tokens_set = set(self.all_special_tokens)
|
||
for token in tokens:
|
||
# make sure that special tokens are not decoded using sentencepiece model
|
||
- if token in self.all_special_tokens:
|
||
+ if token in special_tokens_set:
|
||
out_string += sp_model.decode_pieces(current_sub_tokens) + token + " "
|
||
current_sub_tokens = []
|
||
else:
|
||
|
||
--- a/src/transformers/models/m2m_100/tokenization_m2m_100.py
|
||
+++ b/src/transformers/models/m2m_100/tokenization_m2m_100.py
|
||
@@ -212,8 +212,9 @@ class M2M100Tokenizer(PreTrainedTokenizer):
|
||
def convert_tokens_to_string(self, tokens):
|
||
"""Converts a sequence of tokens (strings) in a single string."""
|
||
current_sub_tokens = []
|
||
out_string = ""
|
||
+ special_tokens_set = set(self.all_special_tokens)
|
||
for token in tokens:
|
||
- if token in self.all_special_tokens:
|
||
+ if token in special_tokens_set:
|
||
out_string += self.sp_model.decode(current_sub_tokens) + token + " "
|
||
current_sub_tokens = []
|
||
else:
|
||
|
||
--- a/src/transformers/models/speech_to_text/tokenization_speech_to_text.py
|
||
+++ b/src/transformers/models/speech_to_text/tokenization_speech_to_text.py
|
||
@@ -189,8 +189,9 @@ class Speech2TextTokenizer(PreTrainedTokenizer):
|
||
def convert_tokens_to_string(self, tokens: list[str]) -> str:
|
||
"""Converts a sequence of tokens (strings for sub-words) in a single string."""
|
||
current_sub_tokens = []
|
||
out_string = ""
|
||
+ special_tokens_set = set(self.all_special_tokens)
|
||
for token in tokens:
|
||
# make sure that special tokens are not decoded using sentencepiece model
|
||
- if token in self.all_special_tokens:
|
||
+ if token in special_tokens_set:
|
||
decoded = self.sp_model.decode(current_sub_tokens)
|
||
out_string += (decoded.upper() if self.do_upper_case else decoded) + token + " "
|
||
current_sub_tokens = []
|