fix(security): bound tokenizer work when explicit truncation_side is set (#47007)

Signed-off-by: jperezde <jperezde@redhat.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Juan Pérez de Algaba
2026-06-30 23:08:51 +08:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent db808b3961
commit c231d1f290
2 changed files with 94 additions and 9 deletions
+81
View File
@@ -64,12 +64,14 @@ class DummyTokenizer:
def __post_init__(self) -> None:
self._captured_encode_kwargs: dict = {}
self._captured_text_len: int = 0
def decode(self, tokens: list[int]):
return str(tokens)
def encode(self, text: str, **kwargs):
self._captured_encode_kwargs = kwargs
self._captured_text_len = len(text)
in_length = len(text)
truncation = kwargs.get("truncation")
@@ -366,6 +368,85 @@ class TestRenderPrompt:
assert results[0]["prompt_token_ids"] == tokens
assert results[0]["prompt"] == "[1, 2, 3, 4]"
def test_explicit_side_tokenizer_unbounded(self):
renderer = _build_renderer(MockModelConfig())
prompts = renderer.render_prompts(
_preprocess_prompt(renderer.model_config, "x" * 500)
)
results = renderer.tokenize_prompts(
prompts,
TokenizeParams(
max_total_tokens=100,
truncate_prompt_tokens=4,
truncation_side="left",
),
)
assert len(results) == 1
assert len(results[0]["prompt_token_ids"]) == 4
kwargs = renderer.tokenizer._captured_encode_kwargs
assert kwargs["truncation"] is False
def test_explicit_side_left_text(self):
renderer = _build_renderer(MockModelConfig())
prompts = renderer.render_prompts(
_preprocess_prompt(renderer.model_config, "x" * 50)
)
results = renderer.tokenize_prompts(
prompts,
TokenizeParams(
max_total_tokens=100,
truncate_prompt_tokens=5,
truncation_side="left",
),
)
assert len(results) == 1
assert len(results[0]["prompt_token_ids"]) == 5
assert results[0]["prompt_token_ids"] == list(range(45, 50))
def test_explicit_side_right_text(self):
renderer = _build_renderer(MockModelConfig())
prompts = renderer.render_prompts(
_preprocess_prompt(renderer.model_config, "x" * 50)
)
results = renderer.tokenize_prompts(
prompts,
TokenizeParams(
max_total_tokens=100,
truncate_prompt_tokens=5,
truncation_side="right",
),
)
assert len(results) == 1
assert len(results[0]["prompt_token_ids"]) == 5
assert results[0]["prompt_token_ids"] == list(range(5))
def test_explicit_side_text_pretokenization_guard(self):
renderer = _build_renderer(MockModelConfig(), max_chars_per_token=1)
prompts = renderer.render_prompts(
_preprocess_prompt(renderer.model_config, "x" * 500)
)
results = renderer.tokenize_prompts(
prompts,
TokenizeParams(
max_total_tokens=100,
truncate_prompt_tokens=4,
truncation_side="left",
),
)
assert len(results) == 1
assert len(results[0]["prompt_token_ids"]) == 4
assert renderer.tokenizer._captured_text_len <= 100
class TestRenderEmbedPrompt:
def _create_test_embed_bytes(self, tensor: torch.Tensor) -> bytes: