fix(security): bound tokenizer work when explicit truncation_side is set (#47007)
Signed-off-by: jperezde <jperezde@redhat.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
db808b3961
commit
c231d1f290
@@ -64,12 +64,14 @@ class DummyTokenizer:
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
self._captured_encode_kwargs: dict = {}
|
||||
self._captured_text_len: int = 0
|
||||
|
||||
def decode(self, tokens: list[int]):
|
||||
return str(tokens)
|
||||
|
||||
def encode(self, text: str, **kwargs):
|
||||
self._captured_encode_kwargs = kwargs
|
||||
self._captured_text_len = len(text)
|
||||
|
||||
in_length = len(text)
|
||||
truncation = kwargs.get("truncation")
|
||||
@@ -366,6 +368,85 @@ class TestRenderPrompt:
|
||||
assert results[0]["prompt_token_ids"] == tokens
|
||||
assert results[0]["prompt"] == "[1, 2, 3, 4]"
|
||||
|
||||
def test_explicit_side_tokenizer_unbounded(self):
|
||||
renderer = _build_renderer(MockModelConfig())
|
||||
|
||||
prompts = renderer.render_prompts(
|
||||
_preprocess_prompt(renderer.model_config, "x" * 500)
|
||||
)
|
||||
results = renderer.tokenize_prompts(
|
||||
prompts,
|
||||
TokenizeParams(
|
||||
max_total_tokens=100,
|
||||
truncate_prompt_tokens=4,
|
||||
truncation_side="left",
|
||||
),
|
||||
)
|
||||
|
||||
assert len(results) == 1
|
||||
assert len(results[0]["prompt_token_ids"]) == 4
|
||||
|
||||
kwargs = renderer.tokenizer._captured_encode_kwargs
|
||||
assert kwargs["truncation"] is False
|
||||
|
||||
def test_explicit_side_left_text(self):
|
||||
renderer = _build_renderer(MockModelConfig())
|
||||
|
||||
prompts = renderer.render_prompts(
|
||||
_preprocess_prompt(renderer.model_config, "x" * 50)
|
||||
)
|
||||
results = renderer.tokenize_prompts(
|
||||
prompts,
|
||||
TokenizeParams(
|
||||
max_total_tokens=100,
|
||||
truncate_prompt_tokens=5,
|
||||
truncation_side="left",
|
||||
),
|
||||
)
|
||||
|
||||
assert len(results) == 1
|
||||
assert len(results[0]["prompt_token_ids"]) == 5
|
||||
assert results[0]["prompt_token_ids"] == list(range(45, 50))
|
||||
|
||||
def test_explicit_side_right_text(self):
|
||||
renderer = _build_renderer(MockModelConfig())
|
||||
|
||||
prompts = renderer.render_prompts(
|
||||
_preprocess_prompt(renderer.model_config, "x" * 50)
|
||||
)
|
||||
results = renderer.tokenize_prompts(
|
||||
prompts,
|
||||
TokenizeParams(
|
||||
max_total_tokens=100,
|
||||
truncate_prompt_tokens=5,
|
||||
truncation_side="right",
|
||||
),
|
||||
)
|
||||
|
||||
assert len(results) == 1
|
||||
assert len(results[0]["prompt_token_ids"]) == 5
|
||||
assert results[0]["prompt_token_ids"] == list(range(5))
|
||||
|
||||
def test_explicit_side_text_pretokenization_guard(self):
|
||||
renderer = _build_renderer(MockModelConfig(), max_chars_per_token=1)
|
||||
|
||||
prompts = renderer.render_prompts(
|
||||
_preprocess_prompt(renderer.model_config, "x" * 500)
|
||||
)
|
||||
results = renderer.tokenize_prompts(
|
||||
prompts,
|
||||
TokenizeParams(
|
||||
max_total_tokens=100,
|
||||
truncate_prompt_tokens=4,
|
||||
truncation_side="left",
|
||||
),
|
||||
)
|
||||
|
||||
assert len(results) == 1
|
||||
assert len(results[0]["prompt_token_ids"]) == 4
|
||||
|
||||
assert renderer.tokenizer._captured_text_len <= 100
|
||||
|
||||
|
||||
class TestRenderEmbedPrompt:
|
||||
def _create_test_embed_bytes(self, tensor: torch.Tensor) -> bytes:
|
||||
|
||||
Reference in New Issue
Block a user