[Bugfix][Frontend] Truncate prompt_token_offsets with the prompt (#54407)
Signed-off-by: hotragn <hotragn.pettugani_2024@woxsen.edu.in>
This commit is contained in:
@@ -347,6 +347,42 @@ async def test_chat_render_default_no_token_offsets(client):
|
|||||||
assert data["token_offsets"] is None
|
assert data["token_offsets"] is None
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_completion_render_truncated_token_offsets(client):
|
||||||
|
"""Truncation must shorten token_offsets together with token_ids.
|
||||||
|
|
||||||
|
An explicit truncation_side turns off tokenizer-level truncation, so the
|
||||||
|
tokenizer returns offsets for the whole prompt and they are reduced
|
||||||
|
afterwards -- separately from token_ids. GenerateRequest documents that the
|
||||||
|
two lists have equal length.
|
||||||
|
"""
|
||||||
|
prompt = "The quick brown fox jumps over the lazy dog."
|
||||||
|
keep = 4
|
||||||
|
response = await client.post(
|
||||||
|
"/v1/completions/render",
|
||||||
|
json={
|
||||||
|
"model": MODEL_NAME,
|
||||||
|
"prompt": prompt,
|
||||||
|
"return_token_offsets": True,
|
||||||
|
"truncate_prompt_tokens": keep,
|
||||||
|
"truncation_side": "left",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert response.status_code == 200
|
||||||
|
data = response.json()
|
||||||
|
token_ids = data[0]["token_ids"]
|
||||||
|
offsets = data[0]["token_offsets"]
|
||||||
|
|
||||||
|
assert len(token_ids) == keep
|
||||||
|
assert len(offsets) == len(token_ids)
|
||||||
|
|
||||||
|
# Equal length is not enough: truncating from the left keeps the *last*
|
||||||
|
# tokens, so the surviving offsets must cover the end of the prompt.
|
||||||
|
assert offsets[-1][1] == len(prompt)
|
||||||
|
assert offsets[0][0] > 0
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_completion_render_multiple_prompts_token_offsets(client):
|
async def test_completion_render_multiple_prompts_token_offsets(client):
|
||||||
"""Each prompt in a batch gets its own offsets aligned with its tokens."""
|
"""Each prompt in a batch gets its own offsets aligned with its tokens."""
|
||||||
|
|||||||
@@ -197,3 +197,45 @@ class TestProcessTokensForwardsOffsets:
|
|||||||
engine_input = renderer._process_tokens(tokens_prompt)
|
engine_input = renderer._process_tokens(tokens_prompt)
|
||||||
|
|
||||||
assert "prompt_token_offsets" not in engine_input
|
assert "prompt_token_offsets" not in engine_input
|
||||||
|
|
||||||
|
|
||||||
|
class TestTruncationKeepsOffsetsAligned:
|
||||||
|
"""``prompt_token_offsets`` runs parallel to ``prompt_token_ids``, and both
|
||||||
|
``TokensPrompt`` and the render API's ``GenerateRequest`` document that the
|
||||||
|
two have equal length. Only ``prompt_token_ids`` was being truncated.
|
||||||
|
"""
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("side", ["left", "right"])
|
||||||
|
def test_explicit_truncation_side_truncates_offsets(self, fast_tokenizer, side):
|
||||||
|
renderer = _make_base_renderer_with(fast_tokenizer)
|
||||||
|
text = "The quick brown fox jumps over the lazy dog."
|
||||||
|
keep = 4
|
||||||
|
|
||||||
|
untruncated = renderer._tokenize_prompt(
|
||||||
|
{"prompt": text},
|
||||||
|
TokenizeParams(max_total_tokens=1024, return_token_offsets=True),
|
||||||
|
)
|
||||||
|
full_offsets = untruncated["prompt_token_offsets"]
|
||||||
|
assert len(full_offsets) > keep
|
||||||
|
|
||||||
|
# An explicit truncation_side disables tokenizer-level truncation (see
|
||||||
|
# get_encode_kwargs), so the tokenizer returns the whole sequence and
|
||||||
|
# truncation happens in apply_post_tokenization.
|
||||||
|
params = TokenizeParams(
|
||||||
|
max_total_tokens=1024,
|
||||||
|
return_token_offsets=True,
|
||||||
|
truncate_prompt_tokens=keep,
|
||||||
|
truncation_side=side,
|
||||||
|
)
|
||||||
|
result = params.apply_post_tokenization(
|
||||||
|
fast_tokenizer, renderer._tokenize_prompt({"prompt": text}, params)
|
||||||
|
)
|
||||||
|
|
||||||
|
offsets = result["prompt_token_offsets"]
|
||||||
|
assert len(result["prompt_token_ids"]) == keep
|
||||||
|
assert len(offsets) == keep
|
||||||
|
|
||||||
|
# Equal length is not enough: the surviving offsets must be the ones
|
||||||
|
# belonging to the surviving tokens.
|
||||||
|
expected = full_offsets[-keep:] if side == "left" else full_offsets[:keep]
|
||||||
|
assert offsets == expected
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
from dataclasses import dataclass, field
|
from dataclasses import dataclass, field
|
||||||
from typing import TYPE_CHECKING, Any, Literal, TypeVar
|
from typing import TYPE_CHECKING, Any, Literal, TypeVar, cast
|
||||||
|
|
||||||
from vllm.exceptions import VLLMValidationError
|
from vllm.exceptions import VLLMValidationError
|
||||||
from vllm.inputs import EmbedsPrompt, TextPrompt, TokensPrompt
|
from vllm.inputs import EmbedsPrompt, TextPrompt, TokensPrompt
|
||||||
@@ -414,24 +414,39 @@ class TokenizeParams:
|
|||||||
|
|
||||||
return tokens + [tokenizer.pad_token_id] * (pad_length - len(tokens))
|
return tokens + [tokenizer.pad_token_id] * (pad_length - len(tokens))
|
||||||
|
|
||||||
def _token_truncation(self, tokenizer: TokenizerLike | None, tokens: _S) -> _S:
|
def _truncation_slice(
|
||||||
"""Apply truncation to prompt tokens if necessary."""
|
self, tokenizer: TokenizerLike | None, length: int
|
||||||
|
) -> slice | None:
|
||||||
|
"""The slice truncation applies to a sequence of `length` tokens.
|
||||||
|
|
||||||
|
`None` means no truncation. Anything parallel to the prompt tokens
|
||||||
|
(see `prompt_token_offsets`) must be reduced with this same slice to
|
||||||
|
stay aligned with them.
|
||||||
|
"""
|
||||||
max_length = self.truncate_prompt_tokens
|
max_length = self.truncate_prompt_tokens
|
||||||
if max_length is not None and max_length < 0:
|
if max_length is not None and max_length < 0:
|
||||||
max_length = self.max_input_tokens
|
max_length = self.max_input_tokens
|
||||||
|
|
||||||
if max_length is None or max_length >= len(tokens):
|
if max_length is None or max_length >= length:
|
||||||
return tokens
|
return None
|
||||||
if max_length == 0:
|
if max_length == 0:
|
||||||
return tokens[:0]
|
return slice(0, 0)
|
||||||
|
|
||||||
side = self.truncation_side or (
|
side = self.truncation_side or (
|
||||||
tokenizer.truncation_side if tokenizer is not None else None
|
tokenizer.truncation_side if tokenizer is not None else None
|
||||||
)
|
)
|
||||||
if side == "left":
|
if side == "left":
|
||||||
return tokens[-max_length:]
|
return slice(-max_length, None)
|
||||||
|
|
||||||
return tokens[:max_length]
|
return slice(0, max_length)
|
||||||
|
|
||||||
|
def _token_truncation(self, tokenizer: TokenizerLike | None, tokens: _S) -> _S:
|
||||||
|
"""Apply truncation to prompt tokens if necessary."""
|
||||||
|
truncation = self._truncation_slice(tokenizer, len(tokens))
|
||||||
|
if truncation is None:
|
||||||
|
return tokens
|
||||||
|
|
||||||
|
return tokens[truncation]
|
||||||
|
|
||||||
def _token_len_check(self, tokenizer: TokenizerLike | None, tokens: _S) -> _S:
|
def _token_len_check(self, tokenizer: TokenizerLike | None, tokens: _S) -> _S:
|
||||||
"""Apply length checks to prompt tokens if necessary."""
|
"""Apply length checks to prompt tokens if necessary."""
|
||||||
@@ -496,5 +511,10 @@ class TokenizeParams:
|
|||||||
tokenizer,
|
tokenizer,
|
||||||
prompt["prompt_embeds"], # type: ignore[typeddict-item]
|
prompt["prompt_embeds"], # type: ignore[typeddict-item]
|
||||||
)
|
)
|
||||||
|
offsets = cast(TokensPrompt, prompt).get("prompt_token_offsets")
|
||||||
|
if offsets is not None:
|
||||||
|
truncation = self._truncation_slice(tokenizer, len(offsets))
|
||||||
|
if truncation is not None:
|
||||||
|
prompt["prompt_token_offsets"] = offsets[truncation] # type: ignore[typeddict-unknown-key]
|
||||||
|
|
||||||
return prompt
|
return prompt
|
||||||
|
|||||||
Reference in New Issue
Block a user