From 8fd9eb85d5a3d8f36cacb2b5d570a7a51c8200d8 Mon Sep 17 00:00:00 2001 From: Hotragn Pettugani <103170876+Hotragn@users.noreply.github.com> Date: Mon, 31 Aug 2026 02:13:37 -0400 Subject: [PATCH] [Bugfix][Frontend] Truncate prompt_token_offsets with the prompt (#54407) Signed-off-by: hotragn --- .../scale_out/render/test_render.py | 36 ++++++++++++++++ tests/renderers/test_token_offsets.py | 42 +++++++++++++++++++ vllm/renderers/params.py | 36 ++++++++++++---- 3 files changed, 106 insertions(+), 8 deletions(-) diff --git a/tests/entrypoints/scale_out/render/test_render.py b/tests/entrypoints/scale_out/render/test_render.py index 444e66f898..5b6bb9d1b4 100644 --- a/tests/entrypoints/scale_out/render/test_render.py +++ b/tests/entrypoints/scale_out/render/test_render.py @@ -347,6 +347,42 @@ async def test_chat_render_default_no_token_offsets(client): assert data["token_offsets"] is None +@pytest.mark.asyncio +async def test_completion_render_truncated_token_offsets(client): + """Truncation must shorten token_offsets together with token_ids. + + An explicit truncation_side turns off tokenizer-level truncation, so the + tokenizer returns offsets for the whole prompt and they are reduced + afterwards -- separately from token_ids. GenerateRequest documents that the + two lists have equal length. + """ + prompt = "The quick brown fox jumps over the lazy dog." + keep = 4 + response = await client.post( + "/v1/completions/render", + json={ + "model": MODEL_NAME, + "prompt": prompt, + "return_token_offsets": True, + "truncate_prompt_tokens": keep, + "truncation_side": "left", + }, + ) + + assert response.status_code == 200 + data = response.json() + token_ids = data[0]["token_ids"] + offsets = data[0]["token_offsets"] + + assert len(token_ids) == keep + assert len(offsets) == len(token_ids) + + # Equal length is not enough: truncating from the left keeps the *last* + # tokens, so the surviving offsets must cover the end of the prompt. + assert offsets[-1][1] == len(prompt) + assert offsets[0][0] > 0 + + @pytest.mark.asyncio async def test_completion_render_multiple_prompts_token_offsets(client): """Each prompt in a batch gets its own offsets aligned with its tokens.""" diff --git a/tests/renderers/test_token_offsets.py b/tests/renderers/test_token_offsets.py index f973e8610b..eca67b142a 100644 --- a/tests/renderers/test_token_offsets.py +++ b/tests/renderers/test_token_offsets.py @@ -197,3 +197,45 @@ class TestProcessTokensForwardsOffsets: engine_input = renderer._process_tokens(tokens_prompt) assert "prompt_token_offsets" not in engine_input + + +class TestTruncationKeepsOffsetsAligned: + """``prompt_token_offsets`` runs parallel to ``prompt_token_ids``, and both + ``TokensPrompt`` and the render API's ``GenerateRequest`` document that the + two have equal length. Only ``prompt_token_ids`` was being truncated. + """ + + @pytest.mark.parametrize("side", ["left", "right"]) + def test_explicit_truncation_side_truncates_offsets(self, fast_tokenizer, side): + renderer = _make_base_renderer_with(fast_tokenizer) + text = "The quick brown fox jumps over the lazy dog." + keep = 4 + + untruncated = renderer._tokenize_prompt( + {"prompt": text}, + TokenizeParams(max_total_tokens=1024, return_token_offsets=True), + ) + full_offsets = untruncated["prompt_token_offsets"] + assert len(full_offsets) > keep + + # An explicit truncation_side disables tokenizer-level truncation (see + # get_encode_kwargs), so the tokenizer returns the whole sequence and + # truncation happens in apply_post_tokenization. + params = TokenizeParams( + max_total_tokens=1024, + return_token_offsets=True, + truncate_prompt_tokens=keep, + truncation_side=side, + ) + result = params.apply_post_tokenization( + fast_tokenizer, renderer._tokenize_prompt({"prompt": text}, params) + ) + + offsets = result["prompt_token_offsets"] + assert len(result["prompt_token_ids"]) == keep + assert len(offsets) == keep + + # Equal length is not enough: the surviving offsets must be the ones + # belonging to the surviving tokens. + expected = full_offsets[-keep:] if side == "left" else full_offsets[:keep] + assert offsets == expected diff --git a/vllm/renderers/params.py b/vllm/renderers/params.py index cf6c3c9fbd..048f450ac1 100644 --- a/vllm/renderers/params.py +++ b/vllm/renderers/params.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from dataclasses import dataclass, field -from typing import TYPE_CHECKING, Any, Literal, TypeVar +from typing import TYPE_CHECKING, Any, Literal, TypeVar, cast from vllm.exceptions import VLLMValidationError from vllm.inputs import EmbedsPrompt, TextPrompt, TokensPrompt @@ -414,24 +414,39 @@ class TokenizeParams: return tokens + [tokenizer.pad_token_id] * (pad_length - len(tokens)) - def _token_truncation(self, tokenizer: TokenizerLike | None, tokens: _S) -> _S: - """Apply truncation to prompt tokens if necessary.""" + def _truncation_slice( + self, tokenizer: TokenizerLike | None, length: int + ) -> slice | None: + """The slice truncation applies to a sequence of `length` tokens. + + `None` means no truncation. Anything parallel to the prompt tokens + (see `prompt_token_offsets`) must be reduced with this same slice to + stay aligned with them. + """ max_length = self.truncate_prompt_tokens if max_length is not None and max_length < 0: max_length = self.max_input_tokens - if max_length is None or max_length >= len(tokens): - return tokens + if max_length is None or max_length >= length: + return None if max_length == 0: - return tokens[:0] + return slice(0, 0) side = self.truncation_side or ( tokenizer.truncation_side if tokenizer is not None else None ) if side == "left": - return tokens[-max_length:] + return slice(-max_length, None) - return tokens[:max_length] + return slice(0, max_length) + + def _token_truncation(self, tokenizer: TokenizerLike | None, tokens: _S) -> _S: + """Apply truncation to prompt tokens if necessary.""" + truncation = self._truncation_slice(tokenizer, len(tokens)) + if truncation is None: + return tokens + + return tokens[truncation] def _token_len_check(self, tokenizer: TokenizerLike | None, tokens: _S) -> _S: """Apply length checks to prompt tokens if necessary.""" @@ -496,5 +511,10 @@ class TokenizeParams: tokenizer, prompt["prompt_embeds"], # type: ignore[typeddict-item] ) + offsets = cast(TokensPrompt, prompt).get("prompt_token_offsets") + if offsets is not None: + truncation = self._truncation_slice(tokenizer, len(offsets)) + if truncation is not None: + prompt["prompt_token_offsets"] = offsets[truncation] # type: ignore[typeddict-unknown-key] return prompt