From ff0c3cb03c5d52fe21ebcf98f3e45325b1689116 Mon Sep 17 00:00:00 2001 From: Hotragn Pettugani <103170876+Hotragn@users.noreply.github.com> Date: Tue, 1 Sep 2026 01:53:22 -0400 Subject: [PATCH] [Bugfix][Frontend] Truncate the assistant tokens mask with the prompt (#54539) Signed-off-by: hotragn --- .../scale_out/render/test_render.py | 46 +++++++++++++++++++ vllm/renderers/params.py | 32 ++++++++----- 2 files changed, 67 insertions(+), 11 deletions(-) diff --git a/tests/entrypoints/scale_out/render/test_render.py b/tests/entrypoints/scale_out/render/test_render.py index 5b6bb9d1b4..40e338c7f0 100644 --- a/tests/entrypoints/scale_out/render/test_render.py +++ b/tests/entrypoints/scale_out/render/test_render.py @@ -531,6 +531,52 @@ async def test_chat_completion_render_assistant_tokens_mask_with_generation_tags assert "Bye" in detok_rest.json()["prompt"] +@pytest.mark.asyncio +async def test_chat_render_assistant_tokens_mask_follows_truncation(client): + """The assistant mask must be truncated with the prompt it describes. + + `assistant_tokens_mask` is positional: entry i labels token i. Truncating + `token_ids` from the left without truncating the mask leaves the two + describing different positions, and the mask ends up marking whichever + tokens happen to sit at the old offsets. + """ + messages = [ + # Deliberately lopsided: a long leading user turn and a short trailing + # one, so keeping the head of the mask is distinguishable from keeping + # its tail. + {"role": "user", "content": "Hello hello hello hello hello hello"}, + {"role": "assistant", "content": "Hi!"}, + {"role": "user", "content": "Bye"}, + ] + body = { + "model": MODEL_NAME, + "messages": messages, + "chat_template": _TEMPLATE_WITH_GENERATION, + "return_assistant_tokens_mask": True, + } + + full = await client.post("/v1/chat/completions/render", json=body) + assert full.status_code == 200 + full_token_ids = full.json()["token_ids"] + full_mask = full.json()["assistant_tokens_mask"] + assert sum(full_mask) > 0 + + keep = len(full_token_ids) - 4 + # Precondition: with this prompt the head and tail slices of the mask + # really do differ, so the assertion below can tell them apart. + assert full_mask[-keep:] != full_mask[:keep] + + truncated = await client.post( + "/v1/chat/completions/render", + json={**body, "truncate_prompt_tokens": keep, "truncation_side": "left"}, + ) + assert truncated.status_code == 200 + data = truncated.json() + + assert data["token_ids"] == full_token_ids[-keep:] + assert data["assistant_tokens_mask"] == full_mask[-keep:] + + @pytest.mark.asyncio async def test_messages_render_basic(client): """Test basic Anthropic Messages render endpoint.""" diff --git a/vllm/renderers/params.py b/vllm/renderers/params.py index 3ecd8f4828..856934c4eb 100644 --- a/vllm/renderers/params.py +++ b/vllm/renderers/params.py @@ -24,6 +24,18 @@ logger = init_logger(__name__) _S = TypeVar("_S", list[int], "torch.Tensor") +# Prompt keys whose entry i describes token i of the prompt, so they must be +# reduced with the same slice as the prompt tokens themselves. Anything added +# here is truncated by `TokenizeParams.apply_post_tokenization`. +# `_assistant_tokens_mask` is renderer-internal: `HfRenderer.render_messages` +# stashes it on the prompt so `_process_tokens` can move it onto the engine +# input. +_PARALLEL_TO_PROMPT_TOKENS = ( + "prompt_token_offsets", + "prompt_is_token_ids", + "_assistant_tokens_mask", +) + def merge_kwargs( defaults: dict[str, Any] | None, @@ -420,8 +432,8 @@ class TokenizeParams: """The slice truncation applies to a sequence of `length` tokens. `None` means no truncation. Anything parallel to the prompt tokens - (see `prompt_token_offsets` and `prompt_is_token_ids`) must be reduced - with this same slice to stay aligned with them. + must be reduced with this same slice to stay aligned with them; list + such keys in `_PARALLEL_TO_PROMPT_TOKENS`. """ max_length = self.truncate_prompt_tokens if max_length is not None and max_length < 0: @@ -511,15 +523,13 @@ class TokenizeParams: tokenizer, prompt["prompt_embeds"], # type: ignore[typeddict-item] ) - offsets = cast(TokensPrompt, prompt).get("prompt_token_offsets") - if offsets is not None: - truncation = self._truncation_slice(tokenizer, len(offsets)) + prompt_dict = cast(dict, prompt) + for key in _PARALLEL_TO_PROMPT_TOKENS: + parallel = prompt_dict.get(key) + if parallel is None: + continue + truncation = self._truncation_slice(tokenizer, len(parallel)) if truncation is not None: - prompt["prompt_token_offsets"] = offsets[truncation] # type: ignore[typeddict-unknown-key] - is_token_ids = cast(EmbedsPrompt, prompt).get("prompt_is_token_ids") - if is_token_ids is not None: - truncation = self._truncation_slice(tokenizer, len(is_token_ids)) - if truncation is not None: - prompt["prompt_is_token_ids"] = is_token_ids[truncation] # type: ignore[typeddict-unknown-key] + prompt_dict[key] = parallel[truncation] return prompt