[Bugfix][Frontend] Truncate the assistant tokens mask with the prompt (#54539)

Signed-off-by: hotragn <hotragn.pettugani_2024@woxsen.edu.in>
This commit is contained in:
Hotragn Pettugani
2026-09-01 05:53:22 +00:00
committed by GitHub
parent 30dd1a7954
commit ff0c3cb03c
2 changed files with 67 additions and 11 deletions
@@ -531,6 +531,52 @@ async def test_chat_completion_render_assistant_tokens_mask_with_generation_tags
assert "Bye" in detok_rest.json()["prompt"]
@pytest.mark.asyncio
async def test_chat_render_assistant_tokens_mask_follows_truncation(client):
"""The assistant mask must be truncated with the prompt it describes.
`assistant_tokens_mask` is positional: entry i labels token i. Truncating
`token_ids` from the left without truncating the mask leaves the two
describing different positions, and the mask ends up marking whichever
tokens happen to sit at the old offsets.
"""
messages = [
# Deliberately lopsided: a long leading user turn and a short trailing
# one, so keeping the head of the mask is distinguishable from keeping
# its tail.
{"role": "user", "content": "Hello hello hello hello hello hello"},
{"role": "assistant", "content": "Hi!"},
{"role": "user", "content": "Bye"},
]
body = {
"model": MODEL_NAME,
"messages": messages,
"chat_template": _TEMPLATE_WITH_GENERATION,
"return_assistant_tokens_mask": True,
}
full = await client.post("/v1/chat/completions/render", json=body)
assert full.status_code == 200
full_token_ids = full.json()["token_ids"]
full_mask = full.json()["assistant_tokens_mask"]
assert sum(full_mask) > 0
keep = len(full_token_ids) - 4
# Precondition: with this prompt the head and tail slices of the mask
# really do differ, so the assertion below can tell them apart.
assert full_mask[-keep:] != full_mask[:keep]
truncated = await client.post(
"/v1/chat/completions/render",
json={**body, "truncate_prompt_tokens": keep, "truncation_side": "left"},
)
assert truncated.status_code == 200
data = truncated.json()
assert data["token_ids"] == full_token_ids[-keep:]
assert data["assistant_tokens_mask"] == full_mask[-keep:]
@pytest.mark.asyncio
async def test_messages_render_basic(client):
"""Test basic Anthropic Messages render endpoint."""
+21 -11
View File
@@ -24,6 +24,18 @@ logger = init_logger(__name__)
_S = TypeVar("_S", list[int], "torch.Tensor")
# Prompt keys whose entry i describes token i of the prompt, so they must be
# reduced with the same slice as the prompt tokens themselves. Anything added
# here is truncated by `TokenizeParams.apply_post_tokenization`.
# `_assistant_tokens_mask` is renderer-internal: `HfRenderer.render_messages`
# stashes it on the prompt so `_process_tokens` can move it onto the engine
# input.
_PARALLEL_TO_PROMPT_TOKENS = (
"prompt_token_offsets",
"prompt_is_token_ids",
"_assistant_tokens_mask",
)
def merge_kwargs(
defaults: dict[str, Any] | None,
@@ -420,8 +432,8 @@ class TokenizeParams:
"""The slice truncation applies to a sequence of `length` tokens.
`None` means no truncation. Anything parallel to the prompt tokens
(see `prompt_token_offsets` and `prompt_is_token_ids`) must be reduced
with this same slice to stay aligned with them.
must be reduced with this same slice to stay aligned with them; list
such keys in `_PARALLEL_TO_PROMPT_TOKENS`.
"""
max_length = self.truncate_prompt_tokens
if max_length is not None and max_length < 0:
@@ -511,15 +523,13 @@ class TokenizeParams:
tokenizer,
prompt["prompt_embeds"], # type: ignore[typeddict-item]
)
offsets = cast(TokensPrompt, prompt).get("prompt_token_offsets")
if offsets is not None:
truncation = self._truncation_slice(tokenizer, len(offsets))
prompt_dict = cast(dict, prompt)
for key in _PARALLEL_TO_PROMPT_TOKENS:
parallel = prompt_dict.get(key)
if parallel is None:
continue
truncation = self._truncation_slice(tokenizer, len(parallel))
if truncation is not None:
prompt["prompt_token_offsets"] = offsets[truncation] # type: ignore[typeddict-unknown-key]
is_token_ids = cast(EmbedsPrompt, prompt).get("prompt_is_token_ids")
if is_token_ids is not None:
truncation = self._truncation_slice(tokenizer, len(is_token_ids))
if truncation is not None:
prompt["prompt_is_token_ids"] = is_token_ids[truncation] # type: ignore[typeddict-unknown-key]
prompt_dict[key] = parallel[truncation]
return prompt