[Deprecation] Remove assistant_token_mask support (#57520)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Cyrus Leung
2026-09-19 00:17:51 +08:00
committed by GitHub
parent 164dca8d1e
commit 56cac80b14
7 changed files with 20 additions and 334 deletions
@@ -594,176 +594,6 @@ async def test_completion_render_multiple_prompts_token_offsets(client):
assert 0 <= start <= end <= len(prompt)
@pytest.mark.asyncio
async def test_chat_completion_render_assistant_tokens_mask_default(client):
"""Without return_assistant_tokens_mask, assistant_tokens_mask should be null."""
response = await client.post(
"/v1/chat/completions/render",
json={
"model": MODEL_NAME,
"messages": [
{"role": "user", "content": "Hello"},
{"role": "assistant", "content": "Hi!"},
{"role": "user", "content": "How are you?"},
],
},
)
assert response.status_code == 200
data = response.json()
assert data.get("assistant_tokens_mask") is None
@pytest.mark.asyncio
async def test_chat_completion_render_assistant_tokens_mask_false(client):
"""Explicitly setting return_assistant_tokens_mask=false gives null."""
response = await client.post(
"/v1/chat/completions/render",
json={
"model": MODEL_NAME,
"messages": [
{"role": "user", "content": "Hello"},
],
"return_assistant_tokens_mask": False,
},
)
assert response.status_code == 200
data = response.json()
assert data.get("assistant_tokens_mask") is None
@pytest.mark.asyncio
async def test_chat_render_assistant_tokens_mask_null_without_gen_tags(
client,
):
"""The tiny test model lacks ``{% generation %}`` tags, so the mask is null."""
response = await client.post(
"/v1/chat/completions/render",
json={
"model": MODEL_NAME,
"messages": [
{"role": "user", "content": "Hello"},
{"role": "assistant", "content": "Hi!"},
],
"return_assistant_tokens_mask": True,
},
)
assert response.status_code == 200
assert response.json().get("assistant_tokens_mask") is None
# A minimal chat template with {% generation %} tags so we can test that
# the mask correctly marks assistant tokens.
_TEMPLATE_WITH_GENERATION = (
"{% for m in messages %}"
"{% if m['role'] == 'user' %}User: {{ m['content'] }}\n"
"{% elif m['role'] == 'assistant' %}"
"{% generation %}Assistant: {{ m['content'] }}\n{% endgeneration %}"
"{% endif %}"
"{% endfor %}"
)
@pytest.mark.asyncio
async def test_chat_completion_render_assistant_tokens_mask_with_generation_tags(
client,
):
"""With a ``{% generation %}``-enabled template, the mask marks assistant
tokens and the masked tokens decode to the assistant content."""
response = await client.post(
"/v1/chat/completions/render",
json={
"model": MODEL_NAME,
"messages": [
{"role": "user", "content": "Hello"},
{"role": "assistant", "content": "Hi!"},
{"role": "user", "content": "Bye"},
],
"chat_template": _TEMPLATE_WITH_GENERATION,
"return_assistant_tokens_mask": True,
},
)
assert response.status_code == 200
data = response.json()
mask = data["assistant_tokens_mask"]
token_ids = data["token_ids"]
assert mask is not None
assert isinstance(mask, list)
assert len(mask) == len(token_ids)
assert all(v in (0, 1) for v in mask)
assert sum(mask) > 0, "mask should mark at least one assistant token"
# Detokenize masked (assistant) and unmasked (non-assistant) tokens
# separately to verify the mask is correct, not just non-empty.
masked_ids = [t for t, m in zip(token_ids, mask, strict=True) if m]
unmasked_ids = [t for t, m in zip(token_ids, mask, strict=True) if not m]
detok = await client.post(
"/detokenize",
json={"model": MODEL_NAME, "tokens": masked_ids},
)
assert detok.status_code == 200
assert "Hi!" in detok.json()["prompt"]
detok_rest = await client.post(
"/detokenize",
json={"model": MODEL_NAME, "tokens": unmasked_ids},
)
assert detok_rest.status_code == 200
assert "Hi!" not in detok_rest.json()["prompt"]
assert "Bye" in detok_rest.json()["prompt"]
@pytest.mark.asyncio
async def test_chat_render_assistant_tokens_mask_follows_truncation(client):
"""The assistant mask must be truncated with the prompt it describes.
`assistant_tokens_mask` is positional: entry i labels token i. Truncating
`token_ids` from the left without truncating the mask leaves the two
describing different positions, and the mask ends up marking whichever
tokens happen to sit at the old offsets.
"""
messages = [
# Deliberately lopsided: a long leading user turn and a short trailing
# one, so keeping the head of the mask is distinguishable from keeping
# its tail.
{"role": "user", "content": "Hello hello hello hello hello hello"},
{"role": "assistant", "content": "Hi!"},
{"role": "user", "content": "Bye"},
]
body = {
"model": MODEL_NAME,
"messages": messages,
"chat_template": _TEMPLATE_WITH_GENERATION,
"return_assistant_tokens_mask": True,
}
full = await client.post("/v1/chat/completions/render", json=body)
assert full.status_code == 200
full_token_ids = full.json()["token_ids"]
full_mask = full.json()["assistant_tokens_mask"]
assert sum(full_mask) > 0
keep = len(full_token_ids) - 4
# Precondition: with this prompt the head and tail slices of the mask
# really do differ, so the assertion below can tell them apart.
assert full_mask[-keep:] != full_mask[:keep]
truncated = await client.post(
"/v1/chat/completions/render",
json={**body, "truncate_prompt_tokens": keep, "truncation_side": "left"},
)
assert truncated.status_code == 200
data = truncated.json()
assert data["token_ids"] == full_token_ids[-keep:]
assert data["assistant_tokens_mask"] == full_mask[-keep:]
@pytest.mark.asyncio
async def test_messages_render_basic(client):
"""Test basic Anthropic Messages render endpoint."""
@@ -453,18 +453,6 @@ class ChatCompletionRequest(OpenAIBaseModel):
),
)
return_assistant_tokens_mask: bool = Field(
default=False,
description=(
"If true, the /render response will include an "
"``assistant_tokens_mask`` field — a per-token list of 0/1 "
"values indicating which tokens were assistant-generated. "
"Requires the chat template to use ``{% generation %}`` "
"tags. When the template does not support it, "
"``assistant_tokens_mask`` will be ``null``."
),
)
cache_salt: str | None = Field(
default=None,
min_length=1,
@@ -605,7 +593,6 @@ class ChatCompletionRequest(OpenAIBaseModel):
extra_kwargs,
),
media_io_kwargs=self.media_io_kwargs,
return_assistant_tokens_mask=bool(self.return_assistant_tokens_mask),
# No-tools requests default to tool_choice="none" at the API
# layer. Collapse that default before rendering, so K3 emits a
# model-visible tool-choice instruction only for requests with a
@@ -126,33 +126,11 @@ class ServingRender(BaseServing):
)
params = request.to_sampling_params(max_tokens, self.default_sampling_params)
assistant_tokens_mask: list[int] | None = engine_input.get( # type: ignore[assignment]
"assistant_tokens_mask"
)
if assistant_tokens_mask is not None and len(assistant_tokens_mask) != len(
token_ids
):
logger.warning(
"assistant_tokens_mask length (%d) != token_ids length (%d); "
"this can happen with multimodal inputs where "
"placeholder expansion changes the token count. "
"The mask may be positionally misaligned.",
len(assistant_tokens_mask),
len(token_ids),
)
if len(assistant_tokens_mask) < len(token_ids):
assistant_tokens_mask.extend(
[0] * (len(token_ids) - len(assistant_tokens_mask))
)
else:
assistant_tokens_mask = assistant_tokens_mask[: len(token_ids)]
request_id = f"chatcmpl-{random_uuid()}"
return GenerateRequest(
request_id=request_id,
token_ids=token_ids,
assistant_tokens_mask=assistant_tokens_mask,
features=self._extract_mm_features(engine_input),
sampling_params=params,
model=request.model,
@@ -150,14 +150,6 @@ class GenerateRequest(BaseModel):
token_ids: list[int] = Field(min_length=1)
"""The token ids to generate text from."""
assistant_tokens_mask: list[int] | None = None
"""Per-token mask (1 = assistant-generated, 0 = not).
Only populated when the render request sets ``return_assistant_tokens_mask=True``
and the chat template supports ``{% generation %}``.
``None`` when the mask was not requested or could not be computed.
"""
@field_validator("token_ids")
@classmethod
def validate_token_ids(cls, v: list[int]) -> list[int]:
-10
View File
@@ -43,11 +43,6 @@ class TokensInput(_InputOptions):
"""Char-level (start, end) offsets per token, propagated from the
renderer's TokensPrompt when offsets were computed."""
assistant_tokens_mask: NotRequired[list[int] | None]
"""Per-token 0/1 mask marking assistant-generated tokens.
Populated when ``return_assistant_tokens_mask=True`` is set on the
render request and the chat template supports ``{% generation %}``."""
def tokens_input(
prompt_token_ids: list[int],
@@ -155,11 +150,6 @@ class MultiModalInput(_InputOptions):
`prompt_token_ids`.
"""
assistant_tokens_mask: NotRequired[list[int] | None]
"""Per-token 0/1 mask marking assistant-generated tokens.
Populated when ``return_assistant_tokens_mask=True`` is set on the
render request and the chat template supports ``{% generation %}``."""
def mm_input(
prompt_token_ids: list[int],
+20 -103
View File
@@ -7,7 +7,7 @@ import inspect
import itertools
import weakref
from collections import defaultdict, deque
from collections.abc import Mapping, Sequence
from collections.abc import Sequence
from functools import lru_cache
from typing import TYPE_CHECKING, Any, Final, Literal, cast, overload
@@ -758,7 +758,6 @@ def safe_apply_chat_template(
tools: list[dict[str, Any]] | None = ...,
chat_template: str | None = ...,
tokenize: Literal[True] = ...,
return_assistant_tokens_mask: Literal[False] = ...,
**kwargs,
) -> list[int]: ...
@overload
@@ -770,20 +769,8 @@ def safe_apply_chat_template(
tools: list[dict[str, Any]] | None = ...,
chat_template: str | None = ...,
tokenize: Literal[False] = ...,
return_assistant_tokens_mask: Literal[False] = ...,
**kwargs,
) -> str: ...
@overload
def safe_apply_chat_template(
model_config: ModelConfig,
tokenizer: HfTokenizer,
conversation: list[ConversationMessage],
*,
tools: list[dict[str, Any]] | None = ...,
chat_template: str | None = ...,
return_assistant_tokens_mask: Literal[True],
**kwargs,
) -> tuple[list[int], list[int] | None]: ...
def safe_apply_chat_template(
model_config: ModelConfig,
tokenizer: HfTokenizer,
@@ -792,9 +779,8 @@ def safe_apply_chat_template(
tools: list[dict[str, Any]] | None = None,
chat_template: str | None = None,
tokenize: bool = True,
return_assistant_tokens_mask: bool = False,
**kwargs,
) -> str | list[int] | tuple[list[int], list[int] | None]:
) -> str | list[int]:
chat_template = resolve_chat_template(
tokenizer,
chat_template=chat_template,
@@ -822,38 +808,6 @@ def safe_apply_chat_template(
chat_template_kwargs=kwargs,
)
# assistant_tokens_mask requires tokenized output — force tokenize=True.
if return_assistant_tokens_mask:
tokenize = True
# When return_assistant_tokens_mask is requested and the template supports it,
# request assistant_tokens_mask via return_dict.
# Check for the actual Jinja tag, not just the word "generation"
# (which also appears in add_generation_prompt).
if return_assistant_tokens_mask and "{% generation %}" in chat_template:
resolved_kwargs["return_assistant_tokens_mask"] = True
resolved_kwargs["return_dict"] = True
resolved_kwargs.pop("tokenize", None)
try:
result = tokenizer.apply_chat_template(
conversation=conversation, # type: ignore[arg-type]
tools=tools, # type: ignore[arg-type]
chat_template=chat_template,
tokenize=True,
**resolved_kwargs,
)
except (TypeError, ValueError) as exc:
logger.warning(
"apply_chat_template failed for assistant_tokens_mask: %s", exc
)
else:
if isinstance(result, Mapping):
token_ids = list(result.get("input_ids", []))
mask_raw = result.get("assistant_masks")
mask = list(mask_raw) if mask_raw is not None else None
return token_ids, mask
return list(result), None
# transformers v5 changed the default of `return_dict` to True, which
# makes `apply_chat_template(tokenize=True)` return a `BatchEncoding`
# instead of `list[int]`. Force `return_dict=False` so downstream code
@@ -879,9 +833,6 @@ def safe_apply_chat_template(
logger.warning("Chat template rejected the request: %s", e)
raise VLLMValidationError(_template_error_reason(e)) from e
if return_assistant_tokens_mask:
assert isinstance(plain, list), f"Expected list[int], got {type(plain)}"
return plain, None
return plain
@@ -1066,22 +1017,12 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
logger.warning_once(_TOKENIZE_OVERRIDE_WARNING)
chat_template_kwargs["tokenize"] = True
assistant_tokens_mask: list[int] | None = None
if params.return_assistant_tokens_mask:
prompt_raw, assistant_tokens_mask = safe_apply_chat_template(
model_config,
tokenizer,
conversation,
return_assistant_tokens_mask=True,
**chat_template_kwargs,
)
else:
prompt_raw = safe_apply_chat_template(
model_config,
tokenizer,
conversation,
**chat_template_kwargs,
)
prompt_raw = safe_apply_chat_template(
model_config,
tokenizer,
conversation,
**chat_template_kwargs,
)
# NOTE: use_unified_vision_chunk is currently specific to Kimi-K2.5
# model which uses unified vision chunks for both images and videos.
@@ -1107,9 +1048,6 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
prompt = parse_dec_only_prompt(prompt_raw)
if assistant_tokens_mask is not None:
cast(dict, prompt)["_assistant_tokens_mask"] = assistant_tokens_mask
# When `prompt_embeds` is mixed with other modality data,
# `_process_tokens` runs `_process_multimodal` first (expanding
# `<|AUDIO|>` / `<|IMAGE|>` placeholders) and then
@@ -1183,30 +1121,12 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
logger.warning_once(_TOKENIZE_OVERRIDE_WARNING)
chat_template_kwargs["tokenize"] = True
assistant_tokens_mask: list[int] | None = None
if params.return_assistant_tokens_mask:
result_with_mask = cast(
tuple[list[int], list[int] | None],
await make_async(
safe_apply_chat_template,
executor=self._executor,
)(
model_config,
tokenizer,
conversation,
return_assistant_tokens_mask=True, # type: ignore[arg-type]
**chat_template_kwargs,
),
)
prompt_raw: str | list[int] = result_with_mask[0]
assistant_tokens_mask = result_with_mask[1]
else:
prompt_raw = await self._apply_chat_template_async(
model_config,
tokenizer,
conversation,
**chat_template_kwargs,
)
prompt_raw = await self._apply_chat_template_async(
model_config,
tokenizer,
conversation,
**chat_template_kwargs,
)
# NOTE: use_unified_vision_chunk is currently specific to Kimi-K2.5
# model which uses unified vision chunks for both images and videos.
@@ -1232,9 +1152,6 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
prompt = parse_dec_only_prompt(prompt_raw)
if assistant_tokens_mask is not None:
cast(dict, prompt)["_assistant_tokens_mask"] = assistant_tokens_mask
# See `render_messages` for the rationale.
if prompt_embeds_tensors and mm_data:
assert prompt_embeds_placeholder_token_id is not None
@@ -1276,7 +1193,6 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
processor records all placeholder offsets in the final (post-expansion)
coordinate space, no offset shifting needed afterwards.
"""
assistant_tokens_mask = cast(dict, prompt).pop("_assistant_tokens_mask", None)
prompt_embeds_info = cast(dict, prompt).pop("_prompt_embeds", None)
if prompt_embeds_info is not None:
tensors, placeholder_token_id = prompt_embeds_info
@@ -1284,7 +1200,9 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
cast(dict, prompt)["prompt_token_ids"] = _expand_prompt_embeds_placeholders(
list(prompt["prompt_token_ids"]), mm_updates
)
engine_input = super()._process_tokens(prompt, skip_mm_cache=skip_mm_cache)
if prompt_embeds_info is not None:
tensors, _ = prompt_embeds_info
self._apply_prompt_embeds_to_engine_input(
@@ -1292,8 +1210,7 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
tensors,
mm_updates,
)
if assistant_tokens_mask is not None:
engine_input["assistant_tokens_mask"] = assistant_tokens_mask
return engine_input
@override
@@ -1304,7 +1221,6 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
skip_mm_cache: bool = False,
) -> TokensInput | MultiModalInput:
"""Async equivalent of `_process_tokens`."""
assistant_tokens_mask = cast(dict, prompt).pop("_assistant_tokens_mask", None)
prompt_embeds_info = cast(dict, prompt).pop("_prompt_embeds", None)
if prompt_embeds_info is not None:
tensors, placeholder_token_id = prompt_embeds_info
@@ -1312,9 +1228,11 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
cast(dict, prompt)["prompt_token_ids"] = _expand_prompt_embeds_placeholders(
list(prompt["prompt_token_ids"]), mm_updates
)
engine_input = await super()._process_tokens_async(
prompt, skip_mm_cache=skip_mm_cache
)
if prompt_embeds_info is not None:
tensors, _ = prompt_embeds_info
self._apply_prompt_embeds_to_engine_input(
@@ -1322,8 +1240,7 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
tensors,
mm_updates,
)
if assistant_tokens_mask is not None:
engine_input["assistant_tokens_mask"] = assistant_tokens_mask
return engine_input
@staticmethod
-8
View File
@@ -27,13 +27,9 @@ _S = TypeVar("_S", list[int], "torch.Tensor")
# Prompt keys whose entry i describes token i of the prompt, so they must be
# reduced with the same slice as the prompt tokens themselves. Anything added
# here is truncated by `TokenizeParams.apply_post_tokenization`.
# `_assistant_tokens_mask` is renderer-internal: `HfRenderer.render_messages`
# stashes it on the prompt so `_process_tokens` can move it onto the engine
# input.
_PARALLEL_TO_PROMPT_TOKENS = (
"prompt_token_offsets",
"prompt_is_token_ids",
"_assistant_tokens_mask",
)
@@ -99,9 +95,6 @@ class ChatParams:
mm_processor_kwargs: dict[str, Any] | None = None
"""The kwargs to pass to the multi-modal processor."""
return_assistant_tokens_mask: bool = False
"""Request a per-token assistant mask from apply_chat_template."""
tool_choice: Any | None = None
"""Request-level tool choice for renderers that need API metadata."""
@@ -136,7 +129,6 @@ class ChatParams:
default_mm_processor_kwargs,
self.mm_processor_kwargs,
),
return_assistant_tokens_mask=self.return_assistant_tokens_mask,
tool_choice=self.tool_choice,
response_format=self.response_format,
)