From 56cac80b145ed1e9814bba519735ad4c71c530f5 Mon Sep 17 00:00:00 2001 From: Cyrus Leung Date: Sat, 19 Sep 2026 00:17:51 +0800 Subject: [PATCH] [Deprecation] Remove assistant_token_mask support (#57520) Signed-off-by: DarkLight1337 --- .../scale_out/render/test_render.py | 170 ------------------ .../openai/chat_completion/protocol.py | 13 -- vllm/entrypoints/scale_out/render/serving.py | 22 --- .../scale_out/token_in_token_out/protocol.py | 8 - vllm/inputs/engine.py | 10 -- vllm/renderers/hf.py | 123 +++---------- vllm/renderers/params.py | 8 - 7 files changed, 20 insertions(+), 334 deletions(-) diff --git a/tests/entrypoints/scale_out/render/test_render.py b/tests/entrypoints/scale_out/render/test_render.py index e21343553d..e65a26a7d1 100644 --- a/tests/entrypoints/scale_out/render/test_render.py +++ b/tests/entrypoints/scale_out/render/test_render.py @@ -594,176 +594,6 @@ async def test_completion_render_multiple_prompts_token_offsets(client): assert 0 <= start <= end <= len(prompt) -@pytest.mark.asyncio -async def test_chat_completion_render_assistant_tokens_mask_default(client): - """Without return_assistant_tokens_mask, assistant_tokens_mask should be null.""" - response = await client.post( - "/v1/chat/completions/render", - json={ - "model": MODEL_NAME, - "messages": [ - {"role": "user", "content": "Hello"}, - {"role": "assistant", "content": "Hi!"}, - {"role": "user", "content": "How are you?"}, - ], - }, - ) - - assert response.status_code == 200 - data = response.json() - assert data.get("assistant_tokens_mask") is None - - -@pytest.mark.asyncio -async def test_chat_completion_render_assistant_tokens_mask_false(client): - """Explicitly setting return_assistant_tokens_mask=false gives null.""" - response = await client.post( - "/v1/chat/completions/render", - json={ - "model": MODEL_NAME, - "messages": [ - {"role": "user", "content": "Hello"}, - ], - "return_assistant_tokens_mask": False, - }, - ) - - assert response.status_code == 200 - data = response.json() - assert data.get("assistant_tokens_mask") is None - - -@pytest.mark.asyncio -async def test_chat_render_assistant_tokens_mask_null_without_gen_tags( - client, -): - """The tiny test model lacks ``{% generation %}`` tags, so the mask is null.""" - response = await client.post( - "/v1/chat/completions/render", - json={ - "model": MODEL_NAME, - "messages": [ - {"role": "user", "content": "Hello"}, - {"role": "assistant", "content": "Hi!"}, - ], - "return_assistant_tokens_mask": True, - }, - ) - - assert response.status_code == 200 - assert response.json().get("assistant_tokens_mask") is None - - -# A minimal chat template with {% generation %} tags so we can test that -# the mask correctly marks assistant tokens. -_TEMPLATE_WITH_GENERATION = ( - "{% for m in messages %}" - "{% if m['role'] == 'user' %}User: {{ m['content'] }}\n" - "{% elif m['role'] == 'assistant' %}" - "{% generation %}Assistant: {{ m['content'] }}\n{% endgeneration %}" - "{% endif %}" - "{% endfor %}" -) - - -@pytest.mark.asyncio -async def test_chat_completion_render_assistant_tokens_mask_with_generation_tags( - client, -): - """With a ``{% generation %}``-enabled template, the mask marks assistant - tokens and the masked tokens decode to the assistant content.""" - response = await client.post( - "/v1/chat/completions/render", - json={ - "model": MODEL_NAME, - "messages": [ - {"role": "user", "content": "Hello"}, - {"role": "assistant", "content": "Hi!"}, - {"role": "user", "content": "Bye"}, - ], - "chat_template": _TEMPLATE_WITH_GENERATION, - "return_assistant_tokens_mask": True, - }, - ) - - assert response.status_code == 200 - data = response.json() - - mask = data["assistant_tokens_mask"] - token_ids = data["token_ids"] - assert mask is not None - assert isinstance(mask, list) - assert len(mask) == len(token_ids) - assert all(v in (0, 1) for v in mask) - assert sum(mask) > 0, "mask should mark at least one assistant token" - - # Detokenize masked (assistant) and unmasked (non-assistant) tokens - # separately to verify the mask is correct, not just non-empty. - masked_ids = [t for t, m in zip(token_ids, mask, strict=True) if m] - unmasked_ids = [t for t, m in zip(token_ids, mask, strict=True) if not m] - - detok = await client.post( - "/detokenize", - json={"model": MODEL_NAME, "tokens": masked_ids}, - ) - assert detok.status_code == 200 - assert "Hi!" in detok.json()["prompt"] - - detok_rest = await client.post( - "/detokenize", - json={"model": MODEL_NAME, "tokens": unmasked_ids}, - ) - assert detok_rest.status_code == 200 - assert "Hi!" not in detok_rest.json()["prompt"] - assert "Bye" in detok_rest.json()["prompt"] - - -@pytest.mark.asyncio -async def test_chat_render_assistant_tokens_mask_follows_truncation(client): - """The assistant mask must be truncated with the prompt it describes. - - `assistant_tokens_mask` is positional: entry i labels token i. Truncating - `token_ids` from the left without truncating the mask leaves the two - describing different positions, and the mask ends up marking whichever - tokens happen to sit at the old offsets. - """ - messages = [ - # Deliberately lopsided: a long leading user turn and a short trailing - # one, so keeping the head of the mask is distinguishable from keeping - # its tail. - {"role": "user", "content": "Hello hello hello hello hello hello"}, - {"role": "assistant", "content": "Hi!"}, - {"role": "user", "content": "Bye"}, - ] - body = { - "model": MODEL_NAME, - "messages": messages, - "chat_template": _TEMPLATE_WITH_GENERATION, - "return_assistant_tokens_mask": True, - } - - full = await client.post("/v1/chat/completions/render", json=body) - assert full.status_code == 200 - full_token_ids = full.json()["token_ids"] - full_mask = full.json()["assistant_tokens_mask"] - assert sum(full_mask) > 0 - - keep = len(full_token_ids) - 4 - # Precondition: with this prompt the head and tail slices of the mask - # really do differ, so the assertion below can tell them apart. - assert full_mask[-keep:] != full_mask[:keep] - - truncated = await client.post( - "/v1/chat/completions/render", - json={**body, "truncate_prompt_tokens": keep, "truncation_side": "left"}, - ) - assert truncated.status_code == 200 - data = truncated.json() - - assert data["token_ids"] == full_token_ids[-keep:] - assert data["assistant_tokens_mask"] == full_mask[-keep:] - - @pytest.mark.asyncio async def test_messages_render_basic(client): """Test basic Anthropic Messages render endpoint.""" diff --git a/vllm/entrypoints/openai/chat_completion/protocol.py b/vllm/entrypoints/openai/chat_completion/protocol.py index badd5a0344..225e992441 100644 --- a/vllm/entrypoints/openai/chat_completion/protocol.py +++ b/vllm/entrypoints/openai/chat_completion/protocol.py @@ -453,18 +453,6 @@ class ChatCompletionRequest(OpenAIBaseModel): ), ) - return_assistant_tokens_mask: bool = Field( - default=False, - description=( - "If true, the /render response will include an " - "``assistant_tokens_mask`` field — a per-token list of 0/1 " - "values indicating which tokens were assistant-generated. " - "Requires the chat template to use ``{% generation %}`` " - "tags. When the template does not support it, " - "``assistant_tokens_mask`` will be ``null``." - ), - ) - cache_salt: str | None = Field( default=None, min_length=1, @@ -605,7 +593,6 @@ class ChatCompletionRequest(OpenAIBaseModel): extra_kwargs, ), media_io_kwargs=self.media_io_kwargs, - return_assistant_tokens_mask=bool(self.return_assistant_tokens_mask), # No-tools requests default to tool_choice="none" at the API # layer. Collapse that default before rendering, so K3 emits a # model-visible tool-choice instruction only for requests with a diff --git a/vllm/entrypoints/scale_out/render/serving.py b/vllm/entrypoints/scale_out/render/serving.py index b79138664f..7973ac3c3d 100644 --- a/vllm/entrypoints/scale_out/render/serving.py +++ b/vllm/entrypoints/scale_out/render/serving.py @@ -126,33 +126,11 @@ class ServingRender(BaseServing): ) params = request.to_sampling_params(max_tokens, self.default_sampling_params) - assistant_tokens_mask: list[int] | None = engine_input.get( # type: ignore[assignment] - "assistant_tokens_mask" - ) - if assistant_tokens_mask is not None and len(assistant_tokens_mask) != len( - token_ids - ): - logger.warning( - "assistant_tokens_mask length (%d) != token_ids length (%d); " - "this can happen with multimodal inputs where " - "placeholder expansion changes the token count. " - "The mask may be positionally misaligned.", - len(assistant_tokens_mask), - len(token_ids), - ) - if len(assistant_tokens_mask) < len(token_ids): - assistant_tokens_mask.extend( - [0] * (len(token_ids) - len(assistant_tokens_mask)) - ) - else: - assistant_tokens_mask = assistant_tokens_mask[: len(token_ids)] - request_id = f"chatcmpl-{random_uuid()}" return GenerateRequest( request_id=request_id, token_ids=token_ids, - assistant_tokens_mask=assistant_tokens_mask, features=self._extract_mm_features(engine_input), sampling_params=params, model=request.model, diff --git a/vllm/entrypoints/scale_out/token_in_token_out/protocol.py b/vllm/entrypoints/scale_out/token_in_token_out/protocol.py index 2bd47b1059..ae0fd62b6e 100644 --- a/vllm/entrypoints/scale_out/token_in_token_out/protocol.py +++ b/vllm/entrypoints/scale_out/token_in_token_out/protocol.py @@ -150,14 +150,6 @@ class GenerateRequest(BaseModel): token_ids: list[int] = Field(min_length=1) """The token ids to generate text from.""" - assistant_tokens_mask: list[int] | None = None - """Per-token mask (1 = assistant-generated, 0 = not). - - Only populated when the render request sets ``return_assistant_tokens_mask=True`` - and the chat template supports ``{% generation %}``. - ``None`` when the mask was not requested or could not be computed. - """ - @field_validator("token_ids") @classmethod def validate_token_ids(cls, v: list[int]) -> list[int]: diff --git a/vllm/inputs/engine.py b/vllm/inputs/engine.py index 429e8a829d..26b840f76f 100644 --- a/vllm/inputs/engine.py +++ b/vllm/inputs/engine.py @@ -43,11 +43,6 @@ class TokensInput(_InputOptions): """Char-level (start, end) offsets per token, propagated from the renderer's TokensPrompt when offsets were computed.""" - assistant_tokens_mask: NotRequired[list[int] | None] - """Per-token 0/1 mask marking assistant-generated tokens. - Populated when ``return_assistant_tokens_mask=True`` is set on the - render request and the chat template supports ``{% generation %}``.""" - def tokens_input( prompt_token_ids: list[int], @@ -155,11 +150,6 @@ class MultiModalInput(_InputOptions): `prompt_token_ids`. """ - assistant_tokens_mask: NotRequired[list[int] | None] - """Per-token 0/1 mask marking assistant-generated tokens. - Populated when ``return_assistant_tokens_mask=True`` is set on the - render request and the chat template supports ``{% generation %}``.""" - def mm_input( prompt_token_ids: list[int], diff --git a/vllm/renderers/hf.py b/vllm/renderers/hf.py index a32824eebd..1c35802d0d 100644 --- a/vllm/renderers/hf.py +++ b/vllm/renderers/hf.py @@ -7,7 +7,7 @@ import inspect import itertools import weakref from collections import defaultdict, deque -from collections.abc import Mapping, Sequence +from collections.abc import Sequence from functools import lru_cache from typing import TYPE_CHECKING, Any, Final, Literal, cast, overload @@ -758,7 +758,6 @@ def safe_apply_chat_template( tools: list[dict[str, Any]] | None = ..., chat_template: str | None = ..., tokenize: Literal[True] = ..., - return_assistant_tokens_mask: Literal[False] = ..., **kwargs, ) -> list[int]: ... @overload @@ -770,20 +769,8 @@ def safe_apply_chat_template( tools: list[dict[str, Any]] | None = ..., chat_template: str | None = ..., tokenize: Literal[False] = ..., - return_assistant_tokens_mask: Literal[False] = ..., **kwargs, ) -> str: ... -@overload -def safe_apply_chat_template( - model_config: ModelConfig, - tokenizer: HfTokenizer, - conversation: list[ConversationMessage], - *, - tools: list[dict[str, Any]] | None = ..., - chat_template: str | None = ..., - return_assistant_tokens_mask: Literal[True], - **kwargs, -) -> tuple[list[int], list[int] | None]: ... def safe_apply_chat_template( model_config: ModelConfig, tokenizer: HfTokenizer, @@ -792,9 +779,8 @@ def safe_apply_chat_template( tools: list[dict[str, Any]] | None = None, chat_template: str | None = None, tokenize: bool = True, - return_assistant_tokens_mask: bool = False, **kwargs, -) -> str | list[int] | tuple[list[int], list[int] | None]: +) -> str | list[int]: chat_template = resolve_chat_template( tokenizer, chat_template=chat_template, @@ -822,38 +808,6 @@ def safe_apply_chat_template( chat_template_kwargs=kwargs, ) - # assistant_tokens_mask requires tokenized output — force tokenize=True. - if return_assistant_tokens_mask: - tokenize = True - - # When return_assistant_tokens_mask is requested and the template supports it, - # request assistant_tokens_mask via return_dict. - # Check for the actual Jinja tag, not just the word "generation" - # (which also appears in add_generation_prompt). - if return_assistant_tokens_mask and "{% generation %}" in chat_template: - resolved_kwargs["return_assistant_tokens_mask"] = True - resolved_kwargs["return_dict"] = True - resolved_kwargs.pop("tokenize", None) - try: - result = tokenizer.apply_chat_template( - conversation=conversation, # type: ignore[arg-type] - tools=tools, # type: ignore[arg-type] - chat_template=chat_template, - tokenize=True, - **resolved_kwargs, - ) - except (TypeError, ValueError) as exc: - logger.warning( - "apply_chat_template failed for assistant_tokens_mask: %s", exc - ) - else: - if isinstance(result, Mapping): - token_ids = list(result.get("input_ids", [])) - mask_raw = result.get("assistant_masks") - mask = list(mask_raw) if mask_raw is not None else None - return token_ids, mask - return list(result), None - # transformers v5 changed the default of `return_dict` to True, which # makes `apply_chat_template(tokenize=True)` return a `BatchEncoding` # instead of `list[int]`. Force `return_dict=False` so downstream code @@ -879,9 +833,6 @@ def safe_apply_chat_template( logger.warning("Chat template rejected the request: %s", e) raise VLLMValidationError(_template_error_reason(e)) from e - if return_assistant_tokens_mask: - assert isinstance(plain, list), f"Expected list[int], got {type(plain)}" - return plain, None return plain @@ -1066,22 +1017,12 @@ class HfRenderer(BaseRenderer[HfTokenizer]): logger.warning_once(_TOKENIZE_OVERRIDE_WARNING) chat_template_kwargs["tokenize"] = True - assistant_tokens_mask: list[int] | None = None - if params.return_assistant_tokens_mask: - prompt_raw, assistant_tokens_mask = safe_apply_chat_template( - model_config, - tokenizer, - conversation, - return_assistant_tokens_mask=True, - **chat_template_kwargs, - ) - else: - prompt_raw = safe_apply_chat_template( - model_config, - tokenizer, - conversation, - **chat_template_kwargs, - ) + prompt_raw = safe_apply_chat_template( + model_config, + tokenizer, + conversation, + **chat_template_kwargs, + ) # NOTE: use_unified_vision_chunk is currently specific to Kimi-K2.5 # model which uses unified vision chunks for both images and videos. @@ -1107,9 +1048,6 @@ class HfRenderer(BaseRenderer[HfTokenizer]): prompt = parse_dec_only_prompt(prompt_raw) - if assistant_tokens_mask is not None: - cast(dict, prompt)["_assistant_tokens_mask"] = assistant_tokens_mask - # When `prompt_embeds` is mixed with other modality data, # `_process_tokens` runs `_process_multimodal` first (expanding # `<|AUDIO|>` / `<|IMAGE|>` placeholders) and then @@ -1183,30 +1121,12 @@ class HfRenderer(BaseRenderer[HfTokenizer]): logger.warning_once(_TOKENIZE_OVERRIDE_WARNING) chat_template_kwargs["tokenize"] = True - assistant_tokens_mask: list[int] | None = None - if params.return_assistant_tokens_mask: - result_with_mask = cast( - tuple[list[int], list[int] | None], - await make_async( - safe_apply_chat_template, - executor=self._executor, - )( - model_config, - tokenizer, - conversation, - return_assistant_tokens_mask=True, # type: ignore[arg-type] - **chat_template_kwargs, - ), - ) - prompt_raw: str | list[int] = result_with_mask[0] - assistant_tokens_mask = result_with_mask[1] - else: - prompt_raw = await self._apply_chat_template_async( - model_config, - tokenizer, - conversation, - **chat_template_kwargs, - ) + prompt_raw = await self._apply_chat_template_async( + model_config, + tokenizer, + conversation, + **chat_template_kwargs, + ) # NOTE: use_unified_vision_chunk is currently specific to Kimi-K2.5 # model which uses unified vision chunks for both images and videos. @@ -1232,9 +1152,6 @@ class HfRenderer(BaseRenderer[HfTokenizer]): prompt = parse_dec_only_prompt(prompt_raw) - if assistant_tokens_mask is not None: - cast(dict, prompt)["_assistant_tokens_mask"] = assistant_tokens_mask - # See `render_messages` for the rationale. if prompt_embeds_tensors and mm_data: assert prompt_embeds_placeholder_token_id is not None @@ -1276,7 +1193,6 @@ class HfRenderer(BaseRenderer[HfTokenizer]): processor records all placeholder offsets in the final (post-expansion) coordinate space, no offset shifting needed afterwards. """ - assistant_tokens_mask = cast(dict, prompt).pop("_assistant_tokens_mask", None) prompt_embeds_info = cast(dict, prompt).pop("_prompt_embeds", None) if prompt_embeds_info is not None: tensors, placeholder_token_id = prompt_embeds_info @@ -1284,7 +1200,9 @@ class HfRenderer(BaseRenderer[HfTokenizer]): cast(dict, prompt)["prompt_token_ids"] = _expand_prompt_embeds_placeholders( list(prompt["prompt_token_ids"]), mm_updates ) + engine_input = super()._process_tokens(prompt, skip_mm_cache=skip_mm_cache) + if prompt_embeds_info is not None: tensors, _ = prompt_embeds_info self._apply_prompt_embeds_to_engine_input( @@ -1292,8 +1210,7 @@ class HfRenderer(BaseRenderer[HfTokenizer]): tensors, mm_updates, ) - if assistant_tokens_mask is not None: - engine_input["assistant_tokens_mask"] = assistant_tokens_mask + return engine_input @override @@ -1304,7 +1221,6 @@ class HfRenderer(BaseRenderer[HfTokenizer]): skip_mm_cache: bool = False, ) -> TokensInput | MultiModalInput: """Async equivalent of `_process_tokens`.""" - assistant_tokens_mask = cast(dict, prompt).pop("_assistant_tokens_mask", None) prompt_embeds_info = cast(dict, prompt).pop("_prompt_embeds", None) if prompt_embeds_info is not None: tensors, placeholder_token_id = prompt_embeds_info @@ -1312,9 +1228,11 @@ class HfRenderer(BaseRenderer[HfTokenizer]): cast(dict, prompt)["prompt_token_ids"] = _expand_prompt_embeds_placeholders( list(prompt["prompt_token_ids"]), mm_updates ) + engine_input = await super()._process_tokens_async( prompt, skip_mm_cache=skip_mm_cache ) + if prompt_embeds_info is not None: tensors, _ = prompt_embeds_info self._apply_prompt_embeds_to_engine_input( @@ -1322,8 +1240,7 @@ class HfRenderer(BaseRenderer[HfTokenizer]): tensors, mm_updates, ) - if assistant_tokens_mask is not None: - engine_input["assistant_tokens_mask"] = assistant_tokens_mask + return engine_input @staticmethod diff --git a/vllm/renderers/params.py b/vllm/renderers/params.py index 86470f8db1..9f76a2ed49 100644 --- a/vllm/renderers/params.py +++ b/vllm/renderers/params.py @@ -27,13 +27,9 @@ _S = TypeVar("_S", list[int], "torch.Tensor") # Prompt keys whose entry i describes token i of the prompt, so they must be # reduced with the same slice as the prompt tokens themselves. Anything added # here is truncated by `TokenizeParams.apply_post_tokenization`. -# `_assistant_tokens_mask` is renderer-internal: `HfRenderer.render_messages` -# stashes it on the prompt so `_process_tokens` can move it onto the engine -# input. _PARALLEL_TO_PROMPT_TOKENS = ( "prompt_token_offsets", "prompt_is_token_ids", - "_assistant_tokens_mask", ) @@ -99,9 +95,6 @@ class ChatParams: mm_processor_kwargs: dict[str, Any] | None = None """The kwargs to pass to the multi-modal processor.""" - return_assistant_tokens_mask: bool = False - """Request a per-token assistant mask from apply_chat_template.""" - tool_choice: Any | None = None """Request-level tool choice for renderers that need API metadata.""" @@ -136,7 +129,6 @@ class ChatParams: default_mm_processor_kwargs, self.mm_processor_kwargs, ), - return_assistant_tokens_mask=self.return_assistant_tokens_mask, tool_choice=self.tool_choice, response_format=self.response_format, )