diff --git a/tests/entrypoints/serve/render/test_render.py b/tests/entrypoints/serve/render/test_render.py index d7339361ff..ffd7f9f30a 100644 --- a/tests/entrypoints/serve/render/test_render.py +++ b/tests/entrypoints/serve/render/test_render.py @@ -14,7 +14,7 @@ MODEL_NAME = "hmellor/tiny-random-LlamaForCausalLM" @pytest.fixture(scope="module") def server(): - args: list[str] = [] + args: list[str] = ["--trust-request-chat-template"] with RemoteLaunchRenderServer(MODEL_NAME, args) as remote_server: yield remote_server @@ -369,3 +369,127 @@ async def test_completion_render_multiple_prompts_token_offsets(client): assert len(offsets) == len(item["token_ids"]) for start, end in offsets: assert 0 <= start <= end <= len(prompt) + + +@pytest.mark.asyncio +async def test_chat_completion_render_assistant_tokens_mask_default(client): + """Without return_assistant_tokens_mask, assistant_tokens_mask should be null.""" + response = await client.post( + "/v1/chat/completions/render", + json={ + "model": MODEL_NAME, + "messages": [ + {"role": "user", "content": "Hello"}, + {"role": "assistant", "content": "Hi!"}, + {"role": "user", "content": "How are you?"}, + ], + }, + ) + + assert response.status_code == 200 + data = response.json() + assert data.get("assistant_tokens_mask") is None + + +@pytest.mark.asyncio +async def test_chat_completion_render_assistant_tokens_mask_false(client): + """Explicitly setting return_assistant_tokens_mask=false gives null.""" + response = await client.post( + "/v1/chat/completions/render", + json={ + "model": MODEL_NAME, + "messages": [ + {"role": "user", "content": "Hello"}, + ], + "return_assistant_tokens_mask": False, + }, + ) + + assert response.status_code == 200 + data = response.json() + assert data.get("assistant_tokens_mask") is None + + +@pytest.mark.asyncio +async def test_chat_render_assistant_tokens_mask_null_without_gen_tags( + client, +): + """The tiny test model lacks ``{% generation %}`` tags, so the mask is null.""" + response = await client.post( + "/v1/chat/completions/render", + json={ + "model": MODEL_NAME, + "messages": [ + {"role": "user", "content": "Hello"}, + {"role": "assistant", "content": "Hi!"}, + ], + "return_assistant_tokens_mask": True, + }, + ) + + assert response.status_code == 200 + assert response.json().get("assistant_tokens_mask") is None + + +# A minimal chat template with {% generation %} tags so we can test that +# the mask correctly marks assistant tokens. +_TEMPLATE_WITH_GENERATION = ( + "{% for m in messages %}" + "{% if m['role'] == 'user' %}User: {{ m['content'] }}\n" + "{% elif m['role'] == 'assistant' %}" + "{% generation %}Assistant: {{ m['content'] }}\n{% endgeneration %}" + "{% endif %}" + "{% endfor %}" +) + + +@pytest.mark.asyncio +async def test_chat_completion_render_assistant_tokens_mask_with_generation_tags( + client, +): + """With a ``{% generation %}``-enabled template, the mask marks assistant + tokens and the masked tokens decode to the assistant content.""" + response = await client.post( + "/v1/chat/completions/render", + json={ + "model": MODEL_NAME, + "messages": [ + {"role": "user", "content": "Hello"}, + {"role": "assistant", "content": "Hi!"}, + {"role": "user", "content": "Bye"}, + ], + "chat_template": _TEMPLATE_WITH_GENERATION, + "return_assistant_tokens_mask": True, + }, + ) + + assert response.status_code == 200 + data = response.json() + + mask = data["assistant_tokens_mask"] + token_ids = data["token_ids"] + assert mask is not None + assert isinstance(mask, list) + assert len(mask) == len(token_ids) + assert all(v in (0, 1) for v in mask) + assert sum(mask) > 0, "mask should mark at least one assistant token" + + # Detokenize masked (assistant) and unmasked (non-assistant) tokens + # separately to verify the mask is correct, not just non-empty. + masked_ids = [t for t, m in zip(token_ids, mask, strict=True) if m] + unmasked_ids = [t for t, m in zip(token_ids, mask, strict=True) if not m] + + detok = await client.post( + "/detokenize", + json={"model": MODEL_NAME, "tokens": masked_ids}, + ) + assert detok.status_code == 200 + assert "Hi!" in detok.json()["prompt"] + + detok_rest = await client.post( + "/detokenize", + json={"model": MODEL_NAME, "tokens": unmasked_ids}, + ) + assert detok_rest.status_code == 200 + assert "Hi!" not in detok_rest.json()["prompt"] + assert "Bye" in detok_rest.json()["prompt"] diff --git a/vllm/entrypoints/openai/chat_completion/protocol.py b/vllm/entrypoints/openai/chat_completion/protocol.py index aa2af69777..36e467f6f3 100644 --- a/vllm/entrypoints/openai/chat_completion/protocol.py +++ b/vllm/entrypoints/openai/chat_completion/protocol.py @@ -407,6 +407,18 @@ class ChatCompletionRequest(OpenAIBaseModel): ), ) + return_assistant_tokens_mask: bool = Field( + default=False, + description=( + "If true, the /render response will include an " + "``assistant_tokens_mask`` field — a per-token list of 0/1 " + "values indicating which tokens were assistant-generated. " + "Requires the chat template to use ``{% generation %}`` " + "tags. When the template does not support it, " + "``assistant_tokens_mask`` will be ``null``." + ), + ) + cache_salt: str | None = Field( default=None, description=( @@ -520,6 +532,7 @@ class ChatCompletionRequest(OpenAIBaseModel): extra_kwargs, ), media_io_kwargs=self.media_io_kwargs, + return_assistant_tokens_mask=bool(self.return_assistant_tokens_mask), ) def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams: diff --git a/vllm/entrypoints/serve/disagg/protocol.py b/vllm/entrypoints/serve/disagg/protocol.py index d20752a906..723c279249 100644 --- a/vllm/entrypoints/serve/disagg/protocol.py +++ b/vllm/entrypoints/serve/disagg/protocol.py @@ -75,6 +75,14 @@ class GenerateRequest(BaseModel): token_ids: list[int] = Field(min_length=1) """The token ids to generate text from.""" + assistant_tokens_mask: list[int] | None = None + """Per-token mask (1 = assistant-generated, 0 = not). + + Only populated when the render request sets ``return_assistant_tokens_mask=True`` + and the chat template supports ``{% generation %}``. + ``None`` when the mask was not requested or could not be computed. + """ + @field_validator("token_ids") @classmethod def validate_token_ids(cls, v: list[int]) -> list[int]: diff --git a/vllm/entrypoints/serve/render/serving.py b/vllm/entrypoints/serve/render/serving.py index 1bba26722b..adcaf8af9a 100644 --- a/vllm/entrypoints/serve/render/serving.py +++ b/vllm/entrypoints/serve/render/serving.py @@ -127,11 +127,33 @@ class ServingRender(BaseServing): ) params = request.to_sampling_params(max_tokens, self.default_sampling_params) + assistant_tokens_mask: list[int] | None = engine_input.get( # type: ignore[assignment] + "assistant_tokens_mask" + ) + if assistant_tokens_mask is not None and len(assistant_tokens_mask) != len( + token_ids + ): + logger.warning( + "assistant_tokens_mask length (%d) != token_ids length (%d); " + "this can happen with multimodal inputs where " + "placeholder expansion changes the token count. " + "The mask may be positionally misaligned.", + len(assistant_tokens_mask), + len(token_ids), + ) + if len(assistant_tokens_mask) < len(token_ids): + assistant_tokens_mask.extend( + [0] * (len(token_ids) - len(assistant_tokens_mask)) + ) + else: + assistant_tokens_mask = assistant_tokens_mask[: len(token_ids)] + request_id = f"chatcmpl-{random_uuid()}" return GenerateRequest( request_id=request_id, token_ids=token_ids, + assistant_tokens_mask=assistant_tokens_mask, features=self._extract_mm_features(engine_input), sampling_params=params, model=request.model, diff --git a/vllm/inputs/engine.py b/vllm/inputs/engine.py index eacadcbc92..f997004d2f 100644 --- a/vllm/inputs/engine.py +++ b/vllm/inputs/engine.py @@ -42,6 +42,11 @@ class TokensInput(_InputOptions): """Char-level (start, end) offsets per token, propagated from the renderer's TokensPrompt when offsets were computed.""" + assistant_tokens_mask: NotRequired[list[int] | None] + """Per-token 0/1 mask marking assistant-generated tokens. + Populated when ``return_assistant_tokens_mask=True`` is set on the + render request and the chat template supports ``{% generation %}``.""" + def tokens_input( prompt_token_ids: list[int], @@ -151,6 +156,11 @@ class MultiModalInput(_InputOptions): `prompt_token_ids`. """ + assistant_tokens_mask: NotRequired[list[int] | None] + """Per-token 0/1 mask marking assistant-generated tokens. + Populated when ``return_assistant_tokens_mask=True`` is set on the + render request and the chat template supports ``{% generation %}``.""" + def mm_input( prompt_token_ids: list[int], diff --git a/vllm/renderers/hf.py b/vllm/renderers/hf.py index ea0902c880..490f589af4 100644 --- a/vllm/renderers/hf.py +++ b/vllm/renderers/hf.py @@ -7,7 +7,7 @@ import inspect import itertools import weakref from collections import defaultdict, deque -from collections.abc import Sequence +from collections.abc import Mapping, Sequence from functools import lru_cache from typing import TYPE_CHECKING, Any, Final, Literal, cast, overload @@ -678,6 +678,7 @@ def safe_apply_chat_template( tools: list[dict[str, Any]] | None = ..., chat_template: str | None = ..., tokenize: Literal[True] = ..., + return_assistant_tokens_mask: Literal[False] = ..., **kwargs, ) -> list[int]: ... @overload @@ -689,8 +690,20 @@ def safe_apply_chat_template( tools: list[dict[str, Any]] | None = ..., chat_template: str | None = ..., tokenize: Literal[False] = ..., + return_assistant_tokens_mask: Literal[False] = ..., **kwargs, ) -> str: ... +@overload +def safe_apply_chat_template( + model_config: ModelConfig, + tokenizer: HfTokenizer, + conversation: list[ConversationMessage], + *, + tools: list[dict[str, Any]] | None = ..., + chat_template: str | None = ..., + return_assistant_tokens_mask: Literal[True], + **kwargs, +) -> tuple[list[int], list[int] | None]: ... def safe_apply_chat_template( model_config: ModelConfig, tokenizer: HfTokenizer, @@ -699,8 +712,9 @@ def safe_apply_chat_template( tools: list[dict[str, Any]] | None = None, chat_template: str | None = None, tokenize: bool = True, + return_assistant_tokens_mask: bool = False, **kwargs, -) -> str | list[int]: +) -> str | list[int] | tuple[list[int], list[int] | None]: chat_template = resolve_chat_template( tokenizer, chat_template=chat_template, @@ -728,6 +742,38 @@ def safe_apply_chat_template( chat_template_kwargs=kwargs, ) + # assistant_tokens_mask requires tokenized output — force tokenize=True. + if return_assistant_tokens_mask: + tokenize = True + + # When return_assistant_tokens_mask is requested and the template supports it, + # request assistant_tokens_mask via return_dict. + # Check for the actual Jinja tag, not just the word "generation" + # (which also appears in add_generation_prompt). + if return_assistant_tokens_mask and "{% generation %}" in chat_template: + resolved_kwargs["return_assistant_tokens_mask"] = True + resolved_kwargs["return_dict"] = True + resolved_kwargs.pop("tokenize", None) + try: + result = tokenizer.apply_chat_template( + conversation=conversation, # type: ignore[arg-type] + tools=tools, # type: ignore[arg-type] + chat_template=chat_template, + tokenize=True, + **resolved_kwargs, + ) + except (TypeError, ValueError) as exc: + logger.warning( + "apply_chat_template failed for assistant_tokens_mask: %s", exc + ) + else: + if isinstance(result, Mapping): + token_ids = list(result.get("input_ids", [])) + mask_raw = result.get("assistant_masks") + mask = list(mask_raw) if mask_raw is not None else None + return token_ids, mask + return list(result), None + # transformers v5 changed the default of `return_dict` to True, which # makes `apply_chat_template(tokenize=True)` return a `BatchEncoding` # instead of `list[int]`. Force `return_dict=False` so downstream code @@ -737,23 +783,24 @@ def safe_apply_chat_template( resolved_kwargs["return_dict"] = False try: - return tokenizer.apply_chat_template( + plain = tokenizer.apply_chat_template( conversation=conversation, # type: ignore[arg-type] tools=tools, # type: ignore[arg-type] chat_template=chat_template, tokenize=tokenize, **resolved_kwargs, ) - # External library exceptions can sometimes occur despite the framework's - # internal exception management capabilities. except Exception as e: - # Log and report any library-related exceptions for further - # investigation. logger.exception( "An error occurred in `transformers` while applying chat template" ) raise ValueError(str(e)) from e + if return_assistant_tokens_mask: + assert isinstance(plain, list), f"Expected list[int], got {type(plain)}" + return plain, None + return plain + def rebuild_mm_uuids_from_mm_data( mm_uuids: MultiModalUUIDDict, @@ -934,12 +981,22 @@ class HfRenderer(BaseRenderer[HfTokenizer]): logger.warning_once(_TOKENIZE_OVERRIDE_WARNING) chat_template_kwargs["tokenize"] = True - prompt_raw = safe_apply_chat_template( - model_config, - tokenizer, - conversation, - **chat_template_kwargs, - ) + assistant_tokens_mask: list[int] | None = None + if params.return_assistant_tokens_mask: + prompt_raw, assistant_tokens_mask = safe_apply_chat_template( + model_config, + tokenizer, + conversation, + return_assistant_tokens_mask=True, + **chat_template_kwargs, + ) + else: + prompt_raw = safe_apply_chat_template( + model_config, + tokenizer, + conversation, + **chat_template_kwargs, + ) # NOTE: use_unified_vision_chunk is currently specific to Kimi-K2.5 # model which uses unified vision chunks for both images and videos. @@ -965,6 +1022,9 @@ class HfRenderer(BaseRenderer[HfTokenizer]): prompt = parse_dec_only_prompt(prompt_raw) + if assistant_tokens_mask is not None: + cast(dict, prompt)["_assistant_tokens_mask"] = assistant_tokens_mask + # When `prompt_embeds` is mixed with other modality data, # `_process_tokens` runs `_process_multimodal` first (expanding # `<|AUDIO|>` / `<|IMAGE|>` placeholders) and then @@ -1038,12 +1098,30 @@ class HfRenderer(BaseRenderer[HfTokenizer]): logger.warning_once(_TOKENIZE_OVERRIDE_WARNING) chat_template_kwargs["tokenize"] = True - prompt_raw = await self._apply_chat_template_async( - model_config, - tokenizer, - conversation, - **chat_template_kwargs, - ) + assistant_tokens_mask: list[int] | None = None + if params.return_assistant_tokens_mask: + result_with_mask = cast( + tuple[list[int], list[int] | None], + await make_async( + safe_apply_chat_template, + executor=self._executor, + )( + model_config, + tokenizer, + conversation, + return_assistant_tokens_mask=True, # type: ignore[arg-type] + **chat_template_kwargs, + ), + ) + prompt_raw: str | list[int] = result_with_mask[0] + assistant_tokens_mask = result_with_mask[1] + else: + prompt_raw = await self._apply_chat_template_async( + model_config, + tokenizer, + conversation, + **chat_template_kwargs, + ) # NOTE: use_unified_vision_chunk is currently specific to Kimi-K2.5 # model which uses unified vision chunks for both images and videos. @@ -1067,6 +1145,9 @@ class HfRenderer(BaseRenderer[HfTokenizer]): prompt = parse_dec_only_prompt(prompt_raw) + if assistant_tokens_mask is not None: + cast(dict, prompt)["_assistant_tokens_mask"] = assistant_tokens_mask + # See `render_messages` for the rationale. if prompt_embeds_tensors and mm_data: assert prompt_embeds_placeholder_token_id is not None @@ -1108,6 +1189,7 @@ class HfRenderer(BaseRenderer[HfTokenizer]): processor records all placeholder offsets in the final (post-expansion) coordinate space, no offset shifting needed afterwards. """ + assistant_tokens_mask = cast(dict, prompt).pop("_assistant_tokens_mask", None) prompt_embeds_info = cast(dict, prompt).pop("_prompt_embeds", None) if prompt_embeds_info is not None: tensors, placeholder_token_id = prompt_embeds_info @@ -1123,6 +1205,8 @@ class HfRenderer(BaseRenderer[HfTokenizer]): tensors, mm_updates, ) + if assistant_tokens_mask is not None: + engine_input["assistant_tokens_mask"] = assistant_tokens_mask return engine_input @override @@ -1133,6 +1217,7 @@ class HfRenderer(BaseRenderer[HfTokenizer]): skip_mm_cache: bool = False, ) -> TokensInput | MultiModalInput: """Async equivalent of `_process_tokens`.""" + assistant_tokens_mask = cast(dict, prompt).pop("_assistant_tokens_mask", None) prompt_embeds_info = cast(dict, prompt).pop("_prompt_embeds", None) if prompt_embeds_info is not None: tensors, placeholder_token_id = prompt_embeds_info @@ -1150,6 +1235,8 @@ class HfRenderer(BaseRenderer[HfTokenizer]): tensors, mm_updates, ) + if assistant_tokens_mask is not None: + engine_input["assistant_tokens_mask"] = assistant_tokens_mask return engine_input @staticmethod diff --git a/vllm/renderers/params.py b/vllm/renderers/params.py index 8e0aaf303c..7e3670c738 100644 --- a/vllm/renderers/params.py +++ b/vllm/renderers/params.py @@ -87,6 +87,9 @@ class ChatParams: mm_processor_kwargs: dict[str, Any] | None = None """The kwargs to pass to the multi-modal processor.""" + return_assistant_tokens_mask: bool = False + """Request a per-token assistant mask from apply_chat_template.""" + def with_defaults( self, default_chat_template_kwargs: dict[str, Any] | None = None, @@ -115,6 +118,7 @@ class ChatParams: default_mm_processor_kwargs, self.mm_processor_kwargs, ), + return_assistant_tokens_mask=self.return_assistant_tokens_mask, ) def get_apply_chat_template_kwargs(self) -> dict[str, Any]: