[Render][Speculator] Add return_loss_mask to render endpoint for training data generation (#46846)

Signed-off-by: Ranran Haoran Zhang <ranzhang@redhat.com>
Co-authored-by: Benjamin Chislett <chislett.ben@gmail.com>
This commit is contained in:
Ranran
2026-06-28 00:07:33 -07:00
committed by GitHub
co-authored by Benjamin Chislett
parent a2a92cbbaa
commit 09841ae705
7 changed files with 288 additions and 20 deletions
+125 -1
View File
@@ -14,7 +14,7 @@ MODEL_NAME = "hmellor/tiny-random-LlamaForCausalLM"
@pytest.fixture(scope="module")
def server():
args: list[str] = []
args: list[str] = ["--trust-request-chat-template"]
with RemoteLaunchRenderServer(MODEL_NAME, args) as remote_server:
yield remote_server
@@ -369,3 +369,127 @@ async def test_completion_render_multiple_prompts_token_offsets(client):
assert len(offsets) == len(item["token_ids"])
for start, end in offsets:
assert 0 <= start <= end <= len(prompt)
@pytest.mark.asyncio
async def test_chat_completion_render_assistant_tokens_mask_default(client):
"""Without return_assistant_tokens_mask, assistant_tokens_mask should be null."""
response = await client.post(
"/v1/chat/completions/render",
json={
"model": MODEL_NAME,
"messages": [
{"role": "user", "content": "Hello"},
{"role": "assistant", "content": "Hi!"},
{"role": "user", "content": "How are you?"},
],
},
)
assert response.status_code == 200
data = response.json()
assert data.get("assistant_tokens_mask") is None
@pytest.mark.asyncio
async def test_chat_completion_render_assistant_tokens_mask_false(client):
"""Explicitly setting return_assistant_tokens_mask=false gives null."""
response = await client.post(
"/v1/chat/completions/render",
json={
"model": MODEL_NAME,
"messages": [
{"role": "user", "content": "Hello"},
],
"return_assistant_tokens_mask": False,
},
)
assert response.status_code == 200
data = response.json()
assert data.get("assistant_tokens_mask") is None
@pytest.mark.asyncio
async def test_chat_render_assistant_tokens_mask_null_without_gen_tags(
client,
):
"""The tiny test model lacks ``{% generation %}`` tags, so the mask is null."""
response = await client.post(
"/v1/chat/completions/render",
json={
"model": MODEL_NAME,
"messages": [
{"role": "user", "content": "Hello"},
{"role": "assistant", "content": "Hi!"},
],
"return_assistant_tokens_mask": True,
},
)
assert response.status_code == 200
assert response.json().get("assistant_tokens_mask") is None
# A minimal chat template with {% generation %} tags so we can test that
# the mask correctly marks assistant tokens.
_TEMPLATE_WITH_GENERATION = (
"{% for m in messages %}"
"{% if m['role'] == 'user' %}User: {{ m['content'] }}\n"
"{% elif m['role'] == 'assistant' %}"
"{% generation %}Assistant: {{ m['content'] }}\n{% endgeneration %}"
"{% endif %}"
"{% endfor %}"
)
@pytest.mark.asyncio
async def test_chat_completion_render_assistant_tokens_mask_with_generation_tags(
client,
):
"""With a ``{% generation %}``-enabled template, the mask marks assistant
tokens and the masked tokens decode to the assistant content."""
response = await client.post(
"/v1/chat/completions/render",
json={
"model": MODEL_NAME,
"messages": [
{"role": "user", "content": "Hello"},
{"role": "assistant", "content": "Hi!"},
{"role": "user", "content": "Bye"},
],
"chat_template": _TEMPLATE_WITH_GENERATION,
"return_assistant_tokens_mask": True,
},
)
assert response.status_code == 200
data = response.json()
mask = data["assistant_tokens_mask"]
token_ids = data["token_ids"]
assert mask is not None
assert isinstance(mask, list)
assert len(mask) == len(token_ids)
assert all(v in (0, 1) for v in mask)
assert sum(mask) > 0, "mask should mark at least one assistant token"
# Detokenize masked (assistant) and unmasked (non-assistant) tokens
# separately to verify the mask is correct, not just non-empty.
masked_ids = [t for t, m in zip(token_ids, mask, strict=True) if m]
unmasked_ids = [t for t, m in zip(token_ids, mask, strict=True) if not m]
detok = await client.post(
"/detokenize",
json={"model": MODEL_NAME, "tokens": masked_ids},
)
assert detok.status_code == 200
assert "Hi!" in detok.json()["prompt"]
detok_rest = await client.post(
"/detokenize",
json={"model": MODEL_NAME, "tokens": unmasked_ids},
)
assert detok_rest.status_code == 200
assert "Hi!" not in detok_rest.json()["prompt"]
assert "Bye" in detok_rest.json()["prompt"]
@@ -407,6 +407,18 @@ class ChatCompletionRequest(OpenAIBaseModel):
),
)
return_assistant_tokens_mask: bool = Field(
default=False,
description=(
"If true, the /render response will include an "
"``assistant_tokens_mask`` field — a per-token list of 0/1 "
"values indicating which tokens were assistant-generated. "
"Requires the chat template to use ``{% generation %}`` "
"tags. When the template does not support it, "
"``assistant_tokens_mask`` will be ``null``."
),
)
cache_salt: str | None = Field(
default=None,
description=(
@@ -520,6 +532,7 @@ class ChatCompletionRequest(OpenAIBaseModel):
extra_kwargs,
),
media_io_kwargs=self.media_io_kwargs,
return_assistant_tokens_mask=bool(self.return_assistant_tokens_mask),
)
def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:
@@ -75,6 +75,14 @@ class GenerateRequest(BaseModel):
token_ids: list[int] = Field(min_length=1)
"""The token ids to generate text from."""
assistant_tokens_mask: list[int] | None = None
"""Per-token mask (1 = assistant-generated, 0 = not).
Only populated when the render request sets ``return_assistant_tokens_mask=True``
and the chat template supports ``{% generation %}``.
``None`` when the mask was not requested or could not be computed.
"""
@field_validator("token_ids")
@classmethod
def validate_token_ids(cls, v: list[int]) -> list[int]:
+22
View File
@@ -127,11 +127,33 @@ class ServingRender(BaseServing):
)
params = request.to_sampling_params(max_tokens, self.default_sampling_params)
assistant_tokens_mask: list[int] | None = engine_input.get( # type: ignore[assignment]
"assistant_tokens_mask"
)
if assistant_tokens_mask is not None and len(assistant_tokens_mask) != len(
token_ids
):
logger.warning(
"assistant_tokens_mask length (%d) != token_ids length (%d); "
"this can happen with multimodal inputs where "
"placeholder expansion changes the token count. "
"The mask may be positionally misaligned.",
len(assistant_tokens_mask),
len(token_ids),
)
if len(assistant_tokens_mask) < len(token_ids):
assistant_tokens_mask.extend(
[0] * (len(token_ids) - len(assistant_tokens_mask))
)
else:
assistant_tokens_mask = assistant_tokens_mask[: len(token_ids)]
request_id = f"chatcmpl-{random_uuid()}"
return GenerateRequest(
request_id=request_id,
token_ids=token_ids,
assistant_tokens_mask=assistant_tokens_mask,
features=self._extract_mm_features(engine_input),
sampling_params=params,
model=request.model,
+10
View File
@@ -42,6 +42,11 @@ class TokensInput(_InputOptions):
"""Char-level (start, end) offsets per token, propagated from the
renderer's TokensPrompt when offsets were computed."""
assistant_tokens_mask: NotRequired[list[int] | None]
"""Per-token 0/1 mask marking assistant-generated tokens.
Populated when ``return_assistant_tokens_mask=True`` is set on the
render request and the chat template supports ``{% generation %}``."""
def tokens_input(
prompt_token_ids: list[int],
@@ -151,6 +156,11 @@ class MultiModalInput(_InputOptions):
`prompt_token_ids`.
"""
assistant_tokens_mask: NotRequired[list[int] | None]
"""Per-token 0/1 mask marking assistant-generated tokens.
Populated when ``return_assistant_tokens_mask=True`` is set on the
render request and the chat template supports ``{% generation %}``."""
def mm_input(
prompt_token_ids: list[int],
+106 -19
View File
@@ -7,7 +7,7 @@ import inspect
import itertools
import weakref
from collections import defaultdict, deque
from collections.abc import Sequence
from collections.abc import Mapping, Sequence
from functools import lru_cache
from typing import TYPE_CHECKING, Any, Final, Literal, cast, overload
@@ -678,6 +678,7 @@ def safe_apply_chat_template(
tools: list[dict[str, Any]] | None = ...,
chat_template: str | None = ...,
tokenize: Literal[True] = ...,
return_assistant_tokens_mask: Literal[False] = ...,
**kwargs,
) -> list[int]: ...
@overload
@@ -689,8 +690,20 @@ def safe_apply_chat_template(
tools: list[dict[str, Any]] | None = ...,
chat_template: str | None = ...,
tokenize: Literal[False] = ...,
return_assistant_tokens_mask: Literal[False] = ...,
**kwargs,
) -> str: ...
@overload
def safe_apply_chat_template(
model_config: ModelConfig,
tokenizer: HfTokenizer,
conversation: list[ConversationMessage],
*,
tools: list[dict[str, Any]] | None = ...,
chat_template: str | None = ...,
return_assistant_tokens_mask: Literal[True],
**kwargs,
) -> tuple[list[int], list[int] | None]: ...
def safe_apply_chat_template(
model_config: ModelConfig,
tokenizer: HfTokenizer,
@@ -699,8 +712,9 @@ def safe_apply_chat_template(
tools: list[dict[str, Any]] | None = None,
chat_template: str | None = None,
tokenize: bool = True,
return_assistant_tokens_mask: bool = False,
**kwargs,
) -> str | list[int]:
) -> str | list[int] | tuple[list[int], list[int] | None]:
chat_template = resolve_chat_template(
tokenizer,
chat_template=chat_template,
@@ -728,6 +742,38 @@ def safe_apply_chat_template(
chat_template_kwargs=kwargs,
)
# assistant_tokens_mask requires tokenized output — force tokenize=True.
if return_assistant_tokens_mask:
tokenize = True
# When return_assistant_tokens_mask is requested and the template supports it,
# request assistant_tokens_mask via return_dict.
# Check for the actual Jinja tag, not just the word "generation"
# (which also appears in add_generation_prompt).
if return_assistant_tokens_mask and "{% generation %}" in chat_template:
resolved_kwargs["return_assistant_tokens_mask"] = True
resolved_kwargs["return_dict"] = True
resolved_kwargs.pop("tokenize", None)
try:
result = tokenizer.apply_chat_template(
conversation=conversation, # type: ignore[arg-type]
tools=tools, # type: ignore[arg-type]
chat_template=chat_template,
tokenize=True,
**resolved_kwargs,
)
except (TypeError, ValueError) as exc:
logger.warning(
"apply_chat_template failed for assistant_tokens_mask: %s", exc
)
else:
if isinstance(result, Mapping):
token_ids = list(result.get("input_ids", []))
mask_raw = result.get("assistant_masks")
mask = list(mask_raw) if mask_raw is not None else None
return token_ids, mask
return list(result), None
# transformers v5 changed the default of `return_dict` to True, which
# makes `apply_chat_template(tokenize=True)` return a `BatchEncoding`
# instead of `list[int]`. Force `return_dict=False` so downstream code
@@ -737,23 +783,24 @@ def safe_apply_chat_template(
resolved_kwargs["return_dict"] = False
try:
return tokenizer.apply_chat_template(
plain = tokenizer.apply_chat_template(
conversation=conversation, # type: ignore[arg-type]
tools=tools, # type: ignore[arg-type]
chat_template=chat_template,
tokenize=tokenize,
**resolved_kwargs,
)
# External library exceptions can sometimes occur despite the framework's
# internal exception management capabilities.
except Exception as e:
# Log and report any library-related exceptions for further
# investigation.
logger.exception(
"An error occurred in `transformers` while applying chat template"
)
raise ValueError(str(e)) from e
if return_assistant_tokens_mask:
assert isinstance(plain, list), f"Expected list[int], got {type(plain)}"
return plain, None
return plain
def rebuild_mm_uuids_from_mm_data(
mm_uuids: MultiModalUUIDDict,
@@ -934,12 +981,22 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
logger.warning_once(_TOKENIZE_OVERRIDE_WARNING)
chat_template_kwargs["tokenize"] = True
prompt_raw = safe_apply_chat_template(
model_config,
tokenizer,
conversation,
**chat_template_kwargs,
)
assistant_tokens_mask: list[int] | None = None
if params.return_assistant_tokens_mask:
prompt_raw, assistant_tokens_mask = safe_apply_chat_template(
model_config,
tokenizer,
conversation,
return_assistant_tokens_mask=True,
**chat_template_kwargs,
)
else:
prompt_raw = safe_apply_chat_template(
model_config,
tokenizer,
conversation,
**chat_template_kwargs,
)
# NOTE: use_unified_vision_chunk is currently specific to Kimi-K2.5
# model which uses unified vision chunks for both images and videos.
@@ -965,6 +1022,9 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
prompt = parse_dec_only_prompt(prompt_raw)
if assistant_tokens_mask is not None:
cast(dict, prompt)["_assistant_tokens_mask"] = assistant_tokens_mask
# When `prompt_embeds` is mixed with other modality data,
# `_process_tokens` runs `_process_multimodal` first (expanding
# `<|AUDIO|>` / `<|IMAGE|>` placeholders) and then
@@ -1038,12 +1098,30 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
logger.warning_once(_TOKENIZE_OVERRIDE_WARNING)
chat_template_kwargs["tokenize"] = True
prompt_raw = await self._apply_chat_template_async(
model_config,
tokenizer,
conversation,
**chat_template_kwargs,
)
assistant_tokens_mask: list[int] | None = None
if params.return_assistant_tokens_mask:
result_with_mask = cast(
tuple[list[int], list[int] | None],
await make_async(
safe_apply_chat_template,
executor=self._executor,
)(
model_config,
tokenizer,
conversation,
return_assistant_tokens_mask=True, # type: ignore[arg-type]
**chat_template_kwargs,
),
)
prompt_raw: str | list[int] = result_with_mask[0]
assistant_tokens_mask = result_with_mask[1]
else:
prompt_raw = await self._apply_chat_template_async(
model_config,
tokenizer,
conversation,
**chat_template_kwargs,
)
# NOTE: use_unified_vision_chunk is currently specific to Kimi-K2.5
# model which uses unified vision chunks for both images and videos.
@@ -1067,6 +1145,9 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
prompt = parse_dec_only_prompt(prompt_raw)
if assistant_tokens_mask is not None:
cast(dict, prompt)["_assistant_tokens_mask"] = assistant_tokens_mask
# See `render_messages` for the rationale.
if prompt_embeds_tensors and mm_data:
assert prompt_embeds_placeholder_token_id is not None
@@ -1108,6 +1189,7 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
processor records all placeholder offsets in the final (post-expansion)
coordinate space, no offset shifting needed afterwards.
"""
assistant_tokens_mask = cast(dict, prompt).pop("_assistant_tokens_mask", None)
prompt_embeds_info = cast(dict, prompt).pop("_prompt_embeds", None)
if prompt_embeds_info is not None:
tensors, placeholder_token_id = prompt_embeds_info
@@ -1123,6 +1205,8 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
tensors,
mm_updates,
)
if assistant_tokens_mask is not None:
engine_input["assistant_tokens_mask"] = assistant_tokens_mask
return engine_input
@override
@@ -1133,6 +1217,7 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
skip_mm_cache: bool = False,
) -> TokensInput | MultiModalInput:
"""Async equivalent of `_process_tokens`."""
assistant_tokens_mask = cast(dict, prompt).pop("_assistant_tokens_mask", None)
prompt_embeds_info = cast(dict, prompt).pop("_prompt_embeds", None)
if prompt_embeds_info is not None:
tensors, placeholder_token_id = prompt_embeds_info
@@ -1150,6 +1235,8 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
tensors,
mm_updates,
)
if assistant_tokens_mask is not None:
engine_input["assistant_tokens_mask"] = assistant_tokens_mask
return engine_input
@staticmethod
+4
View File
@@ -87,6 +87,9 @@ class ChatParams:
mm_processor_kwargs: dict[str, Any] | None = None
"""The kwargs to pass to the multi-modal processor."""
return_assistant_tokens_mask: bool = False
"""Request a per-token assistant mask from apply_chat_template."""
def with_defaults(
self,
default_chat_template_kwargs: dict[str, Any] | None = None,
@@ -115,6 +118,7 @@ class ChatParams:
default_mm_processor_kwargs,
self.mm_processor_kwargs,
),
return_assistant_tokens_mask=self.return_assistant_tokens_mask,
)
def get_apply_chat_template_kwargs(self) -> dict[str, Any]: