[Render][Speculator] Add return_loss_mask to render endpoint for training data generation (#46846)
Signed-off-by: Ranran Haoran Zhang <ranzhang@redhat.com> Co-authored-by: Benjamin Chislett <chislett.ben@gmail.com>
This commit is contained in:
co-authored by
Benjamin Chislett
parent
a2a92cbbaa
commit
09841ae705
@@ -14,7 +14,7 @@ MODEL_NAME = "hmellor/tiny-random-LlamaForCausalLM"
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server():
|
||||
args: list[str] = []
|
||||
args: list[str] = ["--trust-request-chat-template"]
|
||||
|
||||
with RemoteLaunchRenderServer(MODEL_NAME, args) as remote_server:
|
||||
yield remote_server
|
||||
@@ -369,3 +369,127 @@ async def test_completion_render_multiple_prompts_token_offsets(client):
|
||||
assert len(offsets) == len(item["token_ids"])
|
||||
for start, end in offsets:
|
||||
assert 0 <= start <= end <= len(prompt)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chat_completion_render_assistant_tokens_mask_default(client):
|
||||
"""Without return_assistant_tokens_mask, assistant_tokens_mask should be null."""
|
||||
response = await client.post(
|
||||
"/v1/chat/completions/render",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"messages": [
|
||||
{"role": "user", "content": "Hello"},
|
||||
{"role": "assistant", "content": "Hi!"},
|
||||
{"role": "user", "content": "How are you?"},
|
||||
],
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data.get("assistant_tokens_mask") is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chat_completion_render_assistant_tokens_mask_false(client):
|
||||
"""Explicitly setting return_assistant_tokens_mask=false gives null."""
|
||||
response = await client.post(
|
||||
"/v1/chat/completions/render",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"messages": [
|
||||
{"role": "user", "content": "Hello"},
|
||||
],
|
||||
"return_assistant_tokens_mask": False,
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data.get("assistant_tokens_mask") is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chat_render_assistant_tokens_mask_null_without_gen_tags(
|
||||
client,
|
||||
):
|
||||
"""The tiny test model lacks ``{% generation %}`` tags, so the mask is null."""
|
||||
response = await client.post(
|
||||
"/v1/chat/completions/render",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"messages": [
|
||||
{"role": "user", "content": "Hello"},
|
||||
{"role": "assistant", "content": "Hi!"},
|
||||
],
|
||||
"return_assistant_tokens_mask": True,
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.json().get("assistant_tokens_mask") is None
|
||||
|
||||
|
||||
# A minimal chat template with {% generation %} tags so we can test that
|
||||
# the mask correctly marks assistant tokens.
|
||||
_TEMPLATE_WITH_GENERATION = (
|
||||
"{% for m in messages %}"
|
||||
"{% if m['role'] == 'user' %}User: {{ m['content'] }}\n"
|
||||
"{% elif m['role'] == 'assistant' %}"
|
||||
"{% generation %}Assistant: {{ m['content'] }}\n{% endgeneration %}"
|
||||
"{% endif %}"
|
||||
"{% endfor %}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chat_completion_render_assistant_tokens_mask_with_generation_tags(
|
||||
client,
|
||||
):
|
||||
"""With a ``{% generation %}``-enabled template, the mask marks assistant
|
||||
tokens and the masked tokens decode to the assistant content."""
|
||||
response = await client.post(
|
||||
"/v1/chat/completions/render",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"messages": [
|
||||
{"role": "user", "content": "Hello"},
|
||||
{"role": "assistant", "content": "Hi!"},
|
||||
{"role": "user", "content": "Bye"},
|
||||
],
|
||||
"chat_template": _TEMPLATE_WITH_GENERATION,
|
||||
"return_assistant_tokens_mask": True,
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
|
||||
mask = data["assistant_tokens_mask"]
|
||||
token_ids = data["token_ids"]
|
||||
assert mask is not None
|
||||
assert isinstance(mask, list)
|
||||
assert len(mask) == len(token_ids)
|
||||
assert all(v in (0, 1) for v in mask)
|
||||
assert sum(mask) > 0, "mask should mark at least one assistant token"
|
||||
|
||||
# Detokenize masked (assistant) and unmasked (non-assistant) tokens
|
||||
# separately to verify the mask is correct, not just non-empty.
|
||||
masked_ids = [t for t, m in zip(token_ids, mask, strict=True) if m]
|
||||
unmasked_ids = [t for t, m in zip(token_ids, mask, strict=True) if not m]
|
||||
|
||||
detok = await client.post(
|
||||
"/detokenize",
|
||||
json={"model": MODEL_NAME, "tokens": masked_ids},
|
||||
)
|
||||
assert detok.status_code == 200
|
||||
assert "Hi!" in detok.json()["prompt"]
|
||||
|
||||
detok_rest = await client.post(
|
||||
"/detokenize",
|
||||
json={"model": MODEL_NAME, "tokens": unmasked_ids},
|
||||
)
|
||||
assert detok_rest.status_code == 200
|
||||
assert "Hi!" not in detok_rest.json()["prompt"]
|
||||
assert "Bye" in detok_rest.json()["prompt"]
|
||||
|
||||
@@ -407,6 +407,18 @@ class ChatCompletionRequest(OpenAIBaseModel):
|
||||
),
|
||||
)
|
||||
|
||||
return_assistant_tokens_mask: bool = Field(
|
||||
default=False,
|
||||
description=(
|
||||
"If true, the /render response will include an "
|
||||
"``assistant_tokens_mask`` field — a per-token list of 0/1 "
|
||||
"values indicating which tokens were assistant-generated. "
|
||||
"Requires the chat template to use ``{% generation %}`` "
|
||||
"tags. When the template does not support it, "
|
||||
"``assistant_tokens_mask`` will be ``null``."
|
||||
),
|
||||
)
|
||||
|
||||
cache_salt: str | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
@@ -520,6 +532,7 @@ class ChatCompletionRequest(OpenAIBaseModel):
|
||||
extra_kwargs,
|
||||
),
|
||||
media_io_kwargs=self.media_io_kwargs,
|
||||
return_assistant_tokens_mask=bool(self.return_assistant_tokens_mask),
|
||||
)
|
||||
|
||||
def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:
|
||||
|
||||
@@ -75,6 +75,14 @@ class GenerateRequest(BaseModel):
|
||||
token_ids: list[int] = Field(min_length=1)
|
||||
"""The token ids to generate text from."""
|
||||
|
||||
assistant_tokens_mask: list[int] | None = None
|
||||
"""Per-token mask (1 = assistant-generated, 0 = not).
|
||||
|
||||
Only populated when the render request sets ``return_assistant_tokens_mask=True``
|
||||
and the chat template supports ``{% generation %}``.
|
||||
``None`` when the mask was not requested or could not be computed.
|
||||
"""
|
||||
|
||||
@field_validator("token_ids")
|
||||
@classmethod
|
||||
def validate_token_ids(cls, v: list[int]) -> list[int]:
|
||||
|
||||
@@ -127,11 +127,33 @@ class ServingRender(BaseServing):
|
||||
)
|
||||
params = request.to_sampling_params(max_tokens, self.default_sampling_params)
|
||||
|
||||
assistant_tokens_mask: list[int] | None = engine_input.get( # type: ignore[assignment]
|
||||
"assistant_tokens_mask"
|
||||
)
|
||||
if assistant_tokens_mask is not None and len(assistant_tokens_mask) != len(
|
||||
token_ids
|
||||
):
|
||||
logger.warning(
|
||||
"assistant_tokens_mask length (%d) != token_ids length (%d); "
|
||||
"this can happen with multimodal inputs where "
|
||||
"placeholder expansion changes the token count. "
|
||||
"The mask may be positionally misaligned.",
|
||||
len(assistant_tokens_mask),
|
||||
len(token_ids),
|
||||
)
|
||||
if len(assistant_tokens_mask) < len(token_ids):
|
||||
assistant_tokens_mask.extend(
|
||||
[0] * (len(token_ids) - len(assistant_tokens_mask))
|
||||
)
|
||||
else:
|
||||
assistant_tokens_mask = assistant_tokens_mask[: len(token_ids)]
|
||||
|
||||
request_id = f"chatcmpl-{random_uuid()}"
|
||||
|
||||
return GenerateRequest(
|
||||
request_id=request_id,
|
||||
token_ids=token_ids,
|
||||
assistant_tokens_mask=assistant_tokens_mask,
|
||||
features=self._extract_mm_features(engine_input),
|
||||
sampling_params=params,
|
||||
model=request.model,
|
||||
|
||||
@@ -42,6 +42,11 @@ class TokensInput(_InputOptions):
|
||||
"""Char-level (start, end) offsets per token, propagated from the
|
||||
renderer's TokensPrompt when offsets were computed."""
|
||||
|
||||
assistant_tokens_mask: NotRequired[list[int] | None]
|
||||
"""Per-token 0/1 mask marking assistant-generated tokens.
|
||||
Populated when ``return_assistant_tokens_mask=True`` is set on the
|
||||
render request and the chat template supports ``{% generation %}``."""
|
||||
|
||||
|
||||
def tokens_input(
|
||||
prompt_token_ids: list[int],
|
||||
@@ -151,6 +156,11 @@ class MultiModalInput(_InputOptions):
|
||||
`prompt_token_ids`.
|
||||
"""
|
||||
|
||||
assistant_tokens_mask: NotRequired[list[int] | None]
|
||||
"""Per-token 0/1 mask marking assistant-generated tokens.
|
||||
Populated when ``return_assistant_tokens_mask=True`` is set on the
|
||||
render request and the chat template supports ``{% generation %}``."""
|
||||
|
||||
|
||||
def mm_input(
|
||||
prompt_token_ids: list[int],
|
||||
|
||||
+106
-19
@@ -7,7 +7,7 @@ import inspect
|
||||
import itertools
|
||||
import weakref
|
||||
from collections import defaultdict, deque
|
||||
from collections.abc import Sequence
|
||||
from collections.abc import Mapping, Sequence
|
||||
from functools import lru_cache
|
||||
from typing import TYPE_CHECKING, Any, Final, Literal, cast, overload
|
||||
|
||||
@@ -678,6 +678,7 @@ def safe_apply_chat_template(
|
||||
tools: list[dict[str, Any]] | None = ...,
|
||||
chat_template: str | None = ...,
|
||||
tokenize: Literal[True] = ...,
|
||||
return_assistant_tokens_mask: Literal[False] = ...,
|
||||
**kwargs,
|
||||
) -> list[int]: ...
|
||||
@overload
|
||||
@@ -689,8 +690,20 @@ def safe_apply_chat_template(
|
||||
tools: list[dict[str, Any]] | None = ...,
|
||||
chat_template: str | None = ...,
|
||||
tokenize: Literal[False] = ...,
|
||||
return_assistant_tokens_mask: Literal[False] = ...,
|
||||
**kwargs,
|
||||
) -> str: ...
|
||||
@overload
|
||||
def safe_apply_chat_template(
|
||||
model_config: ModelConfig,
|
||||
tokenizer: HfTokenizer,
|
||||
conversation: list[ConversationMessage],
|
||||
*,
|
||||
tools: list[dict[str, Any]] | None = ...,
|
||||
chat_template: str | None = ...,
|
||||
return_assistant_tokens_mask: Literal[True],
|
||||
**kwargs,
|
||||
) -> tuple[list[int], list[int] | None]: ...
|
||||
def safe_apply_chat_template(
|
||||
model_config: ModelConfig,
|
||||
tokenizer: HfTokenizer,
|
||||
@@ -699,8 +712,9 @@ def safe_apply_chat_template(
|
||||
tools: list[dict[str, Any]] | None = None,
|
||||
chat_template: str | None = None,
|
||||
tokenize: bool = True,
|
||||
return_assistant_tokens_mask: bool = False,
|
||||
**kwargs,
|
||||
) -> str | list[int]:
|
||||
) -> str | list[int] | tuple[list[int], list[int] | None]:
|
||||
chat_template = resolve_chat_template(
|
||||
tokenizer,
|
||||
chat_template=chat_template,
|
||||
@@ -728,6 +742,38 @@ def safe_apply_chat_template(
|
||||
chat_template_kwargs=kwargs,
|
||||
)
|
||||
|
||||
# assistant_tokens_mask requires tokenized output — force tokenize=True.
|
||||
if return_assistant_tokens_mask:
|
||||
tokenize = True
|
||||
|
||||
# When return_assistant_tokens_mask is requested and the template supports it,
|
||||
# request assistant_tokens_mask via return_dict.
|
||||
# Check for the actual Jinja tag, not just the word "generation"
|
||||
# (which also appears in add_generation_prompt).
|
||||
if return_assistant_tokens_mask and "{% generation %}" in chat_template:
|
||||
resolved_kwargs["return_assistant_tokens_mask"] = True
|
||||
resolved_kwargs["return_dict"] = True
|
||||
resolved_kwargs.pop("tokenize", None)
|
||||
try:
|
||||
result = tokenizer.apply_chat_template(
|
||||
conversation=conversation, # type: ignore[arg-type]
|
||||
tools=tools, # type: ignore[arg-type]
|
||||
chat_template=chat_template,
|
||||
tokenize=True,
|
||||
**resolved_kwargs,
|
||||
)
|
||||
except (TypeError, ValueError) as exc:
|
||||
logger.warning(
|
||||
"apply_chat_template failed for assistant_tokens_mask: %s", exc
|
||||
)
|
||||
else:
|
||||
if isinstance(result, Mapping):
|
||||
token_ids = list(result.get("input_ids", []))
|
||||
mask_raw = result.get("assistant_masks")
|
||||
mask = list(mask_raw) if mask_raw is not None else None
|
||||
return token_ids, mask
|
||||
return list(result), None
|
||||
|
||||
# transformers v5 changed the default of `return_dict` to True, which
|
||||
# makes `apply_chat_template(tokenize=True)` return a `BatchEncoding`
|
||||
# instead of `list[int]`. Force `return_dict=False` so downstream code
|
||||
@@ -737,23 +783,24 @@ def safe_apply_chat_template(
|
||||
resolved_kwargs["return_dict"] = False
|
||||
|
||||
try:
|
||||
return tokenizer.apply_chat_template(
|
||||
plain = tokenizer.apply_chat_template(
|
||||
conversation=conversation, # type: ignore[arg-type]
|
||||
tools=tools, # type: ignore[arg-type]
|
||||
chat_template=chat_template,
|
||||
tokenize=tokenize,
|
||||
**resolved_kwargs,
|
||||
)
|
||||
# External library exceptions can sometimes occur despite the framework's
|
||||
# internal exception management capabilities.
|
||||
except Exception as e:
|
||||
# Log and report any library-related exceptions for further
|
||||
# investigation.
|
||||
logger.exception(
|
||||
"An error occurred in `transformers` while applying chat template"
|
||||
)
|
||||
raise ValueError(str(e)) from e
|
||||
|
||||
if return_assistant_tokens_mask:
|
||||
assert isinstance(plain, list), f"Expected list[int], got {type(plain)}"
|
||||
return plain, None
|
||||
return plain
|
||||
|
||||
|
||||
def rebuild_mm_uuids_from_mm_data(
|
||||
mm_uuids: MultiModalUUIDDict,
|
||||
@@ -934,12 +981,22 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
|
||||
logger.warning_once(_TOKENIZE_OVERRIDE_WARNING)
|
||||
chat_template_kwargs["tokenize"] = True
|
||||
|
||||
prompt_raw = safe_apply_chat_template(
|
||||
model_config,
|
||||
tokenizer,
|
||||
conversation,
|
||||
**chat_template_kwargs,
|
||||
)
|
||||
assistant_tokens_mask: list[int] | None = None
|
||||
if params.return_assistant_tokens_mask:
|
||||
prompt_raw, assistant_tokens_mask = safe_apply_chat_template(
|
||||
model_config,
|
||||
tokenizer,
|
||||
conversation,
|
||||
return_assistant_tokens_mask=True,
|
||||
**chat_template_kwargs,
|
||||
)
|
||||
else:
|
||||
prompt_raw = safe_apply_chat_template(
|
||||
model_config,
|
||||
tokenizer,
|
||||
conversation,
|
||||
**chat_template_kwargs,
|
||||
)
|
||||
|
||||
# NOTE: use_unified_vision_chunk is currently specific to Kimi-K2.5
|
||||
# model which uses unified vision chunks for both images and videos.
|
||||
@@ -965,6 +1022,9 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
|
||||
|
||||
prompt = parse_dec_only_prompt(prompt_raw)
|
||||
|
||||
if assistant_tokens_mask is not None:
|
||||
cast(dict, prompt)["_assistant_tokens_mask"] = assistant_tokens_mask
|
||||
|
||||
# When `prompt_embeds` is mixed with other modality data,
|
||||
# `_process_tokens` runs `_process_multimodal` first (expanding
|
||||
# `<|AUDIO|>` / `<|IMAGE|>` placeholders) and then
|
||||
@@ -1038,12 +1098,30 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
|
||||
logger.warning_once(_TOKENIZE_OVERRIDE_WARNING)
|
||||
chat_template_kwargs["tokenize"] = True
|
||||
|
||||
prompt_raw = await self._apply_chat_template_async(
|
||||
model_config,
|
||||
tokenizer,
|
||||
conversation,
|
||||
**chat_template_kwargs,
|
||||
)
|
||||
assistant_tokens_mask: list[int] | None = None
|
||||
if params.return_assistant_tokens_mask:
|
||||
result_with_mask = cast(
|
||||
tuple[list[int], list[int] | None],
|
||||
await make_async(
|
||||
safe_apply_chat_template,
|
||||
executor=self._executor,
|
||||
)(
|
||||
model_config,
|
||||
tokenizer,
|
||||
conversation,
|
||||
return_assistant_tokens_mask=True, # type: ignore[arg-type]
|
||||
**chat_template_kwargs,
|
||||
),
|
||||
)
|
||||
prompt_raw: str | list[int] = result_with_mask[0]
|
||||
assistant_tokens_mask = result_with_mask[1]
|
||||
else:
|
||||
prompt_raw = await self._apply_chat_template_async(
|
||||
model_config,
|
||||
tokenizer,
|
||||
conversation,
|
||||
**chat_template_kwargs,
|
||||
)
|
||||
|
||||
# NOTE: use_unified_vision_chunk is currently specific to Kimi-K2.5
|
||||
# model which uses unified vision chunks for both images and videos.
|
||||
@@ -1067,6 +1145,9 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
|
||||
|
||||
prompt = parse_dec_only_prompt(prompt_raw)
|
||||
|
||||
if assistant_tokens_mask is not None:
|
||||
cast(dict, prompt)["_assistant_tokens_mask"] = assistant_tokens_mask
|
||||
|
||||
# See `render_messages` for the rationale.
|
||||
if prompt_embeds_tensors and mm_data:
|
||||
assert prompt_embeds_placeholder_token_id is not None
|
||||
@@ -1108,6 +1189,7 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
|
||||
processor records all placeholder offsets in the final (post-expansion)
|
||||
coordinate space, no offset shifting needed afterwards.
|
||||
"""
|
||||
assistant_tokens_mask = cast(dict, prompt).pop("_assistant_tokens_mask", None)
|
||||
prompt_embeds_info = cast(dict, prompt).pop("_prompt_embeds", None)
|
||||
if prompt_embeds_info is not None:
|
||||
tensors, placeholder_token_id = prompt_embeds_info
|
||||
@@ -1123,6 +1205,8 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
|
||||
tensors,
|
||||
mm_updates,
|
||||
)
|
||||
if assistant_tokens_mask is not None:
|
||||
engine_input["assistant_tokens_mask"] = assistant_tokens_mask
|
||||
return engine_input
|
||||
|
||||
@override
|
||||
@@ -1133,6 +1217,7 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
|
||||
skip_mm_cache: bool = False,
|
||||
) -> TokensInput | MultiModalInput:
|
||||
"""Async equivalent of `_process_tokens`."""
|
||||
assistant_tokens_mask = cast(dict, prompt).pop("_assistant_tokens_mask", None)
|
||||
prompt_embeds_info = cast(dict, prompt).pop("_prompt_embeds", None)
|
||||
if prompt_embeds_info is not None:
|
||||
tensors, placeholder_token_id = prompt_embeds_info
|
||||
@@ -1150,6 +1235,8 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
|
||||
tensors,
|
||||
mm_updates,
|
||||
)
|
||||
if assistant_tokens_mask is not None:
|
||||
engine_input["assistant_tokens_mask"] = assistant_tokens_mask
|
||||
return engine_input
|
||||
|
||||
@staticmethod
|
||||
|
||||
@@ -87,6 +87,9 @@ class ChatParams:
|
||||
mm_processor_kwargs: dict[str, Any] | None = None
|
||||
"""The kwargs to pass to the multi-modal processor."""
|
||||
|
||||
return_assistant_tokens_mask: bool = False
|
||||
"""Request a per-token assistant mask from apply_chat_template."""
|
||||
|
||||
def with_defaults(
|
||||
self,
|
||||
default_chat_template_kwargs: dict[str, Any] | None = None,
|
||||
@@ -115,6 +118,7 @@ class ChatParams:
|
||||
default_mm_processor_kwargs,
|
||||
self.mm_processor_kwargs,
|
||||
),
|
||||
return_assistant_tokens_mask=self.return_assistant_tokens_mask,
|
||||
)
|
||||
|
||||
def get_apply_chat_template_kwargs(self) -> dict[str, Any]:
|
||||
|
||||
Reference in New Issue
Block a user