[Render][Speculator] Add return_loss_mask to render endpoint for training data generation (#46846)

Signed-off-by: Ranran Haoran Zhang <ranzhang@redhat.com>
Co-authored-by: Benjamin Chislett <chislett.ben@gmail.com>
This commit is contained in:
Ranran
2026-06-28 00:07:33 -07:00
committed by GitHub
co-authored by Benjamin Chislett
parent a2a92cbbaa
commit 09841ae705
7 changed files with 288 additions and 20 deletions
@@ -407,6 +407,18 @@ class ChatCompletionRequest(OpenAIBaseModel):
),
)
return_assistant_tokens_mask: bool = Field(
default=False,
description=(
"If true, the /render response will include an "
"``assistant_tokens_mask`` field — a per-token list of 0/1 "
"values indicating which tokens were assistant-generated. "
"Requires the chat template to use ``{% generation %}`` "
"tags. When the template does not support it, "
"``assistant_tokens_mask`` will be ``null``."
),
)
cache_salt: str | None = Field(
default=None,
description=(
@@ -520,6 +532,7 @@ class ChatCompletionRequest(OpenAIBaseModel):
extra_kwargs,
),
media_io_kwargs=self.media_io_kwargs,
return_assistant_tokens_mask=bool(self.return_assistant_tokens_mask),
)
def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:
@@ -75,6 +75,14 @@ class GenerateRequest(BaseModel):
token_ids: list[int] = Field(min_length=1)
"""The token ids to generate text from."""
assistant_tokens_mask: list[int] | None = None
"""Per-token mask (1 = assistant-generated, 0 = not).
Only populated when the render request sets ``return_assistant_tokens_mask=True``
and the chat template supports ``{% generation %}``.
``None`` when the mask was not requested or could not be computed.
"""
@field_validator("token_ids")
@classmethod
def validate_token_ids(cls, v: list[int]) -> list[int]:
+22
View File
@@ -127,11 +127,33 @@ class ServingRender(BaseServing):
)
params = request.to_sampling_params(max_tokens, self.default_sampling_params)
assistant_tokens_mask: list[int] | None = engine_input.get( # type: ignore[assignment]
"assistant_tokens_mask"
)
if assistant_tokens_mask is not None and len(assistant_tokens_mask) != len(
token_ids
):
logger.warning(
"assistant_tokens_mask length (%d) != token_ids length (%d); "
"this can happen with multimodal inputs where "
"placeholder expansion changes the token count. "
"The mask may be positionally misaligned.",
len(assistant_tokens_mask),
len(token_ids),
)
if len(assistant_tokens_mask) < len(token_ids):
assistant_tokens_mask.extend(
[0] * (len(token_ids) - len(assistant_tokens_mask))
)
else:
assistant_tokens_mask = assistant_tokens_mask[: len(token_ids)]
request_id = f"chatcmpl-{random_uuid()}"
return GenerateRequest(
request_id=request_id,
token_ids=token_ids,
assistant_tokens_mask=assistant_tokens_mask,
features=self._extract_mm_features(engine_input),
sampling_params=params,
model=request.model,