[Render][Speculator] Add return_loss_mask to render endpoint for training data generation (#46846)
Signed-off-by: Ranran Haoran Zhang <ranzhang@redhat.com> Co-authored-by: Benjamin Chislett <chislett.ben@gmail.com>
This commit is contained in:
co-authored by
Benjamin Chislett
parent
a2a92cbbaa
commit
09841ae705
@@ -407,6 +407,18 @@ class ChatCompletionRequest(OpenAIBaseModel):
|
||||
),
|
||||
)
|
||||
|
||||
return_assistant_tokens_mask: bool = Field(
|
||||
default=False,
|
||||
description=(
|
||||
"If true, the /render response will include an "
|
||||
"``assistant_tokens_mask`` field — a per-token list of 0/1 "
|
||||
"values indicating which tokens were assistant-generated. "
|
||||
"Requires the chat template to use ``{% generation %}`` "
|
||||
"tags. When the template does not support it, "
|
||||
"``assistant_tokens_mask`` will be ``null``."
|
||||
),
|
||||
)
|
||||
|
||||
cache_salt: str | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
@@ -520,6 +532,7 @@ class ChatCompletionRequest(OpenAIBaseModel):
|
||||
extra_kwargs,
|
||||
),
|
||||
media_io_kwargs=self.media_io_kwargs,
|
||||
return_assistant_tokens_mask=bool(self.return_assistant_tokens_mask),
|
||||
)
|
||||
|
||||
def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:
|
||||
|
||||
@@ -75,6 +75,14 @@ class GenerateRequest(BaseModel):
|
||||
token_ids: list[int] = Field(min_length=1)
|
||||
"""The token ids to generate text from."""
|
||||
|
||||
assistant_tokens_mask: list[int] | None = None
|
||||
"""Per-token mask (1 = assistant-generated, 0 = not).
|
||||
|
||||
Only populated when the render request sets ``return_assistant_tokens_mask=True``
|
||||
and the chat template supports ``{% generation %}``.
|
||||
``None`` when the mask was not requested or could not be computed.
|
||||
"""
|
||||
|
||||
@field_validator("token_ids")
|
||||
@classmethod
|
||||
def validate_token_ids(cls, v: list[int]) -> list[int]:
|
||||
|
||||
@@ -127,11 +127,33 @@ class ServingRender(BaseServing):
|
||||
)
|
||||
params = request.to_sampling_params(max_tokens, self.default_sampling_params)
|
||||
|
||||
assistant_tokens_mask: list[int] | None = engine_input.get( # type: ignore[assignment]
|
||||
"assistant_tokens_mask"
|
||||
)
|
||||
if assistant_tokens_mask is not None and len(assistant_tokens_mask) != len(
|
||||
token_ids
|
||||
):
|
||||
logger.warning(
|
||||
"assistant_tokens_mask length (%d) != token_ids length (%d); "
|
||||
"this can happen with multimodal inputs where "
|
||||
"placeholder expansion changes the token count. "
|
||||
"The mask may be positionally misaligned.",
|
||||
len(assistant_tokens_mask),
|
||||
len(token_ids),
|
||||
)
|
||||
if len(assistant_tokens_mask) < len(token_ids):
|
||||
assistant_tokens_mask.extend(
|
||||
[0] * (len(token_ids) - len(assistant_tokens_mask))
|
||||
)
|
||||
else:
|
||||
assistant_tokens_mask = assistant_tokens_mask[: len(token_ids)]
|
||||
|
||||
request_id = f"chatcmpl-{random_uuid()}"
|
||||
|
||||
return GenerateRequest(
|
||||
request_id=request_id,
|
||||
token_ids=token_ids,
|
||||
assistant_tokens_mask=assistant_tokens_mask,
|
||||
features=self._extract_mm_features(engine_input),
|
||||
sampling_params=params,
|
||||
model=request.model,
|
||||
|
||||
Reference in New Issue
Block a user