[Deprecation] Remove assistant_token_mask support (#57520)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -594,176 +594,6 @@ async def test_completion_render_multiple_prompts_token_offsets(client):
|
||||
assert 0 <= start <= end <= len(prompt)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chat_completion_render_assistant_tokens_mask_default(client):
|
||||
"""Without return_assistant_tokens_mask, assistant_tokens_mask should be null."""
|
||||
response = await client.post(
|
||||
"/v1/chat/completions/render",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"messages": [
|
||||
{"role": "user", "content": "Hello"},
|
||||
{"role": "assistant", "content": "Hi!"},
|
||||
{"role": "user", "content": "How are you?"},
|
||||
],
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data.get("assistant_tokens_mask") is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chat_completion_render_assistant_tokens_mask_false(client):
|
||||
"""Explicitly setting return_assistant_tokens_mask=false gives null."""
|
||||
response = await client.post(
|
||||
"/v1/chat/completions/render",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"messages": [
|
||||
{"role": "user", "content": "Hello"},
|
||||
],
|
||||
"return_assistant_tokens_mask": False,
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data.get("assistant_tokens_mask") is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chat_render_assistant_tokens_mask_null_without_gen_tags(
|
||||
client,
|
||||
):
|
||||
"""The tiny test model lacks ``{% generation %}`` tags, so the mask is null."""
|
||||
response = await client.post(
|
||||
"/v1/chat/completions/render",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"messages": [
|
||||
{"role": "user", "content": "Hello"},
|
||||
{"role": "assistant", "content": "Hi!"},
|
||||
],
|
||||
"return_assistant_tokens_mask": True,
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.json().get("assistant_tokens_mask") is None
|
||||
|
||||
|
||||
# A minimal chat template with {% generation %} tags so we can test that
|
||||
# the mask correctly marks assistant tokens.
|
||||
_TEMPLATE_WITH_GENERATION = (
|
||||
"{% for m in messages %}"
|
||||
"{% if m['role'] == 'user' %}User: {{ m['content'] }}\n"
|
||||
"{% elif m['role'] == 'assistant' %}"
|
||||
"{% generation %}Assistant: {{ m['content'] }}\n{% endgeneration %}"
|
||||
"{% endif %}"
|
||||
"{% endfor %}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chat_completion_render_assistant_tokens_mask_with_generation_tags(
|
||||
client,
|
||||
):
|
||||
"""With a ``{% generation %}``-enabled template, the mask marks assistant
|
||||
tokens and the masked tokens decode to the assistant content."""
|
||||
response = await client.post(
|
||||
"/v1/chat/completions/render",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"messages": [
|
||||
{"role": "user", "content": "Hello"},
|
||||
{"role": "assistant", "content": "Hi!"},
|
||||
{"role": "user", "content": "Bye"},
|
||||
],
|
||||
"chat_template": _TEMPLATE_WITH_GENERATION,
|
||||
"return_assistant_tokens_mask": True,
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
|
||||
mask = data["assistant_tokens_mask"]
|
||||
token_ids = data["token_ids"]
|
||||
assert mask is not None
|
||||
assert isinstance(mask, list)
|
||||
assert len(mask) == len(token_ids)
|
||||
assert all(v in (0, 1) for v in mask)
|
||||
assert sum(mask) > 0, "mask should mark at least one assistant token"
|
||||
|
||||
# Detokenize masked (assistant) and unmasked (non-assistant) tokens
|
||||
# separately to verify the mask is correct, not just non-empty.
|
||||
masked_ids = [t for t, m in zip(token_ids, mask, strict=True) if m]
|
||||
unmasked_ids = [t for t, m in zip(token_ids, mask, strict=True) if not m]
|
||||
|
||||
detok = await client.post(
|
||||
"/detokenize",
|
||||
json={"model": MODEL_NAME, "tokens": masked_ids},
|
||||
)
|
||||
assert detok.status_code == 200
|
||||
assert "Hi!" in detok.json()["prompt"]
|
||||
|
||||
detok_rest = await client.post(
|
||||
"/detokenize",
|
||||
json={"model": MODEL_NAME, "tokens": unmasked_ids},
|
||||
)
|
||||
assert detok_rest.status_code == 200
|
||||
assert "Hi!" not in detok_rest.json()["prompt"]
|
||||
assert "Bye" in detok_rest.json()["prompt"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chat_render_assistant_tokens_mask_follows_truncation(client):
|
||||
"""The assistant mask must be truncated with the prompt it describes.
|
||||
|
||||
`assistant_tokens_mask` is positional: entry i labels token i. Truncating
|
||||
`token_ids` from the left without truncating the mask leaves the two
|
||||
describing different positions, and the mask ends up marking whichever
|
||||
tokens happen to sit at the old offsets.
|
||||
"""
|
||||
messages = [
|
||||
# Deliberately lopsided: a long leading user turn and a short trailing
|
||||
# one, so keeping the head of the mask is distinguishable from keeping
|
||||
# its tail.
|
||||
{"role": "user", "content": "Hello hello hello hello hello hello"},
|
||||
{"role": "assistant", "content": "Hi!"},
|
||||
{"role": "user", "content": "Bye"},
|
||||
]
|
||||
body = {
|
||||
"model": MODEL_NAME,
|
||||
"messages": messages,
|
||||
"chat_template": _TEMPLATE_WITH_GENERATION,
|
||||
"return_assistant_tokens_mask": True,
|
||||
}
|
||||
|
||||
full = await client.post("/v1/chat/completions/render", json=body)
|
||||
assert full.status_code == 200
|
||||
full_token_ids = full.json()["token_ids"]
|
||||
full_mask = full.json()["assistant_tokens_mask"]
|
||||
assert sum(full_mask) > 0
|
||||
|
||||
keep = len(full_token_ids) - 4
|
||||
# Precondition: with this prompt the head and tail slices of the mask
|
||||
# really do differ, so the assertion below can tell them apart.
|
||||
assert full_mask[-keep:] != full_mask[:keep]
|
||||
|
||||
truncated = await client.post(
|
||||
"/v1/chat/completions/render",
|
||||
json={**body, "truncate_prompt_tokens": keep, "truncation_side": "left"},
|
||||
)
|
||||
assert truncated.status_code == 200
|
||||
data = truncated.json()
|
||||
|
||||
assert data["token_ids"] == full_token_ids[-keep:]
|
||||
assert data["assistant_tokens_mask"] == full_mask[-keep:]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_messages_render_basic(client):
|
||||
"""Test basic Anthropic Messages render endpoint."""
|
||||
|
||||
@@ -453,18 +453,6 @@ class ChatCompletionRequest(OpenAIBaseModel):
|
||||
),
|
||||
)
|
||||
|
||||
return_assistant_tokens_mask: bool = Field(
|
||||
default=False,
|
||||
description=(
|
||||
"If true, the /render response will include an "
|
||||
"``assistant_tokens_mask`` field — a per-token list of 0/1 "
|
||||
"values indicating which tokens were assistant-generated. "
|
||||
"Requires the chat template to use ``{% generation %}`` "
|
||||
"tags. When the template does not support it, "
|
||||
"``assistant_tokens_mask`` will be ``null``."
|
||||
),
|
||||
)
|
||||
|
||||
cache_salt: str | None = Field(
|
||||
default=None,
|
||||
min_length=1,
|
||||
@@ -605,7 +593,6 @@ class ChatCompletionRequest(OpenAIBaseModel):
|
||||
extra_kwargs,
|
||||
),
|
||||
media_io_kwargs=self.media_io_kwargs,
|
||||
return_assistant_tokens_mask=bool(self.return_assistant_tokens_mask),
|
||||
# No-tools requests default to tool_choice="none" at the API
|
||||
# layer. Collapse that default before rendering, so K3 emits a
|
||||
# model-visible tool-choice instruction only for requests with a
|
||||
|
||||
@@ -126,33 +126,11 @@ class ServingRender(BaseServing):
|
||||
)
|
||||
params = request.to_sampling_params(max_tokens, self.default_sampling_params)
|
||||
|
||||
assistant_tokens_mask: list[int] | None = engine_input.get( # type: ignore[assignment]
|
||||
"assistant_tokens_mask"
|
||||
)
|
||||
if assistant_tokens_mask is not None and len(assistant_tokens_mask) != len(
|
||||
token_ids
|
||||
):
|
||||
logger.warning(
|
||||
"assistant_tokens_mask length (%d) != token_ids length (%d); "
|
||||
"this can happen with multimodal inputs where "
|
||||
"placeholder expansion changes the token count. "
|
||||
"The mask may be positionally misaligned.",
|
||||
len(assistant_tokens_mask),
|
||||
len(token_ids),
|
||||
)
|
||||
if len(assistant_tokens_mask) < len(token_ids):
|
||||
assistant_tokens_mask.extend(
|
||||
[0] * (len(token_ids) - len(assistant_tokens_mask))
|
||||
)
|
||||
else:
|
||||
assistant_tokens_mask = assistant_tokens_mask[: len(token_ids)]
|
||||
|
||||
request_id = f"chatcmpl-{random_uuid()}"
|
||||
|
||||
return GenerateRequest(
|
||||
request_id=request_id,
|
||||
token_ids=token_ids,
|
||||
assistant_tokens_mask=assistant_tokens_mask,
|
||||
features=self._extract_mm_features(engine_input),
|
||||
sampling_params=params,
|
||||
model=request.model,
|
||||
|
||||
@@ -150,14 +150,6 @@ class GenerateRequest(BaseModel):
|
||||
token_ids: list[int] = Field(min_length=1)
|
||||
"""The token ids to generate text from."""
|
||||
|
||||
assistant_tokens_mask: list[int] | None = None
|
||||
"""Per-token mask (1 = assistant-generated, 0 = not).
|
||||
|
||||
Only populated when the render request sets ``return_assistant_tokens_mask=True``
|
||||
and the chat template supports ``{% generation %}``.
|
||||
``None`` when the mask was not requested or could not be computed.
|
||||
"""
|
||||
|
||||
@field_validator("token_ids")
|
||||
@classmethod
|
||||
def validate_token_ids(cls, v: list[int]) -> list[int]:
|
||||
|
||||
@@ -43,11 +43,6 @@ class TokensInput(_InputOptions):
|
||||
"""Char-level (start, end) offsets per token, propagated from the
|
||||
renderer's TokensPrompt when offsets were computed."""
|
||||
|
||||
assistant_tokens_mask: NotRequired[list[int] | None]
|
||||
"""Per-token 0/1 mask marking assistant-generated tokens.
|
||||
Populated when ``return_assistant_tokens_mask=True`` is set on the
|
||||
render request and the chat template supports ``{% generation %}``."""
|
||||
|
||||
|
||||
def tokens_input(
|
||||
prompt_token_ids: list[int],
|
||||
@@ -155,11 +150,6 @@ class MultiModalInput(_InputOptions):
|
||||
`prompt_token_ids`.
|
||||
"""
|
||||
|
||||
assistant_tokens_mask: NotRequired[list[int] | None]
|
||||
"""Per-token 0/1 mask marking assistant-generated tokens.
|
||||
Populated when ``return_assistant_tokens_mask=True`` is set on the
|
||||
render request and the chat template supports ``{% generation %}``."""
|
||||
|
||||
|
||||
def mm_input(
|
||||
prompt_token_ids: list[int],
|
||||
|
||||
+20
-103
@@ -7,7 +7,7 @@ import inspect
|
||||
import itertools
|
||||
import weakref
|
||||
from collections import defaultdict, deque
|
||||
from collections.abc import Mapping, Sequence
|
||||
from collections.abc import Sequence
|
||||
from functools import lru_cache
|
||||
from typing import TYPE_CHECKING, Any, Final, Literal, cast, overload
|
||||
|
||||
@@ -758,7 +758,6 @@ def safe_apply_chat_template(
|
||||
tools: list[dict[str, Any]] | None = ...,
|
||||
chat_template: str | None = ...,
|
||||
tokenize: Literal[True] = ...,
|
||||
return_assistant_tokens_mask: Literal[False] = ...,
|
||||
**kwargs,
|
||||
) -> list[int]: ...
|
||||
@overload
|
||||
@@ -770,20 +769,8 @@ def safe_apply_chat_template(
|
||||
tools: list[dict[str, Any]] | None = ...,
|
||||
chat_template: str | None = ...,
|
||||
tokenize: Literal[False] = ...,
|
||||
return_assistant_tokens_mask: Literal[False] = ...,
|
||||
**kwargs,
|
||||
) -> str: ...
|
||||
@overload
|
||||
def safe_apply_chat_template(
|
||||
model_config: ModelConfig,
|
||||
tokenizer: HfTokenizer,
|
||||
conversation: list[ConversationMessage],
|
||||
*,
|
||||
tools: list[dict[str, Any]] | None = ...,
|
||||
chat_template: str | None = ...,
|
||||
return_assistant_tokens_mask: Literal[True],
|
||||
**kwargs,
|
||||
) -> tuple[list[int], list[int] | None]: ...
|
||||
def safe_apply_chat_template(
|
||||
model_config: ModelConfig,
|
||||
tokenizer: HfTokenizer,
|
||||
@@ -792,9 +779,8 @@ def safe_apply_chat_template(
|
||||
tools: list[dict[str, Any]] | None = None,
|
||||
chat_template: str | None = None,
|
||||
tokenize: bool = True,
|
||||
return_assistant_tokens_mask: bool = False,
|
||||
**kwargs,
|
||||
) -> str | list[int] | tuple[list[int], list[int] | None]:
|
||||
) -> str | list[int]:
|
||||
chat_template = resolve_chat_template(
|
||||
tokenizer,
|
||||
chat_template=chat_template,
|
||||
@@ -822,38 +808,6 @@ def safe_apply_chat_template(
|
||||
chat_template_kwargs=kwargs,
|
||||
)
|
||||
|
||||
# assistant_tokens_mask requires tokenized output — force tokenize=True.
|
||||
if return_assistant_tokens_mask:
|
||||
tokenize = True
|
||||
|
||||
# When return_assistant_tokens_mask is requested and the template supports it,
|
||||
# request assistant_tokens_mask via return_dict.
|
||||
# Check for the actual Jinja tag, not just the word "generation"
|
||||
# (which also appears in add_generation_prompt).
|
||||
if return_assistant_tokens_mask and "{% generation %}" in chat_template:
|
||||
resolved_kwargs["return_assistant_tokens_mask"] = True
|
||||
resolved_kwargs["return_dict"] = True
|
||||
resolved_kwargs.pop("tokenize", None)
|
||||
try:
|
||||
result = tokenizer.apply_chat_template(
|
||||
conversation=conversation, # type: ignore[arg-type]
|
||||
tools=tools, # type: ignore[arg-type]
|
||||
chat_template=chat_template,
|
||||
tokenize=True,
|
||||
**resolved_kwargs,
|
||||
)
|
||||
except (TypeError, ValueError) as exc:
|
||||
logger.warning(
|
||||
"apply_chat_template failed for assistant_tokens_mask: %s", exc
|
||||
)
|
||||
else:
|
||||
if isinstance(result, Mapping):
|
||||
token_ids = list(result.get("input_ids", []))
|
||||
mask_raw = result.get("assistant_masks")
|
||||
mask = list(mask_raw) if mask_raw is not None else None
|
||||
return token_ids, mask
|
||||
return list(result), None
|
||||
|
||||
# transformers v5 changed the default of `return_dict` to True, which
|
||||
# makes `apply_chat_template(tokenize=True)` return a `BatchEncoding`
|
||||
# instead of `list[int]`. Force `return_dict=False` so downstream code
|
||||
@@ -879,9 +833,6 @@ def safe_apply_chat_template(
|
||||
logger.warning("Chat template rejected the request: %s", e)
|
||||
raise VLLMValidationError(_template_error_reason(e)) from e
|
||||
|
||||
if return_assistant_tokens_mask:
|
||||
assert isinstance(plain, list), f"Expected list[int], got {type(plain)}"
|
||||
return plain, None
|
||||
return plain
|
||||
|
||||
|
||||
@@ -1066,22 +1017,12 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
|
||||
logger.warning_once(_TOKENIZE_OVERRIDE_WARNING)
|
||||
chat_template_kwargs["tokenize"] = True
|
||||
|
||||
assistant_tokens_mask: list[int] | None = None
|
||||
if params.return_assistant_tokens_mask:
|
||||
prompt_raw, assistant_tokens_mask = safe_apply_chat_template(
|
||||
model_config,
|
||||
tokenizer,
|
||||
conversation,
|
||||
return_assistant_tokens_mask=True,
|
||||
**chat_template_kwargs,
|
||||
)
|
||||
else:
|
||||
prompt_raw = safe_apply_chat_template(
|
||||
model_config,
|
||||
tokenizer,
|
||||
conversation,
|
||||
**chat_template_kwargs,
|
||||
)
|
||||
prompt_raw = safe_apply_chat_template(
|
||||
model_config,
|
||||
tokenizer,
|
||||
conversation,
|
||||
**chat_template_kwargs,
|
||||
)
|
||||
|
||||
# NOTE: use_unified_vision_chunk is currently specific to Kimi-K2.5
|
||||
# model which uses unified vision chunks for both images and videos.
|
||||
@@ -1107,9 +1048,6 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
|
||||
|
||||
prompt = parse_dec_only_prompt(prompt_raw)
|
||||
|
||||
if assistant_tokens_mask is not None:
|
||||
cast(dict, prompt)["_assistant_tokens_mask"] = assistant_tokens_mask
|
||||
|
||||
# When `prompt_embeds` is mixed with other modality data,
|
||||
# `_process_tokens` runs `_process_multimodal` first (expanding
|
||||
# `<|AUDIO|>` / `<|IMAGE|>` placeholders) and then
|
||||
@@ -1183,30 +1121,12 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
|
||||
logger.warning_once(_TOKENIZE_OVERRIDE_WARNING)
|
||||
chat_template_kwargs["tokenize"] = True
|
||||
|
||||
assistant_tokens_mask: list[int] | None = None
|
||||
if params.return_assistant_tokens_mask:
|
||||
result_with_mask = cast(
|
||||
tuple[list[int], list[int] | None],
|
||||
await make_async(
|
||||
safe_apply_chat_template,
|
||||
executor=self._executor,
|
||||
)(
|
||||
model_config,
|
||||
tokenizer,
|
||||
conversation,
|
||||
return_assistant_tokens_mask=True, # type: ignore[arg-type]
|
||||
**chat_template_kwargs,
|
||||
),
|
||||
)
|
||||
prompt_raw: str | list[int] = result_with_mask[0]
|
||||
assistant_tokens_mask = result_with_mask[1]
|
||||
else:
|
||||
prompt_raw = await self._apply_chat_template_async(
|
||||
model_config,
|
||||
tokenizer,
|
||||
conversation,
|
||||
**chat_template_kwargs,
|
||||
)
|
||||
prompt_raw = await self._apply_chat_template_async(
|
||||
model_config,
|
||||
tokenizer,
|
||||
conversation,
|
||||
**chat_template_kwargs,
|
||||
)
|
||||
|
||||
# NOTE: use_unified_vision_chunk is currently specific to Kimi-K2.5
|
||||
# model which uses unified vision chunks for both images and videos.
|
||||
@@ -1232,9 +1152,6 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
|
||||
|
||||
prompt = parse_dec_only_prompt(prompt_raw)
|
||||
|
||||
if assistant_tokens_mask is not None:
|
||||
cast(dict, prompt)["_assistant_tokens_mask"] = assistant_tokens_mask
|
||||
|
||||
# See `render_messages` for the rationale.
|
||||
if prompt_embeds_tensors and mm_data:
|
||||
assert prompt_embeds_placeholder_token_id is not None
|
||||
@@ -1276,7 +1193,6 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
|
||||
processor records all placeholder offsets in the final (post-expansion)
|
||||
coordinate space, no offset shifting needed afterwards.
|
||||
"""
|
||||
assistant_tokens_mask = cast(dict, prompt).pop("_assistant_tokens_mask", None)
|
||||
prompt_embeds_info = cast(dict, prompt).pop("_prompt_embeds", None)
|
||||
if prompt_embeds_info is not None:
|
||||
tensors, placeholder_token_id = prompt_embeds_info
|
||||
@@ -1284,7 +1200,9 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
|
||||
cast(dict, prompt)["prompt_token_ids"] = _expand_prompt_embeds_placeholders(
|
||||
list(prompt["prompt_token_ids"]), mm_updates
|
||||
)
|
||||
|
||||
engine_input = super()._process_tokens(prompt, skip_mm_cache=skip_mm_cache)
|
||||
|
||||
if prompt_embeds_info is not None:
|
||||
tensors, _ = prompt_embeds_info
|
||||
self._apply_prompt_embeds_to_engine_input(
|
||||
@@ -1292,8 +1210,7 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
|
||||
tensors,
|
||||
mm_updates,
|
||||
)
|
||||
if assistant_tokens_mask is not None:
|
||||
engine_input["assistant_tokens_mask"] = assistant_tokens_mask
|
||||
|
||||
return engine_input
|
||||
|
||||
@override
|
||||
@@ -1304,7 +1221,6 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
|
||||
skip_mm_cache: bool = False,
|
||||
) -> TokensInput | MultiModalInput:
|
||||
"""Async equivalent of `_process_tokens`."""
|
||||
assistant_tokens_mask = cast(dict, prompt).pop("_assistant_tokens_mask", None)
|
||||
prompt_embeds_info = cast(dict, prompt).pop("_prompt_embeds", None)
|
||||
if prompt_embeds_info is not None:
|
||||
tensors, placeholder_token_id = prompt_embeds_info
|
||||
@@ -1312,9 +1228,11 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
|
||||
cast(dict, prompt)["prompt_token_ids"] = _expand_prompt_embeds_placeholders(
|
||||
list(prompt["prompt_token_ids"]), mm_updates
|
||||
)
|
||||
|
||||
engine_input = await super()._process_tokens_async(
|
||||
prompt, skip_mm_cache=skip_mm_cache
|
||||
)
|
||||
|
||||
if prompt_embeds_info is not None:
|
||||
tensors, _ = prompt_embeds_info
|
||||
self._apply_prompt_embeds_to_engine_input(
|
||||
@@ -1322,8 +1240,7 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
|
||||
tensors,
|
||||
mm_updates,
|
||||
)
|
||||
if assistant_tokens_mask is not None:
|
||||
engine_input["assistant_tokens_mask"] = assistant_tokens_mask
|
||||
|
||||
return engine_input
|
||||
|
||||
@staticmethod
|
||||
|
||||
@@ -27,13 +27,9 @@ _S = TypeVar("_S", list[int], "torch.Tensor")
|
||||
# Prompt keys whose entry i describes token i of the prompt, so they must be
|
||||
# reduced with the same slice as the prompt tokens themselves. Anything added
|
||||
# here is truncated by `TokenizeParams.apply_post_tokenization`.
|
||||
# `_assistant_tokens_mask` is renderer-internal: `HfRenderer.render_messages`
|
||||
# stashes it on the prompt so `_process_tokens` can move it onto the engine
|
||||
# input.
|
||||
_PARALLEL_TO_PROMPT_TOKENS = (
|
||||
"prompt_token_offsets",
|
||||
"prompt_is_token_ids",
|
||||
"_assistant_tokens_mask",
|
||||
)
|
||||
|
||||
|
||||
@@ -99,9 +95,6 @@ class ChatParams:
|
||||
mm_processor_kwargs: dict[str, Any] | None = None
|
||||
"""The kwargs to pass to the multi-modal processor."""
|
||||
|
||||
return_assistant_tokens_mask: bool = False
|
||||
"""Request a per-token assistant mask from apply_chat_template."""
|
||||
|
||||
tool_choice: Any | None = None
|
||||
"""Request-level tool choice for renderers that need API metadata."""
|
||||
|
||||
@@ -136,7 +129,6 @@ class ChatParams:
|
||||
default_mm_processor_kwargs,
|
||||
self.mm_processor_kwargs,
|
||||
),
|
||||
return_assistant_tokens_mask=self.return_assistant_tokens_mask,
|
||||
tool_choice=self.tool_choice,
|
||||
response_format=self.response_format,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user