[Bugfix][Frontend] Truncate the assistant tokens mask with the prompt (#54539)
Signed-off-by: hotragn <hotragn.pettugani_2024@woxsen.edu.in>
This commit is contained in:
+21
-11
@@ -24,6 +24,18 @@ logger = init_logger(__name__)
|
||||
|
||||
_S = TypeVar("_S", list[int], "torch.Tensor")
|
||||
|
||||
# Prompt keys whose entry i describes token i of the prompt, so they must be
|
||||
# reduced with the same slice as the prompt tokens themselves. Anything added
|
||||
# here is truncated by `TokenizeParams.apply_post_tokenization`.
|
||||
# `_assistant_tokens_mask` is renderer-internal: `HfRenderer.render_messages`
|
||||
# stashes it on the prompt so `_process_tokens` can move it onto the engine
|
||||
# input.
|
||||
_PARALLEL_TO_PROMPT_TOKENS = (
|
||||
"prompt_token_offsets",
|
||||
"prompt_is_token_ids",
|
||||
"_assistant_tokens_mask",
|
||||
)
|
||||
|
||||
|
||||
def merge_kwargs(
|
||||
defaults: dict[str, Any] | None,
|
||||
@@ -420,8 +432,8 @@ class TokenizeParams:
|
||||
"""The slice truncation applies to a sequence of `length` tokens.
|
||||
|
||||
`None` means no truncation. Anything parallel to the prompt tokens
|
||||
(see `prompt_token_offsets` and `prompt_is_token_ids`) must be reduced
|
||||
with this same slice to stay aligned with them.
|
||||
must be reduced with this same slice to stay aligned with them; list
|
||||
such keys in `_PARALLEL_TO_PROMPT_TOKENS`.
|
||||
"""
|
||||
max_length = self.truncate_prompt_tokens
|
||||
if max_length is not None and max_length < 0:
|
||||
@@ -511,15 +523,13 @@ class TokenizeParams:
|
||||
tokenizer,
|
||||
prompt["prompt_embeds"], # type: ignore[typeddict-item]
|
||||
)
|
||||
offsets = cast(TokensPrompt, prompt).get("prompt_token_offsets")
|
||||
if offsets is not None:
|
||||
truncation = self._truncation_slice(tokenizer, len(offsets))
|
||||
prompt_dict = cast(dict, prompt)
|
||||
for key in _PARALLEL_TO_PROMPT_TOKENS:
|
||||
parallel = prompt_dict.get(key)
|
||||
if parallel is None:
|
||||
continue
|
||||
truncation = self._truncation_slice(tokenizer, len(parallel))
|
||||
if truncation is not None:
|
||||
prompt["prompt_token_offsets"] = offsets[truncation] # type: ignore[typeddict-unknown-key]
|
||||
is_token_ids = cast(EmbedsPrompt, prompt).get("prompt_is_token_ids")
|
||||
if is_token_ids is not None:
|
||||
truncation = self._truncation_slice(tokenizer, len(is_token_ids))
|
||||
if truncation is not None:
|
||||
prompt["prompt_is_token_ids"] = is_token_ids[truncation] # type: ignore[typeddict-unknown-key]
|
||||
prompt_dict[key] = parallel[truncation]
|
||||
|
||||
return prompt
|
||||
|
||||
Reference in New Issue
Block a user