[Model] Fix Gemma 4 token repetition by dynamic BOS injection for PT models (#39842)
Signed-off-by: Luciano Martins <lucianommartins@users.noreply.github.com> Co-authored-by: Luciano Martins <lucianommartins@users.noreply.github.com>
This commit is contained in:
co-authored by
Luciano Martins
parent
27c0ca50a0
commit
6dc9491406
@@ -167,10 +167,15 @@ class Gemma4ProcessingInfo(BaseProcessingInfo):
|
||||
|
||||
Setting ``add_special_tokens=False`` here prevents the duplicate and
|
||||
ensures both ``llm.generate()`` and the chat/completions API behave
|
||||
correctly.
|
||||
correctly for IT models. For PT models (without chat template), we
|
||||
keep the default (True) to ensure BOS is added for raw prompts.
|
||||
"""
|
||||
tokenizer = self.ctx.get_tokenizer()
|
||||
has_chat_template = getattr(tokenizer, "chat_template", None) is not None
|
||||
|
||||
params = super().get_default_tok_params()
|
||||
params = params.with_kwargs(add_special_tokens=False)
|
||||
if has_chat_template:
|
||||
params = params.with_kwargs(add_special_tokens=False)
|
||||
return params
|
||||
|
||||
def get_hf_processor(self, **kwargs: object) -> Gemma4Processor:
|
||||
|
||||
Reference in New Issue
Block a user