From 6dc9491406938463e44e631610ce83df1d20f8cd Mon Sep 17 00:00:00 2001 From: Luciano Martins Date: Wed, 15 Apr 2026 20:13:07 -0300 Subject: [PATCH] [Model] Fix Gemma 4 token repetition by dynamic BOS injection for PT models (#39842) Signed-off-by: Luciano Martins Co-authored-by: Luciano Martins --- vllm/model_executor/models/gemma4_mm.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/vllm/model_executor/models/gemma4_mm.py b/vllm/model_executor/models/gemma4_mm.py index e22f23c5c8..d88a5f1725 100644 --- a/vllm/model_executor/models/gemma4_mm.py +++ b/vllm/model_executor/models/gemma4_mm.py @@ -167,10 +167,15 @@ class Gemma4ProcessingInfo(BaseProcessingInfo): Setting ``add_special_tokens=False`` here prevents the duplicate and ensures both ``llm.generate()`` and the chat/completions API behave - correctly. + correctly for IT models. For PT models (without chat template), we + keep the default (True) to ensure BOS is added for raw prompts. """ + tokenizer = self.ctx.get_tokenizer() + has_chat_template = getattr(tokenizer, "chat_template", None) is not None + params = super().get_default_tok_params() - params = params.with_kwargs(add_special_tokens=False) + if has_chat_template: + params = params.with_kwargs(add_special_tokens=False) return params def get_hf_processor(self, **kwargs: object) -> Gemma4Processor: