[Model] Add Gemma4 Unified (encoder-free) support (#44429)

Signed-off-by: Luciano Martins <lucianommartins@users.noreply.github.com>
Co-authored-by: Luciano Martins <lucianommartins@users.noreply.github.com>
This commit is contained in:
Luciano Martins
2026-06-03 12:01:39 -07:00
committed by GitHub
co-authored by Luciano Martins
parent 271328e256
commit a248b45d05
14 changed files with 791 additions and 31 deletions
+19 -3
View File
@@ -279,11 +279,19 @@ class Gemma4MTPDecoderLayer(nn.Module):
else config.head_dim
)
use_k_eq_v = is_full_attention and getattr(config, "attention_k_eq_v", False)
if use_k_eq_v:
num_kv_heads = getattr(
config, "num_global_key_value_heads", config.num_key_value_heads
)
else:
num_kv_heads = config.num_key_value_heads
self.self_attn = Gemma4MTPAttention(
config=config,
hidden_size=self.hidden_size,
num_heads=config.num_attention_heads,
num_kv_heads=config.num_key_value_heads,
num_kv_heads=num_kv_heads,
head_dim=head_dim,
max_position_embeddings=config.max_position_embeddings,
cache_config=cache_config,
@@ -545,6 +553,10 @@ class Gemma4MTP(nn.Module):
else:
self.masked_embedding = None
draft_cfg = vllm_config.speculative_config.draft_model_config
gen_cfg = draft_cfg.try_get_generation_config()
self._suppress_token_ids = gen_cfg.get("suppress_tokens") if gen_cfg else None
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.model.embed_input_ids(input_ids)
@@ -589,11 +601,15 @@ class Gemma4MTP(nn.Module):
spec_step_idx: int = 0,
) -> torch.Tensor | None:
if self.masked_embedding is not None:
return self.masked_embedding(
logits = self.masked_embedding(
hidden_states,
self._get_full_lm_head_weight(),
)
return self.logits_processor(self.lm_head, hidden_states)
else:
logits = self.logits_processor(self.lm_head, hidden_states)
if logits is not None and self._suppress_token_ids:
logits[:, self._suppress_token_ids] = -float("inf")
return logits
def get_top_tokens(
self,