[Model] Add Gemma4 Unified (encoder-free) support (#44429)
Signed-off-by: Luciano Martins <lucianommartins@users.noreply.github.com> Co-authored-by: Luciano Martins <lucianommartins@users.noreply.github.com>
This commit is contained in:
co-authored by
Luciano Martins
parent
271328e256
commit
a248b45d05
@@ -279,11 +279,19 @@ class Gemma4MTPDecoderLayer(nn.Module):
|
||||
else config.head_dim
|
||||
)
|
||||
|
||||
use_k_eq_v = is_full_attention and getattr(config, "attention_k_eq_v", False)
|
||||
if use_k_eq_v:
|
||||
num_kv_heads = getattr(
|
||||
config, "num_global_key_value_heads", config.num_key_value_heads
|
||||
)
|
||||
else:
|
||||
num_kv_heads = config.num_key_value_heads
|
||||
|
||||
self.self_attn = Gemma4MTPAttention(
|
||||
config=config,
|
||||
hidden_size=self.hidden_size,
|
||||
num_heads=config.num_attention_heads,
|
||||
num_kv_heads=config.num_key_value_heads,
|
||||
num_kv_heads=num_kv_heads,
|
||||
head_dim=head_dim,
|
||||
max_position_embeddings=config.max_position_embeddings,
|
||||
cache_config=cache_config,
|
||||
@@ -545,6 +553,10 @@ class Gemma4MTP(nn.Module):
|
||||
else:
|
||||
self.masked_embedding = None
|
||||
|
||||
draft_cfg = vllm_config.speculative_config.draft_model_config
|
||||
gen_cfg = draft_cfg.try_get_generation_config()
|
||||
self._suppress_token_ids = gen_cfg.get("suppress_tokens") if gen_cfg else None
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.embed_input_ids(input_ids)
|
||||
|
||||
@@ -589,11 +601,15 @@ class Gemma4MTP(nn.Module):
|
||||
spec_step_idx: int = 0,
|
||||
) -> torch.Tensor | None:
|
||||
if self.masked_embedding is not None:
|
||||
return self.masked_embedding(
|
||||
logits = self.masked_embedding(
|
||||
hidden_states,
|
||||
self._get_full_lm_head_weight(),
|
||||
)
|
||||
return self.logits_processor(self.lm_head, hidden_states)
|
||||
else:
|
||||
logits = self.logits_processor(self.lm_head, hidden_states)
|
||||
if logits is not None and self._suppress_token_ids:
|
||||
logits[:, self._suppress_token_ids] = -float("inf")
|
||||
return logits
|
||||
|
||||
def get_top_tokens(
|
||||
self,
|
||||
|
||||
Reference in New Issue
Block a user