[Model] Gemma3n MM (#20495)
Signed-off-by: ShriKode <shrikode@gmail.com> Signed-off-by: NickLucche <nlucches@redhat.com> Signed-off-by: Roger Wang <hey@rogerw.me> Co-authored-by: ShriKode <shrikode@gmail.com> Co-authored-by: Roger Wang <hey@rogerw.me>
This commit is contained in:
co-authored by
ShriKode
Roger Wang
parent
2d18256e47
commit
5a16fa614c
@@ -331,14 +331,15 @@ class Gemma3nAttention(nn.Module):
|
||||
config.num_kv_shared_layers)
|
||||
self.is_kv_shared = layer_idx >= first_kv_shared_layer_idx
|
||||
|
||||
kv_sharing_target_layer_name = None
|
||||
if self.is_kv_shared:
|
||||
# Last full attention layer is 1 before sharing
|
||||
# Last sliding attention layer is 2 before sharing
|
||||
offset = 2 if self.sliding_window is not None else 1
|
||||
kv_shared_layer_index = first_kv_shared_layer_idx - offset
|
||||
kv_sharing_target_layer_name = f"model.language_model.layers.{kv_shared_layer_index}.self_attn.attn" # noqa: E501
|
||||
else:
|
||||
kv_sharing_target_layer_name = None
|
||||
if kv_shared_layer_index >= 0:
|
||||
# Only the greater layer is required to specify sharing.
|
||||
kv_sharing_target_layer_name = f"language_model.model.layers.{kv_shared_layer_index}.self_attn.attn" # noqa: E501
|
||||
|
||||
self.rotary_emb = get_rope(
|
||||
self.head_dim,
|
||||
@@ -396,6 +397,7 @@ class Gemma3nDecoderLayer(nn.Module):
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
assert isinstance(config, Gemma3nTextConfig)
|
||||
self.altup_active_idx = config.altup_active_idx
|
||||
assert config.altup_correct_scale
|
||||
|
||||
@@ -537,7 +539,7 @@ class Gemma3nTextModel(nn.Module, SupportsQuant):
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
config = vllm_config.model_config.hf_config.text_config
|
||||
config = vllm_config.model_config.hf_config
|
||||
cache_config = vllm_config.cache_config
|
||||
quant_config = vllm_config.quant_config
|
||||
self.config = config
|
||||
@@ -553,6 +555,7 @@ class Gemma3nTextModel(nn.Module, SupportsQuant):
|
||||
config.hidden_size**0.5,
|
||||
dtype=self.embed_tokens.weight.dtype,
|
||||
)
|
||||
# Additional per-layer embeddings (PLE)
|
||||
self.embed_tokens_per_layer = VocabParallelEmbedding(
|
||||
config.vocab_size_per_layer_input,
|
||||
config.num_hidden_layers * config.hidden_size_per_layer_input,
|
||||
@@ -636,6 +639,8 @@ class Gemma3nTextModel(nn.Module, SupportsQuant):
|
||||
self,
|
||||
input_ids: Optional[torch.Tensor],
|
||||
positions: torch.Tensor,
|
||||
per_layer_inputs: torch.Tensor,
|
||||
intermediate_tensors: Optional[IntermediateTensors] = None,
|
||||
inputs_embeds: Optional[torch.Tensor] = None,
|
||||
**kwargs,
|
||||
) -> Union[torch.Tensor, IntermediateTensors]:
|
||||
@@ -644,13 +649,6 @@ class Gemma3nTextModel(nn.Module, SupportsQuant):
|
||||
else:
|
||||
hidden_states_0 = self.get_input_embeddings(input_ids)
|
||||
|
||||
# Per layer inputs.
|
||||
if input_ids is None:
|
||||
raise ValueError("Passing None for input ids is not supported.")
|
||||
per_layer_inputs = self.get_per_layer_input_embeddings(input_ids)
|
||||
per_layer_inputs = per_layer_inputs.reshape(
|
||||
-1, self.config.num_hidden_layers,
|
||||
self.config.hidden_size_per_layer_input)
|
||||
per_layer_projection = self.per_layer_model_projection(hidden_states_0)
|
||||
per_layer_projection = per_layer_projection.reshape(
|
||||
*hidden_states_0.shape[:-1],
|
||||
@@ -659,8 +657,13 @@ class Gemma3nTextModel(nn.Module, SupportsQuant):
|
||||
)
|
||||
per_layer_projection = self.per_layer_projection_norm(
|
||||
per_layer_projection)
|
||||
per_layer_inputs = per_layer_projection + per_layer_inputs
|
||||
per_layer_inputs *= self.per_layer_input_scale
|
||||
|
||||
if per_layer_inputs is not None:
|
||||
# Profiling run does not compute per_layer_inputs
|
||||
per_layer_inputs = per_layer_projection + per_layer_inputs
|
||||
per_layer_inputs *= self.per_layer_input_scale
|
||||
else:
|
||||
per_layer_inputs = per_layer_projection
|
||||
|
||||
# Altup embed.
|
||||
hidden_states = [hidden_states_0] * self.config.altup_num_inputs
|
||||
@@ -760,29 +763,7 @@ class Gemma3nTextModel(nn.Module, SupportsQuant):
|
||||
return loaded_params
|
||||
|
||||
|
||||
class Gemma3nModel(nn.Module):
|
||||
|
||||
def __init__(self, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
self.language_model = Gemma3nTextModel(vllm_config=vllm_config,
|
||||
prefix=maybe_prefix(
|
||||
prefix, "language_model"))
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: Optional[torch.Tensor],
|
||||
positions: torch.Tensor,
|
||||
intermediate_tensors: Optional[IntermediateTensors] = None,
|
||||
inputs_embeds: Optional[torch.Tensor] = None,
|
||||
**kwargs,
|
||||
) -> torch.Tensor:
|
||||
return self.language_model(input_ids=input_ids,
|
||||
positions=positions,
|
||||
inputs_embeds=inputs_embeds,
|
||||
**kwargs)
|
||||
|
||||
|
||||
class Gemma3nForConditionalGeneration(nn.Module, SupportsQuant):
|
||||
class Gemma3nForCausalLM(nn.Module):
|
||||
packed_modules_mapping = {
|
||||
"qkv_proj": [
|
||||
"q_proj",
|
||||
@@ -802,25 +783,33 @@ class Gemma3nForConditionalGeneration(nn.Module, SupportsQuant):
|
||||
super().__init__()
|
||||
self.config = config
|
||||
self.cache_config = vllm_config.cache_config
|
||||
self.model = Gemma3nModel(vllm_config=vllm_config,
|
||||
prefix=maybe_prefix(prefix, "model"))
|
||||
self.model = Gemma3nTextModel(vllm_config=vllm_config,
|
||||
prefix=maybe_prefix(prefix, "model"))
|
||||
self.logits_processor = LogitsProcessor(
|
||||
config.text_config.vocab_size,
|
||||
soft_cap=config.text_config.final_logit_softcapping)
|
||||
config.vocab_size, soft_cap=config.final_logit_softcapping)
|
||||
|
||||
def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.language_model.get_input_embeddings(input_ids)
|
||||
return self.model.get_input_embeddings(input_ids)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
*,
|
||||
per_layer_inputs: Optional[torch.Tensor] = None,
|
||||
intermediate_tensors: Optional[IntermediateTensors] = None,
|
||||
inputs_embeds: Optional[torch.Tensor] = None,
|
||||
**kwargs,
|
||||
) -> Union[torch.Tensor, IntermediateTensors]:
|
||||
hidden_states = self.model(input_ids, positions, intermediate_tensors,
|
||||
inputs_embeds, **kwargs)
|
||||
|
||||
hidden_states = self.model(
|
||||
input_ids,
|
||||
positions,
|
||||
per_layer_inputs=per_layer_inputs,
|
||||
intermediate_tensors=intermediate_tensors,
|
||||
inputs_embeds=inputs_embeds,
|
||||
**kwargs,
|
||||
)
|
||||
return hidden_states
|
||||
|
||||
def compute_logits(
|
||||
@@ -828,8 +817,8 @@ class Gemma3nForConditionalGeneration(nn.Module, SupportsQuant):
|
||||
hidden_states: torch.Tensor,
|
||||
sampling_metadata: Optional[SamplingMetadata],
|
||||
) -> Optional[torch.Tensor]:
|
||||
logits = self.logits_processor(self.model.language_model.embed_tokens,
|
||||
hidden_states, sampling_metadata)
|
||||
logits = self.logits_processor(self.model.embed_tokens, hidden_states,
|
||||
sampling_metadata)
|
||||
return logits
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str,
|
||||
|
||||
Reference in New Issue
Block a user