Convert formatting to use ruff instead of yapf + isort (#26247)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Harry Mellor
2025-10-05 07:06:22 -07:00
committed by GitHub
parent 17edd8a807
commit d6953beb91
1508 changed files with 115205 additions and 94107 deletions
+199 -164
View File
@@ -28,28 +28,38 @@ from vllm.config import CacheConfig, VllmConfig
from vllm.distributed import get_tensor_model_parallel_world_size
from vllm.forward_context import get_forward_context
from vllm.logger import init_logger
from vllm.model_executor.layers.activation import (_ACTIVATION_REGISTRY,
GeluAndMul,
GeluAndMulSparse)
from vllm.model_executor.layers.activation import (
_ACTIVATION_REGISTRY,
GeluAndMul,
GeluAndMulSparse,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
MergedColumnParallelLinear,
QKVParallelLinear,
ReplicatedLinear,
RowParallelLinear)
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
MergedColumnParallelLinear,
QKVParallelLinear,
ReplicatedLinear,
RowParallelLinear,
)
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.rotary_embedding import get_rope
from vllm.model_executor.layers.vocab_parallel_embedding import (
VocabParallelEmbedding)
from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
from vllm.model_executor.model_loader.weight_utils import (
default_weight_loader, maybe_remap_kv_scale_name)
default_weight_loader,
maybe_remap_kv_scale_name,
)
from vllm.sequence import IntermediateTensors
from vllm.v1.attention.backends.utils import KVSharingFastPrefillMetadata
from .interfaces import SupportsQuant
from .utils import (AutoWeightsLoader, extract_layer_index,
is_pp_missing_parameter, make_layers, maybe_prefix)
from .utils import (
AutoWeightsLoader,
extract_layer_index,
is_pp_missing_parameter,
make_layers,
maybe_prefix,
)
logger = init_logger(__name__)
@@ -110,9 +120,11 @@ class Gemma3nAltUp(nn.Module):
eps=rms_norm_eps,
)
self.router_input_scale = torch.tensor(
hidden_size**-1.0, dtype=self.modality_router.weight.dtype)
hidden_size**-1.0, dtype=self.modality_router.weight.dtype
)
self.correct_output_scale = nn.Parameter(
torch.zeros(hidden_size, dtype=torch.float32))
torch.zeros(hidden_size, dtype=torch.float32)
)
def _compute_router_modalities(self, x: torch.Tensor) -> torch.Tensor:
router_inputs = self.router_norm(x) * self.router_input_scale
@@ -120,15 +132,17 @@ class Gemma3nAltUp(nn.Module):
return torch.tanh(routed.float()).type_as(x)
def scale_corrected_output(self, corrected: torch.Tensor) -> torch.Tensor:
return (corrected.type_as(self.correct_output_scale) *
self.correct_output_scale).type_as(corrected)
return (
corrected.type_as(self.correct_output_scale) * self.correct_output_scale
).type_as(corrected)
def predict(self, hidden_states: torch.Tensor) -> torch.Tensor:
# hidden: [altup_num_inputs, num_tokens, hidden_size]
# modalities: [num_tokens, num_altup_inputs]
# all_coefs: [num_tokens, num_altup_inputs ** 2]
modalities = self._compute_router_modalities(
hidden_states[self.altup_active_idx])
hidden_states[self.altup_active_idx]
)
all_coefs = self.prediction_coefs(modalities)
# Reshape and transpose the 2D matrix for the matmul.
@@ -146,8 +160,9 @@ class Gemma3nAltUp(nn.Module):
predictions += hidden_states
return predictions.contiguous()
def correct(self, predictions: torch.Tensor,
activated: torch.Tensor) -> torch.Tensor:
def correct(
self, predictions: torch.Tensor, activated: torch.Tensor
) -> torch.Tensor:
# predictions: [altup_num_inputs, num_tokens, hidden_size]
# activated: [num_tokens, hidden_size]
# modalities: [num_tokens, altup_num_inputs]
@@ -215,7 +230,6 @@ class Gemma3nLaurelBlock(nn.Module):
class Gemma3nMLP(nn.Module):
def __init__(
self,
hidden_size: int,
@@ -244,12 +258,16 @@ class Gemma3nMLP(nn.Module):
raise ValueError(
"Gemma3 uses `gelu_pytorch_tanh` as the hidden activation "
"function. Please set `hidden_act` and `hidden_activation` to "
"`gelu_pytorch_tanh`.")
"`gelu_pytorch_tanh`."
)
self.act_fn = GeluAndMulSparse(
activation_sparsity=activation_sparsity,
approximate="tanh") if activation_sparsity > 0.0 else GeluAndMul(
approximate="tanh")
self.act_fn = (
GeluAndMulSparse(
activation_sparsity=activation_sparsity, approximate="tanh"
)
if activation_sparsity > 0.0
else GeluAndMul(approximate="tanh")
)
def forward(self, x: torch.Tensor) -> torch.Tensor:
gate_up, _ = self.gate_up_proj(x)
@@ -259,17 +277,18 @@ class Gemma3nMLP(nn.Module):
class Gemma3nAttention(nn.Module):
def __init__(self,
config: Gemma3nTextConfig,
hidden_size: int,
num_heads: int,
num_kv_heads: int,
head_dim: int,
max_position_embeddings: int,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
prefix: str = "") -> None:
def __init__(
self,
config: Gemma3nTextConfig,
hidden_size: int,
num_heads: int,
num_kv_heads: int,
head_dim: int,
max_position_embeddings: int,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
prefix: str = "",
) -> None:
super().__init__()
self.config = config
self.hidden_size = hidden_size
@@ -307,13 +326,11 @@ class Gemma3nAttention(nn.Module):
quant_config=quant_config,
prefix=f"{prefix}.o_proj",
)
self.q_norm = RMSNorm(hidden_size=self.head_dim,
eps=config.rms_norm_eps)
self.k_norm = RMSNorm(hidden_size=self.head_dim,
eps=config.rms_norm_eps)
self.v_norm = RMSNorm(hidden_size=self.head_dim,
eps=config.rms_norm_eps,
has_weight=False)
self.q_norm = RMSNorm(hidden_size=self.head_dim, eps=config.rms_norm_eps)
self.k_norm = RMSNorm(hidden_size=self.head_dim, eps=config.rms_norm_eps)
self.v_norm = RMSNorm(
hidden_size=self.head_dim, eps=config.rms_norm_eps, has_weight=False
)
layer_idx = extract_layer_index(prefix)
is_sliding = config.layer_types[layer_idx] == "sliding_attention"
@@ -329,8 +346,9 @@ class Gemma3nAttention(nn.Module):
rope_theta = config.rope_theta
rope_scaling = config.rope_scaling
first_kv_shared_layer_idx = (config.num_hidden_layers -
config.num_kv_shared_layers)
first_kv_shared_layer_idx = (
config.num_hidden_layers - config.num_kv_shared_layers
)
self.is_kv_shared = layer_idx >= first_kv_shared_layer_idx
kv_sharing_target_layer_name = None
@@ -361,7 +379,8 @@ class Gemma3nAttention(nn.Module):
quant_config=quant_config,
per_layer_sliding_window=self.sliding_window,
kv_sharing_target_layer_name=kv_sharing_target_layer_name,
prefix=f"{prefix}.attn")
prefix=f"{prefix}.attn",
)
def forward(
self,
@@ -390,7 +409,6 @@ class Gemma3nAttention(nn.Module):
class Gemma3nDecoderLayer(nn.Module):
def __init__(
self,
config: Gemma3nTextConfig,
@@ -426,12 +444,12 @@ class Gemma3nDecoderLayer(nn.Module):
self.mlp = Gemma3nMLP(
hidden_size=config.hidden_size,
# NOTE: Matformer https://github.com/huggingface/transformers/blob/a52478253bbe522a420e88ea3940d4d98a935300/src/transformers/models/gemma3n/modular_gemma3n.py#L258 # noqa: E501
intermediate_size=config.intermediate_size[extract_layer_index(
prefix)],
intermediate_size=config.intermediate_size[extract_layer_index(prefix)],
hidden_activation=config.hidden_activation,
quant_config=quant_config,
activation_sparsity=config.activation_sparsity_pattern[
extract_layer_index(prefix)],
extract_layer_index(prefix)
],
prefix=f"{prefix}.mlp",
)
self.laurel = Gemma3nLaurelBlock(
@@ -493,7 +511,6 @@ class Gemma3nDecoderLayer(nn.Module):
per_layer_input: torch.Tensor,
**kwargs,
) -> tuple[torch.Tensor, torch.Tensor]:
# ActUp (predict).
predictions = self.altup.predict(hidden_states)
active_prediction = predictions[self.altup_active_idx]
@@ -508,8 +525,7 @@ class Gemma3nDecoderLayer(nn.Module):
)
attn = self.post_attention_layernorm(attn)
attn_gated = attn + active_prediction
attn_laurel = (attn_gated + laurel_output) / torch.sqrt(
torch.tensor(2.0))
attn_laurel = (attn_gated + laurel_output) / torch.sqrt(torch.tensor(2.0))
# MLP.
attn_norm = self.pre_feedforward_layernorm(attn_laurel)
@@ -518,8 +534,7 @@ class Gemma3nDecoderLayer(nn.Module):
attn_ffw_laurel_gated = attn_laurel + attn_ffw_norm
# ActUp (connect).
corrected_predictions = self.altup.correct(predictions,
attn_ffw_laurel_gated)
corrected_predictions = self.altup.correct(predictions, attn_ffw_laurel_gated)
first_prediction = corrected_predictions[self.altup_active_idx]
first_prediction = self.altup.scale_corrected_output(first_prediction)
@@ -537,8 +552,9 @@ class Gemma3nDecoderLayer(nn.Module):
# This enables torch.compile if --kv-sharing-fast-prefill passed
@support_torch_compile(enable_if=lambda vllm_config: vllm_config.cache_config.
kv_sharing_fast_prefill)
@support_torch_compile(
enable_if=lambda vllm_config: vllm_config.cache_config.kv_sharing_fast_prefill
)
class Gemma3nSelfDecoder(nn.Module):
"""
Includes altup embedding and self decoder layers
@@ -595,34 +611,41 @@ class Gemma3nSelfDecoder(nn.Module):
eps=config.rms_norm_eps,
)
self.per_layer_input_scale = torch.rsqrt(torch.tensor(2.0)).to(
self.embed_tokens.weight.dtype)
self.embed_tokens.weight.dtype
)
self.per_layer_projection_scale = torch.tensor(
config.hidden_size**0.5,
dtype=self.embed_tokens.weight.dtype,
)
self.altup_projections = nn.ModuleList([
ColumnParallelLinear(
config.hidden_size,
config.hidden_size,
bias=False,
gather_output=True,
return_bias=False,
quant_config=quant_config,
prefix=f"{prefix}.altup_projections.{idx-1}",
) for idx in range(1, self.config.altup_num_inputs)
])
self.altup_projections = nn.ModuleList(
[
ColumnParallelLinear(
config.hidden_size,
config.hidden_size,
bias=False,
gather_output=True,
return_bias=False,
quant_config=quant_config,
prefix=f"{prefix}.altup_projections.{idx - 1}",
)
for idx in range(1, self.config.altup_num_inputs)
]
)
def get_per_layer_input_embeddings(
self, input_ids: torch.Tensor) -> torch.Tensor:
def get_per_layer_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
# Deal with the fact that vocab_size_per_layer_input < vocab_size
# which causes us to have some out of vocab tokens by setting
# those token ids to 0. This matches the HF implementation.
per_layer_inputs_mask = torch.logical_and(
input_ids >= 0, input_ids < self.config.vocab_size_per_layer_input)
per_layer_inputs_tokens = torch.where(per_layer_inputs_mask, input_ids,
torch.zeros_like(input_ids))
return self.embed_tokens_per_layer(
per_layer_inputs_tokens) * self.embed_scale_per_layer
input_ids >= 0, input_ids < self.config.vocab_size_per_layer_input
)
per_layer_inputs_tokens = torch.where(
per_layer_inputs_mask, input_ids, torch.zeros_like(input_ids)
)
return (
self.embed_tokens_per_layer(per_layer_inputs_tokens)
* self.embed_scale_per_layer
)
def get_per_layer_inputs(
self,
@@ -635,8 +658,7 @@ class Gemma3nSelfDecoder(nn.Module):
self.config.num_hidden_layers,
self.config.hidden_size_per_layer_input,
)
per_layer_projection = self.per_layer_projection_norm(
per_layer_projection)
per_layer_projection = self.per_layer_projection_norm(per_layer_projection)
if per_layer_inputs is not None:
# Profiling run does not compute per_layer_inputs
per_layer_inputs = per_layer_projection + per_layer_inputs
@@ -651,15 +673,13 @@ class Gemma3nSelfDecoder(nn.Module):
def altup_embed(self, hidden_states_0: torch.Tensor) -> torch.Tensor:
# Altup embed.
hidden_states = [hidden_states_0] * self.config.altup_num_inputs
target_magnitude = torch.mean(hidden_states_0**2, dim=-1,
keepdim=True)**0.5
target_magnitude = torch.mean(hidden_states_0**2, dim=-1, keepdim=True) ** 0.5
for i in range(1, self.config.altup_num_inputs):
hidden_states[i] = self.altup_projections[i - 1](hidden_states[i])
new_magnitude = torch.mean(hidden_states[i]**2,
dim=-1,
keepdim=True)**0.5
hidden_states[i] *= target_magnitude / torch.maximum(
new_magnitude, EPS)
new_magnitude = (
torch.mean(hidden_states[i] ** 2, dim=-1, keepdim=True) ** 0.5
)
hidden_states[i] *= target_magnitude / torch.maximum(new_magnitude, EPS)
hidden_states = torch.stack(hidden_states, dim=-1)
return hidden_states
@@ -677,7 +697,8 @@ class Gemma3nSelfDecoder(nn.Module):
hidden_states_0 = self.get_input_embeddings(input_ids)
adjusted_per_layer_inputs = self.get_per_layer_inputs(
hidden_states_0, per_layer_inputs)
hidden_states_0, per_layer_inputs
)
hidden_states = self.altup_embed(hidden_states_0)
# [altnum_inputs, num_tokens, hidden_size]
@@ -700,8 +721,9 @@ class Gemma3nSelfDecoder(nn.Module):
# This enables torch.compile if --kv-sharing-fast-prefill passed
@support_torch_compile(enable_if=lambda vllm_config: vllm_config.cache_config.
kv_sharing_fast_prefill)
@support_torch_compile(
enable_if=lambda vllm_config: vllm_config.cache_config.kv_sharing_fast_prefill
)
class Gemma3nCrossDecoder(nn.Module):
"""
Cross-decoder layers
@@ -743,10 +765,10 @@ class Gemma3nCrossDecoder(nn.Module):
# This disables torch.compile if --kv-sharing-fast-prefill passed
@support_torch_compile(enable_if=lambda vllm_config: not vllm_config.
cache_config.kv_sharing_fast_prefill)
@support_torch_compile(
enable_if=lambda vllm_config: not vllm_config.cache_config.kv_sharing_fast_prefill
)
class Gemma3nTextModel(nn.Module, SupportsQuant):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
config = vllm_config.model_config.hf_config
@@ -755,27 +777,33 @@ class Gemma3nTextModel(nn.Module, SupportsQuant):
self.config = config
self.quant_config = quant_config
self.altup_unembed_projections = nn.ModuleList([
ColumnParallelLinear(
config.hidden_size,
config.hidden_size,
bias=False,
gather_output=True,
return_bias=False,
quant_config=quant_config,
prefix=f"{prefix}.altup_unembed_projections.{idx-1}",
) for idx in range(1, self.config.altup_num_inputs)
])
self.altup_unembed_projections = nn.ModuleList(
[
ColumnParallelLinear(
config.hidden_size,
config.hidden_size,
bias=False,
gather_output=True,
return_bias=False,
quant_config=quant_config,
prefix=f"{prefix}.altup_unembed_projections.{idx - 1}",
)
for idx in range(1, self.config.altup_num_inputs)
]
)
# Allocate config.num_kv_shared_layers layers for self-decoder
self.start_layer, self.end_layer, self.layers = make_layers(
config.num_hidden_layers,
lambda prefix: Gemma3nDecoderLayer(
config, cache_config, quant_config, prefix=prefix),
prefix=f"{prefix}.layers")
config, cache_config, quant_config, prefix=prefix
),
prefix=f"{prefix}.layers",
)
first_kv_shared_layer_idx = (config.num_hidden_layers -
config.num_kv_shared_layers)
first_kv_shared_layer_idx = (
config.num_hidden_layers - config.num_kv_shared_layers
)
# NOTE(sarckk): importing this top level seems to cause issues
# during running of tests.
@@ -810,18 +838,20 @@ class Gemma3nTextModel(nn.Module, SupportsQuant):
# TODO(sarckk): Extract this functionality to interface
max_num_tokens = vllm_config.scheduler_config.max_num_batched_tokens
device = next(self.parameters()).device
self.positions = torch.zeros(max_num_tokens,
dtype=torch.int64,
device=device)
self.positions = torch.zeros(
max_num_tokens, dtype=torch.int64, device=device
)
self.hidden_states = torch.zeros(
(max_num_tokens, config.hidden_size,
self.config.altup_num_inputs),
(max_num_tokens, config.hidden_size, self.config.altup_num_inputs),
dtype=self.embed_tokens.weight.dtype,
device=device,
)
self.per_layer_inputs = torch.zeros(
(max_num_tokens, self.config.num_hidden_layers,
self.config.hidden_size_per_layer_input),
(
max_num_tokens,
self.config.num_hidden_layers,
self.config.hidden_size_per_layer_input,
),
dtype=self.embed_tokens.weight.dtype,
device=device,
)
@@ -830,8 +860,7 @@ class Gemma3nTextModel(nn.Module, SupportsQuant):
def embed_tokens(self):
return self.self_decoder.embed_tokens
def get_per_layer_input_embeddings(
self, input_ids: torch.Tensor) -> torch.Tensor:
def get_per_layer_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.self_decoder.get_per_layer_input_embeddings(input_ids)
def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
@@ -849,27 +878,26 @@ class Gemma3nTextModel(nn.Module, SupportsQuant):
attn_metadata = get_forward_context().attn_metadata
# attn_metadata is None during dummy runs
if (self.fast_prefill_enabled and attn_metadata is not None):
if self.fast_prefill_enabled and attn_metadata is not None:
assert isinstance(attn_metadata, dict)
# Last layer is a KV sharing layer
layer_attn_metadata = attn_metadata[
self.layers[-1].self_attn.attn.layer_name]
if (isinstance(layer_attn_metadata, KVSharingFastPrefillMetadata)):
logits_indices_padded = (
layer_attn_metadata.logits_indices_padded)
self.layers[-1].self_attn.attn.layer_name
]
if isinstance(layer_attn_metadata, KVSharingFastPrefillMetadata):
logits_indices_padded = layer_attn_metadata.logits_indices_padded
num_logits_indices = layer_attn_metadata.num_logits_indices
# Copy inputs for cudagraph
batch_size = positions.size(0)
self.positions[:batch_size].copy_(positions)
self_decoder_hidden_states, per_layer_inputs_adjusted = \
self.self_decoder(
input_ids=input_ids,
positions=self.positions[:batch_size],
inputs_embeds=inputs_embeds,
per_layer_inputs=per_layer_inputs,
**kwargs,
)
self_decoder_hidden_states, per_layer_inputs_adjusted = self.self_decoder(
input_ids=input_ids,
positions=self.positions[:batch_size],
inputs_embeds=inputs_embeds,
per_layer_inputs=per_layer_inputs,
**kwargs,
)
if logits_indices_padded is None:
logits_indices_padded = torch.arange(
@@ -889,11 +917,14 @@ class Gemma3nTextModel(nn.Module, SupportsQuant):
# Copy inputs for cudagraph
num_padded_logits_indices = logits_indices_padded.size(0)
self.positions[:num_padded_logits_indices].copy_(
positions[logits_indices_padded])
positions[logits_indices_padded]
)
self.hidden_states[:num_padded_logits_indices].copy_(
self_decoder_hidden_states[logits_indices_padded])
self_decoder_hidden_states[logits_indices_padded]
)
self.per_layer_inputs[:num_padded_logits_indices].copy_(
per_layer_inputs_adjusted[logits_indices_padded])
per_layer_inputs_adjusted[logits_indices_padded]
)
cross_decoder_hidden_states = self.cross_decoder(
positions=self.positions[:num_padded_logits_indices],
hidden_states=self.hidden_states[:num_padded_logits_indices],
@@ -905,7 +936,8 @@ class Gemma3nTextModel(nn.Module, SupportsQuant):
assert num_logits_indices > 0
# Merge cross-decoder and self-decoder hidden states
hidden_states[logits_indices_padded[:num_logits_indices]] = (
cross_decoder_hidden_states[:num_logits_indices])
cross_decoder_hidden_states[:num_logits_indices]
)
else:
hidden_states = cross_decoder_hidden_states
@@ -939,17 +971,19 @@ class Gemma3nTextModel(nn.Module, SupportsQuant):
hidden_states: torch.Tensor,
) -> torch.Tensor:
# Altup unembed.
target_magnitude = torch.mean(hidden_states[..., 0]**2,
dim=-1,
keepdim=True)**0.5
target_magnitude = (
torch.mean(hidden_states[..., 0] ** 2, dim=-1, keepdim=True) ** 0.5
)
for i in range(1, self.config.altup_num_inputs):
hidden_states[..., i] = self.altup_unembed_projections[i - 1](
hidden_states[..., i])
new_magnitude = torch.mean(hidden_states[..., i]**2,
dim=-1,
keepdim=True)**0.5
hidden_states[..., i]
)
new_magnitude = (
torch.mean(hidden_states[..., i] ** 2, dim=-1, keepdim=True) ** 0.5
)
hidden_states[..., i] *= target_magnitude / torch.maximum(
new_magnitude, EPS)
new_magnitude, EPS
)
# [num_tokens,hidden_size, altup_num_inputs] -> [num_tokens,hidden_size]
hidden_states = torch.mean(hidden_states, dim=-1)
return hidden_states
@@ -982,8 +1016,7 @@ class Gemma3nTextModel(nn.Module, SupportsQuant):
hidden_states = self.altup_unembed(hidden_states)
return self.norm(hidden_states)
def load_weights(self, weights: Iterable[tuple[str,
torch.Tensor]]) -> set[str]:
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
# (param_name, shard_name, shard_id)
("qkv_proj", "q_proj", "q"),
@@ -997,22 +1030,24 @@ class Gemma3nTextModel(nn.Module, SupportsQuant):
for name, loaded_weight in weights:
# decoder layer weights, altup_unembed_projections and rmsnorm
# are initialized in text model, others are in self decoder
if (not name.startswith('layers')
and not name.startswith('altup_unembed_projections')
and not name.startswith('norm')):
if (
not name.startswith("layers")
and not name.startswith("altup_unembed_projections")
and not name.startswith("norm")
):
name = f"self_decoder.{name}"
if (self.quant_config is not None and
(scale_name := self.quant_config.get_cache_scale(name))):
if self.quant_config is not None and (
scale_name := self.quant_config.get_cache_scale(name)
):
# Loading kv cache scales for compressed-tensors quantization
param = params_dict[scale_name]
weight_loader = getattr(param, "weight_loader",
default_weight_loader)
weight_loader = getattr(param, "weight_loader", default_weight_loader)
loaded_weight = loaded_weight[0]
weight_loader(param, loaded_weight)
loaded_params.add(scale_name)
continue
for (param_name, shard_name, shard_id) in stacked_params_mapping:
for param_name, shard_name, shard_id in stacked_params_mapping:
if shard_name not in name:
continue
# Avoid spurious match with ".up_proj".
@@ -1039,8 +1074,7 @@ class Gemma3nTextModel(nn.Module, SupportsQuant):
if is_pp_missing_parameter(name, self):
continue
param = params_dict[name]
weight_loader = getattr(param, "weight_loader",
default_weight_loader)
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
@@ -1067,10 +1101,12 @@ class Gemma3nForCausalLM(nn.Module):
super().__init__()
self.config = config
self.cache_config = vllm_config.cache_config
self.model = Gemma3nTextModel(vllm_config=vllm_config,
prefix=maybe_prefix(prefix, "model"))
self.model = Gemma3nTextModel(
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
)
self.logits_processor = LogitsProcessor(
config.vocab_size, soft_cap=config.final_logit_softcapping)
config.vocab_size, soft_cap=config.final_logit_softcapping
)
def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.model.get_input_embeddings(input_ids)
@@ -1085,7 +1121,6 @@ class Gemma3nForCausalLM(nn.Module):
inputs_embeds: Optional[torch.Tensor] = None,
**kwargs,
) -> Union[torch.Tensor, IntermediateTensors]:
hidden_states = self.model(
input_ids,
positions,
@@ -1103,11 +1138,11 @@ class Gemma3nForCausalLM(nn.Module):
logits = self.logits_processor(self.model.embed_tokens, hidden_states)
return logits
def load_weights(self, weights: Iterable[tuple[str,
torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(self,
skip_substrs=([
"embed_audio.", "embed_vision.",
"audio_tower.", "vision_tower."
]))
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(
self,
skip_substrs=(
["embed_audio.", "embed_vision.", "audio_tower.", "vision_tower."]
),
)
return loader.load_weights(weights)