[Model] Add Gemma4 Unified (encoder-free) support (#44429)

Signed-off-by: Luciano Martins <lucianommartins@users.noreply.github.com>
Co-authored-by: Luciano Martins <lucianommartins@users.noreply.github.com>
This commit is contained in:
Luciano Martins
2026-06-03 12:01:39 -07:00
committed by GitHub
co-authored by Luciano Martins
parent 271328e256
commit a248b45d05
14 changed files with 791 additions and 31 deletions
+64 -19
View File
@@ -121,7 +121,7 @@ class Gemma4ImagePixelInputs(TensorSchema):
- np: Number of patches (max_patches = max_soft_tokens * pooling_kernel_size²)
- pp: Patch pixels (patch_size² * 3)
The HF Gemma4ImageProcessor outputs pixel_values as
The Gemma4 image processor outputs pixel_values as
(batch, max_patches, patch_pixels) — already patchified with
zero-padding for patches beyond the real image content.
pixel_position_ids provides (x, y) coordinates per patch,
@@ -341,6 +341,29 @@ class Gemma4ProcessingInfo(BaseProcessingInfo):
)
return PromptUpdateDetails.select_token_id(token_ids, processor.image_token_id)
@staticmethod
def _compute_audio_num_tokens(
num_samples: int, sampling_rate: int, audio_seq_length: int
) -> int:
"""Replicate the audio encoder's sequence-length arithmetic.
Mirrors: mel framing (_unfold in Gemma4AudioFeatureExtractor)
followed by two Conv2d subsampling layers (kernel=3, stride=2,
semicausal padding top=1, bottom=1), capped at audio_seq_length.
"""
frame_length = int(round(sampling_rate * 20.0 / 1000.0))
hop_length = int(round(sampling_rate * 10.0 / 1000.0))
frame_size_for_unfold = frame_length + 1
pad_left = frame_length // 2
padded_samples = num_samples + pad_left
num_mel_frames = (padded_samples - frame_size_for_unfold) // hop_length + 1
if num_mel_frames <= 0:
return 0
t = num_mel_frames
for _ in range(2):
t = (t + 2 - 3) // 2 + 1
return min(t, audio_seq_length)
def get_audio_repl(
self,
*,
@@ -350,20 +373,21 @@ class Gemma4ProcessingInfo(BaseProcessingInfo):
"""Return the dynamic audio token sequence for this audio.
Computes the number of soft tokens from the audio waveform
length using ``ceil(duration_ms / audio_ms_per_token)``.
length by replicating the audio encoder's sequence-length
arithmetic (mel framing + two Conv2d subsampling layers).
"""
if processor is None:
processor = self.get_hf_processor()
sampling_rate = processor.feature_extractor.sampling_rate
num_tokens = processor._compute_audio_num_tokens(
torch.zeros(audio_len), sampling_rate
num_tokens = self._compute_audio_num_tokens(
audio_len, sampling_rate, processor.audio_seq_length
)
config = self.get_hf_config()
token_ids = (
[config.boa_token_id]
+ [processor.audio_token_id] * num_tokens
+ [config.eoa_token_id]
+ [getattr(config, "eoa_token_id", config.eoa_token_index)]
)
return PromptUpdateDetails.select_token_id(token_ids, processor.audio_token_id)
@@ -988,18 +1012,35 @@ class Gemma4ForConditionalGeneration(
self.quant_config = quant_config
self.multimodal_config = multimodal_config
# Only quantize towers when the quant method supports their
# dimensions. BNB/torchao handle arbitrary sizes; other methods
# (Marlin, FP8, …) require dimensions divisible by 64, which
# the vision tower (intermediate_size=4304) does not satisfy.
if quant_config and quant_config.get_name() in [
"bitsandbytes",
"torchao",
]:
tower_quant = quant_config
else:
vision_cfg = config.vision_config
quantizable = (
vision_cfg.hidden_size % 64 == 0
and vision_cfg.intermediate_size % 64 == 0
)
tower_quant = quant_config if quantizable else None
# ---- Vision tower (shared by image and video) ----
with self._mark_tower_model(vllm_config, {"image", "video"}):
self.vision_tower = AutoModel.from_config(config=config.vision_config)
self.embed_vision = Gemma4MultimodalEmbedder(
config.vision_config,
config.text_config,
quant_config=quant_config,
quant_config=tower_quant,
prefix=maybe_prefix(prefix, "embed_vision"),
)
recursive_replace_linear(
self.vision_tower,
quant_config,
tower_quant,
prefix=maybe_prefix(prefix, "vision_tower"),
)
@@ -1015,12 +1056,12 @@ class Gemma4ForConditionalGeneration(
self.embed_audio = Gemma4MultimodalEmbedder(
config.audio_config,
config.text_config,
quant_config=quant_config,
quant_config=tower_quant,
prefix=maybe_prefix(prefix, "embed_audio"),
)
recursive_replace_linear(
self.audio_tower,
quant_config,
tower_quant,
prefix=maybe_prefix(prefix, "audio_tower"),
)
else:
@@ -1039,13 +1080,13 @@ class Gemma4ForConditionalGeneration(
# Pre-allocate PLE buffer for CUDA graph compatibility.
# Some variants have hidden_size_per_layer_input=None (no PLE).
ple_dim = config.text_config.hidden_size_per_layer_input
if ple_dim is not None:
if ple_dim is not None and ple_dim > 0:
self.per_layer_embeddings = torch.zeros(
vllm_config.scheduler_config.max_num_batched_tokens,
config.text_config.num_hidden_layers,
ple_dim,
device=(self.language_model.model.embed_tokens.weight.device),
dtype=(self.language_model.model.embed_tokens.weight.dtype),
device=self.language_model.model.embed_tokens.weight.device,
dtype=self.language_model.model.embed_tokens.weight.dtype,
)
else:
self.per_layer_embeddings = None
@@ -1076,6 +1117,9 @@ class Gemma4ForConditionalGeneration(
self.num_shared_experts = self.language_model.num_shared_experts
self.num_redundant_experts = self.language_model.num_redundant_experts
gen_cfg = vllm_config.model_config.try_get_generation_config()
self._suppress_token_ids = gen_cfg.get("suppress_tokens") if gen_cfg else None
# ------------------------------------------------------------------ #
# Input parsing
# ------------------------------------------------------------------ #
@@ -1424,8 +1468,7 @@ class Gemma4ForConditionalGeneration(
input_features = audio_input["input_features_padded"].squeeze(1)
input_features_mask = audio_input["input_features_mask"].squeeze(1)
# Run audio tower — mask uses standard HF convention
# (True=valid, False=padding).
# Run audio tower — mask convention: True=valid, False=padding.
audio_outputs = self.audio_tower(input_features, input_features_mask)
if isinstance(audio_outputs, tuple):
audio_encodings, audio_mask = audio_outputs
@@ -1436,8 +1479,8 @@ class Gemma4ForConditionalGeneration(
# Project into LM embedding space.
audio_features = self.embed_audio(inputs_embeds=audio_encodings)
# Strip padding per-batch element: only keep real (non-padding)
# tokens. audio_mask is True for valid positions (HF convention).
# Strip padding per-batch element: only keep valid (non-padding)
# tokens.
per_audio = []
for enc, mask in zip(audio_features, audio_mask, strict=True):
per_audio.append(enc[mask]) # [num_real, hidden_size]
@@ -1559,7 +1602,10 @@ class Gemma4ForConditionalGeneration(
self,
hidden_states: torch.Tensor,
) -> torch.Tensor | None:
return self.language_model.compute_logits(hidden_states)
logits = self.language_model.compute_logits(hidden_states)
if logits is not None and self._suppress_token_ids:
logits[:, self._suppress_token_ids] = -float("inf")
return logits
# ------------------------------------------------------------------ #
# Bidirectional attention helpers
@@ -1617,8 +1663,7 @@ class Gemma4ForConditionalGeneration(
"embed_vision.embedding.",
"embed_audio.embedding.",
]
# Models without audio tower should skip
# audio weights entirely.
# Models without audio tower should skip audio weights entirely.
if self.audio_tower is None:
ignore_prefixes.extend(
[