[Model] Add Gemma4 Unified (encoder-free) support (#44429)
Signed-off-by: Luciano Martins <lucianommartins@users.noreply.github.com> Co-authored-by: Luciano Martins <lucianommartins@users.noreply.github.com>
This commit is contained in:
co-authored by
Luciano Martins
parent
271328e256
commit
a248b45d05
@@ -121,7 +121,7 @@ class Gemma4ImagePixelInputs(TensorSchema):
|
||||
- np: Number of patches (max_patches = max_soft_tokens * pooling_kernel_size²)
|
||||
- pp: Patch pixels (patch_size² * 3)
|
||||
|
||||
The HF Gemma4ImageProcessor outputs pixel_values as
|
||||
The Gemma4 image processor outputs pixel_values as
|
||||
(batch, max_patches, patch_pixels) — already patchified with
|
||||
zero-padding for patches beyond the real image content.
|
||||
pixel_position_ids provides (x, y) coordinates per patch,
|
||||
@@ -341,6 +341,29 @@ class Gemma4ProcessingInfo(BaseProcessingInfo):
|
||||
)
|
||||
return PromptUpdateDetails.select_token_id(token_ids, processor.image_token_id)
|
||||
|
||||
@staticmethod
|
||||
def _compute_audio_num_tokens(
|
||||
num_samples: int, sampling_rate: int, audio_seq_length: int
|
||||
) -> int:
|
||||
"""Replicate the audio encoder's sequence-length arithmetic.
|
||||
|
||||
Mirrors: mel framing (_unfold in Gemma4AudioFeatureExtractor)
|
||||
followed by two Conv2d subsampling layers (kernel=3, stride=2,
|
||||
semicausal padding top=1, bottom=1), capped at audio_seq_length.
|
||||
"""
|
||||
frame_length = int(round(sampling_rate * 20.0 / 1000.0))
|
||||
hop_length = int(round(sampling_rate * 10.0 / 1000.0))
|
||||
frame_size_for_unfold = frame_length + 1
|
||||
pad_left = frame_length // 2
|
||||
padded_samples = num_samples + pad_left
|
||||
num_mel_frames = (padded_samples - frame_size_for_unfold) // hop_length + 1
|
||||
if num_mel_frames <= 0:
|
||||
return 0
|
||||
t = num_mel_frames
|
||||
for _ in range(2):
|
||||
t = (t + 2 - 3) // 2 + 1
|
||||
return min(t, audio_seq_length)
|
||||
|
||||
def get_audio_repl(
|
||||
self,
|
||||
*,
|
||||
@@ -350,20 +373,21 @@ class Gemma4ProcessingInfo(BaseProcessingInfo):
|
||||
"""Return the dynamic audio token sequence for this audio.
|
||||
|
||||
Computes the number of soft tokens from the audio waveform
|
||||
length using ``ceil(duration_ms / audio_ms_per_token)``.
|
||||
length by replicating the audio encoder's sequence-length
|
||||
arithmetic (mel framing + two Conv2d subsampling layers).
|
||||
"""
|
||||
if processor is None:
|
||||
processor = self.get_hf_processor()
|
||||
|
||||
sampling_rate = processor.feature_extractor.sampling_rate
|
||||
num_tokens = processor._compute_audio_num_tokens(
|
||||
torch.zeros(audio_len), sampling_rate
|
||||
num_tokens = self._compute_audio_num_tokens(
|
||||
audio_len, sampling_rate, processor.audio_seq_length
|
||||
)
|
||||
config = self.get_hf_config()
|
||||
token_ids = (
|
||||
[config.boa_token_id]
|
||||
+ [processor.audio_token_id] * num_tokens
|
||||
+ [config.eoa_token_id]
|
||||
+ [getattr(config, "eoa_token_id", config.eoa_token_index)]
|
||||
)
|
||||
return PromptUpdateDetails.select_token_id(token_ids, processor.audio_token_id)
|
||||
|
||||
@@ -988,18 +1012,35 @@ class Gemma4ForConditionalGeneration(
|
||||
self.quant_config = quant_config
|
||||
self.multimodal_config = multimodal_config
|
||||
|
||||
# Only quantize towers when the quant method supports their
|
||||
# dimensions. BNB/torchao handle arbitrary sizes; other methods
|
||||
# (Marlin, FP8, …) require dimensions divisible by 64, which
|
||||
# the vision tower (intermediate_size=4304) does not satisfy.
|
||||
if quant_config and quant_config.get_name() in [
|
||||
"bitsandbytes",
|
||||
"torchao",
|
||||
]:
|
||||
tower_quant = quant_config
|
||||
else:
|
||||
vision_cfg = config.vision_config
|
||||
quantizable = (
|
||||
vision_cfg.hidden_size % 64 == 0
|
||||
and vision_cfg.intermediate_size % 64 == 0
|
||||
)
|
||||
tower_quant = quant_config if quantizable else None
|
||||
|
||||
# ---- Vision tower (shared by image and video) ----
|
||||
with self._mark_tower_model(vllm_config, {"image", "video"}):
|
||||
self.vision_tower = AutoModel.from_config(config=config.vision_config)
|
||||
self.embed_vision = Gemma4MultimodalEmbedder(
|
||||
config.vision_config,
|
||||
config.text_config,
|
||||
quant_config=quant_config,
|
||||
quant_config=tower_quant,
|
||||
prefix=maybe_prefix(prefix, "embed_vision"),
|
||||
)
|
||||
recursive_replace_linear(
|
||||
self.vision_tower,
|
||||
quant_config,
|
||||
tower_quant,
|
||||
prefix=maybe_prefix(prefix, "vision_tower"),
|
||||
)
|
||||
|
||||
@@ -1015,12 +1056,12 @@ class Gemma4ForConditionalGeneration(
|
||||
self.embed_audio = Gemma4MultimodalEmbedder(
|
||||
config.audio_config,
|
||||
config.text_config,
|
||||
quant_config=quant_config,
|
||||
quant_config=tower_quant,
|
||||
prefix=maybe_prefix(prefix, "embed_audio"),
|
||||
)
|
||||
recursive_replace_linear(
|
||||
self.audio_tower,
|
||||
quant_config,
|
||||
tower_quant,
|
||||
prefix=maybe_prefix(prefix, "audio_tower"),
|
||||
)
|
||||
else:
|
||||
@@ -1039,13 +1080,13 @@ class Gemma4ForConditionalGeneration(
|
||||
# Pre-allocate PLE buffer for CUDA graph compatibility.
|
||||
# Some variants have hidden_size_per_layer_input=None (no PLE).
|
||||
ple_dim = config.text_config.hidden_size_per_layer_input
|
||||
if ple_dim is not None:
|
||||
if ple_dim is not None and ple_dim > 0:
|
||||
self.per_layer_embeddings = torch.zeros(
|
||||
vllm_config.scheduler_config.max_num_batched_tokens,
|
||||
config.text_config.num_hidden_layers,
|
||||
ple_dim,
|
||||
device=(self.language_model.model.embed_tokens.weight.device),
|
||||
dtype=(self.language_model.model.embed_tokens.weight.dtype),
|
||||
device=self.language_model.model.embed_tokens.weight.device,
|
||||
dtype=self.language_model.model.embed_tokens.weight.dtype,
|
||||
)
|
||||
else:
|
||||
self.per_layer_embeddings = None
|
||||
@@ -1076,6 +1117,9 @@ class Gemma4ForConditionalGeneration(
|
||||
self.num_shared_experts = self.language_model.num_shared_experts
|
||||
self.num_redundant_experts = self.language_model.num_redundant_experts
|
||||
|
||||
gen_cfg = vllm_config.model_config.try_get_generation_config()
|
||||
self._suppress_token_ids = gen_cfg.get("suppress_tokens") if gen_cfg else None
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# Input parsing
|
||||
# ------------------------------------------------------------------ #
|
||||
@@ -1424,8 +1468,7 @@ class Gemma4ForConditionalGeneration(
|
||||
input_features = audio_input["input_features_padded"].squeeze(1)
|
||||
input_features_mask = audio_input["input_features_mask"].squeeze(1)
|
||||
|
||||
# Run audio tower — mask uses standard HF convention
|
||||
# (True=valid, False=padding).
|
||||
# Run audio tower — mask convention: True=valid, False=padding.
|
||||
audio_outputs = self.audio_tower(input_features, input_features_mask)
|
||||
if isinstance(audio_outputs, tuple):
|
||||
audio_encodings, audio_mask = audio_outputs
|
||||
@@ -1436,8 +1479,8 @@ class Gemma4ForConditionalGeneration(
|
||||
# Project into LM embedding space.
|
||||
audio_features = self.embed_audio(inputs_embeds=audio_encodings)
|
||||
|
||||
# Strip padding per-batch element: only keep real (non-padding)
|
||||
# tokens. audio_mask is True for valid positions (HF convention).
|
||||
# Strip padding per-batch element: only keep valid (non-padding)
|
||||
# tokens.
|
||||
per_audio = []
|
||||
for enc, mask in zip(audio_features, audio_mask, strict=True):
|
||||
per_audio.append(enc[mask]) # [num_real, hidden_size]
|
||||
@@ -1559,7 +1602,10 @@ class Gemma4ForConditionalGeneration(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
) -> torch.Tensor | None:
|
||||
return self.language_model.compute_logits(hidden_states)
|
||||
logits = self.language_model.compute_logits(hidden_states)
|
||||
if logits is not None and self._suppress_token_ids:
|
||||
logits[:, self._suppress_token_ids] = -float("inf")
|
||||
return logits
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# Bidirectional attention helpers
|
||||
@@ -1617,8 +1663,7 @@ class Gemma4ForConditionalGeneration(
|
||||
"embed_vision.embedding.",
|
||||
"embed_audio.embedding.",
|
||||
]
|
||||
# Models without audio tower should skip
|
||||
# audio weights entirely.
|
||||
# Models without audio tower should skip audio weights entirely.
|
||||
if self.audio_tower is None:
|
||||
ignore_prefixes.extend(
|
||||
[
|
||||
|
||||
Reference in New Issue
Block a user