[Bugfix][Model] Gemma3n/Gemma4: pad variable-length audio batches (#50958)

Signed-off-by: TrainToGPB <kaiba0514@gmail.com>
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
TrainToGPB
2026-08-05 12:46:00 +08:00
committed by GitHub
co-authored by Claude
parent c25093a305
commit eb3dce9757
4 changed files with 165 additions and 11 deletions
+56 -4
View File
@@ -83,6 +83,52 @@ class Gemma3nImagePixelInputs(TensorSchema):
pixel_values: Annotated[torch.Tensor, TensorShape("bn", 3, "h", "w")]
def batch_audio_features(
input_features: torch.Tensor | list[torch.Tensor],
input_features_mask: torch.Tensor | list[torch.Tensor],
) -> tuple[torch.Tensor, torch.Tensor]:
"""Return mel features and their validity mask as batched tensors.
Audio features are unpadded per item so that a multimodal cache entry does
not depend on the batch it was first processed in.
[`MultiModalFieldConfig.batched`][vllm.multimodal.inputs.MultiModalFieldConfig.batched]
stacks items only when their shapes agree, so a batch of clips with
differing durations reaches the model as a list and must be re-padded here.
Padded frames are zero-filled and marked invalid. Audio towers consume the
mask, and callers keep only masked-in positions, so padding never reaches
the language model.
Gemma4 shares this helper: its audio path has the same unpad/re-pad
contract and the same fields.
Args:
input_features: `(bn, s, f)` tensor, or a list of `(s_i, f)` tensors
when clip durations differ.
input_features_mask: Matching `(bn, s)` tensor, or list of `(s_i,)`
tensors. `True` marks a valid frame.
Returns:
The `(bn, s_max, f)` features and their `(bn, s_max)` mask.
"""
if isinstance(input_features, torch.Tensor):
return input_features.squeeze(1), input_features_mask.squeeze(1)
max_len = max(features.shape[0] for features in input_features)
batched_features = input_features[0].new_zeros(
(len(input_features), max_len, input_features[0].shape[-1])
)
batched_mask = input_features_mask[0].new_zeros(
(len(input_features_mask), max_len), dtype=torch.bool
)
for i, (features, mask) in enumerate(
zip(input_features, input_features_mask, strict=True)
):
batched_features[i, : features.shape[0]] = features
batched_mask[i, : mask.shape[0]] = mask
return batched_features, batched_mask
class Gemma3nAudioInputs(TensorSchema):
"""
Dimensions:
@@ -92,8 +138,12 @@ class Gemma3nAudioInputs(TensorSchema):
"""
type: Literal["audio"] = "audio"
input_features_padded: Annotated[torch.Tensor, TensorShape("bn", "s", "f")]
input_features_mask: Annotated[torch.Tensor, TensorShape("bn", "s")]
input_features_padded: Annotated[
torch.Tensor, TensorShape("bn", "s", "f", dynamic_dims={"s"})
]
input_features_mask: Annotated[
torch.Tensor, TensorShape("bn", "s", dynamic_dims={"s"})
]
Gemma3nImageInputs = Gemma3nImagePixelInputs
@@ -615,8 +665,10 @@ class Gemma3nForConditionalGeneration(
audio_input: Gemma3nAudioInputs,
) -> list[torch.Tensor]:
# Run on padded features to enable batching
input_features = audio_input["input_features_padded"].squeeze(1)
input_features_mask = audio_input["input_features_mask"].squeeze(1)
input_features, input_features_mask = batch_audio_features(
audio_input["input_features_padded"],
audio_input["input_features_mask"],
)
audio_outputs = self.audio_tower(input_features, ~input_features_mask)
audio_encodings = audio_outputs.last_hidden_state
audio_mask = audio_outputs.audio_mel_mask