[Fix] Gemma4 Mixed-Resolution Image Co-Batching Crash (#42217)
Signed-off-by: zitian.zhao <zitian.zhao@tencentmusic.com>
This commit is contained in:
@@ -4,9 +4,12 @@
|
||||
from collections.abc import Mapping
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from PIL import Image as PILImage
|
||||
|
||||
from vllm.model_executor.models.gemma4_mm import Gemma4ImagePixelInputs
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import MultiModalFieldConfig
|
||||
|
||||
from ....conftest import ImageTestAssets
|
||||
from ...utils import build_model_context
|
||||
@@ -15,6 +18,36 @@ from ...utils import build_model_context
|
||||
GEMMA4_MODEL_ID = "google/gemma-4-E2B-it"
|
||||
|
||||
|
||||
def test_gemma4_image_schema_accepts_variable_patch_counts():
|
||||
Gemma4ImagePixelInputs(
|
||||
pixel_values=[
|
||||
torch.randn(10080, 768),
|
||||
torch.randn(2520, 768),
|
||||
],
|
||||
pixel_position_ids=[
|
||||
torch.zeros(10080, 2, dtype=torch.long),
|
||||
torch.zeros(2520, 2, dtype=torch.long),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def test_gemma4_image_batching_keeps_variable_patch_counts_unstacked():
|
||||
field = MultiModalFieldConfig.batched("image").field
|
||||
elems = field.build_elems(
|
||||
"image",
|
||||
"pixel_values",
|
||||
[torch.randn(10080, 768), torch.randn(2520, 768)],
|
||||
)
|
||||
|
||||
reduced = field.reduce_data(list(elems))
|
||||
|
||||
assert isinstance(reduced, list)
|
||||
assert [tensor.shape for tensor in reduced] == [
|
||||
torch.Size([10080, 768]),
|
||||
torch.Size([2520, 768]),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"image_width,image_height,max_soft_tokens",
|
||||
[
|
||||
|
||||
@@ -124,12 +124,12 @@ class Gemma4ImagePixelInputs(TensorSchema):
|
||||
|
||||
type: Literal["pixel_values"] = "pixel_values"
|
||||
pixel_values: Annotated[
|
||||
torch.Tensor,
|
||||
TensorShape("bn", "np", "pp"),
|
||||
torch.Tensor | list[torch.Tensor],
|
||||
TensorShape("bn", "np", "pp", dynamic_dims={"np"}),
|
||||
]
|
||||
pixel_position_ids: Annotated[
|
||||
torch.Tensor,
|
||||
TensorShape("bn", "np", 2),
|
||||
torch.Tensor | list[torch.Tensor],
|
||||
TensorShape("bn", "np", 2, dynamic_dims={"np"}),
|
||||
]
|
||||
|
||||
|
||||
@@ -1128,15 +1128,20 @@ class Gemma4ForConditionalGeneration(
|
||||
# metadata, and validating numerical equivalence with the
|
||||
# current per-image path.
|
||||
#
|
||||
# Concurrent requests with different image resolutions may
|
||||
# arrive as a list of per-image tensors, while same-resolution
|
||||
# batches may arrive as a stacked tensor. Both forms are
|
||||
# iterable over the per-image dimension.
|
||||
|
||||
# Process each image individually through the vision tower.
|
||||
# The vision tower's forward() strips padding and returns a
|
||||
# flat tensor of valid tokens. We process per-image to get
|
||||
# variable-length outputs matching the dynamic token count
|
||||
# from get_image_repl.
|
||||
per_image_features = []
|
||||
for i in range(pixel_values.shape[0]):
|
||||
pv = pixel_values[i].unsqueeze(0) # (1, max_patches, patch_pixels)
|
||||
pp = pixel_position_ids[i].unsqueeze(0) # (1, max_patches, 2)
|
||||
for pv, pp in zip(pixel_values, pixel_position_ids, strict=True):
|
||||
pv = pv.unsqueeze(0) # (1, max_patches, patch_pixels)
|
||||
pp = pp.unsqueeze(0) # (1, max_patches, 2)
|
||||
|
||||
# Derive the pooler's output_length from the total patch
|
||||
# count (including padding). The vision tower encoder
|
||||
|
||||
Reference in New Issue
Block a user