[CI] Fix "test_awq_load[gemma4-moe-*]" failure (#43296)

Signed-off-by: haosdent <haosdent@gmail.com>
This commit is contained in:
haosdent
2026-05-22 07:13:59 +00:00
committed by GitHub
parent 5ea76fa89a
commit 025d4f5cd2
2 changed files with 116 additions and 31 deletions
@@ -7,9 +7,13 @@ import pytest
import torch
from PIL import Image as PILImage
from vllm.model_executor.models.gemma4_mm import Gemma4ImagePixelInputs
from vllm.model_executor.models.gemma4_mm import (
Gemma4ForConditionalGeneration,
Gemma4ImagePixelInputs,
)
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import MultiModalFieldConfig
from vllm.utils.mem_constants import GiB_bytes
from ....conftest import ImageTestAssets
from ...utils import build_model_context
@@ -224,3 +228,60 @@ def test_limit_mm_per_prompt(
mm_items=processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs={},
)
# Regression guard for PR #43169 follow-up: the batched Gemma4 vision encoder
# admitted ``chunk ~= 53`` on a 22 GiB L4 with a 26B AWQ model loaded,
# allocating 2.43 GiB int64 inside
# ``F.one_hot(num_classes=position_embedding_size)`` and OOMing because only
# 2.41 GiB was actually free. The fix sizes ``chunk`` from currently-free GPU
# memory and counts the ``F.one_hot`` transient as the dominant cost.
_encoder_chunk = Gemma4ForConditionalGeneration._encoder_chunk
# Gemma4 vision config default (HF: configuration_gemma4.py).
_POSITION_EMBEDDING_SIZE = 10240
# Video frame: max_soft_tokens=70, pooling_kernel_size=2 -> 70 * 4 patches.
_VIDEO_PATCHES_PER_FRAME = 280
def test_encoder_chunk_tight_budget_fits_in_free():
free = 3 * GiB_bytes # L4 22 GiB after 26B AWQ load.
total = 22 * GiB_bytes
chunk = _encoder_chunk(
_VIDEO_PATCHES_PER_FRAME, free, total, _POSITION_EMBEDDING_SIZE
)
one_hot_bytes = chunk * _VIDEO_PATCHES_PER_FRAME * 2 * _POSITION_EMBEDDING_SIZE * 8
assert one_hot_bytes <= free // 2
def test_encoder_chunk_roomy_gpu_keeps_batching():
chunk = _encoder_chunk(
_VIDEO_PATCHES_PER_FRAME,
60 * GiB_bytes,
80 * GiB_bytes,
_POSITION_EMBEDDING_SIZE,
)
assert chunk > 8
def test_encoder_chunk_zero_patches_is_safe():
assert (
_encoder_chunk(0, 60 * GiB_bytes, 80 * GiB_bytes, _POSITION_EMBEDDING_SIZE) == 1
)
def test_encoder_chunk_zero_position_embedding_size_is_safe():
# Degenerate config: must not raise ZeroDivisionError.
assert (
_encoder_chunk(_VIDEO_PATCHES_PER_FRAME, 60 * GiB_bytes, 80 * GiB_bytes, 0) == 1
)
def test_encoder_chunk_no_free_memory_falls_back_to_one():
assert (
_encoder_chunk(
_VIDEO_PATCHES_PER_FRAME, 0, 22 * GiB_bytes, _POSITION_EMBEDDING_SIZE
)
== 1
)
+54 -30
View File
@@ -961,9 +961,6 @@ class Gemma4ForConditionalGeneration(
self.embed_vision = Gemma4MultimodalEmbedder(
config.vision_config, config.text_config
)
# Lazy-initialized on first encoder call (see _encoder_max_batch).
self._encoder_budget_bytes = 0
self._encoder_bytes_per_patch = 0
# ---- Audio tower (variants with audio_config) ----
if config.audio_config is not None:
@@ -1104,18 +1101,36 @@ class Gemma4ForConditionalGeneration(
)
return mm_input_by_modality
def _encoder_max_batch(self, patches_per_item: int) -> int:
"""Max items per encoder call given per-item patch count."""
if self._encoder_budget_bytes == 0:
total_mem = current_platform.get_device_total_memory()
self._encoder_budget_bytes = int(total_mem * 0.05)
logger.info(
"Encoder memory budget: %.1fGB (total=%.1fGB)",
self._encoder_budget_bytes / 1024**3,
total_mem / 1024**3,
)
cost = patches_per_item * self._encoder_bytes_per_patch
return max(1, self._encoder_budget_bytes // cost) if cost > 0 else 1
@staticmethod
def _encoder_chunk(
patches_per_item: int,
free_bytes: int,
total_bytes: int,
position_embedding_size: int,
) -> int:
"""Max chunk size whose F.one_hot transient fits in the budget.
The dominant transient inside HF's ``Gemma4VisionPatchEmbedder.
_position_embeddings`` is
``F.one_hot(clamped_positions, num_classes=position_embedding_size)``
with shape ``(chunk, patches, 2, position_embedding_size)``,
int64, plus its simultaneous cast to the position embedding
table dtype. That, not the encoder residual stream, sets peak
memory.
"""
if patches_per_item <= 0:
return 1
# Half of currently-free, capped at 10% of total so we leave room
# for the rest of profile_run / the subsequent encoder + pooler.
budget = min(free_bytes // 2, total_bytes // 10)
if budget <= 0:
return 1
# F.one_hot allocates (chunk, patches, 2, pos_emb_size) int64
# (the inner 2 is the (x, y) coordinate axis, 8 is sizeof(int64)).
# Outer 2x covers the int64 buffer and its concurrent bf16 cast
# plus the matmul output that live alongside it at peak.
cost = patches_per_item * 4 * position_embedding_size * 8
return max(1, budget // cost) if cost > 0 else 1
# ------------------------------------------------------------------ #
# Image processing
@@ -1136,7 +1151,8 @@ class Gemma4ForConditionalGeneration(
pixel_position_ids = image_input["pixel_position_ids"]
vt = self.vision_tower
pooling_k2 = self.config.vision_config.pooling_kernel_size**2
vision_cfg = self.config.vision_config
pooling_k2 = vision_cfg.pooling_kernel_size**2
# Concurrent requests with different image resolutions may
# arrive as a list of per-image tensors, while same-resolution
@@ -1153,10 +1169,18 @@ class Gemma4ForConditionalGeneration(
pp = pixel_position_ids[idx]
buckets.setdefault(pv.shape[0], []).append((idx, pv, pp))
# Encode each resolution bucket in memory-safe chunks.
# Encode each resolution bucket in memory-safe chunks. Re-read
# free memory per bucket because the previous bucket's encoder
# pass has already allocated activations we should account for.
last_hidden_states_map: dict[int, torch.Tensor] = {}
for patches, items in buckets.items():
max_batch_size = min(len(items), self._encoder_max_batch(patches))
free, total = current_platform.mem_get_info()
max_batch_size = min(
len(items),
self._encoder_chunk(
patches, free, total, vision_cfg.position_embedding_size
),
)
for chunk_idx in range(0, len(items), max_batch_size):
chunk_items = items[chunk_idx : chunk_idx + max_batch_size]
@@ -1247,7 +1271,8 @@ class Gemma4ForConditionalGeneration(
frame_counts = video_input["video_frame_counts"]
vt = self.vision_tower
pooling_k2 = self.config.vision_config.pooling_kernel_size**2
vision_cfg = self.config.vision_config
pooling_k2 = vision_cfg.pooling_kernel_size**2
target_dtype = self.embed_vision.embedding_projection.weight.dtype
if isinstance(frame_counts, torch.Tensor):
@@ -1256,13 +1281,20 @@ class Gemma4ForConditionalGeneration(
fc_list = list(frame_counts)
total_frames = pixel_values.shape[0]
free, total = current_platform.mem_get_info()
max_batch_size = min(
total_frames, self._encoder_max_batch(pixel_values.shape[1])
total_frames,
self._encoder_chunk(
pixel_values.shape[1],
free,
total,
vision_cfg.position_embedding_size,
),
)
padding_positions = (pixel_position_ids == -1).all(dim=-1)
# Encode frames in chunks bounded by _encoder_max_batch.
# Encode frames in chunks bounded by _encoder_chunk.
last_hidden_states_list: list[torch.Tensor] = []
for i in range(0, total_frames, max_batch_size):
pv_chunk = pixel_values[i : i + max_batch_size]
@@ -1538,15 +1570,7 @@ class Gemma4ForConditionalGeneration(
self,
ignore_unexpected_prefixes=ignore_prefixes,
)
loaded = loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
# Per-patch activation cost for dynamic encoder batch sizing.
vis_cfg = self.config.vision_config
self._encoder_bytes_per_patch = (
vis_cfg.hidden_size * 2 * vis_cfg.num_hidden_layers
)
return loaded
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
# ------------------------------------------------------------------ #
# LoRA / multimodal mapping