[Model] Support EmbeddingGemma2 multimodal pooling architecture (#60254)

Signed-off-by: Luciano Martins <lucianommartins@users.noreply.github.com>
Co-authored-by: Luciano Martins <lucianommartins@users.noreply.github.com>
This commit is contained in:
Luciano Martins
2026-10-06 13:43:38 -04:00
committed by GitHub
co-authored by Luciano Martins
parent 7b665f7a58
commit 02b83919aa
7 changed files with 2122 additions and 2 deletions
@@ -0,0 +1,687 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import ast
import inspect
import os
import textwrap
import pytest
import torch
MODEL = os.environ.get("EG2_MODEL_PATH")
needs_model = pytest.mark.skipif(not MODEL, reason="EG2_MODEL_PATH not set")
TEXT_COS, MM_COS = 0.999, 0.999
def _cos(a, b):
return torch.nn.functional.cosine_similarity(
torch.tensor(a, dtype=torch.float32),
torch.tensor(b, dtype=torch.float32),
dim=-1,
)
def _ensure_synthetic_video(
path: str, duration: float, fps: float, width: int, height: int
) -> str:
if os.path.exists(path):
os.remove(path)
import cv2
import numpy as np
fourcc_fn = getattr(cv2, "VideoWriter_fourcc", cv2.VideoWriter.fourcc)
fourcc = fourcc_fn(*"mp4v")
num_frames = int(duration * fps)
writer = cv2.VideoWriter(path, fourcc, fps, (width, height))
y_coords, x_coords = np.mgrid[0:height, 0:width]
for i in range(num_frames):
# Spatially varying color pattern that evolves dynamically over frames
frame: np.ndarray = np.zeros((height, width, 3), dtype=np.uint8)
frame[..., 0] = ((x_coords * 255 // max(1, width)) + i * 17) % 256
frame[..., 1] = ((y_coords * 255 // max(1, height)) + i * 29) % 256
frame[..., 2] = (x_coords + y_coords + i * 13) % 256
writer.write(frame)
writer.release()
return path
@pytest.fixture(scope="module")
def st_model():
from sentence_transformers import SentenceTransformer
return SentenceTransformer(
MODEL, device="cuda", model_kwargs={"torch_dtype": torch.bfloat16}
)
@needs_model
def test_structure(vllm_runner, monkeypatch):
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
with vllm_runner(MODEL, runner="pooling", max_model_len=8192) as vm:
def probe(model):
lm = model.language_model
win = [layer.self_attn.attn.impl.sliding_window for layer in lm.layers]
return win, model.config.text_config.sliding_window
win, cfg_w = vm.apply_model(probe)[0]
assert (cfg_w, cfg_w) in win # (512, 512) == W-1 with W = 513
@needs_model
@pytest.mark.parametrize(
"prompt",
[
"short query",
"task: search result | query: what is vllm",
"word " * 700, # > 512: crosses sliding window
"word " * 1500, # > 1024
],
)
def test_text_parity(vllm_runner, st_model, prompt):
ref = st_model.encode([prompt], normalize_embeddings=True)
with vllm_runner(MODEL, runner="pooling", max_model_len=8192) as vm:
out = vm.embed([prompt])
sim = _cos(out, ref).min().item()
assert sim >= TEXT_COS, (
f"Text cosine {sim} < {TEXT_COS} for prompt len {len(prompt)}"
)
@needs_model
def test_mm_recipe(vllm_runner, st_model):
from PIL import Image
img = Image.new("RGB", (224, 224), color=(255, 0, 0))
prompt = "Describe this image: <|image|>"
# SentenceTransformer multimodal encode
ref = st_model.encode([{"image": img, "text": prompt}], normalize_embeddings=True)
with vllm_runner(MODEL, runner="pooling", max_model_len=8192) as vm:
out = vm.embed([{"prompt": prompt, "multi_modal_data": {"image": img}}])
sim = _cos(out, ref).min().item()
assert sim >= MM_COS, f"Image cosine {sim} < {MM_COS}"
@needs_model
def test_natural_image_parity(vllm_runner, st_model):
from PIL import Image
img_path = "/projects/gemma4-vllm/images/cat.jpg"
if not os.path.exists(img_path):
pytest.skip(f"Test asset {img_path} not found")
img = Image.open(img_path)
prompt = "What is this image? <|image|>"
ref = st_model.encode([{"image": img, "text": prompt}], normalize_embeddings=True)
with vllm_runner(MODEL, runner="pooling", max_model_len=8192) as vm:
out = vm.embed([{"prompt": prompt, "multi_modal_data": {"image": img}}])
sim = _cos(out, ref).min().item()
assert sim >= MM_COS, f"Natural image cosine {sim} < {MM_COS}"
@needs_model
def test_audio_parity(vllm_runner, st_model):
import torchaudio
audio_path = "/projects/gemma4-vllm/audio/tellMeAboutTheSun.wav"
if not os.path.exists(audio_path):
pytest.skip(f"Test asset {audio_path} not found")
wav, sr = torchaudio.load(audio_path)
mono_wav = wav.mean(dim=0).numpy()
prompt = "Describe this audio: <|audio|>"
ref = st_model.encode(
[{"audio": audio_path, "text": prompt}], normalize_embeddings=True
)
with vllm_runner(MODEL, runner="pooling", max_model_len=8192) as vm:
out = vm.embed(
[{"prompt": prompt, "multi_modal_data": {"audio": (mono_wav, sr)}}]
)
sim = _cos(out, ref).min().item()
assert sim >= MM_COS, f"Audio cosine {sim} < {MM_COS}"
@needs_model
def test_mixed_parity(vllm_runner, st_model):
import torchaudio
from PIL import Image
img_path = "/projects/gemma4-vllm/images/cat.jpg"
audio_path = "/projects/gemma4-vllm/audio/tellMeAboutTheSun.wav"
if not os.path.exists(img_path) or not os.path.exists(audio_path):
pytest.skip("Test assets for mixed parity not found")
img = Image.open(img_path)
wav, sr = torchaudio.load(audio_path)
mono_wav = wav.mean(dim=0).numpy()
prompt = "Describe image <|image|> and audio <|audio|>"
ref = st_model.encode(
[{"image": img, "audio": audio_path, "text": prompt}],
normalize_embeddings=True,
)
with vllm_runner(MODEL, runner="pooling", max_model_len=8192) as vm:
out = vm.embed(
[
{
"prompt": prompt,
"multi_modal_data": {
"image": img,
"audio": (mono_wav, sr),
},
}
]
)
sim = _cos(out, ref).min().item()
assert sim >= MM_COS, f"Mixed cosine {sim} < {MM_COS}"
@needs_model
def test_video_parity_uniform_overflow(vllm_runner, st_model):
from transformers.video_utils import load_video
video_path = _ensure_synthetic_video(
"/tmp/test_40s.mp4", duration=40.0, fps=10.0, width=160, height=120
)
frames, meta = load_video(video_path, backend="opencv")
prompt = "Describe this video: <|video|>"
ref = st_model.encode(
[{"video": video_path, "text": prompt}], normalize_embeddings=True
)
with vllm_runner(MODEL, runner="pooling", max_model_len=8192) as vm:
out = vm.embed(
[{"prompt": prompt, "multi_modal_data": {"video": (frames, meta)}}]
)
sim = _cos(out, ref).min().item()
assert sim >= MM_COS, f"40s video cosine {sim} < {MM_COS}"
@needs_model
def test_video_parity_portrait(vllm_runner, st_model):
from transformers.video_utils import load_video
video_path = _ensure_synthetic_video(
"/tmp/test_portrait.mp4", duration=3.0, fps=10.0, width=120, height=160
)
frames, meta = load_video(video_path, backend="opencv")
prompt = "Describe this video: <|video|>"
ref = st_model.encode(
[{"video": video_path, "text": prompt}], normalize_embeddings=True
)
with vllm_runner(MODEL, runner="pooling", max_model_len=8192) as vm:
out = vm.embed(
[{"prompt": prompt, "multi_modal_data": {"video": (frames, meta)}}]
)
sim = _cos(out, ref).min().item()
assert sim >= MM_COS, f"Portrait video cosine {sim} < {MM_COS}"
@needs_model
def test_tower_skip(vllm_runner, monkeypatch):
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
def nparams(m):
return sum(p.numel() for p in m.parameters()) / 1e6
with vllm_runner(
MODEL,
runner="pooling",
max_model_len=2048,
limit_mm_per_prompt={"image": 0, "audio": 0, "video": 0},
) as vm:
p_text = vm.apply_model(nparams)[0]
assert abs(p_text - 271.00) <= 0.05, (
f"Expected 271.00M text params, got {p_text}M"
)
with vllm_runner(MODEL, runner="pooling", max_model_len=2048) as vm:
p_full = vm.apply_model(nparams)[0]
assert abs(p_full - 744.37) <= 0.05, (
f"Expected 744.37M full params, got {p_full}M"
)
def test_encoder_contract():
from vllm.model_executor.models.embedding_gemma2 import EmbeddingGemma2Model
from vllm.model_executor.models.gemma4_mm import (
Gemma4ForConditionalGeneration as G,
)
borrowed_methods = [
"_parse_and_validate_multimodal_inputs",
"_parse_and_validate_image_input",
"_parse_and_validate_video_input",
"_parse_and_validate_audio_input",
"_process_image_input",
"_process_video_input",
"_process_audio_input",
"embed_multimodal",
"_encoder_chunk",
]
for name in borrowed_methods:
# Check method identity (L-F6 + T-F4)
assert getattr(EmbeddingGemma2Model, name) is getattr(G, name)
# AST checks: verify no zero-arg super() calls in borrowed methods
func = getattr(EmbeddingGemma2Model, name)
src = textwrap.dedent(inspect.getsource(func))
tree = ast.parse(src)
for node in ast.walk(tree):
if (
isinstance(node, ast.Call)
and isinstance(node.func, ast.Name)
and node.func.id == "super"
):
assert len(node.args) > 0, (
f"Zero-arg super() call found in borrowed method {name}"
)
# AST check: all self.<attr> non-callable accesses are assigned in __init__
init_src = inspect.getsource(EmbeddingGemma2Model.__init__)
for name in borrowed_methods:
func = getattr(EmbeddingGemma2Model, name)
src = textwrap.dedent(inspect.getsource(func))
tree = ast.parse(src)
for node in ast.walk(tree):
if (
isinstance(node, ast.Attribute)
and isinstance(node.value, ast.Name)
and node.value.id == "self"
):
attr = node.attr
assert hasattr(EmbeddingGemma2Model, attr) or attr in init_src, (
f"Attribute {attr} accessed on self in {name} is neither "
f"defined on class nor in __init__"
)
@needs_model
def test_instance_attribute_access(vllm_runner):
"""Check self.<attr> accesses on a real model instance."""
from vllm.model_executor.models.embedding_gemma2 import EmbeddingGemma2Model
borrowed_methods = [
"_parse_and_validate_multimodal_inputs",
"_parse_and_validate_image_input",
"_parse_and_validate_video_input",
"_parse_and_validate_audio_input",
"_process_image_input",
"_process_video_input",
"_process_audio_input",
"embed_multimodal",
"_encoder_chunk",
]
attrs_accessed = set()
for name in borrowed_methods:
func = getattr(EmbeddingGemma2Model, name)
src = textwrap.dedent(inspect.getsource(func))
tree = ast.parse(src)
for node in ast.walk(tree):
if (
isinstance(node, ast.Attribute)
and isinstance(node.value, ast.Name)
and node.value.id == "self"
):
attrs_accessed.add(node.attr)
def check_attrs(model):
for attr in attrs_accessed:
assert hasattr(model, attr), (
f"Real instance of EmbeddingGemma2Model is missing attribute '{attr}' "
f"accessed by borrowed method"
)
return True
with vllm_runner(MODEL, runner="pooling", max_model_len=2048) as vm:
res = vm.apply_model(check_attrs)
assert res[0] is True
@needs_model
def test_video_loader_real_video(vllm_runner, st_model):
"""Decode a real MP4 file through EmbeddingGemma2VideoBackend.load_bytes
and VideoMediaIO / MediaConnector with default num_frames=32.
"""
from vllm.multimodal.media import MediaConnector
from vllm.multimodal.media.video import VideoMediaIO
from vllm.multimodal.video import VIDEO_LOADER_REGISTRY
video_path = "/projects/gemma4-vllm/videos/test_intel.mp4"
backend_cls = VIDEO_LOADER_REGISTRY.name2class["embedding_gemma2"]
with open(video_path, "rb") as f:
data = f.read()
# Direct backend load
frames, meta = backend_cls.load_bytes(data)
# VideoMediaIO with default num_frames=32 (VideoMediaIO passes num_frames=32)
video_io = VideoMediaIO(image_io=None, video_backend="embedding_gemma2")
loaded = video_io.load_bytes(data)
frames_io, meta_io = loaded.media
assert len(frames_io) == len(frames)
assert meta_io.get("do_sample_frames") is False
# MediaConnector with video_processor="EmbeddingGemma2VideoProcessor"
connector = MediaConnector(allowed_local_media_path="/projects/gemma4-vllm/videos")
fetched = connector.fetch_video(
f"file://{video_path}",
video_processor="EmbeddingGemma2VideoProcessor",
)
frames_conn, meta_conn = fetched.media
assert len(frames_conn) == 32
assert meta_conn.get("do_sample_frames") is False
prompt = "Describe this video: <|video|>"
ref = st_model.encode(
[{"video": video_path, "text": prompt}], normalize_embeddings=True
)
with vllm_runner(MODEL, runner="pooling", max_model_len=8192) as vm:
out = vm.embed(
[
{
"prompt": prompt,
"multi_modal_data": {"video": (frames_conn, meta_conn)},
}
]
)
sim = _cos(out, ref).min().item()
assert sim >= MM_COS, f"Real video loader cosine {sim} < {MM_COS}"
@needs_model
def test_video_loader_duplicate_expansion_e2e(vllm_runner, st_model):
"""End-to-end duplicate expansion test with low-fps clip."""
import numpy as np
from vllm.multimodal.video import VIDEO_LOADER_REGISTRY
backend_cls = VIDEO_LOADER_REGISTRY.name2class["embedding_gemma2"]
# 4s clip at 0.5 fps = 2 unique frames
lowfps_path = "/tmp/test_lowfps_05.mp4"
_ensure_synthetic_video(lowfps_path, duration=4.0, fps=0.5, width=64, height=48)
with open(lowfps_path, "rb") as f:
data = f.read()
# Default fps=1.0 -> 4 frames expanding 2 unique frames to [0, 0, 1, 1]
frames, meta = backend_cls.load_bytes(data)
assert frames.shape[0] == 4
# Compare returned frames directly, not only indices
assert np.array_equal(frames[0], frames[1])
assert not np.array_equal(frames[0], frames[2])
assert np.array_equal(frames[2], frames[3])
prompt = "Describe this video: <|video|>"
with vllm_runner(MODEL, runner="pooling", max_model_len=8192) as vm:
out = vm.embed(
[{"prompt": prompt, "multi_modal_data": {"video": (frames, meta)}}]
)
ref = st_model.encode(
[{"video": lowfps_path, "text": prompt}], normalize_embeddings=True
)
sim = _cos(out, ref).min().item()
assert sim >= MM_COS, f"Duplicate expansion cosine {sim} < {MM_COS}"
assert len(out) == 1
assert len(out[0]) == 768
def test_video_loader_indices():
from transformers.models.embedding_gemma2.video_processing_embedding_gemma2 import (
EmbeddingGemma2VideoProcessor,
)
from transformers.video_utils import VideoMetadata
from vllm.multimodal.video import (
VIDEO_LOADER_REGISTRY,
VideoSourceMetadata,
VideoTargetMetadata,
)
backend_cls = VIDEO_LOADER_REGISTRY.name2class["embedding_gemma2"]
hf_proc = EmbeddingGemma2VideoProcessor()
# Case 1: clip shorter than 32s (e.g. 5s @ 24fps = 120 frames)
src_5s = VideoSourceMetadata(total_frames_num=120, original_fps=24.0, duration=5.0)
tgt = VideoTargetMetadata(num_frames=-1, fps=1.0, max_duration=300)
vllm_indices = backend_cls.compute_frames_index_to_sample(src_5s, tgt)
hf_meta_5s = VideoMetadata(fps=24.0, duration=5.0, total_num_frames=120)
hf_indices_5s = hf_proc.sample_frames(
hf_meta_5s, fps=1.0, max_frames=32, overflow_strategy="uniform"
).tolist()
assert vllm_indices == hf_indices_5s
# Case 2: clip shorter than 1s (e.g. 0.5s @ 24fps = 12 frames)
src_sub1s = VideoSourceMetadata(
total_frames_num=12, original_fps=24.0, duration=0.5
)
vllm_sub1s = backend_cls.compute_frames_index_to_sample(src_sub1s, tgt)
hf_meta_sub1s = VideoMetadata(fps=24.0, duration=0.5, total_num_frames=12)
hf_indices_sub1s = hf_proc.sample_frames(
hf_meta_sub1s, fps=1.0, max_frames=32, overflow_strategy="uniform"
).tolist()
assert vllm_sub1s == hf_indices_sub1s
# Case 3: clip longer than 32s (e.g. 40s @ 10fps = 400 frames)
# -> uniform overflow cap 32 frames
src_40s = VideoSourceMetadata(
total_frames_num=400, original_fps=10.0, duration=40.0
)
vllm_40s = backend_cls.compute_frames_index_to_sample(src_40s, tgt)
hf_meta_40s = VideoMetadata(fps=10.0, duration=40.0, total_num_frames=400)
hf_indices_40s = hf_proc.sample_frames(
hf_meta_40s, fps=1.0, max_frames=32, overflow_strategy="uniform"
).tolist()
assert vllm_40s == hf_indices_40s
assert len(vllm_40s) == 32
# Case 4: default target fps (target.fps <= 0 defaults to 1.0)
tgt_default_fps = VideoTargetMetadata(num_frames=-1, fps=-1.0, max_duration=300)
vllm_def = backend_cls.compute_frames_index_to_sample(src_5s, tgt_default_fps)
assert vllm_def == hf_indices_5s
# Case 5 (NEW-2): 60 fps / 300 s (18000 frames) vs HF sample_frames
src_300s = VideoSourceMetadata(
total_frames_num=18000, original_fps=60.0, duration=300.0
)
v_300s = backend_cls.compute_frames_index_to_sample(
src_300s, VideoTargetMetadata(num_frames=-1, fps=60.0, max_duration=300)
)
h_300s = hf_proc.sample_frames(
VideoMetadata(fps=60.0, duration=300.0, total_num_frames=18000),
fps=60.0,
max_frames=32,
overflow_strategy="uniform",
).tolist()
assert v_300s == h_300s, f"60fps/300s mismatch: {v_300s[-3:]} vs {h_300s[-3:]}"
# Case 6 (NEW-2): 1 fps / 4 h (432000 frames @ 30 native fps) vs HF sample_frames
src_4h = VideoSourceMetadata(
total_frames_num=432000, original_fps=30.0, duration=14400.0
)
v_4h = backend_cls.compute_frames_index_to_sample(
src_4h, VideoTargetMetadata(num_frames=-1, fps=1.0, max_duration=300)
)
h_4h = hf_proc.sample_frames(
VideoMetadata(fps=30.0, duration=14400.0, total_num_frames=432000),
fps=1.0,
max_frames=32,
overflow_strategy="uniform",
).tolist()
assert v_4h == h_4h, f"1fps/4h mismatch: {v_4h[-2:]} vs {h_4h[-2:]}"
def test_video_loader_duplicate_expansion():
from transformers.models.embedding_gemma2.video_processing_embedding_gemma2 import (
EmbeddingGemma2VideoProcessor,
)
from transformers.video_utils import VideoMetadata
from vllm.multimodal.video import (
VIDEO_LOADER_REGISTRY,
VideoSourceMetadata,
VideoTargetMetadata,
)
backend_cls = VIDEO_LOADER_REGISTRY.name2class["embedding_gemma2"]
hf_proc = EmbeddingGemma2VideoProcessor()
# 4s clip @ 0.5fps = 2 frames total. Requested target fps = 1.0 -> 4 frames.
# Source has fewer frames than requested sampled frames, requiring duplicates.
src_lowfps = VideoSourceMetadata(total_frames_num=2, original_fps=0.5, duration=4.0)
tgt = VideoTargetMetadata(num_frames=-1, fps=1.0, max_duration=300)
vllm_indices = backend_cls.compute_frames_index_to_sample(src_lowfps, tgt)
hf_meta = VideoMetadata(fps=0.5, duration=4.0, total_num_frames=2)
hf_indices = hf_proc.sample_frames(
hf_meta, fps=1.0, max_frames=32, overflow_strategy="uniform"
).tolist()
assert vllm_indices == hf_indices
assert len(vllm_indices) == 4
assert vllm_indices == [0, 0, 1, 1]
def test_video_media_io_embedding_gemma2():
"""Verify VideoMediaIO with default num_frames=32 loads video bytes for
EmbeddingGemma2 without error.
"""
import os
from unittest.mock import MagicMock
from vllm.multimodal.media import MediaConnector
from vllm.multimodal.media.video import VideoMediaIO
video_path = "/projects/gemma4-vllm/videos/test_intel.mp4"
if not os.path.exists(video_path):
return
with open(video_path, "rb") as f:
data = f.read()
# VideoMediaIO defaults to num_frames=32
video_io = VideoMediaIO(image_io=MagicMock(), video_backend="embedding_gemma2")
loaded = video_io.load_bytes(data)
frames, meta = loaded.media
assert len(frames) == 32
assert meta.get("do_sample_frames") is False
# MediaConnector path
connector = MediaConnector(allowed_local_media_path="/projects/gemma4-vllm/videos")
fetched = connector.fetch_video(
f"file://{video_path}",
video_processor="EmbeddingGemma2VideoProcessor",
)
frames_conn, meta_conn = fetched.media
assert len(frames_conn) == 32
assert meta_conn.get("do_sample_frames") is False
def test_config_capping_and_explicit_limits(caplog):
"""Verify EmbeddingGemma2ModelConfig capping and explicit user overrides."""
import logging
from types import SimpleNamespace
from unittest.mock import patch
from vllm.model_executor.models.config import (
EmbeddingGemma2ModelConfig,
Gemma4Config,
)
from vllm.v1.attention.backends.registry import AttentionBackendEnum
# (a) Defaults: uncapped max_model_len=262144 -> capped to 8192,
# and derived scheduler limits (>= 262144) lowered to 8192.
cfg_default = SimpleNamespace(
attention_config=SimpleNamespace(backend=None),
model_config=SimpleNamespace(
model="dummy", max_model_len=262144, original_max_model_len=None
),
scheduler_config=SimpleNamespace(
max_num_batched_tokens=262144,
max_num_encoder_input_tokens=262144,
encoder_cache_size=262144,
max_num_seqs=256,
verify_max_model_len=lambda x: None,
),
)
with patch.object(Gemma4Config, "verify_and_update_config"):
EmbeddingGemma2ModelConfig.verify_and_update_config(cfg_default)
assert cfg_default.attention_config.backend == AttentionBackendEnum.TRITON_ATTN
assert cfg_default.model_config.max_model_len == 8192
assert cfg_default.scheduler_config.max_num_batched_tokens == 8192
assert cfg_default.scheduler_config.max_num_encoder_input_tokens == 8192
assert cfg_default.scheduler_config.encoder_cache_size == 8192
# (b) Reachable explicit limits: explicit max_model_len=8192 and explicit
# max_num_batched_tokens=32768 stays 32768.
# Note: omitting max_model_len while passing max_num_batched_tokens < 262144
# is rejected earlier by SchedulerConfig validation (32768 < 262144) before
# this hook runs.
cfg_explicit_batched = SimpleNamespace(
attention_config=SimpleNamespace(backend=None),
model_config=SimpleNamespace(
model="dummy", max_model_len=8192, original_max_model_len=8192
),
scheduler_config=SimpleNamespace(
max_num_batched_tokens=32768,
max_num_encoder_input_tokens=32768,
encoder_cache_size=32768,
max_num_seqs=256,
verify_max_model_len=lambda x: None,
),
)
with patch.object(Gemma4Config, "verify_and_update_config"):
EmbeddingGemma2ModelConfig.verify_and_update_config(cfg_explicit_batched)
assert cfg_explicit_batched.model_config.max_model_len == 8192
assert cfg_explicit_batched.scheduler_config.max_num_batched_tokens == 32768
assert cfg_explicit_batched.scheduler_config.max_num_encoder_input_tokens == 32768
assert cfg_explicit_batched.scheduler_config.encoder_cache_size == 32768
# (c) Explicit value > uncapped_len (e.g. 300000) with uncapped model len is lowered
# to 8192 and logs a warning because it exceeds uncapped_len.
cfg_explicit_large = SimpleNamespace(
attention_config=SimpleNamespace(backend=None),
model_config=SimpleNamespace(
model="dummy", max_model_len=262144, original_max_model_len=None
),
scheduler_config=SimpleNamespace(
max_num_batched_tokens=300000,
max_num_encoder_input_tokens=300000,
encoder_cache_size=300000,
max_num_seqs=256,
verify_max_model_len=lambda x: None,
),
)
with (
caplog.at_level(logging.WARNING),
patch.object(Gemma4Config, "verify_and_update_config"),
):
EmbeddingGemma2ModelConfig.verify_and_update_config(cfg_explicit_large)
assert cfg_explicit_large.model_config.max_model_len == 8192
assert cfg_explicit_large.scheduler_config.max_num_batched_tokens == 8192
assert cfg_explicit_large.scheduler_config.max_num_encoder_input_tokens == 8192
assert cfg_explicit_large.scheduler_config.encoder_cache_size == 8192
assert any(
"lowering explicitly set scheduler_config.max_num_batched_tokens"
" from 300000 to 8192" in rec.message
for rec in caplog.records
)
# (d) Explicit max_model_len=16384 is respected and not capped to 8192
cfg_explicit_model_len = SimpleNamespace(
attention_config=SimpleNamespace(backend=None),
model_config=SimpleNamespace(
model="dummy", max_model_len=16384, original_max_model_len=16384
),
scheduler_config=SimpleNamespace(
max_num_batched_tokens=16384,
max_num_encoder_input_tokens=16384,
encoder_cache_size=16384,
max_num_seqs=256,
verify_max_model_len=lambda x: None,
),
)
with patch.object(Gemma4Config, "verify_and_update_config"):
EmbeddingGemma2ModelConfig.verify_and_update_config(cfg_explicit_model_len)
assert cfg_explicit_model_len.model_config.max_model_len == 16384
assert cfg_explicit_model_len.scheduler_config.max_num_batched_tokens == 16384
+5
View File
@@ -581,6 +581,11 @@ _EMBEDDING_EXAMPLE_MODELS = {
trust_remote_code=True, trust_remote_code=True,
hf_overrides={"model_type": "deepseek_v3", "auto_map": None}, hf_overrides={"model_type": "deepseek_v3", "auto_map": None},
), ),
"EmbeddingGemma2Model": _HfExamplesInfo(
"google/embeddinggemma-2",
min_transformers_version="5.18.0",
is_available_online=False,
),
"Gemma2Model": _HfExamplesInfo("BAAI/bge-multilingual-gemma2"), "Gemma2Model": _HfExamplesInfo("BAAI/bge-multilingual-gemma2"),
"Gemma3TextModel": _HfExamplesInfo("google/embeddinggemma-300m"), "Gemma3TextModel": _HfExamplesInfo("google/embeddinggemma-300m"),
"GteModel": _HfExamplesInfo( "GteModel": _HfExamplesInfo(
+106
View File
@@ -282,6 +282,111 @@ class Gemma4Config(VerifyAndUpdateConfig):
) )
class EmbeddingGemma2ModelConfig(Gemma4Config):
@staticmethod
def verify_and_update_config(vllm_config: "VllmConfig") -> None:
Gemma4Config.verify_and_update_config(vllm_config)
from vllm.v1.attention.backends.registry import AttentionBackendEnum
attn_config = vllm_config.attention_config
if attn_config.backend is None:
attn_config.backend = AttentionBackendEnum.TRITON_ATTN
logger.info(
"EmbeddingGemma2: attention backend not specified; defaulting to "
"TRITON_ATTN (validated backend for heterogeneous head dimensions)."
)
else:
logger.info(
"EmbeddingGemma2: using attention backend %s", attn_config.backend
)
model_config = vllm_config.model_config
orig_len = getattr(model_config, "original_max_model_len", None)
if orig_len in (None, -1):
default_len = 8192
# Check sentence_bert_config.json if available
try:
import json
from pathlib import Path
cfg_path = Path(model_config.model) / "sentence_bert_config.json"
if cfg_path.exists():
data = json.loads(cfg_path.read_text())
if "max_seq_length" in data and isinstance(
data["max_seq_length"], int
):
default_len = data["max_seq_length"]
except Exception:
pass
if model_config.max_model_len > default_len:
uncapped_len = model_config.max_model_len
logger.info(
"EmbeddingGemma2: max_model_len not explicitly set; capping from "
"%d to %d.",
model_config.max_model_len,
default_len,
)
model_config.max_model_len = default_len
scheduler_config = vllm_config.scheduler_config
if scheduler_config is not None:
new_batched_tokens = max(
model_config.max_model_len, scheduler_config.max_num_seqs
)
if scheduler_config.max_num_batched_tokens >= uncapped_len:
if scheduler_config.max_num_batched_tokens > uncapped_len:
logger.warning(
"EmbeddingGemma2: lowering explicitly set "
"scheduler_config.max_num_batched_tokens from %d to %d "
"to match capped max_model_len.",
scheduler_config.max_num_batched_tokens,
new_batched_tokens,
)
else:
logger.info(
"EmbeddingGemma2: lowering "
"scheduler_config.max_num_batched_tokens from %d to %d "
"to match capped max_model_len.",
scheduler_config.max_num_batched_tokens,
new_batched_tokens,
)
scheduler_config.max_num_batched_tokens = new_batched_tokens
if hasattr(vllm_config, "_set_compile_ranges"):
vllm_config._set_compile_ranges()
if scheduler_config.max_num_encoder_input_tokens >= uncapped_len:
if scheduler_config.max_num_encoder_input_tokens > uncapped_len:
logger.warning(
"EmbeddingGemma2: lowering explicitly set "
"scheduler_config.max_num_encoder_input_tokens "
"from %d to %d to match capped max_model_len.",
scheduler_config.max_num_encoder_input_tokens,
new_batched_tokens,
)
scheduler_config.max_num_encoder_input_tokens = (
new_batched_tokens
)
if scheduler_config.encoder_cache_size >= uncapped_len:
if scheduler_config.encoder_cache_size > uncapped_len:
logger.warning(
"EmbeddingGemma2: lowering explicitly set "
"scheduler_config.encoder_cache_size from %d to %d "
"to match capped max_model_len.",
scheduler_config.encoder_cache_size,
new_batched_tokens,
)
scheduler_config.encoder_cache_size = new_batched_tokens
if hasattr(scheduler_config, "verify_max_model_len"):
scheduler_config.verify_max_model_len(
model_config.max_model_len
)
elif model_config.max_model_len > 32768:
logger.warning(
"EmbeddingGemma2: max_model_len=%d is large; "
"consider --max-model-len 8192 to bound encoder memory.",
model_config.max_model_len,
)
class DiffusionGemmaModelForBlockDiffusionConfig(VerifyAndUpdateConfig): class DiffusionGemmaModelForBlockDiffusionConfig(VerifyAndUpdateConfig):
@classmethod @classmethod
def verify_and_update_config(cls, vllm_config: "VllmConfig") -> None: def verify_and_update_config(cls, vllm_config: "VllmConfig") -> None:
@@ -1025,6 +1130,7 @@ MODELS_CONFIG_MAP: dict[str, type[VerifyAndUpdateConfig]] = {
"DeepseekV41ForCausalLM": DeepseekV4ForCausalLMConfig, "DeepseekV41ForCausalLM": DeepseekV4ForCausalLMConfig,
"DeepseekV32ForCausalLM": DeepseekV32ForCausalLM, "DeepseekV32ForCausalLM": DeepseekV32ForCausalLM,
"DiffusionGemmaForBlockDiffusion": DiffusionGemmaModelForBlockDiffusionConfig, # noqa: E501 "DiffusionGemmaForBlockDiffusion": DiffusionGemmaModelForBlockDiffusionConfig, # noqa: E501
"EmbeddingGemma2Model": EmbeddingGemma2ModelConfig,
"Ernie4_5_VLMoeForConditionalGeneration": Ernie4_5_VLMoeForConditionalGenerationConfig, # noqa: E501 "Ernie4_5_VLMoeForConditionalGeneration": Ernie4_5_VLMoeForConditionalGenerationConfig, # noqa: E501
"FalconMambaForCausalLM": MambaModelConfig, "FalconMambaForCausalLM": MambaModelConfig,
"Gemma3TextModel": Gemma3TextModelConfig, "Gemma3TextModel": Gemma3TextModelConfig,
File diff suppressed because it is too large Load Diff
+1
View File
@@ -246,6 +246,7 @@ _EMBEDDING_MODELS = {
# [Multimodal] # [Multimodal]
"CLIPModel": ("clip", "CLIPEmbeddingModel"), "CLIPModel": ("clip", "CLIPEmbeddingModel"),
"ColPaliForRetrieval": ("colpali", "ColPaliModel"), "ColPaliForRetrieval": ("colpali", "ColPaliModel"),
"EmbeddingGemma2Model": ("embedding_gemma2", "EmbeddingGemma2Model"),
"LlamaNemotronVLModel": ("nemotron_vl", "LlamaNemotronVLForEmbedding"), "LlamaNemotronVLModel": ("nemotron_vl", "LlamaNemotronVLForEmbedding"),
"LlavaNextForConditionalGeneration": ( "LlavaNextForConditionalGeneration": (
"llava_next", "llava_next",
+178
View File
@@ -405,6 +405,184 @@ class Qwen3VLVideoBackend(VideoBackend):
) )
@VIDEO_LOADER_REGISTRY.register(
"embedding_gemma2",
video_processor="EmbeddingGemma2VideoProcessor",
)
class EmbeddingGemma2VideoBackend(VideoBackend):
"""Samples frames with the same indices as HF EmbeddingGemma2VideoProcessor."""
PROCESSOR_NAMES = ("EmbeddingGemma2Processor", "EmbeddingGemma2VideoProcessor")
_MAX_ALLOWED_FRAMES: ClassVar[int] = 128
_MAX_ALLOWED_FPS: ClassVar[float] = 60.0
@classmethod
def _get_processor_defaults(cls, **kwargs) -> tuple[float, int]:
"""Read default fps and max_frames from video_processor if available."""
proc = kwargs.get("video_processor") or kwargs.get("processor")
if proc is not None:
vp = getattr(proc, "video_processor", proc)
fps = getattr(vp, "fps", None)
max_frames = getattr(vp, "max_frames", None)
if fps is not None and max_frames is not None:
return float(fps), int(max_frames)
try:
from transformers.models.embedding_gemma2 import (
EmbeddingGemma2VideoProcessor,
)
vp = EmbeddingGemma2VideoProcessor()
return float(getattr(vp, "fps", 1.0)), int(getattr(vp, "max_frames", 32))
except Exception:
return 1.0, 32
@classmethod
def compute_frames_index_to_sample(
cls,
source: VideoSourceMetadata,
target: VideoTargetMetadata,
**kwargs,
) -> list[int]:
# Ignores target.num_frames (EmbeddingGemma2 does not accept num_frames)
default_fps, default_max_frames = cls._get_processor_defaults(**kwargs)
fps = kwargs.get("fps")
if fps is None:
fps = target.fps if target.fps > 0 else default_fps
if fps is not None:
if fps <= 0:
raise ValueError(f"fps must be positive, got {fps}")
if fps > cls._MAX_ALLOWED_FPS:
raise ValueError(
f"fps {fps} exceeds maximum allowed {cls._MAX_ALLOWED_FPS}"
)
max_frames = kwargs.get("max_frames", default_max_frames)
if max_frames is not None:
if max_frames <= 0:
raise ValueError(f"max_frames must be positive, got {max_frames}")
if max_frames > cls._MAX_ALLOWED_FRAMES:
raise ValueError(
f"max_frames {max_frames} exceeds maximum allowed "
f"{cls._MAX_ALLOWED_FRAMES}"
)
overflow_strategy = kwargs.get("overflow_strategy", "uniform")
total_frames = source.total_frames_num
original_fps = source.original_fps
duration = source.duration
if fps is not None and (original_fps <= 0 or duration <= 0):
fps = None
if overflow_strategy is not None:
if max_frames is None:
raise ValueError(
"You must pass max_frames when requesting an "
f"overflow_strategy={overflow_strategy}!"
)
if overflow_strategy not in ("truncate", "uniform"):
raise ValueError(
f"You passed overflow_strategy={overflow_strategy} but "
"expected one of ['truncate', 'uniform']"
)
if fps is None:
if overflow_strategy is None or total_frames <= max_frames:
indices = np.arange(total_frames, dtype=int)
elif overflow_strategy == "truncate":
indices = np.arange(max_frames, dtype=int)
elif overflow_strategy == "uniform":
indices = np.linspace(0, total_frames - 1, max_frames, dtype=int)
else:
step = original_fps / fps
num_sampled = max(1, int(duration * fps))
if overflow_strategy is None or num_sampled <= max_frames:
i_arr = np.arange(num_sampled, dtype=float)
indices = np.minimum(total_frames - 1, (i_arr * step).astype(int))
elif overflow_strategy == "truncate":
i_arr = np.arange(max_frames, dtype=float)
indices = np.minimum(total_frames - 1, (i_arr * step).astype(int))
elif overflow_strategy == "uniform":
linspace_idx = np.linspace(0, num_sampled - 1, max_frames, dtype=int)
indices = np.minimum(
total_frames - 1, (linspace_idx * step).astype(int)
)
return indices.tolist()
@classmethod
def load_bytes(
cls,
data: bytes,
num_frames: int = -1,
fps: int | float = -1,
max_duration: int = 300,
frame_recovery: bool = False,
*,
backend: VideoDecoderBackend = "opencv",
**kwargs: Any,
) -> tuple[DecodedFrames, dict[str, Any]]:
# Accept num_frames as required by VideoLoader interface (VideoMediaIO
# passes default num_frames=32), but ignore it since EmbeddingGemma2 uses
# fps/max_frames. Force num_frames=-1 so compute_frames_index_to_sample
# drives sampling via fps/duration.
kwargs.pop("num_frames", None)
frames, metadata = super().load_bytes(
data,
num_frames=-1,
fps=fps, # type: ignore[arg-type]
max_duration=max_duration,
frame_recovery=frame_recovery,
backend=backend,
**kwargs,
)
source = VideoSourceMetadata(
total_frames_num=metadata["total_num_frames"],
original_fps=metadata["fps"],
duration=metadata["duration"],
)
target = VideoTargetMetadata(num_frames=-1, fps=fps, max_duration=max_duration)
sampling_kwargs, _ = resolve_video_backend_kwargs(backend, kwargs)
requested_indices = cls.compute_frames_index_to_sample(
source, target, **sampling_kwargs
)
# Expand deduplicated frames back to requested_indices if needed
# (e.g., OpenCV deduplicates frame_idx using set(frame_idx))
valid_frame_indices = metadata.get("frames_indices")
if valid_frame_indices is not None and requested_indices != valid_frame_indices:
idx_to_pos = {idx: pos for pos, idx in enumerate(valid_frame_indices)}
if all(idx in idx_to_pos for idx in requested_indices):
positions = [idx_to_pos[idx] for idx in requested_indices]
if isinstance(frames, np.ndarray):
frames = frames[positions]
else:
frames = frames[torch.tensor(positions, device=frames.device)]
metadata["frames_indices"] = list(requested_indices)
return frames, metadata
@classmethod
def create_hf_metadata(
cls,
source: VideoSourceMetadata,
valid_frame_indices: list[int],
video_backend: str,
) -> dict[str, Any]:
meta = super().create_hf_metadata(
source=source,
valid_frame_indices=valid_frame_indices,
video_backend=video_backend,
)
meta["do_sample_frames"] = False
return meta
@VIDEO_LOADER_REGISTRY.register( @VIDEO_LOADER_REGISTRY.register(
"qwen2_vl", "qwen2_vl",
video_processor="Qwen2VLVideoProcessor", video_processor="Qwen2VLVideoProcessor",
@@ -243,9 +243,10 @@ def context_attention_fwd(
b_seq_len: [b] b_seq_len: [b]
out: [b * s, head, head_dim] out: [b * s, head, head_dim]
""" """
BLOCK = get_block_size(q.dtype)
Lq, Lk, _ = q.shape[-1], k.shape[-1], v.shape[-1] Lq, Lk, _ = q.shape[-1], k.shape[-1], v.shape[-1]
BLOCK = get_block_size(q.dtype)
if Lk >= 512:
BLOCK = min(BLOCK, 32)
sm_scale = 1.0 / (Lq**0.5) if softmax_scale is None else softmax_scale sm_scale = 1.0 / (Lq**0.5) if softmax_scale is None else softmax_scale
# rescale with 1/ln(2) for triton exp2 # rescale with 1/ln(2) for triton exp2