[Model] Support EmbeddingGemma2 multimodal pooling architecture (#60254)
Signed-off-by: Luciano Martins <lucianommartins@users.noreply.github.com> Co-authored-by: Luciano Martins <lucianommartins@users.noreply.github.com>
This commit is contained in:
co-authored by
Luciano Martins
parent
7b665f7a58
commit
02b83919aa
@@ -0,0 +1,687 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
|
import ast
|
||||||
|
import inspect
|
||||||
|
import os
|
||||||
|
import textwrap
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
import torch
|
||||||
|
|
||||||
|
MODEL = os.environ.get("EG2_MODEL_PATH")
|
||||||
|
needs_model = pytest.mark.skipif(not MODEL, reason="EG2_MODEL_PATH not set")
|
||||||
|
TEXT_COS, MM_COS = 0.999, 0.999
|
||||||
|
|
||||||
|
|
||||||
|
def _cos(a, b):
|
||||||
|
return torch.nn.functional.cosine_similarity(
|
||||||
|
torch.tensor(a, dtype=torch.float32),
|
||||||
|
torch.tensor(b, dtype=torch.float32),
|
||||||
|
dim=-1,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _ensure_synthetic_video(
|
||||||
|
path: str, duration: float, fps: float, width: int, height: int
|
||||||
|
) -> str:
|
||||||
|
if os.path.exists(path):
|
||||||
|
os.remove(path)
|
||||||
|
import cv2
|
||||||
|
import numpy as np
|
||||||
|
|
||||||
|
fourcc_fn = getattr(cv2, "VideoWriter_fourcc", cv2.VideoWriter.fourcc)
|
||||||
|
fourcc = fourcc_fn(*"mp4v")
|
||||||
|
num_frames = int(duration * fps)
|
||||||
|
writer = cv2.VideoWriter(path, fourcc, fps, (width, height))
|
||||||
|
y_coords, x_coords = np.mgrid[0:height, 0:width]
|
||||||
|
for i in range(num_frames):
|
||||||
|
# Spatially varying color pattern that evolves dynamically over frames
|
||||||
|
frame: np.ndarray = np.zeros((height, width, 3), dtype=np.uint8)
|
||||||
|
frame[..., 0] = ((x_coords * 255 // max(1, width)) + i * 17) % 256
|
||||||
|
frame[..., 1] = ((y_coords * 255 // max(1, height)) + i * 29) % 256
|
||||||
|
frame[..., 2] = (x_coords + y_coords + i * 13) % 256
|
||||||
|
writer.write(frame)
|
||||||
|
writer.release()
|
||||||
|
return path
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope="module")
|
||||||
|
def st_model():
|
||||||
|
from sentence_transformers import SentenceTransformer
|
||||||
|
|
||||||
|
return SentenceTransformer(
|
||||||
|
MODEL, device="cuda", model_kwargs={"torch_dtype": torch.bfloat16}
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@needs_model
|
||||||
|
def test_structure(vllm_runner, monkeypatch):
|
||||||
|
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
|
||||||
|
with vllm_runner(MODEL, runner="pooling", max_model_len=8192) as vm:
|
||||||
|
|
||||||
|
def probe(model):
|
||||||
|
lm = model.language_model
|
||||||
|
win = [layer.self_attn.attn.impl.sliding_window for layer in lm.layers]
|
||||||
|
return win, model.config.text_config.sliding_window
|
||||||
|
|
||||||
|
win, cfg_w = vm.apply_model(probe)[0]
|
||||||
|
assert (cfg_w, cfg_w) in win # (512, 512) == W-1 with W = 513
|
||||||
|
|
||||||
|
|
||||||
|
@needs_model
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"prompt",
|
||||||
|
[
|
||||||
|
"short query",
|
||||||
|
"task: search result | query: what is vllm",
|
||||||
|
"word " * 700, # > 512: crosses sliding window
|
||||||
|
"word " * 1500, # > 1024
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_text_parity(vllm_runner, st_model, prompt):
|
||||||
|
ref = st_model.encode([prompt], normalize_embeddings=True)
|
||||||
|
with vllm_runner(MODEL, runner="pooling", max_model_len=8192) as vm:
|
||||||
|
out = vm.embed([prompt])
|
||||||
|
sim = _cos(out, ref).min().item()
|
||||||
|
assert sim >= TEXT_COS, (
|
||||||
|
f"Text cosine {sim} < {TEXT_COS} for prompt len {len(prompt)}"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@needs_model
|
||||||
|
def test_mm_recipe(vllm_runner, st_model):
|
||||||
|
from PIL import Image
|
||||||
|
|
||||||
|
img = Image.new("RGB", (224, 224), color=(255, 0, 0))
|
||||||
|
prompt = "Describe this image: <|image|>"
|
||||||
|
|
||||||
|
# SentenceTransformer multimodal encode
|
||||||
|
ref = st_model.encode([{"image": img, "text": prompt}], normalize_embeddings=True)
|
||||||
|
with vllm_runner(MODEL, runner="pooling", max_model_len=8192) as vm:
|
||||||
|
out = vm.embed([{"prompt": prompt, "multi_modal_data": {"image": img}}])
|
||||||
|
sim = _cos(out, ref).min().item()
|
||||||
|
assert sim >= MM_COS, f"Image cosine {sim} < {MM_COS}"
|
||||||
|
|
||||||
|
|
||||||
|
@needs_model
|
||||||
|
def test_natural_image_parity(vllm_runner, st_model):
|
||||||
|
from PIL import Image
|
||||||
|
|
||||||
|
img_path = "/projects/gemma4-vllm/images/cat.jpg"
|
||||||
|
if not os.path.exists(img_path):
|
||||||
|
pytest.skip(f"Test asset {img_path} not found")
|
||||||
|
img = Image.open(img_path)
|
||||||
|
prompt = "What is this image? <|image|>"
|
||||||
|
ref = st_model.encode([{"image": img, "text": prompt}], normalize_embeddings=True)
|
||||||
|
with vllm_runner(MODEL, runner="pooling", max_model_len=8192) as vm:
|
||||||
|
out = vm.embed([{"prompt": prompt, "multi_modal_data": {"image": img}}])
|
||||||
|
sim = _cos(out, ref).min().item()
|
||||||
|
assert sim >= MM_COS, f"Natural image cosine {sim} < {MM_COS}"
|
||||||
|
|
||||||
|
|
||||||
|
@needs_model
|
||||||
|
def test_audio_parity(vllm_runner, st_model):
|
||||||
|
import torchaudio
|
||||||
|
|
||||||
|
audio_path = "/projects/gemma4-vllm/audio/tellMeAboutTheSun.wav"
|
||||||
|
if not os.path.exists(audio_path):
|
||||||
|
pytest.skip(f"Test asset {audio_path} not found")
|
||||||
|
wav, sr = torchaudio.load(audio_path)
|
||||||
|
mono_wav = wav.mean(dim=0).numpy()
|
||||||
|
prompt = "Describe this audio: <|audio|>"
|
||||||
|
ref = st_model.encode(
|
||||||
|
[{"audio": audio_path, "text": prompt}], normalize_embeddings=True
|
||||||
|
)
|
||||||
|
with vllm_runner(MODEL, runner="pooling", max_model_len=8192) as vm:
|
||||||
|
out = vm.embed(
|
||||||
|
[{"prompt": prompt, "multi_modal_data": {"audio": (mono_wav, sr)}}]
|
||||||
|
)
|
||||||
|
sim = _cos(out, ref).min().item()
|
||||||
|
assert sim >= MM_COS, f"Audio cosine {sim} < {MM_COS}"
|
||||||
|
|
||||||
|
|
||||||
|
@needs_model
|
||||||
|
def test_mixed_parity(vllm_runner, st_model):
|
||||||
|
import torchaudio
|
||||||
|
from PIL import Image
|
||||||
|
|
||||||
|
img_path = "/projects/gemma4-vllm/images/cat.jpg"
|
||||||
|
audio_path = "/projects/gemma4-vllm/audio/tellMeAboutTheSun.wav"
|
||||||
|
if not os.path.exists(img_path) or not os.path.exists(audio_path):
|
||||||
|
pytest.skip("Test assets for mixed parity not found")
|
||||||
|
img = Image.open(img_path)
|
||||||
|
wav, sr = torchaudio.load(audio_path)
|
||||||
|
mono_wav = wav.mean(dim=0).numpy()
|
||||||
|
prompt = "Describe image <|image|> and audio <|audio|>"
|
||||||
|
ref = st_model.encode(
|
||||||
|
[{"image": img, "audio": audio_path, "text": prompt}],
|
||||||
|
normalize_embeddings=True,
|
||||||
|
)
|
||||||
|
with vllm_runner(MODEL, runner="pooling", max_model_len=8192) as vm:
|
||||||
|
out = vm.embed(
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"prompt": prompt,
|
||||||
|
"multi_modal_data": {
|
||||||
|
"image": img,
|
||||||
|
"audio": (mono_wav, sr),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
]
|
||||||
|
)
|
||||||
|
sim = _cos(out, ref).min().item()
|
||||||
|
assert sim >= MM_COS, f"Mixed cosine {sim} < {MM_COS}"
|
||||||
|
|
||||||
|
|
||||||
|
@needs_model
|
||||||
|
def test_video_parity_uniform_overflow(vllm_runner, st_model):
|
||||||
|
from transformers.video_utils import load_video
|
||||||
|
|
||||||
|
video_path = _ensure_synthetic_video(
|
||||||
|
"/tmp/test_40s.mp4", duration=40.0, fps=10.0, width=160, height=120
|
||||||
|
)
|
||||||
|
frames, meta = load_video(video_path, backend="opencv")
|
||||||
|
prompt = "Describe this video: <|video|>"
|
||||||
|
ref = st_model.encode(
|
||||||
|
[{"video": video_path, "text": prompt}], normalize_embeddings=True
|
||||||
|
)
|
||||||
|
with vllm_runner(MODEL, runner="pooling", max_model_len=8192) as vm:
|
||||||
|
out = vm.embed(
|
||||||
|
[{"prompt": prompt, "multi_modal_data": {"video": (frames, meta)}}]
|
||||||
|
)
|
||||||
|
sim = _cos(out, ref).min().item()
|
||||||
|
assert sim >= MM_COS, f"40s video cosine {sim} < {MM_COS}"
|
||||||
|
|
||||||
|
|
||||||
|
@needs_model
|
||||||
|
def test_video_parity_portrait(vllm_runner, st_model):
|
||||||
|
from transformers.video_utils import load_video
|
||||||
|
|
||||||
|
video_path = _ensure_synthetic_video(
|
||||||
|
"/tmp/test_portrait.mp4", duration=3.0, fps=10.0, width=120, height=160
|
||||||
|
)
|
||||||
|
frames, meta = load_video(video_path, backend="opencv")
|
||||||
|
prompt = "Describe this video: <|video|>"
|
||||||
|
ref = st_model.encode(
|
||||||
|
[{"video": video_path, "text": prompt}], normalize_embeddings=True
|
||||||
|
)
|
||||||
|
with vllm_runner(MODEL, runner="pooling", max_model_len=8192) as vm:
|
||||||
|
out = vm.embed(
|
||||||
|
[{"prompt": prompt, "multi_modal_data": {"video": (frames, meta)}}]
|
||||||
|
)
|
||||||
|
sim = _cos(out, ref).min().item()
|
||||||
|
assert sim >= MM_COS, f"Portrait video cosine {sim} < {MM_COS}"
|
||||||
|
|
||||||
|
|
||||||
|
@needs_model
|
||||||
|
def test_tower_skip(vllm_runner, monkeypatch):
|
||||||
|
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
|
||||||
|
|
||||||
|
def nparams(m):
|
||||||
|
return sum(p.numel() for p in m.parameters()) / 1e6
|
||||||
|
|
||||||
|
with vllm_runner(
|
||||||
|
MODEL,
|
||||||
|
runner="pooling",
|
||||||
|
max_model_len=2048,
|
||||||
|
limit_mm_per_prompt={"image": 0, "audio": 0, "video": 0},
|
||||||
|
) as vm:
|
||||||
|
p_text = vm.apply_model(nparams)[0]
|
||||||
|
assert abs(p_text - 271.00) <= 0.05, (
|
||||||
|
f"Expected 271.00M text params, got {p_text}M"
|
||||||
|
)
|
||||||
|
|
||||||
|
with vllm_runner(MODEL, runner="pooling", max_model_len=2048) as vm:
|
||||||
|
p_full = vm.apply_model(nparams)[0]
|
||||||
|
assert abs(p_full - 744.37) <= 0.05, (
|
||||||
|
f"Expected 744.37M full params, got {p_full}M"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_encoder_contract():
|
||||||
|
from vllm.model_executor.models.embedding_gemma2 import EmbeddingGemma2Model
|
||||||
|
from vllm.model_executor.models.gemma4_mm import (
|
||||||
|
Gemma4ForConditionalGeneration as G,
|
||||||
|
)
|
||||||
|
|
||||||
|
borrowed_methods = [
|
||||||
|
"_parse_and_validate_multimodal_inputs",
|
||||||
|
"_parse_and_validate_image_input",
|
||||||
|
"_parse_and_validate_video_input",
|
||||||
|
"_parse_and_validate_audio_input",
|
||||||
|
"_process_image_input",
|
||||||
|
"_process_video_input",
|
||||||
|
"_process_audio_input",
|
||||||
|
"embed_multimodal",
|
||||||
|
"_encoder_chunk",
|
||||||
|
]
|
||||||
|
|
||||||
|
for name in borrowed_methods:
|
||||||
|
# Check method identity (L-F6 + T-F4)
|
||||||
|
assert getattr(EmbeddingGemma2Model, name) is getattr(G, name)
|
||||||
|
|
||||||
|
# AST checks: verify no zero-arg super() calls in borrowed methods
|
||||||
|
func = getattr(EmbeddingGemma2Model, name)
|
||||||
|
src = textwrap.dedent(inspect.getsource(func))
|
||||||
|
tree = ast.parse(src)
|
||||||
|
for node in ast.walk(tree):
|
||||||
|
if (
|
||||||
|
isinstance(node, ast.Call)
|
||||||
|
and isinstance(node.func, ast.Name)
|
||||||
|
and node.func.id == "super"
|
||||||
|
):
|
||||||
|
assert len(node.args) > 0, (
|
||||||
|
f"Zero-arg super() call found in borrowed method {name}"
|
||||||
|
)
|
||||||
|
|
||||||
|
# AST check: all self.<attr> non-callable accesses are assigned in __init__
|
||||||
|
init_src = inspect.getsource(EmbeddingGemma2Model.__init__)
|
||||||
|
for name in borrowed_methods:
|
||||||
|
func = getattr(EmbeddingGemma2Model, name)
|
||||||
|
src = textwrap.dedent(inspect.getsource(func))
|
||||||
|
tree = ast.parse(src)
|
||||||
|
for node in ast.walk(tree):
|
||||||
|
if (
|
||||||
|
isinstance(node, ast.Attribute)
|
||||||
|
and isinstance(node.value, ast.Name)
|
||||||
|
and node.value.id == "self"
|
||||||
|
):
|
||||||
|
attr = node.attr
|
||||||
|
assert hasattr(EmbeddingGemma2Model, attr) or attr in init_src, (
|
||||||
|
f"Attribute {attr} accessed on self in {name} is neither "
|
||||||
|
f"defined on class nor in __init__"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@needs_model
|
||||||
|
def test_instance_attribute_access(vllm_runner):
|
||||||
|
"""Check self.<attr> accesses on a real model instance."""
|
||||||
|
from vllm.model_executor.models.embedding_gemma2 import EmbeddingGemma2Model
|
||||||
|
|
||||||
|
borrowed_methods = [
|
||||||
|
"_parse_and_validate_multimodal_inputs",
|
||||||
|
"_parse_and_validate_image_input",
|
||||||
|
"_parse_and_validate_video_input",
|
||||||
|
"_parse_and_validate_audio_input",
|
||||||
|
"_process_image_input",
|
||||||
|
"_process_video_input",
|
||||||
|
"_process_audio_input",
|
||||||
|
"embed_multimodal",
|
||||||
|
"_encoder_chunk",
|
||||||
|
]
|
||||||
|
attrs_accessed = set()
|
||||||
|
for name in borrowed_methods:
|
||||||
|
func = getattr(EmbeddingGemma2Model, name)
|
||||||
|
src = textwrap.dedent(inspect.getsource(func))
|
||||||
|
tree = ast.parse(src)
|
||||||
|
for node in ast.walk(tree):
|
||||||
|
if (
|
||||||
|
isinstance(node, ast.Attribute)
|
||||||
|
and isinstance(node.value, ast.Name)
|
||||||
|
and node.value.id == "self"
|
||||||
|
):
|
||||||
|
attrs_accessed.add(node.attr)
|
||||||
|
|
||||||
|
def check_attrs(model):
|
||||||
|
for attr in attrs_accessed:
|
||||||
|
assert hasattr(model, attr), (
|
||||||
|
f"Real instance of EmbeddingGemma2Model is missing attribute '{attr}' "
|
||||||
|
f"accessed by borrowed method"
|
||||||
|
)
|
||||||
|
return True
|
||||||
|
|
||||||
|
with vllm_runner(MODEL, runner="pooling", max_model_len=2048) as vm:
|
||||||
|
res = vm.apply_model(check_attrs)
|
||||||
|
assert res[0] is True
|
||||||
|
|
||||||
|
|
||||||
|
@needs_model
|
||||||
|
def test_video_loader_real_video(vllm_runner, st_model):
|
||||||
|
"""Decode a real MP4 file through EmbeddingGemma2VideoBackend.load_bytes
|
||||||
|
|
||||||
|
and VideoMediaIO / MediaConnector with default num_frames=32.
|
||||||
|
"""
|
||||||
|
from vllm.multimodal.media import MediaConnector
|
||||||
|
from vllm.multimodal.media.video import VideoMediaIO
|
||||||
|
from vllm.multimodal.video import VIDEO_LOADER_REGISTRY
|
||||||
|
|
||||||
|
video_path = "/projects/gemma4-vllm/videos/test_intel.mp4"
|
||||||
|
backend_cls = VIDEO_LOADER_REGISTRY.name2class["embedding_gemma2"]
|
||||||
|
with open(video_path, "rb") as f:
|
||||||
|
data = f.read()
|
||||||
|
|
||||||
|
# Direct backend load
|
||||||
|
frames, meta = backend_cls.load_bytes(data)
|
||||||
|
|
||||||
|
# VideoMediaIO with default num_frames=32 (VideoMediaIO passes num_frames=32)
|
||||||
|
video_io = VideoMediaIO(image_io=None, video_backend="embedding_gemma2")
|
||||||
|
loaded = video_io.load_bytes(data)
|
||||||
|
frames_io, meta_io = loaded.media
|
||||||
|
assert len(frames_io) == len(frames)
|
||||||
|
assert meta_io.get("do_sample_frames") is False
|
||||||
|
|
||||||
|
# MediaConnector with video_processor="EmbeddingGemma2VideoProcessor"
|
||||||
|
connector = MediaConnector(allowed_local_media_path="/projects/gemma4-vllm/videos")
|
||||||
|
fetched = connector.fetch_video(
|
||||||
|
f"file://{video_path}",
|
||||||
|
video_processor="EmbeddingGemma2VideoProcessor",
|
||||||
|
)
|
||||||
|
frames_conn, meta_conn = fetched.media
|
||||||
|
assert len(frames_conn) == 32
|
||||||
|
assert meta_conn.get("do_sample_frames") is False
|
||||||
|
|
||||||
|
prompt = "Describe this video: <|video|>"
|
||||||
|
ref = st_model.encode(
|
||||||
|
[{"video": video_path, "text": prompt}], normalize_embeddings=True
|
||||||
|
)
|
||||||
|
with vllm_runner(MODEL, runner="pooling", max_model_len=8192) as vm:
|
||||||
|
out = vm.embed(
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"prompt": prompt,
|
||||||
|
"multi_modal_data": {"video": (frames_conn, meta_conn)},
|
||||||
|
}
|
||||||
|
]
|
||||||
|
)
|
||||||
|
sim = _cos(out, ref).min().item()
|
||||||
|
assert sim >= MM_COS, f"Real video loader cosine {sim} < {MM_COS}"
|
||||||
|
|
||||||
|
|
||||||
|
@needs_model
|
||||||
|
def test_video_loader_duplicate_expansion_e2e(vllm_runner, st_model):
|
||||||
|
"""End-to-end duplicate expansion test with low-fps clip."""
|
||||||
|
import numpy as np
|
||||||
|
|
||||||
|
from vllm.multimodal.video import VIDEO_LOADER_REGISTRY
|
||||||
|
|
||||||
|
backend_cls = VIDEO_LOADER_REGISTRY.name2class["embedding_gemma2"]
|
||||||
|
# 4s clip at 0.5 fps = 2 unique frames
|
||||||
|
lowfps_path = "/tmp/test_lowfps_05.mp4"
|
||||||
|
_ensure_synthetic_video(lowfps_path, duration=4.0, fps=0.5, width=64, height=48)
|
||||||
|
with open(lowfps_path, "rb") as f:
|
||||||
|
data = f.read()
|
||||||
|
# Default fps=1.0 -> 4 frames expanding 2 unique frames to [0, 0, 1, 1]
|
||||||
|
frames, meta = backend_cls.load_bytes(data)
|
||||||
|
assert frames.shape[0] == 4
|
||||||
|
# Compare returned frames directly, not only indices
|
||||||
|
assert np.array_equal(frames[0], frames[1])
|
||||||
|
assert not np.array_equal(frames[0], frames[2])
|
||||||
|
assert np.array_equal(frames[2], frames[3])
|
||||||
|
|
||||||
|
prompt = "Describe this video: <|video|>"
|
||||||
|
with vllm_runner(MODEL, runner="pooling", max_model_len=8192) as vm:
|
||||||
|
out = vm.embed(
|
||||||
|
[{"prompt": prompt, "multi_modal_data": {"video": (frames, meta)}}]
|
||||||
|
)
|
||||||
|
ref = st_model.encode(
|
||||||
|
[{"video": lowfps_path, "text": prompt}], normalize_embeddings=True
|
||||||
|
)
|
||||||
|
sim = _cos(out, ref).min().item()
|
||||||
|
assert sim >= MM_COS, f"Duplicate expansion cosine {sim} < {MM_COS}"
|
||||||
|
assert len(out) == 1
|
||||||
|
assert len(out[0]) == 768
|
||||||
|
|
||||||
|
|
||||||
|
def test_video_loader_indices():
|
||||||
|
from transformers.models.embedding_gemma2.video_processing_embedding_gemma2 import (
|
||||||
|
EmbeddingGemma2VideoProcessor,
|
||||||
|
)
|
||||||
|
from transformers.video_utils import VideoMetadata
|
||||||
|
|
||||||
|
from vllm.multimodal.video import (
|
||||||
|
VIDEO_LOADER_REGISTRY,
|
||||||
|
VideoSourceMetadata,
|
||||||
|
VideoTargetMetadata,
|
||||||
|
)
|
||||||
|
|
||||||
|
backend_cls = VIDEO_LOADER_REGISTRY.name2class["embedding_gemma2"]
|
||||||
|
hf_proc = EmbeddingGemma2VideoProcessor()
|
||||||
|
|
||||||
|
# Case 1: clip shorter than 32s (e.g. 5s @ 24fps = 120 frames)
|
||||||
|
src_5s = VideoSourceMetadata(total_frames_num=120, original_fps=24.0, duration=5.0)
|
||||||
|
tgt = VideoTargetMetadata(num_frames=-1, fps=1.0, max_duration=300)
|
||||||
|
vllm_indices = backend_cls.compute_frames_index_to_sample(src_5s, tgt)
|
||||||
|
hf_meta_5s = VideoMetadata(fps=24.0, duration=5.0, total_num_frames=120)
|
||||||
|
hf_indices_5s = hf_proc.sample_frames(
|
||||||
|
hf_meta_5s, fps=1.0, max_frames=32, overflow_strategy="uniform"
|
||||||
|
).tolist()
|
||||||
|
assert vllm_indices == hf_indices_5s
|
||||||
|
|
||||||
|
# Case 2: clip shorter than 1s (e.g. 0.5s @ 24fps = 12 frames)
|
||||||
|
src_sub1s = VideoSourceMetadata(
|
||||||
|
total_frames_num=12, original_fps=24.0, duration=0.5
|
||||||
|
)
|
||||||
|
vllm_sub1s = backend_cls.compute_frames_index_to_sample(src_sub1s, tgt)
|
||||||
|
hf_meta_sub1s = VideoMetadata(fps=24.0, duration=0.5, total_num_frames=12)
|
||||||
|
hf_indices_sub1s = hf_proc.sample_frames(
|
||||||
|
hf_meta_sub1s, fps=1.0, max_frames=32, overflow_strategy="uniform"
|
||||||
|
).tolist()
|
||||||
|
assert vllm_sub1s == hf_indices_sub1s
|
||||||
|
|
||||||
|
# Case 3: clip longer than 32s (e.g. 40s @ 10fps = 400 frames)
|
||||||
|
# -> uniform overflow cap 32 frames
|
||||||
|
src_40s = VideoSourceMetadata(
|
||||||
|
total_frames_num=400, original_fps=10.0, duration=40.0
|
||||||
|
)
|
||||||
|
vllm_40s = backend_cls.compute_frames_index_to_sample(src_40s, tgt)
|
||||||
|
hf_meta_40s = VideoMetadata(fps=10.0, duration=40.0, total_num_frames=400)
|
||||||
|
hf_indices_40s = hf_proc.sample_frames(
|
||||||
|
hf_meta_40s, fps=1.0, max_frames=32, overflow_strategy="uniform"
|
||||||
|
).tolist()
|
||||||
|
assert vllm_40s == hf_indices_40s
|
||||||
|
assert len(vllm_40s) == 32
|
||||||
|
|
||||||
|
# Case 4: default target fps (target.fps <= 0 defaults to 1.0)
|
||||||
|
tgt_default_fps = VideoTargetMetadata(num_frames=-1, fps=-1.0, max_duration=300)
|
||||||
|
vllm_def = backend_cls.compute_frames_index_to_sample(src_5s, tgt_default_fps)
|
||||||
|
assert vllm_def == hf_indices_5s
|
||||||
|
|
||||||
|
# Case 5 (NEW-2): 60 fps / 300 s (18000 frames) vs HF sample_frames
|
||||||
|
src_300s = VideoSourceMetadata(
|
||||||
|
total_frames_num=18000, original_fps=60.0, duration=300.0
|
||||||
|
)
|
||||||
|
v_300s = backend_cls.compute_frames_index_to_sample(
|
||||||
|
src_300s, VideoTargetMetadata(num_frames=-1, fps=60.0, max_duration=300)
|
||||||
|
)
|
||||||
|
h_300s = hf_proc.sample_frames(
|
||||||
|
VideoMetadata(fps=60.0, duration=300.0, total_num_frames=18000),
|
||||||
|
fps=60.0,
|
||||||
|
max_frames=32,
|
||||||
|
overflow_strategy="uniform",
|
||||||
|
).tolist()
|
||||||
|
assert v_300s == h_300s, f"60fps/300s mismatch: {v_300s[-3:]} vs {h_300s[-3:]}"
|
||||||
|
|
||||||
|
# Case 6 (NEW-2): 1 fps / 4 h (432000 frames @ 30 native fps) vs HF sample_frames
|
||||||
|
src_4h = VideoSourceMetadata(
|
||||||
|
total_frames_num=432000, original_fps=30.0, duration=14400.0
|
||||||
|
)
|
||||||
|
v_4h = backend_cls.compute_frames_index_to_sample(
|
||||||
|
src_4h, VideoTargetMetadata(num_frames=-1, fps=1.0, max_duration=300)
|
||||||
|
)
|
||||||
|
h_4h = hf_proc.sample_frames(
|
||||||
|
VideoMetadata(fps=30.0, duration=14400.0, total_num_frames=432000),
|
||||||
|
fps=1.0,
|
||||||
|
max_frames=32,
|
||||||
|
overflow_strategy="uniform",
|
||||||
|
).tolist()
|
||||||
|
assert v_4h == h_4h, f"1fps/4h mismatch: {v_4h[-2:]} vs {h_4h[-2:]}"
|
||||||
|
|
||||||
|
|
||||||
|
def test_video_loader_duplicate_expansion():
|
||||||
|
from transformers.models.embedding_gemma2.video_processing_embedding_gemma2 import (
|
||||||
|
EmbeddingGemma2VideoProcessor,
|
||||||
|
)
|
||||||
|
from transformers.video_utils import VideoMetadata
|
||||||
|
|
||||||
|
from vllm.multimodal.video import (
|
||||||
|
VIDEO_LOADER_REGISTRY,
|
||||||
|
VideoSourceMetadata,
|
||||||
|
VideoTargetMetadata,
|
||||||
|
)
|
||||||
|
|
||||||
|
backend_cls = VIDEO_LOADER_REGISTRY.name2class["embedding_gemma2"]
|
||||||
|
hf_proc = EmbeddingGemma2VideoProcessor()
|
||||||
|
|
||||||
|
# 4s clip @ 0.5fps = 2 frames total. Requested target fps = 1.0 -> 4 frames.
|
||||||
|
# Source has fewer frames than requested sampled frames, requiring duplicates.
|
||||||
|
src_lowfps = VideoSourceMetadata(total_frames_num=2, original_fps=0.5, duration=4.0)
|
||||||
|
tgt = VideoTargetMetadata(num_frames=-1, fps=1.0, max_duration=300)
|
||||||
|
vllm_indices = backend_cls.compute_frames_index_to_sample(src_lowfps, tgt)
|
||||||
|
hf_meta = VideoMetadata(fps=0.5, duration=4.0, total_num_frames=2)
|
||||||
|
hf_indices = hf_proc.sample_frames(
|
||||||
|
hf_meta, fps=1.0, max_frames=32, overflow_strategy="uniform"
|
||||||
|
).tolist()
|
||||||
|
assert vllm_indices == hf_indices
|
||||||
|
assert len(vllm_indices) == 4
|
||||||
|
assert vllm_indices == [0, 0, 1, 1]
|
||||||
|
|
||||||
|
|
||||||
|
def test_video_media_io_embedding_gemma2():
|
||||||
|
"""Verify VideoMediaIO with default num_frames=32 loads video bytes for
|
||||||
|
|
||||||
|
EmbeddingGemma2 without error.
|
||||||
|
"""
|
||||||
|
import os
|
||||||
|
from unittest.mock import MagicMock
|
||||||
|
|
||||||
|
from vllm.multimodal.media import MediaConnector
|
||||||
|
from vllm.multimodal.media.video import VideoMediaIO
|
||||||
|
|
||||||
|
video_path = "/projects/gemma4-vllm/videos/test_intel.mp4"
|
||||||
|
if not os.path.exists(video_path):
|
||||||
|
return
|
||||||
|
|
||||||
|
with open(video_path, "rb") as f:
|
||||||
|
data = f.read()
|
||||||
|
|
||||||
|
# VideoMediaIO defaults to num_frames=32
|
||||||
|
video_io = VideoMediaIO(image_io=MagicMock(), video_backend="embedding_gemma2")
|
||||||
|
loaded = video_io.load_bytes(data)
|
||||||
|
frames, meta = loaded.media
|
||||||
|
assert len(frames) == 32
|
||||||
|
assert meta.get("do_sample_frames") is False
|
||||||
|
|
||||||
|
# MediaConnector path
|
||||||
|
connector = MediaConnector(allowed_local_media_path="/projects/gemma4-vllm/videos")
|
||||||
|
fetched = connector.fetch_video(
|
||||||
|
f"file://{video_path}",
|
||||||
|
video_processor="EmbeddingGemma2VideoProcessor",
|
||||||
|
)
|
||||||
|
frames_conn, meta_conn = fetched.media
|
||||||
|
assert len(frames_conn) == 32
|
||||||
|
assert meta_conn.get("do_sample_frames") is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_config_capping_and_explicit_limits(caplog):
|
||||||
|
"""Verify EmbeddingGemma2ModelConfig capping and explicit user overrides."""
|
||||||
|
import logging
|
||||||
|
from types import SimpleNamespace
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
from vllm.model_executor.models.config import (
|
||||||
|
EmbeddingGemma2ModelConfig,
|
||||||
|
Gemma4Config,
|
||||||
|
)
|
||||||
|
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||||
|
|
||||||
|
# (a) Defaults: uncapped max_model_len=262144 -> capped to 8192,
|
||||||
|
# and derived scheduler limits (>= 262144) lowered to 8192.
|
||||||
|
cfg_default = SimpleNamespace(
|
||||||
|
attention_config=SimpleNamespace(backend=None),
|
||||||
|
model_config=SimpleNamespace(
|
||||||
|
model="dummy", max_model_len=262144, original_max_model_len=None
|
||||||
|
),
|
||||||
|
scheduler_config=SimpleNamespace(
|
||||||
|
max_num_batched_tokens=262144,
|
||||||
|
max_num_encoder_input_tokens=262144,
|
||||||
|
encoder_cache_size=262144,
|
||||||
|
max_num_seqs=256,
|
||||||
|
verify_max_model_len=lambda x: None,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
with patch.object(Gemma4Config, "verify_and_update_config"):
|
||||||
|
EmbeddingGemma2ModelConfig.verify_and_update_config(cfg_default)
|
||||||
|
|
||||||
|
assert cfg_default.attention_config.backend == AttentionBackendEnum.TRITON_ATTN
|
||||||
|
assert cfg_default.model_config.max_model_len == 8192
|
||||||
|
assert cfg_default.scheduler_config.max_num_batched_tokens == 8192
|
||||||
|
assert cfg_default.scheduler_config.max_num_encoder_input_tokens == 8192
|
||||||
|
assert cfg_default.scheduler_config.encoder_cache_size == 8192
|
||||||
|
|
||||||
|
# (b) Reachable explicit limits: explicit max_model_len=8192 and explicit
|
||||||
|
# max_num_batched_tokens=32768 stays 32768.
|
||||||
|
# Note: omitting max_model_len while passing max_num_batched_tokens < 262144
|
||||||
|
# is rejected earlier by SchedulerConfig validation (32768 < 262144) before
|
||||||
|
# this hook runs.
|
||||||
|
cfg_explicit_batched = SimpleNamespace(
|
||||||
|
attention_config=SimpleNamespace(backend=None),
|
||||||
|
model_config=SimpleNamespace(
|
||||||
|
model="dummy", max_model_len=8192, original_max_model_len=8192
|
||||||
|
),
|
||||||
|
scheduler_config=SimpleNamespace(
|
||||||
|
max_num_batched_tokens=32768,
|
||||||
|
max_num_encoder_input_tokens=32768,
|
||||||
|
encoder_cache_size=32768,
|
||||||
|
max_num_seqs=256,
|
||||||
|
verify_max_model_len=lambda x: None,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
with patch.object(Gemma4Config, "verify_and_update_config"):
|
||||||
|
EmbeddingGemma2ModelConfig.verify_and_update_config(cfg_explicit_batched)
|
||||||
|
|
||||||
|
assert cfg_explicit_batched.model_config.max_model_len == 8192
|
||||||
|
assert cfg_explicit_batched.scheduler_config.max_num_batched_tokens == 32768
|
||||||
|
assert cfg_explicit_batched.scheduler_config.max_num_encoder_input_tokens == 32768
|
||||||
|
assert cfg_explicit_batched.scheduler_config.encoder_cache_size == 32768
|
||||||
|
|
||||||
|
# (c) Explicit value > uncapped_len (e.g. 300000) with uncapped model len is lowered
|
||||||
|
# to 8192 and logs a warning because it exceeds uncapped_len.
|
||||||
|
cfg_explicit_large = SimpleNamespace(
|
||||||
|
attention_config=SimpleNamespace(backend=None),
|
||||||
|
model_config=SimpleNamespace(
|
||||||
|
model="dummy", max_model_len=262144, original_max_model_len=None
|
||||||
|
),
|
||||||
|
scheduler_config=SimpleNamespace(
|
||||||
|
max_num_batched_tokens=300000,
|
||||||
|
max_num_encoder_input_tokens=300000,
|
||||||
|
encoder_cache_size=300000,
|
||||||
|
max_num_seqs=256,
|
||||||
|
verify_max_model_len=lambda x: None,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
with (
|
||||||
|
caplog.at_level(logging.WARNING),
|
||||||
|
patch.object(Gemma4Config, "verify_and_update_config"),
|
||||||
|
):
|
||||||
|
EmbeddingGemma2ModelConfig.verify_and_update_config(cfg_explicit_large)
|
||||||
|
|
||||||
|
assert cfg_explicit_large.model_config.max_model_len == 8192
|
||||||
|
assert cfg_explicit_large.scheduler_config.max_num_batched_tokens == 8192
|
||||||
|
assert cfg_explicit_large.scheduler_config.max_num_encoder_input_tokens == 8192
|
||||||
|
assert cfg_explicit_large.scheduler_config.encoder_cache_size == 8192
|
||||||
|
assert any(
|
||||||
|
"lowering explicitly set scheduler_config.max_num_batched_tokens"
|
||||||
|
" from 300000 to 8192" in rec.message
|
||||||
|
for rec in caplog.records
|
||||||
|
)
|
||||||
|
|
||||||
|
# (d) Explicit max_model_len=16384 is respected and not capped to 8192
|
||||||
|
cfg_explicit_model_len = SimpleNamespace(
|
||||||
|
attention_config=SimpleNamespace(backend=None),
|
||||||
|
model_config=SimpleNamespace(
|
||||||
|
model="dummy", max_model_len=16384, original_max_model_len=16384
|
||||||
|
),
|
||||||
|
scheduler_config=SimpleNamespace(
|
||||||
|
max_num_batched_tokens=16384,
|
||||||
|
max_num_encoder_input_tokens=16384,
|
||||||
|
encoder_cache_size=16384,
|
||||||
|
max_num_seqs=256,
|
||||||
|
verify_max_model_len=lambda x: None,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
with patch.object(Gemma4Config, "verify_and_update_config"):
|
||||||
|
EmbeddingGemma2ModelConfig.verify_and_update_config(cfg_explicit_model_len)
|
||||||
|
|
||||||
|
assert cfg_explicit_model_len.model_config.max_model_len == 16384
|
||||||
|
assert cfg_explicit_model_len.scheduler_config.max_num_batched_tokens == 16384
|
||||||
@@ -581,6 +581,11 @@ _EMBEDDING_EXAMPLE_MODELS = {
|
|||||||
trust_remote_code=True,
|
trust_remote_code=True,
|
||||||
hf_overrides={"model_type": "deepseek_v3", "auto_map": None},
|
hf_overrides={"model_type": "deepseek_v3", "auto_map": None},
|
||||||
),
|
),
|
||||||
|
"EmbeddingGemma2Model": _HfExamplesInfo(
|
||||||
|
"google/embeddinggemma-2",
|
||||||
|
min_transformers_version="5.18.0",
|
||||||
|
is_available_online=False,
|
||||||
|
),
|
||||||
"Gemma2Model": _HfExamplesInfo("BAAI/bge-multilingual-gemma2"),
|
"Gemma2Model": _HfExamplesInfo("BAAI/bge-multilingual-gemma2"),
|
||||||
"Gemma3TextModel": _HfExamplesInfo("google/embeddinggemma-300m"),
|
"Gemma3TextModel": _HfExamplesInfo("google/embeddinggemma-300m"),
|
||||||
"GteModel": _HfExamplesInfo(
|
"GteModel": _HfExamplesInfo(
|
||||||
|
|||||||
@@ -282,6 +282,111 @@ class Gemma4Config(VerifyAndUpdateConfig):
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class EmbeddingGemma2ModelConfig(Gemma4Config):
|
||||||
|
@staticmethod
|
||||||
|
def verify_and_update_config(vllm_config: "VllmConfig") -> None:
|
||||||
|
Gemma4Config.verify_and_update_config(vllm_config)
|
||||||
|
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||||
|
|
||||||
|
attn_config = vllm_config.attention_config
|
||||||
|
if attn_config.backend is None:
|
||||||
|
attn_config.backend = AttentionBackendEnum.TRITON_ATTN
|
||||||
|
logger.info(
|
||||||
|
"EmbeddingGemma2: attention backend not specified; defaulting to "
|
||||||
|
"TRITON_ATTN (validated backend for heterogeneous head dimensions)."
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
logger.info(
|
||||||
|
"EmbeddingGemma2: using attention backend %s", attn_config.backend
|
||||||
|
)
|
||||||
|
|
||||||
|
model_config = vllm_config.model_config
|
||||||
|
orig_len = getattr(model_config, "original_max_model_len", None)
|
||||||
|
if orig_len in (None, -1):
|
||||||
|
default_len = 8192
|
||||||
|
# Check sentence_bert_config.json if available
|
||||||
|
try:
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
cfg_path = Path(model_config.model) / "sentence_bert_config.json"
|
||||||
|
if cfg_path.exists():
|
||||||
|
data = json.loads(cfg_path.read_text())
|
||||||
|
if "max_seq_length" in data and isinstance(
|
||||||
|
data["max_seq_length"], int
|
||||||
|
):
|
||||||
|
default_len = data["max_seq_length"]
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
if model_config.max_model_len > default_len:
|
||||||
|
uncapped_len = model_config.max_model_len
|
||||||
|
logger.info(
|
||||||
|
"EmbeddingGemma2: max_model_len not explicitly set; capping from "
|
||||||
|
"%d to %d.",
|
||||||
|
model_config.max_model_len,
|
||||||
|
default_len,
|
||||||
|
)
|
||||||
|
model_config.max_model_len = default_len
|
||||||
|
scheduler_config = vllm_config.scheduler_config
|
||||||
|
if scheduler_config is not None:
|
||||||
|
new_batched_tokens = max(
|
||||||
|
model_config.max_model_len, scheduler_config.max_num_seqs
|
||||||
|
)
|
||||||
|
if scheduler_config.max_num_batched_tokens >= uncapped_len:
|
||||||
|
if scheduler_config.max_num_batched_tokens > uncapped_len:
|
||||||
|
logger.warning(
|
||||||
|
"EmbeddingGemma2: lowering explicitly set "
|
||||||
|
"scheduler_config.max_num_batched_tokens from %d to %d "
|
||||||
|
"to match capped max_model_len.",
|
||||||
|
scheduler_config.max_num_batched_tokens,
|
||||||
|
new_batched_tokens,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
logger.info(
|
||||||
|
"EmbeddingGemma2: lowering "
|
||||||
|
"scheduler_config.max_num_batched_tokens from %d to %d "
|
||||||
|
"to match capped max_model_len.",
|
||||||
|
scheduler_config.max_num_batched_tokens,
|
||||||
|
new_batched_tokens,
|
||||||
|
)
|
||||||
|
scheduler_config.max_num_batched_tokens = new_batched_tokens
|
||||||
|
if hasattr(vllm_config, "_set_compile_ranges"):
|
||||||
|
vllm_config._set_compile_ranges()
|
||||||
|
if scheduler_config.max_num_encoder_input_tokens >= uncapped_len:
|
||||||
|
if scheduler_config.max_num_encoder_input_tokens > uncapped_len:
|
||||||
|
logger.warning(
|
||||||
|
"EmbeddingGemma2: lowering explicitly set "
|
||||||
|
"scheduler_config.max_num_encoder_input_tokens "
|
||||||
|
"from %d to %d to match capped max_model_len.",
|
||||||
|
scheduler_config.max_num_encoder_input_tokens,
|
||||||
|
new_batched_tokens,
|
||||||
|
)
|
||||||
|
scheduler_config.max_num_encoder_input_tokens = (
|
||||||
|
new_batched_tokens
|
||||||
|
)
|
||||||
|
if scheduler_config.encoder_cache_size >= uncapped_len:
|
||||||
|
if scheduler_config.encoder_cache_size > uncapped_len:
|
||||||
|
logger.warning(
|
||||||
|
"EmbeddingGemma2: lowering explicitly set "
|
||||||
|
"scheduler_config.encoder_cache_size from %d to %d "
|
||||||
|
"to match capped max_model_len.",
|
||||||
|
scheduler_config.encoder_cache_size,
|
||||||
|
new_batched_tokens,
|
||||||
|
)
|
||||||
|
scheduler_config.encoder_cache_size = new_batched_tokens
|
||||||
|
if hasattr(scheduler_config, "verify_max_model_len"):
|
||||||
|
scheduler_config.verify_max_model_len(
|
||||||
|
model_config.max_model_len
|
||||||
|
)
|
||||||
|
elif model_config.max_model_len > 32768:
|
||||||
|
logger.warning(
|
||||||
|
"EmbeddingGemma2: max_model_len=%d is large; "
|
||||||
|
"consider --max-model-len 8192 to bound encoder memory.",
|
||||||
|
model_config.max_model_len,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class DiffusionGemmaModelForBlockDiffusionConfig(VerifyAndUpdateConfig):
|
class DiffusionGemmaModelForBlockDiffusionConfig(VerifyAndUpdateConfig):
|
||||||
@classmethod
|
@classmethod
|
||||||
def verify_and_update_config(cls, vllm_config: "VllmConfig") -> None:
|
def verify_and_update_config(cls, vllm_config: "VllmConfig") -> None:
|
||||||
@@ -1025,6 +1130,7 @@ MODELS_CONFIG_MAP: dict[str, type[VerifyAndUpdateConfig]] = {
|
|||||||
"DeepseekV41ForCausalLM": DeepseekV4ForCausalLMConfig,
|
"DeepseekV41ForCausalLM": DeepseekV4ForCausalLMConfig,
|
||||||
"DeepseekV32ForCausalLM": DeepseekV32ForCausalLM,
|
"DeepseekV32ForCausalLM": DeepseekV32ForCausalLM,
|
||||||
"DiffusionGemmaForBlockDiffusion": DiffusionGemmaModelForBlockDiffusionConfig, # noqa: E501
|
"DiffusionGemmaForBlockDiffusion": DiffusionGemmaModelForBlockDiffusionConfig, # noqa: E501
|
||||||
|
"EmbeddingGemma2Model": EmbeddingGemma2ModelConfig,
|
||||||
"Ernie4_5_VLMoeForConditionalGeneration": Ernie4_5_VLMoeForConditionalGenerationConfig, # noqa: E501
|
"Ernie4_5_VLMoeForConditionalGeneration": Ernie4_5_VLMoeForConditionalGenerationConfig, # noqa: E501
|
||||||
"FalconMambaForCausalLM": MambaModelConfig,
|
"FalconMambaForCausalLM": MambaModelConfig,
|
||||||
"Gemma3TextModel": Gemma3TextModelConfig,
|
"Gemma3TextModel": Gemma3TextModelConfig,
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -246,6 +246,7 @@ _EMBEDDING_MODELS = {
|
|||||||
# [Multimodal]
|
# [Multimodal]
|
||||||
"CLIPModel": ("clip", "CLIPEmbeddingModel"),
|
"CLIPModel": ("clip", "CLIPEmbeddingModel"),
|
||||||
"ColPaliForRetrieval": ("colpali", "ColPaliModel"),
|
"ColPaliForRetrieval": ("colpali", "ColPaliModel"),
|
||||||
|
"EmbeddingGemma2Model": ("embedding_gemma2", "EmbeddingGemma2Model"),
|
||||||
"LlamaNemotronVLModel": ("nemotron_vl", "LlamaNemotronVLForEmbedding"),
|
"LlamaNemotronVLModel": ("nemotron_vl", "LlamaNemotronVLForEmbedding"),
|
||||||
"LlavaNextForConditionalGeneration": (
|
"LlavaNextForConditionalGeneration": (
|
||||||
"llava_next",
|
"llava_next",
|
||||||
|
|||||||
@@ -405,6 +405,184 @@ class Qwen3VLVideoBackend(VideoBackend):
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@VIDEO_LOADER_REGISTRY.register(
|
||||||
|
"embedding_gemma2",
|
||||||
|
video_processor="EmbeddingGemma2VideoProcessor",
|
||||||
|
)
|
||||||
|
class EmbeddingGemma2VideoBackend(VideoBackend):
|
||||||
|
"""Samples frames with the same indices as HF EmbeddingGemma2VideoProcessor."""
|
||||||
|
|
||||||
|
PROCESSOR_NAMES = ("EmbeddingGemma2Processor", "EmbeddingGemma2VideoProcessor")
|
||||||
|
|
||||||
|
_MAX_ALLOWED_FRAMES: ClassVar[int] = 128
|
||||||
|
_MAX_ALLOWED_FPS: ClassVar[float] = 60.0
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _get_processor_defaults(cls, **kwargs) -> tuple[float, int]:
|
||||||
|
"""Read default fps and max_frames from video_processor if available."""
|
||||||
|
proc = kwargs.get("video_processor") or kwargs.get("processor")
|
||||||
|
if proc is not None:
|
||||||
|
vp = getattr(proc, "video_processor", proc)
|
||||||
|
fps = getattr(vp, "fps", None)
|
||||||
|
max_frames = getattr(vp, "max_frames", None)
|
||||||
|
if fps is not None and max_frames is not None:
|
||||||
|
return float(fps), int(max_frames)
|
||||||
|
try:
|
||||||
|
from transformers.models.embedding_gemma2 import (
|
||||||
|
EmbeddingGemma2VideoProcessor,
|
||||||
|
)
|
||||||
|
|
||||||
|
vp = EmbeddingGemma2VideoProcessor()
|
||||||
|
return float(getattr(vp, "fps", 1.0)), int(getattr(vp, "max_frames", 32))
|
||||||
|
except Exception:
|
||||||
|
return 1.0, 32
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def compute_frames_index_to_sample(
|
||||||
|
cls,
|
||||||
|
source: VideoSourceMetadata,
|
||||||
|
target: VideoTargetMetadata,
|
||||||
|
**kwargs,
|
||||||
|
) -> list[int]:
|
||||||
|
# Ignores target.num_frames (EmbeddingGemma2 does not accept num_frames)
|
||||||
|
default_fps, default_max_frames = cls._get_processor_defaults(**kwargs)
|
||||||
|
|
||||||
|
fps = kwargs.get("fps")
|
||||||
|
if fps is None:
|
||||||
|
fps = target.fps if target.fps > 0 else default_fps
|
||||||
|
if fps is not None:
|
||||||
|
if fps <= 0:
|
||||||
|
raise ValueError(f"fps must be positive, got {fps}")
|
||||||
|
if fps > cls._MAX_ALLOWED_FPS:
|
||||||
|
raise ValueError(
|
||||||
|
f"fps {fps} exceeds maximum allowed {cls._MAX_ALLOWED_FPS}"
|
||||||
|
)
|
||||||
|
|
||||||
|
max_frames = kwargs.get("max_frames", default_max_frames)
|
||||||
|
if max_frames is not None:
|
||||||
|
if max_frames <= 0:
|
||||||
|
raise ValueError(f"max_frames must be positive, got {max_frames}")
|
||||||
|
if max_frames > cls._MAX_ALLOWED_FRAMES:
|
||||||
|
raise ValueError(
|
||||||
|
f"max_frames {max_frames} exceeds maximum allowed "
|
||||||
|
f"{cls._MAX_ALLOWED_FRAMES}"
|
||||||
|
)
|
||||||
|
|
||||||
|
overflow_strategy = kwargs.get("overflow_strategy", "uniform")
|
||||||
|
|
||||||
|
total_frames = source.total_frames_num
|
||||||
|
original_fps = source.original_fps
|
||||||
|
duration = source.duration
|
||||||
|
|
||||||
|
if fps is not None and (original_fps <= 0 or duration <= 0):
|
||||||
|
fps = None
|
||||||
|
|
||||||
|
if overflow_strategy is not None:
|
||||||
|
if max_frames is None:
|
||||||
|
raise ValueError(
|
||||||
|
"You must pass max_frames when requesting an "
|
||||||
|
f"overflow_strategy={overflow_strategy}!"
|
||||||
|
)
|
||||||
|
if overflow_strategy not in ("truncate", "uniform"):
|
||||||
|
raise ValueError(
|
||||||
|
f"You passed overflow_strategy={overflow_strategy} but "
|
||||||
|
"expected one of ['truncate', 'uniform']"
|
||||||
|
)
|
||||||
|
|
||||||
|
if fps is None:
|
||||||
|
if overflow_strategy is None or total_frames <= max_frames:
|
||||||
|
indices = np.arange(total_frames, dtype=int)
|
||||||
|
elif overflow_strategy == "truncate":
|
||||||
|
indices = np.arange(max_frames, dtype=int)
|
||||||
|
elif overflow_strategy == "uniform":
|
||||||
|
indices = np.linspace(0, total_frames - 1, max_frames, dtype=int)
|
||||||
|
else:
|
||||||
|
step = original_fps / fps
|
||||||
|
num_sampled = max(1, int(duration * fps))
|
||||||
|
|
||||||
|
if overflow_strategy is None or num_sampled <= max_frames:
|
||||||
|
i_arr = np.arange(num_sampled, dtype=float)
|
||||||
|
indices = np.minimum(total_frames - 1, (i_arr * step).astype(int))
|
||||||
|
elif overflow_strategy == "truncate":
|
||||||
|
i_arr = np.arange(max_frames, dtype=float)
|
||||||
|
indices = np.minimum(total_frames - 1, (i_arr * step).astype(int))
|
||||||
|
elif overflow_strategy == "uniform":
|
||||||
|
linspace_idx = np.linspace(0, num_sampled - 1, max_frames, dtype=int)
|
||||||
|
indices = np.minimum(
|
||||||
|
total_frames - 1, (linspace_idx * step).astype(int)
|
||||||
|
)
|
||||||
|
|
||||||
|
return indices.tolist()
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def load_bytes(
|
||||||
|
cls,
|
||||||
|
data: bytes,
|
||||||
|
num_frames: int = -1,
|
||||||
|
fps: int | float = -1,
|
||||||
|
max_duration: int = 300,
|
||||||
|
frame_recovery: bool = False,
|
||||||
|
*,
|
||||||
|
backend: VideoDecoderBackend = "opencv",
|
||||||
|
**kwargs: Any,
|
||||||
|
) -> tuple[DecodedFrames, dict[str, Any]]:
|
||||||
|
# Accept num_frames as required by VideoLoader interface (VideoMediaIO
|
||||||
|
# passes default num_frames=32), but ignore it since EmbeddingGemma2 uses
|
||||||
|
# fps/max_frames. Force num_frames=-1 so compute_frames_index_to_sample
|
||||||
|
# drives sampling via fps/duration.
|
||||||
|
kwargs.pop("num_frames", None)
|
||||||
|
frames, metadata = super().load_bytes(
|
||||||
|
data,
|
||||||
|
num_frames=-1,
|
||||||
|
fps=fps, # type: ignore[arg-type]
|
||||||
|
max_duration=max_duration,
|
||||||
|
frame_recovery=frame_recovery,
|
||||||
|
backend=backend,
|
||||||
|
**kwargs,
|
||||||
|
)
|
||||||
|
|
||||||
|
source = VideoSourceMetadata(
|
||||||
|
total_frames_num=metadata["total_num_frames"],
|
||||||
|
original_fps=metadata["fps"],
|
||||||
|
duration=metadata["duration"],
|
||||||
|
)
|
||||||
|
target = VideoTargetMetadata(num_frames=-1, fps=fps, max_duration=max_duration)
|
||||||
|
sampling_kwargs, _ = resolve_video_backend_kwargs(backend, kwargs)
|
||||||
|
requested_indices = cls.compute_frames_index_to_sample(
|
||||||
|
source, target, **sampling_kwargs
|
||||||
|
)
|
||||||
|
|
||||||
|
# Expand deduplicated frames back to requested_indices if needed
|
||||||
|
# (e.g., OpenCV deduplicates frame_idx using set(frame_idx))
|
||||||
|
valid_frame_indices = metadata.get("frames_indices")
|
||||||
|
if valid_frame_indices is not None and requested_indices != valid_frame_indices:
|
||||||
|
idx_to_pos = {idx: pos for pos, idx in enumerate(valid_frame_indices)}
|
||||||
|
if all(idx in idx_to_pos for idx in requested_indices):
|
||||||
|
positions = [idx_to_pos[idx] for idx in requested_indices]
|
||||||
|
if isinstance(frames, np.ndarray):
|
||||||
|
frames = frames[positions]
|
||||||
|
else:
|
||||||
|
frames = frames[torch.tensor(positions, device=frames.device)]
|
||||||
|
metadata["frames_indices"] = list(requested_indices)
|
||||||
|
|
||||||
|
return frames, metadata
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def create_hf_metadata(
|
||||||
|
cls,
|
||||||
|
source: VideoSourceMetadata,
|
||||||
|
valid_frame_indices: list[int],
|
||||||
|
video_backend: str,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
meta = super().create_hf_metadata(
|
||||||
|
source=source,
|
||||||
|
valid_frame_indices=valid_frame_indices,
|
||||||
|
video_backend=video_backend,
|
||||||
|
)
|
||||||
|
meta["do_sample_frames"] = False
|
||||||
|
return meta
|
||||||
|
|
||||||
|
|
||||||
@VIDEO_LOADER_REGISTRY.register(
|
@VIDEO_LOADER_REGISTRY.register(
|
||||||
"qwen2_vl",
|
"qwen2_vl",
|
||||||
video_processor="Qwen2VLVideoProcessor",
|
video_processor="Qwen2VLVideoProcessor",
|
||||||
|
|||||||
@@ -243,9 +243,10 @@ def context_attention_fwd(
|
|||||||
b_seq_len: [b]
|
b_seq_len: [b]
|
||||||
out: [b * s, head, head_dim]
|
out: [b * s, head, head_dim]
|
||||||
"""
|
"""
|
||||||
BLOCK = get_block_size(q.dtype)
|
|
||||||
|
|
||||||
Lq, Lk, _ = q.shape[-1], k.shape[-1], v.shape[-1]
|
Lq, Lk, _ = q.shape[-1], k.shape[-1], v.shape[-1]
|
||||||
|
BLOCK = get_block_size(q.dtype)
|
||||||
|
if Lk >= 512:
|
||||||
|
BLOCK = min(BLOCK, 32)
|
||||||
|
|
||||||
sm_scale = 1.0 / (Lq**0.5) if softmax_scale is None else softmax_scale
|
sm_scale = 1.0 / (Lq**0.5) if softmax_scale is None else softmax_scale
|
||||||
# rescale with 1/ln(2) for triton exp2
|
# rescale with 1/ln(2) for triton exp2
|
||||||
|
|||||||
Reference in New Issue
Block a user