[Bugfix][Multimodal] Fix Qwen3-Omni use_audio_in_video with mixed image/video inputs (#46213)
Signed-off-by: wendadawen <wendadawen@qq.com> Signed-off-by: Tianyu Guo <guoty9@mail2.sysu.edu.cn> Co-authored-by: Tianyu Guo <guoty9@mail2.sysu.edu.cn>
This commit is contained in:
@@ -19,6 +19,7 @@ from vllm.model_executor.models.qwen2_5_omni_thinker import (
|
||||
check_interleaved_audio_video,
|
||||
merge_interleaved_embeddings,
|
||||
)
|
||||
from vllm.multimodal.utils import set_mm_embedding_modality
|
||||
|
||||
# Fake token IDs
|
||||
AUDIO_TOKEN_ID = 1001
|
||||
@@ -27,6 +28,10 @@ VIDEO_TOKEN_ID = 1003
|
||||
TEXT_TOKEN_ID = 0
|
||||
|
||||
|
||||
def _mm_embed(shape: tuple[int, ...], value: float, modality: str) -> torch.Tensor:
|
||||
return set_mm_embedding_modality(torch.full(shape, value), modality)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -116,6 +121,24 @@ class TestCheckInterleavedAudioVideo:
|
||||
is_video, is_audio, is_video.sum().item(), is_audio.sum().item()
|
||||
)
|
||||
|
||||
def test_multi_video_with_boundary_tokens(self):
|
||||
"""Two interleaved videos separated by boundary tokens → still True.
|
||||
|
||||
use_audio_in_video expands each video into a local V/A span bounded by
|
||||
non-pad tokens. A global density check would fail across those gaps.
|
||||
"""
|
||||
# [text][V A V A][text boundary][V A V A]
|
||||
first_ids, _ = make_interleaved_seq([2, 2], [2, 2], text_prefix=1)
|
||||
second_ids, _ = make_interleaved_seq([2, 2], [2, 2], text_prefix=0)
|
||||
boundary = torch.tensor([TEXT_TOKEN_ID, TEXT_TOKEN_ID])
|
||||
input_ids = torch.cat([first_ids, boundary, second_ids])
|
||||
is_multimodal = (input_ids == VIDEO_TOKEN_ID) | (input_ids == AUDIO_TOKEN_ID)
|
||||
is_video = is_multimodal & (input_ids == VIDEO_TOKEN_ID)
|
||||
is_audio = is_multimodal & (input_ids == AUDIO_TOKEN_ID)
|
||||
assert check_interleaved_audio_video(
|
||||
is_video, is_audio, is_video.sum().item(), is_audio.sum().item()
|
||||
)
|
||||
|
||||
def test_batched_non_interleaved_no_false_positive(self):
|
||||
"""
|
||||
Regression test for https://github.com/vllm-project/vllm/issues/35394.
|
||||
@@ -323,13 +346,16 @@ class TestEmbedInputIds:
|
||||
# mm_embeds come in [video, audio] order (video feature first in
|
||||
# mm_features when positions are the same for use_audio_in_video)
|
||||
mm_embeds = [
|
||||
torch.full((video_n, hidden), video_val),
|
||||
torch.full((audio_n, hidden), audio_val),
|
||||
_mm_embed((video_n, hidden), video_val, "video"),
|
||||
_mm_embed((audio_n, hidden), audio_val, "audio"),
|
||||
]
|
||||
|
||||
model, _ = make_mock_model(hidden)
|
||||
# Modalities are attached on the embedding tensors (as in encoder gather).
|
||||
result = model.embed_input_ids(
|
||||
input_ids, mm_embeds, is_multimodal=is_multimodal
|
||||
input_ids,
|
||||
mm_embeds,
|
||||
is_multimodal=is_multimodal,
|
||||
)
|
||||
|
||||
video_pos = (input_ids == VIDEO_TOKEN_ID).nonzero(as_tuple=True)[0]
|
||||
@@ -362,8 +388,8 @@ class TestMergeInterleavedEmbeddings:
|
||||
|
||||
inputs_embeds = torch.zeros(len(input_ids), hidden)
|
||||
mm_embeds = [
|
||||
torch.full((num_video, hidden), 30.0),
|
||||
torch.full((num_audio, hidden), 10.0),
|
||||
_mm_embed((num_video, hidden), 30.0, "video"),
|
||||
_mm_embed((num_audio, hidden), 10.0, "audio"),
|
||||
]
|
||||
|
||||
result = merge_interleaved_embeddings(
|
||||
@@ -372,8 +398,6 @@ class TestMergeInterleavedEmbeddings:
|
||||
is_video,
|
||||
is_audio,
|
||||
is_multimodal,
|
||||
num_video,
|
||||
num_audio,
|
||||
)
|
||||
|
||||
video_pos = is_video.nonzero(as_tuple=True)[0]
|
||||
@@ -381,6 +405,62 @@ class TestMergeInterleavedEmbeddings:
|
||||
assert result[video_pos].allclose(torch.full((num_video, hidden), 30.0))
|
||||
assert result[audio_pos].allclose(torch.full((num_audio, hidden), 10.0))
|
||||
|
||||
def test_image_and_video_mixed(self):
|
||||
"""Image embeddings must not be misclassified as video."""
|
||||
hidden = 4
|
||||
# [text][I I][V A V A]
|
||||
tokens = (
|
||||
[TEXT_TOKEN_ID] * 2
|
||||
+ [IMAGE_TOKEN_ID] * 2
|
||||
+ [VIDEO_TOKEN_ID, AUDIO_TOKEN_ID, VIDEO_TOKEN_ID, AUDIO_TOKEN_ID]
|
||||
)
|
||||
input_ids = torch.tensor(tokens)
|
||||
is_multimodal = (
|
||||
(input_ids == IMAGE_TOKEN_ID)
|
||||
| (input_ids == VIDEO_TOKEN_ID)
|
||||
| (input_ids == AUDIO_TOKEN_ID)
|
||||
)
|
||||
is_video = is_multimodal & (input_ids == VIDEO_TOKEN_ID)
|
||||
is_audio = is_multimodal & (input_ids == AUDIO_TOKEN_ID)
|
||||
is_image = is_multimodal & (input_ids == IMAGE_TOKEN_ID)
|
||||
|
||||
inputs_embeds = torch.zeros(len(input_ids), hidden)
|
||||
mm_embeds = [
|
||||
_mm_embed((2, hidden), 20.0, "image"),
|
||||
_mm_embed((2, hidden), 30.0, "video"),
|
||||
_mm_embed((2, hidden), 10.0, "audio"),
|
||||
]
|
||||
result = merge_interleaved_embeddings(
|
||||
inputs_embeds,
|
||||
mm_embeds,
|
||||
is_video,
|
||||
is_audio,
|
||||
is_multimodal,
|
||||
)
|
||||
assert result[is_image.nonzero(as_tuple=True)[0]].allclose(
|
||||
torch.full((2, hidden), 20.0)
|
||||
)
|
||||
assert result[is_video.nonzero(as_tuple=True)[0]].allclose(
|
||||
torch.full((2, hidden), 30.0)
|
||||
)
|
||||
assert result[is_audio.nonzero(as_tuple=True)[0]].allclose(
|
||||
torch.full((2, hidden), 10.0)
|
||||
)
|
||||
|
||||
def test_missing_modality_raises(self):
|
||||
hidden = 2
|
||||
input_ids, is_multimodal = make_interleaved_seq([2], [2])
|
||||
is_video = is_multimodal & (input_ids == VIDEO_TOKEN_ID)
|
||||
is_audio = is_multimodal & (input_ids == AUDIO_TOKEN_ID)
|
||||
with pytest.raises(ValueError, match="Missing modality"):
|
||||
merge_interleaved_embeddings(
|
||||
torch.zeros(len(input_ids), hidden),
|
||||
[torch.zeros(2, hidden), torch.zeros(2, hidden)],
|
||||
is_video,
|
||||
is_audio,
|
||||
is_multimodal,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pytest.main([__file__, "-v"])
|
||||
|
||||
@@ -78,6 +78,7 @@ def test_draft_lookahead_uses_boundary_feature_when_cached():
|
||||
|
||||
# f0 covers positions 0..6 (+1 skew); f1's first embed covers position 7.
|
||||
assert len(mm_embeds) == 2
|
||||
assert [e.modality for e in mm_embeds] == ["image", "image"]
|
||||
assert bool(is_mm_embed[7])
|
||||
assert int(is_mm_embed.sum()) == 8
|
||||
|
||||
@@ -94,6 +95,7 @@ def test_draft_lookahead_tolerates_missing_boundary_feature():
|
||||
|
||||
# Only f0 is gathered; f1's boundary position falls back silently.
|
||||
assert len(mm_embeds) == 1
|
||||
assert [e.modality for e in mm_embeds] == ["image"]
|
||||
assert not bool(is_mm_embed[7])
|
||||
assert int(is_mm_embed.sum()) == 7
|
||||
|
||||
@@ -154,4 +156,28 @@ def test_multi_request_batch_gathers_per_request(draft_lookahead):
|
||||
# Both requests contribute a feature; with the +1 skew each marks 7 of its
|
||||
# 8 positions (the skew drops one), otherwise all 8.
|
||||
assert len(mm_embeds) == 2
|
||||
assert [e.modality for e in mm_embeds] == ["image", "image"]
|
||||
assert int(is_mm_embed.sum()) == (14 if draft_lookahead else 16)
|
||||
|
||||
|
||||
def test_gather_preserves_mixed_modalities():
|
||||
"""Modalities must be attached on tensors in gather order."""
|
||||
video = MultiModalFeatureSpec(
|
||||
data=None,
|
||||
modality="video",
|
||||
identifier="v0",
|
||||
mm_position=PlaceholderRange(offset=0, length=4),
|
||||
)
|
||||
audio = MultiModalFeatureSpec(
|
||||
data=None,
|
||||
modality="audio",
|
||||
identifier="a0",
|
||||
mm_position=PlaceholderRange(offset=4, length=4),
|
||||
)
|
||||
runner = _make_runner([video, audio], cached=[video, audio])
|
||||
|
||||
mm_embeds, is_mm_embed = _gather(runner, num_scheduled=8, draft_lookahead=0)
|
||||
|
||||
assert len(mm_embeds) == 2
|
||||
assert [e.modality for e in mm_embeds] == ["video", "audio"]
|
||||
assert int(is_mm_embed.sum()) == 8
|
||||
|
||||
@@ -138,42 +138,44 @@ def check_interleaved_audio_video(
|
||||
num_audio: int,
|
||||
) -> bool:
|
||||
"""
|
||||
Check if video and audio positions are interleaved in the multimodal region.
|
||||
Check if video and audio positions are interleaved in any per-video span.
|
||||
|
||||
Returns True only for the use_audio_in_video=True case, where video and
|
||||
audio tokens alternate within a single contiguous region with no gaps.
|
||||
|
||||
A simple range-overlap check produces false positives when multiple
|
||||
non-interleaved requests are batched together: audio tokens from request N
|
||||
fall between video tokens from request N and request N+1, making the
|
||||
global ranges overlap even though each individual request is non-interleaved.
|
||||
|
||||
To distinguish true interleaving from this batching artefact we require
|
||||
that every position in the combined [first_VA, last_VA] range is occupied
|
||||
by either a video or an audio token (no text/image gaps).
|
||||
For use_audio_in_video=True, each video placeholder is expanded into one
|
||||
local span containing only video/audio pad tokens, bounded by non-pad
|
||||
tokens such as audio_start/audio_end. Check each contiguous V/A span
|
||||
independently instead of requiring all V/A tokens in the whole sequence to
|
||||
form one global dense range; otherwise multi-video requests can be
|
||||
misclassified as non-interleaved because of the boundary tokens between
|
||||
videos.
|
||||
"""
|
||||
if num_video == 0 or num_audio == 0:
|
||||
return False
|
||||
|
||||
video_pos = is_video.nonzero(as_tuple=True)[0]
|
||||
audio_pos = is_audio.nonzero(as_tuple=True)[0]
|
||||
|
||||
# Quick range-overlap pre-check (necessary but not sufficient).
|
||||
if not (
|
||||
video_pos[0].item() < audio_pos[-1].item()
|
||||
and audio_pos[0].item() < video_pos[-1].item()
|
||||
):
|
||||
va_pos = (is_video | is_audio).nonzero(as_tuple=True)[0]
|
||||
if len(va_pos) == 0:
|
||||
return False
|
||||
|
||||
# Density check: for true use_audio_in_video interleaving every position
|
||||
# in the combined span is a video or audio token. Batched non-interleaved
|
||||
# requests have text/image tokens between the per-request V and A blocks.
|
||||
# combined_start/end encompass all V/A tokens, so num_video + num_audio
|
||||
# equals the number of V/A tokens in range; compare directly to span size.
|
||||
combined_start = min(video_pos[0].item(), audio_pos[0].item())
|
||||
combined_end = max(video_pos[-1].item(), audio_pos[-1].item())
|
||||
total_in_range = combined_end - combined_start + 1
|
||||
return (num_video + num_audio) == total_in_range
|
||||
span_start = 0
|
||||
for span_end in range(1, len(va_pos) + 1):
|
||||
is_last = span_end == len(va_pos)
|
||||
if not is_last and va_pos[span_end].item() == va_pos[span_end - 1].item() + 1:
|
||||
continue
|
||||
|
||||
span = va_pos[span_start:span_end]
|
||||
span_is_video = is_video[span]
|
||||
span_is_audio = is_audio[span]
|
||||
if span_is_video.any() and span_is_audio.any():
|
||||
video_offsets = span_is_video.nonzero(as_tuple=True)[0]
|
||||
audio_offsets = span_is_audio.nonzero(as_tuple=True)[0]
|
||||
if (
|
||||
video_offsets[0].item() < audio_offsets[-1].item()
|
||||
and audio_offsets[0].item() < video_offsets[-1].item()
|
||||
):
|
||||
return True
|
||||
|
||||
span_start = span_end
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def merge_interleaved_embeddings(
|
||||
@@ -182,58 +184,69 @@ def merge_interleaved_embeddings(
|
||||
is_video: torch.Tensor,
|
||||
is_audio: torch.Tensor,
|
||||
is_multimodal: torch.Tensor,
|
||||
num_video: int,
|
||||
num_audio: int,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Merge embeddings for interleaved audio-in-video sequences.
|
||||
|
||||
When use_audio_in_video=True, video and audio tokens are interleaved in
|
||||
the token sequence, but embeddings are provided as separate contiguous
|
||||
tensors (video first, then audio). This function reorders video and audio
|
||||
embeddings to match sequence position order and scatters them efficiently.
|
||||
tensors. This function scatters each modality by the ``modality`` attribute
|
||||
attached to each embedding tensor (set during encoder gather) and also
|
||||
supports image embeddings in the same interleaved request.
|
||||
|
||||
Args:
|
||||
inputs_embeds: The input embeddings tensor to merge into.
|
||||
multimodal_embeddings: List of embedding tensors (video, audio, other).
|
||||
multimodal_embeddings: List of embedding tensors (video, audio, image).
|
||||
is_video: Boolean mask for video token positions.
|
||||
is_audio: Boolean mask for audio token positions.
|
||||
is_multimodal: Boolean mask for all multimodal token positions.
|
||||
num_video: Total count of video tokens.
|
||||
num_audio: Total count of audio tokens.
|
||||
|
||||
Returns:
|
||||
The merged inputs_embeds tensor with multimodal embeddings scattered
|
||||
to their correct positions.
|
||||
"""
|
||||
# Categorize embeddings by modality based on token counts.
|
||||
# Embeddings come grouped by modality but order varies (e.g., image, video, audio
|
||||
# or video, audio depending on input kwargs order).
|
||||
video_embeds: list[torch.Tensor] = []
|
||||
audio_embeds: list[torch.Tensor] = []
|
||||
other_embeds: list[torch.Tensor] = []
|
||||
video_remaining = num_video
|
||||
audio_remaining = num_audio
|
||||
from vllm.multimodal.utils import get_mm_embedding_modalities
|
||||
|
||||
for emb in multimodal_embeddings:
|
||||
n = emb.shape[0]
|
||||
if video_remaining > 0 and n <= video_remaining:
|
||||
video_embeds.append(emb)
|
||||
video_remaining -= n
|
||||
elif audio_remaining > 0 and n <= audio_remaining:
|
||||
audio_embeds.append(emb)
|
||||
audio_remaining -= n
|
||||
else:
|
||||
other_embeds.append(emb)
|
||||
def _merge_embedding_group(
|
||||
mask: torch.Tensor,
|
||||
embeddings: Sequence[torch.Tensor],
|
||||
modality: str,
|
||||
) -> None:
|
||||
num_expected_tokens = mask.sum().item()
|
||||
num_actual_tokens = sum(embedding.shape[0] for embedding in embeddings)
|
||||
if num_actual_tokens != num_expected_tokens:
|
||||
raise ValueError(
|
||||
f"Attempted to assign {num_actual_tokens} {modality} tokens "
|
||||
f"to {num_expected_tokens} placeholders"
|
||||
)
|
||||
if embeddings:
|
||||
inputs_embeds[mask] = torch.cat(list(embeddings), dim=0)
|
||||
|
||||
# Scatter each modality to its positions
|
||||
if video_embeds:
|
||||
inputs_embeds[is_video] = torch.cat(video_embeds, dim=0)
|
||||
if audio_embeds:
|
||||
inputs_embeds[is_audio] = torch.cat(audio_embeds, dim=0)
|
||||
if other_embeds:
|
||||
other_mask = is_multimodal & ~is_video & ~is_audio
|
||||
inputs_embeds[other_mask] = torch.cat(other_embeds, dim=0)
|
||||
# Qwen Omni multimodal placeholders are image/video/audio; after excluding
|
||||
# video and audio positions, the remaining multimodal positions are images.
|
||||
is_image = is_multimodal & ~is_video & ~is_audio
|
||||
|
||||
embedding_modalities = get_mm_embedding_modalities(multimodal_embeddings)
|
||||
|
||||
video_embeds = [
|
||||
embedding
|
||||
for embedding, modality in zip(multimodal_embeddings, embedding_modalities)
|
||||
if modality == "video"
|
||||
]
|
||||
audio_embeds = [
|
||||
embedding
|
||||
for embedding, modality in zip(multimodal_embeddings, embedding_modalities)
|
||||
if modality == "audio"
|
||||
]
|
||||
image_embeds = [
|
||||
embedding
|
||||
for embedding, modality in zip(multimodal_embeddings, embedding_modalities)
|
||||
if modality == "image"
|
||||
]
|
||||
|
||||
_merge_embedding_group(is_video, video_embeds, "video")
|
||||
_merge_embedding_group(is_audio, audio_embeds, "audio")
|
||||
_merge_embedding_group(is_image, image_embeds, "image")
|
||||
|
||||
return inputs_embeds
|
||||
|
||||
@@ -1497,8 +1510,6 @@ class Qwen2_5OmniThinkerForConditionalGeneration(
|
||||
is_video,
|
||||
is_audio,
|
||||
is_multimodal,
|
||||
num_video,
|
||||
num_audio,
|
||||
)
|
||||
|
||||
# Default: standard merge (no interleaving), same as parent class
|
||||
|
||||
@@ -1833,9 +1833,11 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
|
||||
# both the deepstack path and the final embedding merge.
|
||||
video_token_id = self.config.video_token_id
|
||||
audio_token_id = self.config.audio_token_id
|
||||
image_token_id = self.config.image_token_id
|
||||
input_ids_cpu = input_ids.cpu()
|
||||
is_video = is_multimodal & (input_ids_cpu == video_token_id)
|
||||
is_audio = is_multimodal & (input_ids_cpu == audio_token_id)
|
||||
is_image = is_multimodal & (input_ids_cpu == image_token_id)
|
||||
num_video = is_video.sum().item()
|
||||
num_audio = is_audio.sum().item()
|
||||
|
||||
@@ -1856,9 +1858,9 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
|
||||
multimodal_embeddings_multiscale = []
|
||||
|
||||
if is_interleaved:
|
||||
# Use input_ids-based mask for correct vision positions
|
||||
# when audio and video tokens are interleaved.
|
||||
is_vision = is_video.clone()
|
||||
# Use input_ids-based mask for all vision positions when
|
||||
# audio and video tokens are interleaved.
|
||||
is_vision = is_video | is_image
|
||||
else:
|
||||
is_vision = torch.zeros_like(is_multimodal)
|
||||
mm_positions = torch.nonzero(is_multimodal, as_tuple=True)[0]
|
||||
@@ -1917,8 +1919,6 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
|
||||
is_video,
|
||||
is_audio,
|
||||
is_multimodal,
|
||||
num_video,
|
||||
num_audio,
|
||||
)
|
||||
|
||||
# Default: standard merge (no interleaving), same as parent class.
|
||||
|
||||
@@ -346,3 +346,41 @@ def fetch_video(
|
||||
allowed_local_media_path="/",
|
||||
)
|
||||
return media_connector.fetch_video(video_url)
|
||||
|
||||
|
||||
def set_mm_embedding_modality(embed: "torch.Tensor", modality: str) -> "torch.Tensor":
|
||||
"""Attach modality metadata to a gathered multimodal embedding tensor.
|
||||
|
||||
Used by interleaved Omni merge paths that need to group embeddings by
|
||||
modality without threading a parallel modalities list through
|
||||
``embed_input_ids``.
|
||||
"""
|
||||
embed.modality = modality # type: ignore[attr-defined]
|
||||
return embed
|
||||
|
||||
|
||||
def copy_mm_embedding_modality(
|
||||
src: "torch.Tensor", dst: "torch.Tensor"
|
||||
) -> "torch.Tensor":
|
||||
"""Copy ``modality`` from ``src`` onto ``dst`` if present."""
|
||||
modality = getattr(src, "modality", None)
|
||||
if modality is not None:
|
||||
dst.modality = modality # type: ignore[attr-defined]
|
||||
return dst
|
||||
|
||||
|
||||
def get_mm_embedding_modalities(
|
||||
multimodal_embeddings: Sequence["torch.Tensor"],
|
||||
) -> list[str]:
|
||||
"""Collect per-embedding modalities previously set on the tensors."""
|
||||
modalities: list[str] = []
|
||||
for i, emb in enumerate(multimodal_embeddings):
|
||||
modality = getattr(emb, "modality", None)
|
||||
if modality is None:
|
||||
raise ValueError(
|
||||
f"Missing modality on multimodal embedding at index {i}. "
|
||||
"Encoder gather must set embed.modality before interleaved "
|
||||
"audio-in-video merge."
|
||||
)
|
||||
modalities.append(modality)
|
||||
return modalities
|
||||
|
||||
@@ -5,7 +5,11 @@ import torch
|
||||
|
||||
from vllm.model_executor.models.interfaces import SupportsMultiModal, supports_realtime
|
||||
from vllm.multimodal.inputs import MultiModalKwargsItem
|
||||
from vllm.multimodal.utils import get_mm_features_in_window, group_and_batch_mm_kwargs
|
||||
from vllm.multimodal.utils import (
|
||||
get_mm_features_in_window,
|
||||
group_and_batch_mm_kwargs,
|
||||
set_mm_embedding_modality,
|
||||
)
|
||||
from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache
|
||||
from vllm.v1.worker.utils import sanity_check_mm_encoder_outputs
|
||||
|
||||
@@ -136,6 +140,9 @@ class EncoderRunner:
|
||||
else:
|
||||
mm_embeds_item = encoder_output[start_idx:end_idx]
|
||||
|
||||
# Attach modality for Omni interleaved merge (collected on demand).
|
||||
set_mm_embedding_modality(mm_embeds_item, mm_feature.modality)
|
||||
|
||||
req_start_pos = query_start_loc[i] + start_pos - cur_query_start
|
||||
is_mm_embed[req_start_pos + start_idx : req_start_pos + end_idx] |= (
|
||||
True if is_embed is None else is_embed
|
||||
|
||||
@@ -5,7 +5,10 @@ import torch.nn as nn
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.model_executor.models.interfaces import supports_multimodal_pruning
|
||||
from vllm.multimodal.utils import get_mm_features_in_window
|
||||
from vllm.multimodal.utils import (
|
||||
copy_mm_embedding_modality,
|
||||
get_mm_features_in_window,
|
||||
)
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch
|
||||
from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache
|
||||
from vllm.v1.worker.gpu.mm.rope import RopeState
|
||||
@@ -42,7 +45,11 @@ class MultiModalPruner:
|
||||
speculator reuses the target's already-recomputed positions, hence there is
|
||||
no position write-back here.
|
||||
"""
|
||||
return [mm[:, : self.inputs_embeds_size] for mm in mm_embeds]
|
||||
stripped: list[torch.Tensor] = []
|
||||
for mm in mm_embeds:
|
||||
out = mm[:, : self.inputs_embeds_size]
|
||||
stripped.append(copy_mm_embedding_modality(mm, out))
|
||||
return stripped
|
||||
|
||||
def recompute(
|
||||
self,
|
||||
@@ -82,7 +89,10 @@ class MultiModalPruner:
|
||||
num_computed_tokens=num_computed,
|
||||
)
|
||||
self.rope_state.update_prefill_positions(req_idx, new_positions, delta)
|
||||
cleaned.extend(req_cleaned)
|
||||
cleaned.extend(
|
||||
copy_mm_embedding_modality(src, dst)
|
||||
for src, dst in zip(req_embeds, req_cleaned)
|
||||
)
|
||||
|
||||
assert pos == len(mm_embeds)
|
||||
return cleaned
|
||||
|
||||
@@ -105,7 +105,12 @@ from vllm.multimodal.inputs import (
|
||||
MultiModalKwargsItem,
|
||||
PlaceholderRange,
|
||||
)
|
||||
from vllm.multimodal.utils import get_mm_features_in_window, group_and_batch_mm_kwargs
|
||||
from vllm.multimodal.utils import (
|
||||
copy_mm_embedding_modality,
|
||||
get_mm_features_in_window,
|
||||
group_and_batch_mm_kwargs,
|
||||
set_mm_embedding_modality,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.pooling_params import PoolingParams
|
||||
from vllm.sampling_params import SamplingType
|
||||
@@ -3245,11 +3250,13 @@ class GPUModelRunner(
|
||||
is_mm_embed[
|
||||
req_start_pos + start_idx : req_start_pos + end_idx
|
||||
] |= is_embed
|
||||
set_mm_embedding_modality(mm_embeds_item, mm_feature.modality)
|
||||
mm_embeds_req.append(mm_embeds_item)
|
||||
|
||||
if self.is_multimodal_pruning_enabled and self.uses_mrope:
|
||||
assert req_state.mrope_positions is not None
|
||||
should_sync_mrope_positions = True
|
||||
old_mm_embeds_req = mm_embeds_req
|
||||
mm_embeds_req, new_mrope_positions, new_delta = (
|
||||
self.model.recompute_mrope_positions(
|
||||
input_ids=req_state.prompt_token_ids,
|
||||
@@ -3258,6 +3265,10 @@ class GPUModelRunner(
|
||||
num_computed_tokens=req_state.num_computed_tokens,
|
||||
)
|
||||
)
|
||||
mm_embeds_req = [
|
||||
copy_mm_embedding_modality(src, dst)
|
||||
for src, dst in zip(old_mm_embeds_req, mm_embeds_req)
|
||||
]
|
||||
req_state.mrope_positions.copy_(new_mrope_positions)
|
||||
req_state.mrope_position_delta = new_delta
|
||||
|
||||
|
||||
Reference in New Issue
Block a user