[Model Runner V2][MM] Support EVS (#46535)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Nick Hill
2026-06-24 10:18:56 -04:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 061043eaca
commit 84c62e1cbd
10 changed files with 245 additions and 50 deletions
@@ -49,10 +49,12 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.v1.outputs import LogprobsTensors
from vllm.v1.worker.gpu.attn_utils import build_attn_metadata
from vllm.v1.worker.gpu.buffer_utils import UvaBackedTensor, async_copy_to_gpu
from vllm.v1.worker.gpu.input_batch import InputBatch
from vllm.v1.worker.gpu.model_states.interface import ModelState
from vllm.v1.worker.gpu.sample.logprob import compute_topk_logprobs
from vllm.v1.worker.gpu.sample.output import SamplerOutput
from vllm.v1.worker.gpu.sample.penalties import use_penalty
from vllm.v1.worker.gpu.states import RequestState
from .interfaces import (
SupportsMultiModal,
@@ -869,7 +871,12 @@ class DiffusionGemmaModelState(ModelState):
if idx is not None:
self.diffusion_states.remove_request(idx)
def get_mm_embeddings(self, scheduled_encoder_inputs, input_batch):
def get_mm_embeddings(
self,
scheduled_encoder_inputs: dict[str, list[int]],
input_batch: InputBatch,
req_states: RequestState,
) -> torch.Tensor:
if not self.supports_mm_inputs:
return None
@@ -880,14 +887,7 @@ class DiffusionGemmaModelState(ModelState):
encoder_outputs = self.encoder_runner.execute_mm_encoder(mm_kwargs)
self.encoder_cache.encoder_outputs.update(zip(mm_hashes, encoder_outputs))
mm_embeds, is_mm_embed = self.encoder_runner.gather_mm_embeddings(
input_batch.req_ids,
input_batch.num_tokens,
input_batch.num_scheduled_tokens,
input_batch.query_start_loc_np,
input_batch.prefill_len_np,
input_batch.num_computed_prefill_tokens_np,
)
mm_embeds, is_mm_embed = self.gather_mm_embeddings(input_batch)
if not mm_embeds:
# No MM tokens in this batch (e.g. all-decode step).
+6 -5
View File
@@ -421,11 +421,11 @@ class SupportsMultiModalPruning(Protocol):
def recompute_mrope_positions(
self,
input_ids: list[int],
multimodal_embeddings: MultiModalEmbeddings,
input_ids: list[int] | torch.Tensor,
multimodal_embeddings: Sequence[torch.Tensor],
mrope_positions: torch.LongTensor,
num_computed_tokens: int,
) -> tuple[MultiModalEmbeddings, Tensor, int]:
) -> tuple[Sequence[torch.Tensor], Tensor, int]:
"""
Update part of input mrope positions (starting with
num_computed_tokens index). Original mrope_positions are computed
@@ -435,8 +435,9 @@ class SupportsMultiModalPruning(Protocol):
Args:
input_ids: (N,) All input tokens of the prompt containing
entire sequence.
multimodal_embeddings: Tuple of multimodal embeddings that
entire sequence. Either a host-side list or an already
device-resident tensor.
multimodal_embeddings: Sequence of multimodal embeddings that
fits into the prefill chunk that is being processed.
mrope_positions: Existing mrope positions (3, N) for entire
sequence
+10 -6
View File
@@ -1583,11 +1583,11 @@ class Qwen2_5_VLForConditionalGeneration(
def recompute_mrope_positions(
self,
input_ids: list[int],
multimodal_embeddings: tuple[torch.Tensor, ...],
input_ids: list[int] | torch.Tensor,
multimodal_embeddings: Sequence[torch.Tensor],
mrope_positions: torch.LongTensor,
num_computed_tokens: int,
) -> tuple[tuple[torch.Tensor, ...], torch.Tensor, int]:
) -> tuple[Sequence[torch.Tensor], torch.Tensor, int]:
"""
Update part of input mrope positions (starting with
num_computed_tokens index). Original mrope_positions are computed
@@ -1618,8 +1618,12 @@ class Qwen2_5_VLForConditionalGeneration(
else mrope_positions.device
)
# Tensors.
input_ids_t = async_tensor_h2d(input_ids, dtype=torch.long, device=device)
# Tensors. input_ids may already be a (device-side) tensor.
if isinstance(input_ids, torch.Tensor):
assert input_ids.device == device
input_ids_t = input_ids.to(torch.long)
else:
input_ids_t = async_tensor_h2d(input_ids, dtype=torch.long, device=device)
mm_embeddings_out = [mm[:, :-4] for mm in multimodal_embeddings]
mm_embeddings_pos = [
@@ -1636,7 +1640,7 @@ class Qwen2_5_VLForConditionalGeneration(
video_token_id,
)
return tuple(mm_embeddings_out), positions, mrope_positions_delta
return mm_embeddings_out, positions, mrope_positions_delta
def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
mm_input_by_modality = {}
+14 -10
View File
@@ -2657,11 +2657,11 @@ class Qwen3VLForConditionalGeneration(
def recompute_mrope_positions(
self,
input_ids: list[int],
multimodal_embeddings: MultiModalEmbeddings,
input_ids: list[int] | torch.Tensor,
multimodal_embeddings: Sequence[torch.Tensor],
mrope_positions: torch.LongTensor,
num_computed_tokens: int,
) -> tuple[MultiModalEmbeddings, torch.Tensor, int]:
) -> tuple[Sequence[torch.Tensor], torch.Tensor, int]:
"""
Update part of input mrope positions (starting with
num_computed_tokens index). Original mrope_positions are computed
@@ -2672,7 +2672,7 @@ class Qwen3VLForConditionalGeneration(
Args:
input_ids: (N,) All input tokens of the prompt containing
entire sequence.
multimodal_embeddings: Tuple of multimodal embeddings that
multimodal_embeddings: Sequence of multimodal embeddings that
fits into the prefill chunk that is being processed.
mrope_positions: Existing mrope positions (3, N) for entire
sequence
@@ -2694,14 +2694,14 @@ class Qwen3VLForConditionalGeneration(
@staticmethod
def _recompute_mrope_positions(
input_ids: list[int],
multimodal_embeddings: MultiModalEmbeddings,
input_ids: list[int] | torch.Tensor,
multimodal_embeddings: Sequence[torch.Tensor],
mrope_positions: torch.LongTensor,
num_computed_tokens: int,
vision_start_token_id: int,
image_token_id: int,
video_token_id: int,
) -> tuple[MultiModalEmbeddings, torch.Tensor, int]:
) -> tuple[Sequence[torch.Tensor], torch.Tensor, int]:
# Device
device = (
multimodal_embeddings[0].device
@@ -2709,8 +2709,12 @@ class Qwen3VLForConditionalGeneration(
else mrope_positions.device
)
# Tensors
input_ids_t = async_tensor_h2d(input_ids, device=device, dtype=torch.long)
# Tensors. input_ids may already be a (device-side) tensor.
if isinstance(input_ids, torch.Tensor):
assert input_ids.device == device
input_ids_t = input_ids.to(dtype=torch.long)
else:
input_ids_t = async_tensor_h2d(input_ids, device=device, dtype=torch.long)
mm_embeddings_out = []
mm_embeddings_pos = []
@@ -2738,7 +2742,7 @@ class Qwen3VLForConditionalGeneration(
video_token_id,
)
return tuple(mm_embeddings_out), positions, mrope_positions_delta
return mm_embeddings_out, positions, mrope_positions_delta
def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings | None:
mm_input_by_modality = self._parse_and_validate_multimodal_inputs(**kwargs)