[Model Runner V2][MM] Support EVS (#46535)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Nick Hill
2026-06-24 10:18:56 -04:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 061043eaca
commit 84c62e1cbd
10 changed files with 245 additions and 50 deletions
@@ -49,10 +49,12 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.v1.outputs import LogprobsTensors
from vllm.v1.worker.gpu.attn_utils import build_attn_metadata
from vllm.v1.worker.gpu.buffer_utils import UvaBackedTensor, async_copy_to_gpu
from vllm.v1.worker.gpu.input_batch import InputBatch
from vllm.v1.worker.gpu.model_states.interface import ModelState
from vllm.v1.worker.gpu.sample.logprob import compute_topk_logprobs
from vllm.v1.worker.gpu.sample.output import SamplerOutput
from vllm.v1.worker.gpu.sample.penalties import use_penalty
from vllm.v1.worker.gpu.states import RequestState
from .interfaces import (
SupportsMultiModal,
@@ -869,7 +871,12 @@ class DiffusionGemmaModelState(ModelState):
if idx is not None:
self.diffusion_states.remove_request(idx)
def get_mm_embeddings(self, scheduled_encoder_inputs, input_batch):
def get_mm_embeddings(
self,
scheduled_encoder_inputs: dict[str, list[int]],
input_batch: InputBatch,
req_states: RequestState,
) -> torch.Tensor:
if not self.supports_mm_inputs:
return None
@@ -880,14 +887,7 @@ class DiffusionGemmaModelState(ModelState):
encoder_outputs = self.encoder_runner.execute_mm_encoder(mm_kwargs)
self.encoder_cache.encoder_outputs.update(zip(mm_hashes, encoder_outputs))
mm_embeds, is_mm_embed = self.encoder_runner.gather_mm_embeddings(
input_batch.req_ids,
input_batch.num_tokens,
input_batch.num_scheduled_tokens,
input_batch.query_start_loc_np,
input_batch.prefill_len_np,
input_batch.num_computed_prefill_tokens_np,
)
mm_embeds, is_mm_embed = self.gather_mm_embeddings(input_batch)
if not mm_embeds:
# No MM tokens in this batch (e.g. all-decode step).
+6 -5
View File
@@ -421,11 +421,11 @@ class SupportsMultiModalPruning(Protocol):
def recompute_mrope_positions(
self,
input_ids: list[int],
multimodal_embeddings: MultiModalEmbeddings,
input_ids: list[int] | torch.Tensor,
multimodal_embeddings: Sequence[torch.Tensor],
mrope_positions: torch.LongTensor,
num_computed_tokens: int,
) -> tuple[MultiModalEmbeddings, Tensor, int]:
) -> tuple[Sequence[torch.Tensor], Tensor, int]:
"""
Update part of input mrope positions (starting with
num_computed_tokens index). Original mrope_positions are computed
@@ -435,8 +435,9 @@ class SupportsMultiModalPruning(Protocol):
Args:
input_ids: (N,) All input tokens of the prompt containing
entire sequence.
multimodal_embeddings: Tuple of multimodal embeddings that
entire sequence. Either a host-side list or an already
device-resident tensor.
multimodal_embeddings: Sequence of multimodal embeddings that
fits into the prefill chunk that is being processed.
mrope_positions: Existing mrope positions (3, N) for entire
sequence
+10 -6
View File
@@ -1583,11 +1583,11 @@ class Qwen2_5_VLForConditionalGeneration(
def recompute_mrope_positions(
self,
input_ids: list[int],
multimodal_embeddings: tuple[torch.Tensor, ...],
input_ids: list[int] | torch.Tensor,
multimodal_embeddings: Sequence[torch.Tensor],
mrope_positions: torch.LongTensor,
num_computed_tokens: int,
) -> tuple[tuple[torch.Tensor, ...], torch.Tensor, int]:
) -> tuple[Sequence[torch.Tensor], torch.Tensor, int]:
"""
Update part of input mrope positions (starting with
num_computed_tokens index). Original mrope_positions are computed
@@ -1618,8 +1618,12 @@ class Qwen2_5_VLForConditionalGeneration(
else mrope_positions.device
)
# Tensors.
input_ids_t = async_tensor_h2d(input_ids, dtype=torch.long, device=device)
# Tensors. input_ids may already be a (device-side) tensor.
if isinstance(input_ids, torch.Tensor):
assert input_ids.device == device
input_ids_t = input_ids.to(torch.long)
else:
input_ids_t = async_tensor_h2d(input_ids, dtype=torch.long, device=device)
mm_embeddings_out = [mm[:, :-4] for mm in multimodal_embeddings]
mm_embeddings_pos = [
@@ -1636,7 +1640,7 @@ class Qwen2_5_VLForConditionalGeneration(
video_token_id,
)
return tuple(mm_embeddings_out), positions, mrope_positions_delta
return mm_embeddings_out, positions, mrope_positions_delta
def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
mm_input_by_modality = {}
+14 -10
View File
@@ -2657,11 +2657,11 @@ class Qwen3VLForConditionalGeneration(
def recompute_mrope_positions(
self,
input_ids: list[int],
multimodal_embeddings: MultiModalEmbeddings,
input_ids: list[int] | torch.Tensor,
multimodal_embeddings: Sequence[torch.Tensor],
mrope_positions: torch.LongTensor,
num_computed_tokens: int,
) -> tuple[MultiModalEmbeddings, torch.Tensor, int]:
) -> tuple[Sequence[torch.Tensor], torch.Tensor, int]:
"""
Update part of input mrope positions (starting with
num_computed_tokens index). Original mrope_positions are computed
@@ -2672,7 +2672,7 @@ class Qwen3VLForConditionalGeneration(
Args:
input_ids: (N,) All input tokens of the prompt containing
entire sequence.
multimodal_embeddings: Tuple of multimodal embeddings that
multimodal_embeddings: Sequence of multimodal embeddings that
fits into the prefill chunk that is being processed.
mrope_positions: Existing mrope positions (3, N) for entire
sequence
@@ -2694,14 +2694,14 @@ class Qwen3VLForConditionalGeneration(
@staticmethod
def _recompute_mrope_positions(
input_ids: list[int],
multimodal_embeddings: MultiModalEmbeddings,
input_ids: list[int] | torch.Tensor,
multimodal_embeddings: Sequence[torch.Tensor],
mrope_positions: torch.LongTensor,
num_computed_tokens: int,
vision_start_token_id: int,
image_token_id: int,
video_token_id: int,
) -> tuple[MultiModalEmbeddings, torch.Tensor, int]:
) -> tuple[Sequence[torch.Tensor], torch.Tensor, int]:
# Device
device = (
multimodal_embeddings[0].device
@@ -2709,8 +2709,12 @@ class Qwen3VLForConditionalGeneration(
else mrope_positions.device
)
# Tensors
input_ids_t = async_tensor_h2d(input_ids, device=device, dtype=torch.long)
# Tensors. input_ids may already be a (device-side) tensor.
if isinstance(input_ids, torch.Tensor):
assert input_ids.device == device
input_ids_t = input_ids.to(dtype=torch.long)
else:
input_ids_t = async_tensor_h2d(input_ids, device=device, dtype=torch.long)
mm_embeddings_out = []
mm_embeddings_pos = []
@@ -2738,7 +2742,7 @@ class Qwen3VLForConditionalGeneration(
video_token_id,
)
return tuple(mm_embeddings_out), positions, mrope_positions_delta
return mm_embeddings_out, positions, mrope_positions_delta
def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings | None:
mm_input_by_modality = self._parse_and_validate_multimodal_inputs(**kwargs)
+17
View File
@@ -90,6 +90,23 @@ class RopeState:
def get_positions(self, num_tokens: int) -> torch.Tensor:
return self.positions[:, :num_tokens]
def read_prefill_positions(self, req_idx: int, length: int) -> torch.Tensor:
"""Return staged per-request prefill positions as [num_dims, length]."""
base = self.num_dims * req_idx
return self.prefill_positions.gpu[base : base + self.num_dims, :length]
def update_prefill_positions(
self, req_idx: int, positions: torch.Tensor, delta: int
) -> None:
"""Overwrite a request's staged prefill positions with recomputed values."""
base = self.num_dims * req_idx
length = positions.shape[1]
self.prefill_positions.gpu[base : base + self.num_dims, :length].copy_(
positions
)
if self.has_delta:
self.prefill_delta.np[req_idx] = delta
def prepare_positions(
self,
idx_mapping: torch.Tensor,
+4 -10
View File
@@ -1229,7 +1229,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
scheduled_encoder_inputs=scheduler_output.scheduled_encoder_inputs,
)
inputs_embeds = self.model_state.get_mm_embeddings(
scheduler_output.scheduled_encoder_inputs, input_batch
scheduler_output.scheduled_encoder_inputs, input_batch, self.req_states
)
if inputs_embeds is not None and not self.model.requires_raw_input_tokens:
input_ids = None
@@ -1413,15 +1413,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
# Get cached multimodal embeddings for draft forward.
# NOTE: This is done here because postprocess updates
# num_computed_prefill_tokens.
mm_inputs = self.model_state.encoder_runner.gather_mm_embeddings(
input_batch.req_ids,
input_batch.num_tokens,
input_batch.num_scheduled_tokens,
input_batch.query_start_loc_np,
input_batch.prefill_len_np,
input_batch.num_computed_prefill_tokens_np,
# The EAGLE/MTP drafter reads one position ahead of the target.
draft_lookahead=1,
# The EAGLE/MTP drafter reads one position ahead of the target.
mm_inputs = self.model_state.gather_mm_embeddings(
input_batch, draft_lookahead=1
)
# Postprocess results and update request states.
+25 -8
View File
@@ -15,6 +15,7 @@ from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache
from vllm.v1.worker.gpu.mm.encoder_runner import EncoderRunner
from vllm.v1.worker.gpu.mm.rope import get_rope_state
from vllm.v1.worker.gpu.model_states.interface import ModelState
from vllm.v1.worker.gpu.model_states.mm_pruning import maybe_create_mm_pruner
from vllm.v1.worker.gpu.states import RequestState
from vllm.v1.worker.utils import AttentionGroup
@@ -61,6 +62,11 @@ class DefaultModelState(ModelState):
device=self.device,
)
# Pruner is used for multimodal embedding pruning (EVS).
self.mm_pruner = maybe_create_mm_pruner(
self.model_config, model, self.rope_state, encoder_cache
)
def add_request(self, req_index: int, new_req_data: NewRequestData) -> None:
if self.rope_state is not None:
assert new_req_data.prefill_token_ids is not None
@@ -79,6 +85,7 @@ class DefaultModelState(ModelState):
self,
scheduled_encoder_inputs: dict[str, list[int]],
input_batch: InputBatch,
req_states: RequestState,
) -> torch.Tensor:
mm_hashes, mm_kwargs = self.encoder_runner.prepare_mm_inputs(
scheduled_encoder_inputs
@@ -89,14 +96,13 @@ class DefaultModelState(ModelState):
# Cache the encoder outputs by mm_hash
self.encoder_cache.encoder_outputs.update(zip(mm_hashes, encoder_outputs))
mm_embeds, is_mm_embed = self.encoder_runner.gather_mm_embeddings(
input_batch.req_ids,
input_batch.num_tokens,
input_batch.num_scheduled_tokens,
input_batch.query_start_loc_np,
input_batch.prefill_len_np,
input_batch.num_computed_prefill_tokens_np,
)
mm_embeds, is_mm_embed = super().gather_mm_embeddings(input_batch)
if self.mm_pruner is not None and mm_embeds:
# EVS: recompute mrope positions for pruned media.
mm_embeds = self.mm_pruner.recompute(mm_embeds, input_batch, req_states)
# We must flush the staged rope updates for prepare_inputs() to pick up.
self.apply_staged_writes()
# Use unpadded input_ids to match is_mm_embed size (num_tokens).
# input_batch.input_ids may be padded for CUDA graphs.
input_ids_unpadded = input_batch.input_ids[: input_batch.num_tokens]
@@ -105,6 +111,17 @@ class DefaultModelState(ModelState):
)
return inputs_embeds[: input_batch.num_tokens_after_padding]
def gather_mm_embeddings(
self, input_batch: InputBatch, draft_lookahead: int = 0
) -> tuple[list[torch.Tensor], torch.Tensor]:
mm_embeds, is_mm_embed = super().gather_mm_embeddings(
input_batch, draft_lookahead
)
if self.mm_pruner is not None:
# EVS: strip the appended mrope-position channels.
mm_embeds = self.mm_pruner.strip(mm_embeds)
return mm_embeds, is_mm_embed
def prepare_inputs(
self, input_batch: InputBatch, req_states: RequestState
) -> dict[str, torch.Tensor | None]:
@@ -85,7 +85,10 @@ class EncoderDecoderModelState(ModelState):
self.encoder_outputs: list[torch.Tensor] = []
def get_mm_embeddings(
self, scheduled_encoder_inputs: dict[str, list[int]], input_batch: InputBatch
self,
scheduled_encoder_inputs: dict[str, list[int]],
input_batch: InputBatch,
req_states: RequestState,
) -> None:
# Ensure encoder inputs are ordered consistently with input_batch.req_ids.
encoder_inputs: dict[str, list[int]] = {}
+21 -1
View File
@@ -13,6 +13,7 @@ from vllm.v1.core.sched.output import NewRequestData
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.worker.gpu.input_batch import InputBatch
from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache
from vllm.v1.worker.gpu.mm.encoder_runner import EncoderRunner
from vllm.v1.worker.gpu.states import RequestState
from vllm.v1.worker.utils import AttentionGroup
@@ -47,6 +48,8 @@ class ModelState(ABC):
raise NotImplementedError
model: nn.Module
# Set by mm-capable states; used by the default gather_mm_embeddings().
encoder_runner: EncoderRunner
def get_supported_generation_tasks(self) -> tuple[GenerationTask, ...]:
from vllm.model_executor.models.interfaces import (
@@ -82,10 +85,27 @@ class ModelState(ABC):
@abstractmethod
def get_mm_embeddings(
self, scheduled_encoder_inputs: dict[str, list[int]], input_batch: InputBatch
self,
scheduled_encoder_inputs: dict[str, list[int]],
input_batch: InputBatch,
req_states: RequestState,
) -> torch.Tensor | None:
raise NotImplementedError
def gather_mm_embeddings(
self, input_batch: InputBatch, draft_lookahead: int = 0
) -> tuple[list[torch.Tensor], torch.Tensor]:
"""Gather cached multimodal embeddings for a speculator's draft forward."""
return self.encoder_runner.gather_mm_embeddings(
input_batch.req_ids,
input_batch.num_tokens,
input_batch.num_scheduled_tokens,
input_batch.query_start_loc_np,
input_batch.prefill_len_np,
input_batch.num_computed_prefill_tokens_np,
draft_lookahead=draft_lookahead,
)
@abstractmethod
def prepare_inputs(
self, input_batch: InputBatch, req_states: RequestState
@@ -0,0 +1,135 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
import torch.nn as nn
from vllm.config import ModelConfig
from vllm.model_executor.models.interfaces import supports_multimodal_pruning
from vllm.multimodal.utils import get_mm_features_in_window
from vllm.v1.worker.gpu.input_batch import InputBatch
from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache
from vllm.v1.worker.gpu.mm.rope import RopeState
from vllm.v1.worker.gpu.states import RequestState
class MultiModalPruner:
"""Recomputes M-RoPE positions for multimodal models that prune embeddings
(e.g. Qwen2.5-VL / Qwen3-VL / Nemotron-Nano-VL Efficient Video Sampling).
Pruning models append their mrope-position channels to the (variable-count)
media embeddings from `embed_multimodal`. Those channels must be split off and
used to recompute mrope positions before the embeddings are merged.
"""
def __init__(
self,
model: nn.Module,
rope_state: RopeState,
encoder_cache: EncoderCache,
inputs_embeds_size: int,
) -> None:
self.model = model
self.rope_state = rope_state
self.encoder_cache = encoder_cache
# The cleaned embedding width: pruning models append their mrope-position
# channels as trailing columns, so embeds[:, :inputs_embeds_size] strips them.
self.inputs_embeds_size = inputs_embeds_size
def strip(self, mm_embeds: list[torch.Tensor]) -> list[torch.Tensor]:
"""Draft forward: strip the appended position channels only.
Stripping is per-embedding, so no per-request segmentation is needed. The
speculator reuses the target's already-recomputed positions, hence there is
no position write-back here.
"""
return [mm[:, : self.inputs_embeds_size] for mm in mm_embeds]
def recompute(
self,
mm_embeds: list[torch.Tensor],
input_batch: InputBatch,
req_states: RequestState,
) -> list[torch.Tensor]:
"""Target forward: split the appended mrope-position channels off each
request's media embeddings, recompute the corrected mrope positions, and
stage them back into RopeState. Returns the cleaned, flattened embeddings.
"""
cleaned: list[torch.Tensor] = []
pos = 0
req_idx_list = input_batch.idx_mapping_np.tolist()
prefill_lens_list = input_batch.prefill_len_np.tolist()
num_computed_list = input_batch.num_computed_prefill_tokens_np.tolist()
num_scheduled_list = input_batch.num_scheduled_tokens.tolist()
for batch_idx, req_id in enumerate(input_batch.req_ids):
num_computed = num_computed_list[batch_idx]
query_end = num_computed + num_scheduled_list[batch_idx]
num_req_embeds = self._num_window_embeds(req_id, num_computed, query_end)
if num_req_embeds == 0:
continue
req_embeds = mm_embeds[pos : pos + num_req_embeds]
pos += num_req_embeds
req_idx = req_idx_list[batch_idx]
prefill_len = prefill_lens_list[batch_idx]
input_ids = req_states.all_token_ids.gpu[req_idx, :prefill_len]
mrope_positions = self.rope_state.read_prefill_positions(
req_idx, prefill_len
).long()
req_cleaned, new_positions, delta = self.model.recompute_mrope_positions(
input_ids=input_ids,
multimodal_embeddings=req_embeds,
mrope_positions=mrope_positions,
num_computed_tokens=num_computed,
)
self.rope_state.update_prefill_positions(req_idx, new_positions, delta)
cleaned.extend(req_cleaned)
assert pos == len(mm_embeds)
return cleaned
def _num_window_embeds(self, req_id: str, query_start: int, query_end: int) -> int:
"""Count the media items contributing embeddings to [query_start,
query_end), mirroring EncoderRunner.gather_mm_embeddings' per-request
windowing so the flat mm_embeds list can be re-segmented per request.
Note: This logic is intentionally duplicated here rather than being emitted
from gather_mm_embeddings, to keep the main path cleaner, since this is a niche
feature.
"""
mm_features = self.encoder_cache.mm_features[req_id]
lo, hi = get_mm_features_in_window(
mm_features, start=query_start, end=query_end
)
count = 0
for mm_feature in mm_features[lo:hi]:
pos_info = mm_feature.mm_position
start_idx = max(query_start - pos_info.offset, 0)
end_idx = min(query_end - pos_info.offset, pos_info.length)
embeds_start, embeds_end = pos_info.get_embeds_indices_in_range(
start_idx, end_idx
)
if embeds_start != embeds_end:
count += 1
return count
def maybe_create_mm_pruner(
model_config: ModelConfig,
model: nn.Module,
rope_state: RopeState | None,
encoder_cache: EncoderCache | None,
) -> MultiModalPruner | None:
"""Create a MultiModalPruner if the model prunes embeddings and uses M-RoPE."""
if (
not rope_state
or not rope_state.has_delta
or not encoder_cache
or not model_config.multimodal_config
or not model_config.multimodal_config.is_multimodal_pruning_enabled()
or not supports_multimodal_pruning(model)
):
return None
return MultiModalPruner(
model, rope_state, encoder_cache, model_config.get_inputs_embeds_size()
)