From 84c62e1cbdef4250fbfda83782fd250e07ad0256 Mon Sep 17 00:00:00 2001 From: Nick Hill Date: Wed, 24 Jun 2026 07:18:56 -0700 Subject: [PATCH] [Model Runner V2][MM] Support EVS (#46535) Signed-off-by: Nick Hill Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- vllm/model_executor/models/diffusion_gemma.py | 18 +-- vllm/model_executor/models/interfaces.py | 11 +- vllm/model_executor/models/qwen2_5_vl.py | 16 ++- vllm/model_executor/models/qwen3_vl.py | 24 ++-- vllm/v1/worker/gpu/mm/rope.py | 17 +++ vllm/v1/worker/gpu/model_runner.py | 14 +- vllm/v1/worker/gpu/model_states/default.py | 33 +++-- .../gpu/model_states/encoder_decoder.py | 5 +- vllm/v1/worker/gpu/model_states/interface.py | 22 ++- vllm/v1/worker/gpu/model_states/mm_pruning.py | 135 ++++++++++++++++++ 10 files changed, 245 insertions(+), 50 deletions(-) create mode 100644 vllm/v1/worker/gpu/model_states/mm_pruning.py diff --git a/vllm/model_executor/models/diffusion_gemma.py b/vllm/model_executor/models/diffusion_gemma.py index 91dd5e6b6a..85e0ef0467 100644 --- a/vllm/model_executor/models/diffusion_gemma.py +++ b/vllm/model_executor/models/diffusion_gemma.py @@ -49,10 +49,12 @@ from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.v1.outputs import LogprobsTensors from vllm.v1.worker.gpu.attn_utils import build_attn_metadata from vllm.v1.worker.gpu.buffer_utils import UvaBackedTensor, async_copy_to_gpu +from vllm.v1.worker.gpu.input_batch import InputBatch from vllm.v1.worker.gpu.model_states.interface import ModelState from vllm.v1.worker.gpu.sample.logprob import compute_topk_logprobs from vllm.v1.worker.gpu.sample.output import SamplerOutput from vllm.v1.worker.gpu.sample.penalties import use_penalty +from vllm.v1.worker.gpu.states import RequestState from .interfaces import ( SupportsMultiModal, @@ -869,7 +871,12 @@ class DiffusionGemmaModelState(ModelState): if idx is not None: self.diffusion_states.remove_request(idx) - def get_mm_embeddings(self, scheduled_encoder_inputs, input_batch): + def get_mm_embeddings( + self, + scheduled_encoder_inputs: dict[str, list[int]], + input_batch: InputBatch, + req_states: RequestState, + ) -> torch.Tensor: if not self.supports_mm_inputs: return None @@ -880,14 +887,7 @@ class DiffusionGemmaModelState(ModelState): encoder_outputs = self.encoder_runner.execute_mm_encoder(mm_kwargs) self.encoder_cache.encoder_outputs.update(zip(mm_hashes, encoder_outputs)) - mm_embeds, is_mm_embed = self.encoder_runner.gather_mm_embeddings( - input_batch.req_ids, - input_batch.num_tokens, - input_batch.num_scheduled_tokens, - input_batch.query_start_loc_np, - input_batch.prefill_len_np, - input_batch.num_computed_prefill_tokens_np, - ) + mm_embeds, is_mm_embed = self.gather_mm_embeddings(input_batch) if not mm_embeds: # No MM tokens in this batch (e.g. all-decode step). diff --git a/vllm/model_executor/models/interfaces.py b/vllm/model_executor/models/interfaces.py index 66d1fc6a4e..ad3d01ae9e 100644 --- a/vllm/model_executor/models/interfaces.py +++ b/vllm/model_executor/models/interfaces.py @@ -421,11 +421,11 @@ class SupportsMultiModalPruning(Protocol): def recompute_mrope_positions( self, - input_ids: list[int], - multimodal_embeddings: MultiModalEmbeddings, + input_ids: list[int] | torch.Tensor, + multimodal_embeddings: Sequence[torch.Tensor], mrope_positions: torch.LongTensor, num_computed_tokens: int, - ) -> tuple[MultiModalEmbeddings, Tensor, int]: + ) -> tuple[Sequence[torch.Tensor], Tensor, int]: """ Update part of input mrope positions (starting with num_computed_tokens index). Original mrope_positions are computed @@ -435,8 +435,9 @@ class SupportsMultiModalPruning(Protocol): Args: input_ids: (N,) All input tokens of the prompt containing - entire sequence. - multimodal_embeddings: Tuple of multimodal embeddings that + entire sequence. Either a host-side list or an already + device-resident tensor. + multimodal_embeddings: Sequence of multimodal embeddings that fits into the prefill chunk that is being processed. mrope_positions: Existing mrope positions (3, N) for entire sequence diff --git a/vllm/model_executor/models/qwen2_5_vl.py b/vllm/model_executor/models/qwen2_5_vl.py index 986783fa34..ebc51e9683 100644 --- a/vllm/model_executor/models/qwen2_5_vl.py +++ b/vllm/model_executor/models/qwen2_5_vl.py @@ -1583,11 +1583,11 @@ class Qwen2_5_VLForConditionalGeneration( def recompute_mrope_positions( self, - input_ids: list[int], - multimodal_embeddings: tuple[torch.Tensor, ...], + input_ids: list[int] | torch.Tensor, + multimodal_embeddings: Sequence[torch.Tensor], mrope_positions: torch.LongTensor, num_computed_tokens: int, - ) -> tuple[tuple[torch.Tensor, ...], torch.Tensor, int]: + ) -> tuple[Sequence[torch.Tensor], torch.Tensor, int]: """ Update part of input mrope positions (starting with num_computed_tokens index). Original mrope_positions are computed @@ -1618,8 +1618,12 @@ class Qwen2_5_VLForConditionalGeneration( else mrope_positions.device ) - # Tensors. - input_ids_t = async_tensor_h2d(input_ids, dtype=torch.long, device=device) + # Tensors. input_ids may already be a (device-side) tensor. + if isinstance(input_ids, torch.Tensor): + assert input_ids.device == device + input_ids_t = input_ids.to(torch.long) + else: + input_ids_t = async_tensor_h2d(input_ids, dtype=torch.long, device=device) mm_embeddings_out = [mm[:, :-4] for mm in multimodal_embeddings] mm_embeddings_pos = [ @@ -1636,7 +1640,7 @@ class Qwen2_5_VLForConditionalGeneration( video_token_id, ) - return tuple(mm_embeddings_out), positions, mrope_positions_delta + return mm_embeddings_out, positions, mrope_positions_delta def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict: mm_input_by_modality = {} diff --git a/vllm/model_executor/models/qwen3_vl.py b/vllm/model_executor/models/qwen3_vl.py index 0e6ddef7f3..a52f725ccb 100644 --- a/vllm/model_executor/models/qwen3_vl.py +++ b/vllm/model_executor/models/qwen3_vl.py @@ -2657,11 +2657,11 @@ class Qwen3VLForConditionalGeneration( def recompute_mrope_positions( self, - input_ids: list[int], - multimodal_embeddings: MultiModalEmbeddings, + input_ids: list[int] | torch.Tensor, + multimodal_embeddings: Sequence[torch.Tensor], mrope_positions: torch.LongTensor, num_computed_tokens: int, - ) -> tuple[MultiModalEmbeddings, torch.Tensor, int]: + ) -> tuple[Sequence[torch.Tensor], torch.Tensor, int]: """ Update part of input mrope positions (starting with num_computed_tokens index). Original mrope_positions are computed @@ -2672,7 +2672,7 @@ class Qwen3VLForConditionalGeneration( Args: input_ids: (N,) All input tokens of the prompt containing entire sequence. - multimodal_embeddings: Tuple of multimodal embeddings that + multimodal_embeddings: Sequence of multimodal embeddings that fits into the prefill chunk that is being processed. mrope_positions: Existing mrope positions (3, N) for entire sequence @@ -2694,14 +2694,14 @@ class Qwen3VLForConditionalGeneration( @staticmethod def _recompute_mrope_positions( - input_ids: list[int], - multimodal_embeddings: MultiModalEmbeddings, + input_ids: list[int] | torch.Tensor, + multimodal_embeddings: Sequence[torch.Tensor], mrope_positions: torch.LongTensor, num_computed_tokens: int, vision_start_token_id: int, image_token_id: int, video_token_id: int, - ) -> tuple[MultiModalEmbeddings, torch.Tensor, int]: + ) -> tuple[Sequence[torch.Tensor], torch.Tensor, int]: # Device device = ( multimodal_embeddings[0].device @@ -2709,8 +2709,12 @@ class Qwen3VLForConditionalGeneration( else mrope_positions.device ) - # Tensors - input_ids_t = async_tensor_h2d(input_ids, device=device, dtype=torch.long) + # Tensors. input_ids may already be a (device-side) tensor. + if isinstance(input_ids, torch.Tensor): + assert input_ids.device == device + input_ids_t = input_ids.to(dtype=torch.long) + else: + input_ids_t = async_tensor_h2d(input_ids, device=device, dtype=torch.long) mm_embeddings_out = [] mm_embeddings_pos = [] @@ -2738,7 +2742,7 @@ class Qwen3VLForConditionalGeneration( video_token_id, ) - return tuple(mm_embeddings_out), positions, mrope_positions_delta + return mm_embeddings_out, positions, mrope_positions_delta def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings | None: mm_input_by_modality = self._parse_and_validate_multimodal_inputs(**kwargs) diff --git a/vllm/v1/worker/gpu/mm/rope.py b/vllm/v1/worker/gpu/mm/rope.py index 712f58af57..e5de282234 100644 --- a/vllm/v1/worker/gpu/mm/rope.py +++ b/vllm/v1/worker/gpu/mm/rope.py @@ -90,6 +90,23 @@ class RopeState: def get_positions(self, num_tokens: int) -> torch.Tensor: return self.positions[:, :num_tokens] + def read_prefill_positions(self, req_idx: int, length: int) -> torch.Tensor: + """Return staged per-request prefill positions as [num_dims, length].""" + base = self.num_dims * req_idx + return self.prefill_positions.gpu[base : base + self.num_dims, :length] + + def update_prefill_positions( + self, req_idx: int, positions: torch.Tensor, delta: int + ) -> None: + """Overwrite a request's staged prefill positions with recomputed values.""" + base = self.num_dims * req_idx + length = positions.shape[1] + self.prefill_positions.gpu[base : base + self.num_dims, :length].copy_( + positions + ) + if self.has_delta: + self.prefill_delta.np[req_idx] = delta + def prepare_positions( self, idx_mapping: torch.Tensor, diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index 4e1594e806..de8cd476cd 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -1229,7 +1229,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): scheduled_encoder_inputs=scheduler_output.scheduled_encoder_inputs, ) inputs_embeds = self.model_state.get_mm_embeddings( - scheduler_output.scheduled_encoder_inputs, input_batch + scheduler_output.scheduled_encoder_inputs, input_batch, self.req_states ) if inputs_embeds is not None and not self.model.requires_raw_input_tokens: input_ids = None @@ -1413,15 +1413,9 @@ class GPUModelRunner(LoRAModelRunnerMixin): # Get cached multimodal embeddings for draft forward. # NOTE: This is done here because postprocess updates # num_computed_prefill_tokens. - mm_inputs = self.model_state.encoder_runner.gather_mm_embeddings( - input_batch.req_ids, - input_batch.num_tokens, - input_batch.num_scheduled_tokens, - input_batch.query_start_loc_np, - input_batch.prefill_len_np, - input_batch.num_computed_prefill_tokens_np, - # The EAGLE/MTP drafter reads one position ahead of the target. - draft_lookahead=1, + # The EAGLE/MTP drafter reads one position ahead of the target. + mm_inputs = self.model_state.gather_mm_embeddings( + input_batch, draft_lookahead=1 ) # Postprocess results and update request states. diff --git a/vllm/v1/worker/gpu/model_states/default.py b/vllm/v1/worker/gpu/model_states/default.py index 4d71b4d5b3..05ff827886 100644 --- a/vllm/v1/worker/gpu/model_states/default.py +++ b/vllm/v1/worker/gpu/model_states/default.py @@ -15,6 +15,7 @@ from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache from vllm.v1.worker.gpu.mm.encoder_runner import EncoderRunner from vllm.v1.worker.gpu.mm.rope import get_rope_state from vllm.v1.worker.gpu.model_states.interface import ModelState +from vllm.v1.worker.gpu.model_states.mm_pruning import maybe_create_mm_pruner from vllm.v1.worker.gpu.states import RequestState from vllm.v1.worker.utils import AttentionGroup @@ -61,6 +62,11 @@ class DefaultModelState(ModelState): device=self.device, ) + # Pruner is used for multimodal embedding pruning (EVS). + self.mm_pruner = maybe_create_mm_pruner( + self.model_config, model, self.rope_state, encoder_cache + ) + def add_request(self, req_index: int, new_req_data: NewRequestData) -> None: if self.rope_state is not None: assert new_req_data.prefill_token_ids is not None @@ -79,6 +85,7 @@ class DefaultModelState(ModelState): self, scheduled_encoder_inputs: dict[str, list[int]], input_batch: InputBatch, + req_states: RequestState, ) -> torch.Tensor: mm_hashes, mm_kwargs = self.encoder_runner.prepare_mm_inputs( scheduled_encoder_inputs @@ -89,14 +96,13 @@ class DefaultModelState(ModelState): # Cache the encoder outputs by mm_hash self.encoder_cache.encoder_outputs.update(zip(mm_hashes, encoder_outputs)) - mm_embeds, is_mm_embed = self.encoder_runner.gather_mm_embeddings( - input_batch.req_ids, - input_batch.num_tokens, - input_batch.num_scheduled_tokens, - input_batch.query_start_loc_np, - input_batch.prefill_len_np, - input_batch.num_computed_prefill_tokens_np, - ) + mm_embeds, is_mm_embed = super().gather_mm_embeddings(input_batch) + if self.mm_pruner is not None and mm_embeds: + # EVS: recompute mrope positions for pruned media. + mm_embeds = self.mm_pruner.recompute(mm_embeds, input_batch, req_states) + # We must flush the staged rope updates for prepare_inputs() to pick up. + self.apply_staged_writes() + # Use unpadded input_ids to match is_mm_embed size (num_tokens). # input_batch.input_ids may be padded for CUDA graphs. input_ids_unpadded = input_batch.input_ids[: input_batch.num_tokens] @@ -105,6 +111,17 @@ class DefaultModelState(ModelState): ) return inputs_embeds[: input_batch.num_tokens_after_padding] + def gather_mm_embeddings( + self, input_batch: InputBatch, draft_lookahead: int = 0 + ) -> tuple[list[torch.Tensor], torch.Tensor]: + mm_embeds, is_mm_embed = super().gather_mm_embeddings( + input_batch, draft_lookahead + ) + if self.mm_pruner is not None: + # EVS: strip the appended mrope-position channels. + mm_embeds = self.mm_pruner.strip(mm_embeds) + return mm_embeds, is_mm_embed + def prepare_inputs( self, input_batch: InputBatch, req_states: RequestState ) -> dict[str, torch.Tensor | None]: diff --git a/vllm/v1/worker/gpu/model_states/encoder_decoder.py b/vllm/v1/worker/gpu/model_states/encoder_decoder.py index 2a07e4a1a3..6ad9a448be 100644 --- a/vllm/v1/worker/gpu/model_states/encoder_decoder.py +++ b/vllm/v1/worker/gpu/model_states/encoder_decoder.py @@ -85,7 +85,10 @@ class EncoderDecoderModelState(ModelState): self.encoder_outputs: list[torch.Tensor] = [] def get_mm_embeddings( - self, scheduled_encoder_inputs: dict[str, list[int]], input_batch: InputBatch + self, + scheduled_encoder_inputs: dict[str, list[int]], + input_batch: InputBatch, + req_states: RequestState, ) -> None: # Ensure encoder inputs are ordered consistently with input_batch.req_ids. encoder_inputs: dict[str, list[int]] = {} diff --git a/vllm/v1/worker/gpu/model_states/interface.py b/vllm/v1/worker/gpu/model_states/interface.py index be631bb94b..882b38073c 100644 --- a/vllm/v1/worker/gpu/model_states/interface.py +++ b/vllm/v1/worker/gpu/model_states/interface.py @@ -13,6 +13,7 @@ from vllm.v1.core.sched.output import NewRequestData from vllm.v1.kv_cache_interface import KVCacheConfig from vllm.v1.worker.gpu.input_batch import InputBatch from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache +from vllm.v1.worker.gpu.mm.encoder_runner import EncoderRunner from vllm.v1.worker.gpu.states import RequestState from vllm.v1.worker.utils import AttentionGroup @@ -47,6 +48,8 @@ class ModelState(ABC): raise NotImplementedError model: nn.Module + # Set by mm-capable states; used by the default gather_mm_embeddings(). + encoder_runner: EncoderRunner def get_supported_generation_tasks(self) -> tuple[GenerationTask, ...]: from vllm.model_executor.models.interfaces import ( @@ -82,10 +85,27 @@ class ModelState(ABC): @abstractmethod def get_mm_embeddings( - self, scheduled_encoder_inputs: dict[str, list[int]], input_batch: InputBatch + self, + scheduled_encoder_inputs: dict[str, list[int]], + input_batch: InputBatch, + req_states: RequestState, ) -> torch.Tensor | None: raise NotImplementedError + def gather_mm_embeddings( + self, input_batch: InputBatch, draft_lookahead: int = 0 + ) -> tuple[list[torch.Tensor], torch.Tensor]: + """Gather cached multimodal embeddings for a speculator's draft forward.""" + return self.encoder_runner.gather_mm_embeddings( + input_batch.req_ids, + input_batch.num_tokens, + input_batch.num_scheduled_tokens, + input_batch.query_start_loc_np, + input_batch.prefill_len_np, + input_batch.num_computed_prefill_tokens_np, + draft_lookahead=draft_lookahead, + ) + @abstractmethod def prepare_inputs( self, input_batch: InputBatch, req_states: RequestState diff --git a/vllm/v1/worker/gpu/model_states/mm_pruning.py b/vllm/v1/worker/gpu/model_states/mm_pruning.py new file mode 100644 index 0000000000..e1eb098792 --- /dev/null +++ b/vllm/v1/worker/gpu/model_states/mm_pruning.py @@ -0,0 +1,135 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import torch +import torch.nn as nn + +from vllm.config import ModelConfig +from vllm.model_executor.models.interfaces import supports_multimodal_pruning +from vllm.multimodal.utils import get_mm_features_in_window +from vllm.v1.worker.gpu.input_batch import InputBatch +from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache +from vllm.v1.worker.gpu.mm.rope import RopeState +from vllm.v1.worker.gpu.states import RequestState + + +class MultiModalPruner: + """Recomputes M-RoPE positions for multimodal models that prune embeddings + (e.g. Qwen2.5-VL / Qwen3-VL / Nemotron-Nano-VL Efficient Video Sampling). + + Pruning models append their mrope-position channels to the (variable-count) + media embeddings from `embed_multimodal`. Those channels must be split off and + used to recompute mrope positions before the embeddings are merged. + """ + + def __init__( + self, + model: nn.Module, + rope_state: RopeState, + encoder_cache: EncoderCache, + inputs_embeds_size: int, + ) -> None: + self.model = model + self.rope_state = rope_state + self.encoder_cache = encoder_cache + # The cleaned embedding width: pruning models append their mrope-position + # channels as trailing columns, so embeds[:, :inputs_embeds_size] strips them. + self.inputs_embeds_size = inputs_embeds_size + + def strip(self, mm_embeds: list[torch.Tensor]) -> list[torch.Tensor]: + """Draft forward: strip the appended position channels only. + + Stripping is per-embedding, so no per-request segmentation is needed. The + speculator reuses the target's already-recomputed positions, hence there is + no position write-back here. + """ + return [mm[:, : self.inputs_embeds_size] for mm in mm_embeds] + + def recompute( + self, + mm_embeds: list[torch.Tensor], + input_batch: InputBatch, + req_states: RequestState, + ) -> list[torch.Tensor]: + """Target forward: split the appended mrope-position channels off each + request's media embeddings, recompute the corrected mrope positions, and + stage them back into RopeState. Returns the cleaned, flattened embeddings. + """ + cleaned: list[torch.Tensor] = [] + pos = 0 + req_idx_list = input_batch.idx_mapping_np.tolist() + prefill_lens_list = input_batch.prefill_len_np.tolist() + num_computed_list = input_batch.num_computed_prefill_tokens_np.tolist() + num_scheduled_list = input_batch.num_scheduled_tokens.tolist() + for batch_idx, req_id in enumerate(input_batch.req_ids): + num_computed = num_computed_list[batch_idx] + query_end = num_computed + num_scheduled_list[batch_idx] + num_req_embeds = self._num_window_embeds(req_id, num_computed, query_end) + if num_req_embeds == 0: + continue + req_embeds = mm_embeds[pos : pos + num_req_embeds] + pos += num_req_embeds + + req_idx = req_idx_list[batch_idx] + prefill_len = prefill_lens_list[batch_idx] + input_ids = req_states.all_token_ids.gpu[req_idx, :prefill_len] + mrope_positions = self.rope_state.read_prefill_positions( + req_idx, prefill_len + ).long() + req_cleaned, new_positions, delta = self.model.recompute_mrope_positions( + input_ids=input_ids, + multimodal_embeddings=req_embeds, + mrope_positions=mrope_positions, + num_computed_tokens=num_computed, + ) + self.rope_state.update_prefill_positions(req_idx, new_positions, delta) + cleaned.extend(req_cleaned) + + assert pos == len(mm_embeds) + return cleaned + + def _num_window_embeds(self, req_id: str, query_start: int, query_end: int) -> int: + """Count the media items contributing embeddings to [query_start, + query_end), mirroring EncoderRunner.gather_mm_embeddings' per-request + windowing so the flat mm_embeds list can be re-segmented per request. + + Note: This logic is intentionally duplicated here rather than being emitted + from gather_mm_embeddings, to keep the main path cleaner, since this is a niche + feature. + """ + mm_features = self.encoder_cache.mm_features[req_id] + lo, hi = get_mm_features_in_window( + mm_features, start=query_start, end=query_end + ) + count = 0 + for mm_feature in mm_features[lo:hi]: + pos_info = mm_feature.mm_position + start_idx = max(query_start - pos_info.offset, 0) + end_idx = min(query_end - pos_info.offset, pos_info.length) + embeds_start, embeds_end = pos_info.get_embeds_indices_in_range( + start_idx, end_idx + ) + if embeds_start != embeds_end: + count += 1 + return count + + +def maybe_create_mm_pruner( + model_config: ModelConfig, + model: nn.Module, + rope_state: RopeState | None, + encoder_cache: EncoderCache | None, +) -> MultiModalPruner | None: + """Create a MultiModalPruner if the model prunes embeddings and uses M-RoPE.""" + if ( + not rope_state + or not rope_state.has_delta + or not encoder_cache + or not model_config.multimodal_config + or not model_config.multimodal_config.is_multimodal_pruning_enabled() + or not supports_multimodal_pruning(model) + ): + return None + + return MultiModalPruner( + model, rope_state, encoder_cache, model_config.get_inputs_embeds_size() + )