diff --git a/tests/v1/worker/test_encoder_runner.py b/tests/v1/worker/test_encoder_runner.py index 79c13a2b96..70c0426640 100644 --- a/tests/v1/worker/test_encoder_runner.py +++ b/tests/v1/worker/test_encoder_runner.py @@ -53,14 +53,14 @@ def _make_runner( def _gather(runner: EncoderRunner, *, num_scheduled: int, draft_lookahead: int): - # Single prefilling request, computed_prefill=0, prefill_len large. + # Single prefilling request, num_computed_tokens=0, prefill_len large. return runner.gather_mm_embeddings( req_ids=["req0"], total_num_scheduled_tokens=num_scheduled, num_scheduled_tokens=np.array([num_scheduled]), query_start_loc=np.array([0]), prefill_lens=np.array([1000]), - computed_prefill_lens=np.array([0]), + num_computed_tokens=np.array([0]), draft_lookahead=draft_lookahead, ) @@ -147,7 +147,7 @@ def test_multi_request_batch_gathers_per_request(draft_lookahead): num_scheduled_tokens=np.array([8, 8]), query_start_loc=np.array([0, 8]), prefill_lens=np.array([1000, 1000]), - computed_prefill_lens=np.array([0, 0]), + num_computed_tokens=np.array([0, 0]), draft_lookahead=draft_lookahead, ) diff --git a/vllm/model_executor/models/diffusion_gemma.py b/vllm/model_executor/models/diffusion_gemma.py index e28e2720a7..6121e55dab 100644 --- a/vllm/model_executor/models/diffusion_gemma.py +++ b/vllm/model_executor/models/diffusion_gemma.py @@ -878,7 +878,7 @@ class DiffusionGemmaModelState(ModelState): scheduled_encoder_inputs: dict[str, list[int]], input_batch: InputBatch, req_states: RequestState, - ) -> torch.Tensor: + ) -> torch.Tensor | None: if not self.supports_mm_inputs: return None diff --git a/vllm/v1/worker/gpu/mm/encoder_runner.py b/vllm/v1/worker/gpu/mm/encoder_runner.py index f0e99fae1f..48a3af2505 100644 --- a/vllm/v1/worker/gpu/mm/encoder_runner.py +++ b/vllm/v1/worker/gpu/mm/encoder_runner.py @@ -3,7 +3,7 @@ import numpy as np import torch -from vllm.model_executor.models.interfaces import SupportsMultiModal +from vllm.model_executor.models.interfaces import SupportsMultiModal, supports_realtime from vllm.multimodal.inputs import MultiModalKwargsItem from vllm.multimodal.utils import get_mm_features_in_window, group_and_batch_mm_kwargs from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache @@ -26,6 +26,7 @@ class EncoderRunner: self.encoder_cache = encoder_cache self.dtype = dtype self.device = device + self.is_realtime = supports_realtime(model) self.inputs_embeds = torch.zeros( max_num_tokens, hidden_size, dtype=dtype, device=device @@ -67,30 +68,32 @@ class EncoderRunner: num_scheduled_tokens: np.ndarray, query_start_loc: np.ndarray, prefill_lens: np.ndarray, - computed_prefill_lens: np.ndarray, + num_computed_tokens: np.ndarray, draft_lookahead: int = 0, ) -> tuple[list[torch.Tensor], torch.Tensor]: if draft_lookahead: - computed_prefill_lens = computed_prefill_lens + draft_lookahead + num_computed_tokens = num_computed_tokens + draft_lookahead - is_prefilling_np = computed_prefill_lens < prefill_lens - if not is_prefilling_np.any(): - # All decode requests, so no need to gather any embeddings. - return [], torch.zeros( - total_num_scheduled_tokens, dtype=torch.bool, device=self.device - ) - - is_prefilling = is_prefilling_np.tolist() - query_start = computed_prefill_lens.tolist() - query_end = (computed_prefill_lens + num_scheduled_tokens).tolist() - - mm_embeds: list[torch.Tensor] = [] is_mm_embed = torch.zeros( total_num_scheduled_tokens, dtype=torch.bool, device="cpu" ) + + # Whether to gather media embeddings this step. + exclude_embeddings: list[bool] | None = None + if not self.is_realtime: + # Non-realtime models only have media embeddings within the prompt. + is_decode = num_computed_tokens >= prefill_lens + if is_decode.all(): + # All decode requests, so no need to gather any embeddings. + return [], is_mm_embed + exclude_embeddings = is_decode.tolist() + + query_start = num_computed_tokens.tolist() + query_end = (num_computed_tokens + num_scheduled_tokens).tolist() + + mm_embeds: list[torch.Tensor] = [] for i, req_id in enumerate(req_ids): - if not is_prefilling[i]: - # OPTIMIZATION: Skip decode requests. + if exclude_embeddings is not None and exclude_embeddings[i]: continue cur_query_start = query_start[i] diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index cb46ffc3dc..927ece4fba 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -1218,20 +1218,25 @@ class GPUModelRunner(LoRAModelRunnerMixin): if self.supports_mm_inputs and self.is_first_pp_rank: # Run MM encoder (if needed) and get multimodal embeddings. # Only first PP rank prepares multimodal embeddings. - # NOTE(woosuk): We must call get_mm_embeddings even during dummy runs - # to obtain inputs_embeds, because the compiled model expects this input. - if self.lora_config is not None: - set_active_mm_loras( - model=self.model, - lora_manager=self.lora_manager, - encoder_cache=self.encoder_cache, - req_id_to_index=self.req_states.req_id_to_index, - lora_state=self.lora_state, - scheduled_encoder_inputs=scheduler_output.scheduled_encoder_inputs, + if dummy_run: + # Obtain mm embeddings of correct shape for compiled model. + inputs_embeds = self.model_state.dummy_inputs_embeds( + input_batch.num_tokens_after_padding + ) + else: + scheduled_encoder_inputs = scheduler_output.scheduled_encoder_inputs + if self.lora_config is not None: + set_active_mm_loras( + model=self.model, + lora_manager=self.lora_manager, + encoder_cache=self.encoder_cache, + req_id_to_index=self.req_states.req_id_to_index, + lora_state=self.lora_state, + scheduled_encoder_inputs=scheduled_encoder_inputs, + ) + inputs_embeds = self.model_state.get_mm_embeddings( + scheduled_encoder_inputs, input_batch, self.req_states ) - inputs_embeds = self.model_state.get_mm_embeddings( - scheduler_output.scheduled_encoder_inputs, input_batch, self.req_states - ) if inputs_embeds is not None and not self.model.requires_raw_input_tokens: input_ids = None diff --git a/vllm/v1/worker/gpu/model_states/default.py b/vllm/v1/worker/gpu/model_states/default.py index 22e6aa00bc..2e14eb2e7d 100644 --- a/vllm/v1/worker/gpu/model_states/default.py +++ b/vllm/v1/worker/gpu/model_states/default.py @@ -57,6 +57,10 @@ class DefaultModelState(ModelState): if self.rope_state is not None: self.rope_state.apply_staged_writes() + def dummy_inputs_embeds(self, num_tokens: int) -> torch.Tensor: + """Pre-allocated inputs_embeds buffer for dummy runs (contents unused).""" + return self.encoder_runner.inputs_embeds[:num_tokens] + def get_mm_embeddings( self, scheduled_encoder_inputs: dict[str, list[int]], diff --git a/vllm/v1/worker/gpu/model_states/interface.py b/vllm/v1/worker/gpu/model_states/interface.py index a4c436a423..c80e19547c 100644 --- a/vllm/v1/worker/gpu/model_states/interface.py +++ b/vllm/v1/worker/gpu/model_states/interface.py @@ -109,17 +109,21 @@ class ModelState(ABC): ) -> torch.Tensor | None: raise NotImplementedError + def dummy_inputs_embeds(self, num_tokens: int) -> torch.Tensor | None: + """Pre-allocated inputs_embeds buffer for dummy runs (contents unused).""" + return None + def gather_mm_embeddings( self, input_batch: InputBatch, draft_lookahead: int = 0 ) -> tuple[list[torch.Tensor], torch.Tensor]: - """Gather cached multimodal embeddings for a speculator's draft forward.""" + """Gather cached multimodal embeddings.""" return self.encoder_runner.gather_mm_embeddings( input_batch.req_ids, input_batch.num_tokens, input_batch.num_scheduled_tokens, input_batch.query_start_loc_np, input_batch.prefill_len_np, - input_batch.num_computed_prefill_tokens_np, + input_batch.num_computed_tokens_np, draft_lookahead=draft_lookahead, )