[ModelRunner V2] Support realtime embeddings (#46762)

This commit is contained in:
Nick Hill
2026-06-26 19:42:27 -07:00
committed by GitHub
parent af16446bf3
commit c6dd32a810
6 changed files with 52 additions and 36 deletions
+3 -3
View File
@@ -53,14 +53,14 @@ def _make_runner(
def _gather(runner: EncoderRunner, *, num_scheduled: int, draft_lookahead: int):
# Single prefilling request, computed_prefill=0, prefill_len large.
# Single prefilling request, num_computed_tokens=0, prefill_len large.
return runner.gather_mm_embeddings(
req_ids=["req0"],
total_num_scheduled_tokens=num_scheduled,
num_scheduled_tokens=np.array([num_scheduled]),
query_start_loc=np.array([0]),
prefill_lens=np.array([1000]),
computed_prefill_lens=np.array([0]),
num_computed_tokens=np.array([0]),
draft_lookahead=draft_lookahead,
)
@@ -147,7 +147,7 @@ def test_multi_request_batch_gathers_per_request(draft_lookahead):
num_scheduled_tokens=np.array([8, 8]),
query_start_loc=np.array([0, 8]),
prefill_lens=np.array([1000, 1000]),
computed_prefill_lens=np.array([0, 0]),
num_computed_tokens=np.array([0, 0]),
draft_lookahead=draft_lookahead,
)
@@ -878,7 +878,7 @@ class DiffusionGemmaModelState(ModelState):
scheduled_encoder_inputs: dict[str, list[int]],
input_batch: InputBatch,
req_states: RequestState,
) -> torch.Tensor:
) -> torch.Tensor | None:
if not self.supports_mm_inputs:
return None
+20 -17
View File
@@ -3,7 +3,7 @@
import numpy as np
import torch
from vllm.model_executor.models.interfaces import SupportsMultiModal
from vllm.model_executor.models.interfaces import SupportsMultiModal, supports_realtime
from vllm.multimodal.inputs import MultiModalKwargsItem
from vllm.multimodal.utils import get_mm_features_in_window, group_and_batch_mm_kwargs
from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache
@@ -26,6 +26,7 @@ class EncoderRunner:
self.encoder_cache = encoder_cache
self.dtype = dtype
self.device = device
self.is_realtime = supports_realtime(model)
self.inputs_embeds = torch.zeros(
max_num_tokens, hidden_size, dtype=dtype, device=device
@@ -67,30 +68,32 @@ class EncoderRunner:
num_scheduled_tokens: np.ndarray,
query_start_loc: np.ndarray,
prefill_lens: np.ndarray,
computed_prefill_lens: np.ndarray,
num_computed_tokens: np.ndarray,
draft_lookahead: int = 0,
) -> tuple[list[torch.Tensor], torch.Tensor]:
if draft_lookahead:
computed_prefill_lens = computed_prefill_lens + draft_lookahead
num_computed_tokens = num_computed_tokens + draft_lookahead
is_prefilling_np = computed_prefill_lens < prefill_lens
if not is_prefilling_np.any():
# All decode requests, so no need to gather any embeddings.
return [], torch.zeros(
total_num_scheduled_tokens, dtype=torch.bool, device=self.device
)
is_prefilling = is_prefilling_np.tolist()
query_start = computed_prefill_lens.tolist()
query_end = (computed_prefill_lens + num_scheduled_tokens).tolist()
mm_embeds: list[torch.Tensor] = []
is_mm_embed = torch.zeros(
total_num_scheduled_tokens, dtype=torch.bool, device="cpu"
)
# Whether to gather media embeddings this step.
exclude_embeddings: list[bool] | None = None
if not self.is_realtime:
# Non-realtime models only have media embeddings within the prompt.
is_decode = num_computed_tokens >= prefill_lens
if is_decode.all():
# All decode requests, so no need to gather any embeddings.
return [], is_mm_embed
exclude_embeddings = is_decode.tolist()
query_start = num_computed_tokens.tolist()
query_end = (num_computed_tokens + num_scheduled_tokens).tolist()
mm_embeds: list[torch.Tensor] = []
for i, req_id in enumerate(req_ids):
if not is_prefilling[i]:
# OPTIMIZATION: Skip decode requests.
if exclude_embeddings is not None and exclude_embeddings[i]:
continue
cur_query_start = query_start[i]
+18 -13
View File
@@ -1218,20 +1218,25 @@ class GPUModelRunner(LoRAModelRunnerMixin):
if self.supports_mm_inputs and self.is_first_pp_rank:
# Run MM encoder (if needed) and get multimodal embeddings.
# Only first PP rank prepares multimodal embeddings.
# NOTE(woosuk): We must call get_mm_embeddings even during dummy runs
# to obtain inputs_embeds, because the compiled model expects this input.
if self.lora_config is not None:
set_active_mm_loras(
model=self.model,
lora_manager=self.lora_manager,
encoder_cache=self.encoder_cache,
req_id_to_index=self.req_states.req_id_to_index,
lora_state=self.lora_state,
scheduled_encoder_inputs=scheduler_output.scheduled_encoder_inputs,
if dummy_run:
# Obtain mm embeddings of correct shape for compiled model.
inputs_embeds = self.model_state.dummy_inputs_embeds(
input_batch.num_tokens_after_padding
)
else:
scheduled_encoder_inputs = scheduler_output.scheduled_encoder_inputs
if self.lora_config is not None:
set_active_mm_loras(
model=self.model,
lora_manager=self.lora_manager,
encoder_cache=self.encoder_cache,
req_id_to_index=self.req_states.req_id_to_index,
lora_state=self.lora_state,
scheduled_encoder_inputs=scheduled_encoder_inputs,
)
inputs_embeds = self.model_state.get_mm_embeddings(
scheduled_encoder_inputs, input_batch, self.req_states
)
inputs_embeds = self.model_state.get_mm_embeddings(
scheduler_output.scheduled_encoder_inputs, input_batch, self.req_states
)
if inputs_embeds is not None and not self.model.requires_raw_input_tokens:
input_ids = None
@@ -57,6 +57,10 @@ class DefaultModelState(ModelState):
if self.rope_state is not None:
self.rope_state.apply_staged_writes()
def dummy_inputs_embeds(self, num_tokens: int) -> torch.Tensor:
"""Pre-allocated inputs_embeds buffer for dummy runs (contents unused)."""
return self.encoder_runner.inputs_embeds[:num_tokens]
def get_mm_embeddings(
self,
scheduled_encoder_inputs: dict[str, list[int]],
+6 -2
View File
@@ -109,17 +109,21 @@ class ModelState(ABC):
) -> torch.Tensor | None:
raise NotImplementedError
def dummy_inputs_embeds(self, num_tokens: int) -> torch.Tensor | None:
"""Pre-allocated inputs_embeds buffer for dummy runs (contents unused)."""
return None
def gather_mm_embeddings(
self, input_batch: InputBatch, draft_lookahead: int = 0
) -> tuple[list[torch.Tensor], torch.Tensor]:
"""Gather cached multimodal embeddings for a speculator's draft forward."""
"""Gather cached multimodal embeddings."""
return self.encoder_runner.gather_mm_embeddings(
input_batch.req_ids,
input_batch.num_tokens,
input_batch.num_scheduled_tokens,
input_batch.query_start_loc_np,
input_batch.prefill_len_np,
input_batch.num_computed_prefill_tokens_np,
input_batch.num_computed_tokens_np,
draft_lookahead=draft_lookahead,
)