[ModelRunner V2] Support realtime embeddings (#46762)
This commit is contained in:
@@ -53,14 +53,14 @@ def _make_runner(
|
||||
|
||||
|
||||
def _gather(runner: EncoderRunner, *, num_scheduled: int, draft_lookahead: int):
|
||||
# Single prefilling request, computed_prefill=0, prefill_len large.
|
||||
# Single prefilling request, num_computed_tokens=0, prefill_len large.
|
||||
return runner.gather_mm_embeddings(
|
||||
req_ids=["req0"],
|
||||
total_num_scheduled_tokens=num_scheduled,
|
||||
num_scheduled_tokens=np.array([num_scheduled]),
|
||||
query_start_loc=np.array([0]),
|
||||
prefill_lens=np.array([1000]),
|
||||
computed_prefill_lens=np.array([0]),
|
||||
num_computed_tokens=np.array([0]),
|
||||
draft_lookahead=draft_lookahead,
|
||||
)
|
||||
|
||||
@@ -147,7 +147,7 @@ def test_multi_request_batch_gathers_per_request(draft_lookahead):
|
||||
num_scheduled_tokens=np.array([8, 8]),
|
||||
query_start_loc=np.array([0, 8]),
|
||||
prefill_lens=np.array([1000, 1000]),
|
||||
computed_prefill_lens=np.array([0, 0]),
|
||||
num_computed_tokens=np.array([0, 0]),
|
||||
draft_lookahead=draft_lookahead,
|
||||
)
|
||||
|
||||
|
||||
@@ -878,7 +878,7 @@ class DiffusionGemmaModelState(ModelState):
|
||||
scheduled_encoder_inputs: dict[str, list[int]],
|
||||
input_batch: InputBatch,
|
||||
req_states: RequestState,
|
||||
) -> torch.Tensor:
|
||||
) -> torch.Tensor | None:
|
||||
if not self.supports_mm_inputs:
|
||||
return None
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.models.interfaces import SupportsMultiModal
|
||||
from vllm.model_executor.models.interfaces import SupportsMultiModal, supports_realtime
|
||||
from vllm.multimodal.inputs import MultiModalKwargsItem
|
||||
from vllm.multimodal.utils import get_mm_features_in_window, group_and_batch_mm_kwargs
|
||||
from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache
|
||||
@@ -26,6 +26,7 @@ class EncoderRunner:
|
||||
self.encoder_cache = encoder_cache
|
||||
self.dtype = dtype
|
||||
self.device = device
|
||||
self.is_realtime = supports_realtime(model)
|
||||
|
||||
self.inputs_embeds = torch.zeros(
|
||||
max_num_tokens, hidden_size, dtype=dtype, device=device
|
||||
@@ -67,30 +68,32 @@ class EncoderRunner:
|
||||
num_scheduled_tokens: np.ndarray,
|
||||
query_start_loc: np.ndarray,
|
||||
prefill_lens: np.ndarray,
|
||||
computed_prefill_lens: np.ndarray,
|
||||
num_computed_tokens: np.ndarray,
|
||||
draft_lookahead: int = 0,
|
||||
) -> tuple[list[torch.Tensor], torch.Tensor]:
|
||||
if draft_lookahead:
|
||||
computed_prefill_lens = computed_prefill_lens + draft_lookahead
|
||||
num_computed_tokens = num_computed_tokens + draft_lookahead
|
||||
|
||||
is_prefilling_np = computed_prefill_lens < prefill_lens
|
||||
if not is_prefilling_np.any():
|
||||
# All decode requests, so no need to gather any embeddings.
|
||||
return [], torch.zeros(
|
||||
total_num_scheduled_tokens, dtype=torch.bool, device=self.device
|
||||
)
|
||||
|
||||
is_prefilling = is_prefilling_np.tolist()
|
||||
query_start = computed_prefill_lens.tolist()
|
||||
query_end = (computed_prefill_lens + num_scheduled_tokens).tolist()
|
||||
|
||||
mm_embeds: list[torch.Tensor] = []
|
||||
is_mm_embed = torch.zeros(
|
||||
total_num_scheduled_tokens, dtype=torch.bool, device="cpu"
|
||||
)
|
||||
|
||||
# Whether to gather media embeddings this step.
|
||||
exclude_embeddings: list[bool] | None = None
|
||||
if not self.is_realtime:
|
||||
# Non-realtime models only have media embeddings within the prompt.
|
||||
is_decode = num_computed_tokens >= prefill_lens
|
||||
if is_decode.all():
|
||||
# All decode requests, so no need to gather any embeddings.
|
||||
return [], is_mm_embed
|
||||
exclude_embeddings = is_decode.tolist()
|
||||
|
||||
query_start = num_computed_tokens.tolist()
|
||||
query_end = (num_computed_tokens + num_scheduled_tokens).tolist()
|
||||
|
||||
mm_embeds: list[torch.Tensor] = []
|
||||
for i, req_id in enumerate(req_ids):
|
||||
if not is_prefilling[i]:
|
||||
# OPTIMIZATION: Skip decode requests.
|
||||
if exclude_embeddings is not None and exclude_embeddings[i]:
|
||||
continue
|
||||
|
||||
cur_query_start = query_start[i]
|
||||
|
||||
@@ -1218,20 +1218,25 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
if self.supports_mm_inputs and self.is_first_pp_rank:
|
||||
# Run MM encoder (if needed) and get multimodal embeddings.
|
||||
# Only first PP rank prepares multimodal embeddings.
|
||||
# NOTE(woosuk): We must call get_mm_embeddings even during dummy runs
|
||||
# to obtain inputs_embeds, because the compiled model expects this input.
|
||||
if self.lora_config is not None:
|
||||
set_active_mm_loras(
|
||||
model=self.model,
|
||||
lora_manager=self.lora_manager,
|
||||
encoder_cache=self.encoder_cache,
|
||||
req_id_to_index=self.req_states.req_id_to_index,
|
||||
lora_state=self.lora_state,
|
||||
scheduled_encoder_inputs=scheduler_output.scheduled_encoder_inputs,
|
||||
if dummy_run:
|
||||
# Obtain mm embeddings of correct shape for compiled model.
|
||||
inputs_embeds = self.model_state.dummy_inputs_embeds(
|
||||
input_batch.num_tokens_after_padding
|
||||
)
|
||||
else:
|
||||
scheduled_encoder_inputs = scheduler_output.scheduled_encoder_inputs
|
||||
if self.lora_config is not None:
|
||||
set_active_mm_loras(
|
||||
model=self.model,
|
||||
lora_manager=self.lora_manager,
|
||||
encoder_cache=self.encoder_cache,
|
||||
req_id_to_index=self.req_states.req_id_to_index,
|
||||
lora_state=self.lora_state,
|
||||
scheduled_encoder_inputs=scheduled_encoder_inputs,
|
||||
)
|
||||
inputs_embeds = self.model_state.get_mm_embeddings(
|
||||
scheduled_encoder_inputs, input_batch, self.req_states
|
||||
)
|
||||
inputs_embeds = self.model_state.get_mm_embeddings(
|
||||
scheduler_output.scheduled_encoder_inputs, input_batch, self.req_states
|
||||
)
|
||||
if inputs_embeds is not None and not self.model.requires_raw_input_tokens:
|
||||
input_ids = None
|
||||
|
||||
|
||||
@@ -57,6 +57,10 @@ class DefaultModelState(ModelState):
|
||||
if self.rope_state is not None:
|
||||
self.rope_state.apply_staged_writes()
|
||||
|
||||
def dummy_inputs_embeds(self, num_tokens: int) -> torch.Tensor:
|
||||
"""Pre-allocated inputs_embeds buffer for dummy runs (contents unused)."""
|
||||
return self.encoder_runner.inputs_embeds[:num_tokens]
|
||||
|
||||
def get_mm_embeddings(
|
||||
self,
|
||||
scheduled_encoder_inputs: dict[str, list[int]],
|
||||
|
||||
@@ -109,17 +109,21 @@ class ModelState(ABC):
|
||||
) -> torch.Tensor | None:
|
||||
raise NotImplementedError
|
||||
|
||||
def dummy_inputs_embeds(self, num_tokens: int) -> torch.Tensor | None:
|
||||
"""Pre-allocated inputs_embeds buffer for dummy runs (contents unused)."""
|
||||
return None
|
||||
|
||||
def gather_mm_embeddings(
|
||||
self, input_batch: InputBatch, draft_lookahead: int = 0
|
||||
) -> tuple[list[torch.Tensor], torch.Tensor]:
|
||||
"""Gather cached multimodal embeddings for a speculator's draft forward."""
|
||||
"""Gather cached multimodal embeddings."""
|
||||
return self.encoder_runner.gather_mm_embeddings(
|
||||
input_batch.req_ids,
|
||||
input_batch.num_tokens,
|
||||
input_batch.num_scheduled_tokens,
|
||||
input_batch.query_start_loc_np,
|
||||
input_batch.prefill_len_np,
|
||||
input_batch.num_computed_prefill_tokens_np,
|
||||
input_batch.num_computed_tokens_np,
|
||||
draft_lookahead=draft_lookahead,
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user