[ModelRunnerV2] Support prompt embeds (#42963)

Signed-off-by: gcanlin <canlinguosdu@gmail.com>
Signed-off-by: Canlin Guo <961750412@qq.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Canlin Guo
2026-08-17 21:58:50 +00:00
committed by GitHub
co-authored by Nick Hill
parent e68fb75b25
commit 455edc022b
17 changed files with 551 additions and 54 deletions
+14 -10
View File
@@ -46,6 +46,7 @@ from vllm.model_executor.layers.mamba.ops.ssu_dispatch import (
initialize_mamba_ssu_backend,
)
from vllm.model_executor.model_loader import get_model_loader
from vllm.model_executor.models.interfaces import requires_raw_input_tokens
from vllm.model_executor.offloader import (
create_offloader,
get_offloader,
@@ -231,6 +232,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
self.supports_mm_inputs = self.mm_registry.supports_multimodal_inputs(
self.model_config
)
self.uses_inputs_embeds = (
self.supports_mm_inputs or self.model_config.enable_prompt_embeds
)
self.encoder_cache = None
if self.supports_mm_inputs and self.is_first_pp_rank:
self.encoder_cache = EncoderCache()
@@ -956,7 +960,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
def add_requests(self, scheduler_output: SchedulerOutput) -> None:
for new_req_data in scheduler_output.scheduled_new_reqs:
assert new_req_data.prompt_token_ids is not None
assert new_req_data.prefill_token_ids is not None
req_id = new_req_data.req_id
@@ -965,7 +968,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
# with the updated prompt_token_ids and mm_features.
self._remove_request(req_id)
prompt_len = len(new_req_data.prompt_token_ids)
prompt_len = new_req_data.prompt_len
sampling_params = new_req_data.sampling_params
self.req_states.add_request(
req_id=req_id,
@@ -980,6 +983,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
if self.pooling_runner is not None:
assert new_req_data.pooling_params is not None
assert new_req_data.prompt_token_ids is not None
self.pooling_runner.add_request(
req_id,
req_index,
@@ -1550,17 +1554,17 @@ class GPUModelRunner(LoRAModelRunnerMixin):
input_ids = input_batch.input_ids
inputs_embeds = None
ec_connector_output = None
if self.supports_mm_inputs and self.is_first_pp_rank:
# Run MM encoder (if needed) and get multimodal embeddings.
# Only first PP rank prepares multimodal embeddings.
if self.uses_inputs_embeds and self.is_first_pp_rank:
# Prepare inputs_embeds (MM encoder outputs and/or prompt_embeds
# overlay). Only first PP rank prepares them.
if dummy_run:
# Obtain mm embeddings of correct shape for compiled model.
# Obtain embeddings of correct shape for compiled model.
inputs_embeds = self.model_state.dummy_inputs_embeds(
input_batch.num_tokens_after_padding
)
else:
scheduled_encoder_inputs = scheduler_output.scheduled_encoder_inputs
if self.lora_config is not None:
if self.supports_mm_inputs and self.lora_config is not None:
set_active_mm_loras(
model=self.model,
lora_manager=self.lora_manager,
@@ -1574,16 +1578,16 @@ class GPUModelRunner(LoRAModelRunnerMixin):
) as ec_connector_output:
if self.is_encoder_only:
# Encode and publish, nothing else: this instance runs no
# language model, so the gather inside get_mm_embeddings
# language model, so the gather inside prepare_inputs_embeds
# would build an inputs_embeds nobody reads -- and it
# raises "Encoder cache miss" for any scheduled item this
# instance did not encode, taking the engine down with it.
self.model_state.execute_mm_encoder(scheduled_encoder_inputs)
else:
inputs_embeds = self.model_state.get_mm_embeddings(
inputs_embeds = self.model_state.prepare_inputs_embeds(
scheduled_encoder_inputs, input_batch, self.req_states
)
if inputs_embeds is not None and not self.model.requires_raw_input_tokens:
if inputs_embeds is not None and not requires_raw_input_tokens(self.model):
input_ids = None
if self.is_encoder_only: