[ModelRunnerV2] Support prompt embeds (#42963)

Signed-off-by: gcanlin <canlinguosdu@gmail.com>
Signed-off-by: Canlin Guo <961750412@qq.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Canlin Guo
2026-08-17 21:58:50 +00:00
committed by GitHub
co-authored by Nick Hill
parent e68fb75b25
commit 455edc022b
17 changed files with 551 additions and 54 deletions
+12 -1
View File
@@ -18,7 +18,7 @@ from vllm.multimodal.utils import (
group_and_batch_mm_kwargs,
set_mm_embedding_modality,
)
from vllm.utils.torch_utils import PIN_MEMORY
from vllm.utils.torch_utils import PIN_MEMORY, async_tensor_h2d
from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache
from vllm.v1.worker.utils import (
EncoderTimingStats,
@@ -91,6 +91,17 @@ class EncoderRunner:
continue
if mm_feature.identifier in self.encoder_cache.encoder_outputs:
continue
if mm_feature.modality == "prompt_embeds":
# Passthrough modality: the tensor is already in the
# model's embedding space, so no encoder runs. Cache it
# directly so gather_mm_embeddings splices it via the
# standard is_mm_embed path.
embeds = mm_feature.data["embedding"].data
assert isinstance(embeds, torch.Tensor)
self.encoder_cache.encoder_outputs[mm_feature.identifier] = (
async_tensor_h2d(embeds, device=self.device)
)
continue
mm_hashes.append(mm_feature.identifier)
mm_kwargs.append((mm_feature.modality, mm_feature.data))