[ModelRunnerV2] Support prompt embeds (#42963)
Signed-off-by: gcanlin <canlinguosdu@gmail.com> Signed-off-by: Canlin Guo <961750412@qq.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -18,7 +18,7 @@ from vllm.multimodal.utils import (
|
||||
group_and_batch_mm_kwargs,
|
||||
set_mm_embedding_modality,
|
||||
)
|
||||
from vllm.utils.torch_utils import PIN_MEMORY
|
||||
from vllm.utils.torch_utils import PIN_MEMORY, async_tensor_h2d
|
||||
from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache
|
||||
from vllm.v1.worker.utils import (
|
||||
EncoderTimingStats,
|
||||
@@ -91,6 +91,17 @@ class EncoderRunner:
|
||||
continue
|
||||
if mm_feature.identifier in self.encoder_cache.encoder_outputs:
|
||||
continue
|
||||
if mm_feature.modality == "prompt_embeds":
|
||||
# Passthrough modality: the tensor is already in the
|
||||
# model's embedding space, so no encoder runs. Cache it
|
||||
# directly so gather_mm_embeddings splices it via the
|
||||
# standard is_mm_embed path.
|
||||
embeds = mm_feature.data["embedding"].data
|
||||
assert isinstance(embeds, torch.Tensor)
|
||||
self.encoder_cache.encoder_outputs[mm_feature.identifier] = (
|
||||
async_tensor_h2d(embeds, device=self.device)
|
||||
)
|
||||
continue
|
||||
mm_hashes.append(mm_feature.identifier)
|
||||
mm_kwargs.append((mm_feature.modality, mm_feature.data))
|
||||
|
||||
|
||||
Reference in New Issue
Block a user