[ModelRunnerV2] Support prompt embeds (#42963)

Signed-off-by: gcanlin <canlinguosdu@gmail.com>
Signed-off-by: Canlin Guo <961750412@qq.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Canlin Guo
2026-08-17 21:58:50 +00:00
committed by GitHub
co-authored by Nick Hill
parent e68fb75b25
commit 455edc022b
17 changed files with 551 additions and 54 deletions
+6 -2
View File
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from abc import ABC, abstractmethod
from typing import Any, cast
from typing import Any, ClassVar, cast
import torch
import torch.nn as nn
@@ -43,6 +43,9 @@ class ModelSpecificAttnMetadata:
class ModelState(ABC):
supports_prompt_embeds: ClassVar[bool] = False
"""Whether this state implements user-provided prompt embeddings."""
def __init__(
self,
vllm_config: VllmConfig,
@@ -153,12 +156,13 @@ class ModelState(ABC):
return None
@abstractmethod
def get_mm_embeddings(
def prepare_inputs_embeds(
self,
scheduled_encoder_inputs: dict[str, list[int]],
input_batch: InputBatch,
req_states: RequestState,
) -> torch.Tensor | None:
"""Prepare the ``inputs_embeds`` tensor for the current forward pass."""
raise NotImplementedError
def dummy_inputs_embeds(self, num_tokens: int) -> torch.Tensor | None: