[ModelRunnerV2] Support prompt embeds (#42963)
Signed-off-by: gcanlin <canlinguosdu@gmail.com> Signed-off-by: Canlin Guo <961750412@qq.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -1,7 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Any, cast
|
||||
from typing import Any, ClassVar, cast
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
@@ -43,6 +43,9 @@ class ModelSpecificAttnMetadata:
|
||||
|
||||
|
||||
class ModelState(ABC):
|
||||
supports_prompt_embeds: ClassVar[bool] = False
|
||||
"""Whether this state implements user-provided prompt embeddings."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
@@ -153,12 +156,13 @@ class ModelState(ABC):
|
||||
return None
|
||||
|
||||
@abstractmethod
|
||||
def get_mm_embeddings(
|
||||
def prepare_inputs_embeds(
|
||||
self,
|
||||
scheduled_encoder_inputs: dict[str, list[int]],
|
||||
input_batch: InputBatch,
|
||||
req_states: RequestState,
|
||||
) -> torch.Tensor | None:
|
||||
"""Prepare the ``inputs_embeds`` tensor for the current forward pass."""
|
||||
raise NotImplementedError
|
||||
|
||||
def dummy_inputs_embeds(self, num_tokens: int) -> torch.Tensor | None:
|
||||
|
||||
Reference in New Issue
Block a user