[Misc] Reorganize inputs (#35182)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
+51
-58
@@ -11,17 +11,32 @@ from typing import TYPE_CHECKING, Any, Generic, overload
|
||||
from typing_extensions import TypeVar
|
||||
|
||||
from vllm.inputs import (
|
||||
EmbedsInputs,
|
||||
EmbedsInput,
|
||||
EmbedsPrompt,
|
||||
EncoderDecoderInputs,
|
||||
ProcessorInputs,
|
||||
SingletonInputs,
|
||||
EncoderDecoderInput,
|
||||
EngineInput,
|
||||
MultiModalDataDict,
|
||||
MultiModalInput,
|
||||
MultiModalUUIDDict,
|
||||
SingletonInput,
|
||||
TextPrompt,
|
||||
TokenInputs,
|
||||
TokensInput,
|
||||
TokensPrompt,
|
||||
build_enc_dec_input,
|
||||
embeds_input,
|
||||
tokens_input,
|
||||
)
|
||||
from vllm.inputs.data import build_enc_dec_inputs, embeds_inputs, token_inputs
|
||||
from vllm.logger import init_logger
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY as mm_registry
|
||||
from vllm.multimodal.cache import BaseMultiModalProcessorCache
|
||||
from vllm.multimodal.parse import (
|
||||
MultiModalDataItems,
|
||||
MultiModalUUIDItems,
|
||||
parse_mm_uuids,
|
||||
)
|
||||
from vllm.multimodal.processing import BaseMultiModalProcessor
|
||||
from vllm.multimodal.processing import ProcessorInputs as MMProcessorInputs
|
||||
from vllm.multimodal.registry import MultiModalTimingRegistry
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.utils.async_utils import AsyncMicrobatchTokenizer
|
||||
from vllm.utils.counter import AtomicCounter
|
||||
@@ -46,14 +61,6 @@ if TYPE_CHECKING:
|
||||
ChatCompletionMessageParam,
|
||||
ConversationMessage,
|
||||
)
|
||||
from vllm.multimodal.cache import BaseMultiModalProcessorCache
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalDataDict,
|
||||
MultiModalInputs,
|
||||
MultiModalUUIDDict,
|
||||
)
|
||||
from vllm.multimodal.parse import MultiModalDataItems, MultiModalUUIDItems
|
||||
from vllm.multimodal.processing import BaseMultiModalProcessor
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -86,9 +93,6 @@ class BaseRenderer(ABC, Generic[_T]):
|
||||
self.mm_processor: BaseMultiModalProcessor | None = None
|
||||
self._mm_cache_stats: MultiModalCacheStats | None = None
|
||||
if config.model_config.is_multimodal_model:
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY as mm_registry
|
||||
from vllm.multimodal.registry import MultiModalTimingRegistry
|
||||
|
||||
mm_processor_cache = mm_registry.processor_cache_from_config(config)
|
||||
|
||||
# Deep-copy the tokenizer so the multimodal processor gets its
|
||||
@@ -524,9 +528,9 @@ class BaseRenderer(ABC, Generic[_T]):
|
||||
# Step 4: Convert to engine inputs
|
||||
def _validate_mm_uuids(
|
||||
self,
|
||||
mm_data: "MultiModalDataDict",
|
||||
mm_data_items: "MultiModalDataItems",
|
||||
mm_uuid_items: "MultiModalUUIDItems",
|
||||
mm_data: MultiModalDataDict,
|
||||
mm_data_items: MultiModalDataItems,
|
||||
mm_uuid_items: MultiModalUUIDItems,
|
||||
) -> None:
|
||||
# NOTE: Keys corresponding to `None` in `mm_data` don't appear in
|
||||
# `mm_data_items`
|
||||
@@ -560,11 +564,11 @@ class BaseRenderer(ABC, Generic[_T]):
|
||||
|
||||
def _process_mm_uuids(
|
||||
self,
|
||||
mm_data: "MultiModalDataDict",
|
||||
mm_data_items: "MultiModalDataItems",
|
||||
mm_uuid_items: "MultiModalUUIDItems",
|
||||
mm_data: MultiModalDataDict,
|
||||
mm_data_items: MultiModalDataItems,
|
||||
mm_uuid_items: MultiModalUUIDItems,
|
||||
mm_req_id: str,
|
||||
):
|
||||
) -> MultiModalUUIDItems:
|
||||
model_config = self.model_config
|
||||
|
||||
# NOTE: When users explicitly turn off BOTH prefix caching and input
|
||||
@@ -590,14 +594,11 @@ class BaseRenderer(ABC, Generic[_T]):
|
||||
def _process_multimodal(
|
||||
self,
|
||||
prompt: list[int] | str,
|
||||
mm_data: "MultiModalDataDict",
|
||||
mm_uuids: "MultiModalUUIDDict | None",
|
||||
mm_data: MultiModalDataDict,
|
||||
mm_uuids: MultiModalUUIDDict | None,
|
||||
mm_processor_kwargs: Mapping[str, object] | None,
|
||||
tokenization_kwargs: dict[str, Any] | None,
|
||||
) -> "MultiModalInputs":
|
||||
from vllm.multimodal.parse import parse_mm_uuids
|
||||
from vllm.multimodal.processing import ProcessorInputs as MMProcessorInputs
|
||||
|
||||
) -> "MultiModalInput":
|
||||
mm_req_id = f"renderer{self.api_process_rank}-mm-{self._mm_req_counter.inc(1)}"
|
||||
|
||||
mm_processor = self.get_mm_processor()
|
||||
@@ -628,12 +629,12 @@ class BaseRenderer(ABC, Generic[_T]):
|
||||
def _process_tokens(
|
||||
self,
|
||||
prompt: TokensPrompt,
|
||||
) -> "TokenInputs | MultiModalInputs":
|
||||
) -> TokensInput | MultiModalInput:
|
||||
prompt_token_ids = prompt["prompt_token_ids"]
|
||||
|
||||
inputs: TokenInputs | MultiModalInputs
|
||||
engine_input: TokensInput | MultiModalInput
|
||||
if multi_modal_data := prompt.get("multi_modal_data"):
|
||||
inputs = self._process_multimodal(
|
||||
engine_input = self._process_multimodal(
|
||||
prompt_token_ids,
|
||||
multi_modal_data,
|
||||
mm_processor_kwargs=prompt.get("mm_processor_kwargs"),
|
||||
@@ -641,19 +642,16 @@ class BaseRenderer(ABC, Generic[_T]):
|
||||
mm_uuids=prompt.get("multi_modal_uuids"),
|
||||
)
|
||||
else:
|
||||
inputs = token_inputs(prompt_token_ids)
|
||||
engine_input = tokens_input(prompt_token_ids)
|
||||
|
||||
if prompt_text := prompt.get("prompt"):
|
||||
inputs["prompt"] = prompt_text
|
||||
engine_input["prompt"] = prompt_text
|
||||
if cache_salt := prompt.get("cache_salt"):
|
||||
inputs["cache_salt"] = cache_salt
|
||||
engine_input["cache_salt"] = cache_salt
|
||||
|
||||
return inputs
|
||||
return engine_input
|
||||
|
||||
def _process_embeds(
|
||||
self,
|
||||
prompt: EmbedsPrompt,
|
||||
) -> EmbedsInputs:
|
||||
def _process_embeds(self, prompt: EmbedsPrompt) -> EmbedsInput:
|
||||
if not self.model_config.enable_prompt_embeds:
|
||||
raise ValueError(
|
||||
"You must set `--enable-prompt-embeds` to input `prompt_embeds`."
|
||||
@@ -676,15 +674,12 @@ class BaseRenderer(ABC, Generic[_T]):
|
||||
# hidden device transfer in the critical path of generation.
|
||||
prompt_embeds = prompt_embeds.cpu()
|
||||
|
||||
return embeds_inputs(
|
||||
return embeds_input(
|
||||
prompt_embeds=prompt_embeds,
|
||||
cache_salt=prompt.get("cache_salt"),
|
||||
)
|
||||
|
||||
def _process_singleton(
|
||||
self,
|
||||
prompt: SingletonTokPrompt,
|
||||
) -> SingletonInputs:
|
||||
def _process_singleton(self, prompt: SingletonTokPrompt) -> SingletonInput:
|
||||
if "prompt_embeds" in prompt:
|
||||
return self._process_embeds(prompt) # type: ignore[arg-type]
|
||||
|
||||
@@ -693,7 +688,7 @@ class BaseRenderer(ABC, Generic[_T]):
|
||||
def _process_enc_dec(
|
||||
self,
|
||||
prompt: EncoderDecoderTokPrompt,
|
||||
) -> EncoderDecoderInputs:
|
||||
) -> EncoderDecoderInput:
|
||||
enc_prompt = prompt["encoder_prompt"]
|
||||
dec_prompt = prompt["decoder_prompt"]
|
||||
|
||||
@@ -704,27 +699,25 @@ class BaseRenderer(ABC, Generic[_T]):
|
||||
if isinstance(self.mm_processor, EncDecMultiModalProcessor):
|
||||
skip_decoder_start_token = self.mm_processor.skip_decoder_start_token
|
||||
|
||||
return build_enc_dec_inputs(
|
||||
encoder_inputs=self._process_singleton(enc_prompt),
|
||||
decoder_inputs=(
|
||||
return build_enc_dec_input(
|
||||
encoder_input=self._process_singleton(enc_prompt),
|
||||
decoder_input=(
|
||||
None if dec_prompt is None else self._process_singleton(dec_prompt)
|
||||
),
|
||||
decoder_start_token_id=self.get_dec_start_token_id(),
|
||||
skip_decoder_start_token=skip_decoder_start_token,
|
||||
)
|
||||
|
||||
def process_for_engine(
|
||||
self, prompt: TokPrompt, arrival_time: float
|
||||
) -> ProcessorInputs:
|
||||
engine_prompt: ProcessorInputs
|
||||
def process_for_engine(self, prompt: TokPrompt, arrival_time: float) -> EngineInput:
|
||||
engine_input: EngineInput
|
||||
if "encoder_prompt" in prompt:
|
||||
engine_prompt = self._process_enc_dec(prompt) # type: ignore[arg-type]
|
||||
engine_input = self._process_enc_dec(prompt) # type: ignore[arg-type]
|
||||
else:
|
||||
engine_prompt = self._process_singleton(prompt)
|
||||
engine_input = self._process_singleton(prompt)
|
||||
|
||||
engine_prompt["arrival_time"] = arrival_time
|
||||
engine_input["arrival_time"] = arrival_time
|
||||
|
||||
return engine_prompt
|
||||
return engine_input
|
||||
|
||||
# Top-level methods
|
||||
def render_cmpl(
|
||||
|
||||
+7
-13
@@ -5,7 +5,7 @@ import itertools
|
||||
from collections import defaultdict, deque
|
||||
from collections.abc import Set
|
||||
from functools import lru_cache
|
||||
from typing import TYPE_CHECKING, Any, Literal, cast, overload
|
||||
from typing import Any, Literal, cast, overload
|
||||
|
||||
import jinja2
|
||||
import jinja2.ext
|
||||
@@ -25,6 +25,7 @@ from vllm.entrypoints.chat_utils import (
|
||||
parse_chat_messages,
|
||||
parse_chat_messages_async,
|
||||
)
|
||||
from vllm.inputs import MultiModalDataDict, MultiModalUUIDDict
|
||||
from vllm.logger import init_logger
|
||||
from vllm.tokenizers import cached_get_tokenizer
|
||||
from vllm.tokenizers.hf import CachedHfTokenizer, HfTokenizer
|
||||
@@ -37,13 +38,6 @@ from .inputs import DictPrompt
|
||||
from .inputs.preprocess import parse_dec_only_prompt
|
||||
from .params import ChatParams
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.multimodal.inputs import MultiModalDataDict, MultiModalUUIDDict
|
||||
else:
|
||||
MultiModalDataDict = dict[str, Any]
|
||||
MultiModalUUIDDict = dict[str, Any]
|
||||
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@@ -512,9 +506,9 @@ def safe_apply_chat_template(
|
||||
|
||||
|
||||
def rebuild_mm_uuids_from_mm_data(
|
||||
mm_uuids: "MultiModalUUIDDict",
|
||||
mm_data: "MultiModalDataDict",
|
||||
) -> "MultiModalUUIDDict":
|
||||
mm_uuids: MultiModalUUIDDict,
|
||||
mm_data: MultiModalDataDict,
|
||||
) -> MultiModalUUIDDict:
|
||||
"""Rebuild mm_uuids after vision_chunk processing.
|
||||
|
||||
When videos are split into chunks, the original UUIDs need to be updated
|
||||
@@ -547,7 +541,7 @@ def rebuild_mm_uuids_from_mm_data(
|
||||
|
||||
|
||||
def build_video_prompts_from_mm_data(
|
||||
mm_data: "MultiModalDataDict",
|
||||
mm_data: MultiModalDataDict,
|
||||
) -> list[str]:
|
||||
"""Build video prompts from vision_chunk data.
|
||||
|
||||
@@ -585,7 +579,7 @@ def build_video_prompts_from_mm_data(
|
||||
|
||||
def replace_vision_chunk_video_placeholder(
|
||||
prompt_raw: str | list[int],
|
||||
mm_data: "MultiModalDataDict",
|
||||
mm_data: MultiModalDataDict,
|
||||
video_placeholder: str | None,
|
||||
) -> str | list[int]:
|
||||
# get video placeholder, replace it with runtime video-chunk prompts
|
||||
|
||||
@@ -9,8 +9,8 @@ from typing import TYPE_CHECKING, NamedTuple, TypeAlias, TypedDict, overload
|
||||
|
||||
from vllm.inputs import (
|
||||
EmbedsPrompt,
|
||||
EngineInput,
|
||||
ExplicitEncoderDecoderPrompt,
|
||||
ProcessorInputs,
|
||||
PromptType,
|
||||
SingletonPrompt,
|
||||
TextPrompt,
|
||||
@@ -70,28 +70,28 @@ def conversation_to_seq(
|
||||
|
||||
DecoderOnlyDictPrompt: TypeAlias = TextPrompt | TokensPrompt | EmbedsPrompt
|
||||
"""
|
||||
A [`DecoderOnlyPrompt`][vllm.inputs.data.DecoderOnlyPrompt]
|
||||
A [`DecoderOnlyPrompt`][vllm.inputs.llm.DecoderOnlyPrompt]
|
||||
that has been standardized into a dictionary.
|
||||
"""
|
||||
|
||||
|
||||
EncoderDictPrompt: TypeAlias = TextPrompt | TokensPrompt
|
||||
"""
|
||||
A [`EncoderPrompt`][vllm.inputs.data.EncoderPrompt]
|
||||
A [`EncoderPrompt`][vllm.inputs.llm.EncoderPrompt]
|
||||
that has been standardized into a dictionary.
|
||||
"""
|
||||
|
||||
|
||||
DecoderDictPrompt: TypeAlias = TextPrompt | TokensPrompt
|
||||
"""
|
||||
A [`DecoderPrompt`][vllm.inputs.data.DecoderPrompt]
|
||||
A [`DecoderPrompt`][vllm.inputs.llm.DecoderPrompt]
|
||||
that has been standardized into a dictionary.
|
||||
"""
|
||||
|
||||
|
||||
class EncoderDecoderDictPrompt(TypedDict):
|
||||
"""
|
||||
A [`EncoderDecoderPrompt`][vllm.inputs.data.EncoderDecoderPrompt]
|
||||
A [`EncoderDecoderPrompt`][vllm.inputs.llm.EncoderDecoderPrompt]
|
||||
that has been standardized into a dictionary.
|
||||
"""
|
||||
|
||||
@@ -104,14 +104,14 @@ SingletonDictPrompt: TypeAlias = (
|
||||
DecoderOnlyDictPrompt | EncoderDictPrompt | DecoderDictPrompt
|
||||
)
|
||||
"""
|
||||
A [`SingletonPrompt`][vllm.inputs.data.SingletonPrompt]
|
||||
A [`SingletonPrompt`][vllm.inputs.llm.SingletonPrompt]
|
||||
that has been standardized into a dictionary.
|
||||
"""
|
||||
|
||||
|
||||
DictPrompt: TypeAlias = DecoderOnlyDictPrompt | EncoderDecoderDictPrompt
|
||||
"""
|
||||
A [`PromptType`][vllm.inputs.data.PromptType]
|
||||
A [`PromptType`][vllm.inputs.llm.PromptType]
|
||||
that has been standardized into a dictionary.
|
||||
"""
|
||||
|
||||
@@ -236,7 +236,7 @@ def extract_target_prompt(model_config: "ModelConfig", prompt: object):
|
||||
|
||||
def extract_prompt_components(
|
||||
model_config: "ModelConfig",
|
||||
prompt: PromptType | ProcessorInputs,
|
||||
prompt: PromptType | EngineInput,
|
||||
) -> PromptComponents:
|
||||
target_prompt = extract_target_prompt(model_config, prompt)
|
||||
|
||||
@@ -248,7 +248,8 @@ def extract_prompt_components(
|
||||
|
||||
|
||||
def extract_prompt_len(
|
||||
model_config: "ModelConfig", prompt: PromptType | ProcessorInputs
|
||||
model_config: "ModelConfig",
|
||||
prompt: PromptType | EngineInput,
|
||||
):
|
||||
target_prompt = extract_target_prompt(model_config, prompt)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user