[Misc] Reorganize inputs (#35182)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Cyrus Leung
2026-03-25 10:22:54 -07:00
committed by GitHub
parent 678b3c99e8
commit ba2f0acc2d
142 changed files with 1212 additions and 1342 deletions
+6 -4
View File
@@ -11,15 +11,14 @@ from typing import TYPE_CHECKING, Any, overload
import torch
from typing_extensions import TypeVar
from vllm.inputs import MultiModalDataDict
from vllm.logger import init_logger
from vllm.multimodal.inputs import MultiModalDataDict
from vllm.multimodal.parse import (
DictEmbeddingItems,
EmbeddingItems,
MultiModalDataItems,
MultiModalDataParser,
)
from vllm.renderers import TokenizeParams
from vllm.tokenizers import TokenizerLike
from vllm.transformers_utils.processor import cached_processor_from_config
from vllm.utils.func_utils import get_allowed_kwarg_only_overrides
@@ -32,12 +31,14 @@ if TYPE_CHECKING:
from transformers.processing_utils import ProcessorMixin
from vllm.config import ModelConfig
from vllm.renderers import TokenizeParams
else:
PretrainedConfig = object
BatchFeature = object
ProcessorMixin = object
ModelConfig = object
TokenizeParams = object
logger = init_logger(__name__)
@@ -339,6 +340,8 @@ class BaseProcessingInfo:
def get_default_tok_params(self) -> TokenizeParams:
"""Construct the default parameters for tokenization."""
from vllm.renderers import TokenizeParams
model_config = self.ctx.model_config
encoder_config = model_config.encoder_config or {}
@@ -451,8 +454,7 @@ class BaseProcessingInfo:
validate: bool = True,
) -> MultiModalDataItems:
"""
Normalize
[`MultiModalDataDict`][vllm.multimodal.inputs.MultiModalDataDict]
Normalize [`MultiModalDataDict`][vllm.inputs.MultiModalDataDict]
to [`MultiModalDataItems`][vllm.multimodal.parse.MultiModalDataItems]
before passing them to
[`_get_hf_mm_data`][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_data].
+1 -1
View File
@@ -14,9 +14,9 @@ from vllm.config.multimodal import (
ImageDummyOptions,
VideoDummyOptions,
)
from vllm.inputs import MultiModalDataDict
from vllm.logger import init_logger
from ..inputs import MultiModalDataDict
from .context import BaseProcessingInfo
from .inputs import ProcessorInputs
+3 -2
View File
@@ -3,8 +3,9 @@
from collections.abc import Mapping
from dataclasses import dataclass, field
from vllm.inputs import MultiModalHashes
from ..hasher import MultiModalHasher
from ..inputs import MultiModalHashes
from ..parse import MultiModalDataItems, MultiModalUUIDItems
@@ -26,7 +27,7 @@ class ProcessorInputs:
mm_uuid_items = self.mm_uuid_items or {}
hf_processor_mm_kwargs = self.hf_processor_mm_kwargs
mm_hashes: MultiModalHashes = {}
mm_hashes = dict[str, list[str]]()
hasher = MultiModalHasher
for modality, data_items in mm_data_items.items():
+14 -19
View File
@@ -6,34 +6,29 @@ from collections.abc import Callable, Generator, ItemsView, Iterable, Mapping, S
from dataclasses import dataclass, field, replace
from enum import Enum
from functools import lru_cache
from typing import (
TYPE_CHECKING,
Generic,
NamedTuple,
Protocol,
TypeAlias,
cast,
)
from typing import TYPE_CHECKING, Generic, NamedTuple, Protocol, TypeAlias, cast
import regex as re
import torch
from typing_extensions import TypeVar, assert_never
from vllm.inputs import (
MultiModalEncDecInput,
MultiModalHashes,
MultiModalInput,
mm_enc_dec_input,
mm_input,
)
from vllm.logger import init_logger
from vllm.tokenizers import TokenizerLike
from vllm.utils.collection_utils import flatten_2d_lists, full_groupby
from ..inputs import (
MultiModalEncDecInputs,
MultiModalFieldConfig,
MultiModalHashes,
MultiModalInputs,
MultiModalKwargsItem,
MultiModalKwargsItems,
MultiModalKwargsOptionalItems,
PlaceholderRange,
mm_enc_dec_inputs,
mm_inputs,
)
from ..parse import (
DictEmbeddingItems,
@@ -994,7 +989,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
mm_items: MultiModalDataItems,
mm_uuid_items: MultiModalUUIDItems | None = None,
hf_processor_mm_kwargs: Mapping[str, object] | None = None,
) -> MultiModalInputs:
) -> MultiModalInput:
processor_inputs = ProcessorInputs(
prompt,
mm_items,
@@ -1638,7 +1633,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
self,
inputs: ProcessorInputs,
timing_ctx: TimingContext,
) -> MultiModalInputs:
) -> MultiModalInput:
"""
Process multi-modal inputs to be used in vLLM.
@@ -1673,7 +1668,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
for modality, placeholders in mm_placeholders.items()
}
return mm_inputs(
return mm_input(
prompt_token_ids=prompt_ids,
mm_kwargs=mm_info.kwargs,
mm_hashes=mm_info.hashes,
@@ -1708,7 +1703,7 @@ class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):
self,
prompt: str | list[int],
mm_items: MultiModalDataItems,
encoder_inputs: MultiModalInputs,
encoder_inputs: MultiModalInput,
):
tokenizer = self.info.get_tokenizer()
decoder_prompt_raw = self.create_decoder_prompt(prompt, mm_items)
@@ -1721,7 +1716,7 @@ class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):
decoder_prompt_text = None
decoder_prompt_ids = decoder_prompt_raw
return mm_enc_dec_inputs(
return mm_enc_dec_input(
encoder_inputs,
decoder_prompt_ids,
decoder_prompt=decoder_prompt_text,
@@ -1731,7 +1726,7 @@ class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):
self,
inputs: ProcessorInputs,
timing_ctx: TimingContext,
) -> MultiModalEncDecInputs:
) -> MultiModalEncDecInput:
"""
Process multi-modal inputs to be used in vLLM.
The main processing steps are modified to fit encoder-decoder model: