[Doc] [1/N] Initial guide for merged multi-modal processor (#11925)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Cyrus Leung
2025-01-10 14:30:25 +00:00
committed by GitHub
parent 241ad7b301
commit 12664ddda5
19 changed files with 433 additions and 198 deletions
+1 -2
View File
@@ -2124,8 +2124,7 @@ class MultiModalConfig:
limit_per_prompt: Mapping[str, int] = field(default_factory=dict)
"""
The maximum number of multi-modal input instances allowed per prompt
for each :class:`~vllm.multimodal.MultiModalPlugin`.
The maximum number of input items allowed per prompt for each modality.
"""
def compute_hash(self) -> str:
-3
View File
@@ -11,9 +11,6 @@ INPUT_REGISTRY = InputRegistry()
"""
The global :class:`~InputRegistry` which is used by :class:`~vllm.LLMEngine`
to dispatch data processing according to the target model.
See also:
:ref:`input-processing-pipeline`
"""
__all__ = [
+2 -10
View File
@@ -313,9 +313,6 @@ class InputRegistry:
The model is identified by ``model_config``.
See also:
:ref:`enabling-multimodal-inputs`
Note:
This should be called after
:meth:`~MultiModalRegistry.init_mm_limits_per_prompt`.
@@ -384,10 +381,8 @@ class InputRegistry:
Register an input processor to a model class.
The provided function is invoked on each input to the model. This
happens before :meth:`~vllm.multimodal.MultiModalRegistry.map_input`.
See also:
:ref:`input-processing-pipeline`
happens before
:meth:`~vllm.multimodal.registry.MultiModalRegistry.map_input`.
"""
def wrapper(model_cls: N) -> N:
@@ -429,9 +424,6 @@ class InputRegistry:
Apply an input processor to an instance of model inputs.
The model is identified by ``model_config``.
See also:
:ref:`input-processing-pipeline`
"""
# Avoid circular import
from vllm.model_executor.model_loader import get_model_architecture
+2 -2
View File
@@ -8,10 +8,10 @@ from .registry import MultiModalRegistry
MULTIMODAL_REGISTRY = MultiModalRegistry()
"""
The global :class:`~MultiModalRegistry` is used by model runners to
dispatch data processing according to its modality and the target model.
dispatch data processing according to the target model.
See also:
:ref:`input-processing-pipeline`
:ref:`mm-processing`
"""
__all__ = [
-14
View File
@@ -90,10 +90,6 @@ class MultiModalPlugin(ABC):
invoked to transform the data into a dictionary of model inputs.
If `None` is provided, then the default input mapper is used instead.
See also:
- :ref:`input-processing-pipeline`
- :ref:`enabling-multimodal-inputs`
"""
def wrapper(model_cls: N) -> N:
@@ -126,10 +122,6 @@ class MultiModalPlugin(ABC):
Raises:
TypeError: If the data type is not supported.
See also:
- :ref:`input-processing-pipeline`
- :ref:`enabling-multimodal-inputs`
"""
# Avoid circular import
@@ -186,9 +178,6 @@ class MultiModalPlugin(ABC):
for a model class.
If `None` is provided, then the default calculation is used instead.
See also:
:ref:`enabling-multimodal-inputs`
"""
def wrapper(model_cls: N) -> N:
@@ -218,9 +207,6 @@ class MultiModalPlugin(ABC):
If this registry is not applicable to the model, `0` is returned.
The model is identified by ``model_config``.
See also:
:ref:`enabling-multimodal-inputs`
"""
# Avoid circular import
from vllm.model_executor.model_loader import get_model_architecture
+2 -1
View File
@@ -493,7 +493,8 @@ A dictionary containing placeholder ranges for each modality.
class MultiModalInputsV2(TypedDict):
"""
Represents the outputs of :class:`vllm.multimodal.MultiModalProcessor`,
Represents the outputs of
:class:`vllm.multimodal.processing.BaseMultiModalProcessor`,
ready to be passed to vLLM internals.
"""
+8 -4
View File
@@ -100,8 +100,7 @@ class _MultiModalLimits(UserDict["ModelConfig", Dict[str, int]]):
class MultiModalRegistry:
"""
A registry that dispatches data processing to the
:class:`~vllm.multimodal.MultiModalPlugin` for each modality.
A registry that dispatches data processing according to the model.
"""
DEFAULT_PLUGINS = (ImagePlugin(), AudioPlugin(), VideoPlugin())
@@ -367,8 +366,7 @@ class MultiModalRegistry:
invoked to transform the data into a dictionary of model inputs.
See also:
- :ref:`input-processing-pipeline`
- :ref:`enabling-multimodal-inputs`
:ref:`mm-processing`
"""
def wrapper(model_cls: N) -> N:
@@ -398,6 +396,9 @@ class MultiModalRegistry:
def has_processor(self, model_config: "ModelConfig") -> bool:
"""
Test whether a multi-modal processor is defined for a specific model.
See also:
:ref:`mm-processing`
"""
return self._get_model_cls(model_config) in self._processor_factories
@@ -408,6 +409,9 @@ class MultiModalRegistry:
) -> BaseMultiModalProcessor[BaseProcessingInfo]:
"""
Create a multi-modal processor for a specific model and tokenizer.
See also:
:ref:`mm-processing`
"""
model_cls = self._get_model_cls(model_config)
factories = self._processor_factories[model_cls]