[Doc] [1/N] Initial guide for merged multi-modal processor (#11925)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
+1
-2
@@ -2124,8 +2124,7 @@ class MultiModalConfig:
|
||||
|
||||
limit_per_prompt: Mapping[str, int] = field(default_factory=dict)
|
||||
"""
|
||||
The maximum number of multi-modal input instances allowed per prompt
|
||||
for each :class:`~vllm.multimodal.MultiModalPlugin`.
|
||||
The maximum number of input items allowed per prompt for each modality.
|
||||
"""
|
||||
|
||||
def compute_hash(self) -> str:
|
||||
|
||||
@@ -11,9 +11,6 @@ INPUT_REGISTRY = InputRegistry()
|
||||
"""
|
||||
The global :class:`~InputRegistry` which is used by :class:`~vllm.LLMEngine`
|
||||
to dispatch data processing according to the target model.
|
||||
|
||||
See also:
|
||||
:ref:`input-processing-pipeline`
|
||||
"""
|
||||
|
||||
__all__ = [
|
||||
|
||||
+2
-10
@@ -313,9 +313,6 @@ class InputRegistry:
|
||||
|
||||
The model is identified by ``model_config``.
|
||||
|
||||
See also:
|
||||
:ref:`enabling-multimodal-inputs`
|
||||
|
||||
Note:
|
||||
This should be called after
|
||||
:meth:`~MultiModalRegistry.init_mm_limits_per_prompt`.
|
||||
@@ -384,10 +381,8 @@ class InputRegistry:
|
||||
Register an input processor to a model class.
|
||||
|
||||
The provided function is invoked on each input to the model. This
|
||||
happens before :meth:`~vllm.multimodal.MultiModalRegistry.map_input`.
|
||||
|
||||
See also:
|
||||
:ref:`input-processing-pipeline`
|
||||
happens before
|
||||
:meth:`~vllm.multimodal.registry.MultiModalRegistry.map_input`.
|
||||
"""
|
||||
|
||||
def wrapper(model_cls: N) -> N:
|
||||
@@ -429,9 +424,6 @@ class InputRegistry:
|
||||
Apply an input processor to an instance of model inputs.
|
||||
|
||||
The model is identified by ``model_config``.
|
||||
|
||||
See also:
|
||||
:ref:`input-processing-pipeline`
|
||||
"""
|
||||
# Avoid circular import
|
||||
from vllm.model_executor.model_loader import get_model_architecture
|
||||
|
||||
@@ -8,10 +8,10 @@ from .registry import MultiModalRegistry
|
||||
MULTIMODAL_REGISTRY = MultiModalRegistry()
|
||||
"""
|
||||
The global :class:`~MultiModalRegistry` is used by model runners to
|
||||
dispatch data processing according to its modality and the target model.
|
||||
dispatch data processing according to the target model.
|
||||
|
||||
See also:
|
||||
:ref:`input-processing-pipeline`
|
||||
:ref:`mm-processing`
|
||||
"""
|
||||
|
||||
__all__ = [
|
||||
|
||||
@@ -90,10 +90,6 @@ class MultiModalPlugin(ABC):
|
||||
invoked to transform the data into a dictionary of model inputs.
|
||||
|
||||
If `None` is provided, then the default input mapper is used instead.
|
||||
|
||||
See also:
|
||||
- :ref:`input-processing-pipeline`
|
||||
- :ref:`enabling-multimodal-inputs`
|
||||
"""
|
||||
|
||||
def wrapper(model_cls: N) -> N:
|
||||
@@ -126,10 +122,6 @@ class MultiModalPlugin(ABC):
|
||||
|
||||
Raises:
|
||||
TypeError: If the data type is not supported.
|
||||
|
||||
See also:
|
||||
- :ref:`input-processing-pipeline`
|
||||
- :ref:`enabling-multimodal-inputs`
|
||||
"""
|
||||
|
||||
# Avoid circular import
|
||||
@@ -186,9 +178,6 @@ class MultiModalPlugin(ABC):
|
||||
for a model class.
|
||||
|
||||
If `None` is provided, then the default calculation is used instead.
|
||||
|
||||
See also:
|
||||
:ref:`enabling-multimodal-inputs`
|
||||
"""
|
||||
|
||||
def wrapper(model_cls: N) -> N:
|
||||
@@ -218,9 +207,6 @@ class MultiModalPlugin(ABC):
|
||||
If this registry is not applicable to the model, `0` is returned.
|
||||
|
||||
The model is identified by ``model_config``.
|
||||
|
||||
See also:
|
||||
:ref:`enabling-multimodal-inputs`
|
||||
"""
|
||||
# Avoid circular import
|
||||
from vllm.model_executor.model_loader import get_model_architecture
|
||||
|
||||
@@ -493,7 +493,8 @@ A dictionary containing placeholder ranges for each modality.
|
||||
|
||||
class MultiModalInputsV2(TypedDict):
|
||||
"""
|
||||
Represents the outputs of :class:`vllm.multimodal.MultiModalProcessor`,
|
||||
Represents the outputs of
|
||||
:class:`vllm.multimodal.processing.BaseMultiModalProcessor`,
|
||||
ready to be passed to vLLM internals.
|
||||
"""
|
||||
|
||||
|
||||
@@ -100,8 +100,7 @@ class _MultiModalLimits(UserDict["ModelConfig", Dict[str, int]]):
|
||||
|
||||
class MultiModalRegistry:
|
||||
"""
|
||||
A registry that dispatches data processing to the
|
||||
:class:`~vllm.multimodal.MultiModalPlugin` for each modality.
|
||||
A registry that dispatches data processing according to the model.
|
||||
"""
|
||||
|
||||
DEFAULT_PLUGINS = (ImagePlugin(), AudioPlugin(), VideoPlugin())
|
||||
@@ -367,8 +366,7 @@ class MultiModalRegistry:
|
||||
invoked to transform the data into a dictionary of model inputs.
|
||||
|
||||
See also:
|
||||
- :ref:`input-processing-pipeline`
|
||||
- :ref:`enabling-multimodal-inputs`
|
||||
:ref:`mm-processing`
|
||||
"""
|
||||
|
||||
def wrapper(model_cls: N) -> N:
|
||||
@@ -398,6 +396,9 @@ class MultiModalRegistry:
|
||||
def has_processor(self, model_config: "ModelConfig") -> bool:
|
||||
"""
|
||||
Test whether a multi-modal processor is defined for a specific model.
|
||||
|
||||
See also:
|
||||
:ref:`mm-processing`
|
||||
"""
|
||||
return self._get_model_cls(model_config) in self._processor_factories
|
||||
|
||||
@@ -408,6 +409,9 @@ class MultiModalRegistry:
|
||||
) -> BaseMultiModalProcessor[BaseProcessingInfo]:
|
||||
"""
|
||||
Create a multi-modal processor for a specific model and tokenizer.
|
||||
|
||||
See also:
|
||||
:ref:`mm-processing`
|
||||
"""
|
||||
model_cls = self._get_model_cls(model_config)
|
||||
factories = self._processor_factories[model_cls]
|
||||
|
||||
Reference in New Issue
Block a user