Revert "[Renderer] Separate out RendererConfig from ModelConfig (#30145)" (#30199)

This commit is contained in:
Cyrus Leung
2025-12-07 00:00:22 -08:00
committed by GitHub
parent 27f4c2fd46
commit e83b7e379c
105 changed files with 797 additions and 969 deletions
+12 -10
View File
@@ -31,7 +31,7 @@ from .inputs import (
)
if TYPE_CHECKING:
from vllm.config import ModelConfig, RendererConfig, VllmConfig
from vllm.config import ModelConfig, VllmConfig
from .processing import ResolvedPromptUpdate
from .registry import MultiModalRegistry
@@ -561,13 +561,13 @@ class ShmObjectStoreSenderCache(BaseMultiModalProcessorCache):
def _enable_processor_cache(
renderer_config: "RendererConfig",
model_config: "ModelConfig",
mm_registry: "MultiModalRegistry",
) -> bool:
if not mm_registry.supports_multimodal_inputs(renderer_config):
if not mm_registry.supports_multimodal_inputs(model_config):
return False
mm_config = renderer_config.model_config.get_multimodal_config()
mm_config = model_config.get_multimodal_config()
return mm_config.mm_processor_cache_gb > 0
@@ -599,7 +599,7 @@ def processor_cache_from_config(
"""Return a `BaseMultiModalProcessorCache`, if enabled."""
model_config = vllm_config.model_config
if not _enable_processor_cache(vllm_config.renderer_config, mm_registry):
if not _enable_processor_cache(model_config, mm_registry):
return None
if not _enable_ipc_cache(vllm_config):
@@ -611,14 +611,14 @@ def processor_cache_from_config(
def processor_only_cache_from_config(
renderer_config: "RendererConfig",
model_config: "ModelConfig",
mm_registry: "MultiModalRegistry",
):
"""Return a `MultiModalProcessorOnlyCache`, if enabled."""
if not _enable_processor_cache(renderer_config, mm_registry):
if not _enable_processor_cache(model_config, mm_registry):
return None
return MultiModalProcessorOnlyCache(renderer_config.model_config)
return MultiModalProcessorOnlyCache(model_config)
class BaseMultiModalReceiverCache(
@@ -787,7 +787,7 @@ def engine_receiver_cache_from_config(
"""
model_config = vllm_config.model_config
if not _enable_processor_cache(vllm_config.renderer_config, mm_registry):
if not _enable_processor_cache(model_config, mm_registry):
return None
if not _enable_ipc_cache(vllm_config):
@@ -809,7 +809,9 @@ def worker_receiver_cache_from_config(
Return a `BaseMultiModalReceiverCache` only when IPC caching is enabled and
mm_processor_cache_type=="shm".
"""
if not _enable_processor_cache(vllm_config.renderer_config, mm_registry):
model_config = vllm_config.model_config
if not _enable_processor_cache(model_config, mm_registry):
return None
if not _enable_ipc_cache(vllm_config):
+5 -23
View File
@@ -23,7 +23,7 @@ import torch
from typing_extensions import TypeVar, assert_never
from vllm.logger import init_logger
from vllm.tokenizers import TokenizerLike, cached_tokenizer_from_config
from vllm.tokenizers import TokenizerLike
from vllm.transformers_utils.processor import cached_processor_from_config
from vllm.utils.collection_utils import flatten_2d_lists, full_groupby
from vllm.utils.func_utils import get_allowed_kwarg_only_overrides
@@ -53,7 +53,7 @@ if TYPE_CHECKING:
from transformers.feature_extraction_utils import BatchFeature
from transformers.processing_utils import ProcessorMixin
from vllm.config import ModelConfig, RendererConfig
from vllm.config import ModelConfig
from .cache import BaseMultiModalProcessorCache
from .profiling import BaseDummyInputsBuilder
@@ -63,7 +63,6 @@ else:
ProcessorMixin = object
ModelConfig = object
RendererConfig = object
BaseMultiModalProcessorCache = object
@@ -946,29 +945,12 @@ class InputProcessingContext:
modify the inputs.
"""
renderer_config: RendererConfig
"""The configuration of the renderer."""
model_config: ModelConfig
"""The configuration of the model."""
tokenizer: TokenizerLike | None
"""The tokenizer used to tokenize the inputs."""
@classmethod
def from_config(
cls,
renderer_config: RendererConfig,
*,
tokenizer: TokenizerLike | None = None,
):
if tokenizer is None and not renderer_config.skip_tokenizer_init:
tokenizer = cached_tokenizer_from_config(renderer_config)
return cls(renderer_config, tokenizer)
@property
def model_config(self) -> ModelConfig:
"""The configuration of the model."""
return self.renderer_config.model_config
def get_tokenizer(self) -> TokenizerLike:
if self.tokenizer is None:
raise ValueError(
@@ -1065,7 +1047,7 @@ class InputProcessingContext:
typ = ProcessorMixin
return cached_processor_from_config(
self.renderer_config,
self.model_config,
processor_cls=typ,
tokenizer=self.tokenizer,
**kwargs,
+34 -30
View File
@@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, Generic, Protocol, TypeVar, cast
from vllm.config.multimodal import BaseDummyOptions
from vllm.logger import init_logger
from vllm.tokenizers import TokenizerLike
from vllm.tokenizers import TokenizerLike, cached_tokenizer_from_config
from .cache import BaseMultiModalProcessorCache
from .processing import (
@@ -22,7 +22,7 @@ from .profiling import (
)
if TYPE_CHECKING:
from vllm.config import ModelConfig, RendererConfig
from vllm.config import ModelConfig
from vllm.model_executor.models.interfaces import SupportsMultiModal
logger = init_logger(__name__)
@@ -114,18 +114,17 @@ class MultiModalRegistry:
return mm_options if len(mm_options) > 0 else None
def supports_multimodal_inputs(self, renderer_config: "RendererConfig") -> bool:
def supports_multimodal_inputs(self, model_config: "ModelConfig") -> bool:
"""
Checks if the model supports multimodal inputs.
Returns True if the model is multimodal with any non-zero supported
modalities, otherwise returns False, effectively running in
text-only mode.
"""
model_config = renderer_config.model_config
if not model_config.is_multimodal_model:
return False
info = self._create_processing_info(renderer_config, tokenizer=None)
info = self._create_processing_info(model_config, tokenizer=None)
supported_modalities = info.get_supported_mm_limits()
mm_config = model_config.get_multimodal_config()
@@ -145,7 +144,7 @@ class MultiModalRegistry:
def get_max_tokens_per_item_by_modality(
self,
renderer_config: "RendererConfig",
model_config: "ModelConfig",
*,
cache: BaseMultiModalProcessorCache | None = None,
profiler_limits: Mapping[str, int] | None = None,
@@ -154,11 +153,10 @@ class MultiModalRegistry:
Get the maximum number of tokens per data item from each modality based
on underlying model configuration.
"""
model_config = renderer_config.model_config
if not model_config.is_multimodal_model:
return {}
processor = self.create_processor(renderer_config, cache=cache)
processor = self.create_processor(model_config, cache=cache)
profiler: MultiModalProfiler = MultiModalProfiler(processor)
seq_len = model_config.max_model_len
@@ -173,7 +171,7 @@ class MultiModalRegistry:
def get_mm_limits_per_prompt(
self,
renderer_config: "RendererConfig",
model_config: "ModelConfig",
*,
cache: BaseMultiModalProcessorCache | None = None,
) -> Mapping[str, int]:
@@ -181,11 +179,10 @@ class MultiModalRegistry:
Get the maximum number of multi-modal input instances for each modality
that are allowed per prompt for a model class.
"""
model_config = renderer_config.model_config
if not model_config.is_multimodal_model:
return {}
processor = self.create_processor(renderer_config, cache=cache)
processor = self.create_processor(model_config, cache=cache)
profiler: MultiModalProfiler = MultiModalProfiler(processor)
return profiler.get_mm_limits()
@@ -231,21 +228,30 @@ class MultiModalRegistry:
assert hasattr(model_cls, "_processor_factory")
return cast("SupportsMultiModal", model_cls)
def _create_processing_ctx(
self,
model_config: "ModelConfig",
tokenizer: TokenizerLike | None = None,
) -> InputProcessingContext:
if tokenizer is None and not model_config.skip_tokenizer_init:
tokenizer = cached_tokenizer_from_config(model_config)
return InputProcessingContext(model_config, tokenizer)
def _create_processing_info(
self,
renderer_config: "RendererConfig",
model_config: "ModelConfig",
*,
tokenizer: TokenizerLike | None = None,
) -> BaseProcessingInfo:
model_cls = self._get_model_cls(renderer_config.model_config)
model_cls = self._get_model_cls(model_config)
factories = model_cls._processor_factory
ctx = InputProcessingContext.from_config(renderer_config, tokenizer=tokenizer)
ctx = self._create_processing_ctx(model_config, tokenizer)
return factories.info(ctx)
def create_processor(
self,
renderer_config: "RendererConfig",
model_config: "ModelConfig",
*,
tokenizer: TokenizerLike | None = None,
cache: BaseMultiModalProcessorCache | None = None,
@@ -253,19 +259,19 @@ class MultiModalRegistry:
"""
Create a multi-modal processor for a specific model and tokenizer.
"""
model_config = renderer_config.model_config
if not model_config.is_multimodal_model:
raise ValueError(f"{model_config.model} is not a multimodal model")
model_cls = self._get_model_cls(model_config)
factories = model_cls._processor_factory
ctx = InputProcessingContext.from_config(renderer_config, tokenizer=tokenizer)
ctx = self._create_processing_ctx(model_config, tokenizer)
return factories.build_processor(ctx, cache=cache)
def get_decoder_dummy_data(
self,
renderer_config: "RendererConfig",
model_config: "ModelConfig",
seq_len: int,
mm_counts: Mapping[str, int] | None = None,
*,
@@ -274,15 +280,15 @@ class MultiModalRegistry:
"""
Create dummy data for profiling the memory usage of a model.
The model is identified by `renderer_config`.
The model is identified by `model_config`.
"""
processor = self.create_processor(renderer_config, cache=cache)
processor = self.create_processor(model_config, cache=cache)
profiler: MultiModalProfiler = MultiModalProfiler(processor)
# Extract configurable options from multimodal config.
# Only include modalities that use advanced option types so legacy
# count-only behavior remains unchanged.
mm_options = self._extract_mm_options(renderer_config.model_config)
mm_options = self._extract_mm_options(model_config)
dummy_data = profiler.get_decoder_dummy_data(seq_len, mm_counts, mm_options)
@@ -298,7 +304,7 @@ class MultiModalRegistry:
def get_encoder_dummy_data(
self,
renderer_config: "RendererConfig",
model_config: "ModelConfig",
seq_len: int,
mm_counts: Mapping[str, int] | None = None,
*,
@@ -307,15 +313,15 @@ class MultiModalRegistry:
"""
Create dummy data for profiling the memory usage of a model.
The model is identified by `renderer_config`.
The model is identified by `model_config`.
"""
processor = self.create_processor(renderer_config, cache=cache)
processor = self.create_processor(model_config, cache=cache)
profiler: MultiModalProfiler = MultiModalProfiler(processor)
# Extract configurable options from multimodal config.
# Only include modalities that use advanced option types so legacy
# count-only behavior remains unchanged.
mm_options = self._extract_mm_options(renderer_config.model_config)
mm_options = self._extract_mm_options(model_config)
dummy_data = profiler.get_encoder_dummy_data(seq_len, mm_counts, mm_options)
@@ -330,15 +336,13 @@ class MultiModalRegistry:
return dummy_data
def get_encdec_max_encoder_len(self, renderer_config: "RendererConfig") -> int:
def get_encdec_max_encoder_len(self, model_config: "ModelConfig") -> int:
"""
Get the maximum length of the encoder input for encoder-decoder models.
"""
model_config = renderer_config.model_config
if not model_config.is_encoder_decoder:
return 0
max_tokens = self.get_max_tokens_per_item_by_modality(renderer_config)
max_tokens = self.get_max_tokens_per_item_by_modality(model_config)
if not max_tokens:
# TODO - this function assumes encoder-decoder models are
# multimodal. This will need to change when adding support for more