[Core] Keep attention backends eligible for text-only serving of prefix-LM models (#48796)
Signed-off-by: qtris123 <voquangtri2021@gmail.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -1,10 +1,16 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from vllm.config.model import ModelConfig
|
||||
from vllm.config.multimodal import MultiModalConfig
|
||||
from vllm.transformers_utils.model_arch_config_convertor import (
|
||||
ModelArchConfigConvertorBase,
|
||||
)
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
|
||||
|
||||
@@ -68,3 +74,97 @@ def test_mm_encoder_attn_dtype_hash_updates(tmp_path):
|
||||
).compute_hash()
|
||||
assert base_hash != fp8_hash
|
||||
assert fp8_hash != fp8_static_hash
|
||||
|
||||
|
||||
def _make_mm_prefix_model_config(
|
||||
*,
|
||||
language_model_only: bool = False,
|
||||
) -> ModelConfig:
|
||||
model_config = MagicMock(spec=ModelConfig)
|
||||
model_config.multimodal_config = MultiModalConfig(
|
||||
language_model_only=language_model_only
|
||||
)
|
||||
# Bind real helper methods onto the mock.
|
||||
model_config._supports_multimodal_for_mm_prefix = (
|
||||
ModelConfig._supports_multimodal_for_mm_prefix.__get__(
|
||||
model_config, ModelConfig
|
||||
)
|
||||
)
|
||||
return model_config
|
||||
|
||||
|
||||
@pytest.mark.parametrize("supports_mm", [True, False])
|
||||
def test_supports_multimodal_for_mm_prefix_uses_registry(supports_mm: bool):
|
||||
model_config = _make_mm_prefix_model_config()
|
||||
|
||||
with patch(
|
||||
"vllm.multimodal.MULTIMODAL_REGISTRY.supports_multimodal_inputs",
|
||||
return_value=supports_mm,
|
||||
) as mocked:
|
||||
assert model_config._supports_multimodal_for_mm_prefix() is supports_mm
|
||||
mocked.assert_called_once_with(model_config)
|
||||
|
||||
# Sticky cache — registry must not be consulted again.
|
||||
with patch(
|
||||
"vllm.multimodal.MULTIMODAL_REGISTRY.supports_multimodal_inputs",
|
||||
side_effect=AssertionError("should use cache"),
|
||||
):
|
||||
assert model_config._supports_multimodal_for_mm_prefix() is supports_mm
|
||||
|
||||
|
||||
def test_supports_multimodal_for_mm_prefix_before_multimodal_config():
|
||||
model_config = _make_mm_prefix_model_config()
|
||||
model_config.multimodal_config = None
|
||||
|
||||
assert model_config._supports_multimodal_for_mm_prefix() is True
|
||||
assert not hasattr(model_config, "_supports_multimodal_inputs_cached")
|
||||
|
||||
|
||||
def test_language_model_only_disables_via_supports_multimodal_inputs():
|
||||
"""language_model_only zeros all limits, so registry reports text-only."""
|
||||
model_config = _make_mm_prefix_model_config(language_model_only=True)
|
||||
|
||||
with patch(
|
||||
"vllm.multimodal.MULTIMODAL_REGISTRY.supports_multimodal_inputs",
|
||||
return_value=False,
|
||||
):
|
||||
assert model_config._supports_multimodal_for_mm_prefix() is False
|
||||
|
||||
|
||||
def test_convertor_clears_mm_prefix_when_multimodal_disabled():
|
||||
hf_config = PretrainedConfig(
|
||||
model_type="gemma3",
|
||||
architectures=["Gemma3ForConditionalGeneration"],
|
||||
)
|
||||
hf_config.is_mm_prefix_lm = True
|
||||
convertor = ModelArchConfigConvertorBase(hf_config, hf_config)
|
||||
|
||||
assert convertor.is_mm_prefix_lm(supports_multimodal=True) is True
|
||||
assert convertor.is_mm_prefix_lm(supports_multimodal=False) is False
|
||||
|
||||
enabled = convertor.convert(supports_multimodal=True)
|
||||
disabled = convertor.convert(supports_multimodal=False)
|
||||
assert enabled.is_mm_prefix_lm is True
|
||||
assert disabled.is_mm_prefix_lm is False
|
||||
|
||||
|
||||
def test_sticky_cache_survives_text_subconfig_regeneration():
|
||||
"""with_hf_config deepcopies the cached decision onto text submodules."""
|
||||
model_config = _make_mm_prefix_model_config()
|
||||
with patch(
|
||||
"vllm.multimodal.MULTIMODAL_REGISTRY.supports_multimodal_inputs",
|
||||
return_value=False,
|
||||
):
|
||||
assert model_config._supports_multimodal_for_mm_prefix() is False
|
||||
|
||||
# Simulate deepcopy onto a Gemma4ForCausalLM-like config that would
|
||||
# otherwise fail registry lookup / return False incorrectly.
|
||||
text_config = _make_mm_prefix_model_config()
|
||||
text_config._supports_multimodal_inputs_cached = (
|
||||
model_config._supports_multimodal_inputs_cached
|
||||
)
|
||||
with patch(
|
||||
"vllm.multimodal.MULTIMODAL_REGISTRY.supports_multimodal_inputs",
|
||||
side_effect=AssertionError("must not re-query registry"),
|
||||
):
|
||||
assert text_config._supports_multimodal_for_mm_prefix() is False
|
||||
|
||||
@@ -507,6 +507,12 @@ def dummy_hf_overrides(
|
||||
hf_config = _hf_config
|
||||
hf_text_config = text_config
|
||||
|
||||
# Keep architecture conversion on the default multimodal path; this
|
||||
# helper only needs HF-derived fields, not deployment MM limits.
|
||||
@staticmethod
|
||||
def _supports_multimodal_for_mm_prefix() -> bool:
|
||||
return True
|
||||
|
||||
model_arch_config = ModelConfig.get_model_arch_config(DummyConfig)
|
||||
# Only set MoE related config when the model has MoE layers.
|
||||
# Otherwise all models detected as MoE by _get_transformers_backend_cls.
|
||||
|
||||
+43
-1
@@ -758,6 +758,10 @@ class ModelConfig:
|
||||
"disable the cache with --mm-processor-cache-gb 0."
|
||||
)
|
||||
|
||||
# Rebuild after multimodal_config exists so text-only mm_prefix
|
||||
# clearing is applied (and cached for later with_hf_config calls).
|
||||
self.model_arch_config = self.get_model_arch_config()
|
||||
|
||||
if self.disable_sliding_window:
|
||||
# Set after get_and_verify_max_len to ensure that max_model_len
|
||||
# can be correctly capped to sliding window size
|
||||
@@ -770,6 +774,42 @@ class ModelConfig:
|
||||
self._verify_cuda_graph()
|
||||
self._verify_bnb_config()
|
||||
|
||||
def _supports_multimodal_for_mm_prefix(self) -> bool:
|
||||
"""Whether multimodal inputs can still appear for this deployment.
|
||||
|
||||
This runs more than once per config: once early in ``__post_init__``
|
||||
(before ``multimodal_config`` exists), again after it is created, and
|
||||
then for every ``get_model_arch_config`` regeneration -- notably
|
||||
``with_hf_config``, which deep-copies this ``ModelConfig`` and swaps
|
||||
``hf_config`` for a text-only submodule (e.g. ``Gemma4ForCausalLM``).
|
||||
|
||||
The result is cached for correctness, not just to save work: on the
|
||||
``with_hf_config`` copy the submodule architecture has no registered
|
||||
multimodal processor, so re-querying the registry would raise and be
|
||||
treated as text-only, wrongly clearing ``is_mm_prefix_lm`` even when a
|
||||
vision modality is still enabled (e.g. ``image=0`` but video allowed).
|
||||
The deep-copied cache preserves the top-level decision instead.
|
||||
"""
|
||||
cached = getattr(self, "_supports_multimodal_inputs_cached", None)
|
||||
if cached is not None:
|
||||
return cached
|
||||
|
||||
if self.multimodal_config is None:
|
||||
# Early call before multimodal init — do not clear mm_prefix yet.
|
||||
return True
|
||||
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
|
||||
supports_mm = MULTIMODAL_REGISTRY.supports_multimodal_inputs(self)
|
||||
self._supports_multimodal_inputs_cached = supports_mm
|
||||
if not supports_mm:
|
||||
logger.info_once(
|
||||
"Disabled mm_prefix attention mode because multimodal inputs "
|
||||
"are configuration-disabled. Attention backends without "
|
||||
"mm_prefix support may now be selected."
|
||||
)
|
||||
return supports_mm
|
||||
|
||||
def get_model_arch_config(
|
||||
self,
|
||||
) -> ModelArchitectureConfig:
|
||||
@@ -777,7 +817,9 @@ class ModelConfig:
|
||||
self.hf_config.model_type, ModelArchConfigConvertorBase
|
||||
)
|
||||
convertor = convertor_cls(self.hf_config, self.hf_text_config)
|
||||
return convertor.convert()
|
||||
return convertor.convert(
|
||||
supports_multimodal=self._supports_multimodal_for_mm_prefix()
|
||||
)
|
||||
|
||||
@field_validator("tokenizer", "max_model_len", mode="wrap")
|
||||
@classmethod
|
||||
|
||||
@@ -300,8 +300,16 @@ class ModelArchConfigConvertorBase:
|
||||
)
|
||||
return False
|
||||
|
||||
def is_mm_prefix_lm(self) -> bool:
|
||||
"""Whether to use bidirectional attention for mm positions."""
|
||||
def is_mm_prefix_lm(self, supports_multimodal: bool = True) -> bool:
|
||||
"""Whether to use bidirectional attention for mm positions.
|
||||
|
||||
``supports_multimodal`` is False when the deployment is configuration-
|
||||
disabled for multimodal inputs (text-only serving). In that case
|
||||
mm_prefix is unnecessary and must stay off so attention backends
|
||||
without ``supports_mm_prefix()`` remain eligible.
|
||||
"""
|
||||
if not supports_multimodal:
|
||||
return False
|
||||
if hasattr(self.hf_config, "is_mm_prefix_lm"):
|
||||
return bool(self.hf_config.is_mm_prefix_lm)
|
||||
# fallback to list of known models
|
||||
@@ -358,7 +366,7 @@ class ModelArchConfigConvertorBase:
|
||||
derived_max_model_len = tmp_max_len
|
||||
return derived_max_model_len, max_len_key
|
||||
|
||||
def convert(self) -> ModelArchitectureConfig:
|
||||
def convert(self, supports_multimodal: bool = True) -> ModelArchitectureConfig:
|
||||
model_arch_config = ModelArchitectureConfig(
|
||||
architectures=self.get_architectures(),
|
||||
model_type=self.hf_config.model_type,
|
||||
@@ -372,7 +380,7 @@ class ModelArchConfigConvertorBase:
|
||||
num_experts=self.get_num_experts(),
|
||||
quantization_config=self.get_quantization_config(),
|
||||
is_deepseek_mla=self.is_deepseek_mla(),
|
||||
is_mm_prefix_lm=self.is_mm_prefix_lm(),
|
||||
is_mm_prefix_lm=self.is_mm_prefix_lm(supports_multimodal),
|
||||
rswa_window=self.rswa_window(),
|
||||
derived_max_model_len_and_key=self.derive_max_model_len_and_key(),
|
||||
)
|
||||
@@ -401,7 +409,7 @@ class CohereAsrModelArchConfigConvertor(ModelArchConfigConvertorBase):
|
||||
)
|
||||
return enc_num_kv_heads
|
||||
|
||||
def is_mm_prefix_lm(self) -> bool:
|
||||
def is_mm_prefix_lm(self, supports_multimodal: bool = True) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
@@ -584,7 +592,9 @@ class Gemma4MTPModelArchConfigConvertor(ModelArchConfigConvertorBase):
|
||||
|
||||
|
||||
class Gemma4ModelArchConfigConvertor(ModelArchConfigConvertorBase):
|
||||
def is_mm_prefix_lm(self) -> bool:
|
||||
def is_mm_prefix_lm(self, supports_multimodal: bool = True) -> bool:
|
||||
if not supports_multimodal:
|
||||
return False
|
||||
return (
|
||||
getattr(self.hf_text_config, "use_bidirectional_attention", None)
|
||||
== "vision"
|
||||
|
||||
Reference in New Issue
Block a user