[Core] Keep attention backends eligible for text-only serving of prefix-LM models (#48796)

Signed-off-by: qtris123 <voquangtri2021@gmail.com>
Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
Tri Vo
2026-07-25 03:06:14 -07:00
committed by GitHub
co-authored by Cyrus Leung
parent dbcc1cdd0a
commit fe5145765f
4 changed files with 165 additions and 7 deletions
@@ -300,8 +300,16 @@ class ModelArchConfigConvertorBase:
)
return False
def is_mm_prefix_lm(self) -> bool:
"""Whether to use bidirectional attention for mm positions."""
def is_mm_prefix_lm(self, supports_multimodal: bool = True) -> bool:
"""Whether to use bidirectional attention for mm positions.
``supports_multimodal`` is False when the deployment is configuration-
disabled for multimodal inputs (text-only serving). In that case
mm_prefix is unnecessary and must stay off so attention backends
without ``supports_mm_prefix()`` remain eligible.
"""
if not supports_multimodal:
return False
if hasattr(self.hf_config, "is_mm_prefix_lm"):
return bool(self.hf_config.is_mm_prefix_lm)
# fallback to list of known models
@@ -358,7 +366,7 @@ class ModelArchConfigConvertorBase:
derived_max_model_len = tmp_max_len
return derived_max_model_len, max_len_key
def convert(self) -> ModelArchitectureConfig:
def convert(self, supports_multimodal: bool = True) -> ModelArchitectureConfig:
model_arch_config = ModelArchitectureConfig(
architectures=self.get_architectures(),
model_type=self.hf_config.model_type,
@@ -372,7 +380,7 @@ class ModelArchConfigConvertorBase:
num_experts=self.get_num_experts(),
quantization_config=self.get_quantization_config(),
is_deepseek_mla=self.is_deepseek_mla(),
is_mm_prefix_lm=self.is_mm_prefix_lm(),
is_mm_prefix_lm=self.is_mm_prefix_lm(supports_multimodal),
rswa_window=self.rswa_window(),
derived_max_model_len_and_key=self.derive_max_model_len_and_key(),
)
@@ -401,7 +409,7 @@ class CohereAsrModelArchConfigConvertor(ModelArchConfigConvertorBase):
)
return enc_num_kv_heads
def is_mm_prefix_lm(self) -> bool:
def is_mm_prefix_lm(self, supports_multimodal: bool = True) -> bool:
return False
@@ -584,7 +592,9 @@ class Gemma4MTPModelArchConfigConvertor(ModelArchConfigConvertorBase):
class Gemma4ModelArchConfigConvertor(ModelArchConfigConvertorBase):
def is_mm_prefix_lm(self) -> bool:
def is_mm_prefix_lm(self, supports_multimodal: bool = True) -> bool:
if not supports_multimodal:
return False
return (
getattr(self.hf_text_config, "use_bidirectional_attention", None)
== "vision"