[Core] Keep attention backends eligible for text-only serving of prefix-LM models (#48796)
Signed-off-by: qtris123 <voquangtri2021@gmail.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -300,8 +300,16 @@ class ModelArchConfigConvertorBase:
|
||||
)
|
||||
return False
|
||||
|
||||
def is_mm_prefix_lm(self) -> bool:
|
||||
"""Whether to use bidirectional attention for mm positions."""
|
||||
def is_mm_prefix_lm(self, supports_multimodal: bool = True) -> bool:
|
||||
"""Whether to use bidirectional attention for mm positions.
|
||||
|
||||
``supports_multimodal`` is False when the deployment is configuration-
|
||||
disabled for multimodal inputs (text-only serving). In that case
|
||||
mm_prefix is unnecessary and must stay off so attention backends
|
||||
without ``supports_mm_prefix()`` remain eligible.
|
||||
"""
|
||||
if not supports_multimodal:
|
||||
return False
|
||||
if hasattr(self.hf_config, "is_mm_prefix_lm"):
|
||||
return bool(self.hf_config.is_mm_prefix_lm)
|
||||
# fallback to list of known models
|
||||
@@ -358,7 +366,7 @@ class ModelArchConfigConvertorBase:
|
||||
derived_max_model_len = tmp_max_len
|
||||
return derived_max_model_len, max_len_key
|
||||
|
||||
def convert(self) -> ModelArchitectureConfig:
|
||||
def convert(self, supports_multimodal: bool = True) -> ModelArchitectureConfig:
|
||||
model_arch_config = ModelArchitectureConfig(
|
||||
architectures=self.get_architectures(),
|
||||
model_type=self.hf_config.model_type,
|
||||
@@ -372,7 +380,7 @@ class ModelArchConfigConvertorBase:
|
||||
num_experts=self.get_num_experts(),
|
||||
quantization_config=self.get_quantization_config(),
|
||||
is_deepseek_mla=self.is_deepseek_mla(),
|
||||
is_mm_prefix_lm=self.is_mm_prefix_lm(),
|
||||
is_mm_prefix_lm=self.is_mm_prefix_lm(supports_multimodal),
|
||||
rswa_window=self.rswa_window(),
|
||||
derived_max_model_len_and_key=self.derive_max_model_len_and_key(),
|
||||
)
|
||||
@@ -401,7 +409,7 @@ class CohereAsrModelArchConfigConvertor(ModelArchConfigConvertorBase):
|
||||
)
|
||||
return enc_num_kv_heads
|
||||
|
||||
def is_mm_prefix_lm(self) -> bool:
|
||||
def is_mm_prefix_lm(self, supports_multimodal: bool = True) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
@@ -584,7 +592,9 @@ class Gemma4MTPModelArchConfigConvertor(ModelArchConfigConvertorBase):
|
||||
|
||||
|
||||
class Gemma4ModelArchConfigConvertor(ModelArchConfigConvertorBase):
|
||||
def is_mm_prefix_lm(self) -> bool:
|
||||
def is_mm_prefix_lm(self, supports_multimodal: bool = True) -> bool:
|
||||
if not supports_multimodal:
|
||||
return False
|
||||
return (
|
||||
getattr(self.hf_text_config, "use_bidirectional_attention", None)
|
||||
== "vision"
|
||||
|
||||
Reference in New Issue
Block a user