Support MLA properly in the Transformers modeling backend (#48250)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Harry Mellor
2026-08-04 13:24:11 +01:00
committed by GitHub
parent 24c939c47d
commit 59b2fdfc4e
9 changed files with 704 additions and 57 deletions
+109 -38
View File
@@ -16,6 +16,7 @@
# limitations under the License.
"""Transformers modeling backend base class."""
import os
from collections.abc import Callable, Iterable
from itertools import chain
from operator import attrgetter
@@ -40,7 +41,10 @@ from vllm.logger import init_logger
from vllm.model_executor.layers.attention import (
Attention,
EncoderOnlyAttention,
MLAAttention,
)
from vllm.model_executor.layers.attention.mla_attention import get_mla_dims
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
from vllm.model_executor.layers.fused_moe import MoERunner
from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
from vllm.model_executor.models.interfaces import (
@@ -52,11 +56,13 @@ from vllm.model_executor.models.interfaces import (
)
from vllm.model_executor.models.interfaces_base import VllmModel
from vllm.model_executor.models.transformers.fuser import BaseFuser, Fusers
from vllm.model_executor.models.transformers.fusers import MLAFuser
from vllm.model_executor.models.transformers.utils import (
can_enable_torch_compile,
get_feature_request_tip,
init_on_device_without_buffers,
log_replacement,
named_state,
replace_conv_class,
replace_linear_class,
)
@@ -64,11 +70,11 @@ from vllm.model_executor.models.utils import (
AutoWeightsLoader,
PPMissingLayer,
WeightsMapper,
extract_layer_index,
make_empty_intermediate_tensors_factory,
maybe_prefix,
)
from vllm.sequence import IntermediateTensors
from vllm.v1.attention.backend import AttentionType
if TYPE_CHECKING:
from transformers import PreTrainedModel
@@ -129,6 +135,9 @@ class Base(
self.packed_modules_mapping: dict[str, list[str]] = {}
"""Fused module -> constituent projections, populated by `recursive_replace`
for the quantization machinery and loaders (e.g. bitsandbytes)."""
self.fusers: dict[str, BaseFuser] = {}
"""Module qualname -> the fuser applied to it, populated
by `recursive_replace` for `create_attention_instances`."""
# Attrs for Eagle3 (see self.set_aux_hidden_state_layers)
self._target_class: type[nn.Module] = nn.Module
@@ -192,6 +201,9 @@ class Base(
# Initialize any parameters that have not had their modules replaced
self.init_parameters(self.model)
# Upcast weights Transformers always keeps in fp32
self.keep_in_fp32(self.model)
# Pipeline parallel intermediate tensors
self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
["hidden_states"], self.text_config.hidden_size
@@ -457,6 +469,8 @@ class Base(
def register_fusion(fuser: BaseFuser, prefix: str):
"""Register a fused layer's mappings just before it is built."""
self.fusers[prefix] = fuser
orig_to_new_stacked = fuser.orig_to_new_stacked(prefix)
self.hf_to_vllm_mapper.orig_to_new_stacked.update(orig_to_new_stacked)
@@ -522,13 +536,84 @@ class Base(
"""
Create `Attention` instances to inform KV cache allocation.
"""
mla_fusers = {}
attention_instances = {}
text_config = self.text_config
attn_cls = self._get_attn_cls()
# kv_lora_rank indicates that this is an MLA model
if getattr(text_config, "kv_lora_rank", None) is not None:
mla_fusers = {
extract_layer_index(prefix): (prefix, fuser)
for prefix, fuser in self.fusers.items()
if isinstance(fuser, MLAFuser)
}
if attn_cls is MLAAttention:
text_config._attn_implementation = "vllm_mla"
else:
# MLA model not using MLAAttention: recompute head_size for full attn
qk_nope_head_dim = getattr(text_config, "qk_nope_head_dim", 0)
qk_rope_head_dim = getattr(text_config, "qk_rope_head_dim", 0)
if qk_head_dim := qk_nope_head_dim + qk_rope_head_dim:
self.model_config.model_arch_config.head_size = qk_head_dim
num_heads = self.model_config.get_num_attention_heads(self.parallel_config)
head_size = self.model_config.get_head_size()
num_kv_heads = self.model_config.get_num_kv_heads(self.parallel_config)
logits_soft_cap = getattr(text_config, "attn_logit_softcapping", None)
pp_rank = self.pp_group.rank_in_group
pp_size = self.pp_group.world_size
start, end = get_pp_indices(text_config.num_hidden_layers, pp_rank, pp_size)
for i in range(start, end):
kwargs = dict(
num_heads=num_heads,
# NOTE: We use Llama scale as default, if it's set by
# Transformers, it's updated in vllm_attention_forward
scale=head_size**-0.5,
cache_config=self.cache_config,
quant_config=self.quant_config,
prefix=f"{i}.attn",
)
if attn_cls is MLAAttention:
prefix, fuser = mla_fusers[i]
mla_module = self.get_submodule(prefix)
dims = get_mla_dims(self.model_config)
kwargs.update(
scale=mla_module.scaling,
qk_nope_head_dim=dims.qk_nope_head_dim,
qk_rope_head_dim=dims.qk_rope_head_dim,
v_head_dim=dims.v_head_dim,
q_lora_rank=dims.q_lora_rank,
kv_lora_rank=dims.kv_lora_rank,
kv_b_proj=mla_module.get_submodule(fuser.kv_b_proj_name),
)
else:
kwargs.update(
head_size=head_size,
num_kv_heads=num_kv_heads,
logits_soft_cap=logits_soft_cap,
)
# Handle interleaved sliding window attention
if (
hasattr(text_config, "layer_types")
and text_config.layer_types[i] == "sliding_attention"
):
kwargs["per_layer_sliding_window"] = text_config.sliding_window
attn_instance = attn_cls(**kwargs)
if attn_cls is MLAAttention:
# Attach MLA attn_instance to mla_module so it appears in
# model.named_modules() and runs its process_weights_after_loading
mla_module._vllm_mla_attn = attn_instance
attention_instances[i] = attn_instance
return attention_instances
def _get_attn_cls(self) -> type[AttentionLayerBase]:
"""Return the `Attention` class to use for this model's layers."""
# In encoder models, the attention layers will have `is_causal=False`
is_encoder = lambda module: not getattr(module, "is_causal", True)
has_encoder = lambda model: any(is_encoder(m) for m in model.modules())
@@ -537,44 +622,18 @@ class Base(
# found in a text only model, we assume the whole model is an encoder model
if has_encoder(self.model) and not is_multimodal(self.config):
self.check_version("5.0.0", "encoder models support")
attn_type = AttentionType.ENCODER_ONLY
else:
attn_type = AttentionType.DECODER
pp_rank = self.pp_group.rank_in_group
pp_size = self.pp_group.world_size
start, end = get_pp_indices(text_config.num_hidden_layers, pp_rank, pp_size)
attention_instances = {}
for i in range(start, end):
# Handle interleaved sliding window attention
per_layer_sliding_window = None
if (
hasattr(self.config, "layer_types")
and self.config.layer_types[i] == "sliding_attention"
):
per_layer_sliding_window = self.config.sliding_window
attn_cls = (
EncoderOnlyAttention
if attn_type == AttentionType.ENCODER_ONLY
else Attention
return EncoderOnlyAttention
if self.model_config.use_mla:
self.check_version("5.15.0.dev0", "optimized MLA support")
if any(isinstance(fuser, MLAFuser) for fuser in self.fusers.values()):
return MLAAttention
logger.warning_once(
"This model uses MLA but `MLAFuser` failed to match and/or fuse any "
"MLA attention layers. Falling back to full attention with a padded "
"`value` head dimension."
)
attention_instances[i] = attn_cls(
num_heads=num_heads,
head_size=head_size,
# NOTE: We use Llama scale as default, if it's set by
# Transformers, it's updated in vllm_attention_forward
scale=head_size**-0.5,
num_kv_heads=num_kv_heads,
cache_config=self.cache_config,
quant_config=self.quant_config,
logits_soft_cap=logits_soft_cap,
per_layer_sliding_window=per_layer_sliding_window,
prefix=f"{i}.attn",
attn_type=attn_type,
)
return attention_instances
os.environ["VLLM_MLA_DISABLE"] = "1"
return Attention
def init_parameters(self, module: nn.Module, dtype: torch.dtype | None = None):
"""
@@ -604,6 +663,18 @@ class Base(
_init_parameters(module)
def keep_in_fp32(self, module: nn.Module):
"""Honor `_keep_in_fp32_modules_strict` as `from_pretrained` would."""
if self.model_config.dtype not in (torch.float16, torch.bfloat16):
return
fragments = getattr(module, "_keep_in_fp32_modules_strict", None)
if not fragments:
return
pattern = re.compile("|".join(re.escape(f) for f in fragments))
for name, tensor in named_state(module):
if not hasattr(tensor, "weight_loader") and pattern.search(name):
tensor.data = tensor.data.to(torch.float32)
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.model.get_input_embeddings()(input_ids)