Support MLA properly in the Transformers modeling backend (#48250)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -16,6 +16,7 @@
|
||||
# limitations under the License.
|
||||
"""Transformers modeling backend base class."""
|
||||
|
||||
import os
|
||||
from collections.abc import Callable, Iterable
|
||||
from itertools import chain
|
||||
from operator import attrgetter
|
||||
@@ -40,7 +41,10 @@ from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.attention import (
|
||||
Attention,
|
||||
EncoderOnlyAttention,
|
||||
MLAAttention,
|
||||
)
|
||||
from vllm.model_executor.layers.attention.mla_attention import get_mla_dims
|
||||
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
|
||||
from vllm.model_executor.layers.fused_moe import MoERunner
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
|
||||
from vllm.model_executor.models.interfaces import (
|
||||
@@ -52,11 +56,13 @@ from vllm.model_executor.models.interfaces import (
|
||||
)
|
||||
from vllm.model_executor.models.interfaces_base import VllmModel
|
||||
from vllm.model_executor.models.transformers.fuser import BaseFuser, Fusers
|
||||
from vllm.model_executor.models.transformers.fusers import MLAFuser
|
||||
from vllm.model_executor.models.transformers.utils import (
|
||||
can_enable_torch_compile,
|
||||
get_feature_request_tip,
|
||||
init_on_device_without_buffers,
|
||||
log_replacement,
|
||||
named_state,
|
||||
replace_conv_class,
|
||||
replace_linear_class,
|
||||
)
|
||||
@@ -64,11 +70,11 @@ from vllm.model_executor.models.utils import (
|
||||
AutoWeightsLoader,
|
||||
PPMissingLayer,
|
||||
WeightsMapper,
|
||||
extract_layer_index,
|
||||
make_empty_intermediate_tensors_factory,
|
||||
maybe_prefix,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.v1.attention.backend import AttentionType
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from transformers import PreTrainedModel
|
||||
@@ -129,6 +135,9 @@ class Base(
|
||||
self.packed_modules_mapping: dict[str, list[str]] = {}
|
||||
"""Fused module -> constituent projections, populated by `recursive_replace`
|
||||
for the quantization machinery and loaders (e.g. bitsandbytes)."""
|
||||
self.fusers: dict[str, BaseFuser] = {}
|
||||
"""Module qualname -> the fuser applied to it, populated
|
||||
by `recursive_replace` for `create_attention_instances`."""
|
||||
|
||||
# Attrs for Eagle3 (see self.set_aux_hidden_state_layers)
|
||||
self._target_class: type[nn.Module] = nn.Module
|
||||
@@ -192,6 +201,9 @@ class Base(
|
||||
# Initialize any parameters that have not had their modules replaced
|
||||
self.init_parameters(self.model)
|
||||
|
||||
# Upcast weights Transformers always keeps in fp32
|
||||
self.keep_in_fp32(self.model)
|
||||
|
||||
# Pipeline parallel intermediate tensors
|
||||
self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
|
||||
["hidden_states"], self.text_config.hidden_size
|
||||
@@ -457,6 +469,8 @@ class Base(
|
||||
|
||||
def register_fusion(fuser: BaseFuser, prefix: str):
|
||||
"""Register a fused layer's mappings just before it is built."""
|
||||
self.fusers[prefix] = fuser
|
||||
|
||||
orig_to_new_stacked = fuser.orig_to_new_stacked(prefix)
|
||||
self.hf_to_vllm_mapper.orig_to_new_stacked.update(orig_to_new_stacked)
|
||||
|
||||
@@ -522,13 +536,84 @@ class Base(
|
||||
"""
|
||||
Create `Attention` instances to inform KV cache allocation.
|
||||
"""
|
||||
mla_fusers = {}
|
||||
attention_instances = {}
|
||||
text_config = self.text_config
|
||||
attn_cls = self._get_attn_cls()
|
||||
|
||||
# kv_lora_rank indicates that this is an MLA model
|
||||
if getattr(text_config, "kv_lora_rank", None) is not None:
|
||||
mla_fusers = {
|
||||
extract_layer_index(prefix): (prefix, fuser)
|
||||
for prefix, fuser in self.fusers.items()
|
||||
if isinstance(fuser, MLAFuser)
|
||||
}
|
||||
if attn_cls is MLAAttention:
|
||||
text_config._attn_implementation = "vllm_mla"
|
||||
else:
|
||||
# MLA model not using MLAAttention: recompute head_size for full attn
|
||||
qk_nope_head_dim = getattr(text_config, "qk_nope_head_dim", 0)
|
||||
qk_rope_head_dim = getattr(text_config, "qk_rope_head_dim", 0)
|
||||
if qk_head_dim := qk_nope_head_dim + qk_rope_head_dim:
|
||||
self.model_config.model_arch_config.head_size = qk_head_dim
|
||||
|
||||
num_heads = self.model_config.get_num_attention_heads(self.parallel_config)
|
||||
head_size = self.model_config.get_head_size()
|
||||
num_kv_heads = self.model_config.get_num_kv_heads(self.parallel_config)
|
||||
logits_soft_cap = getattr(text_config, "attn_logit_softcapping", None)
|
||||
|
||||
pp_rank = self.pp_group.rank_in_group
|
||||
pp_size = self.pp_group.world_size
|
||||
start, end = get_pp_indices(text_config.num_hidden_layers, pp_rank, pp_size)
|
||||
|
||||
for i in range(start, end):
|
||||
kwargs = dict(
|
||||
num_heads=num_heads,
|
||||
# NOTE: We use Llama scale as default, if it's set by
|
||||
# Transformers, it's updated in vllm_attention_forward
|
||||
scale=head_size**-0.5,
|
||||
cache_config=self.cache_config,
|
||||
quant_config=self.quant_config,
|
||||
prefix=f"{i}.attn",
|
||||
)
|
||||
|
||||
if attn_cls is MLAAttention:
|
||||
prefix, fuser = mla_fusers[i]
|
||||
mla_module = self.get_submodule(prefix)
|
||||
dims = get_mla_dims(self.model_config)
|
||||
kwargs.update(
|
||||
scale=mla_module.scaling,
|
||||
qk_nope_head_dim=dims.qk_nope_head_dim,
|
||||
qk_rope_head_dim=dims.qk_rope_head_dim,
|
||||
v_head_dim=dims.v_head_dim,
|
||||
q_lora_rank=dims.q_lora_rank,
|
||||
kv_lora_rank=dims.kv_lora_rank,
|
||||
kv_b_proj=mla_module.get_submodule(fuser.kv_b_proj_name),
|
||||
)
|
||||
else:
|
||||
kwargs.update(
|
||||
head_size=head_size,
|
||||
num_kv_heads=num_kv_heads,
|
||||
logits_soft_cap=logits_soft_cap,
|
||||
)
|
||||
|
||||
# Handle interleaved sliding window attention
|
||||
if (
|
||||
hasattr(text_config, "layer_types")
|
||||
and text_config.layer_types[i] == "sliding_attention"
|
||||
):
|
||||
kwargs["per_layer_sliding_window"] = text_config.sliding_window
|
||||
|
||||
attn_instance = attn_cls(**kwargs)
|
||||
if attn_cls is MLAAttention:
|
||||
# Attach MLA attn_instance to mla_module so it appears in
|
||||
# model.named_modules() and runs its process_weights_after_loading
|
||||
mla_module._vllm_mla_attn = attn_instance
|
||||
attention_instances[i] = attn_instance
|
||||
return attention_instances
|
||||
|
||||
def _get_attn_cls(self) -> type[AttentionLayerBase]:
|
||||
"""Return the `Attention` class to use for this model's layers."""
|
||||
# In encoder models, the attention layers will have `is_causal=False`
|
||||
is_encoder = lambda module: not getattr(module, "is_causal", True)
|
||||
has_encoder = lambda model: any(is_encoder(m) for m in model.modules())
|
||||
@@ -537,44 +622,18 @@ class Base(
|
||||
# found in a text only model, we assume the whole model is an encoder model
|
||||
if has_encoder(self.model) and not is_multimodal(self.config):
|
||||
self.check_version("5.0.0", "encoder models support")
|
||||
attn_type = AttentionType.ENCODER_ONLY
|
||||
else:
|
||||
attn_type = AttentionType.DECODER
|
||||
|
||||
pp_rank = self.pp_group.rank_in_group
|
||||
pp_size = self.pp_group.world_size
|
||||
start, end = get_pp_indices(text_config.num_hidden_layers, pp_rank, pp_size)
|
||||
|
||||
attention_instances = {}
|
||||
for i in range(start, end):
|
||||
# Handle interleaved sliding window attention
|
||||
per_layer_sliding_window = None
|
||||
if (
|
||||
hasattr(self.config, "layer_types")
|
||||
and self.config.layer_types[i] == "sliding_attention"
|
||||
):
|
||||
per_layer_sliding_window = self.config.sliding_window
|
||||
|
||||
attn_cls = (
|
||||
EncoderOnlyAttention
|
||||
if attn_type == AttentionType.ENCODER_ONLY
|
||||
else Attention
|
||||
return EncoderOnlyAttention
|
||||
if self.model_config.use_mla:
|
||||
self.check_version("5.15.0.dev0", "optimized MLA support")
|
||||
if any(isinstance(fuser, MLAFuser) for fuser in self.fusers.values()):
|
||||
return MLAAttention
|
||||
logger.warning_once(
|
||||
"This model uses MLA but `MLAFuser` failed to match and/or fuse any "
|
||||
"MLA attention layers. Falling back to full attention with a padded "
|
||||
"`value` head dimension."
|
||||
)
|
||||
attention_instances[i] = attn_cls(
|
||||
num_heads=num_heads,
|
||||
head_size=head_size,
|
||||
# NOTE: We use Llama scale as default, if it's set by
|
||||
# Transformers, it's updated in vllm_attention_forward
|
||||
scale=head_size**-0.5,
|
||||
num_kv_heads=num_kv_heads,
|
||||
cache_config=self.cache_config,
|
||||
quant_config=self.quant_config,
|
||||
logits_soft_cap=logits_soft_cap,
|
||||
per_layer_sliding_window=per_layer_sliding_window,
|
||||
prefix=f"{i}.attn",
|
||||
attn_type=attn_type,
|
||||
)
|
||||
return attention_instances
|
||||
os.environ["VLLM_MLA_DISABLE"] = "1"
|
||||
return Attention
|
||||
|
||||
def init_parameters(self, module: nn.Module, dtype: torch.dtype | None = None):
|
||||
"""
|
||||
@@ -604,6 +663,18 @@ class Base(
|
||||
|
||||
_init_parameters(module)
|
||||
|
||||
def keep_in_fp32(self, module: nn.Module):
|
||||
"""Honor `_keep_in_fp32_modules_strict` as `from_pretrained` would."""
|
||||
if self.model_config.dtype not in (torch.float16, torch.bfloat16):
|
||||
return
|
||||
fragments = getattr(module, "_keep_in_fp32_modules_strict", None)
|
||||
if not fragments:
|
||||
return
|
||||
pattern = re.compile("|".join(re.escape(f) for f in fragments))
|
||||
for name, tensor in named_state(module):
|
||||
if not hasattr(tensor, "weight_loader") and pattern.search(name):
|
||||
tensor.data = tensor.data.to(torch.float32)
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.get_input_embeddings()(input_ids)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user