[Transformers backend] Find attention with a fuser and attach vLLM's layer to it (#54941)
Signed-off-by: Thomas Ortner <boh@zurich.ibm.com> Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
co-authored by
Harry Mellor
parent
99a1ab8e4c
commit
8ad2076c4a
@@ -57,7 +57,11 @@ from vllm.model_executor.models.interfaces import (
|
||||
)
|
||||
from vllm.model_executor.models.interfaces_base import VllmModel
|
||||
from vllm.model_executor.models.transformers.fuser import BaseFuser, Fusers
|
||||
from vllm.model_executor.models.transformers.fusers import MLAFuser
|
||||
from vllm.model_executor.models.transformers.fusers import AttentionFuser, MLAFuser
|
||||
from vllm.model_executor.models.transformers.fusers.attention import (
|
||||
VLLM_ATTN_IMPL,
|
||||
VLLM_MLA_ATTN_IMPL,
|
||||
)
|
||||
from vllm.model_executor.models.transformers.utils import (
|
||||
attrsetter,
|
||||
can_enable_torch_compile,
|
||||
@@ -73,7 +77,6 @@ from vllm.model_executor.models.utils import (
|
||||
AutoWeightsLoader,
|
||||
PPMissingLayer,
|
||||
WeightsMapper,
|
||||
extract_layer_index,
|
||||
make_empty_intermediate_tensors_factory,
|
||||
maybe_prefix,
|
||||
)
|
||||
@@ -86,6 +89,11 @@ if TYPE_CHECKING:
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
VLLM_ATTN_ATTR = "attn"
|
||||
"""Attribute in which to store the vLLM `Attention` instance that serves a Transformers
|
||||
attention module. `"attn"` is chosen deliberately to match the mappings generated by
|
||||
`maybe_remap_kv_scale_name`."""
|
||||
|
||||
|
||||
class PreTrainedModelClasses(NamedTuple):
|
||||
decoder: type["PreTrainedModel"]
|
||||
@@ -129,9 +137,13 @@ class Base(
|
||||
self.packed_modules_mapping: dict[str, list[str]] = {}
|
||||
"""Fused module -> constituent projections, populated by `recursive_replace`
|
||||
for the quantization machinery and loaders (e.g. bitsandbytes)."""
|
||||
self.fusers: dict[str, BaseFuser] = {}
|
||||
"""Module qualname -> the fuser applied to it, populated
|
||||
self.fusers: dict[str, list[BaseFuser]] = {}
|
||||
"""Module qualname -> the fusers applied to it, populated
|
||||
by `recursive_replace` for `create_attention_instances`."""
|
||||
self.attention_fusers: dict[int, tuple[str, AttentionFuser]] = {}
|
||||
"""`layer_idx` -> the qualname and fuser of the module computing that
|
||||
layer's attention, populated by `recursive_replace` for
|
||||
`create_attention_instances`."""
|
||||
|
||||
# Attrs for Eagle3 (see self.set_aux_hidden_state_layers)
|
||||
self._target_class: type[nn.Module] = nn.Module
|
||||
@@ -189,7 +201,7 @@ class Base(
|
||||
- Sets the dtype to the default torch dtype set by vLLM because Transformers
|
||||
uses the config dtype when creating the model
|
||||
"""
|
||||
self.text_config._attn_implementation = "vllm"
|
||||
self.text_config._attn_implementation = VLLM_ATTN_IMPL
|
||||
self.config.dtype = torch.get_default_dtype()
|
||||
|
||||
@contextmanager
|
||||
@@ -475,9 +487,22 @@ class Base(
|
||||
|
||||
vocab_embeddings = self._vocab_embeddings()
|
||||
|
||||
def register_fusion(fuser: BaseFuser, prefix: str):
|
||||
def register_fusion(fuser: BaseFuser, prefix: str, module: nn.Module):
|
||||
"""Register a fused layer's mappings just before it is built."""
|
||||
self.fusers[prefix] = fuser
|
||||
self.fusers.setdefault(prefix, []).append(fuser)
|
||||
|
||||
if (
|
||||
isinstance(fuser, AttentionFuser)
|
||||
and (index := fuser.layer_index(module)) is not None
|
||||
):
|
||||
if index in self.attention_fusers:
|
||||
other, _ = self.attention_fusers[index]
|
||||
raise ValueError(
|
||||
f"Layer {index} dispatches attention from more than one "
|
||||
f"module ({other} and {prefix}), which one vLLM attention "
|
||||
"layer cannot serve."
|
||||
)
|
||||
self.attention_fusers[index] = (prefix, fuser)
|
||||
|
||||
orig_to_new_stacked = fuser.orig_to_new_stacked(prefix)
|
||||
self.hf_to_vllm_mapper.orig_to_new_stacked.update(orig_to_new_stacked)
|
||||
@@ -528,10 +553,11 @@ class Base(
|
||||
new_module = replace_embedding_class(
|
||||
child_module, self.quant_config, prefix=qual_name
|
||||
)
|
||||
elif (fuser := fusers[child_module]) is not None:
|
||||
register_fusion(fuser, qual_name)
|
||||
new_module = fuser.fuse(child_module, qual_name, self.vllm_config)
|
||||
logger.info_once(fuser.info(child_name))
|
||||
elif child_module_fusers := fusers[child_module]:
|
||||
for fuser in child_module_fusers:
|
||||
register_fusion(fuser, qual_name, child_module)
|
||||
new_module = fuser.fuse(new_module, qual_name, self.vllm_config)
|
||||
logger.info_once(fuser.info(child_name))
|
||||
_recursive_replace(new_module, prefix=qual_name)
|
||||
elif not isinstance(child_module, MoERunner):
|
||||
# MoERunner can contain aliases of shared experts and gates,
|
||||
@@ -548,20 +574,14 @@ class Base(
|
||||
"""
|
||||
Create `Attention` instances to inform KV cache allocation.
|
||||
"""
|
||||
mla_fusers = {}
|
||||
attention_instances = {}
|
||||
text_config = self.text_config
|
||||
attn_cls = self._get_attn_cls()
|
||||
|
||||
# kv_lora_rank indicates that this is an MLA model
|
||||
if getattr(text_config, "kv_lora_rank", None) is not None:
|
||||
mla_fusers = {
|
||||
extract_layer_index(prefix): (prefix, fuser)
|
||||
for prefix, fuser in self.fusers.items()
|
||||
if isinstance(fuser, MLAFuser)
|
||||
}
|
||||
if attn_cls is MLAAttention:
|
||||
text_config._attn_implementation = "vllm_mla"
|
||||
text_config._attn_implementation = VLLM_MLA_ATTN_IMPL
|
||||
else:
|
||||
# MLA model not using MLAAttention: recompute head_size for full attn
|
||||
qk_nope_head_dim = getattr(text_config, "qk_nope_head_dim", 0)
|
||||
@@ -570,12 +590,23 @@ class Base(
|
||||
self.model_config.model_arch_config.head_size = qk_head_dim
|
||||
|
||||
logits_soft_cap = getattr(text_config, "attn_logit_softcapping", None)
|
||||
layer_types = getattr(text_config, "layer_types", None)
|
||||
|
||||
pp_rank = self.pp_group.rank_in_group
|
||||
pp_size = self.pp_group.world_size
|
||||
start, end = get_pp_indices(text_config.num_hidden_layers, pp_rank, pp_size)
|
||||
|
||||
for i in range(start, end):
|
||||
if i not in self.attention_fusers:
|
||||
in_range = layer_types and i < len(layer_types)
|
||||
layer = f"{i} ({layer_types[i]})" if in_range else str(i)
|
||||
raise ValueError(
|
||||
f"Layer {layer} does not dispatch through the Transformers "
|
||||
"attention interface and vLLM has no other way to handle it."
|
||||
)
|
||||
prefix, attn_fuser = self.attention_fusers[i]
|
||||
attn_module = self.get_submodule(prefix)
|
||||
|
||||
# `[i]` is the whole-model config unless the checkpoint is
|
||||
# heterogeneous, in which case it is this layer's own geometry.
|
||||
arch_config = self.model_config.model_arch_config[i]
|
||||
@@ -583,8 +614,10 @@ class Base(
|
||||
self.parallel_config, arch_config
|
||||
)
|
||||
head_size = arch_config.head_size
|
||||
# Default to Llama scale, maybe updated in vllm_attention_forward
|
||||
scale = head_size**-0.5
|
||||
scale = attn_fuser.scale(attn_module)
|
||||
# Default to Llama scale if AttentionFuser couldn't identify it
|
||||
if scale is None:
|
||||
scale = head_size**-0.5
|
||||
num_kv_heads = self.model_config.get_num_kv_heads(
|
||||
self.parallel_config, arch_config
|
||||
)
|
||||
@@ -598,17 +631,22 @@ class Base(
|
||||
)
|
||||
|
||||
if attn_cls is MLAAttention:
|
||||
prefix, fuser = mla_fusers[i]
|
||||
mla_module = self.get_submodule(prefix)
|
||||
mla_fuser = next(
|
||||
(f for f in self.fusers[prefix] if isinstance(f, MLAFuser)), None
|
||||
)
|
||||
if mla_fuser is None:
|
||||
raise ValueError(
|
||||
f"Layer {i} ({prefix}) did not fuse as MLA, but the rest of "
|
||||
"the model did, so its latent KV cache cannot be allocated."
|
||||
)
|
||||
dims = get_mla_dims(self.model_config)
|
||||
kwargs.update(
|
||||
scale=mla_module.scaling,
|
||||
qk_nope_head_dim=dims.qk_nope_head_dim,
|
||||
qk_rope_head_dim=dims.qk_rope_head_dim,
|
||||
v_head_dim=dims.v_head_dim,
|
||||
q_lora_rank=dims.q_lora_rank,
|
||||
kv_lora_rank=dims.kv_lora_rank,
|
||||
kv_b_proj=mla_module.get_submodule(fuser.kv_b_proj_name),
|
||||
kv_b_proj=attn_module.get_submodule(mla_fuser.kv_b_proj_name),
|
||||
)
|
||||
else:
|
||||
kwargs.update(
|
||||
@@ -618,17 +656,20 @@ class Base(
|
||||
)
|
||||
|
||||
# Handle interleaved sliding window attention
|
||||
if (
|
||||
hasattr(text_config, "layer_types")
|
||||
and text_config.layer_types[i] == "sliding_attention"
|
||||
):
|
||||
if layer_types and layer_types[i] == "sliding_attention":
|
||||
kwargs["per_layer_sliding_window"] = text_config.sliding_window
|
||||
|
||||
attn_instance = attn_cls(**kwargs)
|
||||
if attn_cls is MLAAttention:
|
||||
# Attach MLA attn_instance to mla_module so it appears in
|
||||
# model.named_modules() and runs its process_weights_after_loading
|
||||
mla_module._vllm_mla_attn = attn_instance
|
||||
|
||||
# Attach the instance to the module it serves, which is the only way it
|
||||
# is in the module tree at all: a plain dict is invisible to
|
||||
# `nn.Module.__setattr__`. Being in the tree runs its
|
||||
# `process_weights_after_loading` and moves its KV-scale buffers with
|
||||
# `model.to(device)`, and lets the attention interface read it off the
|
||||
# module rather than index by `module.layer_idx`, an `int` that bakes
|
||||
# layer identity into the traced graph and so costs one compiled
|
||||
# artifact per layer.
|
||||
setattr(attn_module, VLLM_ATTN_ATTR, attn_instance)
|
||||
attention_instances[i] = attn_instance
|
||||
return attention_instances
|
||||
|
||||
@@ -645,7 +686,11 @@ class Base(
|
||||
return EncoderOnlyAttention
|
||||
if self.model_config.use_mla:
|
||||
self.check_version("5.15.0.dev0", "optimized MLA support")
|
||||
if any(isinstance(fuser, MLAFuser) for fuser in self.fusers.values()):
|
||||
if any(
|
||||
isinstance(fuser, MLAFuser)
|
||||
for fusers in self.fusers.values()
|
||||
for fuser in fusers
|
||||
):
|
||||
return MLAAttention
|
||||
logger.warning_once(
|
||||
"This model uses MLA but `MLAFuser` failed to match and/or fuse any "
|
||||
@@ -731,7 +776,6 @@ class Base(
|
||||
inputs_embeds=inputs_embeds,
|
||||
use_cache=False,
|
||||
position_ids=positions,
|
||||
attention_instances=self.attention_instances,
|
||||
return_dict=False,
|
||||
**self._output_aux_hidden_states_kwargs,
|
||||
**kwargs,
|
||||
|
||||
Reference in New Issue
Block a user