[Transformers backend] Find attention with a fuser and attach vLLM's layer to it (#54941)

Signed-off-by: Thomas Ortner <boh@zurich.ibm.com>
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Thomas Ortner
2026-09-04 16:26:01 +00:00
committed by GitHub
co-authored by Harry Mellor
parent 99a1ab8e4c
commit 8ad2076c4a
14 changed files with 521 additions and 150 deletions
@@ -36,6 +36,7 @@ from vllm.model_executor.layers.fused_moe import (
)
from vllm.model_executor.models.interfaces import MixtureOfExperts
from vllm.model_executor.models.transformers.fuser import get_fuser
from vllm.model_executor.models.transformers.fusers.glu import GLUFuser
from vllm.model_executor.models.transformers.fusers.moe import MoEBlockFuser
from vllm.model_executor.models.utils import extract_layer_index, maybe_prefix
from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE, direct_register_custom_op
@@ -319,8 +320,8 @@ class MoEMixin(MixtureOfExperts):
# Store shared experts for later down projection adjustment
if shared_experts is not None:
hf_shared = shared_experts.shared_experts
glu_fuser = get_fuser(hf_shared)
down_name = getattr(glu_fuser, "down_name", None)
glu_fuser = get_fuser(hf_shared, GLUFuser)
down_name = glu_fuser and glu_fuser.down_name
if down_name is not None:
shared_down_projs.append((hf_shared, down_name))
# Prefer config, otherwise read it from fuser.