[Transformers backend] Find attention with a fuser and attach vLLM's layer to it (#54941)
Signed-off-by: Thomas Ortner <boh@zurich.ibm.com> Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
co-authored by
Harry Mellor
parent
99a1ab8e4c
commit
8ad2076c4a
@@ -36,6 +36,7 @@ from vllm.model_executor.layers.fused_moe import (
|
||||
)
|
||||
from vllm.model_executor.models.interfaces import MixtureOfExperts
|
||||
from vllm.model_executor.models.transformers.fuser import get_fuser
|
||||
from vllm.model_executor.models.transformers.fusers.glu import GLUFuser
|
||||
from vllm.model_executor.models.transformers.fusers.moe import MoEBlockFuser
|
||||
from vllm.model_executor.models.utils import extract_layer_index, maybe_prefix
|
||||
from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE, direct_register_custom_op
|
||||
@@ -319,8 +320,8 @@ class MoEMixin(MixtureOfExperts):
|
||||
# Store shared experts for later down projection adjustment
|
||||
if shared_experts is not None:
|
||||
hf_shared = shared_experts.shared_experts
|
||||
glu_fuser = get_fuser(hf_shared)
|
||||
down_name = getattr(glu_fuser, "down_name", None)
|
||||
glu_fuser = get_fuser(hf_shared, GLUFuser)
|
||||
down_name = glu_fuser and glu_fuser.down_name
|
||||
if down_name is not None:
|
||||
shared_down_projs.append((hf_shared, down_name))
|
||||
# Prefer config, otherwise read it from fuser.
|
||||
|
||||
Reference in New Issue
Block a user