Fix MQA with tensor parallelism on transformers modeling backend (#49987)
Signed-off-by: microslaw <milosz.grunwald@intel.com> Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
co-authored by
Harry Mellor
parent
28158b2fc3
commit
ebcef33766
@@ -18,9 +18,10 @@ from vllm.logger import init_logger
|
||||
from vllm.model_executor.models.transformers.fusers import (
|
||||
BaseFuser,
|
||||
GLUFuser,
|
||||
PackedQKVFuser,
|
||||
QKVFuser,
|
||||
RewriteFuser,
|
||||
RMSNormFuser,
|
||||
StackedFuser,
|
||||
)
|
||||
from vllm.model_executor.models.transformers.fx_utils import trace
|
||||
|
||||
@@ -47,9 +48,9 @@ def get_fuser(module: nn.Module) -> BaseFuser | None:
|
||||
return None
|
||||
if (graph := trace(module)) is None:
|
||||
return None
|
||||
for fuser_cls in (GLUFuser, QKVFuser, RMSNormFuser):
|
||||
for fuser_cls in (GLUFuser, QKVFuser, PackedQKVFuser, RMSNormFuser):
|
||||
if (fuser := fuser_cls.match(graph, module)) is not None:
|
||||
if isinstance(fuser, StackedFuser):
|
||||
if isinstance(fuser, RewriteFuser):
|
||||
try:
|
||||
fuser.update_forward(module)
|
||||
except Exception as exc:
|
||||
|
||||
Reference in New Issue
Block a user