[XPU] Enable rms_norm/act quant fusions (#43963)
Signed-off-by: zhenwei-intel <zhenwei.liu@intel.com> Co-authored-by: Kunshang Ji <kunshang.ji@intel.com>
This commit is contained in:
co-authored by
Kunshang Ji
parent
478b49ddec
commit
3f3e2702c2
@@ -70,7 +70,11 @@ class ActivationQuantPattern(VllmPatternReplacement):
|
|||||||
self.silu_and_mul_matcher = MatcherSiluAndMul()
|
self.silu_and_mul_matcher = MatcherSiluAndMul()
|
||||||
|
|
||||||
def empty_quant(self, *args: Any, **kwargs: Any) -> torch.Tensor:
|
def empty_quant(self, *args: Any, **kwargs: Any) -> torch.Tensor:
|
||||||
kwargs = {"dtype": self.quant_dtype, "device": "cuda", **kwargs}
|
kwargs = {
|
||||||
|
"dtype": self.quant_dtype,
|
||||||
|
"device": current_platform.device_type,
|
||||||
|
**kwargs,
|
||||||
|
}
|
||||||
return torch.empty(*args, **kwargs)
|
return torch.empty(*args, **kwargs)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -54,19 +54,27 @@ def _rms_input_weight_dtype_match(match: pm.Match) -> bool:
|
|||||||
|
|
||||||
|
|
||||||
def empty_bf16(*args: Any, **kwargs: Any) -> torch.Tensor:
|
def empty_bf16(*args: Any, **kwargs: Any) -> torch.Tensor:
|
||||||
return torch.empty(*args, **kwargs, dtype=torch.bfloat16, device="cuda")
|
return torch.empty(
|
||||||
|
*args, **kwargs, dtype=torch.bfloat16, device=current_platform.device_type
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def empty_fp32(*args: Any, **kwargs: Any) -> torch.Tensor:
|
def empty_fp32(*args: Any, **kwargs: Any) -> torch.Tensor:
|
||||||
return torch.empty(*args, **kwargs, dtype=torch.float32, device="cuda")
|
return torch.empty(
|
||||||
|
*args, **kwargs, dtype=torch.float32, device=current_platform.device_type
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def empty_i32(*args: Any, **kwargs: Any) -> torch.Tensor:
|
def empty_i32(*args: Any, **kwargs: Any) -> torch.Tensor:
|
||||||
return torch.empty(*args, **kwargs, dtype=torch.int32, device="cuda")
|
return torch.empty(
|
||||||
|
*args, **kwargs, dtype=torch.int32, device=current_platform.device_type
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def empty_i64(*args: Any, **kwargs: Any) -> torch.Tensor:
|
def empty_i64(*args: Any, **kwargs: Any) -> torch.Tensor:
|
||||||
return torch.empty(*args, **kwargs, dtype=torch.int64, device="cuda")
|
return torch.empty(
|
||||||
|
*args, **kwargs, dtype=torch.int64, device=current_platform.device_type
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
RMS_ADD_OP = torch.ops._C.fused_add_rms_norm.default
|
RMS_ADD_OP = torch.ops._C.fused_add_rms_norm.default
|
||||||
|
|||||||
@@ -45,6 +45,10 @@ if current_platform.is_cuda():
|
|||||||
from .fusion.allreduce_rms_fusion import AllReduceFusionPass
|
from .fusion.allreduce_rms_fusion import AllReduceFusionPass
|
||||||
from .fusion.collective_fusion import AsyncTPPass
|
from .fusion.collective_fusion import AsyncTPPass
|
||||||
|
|
||||||
|
if current_platform.is_xpu():
|
||||||
|
from .fusion.act_quant_fusion import ActivationQuantFusionPass
|
||||||
|
from .fusion.rms_quant_fusion import RMSNormQuantFusionPass
|
||||||
|
|
||||||
from .inductor_pass import (
|
from .inductor_pass import (
|
||||||
CustomGraphPass,
|
CustomGraphPass,
|
||||||
InductorPass,
|
InductorPass,
|
||||||
|
|||||||
+10
-9
@@ -197,24 +197,25 @@ class XPUPlatform(Platform):
|
|||||||
)
|
)
|
||||||
|
|
||||||
# Disable fusion passes not yet supported on XPU.
|
# Disable fusion passes not yet supported on XPU.
|
||||||
|
from vllm.config.compilation import CompilationMode
|
||||||
|
|
||||||
pass_config = compilation_config.pass_config
|
pass_config = compilation_config.pass_config
|
||||||
fusion_passes_to_disable = {
|
fusion_passes_to_disable = {
|
||||||
"enable_sp": "Sequence parallelism",
|
"enable_sp": "Sequence parallelism",
|
||||||
"fuse_gemm_comms": "Async TP",
|
"fuse_gemm_comms": "Async TP",
|
||||||
"fuse_allreduce_rms": "AllReduce + RMSNorm fusion",
|
"fuse_allreduce_rms": "AllReduce + RMSNorm fusion",
|
||||||
"fuse_norm_quant": "RMSNorm + quant fusion",
|
|
||||||
"fuse_act_quant": "Activation + quant fusion",
|
|
||||||
"fuse_attn_quant": "Attention + quant fusion",
|
"fuse_attn_quant": "Attention + quant fusion",
|
||||||
"fuse_act_padding": "Activation + padding fusion",
|
"fuse_act_padding": "Activation + padding fusion",
|
||||||
"fuse_rope_kvcache": "RoPE + KV cache fusion",
|
"fuse_rope_kvcache": "RoPE + KV cache fusion",
|
||||||
}
|
}
|
||||||
for flag, feature_name in fusion_passes_to_disable.items():
|
if compilation_config.mode != CompilationMode.NONE:
|
||||||
if getattr(pass_config, flag):
|
for flag, feature_name in fusion_passes_to_disable.items():
|
||||||
logger.warning(
|
if getattr(pass_config, flag):
|
||||||
"Feature %r is not yet supported on XPU and will be disabled.",
|
logger.warning(
|
||||||
feature_name,
|
"Feature %r is not yet supported on XPU and will be disabled.",
|
||||||
)
|
feature_name,
|
||||||
setattr(pass_config, flag, False)
|
)
|
||||||
|
setattr(pass_config, flag, False)
|
||||||
|
|
||||||
# check and update parallel config
|
# check and update parallel config
|
||||||
parallel_config = vllm_config.parallel_config
|
parallel_config = vllm_config.parallel_config
|
||||||
|
|||||||
Reference in New Issue
Block a user