[ROCm][Compile] Support BF16 AsyncTP fusion (#58098)

Signed-off-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
Andreas Karatzas
2026-09-22 20:36:38 -05:00
committed by GitHub
parent f9dce295c9
commit 0eb42dbbfc
9 changed files with 310 additions and 5 deletions
@@ -339,6 +339,8 @@ class BasePattern:
class GEMMReduceScatterPattern(BasePattern):
rocm_gemm = False
def get_inputs(self) -> list[torch.Tensor]:
mul = torch.empty([16, 4], device=self.device, dtype=self.dtype)
mm_weight = torch.empty([4, 4], device=self.device, dtype=self.dtype)
@@ -346,7 +348,10 @@ class GEMMReduceScatterPattern(BasePattern):
def register(self, pm_pass: PatternMatcherPass) -> None:
def pattern(mul: torch.Tensor, mm_weight: torch.Tensor) -> torch.Tensor:
mm = torch.ops.aten.mm.default(mul, mm_weight)
if self.rocm_gemm:
mm = torch.ops.vllm.rocm_unquantized_gemm(mul, mm_weight, None)
else:
mm = torch.ops.aten.mm.default(mul, mm_weight)
reduce_scatter = torch.ops.vllm.reduce_scatter.default(
mm,
dim=0,
@@ -358,7 +363,7 @@ class GEMMReduceScatterPattern(BasePattern):
def replacement(mul: torch.Tensor, mm_weight: torch.Tensor) -> torch.Tensor:
gemm_rs = torch.ops.symm_mem.fused_matmul_reduce_scatter(
mul,
mm_weight,
mm_weight.t() if self.rocm_gemm else mm_weight,
"sum",
scatter_dim=0,
group_name=self.tp.device_group.group_name,
@@ -371,7 +376,13 @@ class GEMMReduceScatterPattern(BasePattern):
)
class RocmGEMMReduceScatterPattern(GEMMReduceScatterPattern):
rocm_gemm = True
class AllGatherGEMMPattern(BasePattern):
rocm_gemm = False
def get_inputs(self) -> list[torch.Tensor]:
x = torch.empty([4, 4], device=self.device, dtype=self.dtype)
weight = torch.empty([4, 4], device=self.device, dtype=self.dtype)
@@ -390,12 +401,14 @@ class AllGatherGEMMPattern(BasePattern):
group_name=self.tp.unique_name,
)
if self.rocm_gemm:
return torch.ops.vllm.rocm_unquantized_gemm(all_gather, weight, None)
return torch.ops.aten.mm.default(all_gather, weight)
def replacement(x: torch.Tensor, weight: torch.Tensor) -> torch.Tensor:
ag_output, mm_outputs = torch.ops.symm_mem.fused_all_gather_matmul(
x,
[weight],
[weight.t() if self.rocm_gemm else weight],
gather_dim=0,
group_name=self.tp.device_group.group_name,
)
@@ -406,6 +419,10 @@ class AllGatherGEMMPattern(BasePattern):
)
class RocmAllGatherGEMMPattern(AllGatherGEMMPattern):
rocm_gemm = True
class ScaledMMReduceScatterPattern(BasePattern):
def get_inputs(self) -> list[torch.Tensor]:
input = torch.empty([16, 16], device=self.device, dtype=FP8_DTYPE)
@@ -906,6 +923,13 @@ class AsyncTPPass(VllmFusionPatternMatcherPass):
GEMMReduceScatterPattern(self.model_dtype, self.device).register(self.pm_pass)
AllGatherGEMMPattern(self.model_dtype, self.device).register(self.pm_pass)
if current_platform.is_rocm() and self.model_dtype == torch.bfloat16:
RocmGEMMReduceScatterPattern(self.model_dtype, self.device).register(
self.pm_pass
)
RocmAllGatherGEMMPattern(self.model_dtype, self.device).register(
self.pm_pass
)
# These fusions are enabled only for bfloat16 models because
# `scaled_mm` or `cutlass_scaled_mm` with per-token (row-wise) scaling
+1 -1
View File
@@ -41,6 +41,7 @@ if current_platform.is_cuda_alike() or current_platform.is_xpu():
if current_platform.is_cuda_alike():
from .fusion.act_quant_fusion import ActivationQuantFusionPass
from .fusion.attn_quant_fusion import AttnQuantFusionPass
from .fusion.collective_fusion import AsyncTPPass
from .fusion.mla_attn_quant_fusion import MLAAttnQuantFusionPass
from .fusion.mla_rope_kvcache_cat_fusion import MLARoPEKVCacheCatFusionPass
from .fusion.qk_norm_rope_kvcache_fusion import QkNormRopeKvCacheFusionPass
@@ -50,7 +51,6 @@ if current_platform.is_cuda_alike():
if current_platform.is_cuda():
from .fusion.allreduce_rms_fusion import AllReduceFusionPass
from .fusion.collective_fusion import AsyncTPPass
if current_platform.is_xpu():
from .fusion.act_quant_fusion import ActivationQuantFusionPass