[Feature] Add async tensor parallelism for scaled mm (#20155)

Signed-off-by: cascade812 <cascade812@outlook.com>
This commit is contained in:
cascade
2025-07-30 17:23:41 -04:00
committed by GitHub
parent f12d9256b3
commit 287f527f54
3 changed files with 381 additions and 8 deletions
+242 -2
View File
@@ -15,10 +15,13 @@ from vllm.distributed import get_tp_group, tensor_model_parallel_all_reduce
from vllm.distributed.parallel_state import (
get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
from vllm.logger import init_logger
from vllm.platforms import current_platform
from vllm.utils import direct_register_custom_op
from .vllm_inductor_pass import VllmInductorPass
FP8_DTYPE = current_platform.fp8_dtype()
if find_spec("flashinfer"):
try:
import flashinfer.comm as flashinfer_comm
@@ -28,7 +31,6 @@ if find_spec("flashinfer"):
flashinfer_comm = None
else:
flashinfer_comm = None
from vllm.platforms import current_platform
logger = init_logger(__name__)
@@ -118,6 +120,230 @@ class AllGatherGEMMPattern(BasePattern):
pm.fwd_only, pm_pass)
class ScaledMMReduceScatterPattern(BasePattern):
def get_inputs(self):
input = torch.empty([16, 16], device=self.device, dtype=FP8_DTYPE)
mm_weight = torch.empty([16, 16], device=self.device,
dtype=FP8_DTYPE).contiguous().transpose(0, 1)
scale_a = torch.empty([16, 1], device=self.device, dtype=torch.float32)
scale_b = torch.empty([1, 16], device=self.device, dtype=torch.float32)
return [input, mm_weight, scale_a, scale_b]
def register(self, pm_pass: PatternMatcherPass):
def pattern(input: torch.Tensor, mat2: torch.Tensor,
scale_a: torch.Tensor,
scale_b: torch.Tensor) -> torch.Tensor:
scaled_mm = torch.ops.aten._scaled_mm.default(input,
mat2=mat2,
scale_a=scale_a,
scale_b=scale_b,
bias=None,
scale_result=None,
out_dtype=self.dtype)
reduce_scatter = torch.ops.vllm.reduce_scatter.default(
scaled_mm,
dim=0,
world_size=self.tp_size,
group_name=self.tp.unique_name)
return reduce_scatter
def replacement(input: torch.Tensor, mat2: torch.Tensor,
scale_a: torch.Tensor,
scale_b: torch.Tensor) -> torch.Tensor:
gemm_rs = torch.ops.symm_mem.fused_scaled_matmul_reduce_scatter(
input,
mat2,
scale_a,
scale_b,
"avg",
scatter_dim=0,
out_dtype=self.dtype,
group_name=self.tp.device_group.group_name,
)
return gemm_rs
pm.register_replacement(pattern, replacement, self.get_inputs(),
pm.fwd_only, pm_pass)
class AllGatherScaledMMPattern(BasePattern):
def get_inputs(self):
x = torch.empty([8, 16], device=self.device, dtype=FP8_DTYPE)
weight = torch.empty([16, 16], device=self.device,
dtype=FP8_DTYPE).contiguous().transpose(0, 1)
s1 = x.shape[0] * self.tp_size
scale_a = torch.empty([s1, 1], device=self.device, dtype=torch.float32)
scale_b = torch.empty([1, 16], device=self.device, dtype=torch.float32)
return [x, weight, scale_a, scale_b]
def register(self, pm_pass: PatternMatcherPass):
def pattern(
x: torch.Tensor,
weight: torch.Tensor,
scale_a: torch.Tensor,
scale_b: torch.Tensor,
) -> torch.Tensor:
all_gather = torch.ops.vllm.all_gather.default(
x,
dim=0,
world_size=self.tp_size,
group_name=self.tp.unique_name)
return torch.ops.aten._scaled_mm.default(all_gather,
mat2=weight,
scale_a=scale_a,
scale_b=scale_b,
bias=None,
scale_result=None,
out_dtype=self.dtype)
def replacement(x: torch.Tensor, weight: torch.Tensor,
scale_a: torch.Tensor,
scale_b: torch.Tensor) -> torch.Tensor:
ag_output, mm_outputs = torch.ops.symm_mem.fused_all_gather_scaled_matmul( # noqa
x,
[weight],
scale_a,
[scale_b],
gather_dim=0,
biases=[None],
result_scales=[None],
out_dtypes=[self.dtype],
use_fast_accum=[False],
group_name=self.tp.device_group.group_name,
)
return mm_outputs
pm.register_replacement(pattern, replacement, self.get_inputs(),
pm.fwd_only, pm_pass)
class CutlassScaledMMReduceScatterPattern(BasePattern):
def get_inputs(self):
input = torch.empty([16, 16], device=self.device, dtype=FP8_DTYPE)
mm_weight = torch.empty([16, 16], device=self.device,
dtype=FP8_DTYPE).contiguous().transpose(0, 1)
scale_a = torch.empty([16, 1], device=self.device, dtype=torch.float32)
scale_b = torch.empty([1, 16], device=self.device, dtype=torch.float32)
cutlass_mm_output = torch.empty([16, 16],
device=self.device,
dtype=self.dtype)
return [input, mm_weight, scale_a, scale_b, cutlass_mm_output]
def register(self, pm_pass: PatternMatcherPass):
def pattern(input: torch.Tensor, weight: torch.Tensor,
scale_a: torch.Tensor, scale_b: torch.Tensor,
cutlass_mm_output: torch.Tensor) -> torch.Tensor:
cutlass_scaled_mm = torch.ops.higher_order.auto_functionalized(
torch.ops._C.cutlass_scaled_mm.default,
out=cutlass_mm_output,
a=input,
b=weight,
a_scales=scale_a,
b_scales=scale_b,
bias=None)
reduce_scatter = torch.ops.vllm.reduce_scatter.default(
cutlass_scaled_mm[1],
dim=0,
world_size=self.tp_size,
group_name=self.tp.unique_name)
return reduce_scatter
def replacement(input: torch.Tensor, mat2: torch.Tensor,
scale_a: torch.Tensor, scale_b: torch.Tensor,
cutlass_mm_output: torch.Tensor) -> torch.Tensor:
gemm_rs = torch.ops.symm_mem.fused_scaled_matmul_reduce_scatter(
input,
mat2,
scale_a,
scale_b,
"avg",
scatter_dim=0,
out_dtype=self.dtype,
group_name=self.tp.device_group.group_name,
)
return gemm_rs
pm.register_replacement(pattern, replacement, self.get_inputs(),
pm.fwd_only, pm_pass)
class AllGatherCutlassScaledMMPattern(BasePattern):
def get_inputs(self):
x = torch.empty([8, 16], device=self.device, dtype=FP8_DTYPE)
weight = torch.empty([16, 16], device=self.device,
dtype=FP8_DTYPE).contiguous().transpose(0, 1)
s1 = x.shape[0] * self.tp_size
scale_a = torch.empty([s1, 1], device=self.device, dtype=torch.float32)
scale_b = torch.empty([1, 16], device=self.device, dtype=torch.float32)
s2 = weight.shape[1]
output = torch.empty([s1, s2], device=self.device, dtype=self.dtype)
return [x, weight, scale_a, scale_b, output]
def register(self, pm_pass: PatternMatcherPass):
def pattern(
x: torch.Tensor,
weight: torch.Tensor,
scale_a: torch.Tensor,
scale_b: torch.Tensor,
output: torch.Tensor,
) -> torch.Tensor:
all_gather = torch.ops.vllm.all_gather.default(
x,
dim=0,
world_size=self.tp_size,
group_name=self.tp.unique_name)
cutlass_scaled_mm = torch.ops.higher_order.auto_functionalized(
torch.ops._C.cutlass_scaled_mm.default,
out=output,
a=all_gather,
b=weight,
a_scales=scale_a,
b_scales=scale_b,
bias=None)
return cutlass_scaled_mm[1]
def replacement(x: torch.Tensor, weight: torch.Tensor,
scale_a: torch.Tensor, scale_b: torch.Tensor,
output: torch.Tensor) -> torch.Tensor:
ag_output, mm_outputs = torch.ops.symm_mem.fused_all_gather_scaled_matmul( # noqa
x,
[weight],
scale_a,
[scale_b],
gather_dim=0,
biases=[None],
result_scales=[None],
out_dtypes=[self.dtype],
use_fast_accum=[False],
group_name=self.tp.device_group.group_name,
)
return mm_outputs
pm.register_replacement(pattern, replacement, self.get_inputs(),
pm.fwd_only, pm_pass)
class AsyncTPPass(VllmInductorPass):
def __init__(self, config: VllmConfig):
@@ -133,6 +359,20 @@ class AsyncTPPass(VllmInductorPass):
AllGatherGEMMPattern(self.model_dtype,
self.device).register(self.patterns)
# These fusions are enabled only for bfloat16 models because
# `scaled_mm` or `cutlass_scaled_mm` with per-token (row-wise) scaling
# only supports bfloat16 as the output dtype.
if self.model_dtype == torch.bfloat16:
ScaledMMReduceScatterPattern(self.model_dtype,
self.device).register(self.patterns)
AllGatherScaledMMPattern(self.model_dtype,
self.device).register(self.patterns)
CutlassScaledMMReduceScatterPattern(
self.model_dtype, self.device).register(self.patterns)
AllGatherCutlassScaledMMPattern(
self.model_dtype, self.device).register(self.patterns)
def is_applicable_for_shape(self, shape: Optional[int]) -> bool:
# only do replace for specific shapes
tp_size = get_tensor_model_parallel_world_size()
@@ -142,7 +382,7 @@ class AsyncTPPass(VllmInductorPass):
self.begin()
self.dump_graph(graph, "before_async_tp_pass")
count = self.patterns.apply(graph)
logger.debug("Replaced %s patterns", count)
logger.debug("Replaced %s patterns with async TP pass.", count)
self.dump_graph(graph, "after_async_tp_pass")
self.end_and_log()
+1 -1
View File
@@ -477,6 +477,6 @@ class SequenceParallelismPass(VllmInductorPass):
self.begin()
self.dump_graph(graph, "before_sequence_parallelism_pass")
count = self.patterns.apply(graph)
logger.debug("Replaced %s patterns", count)
logger.debug("Replaced %s patterns with sequence parallelism", count)
self.dump_graph(graph, "after_sequence_parallelism_pass")
self.end_and_log()