Add NVFP4 all-gather GEMM fusion for AsyncTP (#41882)
Signed-off-by: roG0d <baonudesifeizhai@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
006af4b956
commit
bc5fdc1e6a
@@ -74,6 +74,36 @@ def _flashinfer_scaled_mm_out(
|
||||
)
|
||||
|
||||
|
||||
def _flashinfer_fp4_mm_out(
|
||||
A: torch.Tensor,
|
||||
B: torch.Tensor,
|
||||
*,
|
||||
scale_a: torch.Tensor,
|
||||
scale_b: torch.Tensor,
|
||||
out: torch.Tensor,
|
||||
alpha: torch.Tensor,
|
||||
out_dtype: torch.dtype | None = None,
|
||||
use_8x4_sf_layout: bool = False,
|
||||
backend: str = "cutlass",
|
||||
) -> None:
|
||||
from vllm.utils.flashinfer import flashinfer_scaled_fp4_mm_out
|
||||
|
||||
assert A.ndim == 2 and B.ndim == 2 and out.ndim == 2, (
|
||||
"FlashInfer FP4 symm_mem adapter expects 2D inputs and output"
|
||||
)
|
||||
flashinfer_scaled_fp4_mm_out(
|
||||
A,
|
||||
B,
|
||||
scale_a,
|
||||
scale_b,
|
||||
alpha,
|
||||
out=out,
|
||||
out_dtype=out_dtype or out.dtype,
|
||||
use_8x4_sf_layout=use_8x4_sf_layout,
|
||||
backend=backend,
|
||||
)
|
||||
|
||||
|
||||
def fused_flashinfer_scaled_matmul_reduce_scatter_fake(
|
||||
A: torch.Tensor,
|
||||
B: torch.Tensor,
|
||||
@@ -197,6 +227,90 @@ def fused_all_gather_flashinfer_scaled_matmul(
|
||||
return outputs[0]
|
||||
|
||||
|
||||
def fused_all_gather_flashinfer_fp4_matmul_fake(
|
||||
A_shard: torch.Tensor,
|
||||
B: torch.Tensor,
|
||||
A_scale_shard: torch.Tensor,
|
||||
B_scale: torch.Tensor,
|
||||
alpha: torch.Tensor,
|
||||
gather_dim: int,
|
||||
group_name: str,
|
||||
out_dtype: torch.dtype | None = None,
|
||||
view_a_scale_as_fp8: bool = False,
|
||||
use_8x4_sf_layout: bool = False,
|
||||
backend: str = "cutlass",
|
||||
) -> torch.Tensor:
|
||||
world_size = c10d._resolve_process_group(group_name).size()
|
||||
output_shape = list(A_shard.shape)
|
||||
output_shape[gather_dim] *= world_size
|
||||
output_shape[-1] = B.shape[1]
|
||||
return torch.empty(
|
||||
output_shape,
|
||||
dtype=out_dtype or torch.bfloat16,
|
||||
device=A_shard.device,
|
||||
)
|
||||
|
||||
|
||||
def fused_all_gather_flashinfer_fp4_matmul(
|
||||
A_shard: torch.Tensor,
|
||||
B: torch.Tensor,
|
||||
A_scale_shard: torch.Tensor,
|
||||
B_scale: torch.Tensor,
|
||||
alpha: torch.Tensor,
|
||||
gather_dim: int,
|
||||
group_name: str,
|
||||
out_dtype: torch.dtype | None = None,
|
||||
view_a_scale_as_fp8: bool = False,
|
||||
use_8x4_sf_layout: bool = False,
|
||||
backend: str = "cutlass",
|
||||
) -> torch.Tensor:
|
||||
assert gather_dim == 0, (
|
||||
"FlashInfer FP4 symm_mem adapter currently only supports gather_dim=0"
|
||||
)
|
||||
assert A_shard.ndim == 2 and A_scale_shard.ndim == 2 and B.ndim == 2, (
|
||||
"FlashInfer FP4 symm_mem adapter expects 2D inputs"
|
||||
)
|
||||
if view_a_scale_as_fp8:
|
||||
A_scale_shard = A_scale_shard.view(torch.float8_e4m3fn)
|
||||
|
||||
group = c10d._resolve_process_group(group_name)
|
||||
world_size = group.size()
|
||||
output = A_shard.new_empty(
|
||||
A_shard.shape[0] * world_size,
|
||||
B.shape[1],
|
||||
dtype=out_dtype or torch.bfloat16,
|
||||
)
|
||||
output_shards = output.chunk(world_size)
|
||||
|
||||
A = A_shard.new_empty(A_shard.shape[0] * world_size, A_shard.shape[1])
|
||||
A_scale = A_scale_shard.new_empty(
|
||||
A_scale_shard.shape[0] * world_size,
|
||||
A_scale_shard.shape[1],
|
||||
)
|
||||
|
||||
def fp4_shard_consumer(shards: list[torch.Tensor], rank: int) -> None:
|
||||
_flashinfer_fp4_mm_out(
|
||||
shards[0],
|
||||
B,
|
||||
scale_a=shards[1],
|
||||
scale_b=B_scale,
|
||||
alpha=alpha,
|
||||
out=output_shards[rank],
|
||||
out_dtype=out_dtype,
|
||||
use_8x4_sf_layout=use_8x4_sf_layout,
|
||||
backend=backend,
|
||||
)
|
||||
|
||||
torch.distributed._symmetric_memory._pipelined_multi_all_gather_and_consume(
|
||||
[A_shard, A_scale_shard],
|
||||
fp4_shard_consumer,
|
||||
[A, A_scale],
|
||||
group_name,
|
||||
False,
|
||||
)
|
||||
return output
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="fused_flashinfer_scaled_matmul_reduce_scatter",
|
||||
op_func=fused_flashinfer_scaled_matmul_reduce_scatter,
|
||||
@@ -209,6 +323,12 @@ direct_register_custom_op(
|
||||
fake_impl=fused_all_gather_flashinfer_scaled_matmul_fake,
|
||||
)
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="fused_all_gather_flashinfer_fp4_matmul",
|
||||
op_func=fused_all_gather_flashinfer_fp4_matmul,
|
||||
fake_impl=fused_all_gather_flashinfer_fp4_matmul_fake,
|
||||
)
|
||||
|
||||
|
||||
class BasePattern:
|
||||
def __init__(self, dtype: torch.dtype, device: str | None) -> None:
|
||||
@@ -682,6 +802,101 @@ class FlashInferAllGatherBMMFP8Pattern(
|
||||
return _replacement
|
||||
|
||||
|
||||
class FlashInferAllGatherFP4Pattern(
|
||||
BasePattern, VllmPatternReplacement[..., torch.Tensor]
|
||||
):
|
||||
def __init__(
|
||||
self,
|
||||
dtype: torch.dtype,
|
||||
device: str | None,
|
||||
backend: str,
|
||||
use_8x4_sf_layout: bool,
|
||||
a_scale_view: str,
|
||||
) -> None:
|
||||
super().__init__(dtype, device)
|
||||
self.backend = backend
|
||||
self.use_8x4_sf_layout = use_8x4_sf_layout
|
||||
self.a_scale_view = a_scale_view
|
||||
|
||||
def get_inputs(self) -> list[torch.Tensor]:
|
||||
a_shard_2d = torch.empty([8, 8], device=self.device, dtype=torch.uint8)
|
||||
b_2d = torch.empty([8, 16], device=self.device, dtype=torch.uint8)
|
||||
a_scale_shard = torch.empty([128, 4], device=self.device, dtype=torch.int32)
|
||||
b_scale = torch.empty([4, 128], device=self.device, dtype=torch.uint8)
|
||||
alpha = torch.empty([], device=self.device, dtype=torch.float32)
|
||||
return [
|
||||
a_shard_2d,
|
||||
b_2d,
|
||||
a_scale_shard,
|
||||
b_scale,
|
||||
alpha,
|
||||
]
|
||||
|
||||
@property
|
||||
def pattern(self) -> Callable[..., torch.Tensor]:
|
||||
def _pattern(
|
||||
a_shard_2d: torch.Tensor,
|
||||
b_2d: torch.Tensor,
|
||||
a_scale_shard: torch.Tensor,
|
||||
b_scale: torch.Tensor,
|
||||
alpha: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
all_gather_a = torch.ops.vllm.all_gather.default(
|
||||
a_shard_2d,
|
||||
dim=0,
|
||||
world_size=self.tp_size,
|
||||
group_name=self.tp.unique_name,
|
||||
)
|
||||
all_gather_a_scale = torch.ops.vllm.all_gather.default(
|
||||
a_scale_shard,
|
||||
dim=0,
|
||||
world_size=self.tp_size,
|
||||
group_name=self.tp.unique_name,
|
||||
)
|
||||
a_scale = all_gather_a_scale
|
||||
if self.a_scale_view in ("float8", "float8_uint8"):
|
||||
a_scale = torch.ops.aten.view.dtype(a_scale, torch.float8_e4m3fn)
|
||||
if self.a_scale_view in ("uint8", "float8_uint8"):
|
||||
a_scale = torch.ops.aten.view.dtype(a_scale, torch.uint8)
|
||||
return torch.ops.vllm.flashinfer_mm_fp4.default(
|
||||
all_gather_a,
|
||||
b_2d,
|
||||
a_scale,
|
||||
b_scale,
|
||||
alpha,
|
||||
self.dtype,
|
||||
self.use_8x4_sf_layout,
|
||||
self.backend,
|
||||
)
|
||||
|
||||
return _pattern
|
||||
|
||||
@property
|
||||
def replacement(self) -> Callable[..., torch.Tensor]:
|
||||
def _replacement(
|
||||
a_shard_2d: torch.Tensor,
|
||||
b_2d: torch.Tensor,
|
||||
a_scale_shard: torch.Tensor,
|
||||
b_scale: torch.Tensor,
|
||||
alpha: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
return torch.ops.vllm.fused_all_gather_flashinfer_fp4_matmul.default(
|
||||
a_shard_2d,
|
||||
b_2d,
|
||||
a_scale_shard,
|
||||
b_scale,
|
||||
alpha,
|
||||
0,
|
||||
self.tp.device_group.group_name,
|
||||
self.dtype,
|
||||
self.a_scale_view in ("float8", "float8_uint8"),
|
||||
self.use_8x4_sf_layout,
|
||||
self.backend,
|
||||
)
|
||||
|
||||
return _replacement
|
||||
|
||||
|
||||
class AsyncTPPass(VllmFusionPatternMatcherPass):
|
||||
@enable_fake_mode
|
||||
def __init__(self, config: VllmConfig) -> None:
|
||||
@@ -718,6 +933,34 @@ class AsyncTPPass(VllmFusionPatternMatcherPass):
|
||||
self.register(
|
||||
FlashInferBMMFP8ReduceScatterPattern(self.model_dtype, self.device)
|
||||
)
|
||||
if hasattr(torch.ops.vllm, "flashinfer_mm_fp4"):
|
||||
for backend in ("cutlass", "cudnn"):
|
||||
for a_scale_view in ("float8_uint8", "uint8"):
|
||||
self.register(
|
||||
FlashInferAllGatherFP4Pattern(
|
||||
self.model_dtype,
|
||||
self.device,
|
||||
backend,
|
||||
use_8x4_sf_layout=False,
|
||||
a_scale_view=a_scale_view,
|
||||
)
|
||||
)
|
||||
for use_8x4_sf_layout in (False, True):
|
||||
for a_scale_view in ("float8",):
|
||||
self.register(
|
||||
FlashInferAllGatherFP4Pattern(
|
||||
self.model_dtype,
|
||||
self.device,
|
||||
"trtllm",
|
||||
use_8x4_sf_layout=use_8x4_sf_layout,
|
||||
a_scale_view=a_scale_view,
|
||||
)
|
||||
)
|
||||
# NVFP4 reduce-scatter does not need scale communication: FP4
|
||||
# scales are consumed by the local GEMM and only BF16 partial
|
||||
# outputs are reduced. Keep this PR scoped to the all-gather
|
||||
# path; reduce-scatter needs a dedicated FP4 producer rather
|
||||
# than the existing FP8-style helper.
|
||||
|
||||
self.dump_patterns(config, self.pm_pass)
|
||||
|
||||
|
||||
@@ -8,6 +8,7 @@ from typing import Any
|
||||
import torch
|
||||
import torch._inductor.pattern_matcher as pm
|
||||
import torch.fx as fx
|
||||
from torch._higher_order_ops.auto_functionalize import auto_functionalized
|
||||
from torch._inductor.pattern_matcher import PatternMatcherPass
|
||||
|
||||
import vllm.ir.ops
|
||||
@@ -27,6 +28,10 @@ from .matcher_utils import MatcherQuantFP8
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
if hasattr(torch.ops._C, "scaled_fp4_quant"):
|
||||
SCALED_FP4_QUANT_OUT_OVERLOAD = torch.ops._C.scaled_fp4_quant.out
|
||||
SCALED_FP4_QUANT_DEFAULT_OVERLOAD = torch.ops._C.scaled_fp4_quant.default
|
||||
|
||||
# Min hidden size per device capability for sequence parallelism
|
||||
# Only apply sequence parallelism for models with hidden_size >= threshold
|
||||
SP_MIN_HIDDEN_SIZE: dict[int, int] = {
|
||||
@@ -332,6 +337,129 @@ class MiddleAllReduceRMSNormStaticFP8Pattern(_SequenceParallelPatternHelper):
|
||||
)
|
||||
|
||||
|
||||
class FirstAllReduceRMSNormStaticNVFP4Pattern(_SequenceParallelPatternHelper):
|
||||
def get_inputs(self) -> list[torch.Tensor]:
|
||||
input = self.empty([8, 16])
|
||||
weight = self.empty([16])
|
||||
input_global_scale = self.empty_f32([1, 1])
|
||||
quant_output = torch.empty([8, 8], device=self.device, dtype=torch.uint8)
|
||||
output_scale = torch.empty([128, 4], device=self.device, dtype=torch.int32)
|
||||
return [input, weight, input_global_scale, quant_output, output_scale]
|
||||
|
||||
def register(self, pm_pass: PatternMatcherPass) -> None:
|
||||
def pattern(
|
||||
input: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
input_global_scale: torch.Tensor,
|
||||
quant_output: torch.Tensor,
|
||||
output_scale: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
all_reduce = self._all_reduce(input)
|
||||
rms = vllm.ir.ops.rms_norm(all_reduce, weight, self.epsilon)
|
||||
quant = auto_functionalized(
|
||||
SCALED_FP4_QUANT_OUT_OVERLOAD,
|
||||
input=rms,
|
||||
input_scale=input_global_scale,
|
||||
is_sf_swizzled_layout=True,
|
||||
output=quant_output,
|
||||
output_scale=output_scale,
|
||||
)
|
||||
return quant[1], all_reduce, quant[2]
|
||||
|
||||
def replacement(
|
||||
input: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
input_global_scale: torch.Tensor,
|
||||
quant_output: torch.Tensor,
|
||||
output_scale: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
reduce_scatter = self._reduce_scatter(input)
|
||||
rms = vllm.ir.ops.rms_norm(reduce_scatter, weight, self.epsilon)
|
||||
rms = torch.ops.aten.view.default(rms, [-1, rms.shape[-1]])
|
||||
quant = SCALED_FP4_QUANT_DEFAULT_OVERLOAD(
|
||||
rms,
|
||||
input_global_scale,
|
||||
True,
|
||||
)
|
||||
return (
|
||||
self._all_gather(quant[0]),
|
||||
reduce_scatter,
|
||||
self._all_gather(quant[1]),
|
||||
)
|
||||
|
||||
pm.register_replacement(
|
||||
pattern, replacement, self.get_inputs(), pm.fwd_only, pm_pass
|
||||
)
|
||||
|
||||
|
||||
class MiddleAllReduceRMSNormStaticNVFP4Pattern(_SequenceParallelPatternHelper):
|
||||
def get_inputs(self) -> list[torch.Tensor]:
|
||||
mm_1 = self.empty([8, 16])
|
||||
residual = self.empty([8, 16])
|
||||
rms_norm_weights = self.empty([16])
|
||||
input_global_scale = self.empty_f32([1, 1])
|
||||
quant_output = torch.empty([8, 8], device=self.device, dtype=torch.uint8)
|
||||
output_scale = torch.empty([128, 4], device=self.device, dtype=torch.int32)
|
||||
return [
|
||||
residual,
|
||||
mm_1,
|
||||
rms_norm_weights,
|
||||
input_global_scale,
|
||||
quant_output,
|
||||
output_scale,
|
||||
]
|
||||
|
||||
def register(self, pm_pass: PatternMatcherPass) -> None:
|
||||
def pattern(
|
||||
residual: torch.Tensor,
|
||||
mm_1: torch.Tensor,
|
||||
rms_norm_weights: torch.Tensor,
|
||||
input_global_scale: torch.Tensor,
|
||||
quant_output: torch.Tensor,
|
||||
output_scale: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
all_reduce = self._all_reduce(mm_1)
|
||||
rms, residual_out = vllm.ir.ops.fused_add_rms_norm(
|
||||
all_reduce, residual, rms_norm_weights, self.epsilon
|
||||
)
|
||||
quant = auto_functionalized(
|
||||
SCALED_FP4_QUANT_OUT_OVERLOAD,
|
||||
input=rms,
|
||||
input_scale=input_global_scale,
|
||||
is_sf_swizzled_layout=True,
|
||||
output=quant_output,
|
||||
output_scale=output_scale,
|
||||
)
|
||||
return quant[1], residual_out, quant[2]
|
||||
|
||||
def replacement(
|
||||
residual: torch.Tensor,
|
||||
mm_1: torch.Tensor,
|
||||
rms_norm_weights: torch.Tensor,
|
||||
input_global_scale: torch.Tensor,
|
||||
quant_output: torch.Tensor,
|
||||
output_scale: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
# Keep this slice in sync with the non-quantized SP replacement:
|
||||
# once the previous SP pattern fires, it becomes a no-op.
|
||||
reduce_scatter = self._reduce_scatter(mm_1)
|
||||
residual = residual[0 : reduce_scatter.size(0), ...]
|
||||
rms, residual_out = vllm.ir.ops.fused_add_rms_norm(
|
||||
reduce_scatter, residual, rms_norm_weights, self.epsilon
|
||||
)
|
||||
rms = torch.ops.aten.view.default(rms, [-1, rms.shape[-1]])
|
||||
quant = SCALED_FP4_QUANT_DEFAULT_OVERLOAD(
|
||||
rms,
|
||||
input_global_scale,
|
||||
True,
|
||||
)
|
||||
return self._all_gather(quant[0]), residual_out, self._all_gather(quant[1])
|
||||
|
||||
pm.register_replacement(
|
||||
pattern, replacement, self.get_inputs(), pm.fwd_only, pm_pass
|
||||
)
|
||||
|
||||
|
||||
class SequenceParallelismPass(VllmPatternMatcherPass):
|
||||
"""
|
||||
This pass enables sequence parallelism for models.
|
||||
@@ -404,6 +532,14 @@ class SequenceParallelismPass(VllmPatternMatcherPass):
|
||||
epsilon, self.model_dtype, self.device
|
||||
).register(self.patterns)
|
||||
|
||||
if "SCALED_FP4_QUANT_OUT_OVERLOAD" in globals():
|
||||
FirstAllReduceRMSNormStaticNVFP4Pattern(
|
||||
epsilon, self.model_dtype, self.device
|
||||
).register(self.patterns)
|
||||
MiddleAllReduceRMSNormStaticNVFP4Pattern(
|
||||
epsilon, self.model_dtype, self.device
|
||||
).register(self.patterns)
|
||||
|
||||
# Normal RMSNorm patterns
|
||||
FirstAllReduceRMSNormPattern(
|
||||
epsilon, self.model_dtype, self.device
|
||||
|
||||
Reference in New Issue
Block a user