Add NVFP4 all-gather GEMM fusion for AsyncTP (#41882)

Signed-off-by: roG0d <baonudesifeizhai@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
baonudesifeizhai
2026-05-10 01:13:22 +00:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 006af4b956
commit bc5fdc1e6a
7 changed files with 605 additions and 6 deletions
+42
View File
@@ -685,6 +685,47 @@ def flashinfer_scaled_fp4_mm(
)
def flashinfer_scaled_fp4_mm_out(
a: torch.Tensor,
b: torch.Tensor,
block_scale_a: torch.Tensor,
block_scale_b: torch.Tensor,
alpha: torch.Tensor,
out: torch.Tensor,
out_dtype: torch.dtype | None,
use_8x4_sf_layout: bool,
backend: str,
) -> torch.Tensor:
assert a.ndim == 2 and b.ndim == 2 and out.ndim == 2
assert block_scale_a.ndim == 2 and block_scale_b.ndim == 2
assert a.stride(-1) == 1
assert a.shape[1] == b.shape[0]
assert out.shape == (a.shape[0], b.shape[1])
assert out.device.type == "cuda"
if backend in ("cutlass", "cudnn"):
if block_scale_a.dtype != torch.uint8:
block_scale_a = block_scale_a.view(torch.uint8)
if block_scale_b.dtype != torch.uint8:
block_scale_b = block_scale_b.view(torch.uint8)
from flashinfer import mm_fp4 as flashinfer_mm_fp4_
flashinfer_mm_fp4_(
a,
b,
block_scale_a,
block_scale_b,
alpha,
out_dtype or out.dtype,
out=out,
block_size=16,
use_8x4_sf_layout=use_8x4_sf_layout,
backend=backend,
)
return out
def flashinfer_scaled_fp8_mm(
a: torch.Tensor,
b: torch.Tensor,
@@ -864,6 +905,7 @@ __all__ = [
"can_use_trtllm_attention",
"use_trtllm_attention",
"flashinfer_scaled_fp4_mm",
"flashinfer_scaled_fp4_mm_out",
"flashinfer_scaled_fp8_mm",
"flashinfer_scaled_fp8_mm_out",
"flashinfer_quant_nvfp4_8x4_sf_layout",