[compile] Add FlashInfer FP8 async TP fusion and preserve allreduce fusion ordering #27893 (#39505)

Signed-off-by: baonudesifeizhai <baonudesifeizhai@gmail.com>
Signed-off-by: baonudesifeizhai <85092850+baonudesifeizhai@users.noreply.github.com>
Signed-off-by: roG0d <baonudesifeizhai@gmail.com>
This commit is contained in:
baonudesifeizhai
2026-05-01 05:08:34 +00:00
committed by GitHub
parent 947138b6c2
commit c3868bbbe4
6 changed files with 397 additions and 30 deletions
+33
View File
@@ -715,6 +715,38 @@ def flashinfer_scaled_fp8_mm(
return output
def flashinfer_scaled_fp8_mm_out(
a: torch.Tensor,
b: torch.Tensor,
scale_a: torch.Tensor,
scale_b: torch.Tensor,
out: torch.Tensor,
out_dtype: torch.dtype | None = None,
) -> torch.Tensor:
assert a.ndim == 2 and b.ndim == 2 and out.ndim == 2
assert a.shape[1] == b.shape[0]
assert out.shape == (a.shape[0], b.shape[1])
assert scale_a.numel() == 1 and scale_b.numel() == 1
assert a.dtype == torch.float8_e4m3fn and b.dtype == torch.float8_e4m3fn
assert out.device.type == "cuda"
assert a.is_contiguous()
from flashinfer import bmm_fp8 as bmm_fp8_
bmm_fp8_(
a.unsqueeze(0),
# FlashInfer expects the weight in the same column-major view layout
# consumed by flashinfer_scaled_fp8_mm, so keep the transposed view.
b.unsqueeze(0),
scale_a,
scale_b,
out_dtype or out.dtype,
out.unsqueeze(0),
"auto",
)
return out
def flashinfer_quant_nvfp4_8x4_sf_layout(
a: torch.Tensor, a_global_sf: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor]:
@@ -833,6 +865,7 @@ __all__ = [
"use_trtllm_attention",
"flashinfer_scaled_fp4_mm",
"flashinfer_scaled_fp8_mm",
"flashinfer_scaled_fp8_mm_out",
"flashinfer_quant_nvfp4_8x4_sf_layout",
"flashinfer_fp8_blockscale_gemm",
"should_use_flashinfer_for_blockscale_fp8_gemm",