[DSV4] Add BF16 and MXFP8 A2A support for flashinfer a2a one sided (#40960)

Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
Signed-off-by: Zijing Liu <liuzijing2014@gmail.com>
Co-authored-by: Zijing Liu <liuzijing2014@users.noreply.github.com>
Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Yongye Zhu
2026-04-30 15:33:12 -07:00
committed by GitHub
co-authored by Zijing Liu Claude Opus 4.7
parent 526927be94
commit b4806c8ee1
12 changed files with 107 additions and 68 deletions
@@ -85,7 +85,9 @@ def _mxfp8_e4m3_quantize_torch(
def _mxfp8_e4m3_quantize_impl(
x: torch.Tensor, is_sf_swizzled_layout: bool = False
x: torch.Tensor,
is_sf_swizzled_layout: bool = False,
alignment: int = 0,
) -> tuple[torch.Tensor, torch.Tensor]:
from vllm.platforms import current_platform
@@ -93,7 +95,9 @@ def _mxfp8_e4m3_quantize_impl(
from flashinfer import mxfp8_quantize as flashinfer_mxfp8_quantize
x_q, x_scales = flashinfer_mxfp8_quantize(
x, is_sf_swizzled_layout=is_sf_swizzled_layout
x,
is_sf_swizzled_layout=is_sf_swizzled_layout,
alignment=alignment if alignment > 0 else 32,
)
if x_scales.ndim == 1 and x.ndim == 2 and not is_sf_swizzled_layout:
x_scales = x_scales.view(x.size(0), -1)
@@ -103,9 +107,11 @@ def _mxfp8_e4m3_quantize_impl(
def mxfp8_e4m3_quantize(
x: torch.Tensor, is_sf_swizzled_layout: bool = False
x: torch.Tensor,
is_sf_swizzled_layout: bool = False,
alignment: int = 0,
) -> tuple[torch.Tensor, torch.Tensor]:
return torch.ops.vllm.mxfp8_quantize(x, is_sf_swizzled_layout)
return torch.ops.vllm.mxfp8_quantize(x, is_sf_swizzled_layout, alignment)
def dequant_mxfp8_to_bf16(x: torch.Tensor, scales: torch.Tensor) -> torch.Tensor:
@@ -125,7 +131,9 @@ def dequant_mxfp8_to_bf16(x: torch.Tensor, scales: torch.Tensor) -> torch.Tensor
def mxfp8_e4m3_quantize_fake(
x: torch.Tensor, is_sf_swizzled_layout: bool = False
x: torch.Tensor,
is_sf_swizzled_layout: bool = False,
alignment: int = 0,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Fake implementation for torch.compile tracing."""
fp_data = torch.empty_like(x, dtype=MXFP8_VALUE_DTYPE)