[ROCm][Perf] MXFP8 GEMM on native 32x32 block scales for gfx950 (#58510)

Signed-off-by: fai <fangzhouai@gmail.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Fangzhou Ai
2026-09-24 21:26:30 -05:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 055f150df5
commit 3059155b47
5 changed files with 899 additions and 7 deletions
@@ -117,6 +117,147 @@ def test_rocm_mxfp8_linear_handles_extreme_magnitudes():
assert (out[0] == 0).all(), "an all-zero activation row must give a zero row"
# (name, N, K) of the DeepSeek-V4.1-Flash MXFP8 linears at TP=2 and TP=4. The
# checkpoint stores their scales in 32x32 blocks; shared_down at TP=4 has
# K % 128 != 0, which only the block-scaled GEMM runs natively.
V41_BLOCK32_SHAPES = [
("fused_wqa_wkv", 1280 + 512, 5120),
("wq_b_tp2", 64 * 512 // 2, 1280),
("indexer_wq_b", 32 * 128, 1280),
("wo_b_tp2", 5120, 8192 // 2),
("shared_gate_up_tp2", 2 * 2304 // 2, 5120),
("shared_down_tp2", 5120, 2304 // 2),
("shared_down_tp4", 5120, 2304 // 4),
]
def _make_block32_layer(n: int, k: int, device: str) -> torch.nn.Module:
"""An MXFP8 layer with 32x32 block scales, expanded per row as loaded."""
blocks = (torch.randn(n, k, device=device) * 0.05).view(n // 32, 32, k // 32, 32)
amax = blocks.abs().amax(dim=(1, 3)).clamp(min=torch.finfo(torch.float32).tiny)
bits = (torch.ceil(torch.log2(amax / 448.0)) + 127.0).clamp(0, 254)
weight = (blocks * torch.exp2(127.0 - bits)[:, None, :, None]).view(n, k)
layer = torch.nn.Module()
layer.weight = torch.nn.Parameter(
weight.to(torch.float8_e4m3fn), requires_grad=False
)
layer.weight_scale = torch.nn.Parameter(
bits.to(torch.uint8).repeat_interleave(32, dim=0), requires_grad=False
)
return layer
@pytest.mark.parametrize("name,n,k", V41_BLOCK32_SHAPES, ids=lambda v: str(v))
@pytest.mark.parametrize("num_tokens", [1, 7, 32, 33, 192, 1024])
@torch.inference_mode()
def test_rocm_mxfp8_linear_block32_matches_dequant_reference(name, n, k, num_tokens):
"""32x32 block scales stay compact and the block-scaled GEMM is exact.
The token counts cover the packed, tiled and split-K configurations.
"""
from vllm.model_executor.kernels.linear.mxfp8.Mxfp8LinearKernel import (
Mxfp8LinearLayerConfig,
)
from vllm.model_executor.kernels.linear.mxfp8.rocm_native import (
RocmDotScaledMxfp8LinearKernel,
)
torch.manual_seed(num_tokens)
device = "cuda"
layer = _make_block32_layer(n, k, device)
row_scale = layer.weight_scale.data.clone()
kernel = RocmDotScaledMxfp8LinearKernel(Mxfp8LinearLayerConfig(bmm_batch_size=None))
kernel.process_weights_after_loading(layer)
assert layer.weight.dtype == torch.float8_e4m3fn
assert layer.weight_scale.shape == (n // 32, k // 32)
x = torch.randn(num_tokens, k, device=device, dtype=torch.bfloat16) * 0.5
out = kernel.apply_weights(layer, x)
x_q, x_scale = mxfp8_e4m3_quantize(x)
expected = (
dequant_mxfp8_to_bf16(x_q, x_scale).float()
@ dequant_mxfp8_to_bf16(layer.weight, row_scale).float().T
)
assert out.shape == (num_tokens, n)
rel = (out.float() - expected).norm() / expected.norm()
assert rel < 5e-3, f"{name}: relative error {rel:.4f}"
@torch.inference_mode()
def test_rocm_mxfp8_linear_keeps_per_row_scales():
"""Scales that differ within a 32-row block are not compacted."""
from vllm.model_executor.kernels.linear.mxfp8.Mxfp8LinearKernel import (
Mxfp8LinearLayerConfig,
)
from vllm.model_executor.kernels.linear.mxfp8.rocm_native import (
RocmDotScaledMxfp8LinearKernel,
)
layer = _make_block32_layer(64, 256, "cuda")
layer.weight_scale.data[1, 0] += 1
kernel = RocmDotScaledMxfp8LinearKernel(Mxfp8LinearLayerConfig(bmm_batch_size=None))
kernel.process_weights_after_loading(layer)
assert layer.weight_scale.shape == (64, 256 // 32)
def _split_k_case(packed: bool):
"""A 3-way in-launch split-K on shared_down TP4, with K and M tails."""
from vllm.model_executor.kernels.linear.mxfp8 import rocm_block32_gemm as g
n, k, m = 5120, 576, 19
layer = _make_block32_layer(n, k, "cuda")
x_q, x_scale = mxfp8_e4m3_quantize(
torch.randn(m, k, device="cuda", dtype=torch.bfloat16)
)
expected = (
dequant_mxfp8_to_bf16(x_q, x_scale).float()
@ dequant_mxfp8_to_bf16(layer.weight, layer.weight_scale).float().T
)
if packed:
cfg = g._packed(16, 32, 128, 1, 2, 2, 16, 0, 3)
else:
cfg = g._tiled(32, 64, 128, 1, True, 4, 2, 16, 0, 3)
args = (x_q, x_scale, layer.weight, layer.weight_scale.data[::32].contiguous())
return g, cfg, args, expected
def _rel_err(out: torch.Tensor, expected: torch.Tensor) -> float:
return ((out.float() - expected).norm() / expected.norm()).item()
@pytest.mark.parametrize("packed", [True, False])
@torch.inference_mode()
def test_rocm_mxfp8_block32_split_k_in_launch(packed):
"""Splits summed by the tile's last program are exact and repeatable."""
torch.manual_seed(0)
g, cfg, args, expected = _split_k_case(packed)
out = torch.empty_like(expected, dtype=torch.bfloat16)
g._launch(*args, out, cfg)
assert _rel_err(out, expected) < 5e-3
first = out.clone()
for _ in range(20):
g._launch(*args, out, cfg)
assert torch.equal(out, first)
@pytest.mark.parametrize("packed", [True, False])
@torch.inference_mode()
def test_rocm_mxfp8_block32_split_k_first_use_in_capture(packed):
"""A stream whose first split-K launch is being captured has no counters
yet; it must fall back to an unfused split instead of failing."""
torch.manual_seed(0)
g, cfg, args, expected = _split_k_case(packed)
out = torch.zeros_like(expected, dtype=torch.bfloat16)
stream = torch.cuda.Stream()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph, stream=stream):
g._launch(*args, out, cfg)
graph.replay()
torch.accelerator.synchronize()
assert _rel_err(out, expected) < 5e-3
@pytest.mark.parametrize("num_rows", [1, 64, 65])
@torch.inference_mode()
def test_rocm_mxfp8_quantizer_matches_torch_on_degenerate_blocks(num_rows):