[Perf][DSv4.1] Fuse small-batch WO-A with inverse RoPE and MXFP8 quant on SM100/SM103 (#58634)

Signed-off-by: Juntian Liu <Juntianl777@gmail.com>
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Juntian Liu
2026-09-26 21:16:12 -07:00
committed by GitHub
co-authored by Claude Opus 5.5
parent fba47397f3
commit a9eafde59c
6 changed files with 759 additions and 55 deletions
@@ -21,7 +21,12 @@ Usage:
import pytest
import torch
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
deepgemm_post_process_fp8_weight_block,
)
from vllm.models.deepseek_v4.common.ops import fused_inv_rope_fp8_quant
from vllm.platforms import current_platform
from vllm.utils.deep_gemm import fp8_einsum
# -- Default dimensions matching DeepSeek V3/V4 --------------------------
HEAD_DIM = 512
@@ -33,6 +38,49 @@ FP8_DTYPE = torch.float8_e4m3fn
EPS = 1e-10
@pytest.mark.skipif(
not current_platform.is_device_capability_family(100), reason="Requires SM100/SM103"
)
@pytest.mark.parametrize("groups", [2, 4])
@pytest.mark.parametrize("tokens", [1, 6, 18, 32])
def test_dsv41_fused_wo_a_matches_deepgemm(groups, tokens):
"""Fused WO-A matches inverse RoPE + fp8_einsum + MXFP8 quantize."""
from flashinfer import mxfp8_quantize
from vllm.models.deepseek_v41.nvidia.ops.fused_wo_a import _FUSED_WO_A_KERNEL
torch.manual_seed(0)
n = groups * 1024
x = torch.randn(tokens, 64, 512, device="cuda", dtype=torch.bfloat16)[:, : n // 128]
positions = torch.randint(0, 128, (tokens,), device="cuda")
rope = make_cos_sin_cache(128, device="cuda")
wq = torch.randn(n, 4096, device="cuda").to(FP8_DTYPE)
scale = torch.exp2(torch.randint(-8, 0, (n, 128), device="cuda").float())
wq, ws = deepgemm_post_process_fp8_weight_block(
wq, scale, (1, 32), False, True, groups
)
q, sf = _FUSED_WO_A_KERNEL(
x=x, positions=positions, rope=rope, weight=wq, weight_scale=ws
)
a, sa = fused_inv_rope_fp8_quant(
x, positions, rope, groups, 8, quant_group_size=32, tma_aligned_scales=True
)
y = torch.empty(tokens, groups, 1024, device="cuda", dtype=torch.bfloat16)
fp8_einsum("bhr,hdr->bhd", (a, sa), (wq, ws), y, recipe=(1, 1, 32))
ref, ref_sf = mxfp8_quantize(y.flatten(1), backend="cute-dsl")
def rows(scales): # F8_128x4 bytes -> [128 rows, n / 32 blocks]
return scales.view(n // 128, 32, 4, 4).permute(2, 1, 0, 3).reshape(128, -1)
actual, expected = (
d.float().view(tokens, -1, 32) * torch.exp2(rows(s)[:tokens, :, None] - 127.0)
for d, s in ((q, sf), (ref, ref_sf))
)
assert (actual - expected).norm() / expected.norm() < 3e-3
assert not rows(sf)[tokens:].any(), "padded scale rows must be zero"
# =========================================================================
# Helpers
# =========================================================================