[Perf][DSv4.1] Fuse small-batch WO-A with inverse RoPE and MXFP8 quant on SM100/SM103 (#58634)
Signed-off-by: Juntian Liu <Juntianl777@gmail.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
fba47397f3
commit
a9eafde59c
@@ -21,7 +21,12 @@ Usage:
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
deepgemm_post_process_fp8_weight_block,
|
||||
)
|
||||
from vllm.models.deepseek_v4.common.ops import fused_inv_rope_fp8_quant
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.deep_gemm import fp8_einsum
|
||||
|
||||
# -- Default dimensions matching DeepSeek V3/V4 --------------------------
|
||||
HEAD_DIM = 512
|
||||
@@ -33,6 +38,49 @@ FP8_DTYPE = torch.float8_e4m3fn
|
||||
EPS = 1e-10
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_device_capability_family(100), reason="Requires SM100/SM103"
|
||||
)
|
||||
@pytest.mark.parametrize("groups", [2, 4])
|
||||
@pytest.mark.parametrize("tokens", [1, 6, 18, 32])
|
||||
def test_dsv41_fused_wo_a_matches_deepgemm(groups, tokens):
|
||||
"""Fused WO-A matches inverse RoPE + fp8_einsum + MXFP8 quantize."""
|
||||
from flashinfer import mxfp8_quantize
|
||||
|
||||
from vllm.models.deepseek_v41.nvidia.ops.fused_wo_a import _FUSED_WO_A_KERNEL
|
||||
|
||||
torch.manual_seed(0)
|
||||
n = groups * 1024
|
||||
x = torch.randn(tokens, 64, 512, device="cuda", dtype=torch.bfloat16)[:, : n // 128]
|
||||
positions = torch.randint(0, 128, (tokens,), device="cuda")
|
||||
rope = make_cos_sin_cache(128, device="cuda")
|
||||
wq = torch.randn(n, 4096, device="cuda").to(FP8_DTYPE)
|
||||
scale = torch.exp2(torch.randint(-8, 0, (n, 128), device="cuda").float())
|
||||
wq, ws = deepgemm_post_process_fp8_weight_block(
|
||||
wq, scale, (1, 32), False, True, groups
|
||||
)
|
||||
q, sf = _FUSED_WO_A_KERNEL(
|
||||
x=x, positions=positions, rope=rope, weight=wq, weight_scale=ws
|
||||
)
|
||||
|
||||
a, sa = fused_inv_rope_fp8_quant(
|
||||
x, positions, rope, groups, 8, quant_group_size=32, tma_aligned_scales=True
|
||||
)
|
||||
y = torch.empty(tokens, groups, 1024, device="cuda", dtype=torch.bfloat16)
|
||||
fp8_einsum("bhr,hdr->bhd", (a, sa), (wq, ws), y, recipe=(1, 1, 32))
|
||||
ref, ref_sf = mxfp8_quantize(y.flatten(1), backend="cute-dsl")
|
||||
|
||||
def rows(scales): # F8_128x4 bytes -> [128 rows, n / 32 blocks]
|
||||
return scales.view(n // 128, 32, 4, 4).permute(2, 1, 0, 3).reshape(128, -1)
|
||||
|
||||
actual, expected = (
|
||||
d.float().view(tokens, -1, 32) * torch.exp2(rows(s)[:tokens, :, None] - 127.0)
|
||||
for d, s in ((q, sf), (ref, ref_sf))
|
||||
)
|
||||
assert (actual - expected).norm() / expected.norm() < 3e-3
|
||||
assert not rows(sf)[tokens:].any(), "padded scale rows must be zero"
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# Helpers
|
||||
# =========================================================================
|
||||
|
||||
Reference in New Issue
Block a user