[Perf][MoE] Skip padded work in block-FP8 DeepGEMM experts (#59128)

Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Tyler Michael Smith
2026-10-06 18:49:57 -05:00
committed by GitHub
co-authored by Codex mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent b6d8e8afd9
commit 5ab4e4445b
5 changed files with 431 additions and 76 deletions
+147
View File
@@ -158,6 +158,153 @@ def make_block_quant_fp8_weights(
return w1, w2, w1_s, w2_s
@pytest.mark.parametrize("num_tokens", [16, 129, 512])
@pytest.mark.parametrize("with_metadata", [False, True])
@pytest.mark.skipif(not is_deep_gemm_supported(), reason="Requires deep_gemm kernels")
def test_block_fp8_padding_on_graph_replay(num_tokens, with_metadata, monkeypatch):
"""Both FP8 GEMMs consume only live rows as routing changes on replay."""
from unittest.mock import Mock
import vllm.model_executor.layers.fused_moe.experts.deep_gemm_moe as impl
from vllm.utils.deep_gemm import DeepGemmQuantScaleFMT
if not (
impl.current_platform.is_device_capability_family(90)
or (
impl.current_platform.is_device_capability_family(100)
and DeepGemmQuantScaleFMT.from_oracle() == DeepGemmQuantScaleFMT.UE8M0
)
):
pytest.skip("Prefix-sum FP8 dispatch requires SM90 or SM10.x")
torch.manual_seed(0)
e, n, k, topk = 8, 1024, 512, 2
x = torch.randn(num_tokens, k, device="cuda", dtype=torch.bfloat16) / 10
aq, scales = per_token_group_quant_fp8(x, 128, use_ue8m0=True)
w1, w2, s1, s2 = make_block_quant_fp8_weights(e, n, k, BLOCK_SIZE)
quant = fp8_w8a8_moe_quant_config(w1_scale=s1, w2_scale=s2, block_shape=BLOCK_SIZE)
experts = impl.DeepGemmExperts(make_dummy_moe_config(), quant)
ids = torch.zeros(num_tokens, topk, device="cuda", dtype=torch.int64)
weights = torch.full((num_tokens, topk), 0.5, device="cuda")
counts = torch.zeros(e, device="cuda", dtype=torch.int32)
metadata = mk.ExpertTokensMetadata(counts, None) if with_metadata else None
shapes = experts.workspace_shapes(
num_tokens, 2 * n, k, topk, e, e, metadata, MoEActivation.SILU
)
def route(mode):
ids[:, 0] = torch.arange(num_tokens, device="cuda") % e
ids[:, 1] = (ids[:, 0] + 1) % e
if mode == 1:
ids.fill_(-1)
ids[:1, 0] = e - 1
elif mode == 2:
ids.fill_(-1)
counts.copy_(torch.bincount(ids[ids >= 0], minlength=e).int())
route(0)
variants = []
for dense in (True, False):
ws1, ws2, output = [
torch.empty(shape, device="cuda", dtype=torch.bfloat16) for shape in shapes
]
def invoke(output=output, ws1=ws1, ws2=ws2):
experts.apply(
output,
aq,
w1,
w2,
weights,
ids,
MoEActivation.SILU,
e,
None,
scales,
None,
ws1,
ws2,
metadata,
False,
)
with monkeypatch.context() as mp:
if dense:
platform = Mock(wraps=impl.current_platform)
platform.is_device_capability_family.return_value = False
mp.setattr(impl, "current_platform", platform)
gemm = impl.m_grouped_fp8_gemm_nt_contiguous
calls: list[int] = []
def check_layout(*args, dense=dense, calls=calls, gemm=gemm, **kwargs):
assert kwargs.get("use_psum_layout", False) == (not dense)
calls.append(args[3].numel())
return gemm(*args, **kwargs)
mp.setattr(impl, "m_grouped_fp8_gemm_nt_contiguous", check_layout)
invoke()
assert len(calls) == 2
if not dense:
assert calls == [e, e]
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
invoke()
variants.append((graph, output, ws1, ws2))
for mode in (0, 1, 2, 0):
route(mode)
for graph, _, ws1, ws2 in variants:
ws1.fill_(float("nan"))
ws2.fill_(float("nan"))
graph.replay()
torch.accelerator.synchronize()
torch.testing.assert_close(variants[0][1], variants[1][1], rtol=0, atol=0)
assert torch.isfinite(variants[1][1]).all()
@pytest.mark.parametrize(("n", "k"), [(2048, 512), (512, 1024)])
@pytest.mark.skipif(not is_deep_gemm_supported(), reason="Requires deep_gemm kernels")
def test_deepgemm_prefix_sum_leaves_unused_capacity_untouched(n, k):
"""Both projection shapes skip unused tiles as counts change on replay."""
from vllm.platforms import current_platform
from vllm.utils.deep_gemm import (
m_grouped_fp8_gemm_nt_contiguous,
mk_alignment_scope,
)
if not (
current_platform.is_device_capability_family(90)
or current_platform.is_device_capability_family(100)
):
pytest.skip("Requires SM90 or SM10.x")
capacity, experts, alignment = 512, 4, 128
x = torch.randn(capacity, k, device="cuda", dtype=torch.bfloat16) / 10
aq, scales = per_token_group_quant_fp8(x, 128, use_ue8m0=True)
w, _, ws, _ = make_block_quant_fp8_weights(experts, n // 2, k, BLOCK_SIZE)
ends = torch.tensor([0, 1, 128, 129], device="cuda", dtype=torch.int32)
output = torch.empty(capacity, n, device="cuda", dtype=torch.bfloat16)
def invoke():
with mk_alignment_scope(alignment):
m_grouped_fp8_gemm_nt_contiguous(
(aq, scales), (w, ws), output, ends, use_psum_layout=True
)
invoke()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
invoke()
for bounds, allocated_end in (([0, 1, 128, 129], 256), ([0] * 4, 0)):
ends.copy_(torch.tensor(bounds, device="cuda", dtype=torch.int32))
output.fill_(91)
graph.replay()
torch.accelerator.synchronize()
assert torch.all(output[allocated_end:] == 91)
if allocated_end:
assert torch.all(output[[0, 128]] != 91)
def run_single_case(m, n, k, topk, num_experts, block_size):
"""Run one (M,N,K) configuration on a single GPU and assert DeepGEMM ==
Triton baseline within tolerance.
@@ -7,6 +7,7 @@ import torch
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
_per_token_group_quant_fp8_colmajor,
silu_mul_per_token_group_quant_fp8_colmajor,
silu_mul_quant_fp8_packed_triton,
)
from vllm.platforms import current_platform
from vllm.triton_utils import triton
@@ -108,6 +109,94 @@ def test_silu_mul_fp8_quant_deep_gemm(T: int, N: int):
torch.testing.assert_close(output_scales, ref_output_scales)
@pytest.mark.parametrize(
"counts", [[0, 1, 127, 128, 129, 0, 257, 13], [128] * 8, [0] * 8]
)
@pytest.mark.parametrize("group_size", [32, 128])
@pytest.mark.parametrize("scale_format", ["packed", "float32", "ceil_ue8m0"])
@pytest.mark.parametrize("hidden_size", [256, 2048])
@pytest.mark.parametrize("clamp_limit", [None, 10.0])
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA graph test")
def test_silu_quant_skips_expert_padding_on_graph_replay(
counts: list[int],
group_size: int,
scale_format: str,
hidden_size: int,
clamp_limit: float | None,
):
"""Live rows match dense quantization; gaps stay untouched as counts change."""
alignment = 128
capacity = sum(triton.cdiv(n, alignment) * alignment for n in counts)
# Exercise unused capacity as well as the gaps between experts.
capacity = max(capacity, alignment)
if counts != [128] * 8:
capacity += alignment
x = torch.randn((capacity, 2 * hidden_size), device=DEVICE, dtype=torch.bfloat16)
x *= 16
clean_x = x.clone()
ends = torch.empty(len(counts), device=DEVICE, dtype=torch.int32)
output = torch.empty((capacity, hidden_size), device=DEVICE, dtype=FLOAT8_DTYPE)
def set_counts(values):
positions = []
mask = torch.zeros(capacity, device=DEVICE, dtype=torch.bool)
start = 0
for count in values:
mask[start : start + count] = True
positions.append(start + count)
start += triton.cdiv(count, alignment) * alignment
ends.copy_(torch.tensor(positions, device=DEVICE, dtype=torch.int32))
return mask
def quantize(x, output=None, **kwargs):
if scale_format == "packed":
return silu_mul_quant_fp8_packed_triton(
x,
group_size=group_size,
output_q=output,
clamp_limit=clamp_limit,
**kwargs,
)
return silu_mul_per_token_group_quant_fp8_colmajor(
x,
group_size=group_size,
output=output,
clamp_limit=clamp_limit,
use_ue8m0=scale_format == "ceil_ue8m0",
**kwargs,
)
def invoke():
return quantize(
x,
output=output,
expert_ends=ends,
expert_alignment=alignment,
)
set_counts(counts)
invoke()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
actual_q, actual_s = invoke()
for values in (counts, counts[::-1]):
live = set_counts(values)
x.copy_(clean_x)
x[~live] = float("nan")
output.view(torch.uint8).fill_(91)
graph.replay()
reference_q, reference_s = quantize(x)
torch.testing.assert_close(
actual_q.view(torch.uint8)[live],
reference_q.view(torch.uint8)[live],
rtol=0,
atol=0,
)
torch.testing.assert_close(actual_s[live], reference_s[live], rtol=0, atol=0)
assert torch.all(actual_q.view(torch.uint8)[~live] == 91)
@pytest.mark.parametrize("T", [128, 256, 512])
@pytest.mark.parametrize("N", [128 * 2, 256 * 2, 768 * 2, 2048 * 2, 7168 * 2])
@pytest.mark.parametrize("clamp_limit", [7.0, 10.0])