[Perf][GLM-5.2] Blackwell decode optimizations (#48597)

Signed-off-by: Peiyuan Zhou <peiyuanzhou1994@gmail.com>
Signed-off-by: zhou <zhou@zhoudeMacBook-Pro.local>
Signed-off-by: Stefan Koncarevic <stefan.koncarevic@amd.com>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: Giancarlo Delfin <32987265+TheEpicDolphin@users.noreply.github.com>
Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: zhou <zhou@zhoudeMacBook-Pro.local>
Co-authored-by: stefankoncarevic <Stefan.Koncarevic@amd.com>
This commit is contained in:
xiaozhoupy
2026-07-23 21:36:27 -07:00
committed by GitHub
co-authored by Claude Giancarlo Delfin Thien Tran OpenAI Codex zhou stefankoncarevic
parent 0d77325b10
commit dd72658e7d
29 changed files with 2139 additions and 138 deletions
+70
View File
@@ -0,0 +1,70 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Numerical correctness for the bf16 skinny GEMM (decode-M GEMV) kernel.
Covers every supported (N, K) shape across M = 0/1/2/8/16/32 (M=32 exercises
the tile_m=32 MMA path), plus strided (column-slice) output. M=0 must return
an empty result without launching the kernel (empty ranks at DP/PP
boundaries)."""
import pytest
import torch
from vllm import _custom_ops as ops
from vllm.platforms import current_platform
# (N, K) pairs must match bf16_skinny_gemm_supported() in the entry .cu.
SHAPES = [
(768, 12288),
(1536, 12288),
(6144, 12288),
(2048, 2048),
(2624, 6144),
(2112, 7168),
(7168, 14336),
(512, 6144),
]
MS = [0, 1, 2, 8, 16, 32]
pytestmark = pytest.mark.skipif(
not current_platform.is_cuda() or not current_platform.has_device_capability(90),
reason="bf16_skinny_gemm requires CUDA SM90+",
)
def _rel_err(out: torch.Tensor, ref: torch.Tensor) -> float:
return (out.float() - ref).norm().item() / ref.norm().clamp_min(1e-6).item()
@pytest.mark.parametrize("n,k", SHAPES)
@pytest.mark.parametrize("m", MS)
def test_bf16_skinny_gemm_matches_reference(n: int, k: int, m: int):
torch.manual_seed(0)
x = torch.randn(m, k, dtype=torch.bfloat16, device="cuda")
w = torch.randn(n, k, dtype=torch.bfloat16, device="cuda")
out = ops.bf16_skinny_gemm(x, w)
assert out.shape == (m, n)
assert out.dtype == torch.bfloat16
if m == 0: # empty batch: nothing to check beyond the empty shape
return
ref = x.float() @ w.float().t()
rel = _rel_err(out, ref)
assert rel < 2e-2, f"rel err {rel:.4f} for (m,n,k)=({m},{n},{k})"
@pytest.mark.parametrize("n,k", [(2048, 2048), (512, 6144)])
def test_bf16_skinny_gemm_strided_output(n: int, k: int):
"""Output is a column-slice of a wider padded buffer (row stride > N)."""
torch.manual_seed(0)
m = 8
x = torch.randn(m, k, dtype=torch.bfloat16, device="cuda")
w = torch.randn(n, k, dtype=torch.bfloat16, device="cuda")
wide = torch.empty(m, n + 64, dtype=torch.bfloat16, device="cuda")
view = wide[:, :n]
torch.ops._C.bf16_skinny_gemm(view, x, w)
ref = x.float() @ w.float().t()
assert _rel_err(view, ref) < 2e-2
@@ -49,6 +49,32 @@ pytestmark = pytest.mark.skipif(
)
def test_platform_capability_queries_are_constant_during_compile(monkeypatch):
monkeypatch.setattr(
K.current_platform, "has_device_capability", lambda capability: True
)
monkeypatch.setattr(K, "has_cutedsl", lambda: True)
monkeypatch.setattr(K.current_platform, "is_arch_support_pdl", lambda: True)
def capability_branches(x: torch.Tensor) -> torch.Tensor:
if K._can_use_fused_q_cutedsl() and K._is_arch_support_pdl():
return x + 1
return x - 1
compiled = torch.compile(capability_branches, backend="eager", fullgraph=True)
x = torch.zeros(1, device="cuda")
torch.testing.assert_close(compiled(x), torch.ones_like(x))
def test_pdl_is_disabled_before_blackwell(monkeypatch):
monkeypatch.setattr(
K.current_platform, "has_device_capability", lambda capability: False
)
monkeypatch.setattr(K.current_platform, "is_arch_support_pdl", lambda: True)
assert not K._is_arch_support_pdl()
# ── reference helpers ────────────────────────────────────────────────────────
@@ -292,6 +318,57 @@ def test_fused_norm_rope_no_indexer(num_tokens: int):
assert (topk == 7).all(), "topk buffer should be untouched on shared layer"
def test_fused_norm_rope_profile_without_cache_compiles():
"""The cache-free profiling path must still compile and produce Q."""
torch.manual_seed(2)
dev = "cuda"
num_tokens = 4
pos = torch.arange(num_tokens, device=dev, dtype=torch.int64)
q_c = torch.randn(num_tokens, Q_LORA, device=dev, dtype=torch.bfloat16)
kv_c = torch.randn(num_tokens, KV_LORA, device=dev, dtype=torch.bfloat16)
k_pe = torch.randn(num_tokens, ROPE_DIM, device=dev, dtype=torch.bfloat16)
qw = torch.randn(Q_LORA, device=dev, dtype=torch.bfloat16)
kvw = torch.randn(KV_LORA, device=dev, dtype=torch.bfloat16)
index_k = torch.randn(num_tokens, INDEX_HEAD_DIM, device=dev, dtype=torch.bfloat16)
index_w = torch.randn(INDEX_HEAD_DIM, device=dev, dtype=torch.float32)
index_b = torch.randn(INDEX_HEAD_DIM, device=dev, dtype=torch.float32)
cos_sin = make_cos_sin(64, ROPE_DIM, dev)
topk = torch.empty(num_tokens, 2048, device=dev, dtype=torch.int32)
def profile_run(
q_c: torch.Tensor,
kv_c: torch.Tensor,
k_pe: torch.Tensor,
index_k: torch.Tensor,
) -> torch.Tensor:
return K.fused_norm_rope(
pos,
q_c,
qw,
EPS,
kv_c,
kvw,
EPS,
k_pe,
cos_sin,
index_k,
index_w,
index_b,
EPS,
cos_sin,
topk,
slot_mapping=None,
indexer_k_cache=None,
mla_kv_cache=None,
has_indexer=True,
index_rope_interleave=False,
)
compiled = torch.compile(profile_run, fullgraph=True)
q_out = compiled(q_c, kv_c, k_pe, index_k)
assert_bf16(q_out, rms_norm(q_c, qw), "q_c rmsnorm (profiling)")
@pytest.mark.parametrize("num_tokens", [1, 4, 17, 512])
def test_fused_norm_rope_ds_mla(num_tokens: int):
"""fp8_ds_mla MLA cache layout (FlashMLA sparse, bf16-query path; SM90/SM100).