[Perf][GLM-5.2] Blackwell decode optimizations (#48597)
Signed-off-by: Peiyuan Zhou <peiyuanzhou1994@gmail.com> Signed-off-by: zhou <zhou@zhoudeMacBook-Pro.local> Signed-off-by: Stefan Koncarevic <stefan.koncarevic@amd.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Giancarlo Delfin <32987265+TheEpicDolphin@users.noreply.github.com> Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: zhou <zhou@zhoudeMacBook-Pro.local> Co-authored-by: stefankoncarevic <Stefan.Koncarevic@amd.com>
This commit is contained in:
co-authored by
Claude
Giancarlo Delfin
Thien Tran
OpenAI Codex
zhou
stefankoncarevic
parent
0d77325b10
commit
dd72658e7d
@@ -0,0 +1,70 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Numerical correctness for the bf16 skinny GEMM (decode-M GEMV) kernel.
|
||||
|
||||
Covers every supported (N, K) shape across M = 0/1/2/8/16/32 (M=32 exercises
|
||||
the tile_m=32 MMA path), plus strided (column-slice) output. M=0 must return
|
||||
an empty result without launching the kernel (empty ranks at DP/PP
|
||||
boundaries)."""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
# (N, K) pairs must match bf16_skinny_gemm_supported() in the entry .cu.
|
||||
SHAPES = [
|
||||
(768, 12288),
|
||||
(1536, 12288),
|
||||
(6144, 12288),
|
||||
(2048, 2048),
|
||||
(2624, 6144),
|
||||
(2112, 7168),
|
||||
(7168, 14336),
|
||||
(512, 6144),
|
||||
]
|
||||
MS = [0, 1, 2, 8, 16, 32]
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_cuda() or not current_platform.has_device_capability(90),
|
||||
reason="bf16_skinny_gemm requires CUDA SM90+",
|
||||
)
|
||||
|
||||
|
||||
def _rel_err(out: torch.Tensor, ref: torch.Tensor) -> float:
|
||||
return (out.float() - ref).norm().item() / ref.norm().clamp_min(1e-6).item()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("n,k", SHAPES)
|
||||
@pytest.mark.parametrize("m", MS)
|
||||
def test_bf16_skinny_gemm_matches_reference(n: int, k: int, m: int):
|
||||
torch.manual_seed(0)
|
||||
x = torch.randn(m, k, dtype=torch.bfloat16, device="cuda")
|
||||
w = torch.randn(n, k, dtype=torch.bfloat16, device="cuda")
|
||||
|
||||
out = ops.bf16_skinny_gemm(x, w)
|
||||
assert out.shape == (m, n)
|
||||
assert out.dtype == torch.bfloat16
|
||||
|
||||
if m == 0: # empty batch: nothing to check beyond the empty shape
|
||||
return
|
||||
ref = x.float() @ w.float().t()
|
||||
rel = _rel_err(out, ref)
|
||||
assert rel < 2e-2, f"rel err {rel:.4f} for (m,n,k)=({m},{n},{k})"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("n,k", [(2048, 2048), (512, 6144)])
|
||||
def test_bf16_skinny_gemm_strided_output(n: int, k: int):
|
||||
"""Output is a column-slice of a wider padded buffer (row stride > N)."""
|
||||
torch.manual_seed(0)
|
||||
m = 8
|
||||
x = torch.randn(m, k, dtype=torch.bfloat16, device="cuda")
|
||||
w = torch.randn(n, k, dtype=torch.bfloat16, device="cuda")
|
||||
|
||||
wide = torch.empty(m, n + 64, dtype=torch.bfloat16, device="cuda")
|
||||
view = wide[:, :n]
|
||||
torch.ops._C.bf16_skinny_gemm(view, x, w)
|
||||
|
||||
ref = x.float() @ w.float().t()
|
||||
assert _rel_err(view, ref) < 2e-2
|
||||
@@ -49,6 +49,32 @@ pytestmark = pytest.mark.skipif(
|
||||
)
|
||||
|
||||
|
||||
def test_platform_capability_queries_are_constant_during_compile(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
K.current_platform, "has_device_capability", lambda capability: True
|
||||
)
|
||||
monkeypatch.setattr(K, "has_cutedsl", lambda: True)
|
||||
monkeypatch.setattr(K.current_platform, "is_arch_support_pdl", lambda: True)
|
||||
|
||||
def capability_branches(x: torch.Tensor) -> torch.Tensor:
|
||||
if K._can_use_fused_q_cutedsl() and K._is_arch_support_pdl():
|
||||
return x + 1
|
||||
return x - 1
|
||||
|
||||
compiled = torch.compile(capability_branches, backend="eager", fullgraph=True)
|
||||
x = torch.zeros(1, device="cuda")
|
||||
torch.testing.assert_close(compiled(x), torch.ones_like(x))
|
||||
|
||||
|
||||
def test_pdl_is_disabled_before_blackwell(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
K.current_platform, "has_device_capability", lambda capability: False
|
||||
)
|
||||
monkeypatch.setattr(K.current_platform, "is_arch_support_pdl", lambda: True)
|
||||
|
||||
assert not K._is_arch_support_pdl()
|
||||
|
||||
|
||||
# ── reference helpers ────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@@ -292,6 +318,57 @@ def test_fused_norm_rope_no_indexer(num_tokens: int):
|
||||
assert (topk == 7).all(), "topk buffer should be untouched on shared layer"
|
||||
|
||||
|
||||
def test_fused_norm_rope_profile_without_cache_compiles():
|
||||
"""The cache-free profiling path must still compile and produce Q."""
|
||||
torch.manual_seed(2)
|
||||
dev = "cuda"
|
||||
num_tokens = 4
|
||||
pos = torch.arange(num_tokens, device=dev, dtype=torch.int64)
|
||||
q_c = torch.randn(num_tokens, Q_LORA, device=dev, dtype=torch.bfloat16)
|
||||
kv_c = torch.randn(num_tokens, KV_LORA, device=dev, dtype=torch.bfloat16)
|
||||
k_pe = torch.randn(num_tokens, ROPE_DIM, device=dev, dtype=torch.bfloat16)
|
||||
qw = torch.randn(Q_LORA, device=dev, dtype=torch.bfloat16)
|
||||
kvw = torch.randn(KV_LORA, device=dev, dtype=torch.bfloat16)
|
||||
index_k = torch.randn(num_tokens, INDEX_HEAD_DIM, device=dev, dtype=torch.bfloat16)
|
||||
index_w = torch.randn(INDEX_HEAD_DIM, device=dev, dtype=torch.float32)
|
||||
index_b = torch.randn(INDEX_HEAD_DIM, device=dev, dtype=torch.float32)
|
||||
cos_sin = make_cos_sin(64, ROPE_DIM, dev)
|
||||
topk = torch.empty(num_tokens, 2048, device=dev, dtype=torch.int32)
|
||||
|
||||
def profile_run(
|
||||
q_c: torch.Tensor,
|
||||
kv_c: torch.Tensor,
|
||||
k_pe: torch.Tensor,
|
||||
index_k: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
return K.fused_norm_rope(
|
||||
pos,
|
||||
q_c,
|
||||
qw,
|
||||
EPS,
|
||||
kv_c,
|
||||
kvw,
|
||||
EPS,
|
||||
k_pe,
|
||||
cos_sin,
|
||||
index_k,
|
||||
index_w,
|
||||
index_b,
|
||||
EPS,
|
||||
cos_sin,
|
||||
topk,
|
||||
slot_mapping=None,
|
||||
indexer_k_cache=None,
|
||||
mla_kv_cache=None,
|
||||
has_indexer=True,
|
||||
index_rope_interleave=False,
|
||||
)
|
||||
|
||||
compiled = torch.compile(profile_run, fullgraph=True)
|
||||
q_out = compiled(q_c, kv_c, k_pe, index_k)
|
||||
assert_bf16(q_out, rms_norm(q_c, qw), "q_c rmsnorm (profiling)")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [1, 4, 17, 512])
|
||||
def test_fused_norm_rope_ds_mla(num_tokens: int):
|
||||
"""fp8_ds_mla MLA cache layout (FlashMLA sparse, bf16-query path; SM90/SM100).
|
||||
|
||||
Reference in New Issue
Block a user