[GLM 5.3 Perf] Skip qlnorm calculation for MHA, 4.4~7.7% E2E TTFT Improvement (#58845)

Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
Wentao Ye
2026-10-01 15:13:57 +08:00
committed by GitHub
parent aee8fe202a
commit 4d6874d137
5 changed files with 99 additions and 26 deletions
@@ -24,6 +24,7 @@ outputs are checked within 1 representable-step (ULP); bf16 norm/RoPE outputs us
rtol/atol=1e-2 (the tolerance the sibling deepseek_v4 fused-kernel test uses).
"""
from functools import partial
from typing import NamedTuple
import pytest
@@ -880,22 +881,36 @@ def test_fused_q_no_indexer(num_tokens: int, cfg: ModelConfig):
@pytest.mark.parametrize("cfg", MODEL_CONFIGS, ids=MODEL_IDS)
@pytest.mark.parametrize("num_tokens", [1, 17, 512])
@pytest.mark.parametrize("has_indexer", [True, False])
def test_fused_q_bf16_query(num_tokens: int, has_indexer: bool, cfg: ModelConfig):
@pytest.mark.parametrize("use_mha", [True, False])
@pytest.mark.parametrize(
"index_interleave,cache_dtype", [(False, torch.float32), (True, torch.bfloat16)]
)
@pytest.mark.parametrize("capture", [False, True])
def test_fused_q_bf16_query(
num_tokens: int,
has_indexer: bool,
use_mha: bool,
index_interleave: bool,
cache_dtype: torch.dtype,
capture: bool,
cfg: ModelConfig,
):
"""bf16-query path (FlashMLA sparse, SM90/SM100): only the RoPE'd q_pe is
produced (bf16, unquantized); ql_nope is consumed directly by the caller."""
produced; MHA does not need an absorbed ql_nope projection."""
torch.manual_seed(6)
dev = "cuda"
max_pos = 8192
pos = torch.arange(num_tokens, device=dev, dtype=torch.int64) % max_pos
# Attention takes q_pe as a strided slice of the full Q projection.
q_pe = torch.randn(
num_tokens, cfg.num_heads, cfg.rope_dim, device=dev, dtype=torch.bfloat16
)
num_tokens, cfg.num_heads, 4 * cfg.rope_dim, device=dev, dtype=torch.bfloat16
)[..., -cfg.rope_dim :]
ql_nope = torch.randn(
num_tokens, cfg.num_heads, cfg.kv_lora, device=dev, dtype=torch.bfloat16
)
q_scale = torch.tensor([0.37], device=dev, dtype=torch.float32)
q_cos_sin = make_cos_sin(max_pos, cfg.rope_dim, dev)
q_cos_sin = make_cos_sin(max_pos, cfg.rope_dim, dev).to(cache_dtype)
index_q = index_w = idx_cos_sin = None
if has_indexer:
@@ -909,24 +924,35 @@ def test_fused_q_bf16_query(num_tokens: int, has_indexer: bool, cfg: ModelConfig
index_w = torch.randn(
num_tokens, cfg.index_heads, device=dev, dtype=torch.float32
)
idx_cos_sin = make_cos_sin(max_pos, cfg.rope_dim, dev)
idx_cos_sin = make_cos_sin(max_pos, cfg.rope_dim, dev).to(cache_dtype)
iq_fp8, iw_out, q_pe_out = K.fused_q(
run = partial(
K.fused_q,
pos,
q_pe,
q_cos_sin,
index_q,
idx_cos_sin,
ql_nope,
q_pe if use_mha else ql_nope,
q_scale,
index_w,
cfg.index_head_dim**-0.5,
cfg.index_heads**-0.5,
has_indexer=has_indexer,
index_rope_interleave=False,
index_rope_interleave=index_interleave,
quantize_mqa=False,
)
iq_fp8, iw_out, q_pe_out = run()
if capture:
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
iq_fp8, iw_out, q_pe_out = run()
q_pe.mul_(0.5)
if index_q is not None:
index_q.mul_(1.5)
graph.replay()
# MQA query: only the RoPE'd q_pe, bf16, unquantized.
assert q_pe_out.dtype == torch.bfloat16
assert q_pe_out.shape == (num_tokens, cfg.num_heads, cfg.rope_dim)
@@ -945,7 +971,7 @@ def test_fused_q_bf16_query(num_tokens: int, has_indexer: bool, cfg: ModelConfig
index_q.float(),
pos.unsqueeze(-1).expand(num_tokens, cfg.index_heads),
idx_cos_sin,
interleave=False,
interleave=index_interleave,
)
q_ref, scale_ref = ue8m0_quant(iq_ref)
assert_fp8(iq_fp8, q_ref, "indexer-Q fp8 (bf16-query path)")
@@ -267,9 +267,11 @@ def test_skipped_k_cache_insert_accepts_no_k(
@pytest.mark.parametrize("fp8_query", [False, True])
@pytest.mark.parametrize("prepared_mha", [False, True])
def test_deepseek_v32_dispatches_selected_mha(
monkeypatch: pytest.MonkeyPatch,
fp8_query: bool,
prepared_mha: bool,
) -> None:
attn_metadata = SimpleNamespace(num_actual_tokens=2)
kv_cache = torch.empty(1)
@@ -301,7 +303,11 @@ def test_deepseek_v32_dispatches_selected_mha(
q_nope = torch.randn(2, 1, 2)
q_pe = torch.randn(2, 1, 2)
mqa_q = torch.randn(2, 1, 2)
mha_q = torch.cat((q_nope, q_pe + 1 if fp8_query else mqa_q), dim=-1)
if fp8_query and not prepared_mha:
mqa_q = mqa_q.to(torch.float8_e4m3fn)
mha_q = torch.cat(
(q_nope, q_pe + 1 if fp8_query and not prepared_mha else mqa_q), dim=-1
)
kv_c = torch.empty(2, 2)
k_pe = torch.empty(2, 2)
output = torch.empty(2, 2)
+26 -4
View File
@@ -7,7 +7,7 @@ import torch.nn as nn
from transformers import DeepseekV2Config, DeepseekV3Config
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
from vllm.config import CacheConfig, VllmConfig
from vllm.config import CacheConfig, CUDAGraphMode, VllmConfig
from vllm.distributed import get_tensor_model_parallel_world_size
from vllm.distributed.parallel_state import get_tp_group
from vllm.forward_context import get_forward_context
@@ -283,6 +283,18 @@ class DeepseekV32Attention(MLAAttention):
is_neox_style=not getattr(config, "indexer_rope_interleave", False),
)
def _should_prepare_mqa_query(
self,
attn_metadata: "MLACommonMetadata | None",
cudagraph_runtime_mode: CUDAGraphMode,
) -> bool:
return (
attn_metadata is None
or not self._use_sparse_mha(attn_metadata)
or cudagraph_runtime_mode == CUDAGraphMode.FULL
or torch.cuda.is_current_stream_capturing()
)
def forward( # type: ignore[override]
self,
positions: torch.Tensor,
@@ -321,6 +333,9 @@ class DeepseekV32Attention(MLAAttention):
layer_attn_metadata, _, _, _ = get_attention_context(self.layer_name)
self.impl.prepare_for_batch(layer_attn_metadata)
prepare_mqa_query = self._should_prepare_mqa_query(
layer_attn_metadata, forward_context.cudagraph_runtime_mode
)
if self.indexer is not None and not self.skip_topk:
has_indexer = True
indexer_k_norm_w = self.indexer.k_norm.weight
@@ -404,7 +419,10 @@ class DeepseekV32Attention(MLAAttention):
q = self.q_b_proj(q_c)[0].view(-1, self.num_local_heads, self.qk_head_dim)
q_nope, q_pe = q.split([self.qk_nope_head_dim, self.qk_rope_head_dim], dim=-1)
ql_nope = torch.bmm(q_nope.transpose(0, 1), self.W_UK_T).transpose(0, 1)
if prepare_mqa_query:
ql_nope = torch.bmm(q_nope.transpose(0, 1), self.W_UK_T).transpose(0, 1)
else:
ql_nope = q_nope
if self.indexer is not None and not self.skip_topk:
index_q = self.indexer.wq_b(q_c)[0]
@@ -425,7 +443,7 @@ class DeepseekV32Attention(MLAAttention):
indexer_n_head_scale,
has_indexer=has_indexer,
index_rope_interleave=self._index_rope_interleave,
quantize_mqa=self._fp8_query,
quantize_mqa=self._fp8_query and prepare_mqa_query,
)
self._sparse_indexer_and_attn(
@@ -535,7 +553,11 @@ class DeepseekV32Attention(MLAAttention):
if self._use_sparse_mha(attn_metadata):
assert kv_c is not None and k_pe is not None
mha_q_pe = self.rotary_emb(positions, q_pe)[0] if self._fp8_query else mqa_q
mha_q_pe = (
self.rotary_emb(positions, q_pe)[0]
if mqa_q.dtype != q_pe.dtype
else mqa_q
)
mha_q = torch.cat((q_nope, mha_q_pe), dim=-1)
self.forward_impl(
mha_q,
@@ -1052,6 +1052,7 @@ def fused_q(
index_weights_out,
has_indexer=has_indexer,
index_rope_interleave=index_rope_interleave,
quantize_mqa=quantize_mqa,
)
return index_q_fp8, index_weights_out, mqa_q
@@ -35,10 +35,12 @@ def is_fused_q_cutedsl_supported(
) -> bool:
if not (
current_platform.has_device_capability(100)
and quantize_mqa
and q_pe.dtype == ql_nope.dtype == torch.bfloat16
and q_pe.dtype == torch.bfloat16
and q_pe.shape[-1] == 64
and ql_nope.shape[-1] == 512
and (
not quantize_mqa
or (ql_nope.dtype == torch.bfloat16 and ql_nope.shape[-1] == 512)
)
# One warp per head group; a non-multiple would trip the kernel's own
# assert instead of falling back to Triton.
and q_pe.shape[1] % 4 == 0
@@ -68,9 +70,10 @@ def fused_q_cutedsl(
idx_weights_out: torch.Tensor,
has_indexer: bool = True,
index_rope_interleave: bool = True,
quantize_mqa: bool = True,
) -> None:
_, num_heads, rope_dim = q_pe.shape
_, _, nope_dim = ql_nope.shape
nope_dim = ql_nope.shape[-1] if quantize_mqa else 0
_, num_idx_heads, idx_dim = idx_q.shape
if has_indexer:
@@ -98,8 +101,8 @@ def fused_q_cutedsl(
positions,
q_pe,
rope_cache,
ql_nope,
q_scale.view(1),
ql_nope if quantize_mqa else None,
q_scale.view(1) if quantize_mqa else None,
mqa_output,
idx_q,
idx_rope_cache,
@@ -121,7 +124,7 @@ class FusedQKernel:
index_rope_interleave: bool,
) -> None:
assert rope_dim == 64
assert nope_dim == 512
assert nope_dim in (0, 512)
assert idx_dim in (128, 0)
self.rope_dim = rope_dim
@@ -262,6 +265,17 @@ class FusedQKernel:
cute.arch.griddepcontrol_wait()
pos = positions[token_id]
# no need to do Q calculation for MHA
if cutlass.const_expr(self.nope_dim == 0):
q0, q1 = cvt.bf16x2_to_fp32x2(
cute.recast_tensor(q_pe, Uint32)[token_id, head_id, lane_id]
)
cos = q_pe_rope_cache[pos, lane_id].to(Float32)
sin = q_pe_rope_cache[pos, 32 + lane_id].to(Float32)
result = cvt.fp32x2_to_bf16x2(q0 * cos - q1 * sin, q1 * cos + q0 * sin)
cute.arch.griddepcontrol_launch_dependents()
cute.recast_tensor(mqa_output, Uint32)[token_id, head_id, lane_id] = result
return
inv_scale = 1.0 / q_scale[0]
cp_op = cute.nvgpu.CopyUniversalOp()
@@ -479,12 +493,16 @@ class FusedQKernel:
BFloat16, (num_tokens, num_heads, rope_dim), divisibility=16
)
rope_cache = _make_fake_tensor(rope_type, (max_pos, rope_dim), divisibility=8)
ql_nope = _make_fake_tensor(
BFloat16, (num_tokens, num_heads, nope_dim), divisibility=16
ql_nope = (
_make_fake_tensor(
BFloat16, (num_tokens, num_heads, nope_dim), divisibility=16
)
if nope_dim
else None
)
q_scale = _make_fake_tensor(Float32, (1,), divisibility=4)
q_scale = _make_fake_tensor(Float32, (1,), divisibility=4) if nope_dim else None
mqa_output = _make_fake_tensor(
Float8E4M3FN,
Float8E4M3FN if nope_dim else BFloat16,
(num_tokens, num_heads, nope_dim + rope_dim),
divisibility=16,
)