[GLM 5.3 Perf] Skip qlnorm calculation for MHA, 4.4~7.7% E2E TTFT Improvement (#58845)
Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
@@ -24,6 +24,7 @@ outputs are checked within 1 representable-step (ULP); bf16 norm/RoPE outputs us
|
||||
rtol/atol=1e-2 (the tolerance the sibling deepseek_v4 fused-kernel test uses).
|
||||
"""
|
||||
|
||||
from functools import partial
|
||||
from typing import NamedTuple
|
||||
|
||||
import pytest
|
||||
@@ -880,22 +881,36 @@ def test_fused_q_no_indexer(num_tokens: int, cfg: ModelConfig):
|
||||
@pytest.mark.parametrize("cfg", MODEL_CONFIGS, ids=MODEL_IDS)
|
||||
@pytest.mark.parametrize("num_tokens", [1, 17, 512])
|
||||
@pytest.mark.parametrize("has_indexer", [True, False])
|
||||
def test_fused_q_bf16_query(num_tokens: int, has_indexer: bool, cfg: ModelConfig):
|
||||
@pytest.mark.parametrize("use_mha", [True, False])
|
||||
@pytest.mark.parametrize(
|
||||
"index_interleave,cache_dtype", [(False, torch.float32), (True, torch.bfloat16)]
|
||||
)
|
||||
@pytest.mark.parametrize("capture", [False, True])
|
||||
def test_fused_q_bf16_query(
|
||||
num_tokens: int,
|
||||
has_indexer: bool,
|
||||
use_mha: bool,
|
||||
index_interleave: bool,
|
||||
cache_dtype: torch.dtype,
|
||||
capture: bool,
|
||||
cfg: ModelConfig,
|
||||
):
|
||||
"""bf16-query path (FlashMLA sparse, SM90/SM100): only the RoPE'd q_pe is
|
||||
produced (bf16, unquantized); ql_nope is consumed directly by the caller."""
|
||||
produced; MHA does not need an absorbed ql_nope projection."""
|
||||
torch.manual_seed(6)
|
||||
dev = "cuda"
|
||||
max_pos = 8192
|
||||
pos = torch.arange(num_tokens, device=dev, dtype=torch.int64) % max_pos
|
||||
|
||||
# Attention takes q_pe as a strided slice of the full Q projection.
|
||||
q_pe = torch.randn(
|
||||
num_tokens, cfg.num_heads, cfg.rope_dim, device=dev, dtype=torch.bfloat16
|
||||
)
|
||||
num_tokens, cfg.num_heads, 4 * cfg.rope_dim, device=dev, dtype=torch.bfloat16
|
||||
)[..., -cfg.rope_dim :]
|
||||
ql_nope = torch.randn(
|
||||
num_tokens, cfg.num_heads, cfg.kv_lora, device=dev, dtype=torch.bfloat16
|
||||
)
|
||||
q_scale = torch.tensor([0.37], device=dev, dtype=torch.float32)
|
||||
q_cos_sin = make_cos_sin(max_pos, cfg.rope_dim, dev)
|
||||
q_cos_sin = make_cos_sin(max_pos, cfg.rope_dim, dev).to(cache_dtype)
|
||||
|
||||
index_q = index_w = idx_cos_sin = None
|
||||
if has_indexer:
|
||||
@@ -909,24 +924,35 @@ def test_fused_q_bf16_query(num_tokens: int, has_indexer: bool, cfg: ModelConfig
|
||||
index_w = torch.randn(
|
||||
num_tokens, cfg.index_heads, device=dev, dtype=torch.float32
|
||||
)
|
||||
idx_cos_sin = make_cos_sin(max_pos, cfg.rope_dim, dev)
|
||||
idx_cos_sin = make_cos_sin(max_pos, cfg.rope_dim, dev).to(cache_dtype)
|
||||
|
||||
iq_fp8, iw_out, q_pe_out = K.fused_q(
|
||||
run = partial(
|
||||
K.fused_q,
|
||||
pos,
|
||||
q_pe,
|
||||
q_cos_sin,
|
||||
index_q,
|
||||
idx_cos_sin,
|
||||
ql_nope,
|
||||
q_pe if use_mha else ql_nope,
|
||||
q_scale,
|
||||
index_w,
|
||||
cfg.index_head_dim**-0.5,
|
||||
cfg.index_heads**-0.5,
|
||||
has_indexer=has_indexer,
|
||||
index_rope_interleave=False,
|
||||
index_rope_interleave=index_interleave,
|
||||
quantize_mqa=False,
|
||||
)
|
||||
|
||||
iq_fp8, iw_out, q_pe_out = run()
|
||||
if capture:
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
iq_fp8, iw_out, q_pe_out = run()
|
||||
q_pe.mul_(0.5)
|
||||
if index_q is not None:
|
||||
index_q.mul_(1.5)
|
||||
graph.replay()
|
||||
|
||||
# MQA query: only the RoPE'd q_pe, bf16, unquantized.
|
||||
assert q_pe_out.dtype == torch.bfloat16
|
||||
assert q_pe_out.shape == (num_tokens, cfg.num_heads, cfg.rope_dim)
|
||||
@@ -945,7 +971,7 @@ def test_fused_q_bf16_query(num_tokens: int, has_indexer: bool, cfg: ModelConfig
|
||||
index_q.float(),
|
||||
pos.unsqueeze(-1).expand(num_tokens, cfg.index_heads),
|
||||
idx_cos_sin,
|
||||
interleave=False,
|
||||
interleave=index_interleave,
|
||||
)
|
||||
q_ref, scale_ref = ue8m0_quant(iq_ref)
|
||||
assert_fp8(iq_fp8, q_ref, "indexer-Q fp8 (bf16-query path)")
|
||||
|
||||
@@ -267,9 +267,11 @@ def test_skipped_k_cache_insert_accepts_no_k(
|
||||
|
||||
|
||||
@pytest.mark.parametrize("fp8_query", [False, True])
|
||||
@pytest.mark.parametrize("prepared_mha", [False, True])
|
||||
def test_deepseek_v32_dispatches_selected_mha(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
fp8_query: bool,
|
||||
prepared_mha: bool,
|
||||
) -> None:
|
||||
attn_metadata = SimpleNamespace(num_actual_tokens=2)
|
||||
kv_cache = torch.empty(1)
|
||||
@@ -301,7 +303,11 @@ def test_deepseek_v32_dispatches_selected_mha(
|
||||
q_nope = torch.randn(2, 1, 2)
|
||||
q_pe = torch.randn(2, 1, 2)
|
||||
mqa_q = torch.randn(2, 1, 2)
|
||||
mha_q = torch.cat((q_nope, q_pe + 1 if fp8_query else mqa_q), dim=-1)
|
||||
if fp8_query and not prepared_mha:
|
||||
mqa_q = mqa_q.to(torch.float8_e4m3fn)
|
||||
mha_q = torch.cat(
|
||||
(q_nope, q_pe + 1 if fp8_query and not prepared_mha else mqa_q), dim=-1
|
||||
)
|
||||
kv_c = torch.empty(2, 2)
|
||||
k_pe = torch.empty(2, 2)
|
||||
output = torch.empty(2, 2)
|
||||
|
||||
@@ -7,7 +7,7 @@ import torch.nn as nn
|
||||
from transformers import DeepseekV2Config, DeepseekV3Config
|
||||
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.config import CacheConfig, VllmConfig
|
||||
from vllm.config import CacheConfig, CUDAGraphMode, VllmConfig
|
||||
from vllm.distributed import get_tensor_model_parallel_world_size
|
||||
from vllm.distributed.parallel_state import get_tp_group
|
||||
from vllm.forward_context import get_forward_context
|
||||
@@ -283,6 +283,18 @@ class DeepseekV32Attention(MLAAttention):
|
||||
is_neox_style=not getattr(config, "indexer_rope_interleave", False),
|
||||
)
|
||||
|
||||
def _should_prepare_mqa_query(
|
||||
self,
|
||||
attn_metadata: "MLACommonMetadata | None",
|
||||
cudagraph_runtime_mode: CUDAGraphMode,
|
||||
) -> bool:
|
||||
return (
|
||||
attn_metadata is None
|
||||
or not self._use_sparse_mha(attn_metadata)
|
||||
or cudagraph_runtime_mode == CUDAGraphMode.FULL
|
||||
or torch.cuda.is_current_stream_capturing()
|
||||
)
|
||||
|
||||
def forward( # type: ignore[override]
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
@@ -321,6 +333,9 @@ class DeepseekV32Attention(MLAAttention):
|
||||
layer_attn_metadata, _, _, _ = get_attention_context(self.layer_name)
|
||||
self.impl.prepare_for_batch(layer_attn_metadata)
|
||||
|
||||
prepare_mqa_query = self._should_prepare_mqa_query(
|
||||
layer_attn_metadata, forward_context.cudagraph_runtime_mode
|
||||
)
|
||||
if self.indexer is not None and not self.skip_topk:
|
||||
has_indexer = True
|
||||
indexer_k_norm_w = self.indexer.k_norm.weight
|
||||
@@ -404,7 +419,10 @@ class DeepseekV32Attention(MLAAttention):
|
||||
|
||||
q = self.q_b_proj(q_c)[0].view(-1, self.num_local_heads, self.qk_head_dim)
|
||||
q_nope, q_pe = q.split([self.qk_nope_head_dim, self.qk_rope_head_dim], dim=-1)
|
||||
ql_nope = torch.bmm(q_nope.transpose(0, 1), self.W_UK_T).transpose(0, 1)
|
||||
if prepare_mqa_query:
|
||||
ql_nope = torch.bmm(q_nope.transpose(0, 1), self.W_UK_T).transpose(0, 1)
|
||||
else:
|
||||
ql_nope = q_nope
|
||||
|
||||
if self.indexer is not None and not self.skip_topk:
|
||||
index_q = self.indexer.wq_b(q_c)[0]
|
||||
@@ -425,7 +443,7 @@ class DeepseekV32Attention(MLAAttention):
|
||||
indexer_n_head_scale,
|
||||
has_indexer=has_indexer,
|
||||
index_rope_interleave=self._index_rope_interleave,
|
||||
quantize_mqa=self._fp8_query,
|
||||
quantize_mqa=self._fp8_query and prepare_mqa_query,
|
||||
)
|
||||
|
||||
self._sparse_indexer_and_attn(
|
||||
@@ -535,7 +553,11 @@ class DeepseekV32Attention(MLAAttention):
|
||||
|
||||
if self._use_sparse_mha(attn_metadata):
|
||||
assert kv_c is not None and k_pe is not None
|
||||
mha_q_pe = self.rotary_emb(positions, q_pe)[0] if self._fp8_query else mqa_q
|
||||
mha_q_pe = (
|
||||
self.rotary_emb(positions, q_pe)[0]
|
||||
if mqa_q.dtype != q_pe.dtype
|
||||
else mqa_q
|
||||
)
|
||||
mha_q = torch.cat((q_nope, mha_q_pe), dim=-1)
|
||||
self.forward_impl(
|
||||
mha_q,
|
||||
|
||||
@@ -1052,6 +1052,7 @@ def fused_q(
|
||||
index_weights_out,
|
||||
has_indexer=has_indexer,
|
||||
index_rope_interleave=index_rope_interleave,
|
||||
quantize_mqa=quantize_mqa,
|
||||
)
|
||||
return index_q_fp8, index_weights_out, mqa_q
|
||||
|
||||
|
||||
@@ -35,10 +35,12 @@ def is_fused_q_cutedsl_supported(
|
||||
) -> bool:
|
||||
if not (
|
||||
current_platform.has_device_capability(100)
|
||||
and quantize_mqa
|
||||
and q_pe.dtype == ql_nope.dtype == torch.bfloat16
|
||||
and q_pe.dtype == torch.bfloat16
|
||||
and q_pe.shape[-1] == 64
|
||||
and ql_nope.shape[-1] == 512
|
||||
and (
|
||||
not quantize_mqa
|
||||
or (ql_nope.dtype == torch.bfloat16 and ql_nope.shape[-1] == 512)
|
||||
)
|
||||
# One warp per head group; a non-multiple would trip the kernel's own
|
||||
# assert instead of falling back to Triton.
|
||||
and q_pe.shape[1] % 4 == 0
|
||||
@@ -68,9 +70,10 @@ def fused_q_cutedsl(
|
||||
idx_weights_out: torch.Tensor,
|
||||
has_indexer: bool = True,
|
||||
index_rope_interleave: bool = True,
|
||||
quantize_mqa: bool = True,
|
||||
) -> None:
|
||||
_, num_heads, rope_dim = q_pe.shape
|
||||
_, _, nope_dim = ql_nope.shape
|
||||
nope_dim = ql_nope.shape[-1] if quantize_mqa else 0
|
||||
_, num_idx_heads, idx_dim = idx_q.shape
|
||||
|
||||
if has_indexer:
|
||||
@@ -98,8 +101,8 @@ def fused_q_cutedsl(
|
||||
positions,
|
||||
q_pe,
|
||||
rope_cache,
|
||||
ql_nope,
|
||||
q_scale.view(1),
|
||||
ql_nope if quantize_mqa else None,
|
||||
q_scale.view(1) if quantize_mqa else None,
|
||||
mqa_output,
|
||||
idx_q,
|
||||
idx_rope_cache,
|
||||
@@ -121,7 +124,7 @@ class FusedQKernel:
|
||||
index_rope_interleave: bool,
|
||||
) -> None:
|
||||
assert rope_dim == 64
|
||||
assert nope_dim == 512
|
||||
assert nope_dim in (0, 512)
|
||||
assert idx_dim in (128, 0)
|
||||
|
||||
self.rope_dim = rope_dim
|
||||
@@ -262,6 +265,17 @@ class FusedQKernel:
|
||||
cute.arch.griddepcontrol_wait()
|
||||
|
||||
pos = positions[token_id]
|
||||
# no need to do Q calculation for MHA
|
||||
if cutlass.const_expr(self.nope_dim == 0):
|
||||
q0, q1 = cvt.bf16x2_to_fp32x2(
|
||||
cute.recast_tensor(q_pe, Uint32)[token_id, head_id, lane_id]
|
||||
)
|
||||
cos = q_pe_rope_cache[pos, lane_id].to(Float32)
|
||||
sin = q_pe_rope_cache[pos, 32 + lane_id].to(Float32)
|
||||
result = cvt.fp32x2_to_bf16x2(q0 * cos - q1 * sin, q1 * cos + q0 * sin)
|
||||
cute.arch.griddepcontrol_launch_dependents()
|
||||
cute.recast_tensor(mqa_output, Uint32)[token_id, head_id, lane_id] = result
|
||||
return
|
||||
inv_scale = 1.0 / q_scale[0]
|
||||
|
||||
cp_op = cute.nvgpu.CopyUniversalOp()
|
||||
@@ -479,12 +493,16 @@ class FusedQKernel:
|
||||
BFloat16, (num_tokens, num_heads, rope_dim), divisibility=16
|
||||
)
|
||||
rope_cache = _make_fake_tensor(rope_type, (max_pos, rope_dim), divisibility=8)
|
||||
ql_nope = _make_fake_tensor(
|
||||
BFloat16, (num_tokens, num_heads, nope_dim), divisibility=16
|
||||
ql_nope = (
|
||||
_make_fake_tensor(
|
||||
BFloat16, (num_tokens, num_heads, nope_dim), divisibility=16
|
||||
)
|
||||
if nope_dim
|
||||
else None
|
||||
)
|
||||
q_scale = _make_fake_tensor(Float32, (1,), divisibility=4)
|
||||
q_scale = _make_fake_tensor(Float32, (1,), divisibility=4) if nope_dim else None
|
||||
mqa_output = _make_fake_tensor(
|
||||
Float8E4M3FN,
|
||||
Float8E4M3FN if nope_dim else BFloat16,
|
||||
(num_tokens, num_heads, nope_dim + rope_dim),
|
||||
divisibility=16,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user