diff --git a/tests/kernels/test_fused_deepseek_v32_norm_rope.py b/tests/kernels/test_fused_deepseek_v32_norm_rope.py index df9d45a99d..63bbe0c95f 100644 --- a/tests/kernels/test_fused_deepseek_v32_norm_rope.py +++ b/tests/kernels/test_fused_deepseek_v32_norm_rope.py @@ -24,6 +24,7 @@ outputs are checked within 1 representable-step (ULP); bf16 norm/RoPE outputs us rtol/atol=1e-2 (the tolerance the sibling deepseek_v4 fused-kernel test uses). """ +from functools import partial from typing import NamedTuple import pytest @@ -880,22 +881,36 @@ def test_fused_q_no_indexer(num_tokens: int, cfg: ModelConfig): @pytest.mark.parametrize("cfg", MODEL_CONFIGS, ids=MODEL_IDS) @pytest.mark.parametrize("num_tokens", [1, 17, 512]) @pytest.mark.parametrize("has_indexer", [True, False]) -def test_fused_q_bf16_query(num_tokens: int, has_indexer: bool, cfg: ModelConfig): +@pytest.mark.parametrize("use_mha", [True, False]) +@pytest.mark.parametrize( + "index_interleave,cache_dtype", [(False, torch.float32), (True, torch.bfloat16)] +) +@pytest.mark.parametrize("capture", [False, True]) +def test_fused_q_bf16_query( + num_tokens: int, + has_indexer: bool, + use_mha: bool, + index_interleave: bool, + cache_dtype: torch.dtype, + capture: bool, + cfg: ModelConfig, +): """bf16-query path (FlashMLA sparse, SM90/SM100): only the RoPE'd q_pe is - produced (bf16, unquantized); ql_nope is consumed directly by the caller.""" + produced; MHA does not need an absorbed ql_nope projection.""" torch.manual_seed(6) dev = "cuda" max_pos = 8192 pos = torch.arange(num_tokens, device=dev, dtype=torch.int64) % max_pos + # Attention takes q_pe as a strided slice of the full Q projection. q_pe = torch.randn( - num_tokens, cfg.num_heads, cfg.rope_dim, device=dev, dtype=torch.bfloat16 - ) + num_tokens, cfg.num_heads, 4 * cfg.rope_dim, device=dev, dtype=torch.bfloat16 + )[..., -cfg.rope_dim :] ql_nope = torch.randn( num_tokens, cfg.num_heads, cfg.kv_lora, device=dev, dtype=torch.bfloat16 ) q_scale = torch.tensor([0.37], device=dev, dtype=torch.float32) - q_cos_sin = make_cos_sin(max_pos, cfg.rope_dim, dev) + q_cos_sin = make_cos_sin(max_pos, cfg.rope_dim, dev).to(cache_dtype) index_q = index_w = idx_cos_sin = None if has_indexer: @@ -909,24 +924,35 @@ def test_fused_q_bf16_query(num_tokens: int, has_indexer: bool, cfg: ModelConfig index_w = torch.randn( num_tokens, cfg.index_heads, device=dev, dtype=torch.float32 ) - idx_cos_sin = make_cos_sin(max_pos, cfg.rope_dim, dev) + idx_cos_sin = make_cos_sin(max_pos, cfg.rope_dim, dev).to(cache_dtype) - iq_fp8, iw_out, q_pe_out = K.fused_q( + run = partial( + K.fused_q, pos, q_pe, q_cos_sin, index_q, idx_cos_sin, - ql_nope, + q_pe if use_mha else ql_nope, q_scale, index_w, cfg.index_head_dim**-0.5, cfg.index_heads**-0.5, has_indexer=has_indexer, - index_rope_interleave=False, + index_rope_interleave=index_interleave, quantize_mqa=False, ) + iq_fp8, iw_out, q_pe_out = run() + if capture: + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + iq_fp8, iw_out, q_pe_out = run() + q_pe.mul_(0.5) + if index_q is not None: + index_q.mul_(1.5) + graph.replay() + # MQA query: only the RoPE'd q_pe, bf16, unquantized. assert q_pe_out.dtype == torch.bfloat16 assert q_pe_out.shape == (num_tokens, cfg.num_heads, cfg.rope_dim) @@ -945,7 +971,7 @@ def test_fused_q_bf16_query(num_tokens: int, has_indexer: bool, cfg: ModelConfig index_q.float(), pos.unsqueeze(-1).expand(num_tokens, cfg.index_heads), idx_cos_sin, - interleave=False, + interleave=index_interleave, ) q_ref, scale_ref = ue8m0_quant(iq_ref) assert_fp8(iq_fp8, q_ref, "indexer-Q fp8 (bf16-query path)") diff --git a/tests/model_executor/layers/test_mla_short_prefill_indexer.py b/tests/model_executor/layers/test_mla_short_prefill_indexer.py index 77c52fb291..2e4d4d3d67 100644 --- a/tests/model_executor/layers/test_mla_short_prefill_indexer.py +++ b/tests/model_executor/layers/test_mla_short_prefill_indexer.py @@ -267,9 +267,11 @@ def test_skipped_k_cache_insert_accepts_no_k( @pytest.mark.parametrize("fp8_query", [False, True]) +@pytest.mark.parametrize("prepared_mha", [False, True]) def test_deepseek_v32_dispatches_selected_mha( monkeypatch: pytest.MonkeyPatch, fp8_query: bool, + prepared_mha: bool, ) -> None: attn_metadata = SimpleNamespace(num_actual_tokens=2) kv_cache = torch.empty(1) @@ -301,7 +303,11 @@ def test_deepseek_v32_dispatches_selected_mha( q_nope = torch.randn(2, 1, 2) q_pe = torch.randn(2, 1, 2) mqa_q = torch.randn(2, 1, 2) - mha_q = torch.cat((q_nope, q_pe + 1 if fp8_query else mqa_q), dim=-1) + if fp8_query and not prepared_mha: + mqa_q = mqa_q.to(torch.float8_e4m3fn) + mha_q = torch.cat( + (q_nope, q_pe + 1 if fp8_query and not prepared_mha else mqa_q), dim=-1 + ) kv_c = torch.empty(2, 2) k_pe = torch.empty(2, 2) output = torch.empty(2, 2) diff --git a/vllm/models/deepseek_v32/attention.py b/vllm/models/deepseek_v32/attention.py index 4e63076414..d352002009 100644 --- a/vllm/models/deepseek_v32/attention.py +++ b/vllm/models/deepseek_v32/attention.py @@ -7,7 +7,7 @@ import torch.nn as nn from transformers import DeepseekV2Config, DeepseekV3Config from vllm.compilation.breakable_cudagraph import eager_break_during_capture -from vllm.config import CacheConfig, VllmConfig +from vllm.config import CacheConfig, CUDAGraphMode, VllmConfig from vllm.distributed import get_tensor_model_parallel_world_size from vllm.distributed.parallel_state import get_tp_group from vllm.forward_context import get_forward_context @@ -283,6 +283,18 @@ class DeepseekV32Attention(MLAAttention): is_neox_style=not getattr(config, "indexer_rope_interleave", False), ) + def _should_prepare_mqa_query( + self, + attn_metadata: "MLACommonMetadata | None", + cudagraph_runtime_mode: CUDAGraphMode, + ) -> bool: + return ( + attn_metadata is None + or not self._use_sparse_mha(attn_metadata) + or cudagraph_runtime_mode == CUDAGraphMode.FULL + or torch.cuda.is_current_stream_capturing() + ) + def forward( # type: ignore[override] self, positions: torch.Tensor, @@ -321,6 +333,9 @@ class DeepseekV32Attention(MLAAttention): layer_attn_metadata, _, _, _ = get_attention_context(self.layer_name) self.impl.prepare_for_batch(layer_attn_metadata) + prepare_mqa_query = self._should_prepare_mqa_query( + layer_attn_metadata, forward_context.cudagraph_runtime_mode + ) if self.indexer is not None and not self.skip_topk: has_indexer = True indexer_k_norm_w = self.indexer.k_norm.weight @@ -404,7 +419,10 @@ class DeepseekV32Attention(MLAAttention): q = self.q_b_proj(q_c)[0].view(-1, self.num_local_heads, self.qk_head_dim) q_nope, q_pe = q.split([self.qk_nope_head_dim, self.qk_rope_head_dim], dim=-1) - ql_nope = torch.bmm(q_nope.transpose(0, 1), self.W_UK_T).transpose(0, 1) + if prepare_mqa_query: + ql_nope = torch.bmm(q_nope.transpose(0, 1), self.W_UK_T).transpose(0, 1) + else: + ql_nope = q_nope if self.indexer is not None and not self.skip_topk: index_q = self.indexer.wq_b(q_c)[0] @@ -425,7 +443,7 @@ class DeepseekV32Attention(MLAAttention): indexer_n_head_scale, has_indexer=has_indexer, index_rope_interleave=self._index_rope_interleave, - quantize_mqa=self._fp8_query, + quantize_mqa=self._fp8_query and prepare_mqa_query, ) self._sparse_indexer_and_attn( @@ -535,7 +553,11 @@ class DeepseekV32Attention(MLAAttention): if self._use_sparse_mha(attn_metadata): assert kv_c is not None and k_pe is not None - mha_q_pe = self.rotary_emb(positions, q_pe)[0] if self._fp8_query else mqa_q + mha_q_pe = ( + self.rotary_emb(positions, q_pe)[0] + if mqa_q.dtype != q_pe.dtype + else mqa_q + ) mha_q = torch.cat((q_nope, mha_q_pe), dim=-1) self.forward_impl( mha_q, diff --git a/vllm/models/deepseek_v32/common/kernels.py b/vllm/models/deepseek_v32/common/kernels.py index e0259c46fa..255b1f5aaf 100644 --- a/vllm/models/deepseek_v32/common/kernels.py +++ b/vllm/models/deepseek_v32/common/kernels.py @@ -1052,6 +1052,7 @@ def fused_q( index_weights_out, has_indexer=has_indexer, index_rope_interleave=index_rope_interleave, + quantize_mqa=quantize_mqa, ) return index_q_fp8, index_weights_out, mqa_q diff --git a/vllm/models/deepseek_v32/nvidia/ops/fused_q_cutedsl.py b/vllm/models/deepseek_v32/nvidia/ops/fused_q_cutedsl.py index dec1e6f034..e0c58a7e78 100644 --- a/vllm/models/deepseek_v32/nvidia/ops/fused_q_cutedsl.py +++ b/vllm/models/deepseek_v32/nvidia/ops/fused_q_cutedsl.py @@ -35,10 +35,12 @@ def is_fused_q_cutedsl_supported( ) -> bool: if not ( current_platform.has_device_capability(100) - and quantize_mqa - and q_pe.dtype == ql_nope.dtype == torch.bfloat16 + and q_pe.dtype == torch.bfloat16 and q_pe.shape[-1] == 64 - and ql_nope.shape[-1] == 512 + and ( + not quantize_mqa + or (ql_nope.dtype == torch.bfloat16 and ql_nope.shape[-1] == 512) + ) # One warp per head group; a non-multiple would trip the kernel's own # assert instead of falling back to Triton. and q_pe.shape[1] % 4 == 0 @@ -68,9 +70,10 @@ def fused_q_cutedsl( idx_weights_out: torch.Tensor, has_indexer: bool = True, index_rope_interleave: bool = True, + quantize_mqa: bool = True, ) -> None: _, num_heads, rope_dim = q_pe.shape - _, _, nope_dim = ql_nope.shape + nope_dim = ql_nope.shape[-1] if quantize_mqa else 0 _, num_idx_heads, idx_dim = idx_q.shape if has_indexer: @@ -98,8 +101,8 @@ def fused_q_cutedsl( positions, q_pe, rope_cache, - ql_nope, - q_scale.view(1), + ql_nope if quantize_mqa else None, + q_scale.view(1) if quantize_mqa else None, mqa_output, idx_q, idx_rope_cache, @@ -121,7 +124,7 @@ class FusedQKernel: index_rope_interleave: bool, ) -> None: assert rope_dim == 64 - assert nope_dim == 512 + assert nope_dim in (0, 512) assert idx_dim in (128, 0) self.rope_dim = rope_dim @@ -262,6 +265,17 @@ class FusedQKernel: cute.arch.griddepcontrol_wait() pos = positions[token_id] + # no need to do Q calculation for MHA + if cutlass.const_expr(self.nope_dim == 0): + q0, q1 = cvt.bf16x2_to_fp32x2( + cute.recast_tensor(q_pe, Uint32)[token_id, head_id, lane_id] + ) + cos = q_pe_rope_cache[pos, lane_id].to(Float32) + sin = q_pe_rope_cache[pos, 32 + lane_id].to(Float32) + result = cvt.fp32x2_to_bf16x2(q0 * cos - q1 * sin, q1 * cos + q0 * sin) + cute.arch.griddepcontrol_launch_dependents() + cute.recast_tensor(mqa_output, Uint32)[token_id, head_id, lane_id] = result + return inv_scale = 1.0 / q_scale[0] cp_op = cute.nvgpu.CopyUniversalOp() @@ -479,12 +493,16 @@ class FusedQKernel: BFloat16, (num_tokens, num_heads, rope_dim), divisibility=16 ) rope_cache = _make_fake_tensor(rope_type, (max_pos, rope_dim), divisibility=8) - ql_nope = _make_fake_tensor( - BFloat16, (num_tokens, num_heads, nope_dim), divisibility=16 + ql_nope = ( + _make_fake_tensor( + BFloat16, (num_tokens, num_heads, nope_dim), divisibility=16 + ) + if nope_dim + else None ) - q_scale = _make_fake_tensor(Float32, (1,), divisibility=4) + q_scale = _make_fake_tensor(Float32, (1,), divisibility=4) if nope_dim else None mqa_output = _make_fake_tensor( - Float8E4M3FN, + Float8E4M3FN if nope_dim else BFloat16, (num_tokens, num_heads, nope_dim + rope_dim), divisibility=16, )