From 5f7a20b3162ef88d531ada03aa1174643bb97c11 Mon Sep 17 00:00:00 2001 From: Summer Yang Date: Tue, 18 Aug 2026 14:59:08 -0700 Subject: [PATCH] [nv] add pcp support in dsv3.2 (#52046) Signed-off-by: Summer Yang --- .../test_fused_deepseek_v32_norm_rope.py | 96 +++++++- .../layers/test_mla_short_prefill_indexer.py | 45 ++++ .../layers/sparse_attn_indexer.py | 12 +- vllm/models/deepseek_v32/attention.py | 123 ++++++++-- vllm/models/deepseek_v32/common/kernels.py | 215 +++++++++++------- .../v1/attention/ops/rocm_aiter_mla_sparse.py | 4 +- 6 files changed, 376 insertions(+), 119 deletions(-) diff --git a/tests/kernels/test_fused_deepseek_v32_norm_rope.py b/tests/kernels/test_fused_deepseek_v32_norm_rope.py index d72c1b31c4..7a00d0c3d6 100644 --- a/tests/kernels/test_fused_deepseek_v32_norm_rope.py +++ b/tests/kernels/test_fused_deepseek_v32_norm_rope.py @@ -292,6 +292,81 @@ def test_fused_norm_rope_no_indexer(num_tokens: int): assert (topk == 7).all(), "topk buffer should be untouched on shared layer" +@pytest.mark.parametrize("has_indexer", [False, True]) +def test_fused_norm_rope_materializes_pcp_cache_inputs(has_indexer: bool): + """PCP gets local normalized/rotated K rows without direct cache writes.""" + torch.manual_seed(6) + dev = "cuda" + num_tokens = 17 + max_pos = 8192 + pos = torch.arange(num_tokens, device=dev, dtype=torch.int64) + q_c = torch.randn(num_tokens, Q_LORA, device=dev, dtype=torch.bfloat16) + kv_c = torch.randn(num_tokens, KV_LORA, device=dev, dtype=torch.bfloat16) + k_pe = torch.randn(num_tokens, ROPE_DIM, device=dev, dtype=torch.bfloat16) + qw = torch.randn(Q_LORA, device=dev, dtype=torch.bfloat16) + kvw = torch.randn(KV_LORA, device=dev, dtype=torch.bfloat16) + ik = ( + torch.randn(num_tokens, INDEX_HEAD_DIM, device=dev, dtype=torch.bfloat16) + if has_indexer + else None + ) + ikw = ( + torch.randn(INDEX_HEAD_DIM, device=dev, dtype=torch.float32) + if has_indexer + else None + ) + ikb = ( + torch.randn(INDEX_HEAD_DIM, device=dev, dtype=torch.float32) + if has_indexer + else None + ) + mla_cos_sin = make_cos_sin(max_pos, ROPE_DIM, dev) + idx_cos_sin = make_cos_sin(max_pos, ROPE_DIM, dev) if has_indexer else None + q_out = torch.empty_like(q_c) + kv_out = torch.empty_like(kv_c) + kpe_out = torch.empty_like(k_pe) + ik_out = torch.empty_like(ik) if ik is not None else None + topk = torch.full((num_tokens, 2048), 7, device=dev, dtype=torch.int32) + + actual_q = K.fused_norm_rope( + pos, + q_c, + qw, + EPS, + kv_c, + kvw, + EPS, + k_pe, + mla_cos_sin, + ik, + ikw, + ikb, + EPS, + idx_cos_sin, + topk, + has_indexer=has_indexer, + index_rope_interleave=True, + q_c_out=q_out, + kv_c_out=kv_out, + k_pe_out=kpe_out, + index_k_out=ik_out, + ) + + assert actual_q.data_ptr() == q_out.data_ptr() + assert_bf16(actual_q, rms_norm(q_c, qw), "PCP q norm") + assert_bf16(kv_out, rms_norm(kv_c, kvw), "PCP kv norm") + assert_bf16( + kpe_out, + rope(k_pe.float(), pos, mla_cos_sin, interleave=True), + "PCP k_pe RoPE", + ) + if has_indexer: + assert ik is not None and ikw is not None and ikb is not None + assert ik_out is not None and idx_cos_sin is not None + ik_ref = rope(layer_norm(ik, ikw, ikb), pos, idx_cos_sin, interleave=True) + assert_bf16(ik_out, ik_ref, "PCP indexer-K") + + @pytest.mark.parametrize("num_tokens", [1, 4, 17, 512]) def test_fused_norm_rope_ds_mla(num_tokens: int): """fp8_ds_mla MLA cache layout (FlashMLA sparse, bf16-query path; SM90/SM100). @@ -405,19 +480,29 @@ def test_fused_norm_rope_supports_large_token_count(): # ── fused_q ────────────────────────────────────────────────────────────────── -@pytest.mark.parametrize("num_tokens", [1, 4, 17, 512, 4096]) +@pytest.mark.parametrize( + ("num_tokens", "num_q_heads"), + [ + (1, NUM_HEADS), + (4, NUM_HEADS), + (17, NUM_HEADS), + (512, NUM_HEADS), + (4096, NUM_HEADS), + (17, 64), + ], +) @pytest.mark.parametrize("index_interleave", [True, False]) -def test_fused_q(num_tokens: int, index_interleave: bool): +def test_fused_q(num_tokens: int, num_q_heads: int, index_interleave: bool): torch.manual_seed(2) dev = "cuda" max_pos = 8192 pos = torch.arange(num_tokens, device=dev, dtype=torch.int64) % max_pos q_pe = torch.randn( - num_tokens, NUM_HEADS, ROPE_DIM, device=dev, dtype=torch.bfloat16 + num_tokens, num_q_heads, ROPE_DIM, device=dev, dtype=torch.bfloat16 ) ql_nope = torch.randn( - num_tokens, NUM_HEADS, KV_LORA, device=dev, dtype=torch.bfloat16 + num_tokens, num_q_heads, KV_LORA, device=dev, dtype=torch.bfloat16 ) index_q = torch.randn( num_tokens, INDEX_HEADS, INDEX_HEAD_DIM, device=dev, dtype=torch.bfloat16 @@ -449,7 +534,7 @@ def test_fused_q(num_tokens: int, index_interleave: bool): mqa_nope_ref = (ql_nope.float() / s).to(FP8) qpe_ref = rope( q_pe.float(), - pos.unsqueeze(-1).expand(num_tokens, NUM_HEADS), + pos.unsqueeze(-1).expand(num_tokens, num_q_heads), q_cos_sin, interleave=True, ) @@ -484,7 +569,6 @@ def test_fused_q_no_indexer(num_tokens: int): ) q_scale = torch.tensor([0.5], device=dev, dtype=torch.float32) q_cos_sin = make_cos_sin(max_pos, ROPE_DIM, dev) - _, _, mqa = K.fused_q( pos, q_pe, diff --git a/tests/model_executor/layers/test_mla_short_prefill_indexer.py b/tests/model_executor/layers/test_mla_short_prefill_indexer.py index 47bfcd0b38..e798d8c7f5 100644 --- a/tests/model_executor/layers/test_mla_short_prefill_indexer.py +++ b/tests/model_executor/layers/test_mla_short_prefill_indexer.py @@ -177,3 +177,48 @@ def test_short_prefill_updates_k_cache_before_scoring_decision( # K cache is always updated before the scoring decision. torch.testing.assert_close(observed["k"], k[: slot_mapping.numel()]) assert observed["slots"] is slot_mapping + + +def test_skipped_k_cache_insert_accepts_no_k( + monkeypatch: pytest.MonkeyPatch, +) -> None: + indexer_metadata = make_indexer_metadata( + num_prefills=0, + num_prefill_tokens=0, + slot_mapping=torch.empty(0, dtype=torch.long), + ) + monkeypatch.setattr( + sparse_indexer, + "get_forward_context", + lambda: SimpleNamespace( + attn_metadata={INDEXER_LAYER: indexer_metadata}, + cudagraph_runtime_mode=CUDAGraphMode.PIECEWISE, + ), + ) + monkeypatch.setattr( + sparse_indexer.current_platform, "fp8_dtype", lambda: torch.float16 + ) + + topk_indices = torch.full((1, 2048), 17, dtype=torch.int32) + result = sparse_indexer.sparse_attn_indexer( + torch.empty(1, 1), + INDEXER_LAYER, + torch.empty(1), + torch.empty(1, 1), + None, + None, + torch.empty(1, 1), + 128, + "ue8m0", + 2048, + 4, + 4096, + 4096, + topk_indices, + True, + False, + "", + ) + + assert result is topk_indices + assert torch.all(topk_indices == -1) diff --git a/vllm/model_executor/layers/sparse_attn_indexer.py b/vllm/model_executor/layers/sparse_attn_indexer.py index d94b377dbe..4f0e00cfab 100644 --- a/vllm/model_executor/layers/sparse_attn_indexer.py +++ b/vllm/model_executor/layers/sparse_attn_indexer.py @@ -299,7 +299,7 @@ def sparse_attn_indexer( kv_cache: torch.Tensor, q_quant: torch.Tensor, q_scale: torch.Tensor | None, - k: torch.Tensor, + k: torch.Tensor | None, weights: torch.Tensor, quant_block_size: int, scale_fmt: str | None, @@ -695,7 +695,7 @@ def sparse_attn_indexer_fake( kv_cache: torch.Tensor, q_quant: torch.Tensor, q_scale: torch.Tensor | None, - k: torch.Tensor, + k: torch.Tensor | None, weights: torch.Tensor, quant_block_size: int, scale_fmt: str | None, @@ -781,7 +781,7 @@ class SparseAttnIndexer(CustomOp): self, hidden_states: torch.Tensor, q_quant: torch.Tensor | tuple[torch.Tensor, torch.Tensor], - k: torch.Tensor, + k: torch.Tensor | None, weights: torch.Tensor, ): if current_platform.is_cuda() or current_platform.is_xpu(): @@ -798,7 +798,7 @@ class SparseAttnIndexer(CustomOp): self, hidden_states: torch.Tensor, q_quant: torch.Tensor | tuple[torch.Tensor, torch.Tensor], - k: torch.Tensor, + k: torch.Tensor | None, weights: torch.Tensor, ): # FP8 path: single tensor (per-token scale is folded into `weights`). @@ -835,7 +835,7 @@ class SparseAttnIndexer(CustomOp): self, hidden_states: torch.Tensor, q_fp8: torch.Tensor, - k: torch.Tensor, + k: torch.Tensor | None, weights: torch.Tensor, ): return self.forward_cuda(hidden_states, q_fp8, k, weights) @@ -844,7 +844,7 @@ class SparseAttnIndexer(CustomOp): self, hidden_states: torch.Tensor, q_quant: torch.Tensor | tuple[torch.Tensor, torch.Tensor], - k: torch.Tensor, + k: torch.Tensor | None, weights: torch.Tensor, ): assert not self.use_fp4_cache, "AMD platform doesn't support fp4 cache yet" diff --git a/vllm/models/deepseek_v32/attention.py b/vllm/models/deepseek_v32/attention.py index cc27061989..253971b7bd 100644 --- a/vllm/models/deepseek_v32/attention.py +++ b/vllm/models/deepseek_v32/attention.py @@ -1,5 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from typing import TYPE_CHECKING, cast + import torch import torch.nn as nn from transformers import DeepseekV2Config, DeepseekV3Config @@ -7,8 +9,14 @@ from transformers import DeepseekV2Config, DeepseekV3Config from vllm.compilation.breakable_cudagraph import eager_break_during_capture from vllm.config import CacheConfig, VllmConfig from vllm.distributed import get_tensor_model_parallel_world_size +from vllm.distributed.parallel_state import get_tp_group from vllm.forward_context import get_forward_context from vllm.model_executor.layers.attention import MLAAttention +from vllm.model_executor.layers.attention.attention import get_attention_context +from vllm.model_executor.layers.attention.pcp import ( + finalize_mla_pcp_decode, + maybe_gather_mla_latent_cache_inputs, +) from vllm.model_executor.layers.layernorm import LayerNorm, RMSNorm from vllm.model_executor.layers.linear import ( ColumnParallelLinear, @@ -35,6 +43,9 @@ from vllm.models.deepseek_v32.common.kernels import fused_norm_rope, fused_q from vllm.platforms import current_platform from vllm.utils.torch_utils import is_quantized_kv_cache +if TYPE_CHECKING: + from vllm.model_executor.layers.attention.mla_attention import MLACommonMetadata + class DeepseekV32Indexer(nn.Module): indexer_cache_cls = DeepseekV32IndexerCache @@ -363,6 +374,7 @@ class DeepseekV32Attention(MLAAttention): attn_metadata = attn_metadata_raw[0].get(self.layer_name) else: attn_metadata = attn_metadata_raw + attn_metadata = cast("MLACommonMetadata | None", attn_metadata) slot_mapping = forward_context.slot_mapping assert isinstance(slot_mapping, dict) @@ -374,7 +386,8 @@ class DeepseekV32Attention(MLAAttention): indexer_k_norm_bias = self.indexer.k_norm.bias indexer_k_norm_eps = self.indexer.k_norm.eps indexer_k_rope_cos_sin_cache = self.indexer_rope_emb.cos_sin_cache - indexer_k_cache = self.indexer.k_cache.kv_cache + indexer_k_cache = None if self.use_pcp else self.indexer.k_cache.kv_cache + index_k_out = torch.empty_like(index_k) if self.use_pcp else None indexer_softmax_scale = self.indexer.softmax_scale indexer_n_head_scale = self.indexer.n_head**-0.5 else: @@ -384,10 +397,11 @@ class DeepseekV32Attention(MLAAttention): indexer_k_norm_eps = 1e-6 indexer_k_rope_cos_sin_cache = None indexer_k_cache = None + index_k_out = None indexer_softmax_scale = 0.0 indexer_n_head_scale = 0.0 - if attn_metadata is None: + if attn_metadata is None or self.use_pcp: mla_kv_cache = None mla_k_scale = None indexer_k_cache = None @@ -396,6 +410,8 @@ class DeepseekV32Attention(MLAAttention): mla_kv_cache = self.kv_cache mla_k_scale = self._k_scale + kv_c_out = torch.empty_like(kv_c) if self.use_pcp else None + k_pe_out = torch.empty_like(k_pe) if self.use_pcp else None q_c = fused_norm_rope( positions, q_c, @@ -419,12 +435,14 @@ class DeepseekV32Attention(MLAAttention): mla_k_scale=mla_k_scale, has_indexer=has_indexer, index_rope_interleave=self._index_rope_interleave, + kv_c_out=kv_c_out, + k_pe_out=k_pe_out, + index_k_out=index_k_out, ) q = self.q_b_proj(q_c)[0].view(-1, self.num_local_heads, self.qk_head_dim) q_nope, q_pe = q.split([self.qk_nope_head_dim, self.qk_rope_head_dim], dim=-1) - q_nope = q_nope.transpose(0, 1) - ql_nope = torch.bmm(q_nope, self.W_UK_T).transpose(0, 1) + ql_nope = torch.bmm(q_nope.transpose(0, 1), self.W_UK_T).transpose(0, 1) if self.indexer is not None and not self.skip_topk: index_q = self.indexer.wq_b(q_c)[0] @@ -451,7 +469,10 @@ class DeepseekV32Attention(MLAAttention): self._sparse_indexer_and_attn( q_c, index_q_fp8, + index_k_out, index_weights_out, + kv_c_out, + k_pe_out, ql_nope, mqa_q, output, @@ -463,19 +484,26 @@ class DeepseekV32Attention(MLAAttention): self, q_c: torch.Tensor, index_q_fp8: torch.Tensor | None, + index_k: torch.Tensor | None, index_weights_out: torch.Tensor | None, + kv_c: torch.Tensor | None, + k_pe: torch.Tensor | None, ql_nope: torch.Tensor, mqa_q: torch.Tensor, output: torch.Tensor, ) -> None: if self.indexer is not None and not self.skip_topk: + assert index_q_fp8 is not None + assert index_weights_out is not None + if self.use_pcp: + assert index_k is not None sparse_attn_indexer( q_c, self.indexer.k_cache.prefix, self.indexer.k_cache.kv_cache, index_q_fp8, - None, # q_scale folded into weights on the fp8 path - None, # k unused when skip_k_cache_insert=True + None, + index_k, index_weights_out, self.indexer.quant_block_size, self.indexer.scale_fmt, @@ -484,28 +512,56 @@ class DeepseekV32Attention(MLAAttention): self.indexer.max_model_len, self.indexer.max_total_seq_len, self.topk_indices_buffer, - skip_k_cache_insert=True, - use_pcp=False, + skip_k_cache_insert=not self.use_pcp, + use_pcp=self.use_pcp, dense_mha_metadata_layer_name=self._dense_mha_metadata_layer_name, - use_fp4_cache=False, - # fused_norm_rope already cleared the topk buffer this forward. + dcp_rank=( + self.dcp_manager.group.rank_in_group + if self.dcp_manager is not None + else 0 + ), + dcp_world_size=( + self._vllm_config.parallel_config.decode_context_parallel_size + ), + cp_kv_cache_interleave_size=( + self._vllm_config.parallel_config.cp_kv_cache_interleave_size + ), skip_topk_buffer_clear=True, ) - attn_metadata_raw = get_forward_context().attn_metadata - if isinstance(attn_metadata_raw, dict): - attn_metadata = attn_metadata_raw.get(self.layer_name) - elif isinstance(attn_metadata_raw, list): - attn_metadata = attn_metadata_raw[0].get(self.layer_name) - else: - attn_metadata = attn_metadata_raw - + attn_metadata, _, kv_cache, layer_slot_mapping = get_attention_context( + self.layer_name + ) if attn_metadata is None: output.zero_() return + attn_metadata = cast("MLACommonMetadata", attn_metadata) + + if self.use_pcp: + assert kv_c is not None and k_pe is not None + kv_for_cache, kpe_for_cache, cache_slot_mapping = ( + maybe_gather_mla_latent_cache_inputs( + kv_c, + k_pe.unsqueeze(1), + layer_slot_mapping, + attn_metadata.num_decode_tokens, + True, + ) + ) + self.impl.do_kv_cache_update( # type: ignore[attr-defined] + kv_for_cache, + kpe_for_cache, + kv_cache, + cache_slot_mapping, + self.kv_cache_dtype, + self._k_scale, + ) num_actual = attn_metadata.num_actual_tokens # type: ignore[attr-defined] - kv_cache = self.kv_cache + if num_actual == 0: + output.zero_() + return + if self._fp8_kv_needs_view: kv_cache = kv_cache.view(torch.float8_e4m3fn) if self._fp8_query: @@ -515,10 +571,35 @@ class DeepseekV32Attention(MLAAttention): ] else: mqa_q_arg = (ql_nope[:num_actual], mqa_q[:num_actual]) - attn_out, _ = self.impl.forward_mqa( # type: ignore[attr-defined] + + if self.use_pcp and self.impl.dcp_world_size > self.impl.pcp_world_size: + if isinstance(mqa_q_arg, tuple): + mqa_q_arg = torch.cat(mqa_q_arg, dim=-1) + mqa_q_arg = get_tp_group().all_gather(mqa_q_arg, dim=1) + attn_out, lse = self.impl.forward_mqa( # type: ignore[attr-defined] mqa_q_arg, kv_cache, attn_metadata, self ) + if self.use_pcp and self.impl.dcp_world_size > 1: + assert lse is not None and self.dcp_manager is not None + seq_lens = ( + attn_metadata.decode.seq_lens + if attn_metadata.decode is not None + else cast(torch.Tensor, attn_metadata.seq_lens)[ # type: ignore[attr-defined] + : attn_metadata.num_decodes + ] + ) + query_start_loc = attn_metadata.query_start_loc[ + : attn_metadata.num_decodes + 1 + ] + attn_out = self.dcp_manager.combine( + attn_out, + lse, + seq_lens=seq_lens, + query_start_loc=query_start_loc, + ) + attn_out = finalize_mla_pcp_decode(attn_out, self.num_heads) + # NOTE(woosuk): While the below does not need to be in the eager region, # we put it here to avoid copying the attention output. Move this back to the # captured region once forward_mqa supports `out` argument. @@ -531,3 +612,5 @@ class DeepseekV32Attention(MLAAttention): .transpose(0, 1) ) torch.bmm(x, self.W_UV, out=out) + if self.use_pcp and num_actual < output.shape[0]: + output[num_actual:].zero_() diff --git a/vllm/models/deepseek_v32/common/kernels.py b/vllm/models/deepseek_v32/common/kernels.py index ec642409cb..c5c6c97d55 100644 --- a/vllm/models/deepseek_v32/common/kernels.py +++ b/vllm/models/deepseek_v32/common/kernels.py @@ -103,12 +103,16 @@ def _fused_norm_rope_kernel( kv_stride, kv_rms_norm_w_ptr, kv_rms_eps, + kv_out_ptr, + kv_out_stride, KV_DIM: tl.constexpr, # KV RoPE kpe_ptr, kpe_stride, kpe_rope_cos_sin_cache_ptr, kpe_rope_cos_sin_cache_stride, + kpe_out_ptr, + kpe_out_stride, KPE_HALF_ROT_DIM: tl.constexpr, # Index K layer norm index_k_ptr, @@ -121,6 +125,8 @@ def _fused_norm_rope_kernel( # Index K RoPE index_k_rope_cos_sin_cache_ptr, index_k_rope_cos_sin_cache_stride, + index_k_out_ptr, + index_k_out_stride, INDEX_K_HALF_ROT_DIM: tl.constexpr, # Cache params (shared by indexer K and MLA) slot_mapping_ptr, @@ -174,10 +180,9 @@ def _fused_norm_rope_kernel( return if slot_mapping_ptr is None: - # Memory profiling run. - return - slot_idx = tl.load(slot_mapping_ptr + tok_idx) - if slot_idx < 0: + if kv_out_ptr is None and kpe_out_ptr is None and index_k_out_ptr is None: + return + elif tl.load(slot_mapping_ptr + tok_idx) < 0: # Padding return @@ -217,63 +222,81 @@ def _fused_norm_rope_kernel( r1 = x1 * cos - x2 * sin r2 = x2 * cos + x1 * sin - # MLA concat_and_cache: write [kv_c_normed, k_pe_roped] to cache. - if mla_cache_entry_stride == 0: - return + # PCP materializes K for the cross-rank gather, then inserts it into cache. + if kv_out_ptr is not None: + tl.store(kv_out_ptr + tok_idx * kv_out_stride + kv_block, kv_c) + if kpe_out_ptr is not None: + tl.store( + kpe_out_ptr + tok_idx * kpe_out_stride + dim_off * 2, + r1.to(kpe_out_ptr.dtype.element_ty), + ) + tl.store( + kpe_out_ptr + tok_idx * kpe_out_stride + dim_off * 2 + 1, + r2.to(kpe_out_ptr.dtype.element_ty), + ) - mla_block_size = mla_cache_block_stride // mla_cache_entry_stride - mla_block_idx = slot_idx // mla_block_size - mla_block_off = slot_idx % mla_block_size + if slot_mapping_ptr is not None: + # MLA concat_and_cache: write [kv_c_normed, k_pe_roped] to cache. + if mla_cache_entry_stride == 0: + return - if MLA_CACHE_DS_MLA: - # fp8_ds_mla layout (DeepSeek-V3.2, KV_DIM == 512): per-128-element - # tile of the NoPE is dynamically quantized to fp8 with its own - # float32 scale; the RoPE tail is stored unquantized in bf16. - # bytes [0, KV_DIM) : KV_DIM fp8 NoPE values - # bytes [KV_DIM, KV_DIM + 16) : MLA_NUM_TILES float32 scales - # bytes [KV_DIM + 16, ...) : 2 * KPE_HALF_ROT_DIM bf16 RoPE - # mla_cache_block_stride / mla_cache_entry_stride are byte strides - # (mla_cache_ptr is the 1-byte fp8 view of the uint8 cache). - byte_base = ( - mla_block_idx * mla_cache_block_stride + slot_idx = tl.load(slot_mapping_ptr + tok_idx) + mla_block_size = mla_cache_block_stride // mla_cache_entry_stride + mla_block_idx = slot_idx // mla_block_size + mla_block_off = slot_idx % mla_block_size + + if MLA_CACHE_DS_MLA: + # fp8_ds_mla layout (DeepSeek-V3.2, KV_DIM == 512): per-128-element + # tile of the NoPE is dynamically quantized to fp8 with its own + # float32 scale; the RoPE tail is stored unquantized in bf16. + # bytes [0, KV_DIM) : KV_DIM fp8 NoPE values + # bytes [KV_DIM, KV_DIM + 16) : MLA_NUM_TILES float32 scales + # bytes [KV_DIM + 16, ...) : 2 * KPE_HALF_ROT_DIM bf16 RoPE + # mla_cache_block_stride / mla_cache_entry_stride are byte strides + # (mla_cache_ptr is the 1-byte fp8 view of the uint8 cache). + byte_base = ( + mla_block_idx * mla_cache_block_stride + + mla_block_off * mla_cache_entry_stride + ) + kv_2d = tl.reshape(kv_c, (MLA_NUM_TILES, MLA_TILE_DIM)) + tile_amax = tl.max(tl.abs(kv_2d), axis=1, keep_dims=True) + # scale = amax / 448 (fp8 e4m3 max), matching the reference + # concat_and_cache_ds_mla kernel; floored to FLT_MIN. + tile_scale = tl.maximum(tile_amax * (1.0 / 448.0), 1.1754944e-38) + kv_c_fp8 = tl.reshape((kv_2d / tile_scale).to(tl.float8e4nv), (KV_DIM,)) + tl.store(mla_cache_ptr + byte_base + kv_block, kv_c_fp8) + tile_off = tl.arange(0, MLA_NUM_TILES) + tl.store( + mla_cache_ds_scale_ptr + byte_base // 4 + KV_DIM // 4 + tile_off, + tl.reshape(tile_scale, (MLA_NUM_TILES,)), + ) + rope_dst = mla_cache_ds_rope_ptr + byte_base // 2 + (KV_DIM // 2 + 8) + tl.store(rope_dst + dim_off * 2, r1.to(tl.bfloat16)) + tl.store(rope_dst + dim_off * 2 + 1, r2.to(tl.bfloat16)) + return + + dst = ( + mla_cache_ptr + + mla_block_idx * mla_cache_block_stride + mla_block_off * mla_cache_entry_stride ) - kv_2d = tl.reshape(kv_c, (MLA_NUM_TILES, MLA_TILE_DIM)) - tile_amax = tl.max(tl.abs(kv_2d), axis=1, keep_dims=True) - # scale = amax / 448 (fp8 e4m3 max), matching the reference - # concat_and_cache_ds_mla kernel; floored to FLT_MIN. - tile_scale = tl.maximum(tile_amax * (1.0 / 448.0), 1.1754944e-38) - kv_c_fp8 = tl.reshape((kv_2d / tile_scale).to(tl.float8e4nv), (KV_DIM,)) - tl.store(mla_cache_ptr + byte_base + kv_block, kv_c_fp8) - tile_off = tl.arange(0, MLA_NUM_TILES) - tl.store( - mla_cache_ds_scale_ptr + byte_base // 4 + KV_DIM // 4 + tile_off, - tl.reshape(tile_scale, (MLA_NUM_TILES,)), - ) - rope_dst = mla_cache_ds_rope_ptr + byte_base // 2 + (KV_DIM // 2 + 8) - tl.store(rope_dst + dim_off * 2, r1.to(tl.bfloat16)) - tl.store(rope_dst + dim_off * 2 + 1, r2.to(tl.bfloat16)) - return - - dst = ( - mla_cache_ptr - + mla_block_idx * mla_cache_block_stride - + mla_block_off * mla_cache_entry_stride - ) - # kv_c_normed (KV_DIM elements) - if MLA_CACHE_FP8: - scale = tl.load(mla_cache_scale_ptr) - kv_c_fp8 = (kv_c.to(tl.float32) / scale).to(tl.float8e4nv) - tl.store(dst + kv_block, kv_c_fp8) - else: - tl.store(dst + kv_block, kv_c) - # k_pe_roped (from registers, interleaved layout) - if MLA_CACHE_FP8: - tl.store(dst + KV_DIM + dim_off * 2, (r1 / scale).to(tl.float8e4nv)) - tl.store(dst + KV_DIM + dim_off * 2 + 1, (r2 / scale).to(tl.float8e4nv)) - else: - tl.store(dst + KV_DIM + dim_off * 2, r1) - tl.store(dst + KV_DIM + dim_off * 2 + 1, r2) + # kv_c_normed (KV_DIM elements) + if MLA_CACHE_FP8: + scale = tl.load(mla_cache_scale_ptr) + kv_c_fp8 = (kv_c.to(tl.float32) / scale).to(tl.float8e4nv) + tl.store(dst + kv_block, kv_c_fp8) + else: + tl.store(dst + kv_block, kv_c) + # k_pe_roped (from registers, interleaved layout) + if MLA_CACHE_FP8: + tl.store(dst + KV_DIM + dim_off * 2, (r1 / scale).to(tl.float8e4nv)) + tl.store( + dst + KV_DIM + dim_off * 2 + 1, + (r2 / scale).to(tl.float8e4nv), + ) + else: + tl.store(dst + KV_DIM + dim_off * 2, r1) + tl.store(dst + KV_DIM + dim_off * 2 + 1, r2) elif pid == 0: if not HAS_INDEXER: # Shared layer: no indexer K to process. @@ -354,20 +377,27 @@ def _fused_norm_rope_kernel( roped = normed * cos_full + sign * normed_partner * sin_full result = tl.where(in_rope, roped, normed) - # 3. FP8 quantize + cache write from registers. - # No need to write back to index_k_ptr — the only consumer - # (sparse_attn_indexer) reads from the cache, not index_k. - _fp8_quant_and_cache_write( - result, - index_k_mask, - slot_idx, - indexer_cache_ptr, - indexer_cache_scale_ptr, - indexer_cache_block_size, - indexer_cache_stride, - index_k_block, - INDEX_K_DIM, - ) + if index_k_out_ptr is not None: + tl.store( + index_k_out_ptr + tok_idx * index_k_out_stride + index_k_block, + result.to(index_k_out_ptr.dtype.element_ty), + mask=index_k_mask, + ) + + # PCP inserts index K after gathering; other paths write it directly. + if indexer_cache_ptr is not None and slot_mapping_ptr is not None: + slot_idx = tl.load(slot_mapping_ptr + tok_idx) + _fp8_quant_and_cache_write( + result, + index_k_mask, + slot_idx, + indexer_cache_ptr, + indexer_cache_scale_ptr, + indexer_cache_block_size, + indexer_cache_stride, + index_k_block, + INDEX_K_DIM, + ) def fused_norm_rope( @@ -395,6 +425,9 @@ def fused_norm_rope( has_indexer: bool = True, index_rope_interleave: bool = False, q_c_out: torch.Tensor | None = None, + kv_c_out: torch.Tensor | None = None, + k_pe_out: torch.Tensor | None = None, + index_k_out: torch.Tensor | None = None, ) -> torch.Tensor: assert positions.ndim == 1 assert q_c.ndim == 2 @@ -420,6 +453,10 @@ def fused_norm_rope( assert index_k_rope_cos_sin_cache is not None index_k_dim = index_k.shape[-1] topk = topk_indices_buffer.shape[-1] + if indexer_k_cache is not None or mla_kv_cache is not None: + assert slot_mapping is not None + else: + slot_mapping = None # --- Indexer K cache setup --- if indexer_k_cache is not None: @@ -430,17 +467,9 @@ def fused_norm_rope( if indexer_k_cache.dtype == torch.uint8: indexer_k_cache = indexer_k_cache.view(torch.float8_e4m3fn) else: - # No indexer cache (shared layer / MLA-only fusion). Use dummies but - # KEEP the caller's slot_mapping so the MLA write (pid 1) still runs. - idx_cache_scale_view = torch.empty(0, dtype=torch.float32, device=device) - indexer_k_cache = torch.empty(0, dtype=torch.float8_e4m3fn, device=device) + idx_cache_scale_view = None idx_cache_block_size = 1 - idx_cache_stride = 1 - if mla_kv_cache is None: - # Pure profiling run (no caches at all): skip all per-token writes. - slot_mapping = torch.full( - (num_tokens,), -1, dtype=torch.int64, device=device - ) + idx_cache_stride = 0 # --- MLA KV cache setup --- mla_cache_ds_mla = mla_kv_cache_dtype == "fp8_ds_mla" @@ -469,15 +498,26 @@ def fused_norm_rope( if mla_k_scale is None: mla_k_scale = torch.ones(1, dtype=torch.float32, device=device) else: - # Dummy values — pid 2 will skip the MLA cache write because - # slot_mapping is all -1. + # Dummy cache values; a zero entry stride disables the cache write. mla_kv_cache = torch.empty(0, dtype=torch.bfloat16, device=device) mla_block_stride = 0 mla_entry_stride = 0 - mla_k_scale = torch.ones(1, dtype=torch.float32, device=device) + mla_k_scale = _dummy((1,), torch.float32, device) if q_c_out is None: q_c_out = torch.empty_like(q_c) + kv_c_out_stride = 0 + k_pe_out_stride = 0 + index_k_out_stride = 0 + if kv_c_out is not None: + assert kv_c_out.shape == kv_c.shape + kv_c_out_stride = kv_c_out.stride(0) + if k_pe_out is not None: + assert k_pe_out.shape == k_pe.shape + k_pe_out_stride = k_pe_out.stride(0) + if index_k_out is not None: + assert index_k_out.shape == index_k.shape + index_k_out_stride = index_k_out.stride(0) use_pdl = current_platform.is_arch_support_pdl() _fused_norm_rope_kernel[(num_tokens, 4)]( positions, @@ -495,12 +535,16 @@ def fused_norm_rope( kv_c.stride(0), kv_rms_norm_w, kv_rms_eps, + kv_c_out, + kv_c_out_stride, kv_dim, # KV RoPE k_pe, k_pe.stride(0), k_rope_cos_sin_cache, k_rope_cos_sin_cache.stride(0), + k_pe_out, + k_pe_out_stride, k_rope_cos_sin_cache.shape[-1] // 2, # Index K layer norm + RoPE + FP8 quant index_k, @@ -512,6 +556,8 @@ def fused_norm_rope( triton.next_power_of_2(index_k_dim), index_k_rope_cos_sin_cache, index_k_rope_cos_sin_cache.stride(0), + index_k_out, + index_k_out_stride, index_k_rope_cos_sin_cache.shape[-1] // 2, # Cache params slot_mapping, @@ -793,7 +839,6 @@ def fused_q( assert ql_nope.ndim == 3 assert ql_nope.shape[:2] == q_pe.shape[:2] assert q_scale.dtype == torch.float32 and q_scale.numel() == 1 - num_tokens = positions.shape[0] num_q_heads = q_pe.shape[1] # Grid's 3rd dim must cover the MQA-pack heads (pid 0/2 iterate 2 heads diff --git a/vllm/v1/attention/ops/rocm_aiter_mla_sparse.py b/vllm/v1/attention/ops/rocm_aiter_mla_sparse.py index 5c9ab386d4..0f55556c5d 100644 --- a/vllm/v1/attention/ops/rocm_aiter_mla_sparse.py +++ b/vllm/v1/attention/ops/rocm_aiter_mla_sparse.py @@ -682,7 +682,7 @@ def rocm_aiter_sparse_attn_indexer_fake( k_cache_prefix: LayerNameType, kv_cache: torch.Tensor, q_fp8: torch.Tensor, - k: torch.Tensor, + k: torch.Tensor | None, weights: torch.Tensor, quant_block_size: int, scale_fmt: str | None, @@ -702,7 +702,7 @@ def rocm_aiter_sparse_attn_indexer( k_cache_prefix: LayerNameType, kv_cache: torch.Tensor, q_fp8: torch.Tensor, - k: torch.Tensor, + k: torch.Tensor | None, weights: torch.Tensor, quant_block_size: int, scale_fmt: str | None,