[nv] add pcp support in dsv3.2 (#52046)

Signed-off-by: Summer Yang <girasoleyang@gmail.com>
This commit is contained in:
Summer Yang
2026-08-18 14:59:08 -07:00
committed by GitHub
parent d3fafe0c27
commit 5f7a20b316
6 changed files with 376 additions and 119 deletions
@@ -292,6 +292,81 @@ def test_fused_norm_rope_no_indexer(num_tokens: int):
assert (topk == 7).all(), "topk buffer should be untouched on shared layer"
@pytest.mark.parametrize("has_indexer", [False, True])
def test_fused_norm_rope_materializes_pcp_cache_inputs(has_indexer: bool):
"""PCP gets local normalized/rotated K rows without direct cache writes."""
torch.manual_seed(6)
dev = "cuda"
num_tokens = 17
max_pos = 8192
pos = torch.arange(num_tokens, device=dev, dtype=torch.int64)
q_c = torch.randn(num_tokens, Q_LORA, device=dev, dtype=torch.bfloat16)
kv_c = torch.randn(num_tokens, KV_LORA, device=dev, dtype=torch.bfloat16)
k_pe = torch.randn(num_tokens, ROPE_DIM, device=dev, dtype=torch.bfloat16)
qw = torch.randn(Q_LORA, device=dev, dtype=torch.bfloat16)
kvw = torch.randn(KV_LORA, device=dev, dtype=torch.bfloat16)
ik = (
torch.randn(num_tokens, INDEX_HEAD_DIM, device=dev, dtype=torch.bfloat16)
if has_indexer
else None
)
ikw = (
torch.randn(INDEX_HEAD_DIM, device=dev, dtype=torch.float32)
if has_indexer
else None
)
ikb = (
torch.randn(INDEX_HEAD_DIM, device=dev, dtype=torch.float32)
if has_indexer
else None
)
mla_cos_sin = make_cos_sin(max_pos, ROPE_DIM, dev)
idx_cos_sin = make_cos_sin(max_pos, ROPE_DIM, dev) if has_indexer else None
q_out = torch.empty_like(q_c)
kv_out = torch.empty_like(kv_c)
kpe_out = torch.empty_like(k_pe)
ik_out = torch.empty_like(ik) if ik is not None else None
topk = torch.full((num_tokens, 2048), 7, device=dev, dtype=torch.int32)
actual_q = K.fused_norm_rope(
pos,
q_c,
qw,
EPS,
kv_c,
kvw,
EPS,
k_pe,
mla_cos_sin,
ik,
ikw,
ikb,
EPS,
idx_cos_sin,
topk,
has_indexer=has_indexer,
index_rope_interleave=True,
q_c_out=q_out,
kv_c_out=kv_out,
k_pe_out=kpe_out,
index_k_out=ik_out,
)
assert actual_q.data_ptr() == q_out.data_ptr()
assert_bf16(actual_q, rms_norm(q_c, qw), "PCP q norm")
assert_bf16(kv_out, rms_norm(kv_c, kvw), "PCP kv norm")
assert_bf16(
kpe_out,
rope(k_pe.float(), pos, mla_cos_sin, interleave=True),
"PCP k_pe RoPE",
)
if has_indexer:
assert ik is not None and ikw is not None and ikb is not None
assert ik_out is not None and idx_cos_sin is not None
ik_ref = rope(layer_norm(ik, ikw, ikb), pos, idx_cos_sin, interleave=True)
assert_bf16(ik_out, ik_ref, "PCP indexer-K")
@pytest.mark.parametrize("num_tokens", [1, 4, 17, 512])
def test_fused_norm_rope_ds_mla(num_tokens: int):
"""fp8_ds_mla MLA cache layout (FlashMLA sparse, bf16-query path; SM90/SM100).
@@ -405,19 +480,29 @@ def test_fused_norm_rope_supports_large_token_count():
# ── fused_q ──────────────────────────────────────────────────────────────────
@pytest.mark.parametrize("num_tokens", [1, 4, 17, 512, 4096])
@pytest.mark.parametrize(
("num_tokens", "num_q_heads"),
[
(1, NUM_HEADS),
(4, NUM_HEADS),
(17, NUM_HEADS),
(512, NUM_HEADS),
(4096, NUM_HEADS),
(17, 64),
],
)
@pytest.mark.parametrize("index_interleave", [True, False])
def test_fused_q(num_tokens: int, index_interleave: bool):
def test_fused_q(num_tokens: int, num_q_heads: int, index_interleave: bool):
torch.manual_seed(2)
dev = "cuda"
max_pos = 8192
pos = torch.arange(num_tokens, device=dev, dtype=torch.int64) % max_pos
q_pe = torch.randn(
num_tokens, NUM_HEADS, ROPE_DIM, device=dev, dtype=torch.bfloat16
num_tokens, num_q_heads, ROPE_DIM, device=dev, dtype=torch.bfloat16
)
ql_nope = torch.randn(
num_tokens, NUM_HEADS, KV_LORA, device=dev, dtype=torch.bfloat16
num_tokens, num_q_heads, KV_LORA, device=dev, dtype=torch.bfloat16
)
index_q = torch.randn(
num_tokens, INDEX_HEADS, INDEX_HEAD_DIM, device=dev, dtype=torch.bfloat16
@@ -449,7 +534,7 @@ def test_fused_q(num_tokens: int, index_interleave: bool):
mqa_nope_ref = (ql_nope.float() / s).to(FP8)
qpe_ref = rope(
q_pe.float(),
pos.unsqueeze(-1).expand(num_tokens, NUM_HEADS),
pos.unsqueeze(-1).expand(num_tokens, num_q_heads),
q_cos_sin,
interleave=True,
)
@@ -484,7 +569,6 @@ def test_fused_q_no_indexer(num_tokens: int):
)
q_scale = torch.tensor([0.5], device=dev, dtype=torch.float32)
q_cos_sin = make_cos_sin(max_pos, ROPE_DIM, dev)
_, _, mqa = K.fused_q(
pos,
q_pe,
@@ -177,3 +177,48 @@ def test_short_prefill_updates_k_cache_before_scoring_decision(
# K cache is always updated before the scoring decision.
torch.testing.assert_close(observed["k"], k[: slot_mapping.numel()])
assert observed["slots"] is slot_mapping
def test_skipped_k_cache_insert_accepts_no_k(
monkeypatch: pytest.MonkeyPatch,
) -> None:
indexer_metadata = make_indexer_metadata(
num_prefills=0,
num_prefill_tokens=0,
slot_mapping=torch.empty(0, dtype=torch.long),
)
monkeypatch.setattr(
sparse_indexer,
"get_forward_context",
lambda: SimpleNamespace(
attn_metadata={INDEXER_LAYER: indexer_metadata},
cudagraph_runtime_mode=CUDAGraphMode.PIECEWISE,
),
)
monkeypatch.setattr(
sparse_indexer.current_platform, "fp8_dtype", lambda: torch.float16
)
topk_indices = torch.full((1, 2048), 17, dtype=torch.int32)
result = sparse_indexer.sparse_attn_indexer(
torch.empty(1, 1),
INDEXER_LAYER,
torch.empty(1),
torch.empty(1, 1),
None,
None,
torch.empty(1, 1),
128,
"ue8m0",
2048,
4,
4096,
4096,
topk_indices,
True,
False,
"",
)
assert result is topk_indices
assert torch.all(topk_indices == -1)
@@ -299,7 +299,7 @@ def sparse_attn_indexer(
kv_cache: torch.Tensor,
q_quant: torch.Tensor,
q_scale: torch.Tensor | None,
k: torch.Tensor,
k: torch.Tensor | None,
weights: torch.Tensor,
quant_block_size: int,
scale_fmt: str | None,
@@ -695,7 +695,7 @@ def sparse_attn_indexer_fake(
kv_cache: torch.Tensor,
q_quant: torch.Tensor,
q_scale: torch.Tensor | None,
k: torch.Tensor,
k: torch.Tensor | None,
weights: torch.Tensor,
quant_block_size: int,
scale_fmt: str | None,
@@ -781,7 +781,7 @@ class SparseAttnIndexer(CustomOp):
self,
hidden_states: torch.Tensor,
q_quant: torch.Tensor | tuple[torch.Tensor, torch.Tensor],
k: torch.Tensor,
k: torch.Tensor | None,
weights: torch.Tensor,
):
if current_platform.is_cuda() or current_platform.is_xpu():
@@ -798,7 +798,7 @@ class SparseAttnIndexer(CustomOp):
self,
hidden_states: torch.Tensor,
q_quant: torch.Tensor | tuple[torch.Tensor, torch.Tensor],
k: torch.Tensor,
k: torch.Tensor | None,
weights: torch.Tensor,
):
# FP8 path: single tensor (per-token scale is folded into `weights`).
@@ -835,7 +835,7 @@ class SparseAttnIndexer(CustomOp):
self,
hidden_states: torch.Tensor,
q_fp8: torch.Tensor,
k: torch.Tensor,
k: torch.Tensor | None,
weights: torch.Tensor,
):
return self.forward_cuda(hidden_states, q_fp8, k, weights)
@@ -844,7 +844,7 @@ class SparseAttnIndexer(CustomOp):
self,
hidden_states: torch.Tensor,
q_quant: torch.Tensor | tuple[torch.Tensor, torch.Tensor],
k: torch.Tensor,
k: torch.Tensor | None,
weights: torch.Tensor,
):
assert not self.use_fp4_cache, "AMD platform doesn't support fp4 cache yet"
+103 -20
View File
@@ -1,5 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import TYPE_CHECKING, cast
import torch
import torch.nn as nn
from transformers import DeepseekV2Config, DeepseekV3Config
@@ -7,8 +9,14 @@ from transformers import DeepseekV2Config, DeepseekV3Config
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
from vllm.config import CacheConfig, VllmConfig
from vllm.distributed import get_tensor_model_parallel_world_size
from vllm.distributed.parallel_state import get_tp_group
from vllm.forward_context import get_forward_context
from vllm.model_executor.layers.attention import MLAAttention
from vllm.model_executor.layers.attention.attention import get_attention_context
from vllm.model_executor.layers.attention.pcp import (
finalize_mla_pcp_decode,
maybe_gather_mla_latent_cache_inputs,
)
from vllm.model_executor.layers.layernorm import LayerNorm, RMSNorm
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
@@ -35,6 +43,9 @@ from vllm.models.deepseek_v32.common.kernels import fused_norm_rope, fused_q
from vllm.platforms import current_platform
from vllm.utils.torch_utils import is_quantized_kv_cache
if TYPE_CHECKING:
from vllm.model_executor.layers.attention.mla_attention import MLACommonMetadata
class DeepseekV32Indexer(nn.Module):
indexer_cache_cls = DeepseekV32IndexerCache
@@ -363,6 +374,7 @@ class DeepseekV32Attention(MLAAttention):
attn_metadata = attn_metadata_raw[0].get(self.layer_name)
else:
attn_metadata = attn_metadata_raw
attn_metadata = cast("MLACommonMetadata | None", attn_metadata)
slot_mapping = forward_context.slot_mapping
assert isinstance(slot_mapping, dict)
@@ -374,7 +386,8 @@ class DeepseekV32Attention(MLAAttention):
indexer_k_norm_bias = self.indexer.k_norm.bias
indexer_k_norm_eps = self.indexer.k_norm.eps
indexer_k_rope_cos_sin_cache = self.indexer_rope_emb.cos_sin_cache
indexer_k_cache = self.indexer.k_cache.kv_cache
indexer_k_cache = None if self.use_pcp else self.indexer.k_cache.kv_cache
index_k_out = torch.empty_like(index_k) if self.use_pcp else None
indexer_softmax_scale = self.indexer.softmax_scale
indexer_n_head_scale = self.indexer.n_head**-0.5
else:
@@ -384,10 +397,11 @@ class DeepseekV32Attention(MLAAttention):
indexer_k_norm_eps = 1e-6
indexer_k_rope_cos_sin_cache = None
indexer_k_cache = None
index_k_out = None
indexer_softmax_scale = 0.0
indexer_n_head_scale = 0.0
if attn_metadata is None:
if attn_metadata is None or self.use_pcp:
mla_kv_cache = None
mla_k_scale = None
indexer_k_cache = None
@@ -396,6 +410,8 @@ class DeepseekV32Attention(MLAAttention):
mla_kv_cache = self.kv_cache
mla_k_scale = self._k_scale
kv_c_out = torch.empty_like(kv_c) if self.use_pcp else None
k_pe_out = torch.empty_like(k_pe) if self.use_pcp else None
q_c = fused_norm_rope(
positions,
q_c,
@@ -419,12 +435,14 @@ class DeepseekV32Attention(MLAAttention):
mla_k_scale=mla_k_scale,
has_indexer=has_indexer,
index_rope_interleave=self._index_rope_interleave,
kv_c_out=kv_c_out,
k_pe_out=k_pe_out,
index_k_out=index_k_out,
)
q = self.q_b_proj(q_c)[0].view(-1, self.num_local_heads, self.qk_head_dim)
q_nope, q_pe = q.split([self.qk_nope_head_dim, self.qk_rope_head_dim], dim=-1)
q_nope = q_nope.transpose(0, 1)
ql_nope = torch.bmm(q_nope, self.W_UK_T).transpose(0, 1)
ql_nope = torch.bmm(q_nope.transpose(0, 1), self.W_UK_T).transpose(0, 1)
if self.indexer is not None and not self.skip_topk:
index_q = self.indexer.wq_b(q_c)[0]
@@ -451,7 +469,10 @@ class DeepseekV32Attention(MLAAttention):
self._sparse_indexer_and_attn(
q_c,
index_q_fp8,
index_k_out,
index_weights_out,
kv_c_out,
k_pe_out,
ql_nope,
mqa_q,
output,
@@ -463,19 +484,26 @@ class DeepseekV32Attention(MLAAttention):
self,
q_c: torch.Tensor,
index_q_fp8: torch.Tensor | None,
index_k: torch.Tensor | None,
index_weights_out: torch.Tensor | None,
kv_c: torch.Tensor | None,
k_pe: torch.Tensor | None,
ql_nope: torch.Tensor,
mqa_q: torch.Tensor,
output: torch.Tensor,
) -> None:
if self.indexer is not None and not self.skip_topk:
assert index_q_fp8 is not None
assert index_weights_out is not None
if self.use_pcp:
assert index_k is not None
sparse_attn_indexer(
q_c,
self.indexer.k_cache.prefix,
self.indexer.k_cache.kv_cache,
index_q_fp8,
None, # q_scale folded into weights on the fp8 path
None, # k unused when skip_k_cache_insert=True
None,
index_k,
index_weights_out,
self.indexer.quant_block_size,
self.indexer.scale_fmt,
@@ -484,28 +512,56 @@ class DeepseekV32Attention(MLAAttention):
self.indexer.max_model_len,
self.indexer.max_total_seq_len,
self.topk_indices_buffer,
skip_k_cache_insert=True,
use_pcp=False,
skip_k_cache_insert=not self.use_pcp,
use_pcp=self.use_pcp,
dense_mha_metadata_layer_name=self._dense_mha_metadata_layer_name,
use_fp4_cache=False,
# fused_norm_rope already cleared the topk buffer this forward.
dcp_rank=(
self.dcp_manager.group.rank_in_group
if self.dcp_manager is not None
else 0
),
dcp_world_size=(
self._vllm_config.parallel_config.decode_context_parallel_size
),
cp_kv_cache_interleave_size=(
self._vllm_config.parallel_config.cp_kv_cache_interleave_size
),
skip_topk_buffer_clear=True,
)
attn_metadata_raw = get_forward_context().attn_metadata
if isinstance(attn_metadata_raw, dict):
attn_metadata = attn_metadata_raw.get(self.layer_name)
elif isinstance(attn_metadata_raw, list):
attn_metadata = attn_metadata_raw[0].get(self.layer_name)
else:
attn_metadata = attn_metadata_raw
attn_metadata, _, kv_cache, layer_slot_mapping = get_attention_context(
self.layer_name
)
if attn_metadata is None:
output.zero_()
return
attn_metadata = cast("MLACommonMetadata", attn_metadata)
if self.use_pcp:
assert kv_c is not None and k_pe is not None
kv_for_cache, kpe_for_cache, cache_slot_mapping = (
maybe_gather_mla_latent_cache_inputs(
kv_c,
k_pe.unsqueeze(1),
layer_slot_mapping,
attn_metadata.num_decode_tokens,
True,
)
)
self.impl.do_kv_cache_update( # type: ignore[attr-defined]
kv_for_cache,
kpe_for_cache,
kv_cache,
cache_slot_mapping,
self.kv_cache_dtype,
self._k_scale,
)
num_actual = attn_metadata.num_actual_tokens # type: ignore[attr-defined]
kv_cache = self.kv_cache
if num_actual == 0:
output.zero_()
return
if self._fp8_kv_needs_view:
kv_cache = kv_cache.view(torch.float8_e4m3fn)
if self._fp8_query:
@@ -515,10 +571,35 @@ class DeepseekV32Attention(MLAAttention):
]
else:
mqa_q_arg = (ql_nope[:num_actual], mqa_q[:num_actual])
attn_out, _ = self.impl.forward_mqa( # type: ignore[attr-defined]
if self.use_pcp and self.impl.dcp_world_size > self.impl.pcp_world_size:
if isinstance(mqa_q_arg, tuple):
mqa_q_arg = torch.cat(mqa_q_arg, dim=-1)
mqa_q_arg = get_tp_group().all_gather(mqa_q_arg, dim=1)
attn_out, lse = self.impl.forward_mqa( # type: ignore[attr-defined]
mqa_q_arg, kv_cache, attn_metadata, self
)
if self.use_pcp and self.impl.dcp_world_size > 1:
assert lse is not None and self.dcp_manager is not None
seq_lens = (
attn_metadata.decode.seq_lens
if attn_metadata.decode is not None
else cast(torch.Tensor, attn_metadata.seq_lens)[ # type: ignore[attr-defined]
: attn_metadata.num_decodes
]
)
query_start_loc = attn_metadata.query_start_loc[
: attn_metadata.num_decodes + 1
]
attn_out = self.dcp_manager.combine(
attn_out,
lse,
seq_lens=seq_lens,
query_start_loc=query_start_loc,
)
attn_out = finalize_mla_pcp_decode(attn_out, self.num_heads)
# NOTE(woosuk): While the below does not need to be in the eager region,
# we put it here to avoid copying the attention output. Move this back to the
# captured region once forward_mqa supports `out` argument.
@@ -531,3 +612,5 @@ class DeepseekV32Attention(MLAAttention):
.transpose(0, 1)
)
torch.bmm(x, self.W_UV, out=out)
if self.use_pcp and num_actual < output.shape[0]:
output[num_actual:].zero_()
+130 -85
View File
@@ -103,12 +103,16 @@ def _fused_norm_rope_kernel(
kv_stride,
kv_rms_norm_w_ptr,
kv_rms_eps,
kv_out_ptr,
kv_out_stride,
KV_DIM: tl.constexpr,
# KV RoPE
kpe_ptr,
kpe_stride,
kpe_rope_cos_sin_cache_ptr,
kpe_rope_cos_sin_cache_stride,
kpe_out_ptr,
kpe_out_stride,
KPE_HALF_ROT_DIM: tl.constexpr,
# Index K layer norm
index_k_ptr,
@@ -121,6 +125,8 @@ def _fused_norm_rope_kernel(
# Index K RoPE
index_k_rope_cos_sin_cache_ptr,
index_k_rope_cos_sin_cache_stride,
index_k_out_ptr,
index_k_out_stride,
INDEX_K_HALF_ROT_DIM: tl.constexpr,
# Cache params (shared by indexer K and MLA)
slot_mapping_ptr,
@@ -174,10 +180,9 @@ def _fused_norm_rope_kernel(
return
if slot_mapping_ptr is None:
# Memory profiling run.
return
slot_idx = tl.load(slot_mapping_ptr + tok_idx)
if slot_idx < 0:
if kv_out_ptr is None and kpe_out_ptr is None and index_k_out_ptr is None:
return
elif tl.load(slot_mapping_ptr + tok_idx) < 0:
# Padding
return
@@ -217,63 +222,81 @@ def _fused_norm_rope_kernel(
r1 = x1 * cos - x2 * sin
r2 = x2 * cos + x1 * sin
# MLA concat_and_cache: write [kv_c_normed, k_pe_roped] to cache.
if mla_cache_entry_stride == 0:
return
# PCP materializes K for the cross-rank gather, then inserts it into cache.
if kv_out_ptr is not None:
tl.store(kv_out_ptr + tok_idx * kv_out_stride + kv_block, kv_c)
if kpe_out_ptr is not None:
tl.store(
kpe_out_ptr + tok_idx * kpe_out_stride + dim_off * 2,
r1.to(kpe_out_ptr.dtype.element_ty),
)
tl.store(
kpe_out_ptr + tok_idx * kpe_out_stride + dim_off * 2 + 1,
r2.to(kpe_out_ptr.dtype.element_ty),
)
mla_block_size = mla_cache_block_stride // mla_cache_entry_stride
mla_block_idx = slot_idx // mla_block_size
mla_block_off = slot_idx % mla_block_size
if slot_mapping_ptr is not None:
# MLA concat_and_cache: write [kv_c_normed, k_pe_roped] to cache.
if mla_cache_entry_stride == 0:
return
if MLA_CACHE_DS_MLA:
# fp8_ds_mla layout (DeepSeek-V3.2, KV_DIM == 512): per-128-element
# tile of the NoPE is dynamically quantized to fp8 with its own
# float32 scale; the RoPE tail is stored unquantized in bf16.
# bytes [0, KV_DIM) : KV_DIM fp8 NoPE values
# bytes [KV_DIM, KV_DIM + 16) : MLA_NUM_TILES float32 scales
# bytes [KV_DIM + 16, ...) : 2 * KPE_HALF_ROT_DIM bf16 RoPE
# mla_cache_block_stride / mla_cache_entry_stride are byte strides
# (mla_cache_ptr is the 1-byte fp8 view of the uint8 cache).
byte_base = (
mla_block_idx * mla_cache_block_stride
slot_idx = tl.load(slot_mapping_ptr + tok_idx)
mla_block_size = mla_cache_block_stride // mla_cache_entry_stride
mla_block_idx = slot_idx // mla_block_size
mla_block_off = slot_idx % mla_block_size
if MLA_CACHE_DS_MLA:
# fp8_ds_mla layout (DeepSeek-V3.2, KV_DIM == 512): per-128-element
# tile of the NoPE is dynamically quantized to fp8 with its own
# float32 scale; the RoPE tail is stored unquantized in bf16.
# bytes [0, KV_DIM) : KV_DIM fp8 NoPE values
# bytes [KV_DIM, KV_DIM + 16) : MLA_NUM_TILES float32 scales
# bytes [KV_DIM + 16, ...) : 2 * KPE_HALF_ROT_DIM bf16 RoPE
# mla_cache_block_stride / mla_cache_entry_stride are byte strides
# (mla_cache_ptr is the 1-byte fp8 view of the uint8 cache).
byte_base = (
mla_block_idx * mla_cache_block_stride
+ mla_block_off * mla_cache_entry_stride
)
kv_2d = tl.reshape(kv_c, (MLA_NUM_TILES, MLA_TILE_DIM))
tile_amax = tl.max(tl.abs(kv_2d), axis=1, keep_dims=True)
# scale = amax / 448 (fp8 e4m3 max), matching the reference
# concat_and_cache_ds_mla kernel; floored to FLT_MIN.
tile_scale = tl.maximum(tile_amax * (1.0 / 448.0), 1.1754944e-38)
kv_c_fp8 = tl.reshape((kv_2d / tile_scale).to(tl.float8e4nv), (KV_DIM,))
tl.store(mla_cache_ptr + byte_base + kv_block, kv_c_fp8)
tile_off = tl.arange(0, MLA_NUM_TILES)
tl.store(
mla_cache_ds_scale_ptr + byte_base // 4 + KV_DIM // 4 + tile_off,
tl.reshape(tile_scale, (MLA_NUM_TILES,)),
)
rope_dst = mla_cache_ds_rope_ptr + byte_base // 2 + (KV_DIM // 2 + 8)
tl.store(rope_dst + dim_off * 2, r1.to(tl.bfloat16))
tl.store(rope_dst + dim_off * 2 + 1, r2.to(tl.bfloat16))
return
dst = (
mla_cache_ptr
+ mla_block_idx * mla_cache_block_stride
+ mla_block_off * mla_cache_entry_stride
)
kv_2d = tl.reshape(kv_c, (MLA_NUM_TILES, MLA_TILE_DIM))
tile_amax = tl.max(tl.abs(kv_2d), axis=1, keep_dims=True)
# scale = amax / 448 (fp8 e4m3 max), matching the reference
# concat_and_cache_ds_mla kernel; floored to FLT_MIN.
tile_scale = tl.maximum(tile_amax * (1.0 / 448.0), 1.1754944e-38)
kv_c_fp8 = tl.reshape((kv_2d / tile_scale).to(tl.float8e4nv), (KV_DIM,))
tl.store(mla_cache_ptr + byte_base + kv_block, kv_c_fp8)
tile_off = tl.arange(0, MLA_NUM_TILES)
tl.store(
mla_cache_ds_scale_ptr + byte_base // 4 + KV_DIM // 4 + tile_off,
tl.reshape(tile_scale, (MLA_NUM_TILES,)),
)
rope_dst = mla_cache_ds_rope_ptr + byte_base // 2 + (KV_DIM // 2 + 8)
tl.store(rope_dst + dim_off * 2, r1.to(tl.bfloat16))
tl.store(rope_dst + dim_off * 2 + 1, r2.to(tl.bfloat16))
return
dst = (
mla_cache_ptr
+ mla_block_idx * mla_cache_block_stride
+ mla_block_off * mla_cache_entry_stride
)
# kv_c_normed (KV_DIM elements)
if MLA_CACHE_FP8:
scale = tl.load(mla_cache_scale_ptr)
kv_c_fp8 = (kv_c.to(tl.float32) / scale).to(tl.float8e4nv)
tl.store(dst + kv_block, kv_c_fp8)
else:
tl.store(dst + kv_block, kv_c)
# k_pe_roped (from registers, interleaved layout)
if MLA_CACHE_FP8:
tl.store(dst + KV_DIM + dim_off * 2, (r1 / scale).to(tl.float8e4nv))
tl.store(dst + KV_DIM + dim_off * 2 + 1, (r2 / scale).to(tl.float8e4nv))
else:
tl.store(dst + KV_DIM + dim_off * 2, r1)
tl.store(dst + KV_DIM + dim_off * 2 + 1, r2)
# kv_c_normed (KV_DIM elements)
if MLA_CACHE_FP8:
scale = tl.load(mla_cache_scale_ptr)
kv_c_fp8 = (kv_c.to(tl.float32) / scale).to(tl.float8e4nv)
tl.store(dst + kv_block, kv_c_fp8)
else:
tl.store(dst + kv_block, kv_c)
# k_pe_roped (from registers, interleaved layout)
if MLA_CACHE_FP8:
tl.store(dst + KV_DIM + dim_off * 2, (r1 / scale).to(tl.float8e4nv))
tl.store(
dst + KV_DIM + dim_off * 2 + 1,
(r2 / scale).to(tl.float8e4nv),
)
else:
tl.store(dst + KV_DIM + dim_off * 2, r1)
tl.store(dst + KV_DIM + dim_off * 2 + 1, r2)
elif pid == 0:
if not HAS_INDEXER:
# Shared layer: no indexer K to process.
@@ -354,20 +377,27 @@ def _fused_norm_rope_kernel(
roped = normed * cos_full + sign * normed_partner * sin_full
result = tl.where(in_rope, roped, normed)
# 3. FP8 quantize + cache write from registers.
# No need to write back to index_k_ptr — the only consumer
# (sparse_attn_indexer) reads from the cache, not index_k.
_fp8_quant_and_cache_write(
result,
index_k_mask,
slot_idx,
indexer_cache_ptr,
indexer_cache_scale_ptr,
indexer_cache_block_size,
indexer_cache_stride,
index_k_block,
INDEX_K_DIM,
)
if index_k_out_ptr is not None:
tl.store(
index_k_out_ptr + tok_idx * index_k_out_stride + index_k_block,
result.to(index_k_out_ptr.dtype.element_ty),
mask=index_k_mask,
)
# PCP inserts index K after gathering; other paths write it directly.
if indexer_cache_ptr is not None and slot_mapping_ptr is not None:
slot_idx = tl.load(slot_mapping_ptr + tok_idx)
_fp8_quant_and_cache_write(
result,
index_k_mask,
slot_idx,
indexer_cache_ptr,
indexer_cache_scale_ptr,
indexer_cache_block_size,
indexer_cache_stride,
index_k_block,
INDEX_K_DIM,
)
def fused_norm_rope(
@@ -395,6 +425,9 @@ def fused_norm_rope(
has_indexer: bool = True,
index_rope_interleave: bool = False,
q_c_out: torch.Tensor | None = None,
kv_c_out: torch.Tensor | None = None,
k_pe_out: torch.Tensor | None = None,
index_k_out: torch.Tensor | None = None,
) -> torch.Tensor:
assert positions.ndim == 1
assert q_c.ndim == 2
@@ -420,6 +453,10 @@ def fused_norm_rope(
assert index_k_rope_cos_sin_cache is not None
index_k_dim = index_k.shape[-1]
topk = topk_indices_buffer.shape[-1]
if indexer_k_cache is not None or mla_kv_cache is not None:
assert slot_mapping is not None
else:
slot_mapping = None
# --- Indexer K cache setup ---
if indexer_k_cache is not None:
@@ -430,17 +467,9 @@ def fused_norm_rope(
if indexer_k_cache.dtype == torch.uint8:
indexer_k_cache = indexer_k_cache.view(torch.float8_e4m3fn)
else:
# No indexer cache (shared layer / MLA-only fusion). Use dummies but
# KEEP the caller's slot_mapping so the MLA write (pid 1) still runs.
idx_cache_scale_view = torch.empty(0, dtype=torch.float32, device=device)
indexer_k_cache = torch.empty(0, dtype=torch.float8_e4m3fn, device=device)
idx_cache_scale_view = None
idx_cache_block_size = 1
idx_cache_stride = 1
if mla_kv_cache is None:
# Pure profiling run (no caches at all): skip all per-token writes.
slot_mapping = torch.full(
(num_tokens,), -1, dtype=torch.int64, device=device
)
idx_cache_stride = 0
# --- MLA KV cache setup ---
mla_cache_ds_mla = mla_kv_cache_dtype == "fp8_ds_mla"
@@ -469,15 +498,26 @@ def fused_norm_rope(
if mla_k_scale is None:
mla_k_scale = torch.ones(1, dtype=torch.float32, device=device)
else:
# Dummy values — pid 2 will skip the MLA cache write because
# slot_mapping is all -1.
# Dummy cache values; a zero entry stride disables the cache write.
mla_kv_cache = torch.empty(0, dtype=torch.bfloat16, device=device)
mla_block_stride = 0
mla_entry_stride = 0
mla_k_scale = torch.ones(1, dtype=torch.float32, device=device)
mla_k_scale = _dummy((1,), torch.float32, device)
if q_c_out is None:
q_c_out = torch.empty_like(q_c)
kv_c_out_stride = 0
k_pe_out_stride = 0
index_k_out_stride = 0
if kv_c_out is not None:
assert kv_c_out.shape == kv_c.shape
kv_c_out_stride = kv_c_out.stride(0)
if k_pe_out is not None:
assert k_pe_out.shape == k_pe.shape
k_pe_out_stride = k_pe_out.stride(0)
if index_k_out is not None:
assert index_k_out.shape == index_k.shape
index_k_out_stride = index_k_out.stride(0)
use_pdl = current_platform.is_arch_support_pdl()
_fused_norm_rope_kernel[(num_tokens, 4)](
positions,
@@ -495,12 +535,16 @@ def fused_norm_rope(
kv_c.stride(0),
kv_rms_norm_w,
kv_rms_eps,
kv_c_out,
kv_c_out_stride,
kv_dim,
# KV RoPE
k_pe,
k_pe.stride(0),
k_rope_cos_sin_cache,
k_rope_cos_sin_cache.stride(0),
k_pe_out,
k_pe_out_stride,
k_rope_cos_sin_cache.shape[-1] // 2,
# Index K layer norm + RoPE + FP8 quant
index_k,
@@ -512,6 +556,8 @@ def fused_norm_rope(
triton.next_power_of_2(index_k_dim),
index_k_rope_cos_sin_cache,
index_k_rope_cos_sin_cache.stride(0),
index_k_out,
index_k_out_stride,
index_k_rope_cos_sin_cache.shape[-1] // 2,
# Cache params
slot_mapping,
@@ -793,7 +839,6 @@ def fused_q(
assert ql_nope.ndim == 3
assert ql_nope.shape[:2] == q_pe.shape[:2]
assert q_scale.dtype == torch.float32 and q_scale.numel() == 1
num_tokens = positions.shape[0]
num_q_heads = q_pe.shape[1]
# Grid's 3rd dim must cover the MQA-pack heads (pid 0/2 iterate 2 heads
@@ -682,7 +682,7 @@ def rocm_aiter_sparse_attn_indexer_fake(
k_cache_prefix: LayerNameType,
kv_cache: torch.Tensor,
q_fp8: torch.Tensor,
k: torch.Tensor,
k: torch.Tensor | None,
weights: torch.Tensor,
quant_block_size: int,
scale_fmt: str | None,
@@ -702,7 +702,7 @@ def rocm_aiter_sparse_attn_indexer(
k_cache_prefix: LayerNameType,
kv_cache: torch.Tensor,
q_fp8: torch.Tensor,
k: torch.Tensor,
k: torch.Tensor | None,
weights: torch.Tensor,
quant_block_size: int,
scale_fmt: str | None,