[nv] add pcp support in dsv3.2 (#52046)
Signed-off-by: Summer Yang <girasoleyang@gmail.com>
This commit is contained in:
@@ -292,6 +292,81 @@ def test_fused_norm_rope_no_indexer(num_tokens: int):
|
||||
assert (topk == 7).all(), "topk buffer should be untouched on shared layer"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("has_indexer", [False, True])
|
||||
def test_fused_norm_rope_materializes_pcp_cache_inputs(has_indexer: bool):
|
||||
"""PCP gets local normalized/rotated K rows without direct cache writes."""
|
||||
torch.manual_seed(6)
|
||||
dev = "cuda"
|
||||
num_tokens = 17
|
||||
max_pos = 8192
|
||||
pos = torch.arange(num_tokens, device=dev, dtype=torch.int64)
|
||||
q_c = torch.randn(num_tokens, Q_LORA, device=dev, dtype=torch.bfloat16)
|
||||
kv_c = torch.randn(num_tokens, KV_LORA, device=dev, dtype=torch.bfloat16)
|
||||
k_pe = torch.randn(num_tokens, ROPE_DIM, device=dev, dtype=torch.bfloat16)
|
||||
qw = torch.randn(Q_LORA, device=dev, dtype=torch.bfloat16)
|
||||
kvw = torch.randn(KV_LORA, device=dev, dtype=torch.bfloat16)
|
||||
ik = (
|
||||
torch.randn(num_tokens, INDEX_HEAD_DIM, device=dev, dtype=torch.bfloat16)
|
||||
if has_indexer
|
||||
else None
|
||||
)
|
||||
ikw = (
|
||||
torch.randn(INDEX_HEAD_DIM, device=dev, dtype=torch.float32)
|
||||
if has_indexer
|
||||
else None
|
||||
)
|
||||
ikb = (
|
||||
torch.randn(INDEX_HEAD_DIM, device=dev, dtype=torch.float32)
|
||||
if has_indexer
|
||||
else None
|
||||
)
|
||||
mla_cos_sin = make_cos_sin(max_pos, ROPE_DIM, dev)
|
||||
idx_cos_sin = make_cos_sin(max_pos, ROPE_DIM, dev) if has_indexer else None
|
||||
q_out = torch.empty_like(q_c)
|
||||
kv_out = torch.empty_like(kv_c)
|
||||
kpe_out = torch.empty_like(k_pe)
|
||||
ik_out = torch.empty_like(ik) if ik is not None else None
|
||||
topk = torch.full((num_tokens, 2048), 7, device=dev, dtype=torch.int32)
|
||||
|
||||
actual_q = K.fused_norm_rope(
|
||||
pos,
|
||||
q_c,
|
||||
qw,
|
||||
EPS,
|
||||
kv_c,
|
||||
kvw,
|
||||
EPS,
|
||||
k_pe,
|
||||
mla_cos_sin,
|
||||
ik,
|
||||
ikw,
|
||||
ikb,
|
||||
EPS,
|
||||
idx_cos_sin,
|
||||
topk,
|
||||
has_indexer=has_indexer,
|
||||
index_rope_interleave=True,
|
||||
q_c_out=q_out,
|
||||
kv_c_out=kv_out,
|
||||
k_pe_out=kpe_out,
|
||||
index_k_out=ik_out,
|
||||
)
|
||||
|
||||
assert actual_q.data_ptr() == q_out.data_ptr()
|
||||
assert_bf16(actual_q, rms_norm(q_c, qw), "PCP q norm")
|
||||
assert_bf16(kv_out, rms_norm(kv_c, kvw), "PCP kv norm")
|
||||
assert_bf16(
|
||||
kpe_out,
|
||||
rope(k_pe.float(), pos, mla_cos_sin, interleave=True),
|
||||
"PCP k_pe RoPE",
|
||||
)
|
||||
if has_indexer:
|
||||
assert ik is not None and ikw is not None and ikb is not None
|
||||
assert ik_out is not None and idx_cos_sin is not None
|
||||
ik_ref = rope(layer_norm(ik, ikw, ikb), pos, idx_cos_sin, interleave=True)
|
||||
assert_bf16(ik_out, ik_ref, "PCP indexer-K")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [1, 4, 17, 512])
|
||||
def test_fused_norm_rope_ds_mla(num_tokens: int):
|
||||
"""fp8_ds_mla MLA cache layout (FlashMLA sparse, bf16-query path; SM90/SM100).
|
||||
@@ -405,19 +480,29 @@ def test_fused_norm_rope_supports_large_token_count():
|
||||
# ── fused_q ──────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [1, 4, 17, 512, 4096])
|
||||
@pytest.mark.parametrize(
|
||||
("num_tokens", "num_q_heads"),
|
||||
[
|
||||
(1, NUM_HEADS),
|
||||
(4, NUM_HEADS),
|
||||
(17, NUM_HEADS),
|
||||
(512, NUM_HEADS),
|
||||
(4096, NUM_HEADS),
|
||||
(17, 64),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("index_interleave", [True, False])
|
||||
def test_fused_q(num_tokens: int, index_interleave: bool):
|
||||
def test_fused_q(num_tokens: int, num_q_heads: int, index_interleave: bool):
|
||||
torch.manual_seed(2)
|
||||
dev = "cuda"
|
||||
max_pos = 8192
|
||||
pos = torch.arange(num_tokens, device=dev, dtype=torch.int64) % max_pos
|
||||
|
||||
q_pe = torch.randn(
|
||||
num_tokens, NUM_HEADS, ROPE_DIM, device=dev, dtype=torch.bfloat16
|
||||
num_tokens, num_q_heads, ROPE_DIM, device=dev, dtype=torch.bfloat16
|
||||
)
|
||||
ql_nope = torch.randn(
|
||||
num_tokens, NUM_HEADS, KV_LORA, device=dev, dtype=torch.bfloat16
|
||||
num_tokens, num_q_heads, KV_LORA, device=dev, dtype=torch.bfloat16
|
||||
)
|
||||
index_q = torch.randn(
|
||||
num_tokens, INDEX_HEADS, INDEX_HEAD_DIM, device=dev, dtype=torch.bfloat16
|
||||
@@ -449,7 +534,7 @@ def test_fused_q(num_tokens: int, index_interleave: bool):
|
||||
mqa_nope_ref = (ql_nope.float() / s).to(FP8)
|
||||
qpe_ref = rope(
|
||||
q_pe.float(),
|
||||
pos.unsqueeze(-1).expand(num_tokens, NUM_HEADS),
|
||||
pos.unsqueeze(-1).expand(num_tokens, num_q_heads),
|
||||
q_cos_sin,
|
||||
interleave=True,
|
||||
)
|
||||
@@ -484,7 +569,6 @@ def test_fused_q_no_indexer(num_tokens: int):
|
||||
)
|
||||
q_scale = torch.tensor([0.5], device=dev, dtype=torch.float32)
|
||||
q_cos_sin = make_cos_sin(max_pos, ROPE_DIM, dev)
|
||||
|
||||
_, _, mqa = K.fused_q(
|
||||
pos,
|
||||
q_pe,
|
||||
|
||||
@@ -177,3 +177,48 @@ def test_short_prefill_updates_k_cache_before_scoring_decision(
|
||||
# K cache is always updated before the scoring decision.
|
||||
torch.testing.assert_close(observed["k"], k[: slot_mapping.numel()])
|
||||
assert observed["slots"] is slot_mapping
|
||||
|
||||
|
||||
def test_skipped_k_cache_insert_accepts_no_k(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
indexer_metadata = make_indexer_metadata(
|
||||
num_prefills=0,
|
||||
num_prefill_tokens=0,
|
||||
slot_mapping=torch.empty(0, dtype=torch.long),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
sparse_indexer,
|
||||
"get_forward_context",
|
||||
lambda: SimpleNamespace(
|
||||
attn_metadata={INDEXER_LAYER: indexer_metadata},
|
||||
cudagraph_runtime_mode=CUDAGraphMode.PIECEWISE,
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
sparse_indexer.current_platform, "fp8_dtype", lambda: torch.float16
|
||||
)
|
||||
|
||||
topk_indices = torch.full((1, 2048), 17, dtype=torch.int32)
|
||||
result = sparse_indexer.sparse_attn_indexer(
|
||||
torch.empty(1, 1),
|
||||
INDEXER_LAYER,
|
||||
torch.empty(1),
|
||||
torch.empty(1, 1),
|
||||
None,
|
||||
None,
|
||||
torch.empty(1, 1),
|
||||
128,
|
||||
"ue8m0",
|
||||
2048,
|
||||
4,
|
||||
4096,
|
||||
4096,
|
||||
topk_indices,
|
||||
True,
|
||||
False,
|
||||
"",
|
||||
)
|
||||
|
||||
assert result is topk_indices
|
||||
assert torch.all(topk_indices == -1)
|
||||
|
||||
@@ -299,7 +299,7 @@ def sparse_attn_indexer(
|
||||
kv_cache: torch.Tensor,
|
||||
q_quant: torch.Tensor,
|
||||
q_scale: torch.Tensor | None,
|
||||
k: torch.Tensor,
|
||||
k: torch.Tensor | None,
|
||||
weights: torch.Tensor,
|
||||
quant_block_size: int,
|
||||
scale_fmt: str | None,
|
||||
@@ -695,7 +695,7 @@ def sparse_attn_indexer_fake(
|
||||
kv_cache: torch.Tensor,
|
||||
q_quant: torch.Tensor,
|
||||
q_scale: torch.Tensor | None,
|
||||
k: torch.Tensor,
|
||||
k: torch.Tensor | None,
|
||||
weights: torch.Tensor,
|
||||
quant_block_size: int,
|
||||
scale_fmt: str | None,
|
||||
@@ -781,7 +781,7 @@ class SparseAttnIndexer(CustomOp):
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
q_quant: torch.Tensor | tuple[torch.Tensor, torch.Tensor],
|
||||
k: torch.Tensor,
|
||||
k: torch.Tensor | None,
|
||||
weights: torch.Tensor,
|
||||
):
|
||||
if current_platform.is_cuda() or current_platform.is_xpu():
|
||||
@@ -798,7 +798,7 @@ class SparseAttnIndexer(CustomOp):
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
q_quant: torch.Tensor | tuple[torch.Tensor, torch.Tensor],
|
||||
k: torch.Tensor,
|
||||
k: torch.Tensor | None,
|
||||
weights: torch.Tensor,
|
||||
):
|
||||
# FP8 path: single tensor (per-token scale is folded into `weights`).
|
||||
@@ -835,7 +835,7 @@ class SparseAttnIndexer(CustomOp):
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
q_fp8: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
k: torch.Tensor | None,
|
||||
weights: torch.Tensor,
|
||||
):
|
||||
return self.forward_cuda(hidden_states, q_fp8, k, weights)
|
||||
@@ -844,7 +844,7 @@ class SparseAttnIndexer(CustomOp):
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
q_quant: torch.Tensor | tuple[torch.Tensor, torch.Tensor],
|
||||
k: torch.Tensor,
|
||||
k: torch.Tensor | None,
|
||||
weights: torch.Tensor,
|
||||
):
|
||||
assert not self.use_fp4_cache, "AMD platform doesn't support fp4 cache yet"
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from typing import TYPE_CHECKING, cast
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from transformers import DeepseekV2Config, DeepseekV3Config
|
||||
@@ -7,8 +9,14 @@ from transformers import DeepseekV2Config, DeepseekV3Config
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.config import CacheConfig, VllmConfig
|
||||
from vllm.distributed import get_tensor_model_parallel_world_size
|
||||
from vllm.distributed.parallel_state import get_tp_group
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.model_executor.layers.attention import MLAAttention
|
||||
from vllm.model_executor.layers.attention.attention import get_attention_context
|
||||
from vllm.model_executor.layers.attention.pcp import (
|
||||
finalize_mla_pcp_decode,
|
||||
maybe_gather_mla_latent_cache_inputs,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import LayerNorm, RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
@@ -35,6 +43,9 @@ from vllm.models.deepseek_v32.common.kernels import fused_norm_rope, fused_q
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import is_quantized_kv_cache
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.model_executor.layers.attention.mla_attention import MLACommonMetadata
|
||||
|
||||
|
||||
class DeepseekV32Indexer(nn.Module):
|
||||
indexer_cache_cls = DeepseekV32IndexerCache
|
||||
@@ -363,6 +374,7 @@ class DeepseekV32Attention(MLAAttention):
|
||||
attn_metadata = attn_metadata_raw[0].get(self.layer_name)
|
||||
else:
|
||||
attn_metadata = attn_metadata_raw
|
||||
attn_metadata = cast("MLACommonMetadata | None", attn_metadata)
|
||||
|
||||
slot_mapping = forward_context.slot_mapping
|
||||
assert isinstance(slot_mapping, dict)
|
||||
@@ -374,7 +386,8 @@ class DeepseekV32Attention(MLAAttention):
|
||||
indexer_k_norm_bias = self.indexer.k_norm.bias
|
||||
indexer_k_norm_eps = self.indexer.k_norm.eps
|
||||
indexer_k_rope_cos_sin_cache = self.indexer_rope_emb.cos_sin_cache
|
||||
indexer_k_cache = self.indexer.k_cache.kv_cache
|
||||
indexer_k_cache = None if self.use_pcp else self.indexer.k_cache.kv_cache
|
||||
index_k_out = torch.empty_like(index_k) if self.use_pcp else None
|
||||
indexer_softmax_scale = self.indexer.softmax_scale
|
||||
indexer_n_head_scale = self.indexer.n_head**-0.5
|
||||
else:
|
||||
@@ -384,10 +397,11 @@ class DeepseekV32Attention(MLAAttention):
|
||||
indexer_k_norm_eps = 1e-6
|
||||
indexer_k_rope_cos_sin_cache = None
|
||||
indexer_k_cache = None
|
||||
index_k_out = None
|
||||
indexer_softmax_scale = 0.0
|
||||
indexer_n_head_scale = 0.0
|
||||
|
||||
if attn_metadata is None:
|
||||
if attn_metadata is None or self.use_pcp:
|
||||
mla_kv_cache = None
|
||||
mla_k_scale = None
|
||||
indexer_k_cache = None
|
||||
@@ -396,6 +410,8 @@ class DeepseekV32Attention(MLAAttention):
|
||||
mla_kv_cache = self.kv_cache
|
||||
mla_k_scale = self._k_scale
|
||||
|
||||
kv_c_out = torch.empty_like(kv_c) if self.use_pcp else None
|
||||
k_pe_out = torch.empty_like(k_pe) if self.use_pcp else None
|
||||
q_c = fused_norm_rope(
|
||||
positions,
|
||||
q_c,
|
||||
@@ -419,12 +435,14 @@ class DeepseekV32Attention(MLAAttention):
|
||||
mla_k_scale=mla_k_scale,
|
||||
has_indexer=has_indexer,
|
||||
index_rope_interleave=self._index_rope_interleave,
|
||||
kv_c_out=kv_c_out,
|
||||
k_pe_out=k_pe_out,
|
||||
index_k_out=index_k_out,
|
||||
)
|
||||
|
||||
q = self.q_b_proj(q_c)[0].view(-1, self.num_local_heads, self.qk_head_dim)
|
||||
q_nope, q_pe = q.split([self.qk_nope_head_dim, self.qk_rope_head_dim], dim=-1)
|
||||
q_nope = q_nope.transpose(0, 1)
|
||||
ql_nope = torch.bmm(q_nope, self.W_UK_T).transpose(0, 1)
|
||||
ql_nope = torch.bmm(q_nope.transpose(0, 1), self.W_UK_T).transpose(0, 1)
|
||||
|
||||
if self.indexer is not None and not self.skip_topk:
|
||||
index_q = self.indexer.wq_b(q_c)[0]
|
||||
@@ -451,7 +469,10 @@ class DeepseekV32Attention(MLAAttention):
|
||||
self._sparse_indexer_and_attn(
|
||||
q_c,
|
||||
index_q_fp8,
|
||||
index_k_out,
|
||||
index_weights_out,
|
||||
kv_c_out,
|
||||
k_pe_out,
|
||||
ql_nope,
|
||||
mqa_q,
|
||||
output,
|
||||
@@ -463,19 +484,26 @@ class DeepseekV32Attention(MLAAttention):
|
||||
self,
|
||||
q_c: torch.Tensor,
|
||||
index_q_fp8: torch.Tensor | None,
|
||||
index_k: torch.Tensor | None,
|
||||
index_weights_out: torch.Tensor | None,
|
||||
kv_c: torch.Tensor | None,
|
||||
k_pe: torch.Tensor | None,
|
||||
ql_nope: torch.Tensor,
|
||||
mqa_q: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
) -> None:
|
||||
if self.indexer is not None and not self.skip_topk:
|
||||
assert index_q_fp8 is not None
|
||||
assert index_weights_out is not None
|
||||
if self.use_pcp:
|
||||
assert index_k is not None
|
||||
sparse_attn_indexer(
|
||||
q_c,
|
||||
self.indexer.k_cache.prefix,
|
||||
self.indexer.k_cache.kv_cache,
|
||||
index_q_fp8,
|
||||
None, # q_scale folded into weights on the fp8 path
|
||||
None, # k unused when skip_k_cache_insert=True
|
||||
None,
|
||||
index_k,
|
||||
index_weights_out,
|
||||
self.indexer.quant_block_size,
|
||||
self.indexer.scale_fmt,
|
||||
@@ -484,28 +512,56 @@ class DeepseekV32Attention(MLAAttention):
|
||||
self.indexer.max_model_len,
|
||||
self.indexer.max_total_seq_len,
|
||||
self.topk_indices_buffer,
|
||||
skip_k_cache_insert=True,
|
||||
use_pcp=False,
|
||||
skip_k_cache_insert=not self.use_pcp,
|
||||
use_pcp=self.use_pcp,
|
||||
dense_mha_metadata_layer_name=self._dense_mha_metadata_layer_name,
|
||||
use_fp4_cache=False,
|
||||
# fused_norm_rope already cleared the topk buffer this forward.
|
||||
dcp_rank=(
|
||||
self.dcp_manager.group.rank_in_group
|
||||
if self.dcp_manager is not None
|
||||
else 0
|
||||
),
|
||||
dcp_world_size=(
|
||||
self._vllm_config.parallel_config.decode_context_parallel_size
|
||||
),
|
||||
cp_kv_cache_interleave_size=(
|
||||
self._vllm_config.parallel_config.cp_kv_cache_interleave_size
|
||||
),
|
||||
skip_topk_buffer_clear=True,
|
||||
)
|
||||
|
||||
attn_metadata_raw = get_forward_context().attn_metadata
|
||||
if isinstance(attn_metadata_raw, dict):
|
||||
attn_metadata = attn_metadata_raw.get(self.layer_name)
|
||||
elif isinstance(attn_metadata_raw, list):
|
||||
attn_metadata = attn_metadata_raw[0].get(self.layer_name)
|
||||
else:
|
||||
attn_metadata = attn_metadata_raw
|
||||
|
||||
attn_metadata, _, kv_cache, layer_slot_mapping = get_attention_context(
|
||||
self.layer_name
|
||||
)
|
||||
if attn_metadata is None:
|
||||
output.zero_()
|
||||
return
|
||||
attn_metadata = cast("MLACommonMetadata", attn_metadata)
|
||||
|
||||
if self.use_pcp:
|
||||
assert kv_c is not None and k_pe is not None
|
||||
kv_for_cache, kpe_for_cache, cache_slot_mapping = (
|
||||
maybe_gather_mla_latent_cache_inputs(
|
||||
kv_c,
|
||||
k_pe.unsqueeze(1),
|
||||
layer_slot_mapping,
|
||||
attn_metadata.num_decode_tokens,
|
||||
True,
|
||||
)
|
||||
)
|
||||
self.impl.do_kv_cache_update( # type: ignore[attr-defined]
|
||||
kv_for_cache,
|
||||
kpe_for_cache,
|
||||
kv_cache,
|
||||
cache_slot_mapping,
|
||||
self.kv_cache_dtype,
|
||||
self._k_scale,
|
||||
)
|
||||
|
||||
num_actual = attn_metadata.num_actual_tokens # type: ignore[attr-defined]
|
||||
kv_cache = self.kv_cache
|
||||
if num_actual == 0:
|
||||
output.zero_()
|
||||
return
|
||||
|
||||
if self._fp8_kv_needs_view:
|
||||
kv_cache = kv_cache.view(torch.float8_e4m3fn)
|
||||
if self._fp8_query:
|
||||
@@ -515,10 +571,35 @@ class DeepseekV32Attention(MLAAttention):
|
||||
]
|
||||
else:
|
||||
mqa_q_arg = (ql_nope[:num_actual], mqa_q[:num_actual])
|
||||
attn_out, _ = self.impl.forward_mqa( # type: ignore[attr-defined]
|
||||
|
||||
if self.use_pcp and self.impl.dcp_world_size > self.impl.pcp_world_size:
|
||||
if isinstance(mqa_q_arg, tuple):
|
||||
mqa_q_arg = torch.cat(mqa_q_arg, dim=-1)
|
||||
mqa_q_arg = get_tp_group().all_gather(mqa_q_arg, dim=1)
|
||||
attn_out, lse = self.impl.forward_mqa( # type: ignore[attr-defined]
|
||||
mqa_q_arg, kv_cache, attn_metadata, self
|
||||
)
|
||||
|
||||
if self.use_pcp and self.impl.dcp_world_size > 1:
|
||||
assert lse is not None and self.dcp_manager is not None
|
||||
seq_lens = (
|
||||
attn_metadata.decode.seq_lens
|
||||
if attn_metadata.decode is not None
|
||||
else cast(torch.Tensor, attn_metadata.seq_lens)[ # type: ignore[attr-defined]
|
||||
: attn_metadata.num_decodes
|
||||
]
|
||||
)
|
||||
query_start_loc = attn_metadata.query_start_loc[
|
||||
: attn_metadata.num_decodes + 1
|
||||
]
|
||||
attn_out = self.dcp_manager.combine(
|
||||
attn_out,
|
||||
lse,
|
||||
seq_lens=seq_lens,
|
||||
query_start_loc=query_start_loc,
|
||||
)
|
||||
attn_out = finalize_mla_pcp_decode(attn_out, self.num_heads)
|
||||
|
||||
# NOTE(woosuk): While the below does not need to be in the eager region,
|
||||
# we put it here to avoid copying the attention output. Move this back to the
|
||||
# captured region once forward_mqa supports `out` argument.
|
||||
@@ -531,3 +612,5 @@ class DeepseekV32Attention(MLAAttention):
|
||||
.transpose(0, 1)
|
||||
)
|
||||
torch.bmm(x, self.W_UV, out=out)
|
||||
if self.use_pcp and num_actual < output.shape[0]:
|
||||
output[num_actual:].zero_()
|
||||
|
||||
@@ -103,12 +103,16 @@ def _fused_norm_rope_kernel(
|
||||
kv_stride,
|
||||
kv_rms_norm_w_ptr,
|
||||
kv_rms_eps,
|
||||
kv_out_ptr,
|
||||
kv_out_stride,
|
||||
KV_DIM: tl.constexpr,
|
||||
# KV RoPE
|
||||
kpe_ptr,
|
||||
kpe_stride,
|
||||
kpe_rope_cos_sin_cache_ptr,
|
||||
kpe_rope_cos_sin_cache_stride,
|
||||
kpe_out_ptr,
|
||||
kpe_out_stride,
|
||||
KPE_HALF_ROT_DIM: tl.constexpr,
|
||||
# Index K layer norm
|
||||
index_k_ptr,
|
||||
@@ -121,6 +125,8 @@ def _fused_norm_rope_kernel(
|
||||
# Index K RoPE
|
||||
index_k_rope_cos_sin_cache_ptr,
|
||||
index_k_rope_cos_sin_cache_stride,
|
||||
index_k_out_ptr,
|
||||
index_k_out_stride,
|
||||
INDEX_K_HALF_ROT_DIM: tl.constexpr,
|
||||
# Cache params (shared by indexer K and MLA)
|
||||
slot_mapping_ptr,
|
||||
@@ -174,10 +180,9 @@ def _fused_norm_rope_kernel(
|
||||
return
|
||||
|
||||
if slot_mapping_ptr is None:
|
||||
# Memory profiling run.
|
||||
return
|
||||
slot_idx = tl.load(slot_mapping_ptr + tok_idx)
|
||||
if slot_idx < 0:
|
||||
if kv_out_ptr is None and kpe_out_ptr is None and index_k_out_ptr is None:
|
||||
return
|
||||
elif tl.load(slot_mapping_ptr + tok_idx) < 0:
|
||||
# Padding
|
||||
return
|
||||
|
||||
@@ -217,63 +222,81 @@ def _fused_norm_rope_kernel(
|
||||
r1 = x1 * cos - x2 * sin
|
||||
r2 = x2 * cos + x1 * sin
|
||||
|
||||
# MLA concat_and_cache: write [kv_c_normed, k_pe_roped] to cache.
|
||||
if mla_cache_entry_stride == 0:
|
||||
return
|
||||
# PCP materializes K for the cross-rank gather, then inserts it into cache.
|
||||
if kv_out_ptr is not None:
|
||||
tl.store(kv_out_ptr + tok_idx * kv_out_stride + kv_block, kv_c)
|
||||
if kpe_out_ptr is not None:
|
||||
tl.store(
|
||||
kpe_out_ptr + tok_idx * kpe_out_stride + dim_off * 2,
|
||||
r1.to(kpe_out_ptr.dtype.element_ty),
|
||||
)
|
||||
tl.store(
|
||||
kpe_out_ptr + tok_idx * kpe_out_stride + dim_off * 2 + 1,
|
||||
r2.to(kpe_out_ptr.dtype.element_ty),
|
||||
)
|
||||
|
||||
mla_block_size = mla_cache_block_stride // mla_cache_entry_stride
|
||||
mla_block_idx = slot_idx // mla_block_size
|
||||
mla_block_off = slot_idx % mla_block_size
|
||||
if slot_mapping_ptr is not None:
|
||||
# MLA concat_and_cache: write [kv_c_normed, k_pe_roped] to cache.
|
||||
if mla_cache_entry_stride == 0:
|
||||
return
|
||||
|
||||
if MLA_CACHE_DS_MLA:
|
||||
# fp8_ds_mla layout (DeepSeek-V3.2, KV_DIM == 512): per-128-element
|
||||
# tile of the NoPE is dynamically quantized to fp8 with its own
|
||||
# float32 scale; the RoPE tail is stored unquantized in bf16.
|
||||
# bytes [0, KV_DIM) : KV_DIM fp8 NoPE values
|
||||
# bytes [KV_DIM, KV_DIM + 16) : MLA_NUM_TILES float32 scales
|
||||
# bytes [KV_DIM + 16, ...) : 2 * KPE_HALF_ROT_DIM bf16 RoPE
|
||||
# mla_cache_block_stride / mla_cache_entry_stride are byte strides
|
||||
# (mla_cache_ptr is the 1-byte fp8 view of the uint8 cache).
|
||||
byte_base = (
|
||||
mla_block_idx * mla_cache_block_stride
|
||||
slot_idx = tl.load(slot_mapping_ptr + tok_idx)
|
||||
mla_block_size = mla_cache_block_stride // mla_cache_entry_stride
|
||||
mla_block_idx = slot_idx // mla_block_size
|
||||
mla_block_off = slot_idx % mla_block_size
|
||||
|
||||
if MLA_CACHE_DS_MLA:
|
||||
# fp8_ds_mla layout (DeepSeek-V3.2, KV_DIM == 512): per-128-element
|
||||
# tile of the NoPE is dynamically quantized to fp8 with its own
|
||||
# float32 scale; the RoPE tail is stored unquantized in bf16.
|
||||
# bytes [0, KV_DIM) : KV_DIM fp8 NoPE values
|
||||
# bytes [KV_DIM, KV_DIM + 16) : MLA_NUM_TILES float32 scales
|
||||
# bytes [KV_DIM + 16, ...) : 2 * KPE_HALF_ROT_DIM bf16 RoPE
|
||||
# mla_cache_block_stride / mla_cache_entry_stride are byte strides
|
||||
# (mla_cache_ptr is the 1-byte fp8 view of the uint8 cache).
|
||||
byte_base = (
|
||||
mla_block_idx * mla_cache_block_stride
|
||||
+ mla_block_off * mla_cache_entry_stride
|
||||
)
|
||||
kv_2d = tl.reshape(kv_c, (MLA_NUM_TILES, MLA_TILE_DIM))
|
||||
tile_amax = tl.max(tl.abs(kv_2d), axis=1, keep_dims=True)
|
||||
# scale = amax / 448 (fp8 e4m3 max), matching the reference
|
||||
# concat_and_cache_ds_mla kernel; floored to FLT_MIN.
|
||||
tile_scale = tl.maximum(tile_amax * (1.0 / 448.0), 1.1754944e-38)
|
||||
kv_c_fp8 = tl.reshape((kv_2d / tile_scale).to(tl.float8e4nv), (KV_DIM,))
|
||||
tl.store(mla_cache_ptr + byte_base + kv_block, kv_c_fp8)
|
||||
tile_off = tl.arange(0, MLA_NUM_TILES)
|
||||
tl.store(
|
||||
mla_cache_ds_scale_ptr + byte_base // 4 + KV_DIM // 4 + tile_off,
|
||||
tl.reshape(tile_scale, (MLA_NUM_TILES,)),
|
||||
)
|
||||
rope_dst = mla_cache_ds_rope_ptr + byte_base // 2 + (KV_DIM // 2 + 8)
|
||||
tl.store(rope_dst + dim_off * 2, r1.to(tl.bfloat16))
|
||||
tl.store(rope_dst + dim_off * 2 + 1, r2.to(tl.bfloat16))
|
||||
return
|
||||
|
||||
dst = (
|
||||
mla_cache_ptr
|
||||
+ mla_block_idx * mla_cache_block_stride
|
||||
+ mla_block_off * mla_cache_entry_stride
|
||||
)
|
||||
kv_2d = tl.reshape(kv_c, (MLA_NUM_TILES, MLA_TILE_DIM))
|
||||
tile_amax = tl.max(tl.abs(kv_2d), axis=1, keep_dims=True)
|
||||
# scale = amax / 448 (fp8 e4m3 max), matching the reference
|
||||
# concat_and_cache_ds_mla kernel; floored to FLT_MIN.
|
||||
tile_scale = tl.maximum(tile_amax * (1.0 / 448.0), 1.1754944e-38)
|
||||
kv_c_fp8 = tl.reshape((kv_2d / tile_scale).to(tl.float8e4nv), (KV_DIM,))
|
||||
tl.store(mla_cache_ptr + byte_base + kv_block, kv_c_fp8)
|
||||
tile_off = tl.arange(0, MLA_NUM_TILES)
|
||||
tl.store(
|
||||
mla_cache_ds_scale_ptr + byte_base // 4 + KV_DIM // 4 + tile_off,
|
||||
tl.reshape(tile_scale, (MLA_NUM_TILES,)),
|
||||
)
|
||||
rope_dst = mla_cache_ds_rope_ptr + byte_base // 2 + (KV_DIM // 2 + 8)
|
||||
tl.store(rope_dst + dim_off * 2, r1.to(tl.bfloat16))
|
||||
tl.store(rope_dst + dim_off * 2 + 1, r2.to(tl.bfloat16))
|
||||
return
|
||||
|
||||
dst = (
|
||||
mla_cache_ptr
|
||||
+ mla_block_idx * mla_cache_block_stride
|
||||
+ mla_block_off * mla_cache_entry_stride
|
||||
)
|
||||
# kv_c_normed (KV_DIM elements)
|
||||
if MLA_CACHE_FP8:
|
||||
scale = tl.load(mla_cache_scale_ptr)
|
||||
kv_c_fp8 = (kv_c.to(tl.float32) / scale).to(tl.float8e4nv)
|
||||
tl.store(dst + kv_block, kv_c_fp8)
|
||||
else:
|
||||
tl.store(dst + kv_block, kv_c)
|
||||
# k_pe_roped (from registers, interleaved layout)
|
||||
if MLA_CACHE_FP8:
|
||||
tl.store(dst + KV_DIM + dim_off * 2, (r1 / scale).to(tl.float8e4nv))
|
||||
tl.store(dst + KV_DIM + dim_off * 2 + 1, (r2 / scale).to(tl.float8e4nv))
|
||||
else:
|
||||
tl.store(dst + KV_DIM + dim_off * 2, r1)
|
||||
tl.store(dst + KV_DIM + dim_off * 2 + 1, r2)
|
||||
# kv_c_normed (KV_DIM elements)
|
||||
if MLA_CACHE_FP8:
|
||||
scale = tl.load(mla_cache_scale_ptr)
|
||||
kv_c_fp8 = (kv_c.to(tl.float32) / scale).to(tl.float8e4nv)
|
||||
tl.store(dst + kv_block, kv_c_fp8)
|
||||
else:
|
||||
tl.store(dst + kv_block, kv_c)
|
||||
# k_pe_roped (from registers, interleaved layout)
|
||||
if MLA_CACHE_FP8:
|
||||
tl.store(dst + KV_DIM + dim_off * 2, (r1 / scale).to(tl.float8e4nv))
|
||||
tl.store(
|
||||
dst + KV_DIM + dim_off * 2 + 1,
|
||||
(r2 / scale).to(tl.float8e4nv),
|
||||
)
|
||||
else:
|
||||
tl.store(dst + KV_DIM + dim_off * 2, r1)
|
||||
tl.store(dst + KV_DIM + dim_off * 2 + 1, r2)
|
||||
elif pid == 0:
|
||||
if not HAS_INDEXER:
|
||||
# Shared layer: no indexer K to process.
|
||||
@@ -354,20 +377,27 @@ def _fused_norm_rope_kernel(
|
||||
roped = normed * cos_full + sign * normed_partner * sin_full
|
||||
result = tl.where(in_rope, roped, normed)
|
||||
|
||||
# 3. FP8 quantize + cache write from registers.
|
||||
# No need to write back to index_k_ptr — the only consumer
|
||||
# (sparse_attn_indexer) reads from the cache, not index_k.
|
||||
_fp8_quant_and_cache_write(
|
||||
result,
|
||||
index_k_mask,
|
||||
slot_idx,
|
||||
indexer_cache_ptr,
|
||||
indexer_cache_scale_ptr,
|
||||
indexer_cache_block_size,
|
||||
indexer_cache_stride,
|
||||
index_k_block,
|
||||
INDEX_K_DIM,
|
||||
)
|
||||
if index_k_out_ptr is not None:
|
||||
tl.store(
|
||||
index_k_out_ptr + tok_idx * index_k_out_stride + index_k_block,
|
||||
result.to(index_k_out_ptr.dtype.element_ty),
|
||||
mask=index_k_mask,
|
||||
)
|
||||
|
||||
# PCP inserts index K after gathering; other paths write it directly.
|
||||
if indexer_cache_ptr is not None and slot_mapping_ptr is not None:
|
||||
slot_idx = tl.load(slot_mapping_ptr + tok_idx)
|
||||
_fp8_quant_and_cache_write(
|
||||
result,
|
||||
index_k_mask,
|
||||
slot_idx,
|
||||
indexer_cache_ptr,
|
||||
indexer_cache_scale_ptr,
|
||||
indexer_cache_block_size,
|
||||
indexer_cache_stride,
|
||||
index_k_block,
|
||||
INDEX_K_DIM,
|
||||
)
|
||||
|
||||
|
||||
def fused_norm_rope(
|
||||
@@ -395,6 +425,9 @@ def fused_norm_rope(
|
||||
has_indexer: bool = True,
|
||||
index_rope_interleave: bool = False,
|
||||
q_c_out: torch.Tensor | None = None,
|
||||
kv_c_out: torch.Tensor | None = None,
|
||||
k_pe_out: torch.Tensor | None = None,
|
||||
index_k_out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
assert positions.ndim == 1
|
||||
assert q_c.ndim == 2
|
||||
@@ -420,6 +453,10 @@ def fused_norm_rope(
|
||||
assert index_k_rope_cos_sin_cache is not None
|
||||
index_k_dim = index_k.shape[-1]
|
||||
topk = topk_indices_buffer.shape[-1]
|
||||
if indexer_k_cache is not None or mla_kv_cache is not None:
|
||||
assert slot_mapping is not None
|
||||
else:
|
||||
slot_mapping = None
|
||||
|
||||
# --- Indexer K cache setup ---
|
||||
if indexer_k_cache is not None:
|
||||
@@ -430,17 +467,9 @@ def fused_norm_rope(
|
||||
if indexer_k_cache.dtype == torch.uint8:
|
||||
indexer_k_cache = indexer_k_cache.view(torch.float8_e4m3fn)
|
||||
else:
|
||||
# No indexer cache (shared layer / MLA-only fusion). Use dummies but
|
||||
# KEEP the caller's slot_mapping so the MLA write (pid 1) still runs.
|
||||
idx_cache_scale_view = torch.empty(0, dtype=torch.float32, device=device)
|
||||
indexer_k_cache = torch.empty(0, dtype=torch.float8_e4m3fn, device=device)
|
||||
idx_cache_scale_view = None
|
||||
idx_cache_block_size = 1
|
||||
idx_cache_stride = 1
|
||||
if mla_kv_cache is None:
|
||||
# Pure profiling run (no caches at all): skip all per-token writes.
|
||||
slot_mapping = torch.full(
|
||||
(num_tokens,), -1, dtype=torch.int64, device=device
|
||||
)
|
||||
idx_cache_stride = 0
|
||||
|
||||
# --- MLA KV cache setup ---
|
||||
mla_cache_ds_mla = mla_kv_cache_dtype == "fp8_ds_mla"
|
||||
@@ -469,15 +498,26 @@ def fused_norm_rope(
|
||||
if mla_k_scale is None:
|
||||
mla_k_scale = torch.ones(1, dtype=torch.float32, device=device)
|
||||
else:
|
||||
# Dummy values — pid 2 will skip the MLA cache write because
|
||||
# slot_mapping is all -1.
|
||||
# Dummy cache values; a zero entry stride disables the cache write.
|
||||
mla_kv_cache = torch.empty(0, dtype=torch.bfloat16, device=device)
|
||||
mla_block_stride = 0
|
||||
mla_entry_stride = 0
|
||||
mla_k_scale = torch.ones(1, dtype=torch.float32, device=device)
|
||||
mla_k_scale = _dummy((1,), torch.float32, device)
|
||||
|
||||
if q_c_out is None:
|
||||
q_c_out = torch.empty_like(q_c)
|
||||
kv_c_out_stride = 0
|
||||
k_pe_out_stride = 0
|
||||
index_k_out_stride = 0
|
||||
if kv_c_out is not None:
|
||||
assert kv_c_out.shape == kv_c.shape
|
||||
kv_c_out_stride = kv_c_out.stride(0)
|
||||
if k_pe_out is not None:
|
||||
assert k_pe_out.shape == k_pe.shape
|
||||
k_pe_out_stride = k_pe_out.stride(0)
|
||||
if index_k_out is not None:
|
||||
assert index_k_out.shape == index_k.shape
|
||||
index_k_out_stride = index_k_out.stride(0)
|
||||
use_pdl = current_platform.is_arch_support_pdl()
|
||||
_fused_norm_rope_kernel[(num_tokens, 4)](
|
||||
positions,
|
||||
@@ -495,12 +535,16 @@ def fused_norm_rope(
|
||||
kv_c.stride(0),
|
||||
kv_rms_norm_w,
|
||||
kv_rms_eps,
|
||||
kv_c_out,
|
||||
kv_c_out_stride,
|
||||
kv_dim,
|
||||
# KV RoPE
|
||||
k_pe,
|
||||
k_pe.stride(0),
|
||||
k_rope_cos_sin_cache,
|
||||
k_rope_cos_sin_cache.stride(0),
|
||||
k_pe_out,
|
||||
k_pe_out_stride,
|
||||
k_rope_cos_sin_cache.shape[-1] // 2,
|
||||
# Index K layer norm + RoPE + FP8 quant
|
||||
index_k,
|
||||
@@ -512,6 +556,8 @@ def fused_norm_rope(
|
||||
triton.next_power_of_2(index_k_dim),
|
||||
index_k_rope_cos_sin_cache,
|
||||
index_k_rope_cos_sin_cache.stride(0),
|
||||
index_k_out,
|
||||
index_k_out_stride,
|
||||
index_k_rope_cos_sin_cache.shape[-1] // 2,
|
||||
# Cache params
|
||||
slot_mapping,
|
||||
@@ -793,7 +839,6 @@ def fused_q(
|
||||
assert ql_nope.ndim == 3
|
||||
assert ql_nope.shape[:2] == q_pe.shape[:2]
|
||||
assert q_scale.dtype == torch.float32 and q_scale.numel() == 1
|
||||
|
||||
num_tokens = positions.shape[0]
|
||||
num_q_heads = q_pe.shape[1]
|
||||
# Grid's 3rd dim must cover the MQA-pack heads (pid 0/2 iterate 2 heads
|
||||
|
||||
@@ -682,7 +682,7 @@ def rocm_aiter_sparse_attn_indexer_fake(
|
||||
k_cache_prefix: LayerNameType,
|
||||
kv_cache: torch.Tensor,
|
||||
q_fp8: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
k: torch.Tensor | None,
|
||||
weights: torch.Tensor,
|
||||
quant_block_size: int,
|
||||
scale_fmt: str | None,
|
||||
@@ -702,7 +702,7 @@ def rocm_aiter_sparse_attn_indexer(
|
||||
k_cache_prefix: LayerNameType,
|
||||
kv_cache: torch.Tensor,
|
||||
q_fp8: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
k: torch.Tensor | None,
|
||||
weights: torch.Tensor,
|
||||
quant_block_size: int,
|
||||
scale_fmt: str | None,
|
||||
|
||||
Reference in New Issue
Block a user