[ROCm][Model][Bugfix] Enable GLM-5.2-MXFP4 on the deepseek_v32 path and fix sparse attention correctness (#51915)

Signed-off-by: Jack Hu <Jack.Hu@amd.com>
Signed-off-by: Jack Hu <jack.hu@amd.com>
Signed-off-by: Douglas Lehr <Doug.Lehr@amd.com>
Co-authored-by: James E T Smith <jamesETsmith@users.noreply.github.com>
Co-authored-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com>
Co-authored-by: Douglas Lehr <Doug.Lehr@amd.com>
This commit is contained in:
Jack Hu
2026-09-22 18:08:51 -05:00
committed by GitHub
co-authored by James E T Smith Douglas Lehr Douglas Lehr
parent 9646f53064
commit 1c0eee919d
8 changed files with 318 additions and 211 deletions
+186 -150
View File
@@ -24,6 +24,8 @@ outputs are checked within 1 representable-step (ULP); bf16 norm/RoPE outputs us
rtol/atol=1e-2 (the tolerance the sibling deepseek_v4 fused-kernel test uses). rtol/atol=1e-2 (the tolerance the sibling deepseek_v4 fused-kernel test uses).
""" """
from typing import NamedTuple
import pytest import pytest
import torch import torch
@@ -33,19 +35,30 @@ from vllm.platforms import current_platform
FP8 = torch.float8_e4m3fn FP8 = torch.float8_e4m3fn
FP8_MAX = 448.0 FP8_MAX = 448.0
# GLM-5.2 / DeepSeek-V3.2 shapes (TP8 local heads).
Q_LORA = 2048
KV_LORA = 512
ROPE_DIM = 64
NUM_HEADS = 8
INDEX_HEADS = 32
INDEX_HEAD_DIM = 128
HIDDEN = 6144
EPS = 1e-6 EPS = 1e-6
class ModelConfig(NamedTuple):
"""Per-model attention dims (num_heads is the TP8 local count)."""
q_lora: int
kv_lora: int
rope_dim: int
num_heads: int
index_heads: int
index_head_dim: int
hidden: int
MODEL_CONFIGS = [
ModelConfig(2048, 512, 64, 8, 32, 128, 6144), # GLM-5.2
ModelConfig(1536, 512, 64, 16, 64, 128, 7168), # DeepSeek-V3.2
]
MODEL_IDS = ["glm5.2", "dsv3.2"]
pytestmark = pytest.mark.skipif( pytestmark = pytest.mark.skipif(
not current_platform.is_cuda() or not current_platform.has_device_capability(89), not current_platform.supports_fp8(),
reason="deepseek_v32 fused kernels require CUDA with fp8 (SM89+)", reason="deepseek_v32 fused kernels require fp8 support",
) )
@@ -144,29 +157,32 @@ def assert_fp8(got: torch.Tensor, ref: torch.Tensor, msg: str):
# ── fused_norm_rope ────────────────────────────────────────────────────────── # ── fused_norm_rope ──────────────────────────────────────────────────────────
@pytest.mark.parametrize("cfg", MODEL_CONFIGS, ids=MODEL_IDS)
@pytest.mark.parametrize("num_tokens", [1, 4, 17, 512, 4096]) @pytest.mark.parametrize("num_tokens", [1, 4, 17, 512, 4096])
@pytest.mark.parametrize("index_interleave", [True, False]) @pytest.mark.parametrize("index_interleave", [True, False])
@pytest.mark.parametrize("mla_dtype", ["auto", "bfloat16", "fp8"]) @pytest.mark.parametrize("mla_dtype", ["auto", "bfloat16", "fp8"])
def test_fused_norm_rope(num_tokens: int, index_interleave: bool, mla_dtype: str): def test_fused_norm_rope(
num_tokens: int, index_interleave: bool, mla_dtype: str, cfg: ModelConfig
):
torch.manual_seed(0) torch.manual_seed(0)
dev = "cuda" dev = "cuda"
max_pos = 8192 max_pos = 8192
pos = torch.arange(num_tokens, device=dev, dtype=torch.int64) % max_pos pos = torch.arange(num_tokens, device=dev, dtype=torch.int64) % max_pos
q_c = torch.randn(num_tokens, Q_LORA, device=dev, dtype=torch.bfloat16) q_c = torch.randn(num_tokens, cfg.q_lora, device=dev, dtype=torch.bfloat16)
kv_c = torch.randn(num_tokens, KV_LORA, device=dev, dtype=torch.bfloat16) kv_c = torch.randn(num_tokens, cfg.kv_lora, device=dev, dtype=torch.bfloat16)
k_pe = torch.randn(num_tokens, ROPE_DIM, device=dev, dtype=torch.bfloat16) k_pe = torch.randn(num_tokens, cfg.rope_dim, device=dev, dtype=torch.bfloat16)
qw = torch.randn(Q_LORA, device=dev, dtype=torch.bfloat16) qw = torch.randn(cfg.q_lora, device=dev, dtype=torch.bfloat16)
kvw = torch.randn(KV_LORA, device=dev, dtype=torch.bfloat16) kvw = torch.randn(cfg.kv_lora, device=dev, dtype=torch.bfloat16)
ik = torch.randn(num_tokens, INDEX_HEAD_DIM, device=dev, dtype=torch.bfloat16) ik = torch.randn(num_tokens, cfg.index_head_dim, device=dev, dtype=torch.bfloat16)
ikw = torch.randn(INDEX_HEAD_DIM, device=dev, dtype=torch.float32) ikw = torch.randn(cfg.index_head_dim, device=dev, dtype=torch.float32)
ikb = torch.randn(INDEX_HEAD_DIM, device=dev, dtype=torch.float32) ikb = torch.randn(cfg.index_head_dim, device=dev, dtype=torch.float32)
mla_cos_sin = make_cos_sin(max_pos, ROPE_DIM, dev) # MLA k_pe: interleaved mla_cos_sin = make_cos_sin(max_pos, cfg.rope_dim, dev) # MLA k_pe: interleaved
idx_cos_sin = make_cos_sin(max_pos, ROPE_DIM, dev) idx_cos_sin = make_cos_sin(max_pos, cfg.rope_dim, dev)
bs = max_pos # single block covering all tokens bs = max_pos # single block covering all tokens
mla_dim = KV_LORA + ROPE_DIM mla_dim = cfg.kv_lora + cfg.rope_dim
mla_fp8 = mla_dtype == "fp8" mla_fp8 = mla_dtype == "fp8"
if mla_fp8: if mla_fp8:
mla_cache = torch.zeros(1, bs, mla_dim, device=dev, dtype=torch.uint8) mla_cache = torch.zeros(1, bs, mla_dim, device=dev, dtype=torch.uint8)
@@ -174,7 +190,7 @@ def test_fused_norm_rope(num_tokens: int, index_interleave: bool, mla_dtype: str
else: else:
mla_cache = torch.zeros(1, bs, mla_dim, device=dev, dtype=torch.bfloat16) mla_cache = torch.zeros(1, bs, mla_dim, device=dev, dtype=torch.bfloat16)
mla_k_scale = None mla_k_scale = None
idx_row = INDEX_HEAD_DIM + INDEX_HEAD_DIM // 128 * 4 # 132 idx_row = cfg.index_head_dim + cfg.index_head_dim // 128 * 4 # 132
idx_cache = torch.zeros(1, bs, idx_row, device=dev, dtype=torch.uint8) idx_cache = torch.zeros(1, bs, idx_row, device=dev, dtype=torch.uint8)
slot = torch.arange(num_tokens, device=dev, dtype=torch.int64) slot = torch.arange(num_tokens, device=dev, dtype=torch.int64)
topk = torch.full((num_tokens, 2048), 7, device=dev, dtype=torch.int32) topk = torch.full((num_tokens, 2048), 7, device=dev, dtype=torch.int32)
@@ -213,20 +229,20 @@ def test_fused_norm_rope(num_tokens: int, index_interleave: bool, mla_dtype: str
if mla_fp8: if mla_fp8:
cache = mla_cache.view(FP8)[0, :num_tokens] cache = mla_cache.view(FP8)[0, :num_tokens]
s = mla_k_scale.item() s = mla_k_scale.item()
assert_fp8(cache[:, :KV_LORA], (kv_ref / s).to(FP8), "MLA kv fp8") assert_fp8(cache[:, : cfg.kv_lora], (kv_ref / s).to(FP8), "MLA kv fp8")
assert_fp8(cache[:, KV_LORA:], (kpe_ref / s).to(FP8), "MLA k_pe fp8") assert_fp8(cache[:, cfg.kv_lora :], (kpe_ref / s).to(FP8), "MLA k_pe fp8")
else: else:
cache = mla_cache[0, :num_tokens] cache = mla_cache[0, :num_tokens]
assert_bf16(cache[:, :KV_LORA], kv_ref, "MLA kv bf16") assert_bf16(cache[:, : cfg.kv_lora], kv_ref, "MLA kv bf16")
assert_bf16(cache[:, KV_LORA:], kpe_ref, "MLA k_pe bf16") assert_bf16(cache[:, cfg.kv_lora :], kpe_ref, "MLA k_pe bf16")
# Indexer-K cache (packed [bs*head_dim fp8 | bs*4 fp32 scale]). # Indexer-K cache (packed [bs*head_dim fp8 | bs*4 fp32 scale]).
ik_ref = layer_norm(ik, ikw, ikb) ik_ref = layer_norm(ik, ikw, ikb)
ik_ref = rope(ik_ref, pos, idx_cos_sin, interleave=index_interleave) ik_ref = rope(ik_ref, pos, idx_cos_sin, interleave=index_interleave)
q_ref, s_ref = ue8m0_quant(ik_ref) q_ref, s_ref = ue8m0_quant(ik_ref)
flat = idx_cache[0].reshape(-1) flat = idx_cache[0].reshape(-1)
vals = flat[: bs * INDEX_HEAD_DIM].view(FP8).reshape(bs, INDEX_HEAD_DIM) vals = flat[: bs * cfg.index_head_dim].view(FP8).reshape(bs, cfg.index_head_dim)
scales = flat[bs * INDEX_HEAD_DIM :].view(torch.float32) scales = flat[bs * cfg.index_head_dim :].view(torch.float32)
assert_fp8(vals[:num_tokens], q_ref, "indexer-K fp8") assert_fp8(vals[:num_tokens], q_ref, "indexer-K fp8")
torch.testing.assert_close(scales[:num_tokens], s_ref, rtol=0, atol=0) torch.testing.assert_close(scales[:num_tokens], s_ref, rtol=0, atol=0)
@@ -234,23 +250,24 @@ def test_fused_norm_rope(num_tokens: int, index_interleave: bool, mla_dtype: str
assert (topk == -1).all(), "topk buffer not cleared on indexer layer" assert (topk == -1).all(), "topk buffer not cleared on indexer layer"
def test_fused_norm_rope_packed_indexer_block_stride(): @pytest.mark.parametrize("cfg", MODEL_CONFIGS, ids=MODEL_IDS)
def test_fused_norm_rope_packed_indexer_block_stride(cfg: ModelConfig):
"""Indexer writes use their HMA block stride and independent slot map.""" """Indexer writes use their HMA block stride and independent slot map."""
torch.manual_seed(7) torch.manual_seed(7)
dev = "cuda" dev = "cuda"
num_tokens = block_size = 2 num_tokens = block_size = 2
pos = torch.arange(num_tokens, device=dev, dtype=torch.int64) pos = torch.arange(num_tokens, device=dev, dtype=torch.int64)
q_c = torch.randn(num_tokens, Q_LORA, device=dev, dtype=torch.bfloat16) q_c = torch.randn(num_tokens, cfg.q_lora, device=dev, dtype=torch.bfloat16)
kv_c = torch.randn(num_tokens, KV_LORA, device=dev, dtype=torch.bfloat16) kv_c = torch.randn(num_tokens, cfg.kv_lora, device=dev, dtype=torch.bfloat16)
k_pe = torch.randn(num_tokens, ROPE_DIM, device=dev, dtype=torch.bfloat16) k_pe = torch.randn(num_tokens, cfg.rope_dim, device=dev, dtype=torch.bfloat16)
qw = torch.randn(Q_LORA, device=dev, dtype=torch.bfloat16) qw = torch.randn(cfg.q_lora, device=dev, dtype=torch.bfloat16)
kvw = torch.randn(KV_LORA, device=dev, dtype=torch.bfloat16) kvw = torch.randn(cfg.kv_lora, device=dev, dtype=torch.bfloat16)
ik = torch.randn(num_tokens, INDEX_HEAD_DIM, device=dev, dtype=torch.bfloat16) ik = torch.randn(num_tokens, cfg.index_head_dim, device=dev, dtype=torch.bfloat16)
ikw = torch.randn(INDEX_HEAD_DIM, device=dev, dtype=torch.float32) ikw = torch.randn(cfg.index_head_dim, device=dev, dtype=torch.float32)
ikb = torch.randn(INDEX_HEAD_DIM, device=dev, dtype=torch.float32) ikb = torch.randn(cfg.index_head_dim, device=dev, dtype=torch.float32)
cos_sin = make_cos_sin(32, ROPE_DIM, dev) cos_sin = make_cos_sin(32, cfg.rope_dim, dev)
idx_row = INDEX_HEAD_DIM + INDEX_HEAD_DIM // 128 * 4 idx_row = cfg.index_head_dim + cfg.index_head_dim // 128 * 4
packed_block_stride = block_size * idx_row + 64 packed_block_stride = block_size * idx_row + 64
backing = torch.zeros(2 * packed_block_stride, device=dev, dtype=torch.uint8) backing = torch.zeros(2 * packed_block_stride, device=dev, dtype=torch.uint8)
idx_cache = torch.as_strided( idx_cache = torch.as_strided(
@@ -261,7 +278,7 @@ def test_fused_norm_rope_packed_indexer_block_stride():
mla_cache = torch.zeros( mla_cache = torch.zeros(
1, 1,
block_size, block_size,
KV_LORA + ROPE_DIM, cfg.kv_lora + cfg.rope_dim,
device=dev, device=dev,
dtype=torch.bfloat16, dtype=torch.bfloat16,
) )
@@ -296,15 +313,20 @@ def test_fused_norm_rope_packed_indexer_block_stride():
q_ref, s_ref = ue8m0_quant(ik_ref) q_ref, s_ref = ue8m0_quant(ik_ref)
packed = idx_cache[1].reshape(-1) packed = idx_cache[1].reshape(-1)
values = ( values = (
packed[: block_size * INDEX_HEAD_DIM].view(FP8).view(block_size, INDEX_HEAD_DIM) packed[: block_size * cfg.index_head_dim]
.view(FP8)
.view(block_size, cfg.index_head_dim)
) )
scales = packed[block_size * INDEX_HEAD_DIM :].view(torch.float32) scales = packed[block_size * cfg.index_head_dim :].view(torch.float32)
assert_fp8(values, q_ref, "packed indexer-K fp8") assert_fp8(values, q_ref, "packed indexer-K fp8")
torch.testing.assert_close(scales, s_ref, rtol=0, atol=0) torch.testing.assert_close(scales, s_ref, rtol=0, atol=0)
assert (backing[block_size * idx_row : packed_block_stride] == 0).all() assert (backing[block_size * idx_row : packed_block_stride] == 0).all()
def test_fused_norm_rope_materializes_qk_without_local_cache_slots(): @pytest.mark.parametrize("cfg", MODEL_CONFIGS, ids=MODEL_IDS)
def test_fused_norm_rope_materializes_qk_without_local_cache_slots(
cfg: ModelConfig,
):
"""DCP non-owner ranks still need valid query shards for query AllGather.""" """DCP non-owner ranks still need valid query shards for query AllGather."""
torch.manual_seed(7) torch.manual_seed(7)
dev = "cuda" dev = "cuda"
@@ -312,20 +334,20 @@ def test_fused_norm_rope_materializes_qk_without_local_cache_slots():
max_pos = 16 max_pos = 16
pos = torch.arange(num_tokens, device=dev, dtype=torch.int64) pos = torch.arange(num_tokens, device=dev, dtype=torch.int64)
q_c = torch.randn(num_tokens, Q_LORA, device=dev, dtype=torch.bfloat16) q_c = torch.randn(num_tokens, cfg.q_lora, device=dev, dtype=torch.bfloat16)
kv_c = torch.randn(num_tokens, KV_LORA, device=dev, dtype=torch.bfloat16) kv_c = torch.randn(num_tokens, cfg.kv_lora, device=dev, dtype=torch.bfloat16)
k_pe = torch.randn(num_tokens, ROPE_DIM, device=dev, dtype=torch.bfloat16) k_pe = torch.randn(num_tokens, cfg.rope_dim, device=dev, dtype=torch.bfloat16)
qw = torch.randn(Q_LORA, device=dev, dtype=torch.bfloat16) qw = torch.randn(cfg.q_lora, device=dev, dtype=torch.bfloat16)
kvw = torch.randn(KV_LORA, device=dev, dtype=torch.bfloat16) kvw = torch.randn(cfg.kv_lora, device=dev, dtype=torch.bfloat16)
ik = torch.randn(num_tokens, INDEX_HEAD_DIM, device=dev, dtype=torch.bfloat16) ik = torch.randn(num_tokens, cfg.index_head_dim, device=dev, dtype=torch.bfloat16)
ikw = torch.randn(INDEX_HEAD_DIM, device=dev, dtype=torch.float32) ikw = torch.randn(cfg.index_head_dim, device=dev, dtype=torch.float32)
ikb = torch.randn(INDEX_HEAD_DIM, device=dev, dtype=torch.float32) ikb = torch.randn(cfg.index_head_dim, device=dev, dtype=torch.float32)
cos_sin = make_cos_sin(max_pos, ROPE_DIM, dev) cos_sin = make_cos_sin(max_pos, cfg.rope_dim, dev)
mla_cache = torch.zeros( mla_cache = torch.zeros(
1, max_pos, KV_LORA + ROPE_DIM, device=dev, dtype=torch.bfloat16 1, max_pos, cfg.kv_lora + cfg.rope_dim, device=dev, dtype=torch.bfloat16
) )
idx_row = INDEX_HEAD_DIM + INDEX_HEAD_DIM // 128 * 4 idx_row = cfg.index_head_dim + cfg.index_head_dim // 128 * 4
idx_cache = torch.zeros(1, max_pos, idx_row, device=dev, dtype=torch.uint8) idx_cache = torch.zeros(1, max_pos, idx_row, device=dev, dtype=torch.uint8)
no_local_slots = torch.full((num_tokens,), -1, device=dev, dtype=torch.int64) no_local_slots = torch.full((num_tokens,), -1, device=dev, dtype=torch.int64)
topk = torch.full((num_tokens, 2048), 7, device=dev, dtype=torch.int32) topk = torch.full((num_tokens, 2048), 7, device=dev, dtype=torch.int32)
@@ -371,23 +393,26 @@ def test_fused_norm_rope_materializes_qk_without_local_cache_slots():
assert (topk == -1).all(), "topk buffer not cleared on non-owner rank" assert (topk == -1).all(), "topk buffer not cleared on non-owner rank"
@pytest.mark.parametrize("cfg", MODEL_CONFIGS, ids=MODEL_IDS)
@pytest.mark.parametrize("num_tokens", [1, 17, 512]) @pytest.mark.parametrize("num_tokens", [1, 17, 512])
def test_fused_norm_rope_no_indexer(num_tokens: int): def test_fused_norm_rope_no_indexer(num_tokens: int, cfg: ModelConfig):
"""Shared (no-indexer) layer: q + kv/MLA only; top-k buffer untouched.""" """Shared (no-indexer) layer: q + kv/MLA only; top-k buffer untouched."""
torch.manual_seed(1) torch.manual_seed(1)
dev = "cuda" dev = "cuda"
max_pos = 8192 max_pos = 8192
pos = torch.arange(num_tokens, device=dev, dtype=torch.int64) pos = torch.arange(num_tokens, device=dev, dtype=torch.int64)
q_c = torch.randn(num_tokens, Q_LORA, device=dev, dtype=torch.bfloat16) q_c = torch.randn(num_tokens, cfg.q_lora, device=dev, dtype=torch.bfloat16)
kv_c = torch.randn(num_tokens, KV_LORA, device=dev, dtype=torch.bfloat16) kv_c = torch.randn(num_tokens, cfg.kv_lora, device=dev, dtype=torch.bfloat16)
k_pe = torch.randn(num_tokens, ROPE_DIM, device=dev, dtype=torch.bfloat16) k_pe = torch.randn(num_tokens, cfg.rope_dim, device=dev, dtype=torch.bfloat16)
qw = torch.randn(Q_LORA, device=dev, dtype=torch.bfloat16) qw = torch.randn(cfg.q_lora, device=dev, dtype=torch.bfloat16)
kvw = torch.randn(KV_LORA, device=dev, dtype=torch.bfloat16) kvw = torch.randn(cfg.kv_lora, device=dev, dtype=torch.bfloat16)
mla_cos_sin = make_cos_sin(max_pos, ROPE_DIM, dev) mla_cos_sin = make_cos_sin(max_pos, cfg.rope_dim, dev)
bs = max_pos bs = max_pos
mla_cache = torch.zeros(1, bs, KV_LORA + ROPE_DIM, device=dev, dtype=torch.bfloat16) mla_cache = torch.zeros(
1, bs, cfg.kv_lora + cfg.rope_dim, device=dev, dtype=torch.bfloat16
)
slot = torch.arange(num_tokens, device=dev, dtype=torch.int64) slot = torch.arange(num_tokens, device=dev, dtype=torch.int64)
topk = torch.full((num_tokens, 2048), 7, device=dev, dtype=torch.int32) topk = torch.full((num_tokens, 2048), 7, device=dev, dtype=torch.int32)
@@ -418,9 +443,9 @@ def test_fused_norm_rope_no_indexer(num_tokens: int):
assert_bf16(q_out, rms_norm(q_c, qw), "q_c rmsnorm (no-indexer)") assert_bf16(q_out, rms_norm(q_c, qw), "q_c rmsnorm (no-indexer)")
cache = mla_cache[0, :num_tokens] cache = mla_cache[0, :num_tokens]
assert_bf16(cache[:, :KV_LORA], rms_norm(kv_c, kvw), "MLA kv (no-indexer)") assert_bf16(cache[:, : cfg.kv_lora], rms_norm(kv_c, kvw), "MLA kv (no-indexer)")
assert_bf16( assert_bf16(
cache[:, KV_LORA:], cache[:, cfg.kv_lora :],
rope(k_pe.float(), pos, mla_cos_sin, interleave=True), rope(k_pe.float(), pos, mla_cos_sin, interleave=True),
"MLA k_pe (no-indexer)", "MLA k_pe (no-indexer)",
) )
@@ -428,36 +453,39 @@ def test_fused_norm_rope_no_indexer(num_tokens: int):
assert (topk == 7).all(), "topk buffer should be untouched on shared layer" assert (topk == 7).all(), "topk buffer should be untouched on shared layer"
@pytest.mark.parametrize("cfg", MODEL_CONFIGS, ids=MODEL_IDS)
@pytest.mark.parametrize("has_indexer", [False, True]) @pytest.mark.parametrize("has_indexer", [False, True])
def test_fused_norm_rope_materializes_pcp_cache_inputs(has_indexer: bool): def test_fused_norm_rope_materializes_pcp_cache_inputs(
has_indexer: bool, cfg: ModelConfig
):
"""PCP gets local normalized/rotated K rows without direct cache writes.""" """PCP gets local normalized/rotated K rows without direct cache writes."""
torch.manual_seed(6) torch.manual_seed(6)
dev = "cuda" dev = "cuda"
num_tokens = 17 num_tokens = 17
max_pos = 8192 max_pos = 8192
pos = torch.arange(num_tokens, device=dev, dtype=torch.int64) pos = torch.arange(num_tokens, device=dev, dtype=torch.int64)
q_c = torch.randn(num_tokens, Q_LORA, device=dev, dtype=torch.bfloat16) q_c = torch.randn(num_tokens, cfg.q_lora, device=dev, dtype=torch.bfloat16)
kv_c = torch.randn(num_tokens, KV_LORA, device=dev, dtype=torch.bfloat16) kv_c = torch.randn(num_tokens, cfg.kv_lora, device=dev, dtype=torch.bfloat16)
k_pe = torch.randn(num_tokens, ROPE_DIM, device=dev, dtype=torch.bfloat16) k_pe = torch.randn(num_tokens, cfg.rope_dim, device=dev, dtype=torch.bfloat16)
qw = torch.randn(Q_LORA, device=dev, dtype=torch.bfloat16) qw = torch.randn(cfg.q_lora, device=dev, dtype=torch.bfloat16)
kvw = torch.randn(KV_LORA, device=dev, dtype=torch.bfloat16) kvw = torch.randn(cfg.kv_lora, device=dev, dtype=torch.bfloat16)
ik = ( ik = (
torch.randn(num_tokens, INDEX_HEAD_DIM, device=dev, dtype=torch.bfloat16) torch.randn(num_tokens, cfg.index_head_dim, device=dev, dtype=torch.bfloat16)
if has_indexer if has_indexer
else None else None
) )
ikw = ( ikw = (
torch.randn(INDEX_HEAD_DIM, device=dev, dtype=torch.float32) torch.randn(cfg.index_head_dim, device=dev, dtype=torch.float32)
if has_indexer if has_indexer
else None else None
) )
ikb = ( ikb = (
torch.randn(INDEX_HEAD_DIM, device=dev, dtype=torch.float32) torch.randn(cfg.index_head_dim, device=dev, dtype=torch.float32)
if has_indexer if has_indexer
else None else None
) )
mla_cos_sin = make_cos_sin(max_pos, ROPE_DIM, dev) mla_cos_sin = make_cos_sin(max_pos, cfg.rope_dim, dev)
idx_cos_sin = make_cos_sin(max_pos, ROPE_DIM, dev) if has_indexer else None idx_cos_sin = make_cos_sin(max_pos, cfg.rope_dim, dev) if has_indexer else None
q_out = torch.empty_like(q_c) q_out = torch.empty_like(q_c)
kv_out = torch.empty_like(kv_c) kv_out = torch.empty_like(kv_c)
kpe_out = torch.empty_like(k_pe) kpe_out = torch.empty_like(k_pe)
@@ -503,8 +531,9 @@ def test_fused_norm_rope_materializes_pcp_cache_inputs(has_indexer: bool):
assert_bf16(ik_out, ik_ref, "PCP indexer-K") assert_bf16(ik_out, ik_ref, "PCP indexer-K")
@pytest.mark.parametrize("cfg", MODEL_CONFIGS, ids=MODEL_IDS)
@pytest.mark.parametrize("num_tokens", [1, 4, 17, 512]) @pytest.mark.parametrize("num_tokens", [1, 4, 17, 512])
def test_fused_norm_rope_ds_mla(num_tokens: int): def test_fused_norm_rope_ds_mla(num_tokens: int, cfg: ModelConfig):
"""fp8_ds_mla MLA cache layout (FlashMLA sparse, bf16-query path; SM90/SM100). """fp8_ds_mla MLA cache layout (FlashMLA sparse, bf16-query path; SM90/SM100).
Per-token 656-byte entry: 512 fp8 NoPE (4 per-128 tiles, power-of-two scales Per-token 656-byte entry: 512 fp8 NoPE (4 per-128 tiles, power-of-two scales
@@ -515,12 +544,12 @@ def test_fused_norm_rope_ds_mla(num_tokens: int):
max_pos = 8192 max_pos = 8192
pos = torch.arange(num_tokens, device=dev, dtype=torch.int64) % max_pos pos = torch.arange(num_tokens, device=dev, dtype=torch.int64) % max_pos
q_c = torch.randn(num_tokens, Q_LORA, device=dev, dtype=torch.bfloat16) q_c = torch.randn(num_tokens, cfg.q_lora, device=dev, dtype=torch.bfloat16)
kv_c = torch.randn(num_tokens, KV_LORA, device=dev, dtype=torch.bfloat16) kv_c = torch.randn(num_tokens, cfg.kv_lora, device=dev, dtype=torch.bfloat16)
k_pe = torch.randn(num_tokens, ROPE_DIM, device=dev, dtype=torch.bfloat16) k_pe = torch.randn(num_tokens, cfg.rope_dim, device=dev, dtype=torch.bfloat16)
qw = torch.randn(Q_LORA, device=dev, dtype=torch.bfloat16) qw = torch.randn(cfg.q_lora, device=dev, dtype=torch.bfloat16)
kvw = torch.randn(KV_LORA, device=dev, dtype=torch.bfloat16) kvw = torch.randn(cfg.kv_lora, device=dev, dtype=torch.bfloat16)
mla_cos_sin = make_cos_sin(max_pos, ROPE_DIM, dev) mla_cos_sin = make_cos_sin(max_pos, cfg.rope_dim, dev)
bs = max_pos bs = max_pos
mla_cache = torch.zeros(1, bs, 656, device=dev, dtype=torch.uint8) mla_cache = torch.zeros(1, bs, 656, device=dev, dtype=torch.uint8)
@@ -559,13 +588,13 @@ def test_fused_norm_rope_ds_mla(num_tokens: int):
tiles = kv_ref.view(num_tokens, 4, 128) tiles = kv_ref.view(num_tokens, 4, 128)
ref_scale = torch.clamp(tiles.abs().amax(dim=-1) / FP8_MAX, min=1e-4) ref_scale = torch.clamp(tiles.abs().amax(dim=-1) / FP8_MAX, min=1e-4)
ref_scale = torch.exp2(torch.ceil(torch.log2(ref_scale))) ref_scale = torch.exp2(torch.ceil(torch.log2(ref_scale)))
ref_nope = (tiles / ref_scale[..., None]).reshape(num_tokens, KV_LORA).to(FP8) ref_nope = (tiles / ref_scale[..., None]).reshape(num_tokens, cfg.kv_lora).to(FP8)
cache = mla_cache[0, :num_tokens] # [N, 656] uint8 cache = mla_cache[0, :num_tokens] # [N, 656] uint8
nope = cache[:, :KV_LORA].view(FP8) nope = cache[:, : cfg.kv_lora].view(FP8)
scales = cache.view(torch.float32)[:, KV_LORA // 4 : KV_LORA // 4 + 4] scales = cache.view(torch.float32)[:, cfg.kv_lora // 4 : cfg.kv_lora // 4 + 4]
rope_off = KV_LORA // 2 + 8 rope_off = cfg.kv_lora // 2 + 8
rope_vals = cache.view(torch.bfloat16)[:, rope_off : rope_off + ROPE_DIM] rope_vals = cache.view(torch.bfloat16)[:, rope_off : rope_off + cfg.rope_dim]
torch.testing.assert_close(scales, ref_scale, rtol=0, atol=0) torch.testing.assert_close(scales, ref_scale, rtol=0, atol=0)
assert_fp8(nope, ref_nope, "ds_mla NoPE fp8") assert_fp8(nope, ref_nope, "ds_mla NoPE fp8")
@@ -598,8 +627,9 @@ def nvfp4_sf_byte(s: torch.Tensor) -> torch.Tensor:
not current_platform.is_device_capability(100), not current_platform.is_device_capability(100),
reason="nvfp4_ds_mla requires SM100 (Blackwell)", reason="nvfp4_ds_mla requires SM100 (Blackwell)",
) )
@pytest.mark.parametrize("cfg", MODEL_CONFIGS, ids=MODEL_IDS)
@pytest.mark.parametrize("num_tokens", [1, 4, 17, 512]) @pytest.mark.parametrize("num_tokens", [1, 4, 17, 512])
def test_fused_norm_rope_nvfp4_ds_mla(num_tokens: int): def test_fused_norm_rope_nvfp4_ds_mla(num_tokens: int, cfg: ModelConfig):
"""nvfp4_ds_mla MLA cache layout (FlashMLA sparse, SM100 only). """nvfp4_ds_mla MLA cache layout (FlashMLA sparse, SM100 only).
Per-token 352-byte entry: 256 B of 512 e2m1 NoPE packed 2/byte (low nibble Per-token 352-byte entry: 256 B of 512 e2m1 NoPE packed 2/byte (low nibble
@@ -611,8 +641,8 @@ def test_fused_norm_rope_nvfp4_ds_mla(num_tokens: int):
max_pos = 8192 max_pos = 8192
pos = torch.arange(num_tokens, device=dev, dtype=torch.int64) % max_pos pos = torch.arange(num_tokens, device=dev, dtype=torch.int64) % max_pos
q_c = torch.randn(num_tokens, Q_LORA, device=dev, dtype=torch.bfloat16) q_c = torch.randn(num_tokens, cfg.q_lora, device=dev, dtype=torch.bfloat16)
kv_c = torch.randn(num_tokens, KV_LORA, device=dev, dtype=torch.bfloat16) kv_c = torch.randn(num_tokens, cfg.kv_lora, device=dev, dtype=torch.bfloat16)
# Tile 0 is deliberately low-magnitude: amax/6 lands between two e4m3 # Tile 0 is deliberately low-magnitude: amax/6 lands between two e4m3
# subnormals (spaced a flat 2^-9 there), so round-to-nearest picks a scale # subnormals (spaced a flat 2^-9 there), so round-to-nearest picks a scale
# BELOW amax/6 and the tile's peak saturates at +-6. Asserted below, so a # BELOW amax/6 and the tile's peak saturates at +-6. Asserted below, so a
@@ -622,10 +652,10 @@ def test_fused_norm_rope_nvfp4_ds_mla(num_tokens: int):
kv_c[:, :16] = ( kv_c[:, :16] = (
7.5 * (2.0**-9) * torch.linspace(0.2, 1.0, 16, device=dev, dtype=torch.bfloat16) 7.5 * (2.0**-9) * torch.linspace(0.2, 1.0, 16, device=dev, dtype=torch.bfloat16)
) )
k_pe = torch.randn(num_tokens, ROPE_DIM, device=dev, dtype=torch.bfloat16) k_pe = torch.randn(num_tokens, cfg.rope_dim, device=dev, dtype=torch.bfloat16)
qw = torch.randn(Q_LORA, device=dev, dtype=torch.bfloat16) qw = torch.randn(cfg.q_lora, device=dev, dtype=torch.bfloat16)
kvw = torch.ones(KV_LORA, device=dev, dtype=torch.bfloat16) kvw = torch.ones(cfg.kv_lora, device=dev, dtype=torch.bfloat16)
mla_cos_sin = make_cos_sin(max_pos, ROPE_DIM, dev) mla_cos_sin = make_cos_sin(max_pos, cfg.rope_dim, dev)
bs = max_pos bs = max_pos
mla_cache = torch.zeros(1, bs, 352, device=dev, dtype=torch.uint8) mla_cache = torch.zeros(1, bs, 352, device=dev, dtype=torch.uint8)
@@ -675,19 +705,19 @@ def test_fused_norm_rope_nvfp4_ds_mla(num_tokens: int):
) )
ref_codes = quantize_to_e2m1(tiles / ref_scale.float().unsqueeze(-1)) ref_codes = quantize_to_e2m1(tiles / ref_scale.float().unsqueeze(-1))
ref_codes = ref_codes.reshape(num_tokens, KV_LORA) ref_codes = ref_codes.reshape(num_tokens, cfg.kv_lora)
cache = mla_cache[0, :num_tokens] # [N, 352] uint8 cache = mla_cache[0, :num_tokens] # [N, 352] uint8
packed = cache[:, : KV_LORA // 2] packed = cache[:, : cfg.kv_lora // 2]
got_codes = torch.stack([packed & 0xF, packed >> 4], dim=-1) got_codes = torch.stack([packed & 0xF, packed >> 4], dim=-1)
got_codes = got_codes.reshape(num_tokens, KV_LORA) # low nibble = even elem got_codes = got_codes.reshape(num_tokens, cfg.kv_lora) # low nibble = even elem
torch.testing.assert_close(got_codes, ref_codes, rtol=0, atol=0) torch.testing.assert_close(got_codes, ref_codes, rtol=0, atol=0)
got_rope = cache[:, KV_LORA // 2 : KV_LORA // 2 + ROPE_DIM].view(FP8) got_rope = cache[:, cfg.kv_lora // 2 : cfg.kv_lora // 2 + cfg.rope_dim].view(FP8)
assert_fp8(got_rope, kpe_ref.to(FP8), "nvfp4_ds_mla RoPE e4m3") assert_fp8(got_rope, kpe_ref.to(FP8), "nvfp4_ds_mla RoPE e4m3")
perm = nvfp4_sf_byte(torch.arange(32, device=dev)) perm = nvfp4_sf_byte(torch.arange(32, device=dev))
got_scale = cache[:, KV_LORA // 2 + ROPE_DIM :].view(FP8)[:, perm] got_scale = cache[:, cfg.kv_lora // 2 + cfg.rope_dim :].view(FP8)[:, perm]
torch.testing.assert_close(got_scale.float(), ref_scale.float(), rtol=0, atol=0) torch.testing.assert_close(got_scale.float(), ref_scale.float(), rtol=0, atol=0)
# No indexer on this call: top-k buffer must be untouched. # No indexer on this call: top-k buffer must be untouched.
@@ -737,39 +767,34 @@ def test_fused_norm_rope_supports_large_token_count():
# ── fused_q ────────────────────────────────────────────────────────────────── # ── fused_q ──────────────────────────────────────────────────────────────────
@pytest.mark.parametrize( @pytest.mark.parametrize("cfg", MODEL_CONFIGS, ids=MODEL_IDS)
("num_tokens", "num_q_heads"), @pytest.mark.parametrize("num_tokens", [1, 4, 17, 512, 4096])
[
(1, NUM_HEADS),
(4, NUM_HEADS),
(17, NUM_HEADS),
(512, NUM_HEADS),
(4096, NUM_HEADS),
(17, 64),
],
)
@pytest.mark.parametrize("index_interleave", [True, False]) @pytest.mark.parametrize("index_interleave", [True, False])
def test_fused_q(num_tokens: int, num_q_heads: int, index_interleave: bool): def test_fused_q(num_tokens: int, index_interleave: bool, cfg: ModelConfig):
torch.manual_seed(2) torch.manual_seed(2)
dev = "cuda" dev = "cuda"
max_pos = 8192 max_pos = 8192
pos = torch.arange(num_tokens, device=dev, dtype=torch.int64) % max_pos pos = torch.arange(num_tokens, device=dev, dtype=torch.int64) % max_pos
q_pe = torch.randn( q_pe = torch.randn(
num_tokens, num_q_heads, ROPE_DIM, device=dev, dtype=torch.bfloat16 num_tokens, cfg.num_heads, cfg.rope_dim, device=dev, dtype=torch.bfloat16
) )
ql_nope = torch.randn( ql_nope = torch.randn(
num_tokens, num_q_heads, KV_LORA, device=dev, dtype=torch.bfloat16 num_tokens, cfg.num_heads, cfg.kv_lora, device=dev, dtype=torch.bfloat16
) )
index_q = torch.randn( index_q = torch.randn(
num_tokens, INDEX_HEADS, INDEX_HEAD_DIM, device=dev, dtype=torch.bfloat16 num_tokens,
cfg.index_heads,
cfg.index_head_dim,
device=dev,
dtype=torch.bfloat16,
) )
index_w = torch.randn(num_tokens, INDEX_HEADS, device=dev, dtype=torch.float32) index_w = torch.randn(num_tokens, cfg.index_heads, device=dev, dtype=torch.float32)
q_scale = torch.tensor([0.37], device=dev, dtype=torch.float32) q_scale = torch.tensor([0.37], device=dev, dtype=torch.float32)
softmax_scale = INDEX_HEAD_DIM**-0.5 softmax_scale = cfg.index_head_dim**-0.5
head_scale = INDEX_HEADS**-0.5 head_scale = cfg.index_heads**-0.5
q_cos_sin = make_cos_sin(max_pos, ROPE_DIM, dev) # q_pe: interleaved q_cos_sin = make_cos_sin(max_pos, cfg.rope_dim, dev) # q_pe: interleaved
idx_cos_sin = make_cos_sin(max_pos, ROPE_DIM, dev) idx_cos_sin = make_cos_sin(max_pos, cfg.rope_dim, dev)
iq_fp8, iw_out, mqa = K.fused_q( iq_fp8, iw_out, mqa = K.fused_q(
pos, pos,
@@ -791,18 +816,18 @@ def test_fused_q(num_tokens: int, num_q_heads: int, index_interleave: bool):
mqa_nope_ref = (ql_nope.float() / s).to(FP8) mqa_nope_ref = (ql_nope.float() / s).to(FP8)
qpe_ref = rope( qpe_ref = rope(
q_pe.float(), q_pe.float(),
pos.unsqueeze(-1).expand(num_tokens, num_q_heads), pos.unsqueeze(-1).expand(num_tokens, cfg.num_heads),
q_cos_sin, q_cos_sin,
interleave=True, interleave=True,
) )
mqa_pe_ref = (qpe_ref / s).to(FP8) mqa_pe_ref = (qpe_ref / s).to(FP8)
assert_fp8(mqa[:, :, :KV_LORA], mqa_nope_ref, "mqa ql_nope") assert_fp8(mqa[:, :, : cfg.kv_lora], mqa_nope_ref, "mqa ql_nope")
assert_fp8(mqa[:, :, KV_LORA:], mqa_pe_ref, "mqa q_pe") assert_fp8(mqa[:, :, cfg.kv_lora :], mqa_pe_ref, "mqa q_pe")
# Indexer-Q: RoPE + UE8M0 fp8 quant; index weights fold in q-scale. # Indexer-Q: RoPE + UE8M0 fp8 quant; index weights fold in q-scale.
iq_ref = rope( iq_ref = rope(
index_q.float(), index_q.float(),
pos.unsqueeze(-1).expand(num_tokens, INDEX_HEADS), pos.unsqueeze(-1).expand(num_tokens, cfg.index_heads),
idx_cos_sin, idx_cos_sin,
interleave=index_interleave, interleave=index_interleave,
) )
@@ -812,20 +837,21 @@ def test_fused_q(num_tokens: int, num_q_heads: int, index_interleave: bool):
torch.testing.assert_close(iw_out, iw_ref, rtol=1e-3, atol=1e-3) torch.testing.assert_close(iw_out, iw_ref, rtol=1e-3, atol=1e-3)
@pytest.mark.parametrize("cfg", MODEL_CONFIGS, ids=MODEL_IDS)
@pytest.mark.parametrize("num_tokens", [1, 17, 512]) @pytest.mark.parametrize("num_tokens", [1, 17, 512])
def test_fused_q_no_indexer(num_tokens: int): def test_fused_q_no_indexer(num_tokens: int, cfg: ModelConfig):
torch.manual_seed(3) torch.manual_seed(3)
dev = "cuda" dev = "cuda"
max_pos = 8192 max_pos = 8192
pos = torch.arange(num_tokens, device=dev, dtype=torch.int64) pos = torch.arange(num_tokens, device=dev, dtype=torch.int64)
q_pe = torch.randn( q_pe = torch.randn(
num_tokens, NUM_HEADS, ROPE_DIM, device=dev, dtype=torch.bfloat16 num_tokens, cfg.num_heads, cfg.rope_dim, device=dev, dtype=torch.bfloat16
) )
ql_nope = torch.randn( ql_nope = torch.randn(
num_tokens, NUM_HEADS, KV_LORA, device=dev, dtype=torch.bfloat16 num_tokens, cfg.num_heads, cfg.kv_lora, device=dev, dtype=torch.bfloat16
) )
q_scale = torch.tensor([0.5], device=dev, dtype=torch.float32) q_scale = torch.tensor([0.5], device=dev, dtype=torch.float32)
q_cos_sin = make_cos_sin(max_pos, ROPE_DIM, dev) q_cos_sin = make_cos_sin(max_pos, cfg.rope_dim, dev)
_, _, mqa = K.fused_q( _, _, mqa = K.fused_q(
pos, pos,
q_pe, q_pe,
@@ -841,19 +867,20 @@ def test_fused_q_no_indexer(num_tokens: int):
index_rope_interleave=False, index_rope_interleave=False,
) )
s = q_scale.item() s = q_scale.item()
assert_fp8(mqa[:, :, :KV_LORA], (ql_nope.float() / s).to(FP8), "mqa ql_nope") assert_fp8(mqa[:, :, : cfg.kv_lora], (ql_nope.float() / s).to(FP8), "mqa ql_nope")
qpe_ref = rope( qpe_ref = rope(
q_pe.float(), q_pe.float(),
pos.unsqueeze(-1).expand(num_tokens, NUM_HEADS), pos.unsqueeze(-1).expand(num_tokens, cfg.num_heads),
q_cos_sin, q_cos_sin,
interleave=True, interleave=True,
) )
assert_fp8(mqa[:, :, KV_LORA:], (qpe_ref / s).to(FP8), "mqa q_pe") assert_fp8(mqa[:, :, cfg.kv_lora :], (qpe_ref / s).to(FP8), "mqa q_pe")
@pytest.mark.parametrize("cfg", MODEL_CONFIGS, ids=MODEL_IDS)
@pytest.mark.parametrize("num_tokens", [1, 17, 512]) @pytest.mark.parametrize("num_tokens", [1, 17, 512])
@pytest.mark.parametrize("has_indexer", [True, False]) @pytest.mark.parametrize("has_indexer", [True, False])
def test_fused_q_bf16_query(num_tokens: int, has_indexer: bool): def test_fused_q_bf16_query(num_tokens: int, has_indexer: bool, cfg: ModelConfig):
"""bf16-query path (FlashMLA sparse, SM90/SM100): only the RoPE'd q_pe is """bf16-query path (FlashMLA sparse, SM90/SM100): only the RoPE'd q_pe is
produced (bf16, unquantized); ql_nope is consumed directly by the caller.""" produced (bf16, unquantized); ql_nope is consumed directly by the caller."""
torch.manual_seed(6) torch.manual_seed(6)
@@ -862,21 +889,27 @@ def test_fused_q_bf16_query(num_tokens: int, has_indexer: bool):
pos = torch.arange(num_tokens, device=dev, dtype=torch.int64) % max_pos pos = torch.arange(num_tokens, device=dev, dtype=torch.int64) % max_pos
q_pe = torch.randn( q_pe = torch.randn(
num_tokens, NUM_HEADS, ROPE_DIM, device=dev, dtype=torch.bfloat16 num_tokens, cfg.num_heads, cfg.rope_dim, device=dev, dtype=torch.bfloat16
) )
ql_nope = torch.randn( ql_nope = torch.randn(
num_tokens, NUM_HEADS, KV_LORA, device=dev, dtype=torch.bfloat16 num_tokens, cfg.num_heads, cfg.kv_lora, device=dev, dtype=torch.bfloat16
) )
q_scale = torch.tensor([0.37], device=dev, dtype=torch.float32) q_scale = torch.tensor([0.37], device=dev, dtype=torch.float32)
q_cos_sin = make_cos_sin(max_pos, ROPE_DIM, dev) q_cos_sin = make_cos_sin(max_pos, cfg.rope_dim, dev)
index_q = index_w = idx_cos_sin = None index_q = index_w = idx_cos_sin = None
if has_indexer: if has_indexer:
index_q = torch.randn( index_q = torch.randn(
num_tokens, INDEX_HEADS, INDEX_HEAD_DIM, device=dev, dtype=torch.bfloat16 num_tokens,
cfg.index_heads,
cfg.index_head_dim,
device=dev,
dtype=torch.bfloat16,
) )
index_w = torch.randn(num_tokens, INDEX_HEADS, device=dev, dtype=torch.float32) index_w = torch.randn(
idx_cos_sin = make_cos_sin(max_pos, ROPE_DIM, dev) num_tokens, cfg.index_heads, device=dev, dtype=torch.float32
)
idx_cos_sin = make_cos_sin(max_pos, cfg.rope_dim, dev)
iq_fp8, iw_out, q_pe_out = K.fused_q( iq_fp8, iw_out, q_pe_out = K.fused_q(
pos, pos,
@@ -887,8 +920,8 @@ def test_fused_q_bf16_query(num_tokens: int, has_indexer: bool):
ql_nope, ql_nope,
q_scale, q_scale,
index_w, index_w,
INDEX_HEAD_DIM**-0.5, cfg.index_head_dim**-0.5,
INDEX_HEADS**-0.5, cfg.index_heads**-0.5,
has_indexer=has_indexer, has_indexer=has_indexer,
index_rope_interleave=False, index_rope_interleave=False,
quantize_mqa=False, quantize_mqa=False,
@@ -896,10 +929,10 @@ def test_fused_q_bf16_query(num_tokens: int, has_indexer: bool):
# MQA query: only the RoPE'd q_pe, bf16, unquantized. # MQA query: only the RoPE'd q_pe, bf16, unquantized.
assert q_pe_out.dtype == torch.bfloat16 assert q_pe_out.dtype == torch.bfloat16
assert q_pe_out.shape == (num_tokens, NUM_HEADS, ROPE_DIM) assert q_pe_out.shape == (num_tokens, cfg.num_heads, cfg.rope_dim)
qpe_ref = rope( qpe_ref = rope(
q_pe.float(), q_pe.float(),
pos.unsqueeze(-1).expand(num_tokens, NUM_HEADS), pos.unsqueeze(-1).expand(num_tokens, cfg.num_heads),
q_cos_sin, q_cos_sin,
interleave=True, interleave=True,
) )
@@ -910,13 +943,15 @@ def test_fused_q_bf16_query(num_tokens: int, has_indexer: bool):
assert index_q is not None assert index_q is not None
iq_ref = rope( iq_ref = rope(
index_q.float(), index_q.float(),
pos.unsqueeze(-1).expand(num_tokens, INDEX_HEADS), pos.unsqueeze(-1).expand(num_tokens, cfg.index_heads),
idx_cos_sin, idx_cos_sin,
interleave=False, interleave=False,
) )
q_ref, scale_ref = ue8m0_quant(iq_ref) q_ref, scale_ref = ue8m0_quant(iq_ref)
assert_fp8(iq_fp8, q_ref, "indexer-Q fp8 (bf16-query path)") assert_fp8(iq_fp8, q_ref, "indexer-Q fp8 (bf16-query path)")
iw_ref = index_w * scale_ref * (INDEX_HEAD_DIM**-0.5) * (INDEX_HEADS**-0.5) iw_ref = (
index_w * scale_ref * (cfg.index_head_dim**-0.5) * (cfg.index_heads**-0.5)
)
torch.testing.assert_close(iw_out, iw_ref, rtol=1e-3, atol=1e-3) torch.testing.assert_close(iw_out, iw_ref, rtol=1e-3, atol=1e-3)
@@ -956,21 +991,22 @@ def test_fused_q_triton_supports_large_token_count():
# ── fused_eh_norm (MTP) ────────────────────────────────────────────────────── # ── fused_eh_norm (MTP) ──────────────────────────────────────────────────────
@pytest.mark.parametrize("cfg", MODEL_CONFIGS, ids=MODEL_IDS)
@pytest.mark.parametrize("num_tokens", [1, 4, 17, 512]) @pytest.mark.parametrize("num_tokens", [1, 4, 17, 512])
def test_fused_eh_norm(num_tokens: int): def test_fused_eh_norm(num_tokens: int, cfg: ModelConfig):
torch.manual_seed(4) torch.manual_seed(4)
dev = "cuda" dev = "cuda"
# Mix in a position-0 token to exercise the embeds-zeroing branch. # Mix in a position-0 token to exercise the embeds-zeroing branch.
pos = torch.arange(num_tokens, device=dev, dtype=torch.int64) pos = torch.arange(num_tokens, device=dev, dtype=torch.int64)
pos[0] = 0 pos[0] = 0
embeds = torch.randn(num_tokens, HIDDEN, device=dev, dtype=torch.bfloat16) embeds = torch.randn(num_tokens, cfg.hidden, device=dev, dtype=torch.bfloat16)
prev = torch.randn(num_tokens, HIDDEN, device=dev, dtype=torch.bfloat16) prev = torch.randn(num_tokens, cfg.hidden, device=dev, dtype=torch.bfloat16)
ew = torch.randn(HIDDEN, device=dev, dtype=torch.bfloat16) ew = torch.randn(cfg.hidden, device=dev, dtype=torch.bfloat16)
hw = torch.randn(HIDDEN, device=dev, dtype=torch.bfloat16) hw = torch.randn(cfg.hidden, device=dev, dtype=torch.bfloat16)
out = K.fused_eh_norm(pos, embeds, prev, ew, hw, EPS) out = K.fused_eh_norm(pos, embeds, prev, ew, hw, EPS)
masked = torch.where(pos.unsqueeze(-1) == 0, torch.zeros_like(embeds), embeds) masked = torch.where(pos.unsqueeze(-1) == 0, torch.zeros_like(embeds), embeds)
ref = torch.cat([rms_norm(masked, ew), rms_norm(prev, hw)], dim=-1) ref = torch.cat([rms_norm(masked, ew), rms_norm(prev, hw)], dim=-1)
assert out.shape == (num_tokens, 2 * HIDDEN) assert out.shape == (num_tokens, 2 * cfg.hidden)
assert_bf16(out, ref, "eh_norm") assert_bf16(out, ref, "eh_norm")
+1 -1
View File
@@ -3406,7 +3406,7 @@ class rocm_aiter_ops:
X: torch.Tensor, X: torch.Tensor,
W: torch.Tensor, W: torch.Tensor,
w_scale: torch.Tensor, w_scale: torch.Tensor,
Y: torch.Tensor, Y: torch.Tensor | None = None,
transpose_bm: bool | None = False, transpose_bm: bool | None = False,
prequant: bool | None = False, prequant: bool | None = False,
y_scale: torch.Tensor | None = None, y_scale: torch.Tensor | None = None,
@@ -1025,9 +1025,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
mqa_q_pe = mqa_pe_padded mqa_q_pe = mqa_pe_padded
if self.is_aiter_triton_fp4_bmm_enabled: if self.is_aiter_triton_fp4_bmm_enabled:
from aiter.ops.triton.batched_gemm_a16wfp4 import batched_gemm_a16wfp4 mqa_ql_nope = rocm_aiter_ops.batched_gemm_a16wfp4(
mqa_ql_nope = batched_gemm_a16wfp4(
mqa_q_nope, mqa_q_nope,
self.W_K, self.W_K,
self.W_K_scale, self.W_K_scale,
@@ -665,6 +665,11 @@ class DeepseekV32IndexerCache(torch.nn.Module, AttentionLayerBase):
def forward(self): ... def forward(self): ...
@property
def uses_shuffled_layout(self) -> bool:
"""Whether this cache's reader expects the shuffled value layout."""
return False
def get_attn_backend(self) -> type[AttentionBackend]: def get_attn_backend(self) -> type[AttentionBackend]:
return DeepseekV32IndexerBackend return DeepseekV32IndexerBackend
+8 -4
View File
@@ -7,9 +7,9 @@ MLA + a "lightning indexer" that selects the top-k tokens for a sparse MLA
attend. The same model code serves any DSA checkpoint, including GLM-5.2 attend. The same model code serves any DSA checkpoint, including GLM-5.2
(``glm_moe_dsa``), which reuses this architecture. (``glm_moe_dsa``), which reuses this architecture.
The CUDA implementation selects capability-specific kernels internally and The CUDA and ROCm implementations select capability-specific kernels
falls back when an optimization is unavailable. Other platforms use the internally and fall back when an optimization is unavailable. Other platforms
generic implementation by default. use the generic implementation by default.
""" """
from vllm.platforms import current_platform from vllm.platforms import current_platform
@@ -20,8 +20,12 @@ if current_platform.is_cuda():
from .nvidia.model import DeepseekV32ForCausalLM from .nvidia.model import DeepseekV32ForCausalLM
from .nvidia.model import DeepseekV32ForCausalLM as GlmMoeDsaForCausalLM from .nvidia.model import DeepseekV32ForCausalLM as GlmMoeDsaForCausalLM
from .nvidia.mtp import DeepseekV32MTP from .nvidia.mtp import DeepseekV32MTP
elif current_platform.is_rocm():
from .amd.model import DeepseekV32ForCausalLM
from .amd.model import DeepseekV32ForCausalLM as GlmMoeDsaForCausalLM
from .amd.mtp import DeepseekV32MTP
else: else:
# ROCm, XPU, and CPU keep the generic implementation. # XPU and CPU keep the generic implementation.
from vllm.model_executor.models.deepseek_mtp import DeepSeekMTP as DeepseekV32MTP from vllm.model_executor.models.deepseek_mtp import DeepSeekMTP as DeepseekV32MTP
from vllm.model_executor.models.deepseek_v2 import ( from vllm.model_executor.models.deepseek_v2 import (
DeepseekV3ForCausalLM as DeepseekV32ForCausalLM, DeepseekV3ForCausalLM as DeepseekV32ForCausalLM,
+16 -25
View File
@@ -3,12 +3,14 @@
import torch import torch
from vllm._aiter_ops import rocm_aiter_ops
from vllm.compilation.breakable_cudagraph import eager_break_during_capture from vllm.compilation.breakable_cudagraph import eager_break_during_capture
from vllm.forward_context import get_forward_context from vllm.forward_context import get_forward_context
from vllm.model_executor.layers.sparse_attn_indexer import SparseAttnIndexer from vllm.model_executor.layers.sparse_attn_indexer import SparseAttnIndexer
from vllm.model_executor.models.deepseek_v2 import DeepseekV32IndexerCache from vllm.model_executor.models.deepseek_v2 import DeepseekV32IndexerCache
from vllm.models.deepseek_v32.attention import DeepseekV32Attention, DeepseekV32Indexer from vllm.models.deepseek_v32.attention import DeepseekV32Attention, DeepseekV32Indexer
from vllm.models.deepseek_v32.common.kernels import fused_norm_rope, fused_q from vllm.models.deepseek_v32.common.kernels import fused_norm_rope, fused_q
from vllm.platforms import current_platform
from vllm.utils.torch_utils import is_quantized_kv_cache from vllm.utils.torch_utils import is_quantized_kv_cache
from vllm.v1.attention.backends.mla.indexer import DeepseekV32IndexerBackend from vllm.v1.attention.backends.mla.indexer import DeepseekV32IndexerBackend
from vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse import ( from vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse import (
@@ -16,21 +18,15 @@ from vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse import (
) )
class DeepseekV32MLASparseBackend(ROCMAiterMLASparseBackend):
@staticmethod
def get_supported_kernel_block_sizes() -> list:
return [16, 32]
class DeepseekV32ROCmIndexerBackend(DeepseekV32IndexerBackend):
@staticmethod
def get_supported_kernel_block_sizes() -> list:
return [16, 32]
class DeepseekV32ROCmIndexerCache(DeepseekV32IndexerCache): class DeepseekV32ROCmIndexerCache(DeepseekV32IndexerCache):
def get_attn_backend(self): def get_attn_backend(self):
return DeepseekV32ROCmIndexerBackend return DeepseekV32IndexerBackend
@property
def uses_shuffled_layout(self) -> bool:
# aiter's gather/insert pair shuffles the cache above block size 1:
# [n_blocks, blk/16, head_dim/16, 16, 16] instead of [n_blocks, blk, head_dim].
return self.kv_cache.ndim == 3 and self.kv_cache.shape[1] != 1
class DeepseekV32ROCmIndexer(DeepseekV32Indexer): class DeepseekV32ROCmIndexer(DeepseekV32Indexer):
@@ -46,7 +42,7 @@ class DeepseekV32MLAAttention(DeepseekV32Attention):
config, config,
prefix, prefix,
topk_indices_buffer, topk_indices_buffer,
attn_backend=DeepseekV32MLASparseBackend, attn_backend=ROCMAiterMLASparseBackend,
) )
self.indexer_op: SparseAttnIndexer | None = None self.indexer_op: SparseAttnIndexer | None = None
@@ -100,14 +96,10 @@ class DeepseekV32MLAAttention(DeepseekV32Attention):
q_nope = q_nope.transpose(0, 1) # (N, tokens, P) q_nope = q_nope.transpose(0, 1) # (N, tokens, P)
if self.is_aiter_triton_fp4_bmm_enabled: if self.is_aiter_triton_fp4_bmm_enabled:
from aiter.ops.triton.batched_gemm_a16wfp4 import batched_gemm_a16wfp4 ql_nope = rocm_aiter_ops.batched_gemm_a16wfp4(
ql_nope = batched_gemm_a16wfp4(
q_nope, self.W_K, self.W_K_scale, transpose_bm=True, prequant=True q_nope, self.W_K, self.W_K_scale, transpose_bm=True, prequant=True
) )
elif self.is_aiter_triton_fp8_bmm_enabled: elif self.is_aiter_triton_fp8_bmm_enabled:
from vllm._aiter_ops import rocm_aiter_ops
ql_nope = rocm_aiter_ops.triton_fp8_bmm( ql_nope = rocm_aiter_ops.triton_fp8_bmm(
q_nope, self.W_K, self.W_K_scale, group_size=128, transpose_bm=True q_nope, self.W_K, self.W_K_scale, group_size=128, transpose_bm=True
) )
@@ -152,14 +144,10 @@ class DeepseekV32MLAAttention(DeepseekV32Attention):
) )
if self.is_aiter_triton_fp4_bmm_enabled: if self.is_aiter_triton_fp4_bmm_enabled:
from aiter.ops.triton.batched_gemm_a16wfp4 import batched_gemm_a16wfp4 rocm_aiter_ops.batched_gemm_a16wfp4(
batched_gemm_a16wfp4(
x, self.W_V, self.W_V_scale, out_view, transpose_bm=True, prequant=True x, self.W_V, self.W_V_scale, out_view, transpose_bm=True, prequant=True
) )
elif self.is_aiter_triton_fp8_bmm_enabled: elif self.is_aiter_triton_fp8_bmm_enabled:
from vllm._aiter_ops import rocm_aiter_ops
rocm_aiter_ops.triton_fp8_bmm( rocm_aiter_ops.triton_fp8_bmm(
x, x,
self.W_V, self.W_V,
@@ -202,6 +190,7 @@ class DeepseekV32MLAAttention(DeepseekV32Attention):
indexer_k_norm_eps = self.indexer.k_norm.eps indexer_k_norm_eps = self.indexer.k_norm.eps
indexer_k_rope_cos_sin_cache = self.indexer_rope_emb.cos_sin_cache indexer_k_rope_cos_sin_cache = self.indexer_rope_emb.cos_sin_cache
indexer_k_cache = self.indexer.k_cache.kv_cache indexer_k_cache = self.indexer.k_cache.kv_cache
indexer_cache_shuffled = self.indexer.k_cache.uses_shuffled_layout
indexer_softmax_scale = self.indexer.softmax_scale indexer_softmax_scale = self.indexer.softmax_scale
indexer_n_head_scale = self.indexer.n_head**-0.5 indexer_n_head_scale = self.indexer.n_head**-0.5
else: else:
@@ -211,6 +200,7 @@ class DeepseekV32MLAAttention(DeepseekV32Attention):
indexer_k_norm_eps = 1e-6 indexer_k_norm_eps = 1e-6
indexer_k_rope_cos_sin_cache = None indexer_k_rope_cos_sin_cache = None
indexer_k_cache = None indexer_k_cache = None
indexer_cache_shuffled = False
indexer_softmax_scale = 0.0 indexer_softmax_scale = 0.0
indexer_n_head_scale = 0.0 indexer_n_head_scale = 0.0
@@ -241,6 +231,7 @@ class DeepseekV32MLAAttention(DeepseekV32Attention):
self.topk_indices_buffer, self.topk_indices_buffer,
slot_mapping=mla_slot, slot_mapping=mla_slot,
indexer_k_cache=indexer_k_cache, indexer_k_cache=indexer_k_cache,
indexer_cache_shuffled=indexer_cache_shuffled,
mla_kv_cache=mla_kv_cache, mla_kv_cache=mla_kv_cache,
mla_kv_cache_dtype=self.kv_cache_dtype, mla_kv_cache_dtype=self.kv_cache_dtype,
mla_k_scale=mla_k_scale, mla_k_scale=mla_k_scale,
@@ -282,7 +273,7 @@ class DeepseekV32MLAAttention(DeepseekV32Attention):
num_actual = attn_metadata.num_actual_tokens # type: ignore[attr-defined] num_actual = attn_metadata.num_actual_tokens # type: ignore[attr-defined]
kv_cache = self.kv_cache kv_cache = self.kv_cache
if self._fp8_kv_needs_view: if self._fp8_kv_needs_view:
kv_cache = kv_cache.view(torch.float8_e4m3fn) kv_cache = kv_cache.view(current_platform.fp8_dtype())
q_for_attn = self._build_q_for_attn(ql_nope, mqa_q, num_actual) q_for_attn = self._build_q_for_attn(ql_nope, mqa_q, num_actual)
attn_out, _ = self.impl.forward_mqa( # type: ignore[attr-defined] attn_out, _ = self.impl.forward_mqa( # type: ignore[attr-defined]
+4 -1
View File
@@ -327,6 +327,7 @@ class DeepseekV32Attention(MLAAttention):
indexer_k_norm_bias = self.indexer.k_norm.bias indexer_k_norm_bias = self.indexer.k_norm.bias
indexer_k_norm_eps = self.indexer.k_norm.eps indexer_k_norm_eps = self.indexer.k_norm.eps
indexer_k_rope_cos_sin_cache = self.indexer_rope_emb.cos_sin_cache indexer_k_rope_cos_sin_cache = self.indexer_rope_emb.cos_sin_cache
indexer_cache_shuffled = self.indexer.k_cache.uses_shuffled_layout
indexer_k_cache = None if self.use_pcp else self.indexer.k_cache.kv_cache indexer_k_cache = None if self.use_pcp else self.indexer.k_cache.kv_cache
index_k_out = torch.empty_like(index_k) if self.use_pcp else None index_k_out = torch.empty_like(index_k) if self.use_pcp else None
indexer_softmax_scale = self.indexer.softmax_scale indexer_softmax_scale = self.indexer.softmax_scale
@@ -338,6 +339,7 @@ class DeepseekV32Attention(MLAAttention):
indexer_k_norm_eps = 1e-6 indexer_k_norm_eps = 1e-6
indexer_k_rope_cos_sin_cache = None indexer_k_rope_cos_sin_cache = None
indexer_k_cache = None indexer_k_cache = None
indexer_cache_shuffled = False
index_k_out = None index_k_out = None
indexer_softmax_scale = 0.0 indexer_softmax_scale = 0.0
indexer_n_head_scale = 0.0 indexer_n_head_scale = 0.0
@@ -377,6 +379,7 @@ class DeepseekV32Attention(MLAAttention):
slot_mapping=mla_slot, slot_mapping=mla_slot,
indexer_slot_mapping=indexer_slot, indexer_slot_mapping=indexer_slot,
indexer_k_cache=indexer_k_cache, indexer_k_cache=indexer_k_cache,
indexer_cache_shuffled=indexer_cache_shuffled,
mla_kv_cache=mla_kv_cache, mla_kv_cache=mla_kv_cache,
mla_kv_cache_dtype=self.kv_cache_dtype, mla_kv_cache_dtype=self.kv_cache_dtype,
mla_k_scale=mla_k_scale, mla_k_scale=mla_k_scale,
@@ -544,7 +547,7 @@ class DeepseekV32Attention(MLAAttention):
return return
if self._fp8_kv_needs_view: if self._fp8_kv_needs_view:
kv_cache = kv_cache.view(torch.float8_e4m3fn) kv_cache = kv_cache.view(current_platform.fp8_dtype())
if self._fp8_query: if self._fp8_query:
# FlashInfer sparse: single packed fp8 query. # FlashInfer sparse: single packed fp8 query.
mqa_q_arg: torch.Tensor | tuple[torch.Tensor, torch.Tensor] = mqa_q[ mqa_q_arg: torch.Tensor | tuple[torch.Tensor, torch.Tensor] = mqa_q[
+97 -27
View File
@@ -12,6 +12,18 @@ from vllm.utils.torch_utils import is_quantized_kv_cache
# has_indexer=False path so the indexer args don't allocate every call. # has_indexer=False path so the indexer args don't allocate every call.
_DUMMY_CACHE: dict[tuple, torch.Tensor] = {} _DUMMY_CACHE: dict[tuple, torch.Tensor] = {}
# Tile shape of the indexer-K cache's shuffled layout, used when the cache reports
# uses_shuffled_layout. The block tile is a token count; the head tile is a byte
# count, which the kernel converts to cache elements.
_INDEXER_CACHE_BLOCK_TILE = 16
_INDEXER_CACHE_HEAD_TILE_BYTES = 16
# FNUZ (e4m3fnuz) on gfx942, OCP (e4m3fn) elsewhere. 224.0 keeps the ue8m0
# power-of-two scale inside the FNUZ range after its ceil.
_FP8_DTYPE = current_platform.fp8_dtype()
_USE_FNUZ = torch.float8_e4m3fnuz == _FP8_DTYPE
_FP8_MAX = 224.0 if _USE_FNUZ else 448.0
def _dummy(shape: tuple, dtype: torch.dtype, device: torch.device) -> torch.Tensor: def _dummy(shape: tuple, dtype: torch.dtype, device: torch.device) -> torch.Tensor:
key = (shape, dtype, device) key = (shape, dtype, device)
@@ -47,16 +59,17 @@ def _get_cos_sin(
@triton.jit @triton.jit
def _fp8_ue8m0_quantize(vals): def _fp8_ue8m0_quantize(vals, FP8_MAX: tl.constexpr, USE_FNUZ: tl.constexpr):
"""Quantize float32 values to FP8 E4M3 with a ue8m0 (power-of-2) scale. """Quantize float32 values to FP8 E4M3 with a ue8m0 (power-of-2) scale.
Returns (fp8_vals, scale) so the caller can store them or reuse the scale. Returns (fp8_vals, scale) so the caller can store them or reuse the scale.
""" """
vals = vals.to(tl.float32) vals = vals.to(tl.float32)
amax = tl.max(tl.abs(vals)) amax = tl.max(tl.abs(vals))
scale = tl.div_rn(tl.maximum(amax, 1e-4), 448.0) scale = tl.div_rn(tl.maximum(amax, 1e-4), FP8_MAX)
scale = tl.math.exp2(tl.math.ceil(tl.math.log2(scale))) scale = tl.math.exp2(tl.math.ceil(tl.math.log2(scale)))
fp8_vals = tl.div_rn(vals, scale).to(tl.float8e4nv) fp8_dtype = tl.float8e4b8 if USE_FNUZ else tl.float8e4nv
fp8_vals = tl.div_rn(vals, scale).to(fp8_dtype)
return fp8_vals, scale return fp8_vals, scale
@@ -71,18 +84,32 @@ def _fp8_quant_and_cache_write(
cache_block_stride, cache_block_stride,
offsets, offsets,
HEAD_DIM: tl.constexpr, HEAD_DIM: tl.constexpr,
SHUFFLE: tl.constexpr,
BLOCK_TILE: tl.constexpr,
HEAD_TILE: tl.constexpr,
FP8_MAX: tl.constexpr,
USE_FNUZ: tl.constexpr,
): ):
k_fp8, scale = _fp8_ue8m0_quantize(vals) k_fp8, scale = _fp8_ue8m0_quantize(vals, FP8_MAX, USE_FNUZ)
block_idx = slot_idx // cache_block_size block_idx = slot_idx // cache_block_size
block_offset = slot_idx % cache_block_size block_offset = slot_idx % cache_block_size
block_start = block_idx * cache_block_stride block_start = block_idx * cache_block_stride
tl.store( # Shuffled layout: [blk/BLOCK_TILE, HEAD_DIM/HEAD_TILE, BLOCK_TILE, HEAD_TILE],
kv_cache_ptr + block_start + block_offset * HEAD_DIM + offsets, # so one contiguous run holds BLOCK_TILE tokens x HEAD_TILE bytes and the
k_fp8, # reader's coalesced loads land on the bytes we wrote.
mask=mask, if SHUFFLE:
) value_off = (
block_offset // BLOCK_TILE * BLOCK_TILE * HEAD_DIM
+ block_offset % BLOCK_TILE * HEAD_TILE
+ offsets // HEAD_TILE * BLOCK_TILE * HEAD_TILE
+ offsets % HEAD_TILE
)
else:
value_off = block_offset * HEAD_DIM + offsets
tl.store(kv_cache_ptr + block_start + value_off, k_fp8, mask=mask)
scale_byte_off = block_start + cache_block_size * HEAD_DIM + block_offset * 4 scale_byte_off = block_start + cache_block_size * HEAD_DIM + block_offset * 4
tl.store(kv_cache_scale_ptr + scale_byte_off // 4, scale) tl.store(kv_cache_scale_ptr + scale_byte_off // 4, scale)
@@ -137,6 +164,9 @@ def _fused_norm_rope_kernel(
indexer_cache_scale_ptr, indexer_cache_scale_ptr,
indexer_cache_block_size, indexer_cache_block_size,
indexer_cache_stride, indexer_cache_stride,
INDEXER_CACHE_SHUFFLE: tl.constexpr,
INDEXER_CACHE_BLOCK_TILE: tl.constexpr,
INDEXER_CACHE_HEAD_TILE: tl.constexpr,
# MLA KV cache (concat kv_c_normed + k_pe_roped, uses slot_mapping_ptr) # MLA KV cache (concat kv_c_normed + k_pe_roped, uses slot_mapping_ptr)
mla_cache_ptr, mla_cache_ptr,
mla_cache_block_stride, mla_cache_block_stride,
@@ -161,12 +191,15 @@ def _fused_norm_rope_kernel(
HAS_INDEXER: tl.constexpr, HAS_INDEXER: tl.constexpr,
INDEX_ROPE_INTERLEAVE: tl.constexpr, INDEX_ROPE_INTERLEAVE: tl.constexpr,
USE_PDL: tl.constexpr, USE_PDL: tl.constexpr,
FP8_MAX: tl.constexpr,
USE_FNUZ: tl.constexpr,
): ):
tok_idx = tl.program_id(0).to(tl.int64) tok_idx = tl.program_id(0).to(tl.int64)
pid = tl.program_id(1) pid = tl.program_id(1)
if USE_PDL: if USE_PDL:
tl.extra.cuda.gdc_wait() tl.extra.cuda.gdc_wait()
tl.extra.cuda.gdc_launch_dependents() tl.extra.cuda.gdc_launch_dependents()
fp8_dtype = tl.float8e4b8 if USE_FNUZ else tl.float8e4nv
if pid == 3: if pid == 3:
if not HAS_INDEXER: if not HAS_INDEXER:
# Shared layer: reuse the previous indexer layer's top-k; do not # Shared layer: reuse the previous indexer layer's top-k; do not
@@ -327,9 +360,11 @@ def _fused_norm_rope_kernel(
) )
kv_2d = tl.reshape(kv_c, (MLA_NUM_TILES, MLA_TILE_DIM)) kv_2d = tl.reshape(kv_c, (MLA_NUM_TILES, MLA_TILE_DIM))
tile_amax = tl.max(tl.abs(kv_2d), axis=1, keep_dims=True) tile_amax = tl.max(tl.abs(kv_2d), axis=1, keep_dims=True)
tile_scale = tl.maximum(tile_amax * (1.0 / 448.0), 1e-4) # scale = amax / FP8_MAX (fp8 e4m3 max), matching the reference
# concat_and_cache_ds_mla kernel; floored to FLT_MIN.
tile_scale = tl.maximum(tile_amax * (1.0 / FP8_MAX), 1e-4)
tile_scale = tl.math.exp2(tl.math.ceil(tl.math.log2(tile_scale))) tile_scale = tl.math.exp2(tl.math.ceil(tl.math.log2(tile_scale)))
kv_c_fp8 = tl.reshape((kv_2d / tile_scale).to(tl.float8e4nv), (KV_DIM,)) kv_c_fp8 = tl.reshape((kv_2d / tile_scale).to(fp8_dtype), (KV_DIM,))
tl.store(mla_cache_ptr + byte_base + kv_block, kv_c_fp8) tl.store(mla_cache_ptr + byte_base + kv_block, kv_c_fp8)
tile_off = tl.arange(0, MLA_NUM_TILES) tile_off = tl.arange(0, MLA_NUM_TILES)
tl.store( tl.store(
@@ -349,16 +384,16 @@ def _fused_norm_rope_kernel(
# kv_c_normed (KV_DIM elements) # kv_c_normed (KV_DIM elements)
if MLA_CACHE_FP8: if MLA_CACHE_FP8:
scale = tl.load(mla_cache_scale_ptr) scale = tl.load(mla_cache_scale_ptr)
kv_c_fp8 = (kv_c.to(tl.float32) / scale).to(tl.float8e4nv) kv_c_fp8 = (kv_c.to(tl.float32) / scale).to(fp8_dtype)
tl.store(dst + kv_block, kv_c_fp8) tl.store(dst + kv_block, kv_c_fp8)
else: else:
tl.store(dst + kv_block, kv_c) tl.store(dst + kv_block, kv_c)
# k_pe_roped (from registers, interleaved layout) # k_pe_roped (from registers, interleaved layout)
if MLA_CACHE_FP8: if MLA_CACHE_FP8:
tl.store(dst + KV_DIM + dim_off * 2, (r1 / scale).to(tl.float8e4nv)) tl.store(dst + KV_DIM + dim_off * 2, (r1 / scale).to(fp8_dtype))
tl.store( tl.store(
dst + KV_DIM + dim_off * 2 + 1, dst + KV_DIM + dim_off * 2 + 1,
(r2 / scale).to(tl.float8e4nv), (r2 / scale).to(fp8_dtype),
) )
else: else:
tl.store(dst + KV_DIM + dim_off * 2, r1) tl.store(dst + KV_DIM + dim_off * 2, r1)
@@ -463,6 +498,11 @@ def _fused_norm_rope_kernel(
indexer_cache_stride, indexer_cache_stride,
index_k_block, index_k_block,
INDEX_K_DIM, INDEX_K_DIM,
INDEXER_CACHE_SHUFFLE,
INDEXER_CACHE_BLOCK_TILE,
INDEXER_CACHE_HEAD_TILE,
FP8_MAX,
USE_FNUZ,
) )
@@ -486,6 +526,7 @@ def fused_norm_rope(
slot_mapping: torch.Tensor | None = None, slot_mapping: torch.Tensor | None = None,
indexer_slot_mapping: torch.Tensor | None = None, indexer_slot_mapping: torch.Tensor | None = None,
indexer_k_cache: torch.Tensor | None = None, indexer_k_cache: torch.Tensor | None = None,
indexer_cache_shuffled: bool = False,
mla_kv_cache: torch.Tensor | None = None, mla_kv_cache: torch.Tensor | None = None,
mla_kv_cache_dtype: str = "auto", mla_kv_cache_dtype: str = "auto",
mla_k_scale: torch.Tensor | None = None, mla_k_scale: torch.Tensor | None = None,
@@ -532,13 +573,28 @@ def fused_norm_rope(
indexer_slot_mapping = slot_mapping indexer_slot_mapping = slot_mapping
idx_cache_scale_view = indexer_k_cache.view(torch.uint8).view(torch.float32) idx_cache_scale_view = indexer_k_cache.view(torch.uint8).view(torch.float32)
idx_cache_block_size = indexer_k_cache.shape[1] idx_cache_block_size = indexer_k_cache.shape[1]
idx_cache_stride = indexer_k_cache.shape[2]
# The caller's cache reports whether its reader expects the shuffled
# layout; see DeepseekV32IndexerCache.uses_shuffled_layout.
idx_cache_shuffle = indexer_cache_shuffled
if idx_cache_shuffle:
assert idx_cache_block_size % _INDEXER_CACHE_BLOCK_TILE == 0, (
f"indexer K cache block size {idx_cache_block_size} must be a "
f"multiple of {_INDEXER_CACHE_BLOCK_TILE} for the shuffled layout"
)
idx_cache_stride = indexer_k_cache.stride(0) idx_cache_stride = indexer_k_cache.stride(0)
if indexer_k_cache.dtype == torch.uint8: if indexer_k_cache.dtype == torch.uint8:
indexer_k_cache = indexer_k_cache.view(torch.float8_e4m3fn) indexer_k_cache = indexer_k_cache.view(_FP8_DTYPE)
# The head tile is a byte count; the kernel indexes the cache in elements.
idx_cache_head_tile = (
_INDEXER_CACHE_HEAD_TILE_BYTES // indexer_k_cache.element_size()
)
else: else:
idx_cache_scale_view = None idx_cache_scale_view = None
idx_cache_block_size = 1 idx_cache_block_size = 1
idx_cache_stride = 0 idx_cache_stride = 0
idx_cache_shuffle = False
idx_cache_head_tile = _INDEXER_CACHE_HEAD_TILE_BYTES
# --- MLA KV cache setup --- # --- MLA KV cache setup ---
mla_cache_nvfp4 = mla_kv_cache_dtype == "nvfp4_ds_mla" mla_cache_nvfp4 = mla_kv_cache_dtype == "nvfp4_ds_mla"
@@ -561,12 +617,12 @@ def fused_norm_rope(
mla_entry_stride = u8_cache.stride(1) mla_entry_stride = u8_cache.stride(1)
mla_ds_scale_view = u8_cache.view(torch.float32) mla_ds_scale_view = u8_cache.view(torch.float32)
mla_ds_rope_view = u8_cache.view(torch.bfloat16) mla_ds_rope_view = u8_cache.view(torch.bfloat16)
mla_kv_cache = u8_cache.view(torch.float8_e4m3fn) mla_kv_cache = u8_cache.view(_FP8_DTYPE)
else: else:
mla_block_stride = mla_kv_cache.stride(0) mla_block_stride = mla_kv_cache.stride(0)
mla_entry_stride = mla_kv_cache.stride(1) mla_entry_stride = mla_kv_cache.stride(1)
if mla_cache_fp8 and mla_kv_cache.dtype == torch.uint8: if mla_cache_fp8 and mla_kv_cache.dtype == torch.uint8:
mla_kv_cache = mla_kv_cache.view(torch.float8_e4m3fn) mla_kv_cache = mla_kv_cache.view(_FP8_DTYPE)
if mla_k_scale is None: if mla_k_scale is None:
mla_k_scale = torch.ones(1, dtype=torch.float32, device=device) mla_k_scale = torch.ones(1, dtype=torch.float32, device=device)
else: else:
@@ -592,6 +648,9 @@ def fused_norm_rope(
assert index_k_out.shape == index_k.shape assert index_k_out.shape == index_k.shape
index_k_out_stride = index_k_out.stride(0) index_k_out_stride = index_k_out.stride(0)
use_pdl = current_platform.is_arch_support_pdl() use_pdl = current_platform.is_arch_support_pdl()
pdl_kwargs = (
{"USE_PDL": use_pdl, "launch_pdl": True} if use_pdl else {"USE_PDL": use_pdl}
)
_fused_norm_rope_kernel[(num_tokens, 4)]( _fused_norm_rope_kernel[(num_tokens, 4)](
positions, positions,
# Q RMS norm # Q RMS norm
@@ -639,6 +698,9 @@ def fused_norm_rope(
idx_cache_scale_view, idx_cache_scale_view,
idx_cache_block_size, idx_cache_block_size,
idx_cache_stride, idx_cache_stride,
idx_cache_shuffle,
_INDEXER_CACHE_BLOCK_TILE,
idx_cache_head_tile,
# MLA KV cache (uses same slot_mapping) # MLA KV cache (uses same slot_mapping)
mla_kv_cache, mla_kv_cache,
mla_block_stride, mla_block_stride,
@@ -659,8 +721,9 @@ def fused_norm_rope(
TOPK_BLOCK_SIZE=1024, TOPK_BLOCK_SIZE=1024,
HAS_INDEXER=has_indexer, HAS_INDEXER=has_indexer,
INDEX_ROPE_INTERLEAVE=index_rope_interleave, INDEX_ROPE_INTERLEAVE=index_rope_interleave,
USE_PDL=use_pdl, FP8_MAX=_FP8_MAX,
launch_pdl=use_pdl, USE_FNUZ=_USE_FNUZ,
**pdl_kwargs,
) )
return q_c_out return q_c_out
@@ -715,6 +778,8 @@ def _fused_q_kernel(
INDEX_ROPE_INTERLEAVE: tl.constexpr, INDEX_ROPE_INTERLEAVE: tl.constexpr,
QUANTIZE_MQA: tl.constexpr, QUANTIZE_MQA: tl.constexpr,
USE_PDL: tl.constexpr, USE_PDL: tl.constexpr,
FP8_MAX: tl.constexpr,
USE_FNUZ: tl.constexpr,
): ):
tok_idx = tl.program_id(0).to(tl.int64) tok_idx = tl.program_id(0).to(tl.int64)
pid = tl.program_id(1) pid = tl.program_id(1)
@@ -722,6 +787,7 @@ def _fused_q_kernel(
if USE_PDL: if USE_PDL:
tl.extra.cuda.gdc_wait() tl.extra.cuda.gdc_wait()
tl.extra.cuda.gdc_launch_dependents() tl.extra.cuda.gdc_launch_dependents()
fp8_dtype = tl.float8e4b8 if USE_FNUZ else tl.float8e4nv
if pid == 2: if pid == 2:
# ql_nope quantize + pack into the front of mqa_q_fp8. On the bf16 # ql_nope quantize + pack into the front of mqa_q_fp8. On the bf16
@@ -744,7 +810,7 @@ def _fused_q_kernel(
+ ql_nope_off, + ql_nope_off,
mask=ql_nope_mask, mask=ql_nope_mask,
).to(tl.float32) ).to(tl.float32)
ql_nope_fp8 = (ql_nope / scale).to(tl.float8e4nv) ql_nope_fp8 = (ql_nope / scale).to(fp8_dtype)
tl.store( tl.store(
mqa_q_fp8_ptr mqa_q_fp8_ptr
+ tok_idx * mqa_q_fp8_stride0 + tok_idx * mqa_q_fp8_stride0
@@ -794,7 +860,7 @@ def _fused_q_kernel(
+ q_head_idx * mqa_q_fp8_stride1 + q_head_idx * mqa_q_fp8_stride1
+ QL_NOPE_DIM + QL_NOPE_DIM
+ rot_off * 2, + rot_off * 2,
(r1 / scale).to(tl.float8e4nv), (r1 / scale).to(fp8_dtype),
) )
tl.store( tl.store(
mqa_q_fp8_ptr mqa_q_fp8_ptr
@@ -803,7 +869,7 @@ def _fused_q_kernel(
+ QL_NOPE_DIM + QL_NOPE_DIM
+ rot_off * 2 + rot_off * 2
+ 1, + 1,
(r2 / scale).to(tl.float8e4nv), (r2 / scale).to(fp8_dtype),
) )
else: else:
# bf16 query: write the RoPE'd q_pe unquantized. # bf16 query: write the RoPE'd q_pe unquantized.
@@ -861,7 +927,7 @@ def _fused_q_kernel(
index_q = tl.where(in_rope, roped, index_q) index_q = tl.where(in_rope, roped, index_q)
# Index Q Quantize (from registers) # Index Q Quantize (from registers)
index_q_fp8, index_q_scale = _fp8_ue8m0_quantize(index_q) index_q_fp8, index_q_scale = _fp8_ue8m0_quantize(index_q, FP8_MAX, USE_FNUZ)
tl.store( tl.store(
index_q_fp8_ptr index_q_fp8_ptr
+ tok_idx * index_q_fp8_stride0 + tok_idx * index_q_fp8_stride0
@@ -955,7 +1021,7 @@ def fused_q(
q_pe.shape[0], q_pe.shape[0],
q_pe.shape[1], q_pe.shape[1],
ql_nope.shape[2] + q_pe.shape[2], ql_nope.shape[2] + q_pe.shape[2],
dtype=torch.float8_e4m3fn, dtype=_FP8_DTYPE,
device=q_pe.device, device=q_pe.device,
) )
# Placeholder; pid 0 packs q_pe into mqa_q_fp8 instead. # Placeholder; pid 0 packs q_pe into mqa_q_fp8 instead.
@@ -967,7 +1033,7 @@ def fused_q(
mqa_q_fp8 = q_pe_out # unused placeholder for the fp8 pack pointer mqa_q_fp8 = q_pe_out # unused placeholder for the fp8 pack pointer
mqa_q = q_pe_out mqa_q = q_pe_out
index_q_fp8 = torch.empty_like(index_q, dtype=torch.float8_e4m3fn) index_q_fp8 = torch.empty_like(index_q, dtype=_FP8_DTYPE)
index_weights_out = torch.empty_like(index_weights, dtype=torch.float32) index_weights_out = torch.empty_like(index_weights, dtype=torch.float32)
if cutedsl_kernel is not None: if cutedsl_kernel is not None:
cutedsl_kernel( cutedsl_kernel(
@@ -990,6 +1056,9 @@ def fused_q(
return index_q_fp8, index_weights_out, mqa_q return index_q_fp8, index_weights_out, mqa_q
use_pdl = current_platform.is_arch_support_pdl() use_pdl = current_platform.is_arch_support_pdl()
pdl_kwargs = (
{"USE_PDL": use_pdl, "launch_pdl": True} if use_pdl else {"USE_PDL": use_pdl}
)
_fused_q_kernel[(num_tokens, 3, grid_heads)]( _fused_q_kernel[(num_tokens, 3, grid_heads)](
positions, positions,
q_pe, q_pe,
@@ -1031,8 +1100,9 @@ def fused_q(
HAS_INDEXER=has_indexer, HAS_INDEXER=has_indexer,
INDEX_ROPE_INTERLEAVE=index_rope_interleave, INDEX_ROPE_INTERLEAVE=index_rope_interleave,
QUANTIZE_MQA=quantize_mqa, QUANTIZE_MQA=quantize_mqa,
USE_PDL=use_pdl, FP8_MAX=_FP8_MAX,
launch_pdl=use_pdl, USE_FNUZ=_USE_FNUZ,
**pdl_kwargs,
# num_warps=1 is optimal here: each program is a single 128-element # num_warps=1 is optimal here: each program is a single 128-element
# rope+quant, so the kernel is program-count/occupancy bound, not # rope+quant, so the kernel is program-count/occupancy bound, not
# per-program compute bound (swept 1/2/4/8 — 1 wins or ties everywhere). # per-program compute bound (swept 1/2/4/8 — 1 wins or ties everywhere).