[Bugfix][DSA] Write nvfp4_ds_mla from the fused norm+rope kernel (#55538)

Signed-off-by: Shiyang Chen <shiychen@nvidia.com>
This commit is contained in:
sychen52
2026-09-15 00:25:33 +04:00
committed by GitHub
parent d4ee7fe7a9
commit 7702ee87db
4 changed files with 186 additions and 20 deletions
@@ -109,9 +109,6 @@ __device__ __forceinline__ void nvfp4_unpack16_e2m1(uint64_t raw,
#endif
}
// Rounded UP to the next e4m3 value so amax / sf never exceeds the payload
// range: round-to-nearest can land up to 33% low near e4m3's 2^-9 floor, which
// would saturate the tile's largest values.
__device__ __forceinline__ float nvfp4_tile_scale(const float* vals,
float max_val,
uint8_t* sf_out) {
@@ -121,15 +118,8 @@ __device__ __forceinline__ float nvfp4_tile_scale(const float* vals,
amax = fmaxf(amax, fabsf(vals[i]));
}
const float sf_f = fmaxf(amax / max_val, 0.001953125f); // 2^-9
__nv_fp8_e4m3 sf8(sf_f);
uint8_t sf_bits = *reinterpret_cast<const uint8_t*>(&sf8);
if (float(sf8) < sf_f && sf_bits < 0x7E) {
// Bump to the next e4m3 value (bit patterns of positive e4m3 values are
// monotonic; 0x7E = 448 is the max finite value).
sf_bits += 1;
sf8 = *reinterpret_cast<const __nv_fp8_e4m3*>(&sf_bits);
}
*sf_out = sf_bits;
const __nv_fp8_e4m3 sf8(sf_f);
*sf_out = *reinterpret_cast<const uint8_t*>(&sf8);
return float(sf8);
}
@@ -573,6 +573,126 @@ def test_fused_norm_rope_ds_mla(num_tokens: int):
assert (topk == 7).all(), "topk buffer should be untouched (no indexer)"
E2M1_MAGNITUDES = [0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0]
def quantize_to_e2m1(x: torch.Tensor) -> torch.Tensor:
"""Round to nearest e2m1, saturating to +-6 like cvt.rn.satfinite.e2m1x2.f32."""
mags = torch.tensor(E2M1_MAGNITUDES, dtype=torch.float32, device=x.device)
a = x.float().abs().clamp_max(6.0)
mids = (mags[:-1] + mags[1:]) / 2
code = torch.bucketize(a, mids, right=True)
on_tie = (a.unsqueeze(-1) == mids).any(dim=-1)
tie_code = torch.bucketize(a, mids, right=False)
code = torch.where(on_tie, tie_code + (tie_code & 1), code)
return (torch.signbit(x).to(torch.uint8) << 3) | code.to(torch.uint8)
def nvfp4_sf_byte(s: torch.Tensor) -> torch.Tensor:
"""Scale-factor byte permutation shared with FlashMLA: 8*(s&3) + (s>>2)."""
return 8 * (s % 4) + (s // 4)
@pytest.mark.skipif(
not current_platform.is_device_capability(100),
reason="nvfp4_ds_mla requires SM100 (Blackwell)",
)
@pytest.mark.parametrize("num_tokens", [1, 4, 17, 512])
def test_fused_norm_rope_nvfp4_ds_mla(num_tokens: int):
"""nvfp4_ds_mla MLA cache layout (FlashMLA sparse, SM100 only).
Per-token 352-byte entry: 256 B of 512 e2m1 NoPE packed 2/byte (low nibble
= even element) | 64 B unscaled e4m3 RoPE | 32 B byte-permuted e4m3 tile
scales, one per 16 NoPE elements.
"""
torch.manual_seed(5)
dev = "cuda"
max_pos = 8192
pos = torch.arange(num_tokens, device=dev, dtype=torch.int64) % max_pos
q_c = torch.randn(num_tokens, Q_LORA, device=dev, dtype=torch.bfloat16)
kv_c = torch.randn(num_tokens, KV_LORA, device=dev, dtype=torch.bfloat16)
# Tile 0 is deliberately low-magnitude: amax/6 lands between two e4m3
# subnormals (spaced a flat 2^-9 there), so round-to-nearest picks a scale
# BELOW amax/6 and the tile's peak saturates at +-6. Asserted below, so a
# future change to the scale rule cannot silently stop covering this.
# amax must land in (6, 9) * 2^-9 after rms_norm; 7.5 sits mid-band, with
# margin for the per-token RMS to vary.
kv_c[:, :16] = (
7.5 * (2.0**-9) * torch.linspace(0.2, 1.0, 16, device=dev, dtype=torch.bfloat16)
)
k_pe = torch.randn(num_tokens, ROPE_DIM, device=dev, dtype=torch.bfloat16)
qw = torch.randn(Q_LORA, device=dev, dtype=torch.bfloat16)
kvw = torch.ones(KV_LORA, device=dev, dtype=torch.bfloat16)
mla_cos_sin = make_cos_sin(max_pos, ROPE_DIM, dev)
bs = max_pos
mla_cache = torch.zeros(1, bs, 352, device=dev, dtype=torch.uint8)
slot = torch.arange(num_tokens, device=dev, dtype=torch.int64)
topk = torch.full((num_tokens, 2048), 7, device=dev, dtype=torch.int32)
q_out = K.fused_norm_rope(
pos,
q_c,
qw,
EPS,
kv_c,
kvw,
EPS,
k_pe,
mla_cos_sin,
None,
None,
None,
EPS,
None,
topk,
slot_mapping=slot,
indexer_k_cache=None,
mla_kv_cache=mla_cache,
mla_kv_cache_dtype="nvfp4_ds_mla",
mla_k_scale=None,
has_indexer=False,
index_rope_interleave=False,
)
assert_bf16(q_out, rms_norm(q_c, qw), "q_c rmsnorm (nvfp4_ds_mla)")
kv_ref = rms_norm(kv_c, kvw) # [N, 512] fp32
kpe_ref = rope(k_pe.float(), pos, mla_cos_sin, interleave=True) # [N, 64]
tiles = kv_ref.view(num_tokens, 32, 16)
amax = tiles.abs().amax(dim=-1)
scale_target = torch.clamp_min(amax / 6.0, 2.0**-9)
ref_scale = scale_target.to(FP8) # round-to-nearest
# The crafted tile must actually exercise the saturating path.
assert (ref_scale[:, 0].float() < scale_target[:, 0]).all(), (
"tile 0 should round its scale DOWN; adjust the crafted magnitude"
)
assert (amax[:, 0] / ref_scale[:, 0].float() > 6.0).all(), (
"tile 0 should saturate e2m1; adjust the crafted magnitude"
)
ref_codes = quantize_to_e2m1(tiles / ref_scale.float().unsqueeze(-1))
ref_codes = ref_codes.reshape(num_tokens, KV_LORA)
cache = mla_cache[0, :num_tokens] # [N, 352] uint8
packed = cache[:, : KV_LORA // 2]
got_codes = torch.stack([packed & 0xF, packed >> 4], dim=-1)
got_codes = got_codes.reshape(num_tokens, KV_LORA) # low nibble = even elem
torch.testing.assert_close(got_codes, ref_codes, rtol=0, atol=0)
got_rope = cache[:, KV_LORA // 2 : KV_LORA // 2 + ROPE_DIM].view(FP8)
assert_fp8(got_rope, kpe_ref.to(FP8), "nvfp4_ds_mla RoPE e4m3")
perm = nvfp4_sf_byte(torch.arange(32, device=dev))
got_scale = cache[:, KV_LORA // 2 + ROPE_DIM :].view(FP8)[:, perm]
torch.testing.assert_close(got_scale.float(), ref_scale.float(), rtol=0, atol=0)
# No indexer on this call: top-k buffer must be untouched.
assert (topk == 7).all(), "topk buffer should be untouched (no indexer)"
def test_fused_norm_rope_supports_large_token_count():
"""Keep the token count off CUDA grid-y at its 65,536-block boundary."""
num_tokens = 65536
+3 -1
View File
@@ -237,7 +237,9 @@ class DeepseekV32Attention(MLAAttention):
fp8_attention = is_quantized_kv_cache(self.kv_cache_dtype)
self._fp8_query = fp8_attention and self.impl.supports_quant_query_input
self._fp8_kv_needs_view = fp8_attention and self.kv_cache_dtype != "fp8_ds_mla"
self._fp8_kv_needs_view = fp8_attention and not self.kv_cache_dtype.endswith(
"_ds_mla"
)
self._index_rope_interleave = getattr(config, "indexer_rope_interleave", False)
+61 -7
View File
@@ -150,6 +150,7 @@ def _fused_norm_rope_kernel(
mla_cache_ds_scale_ptr,
mla_cache_ds_rope_ptr,
MLA_CACHE_DS_MLA: tl.constexpr,
MLA_CACHE_NVFP4: tl.constexpr,
MLA_NUM_TILES: tl.constexpr,
MLA_TILE_DIM: tl.constexpr,
# Top k indices
@@ -261,6 +262,56 @@ def _fused_norm_rope_kernel(
mla_block_idx = slot_idx // mla_block_size
mla_block_off = slot_idx % mla_block_size
if MLA_CACHE_NVFP4:
# nvfp4_ds_mla layout (352 B/token, KV_DIM == 512):
# [0, 256) 512 e2m1 NoPE, packed 2/byte (low nibble = even)
# [256, 320) 64 e4m3 RoPE, unscaled
# [320, 352) 32 e4m3 NoPE scales, byte-permuted
# Keep in lockstep with concat_and_cache_nvfp4_ds_mla (CUDA) and
# nvfp4_sf_byte() in FlashMLA: byte(s) = 8*(s&3) + (s>>2).
byte_base = (
mla_block_idx * mla_cache_block_stride
+ mla_block_off * mla_cache_entry_stride
)
kv_2d = tl.reshape(kv_c.to(tl.float32), (MLA_NUM_TILES, MLA_TILE_DIM))
amax = tl.max(tl.abs(kv_2d), axis=1, keep_dims=True)
# sf = e4m3(max(amax/6, 2^-9)), round-to-nearest like
# cvt_warp_fp16_to_fp4.
sf8 = tl.maximum(amax * (1.0 / 6.0), 0.001953125).to(tl.float8e4nv)
q = kv_2d * (1.0 / sf8.to(tl.float32))
qp = tl.reshape(q, (KV_DIM // 2, 2))
sel = tl.arange(0, 2)[None, :]
even = tl.sum(tl.where(sel == 0, qp, 0.0), axis=1)
odd = tl.sum(tl.where(sel == 1, qp, 0.0), axis=1)
# Same instruction the CUDA writer uses, so the two are bit-exact
# by construction: dest byte = {high: odd, low: even}.
packed = tl.inline_asm_elementwise(
"{ .reg .b8 t; .reg .b32 w;"
" cvt.rn.satfinite.e2m1x2.f32 t, $2, $1;"
" mov.b32 w, {t, t, t, t}; and.b32 $0, w, 255; }",
"=r,f,f",
[even, odd],
dtype=tl.int32,
is_pure=True,
pack=1,
).to(tl.uint8)
tl.store(
mla_cache_ptr + byte_base + tl.arange(0, KV_DIM // 2),
packed.to(tl.float8e4nv, bitcast=True),
)
# RoPE: unscaled e4m3, interleaved like the fp8_ds_mla path
rope_base = mla_cache_ptr + byte_base + KV_DIM // 2
tl.store(rope_base + dim_off * 2, r1.to(tl.float8e4nv))
tl.store(rope_base + dim_off * 2 + 1, r2.to(tl.float8e4nv))
# scales, permuted: byte(s) = 8*(s&3) + (s>>2)
sf_tile = tl.arange(0, MLA_NUM_TILES)
sf_perm = 8 * (sf_tile % 4) + (sf_tile // 4)
tl.store(
mla_cache_ptr + byte_base + (KV_DIM // 2 + 64) + sf_perm,
tl.reshape(sf8, (MLA_NUM_TILES,)),
)
return
if MLA_CACHE_DS_MLA:
# fp8_ds_mla layout (DeepSeek-V3.2, KV_DIM == 512): per-128-element
# tile of the NoPE is dynamically quantized to fp8 with its own
@@ -491,7 +542,8 @@ def fused_norm_rope(
idx_cache_stride = 0
# --- MLA KV cache setup ---
mla_cache_ds_mla = mla_kv_cache_dtype == "fp8_ds_mla"
mla_cache_nvfp4 = mla_kv_cache_dtype == "nvfp4_ds_mla"
mla_cache_ds_mla = mla_kv_cache_dtype == "fp8_ds_mla" or mla_cache_nvfp4
mla_cache_fp8 = is_quantized_kv_cache(mla_kv_cache_dtype) and not mla_cache_ds_mla
mla_num_tiles = 1
mla_ds_scale_view = torch.empty(0, dtype=torch.float32, device=device)
@@ -499,11 +551,12 @@ def fused_norm_rope(
if mla_kv_cache is not None:
mla_block_size = mla_kv_cache.shape[1]
if mla_cache_ds_mla:
# 656-byte custom layout addressed in bytes; mla_cache_ptr is the
# 1-byte fp8 view, so block/entry strides are byte offsets and the
# fp32/bf16 views share the same buffer.
assert kv_dim == 512, "fp8_ds_mla requires kv_lora_rank == 512"
mla_num_tiles = kv_dim // 128
# Custom byte-addressed layout: 656 B/token for fp8_ds_mla, 352 B
# for nvfp4_ds_mla. mla_cache_ptr is the 1-byte fp8 view, so
# block/entry strides are byte offsets and the fp32/bf16 views
# (fp8_ds_mla only) share the same buffer.
assert kv_dim == 512, "ds_mla layouts require kv_lora_rank == 512"
mla_num_tiles = kv_dim // 16 if mla_cache_nvfp4 else kv_dim // 128
u8_cache = mla_kv_cache.view(torch.uint8)
mla_block_stride = u8_cache.stride(0)
mla_entry_stride = u8_cache.stride(1)
@@ -596,7 +649,8 @@ def fused_norm_rope(
mla_k_scale,
mla_ds_scale_view,
mla_ds_rope_view,
mla_cache_ds_mla,
mla_cache_ds_mla and not mla_cache_nvfp4,
mla_cache_nvfp4,
mla_num_tiles,
kv_dim // mla_num_tiles if mla_cache_ds_mla else 1,
# Top k indices buffer