From 7702ee87dba0d8eed7f201e77a0a6613aac738a8 Mon Sep 17 00:00:00 2001 From: sychen52 <41452870+sychen52@users.noreply.github.com> Date: Mon, 14 Sep 2026 13:25:33 -0700 Subject: [PATCH] [Bugfix][DSA] Write nvfp4_ds_mla from the fused norm+rope kernel (#55538) Signed-off-by: Shiyang Chen --- .../nvfp4_ds_mla_cache_kernels.cu | 14 +- .../test_fused_deepseek_v32_norm_rope.py | 120 ++++++++++++++++++ vllm/models/deepseek_v32/attention.py | 4 +- vllm/models/deepseek_v32/common/kernels.py | 68 +++++++++- 4 files changed, 186 insertions(+), 20 deletions(-) diff --git a/csrc/libtorch_stable/nvfp4_ds_mla_cache_kernels.cu b/csrc/libtorch_stable/nvfp4_ds_mla_cache_kernels.cu index c2faf7b760..b1f369e5ab 100644 --- a/csrc/libtorch_stable/nvfp4_ds_mla_cache_kernels.cu +++ b/csrc/libtorch_stable/nvfp4_ds_mla_cache_kernels.cu @@ -109,9 +109,6 @@ __device__ __forceinline__ void nvfp4_unpack16_e2m1(uint64_t raw, #endif } -// Rounded UP to the next e4m3 value so amax / sf never exceeds the payload -// range: round-to-nearest can land up to 33% low near e4m3's 2^-9 floor, which -// would saturate the tile's largest values. __device__ __forceinline__ float nvfp4_tile_scale(const float* vals, float max_val, uint8_t* sf_out) { @@ -121,15 +118,8 @@ __device__ __forceinline__ float nvfp4_tile_scale(const float* vals, amax = fmaxf(amax, fabsf(vals[i])); } const float sf_f = fmaxf(amax / max_val, 0.001953125f); // 2^-9 - __nv_fp8_e4m3 sf8(sf_f); - uint8_t sf_bits = *reinterpret_cast(&sf8); - if (float(sf8) < sf_f && sf_bits < 0x7E) { - // Bump to the next e4m3 value (bit patterns of positive e4m3 values are - // monotonic; 0x7E = 448 is the max finite value). - sf_bits += 1; - sf8 = *reinterpret_cast(&sf_bits); - } - *sf_out = sf_bits; + const __nv_fp8_e4m3 sf8(sf_f); + *sf_out = *reinterpret_cast(&sf8); return float(sf8); } diff --git a/tests/kernels/test_fused_deepseek_v32_norm_rope.py b/tests/kernels/test_fused_deepseek_v32_norm_rope.py index fbe22af2f8..7f74cde05d 100644 --- a/tests/kernels/test_fused_deepseek_v32_norm_rope.py +++ b/tests/kernels/test_fused_deepseek_v32_norm_rope.py @@ -573,6 +573,126 @@ def test_fused_norm_rope_ds_mla(num_tokens: int): assert (topk == 7).all(), "topk buffer should be untouched (no indexer)" +E2M1_MAGNITUDES = [0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0] + + +def quantize_to_e2m1(x: torch.Tensor) -> torch.Tensor: + """Round to nearest e2m1, saturating to +-6 like cvt.rn.satfinite.e2m1x2.f32.""" + mags = torch.tensor(E2M1_MAGNITUDES, dtype=torch.float32, device=x.device) + a = x.float().abs().clamp_max(6.0) + mids = (mags[:-1] + mags[1:]) / 2 + code = torch.bucketize(a, mids, right=True) + on_tie = (a.unsqueeze(-1) == mids).any(dim=-1) + tie_code = torch.bucketize(a, mids, right=False) + code = torch.where(on_tie, tie_code + (tie_code & 1), code) + return (torch.signbit(x).to(torch.uint8) << 3) | code.to(torch.uint8) + + +def nvfp4_sf_byte(s: torch.Tensor) -> torch.Tensor: + """Scale-factor byte permutation shared with FlashMLA: 8*(s&3) + (s>>2).""" + return 8 * (s % 4) + (s // 4) + + +@pytest.mark.skipif( + not current_platform.is_device_capability(100), + reason="nvfp4_ds_mla requires SM100 (Blackwell)", +) +@pytest.mark.parametrize("num_tokens", [1, 4, 17, 512]) +def test_fused_norm_rope_nvfp4_ds_mla(num_tokens: int): + """nvfp4_ds_mla MLA cache layout (FlashMLA sparse, SM100 only). + + Per-token 352-byte entry: 256 B of 512 e2m1 NoPE packed 2/byte (low nibble + = even element) | 64 B unscaled e4m3 RoPE | 32 B byte-permuted e4m3 tile + scales, one per 16 NoPE elements. + """ + torch.manual_seed(5) + dev = "cuda" + max_pos = 8192 + pos = torch.arange(num_tokens, device=dev, dtype=torch.int64) % max_pos + + q_c = torch.randn(num_tokens, Q_LORA, device=dev, dtype=torch.bfloat16) + kv_c = torch.randn(num_tokens, KV_LORA, device=dev, dtype=torch.bfloat16) + # Tile 0 is deliberately low-magnitude: amax/6 lands between two e4m3 + # subnormals (spaced a flat 2^-9 there), so round-to-nearest picks a scale + # BELOW amax/6 and the tile's peak saturates at +-6. Asserted below, so a + # future change to the scale rule cannot silently stop covering this. + # amax must land in (6, 9) * 2^-9 after rms_norm; 7.5 sits mid-band, with + # margin for the per-token RMS to vary. + kv_c[:, :16] = ( + 7.5 * (2.0**-9) * torch.linspace(0.2, 1.0, 16, device=dev, dtype=torch.bfloat16) + ) + k_pe = torch.randn(num_tokens, ROPE_DIM, device=dev, dtype=torch.bfloat16) + qw = torch.randn(Q_LORA, device=dev, dtype=torch.bfloat16) + kvw = torch.ones(KV_LORA, device=dev, dtype=torch.bfloat16) + mla_cos_sin = make_cos_sin(max_pos, ROPE_DIM, dev) + + bs = max_pos + mla_cache = torch.zeros(1, bs, 352, device=dev, dtype=torch.uint8) + slot = torch.arange(num_tokens, device=dev, dtype=torch.int64) + topk = torch.full((num_tokens, 2048), 7, device=dev, dtype=torch.int32) + + q_out = K.fused_norm_rope( + pos, + q_c, + qw, + EPS, + kv_c, + kvw, + EPS, + k_pe, + mla_cos_sin, + None, + None, + None, + EPS, + None, + topk, + slot_mapping=slot, + indexer_k_cache=None, + mla_kv_cache=mla_cache, + mla_kv_cache_dtype="nvfp4_ds_mla", + mla_k_scale=None, + has_indexer=False, + index_rope_interleave=False, + ) + + assert_bf16(q_out, rms_norm(q_c, qw), "q_c rmsnorm (nvfp4_ds_mla)") + + kv_ref = rms_norm(kv_c, kvw) # [N, 512] fp32 + kpe_ref = rope(k_pe.float(), pos, mla_cos_sin, interleave=True) # [N, 64] + tiles = kv_ref.view(num_tokens, 32, 16) + amax = tiles.abs().amax(dim=-1) + scale_target = torch.clamp_min(amax / 6.0, 2.0**-9) + ref_scale = scale_target.to(FP8) # round-to-nearest + + # The crafted tile must actually exercise the saturating path. + assert (ref_scale[:, 0].float() < scale_target[:, 0]).all(), ( + "tile 0 should round its scale DOWN; adjust the crafted magnitude" + ) + assert (amax[:, 0] / ref_scale[:, 0].float() > 6.0).all(), ( + "tile 0 should saturate e2m1; adjust the crafted magnitude" + ) + + ref_codes = quantize_to_e2m1(tiles / ref_scale.float().unsqueeze(-1)) + ref_codes = ref_codes.reshape(num_tokens, KV_LORA) + + cache = mla_cache[0, :num_tokens] # [N, 352] uint8 + packed = cache[:, : KV_LORA // 2] + got_codes = torch.stack([packed & 0xF, packed >> 4], dim=-1) + got_codes = got_codes.reshape(num_tokens, KV_LORA) # low nibble = even elem + torch.testing.assert_close(got_codes, ref_codes, rtol=0, atol=0) + + got_rope = cache[:, KV_LORA // 2 : KV_LORA // 2 + ROPE_DIM].view(FP8) + assert_fp8(got_rope, kpe_ref.to(FP8), "nvfp4_ds_mla RoPE e4m3") + + perm = nvfp4_sf_byte(torch.arange(32, device=dev)) + got_scale = cache[:, KV_LORA // 2 + ROPE_DIM :].view(FP8)[:, perm] + torch.testing.assert_close(got_scale.float(), ref_scale.float(), rtol=0, atol=0) + + # No indexer on this call: top-k buffer must be untouched. + assert (topk == 7).all(), "topk buffer should be untouched (no indexer)" + + def test_fused_norm_rope_supports_large_token_count(): """Keep the token count off CUDA grid-y at its 65,536-block boundary.""" num_tokens = 65536 diff --git a/vllm/models/deepseek_v32/attention.py b/vllm/models/deepseek_v32/attention.py index 00dd2e18a8..87cd618524 100644 --- a/vllm/models/deepseek_v32/attention.py +++ b/vllm/models/deepseek_v32/attention.py @@ -237,7 +237,9 @@ class DeepseekV32Attention(MLAAttention): fp8_attention = is_quantized_kv_cache(self.kv_cache_dtype) self._fp8_query = fp8_attention and self.impl.supports_quant_query_input - self._fp8_kv_needs_view = fp8_attention and self.kv_cache_dtype != "fp8_ds_mla" + self._fp8_kv_needs_view = fp8_attention and not self.kv_cache_dtype.endswith( + "_ds_mla" + ) self._index_rope_interleave = getattr(config, "indexer_rope_interleave", False) diff --git a/vllm/models/deepseek_v32/common/kernels.py b/vllm/models/deepseek_v32/common/kernels.py index ef7fcde5d3..fd02a51076 100644 --- a/vllm/models/deepseek_v32/common/kernels.py +++ b/vllm/models/deepseek_v32/common/kernels.py @@ -150,6 +150,7 @@ def _fused_norm_rope_kernel( mla_cache_ds_scale_ptr, mla_cache_ds_rope_ptr, MLA_CACHE_DS_MLA: tl.constexpr, + MLA_CACHE_NVFP4: tl.constexpr, MLA_NUM_TILES: tl.constexpr, MLA_TILE_DIM: tl.constexpr, # Top k indices @@ -261,6 +262,56 @@ def _fused_norm_rope_kernel( mla_block_idx = slot_idx // mla_block_size mla_block_off = slot_idx % mla_block_size + if MLA_CACHE_NVFP4: + # nvfp4_ds_mla layout (352 B/token, KV_DIM == 512): + # [0, 256) 512 e2m1 NoPE, packed 2/byte (low nibble = even) + # [256, 320) 64 e4m3 RoPE, unscaled + # [320, 352) 32 e4m3 NoPE scales, byte-permuted + # Keep in lockstep with concat_and_cache_nvfp4_ds_mla (CUDA) and + # nvfp4_sf_byte() in FlashMLA: byte(s) = 8*(s&3) + (s>>2). + byte_base = ( + mla_block_idx * mla_cache_block_stride + + mla_block_off * mla_cache_entry_stride + ) + kv_2d = tl.reshape(kv_c.to(tl.float32), (MLA_NUM_TILES, MLA_TILE_DIM)) + amax = tl.max(tl.abs(kv_2d), axis=1, keep_dims=True) + # sf = e4m3(max(amax/6, 2^-9)), round-to-nearest like + # cvt_warp_fp16_to_fp4. + sf8 = tl.maximum(amax * (1.0 / 6.0), 0.001953125).to(tl.float8e4nv) + q = kv_2d * (1.0 / sf8.to(tl.float32)) + qp = tl.reshape(q, (KV_DIM // 2, 2)) + sel = tl.arange(0, 2)[None, :] + even = tl.sum(tl.where(sel == 0, qp, 0.0), axis=1) + odd = tl.sum(tl.where(sel == 1, qp, 0.0), axis=1) + # Same instruction the CUDA writer uses, so the two are bit-exact + # by construction: dest byte = {high: odd, low: even}. + packed = tl.inline_asm_elementwise( + "{ .reg .b8 t; .reg .b32 w;" + " cvt.rn.satfinite.e2m1x2.f32 t, $2, $1;" + " mov.b32 w, {t, t, t, t}; and.b32 $0, w, 255; }", + "=r,f,f", + [even, odd], + dtype=tl.int32, + is_pure=True, + pack=1, + ).to(tl.uint8) + tl.store( + mla_cache_ptr + byte_base + tl.arange(0, KV_DIM // 2), + packed.to(tl.float8e4nv, bitcast=True), + ) + # RoPE: unscaled e4m3, interleaved like the fp8_ds_mla path + rope_base = mla_cache_ptr + byte_base + KV_DIM // 2 + tl.store(rope_base + dim_off * 2, r1.to(tl.float8e4nv)) + tl.store(rope_base + dim_off * 2 + 1, r2.to(tl.float8e4nv)) + # scales, permuted: byte(s) = 8*(s&3) + (s>>2) + sf_tile = tl.arange(0, MLA_NUM_TILES) + sf_perm = 8 * (sf_tile % 4) + (sf_tile // 4) + tl.store( + mla_cache_ptr + byte_base + (KV_DIM // 2 + 64) + sf_perm, + tl.reshape(sf8, (MLA_NUM_TILES,)), + ) + return + if MLA_CACHE_DS_MLA: # fp8_ds_mla layout (DeepSeek-V3.2, KV_DIM == 512): per-128-element # tile of the NoPE is dynamically quantized to fp8 with its own @@ -491,7 +542,8 @@ def fused_norm_rope( idx_cache_stride = 0 # --- MLA KV cache setup --- - mla_cache_ds_mla = mla_kv_cache_dtype == "fp8_ds_mla" + mla_cache_nvfp4 = mla_kv_cache_dtype == "nvfp4_ds_mla" + mla_cache_ds_mla = mla_kv_cache_dtype == "fp8_ds_mla" or mla_cache_nvfp4 mla_cache_fp8 = is_quantized_kv_cache(mla_kv_cache_dtype) and not mla_cache_ds_mla mla_num_tiles = 1 mla_ds_scale_view = torch.empty(0, dtype=torch.float32, device=device) @@ -499,11 +551,12 @@ def fused_norm_rope( if mla_kv_cache is not None: mla_block_size = mla_kv_cache.shape[1] if mla_cache_ds_mla: - # 656-byte custom layout addressed in bytes; mla_cache_ptr is the - # 1-byte fp8 view, so block/entry strides are byte offsets and the - # fp32/bf16 views share the same buffer. - assert kv_dim == 512, "fp8_ds_mla requires kv_lora_rank == 512" - mla_num_tiles = kv_dim // 128 + # Custom byte-addressed layout: 656 B/token for fp8_ds_mla, 352 B + # for nvfp4_ds_mla. mla_cache_ptr is the 1-byte fp8 view, so + # block/entry strides are byte offsets and the fp32/bf16 views + # (fp8_ds_mla only) share the same buffer. + assert kv_dim == 512, "ds_mla layouts require kv_lora_rank == 512" + mla_num_tiles = kv_dim // 16 if mla_cache_nvfp4 else kv_dim // 128 u8_cache = mla_kv_cache.view(torch.uint8) mla_block_stride = u8_cache.stride(0) mla_entry_stride = u8_cache.stride(1) @@ -596,7 +649,8 @@ def fused_norm_rope( mla_k_scale, mla_ds_scale_view, mla_ds_rope_view, - mla_cache_ds_mla, + mla_cache_ds_mla and not mla_cache_nvfp4, + mla_cache_nvfp4, mla_num_tiles, kv_dim // mla_num_tiles if mla_cache_ds_mla else 1, # Top k indices buffer