From c843f0cab78b413f2b0a5c2282cb71e3a3937b11 Mon Sep 17 00:00:00 2001 From: Gabriel Wu <13583761+lucifer1004@users.noreply.github.com> Date: Wed, 23 Sep 2026 17:03:20 +0800 Subject: [PATCH] [Bugfix][SM120][MLA] Support NoPE sparse MLA (GLM-5.3-Flash) on the FlashInfer SM120 backend (#55277) Signed-off-by: Zihua Wu <13583761+lucifer1004@users.noreply.github.com> Co-authored-by: Kimi Code --- csrc/libtorch_stable/cache_kernels.cu | 17 +++++++---- tests/kernels/attention/test_cache.py | 44 +++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 6 deletions(-) diff --git a/csrc/libtorch_stable/cache_kernels.cu b/csrc/libtorch_stable/cache_kernels.cu index 4b8f4c23a5..9130adf87d 100644 --- a/csrc/libtorch_stable/cache_kernels.cu +++ b/csrc/libtorch_stable/cache_kernels.cu @@ -549,16 +549,20 @@ __global__ void concat_and_cache_ds_mla_kernel( scalar_t* kv_cache_16bit = reinterpret_cast(&kv_cache[dst_idx_start]); - // The last warp handles the RoPE part + // Zero the reserved RoPE tail for NoPE rows. if (threadIdx.x >= 64) { // Each thread handles two elements of RoPE const int8_t pe_idx_start = (threadIdx.x - 64) * 2; - const int64_t src_idx = token_idx * k_pe_stride + pe_idx_start; - // Vectorized load of two 16-bit values, performed as one 32-bit load - const int32_t vals = *reinterpret_cast(&k_pe[src_idx]); // RoPE values start after the packed 8-bit NoPE values and the // 32-bit scales const int64_t dst_idx = kv_lora_rank / 2 + 8 + pe_idx_start; + if (pe_dim == 0) { + *reinterpret_cast(&kv_cache_16bit[dst_idx]) = 0; + return; + } + const int64_t src_idx = token_idx * k_pe_stride + pe_idx_start; + // Vectorized load of two 16-bit values, performed as one 32-bit load + const int32_t vals = *reinterpret_cast(&k_pe[src_idx]); // Vectorized store of two 16-bit values, performed as one 32-bit store *reinterpret_cast(&kv_cache_16bit[dst_idx]) = vals; return; @@ -600,7 +604,7 @@ __global__ void concat_and_cache_ds_mla_kernel( } // Now all threads in the block scale and write their elements - // NoPE data is packed in the first kv_lora_rank/2 bytes (first 256 bytes) + // NoPE data occupies the first kv_lora_rank bytes (512 bytes). const int64_t dst_idx_base = dst_idx_start + (threadIdx.x * 8); uint8_t result[8]; @@ -936,7 +940,8 @@ void concat_and_cache_mla( if (kv_cache_dtype == "fp8_ds_mla") { STD_TORCH_CHECK(kv_lora_rank == 512, "kv_lora_rank must be 512 for fp8_ds_mla"); - STD_TORCH_CHECK(pe_dim == 64, "pe_dim must be 64 for fp8_ds_mla"); + STD_TORCH_CHECK(pe_dim == 64 || pe_dim == 0, + "pe_dim must be 64 or 0 for fp8_ds_mla"); STD_TORCH_CHECK(kv_cache.size(2) == 656 / kv_cache.element_size(), "kv_cache.size(2) must be 656 bytes for fp8_ds_mla"); STD_TORCH_CHECK(kv_c.element_size() == 2, diff --git a/tests/kernels/attention/test_cache.py b/tests/kernels/attention/test_cache.py index 41f0a1138b..4bc0ce1a67 100644 --- a/tests/kernels/attention/test_cache.py +++ b/tests/kernels/attention/test_cache.py @@ -1012,6 +1012,50 @@ def test_concat_and_cache_ds_mla( torch.testing.assert_close(kv_rope, ref_rope, atol=0.001, rtol=0.1) +@pytest.mark.parametrize("device", CUDA_DEVICES) +@pytest.mark.parametrize("block_size", [64, 256]) +@torch.inference_mode() +def test_concat_and_cache_ds_mla_nope(device: str, block_size: int) -> None: + """NoPE matches zero RoPE, clears valid tails, and preserves unused slots.""" + dtype = torch.bfloat16 + if current_platform.is_rocm(): + pytest.skip("concat_and_cache_mla doesn't support fp8_ds_mla on ROCm") + num_tokens, num_blocks = 3, 2 + set_random_seed(0) + torch.set_default_device(device) + torch.accelerator.set_device_index(device) + + slot_mapping = torch.tensor( + [block_size - 1, block_size, -1], dtype=torch.long, device=device + ) + kv_c = torch.randn(num_tokens, 512, dtype=dtype, device=device) + k_pe = torch.empty(num_tokens, 0, dtype=dtype, device=device) + zero_k_pe = torch.zeros(num_tokens, 64, dtype=dtype, device=device) + scale = torch.tensor(1.0, dtype=torch.float32, device=device) + kv_cache = torch.full( + (num_blocks, block_size, 656), 0xFF, dtype=torch.uint8, device=device + ) + ref_cache = kv_cache.clone() + + opcheck( + torch.ops._C_cache_ops.concat_and_cache_mla, + (kv_c, k_pe, kv_cache, slot_mapping, "fp8_ds_mla", scale), + test_utils=DEFAULT_OPCHECK_TEST_UTILS, + ) + kv_cache.fill_(0xFF) + ops.concat_and_cache_mla(kv_c, k_pe, kv_cache, slot_mapping, "fp8_ds_mla", scale) + ops.concat_and_cache_mla( + kv_c, zero_k_pe, ref_cache, slot_mapping, "fp8_ds_mla", scale + ) + + torch.testing.assert_close(kv_cache, ref_cache, atol=0, rtol=0) + rows = kv_cache.view(num_blocks * block_size, 656) + assert (rows[slot_mapping[:2], 528:] == 0).all() + untouched = torch.ones(num_blocks * block_size, dtype=torch.bool, device=device) + untouched[slot_mapping[:2]] = False + assert (rows[untouched] == 0xFF).all() + + # Bytes per token for the nvfp4_ds_mla cache layout (see flashmla_sparse.py). NVFP4_DS_MLA_ENTRY_SIZE = 352