From 7c5dc571cbd1064ecc8a9b1045637ff647aa22cb Mon Sep 17 00:00:00 2001 From: sychen52 <41452870+sychen52@users.noreply.github.com> Date: Mon, 31 Aug 2026 21:37:47 -0700 Subject: [PATCH] [Attention][DSA] Enable W4A16 DSA (#51724) Signed-off-by: Shiyang Chen --- CMakeLists.txt | 3 +- cmake/external_projects/flashmla.cmake | 3 +- csrc/libtorch_stable/cache_kernels.cu | 104 +++++++ csrc/libtorch_stable/nvfp4_ds_mla_cache.h | 37 +++ .../nvfp4_ds_mla_cache_kernels.cu | 281 ++++++++++++++++++ csrc/libtorch_stable/ops.h | 9 + csrc/libtorch_stable/torch_bindings.cpp | 6 + tests/kernels/attention/test_cache.py | 151 ++++++++++ tests/v1/attention/test_mla_backends.py | 13 +- .../v1/attention/test_sparse_mla_backends.py | 102 ++++++- vllm/_custom_ops.py | 25 ++ vllm/config/cache.py | 1 + vllm/config/vllm.py | 6 +- .../layers/attention/mla_attention.py | 19 +- vllm/utils/torch_utils.py | 1 + .../attention/backends/mla/flashmla_sparse.py | 79 ++++- vllm/v1/kv_cache_interface.py | 6 + 17 files changed, 824 insertions(+), 22 deletions(-) create mode 100644 csrc/libtorch_stable/nvfp4_ds_mla_cache.h create mode 100644 csrc/libtorch_stable/nvfp4_ds_mla_cache_kernels.cu diff --git a/CMakeLists.txt b/CMakeLists.txt index bf7a58e572..e24f4327c6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1042,7 +1042,8 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP") "csrc/libtorch_stable/quantization/fp4/nvfp4_blockwise_moe_kernel.cu" "csrc/libtorch_stable/quantization/fp4/mxfp4_experts_quant.cu" "csrc/libtorch_stable/quantization/fp4/mxfp4_blockwise_moe_kernel.cu" - "csrc/libtorch_stable/nvfp4_kv_cache_kernels.cu") + "csrc/libtorch_stable/nvfp4_kv_cache_kernels.cu" + "csrc/libtorch_stable/nvfp4_ds_mla_cache_kernels.cu") if(NOT ${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 12.9) message(STATUS "Building mxfp4_experts_quant unsupported stubs because CUDA compiler version is not >= 12.9 (found ${CMAKE_CUDA_COMPILER_VERSION}).") diff --git a/cmake/external_projects/flashmla.cmake b/cmake/external_projects/flashmla.cmake index d5e2625d38..4cf1e421f0 100644 --- a/cmake/external_projects/flashmla.cmake +++ b/cmake/external_projects/flashmla.cmake @@ -19,7 +19,7 @@ else() FetchContent_Declare( flashmla GIT_REPOSITORY https://github.com/vllm-project/FlashMLA - GIT_TAG 0397728d511c4e3d94ea3a01d8dda8654525a611 + GIT_TAG 6bc49418c5ead572ff0339191ddf3b155749e183 GIT_PROGRESS TRUE CONFIGURE_COMMAND "" BUILD_COMMAND "" @@ -111,6 +111,7 @@ if(FLASH_MLA_ARCHS) # sm100 sparse decode ${flashmla_SOURCE_DIR}/csrc/sm100/decode/head64/instantiations/v32.cu ${flashmla_SOURCE_DIR}/csrc/sm100/decode/head64/instantiations/model1.cu + ${flashmla_SOURCE_DIR}/csrc/sm100/decode/head64/instantiations/v32_nvfp4_fp8rope.cu ${flashmla_SOURCE_DIR}/csrc/sm100/prefill/sparse/fwd_for_small_topk/head128/instantiations/phase1_decode_k512.cu ) diff --git a/csrc/libtorch_stable/cache_kernels.cu b/csrc/libtorch_stable/cache_kernels.cu index 6b8908d2cd..1807a81f4c 100644 --- a/csrc/libtorch_stable/cache_kernels.cu +++ b/csrc/libtorch_stable/cache_kernels.cu @@ -7,6 +7,10 @@ #include "quantization/vectorization_utils.cuh" #include "concat_mla_q.cuh" +#if !defined(USE_ROCM) && defined(ENABLE_NVFP4_SM100) && ENABLE_NVFP4_SM100 + #include "nvfp4_ds_mla_cache.h" +#endif + #ifdef USE_ROCM #include "../quantization/w8a8/fp8/amd/quant_utils.cuh" #else @@ -917,6 +921,7 @@ void concat_and_cache_mla( int pe_dim = k_pe.size(1); int block_size = kv_cache.size(1); + const bool is_nvfp4_ds_mla = kv_cache_dtype == "nvfp4_ds_mla"; if (kv_cache_dtype == "fp8_ds_mla") { STD_TORCH_CHECK(kv_lora_rank == 512, "kv_lora_rank must be 512 for fp8_ds_mla"); @@ -927,6 +932,21 @@ void concat_and_cache_mla( "kv_c.element_size() must be 2 for fp8_ds_mla"); STD_TORCH_CHECK(k_pe.element_size() == 2, "k_pe.element_size() must be 2 for fp8_ds_mla"); + } else if (is_nvfp4_ds_mla) { + constexpr int bytes_per_token = 352; + STD_TORCH_CHECK(kv_lora_rank == 512, "kv_lora_rank must be 512 for ", + kv_cache_dtype); + STD_TORCH_CHECK(pe_dim == 64, "pe_dim must be 64 for ", kv_cache_dtype); + STD_TORCH_CHECK( + kv_cache.size(2) == bytes_per_token / kv_cache.element_size(), + "kv_cache.size(2) must be ", bytes_per_token, " bytes for ", + kv_cache_dtype); + STD_TORCH_CHECK( + kv_c.scalar_type() == torch::headeronly::ScalarType::BFloat16, + "kv_c must be bfloat16 for ", kv_cache_dtype); + STD_TORCH_CHECK( + k_pe.scalar_type() == torch::headeronly::ScalarType::BFloat16, + "k_pe must be bfloat16 for ", kv_cache_dtype); } else { STD_TORCH_CHECK(kv_cache.size(2) == kv_lora_rank + pe_dim); } @@ -950,6 +970,24 @@ void concat_and_cache_mla( dim3 block(96); DISPATCH_BY_KV_CACHE_DTYPE(kv_c.scalar_type(), kv_cache_dtype, CALL_CONCAT_AND_CACHE_DS_MLA); + } else if (is_nvfp4_ds_mla) { + // The kernels live in nvfp4_ds_mla_cache_kernels.cu, which is only built + // when the target list contains an SM100 arch. +#if !defined(USE_ROCM) && defined(ENABLE_NVFP4_SM100) && ENABLE_NVFP4_SM100 + const cudaDeviceProp* props = get_device_prop(); + STD_TORCH_CHECK(props->major == 10, + "nvfp4_ds_mla KV cache requires SM100 (Blackwell); " + "got SM", + props->major, props->minor); + vllm::launch_concat_and_cache_nvfp4_ds_mla( + kv_c.data_ptr(), k_pe.data_ptr(), kv_cache.data_ptr(), + slot_mapping.const_data_ptr(), block_stride, entry_stride, + kv_c_stride, k_pe_stride, block_size, num_tokens, stream); +#else + STD_TORCH_CHECK(false, + "the nvfp4 ds_mla kv-cache format requires a CUDA build " + "targeting SM100 (Blackwell)"); +#endif } else { dim3 grid(num_tokens); dim3 block(std::min(kv_lora_rank, 512)); @@ -1659,6 +1697,72 @@ void cp_gather_and_upconvert_fp8_kv_cache( dst_entry_stride, seq_starts_ptr); } +void cp_gather_and_upconvert_nvfp4_kv_cache( + torch::stable::Tensor const& src_cache, // [NUM_BLOCKS, BLOCK_SIZE, + // 352] + torch::stable::Tensor const& dst, // [TOT_TOKENS, 576] + torch::stable::Tensor const& block_table, // [BATCH, BLOCK_INDICES] + torch::stable::Tensor const& workspace_starts, // [BATCH] + int64_t batch_size) { + // The kernel lives in nvfp4_ds_mla_cache_kernels.cu, which is only built when + // the target list contains an SM100 arch. +#if !defined(USE_ROCM) && defined(ENABLE_NVFP4_SM100) && ENABLE_NVFP4_SM100 + torch::stable::accelerator::DeviceGuard device_guard( + src_cache.get_device_index()); + const cudaStream_t stream = get_current_cuda_stream(); + + const cudaDeviceProp* props = get_device_prop(); + STD_TORCH_CHECK(props->major == 10, + "nvfp4_ds_mla KV cache requires SM100 (Blackwell); " + "got SM", + props->major, props->minor); + + int32_t block_size = src_cache.size(1); + int32_t head_dim = dst.size(1); + + STD_TORCH_CHECK( + block_table.scalar_type() == torch::headeronly::ScalarType::Int, + "block_table must be int32"); + STD_TORCH_CHECK( + workspace_starts.scalar_type() == torch::headeronly::ScalarType::Int, + "workspace_starts must be int32"); + STD_TORCH_CHECK(src_cache.device() == dst.device(), + "src_cache and dst must be on the same device"); + STD_TORCH_CHECK(src_cache.device() == block_table.device(), + "src_cache and block_table must be on the same device"); + STD_TORCH_CHECK(src_cache.device() == workspace_starts.device(), + "src_cache and workspace_starts must be on the same device"); + STD_TORCH_CHECK( + src_cache.scalar_type() == torch::headeronly::ScalarType::Byte, + "src_cache must be uint8 for the nvfp4 ds_mla format, but got ", + src_cache.scalar_type()); + STD_TORCH_CHECK(dst.scalar_type() == torch::headeronly::ScalarType::BFloat16, + "dst must be bfloat16"); + STD_TORCH_CHECK(head_dim == 576, "head_dim must be 576 for MLA"); + STD_TORCH_CHECK(src_cache.size(2) == 352, + "src_cache.size(2) must be 352 for the nvfp4 ds_mla format"); + + int64_t block_table_stride = block_table.stride(0); + int64_t cache_block_stride = src_cache.stride(0); + int64_t cache_entry_stride = src_cache.stride(1); + int64_t dst_entry_stride = dst.stride(0); + + const int total_tokens = dst.size(0); + + vllm::launch_cp_gather_and_upconvert_nvfp4_kv_cache( + src_cache.const_data_ptr(), dst.data_ptr(), + block_table.const_data_ptr(), + workspace_starts.const_data_ptr(), + static_cast(batch_size), block_size, total_tokens, + block_table_stride, cache_block_stride, cache_entry_stride, + dst_entry_stride, stream); +#else + STD_TORCH_CHECK(false, + "the nvfp4 ds_mla kv-cache format requires a CUDA build " + "targeting SM100 (Blackwell)"); +#endif +} + // Macro to dispatch the kernel based on the data type. #define CALL_INDEXER_K_QUANT_AND_CACHE(KV_T, CACHE_T, KV_DTYPE) \ vllm::indexer_k_quant_and_cache_kernel \ diff --git a/csrc/libtorch_stable/nvfp4_ds_mla_cache.h b/csrc/libtorch_stable/nvfp4_ds_mla_cache.h new file mode 100644 index 0000000000..2b5f757846 --- /dev/null +++ b/csrc/libtorch_stable/nvfp4_ds_mla_cache.h @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project +#pragma once + +#include +#include + +// Launchers for the nvfp4_ds_mla KV cache kernels. Those kernels live in +// their own TU (nvfp4_ds_mla_cache_kernels.cu) so that only that TU is built +// for the arch-conditional Blackwell targets their e2m1 conversions require; +// these declarations are what keeps cache_kernels.cu generic. +// +// Only defined when the build has an SM100 target (-DENABLE_NVFP4_SM100=1); +// callers must guard on the same macro. +namespace vllm { + +// concat_and_cache_mla for the nvfp4_ds_mla layout. +// kv_c: bf16 [num_tokens, 512] +// k_pe: bf16 [num_tokens, 64] +// kv_cache: uint8 [num_blocks, block_size, 352] +void launch_concat_and_cache_nvfp4_ds_mla(const void* kv_c, const void* k_pe, + void* kv_cache, + const int64_t* slot_mapping, + int block_stride, int entry_stride, + int kv_c_stride, int k_pe_stride, + int block_size, int num_tokens, + cudaStream_t stream); + +// Gather an nvfp4_ds_mla cache into a bf16 [total_tokens, 576] workspace. +void launch_cp_gather_and_upconvert_nvfp4_kv_cache( + const uint8_t* src_cache, void* dst, const int32_t* block_table, + const int32_t* workspace_starts, int32_t num_reqs, int32_t block_size, + int32_t total_tokens, int64_t block_table_stride, + int64_t cache_block_stride, int64_t cache_entry_stride, + int64_t dst_entry_stride, cudaStream_t stream); + +} // namespace vllm diff --git a/csrc/libtorch_stable/nvfp4_ds_mla_cache_kernels.cu b/csrc/libtorch_stable/nvfp4_ds_mla_cache_kernels.cu new file mode 100644 index 0000000000..c2faf7b760 --- /dev/null +++ b/csrc/libtorch_stable/nvfp4_ds_mla_cache_kernels.cu @@ -0,0 +1,281 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +// nvfp4_ds_mla KV cache writer and gather/upconvert reader. +// +// Separate TU because its e2m1 conversions are only accepted by ptxas for a +// suffixed Blackwell target (sm_100a / sm_100f); only this file is compiled +// with the suffixed arch list. + +#include "nvfp4_ds_mla_cache.h" + +#include "quantization/fp4/nvfp4_utils.cuh" // vllm::fp32_vec16_to_e2m1() + +#include +#include + +#if defined(CUDART_VERSION) && CUDART_VERSION >= 12080 + #include + #define VLLM_HAS_CUDA_FP4 1 +#else + #define VLLM_HAS_CUDA_FP4 0 +#endif + +// __CUDA_ARCH_FAMILY_SPECIFIC__ is defined only for the suffixed targets. The +// non-native paths trap rather than emulate; the host entry points reject this +// format off SM100, so they are unreachable. +#if VLLM_HAS_CUDA_FP4 && defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 1000 && \ + defined(__CUDA_ARCH_FAMILY_SPECIFIC__) + #define VLLM_NVFP4_NATIVE_CVT 1 +#else + #define VLLM_NVFP4_NATIVE_CVT 0 +#endif + +namespace vllm { + +// nvfp4_ds_mla cache format, 352 B/token: +// [0, 256) 512 x e2m1 NoPE, packed 2/byte (low nibble = even element) +// [256, 320) 64 x e4m3 RoPE (unscaled: e4m3's 4 exponent bits span the +// RoPE range unaided) +// [320, 352) 32 x e4m3 NoPE scale factors (one per 16 elements), permuted +// Quantization: sf = e4m3(max(amax_16 / 6, 2^-9)), q = round(x / float(sf)) +// Dequantization: x = float(q) * float(sf) +// +// The scale-factor region is an 8x4 -> 4x8 transpose: element block s -- NoPE +// dims [16s, 16s + 16) -- has its scale at byte kSfNopeOff + nvfp4_sf_byte(s), +// and byte p holds the scale of element block 4 * (p & 7) + (p >> 3). Keep +// nvfp4_sf_byte() in lockstep with its counterpart in FlashMLA +// (csrc/sm100/decode/head64/config.h and tests/quant.py). +constexpr int kSfNopeOff = 256 + 64; + +__host__ __device__ constexpr int nvfp4_sf_byte(int s) { + return 8 * (s & 3) + (s >> 2); +} + +// Thread q's scale groups {4c + q} must land on the contiguous bytes 8q + c. +// This also makes it a permutation of [0, 32), since both sides cover that +// range exactly once. +constexpr bool nvfp4_sf_perm_ok() { + for (int q = 0; q < 4; ++q) { + for (int c = 0; c < 8; ++c) { + if (nvfp4_sf_byte(4 * c + q) != 8 * q + c) { + return false; + } + } + } + return true; +} +static_assert(nvfp4_sf_perm_ok(), + "NVFP4 SF byte order must match FlashMLA's nvfp4_sf_byte()"); + +// `cvt.rn.satfinite.e2m1x2.f32 d, a, b` puts a in d's HIGH nibble and b in its +// LOW nibble, so the odd element of a pair is the first source. +__device__ __forceinline__ uint64_t +nvfp4_pack16_e2m1_rn(const float (&vals)[16], float inv_sf) { +#if VLLM_NVFP4_NATIVE_CVT + float2 pairs[8]; + #pragma unroll + for (int i = 0; i < 8; i++) { + pairs[i] = make_float2(vals[2 * i] * inv_sf, vals[2 * i + 1] * inv_sf); + } + const u32x2 packed = fp32_vec16_to_e2m1(pairs); + return (static_cast(packed.hi) << 32) | packed.lo; +#else + __trap(); + return 0; +#endif +} + +// The e2m1 data and its e4m3 scale both convert to f16 and multiply there; the +// product then goes f16 -> f32 -> bf16, as no f16 -> bf16 instruction exists. +__device__ __forceinline__ void nvfp4_unpack16_e2m1(uint64_t raw, + uint8_t sf_byte, + __nv_bfloat16 (&out)[16]) { +#if VLLM_NVFP4_NATIVE_CVT + const __half2_raw sf2_raw = __nv_cvt_fp8x2_to_halfraw2( + static_cast<__nv_fp8x2_storage_t>(sf_byte * 0x0101u), __NV_E4M3); + const __half2 sf2 = *reinterpret_cast(&sf2_raw); + __nv_bfloat162* out2 = reinterpret_cast<__nv_bfloat162*>(out); + #pragma unroll + for (int i = 0; i < 8; i++) { + // low nibble -> .x, so .x = element 2i, matching the writer. + const __half2_raw v_raw = __nv_cvt_fp4x2_to_halfraw2( + static_cast<__nv_fp4x2_storage_t>(raw >> (8 * i)), __NV_E2M1); + const __half2 v = __hmul2(*reinterpret_cast(&v_raw), sf2); + out2[i] = __float22bfloat162_rn(__half22float2(v)); + } +#else + __trap(); +#endif +} + +// Rounded UP to the next e4m3 value so amax / sf never exceeds the payload +// range: round-to-nearest can land up to 33% low near e4m3's 2^-9 floor, which +// would saturate the tile's largest values. +__device__ __forceinline__ float nvfp4_tile_scale(const float* vals, + float max_val, + uint8_t* sf_out) { + float amax = 0.0f; +#pragma unroll + for (int i = 0; i < 16; i++) { + amax = fmaxf(amax, fabsf(vals[i])); + } + const float sf_f = fmaxf(amax / max_val, 0.001953125f); // 2^-9 + __nv_fp8_e4m3 sf8(sf_f); + uint8_t sf_bits = *reinterpret_cast(&sf8); + if (float(sf8) < sf_f && sf_bits < 0x7E) { + // Bump to the next e4m3 value (bit patterns of positive e4m3 values are + // monotonic; 0x7E = 448 is the max finite value). + sf_bits += 1; + sf8 = *reinterpret_cast(&sf_bits); + } + *sf_out = sf_bits; + return float(sf8); +} + +// One CUDA block (64 threads) per token. +// Warp 0: 32 threads, one 16-element NoPE tile each. +// Warp 1: 32 threads, two RoPE elements each. +__global__ void concat_and_cache_nvfp4_ds_mla_kernel( + const __nv_bfloat16* __restrict__ kv_c, // [num_tokens, 512] + const __nv_bfloat16* __restrict__ k_pe, // [num_tokens, 64] + uint8_t* __restrict__ kv_cache, // [num_blocks, block_size, 352] + const int64_t* __restrict__ slot_mapping, // [num_tokens] + const int block_stride, // + const int entry_stride, // + const int kv_c_stride, // + const int k_pe_stride, // + const int block_size) { + const int64_t token_idx = blockIdx.x; + const int64_t slot_idx = slot_mapping[token_idx]; + // NOTE: slot_idx can be -1 if the token is padded + if (slot_idx < 0) { + return; + } + const int64_t block_idx = slot_idx / block_size; + const int64_t block_offset = slot_idx % block_size; + uint8_t* token_ptr = + kv_cache + block_idx * block_stride + block_offset * entry_stride; + + if (threadIdx.x < 32) { + const int tile = threadIdx.x; + const __nv_bfloat16* src = kv_c + token_idx * kv_c_stride + tile * 16; + float vals[16]; +#pragma unroll + for (int i = 0; i < 16; i++) { + vals[i] = __bfloat162float(src[i]); + } + // `tile` is the element-block index; its scale byte is permuted. + const float sf = nvfp4_tile_scale( + vals, 6.0f, token_ptr + kSfNopeOff + nvfp4_sf_byte(tile)); + const float inv_sf = 1.0f / sf; + *reinterpret_cast(token_ptr + tile * 8) = + nvfp4_pack16_e2m1_rn(vals, inv_sf); + } else { + // Plain e4m3, unscaled. All 32 lanes take 2 elements each, matching the + // reader and concat_and_cache_ds_mla_kernel. + const int lane = threadIdx.x - 32; + const __nv_bfloat16* src = k_pe + token_idx * k_pe_stride + lane * 2; + uint8_t q[2]; +#pragma unroll + for (int i = 0; i < 2; i++) { + const __nv_fp8_e4m3 v8(__bfloat162float(src[i])); + q[i] = *reinterpret_cast(&v8); + } + *reinterpret_cast(token_ptr + 256 + lane * 2) = + *reinterpret_cast(q); + } +} + +// One warp per token (mirrors cp_gather_and_upconvert_fp8_kv_cache). +__global__ void cp_gather_and_upconvert_nvfp4_kv_cache_kernel( + const uint8_t* __restrict__ src_cache, // [NUM_BLOCKS, BLOCK_SIZE, 352] + __nv_bfloat16* __restrict__ dst, // [total_tokens, 576] + const int32_t* __restrict__ block_table, // [num_reqs, BLOCK_INDICES] + const int32_t* __restrict__ workspace_starts, // [num_reqs] + const int32_t num_reqs, const int32_t block_size, + const int32_t total_tokens, const int64_t block_table_stride, + const int64_t cache_block_stride, const int64_t cache_entry_stride, + const int64_t dst_entry_stride) { + const int flat_warp_id = (blockIdx.x * blockDim.x + threadIdx.x) >> 5; + if (flat_warp_id >= total_tokens) return; + const int lane_id = threadIdx.x & 31; + + // Binary search to find which request owns this output token + int lo = 0, hi = num_reqs - 1; + while (lo < hi) { + int mid = (lo + hi + 1) >> 1; + if (workspace_starts[mid] <= flat_warp_id) + lo = mid; + else + hi = mid - 1; + } + const int req_id = lo; + + const int out_token_id = flat_warp_id; + const int token_offset = out_token_id - workspace_starts[req_id]; + const int cache_block_idx = token_offset / block_size; + const int offset_in_block = token_offset % block_size; + const int physical_block = + block_table[req_id * block_table_stride + cache_block_idx]; + + const uint8_t* token_ptr = src_cache + physical_block * cache_block_stride + + offset_in_block * cache_entry_stride; + __nv_bfloat16* dst_ptr = dst + out_token_id * dst_entry_stride; + + // `lane_id` is the element-block index, so its scale byte is permuted. + { + const uint64_t raw = + *reinterpret_cast(token_ptr + lane_id * 8); + const uint8_t sf_byte = token_ptr[kSfNopeOff + nvfp4_sf_byte(lane_id)]; + __nv_bfloat16 out[16]; + nvfp4_unpack16_e2m1(raw, sf_byte, out); + int4* nope_dst = reinterpret_cast(dst_ptr) + lane_id * 2; + nope_dst[0] = *reinterpret_cast(&out[0]); + nope_dst[1] = *reinterpret_cast(&out[8]); + } + + // RoPE is unscaled e4m3. + { + const __nv_fp8_e4m3* raw = + reinterpret_cast(token_ptr + 256 + lane_id * 2); + const __nv_bfloat16 out[2] = {__float2bfloat16_rn(float(raw[0])), + __float2bfloat16_rn(float(raw[1]))}; + *reinterpret_cast(dst_ptr + 512 + lane_id * 2) = + *reinterpret_cast(out); + } +} + +void launch_concat_and_cache_nvfp4_ds_mla(const void* kv_c, const void* k_pe, + void* kv_cache, + const int64_t* slot_mapping, + int block_stride, int entry_stride, + int kv_c_stride, int k_pe_stride, + int block_size, int num_tokens, + cudaStream_t stream) { + dim3 grid(num_tokens); + dim3 block(64); // warp 0: 32 NoPE tiles; warp 1: 4 RoPE tiles + concat_and_cache_nvfp4_ds_mla_kernel<<>>( + reinterpret_cast(kv_c), + reinterpret_cast(k_pe), + reinterpret_cast(kv_cache), slot_mapping, block_stride, + entry_stride, kv_c_stride, k_pe_stride, block_size); +} + +void launch_cp_gather_and_upconvert_nvfp4_kv_cache( + const uint8_t* src_cache, void* dst, const int32_t* block_table, + const int32_t* workspace_starts, int32_t num_reqs, int32_t block_size, + int32_t total_tokens, int64_t block_table_stride, + int64_t cache_block_stride, int64_t cache_entry_stride, + int64_t dst_entry_stride, cudaStream_t stream) { + constexpr int warps_per_block = 8; + const int grid_size = (total_tokens + warps_per_block - 1) / warps_per_block; + const int block_size_threads = warps_per_block * 32; // 256 threads + cp_gather_and_upconvert_nvfp4_kv_cache_kernel<<>>( + src_cache, reinterpret_cast<__nv_bfloat16*>(dst), block_table, + workspace_starts, num_reqs, block_size, total_tokens, block_table_stride, + cache_block_stride, cache_entry_stride, dst_entry_stride); +} + +} // namespace vllm diff --git a/csrc/libtorch_stable/ops.h b/csrc/libtorch_stable/ops.h index 08e5fe3226..8ac50538db 100644 --- a/csrc/libtorch_stable/ops.h +++ b/csrc/libtorch_stable/ops.h @@ -672,6 +672,15 @@ void cp_gather_and_upconvert_fp8_kv_cache( int64_t batch_size, std::optional seq_starts = std::nullopt); +// Gather and upconvert an nvfp4_ds_mla KV cache to a BF16 workspace +void cp_gather_and_upconvert_nvfp4_kv_cache( + torch::stable::Tensor const& src_cache, // [NUM_BLOCKS, BLOCK_SIZE, + // 352] + torch::stable::Tensor const& dst, // [TOT_TOKENS, 576] + torch::stable::Tensor const& block_table, // [BATCH, BLOCK_INDICES] + torch::stable::Tensor const& workspace_starts, // [BATCH] + int64_t batch_size); + // Indexer K quantization and cache function void indexer_k_quant_and_cache( torch::stable::Tensor& k, // [num_tokens, head_dim] diff --git a/csrc/libtorch_stable/torch_bindings.cpp b/csrc/libtorch_stable/torch_bindings.cpp index 2247281e72..505ec6588a 100644 --- a/csrc/libtorch_stable/torch_bindings.cpp +++ b/csrc/libtorch_stable/torch_bindings.cpp @@ -996,6 +996,10 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C_cache_ops, ops) { "Tensor block_table, Tensor workspace_starts, int batch_size, Tensor? " "seq_starts) -> ()"); + ops.def( + "cp_gather_and_upconvert_nvfp4_kv_cache(Tensor src_cache, Tensor! dst, " + "Tensor block_table, Tensor workspace_starts, int batch_size) -> ()"); + ops.def( "indexer_k_quant_and_cache(Tensor k, Tensor! kv_cache, Tensor " "slot_mapping, " @@ -1066,6 +1070,8 @@ STABLE_TORCH_LIBRARY_IMPL(_C_cache_ops, CUDA, ops) { ops.impl("cp_gather_cache", TORCH_BOX(&cp_gather_cache)); ops.impl("cp_gather_and_upconvert_fp8_kv_cache", TORCH_BOX(&cp_gather_and_upconvert_fp8_kv_cache)); + ops.impl("cp_gather_and_upconvert_nvfp4_kv_cache", + TORCH_BOX(&cp_gather_and_upconvert_nvfp4_kv_cache)); ops.impl("indexer_k_quant_and_cache", TORCH_BOX(&indexer_k_quant_and_cache)); ops.impl("concat_mla_q", TORCH_BOX(&concat_mla_q)); ops.impl("cp_gather_indexer_k_quant_cache", diff --git a/tests/kernels/attention/test_cache.py b/tests/kernels/attention/test_cache.py index 3e3bfe9dac..61772dbaa0 100644 --- a/tests/kernels/attention/test_cache.py +++ b/tests/kernels/attention/test_cache.py @@ -907,6 +907,157 @@ def test_concat_and_cache_ds_mla( torch.testing.assert_close(kv_rope, ref_rope, atol=0.001, rtol=0.1) +# Bytes per token for the nvfp4_ds_mla cache layout (see flashmla_sparse.py). +NVFP4_DS_MLA_ENTRY_SIZE = 352 + +# e2m1 magnitude table; the sign lives in bit 3 of each 4-bit code. +E2M1_VALUES = [0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0] + + +def _unpack_e2m1(packed: torch.Tensor) -> torch.Tensor: + """Unpack uint8 bytes holding two e2m1 codes each into float32 values. + + The low nibble holds the even element, the high nibble the odd one. + """ + low = packed & 0x0F + high = packed >> 4 + codes = torch.stack([low, high], dim=-1).reshape(-1).long() + table = torch.tensor(E2M1_VALUES, dtype=torch.float32, device=packed.device) + magnitude = table[codes & 0x7] + sign = torch.where((codes & 0x8) != 0, -1.0, 1.0) + return sign * magnitude + + +def _ref_nvfp4_sf_bytes(vals: torch.Tensor, divisor: float) -> torch.Tensor: + """Reference e4m3 scale-factor bytes: e4m3(max(amax_per_16 / divisor, 2^-9)). + + In element-block order (byte s scales elements [16s, 16s + 16)). + """ + tiles = vals.to(torch.float32).reshape(-1, 16) + sf = torch.clamp(tiles.abs().amax(dim=1) / divisor, min=2.0**-9) + return sf.to(torch.float8_e4m3fn).view(torch.uint8) + + +def _nvfp4_sf_element_order(wire_sf_bytes: torch.Tensor) -> torch.Tensor: + """Stored NVFP4 scale-factor byte order -> element-block order. + + The 32 NoPE scale-factor bytes of an nvfp4_ds_mla entry are stored + permuted (an 8x4 -> 4x8 transpose): the scale for element block s lives at + byte 8 * (s & 3) + (s >> 2), so that the 8 scales one FlashMLA dequant + thread needs are contiguous. This undoes that permutation. + """ + num_sf = wire_sf_bytes.shape[-1] + return wire_sf_bytes.unflatten(-1, (4, num_sf // 4)).transpose(-1, -2).flatten(-2) + + +@pytest.mark.parametrize("kv_lora_rank", KV_LORA_RANKS) +@pytest.mark.parametrize("qk_rope_head_dim", QK_ROPE_HEAD_DIMS) +@pytest.mark.parametrize("num_tokens", NUM_TOKENS_MLA) +@pytest.mark.parametrize("block_size", BLOCK_SIZES_MLA) +@pytest.mark.parametrize("num_blocks", NUM_BLOCKS_MLA) +@pytest.mark.parametrize("seed", SEEDS) +@pytest.mark.parametrize("device", CUDA_DEVICES) +@torch.inference_mode() +def test_concat_and_cache_nvfp4_ds_mla( + kv_lora_rank: int, + qk_rope_head_dim: int, + num_tokens: int, + block_size: int, + num_blocks: int, + seed: int, + device: str, +) -> None: + if current_platform.is_rocm(): + pytest.skip("concat_and_cache_mla doesn't support NVFP4 DS-MLA on ROCm") + device_capability = current_platform.get_device_capability() + if device_capability is None or device_capability.major != 10: + pytest.skip("The NVFP4 DS-MLA kv-cache dtype requires SM 10.x") + if kv_lora_rank != 512: + pytest.skip("The NVFP4 DS-MLA layout requires kv_lora_rank == 512") + dtype = torch.bfloat16 + set_random_seed(seed) + torch.set_default_device(device) + torch.accelerator.set_device_index(device) + + kv_cache_dtype = "nvfp4_ds_mla" + entry_size = NVFP4_DS_MLA_ENTRY_SIZE + + total_slots = num_blocks * block_size + slot_mapping_lst = random.sample(range(total_slots), num_tokens) + slot_mapping = torch.tensor(slot_mapping_lst, dtype=torch.long, device=device) + + kv_c = torch.randn(num_tokens, kv_lora_rank, dtype=dtype, device=device) + k_pe = torch.randn(num_tokens, qk_rope_head_dim, dtype=dtype, device=device) + + scale = torch.tensor(1.0, dtype=torch.float32, device=device) + kv_cache = _create_mla_cache( + num_blocks, + block_size, + entry_size, + dtype=torch.uint8, + kv_cache_dtype=kv_cache_dtype, + device=device, + ) + + opcheck( + torch.ops._C_cache_ops.concat_and_cache_mla, + (kv_c, k_pe, kv_cache, slot_mapping, kv_cache_dtype, scale), + test_utils=DEFAULT_OPCHECK_TEST_UTILS, + ) + + ops.concat_and_cache_mla(kv_c, k_pe, kv_cache, slot_mapping, kv_cache_dtype, scale) + + # Byte layout of a single cache entry: + # [0, 256) 512 e2m1 NoPE values packed 2/byte + # [256, 320) 64 unscaled e4m3 RoPE values + # [nope_sf_off, 352) 32 e4m3 NoPE SFs (one per 16 elements), stored + # permuted (see _nvfp4_sf_element_order) + nope_bytes = kv_lora_rank // 2 + rope_bytes = qk_rope_head_dim + nope_sf_off = nope_bytes + rope_bytes + num_nope_sf = kv_lora_rank // 16 + assert nope_sf_off + num_nope_sf == entry_size + + for i in range(num_tokens): + slot = slot_mapping[i].item() + block_idx = slot // block_size + block_offset = slot % block_size + entry = kv_cache[block_idx, block_offset] + + kv_c_ref = kv_c[i].to(torch.float32) + k_pe_ref = k_pe[i].to(torch.float32) + + # Scale-factor bytes must match e4m3(max(amax_per_16 / divisor, 2^-9)) + # computed in python, allowing a 1-ulp e4m3 difference (all SFs are + # positive, so e4m3 bit patterns are monotonic and adjacent codes + # differ by 1). + nope_sf_bytes = _nvfp4_sf_element_order( + entry[nope_sf_off : nope_sf_off + num_nope_sf] + ) + ref_nope_sf_bytes = _ref_nvfp4_sf_bytes(kv_c_ref, 6.0) + assert ( + (nope_sf_bytes.to(torch.int16) - ref_nope_sf_bytes.to(torch.int16)).abs() + <= 1 + ).all() + + # Dequantize with the *stored* scale factors: x = float(code) * float(sf). + nope_sf = nope_sf_bytes.view(torch.float8_e4m3fn).to(torch.float32) + nope_sf_per_elem = nope_sf.repeat_interleave(16) + nope_vals = _unpack_e2m1(entry[:nope_bytes]) * nope_sf_per_elem + # e2m1 error bound: the widest grid spacing is 2 (between codes 4 and + # 6), so round-to-nearest is off by at most 1.0*sf; the e4m3 rounding + # of the SF itself adds at most ~2^-4 relative, so 1.5*sf is safe for + # either encode convention (quantized or unquantized SF). + assert ((nope_vals - kv_c_ref).abs() <= 1.5 * nope_sf_per_elem).all() + + # Plain unscaled e4m3: ~2^-4 relative rounding, with a small absolute + # floor for values in the subnormal range. + rope_payload = entry[nope_bytes:nope_sf_off] + rope_vals = rope_payload.view(torch.float8_e4m3fn).to(torch.float32) + rope_tol = k_pe_ref.abs() * 2.0**-3 + 2.0**-9 + assert ((rope_vals - k_pe_ref).abs() <= rope_tol).all() + + @pytest.mark.parametrize("kv_lora_rank", KV_LORA_RANKS) @pytest.mark.parametrize("qk_rope_head_dim", QK_ROPE_HEAD_DIMS) @pytest.mark.parametrize("block_size", BLOCK_SIZES_MLA) diff --git a/tests/v1/attention/test_mla_backends.py b/tests/v1/attention/test_mla_backends.py index c096593b4f..edc6cbaded 100644 --- a/tests/v1/attention/test_mla_backends.py +++ b/tests/v1/attention/test_mla_backends.py @@ -505,7 +505,10 @@ def create_and_prepopulate_kv_cache( block_table = common_attn_metadata.block_table_tensor slot_mapping = common_attn_metadata.slot_mapping - fp8_attention = kv_cache_dtype and kv_cache_dtype.startswith("fp8") + use_nvfp4_ds_mla = kv_cache_dtype == "nvfp4_ds_mla" + fp8_attention = ( + bool(kv_cache_dtype and kv_cache_dtype.startswith("fp8")) or use_nvfp4_ds_mla + ) use_fp8_ds_mla = kv_cache_dtype == "fp8_ds_mla" if fp8_attention: @@ -515,6 +518,14 @@ def create_and_prepopulate_kv_cache( # 4 * 4: 4 float32 scale values for 128-element tiles # 2 * rope_dim: 16-bit RoPE values kv_entry_size = kv_lora_rank + 4 * 4 + 2 * rope_dim + elif use_nvfp4_ds_mla: + kv_lora_rank = kv_c_contexts[0].shape[-1] + rope_dim = k_pe_contexts[0].shape[-1] + # e2m1-packed NoPE + unscaled e4m3 rope + per-16 e4m3 NoPE SFs, + # padded to 16B + kv_entry_size = ( + cdiv(kv_lora_rank // 2 + rope_dim + kv_lora_rank // 16, 16) * 16 + ) else: kv_entry_size = head_size diff --git a/tests/v1/attention/test_sparse_mla_backends.py b/tests/v1/attention/test_sparse_mla_backends.py index cb1c6faaf7..31192f3e76 100644 --- a/tests/v1/attention/test_sparse_mla_backends.py +++ b/tests/v1/attention/test_sparse_mla_backends.py @@ -40,6 +40,9 @@ if not current_platform.is_cuda(): allow_module_level=True, ) +from vllm.model_executor.layers.attention.mla_attention import ( + _canonicalize_sparse_mla_kv_cache_dtype, +) from vllm.utils.math_utils import cdiv from vllm.v1.attention.backends.mla.flashinfer_mla_sparse import ( FlashInferMLASparseTRTLLMBackend, @@ -184,13 +187,89 @@ def _quantize_dequantize_fp8_ds_mla( return dequant_kv_c, dequant_k_pe +_E2M1_TABLE = [0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0] + + +def _quantize_dequantize_nvfp4_ds_mla( + kv_c: torch.Tensor, + k_pe: torch.Tensor, + block_size: int, + kv_cache_dtype: str, + scale: torch.Tensor, +) -> tuple[torch.Tensor, torch.Tensor]: + """Round-trip kv_c/k_pe through the nvfp4_ds_mla cache layout. + + Layout (per token, uint8): [e2m1-packed NoPE | unscaled e4m3 RoPE | + per-16 e4m3 NoPE SFs]. The SF bytes are stored permuted (an 8x4 -> 4x8 + transpose: the scale for element block s lives at byte 8 * (s & 3) + + (s >> 2)) so that the 8 scales one FlashMLA dequant thread needs are + contiguous. Dequant: x = value * float(sf), which is exact in bf16, so + this python dequant matches the kernels bit-for-bit. + """ + if kv_c.numel() == 0: + return kv_c.clone(), k_pe.clone() + + kv_lora_rank = kv_c.shape[-1] + rope_dim = k_pe.shape[-1] + num_tokens = kv_c.shape[0] + num_blocks = max(1, math.ceil(num_tokens / block_size)) + entry_size = ( + math.ceil((kv_lora_rank // 2 + rope_dim + kv_lora_rank // 16) / 16) * 16 + ) + sf_nope_off = kv_lora_rank // 2 + rope_dim + + tmp_cache = torch.zeros( + num_blocks, block_size, entry_size, dtype=torch.uint8, device=kv_c.device + ) + slot_mapping = torch.arange(num_tokens, dtype=torch.long, device=kv_c.device) + ops.concat_and_cache_mla( + kv_c, k_pe, tmp_cache, slot_mapping, kv_cache_dtype=kv_cache_dtype, scale=scale + ) + + tokens = tmp_cache.view(-1, entry_size)[:num_tokens] + table = torch.tensor( + _E2M1_TABLE + [-v for v in _E2M1_TABLE], + dtype=torch.float32, + device=kv_c.device, + ) + + def unpack_e2m1(packed: torch.Tensor) -> torch.Tensor: + lo = (packed & 0xF).long() + hi = (packed >> 4).long() + return table[torch.stack([lo, hi], dim=-1).flatten(-2)] + + nope_vals = unpack_e2m1(tokens[:, : kv_lora_rank // 2]) + num_nope_sf = kv_lora_rank // 16 + nope_sf = ( + # undo the on-wire SF permutation -> element-block order + tokens[:, sf_nope_off : sf_nope_off + num_nope_sf] + .unflatten(-1, (4, num_nope_sf // 4)) + .transpose(-1, -2) + .flatten(-2) + .view(torch.float8_e4m3fn) + .float() + ) + dequant_kv_c = ( + (nope_vals.unflatten(-1, (-1, 16)) * nope_sf.unsqueeze(-1)).flatten(-2) + ).to(kv_c.dtype) + + # RoPE: plain e4m3, no scale factor + rope_raw = tokens[:, kv_lora_rank // 2 : sf_nope_off] + dequant_k_pe = rope_raw.view(torch.float8_e4m3fn).to(k_pe.dtype) + + return dequant_kv_c, dequant_k_pe + + @pytest.mark.parametrize( "backend_cls", [FlashMLASparseBackend, FlashInferMLASparseTRTLLMBackend], ids=["FlashMLA", "FlashInferTRTLLM"], ) @pytest.mark.parametrize("batch_name", list(SPARSE_BACKEND_BATCH_SPECS.keys())) -@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8", "fp8_ds_mla"]) +@pytest.mark.parametrize( + "kv_cache_dtype", + ["auto", "fp8", "fp8_ds_mla", "nvfp4_ds_mla"], +) @pytest.mark.parametrize("tensor_parallel_size", [1, 2, 4]) @pytest.mark.parametrize("block_size", [32, 64]) @pytest.mark.parametrize(("q_scale", "k_scale"), [(1.0, 1.0), (2.0, 3.0)]) @@ -219,6 +298,17 @@ def test_sparse_backend_decode_correctness( "fp8_ds_mla kv-cache dtype" ) + if kv_cache_dtype == "nvfp4_ds_mla": + # Unlike "fp8" (an alias canonicalized to "fp8_ds_mla" above), the + # NVFP4 DS-MLA dtype must reach the backend unchanged. + assert ( + _canonicalize_sparse_mla_kv_cache_dtype(backend_cls, kv_cache_dtype) + == kv_cache_dtype + ) + device_capability = current_platform.get_device_capability() + if device_capability is None or device_capability.major != 10: + pytest.skip("The NVFP4 DS-MLA kv-cache dtype requires SM 10.x") + supported_block_sizes = backend_cls.get_supported_kernel_block_sizes() if block_size not in supported_block_sizes: pytest.skip( @@ -238,6 +328,7 @@ def test_sparse_backend_decode_correctness( batch_spec = SPARSE_BACKEND_BATCH_SPECS[batch_name] use_fp8_ds_mla_quantization = kv_cache_dtype == "fp8_ds_mla" + use_nvfp4_ds_mla_quantization = kv_cache_dtype == "nvfp4_ds_mla" device = torch.device(DEVICE_TYPE) dtype = torch.bfloat16 @@ -381,6 +472,15 @@ def test_sparse_backend_decode_correctness( simulate_sm100_e8m0_scales=is_sm100, ) k_pe_full = k_pe_squeezed.unsqueeze(1) + elif use_nvfp4_ds_mla_quantization: + kv_c_full, k_pe_squeezed = _quantize_dequantize_nvfp4_ds_mla( + kv_c_full, + k_pe_full.squeeze(1), + block_size=block_size, + kv_cache_dtype=kv_cache_dtype, + scale=kv_cache_scale, + ) + k_pe_full = k_pe_squeezed.unsqueeze(1) q_nope, q_pe = q_c.split([qk_nope_head_dim, qk_rope_head_dim], dim=-1) ql_nope = torch.einsum("qnh,lnh->qnl", q_nope, W_UK) diff --git a/vllm/_custom_ops.py b/vllm/_custom_ops.py index 080893d648..b01489a447 100644 --- a/vllm/_custom_ops.py +++ b/vllm/_custom_ops.py @@ -3082,6 +3082,31 @@ def cp_gather_and_upconvert_fp8_kv_cache( ) +def cp_gather_and_upconvert_nvfp4_kv_cache( + src_cache: torch.Tensor, + dst: torch.Tensor, + block_table: torch.Tensor, + workspace_starts: torch.Tensor, + batch_size: int, +) -> None: + """Gather and upconvert an nvfp4_ds_mla KV cache to a BF16 workspace. + + Args: + src_cache: NVFP4 KV cache [num_blocks, block_size, 352] (uint8) + dst: BF16 output workspace [total_tokens, 576] + block_table: Block indices [num_reqs, max_blocks] + workspace_starts: Workspace start offsets [num_reqs] + batch_size: Number of requests + """ + torch.ops._C_cache_ops.cp_gather_and_upconvert_nvfp4_kv_cache( + src_cache, + dst, + block_table, + workspace_starts, + batch_size, + ) + + def concat_mla_q( ql_nope: torch.Tensor, q_pe: torch.Tensor, diff --git a/vllm/config/cache.py b/vllm/config/cache.py index dc4e6faf3c..3a853000c3 100644 --- a/vllm/config/cache.py +++ b/vllm/config/cache.py @@ -45,6 +45,7 @@ CacheDType = Literal[ "fp8_e5m2", "fp8_inc", "fp8_ds_mla", + "nvfp4_ds_mla", "turboquant_k8v4", "turboquant_4bit_nc", "turboquant_k3v4_nc", diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index 033267baf6..3fcbb83685 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -2821,14 +2821,18 @@ class VllmConfig: def validate_nvfp4_kv_cache_with_mla(self) -> "VllmConfig": if self.model_config is None: return self + # The ds_mla layouts are MLA-only by construction; the plain nvfp4 + # layout (head_size//2 + head_size//16) does not apply to MLA. if ( self.cache_config.cache_dtype.startswith("nvfp4") + and not self.cache_config.cache_dtype.endswith("_ds_mla") and self.model_config.use_mla ): raise ValueError( "nvfp4 KV cache is not supported with MLA (Multi-head Latent " "Attention) backends. Please use a different --kv-cache-dtype " - "(e.g., 'fp8' or 'auto') for MLA models such as DeepSeek." + "(e.g., 'fp8', 'auto', or 'nvfp4_ds_mla' with a sparse MLA " + "backend) for MLA models such as DeepSeek." ) return self diff --git a/vllm/model_executor/layers/attention/mla_attention.py b/vllm/model_executor/layers/attention/mla_attention.py index 9f06b64177..479da9721c 100644 --- a/vllm/model_executor/layers/attention/mla_attention.py +++ b/vllm/model_executor/layers/attention/mla_attention.py @@ -352,6 +352,10 @@ def _canonicalize_sparse_mla_kv_cache_dtype( ) -> CacheDType: backend_name = attn_backend.get_name() if backend_name == "FLASHMLA_SPARSE" and is_quantized_kv_cache(kv_cache_dtype): + # The NVFP4 DS-MLA format is used as-is; any other quantized dtype + # (fp8, fp8_e4m3, ...) is served via the fp8_ds_mla format. + if kv_cache_dtype == "nvfp4_ds_mla": + return kv_cache_dtype return "fp8_ds_mla" if backend_name == "FLASHINFER_MLA_SPARSE_SM120" and kv_cache_dtype in ( "auto", @@ -861,7 +865,11 @@ class MLAAttention(nn.Module, AttentionLayerBase): k_c_normed = k_c_normed[:num_actual_toks, ...] k_pe = k_pe[:num_actual_toks, ...] - if fp8_attention and self.kv_cache_dtype != "fp8_ds_mla": + if fp8_attention and self.kv_cache_dtype not in ( + # Opaque per-token byte formats stay as raw uint8 + "fp8_ds_mla", + "nvfp4_ds_mla", + ): kv_cache = kv_cache.view(current_platform.fp8_dtype()) assert ( @@ -1262,9 +1270,12 @@ class MLAAttention(nn.Module, AttentionLayerBase): dtype=kv_cache_dtype, cache_dtype_str=self.kv_cache_dtype, kv_quant_mode=get_kv_quant_mode(self.kv_cache_dtype), - # fp8_ds_mla: 656-byte custom layout (kv_lora_rank=512 + - # qk_rope_head_dim=64, head_size=576). See flashmla_sparse.py. - state_content_bytes=656 if self.kv_cache_dtype == "fp8_ds_mla" else None, + # ds_mla layouts pack NoPE + RoPE + scales into one opaque per-token + # blob, so the size is not derivable from head_size. + # See flashmla_sparse.py. + state_content_bytes={"fp8_ds_mla": 656, "nvfp4_ds_mla": 352}.get( + self.kv_cache_dtype + ), ) if self.sliding_window is not None: return SlidingWindowMLASpec( diff --git a/vllm/utils/torch_utils.py b/vllm/utils/torch_utils.py index faee4305c6..2d428e46c8 100644 --- a/vllm/utils/torch_utils.py +++ b/vllm/utils/torch_utils.py @@ -45,6 +45,7 @@ STR_DTYPE_TO_TORCH_DTYPE = { "fp8_per_token_head": torch.uint8, "fp8_inc": torch.float8_e4m3fn, "fp8_ds_mla": torch.uint8, + "nvfp4_ds_mla": torch.uint8, "turboquant_k8v4": torch.uint8, "turboquant_4bit_nc": torch.uint8, "turboquant_k3v4_nc": torch.uint8, diff --git a/vllm/v1/attention/backends/mla/flashmla_sparse.py b/vllm/v1/attention/backends/mla/flashmla_sparse.py index e134cd1754..d8d8246423 100644 --- a/vllm/v1/attention/backends/mla/flashmla_sparse.py +++ b/vllm/v1/attention/backends/mla/flashmla_sparse.py @@ -84,8 +84,28 @@ Bytes, structured as: - **Last 8 bytes:** Scale factors, containing 7 `ue8m0` values + 1B pad. The first `ue8m0` is the scale for the first 64 `float8_e4m3` values, the second for the next 64, and so on. + +In the "nvfp4_ds_mla" format (SM100 only, DeepSeek V3.2 geometry), each +token's KV cache is 352 Bytes, structured as: +- **First 256 bytes:** 512 `e2m1` NoPE values packed 2/byte (low nibble = + even element). +- **Next 64 bytes:** 64 `float8_e4m3` RoPE values. These carry no scale + factor: `e4m3`'s 4 exponent bits span the RoPE magnitude range unaided. +- **Last 32 bytes:** 32 `float8_e4m3` NoPE scale factors, one per 16 + elements, stored permuted (an 8x4 -> 4x8 transpose: the scale for element + block `s` lives at byte `8 * (s & 3) + (s >> 2)`) so that the 8 scales one + FlashMLA dequant thread needs are contiguous. See the layout comment in + `csrc/libtorch_stable/cache_kernels.cu`. + """ +# Quantized DS-MLA cache formats served by the FP8/NVFP4 sparse decode kernel +# path (as opposed to the plain bf16 cache). FlashMLA infers which of these the +# cache holds from its bytes-per-token, so nothing else needs to be passed down. +QUANTIZED_DS_MLA_CACHE_FORMATS: frozenset[str] = frozenset( + {"fp8_ds_mla", "nvfp4_ds_mla"} +) + class FlashMLASparseBackend(AttentionBackend): supported_dtypes: ClassVar[list[torch.dtype]] = [torch.bfloat16] @@ -94,6 +114,7 @@ class FlashMLASparseBackend(AttentionBackend): "bfloat16", "fp8_ds_mla", "fp8", # alias for fp8_ds_mla + "nvfp4_ds_mla", # NVFP4 NoPE + FP8 RoPE (SM100 only) ] @staticmethod @@ -129,6 +150,26 @@ class FlashMLASparseBackend(AttentionBackend): def supports_compute_capability(cls, capability: DeviceCapability) -> bool: return capability.major in [9, 10] + @classmethod + def supports_combination( + cls, + head_size: int, + dtype: torch.dtype, + kv_cache_dtype: "CacheDType | None", + block_size: int | None, + use_mla: bool, + has_sink: bool, + use_sparse: bool, + use_mm_prefix: bool, + device_capability: DeviceCapability, + ) -> str | None: + if kv_cache_dtype == "nvfp4_ds_mla" and device_capability.major != 10: + return ( + f"FLASHMLA_SPARSE only supports the {kv_cache_dtype} kv-cache " + "dtype on SM100 (Blackwell)" + ) + return None + @dataclass class FlashMLASparseMetadata(AttentionMetadata): @@ -258,7 +299,9 @@ class FlashMLASparseMetadataBuilder( FlashMLASparseImpl._compute_fp8_decode_padded_heads(self.num_heads) ) - self.use_fp8_kv_cache = cache_config.cache_dtype == "fp8_ds_mla" + self.use_fp8_kv_cache = ( + cache_config.cache_dtype in QUANTIZED_DS_MLA_CACHE_FORMATS + ) max_num_seqs = vllm_config.scheduler_config.max_num_seqs # Shape: [max_num_seqs], all elements = topk_tokens (constant for full-CG) self.topk_tokens_tensor = torch.full( @@ -596,9 +639,10 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]): ) q_concat_shape = (max_tokens, q_concat_heads, head_size) if is_quantized_kv_cache(kv_cache_dtype): - assert kv_cache_dtype == "fp8_ds_mla", ( - "FlashMLA Sparse Attention backend fp8 only supports " - "fp8_ds_mla kv-cache dtype" + assert kv_cache_dtype in QUANTIZED_DS_MLA_CACHE_FORMATS, ( + "FlashMLA Sparse Attention backend only supports the " + f"{sorted(QUANTIZED_DS_MLA_CACHE_FORMATS)} quantized kv-cache " + f"dtypes, got {kv_cache_dtype}" ) if self.need_to_return_lse_for_decode and not is_quantized_kv_cache( @@ -609,7 +653,7 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]): "the bf16 sparse path is not supported under DCP." ) - if kv_cache_dtype == "fp8_ds_mla": + if kv_cache_dtype in QUANTIZED_DS_MLA_CACHE_FORMATS: # Reserve workspace during initialization assert vllm_config is not None and vllm_config.model_config is not None prefill_workspace_size = get_prefill_workspace_size( @@ -749,13 +793,22 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]): assert fp8_metadata.prefill is not None for chunk in fp8_metadata.prefill.chunks: chunk_workspace = self.prefill_bf16_workspace[: chunk.chunk_tot_seqlen] - ops.cp_gather_and_upconvert_fp8_kv_cache( - kv_c_and_k_pe_cache, - chunk_workspace, - chunk.block_table, - chunk.workspace_starts, - len(chunk.block_table), - ) + if self.kv_cache_dtype == "fp8_ds_mla": + ops.cp_gather_and_upconvert_fp8_kv_cache( + kv_c_and_k_pe_cache, + chunk_workspace, + chunk.block_table, + chunk.workspace_starts, + len(chunk.block_table), + ) + else: + ops.cp_gather_and_upconvert_nvfp4_kv_cache( + kv_c_and_k_pe_cache.view(torch.uint8), + chunk_workspace, + chunk.block_table, + chunk.workspace_starts, + len(chunk.block_table), + ) chunk_q = q[chunk.tokens_slice] chunk_topk_indices_workspace = topk_indices[chunk.tokens_slice] @@ -953,7 +1006,7 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]): assert self.topk_indices_buffer is not None topk_indices = self.topk_indices_buffer[:num_actual_toks] - use_fp8_cache = self.kv_cache_dtype == "fp8_ds_mla" + use_fp8_cache = self.kv_cache_dtype in QUANTIZED_DS_MLA_CACHE_FORMATS lse: torch.Tensor | None = None diff --git a/vllm/v1/kv_cache_interface.py b/vllm/v1/kv_cache_interface.py index 1007c48a21..fbb3a22611 100644 --- a/vllm/v1/kv_cache_interface.py +++ b/vllm/v1/kv_cache_interface.py @@ -54,6 +54,7 @@ class KVQuantMode(IntEnum): TURBOQUANT_4BIT_NC = 7 TURBOQUANT_K3V4_NC = 8 TURBOQUANT_3BIT_NC = 9 + NVFP4_DS_MLA = 10 # opaque-bytes NVFP4 DS-MLA layouts (FlashMLA sparse) @property def is_per_token_head(self) -> bool: @@ -88,6 +89,11 @@ def get_kv_quant_mode(kv_cache_dtype: str) -> KVQuantMode: return KVQuantMode.INT8_PER_TOKEN_HEAD if kv_cache_dtype == "fp8_per_token_head": return KVQuantMode.FP8_PER_TOKEN_HEAD + # Must precede the ``nvfp4`` prefix test below, which would otherwise match. + if kv_cache_dtype == "nvfp4_ds_mla": + # Page size is keyed on cache_dtype_str in the MLA specs, not + # nvfp4_kv_cache_full_dim. + return KVQuantMode.NVFP4_DS_MLA if kv_cache_dtype.startswith("nvfp4"): return KVQuantMode.NVFP4 if isinstance(kv_cache_dtype, str) and kv_cache_dtype.startswith("turboquant_"):