[Attention][DSA] Enable W4A16 DSA (#51724)
Signed-off-by: Shiyang Chen <shiychen@nvidia.com>
This commit is contained in:
+2
-1
@@ -1042,7 +1042,8 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP")
|
|||||||
"csrc/libtorch_stable/quantization/fp4/nvfp4_blockwise_moe_kernel.cu"
|
"csrc/libtorch_stable/quantization/fp4/nvfp4_blockwise_moe_kernel.cu"
|
||||||
"csrc/libtorch_stable/quantization/fp4/mxfp4_experts_quant.cu"
|
"csrc/libtorch_stable/quantization/fp4/mxfp4_experts_quant.cu"
|
||||||
"csrc/libtorch_stable/quantization/fp4/mxfp4_blockwise_moe_kernel.cu"
|
"csrc/libtorch_stable/quantization/fp4/mxfp4_blockwise_moe_kernel.cu"
|
||||||
"csrc/libtorch_stable/nvfp4_kv_cache_kernels.cu")
|
"csrc/libtorch_stable/nvfp4_kv_cache_kernels.cu"
|
||||||
|
"csrc/libtorch_stable/nvfp4_ds_mla_cache_kernels.cu")
|
||||||
if(NOT ${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 12.9)
|
if(NOT ${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 12.9)
|
||||||
message(STATUS
|
message(STATUS
|
||||||
"Building mxfp4_experts_quant unsupported stubs because CUDA compiler version is not >= 12.9 (found ${CMAKE_CUDA_COMPILER_VERSION}).")
|
"Building mxfp4_experts_quant unsupported stubs because CUDA compiler version is not >= 12.9 (found ${CMAKE_CUDA_COMPILER_VERSION}).")
|
||||||
|
|||||||
@@ -19,7 +19,7 @@ else()
|
|||||||
FetchContent_Declare(
|
FetchContent_Declare(
|
||||||
flashmla
|
flashmla
|
||||||
GIT_REPOSITORY https://github.com/vllm-project/FlashMLA
|
GIT_REPOSITORY https://github.com/vllm-project/FlashMLA
|
||||||
GIT_TAG 0397728d511c4e3d94ea3a01d8dda8654525a611
|
GIT_TAG 6bc49418c5ead572ff0339191ddf3b155749e183
|
||||||
GIT_PROGRESS TRUE
|
GIT_PROGRESS TRUE
|
||||||
CONFIGURE_COMMAND ""
|
CONFIGURE_COMMAND ""
|
||||||
BUILD_COMMAND ""
|
BUILD_COMMAND ""
|
||||||
@@ -111,6 +111,7 @@ if(FLASH_MLA_ARCHS)
|
|||||||
# sm100 sparse decode
|
# sm100 sparse decode
|
||||||
${flashmla_SOURCE_DIR}/csrc/sm100/decode/head64/instantiations/v32.cu
|
${flashmla_SOURCE_DIR}/csrc/sm100/decode/head64/instantiations/v32.cu
|
||||||
${flashmla_SOURCE_DIR}/csrc/sm100/decode/head64/instantiations/model1.cu
|
${flashmla_SOURCE_DIR}/csrc/sm100/decode/head64/instantiations/model1.cu
|
||||||
|
${flashmla_SOURCE_DIR}/csrc/sm100/decode/head64/instantiations/v32_nvfp4_fp8rope.cu
|
||||||
${flashmla_SOURCE_DIR}/csrc/sm100/prefill/sparse/fwd_for_small_topk/head128/instantiations/phase1_decode_k512.cu
|
${flashmla_SOURCE_DIR}/csrc/sm100/prefill/sparse/fwd_for_small_topk/head128/instantiations/phase1_decode_k512.cu
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -7,6 +7,10 @@
|
|||||||
#include "quantization/vectorization_utils.cuh"
|
#include "quantization/vectorization_utils.cuh"
|
||||||
#include "concat_mla_q.cuh"
|
#include "concat_mla_q.cuh"
|
||||||
|
|
||||||
|
#if !defined(USE_ROCM) && defined(ENABLE_NVFP4_SM100) && ENABLE_NVFP4_SM100
|
||||||
|
#include "nvfp4_ds_mla_cache.h"
|
||||||
|
#endif
|
||||||
|
|
||||||
#ifdef USE_ROCM
|
#ifdef USE_ROCM
|
||||||
#include "../quantization/w8a8/fp8/amd/quant_utils.cuh"
|
#include "../quantization/w8a8/fp8/amd/quant_utils.cuh"
|
||||||
#else
|
#else
|
||||||
@@ -917,6 +921,7 @@ void concat_and_cache_mla(
|
|||||||
int pe_dim = k_pe.size(1);
|
int pe_dim = k_pe.size(1);
|
||||||
int block_size = kv_cache.size(1);
|
int block_size = kv_cache.size(1);
|
||||||
|
|
||||||
|
const bool is_nvfp4_ds_mla = kv_cache_dtype == "nvfp4_ds_mla";
|
||||||
if (kv_cache_dtype == "fp8_ds_mla") {
|
if (kv_cache_dtype == "fp8_ds_mla") {
|
||||||
STD_TORCH_CHECK(kv_lora_rank == 512,
|
STD_TORCH_CHECK(kv_lora_rank == 512,
|
||||||
"kv_lora_rank must be 512 for fp8_ds_mla");
|
"kv_lora_rank must be 512 for fp8_ds_mla");
|
||||||
@@ -927,6 +932,21 @@ void concat_and_cache_mla(
|
|||||||
"kv_c.element_size() must be 2 for fp8_ds_mla");
|
"kv_c.element_size() must be 2 for fp8_ds_mla");
|
||||||
STD_TORCH_CHECK(k_pe.element_size() == 2,
|
STD_TORCH_CHECK(k_pe.element_size() == 2,
|
||||||
"k_pe.element_size() must be 2 for fp8_ds_mla");
|
"k_pe.element_size() must be 2 for fp8_ds_mla");
|
||||||
|
} else if (is_nvfp4_ds_mla) {
|
||||||
|
constexpr int bytes_per_token = 352;
|
||||||
|
STD_TORCH_CHECK(kv_lora_rank == 512, "kv_lora_rank must be 512 for ",
|
||||||
|
kv_cache_dtype);
|
||||||
|
STD_TORCH_CHECK(pe_dim == 64, "pe_dim must be 64 for ", kv_cache_dtype);
|
||||||
|
STD_TORCH_CHECK(
|
||||||
|
kv_cache.size(2) == bytes_per_token / kv_cache.element_size(),
|
||||||
|
"kv_cache.size(2) must be ", bytes_per_token, " bytes for ",
|
||||||
|
kv_cache_dtype);
|
||||||
|
STD_TORCH_CHECK(
|
||||||
|
kv_c.scalar_type() == torch::headeronly::ScalarType::BFloat16,
|
||||||
|
"kv_c must be bfloat16 for ", kv_cache_dtype);
|
||||||
|
STD_TORCH_CHECK(
|
||||||
|
k_pe.scalar_type() == torch::headeronly::ScalarType::BFloat16,
|
||||||
|
"k_pe must be bfloat16 for ", kv_cache_dtype);
|
||||||
} else {
|
} else {
|
||||||
STD_TORCH_CHECK(kv_cache.size(2) == kv_lora_rank + pe_dim);
|
STD_TORCH_CHECK(kv_cache.size(2) == kv_lora_rank + pe_dim);
|
||||||
}
|
}
|
||||||
@@ -950,6 +970,24 @@ void concat_and_cache_mla(
|
|||||||
dim3 block(96);
|
dim3 block(96);
|
||||||
DISPATCH_BY_KV_CACHE_DTYPE(kv_c.scalar_type(), kv_cache_dtype,
|
DISPATCH_BY_KV_CACHE_DTYPE(kv_c.scalar_type(), kv_cache_dtype,
|
||||||
CALL_CONCAT_AND_CACHE_DS_MLA);
|
CALL_CONCAT_AND_CACHE_DS_MLA);
|
||||||
|
} else if (is_nvfp4_ds_mla) {
|
||||||
|
// The kernels live in nvfp4_ds_mla_cache_kernels.cu, which is only built
|
||||||
|
// when the target list contains an SM100 arch.
|
||||||
|
#if !defined(USE_ROCM) && defined(ENABLE_NVFP4_SM100) && ENABLE_NVFP4_SM100
|
||||||
|
const cudaDeviceProp* props = get_device_prop();
|
||||||
|
STD_TORCH_CHECK(props->major == 10,
|
||||||
|
"nvfp4_ds_mla KV cache requires SM100 (Blackwell); "
|
||||||
|
"got SM",
|
||||||
|
props->major, props->minor);
|
||||||
|
vllm::launch_concat_and_cache_nvfp4_ds_mla(
|
||||||
|
kv_c.data_ptr(), k_pe.data_ptr(), kv_cache.data_ptr(),
|
||||||
|
slot_mapping.const_data_ptr<int64_t>(), block_stride, entry_stride,
|
||||||
|
kv_c_stride, k_pe_stride, block_size, num_tokens, stream);
|
||||||
|
#else
|
||||||
|
STD_TORCH_CHECK(false,
|
||||||
|
"the nvfp4 ds_mla kv-cache format requires a CUDA build "
|
||||||
|
"targeting SM100 (Blackwell)");
|
||||||
|
#endif
|
||||||
} else {
|
} else {
|
||||||
dim3 grid(num_tokens);
|
dim3 grid(num_tokens);
|
||||||
dim3 block(std::min(kv_lora_rank, 512));
|
dim3 block(std::min(kv_lora_rank, 512));
|
||||||
@@ -1659,6 +1697,72 @@ void cp_gather_and_upconvert_fp8_kv_cache(
|
|||||||
dst_entry_stride, seq_starts_ptr);
|
dst_entry_stride, seq_starts_ptr);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void cp_gather_and_upconvert_nvfp4_kv_cache(
|
||||||
|
torch::stable::Tensor const& src_cache, // [NUM_BLOCKS, BLOCK_SIZE,
|
||||||
|
// 352]
|
||||||
|
torch::stable::Tensor const& dst, // [TOT_TOKENS, 576]
|
||||||
|
torch::stable::Tensor const& block_table, // [BATCH, BLOCK_INDICES]
|
||||||
|
torch::stable::Tensor const& workspace_starts, // [BATCH]
|
||||||
|
int64_t batch_size) {
|
||||||
|
// The kernel lives in nvfp4_ds_mla_cache_kernels.cu, which is only built when
|
||||||
|
// the target list contains an SM100 arch.
|
||||||
|
#if !defined(USE_ROCM) && defined(ENABLE_NVFP4_SM100) && ENABLE_NVFP4_SM100
|
||||||
|
torch::stable::accelerator::DeviceGuard device_guard(
|
||||||
|
src_cache.get_device_index());
|
||||||
|
const cudaStream_t stream = get_current_cuda_stream();
|
||||||
|
|
||||||
|
const cudaDeviceProp* props = get_device_prop();
|
||||||
|
STD_TORCH_CHECK(props->major == 10,
|
||||||
|
"nvfp4_ds_mla KV cache requires SM100 (Blackwell); "
|
||||||
|
"got SM",
|
||||||
|
props->major, props->minor);
|
||||||
|
|
||||||
|
int32_t block_size = src_cache.size(1);
|
||||||
|
int32_t head_dim = dst.size(1);
|
||||||
|
|
||||||
|
STD_TORCH_CHECK(
|
||||||
|
block_table.scalar_type() == torch::headeronly::ScalarType::Int,
|
||||||
|
"block_table must be int32");
|
||||||
|
STD_TORCH_CHECK(
|
||||||
|
workspace_starts.scalar_type() == torch::headeronly::ScalarType::Int,
|
||||||
|
"workspace_starts must be int32");
|
||||||
|
STD_TORCH_CHECK(src_cache.device() == dst.device(),
|
||||||
|
"src_cache and dst must be on the same device");
|
||||||
|
STD_TORCH_CHECK(src_cache.device() == block_table.device(),
|
||||||
|
"src_cache and block_table must be on the same device");
|
||||||
|
STD_TORCH_CHECK(src_cache.device() == workspace_starts.device(),
|
||||||
|
"src_cache and workspace_starts must be on the same device");
|
||||||
|
STD_TORCH_CHECK(
|
||||||
|
src_cache.scalar_type() == torch::headeronly::ScalarType::Byte,
|
||||||
|
"src_cache must be uint8 for the nvfp4 ds_mla format, but got ",
|
||||||
|
src_cache.scalar_type());
|
||||||
|
STD_TORCH_CHECK(dst.scalar_type() == torch::headeronly::ScalarType::BFloat16,
|
||||||
|
"dst must be bfloat16");
|
||||||
|
STD_TORCH_CHECK(head_dim == 576, "head_dim must be 576 for MLA");
|
||||||
|
STD_TORCH_CHECK(src_cache.size(2) == 352,
|
||||||
|
"src_cache.size(2) must be 352 for the nvfp4 ds_mla format");
|
||||||
|
|
||||||
|
int64_t block_table_stride = block_table.stride(0);
|
||||||
|
int64_t cache_block_stride = src_cache.stride(0);
|
||||||
|
int64_t cache_entry_stride = src_cache.stride(1);
|
||||||
|
int64_t dst_entry_stride = dst.stride(0);
|
||||||
|
|
||||||
|
const int total_tokens = dst.size(0);
|
||||||
|
|
||||||
|
vllm::launch_cp_gather_and_upconvert_nvfp4_kv_cache(
|
||||||
|
src_cache.const_data_ptr<uint8_t>(), dst.data_ptr(),
|
||||||
|
block_table.const_data_ptr<int32_t>(),
|
||||||
|
workspace_starts.const_data_ptr<int32_t>(),
|
||||||
|
static_cast<int32_t>(batch_size), block_size, total_tokens,
|
||||||
|
block_table_stride, cache_block_stride, cache_entry_stride,
|
||||||
|
dst_entry_stride, stream);
|
||||||
|
#else
|
||||||
|
STD_TORCH_CHECK(false,
|
||||||
|
"the nvfp4 ds_mla kv-cache format requires a CUDA build "
|
||||||
|
"targeting SM100 (Blackwell)");
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
// Macro to dispatch the kernel based on the data type.
|
// Macro to dispatch the kernel based on the data type.
|
||||||
#define CALL_INDEXER_K_QUANT_AND_CACHE(KV_T, CACHE_T, KV_DTYPE) \
|
#define CALL_INDEXER_K_QUANT_AND_CACHE(KV_T, CACHE_T, KV_DTYPE) \
|
||||||
vllm::indexer_k_quant_and_cache_kernel<KV_T, CACHE_T, KV_DTYPE> \
|
vllm::indexer_k_quant_and_cache_kernel<KV_T, CACHE_T, KV_DTYPE> \
|
||||||
|
|||||||
@@ -0,0 +1,37 @@
|
|||||||
|
// SPDX-License-Identifier: Apache-2.0
|
||||||
|
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
#pragma once
|
||||||
|
|
||||||
|
#include <cuda_runtime_api.h>
|
||||||
|
#include <cstdint>
|
||||||
|
|
||||||
|
// Launchers for the nvfp4_ds_mla KV cache kernels. Those kernels live in
|
||||||
|
// their own TU (nvfp4_ds_mla_cache_kernels.cu) so that only that TU is built
|
||||||
|
// for the arch-conditional Blackwell targets their e2m1 conversions require;
|
||||||
|
// these declarations are what keeps cache_kernels.cu generic.
|
||||||
|
//
|
||||||
|
// Only defined when the build has an SM100 target (-DENABLE_NVFP4_SM100=1);
|
||||||
|
// callers must guard on the same macro.
|
||||||
|
namespace vllm {
|
||||||
|
|
||||||
|
// concat_and_cache_mla for the nvfp4_ds_mla layout.
|
||||||
|
// kv_c: bf16 [num_tokens, 512]
|
||||||
|
// k_pe: bf16 [num_tokens, 64]
|
||||||
|
// kv_cache: uint8 [num_blocks, block_size, 352]
|
||||||
|
void launch_concat_and_cache_nvfp4_ds_mla(const void* kv_c, const void* k_pe,
|
||||||
|
void* kv_cache,
|
||||||
|
const int64_t* slot_mapping,
|
||||||
|
int block_stride, int entry_stride,
|
||||||
|
int kv_c_stride, int k_pe_stride,
|
||||||
|
int block_size, int num_tokens,
|
||||||
|
cudaStream_t stream);
|
||||||
|
|
||||||
|
// Gather an nvfp4_ds_mla cache into a bf16 [total_tokens, 576] workspace.
|
||||||
|
void launch_cp_gather_and_upconvert_nvfp4_kv_cache(
|
||||||
|
const uint8_t* src_cache, void* dst, const int32_t* block_table,
|
||||||
|
const int32_t* workspace_starts, int32_t num_reqs, int32_t block_size,
|
||||||
|
int32_t total_tokens, int64_t block_table_stride,
|
||||||
|
int64_t cache_block_stride, int64_t cache_entry_stride,
|
||||||
|
int64_t dst_entry_stride, cudaStream_t stream);
|
||||||
|
|
||||||
|
} // namespace vllm
|
||||||
@@ -0,0 +1,281 @@
|
|||||||
|
// SPDX-License-Identifier: Apache-2.0
|
||||||
|
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
|
// nvfp4_ds_mla KV cache writer and gather/upconvert reader.
|
||||||
|
//
|
||||||
|
// Separate TU because its e2m1 conversions are only accepted by ptxas for a
|
||||||
|
// suffixed Blackwell target (sm_100a / sm_100f); only this file is compiled
|
||||||
|
// with the suffixed arch list.
|
||||||
|
|
||||||
|
#include "nvfp4_ds_mla_cache.h"
|
||||||
|
|
||||||
|
#include "quantization/fp4/nvfp4_utils.cuh" // vllm::fp32_vec16_to_e2m1()
|
||||||
|
|
||||||
|
#include <cuda_bf16.h>
|
||||||
|
#include <cuda_fp8.h>
|
||||||
|
|
||||||
|
#if defined(CUDART_VERSION) && CUDART_VERSION >= 12080
|
||||||
|
#include <cuda_fp4.h>
|
||||||
|
#define VLLM_HAS_CUDA_FP4 1
|
||||||
|
#else
|
||||||
|
#define VLLM_HAS_CUDA_FP4 0
|
||||||
|
#endif
|
||||||
|
|
||||||
|
// __CUDA_ARCH_FAMILY_SPECIFIC__ is defined only for the suffixed targets. The
|
||||||
|
// non-native paths trap rather than emulate; the host entry points reject this
|
||||||
|
// format off SM100, so they are unreachable.
|
||||||
|
#if VLLM_HAS_CUDA_FP4 && defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 1000 && \
|
||||||
|
defined(__CUDA_ARCH_FAMILY_SPECIFIC__)
|
||||||
|
#define VLLM_NVFP4_NATIVE_CVT 1
|
||||||
|
#else
|
||||||
|
#define VLLM_NVFP4_NATIVE_CVT 0
|
||||||
|
#endif
|
||||||
|
|
||||||
|
namespace vllm {
|
||||||
|
|
||||||
|
// nvfp4_ds_mla cache format, 352 B/token:
|
||||||
|
// [0, 256) 512 x e2m1 NoPE, packed 2/byte (low nibble = even element)
|
||||||
|
// [256, 320) 64 x e4m3 RoPE (unscaled: e4m3's 4 exponent bits span the
|
||||||
|
// RoPE range unaided)
|
||||||
|
// [320, 352) 32 x e4m3 NoPE scale factors (one per 16 elements), permuted
|
||||||
|
// Quantization: sf = e4m3(max(amax_16 / 6, 2^-9)), q = round(x / float(sf))
|
||||||
|
// Dequantization: x = float(q) * float(sf)
|
||||||
|
//
|
||||||
|
// The scale-factor region is an 8x4 -> 4x8 transpose: element block s -- NoPE
|
||||||
|
// dims [16s, 16s + 16) -- has its scale at byte kSfNopeOff + nvfp4_sf_byte(s),
|
||||||
|
// and byte p holds the scale of element block 4 * (p & 7) + (p >> 3). Keep
|
||||||
|
// nvfp4_sf_byte() in lockstep with its counterpart in FlashMLA
|
||||||
|
// (csrc/sm100/decode/head64/config.h and tests/quant.py).
|
||||||
|
constexpr int kSfNopeOff = 256 + 64;
|
||||||
|
|
||||||
|
__host__ __device__ constexpr int nvfp4_sf_byte(int s) {
|
||||||
|
return 8 * (s & 3) + (s >> 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Thread q's scale groups {4c + q} must land on the contiguous bytes 8q + c.
|
||||||
|
// This also makes it a permutation of [0, 32), since both sides cover that
|
||||||
|
// range exactly once.
|
||||||
|
constexpr bool nvfp4_sf_perm_ok() {
|
||||||
|
for (int q = 0; q < 4; ++q) {
|
||||||
|
for (int c = 0; c < 8; ++c) {
|
||||||
|
if (nvfp4_sf_byte(4 * c + q) != 8 * q + c) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
static_assert(nvfp4_sf_perm_ok(),
|
||||||
|
"NVFP4 SF byte order must match FlashMLA's nvfp4_sf_byte()");
|
||||||
|
|
||||||
|
// `cvt.rn.satfinite.e2m1x2.f32 d, a, b` puts a in d's HIGH nibble and b in its
|
||||||
|
// LOW nibble, so the odd element of a pair is the first source.
|
||||||
|
__device__ __forceinline__ uint64_t
|
||||||
|
nvfp4_pack16_e2m1_rn(const float (&vals)[16], float inv_sf) {
|
||||||
|
#if VLLM_NVFP4_NATIVE_CVT
|
||||||
|
float2 pairs[8];
|
||||||
|
#pragma unroll
|
||||||
|
for (int i = 0; i < 8; i++) {
|
||||||
|
pairs[i] = make_float2(vals[2 * i] * inv_sf, vals[2 * i + 1] * inv_sf);
|
||||||
|
}
|
||||||
|
const u32x2 packed = fp32_vec16_to_e2m1(pairs);
|
||||||
|
return (static_cast<uint64_t>(packed.hi) << 32) | packed.lo;
|
||||||
|
#else
|
||||||
|
__trap();
|
||||||
|
return 0;
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
|
// The e2m1 data and its e4m3 scale both convert to f16 and multiply there; the
|
||||||
|
// product then goes f16 -> f32 -> bf16, as no f16 -> bf16 instruction exists.
|
||||||
|
__device__ __forceinline__ void nvfp4_unpack16_e2m1(uint64_t raw,
|
||||||
|
uint8_t sf_byte,
|
||||||
|
__nv_bfloat16 (&out)[16]) {
|
||||||
|
#if VLLM_NVFP4_NATIVE_CVT
|
||||||
|
const __half2_raw sf2_raw = __nv_cvt_fp8x2_to_halfraw2(
|
||||||
|
static_cast<__nv_fp8x2_storage_t>(sf_byte * 0x0101u), __NV_E4M3);
|
||||||
|
const __half2 sf2 = *reinterpret_cast<const __half2*>(&sf2_raw);
|
||||||
|
__nv_bfloat162* out2 = reinterpret_cast<__nv_bfloat162*>(out);
|
||||||
|
#pragma unroll
|
||||||
|
for (int i = 0; i < 8; i++) {
|
||||||
|
// low nibble -> .x, so .x = element 2i, matching the writer.
|
||||||
|
const __half2_raw v_raw = __nv_cvt_fp4x2_to_halfraw2(
|
||||||
|
static_cast<__nv_fp4x2_storage_t>(raw >> (8 * i)), __NV_E2M1);
|
||||||
|
const __half2 v = __hmul2(*reinterpret_cast<const __half2*>(&v_raw), sf2);
|
||||||
|
out2[i] = __float22bfloat162_rn(__half22float2(v));
|
||||||
|
}
|
||||||
|
#else
|
||||||
|
__trap();
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
|
// Rounded UP to the next e4m3 value so amax / sf never exceeds the payload
|
||||||
|
// range: round-to-nearest can land up to 33% low near e4m3's 2^-9 floor, which
|
||||||
|
// would saturate the tile's largest values.
|
||||||
|
__device__ __forceinline__ float nvfp4_tile_scale(const float* vals,
|
||||||
|
float max_val,
|
||||||
|
uint8_t* sf_out) {
|
||||||
|
float amax = 0.0f;
|
||||||
|
#pragma unroll
|
||||||
|
for (int i = 0; i < 16; i++) {
|
||||||
|
amax = fmaxf(amax, fabsf(vals[i]));
|
||||||
|
}
|
||||||
|
const float sf_f = fmaxf(amax / max_val, 0.001953125f); // 2^-9
|
||||||
|
__nv_fp8_e4m3 sf8(sf_f);
|
||||||
|
uint8_t sf_bits = *reinterpret_cast<const uint8_t*>(&sf8);
|
||||||
|
if (float(sf8) < sf_f && sf_bits < 0x7E) {
|
||||||
|
// Bump to the next e4m3 value (bit patterns of positive e4m3 values are
|
||||||
|
// monotonic; 0x7E = 448 is the max finite value).
|
||||||
|
sf_bits += 1;
|
||||||
|
sf8 = *reinterpret_cast<const __nv_fp8_e4m3*>(&sf_bits);
|
||||||
|
}
|
||||||
|
*sf_out = sf_bits;
|
||||||
|
return float(sf8);
|
||||||
|
}
|
||||||
|
|
||||||
|
// One CUDA block (64 threads) per token.
|
||||||
|
// Warp 0: 32 threads, one 16-element NoPE tile each.
|
||||||
|
// Warp 1: 32 threads, two RoPE elements each.
|
||||||
|
__global__ void concat_and_cache_nvfp4_ds_mla_kernel(
|
||||||
|
const __nv_bfloat16* __restrict__ kv_c, // [num_tokens, 512]
|
||||||
|
const __nv_bfloat16* __restrict__ k_pe, // [num_tokens, 64]
|
||||||
|
uint8_t* __restrict__ kv_cache, // [num_blocks, block_size, 352]
|
||||||
|
const int64_t* __restrict__ slot_mapping, // [num_tokens]
|
||||||
|
const int block_stride, //
|
||||||
|
const int entry_stride, //
|
||||||
|
const int kv_c_stride, //
|
||||||
|
const int k_pe_stride, //
|
||||||
|
const int block_size) {
|
||||||
|
const int64_t token_idx = blockIdx.x;
|
||||||
|
const int64_t slot_idx = slot_mapping[token_idx];
|
||||||
|
// NOTE: slot_idx can be -1 if the token is padded
|
||||||
|
if (slot_idx < 0) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const int64_t block_idx = slot_idx / block_size;
|
||||||
|
const int64_t block_offset = slot_idx % block_size;
|
||||||
|
uint8_t* token_ptr =
|
||||||
|
kv_cache + block_idx * block_stride + block_offset * entry_stride;
|
||||||
|
|
||||||
|
if (threadIdx.x < 32) {
|
||||||
|
const int tile = threadIdx.x;
|
||||||
|
const __nv_bfloat16* src = kv_c + token_idx * kv_c_stride + tile * 16;
|
||||||
|
float vals[16];
|
||||||
|
#pragma unroll
|
||||||
|
for (int i = 0; i < 16; i++) {
|
||||||
|
vals[i] = __bfloat162float(src[i]);
|
||||||
|
}
|
||||||
|
// `tile` is the element-block index; its scale byte is permuted.
|
||||||
|
const float sf = nvfp4_tile_scale(
|
||||||
|
vals, 6.0f, token_ptr + kSfNopeOff + nvfp4_sf_byte(tile));
|
||||||
|
const float inv_sf = 1.0f / sf;
|
||||||
|
*reinterpret_cast<uint64_t*>(token_ptr + tile * 8) =
|
||||||
|
nvfp4_pack16_e2m1_rn(vals, inv_sf);
|
||||||
|
} else {
|
||||||
|
// Plain e4m3, unscaled. All 32 lanes take 2 elements each, matching the
|
||||||
|
// reader and concat_and_cache_ds_mla_kernel.
|
||||||
|
const int lane = threadIdx.x - 32;
|
||||||
|
const __nv_bfloat16* src = k_pe + token_idx * k_pe_stride + lane * 2;
|
||||||
|
uint8_t q[2];
|
||||||
|
#pragma unroll
|
||||||
|
for (int i = 0; i < 2; i++) {
|
||||||
|
const __nv_fp8_e4m3 v8(__bfloat162float(src[i]));
|
||||||
|
q[i] = *reinterpret_cast<const uint8_t*>(&v8);
|
||||||
|
}
|
||||||
|
*reinterpret_cast<uint16_t*>(token_ptr + 256 + lane * 2) =
|
||||||
|
*reinterpret_cast<const uint16_t*>(q);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// One warp per token (mirrors cp_gather_and_upconvert_fp8_kv_cache).
|
||||||
|
__global__ void cp_gather_and_upconvert_nvfp4_kv_cache_kernel(
|
||||||
|
const uint8_t* __restrict__ src_cache, // [NUM_BLOCKS, BLOCK_SIZE, 352]
|
||||||
|
__nv_bfloat16* __restrict__ dst, // [total_tokens, 576]
|
||||||
|
const int32_t* __restrict__ block_table, // [num_reqs, BLOCK_INDICES]
|
||||||
|
const int32_t* __restrict__ workspace_starts, // [num_reqs]
|
||||||
|
const int32_t num_reqs, const int32_t block_size,
|
||||||
|
const int32_t total_tokens, const int64_t block_table_stride,
|
||||||
|
const int64_t cache_block_stride, const int64_t cache_entry_stride,
|
||||||
|
const int64_t dst_entry_stride) {
|
||||||
|
const int flat_warp_id = (blockIdx.x * blockDim.x + threadIdx.x) >> 5;
|
||||||
|
if (flat_warp_id >= total_tokens) return;
|
||||||
|
const int lane_id = threadIdx.x & 31;
|
||||||
|
|
||||||
|
// Binary search to find which request owns this output token
|
||||||
|
int lo = 0, hi = num_reqs - 1;
|
||||||
|
while (lo < hi) {
|
||||||
|
int mid = (lo + hi + 1) >> 1;
|
||||||
|
if (workspace_starts[mid] <= flat_warp_id)
|
||||||
|
lo = mid;
|
||||||
|
else
|
||||||
|
hi = mid - 1;
|
||||||
|
}
|
||||||
|
const int req_id = lo;
|
||||||
|
|
||||||
|
const int out_token_id = flat_warp_id;
|
||||||
|
const int token_offset = out_token_id - workspace_starts[req_id];
|
||||||
|
const int cache_block_idx = token_offset / block_size;
|
||||||
|
const int offset_in_block = token_offset % block_size;
|
||||||
|
const int physical_block =
|
||||||
|
block_table[req_id * block_table_stride + cache_block_idx];
|
||||||
|
|
||||||
|
const uint8_t* token_ptr = src_cache + physical_block * cache_block_stride +
|
||||||
|
offset_in_block * cache_entry_stride;
|
||||||
|
__nv_bfloat16* dst_ptr = dst + out_token_id * dst_entry_stride;
|
||||||
|
|
||||||
|
// `lane_id` is the element-block index, so its scale byte is permuted.
|
||||||
|
{
|
||||||
|
const uint64_t raw =
|
||||||
|
*reinterpret_cast<const uint64_t*>(token_ptr + lane_id * 8);
|
||||||
|
const uint8_t sf_byte = token_ptr[kSfNopeOff + nvfp4_sf_byte(lane_id)];
|
||||||
|
__nv_bfloat16 out[16];
|
||||||
|
nvfp4_unpack16_e2m1(raw, sf_byte, out);
|
||||||
|
int4* nope_dst = reinterpret_cast<int4*>(dst_ptr) + lane_id * 2;
|
||||||
|
nope_dst[0] = *reinterpret_cast<const int4*>(&out[0]);
|
||||||
|
nope_dst[1] = *reinterpret_cast<const int4*>(&out[8]);
|
||||||
|
}
|
||||||
|
|
||||||
|
// RoPE is unscaled e4m3.
|
||||||
|
{
|
||||||
|
const __nv_fp8_e4m3* raw =
|
||||||
|
reinterpret_cast<const __nv_fp8_e4m3*>(token_ptr + 256 + lane_id * 2);
|
||||||
|
const __nv_bfloat16 out[2] = {__float2bfloat16_rn(float(raw[0])),
|
||||||
|
__float2bfloat16_rn(float(raw[1]))};
|
||||||
|
*reinterpret_cast<uint32_t*>(dst_ptr + 512 + lane_id * 2) =
|
||||||
|
*reinterpret_cast<const uint32_t*>(out);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void launch_concat_and_cache_nvfp4_ds_mla(const void* kv_c, const void* k_pe,
|
||||||
|
void* kv_cache,
|
||||||
|
const int64_t* slot_mapping,
|
||||||
|
int block_stride, int entry_stride,
|
||||||
|
int kv_c_stride, int k_pe_stride,
|
||||||
|
int block_size, int num_tokens,
|
||||||
|
cudaStream_t stream) {
|
||||||
|
dim3 grid(num_tokens);
|
||||||
|
dim3 block(64); // warp 0: 32 NoPE tiles; warp 1: 4 RoPE tiles
|
||||||
|
concat_and_cache_nvfp4_ds_mla_kernel<<<grid, block, 0, stream>>>(
|
||||||
|
reinterpret_cast<const __nv_bfloat16*>(kv_c),
|
||||||
|
reinterpret_cast<const __nv_bfloat16*>(k_pe),
|
||||||
|
reinterpret_cast<uint8_t*>(kv_cache), slot_mapping, block_stride,
|
||||||
|
entry_stride, kv_c_stride, k_pe_stride, block_size);
|
||||||
|
}
|
||||||
|
|
||||||
|
void launch_cp_gather_and_upconvert_nvfp4_kv_cache(
|
||||||
|
const uint8_t* src_cache, void* dst, const int32_t* block_table,
|
||||||
|
const int32_t* workspace_starts, int32_t num_reqs, int32_t block_size,
|
||||||
|
int32_t total_tokens, int64_t block_table_stride,
|
||||||
|
int64_t cache_block_stride, int64_t cache_entry_stride,
|
||||||
|
int64_t dst_entry_stride, cudaStream_t stream) {
|
||||||
|
constexpr int warps_per_block = 8;
|
||||||
|
const int grid_size = (total_tokens + warps_per_block - 1) / warps_per_block;
|
||||||
|
const int block_size_threads = warps_per_block * 32; // 256 threads
|
||||||
|
cp_gather_and_upconvert_nvfp4_kv_cache_kernel<<<grid_size, block_size_threads,
|
||||||
|
0, stream>>>(
|
||||||
|
src_cache, reinterpret_cast<__nv_bfloat16*>(dst), block_table,
|
||||||
|
workspace_starts, num_reqs, block_size, total_tokens, block_table_stride,
|
||||||
|
cache_block_stride, cache_entry_stride, dst_entry_stride);
|
||||||
|
}
|
||||||
|
|
||||||
|
} // namespace vllm
|
||||||
@@ -672,6 +672,15 @@ void cp_gather_and_upconvert_fp8_kv_cache(
|
|||||||
int64_t batch_size,
|
int64_t batch_size,
|
||||||
std::optional<torch::stable::Tensor> seq_starts = std::nullopt);
|
std::optional<torch::stable::Tensor> seq_starts = std::nullopt);
|
||||||
|
|
||||||
|
// Gather and upconvert an nvfp4_ds_mla KV cache to a BF16 workspace
|
||||||
|
void cp_gather_and_upconvert_nvfp4_kv_cache(
|
||||||
|
torch::stable::Tensor const& src_cache, // [NUM_BLOCKS, BLOCK_SIZE,
|
||||||
|
// 352]
|
||||||
|
torch::stable::Tensor const& dst, // [TOT_TOKENS, 576]
|
||||||
|
torch::stable::Tensor const& block_table, // [BATCH, BLOCK_INDICES]
|
||||||
|
torch::stable::Tensor const& workspace_starts, // [BATCH]
|
||||||
|
int64_t batch_size);
|
||||||
|
|
||||||
// Indexer K quantization and cache function
|
// Indexer K quantization and cache function
|
||||||
void indexer_k_quant_and_cache(
|
void indexer_k_quant_and_cache(
|
||||||
torch::stable::Tensor& k, // [num_tokens, head_dim]
|
torch::stable::Tensor& k, // [num_tokens, head_dim]
|
||||||
|
|||||||
@@ -996,6 +996,10 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C_cache_ops, ops) {
|
|||||||
"Tensor block_table, Tensor workspace_starts, int batch_size, Tensor? "
|
"Tensor block_table, Tensor workspace_starts, int batch_size, Tensor? "
|
||||||
"seq_starts) -> ()");
|
"seq_starts) -> ()");
|
||||||
|
|
||||||
|
ops.def(
|
||||||
|
"cp_gather_and_upconvert_nvfp4_kv_cache(Tensor src_cache, Tensor! dst, "
|
||||||
|
"Tensor block_table, Tensor workspace_starts, int batch_size) -> ()");
|
||||||
|
|
||||||
ops.def(
|
ops.def(
|
||||||
"indexer_k_quant_and_cache(Tensor k, Tensor! kv_cache, Tensor "
|
"indexer_k_quant_and_cache(Tensor k, Tensor! kv_cache, Tensor "
|
||||||
"slot_mapping, "
|
"slot_mapping, "
|
||||||
@@ -1066,6 +1070,8 @@ STABLE_TORCH_LIBRARY_IMPL(_C_cache_ops, CUDA, ops) {
|
|||||||
ops.impl("cp_gather_cache", TORCH_BOX(&cp_gather_cache));
|
ops.impl("cp_gather_cache", TORCH_BOX(&cp_gather_cache));
|
||||||
ops.impl("cp_gather_and_upconvert_fp8_kv_cache",
|
ops.impl("cp_gather_and_upconvert_fp8_kv_cache",
|
||||||
TORCH_BOX(&cp_gather_and_upconvert_fp8_kv_cache));
|
TORCH_BOX(&cp_gather_and_upconvert_fp8_kv_cache));
|
||||||
|
ops.impl("cp_gather_and_upconvert_nvfp4_kv_cache",
|
||||||
|
TORCH_BOX(&cp_gather_and_upconvert_nvfp4_kv_cache));
|
||||||
ops.impl("indexer_k_quant_and_cache", TORCH_BOX(&indexer_k_quant_and_cache));
|
ops.impl("indexer_k_quant_and_cache", TORCH_BOX(&indexer_k_quant_and_cache));
|
||||||
ops.impl("concat_mla_q", TORCH_BOX(&concat_mla_q));
|
ops.impl("concat_mla_q", TORCH_BOX(&concat_mla_q));
|
||||||
ops.impl("cp_gather_indexer_k_quant_cache",
|
ops.impl("cp_gather_indexer_k_quant_cache",
|
||||||
|
|||||||
@@ -907,6 +907,157 @@ def test_concat_and_cache_ds_mla(
|
|||||||
torch.testing.assert_close(kv_rope, ref_rope, atol=0.001, rtol=0.1)
|
torch.testing.assert_close(kv_rope, ref_rope, atol=0.001, rtol=0.1)
|
||||||
|
|
||||||
|
|
||||||
|
# Bytes per token for the nvfp4_ds_mla cache layout (see flashmla_sparse.py).
|
||||||
|
NVFP4_DS_MLA_ENTRY_SIZE = 352
|
||||||
|
|
||||||
|
# e2m1 magnitude table; the sign lives in bit 3 of each 4-bit code.
|
||||||
|
E2M1_VALUES = [0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0]
|
||||||
|
|
||||||
|
|
||||||
|
def _unpack_e2m1(packed: torch.Tensor) -> torch.Tensor:
|
||||||
|
"""Unpack uint8 bytes holding two e2m1 codes each into float32 values.
|
||||||
|
|
||||||
|
The low nibble holds the even element, the high nibble the odd one.
|
||||||
|
"""
|
||||||
|
low = packed & 0x0F
|
||||||
|
high = packed >> 4
|
||||||
|
codes = torch.stack([low, high], dim=-1).reshape(-1).long()
|
||||||
|
table = torch.tensor(E2M1_VALUES, dtype=torch.float32, device=packed.device)
|
||||||
|
magnitude = table[codes & 0x7]
|
||||||
|
sign = torch.where((codes & 0x8) != 0, -1.0, 1.0)
|
||||||
|
return sign * magnitude
|
||||||
|
|
||||||
|
|
||||||
|
def _ref_nvfp4_sf_bytes(vals: torch.Tensor, divisor: float) -> torch.Tensor:
|
||||||
|
"""Reference e4m3 scale-factor bytes: e4m3(max(amax_per_16 / divisor, 2^-9)).
|
||||||
|
|
||||||
|
In element-block order (byte s scales elements [16s, 16s + 16)).
|
||||||
|
"""
|
||||||
|
tiles = vals.to(torch.float32).reshape(-1, 16)
|
||||||
|
sf = torch.clamp(tiles.abs().amax(dim=1) / divisor, min=2.0**-9)
|
||||||
|
return sf.to(torch.float8_e4m3fn).view(torch.uint8)
|
||||||
|
|
||||||
|
|
||||||
|
def _nvfp4_sf_element_order(wire_sf_bytes: torch.Tensor) -> torch.Tensor:
|
||||||
|
"""Stored NVFP4 scale-factor byte order -> element-block order.
|
||||||
|
|
||||||
|
The 32 NoPE scale-factor bytes of an nvfp4_ds_mla entry are stored
|
||||||
|
permuted (an 8x4 -> 4x8 transpose): the scale for element block s lives at
|
||||||
|
byte 8 * (s & 3) + (s >> 2), so that the 8 scales one FlashMLA dequant
|
||||||
|
thread needs are contiguous. This undoes that permutation.
|
||||||
|
"""
|
||||||
|
num_sf = wire_sf_bytes.shape[-1]
|
||||||
|
return wire_sf_bytes.unflatten(-1, (4, num_sf // 4)).transpose(-1, -2).flatten(-2)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("kv_lora_rank", KV_LORA_RANKS)
|
||||||
|
@pytest.mark.parametrize("qk_rope_head_dim", QK_ROPE_HEAD_DIMS)
|
||||||
|
@pytest.mark.parametrize("num_tokens", NUM_TOKENS_MLA)
|
||||||
|
@pytest.mark.parametrize("block_size", BLOCK_SIZES_MLA)
|
||||||
|
@pytest.mark.parametrize("num_blocks", NUM_BLOCKS_MLA)
|
||||||
|
@pytest.mark.parametrize("seed", SEEDS)
|
||||||
|
@pytest.mark.parametrize("device", CUDA_DEVICES)
|
||||||
|
@torch.inference_mode()
|
||||||
|
def test_concat_and_cache_nvfp4_ds_mla(
|
||||||
|
kv_lora_rank: int,
|
||||||
|
qk_rope_head_dim: int,
|
||||||
|
num_tokens: int,
|
||||||
|
block_size: int,
|
||||||
|
num_blocks: int,
|
||||||
|
seed: int,
|
||||||
|
device: str,
|
||||||
|
) -> None:
|
||||||
|
if current_platform.is_rocm():
|
||||||
|
pytest.skip("concat_and_cache_mla doesn't support NVFP4 DS-MLA on ROCm")
|
||||||
|
device_capability = current_platform.get_device_capability()
|
||||||
|
if device_capability is None or device_capability.major != 10:
|
||||||
|
pytest.skip("The NVFP4 DS-MLA kv-cache dtype requires SM 10.x")
|
||||||
|
if kv_lora_rank != 512:
|
||||||
|
pytest.skip("The NVFP4 DS-MLA layout requires kv_lora_rank == 512")
|
||||||
|
dtype = torch.bfloat16
|
||||||
|
set_random_seed(seed)
|
||||||
|
torch.set_default_device(device)
|
||||||
|
torch.accelerator.set_device_index(device)
|
||||||
|
|
||||||
|
kv_cache_dtype = "nvfp4_ds_mla"
|
||||||
|
entry_size = NVFP4_DS_MLA_ENTRY_SIZE
|
||||||
|
|
||||||
|
total_slots = num_blocks * block_size
|
||||||
|
slot_mapping_lst = random.sample(range(total_slots), num_tokens)
|
||||||
|
slot_mapping = torch.tensor(slot_mapping_lst, dtype=torch.long, device=device)
|
||||||
|
|
||||||
|
kv_c = torch.randn(num_tokens, kv_lora_rank, dtype=dtype, device=device)
|
||||||
|
k_pe = torch.randn(num_tokens, qk_rope_head_dim, dtype=dtype, device=device)
|
||||||
|
|
||||||
|
scale = torch.tensor(1.0, dtype=torch.float32, device=device)
|
||||||
|
kv_cache = _create_mla_cache(
|
||||||
|
num_blocks,
|
||||||
|
block_size,
|
||||||
|
entry_size,
|
||||||
|
dtype=torch.uint8,
|
||||||
|
kv_cache_dtype=kv_cache_dtype,
|
||||||
|
device=device,
|
||||||
|
)
|
||||||
|
|
||||||
|
opcheck(
|
||||||
|
torch.ops._C_cache_ops.concat_and_cache_mla,
|
||||||
|
(kv_c, k_pe, kv_cache, slot_mapping, kv_cache_dtype, scale),
|
||||||
|
test_utils=DEFAULT_OPCHECK_TEST_UTILS,
|
||||||
|
)
|
||||||
|
|
||||||
|
ops.concat_and_cache_mla(kv_c, k_pe, kv_cache, slot_mapping, kv_cache_dtype, scale)
|
||||||
|
|
||||||
|
# Byte layout of a single cache entry:
|
||||||
|
# [0, 256) 512 e2m1 NoPE values packed 2/byte
|
||||||
|
# [256, 320) 64 unscaled e4m3 RoPE values
|
||||||
|
# [nope_sf_off, 352) 32 e4m3 NoPE SFs (one per 16 elements), stored
|
||||||
|
# permuted (see _nvfp4_sf_element_order)
|
||||||
|
nope_bytes = kv_lora_rank // 2
|
||||||
|
rope_bytes = qk_rope_head_dim
|
||||||
|
nope_sf_off = nope_bytes + rope_bytes
|
||||||
|
num_nope_sf = kv_lora_rank // 16
|
||||||
|
assert nope_sf_off + num_nope_sf == entry_size
|
||||||
|
|
||||||
|
for i in range(num_tokens):
|
||||||
|
slot = slot_mapping[i].item()
|
||||||
|
block_idx = slot // block_size
|
||||||
|
block_offset = slot % block_size
|
||||||
|
entry = kv_cache[block_idx, block_offset]
|
||||||
|
|
||||||
|
kv_c_ref = kv_c[i].to(torch.float32)
|
||||||
|
k_pe_ref = k_pe[i].to(torch.float32)
|
||||||
|
|
||||||
|
# Scale-factor bytes must match e4m3(max(amax_per_16 / divisor, 2^-9))
|
||||||
|
# computed in python, allowing a 1-ulp e4m3 difference (all SFs are
|
||||||
|
# positive, so e4m3 bit patterns are monotonic and adjacent codes
|
||||||
|
# differ by 1).
|
||||||
|
nope_sf_bytes = _nvfp4_sf_element_order(
|
||||||
|
entry[nope_sf_off : nope_sf_off + num_nope_sf]
|
||||||
|
)
|
||||||
|
ref_nope_sf_bytes = _ref_nvfp4_sf_bytes(kv_c_ref, 6.0)
|
||||||
|
assert (
|
||||||
|
(nope_sf_bytes.to(torch.int16) - ref_nope_sf_bytes.to(torch.int16)).abs()
|
||||||
|
<= 1
|
||||||
|
).all()
|
||||||
|
|
||||||
|
# Dequantize with the *stored* scale factors: x = float(code) * float(sf).
|
||||||
|
nope_sf = nope_sf_bytes.view(torch.float8_e4m3fn).to(torch.float32)
|
||||||
|
nope_sf_per_elem = nope_sf.repeat_interleave(16)
|
||||||
|
nope_vals = _unpack_e2m1(entry[:nope_bytes]) * nope_sf_per_elem
|
||||||
|
# e2m1 error bound: the widest grid spacing is 2 (between codes 4 and
|
||||||
|
# 6), so round-to-nearest is off by at most 1.0*sf; the e4m3 rounding
|
||||||
|
# of the SF itself adds at most ~2^-4 relative, so 1.5*sf is safe for
|
||||||
|
# either encode convention (quantized or unquantized SF).
|
||||||
|
assert ((nope_vals - kv_c_ref).abs() <= 1.5 * nope_sf_per_elem).all()
|
||||||
|
|
||||||
|
# Plain unscaled e4m3: ~2^-4 relative rounding, with a small absolute
|
||||||
|
# floor for values in the subnormal range.
|
||||||
|
rope_payload = entry[nope_bytes:nope_sf_off]
|
||||||
|
rope_vals = rope_payload.view(torch.float8_e4m3fn).to(torch.float32)
|
||||||
|
rope_tol = k_pe_ref.abs() * 2.0**-3 + 2.0**-9
|
||||||
|
assert ((rope_vals - k_pe_ref).abs() <= rope_tol).all()
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("kv_lora_rank", KV_LORA_RANKS)
|
@pytest.mark.parametrize("kv_lora_rank", KV_LORA_RANKS)
|
||||||
@pytest.mark.parametrize("qk_rope_head_dim", QK_ROPE_HEAD_DIMS)
|
@pytest.mark.parametrize("qk_rope_head_dim", QK_ROPE_HEAD_DIMS)
|
||||||
@pytest.mark.parametrize("block_size", BLOCK_SIZES_MLA)
|
@pytest.mark.parametrize("block_size", BLOCK_SIZES_MLA)
|
||||||
|
|||||||
@@ -505,7 +505,10 @@ def create_and_prepopulate_kv_cache(
|
|||||||
block_table = common_attn_metadata.block_table_tensor
|
block_table = common_attn_metadata.block_table_tensor
|
||||||
slot_mapping = common_attn_metadata.slot_mapping
|
slot_mapping = common_attn_metadata.slot_mapping
|
||||||
|
|
||||||
fp8_attention = kv_cache_dtype and kv_cache_dtype.startswith("fp8")
|
use_nvfp4_ds_mla = kv_cache_dtype == "nvfp4_ds_mla"
|
||||||
|
fp8_attention = (
|
||||||
|
bool(kv_cache_dtype and kv_cache_dtype.startswith("fp8")) or use_nvfp4_ds_mla
|
||||||
|
)
|
||||||
use_fp8_ds_mla = kv_cache_dtype == "fp8_ds_mla"
|
use_fp8_ds_mla = kv_cache_dtype == "fp8_ds_mla"
|
||||||
|
|
||||||
if fp8_attention:
|
if fp8_attention:
|
||||||
@@ -515,6 +518,14 @@ def create_and_prepopulate_kv_cache(
|
|||||||
# 4 * 4: 4 float32 scale values for 128-element tiles
|
# 4 * 4: 4 float32 scale values for 128-element tiles
|
||||||
# 2 * rope_dim: 16-bit RoPE values
|
# 2 * rope_dim: 16-bit RoPE values
|
||||||
kv_entry_size = kv_lora_rank + 4 * 4 + 2 * rope_dim
|
kv_entry_size = kv_lora_rank + 4 * 4 + 2 * rope_dim
|
||||||
|
elif use_nvfp4_ds_mla:
|
||||||
|
kv_lora_rank = kv_c_contexts[0].shape[-1]
|
||||||
|
rope_dim = k_pe_contexts[0].shape[-1]
|
||||||
|
# e2m1-packed NoPE + unscaled e4m3 rope + per-16 e4m3 NoPE SFs,
|
||||||
|
# padded to 16B
|
||||||
|
kv_entry_size = (
|
||||||
|
cdiv(kv_lora_rank // 2 + rope_dim + kv_lora_rank // 16, 16) * 16
|
||||||
|
)
|
||||||
else:
|
else:
|
||||||
kv_entry_size = head_size
|
kv_entry_size = head_size
|
||||||
|
|
||||||
|
|||||||
@@ -40,6 +40,9 @@ if not current_platform.is_cuda():
|
|||||||
allow_module_level=True,
|
allow_module_level=True,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
from vllm.model_executor.layers.attention.mla_attention import (
|
||||||
|
_canonicalize_sparse_mla_kv_cache_dtype,
|
||||||
|
)
|
||||||
from vllm.utils.math_utils import cdiv
|
from vllm.utils.math_utils import cdiv
|
||||||
from vllm.v1.attention.backends.mla.flashinfer_mla_sparse import (
|
from vllm.v1.attention.backends.mla.flashinfer_mla_sparse import (
|
||||||
FlashInferMLASparseTRTLLMBackend,
|
FlashInferMLASparseTRTLLMBackend,
|
||||||
@@ -184,13 +187,89 @@ def _quantize_dequantize_fp8_ds_mla(
|
|||||||
return dequant_kv_c, dequant_k_pe
|
return dequant_kv_c, dequant_k_pe
|
||||||
|
|
||||||
|
|
||||||
|
_E2M1_TABLE = [0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0]
|
||||||
|
|
||||||
|
|
||||||
|
def _quantize_dequantize_nvfp4_ds_mla(
|
||||||
|
kv_c: torch.Tensor,
|
||||||
|
k_pe: torch.Tensor,
|
||||||
|
block_size: int,
|
||||||
|
kv_cache_dtype: str,
|
||||||
|
scale: torch.Tensor,
|
||||||
|
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||||
|
"""Round-trip kv_c/k_pe through the nvfp4_ds_mla cache layout.
|
||||||
|
|
||||||
|
Layout (per token, uint8): [e2m1-packed NoPE | unscaled e4m3 RoPE |
|
||||||
|
per-16 e4m3 NoPE SFs]. The SF bytes are stored permuted (an 8x4 -> 4x8
|
||||||
|
transpose: the scale for element block s lives at byte 8 * (s & 3) +
|
||||||
|
(s >> 2)) so that the 8 scales one FlashMLA dequant thread needs are
|
||||||
|
contiguous. Dequant: x = value * float(sf), which is exact in bf16, so
|
||||||
|
this python dequant matches the kernels bit-for-bit.
|
||||||
|
"""
|
||||||
|
if kv_c.numel() == 0:
|
||||||
|
return kv_c.clone(), k_pe.clone()
|
||||||
|
|
||||||
|
kv_lora_rank = kv_c.shape[-1]
|
||||||
|
rope_dim = k_pe.shape[-1]
|
||||||
|
num_tokens = kv_c.shape[0]
|
||||||
|
num_blocks = max(1, math.ceil(num_tokens / block_size))
|
||||||
|
entry_size = (
|
||||||
|
math.ceil((kv_lora_rank // 2 + rope_dim + kv_lora_rank // 16) / 16) * 16
|
||||||
|
)
|
||||||
|
sf_nope_off = kv_lora_rank // 2 + rope_dim
|
||||||
|
|
||||||
|
tmp_cache = torch.zeros(
|
||||||
|
num_blocks, block_size, entry_size, dtype=torch.uint8, device=kv_c.device
|
||||||
|
)
|
||||||
|
slot_mapping = torch.arange(num_tokens, dtype=torch.long, device=kv_c.device)
|
||||||
|
ops.concat_and_cache_mla(
|
||||||
|
kv_c, k_pe, tmp_cache, slot_mapping, kv_cache_dtype=kv_cache_dtype, scale=scale
|
||||||
|
)
|
||||||
|
|
||||||
|
tokens = tmp_cache.view(-1, entry_size)[:num_tokens]
|
||||||
|
table = torch.tensor(
|
||||||
|
_E2M1_TABLE + [-v for v in _E2M1_TABLE],
|
||||||
|
dtype=torch.float32,
|
||||||
|
device=kv_c.device,
|
||||||
|
)
|
||||||
|
|
||||||
|
def unpack_e2m1(packed: torch.Tensor) -> torch.Tensor:
|
||||||
|
lo = (packed & 0xF).long()
|
||||||
|
hi = (packed >> 4).long()
|
||||||
|
return table[torch.stack([lo, hi], dim=-1).flatten(-2)]
|
||||||
|
|
||||||
|
nope_vals = unpack_e2m1(tokens[:, : kv_lora_rank // 2])
|
||||||
|
num_nope_sf = kv_lora_rank // 16
|
||||||
|
nope_sf = (
|
||||||
|
# undo the on-wire SF permutation -> element-block order
|
||||||
|
tokens[:, sf_nope_off : sf_nope_off + num_nope_sf]
|
||||||
|
.unflatten(-1, (4, num_nope_sf // 4))
|
||||||
|
.transpose(-1, -2)
|
||||||
|
.flatten(-2)
|
||||||
|
.view(torch.float8_e4m3fn)
|
||||||
|
.float()
|
||||||
|
)
|
||||||
|
dequant_kv_c = (
|
||||||
|
(nope_vals.unflatten(-1, (-1, 16)) * nope_sf.unsqueeze(-1)).flatten(-2)
|
||||||
|
).to(kv_c.dtype)
|
||||||
|
|
||||||
|
# RoPE: plain e4m3, no scale factor
|
||||||
|
rope_raw = tokens[:, kv_lora_rank // 2 : sf_nope_off]
|
||||||
|
dequant_k_pe = rope_raw.view(torch.float8_e4m3fn).to(k_pe.dtype)
|
||||||
|
|
||||||
|
return dequant_kv_c, dequant_k_pe
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
"backend_cls",
|
"backend_cls",
|
||||||
[FlashMLASparseBackend, FlashInferMLASparseTRTLLMBackend],
|
[FlashMLASparseBackend, FlashInferMLASparseTRTLLMBackend],
|
||||||
ids=["FlashMLA", "FlashInferTRTLLM"],
|
ids=["FlashMLA", "FlashInferTRTLLM"],
|
||||||
)
|
)
|
||||||
@pytest.mark.parametrize("batch_name", list(SPARSE_BACKEND_BATCH_SPECS.keys()))
|
@pytest.mark.parametrize("batch_name", list(SPARSE_BACKEND_BATCH_SPECS.keys()))
|
||||||
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8", "fp8_ds_mla"])
|
@pytest.mark.parametrize(
|
||||||
|
"kv_cache_dtype",
|
||||||
|
["auto", "fp8", "fp8_ds_mla", "nvfp4_ds_mla"],
|
||||||
|
)
|
||||||
@pytest.mark.parametrize("tensor_parallel_size", [1, 2, 4])
|
@pytest.mark.parametrize("tensor_parallel_size", [1, 2, 4])
|
||||||
@pytest.mark.parametrize("block_size", [32, 64])
|
@pytest.mark.parametrize("block_size", [32, 64])
|
||||||
@pytest.mark.parametrize(("q_scale", "k_scale"), [(1.0, 1.0), (2.0, 3.0)])
|
@pytest.mark.parametrize(("q_scale", "k_scale"), [(1.0, 1.0), (2.0, 3.0)])
|
||||||
@@ -219,6 +298,17 @@ def test_sparse_backend_decode_correctness(
|
|||||||
"fp8_ds_mla kv-cache dtype"
|
"fp8_ds_mla kv-cache dtype"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
if kv_cache_dtype == "nvfp4_ds_mla":
|
||||||
|
# Unlike "fp8" (an alias canonicalized to "fp8_ds_mla" above), the
|
||||||
|
# NVFP4 DS-MLA dtype must reach the backend unchanged.
|
||||||
|
assert (
|
||||||
|
_canonicalize_sparse_mla_kv_cache_dtype(backend_cls, kv_cache_dtype)
|
||||||
|
== kv_cache_dtype
|
||||||
|
)
|
||||||
|
device_capability = current_platform.get_device_capability()
|
||||||
|
if device_capability is None or device_capability.major != 10:
|
||||||
|
pytest.skip("The NVFP4 DS-MLA kv-cache dtype requires SM 10.x")
|
||||||
|
|
||||||
supported_block_sizes = backend_cls.get_supported_kernel_block_sizes()
|
supported_block_sizes = backend_cls.get_supported_kernel_block_sizes()
|
||||||
if block_size not in supported_block_sizes:
|
if block_size not in supported_block_sizes:
|
||||||
pytest.skip(
|
pytest.skip(
|
||||||
@@ -238,6 +328,7 @@ def test_sparse_backend_decode_correctness(
|
|||||||
|
|
||||||
batch_spec = SPARSE_BACKEND_BATCH_SPECS[batch_name]
|
batch_spec = SPARSE_BACKEND_BATCH_SPECS[batch_name]
|
||||||
use_fp8_ds_mla_quantization = kv_cache_dtype == "fp8_ds_mla"
|
use_fp8_ds_mla_quantization = kv_cache_dtype == "fp8_ds_mla"
|
||||||
|
use_nvfp4_ds_mla_quantization = kv_cache_dtype == "nvfp4_ds_mla"
|
||||||
|
|
||||||
device = torch.device(DEVICE_TYPE)
|
device = torch.device(DEVICE_TYPE)
|
||||||
dtype = torch.bfloat16
|
dtype = torch.bfloat16
|
||||||
@@ -381,6 +472,15 @@ def test_sparse_backend_decode_correctness(
|
|||||||
simulate_sm100_e8m0_scales=is_sm100,
|
simulate_sm100_e8m0_scales=is_sm100,
|
||||||
)
|
)
|
||||||
k_pe_full = k_pe_squeezed.unsqueeze(1)
|
k_pe_full = k_pe_squeezed.unsqueeze(1)
|
||||||
|
elif use_nvfp4_ds_mla_quantization:
|
||||||
|
kv_c_full, k_pe_squeezed = _quantize_dequantize_nvfp4_ds_mla(
|
||||||
|
kv_c_full,
|
||||||
|
k_pe_full.squeeze(1),
|
||||||
|
block_size=block_size,
|
||||||
|
kv_cache_dtype=kv_cache_dtype,
|
||||||
|
scale=kv_cache_scale,
|
||||||
|
)
|
||||||
|
k_pe_full = k_pe_squeezed.unsqueeze(1)
|
||||||
|
|
||||||
q_nope, q_pe = q_c.split([qk_nope_head_dim, qk_rope_head_dim], dim=-1)
|
q_nope, q_pe = q_c.split([qk_nope_head_dim, qk_rope_head_dim], dim=-1)
|
||||||
ql_nope = torch.einsum("qnh,lnh->qnl", q_nope, W_UK)
|
ql_nope = torch.einsum("qnh,lnh->qnl", q_nope, W_UK)
|
||||||
|
|||||||
@@ -3082,6 +3082,31 @@ def cp_gather_and_upconvert_fp8_kv_cache(
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def cp_gather_and_upconvert_nvfp4_kv_cache(
|
||||||
|
src_cache: torch.Tensor,
|
||||||
|
dst: torch.Tensor,
|
||||||
|
block_table: torch.Tensor,
|
||||||
|
workspace_starts: torch.Tensor,
|
||||||
|
batch_size: int,
|
||||||
|
) -> None:
|
||||||
|
"""Gather and upconvert an nvfp4_ds_mla KV cache to a BF16 workspace.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
src_cache: NVFP4 KV cache [num_blocks, block_size, 352] (uint8)
|
||||||
|
dst: BF16 output workspace [total_tokens, 576]
|
||||||
|
block_table: Block indices [num_reqs, max_blocks]
|
||||||
|
workspace_starts: Workspace start offsets [num_reqs]
|
||||||
|
batch_size: Number of requests
|
||||||
|
"""
|
||||||
|
torch.ops._C_cache_ops.cp_gather_and_upconvert_nvfp4_kv_cache(
|
||||||
|
src_cache,
|
||||||
|
dst,
|
||||||
|
block_table,
|
||||||
|
workspace_starts,
|
||||||
|
batch_size,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def concat_mla_q(
|
def concat_mla_q(
|
||||||
ql_nope: torch.Tensor,
|
ql_nope: torch.Tensor,
|
||||||
q_pe: torch.Tensor,
|
q_pe: torch.Tensor,
|
||||||
|
|||||||
@@ -45,6 +45,7 @@ CacheDType = Literal[
|
|||||||
"fp8_e5m2",
|
"fp8_e5m2",
|
||||||
"fp8_inc",
|
"fp8_inc",
|
||||||
"fp8_ds_mla",
|
"fp8_ds_mla",
|
||||||
|
"nvfp4_ds_mla",
|
||||||
"turboquant_k8v4",
|
"turboquant_k8v4",
|
||||||
"turboquant_4bit_nc",
|
"turboquant_4bit_nc",
|
||||||
"turboquant_k3v4_nc",
|
"turboquant_k3v4_nc",
|
||||||
|
|||||||
+5
-1
@@ -2821,14 +2821,18 @@ class VllmConfig:
|
|||||||
def validate_nvfp4_kv_cache_with_mla(self) -> "VllmConfig":
|
def validate_nvfp4_kv_cache_with_mla(self) -> "VllmConfig":
|
||||||
if self.model_config is None:
|
if self.model_config is None:
|
||||||
return self
|
return self
|
||||||
|
# The ds_mla layouts are MLA-only by construction; the plain nvfp4
|
||||||
|
# layout (head_size//2 + head_size//16) does not apply to MLA.
|
||||||
if (
|
if (
|
||||||
self.cache_config.cache_dtype.startswith("nvfp4")
|
self.cache_config.cache_dtype.startswith("nvfp4")
|
||||||
|
and not self.cache_config.cache_dtype.endswith("_ds_mla")
|
||||||
and self.model_config.use_mla
|
and self.model_config.use_mla
|
||||||
):
|
):
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
"nvfp4 KV cache is not supported with MLA (Multi-head Latent "
|
"nvfp4 KV cache is not supported with MLA (Multi-head Latent "
|
||||||
"Attention) backends. Please use a different --kv-cache-dtype "
|
"Attention) backends. Please use a different --kv-cache-dtype "
|
||||||
"(e.g., 'fp8' or 'auto') for MLA models such as DeepSeek."
|
"(e.g., 'fp8', 'auto', or 'nvfp4_ds_mla' with a sparse MLA "
|
||||||
|
"backend) for MLA models such as DeepSeek."
|
||||||
)
|
)
|
||||||
return self
|
return self
|
||||||
|
|
||||||
|
|||||||
@@ -352,6 +352,10 @@ def _canonicalize_sparse_mla_kv_cache_dtype(
|
|||||||
) -> CacheDType:
|
) -> CacheDType:
|
||||||
backend_name = attn_backend.get_name()
|
backend_name = attn_backend.get_name()
|
||||||
if backend_name == "FLASHMLA_SPARSE" and is_quantized_kv_cache(kv_cache_dtype):
|
if backend_name == "FLASHMLA_SPARSE" and is_quantized_kv_cache(kv_cache_dtype):
|
||||||
|
# The NVFP4 DS-MLA format is used as-is; any other quantized dtype
|
||||||
|
# (fp8, fp8_e4m3, ...) is served via the fp8_ds_mla format.
|
||||||
|
if kv_cache_dtype == "nvfp4_ds_mla":
|
||||||
|
return kv_cache_dtype
|
||||||
return "fp8_ds_mla"
|
return "fp8_ds_mla"
|
||||||
if backend_name == "FLASHINFER_MLA_SPARSE_SM120" and kv_cache_dtype in (
|
if backend_name == "FLASHINFER_MLA_SPARSE_SM120" and kv_cache_dtype in (
|
||||||
"auto",
|
"auto",
|
||||||
@@ -861,7 +865,11 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
|||||||
k_c_normed = k_c_normed[:num_actual_toks, ...]
|
k_c_normed = k_c_normed[:num_actual_toks, ...]
|
||||||
k_pe = k_pe[:num_actual_toks, ...]
|
k_pe = k_pe[:num_actual_toks, ...]
|
||||||
|
|
||||||
if fp8_attention and self.kv_cache_dtype != "fp8_ds_mla":
|
if fp8_attention and self.kv_cache_dtype not in (
|
||||||
|
# Opaque per-token byte formats stay as raw uint8
|
||||||
|
"fp8_ds_mla",
|
||||||
|
"nvfp4_ds_mla",
|
||||||
|
):
|
||||||
kv_cache = kv_cache.view(current_platform.fp8_dtype())
|
kv_cache = kv_cache.view(current_platform.fp8_dtype())
|
||||||
|
|
||||||
assert (
|
assert (
|
||||||
@@ -1262,9 +1270,12 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
|||||||
dtype=kv_cache_dtype,
|
dtype=kv_cache_dtype,
|
||||||
cache_dtype_str=self.kv_cache_dtype,
|
cache_dtype_str=self.kv_cache_dtype,
|
||||||
kv_quant_mode=get_kv_quant_mode(self.kv_cache_dtype),
|
kv_quant_mode=get_kv_quant_mode(self.kv_cache_dtype),
|
||||||
# fp8_ds_mla: 656-byte custom layout (kv_lora_rank=512 +
|
# ds_mla layouts pack NoPE + RoPE + scales into one opaque per-token
|
||||||
# qk_rope_head_dim=64, head_size=576). See flashmla_sparse.py.
|
# blob, so the size is not derivable from head_size.
|
||||||
state_content_bytes=656 if self.kv_cache_dtype == "fp8_ds_mla" else None,
|
# See flashmla_sparse.py.
|
||||||
|
state_content_bytes={"fp8_ds_mla": 656, "nvfp4_ds_mla": 352}.get(
|
||||||
|
self.kv_cache_dtype
|
||||||
|
),
|
||||||
)
|
)
|
||||||
if self.sliding_window is not None:
|
if self.sliding_window is not None:
|
||||||
return SlidingWindowMLASpec(
|
return SlidingWindowMLASpec(
|
||||||
|
|||||||
@@ -45,6 +45,7 @@ STR_DTYPE_TO_TORCH_DTYPE = {
|
|||||||
"fp8_per_token_head": torch.uint8,
|
"fp8_per_token_head": torch.uint8,
|
||||||
"fp8_inc": torch.float8_e4m3fn,
|
"fp8_inc": torch.float8_e4m3fn,
|
||||||
"fp8_ds_mla": torch.uint8,
|
"fp8_ds_mla": torch.uint8,
|
||||||
|
"nvfp4_ds_mla": torch.uint8,
|
||||||
"turboquant_k8v4": torch.uint8,
|
"turboquant_k8v4": torch.uint8,
|
||||||
"turboquant_4bit_nc": torch.uint8,
|
"turboquant_4bit_nc": torch.uint8,
|
||||||
"turboquant_k3v4_nc": torch.uint8,
|
"turboquant_k3v4_nc": torch.uint8,
|
||||||
|
|||||||
@@ -84,8 +84,28 @@ Bytes, structured as:
|
|||||||
- **Last 8 bytes:** Scale factors, containing 7 `ue8m0` values + 1B pad.
|
- **Last 8 bytes:** Scale factors, containing 7 `ue8m0` values + 1B pad.
|
||||||
The first `ue8m0` is the scale for the first 64 `float8_e4m3` values,
|
The first `ue8m0` is the scale for the first 64 `float8_e4m3` values,
|
||||||
the second for the next 64, and so on.
|
the second for the next 64, and so on.
|
||||||
|
|
||||||
|
In the "nvfp4_ds_mla" format (SM100 only, DeepSeek V3.2 geometry), each
|
||||||
|
token's KV cache is 352 Bytes, structured as:
|
||||||
|
- **First 256 bytes:** 512 `e2m1` NoPE values packed 2/byte (low nibble =
|
||||||
|
even element).
|
||||||
|
- **Next 64 bytes:** 64 `float8_e4m3` RoPE values. These carry no scale
|
||||||
|
factor: `e4m3`'s 4 exponent bits span the RoPE magnitude range unaided.
|
||||||
|
- **Last 32 bytes:** 32 `float8_e4m3` NoPE scale factors, one per 16
|
||||||
|
elements, stored permuted (an 8x4 -> 4x8 transpose: the scale for element
|
||||||
|
block `s` lives at byte `8 * (s & 3) + (s >> 2)`) so that the 8 scales one
|
||||||
|
FlashMLA dequant thread needs are contiguous. See the layout comment in
|
||||||
|
`csrc/libtorch_stable/cache_kernels.cu`.
|
||||||
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
# Quantized DS-MLA cache formats served by the FP8/NVFP4 sparse decode kernel
|
||||||
|
# path (as opposed to the plain bf16 cache). FlashMLA infers which of these the
|
||||||
|
# cache holds from its bytes-per-token, so nothing else needs to be passed down.
|
||||||
|
QUANTIZED_DS_MLA_CACHE_FORMATS: frozenset[str] = frozenset(
|
||||||
|
{"fp8_ds_mla", "nvfp4_ds_mla"}
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class FlashMLASparseBackend(AttentionBackend):
|
class FlashMLASparseBackend(AttentionBackend):
|
||||||
supported_dtypes: ClassVar[list[torch.dtype]] = [torch.bfloat16]
|
supported_dtypes: ClassVar[list[torch.dtype]] = [torch.bfloat16]
|
||||||
@@ -94,6 +114,7 @@ class FlashMLASparseBackend(AttentionBackend):
|
|||||||
"bfloat16",
|
"bfloat16",
|
||||||
"fp8_ds_mla",
|
"fp8_ds_mla",
|
||||||
"fp8", # alias for fp8_ds_mla
|
"fp8", # alias for fp8_ds_mla
|
||||||
|
"nvfp4_ds_mla", # NVFP4 NoPE + FP8 RoPE (SM100 only)
|
||||||
]
|
]
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
@@ -129,6 +150,26 @@ class FlashMLASparseBackend(AttentionBackend):
|
|||||||
def supports_compute_capability(cls, capability: DeviceCapability) -> bool:
|
def supports_compute_capability(cls, capability: DeviceCapability) -> bool:
|
||||||
return capability.major in [9, 10]
|
return capability.major in [9, 10]
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def supports_combination(
|
||||||
|
cls,
|
||||||
|
head_size: int,
|
||||||
|
dtype: torch.dtype,
|
||||||
|
kv_cache_dtype: "CacheDType | None",
|
||||||
|
block_size: int | None,
|
||||||
|
use_mla: bool,
|
||||||
|
has_sink: bool,
|
||||||
|
use_sparse: bool,
|
||||||
|
use_mm_prefix: bool,
|
||||||
|
device_capability: DeviceCapability,
|
||||||
|
) -> str | None:
|
||||||
|
if kv_cache_dtype == "nvfp4_ds_mla" and device_capability.major != 10:
|
||||||
|
return (
|
||||||
|
f"FLASHMLA_SPARSE only supports the {kv_cache_dtype} kv-cache "
|
||||||
|
"dtype on SM100 (Blackwell)"
|
||||||
|
)
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class FlashMLASparseMetadata(AttentionMetadata):
|
class FlashMLASparseMetadata(AttentionMetadata):
|
||||||
@@ -258,7 +299,9 @@ class FlashMLASparseMetadataBuilder(
|
|||||||
FlashMLASparseImpl._compute_fp8_decode_padded_heads(self.num_heads)
|
FlashMLASparseImpl._compute_fp8_decode_padded_heads(self.num_heads)
|
||||||
)
|
)
|
||||||
|
|
||||||
self.use_fp8_kv_cache = cache_config.cache_dtype == "fp8_ds_mla"
|
self.use_fp8_kv_cache = (
|
||||||
|
cache_config.cache_dtype in QUANTIZED_DS_MLA_CACHE_FORMATS
|
||||||
|
)
|
||||||
max_num_seqs = vllm_config.scheduler_config.max_num_seqs
|
max_num_seqs = vllm_config.scheduler_config.max_num_seqs
|
||||||
# Shape: [max_num_seqs], all elements = topk_tokens (constant for full-CG)
|
# Shape: [max_num_seqs], all elements = topk_tokens (constant for full-CG)
|
||||||
self.topk_tokens_tensor = torch.full(
|
self.topk_tokens_tensor = torch.full(
|
||||||
@@ -596,9 +639,10 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
|
|||||||
)
|
)
|
||||||
q_concat_shape = (max_tokens, q_concat_heads, head_size)
|
q_concat_shape = (max_tokens, q_concat_heads, head_size)
|
||||||
if is_quantized_kv_cache(kv_cache_dtype):
|
if is_quantized_kv_cache(kv_cache_dtype):
|
||||||
assert kv_cache_dtype == "fp8_ds_mla", (
|
assert kv_cache_dtype in QUANTIZED_DS_MLA_CACHE_FORMATS, (
|
||||||
"FlashMLA Sparse Attention backend fp8 only supports "
|
"FlashMLA Sparse Attention backend only supports the "
|
||||||
"fp8_ds_mla kv-cache dtype"
|
f"{sorted(QUANTIZED_DS_MLA_CACHE_FORMATS)} quantized kv-cache "
|
||||||
|
f"dtypes, got {kv_cache_dtype}"
|
||||||
)
|
)
|
||||||
|
|
||||||
if self.need_to_return_lse_for_decode and not is_quantized_kv_cache(
|
if self.need_to_return_lse_for_decode and not is_quantized_kv_cache(
|
||||||
@@ -609,7 +653,7 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
|
|||||||
"the bf16 sparse path is not supported under DCP."
|
"the bf16 sparse path is not supported under DCP."
|
||||||
)
|
)
|
||||||
|
|
||||||
if kv_cache_dtype == "fp8_ds_mla":
|
if kv_cache_dtype in QUANTIZED_DS_MLA_CACHE_FORMATS:
|
||||||
# Reserve workspace during initialization
|
# Reserve workspace during initialization
|
||||||
assert vllm_config is not None and vllm_config.model_config is not None
|
assert vllm_config is not None and vllm_config.model_config is not None
|
||||||
prefill_workspace_size = get_prefill_workspace_size(
|
prefill_workspace_size = get_prefill_workspace_size(
|
||||||
@@ -749,13 +793,22 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
|
|||||||
assert fp8_metadata.prefill is not None
|
assert fp8_metadata.prefill is not None
|
||||||
for chunk in fp8_metadata.prefill.chunks:
|
for chunk in fp8_metadata.prefill.chunks:
|
||||||
chunk_workspace = self.prefill_bf16_workspace[: chunk.chunk_tot_seqlen]
|
chunk_workspace = self.prefill_bf16_workspace[: chunk.chunk_tot_seqlen]
|
||||||
ops.cp_gather_and_upconvert_fp8_kv_cache(
|
if self.kv_cache_dtype == "fp8_ds_mla":
|
||||||
kv_c_and_k_pe_cache,
|
ops.cp_gather_and_upconvert_fp8_kv_cache(
|
||||||
chunk_workspace,
|
kv_c_and_k_pe_cache,
|
||||||
chunk.block_table,
|
chunk_workspace,
|
||||||
chunk.workspace_starts,
|
chunk.block_table,
|
||||||
len(chunk.block_table),
|
chunk.workspace_starts,
|
||||||
)
|
len(chunk.block_table),
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
ops.cp_gather_and_upconvert_nvfp4_kv_cache(
|
||||||
|
kv_c_and_k_pe_cache.view(torch.uint8),
|
||||||
|
chunk_workspace,
|
||||||
|
chunk.block_table,
|
||||||
|
chunk.workspace_starts,
|
||||||
|
len(chunk.block_table),
|
||||||
|
)
|
||||||
|
|
||||||
chunk_q = q[chunk.tokens_slice]
|
chunk_q = q[chunk.tokens_slice]
|
||||||
chunk_topk_indices_workspace = topk_indices[chunk.tokens_slice]
|
chunk_topk_indices_workspace = topk_indices[chunk.tokens_slice]
|
||||||
@@ -953,7 +1006,7 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
|
|||||||
assert self.topk_indices_buffer is not None
|
assert self.topk_indices_buffer is not None
|
||||||
topk_indices = self.topk_indices_buffer[:num_actual_toks]
|
topk_indices = self.topk_indices_buffer[:num_actual_toks]
|
||||||
|
|
||||||
use_fp8_cache = self.kv_cache_dtype == "fp8_ds_mla"
|
use_fp8_cache = self.kv_cache_dtype in QUANTIZED_DS_MLA_CACHE_FORMATS
|
||||||
|
|
||||||
lse: torch.Tensor | None = None
|
lse: torch.Tensor | None = None
|
||||||
|
|
||||||
|
|||||||
@@ -54,6 +54,7 @@ class KVQuantMode(IntEnum):
|
|||||||
TURBOQUANT_4BIT_NC = 7
|
TURBOQUANT_4BIT_NC = 7
|
||||||
TURBOQUANT_K3V4_NC = 8
|
TURBOQUANT_K3V4_NC = 8
|
||||||
TURBOQUANT_3BIT_NC = 9
|
TURBOQUANT_3BIT_NC = 9
|
||||||
|
NVFP4_DS_MLA = 10 # opaque-bytes NVFP4 DS-MLA layouts (FlashMLA sparse)
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def is_per_token_head(self) -> bool:
|
def is_per_token_head(self) -> bool:
|
||||||
@@ -88,6 +89,11 @@ def get_kv_quant_mode(kv_cache_dtype: str) -> KVQuantMode:
|
|||||||
return KVQuantMode.INT8_PER_TOKEN_HEAD
|
return KVQuantMode.INT8_PER_TOKEN_HEAD
|
||||||
if kv_cache_dtype == "fp8_per_token_head":
|
if kv_cache_dtype == "fp8_per_token_head":
|
||||||
return KVQuantMode.FP8_PER_TOKEN_HEAD
|
return KVQuantMode.FP8_PER_TOKEN_HEAD
|
||||||
|
# Must precede the ``nvfp4`` prefix test below, which would otherwise match.
|
||||||
|
if kv_cache_dtype == "nvfp4_ds_mla":
|
||||||
|
# Page size is keyed on cache_dtype_str in the MLA specs, not
|
||||||
|
# nvfp4_kv_cache_full_dim.
|
||||||
|
return KVQuantMode.NVFP4_DS_MLA
|
||||||
if kv_cache_dtype.startswith("nvfp4"):
|
if kv_cache_dtype.startswith("nvfp4"):
|
||||||
return KVQuantMode.NVFP4
|
return KVQuantMode.NVFP4
|
||||||
if isinstance(kv_cache_dtype, str) and kv_cache_dtype.startswith("turboquant_"):
|
if isinstance(kv_cache_dtype, str) and kv_cache_dtype.startswith("turboquant_"):
|
||||||
|
|||||||
Reference in New Issue
Block a user