[Model][DSv4.1] Store the whole KV in MXFP8 (FlashMLA V4.1 record) (#56893)

Signed-off-by: Yongye Zhu <yongye@inferact.ai>
Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: Yongye Zhu <yongye@inferact.ai>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Yongye Zhu
2026-09-14 20:12:55 -07:00
committed by GitHub
co-authored by Yongye Zhu Claude Opus 5
parent a529c1a748
commit e6eb0d120c
14 changed files with 709 additions and 110 deletions
@@ -23,9 +23,15 @@
* cos_sin_cache layout [max_pos, rope_dim] = cos || sin (cos first, sin
* second along last dim; each half is rope_dim/2 = 32 values)
*
* Cache layout per paged-cache block (block_size tokens):
* [0, bs*576): token data, 448 fp8 + 128 bf16 each
* [bs*576, bs*576 + bs*8): UE8M0 scales, 7 real + 1 pad per token
* Cache layout per paged-cache block (block_size tokens), selected by the
* `kv_mxfp8` argument:
* V4 (584 B/token, kv_mxfp8=false)
* [0, bs*576): token data, 448 fp8 + 128 bf16 each
* [bs*576, bs*576 + bs*8): UE8M0 scales, 7 real + 1 pad per token
* V4.1 (528 B/token, kv_mxfp8=true) -- RoPE is quantized as well, so the
* whole row is MXFP8 (fp8 e4m3 with one UE8M0 scale per 32 dims)
* [0, bs*512): token data, 512 fp8 each
* [bs*512, bs*512 + bs*16): UE8M0 scales, 16 per token
*/
#include "torch_utils.h"
@@ -91,6 +97,10 @@ constexpr int kQuantBlock = 64;
constexpr int kNumQuantBlocks = kNopeDim / kQuantBlock; // 7
constexpr int kScaleBytesPerToken = kNumQuantBlocks + 1; // 8 (7 real + 1 pad)
constexpr int kTokenDataBytes = kNopeDim + kRopeDim * 2; // 448 + 128 = 576
// V4.1 MXFP8 record: every dim quantized, one UE8M0 scale per 32 dims.
constexpr int kMxfp8QuantBlock = 32;
constexpr int kMxfp8NumQuantBlocks = kHeadDim / kMxfp8QuantBlock; // 16
constexpr int kMxfp8TokenDataBytes = kHeadDim; // 512
// FNUZ on gfx942 / OCP elsewhere. FNUZ uses 224.0 (not the dtype's raw
// 240.0) to match the rest of vLLM's FNUZ pipeline.
#if defined(USE_ROCM) && defined(__gfx942__)
@@ -141,6 +151,11 @@ __device__ __forceinline__ uint4 packFp8E4M3x16(float const* values,
// ────────────────────────────────────────────────────────────────────────────
// Small inline helpers
// ────────────────────────────────────────────────────────────────────────────
__device__ __forceinline__ float warp2MaxAbs(float val) {
// Reduce absolute max across 2 consecutive lanes (32 dims = one MXFP8 block).
return fmaxf(val, __shfl_xor_sync(FINAL_MASK, val, 1));
}
__device__ __forceinline__ float warp4MaxAbs(float val) {
// Reduce absolute max across 4 consecutive lanes (lane id & 3 group).
float peer = __shfl_xor_sync(FINAL_MASK, val, 1);
@@ -176,6 +191,10 @@ __device__ __forceinline__ float warpSum(float val) {
// slot == kNumHeadsQPadded → KV (RoPE + UE8M0 quant
// + paged-cache
// insert)
//
// `kv_mxfp8` is grid-uniform and only steers the KV branch, so it stays a
// runtime argument rather than doubling every (head count, q-norm)
// instantiation of a kernel whose hot path is the Q branch.
template <typename scalar_t_in, int kNumHeadsQPadded, bool APPLY_Q_NORM>
__device__ __forceinline__ void processDeepseekV4Slot(
uint4 v0, uint4 v1, int const tokenIdx, int const slotIdx,
@@ -184,7 +203,7 @@ __device__ __forceinline__ void processDeepseekV4Slot(
uint8_t* __restrict__ k_cache, int64_t const* __restrict__ slot_mapping,
int64_t const* __restrict__ position_ids,
float const* __restrict__ cos_sin_cache, int const cache_block_size,
int const kv_block_stride) {
int const kv_block_stride, bool const kv_mxfp8) {
using Converter = vllm::_typeConvert<scalar_t_in>;
bool const isKV = (slotIdx == kNumHeadsQPadded);
bool const isPadQ = !isKV && (slotIdx >= num_heads_q);
@@ -302,19 +321,24 @@ __device__ __forceinline__ void processDeepseekV4Slot(
*reinterpret_cast<uint4*>(dst) = out0;
*reinterpret_cast<uint4*>(dst + 8) = out1;
} else {
// ── KV: FP8 quant on NoPE + bf16 store on RoPE + cache insert.
// ── KV: FP8 quant + cache insert. V4 keeps the RoPE dims in bf16;
// V4.1 (kv_mxfp8) quantizes them too.
int64_t const slot_id = slot_mapping[tokenIdx];
if (slot_id >= 0) {
int const token_data_bytes =
kv_mxfp8 ? kMxfp8TokenDataBytes : kTokenDataBytes;
int const scale_bytes =
kv_mxfp8 ? kMxfp8NumQuantBlocks : kScaleBytesPerToken;
int64_t const block_idx = slot_id / cache_block_size;
int64_t const pos_in_block = slot_id % cache_block_size;
uint8_t* block_base =
k_cache + block_idx * static_cast<int64_t>(kv_block_stride);
uint8_t* token_fp8_ptr = block_base + pos_in_block * kTokenDataBytes;
uint8_t* token_fp8_ptr = block_base + pos_in_block * token_data_bytes;
uint8_t* token_bf16_ptr = token_fp8_ptr + kNopeDim;
uint8_t* token_scale_ptr =
block_base +
static_cast<int64_t>(cache_block_size) * kTokenDataBytes +
pos_in_block * kScaleBytesPerToken;
static_cast<int64_t>(cache_block_size) * token_data_bytes +
pos_in_block * scale_bytes;
#pragma unroll
for (int i = 0; i < kElemsPerLane; i++) {
@@ -326,11 +350,13 @@ __device__ __forceinline__ void processDeepseekV4Slot(
for (int i = 0; i < kElemsPerLane; i++) {
local_absmax = fmaxf(local_absmax, fabsf(elements[i]));
}
float const absmax = fmaxf(warp4MaxAbs(local_absmax), 1e-4f);
float const absmax = fmaxf(
kv_mxfp8 ? warp2MaxAbs(local_absmax) : warp4MaxAbs(local_absmax),
1e-4f);
float const exponent = ceilf(log2f(absmax / kFp8Max));
float const inv_scale = exp2f(-exponent);
if (!is_rope_lane) {
if (kv_mxfp8 || !is_rope_lane) {
uint8_t out_bytes[kElemsPerLane];
#pragma unroll
for (int i = 0; i < kElemsPerLane; i++) {
@@ -347,13 +373,19 @@ __device__ __forceinline__ void processDeepseekV4Slot(
*reinterpret_cast<uint4*>(token_fp8_ptr + dim_base) =
*reinterpret_cast<uint4 const*>(out_bytes);
if ((laneId & 3) == 0) {
int const q_block_idx = laneId >> 2;
float encoded = fmaxf(fminf(exponent + 127.0f, 255.0f), 0.0f);
token_scale_ptr[q_block_idx] = static_cast<uint8_t>(encoded);
}
if (laneId == 0) {
token_scale_ptr[kNumQuantBlocks] = 0;
float const encoded = fmaxf(fminf(exponent + 127.0f, 255.0f), 0.0f);
if (kv_mxfp8) {
// One scale per 32 dims = one per lane pair.
if ((laneId & 1) == 0) {
token_scale_ptr[laneId >> 1] = static_cast<uint8_t>(encoded);
}
} else {
if ((laneId & 3) == 0) {
token_scale_ptr[laneId >> 2] = static_cast<uint8_t>(encoded);
}
if (laneId == 0) {
token_scale_ptr[kNumQuantBlocks] = 0;
}
}
} else {
uint4 out0, out1;
@@ -421,7 +453,8 @@ __global__ void fusedDeepseekV4QNormRopeKVRopeQuantInsertKernel(
int const num_tokens_insert, // = slot_mapping.size(0), ≤ num_tokens_full
int const num_heads_q, // live Q heads (input layout)
int const cache_block_size, // tokens per paged-cache block
int const kv_block_stride) { // bytes per paged-cache block
int const kv_block_stride, // bytes per paged-cache block
bool const kv_mxfp8) { // V4.1 all-dims MXFP8 record
#if (!defined(__CUDA_ARCH__) || __CUDA_ARCH__ < 800) && !defined(USE_ROCM)
// BF16 _typeConvert specialization is unavailable on pre-Ampere. The
// DeepseekV4 kernel only runs with bf16 inputs in practice, so compile a
@@ -477,7 +510,7 @@ __global__ void fusedDeepseekV4QNormRopeKVRopeQuantInsertKernel(
processDeepseekV4Slot<scalar_t_in, kNumHeadsQPadded, APPLY_Q_NORM>(
v0, v1, tokenIdx, slotIdx, dim_base, laneId, num_heads_q, eps, q_out,
k_cache, slot_mapping, position_ids, cos_sin_cache, cache_block_size,
kv_block_stride);
kv_block_stride, kv_mxfp8);
#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)
cudaTriggerProgrammaticLaunchCompletion();
@@ -506,7 +539,7 @@ __global__ void fusedDeepseekV4QNormRopeKVRopeQuantInsertKernelReducedGrid(
float const* __restrict__ cos_sin_cache, float const eps,
int const num_tokens_full, int const num_tokens_insert,
int const num_heads_q, int const cache_block_size,
int const kv_block_stride) {
int const kv_block_stride, bool const kv_mxfp8) {
#if (!defined(__CUDA_ARCH__) || __CUDA_ARCH__ < 800) && !defined(USE_ROCM)
if constexpr (std::is_same_v<scalar_t_in, c10::BFloat16>) {
return;
@@ -564,7 +597,7 @@ __global__ void fusedDeepseekV4QNormRopeKVRopeQuantInsertKernelReducedGrid(
processDeepseekV4Slot<scalar_t_in, kNumHeadsQPadded, APPLY_Q_NORM>(
v0_curr, v1_curr, tokenIdx, curr_slot, dim_base, laneId,
num_heads_q, eps, q_out, k_cache, slot_mapping, position_ids,
cos_sin_cache, cache_block_size, kv_block_stride);
cos_sin_cache, cache_block_size, kv_block_stride, kv_mxfp8);
// ── Buffer rotation: hand the prefetched LDGs to the next iter.
v0_curr = v0_next;
@@ -590,7 +623,7 @@ static void launchFusedDeepseekV4Templated(
uint8_t* k_cache, int64_t const* slot_mapping, int64_t const* position_ids,
float const* cos_sin_cache, float const eps, int const num_tokens_full,
int const num_tokens_insert, int const num_heads_q,
int const cache_block_size, int const kv_block_stride,
int const cache_block_size, int const kv_block_stride, bool const kv_mxfp8,
cudaStream_t stream) {
constexpr int kBlockSize = 256;
constexpr int kWarpsPerBlock = kBlockSize / 32;
@@ -630,7 +663,8 @@ static void launchFusedDeepseekV4Templated(
scalar_t_in, kNumHeadsQPadded, APPLY_Q_NORM>,
q_in, q_out, kv_in, k_cache, slot_mapping, position_ids,
cos_sin_cache, eps, num_tokens_full, num_tokens_insert,
num_heads_q, cache_block_size, kv_block_stride);
num_heads_q, cache_block_size, kv_block_stride,
kv_mxfp8);
} else {
config.gridDim = dim3(num_tokens_full);
cudaLaunchKernelEx(
@@ -639,7 +673,7 @@ static void launchFusedDeepseekV4Templated(
scalar_t_in, kNumHeadsQPadded, APPLY_Q_NORM>,
q_in, q_out, kv_in, k_cache, slot_mapping, position_ids, cos_sin_cache,
eps, num_tokens_full, num_tokens_insert, num_heads_q, cache_block_size,
kv_block_stride);
kv_block_stride, kv_mxfp8);
}
#else
// ROCm: use standard kernel launch syntax (no PDL/stream serialization)
@@ -649,7 +683,7 @@ static void launchFusedDeepseekV4Templated(
<<<grid, kBlockSize, 0, stream>>>(
q_in, q_out, kv_in, k_cache, slot_mapping, position_ids,
cos_sin_cache, eps, num_tokens_full, num_tokens_insert, num_heads_q,
cache_block_size, kv_block_stride);
cache_block_size, kv_block_stride, kv_mxfp8);
#endif
}
@@ -663,19 +697,19 @@ void launchFusedDeepseekV4QNormRopeKVRopeQuantInsert(
int const num_tokens_full, int const num_tokens_insert,
int const num_heads_q, int const num_heads_q_padded,
int const cache_block_size, int const kv_block_stride,
bool const apply_q_norm, cudaStream_t stream) {
bool const apply_q_norm, bool const kv_mxfp8, cudaStream_t stream) {
#define DISPATCH(N) \
case N: \
if (apply_q_norm) { \
launchFusedDeepseekV4Templated<scalar_t_in, N, true>( \
q_in, q_out, kv_in, k_cache, slot_mapping, position_ids, \
cos_sin_cache, eps, num_tokens_full, num_tokens_insert, \
num_heads_q, cache_block_size, kv_block_stride, stream); \
num_heads_q, cache_block_size, kv_block_stride, kv_mxfp8, stream); \
} else { \
launchFusedDeepseekV4Templated<scalar_t_in, N, false>( \
q_in, q_out, kv_in, k_cache, slot_mapping, position_ids, \
cos_sin_cache, eps, num_tokens_full, num_tokens_insert, \
num_heads_q, cache_block_size, kv_block_stride, stream); \
num_heads_q, cache_block_size, kv_block_stride, kv_mxfp8, stream); \
} \
return;
@@ -967,7 +1001,7 @@ torch::stable::Tensor fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
torch::stable::Tensor const& position_ids, // [N] int64
torch::stable::Tensor const& cos_sin_cache, // [max_pos, rope_dim] bf16
int64_t q_head_padded, // padded Q head count for output
double eps, int64_t cache_block_size, bool apply_q_norm) {
double eps, int64_t cache_block_size, bool apply_q_norm, bool kv_mxfp8) {
STD_TORCH_CHECK(q_in.device().is_cuda() && q_in.is_contiguous(),
"q_in must be contiguous CUDA");
STD_TORCH_CHECK(kv.device().is_cuda() && kv.is_contiguous(),
@@ -1036,7 +1070,7 @@ torch::stable::Tensor fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
cos_sin_cache.const_data_ptr<float>(), static_cast<float>(eps),
num_tokens_full, num_tokens_insert, num_heads_q,
num_heads_q_padded, cache_block_size_i, kv_block_stride,
apply_q_norm, stream);
apply_q_norm, kv_mxfp8, stream);
});
return q_out;
}
+1 -1
View File
@@ -264,7 +264,7 @@ torch::stable::Tensor fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
torch::stable::Tensor& k_cache, torch::stable::Tensor const& slot_mapping,
torch::stable::Tensor const& position_ids,
torch::stable::Tensor const& cos_sin_cache, int64_t q_head_padded,
double eps, int64_t cache_block_size, bool apply_q_norm);
double eps, int64_t cache_block_size, bool apply_q_norm, bool kv_mxfp8);
void fused_deepseek_v4_qnorm_rope_kv_rope_full_cache_bf16_insert(
torch::stable::Tensor& q, torch::stable::Tensor const& kv,
+1 -1
View File
@@ -430,7 +430,7 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
"Tensor q_in, Tensor kv, Tensor! k_cache, "
"Tensor slot_mapping, Tensor position_ids, Tensor cos_sin_cache, "
"int q_head_padded, float eps, int cache_block_size, "
"bool apply_q_norm=True) -> Tensor");
"bool apply_q_norm=True, bool kv_mxfp8=False) -> Tensor");
// FlashInfer V4 full-cache variants: write Q in place (bf16) or to a separate
// FP8 tensor, and KV into a contiguous 512-wide token-strided cache.