[Model][DSv4.1] FlashMLA mega attention and the NVFP4 compressed KV cache (#56935)

Signed-off-by: Yongye Zhu <yongye@inferact.ai>
Signed-off-by: 云挚 <ningyunxiao.nyx@antgroup.com>
Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: Yongye Zhu <yongye@inferact.ai>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: Yunxiao Ning <73625538+foraxe@users.noreply.github.com>
Co-authored-by: AI Assistant <noreply@openai.com>
This commit is contained in:
Yongye Zhu
2026-09-16 03:07:50 -07:00
committed by GitHub
co-authored by Yongye Zhu Claude Opus 5 Yunxiao Ning AI Assistant
parent ceb87de065
commit d6a1677d55
20 changed files with 1811 additions and 156 deletions
@@ -0,0 +1,272 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""FlashMLA mega attention vs the split-KV decode pipeline (DeepSeek V4.1).
Times the full set of per-layer ops each path runs in a decode step, captured
in a CUDA graph -- eager timing is dominated by launch overhead and flatters
whichever path launches fewer kernels.
mega: fused_qnorm_rope_kv_rope_quant_insert (Q pad + KV insert)
+ mega kernel
split-KV: fused_qnorm_rope_kv_rope_quant_insert (Q RoPE + pad + KV insert)
+ flash_mla_with_kvcache + fused_inv_rope_fp8_quant
Both arms fuse their Q preparation into the KV insert, so that op is charged
to both; the mega arm's Q side is only a zero-pad (nothing at all when the
shard is already at the kernel head count) because its kernel does the Q RoPE,
the inverse RoPE and the FP8 cast.
``--local-heads`` is the variable that decides the outcome: the mega kernel
absorbs work proportional to the live head count while paying for the padded
one, so it wins at 64 live heads and is a wash at 16 (TP4 on a 64-head model).
Both arms use the same KV records so the comparison is kernel-for-kernel:
a V4.1 fp8 (528 B) sliding-window cache and a compressed cache that is either
V4.1 fp8 or V4.1 NVFP4 (288 B) -- FlashMLA's SM100 sparse decode reads the
NVFP4 compressed record in both the fused and unfused kernels.
Run: .venv/bin/python benchmarks/kernels/benchmark_dsv41_mega_attn.py
"""
import argparse
import torch
import vllm.v1.attention.ops.flashmla as fm
from vllm.models.deepseek_v4.common.ops.fused_inv_rope_fp8_quant import (
fused_inv_rope_fp8_quant,
)
from vllm.models.deepseek_v41.common.ops import quantize_and_insert_k_cache
from vllm.models.deepseek_v41.common.ops.fused_compress_quant_cache import (
rope_quant_insert,
)
from vllm.models.deepseek_v41.common.ops.fused_layout import permute_q_to_fused
from vllm.models.deepseek_v41.nvidia.flash_mla_mega_attn import (
alloc_mega_attn_output,
is_flashmla_mega_attn_supported,
)
from vllm.utils.math_utils import round_up
HEAD_DIM, ROPE_DIM = 512, 64
V41_BYTES, V41_FP4_BYTES = 528, 288
SWA_BLOCK, COMPRESSED_BLOCK = 32, 128
def make_cos_sin_cache(max_pos, device):
inv_freq = 1.0 / (
10000 ** (torch.arange(0, ROPE_DIM, 2, device=device).float() / ROPE_DIM)
)
freqs = torch.outer(torch.arange(max_pos, device=device).float(), inv_freq)
return torch.cat([freqs.cos(), freqs.sin()], -1)
def empty_paged_cache(num_rows, block_size, bytes_per_token, device):
"""A zeroed paged cache view [num_blocks, block_size, bytes_per_token]."""
num_blocks = (num_rows + block_size - 1) // block_size + 1
page = round_up(block_size * bytes_per_token, 512)
backing = torch.zeros(num_blocks, page, dtype=torch.uint8, device=device)
return backing, backing.as_strided(
(num_blocks, block_size, bytes_per_token), (page, bytes_per_token, 1)
)
def fill_cache(view, bytes_per_token, cos_sin, device):
n = view.shape[0] * view.shape[1]
rows = torch.randn(n, HEAD_DIM, device=device, dtype=torch.bfloat16)
slots = torch.arange(n, dtype=torch.int64, device=device)
if bytes_per_token == V41_FP4_BYTES:
# A page-aligned cache has more slots than the cos_sin table has rows;
# wrap so the insert never indexes past it.
positions = slots % cos_sin.shape[0]
rope_quant_insert(rows, positions, cos_sin, view, slots, 1)
else:
quantize_and_insert_k_cache(
rows,
view.reshape(view.shape[0], -1),
slots,
block_size=view.shape[1],
bytes_per_token=bytes_per_token,
)
def time_graph(fn, iters=100, warmup=5):
"""Capture ``fn`` in a CUDA graph and time its replay."""
s = torch.cuda.Stream()
s.wait_stream(torch.cuda.current_stream())
with torch.cuda.stream(s):
for _ in range(3):
fn()
torch.cuda.current_stream().wait_stream(s)
g = torch.cuda.CUDAGraph()
with torch.cuda.graph(g):
fn()
for _ in range(warmup):
g.replay()
torch.accelerator.synchronize()
start = torch.cuda.Event(enable_timing=True)
end = torch.cuda.Event(enable_timing=True)
start.record()
for _ in range(iters):
g.replay()
end.record()
torch.accelerator.synchronize()
return start.elapsed_time(end) * 1000 / iters
def bench_decode(s_q, device, topk_extra, extra_bytes, local_heads):
padded_heads, scale = 64, HEAD_DIM**-0.5
n_wv_group = padded_heads // 8
cos_sin = make_cos_sin_cache(65536, device)
positions = torch.randint(0, 65536, (s_q,), device=device, dtype=torch.int64)
pos32 = positions.to(torch.int32)
# Per-step inputs: the wq_b output (local heads) and this step's KV row.
q_local = torch.randn(
s_q, local_heads, HEAD_DIM, device=device, dtype=torch.bfloat16
)
q_fused = permute_q_to_fused(q_local)
kv = torch.randn(s_q, HEAD_DIM, device=device, dtype=torch.bfloat16)
n_swa = s_q * 128 + 128
swa_backing, swa = empty_paged_cache(n_swa, SWA_BLOCK, V41_BYTES, device)
fill_cache(swa, V41_BYTES, cos_sin, device)
_, extra = empty_paged_cache(65536, COMPRESSED_BLOCK, extra_bytes, device)
fill_cache(extra, extra_bytes, cos_sin, device)
slot_mapping = torch.arange(s_q, dtype=torch.int64, device=device)
swa_idx = torch.randint(0, n_swa, (s_q, 128), device=device, dtype=torch.int32)
swa_len = torch.full((s_q,), 128, device=device, dtype=torch.int32)
ex_idx = torch.randint(
0, 65536, (s_q, topk_extra), device=device, dtype=torch.int32
)
ex_len = torch.full((s_q,), topk_extra, device=device, dtype=torch.int32)
sink = torch.zeros(padded_heads, device=device)
sched = fm.FlashMLASchedMeta()
out = alloc_mega_attn_output(s_q, n_wv_group, device)
swa4 = swa.unsqueeze(-2)
extra4 = extra.unsqueeze(-2)
swa_2d = swa_backing
def mega():
# One launch zero-pads the fused-layout Q and inserts the SWA KV.
q_pad = torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
q_fused,
kv,
swa_2d,
slot_mapping,
positions,
cos_sin,
0 if padded_heads == local_heads else padded_heads,
1e-6,
SWA_BLOCK,
False, # apply_q_norm
True, # kv_mxfp8
False, # apply_q_rope: the mega kernel rotates Q itself
True, # is_q_interleaved
)
torch.ops._flashmla_C.fused_norm_rope_attn_rope_cast_decode(
q_fused if padded_heads == local_heads else q_pad,
swa4,
swa_idx,
scale,
HEAD_DIM,
sink,
swa_len,
extra4,
ex_idx,
ex_len,
False,
0.0,
pos32,
False,
64,
cos_sin,
n_wv_group,
32,
True,
True,
True,
out.data,
out.scale,
)
def split_kv():
q_pad = torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
q_local,
kv,
swa_2d,
slot_mapping,
positions,
cos_sin,
padded_heads,
1e-6,
SWA_BLOCK,
False,
True,
)
o = fm.flash_mla_with_kvcache(
q=q_pad.unsqueeze(1),
k_cache=swa4,
block_table=None,
head_dim_v=HEAD_DIM,
tile_scheduler_metadata=sched,
cache_seqlens=None,
is_fp8_kvcache=True,
indices=swa_idx.view(s_q, 1, -1),
topk_length=swa_len,
softmax_scale=scale,
attn_sink=sink,
extra_k_cache=extra4,
extra_indices_in_kvcache=ex_idx.view(s_q, 1, -1),
extra_topk_length=ex_len,
)[0]
fused_inv_rope_fp8_quant(
o.squeeze(1)[:, :local_heads],
positions,
cos_sin,
n_groups=local_heads // 8,
heads_per_group=8,
quant_group_size=32,
tma_aligned_scales=True,
)
return time_graph(mega), time_graph(split_kv)
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
"--decode-s-q",
nargs="+",
type=int,
default=[1, 2, 4, 8, 16, 32, 64, 128, 256, 512],
)
parser.add_argument("--topk-extra", type=int, default=512)
parser.add_argument("--local-heads", type=int, default=16)
parser.add_argument(
"--extra-bytes",
type=int,
choices=[V41_BYTES, V41_FP4_BYTES],
default=V41_BYTES,
help="compressed-cache record: 528 (V4.1 fp8) or 288 (V4.1 NVFP4)",
)
args = parser.parse_args()
ok, reason = is_flashmla_mega_attn_supported()
if not ok:
raise SystemExit(reason)
device = torch.device("cuda")
torch.manual_seed(0)
print(
f"decode, cudagraph (topk_swa=128, topk_extra={args.topk_extra}, "
f"local_heads={args.local_heads}, extra={args.extra_bytes}B)"
)
print(f"{'s_q':>6} {'mega_us':>9} {'split_kv_us':>12} {'speedup':>8}")
for s_q in args.decode_s_q:
m, s = bench_decode(
s_q, device, args.topk_extra, args.extra_bytes, args.local_heads
)
print(f"{s_q:>6d} {m:9.1f} {s:12.1f} {s / m:7.2f}x")
if __name__ == "__main__":
main()
@@ -3,7 +3,10 @@
* SPDX-FileCopyrightText: Copyright contributors to the vLLM project
*
* Horizontally-fused DeepseekV4-MLA kernel:
* - Q side: optional per-head RMSNorm + GPT-J RoPE on last ROPE_DIM
* - Q side: optional per-head RMSNorm (apply_q_norm) and GPT-J RoPE on the
* last ROPE_DIM (apply_q_rope), plus the zero-pad to
* q_head_padded, in the head-major layout or (is_q_interleaved)
* FlashMLA's mega-attention chunk-interleaved one
* - KV side: GPT-J RoPE on last ROPE_DIM + UE8M0 FP8 quant on NoPE + paged
* cache insert
*
@@ -174,6 +177,30 @@ __device__ __forceinline__ float warpSum(float val) {
return val;
}
// Offset of the 16 elements a lane owns inside a Q tensor of `num_heads`
// heads. The default layout is head-major: a head's 512 dims are contiguous.
// `Q_INTERLEAVED` is FlashMLA's mega-attention layout, 32 chunks of
// [num_heads, 16] per token, which changes the address only -- lane L still
// owns dims [16L, 16L+16) of its head either way, so `dim_base` stays the
// true dim offset and the norm and RoPE below are equally valid in both.
// What it does change is where the padding heads land: at the tail of every
// chunk rather than after the live heads.
template <bool Q_INTERLEAVED>
__device__ __forceinline__ int64_t qElemOffset(int const tokenIdx,
int const slotIdx,
int const laneId,
int const dim_base,
int const num_heads) {
if constexpr (Q_INTERLEAVED) {
return static_cast<int64_t>(tokenIdx) * num_heads * kHeadDim +
static_cast<int64_t>(laneId) * num_heads * kElemsPerLane +
static_cast<int64_t>(slotIdx) * kElemsPerLane;
} else {
return (static_cast<int64_t>(tokenIdx) * num_heads + slotIdx) * kHeadDim +
dim_base;
}
}
// ────────────────────────────────────────────────────────────────────────────
// Per-slot inner pipeline
// ────────────────────────────────────────────────────────────────────────────
@@ -192,10 +219,16 @@ __device__ __forceinline__ float warpSum(float val) {
// + paged-cache
// insert)
//
// `kv_mxfp8` is grid-uniform and only steers the KV branch, so it stays a
// runtime argument rather than doubling every (head count, q-norm)
// instantiation of a kernel whose hot path is the Q branch.
template <typename scalar_t_in, int kNumHeadsQPadded, bool APPLY_Q_NORM>
// `kv_mxfp8` and `apply_q_rope` are grid-uniform, so they stay runtime
// arguments rather than multiplying every (head count, q-norm, q-layout)
// instantiation of a kernel whose hot path is the Q branch. `Q_INTERLEAVED`
// cannot: it picks the address arithmetic, which has to fold.
//
// The three Q knobs are independent. The mega-attention layer uses
// Q_INTERLEAVED with both norm and RoPE off, because its attention kernel
// applies those itself -- not because the layout forbids them.
template <typename scalar_t_in, int kNumHeadsQPadded, bool APPLY_Q_NORM,
bool Q_INTERLEAVED = false>
__device__ __forceinline__ void processDeepseekV4Slot(
uint4 v0, uint4 v1, int const tokenIdx, int const slotIdx,
int const dim_base, int const laneId, int const num_heads_q,
@@ -203,7 +236,7 @@ __device__ __forceinline__ void processDeepseekV4Slot(
uint8_t* __restrict__ k_cache, int64_t const* __restrict__ slot_mapping,
int64_t const* __restrict__ position_ids,
float const* __restrict__ cos_sin_cache, int const cache_block_size,
int const kv_block_stride, bool const kv_mxfp8) {
int const kv_block_stride, bool const kv_mxfp8, bool const apply_q_rope) {
using Converter = vllm::_typeConvert<scalar_t_in>;
bool const isKV = (slotIdx == kNumHeadsQPadded);
bool const isPadQ = !isKV && (slotIdx >= num_heads_q);
@@ -213,10 +246,8 @@ __device__ __forceinline__ void processDeepseekV4Slot(
// zero literal is correct. Matches the live-Q branch's vectorized store.
if (isPadQ) {
scalar_t_in* dst =
q_out +
(static_cast<int64_t>(tokenIdx) * kNumHeadsQPadded + slotIdx) *
kHeadDim +
dim_base;
q_out + qElemOffset<Q_INTERLEAVED>(tokenIdx, slotIdx, laneId, dim_base,
kNumHeadsQPadded);
uint4 const zero4 = {0u, 0u, 0u, 0u};
*reinterpret_cast<uint4*>(dst) = zero4;
*reinterpret_cast<uint4*>(dst + 8) = zero4;
@@ -264,7 +295,9 @@ __device__ __forceinline__ void processDeepseekV4Slot(
// ── GPT-J RoPE on dims [NOPE_DIM, HEAD_DIM) ─────────────────────────────
// All math in fp32. cos_sin_cache is loaded as fp32 (its native storage).
bool const is_rope_lane = dim_base >= kNopeDim;
// KV is always rotated; Q only when the caller has not already had it
// rotated elsewhere (the mega-attention kernel does its own).
bool const is_rope_lane = dim_base >= kNopeDim && (isKV || apply_q_rope);
if (is_rope_lane) {
int64_t const pos = position_ids[tokenIdx];
constexpr int kHalfRope = kRopeDim / 2;
@@ -314,10 +347,8 @@ __device__ __forceinline__ void processDeepseekV4Slot(
make_float2(elements[8 + 2 * i], elements[8 + 2 * i + 1]));
}
scalar_t_in* dst =
q_out +
(static_cast<int64_t>(tokenIdx) * kNumHeadsQPadded + slotIdx) *
kHeadDim +
dim_base;
q_out + qElemOffset<Q_INTERLEAVED>(tokenIdx, slotIdx, laneId, dim_base,
kNumHeadsQPadded);
*reinterpret_cast<uint4*>(dst) = out0;
*reinterpret_cast<uint4*>(dst + 8) = out1;
} else {
@@ -432,14 +463,17 @@ __device__ __forceinline__ void processDeepseekV4Slot(
// `kNumHeadsQPadded` is a template parameter (compile-time constant) so the
// divisions in the grid math and the KV-sentinel comparison fold to fast
// constant operations. The launch wrapper dispatches the runtime value to
// the matching instantiation.
// the matching instantiation. `kNumHeadsQPadded == 0` leaves one slot per
// token and is the KV-insert-only instantiation, for a Q that is already in
// the shape its attention kernel wants.
//
// With DP padding, q/kv/position_ids can have more rows than slot_mapping.
// The live-Q and pad-Q branches cover all `num_tokens_full` rows (downstream
// attention uses them). The KV branch only inserts the first
// `num_tokens_insert` tokens (= slot_mapping length) into the paged cache.
//
template <typename scalar_t_in, int kNumHeadsQPadded, bool APPLY_Q_NORM>
template <typename scalar_t_in, int kNumHeadsQPadded, bool APPLY_Q_NORM,
bool Q_INTERLEAVED>
__global__ void fusedDeepseekV4QNormRopeKVRopeQuantInsertKernel(
scalar_t_in const* __restrict__ q_in, // [N, num_heads_q, 512]
scalar_t_in* __restrict__ q_out, // [N, kNumHeadsQPadded, 512]
@@ -454,7 +488,8 @@ __global__ void fusedDeepseekV4QNormRopeKVRopeQuantInsertKernel(
int const num_heads_q, // live Q heads (input layout)
int const cache_block_size, // tokens per paged-cache block
int const kv_block_stride, // bytes per paged-cache block
bool const kv_mxfp8) { // V4.1 all-dims MXFP8 record
bool const kv_mxfp8, // V4.1 all-dims MXFP8 record
bool const apply_q_rope) { // rotate Q too, not just KV
#if (!defined(__CUDA_ARCH__) || __CUDA_ARCH__ < 800) && !defined(USE_ROCM)
// BF16 _typeConvert specialization is unavailable on pre-Ampere. The
// DeepseekV4 kernel only runs with bf16 inputs in practice, so compile a
@@ -497,20 +532,18 @@ __global__ void fusedDeepseekV4QNormRopeKVRopeQuantInsertKernel(
if (isKV) {
src_ptr = kv_in + static_cast<int64_t>(tokenIdx) * kHeadDim + dim_base;
} else {
int64_t const q_row_offset =
(static_cast<int64_t>(tokenIdx) * num_heads_q + slotIdx) *
kHeadDim +
dim_base;
src_ptr = q_in + q_row_offset;
src_ptr = q_in + qElemOffset<Q_INTERLEAVED>(tokenIdx, slotIdx, laneId,
dim_base, num_heads_q);
}
v0 = *reinterpret_cast<uint4 const*>(src_ptr);
v1 = *reinterpret_cast<uint4 const*>(src_ptr + 8);
}
processDeepseekV4Slot<scalar_t_in, kNumHeadsQPadded, APPLY_Q_NORM>(
processDeepseekV4Slot<scalar_t_in, kNumHeadsQPadded, APPLY_Q_NORM,
Q_INTERLEAVED>(
v0, v1, tokenIdx, slotIdx, dim_base, laneId, num_heads_q, eps, q_out,
k_cache, slot_mapping, position_ids, cos_sin_cache, cache_block_size,
kv_block_stride, kv_mxfp8);
kv_block_stride, kv_mxfp8, apply_q_rope);
#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)
cudaTriggerProgrammaticLaunchCompletion();
@@ -530,7 +563,8 @@ __global__ void fusedDeepseekV4QNormRopeKVRopeQuantInsertKernel(
// Q branch (optional RMSNorm + RoPE, in place) head_slot == num_heads_q
// KV branch (RoPE + UE8M0 quant + insert)
//
template <typename scalar_t_in, int kNumHeadsQPadded, bool APPLY_Q_NORM>
template <typename scalar_t_in, int kNumHeadsQPadded, bool APPLY_Q_NORM,
bool Q_INTERLEAVED>
__global__ void fusedDeepseekV4QNormRopeKVRopeQuantInsertKernelReducedGrid(
scalar_t_in const* __restrict__ q_in, scalar_t_in* __restrict__ q_out,
scalar_t_in const* __restrict__ kv_in, uint8_t* __restrict__ k_cache,
@@ -539,7 +573,7 @@ __global__ void fusedDeepseekV4QNormRopeKVRopeQuantInsertKernelReducedGrid(
float const* __restrict__ cos_sin_cache, float const eps,
int const num_tokens_full, int const num_tokens_insert,
int const num_heads_q, int const cache_block_size,
int const kv_block_stride, bool const kv_mxfp8) {
int const kv_block_stride, bool const kv_mxfp8, bool const apply_q_rope) {
#if (!defined(__CUDA_ARCH__) || __CUDA_ARCH__ < 800) && !defined(USE_ROCM)
if constexpr (std::is_same_v<scalar_t_in, c10::BFloat16>) {
return;
@@ -569,11 +603,8 @@ __global__ void fusedDeepseekV4QNormRopeKVRopeQuantInsertKernelReducedGrid(
if (s == kNumHeadsQPadded) {
src = kv_in + static_cast<int64_t>(tokenIdx) * kHeadDim + dim_base;
} else {
src = q_in +
(static_cast<int64_t>(tokenIdx) * num_heads_q +
static_cast<int64_t>(s)) *
kHeadDim +
dim_base;
src = q_in + qElemOffset<Q_INTERLEAVED>(tokenIdx, s, laneId, dim_base,
num_heads_q);
}
va = *reinterpret_cast<uint4 const*>(src);
vb = *reinterpret_cast<uint4 const*>(src + 8);
@@ -594,10 +625,12 @@ __global__ void fusedDeepseekV4QNormRopeKVRopeQuantInsertKernelReducedGrid(
load_slot(next_slot, v0_next, v1_next);
}
processDeepseekV4Slot<scalar_t_in, kNumHeadsQPadded, APPLY_Q_NORM>(
processDeepseekV4Slot<scalar_t_in, kNumHeadsQPadded, APPLY_Q_NORM,
Q_INTERLEAVED>(
v0_curr, v1_curr, tokenIdx, curr_slot, dim_base, laneId,
num_heads_q, eps, q_out, k_cache, slot_mapping, position_ids,
cos_sin_cache, cache_block_size, kv_block_stride, kv_mxfp8);
cos_sin_cache, cache_block_size, kv_block_stride, kv_mxfp8,
apply_q_rope);
// ── Buffer rotation: hand the prefetched LDGs to the next iter.
v0_curr = v0_next;
@@ -617,14 +650,15 @@ __global__ void fusedDeepseekV4QNormRopeKVRopeQuantInsertKernelReducedGrid(
// ────────────────────────────────────────────────────────────────────────────
// Launch wrapper
// ────────────────────────────────────────────────────────────────────────────
template <typename scalar_t_in, int kNumHeadsQPadded, bool APPLY_Q_NORM>
template <typename scalar_t_in, int kNumHeadsQPadded, bool APPLY_Q_NORM,
bool Q_INTERLEAVED>
static void launchFusedDeepseekV4Templated(
scalar_t_in const* q_in, scalar_t_in* q_out, scalar_t_in const* kv_in,
uint8_t* k_cache, int64_t const* slot_mapping, int64_t const* position_ids,
float const* cos_sin_cache, float const eps, int const num_tokens_full,
int const num_tokens_insert, int const num_heads_q,
int const cache_block_size, int const kv_block_stride, bool const kv_mxfp8,
cudaStream_t stream) {
bool const apply_q_rope, cudaStream_t stream) {
constexpr int kBlockSize = 256;
constexpr int kWarpsPerBlock = kBlockSize / 32;
int64_t const total_warps =
@@ -663,37 +697,40 @@ static void launchFusedDeepseekV4Templated(
// grid instead. Only reachable above NUM_TOKEN_CUTOFF tokens in a single
// insert, where it is worth ~2x.
if (kNumHeadsQPadded == 0 || num_tokens_full < NUM_TOKEN_CUTOFF) {
cudaLaunchKernelEx(&config,
fusedDeepseekV4QNormRopeKVRopeQuantInsertKernel<
scalar_t_in, kNumHeadsQPadded, APPLY_Q_NORM>,
q_in, q_out, kv_in, k_cache, slot_mapping, position_ids,
cos_sin_cache, eps, num_tokens_full, num_tokens_insert,
num_heads_q, cache_block_size, kv_block_stride,
kv_mxfp8);
cudaLaunchKernelEx(
&config,
fusedDeepseekV4QNormRopeKVRopeQuantInsertKernel<
scalar_t_in, kNumHeadsQPadded, APPLY_Q_NORM, Q_INTERLEAVED>,
q_in, q_out, kv_in, k_cache, slot_mapping, position_ids, cos_sin_cache,
eps, num_tokens_full, num_tokens_insert, num_heads_q, cache_block_size,
kv_block_stride, kv_mxfp8, apply_q_rope);
} else {
config.gridDim = dim3(num_tokens_full);
cudaLaunchKernelEx(
&config,
fusedDeepseekV4QNormRopeKVRopeQuantInsertKernelReducedGrid<
scalar_t_in, kNumHeadsQPadded, APPLY_Q_NORM>,
scalar_t_in, kNumHeadsQPadded, APPLY_Q_NORM, Q_INTERLEAVED>,
q_in, q_out, kv_in, k_cache, slot_mapping, position_ids, cos_sin_cache,
eps, num_tokens_full, num_tokens_insert, num_heads_q, cache_block_size,
kv_block_stride, kv_mxfp8);
kv_block_stride, kv_mxfp8, apply_q_rope);
}
#else
// ROCm: use standard kernel launch syntax (no PDL/stream serialization)
// clang-format off
fusedDeepseekV4QNormRopeKVRopeQuantInsertKernel<
scalar_t_in, kNumHeadsQPadded, APPLY_Q_NORM>
scalar_t_in, kNumHeadsQPadded, APPLY_Q_NORM, Q_INTERLEAVED>
<<<grid, kBlockSize, 0, stream>>>(
q_in, q_out, kv_in, k_cache, slot_mapping, position_ids,
cos_sin_cache, eps, num_tokens_full, num_tokens_insert, num_heads_q,
cache_block_size, kv_block_stride, kv_mxfp8);
cache_block_size, kv_block_stride, kv_mxfp8, apply_q_rope);
#endif
}
// Runtime dispatch into one of the precompiled `kNumHeadsQPadded`
// instantiations. Supported padded head counts: 8, 16, 32, 64, 128.
// instantiations. Supported padded head counts: 8, 16, 32, 64, 128, plus 0
// for a KV-only launch that writes no Q at all. `apply_q_norm` and
// `is_q_interleaved` are template parameters, `apply_q_rope` is not (see
// processDeepseekV4Slot), so each head count carries four instantiations.
template <typename scalar_t_in>
void launchFusedDeepseekV4QNormRopeKVRopeQuantInsert(
scalar_t_in const* q_in, scalar_t_in* q_out, scalar_t_in const* kv_in,
@@ -702,22 +739,32 @@ void launchFusedDeepseekV4QNormRopeKVRopeQuantInsert(
int const num_tokens_full, int const num_tokens_insert,
int const num_heads_q, int const num_heads_q_padded,
int const cache_block_size, int const kv_block_stride,
bool const apply_q_norm, bool const kv_mxfp8, cudaStream_t stream) {
#define DISPATCH(N) \
case N: \
if (apply_q_norm) { \
launchFusedDeepseekV4Templated<scalar_t_in, N, true>( \
q_in, q_out, kv_in, k_cache, slot_mapping, position_ids, \
cos_sin_cache, eps, num_tokens_full, num_tokens_insert, \
num_heads_q, cache_block_size, kv_block_stride, kv_mxfp8, stream); \
} else { \
launchFusedDeepseekV4Templated<scalar_t_in, N, false>( \
q_in, q_out, kv_in, k_cache, slot_mapping, position_ids, \
cos_sin_cache, eps, num_tokens_full, num_tokens_insert, \
num_heads_q, cache_block_size, kv_block_stride, kv_mxfp8, stream); \
} \
bool const apply_q_norm, bool const kv_mxfp8, bool const apply_q_rope,
bool const is_q_interleaved, cudaStream_t stream) {
#define LAUNCH(N, NORM, INTERLEAVED) \
launchFusedDeepseekV4Templated<scalar_t_in, N, NORM, INTERLEAVED>( \
q_in, q_out, kv_in, k_cache, slot_mapping, position_ids, cos_sin_cache, \
eps, num_tokens_full, num_tokens_insert, num_heads_q, cache_block_size, \
kv_block_stride, kv_mxfp8, apply_q_rope, stream)
#define DISPATCH(N) \
case N: \
if (is_q_interleaved) { \
if (apply_q_norm) { \
LAUNCH(N, true, true); \
} else { \
LAUNCH(N, false, true); \
} \
} else if (apply_q_norm) { \
LAUNCH(N, true, false); \
} else { \
LAUNCH(N, false, false); \
} \
return;
if (num_heads_q_padded == 0) {
LAUNCH(0, false, false);
return;
}
switch (num_heads_q_padded) {
DISPATCH(8)
DISPATCH(16)
@@ -729,9 +776,10 @@ void launchFusedDeepseekV4QNormRopeKVRopeQuantInsert(
"fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert: "
"unsupported num_heads_q_padded=",
num_heads_q_padded,
" (compiled instantiations: 8, 16, 32, 64, 128).");
" (compiled instantiations: 0, 8, 16, 32, 64, 128).");
}
#undef DISPATCH
#undef LAUNCH
}
// ────────────────────────────────────────────────────────────────────────────
@@ -1059,10 +1107,15 @@ void fused_deepseek_v4_kv_rope_insert(
// Zero query heads schedules only the existing KV branch, preserving its
// RoPE rounding and quantization without allocating any query tensors.
if (packed) {
vllm::deepseek_v4_fused_ops::launchFusedDeepseekV4Templated<scalar_t, 0, false>(
// With zero query heads no Q slot is scheduled, so neither Q knob is
// reachable: Q_INTERLEAVED picks addressing nothing uses, and apply_q_rope
// gates a branch only a Q slot takes (KV is rotated unconditionally).
vllm::deepseek_v4_fused_ops::launchFusedDeepseekV4Templated<scalar_t, 0,
false, false>(
nullptr, nullptr, input, cache, slots, positions, cos_sin, 0.0f,
num_tokens, num_tokens, 0, static_cast<int>(cache_block_size),
static_cast<int>(k_cache.stride(0)), kv_mxfp8, stream);
static_cast<int>(k_cache.stride(0)), kv_mxfp8, /*apply_q_rope=*/true,
stream);
} else if (fp8) {
vllm::deepseek_v4_fused_ops::launchFullCacheKernel<scalar_t, false, true>(
nullptr, nullptr, 0, 0, input, cache, slots, positions, cos_sin,
@@ -1085,8 +1138,9 @@ torch::stable::Tensor fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
torch::stable::Tensor const& slot_mapping, // [N] int64
torch::stable::Tensor const& position_ids, // [N] int64
torch::stable::Tensor const& cos_sin_cache, // [max_pos, rope_dim] bf16
int64_t q_head_padded, // padded Q head count for output
double eps, int64_t cache_block_size, bool apply_q_norm, bool kv_mxfp8) {
int64_t q_head_padded, // padded Q head count for output, 0 for no Q
double eps, int64_t cache_block_size, bool apply_q_norm, bool kv_mxfp8,
bool apply_q_rope, bool is_q_interleaved) {
STD_TORCH_CHECK(q_in.device().is_cuda() && q_in.is_contiguous(),
"q_in must be contiguous CUDA");
STD_TORCH_CHECK(kv.device().is_cuda() && kv.is_contiguous(),
@@ -1106,8 +1160,11 @@ torch::stable::Tensor fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
STD_TORCH_CHECK(kv.dim() == 2 && kv.size(1) == 512, "kv shape [N, 512]");
STD_TORCH_CHECK(q_in.scalar_type() == kv.scalar_type(),
"q_in and kv dtype must match");
STD_TORCH_CHECK(q_head_padded >= q_in.size(1),
"q_head_padded must be >= q_in.size(1) (num_heads_q)");
// q_head_padded == 0 asks for the KV insert alone: Q is left untouched and
// an empty tensor comes back, for a caller whose Q is already in the shape
// its attention kernel reads.
STD_TORCH_CHECK(q_head_padded == 0 || q_head_padded >= q_in.size(1),
"q_head_padded must be 0 or >= q_in.size(1) (num_heads_q)");
STD_TORCH_CHECK(k_cache.scalar_type() == torch::headeronly::ScalarType::Byte,
"k_cache must be uint8");
STD_TORCH_CHECK(cos_sin_cache.dim() == 2 && cos_sin_cache.size(1) == 64,
@@ -1138,8 +1195,12 @@ torch::stable::Tensor fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
// Allocate the padded q output. The kernel writes every element (live
// region gets optional RMSNorm+RoPE; pad region gets zeros), so `empty` is
// safe.
auto q_out = torch::stable::new_empty(
q_in, {q_in.size(0), q_head_padded, q_in.size(2)}, q_in.scalar_type());
auto q_out =
q_head_padded == 0
? torch::stable::new_empty(q_in, {int64_t{0}}, q_in.scalar_type())
: torch::stable::new_empty(
q_in, {q_in.size(0), q_head_padded, q_in.size(2)},
q_in.scalar_type());
VLLM_STABLE_DISPATCH_HALF_TYPES(
q_in.scalar_type(), "fused_deepseek_v4_qnorm_rope_kv_insert", [&] {
@@ -1155,7 +1216,8 @@ torch::stable::Tensor fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
cos_sin_cache.const_data_ptr<float>(), static_cast<float>(eps),
num_tokens_full, num_tokens_insert, num_heads_q,
num_heads_q_padded, cache_block_size_i, kv_block_stride,
apply_q_norm, kv_mxfp8, stream);
apply_q_norm, kv_mxfp8, apply_q_rope, is_q_interleaved,
stream);
});
return q_out;
}
+2 -1
View File
@@ -271,7 +271,8 @@ torch::stable::Tensor fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
torch::stable::Tensor& k_cache, torch::stable::Tensor const& slot_mapping,
torch::stable::Tensor const& position_ids,
torch::stable::Tensor const& cos_sin_cache, int64_t q_head_padded,
double eps, int64_t cache_block_size, bool apply_q_norm, bool kv_mxfp8);
double eps, int64_t cache_block_size, bool apply_q_norm, bool kv_mxfp8,
bool apply_q_rope, bool is_q_interleaved);
void fused_deepseek_v4_qnorm_rope_kv_rope_full_cache_bf16_insert(
torch::stable::Tensor& q, torch::stable::Tensor const& kv,
+8 -1
View File
@@ -425,6 +425,12 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
"bool is_neox, Tensor position_ids, "
"int forced_token_heads_per_warp=-1) -> ()");
// q_head_padded is the padded Q head count of the returned tensor, or 0 to
// do the KV insert alone and return an empty tensor. The Q knobs are
// independent: apply_q_norm and apply_q_rope each drop that step for Q
// alone (KV is always rotated), and is_q_interleaved reads and writes Q in
// FlashMLA's mega-attention chunk-interleaved layout, which moves the
// padding heads to the tail of every head-dim chunk.
ops.def(
"fused_deepseek_v4_kv_rope_insert("
"Tensor kv, Tensor! k_cache, Tensor slot_mapping, Tensor position_ids, "
@@ -436,7 +442,8 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
"Tensor q_in, Tensor kv, Tensor! k_cache, "
"Tensor slot_mapping, Tensor position_ids, Tensor cos_sin_cache, "
"int q_head_padded, float eps, int cache_block_size, "
"bool apply_q_norm=True, bool kv_mxfp8=False) -> Tensor");
"bool apply_q_norm=True, bool kv_mxfp8=False, bool apply_q_rope=True, "
"bool is_q_interleaved=False) -> Tensor");
// FlashInfer V4 full-cache variants: write Q in place (bf16) or to a separate
// FP8 tensor, and KV into a contiguous 512-wide token-strided cache.
+133 -6
View File
@@ -331,6 +331,19 @@ def test_v41_fused_save_compress_and_insert(
torch.testing.assert_close(cache_backing, expected_cache, rtol=0, atol=0)
def _rotate_rope_tail(
latent_row: torch.Tensor, cos_sin_row: torch.Tensor
) -> torch.Tensor:
"""GPT-J RoPE of one latent row's last 64 dims, in fp32."""
row = latent_row.float()
c, s = cos_sin_row.chunk(2)
rotated = row.clone()
even, odd = row[448::2], row[449::2]
rotated[448::2] = even * c - odd * s
rotated[449::2] = odd * c + even * s
return rotated
def _mxfp8_record_reference(
latent_row: torch.Tensor, cos_sin_row: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor]:
@@ -339,12 +352,7 @@ def _mxfp8_record_reference(
Mirrors FlashMLA's ``KVCacheLayout.V41_FP8Sparse`` quantizer: rotate the
RoPE tail first, then scale every 32-dim tile, RoPE tiles included.
"""
row = latent_row.float()
c, s = cos_sin_row.chunk(2)
rotated = row.clone()
even, odd = row[448::2], row[449::2]
rotated[448::2] = even * c - odd * s
rotated[449::2] = odd * c + even * s
rotated = _rotate_rope_tail(latent_row, cos_sin_row)
quantized, scales = _ue8m0_reference(rotated, 32, 448.0)
return quantized.view(torch.uint8), (scales.log2() + 127).to(torch.uint8)
@@ -461,6 +469,125 @@ def test_v41_mxfp8_cache_round_trip():
assert (error <= tolerance.repeat_interleave(32, dim=-1)).all()
# e2m1 magnitudes indexed by the low 3 bits of a code; bit 3 is the sign.
_E2M1_MAGNITUDES = [0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0]
def _decode_nvfp4_row(packed: torch.Tensor, scales: torch.Tensor) -> torch.Tensor:
"""Unpack one 256-byte e2m1 row and apply its 32 e4m3 tile scales."""
mags = torch.tensor(_E2M1_MAGNITUDES, device=packed.device)
codes = torch.empty(512, dtype=torch.uint8, device=packed.device)
codes[0::2] = packed & 0xF # even element in the low nibble
codes[1::2] = packed >> 4
vals = mags[(codes & 7).long()] * torch.where(codes >= 8, -1.0, 1.0)
return (
vals.view(32, 16) * scales.view(torch.float8_e4m3fn).float().view(32, 1)
).flatten()
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA only")
@pytest.mark.parametrize("compress_ratio", [1, 2])
def test_v41_rope_insert_nvfp4_record(compress_ratio: int):
"""The 288-byte V4.1 NVFP4 record: 256 B of e2m1 pairs then 32 e4m3 scales.
Scales are byte-exact against the reference (``amax / 6`` clamped to the
e4m3 range); values are checked as round-to-nearest on the e2m1 grid, whose
coarsest step is 2 between 4 and 6, so half a step is ``1.0 * scale``.
"""
from vllm.models.deepseek_v41.common.ops.fused_compress_quant_cache import (
rope_quant_insert,
)
torch.manual_seed(13)
device = "cuda"
cache_block = 64
cache_stride = math.ceil(cache_block * 288 / 512) * 512
num_tokens = 12
positions = torch.arange(num_tokens, dtype=torch.int64, device=device)
latent = torch.randn(num_tokens, 512, dtype=torch.bfloat16, device=device)
angles = torch.randn(64, 32, device=device)
cos_sin = torch.cat((angles.cos(), angles.sin()), dim=-1)
cache_backing = torch.full((2, cache_stride), 165, dtype=torch.uint8, device=device)
cache = cache_backing.as_strided((2, cache_block, 288), (cache_stride, 288, 1))
slots = torch.arange(num_tokens, dtype=torch.int64, device=device)
slots[3] = -1 # covers the negative-slot skip
rope_quant_insert(latent, positions, cos_sin, cache, slots, compress_ratio)
untouched = torch.full_like(cache_backing, 165)
for t in range(num_tokens):
slot = slots[t].item()
pos = positions[t].item()
if slot < 0 or (pos + 1) % compress_ratio:
continue
page, row = divmod(slot, cache_block)
cs = cos_sin[pos // compress_ratio * compress_ratio]
rotated = _rotate_rope_tail(latent[t], cs)
amax = rotated.view(32, 16).abs().amax(-1)
scale = (amax / 6.0).clamp(2.0**-9, 448.0).to(torch.float8_e4m3fn)
got_scales = cache_backing[page, cache_block * 256 + row * 32 :][:32]
torch.testing.assert_close(got_scales, scale.view(torch.uint8), rtol=0, atol=0)
decoded = _decode_nvfp4_row(
cache_backing[page, row * 256 : (row + 1) * 256], got_scales
)
step = scale.float().repeat_interleave(16)
assert ((decoded - rotated).abs() <= step).all()
untouched[page, row * 256 : (row + 1) * 256] = cache_backing[
page, row * 256 : (row + 1) * 256
]
untouched[page, cache_block * 256 + row * 32 :][:32] = got_scales
# Rows the kernel must not have touched, page padding included.
torch.testing.assert_close(cache_backing, untouched, rtol=0, atol=0)
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA only")
def test_v41_nvfp4_gather_matches_insert():
"""The NVFP4 gather dequantizes exactly what the insert kernel wrote."""
from vllm.models.deepseek_v41.common.ops import dequantize_and_gather_k_cache
from vllm.models.deepseek_v41.common.ops.fused_compress_quant_cache import (
rope_quant_insert,
)
torch.manual_seed(17)
device = "cuda"
block_size = 64
num_tokens = 70
num_blocks = 4
page_bytes = math.ceil(block_size * 288 / 512) * 512
positions = torch.arange(num_tokens, dtype=torch.int64, device=device)
latent = torch.randn(num_tokens, 512, dtype=torch.bfloat16, device=device)
angles = torch.randn(128, 32, device=device)
cos_sin = torch.cat((angles.cos(), angles.sin()), dim=-1)
backing = torch.zeros(num_blocks, page_bytes, dtype=torch.uint8, device=device)
cache = backing.as_strided((num_blocks, block_size, 288), (page_bytes, 288, 1))
slots = torch.arange(num_tokens, dtype=torch.int64, device=device)
rope_quant_insert(latent, positions, cos_sin, cache, slots, 1)
out = torch.zeros(1, num_tokens, 512, dtype=torch.bfloat16, device=device)
dequantize_and_gather_k_cache(
out,
cache,
seq_lens=torch.tensor([num_tokens], dtype=torch.int32, device=device),
gather_lens=None,
block_table=torch.arange(num_blocks, dtype=torch.int32, device=device).view(
1, -1
),
block_size=block_size,
offset=0,
)
for t in (0, 1, block_size, num_tokens - 1):
page, row = divmod(t, block_size)
expected = _decode_nvfp4_row(
backing[page, row * 256 : (row + 1) * 256],
backing[page, block_size * 256 + row * 32 :][:32],
)
torch.testing.assert_close(out[0, t].float(), expected, rtol=0, atol=0)
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA only")
@pytest.mark.parametrize("compress_ratio", [1, 2])
@pytest.mark.parametrize("store_fp8", [False, True])
@@ -0,0 +1,71 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Layout plumbing for DeepSeek V4.1 mega attention.
The kernel's Q and O layouts are produced by permuting ``wq_b`` rows and
``wo_a`` columns once at load, so what has to hold is that the permuted GEMMs
agree with a reference that permutes the activation instead.
"""
import torch
from vllm.models.deepseek_v41.common.ops.fused_layout import (
WV_GROUP_SIZE,
o_fused_chunk_permutation,
o_fused_permutation,
permute_q_to_fused,
permute_wo_a_,
permute_wq_b_,
q_fused_permutation,
)
HEAD_DIM = 512
def test_permuted_wq_b_gemm_matches_permuted_activation():
"""Permuting wq_b's rows makes the Q GEMM emit the fused layout directly."""
torch.manual_seed(0)
num_heads, q_lora_rank, num_tokens = 16, 64, 5
weight = torch.randn(num_heads * HEAD_DIM, q_lora_rank)
# One scale row per weight row, as an MXFP8 wq_b shard carries.
scale = torch.randint(
0, 256, (num_heads * HEAD_DIM, q_lora_rank // 32), dtype=torch.uint8
)
qr = torch.randn(num_tokens, q_lora_rank)
orig_weight, orig_scale = weight.clone(), scale.clone()
standard = (qr @ weight.T).view(num_tokens, num_heads, HEAD_DIM)
expected = permute_q_to_fused(standard)
permute_wq_b_(weight, scale, num_heads)
fused = (qr @ weight.T).view(num_tokens, num_heads, HEAD_DIM)
torch.testing.assert_close(fused, expected)
# The scale has to follow its row, or dequant reads another row's scale.
perm = q_fused_permutation(num_heads, HEAD_DIM)
torch.testing.assert_close(weight, orig_weight[perm])
torch.testing.assert_close(scale, orig_scale[perm])
def test_permuted_wo_a_consumes_fused_output():
"""Permuting wo_a's input columns lets it read the kernel's O layout."""
torch.manual_seed(1)
out_features, num_tokens = 32, 4
in_features = WV_GROUP_SIZE * HEAD_DIM
weight = torch.randn(out_features, in_features)
scale = torch.randint(0, 256, (out_features, in_features // 32), dtype=torch.uint8)
standard_o = torch.randn(num_tokens, in_features)
orig_weight, orig_scale = weight.clone(), scale.clone()
expected = standard_o @ weight.T
# The kernel emits the same values in the fused chunk order.
perm = o_fused_permutation(WV_GROUP_SIZE, HEAD_DIM)
fused_o = standard_o[:, perm]
permute_wo_a_(weight, scale, WV_GROUP_SIZE)
torch.testing.assert_close(weight, orig_weight[:, perm])
torch.testing.assert_close(
scale, orig_scale[:, o_fused_chunk_permutation(WV_GROUP_SIZE, HEAD_DIM)]
)
# Unlike wq_b, this permutation sits inside the 4096-term reduction, so the
# summation order changes and the result differs in the last few ulps.
torch.testing.assert_close(fused_o @ weight.T, expected, rtol=1e-3, atol=1e-3)
@@ -522,6 +522,189 @@ def test_kv_path_mxfp8_record_matches_reference(num_tokens: int, block_size: int
)
# ── Test 2d: mega-attention Q layout (is_q_interleaved) ──────────────────────
Q_CHUNK = 16
NUM_Q_CHUNKS = HEAD_DIM // Q_CHUNK # 32
@pytest.mark.parametrize("num_tokens", [1, 17, 2048])
@pytest.mark.parametrize("n_heads,q_head_padded", [(16, 64), (64, 128), (64, 0)])
def test_q_interleaved_pads_without_rope(
num_tokens: int, n_heads: int, q_head_padded: int
):
"""``is_q_interleaved`` with no norm or RoPE only zero-pads Q.
The mega-attention kernel norms and rotates Q itself, so every live
element must survive bit-exact while the padding heads -- which interleave
with the live ones inside each of the 32 head-dim chunks -- come back
zero. ``q_head_padded=0`` drops the Q pass entirely and returns nothing,
for a shard already at the kernel's head count.
"""
torch.manual_seed(5)
device = "cuda"
dtype = torch.bfloat16
block_size = 64
q = torch.randn(num_tokens, n_heads, HEAD_DIM, dtype=dtype, device=device)
kv = torch.randn(num_tokens, HEAD_DIM, dtype=dtype, device=device)
positions = torch.arange(num_tokens, dtype=torch.int64, device=device)
cos_sin_cache = make_cos_sin_cache(4096, ROPE_DIM, torch.float32, device)
num_blocks = (num_tokens + block_size - 1) // block_size + 1
slot_mapping = torch.arange(num_tokens, dtype=torch.int64, device=device)
k_cache = torch.zeros(
num_blocks, block_size * V41_HEAD_BYTES, dtype=torch.uint8, device=device
)
q_out = torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
q,
kv,
k_cache,
slot_mapping,
positions,
cos_sin_cache,
q_head_padded,
1e-6,
block_size,
False, # apply_q_norm
True, # kv_mxfp8
False, # apply_q_rope
True, # is_q_interleaved
)
# The KV half is unaffected by the Q mode: it must still be inserted.
assert k_cache.any()
if q_head_padded == 0:
assert q_out.numel() == 0
return
assert q_out.shape == (num_tokens, q_head_padded, HEAD_DIM)
got = q_out.view(num_tokens, NUM_Q_CHUNKS, q_head_padded, Q_CHUNK)
want = q.view(num_tokens, NUM_Q_CHUNKS, n_heads, Q_CHUNK)
torch.testing.assert_close(got[:, :, :n_heads], want, rtol=0, atol=0)
assert (got[:, :, n_heads:] == 0).all()
@pytest.mark.parametrize("n_heads", [8, 16, 32])
def test_q_interleaved_survives_graph_replay(n_heads: int):
"""A captured fused insert refills its padded Q buffer on every replay.
The op allocates ``q_out`` itself, so the tensor a capture hands back has
to keep receiving the new padding when the graph is replayed over changed
input -- that is what lets the layer prepare Q inside the captured region
rather than in ``forward_mqa``.
"""
torch.manual_seed(6)
device = "cuda"
dtype = torch.bfloat16
block_size, num_tokens, q_head_padded = 64, 7, 64
q = torch.randn(num_tokens, n_heads, HEAD_DIM, dtype=dtype, device=device)
kv = torch.randn(num_tokens, HEAD_DIM, dtype=dtype, device=device)
positions = torch.arange(num_tokens, dtype=torch.int64, device=device)
cos_sin_cache = make_cos_sin_cache(4096, ROPE_DIM, torch.float32, device)
num_blocks = (num_tokens + block_size - 1) // block_size + 1
slot_mapping = torch.arange(num_tokens, dtype=torch.int64, device=device)
k_cache = torch.zeros(
num_blocks, block_size * V41_HEAD_BYTES, dtype=torch.uint8, device=device
)
def prepare():
return torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
q,
kv,
k_cache,
slot_mapping,
positions,
cos_sin_cache,
q_head_padded,
1e-6,
block_size,
False, # apply_q_norm
True, # kv_mxfp8
False, # apply_q_rope
True, # is_q_interleaved
)
# Capture takes its warmup on a side stream.
side = torch.cuda.Stream()
side.wait_stream(torch.cuda.current_stream())
with torch.cuda.stream(side):
for _ in range(3):
prepare()
torch.cuda.current_stream().wait_stream(side)
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
q_out = prepare()
for factor in (0.25, -0.75):
q.copy_(torch.randn_like(q) * factor)
graph.replay()
torch.accelerator.synchronize()
got = q_out.view(num_tokens, NUM_Q_CHUNKS, q_head_padded, Q_CHUNK)
want = q.view(num_tokens, NUM_Q_CHUNKS, n_heads, Q_CHUNK)
torch.testing.assert_close(got[:, :, :n_heads], want, rtol=0, atol=0)
assert (got[:, :, n_heads:] == 0).all()
@pytest.mark.parametrize("n_heads", [16, 64])
@pytest.mark.parametrize("apply_q_norm", [False, True])
@pytest.mark.parametrize("apply_q_rope", [False, True])
def test_q_interleaved_is_orthogonal_to_norm_and_rope(
n_heads: int, apply_q_norm: bool, apply_q_rope: bool
):
"""The Q layout composes with whatever norm and RoPE it is given.
A lane owns the same 512-dim slice of its head in either layout -- only
the address differs -- so the head-major result permuted into the
interleaved layout must equal running the interleaved path on a permuted
input, bit for bit, for every norm/RoPE combination.
"""
from vllm.models.deepseek_v41.common.ops.fused_layout import permute_q_to_fused
torch.manual_seed(7)
device = "cuda"
dtype = torch.bfloat16
block_size, num_tokens, q_head_padded = 64, 5, 64
q = torch.randn(num_tokens, n_heads, HEAD_DIM, dtype=dtype, device=device)
kv = torch.randn(num_tokens, HEAD_DIM, dtype=dtype, device=device)
positions = torch.arange(num_tokens, dtype=torch.int64, device=device)
cos_sin_cache = make_cos_sin_cache(4096, ROPE_DIM, torch.float32, device)
num_blocks = (num_tokens + block_size - 1) // block_size + 1
slot_mapping = torch.arange(num_tokens, dtype=torch.int64, device=device)
def run(q_in, is_q_interleaved):
cache = torch.zeros(
num_blocks, block_size * V41_HEAD_BYTES, dtype=torch.uint8, device=device
)
out = torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
q_in,
kv,
cache,
slot_mapping,
positions,
cos_sin_cache,
q_head_padded,
1e-6,
block_size,
apply_q_norm,
True, # kv_mxfp8
apply_q_rope,
is_q_interleaved,
)
return out, cache
head_major, cache_major = run(q, False)
interleaved, cache_inter = run(permute_q_to_fused(q), True)
torch.testing.assert_close(
interleaved, permute_q_to_fused(head_major), rtol=0, atol=0
)
# The Q layout must not leak into the KV half.
torch.testing.assert_close(cache_inter, cache_major, rtol=0, atol=0)
# ── Test 2b: DP padding (slot_mapping shorter than q/kv) ─────────────────────
+2 -1
View File
@@ -636,7 +636,8 @@ class DeepseekV41ROCMAiterMLAAttention(DeepseekV4Attention):
transpose_scale=False,
)
def _o_proj(self, o: torch.Tensor, positions: torch.Tensor) -> torch.Tensor:
def _o_proj(self, attn_out: torch.Tensor, positions: torch.Tensor) -> torch.Tensor:
o = attn_out[:, : self.n_local_heads, :]
# ROCm BF16 reference wo_a path (inverse RoPE + einsum) + wo_b.
z = rocm_inv_rope_einsum(
self.rotary_emb,
+150 -54
View File
@@ -44,6 +44,7 @@ from vllm.config import (
VllmConfig,
get_current_vllm_config,
)
from vllm.config.cache import CacheDType
from vllm.distributed import get_tensor_model_parallel_world_size
from vllm.forward_context import get_forward_context
from vllm.logger import init_logger
@@ -127,34 +128,40 @@ def _use_v41_mxfp8_kv_record() -> bool:
def _resolve_dsv4_kv_cache_dtype(
use_fp8_ds_mla_layout: bool,
kv_cache_dtype: str,
kv_cache_dtype: CacheDType,
cache_config: CacheConfig | None,
) -> tuple[str, torch.dtype]:
packed_kv_cache_dtype: CacheDType = "fp8_ds_mla",
) -> tuple[CacheDType, torch.dtype]:
"""Map ``(layout, --kv-cache-dtype)`` to ``(cache_dtype_str, torch_dtype)``.
Both layouts are paged; they differ in the per-token block format. The
``fp8_ds_mla`` format is UE8M0 block-scaled fp8 packed as ``uint8`` (the
canonical ``fp8_ds_mla`` string is written back onto ``cache_config`` so the
page-size specs pick the 576B per-token slot). Plain-row backends store each
token's KV row in its element dtype: bf16 or per-tensor FP8 E4M3.
packed formats are ``uint8``-backed: ``fp8_ds_mla`` is UE8M0 block-scaled
fp8 throughout, ``nvfp4_ds_mla`` keeps that sliding-window record and
stores the compressed cache as NVFP4. An unspecific ``--kv-cache-dtype``
(``auto`` / ``fp8``) resolves to ``packed_kv_cache_dtype``, the record this
layer's kernel prefers, and the canonical string is written back onto
``cache_config`` so the page-size specs pick the right per-token slot.
Plain-row backends store each token's KV row in its element dtype: bf16 or
per-tensor FP8 E4M3.
"""
if use_fp8_ds_mla_layout:
# fp8_ds_mla block format: UE8M0 block-scaled fp8 packed as uint8.
if kv_cache_dtype == "auto":
kv_cache_dtype = "fp8"
if not kv_cache_dtype.startswith("fp8"):
if kv_cache_dtype in ("auto", "fp8"):
kv_cache_dtype = packed_kv_cache_dtype
elif not kv_cache_dtype.endswith("_ds_mla"):
raise ValueError(
"DeepseekV4 fp8_ds_mla layout only supports fp8 "
f"kv-cache, got {kv_cache_dtype}. Please set "
"`--kv-cache-dtype fp8` or select a backend that supports "
"bfloat16 KV cache."
"DeepseekV4 packed KV layouts only support fp8 kv-cache, got "
f"{kv_cache_dtype}. Please set `--kv-cache-dtype fp8` or "
"select a backend that supports bfloat16 KV cache."
)
if kv_cache_dtype != "fp8_ds_mla":
if cache_config is not None:
cache_config.cache_dtype = "fp8_ds_mla"
kv_cache_dtype = "fp8_ds_mla"
logger.info_once("Using DeepSeek's fp8_ds_mla KV cache format.")
if cache_config is not None and cache_config.cache_dtype != kv_cache_dtype:
cache_config.cache_dtype = kv_cache_dtype
logger.info_once("Using DeepSeek's %s KV cache format.", kv_cache_dtype)
return kv_cache_dtype, torch.uint8
if kv_cache_dtype.endswith("_ds_mla"):
raise ValueError(
f"{kv_cache_dtype} is a packed FlashMLA DeepSeek V4.1 KV cache "
"format; the selected backend stores plain KV rows."
)
# Plain bf16 / per-tensor fp8 KV row (FlashInfer).
if kv_cache_dtype.startswith("fp8"):
@@ -189,6 +196,28 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
# path to pre-reserve that workspace.
PREFILL_CHUNK_SIZE: ClassVar[int] = 4
# ---- attention-interface contract, declared by the platform subclass ----
@property
def accepts_unnormed_unroped_query(self) -> bool:
"""Whether ``forward_mqa``'s ``q`` is the raw ``wq_b`` output.
True when the attention kernel applies the Q norm and RoPE itself, and
reads Q in its own chunk-interleaved layout, so the layer only
zero-pads Q to ``padded_heads`` and inserts KV before calling it.
"""
return False
@property
def packed_kv_cache_dtype(self) -> CacheDType:
"""The packed KV record this layer's kernel prefers.
What an unspecific ``--kv-cache-dtype`` (``auto`` / ``fp8``) resolves
to. Mega attention overrides it: its kernel is the one that can read
an NVFP4 compressed cache.
"""
return "fp8_ds_mla"
@classmethod
@abstractmethod
def get_padded_num_q_heads(cls, num_heads: int) -> int:
@@ -213,8 +242,19 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
raise NotImplementedError
@abstractmethod
def _o_proj(self, o: torch.Tensor, positions: torch.Tensor) -> torch.Tensor:
"""Inverse-RoPE + wo_a + wo_b output projection (platform-specific)."""
def _o_proj(
self,
attn_out: "torch.Tensor | QuantizedActivation",
positions: torch.Tensor,
) -> torch.Tensor:
"""Project whatever ``_alloc_attn_out`` produced through wo_a and wo_b.
Takes the buffer whole, so each layer owns the shape it allocated: the
bf16 layers slice off their padding heads and apply the inverse RoPE,
while a layer whose attention kernel already did the inverse RoPE and
the FP8 cast gets a QuantizedActivation and has only wo_a and wo_b
left.
"""
raise NotImplementedError
def _uses_fp8_ds_mla_layout(self) -> bool:
@@ -459,11 +499,27 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
# Resolve the kv-cache dtype from this backend's block format. The same
# resolution drives the SWA cache tensor dtype below.
self.kv_cache_dtype, self.kv_cache_torch_dtype = _resolve_dsv4_kv_cache_dtype(
self._uses_fp8_ds_mla_layout(), cache_config.cache_dtype, cache_config
self._uses_fp8_ds_mla_layout(),
cache_config.cache_dtype,
cache_config,
self.packed_kv_cache_dtype,
)
self.kv_mxfp8 = _use_v41_mxfp8_kv_record()
self.kv_bytes_per_token = 528 if self.kv_mxfp8 else 584
self.swa_bytes_per_token = 528 if self.kv_mxfp8 else 584
# nvfp4_ds_mla keeps the MXFP8 sliding-window record and stores the
# compressed cache as NVFP4 (256 B of e2m1 pairs + 32 e4m3 scales).
self.compressed_bytes_per_token = (
288 if self.kv_cache_dtype == "nvfp4_ds_mla" else self.swa_bytes_per_token
)
# One alignment for every page in the block: the block stride is their
# sum, and 512 satisfies both TMA strides in play (512 for the V4.1
# fp8 record, 256 for NVFP4).
self.kv_page_alignment = 512 if self.kv_mxfp8 else 576
if self.kv_cache_dtype == "nvfp4_ds_mla" and not self.kv_mxfp8:
raise ValueError(
"nvfp4_ds_mla needs the V4.1 KV records, which FlashMLA "
"decodes only on SM100."
)
self.swa_cache_layer = DeepseekV4SWACache(
head_dim=self.head_dim,
@@ -473,7 +529,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
cache_config=cache_config,
backend_cls=self.swa_backend_cls,
block_size=32,
packed_bytes_per_token=self.kv_bytes_per_token,
packed_bytes_per_token=self.swa_bytes_per_token,
packed_page_alignment=self.kv_page_alignment,
)
@@ -553,8 +609,12 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
block_size=self.swa_cache_layer.block_size,
)
# Every backend that gathers a chunk's KV through
# combine_topk_swa_indices needs its Triton kernel warmed, mega
# attention included -- it calls it from _forward_prefill_mega.
if self.backend_cls.get_name() in (
"FLASHMLA_SPARSE_DSV41",
"FLASHMLA_MEGA_ATTN_DSV41",
"ROCM_FLASHMLA_SPARSE_DSV4",
):
from vllm.models.deepseek_v41.common.ops.cache_utils import (
@@ -569,14 +629,10 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
hidden_states: torch.Tensor,
llama_4_scaling: torch.Tensor | None = None,
) -> torch.Tensor:
# Pre-allocate attention output with FlashMLA-padded head count.
# The op writes into `o_padded`; we slice to n_local_heads after.
num_tokens = hidden_states.shape[0]
o_padded = torch.empty(
(num_tokens, self.padded_heads, self.head_dim),
dtype=hidden_states.dtype,
device=hidden_states.device,
)
# The eager attention region writes into a caller-owned buffer
# (breakable_cudagraph needs in-place outputs); its shape and how it is
# projected afterwards follow the interface contract above.
attn_out = self._alloc_attn_out(hidden_states.shape[0], hidden_states)
# Keep the attention input preparation in the captured graph. Only the
# sparse indexer and MLA attention run in the eager break below.
@@ -593,12 +649,19 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
kv_score,
indexer_weights,
positions,
o_padded,
attn_out,
)
o = o_padded[:, : self.n_local_heads, :]
return self._o_proj(attn_out, positions)
# Inverse-RoPE + wo_a + wo_b output projection (platform-specific).
return self._o_proj(o, positions)
def _alloc_attn_out(
self, num_tokens: int, hidden_states: torch.Tensor
) -> "torch.Tensor | QuantizedActivation":
"""The buffer ``forward_mqa`` fills, per the interface contract."""
return torch.empty(
(num_tokens, self.padded_heads, self.head_dim),
dtype=hidden_states.dtype,
device=hidden_states.device,
)
@cached_property
def _can_fuse_query_quant(self) -> bool:
@@ -652,7 +715,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
kv_score: torch.Tensor,
indexer_weights: torch.Tensor,
positions: torch.Tensor,
o_padded: torch.Tensor,
attn_out: "torch.Tensor | QuantizedActivation",
) -> None:
"""Wide eager region: the whole of ``_prepare_and_attn`` runs eagerly.
@@ -667,7 +730,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
kv_score,
indexer_weights,
positions,
o_padded,
attn_out,
)
def _prepare_and_attn(
@@ -679,7 +742,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
kv_score: torch.Tensor,
indexer_weights: torch.Tensor,
positions: torch.Tensor,
o_padded: torch.Tensor,
attn_out: "torch.Tensor | QuantizedActivation",
) -> None:
"""Attention input preparation followed by the sparse indexer and MLA.
@@ -751,7 +814,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
q,
kv,
positions,
o_padded,
attn_out,
)
def _fused_wqa_wkv_gemm(self, hidden_states: torch.Tensor) -> torch.Tensor:
@@ -858,16 +921,28 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
dict[str, AttentionMetadata] | list[dict[str, AttentionMetadata]] | None
),
) -> torch.Tensor:
"""Ready ``q`` for the attention kernel and publish this step's KV.
One launch does both. With ``accepts_unnormed_unroped_query`` the
attention kernel norms and rotates Q itself and reads it in its own
chunk-interleaved layout, so the Q half of the launch is a zero-pad to
``padded_heads`` -- and nothing at all once the shard is that wide.
"""
if not isinstance(attn_metadata, dict):
# Profile run: kernel doesn't fire; produce a padded tensor so
# downstream FlashMLA gets the right shape.
if self.n_local_heads < self.padded_heads:
return F.pad(
q,
(0, 0, 0, self.padded_heads - self.n_local_heads),
value=0.0,
)
return q
if self.n_local_heads >= self.padded_heads:
return q
if self.accepts_unnormed_unroped_query:
# Padding heads sit at the tail of every head-dim chunk in
# that layout, so no head-major pad of `q` reproduces it --
# and nothing reads it on a profile run.
return q.new_zeros((q.shape[0], self.padded_heads, q.shape[2]))
return F.pad(
q,
(0, 0, 0, self.padded_heads - self.n_local_heads),
value=0.0,
)
swa_metadata = cast(
"DeepseekSparseSWAMetadata | None",
@@ -886,23 +961,38 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
if cache_dtype == torch.uint8:
# fp8_ds_mla UE8M0 paged path. Horizontally fused:
# Q side: GPT-J RoPE, zero-filling the padding head slots; the
# kernel allocates and returns the padded q tensor.
# kernel allocates and returns the padded q tensor. An
# interleaved Q skips the RoPE its attention kernel owns
# and keeps only the pad, which q_head_padded=0 drops
# too once the shard is already padded_heads wide.
# KV side: GPT-J RoPE + UE8M0 FP8 quant + paged cache insert.
swa_kv_cache_2d = swa_kv_cache.view(swa_kv_cache.shape[0], -1)
return torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
pad_to = (
0
if self.accepts_unnormed_unroped_query
and self.n_local_heads == self.padded_heads
else self.padded_heads
)
q_padded = torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
q,
kv,
swa_kv_cache_2d,
swa_metadata.slot_mapping,
positions,
cos_sin_cache,
self.padded_heads,
pad_to,
self.eps,
swa_metadata.block_size,
False,
False, # apply_q_norm: qr is normed before wq_b
self.kv_mxfp8,
not self.accepts_unnormed_unroped_query, # apply_q_rope
self.accepts_unnormed_unroped_query, # is_q_interleaved
)
return q if pad_to == 0 else q_padded
assert not self.accepts_unnormed_unroped_query, (
"the chunk-interleaved Q layout only pairs with a packed KV record"
)
# Plain-row path: the [num_blocks, block_size, 512] cache stores the KV
# row in its element dtype (no Q padding). bf16 rewrites q in place;
# per-tensor fp8 writes a separately-allocated fp8 q and quantizes the
@@ -959,7 +1049,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
# fp8_ds_mla is a UE8M0 block-scaled uint8 layout whose page rounds up
# to the decode kernel's TMA stride; plain bf16 / per-tensor fp8 rows
# use natural element-size pages.
uses_fp8_ds_mla_layout = self.kv_cache_dtype == "fp8_ds_mla"
uses_fp8_ds_mla_layout = self.kv_cache_dtype in ("fp8_ds_mla", "nvfp4_ds_mla")
return MLAAttentionSpec(
block_size=vllm_config.cache_config.block_size,
num_kv_heads=1,
@@ -972,7 +1062,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
kv_quant_mode=get_kv_quant_mode(self.kv_cache_dtype),
# Packed record width; head_size stays semantic (512).
state_content_bytes=(
self.kv_bytes_per_token if uses_fp8_ds_mla_layout else None
self.compressed_bytes_per_token if uses_fp8_ds_mla_layout else None
),
)
@@ -1016,7 +1106,13 @@ class DeepseekV4IndexerCache(torch.nn.Module, AttentionLayerBase):
def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec:
# head_dim already carries the fp8 scale padding
# tokens_per_state=1 for V3.2, >1 for DeepseekV4; same cache layout.
uses_fp8_ds_mla_layout = vllm_config.cache_config.cache_dtype == "fp8_ds_mla"
# nvfp4_ds_mla is packed too: its compressed record is NVFP4 but the
# sliding-window record stays the V4.1 MXFP8 one, so the indexer page
# takes the same alignment either way.
uses_fp8_ds_mla_layout = vllm_config.cache_config.cache_dtype in (
"fp8_ds_mla",
"nvfp4_ds_mla",
)
page_alignment = (
576 if uses_fp8_ds_mla_layout and not _use_v41_mxfp8_kv_record() else 512
)
@@ -43,10 +43,16 @@ from vllm.utils.math_utils import next_power_of_2
# dims each plus a pad byte.
# V4.1 (528 B): all 512 dims as fp8 e4m3 (RoPE included), then 16 UE8M0
# scales of 32 dims each. FlashMLA's ``ModelType::V41``.
# V4.1 NVFP4 (288 B): 512 e2m1 values packed two per byte (even element in
# the low nibble), then 32 e4m3 scales of 16 dims each.
# FlashMLA's ``ModelType::V41_FP4``, compressed cache only.
V4_BYTES_PER_TOKEN = 584
V41_BYTES_PER_TOKEN = 528
V41_QUANT_BLOCK = 32
V41_NUM_SCALES = 512 // V41_QUANT_BLOCK # 16
V41_NVFP4_BYTES_PER_TOKEN = 288
V41_NVFP4_QUANT_BLOCK = 16
V41_NVFP4_NUM_SCALES = 512 // V41_NVFP4_QUANT_BLOCK # 32
@triton.jit
@@ -496,6 +502,73 @@ def _dequantize_and_gather_k_mxfp8_kernel(
tl.store(output_row_ptr + d, dequant.to(tl.bfloat16))
@triton.jit
def _dequantize_and_gather_k_nvfp4_kernel(
out_ptr,
out_stride0,
out_stride1,
k_cache_ptr,
seq_lens_ptr,
block_table_ptr,
offset,
gather_lens_ptr,
max_blocks_per_seq: tl.constexpr,
head_dim: tl.constexpr, # 512
scale_dim: tl.constexpr, # 32
quant_block: tl.constexpr, # 16
cache_block_size: tl.constexpr,
block_stride: tl.constexpr,
):
"""Gather and dequantize V4.1 NVFP4 rows into a bf16 workspace."""
batch_idx = tl.program_id(0)
worker_id = tl.program_id(1)
num_workers = tl.num_programs(1)
seq_len = tl.load(seq_lens_ptr + batch_idx)
if gather_lens_ptr is not None: # noqa: SIM108
gather_len = tl.load(gather_lens_ptr + batch_idx)
else:
gather_len = seq_len
start_pos = seq_len - gather_len
packed_bytes: tl.constexpr = head_dim // 2
d = tl.arange(0, head_dim)
for i in range(worker_id, gather_len, num_workers):
pos = start_pos + i
block_in_seq = pos // cache_block_size
pos_in_block = pos % cache_block_size
physical_block_idx = tl.load(
block_table_ptr + batch_idx * max_blocks_per_seq + block_in_seq
)
page = k_cache_ptr + physical_block_idx.to(tl.int64) * block_stride
packed = tl.load(
page + pos_in_block * packed_bytes + tl.arange(0, packed_bytes)
)
# Even element in the low nibble, odd in the high nibble.
codes = tl.interleave((packed & 0xF).to(tl.int32), (packed >> 4).to(tl.int32))
# e2m1 magnitudes: 0, 0.5, 1, 1.5, 2, 3, 4, 6.
mag_code = codes & 7
e = (mag_code >> 1).to(tl.float32)
m = (mag_code & 1).to(tl.float32)
mag = tl.where(mag_code < 2, m * 0.5, (1.0 + m * 0.5) * tl.exp2(e - 1.0))
vals = tl.where(codes >= 8, -mag, mag)
sf = tl.load(
page
+ cache_block_size * packed_bytes
+ pos_in_block * scale_dim
+ tl.arange(0, scale_dim)
)
scale = sf.to(tl.float8e4nv, bitcast=True).to(tl.float32)
tiles = tl.reshape(vals, (scale_dim, quant_block))
dequant = tl.reshape(tiles * tl.reshape(scale, (scale_dim, 1)), (head_dim,))
output_row_ptr = out_ptr + batch_idx * out_stride0 + (offset + i) * out_stride1
tl.store(output_row_ptr + d, dequant.to(tl.bfloat16))
def dequantize_and_gather_k_cache_triton(
# [num_reqs, max_num_tokens, head_size]
out: torch.Tensor,
@@ -513,6 +586,25 @@ def dequantize_and_gather_k_cache_triton(
) -> None:
num_reqs = seq_lens.shape[0]
NUM_WORKERS = 128
if k_cache.shape[-1] == V41_NVFP4_BYTES_PER_TOKEN:
_dequantize_and_gather_k_nvfp4_kernel[(num_reqs, NUM_WORKERS)](
out,
out.stride(0),
out.stride(1),
k_cache,
seq_lens,
block_table,
offset,
gather_lens,
max_blocks_per_seq=block_table.shape[-1],
head_dim=512,
scale_dim=V41_NVFP4_NUM_SCALES,
quant_block=V41_NVFP4_QUANT_BLOCK,
cache_block_size=block_size,
block_stride=k_cache.stride(0),
)
return
if k_cache.shape[-1] == V41_BYTES_PER_TOKEN:
_dequantize_and_gather_k_mxfp8_kernel[(num_reqs, NUM_WORKERS)](
out,
@@ -581,14 +673,16 @@ def dequantize_and_gather_k_cache(
) -> None:
"""Dequantize and gather a paged DSv4 K cache.
The record is read off ``k_cache.shape[-1]``; see the module header.
The record is read off ``k_cache.shape[-1]``; see the module header. Only
the fp8 records have a CuteDSL gather, so NVFP4 always takes the Triton
path.
``use_fnuz`` MUST match the encoder of the specific cache being read:
``False`` for ``compressed_k_cache`` (Triton encoder is OCP everywhere),
``current_platform.is_fp8_fnuz()`` for ``swa_k_cache`` (C++ encoder
writes FNUZ on gfx942 and OCP on gfx950).
"""
if has_cutedsl():
if has_cutedsl() and k_cache.shape[-1] != V41_NVFP4_BYTES_PER_TOKEN:
# lazily import, otherwise some tests fail due to CUDA driver init failure.
from vllm.models.deepseek_v4.nvidia.ops.dequant_gather_k_cutedsl import (
_DEQUANT_GATHER_K_CACHE_CUTEDSL_KERNEL,
@@ -4,6 +4,7 @@
import torch
from vllm.models.deepseek_v4.common.ops.fused_indexer_q import _fp32x2_to_fp4x2
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
@@ -232,13 +233,14 @@ def rope_quant_insert(
The BF16 latent supplies both NoPE quantization and RoPE input. It is read
only for valid slots at group boundaries. The cache dtype selects the
layout: ``uint8`` is the fp8_ds_mla paged layout, whose record the
per-token byte width names -- 584 B for V4 (576 value bytes and eight
segregated UE8M0 scale bytes, including one zero padding scale) or 528 B
for V4.1 (512 MXFP8 value bytes covering the RoPE dims too, then 16 UE8M0
scales of 32 dims each). ``bfloat16`` and ``float8_e4m3fn`` are the plain
[448 NoPE | 64 RoPE] rows read by FlashInfer, the latter scaled by the
per-tensor ``fp8_scale``.
layout: ``uint8`` is a paged FlashMLA layout, whose record the per-token
byte width names -- 584 B for V4 (576 value bytes and eight segregated
UE8M0 scale bytes, including one zero padding scale), 528 B for V4.1
(512 MXFP8 value bytes covering the RoPE dims too, then 16 UE8M0 scales of
32 dims each), or 288 B for V4.1 NVFP4 (256 bytes of e2m1 pairs then 32
e4m3 scales of 16 dims each), which only the compressed cache uses.
``bfloat16`` and ``float8_e4m3fn`` are the plain [448 NoPE | 64 RoPE] rows
read by FlashInfer, the latter scaled by the per-tensor ``fp8_scale``.
"""
assert compress_ratio in (1, 2)
assert latent.shape[1] == 512 and latent.dtype == torch.bfloat16
@@ -249,14 +251,14 @@ def rope_quant_insert(
return
launch_kwargs = {"launch_pdl": False} if current_platform.is_cuda() else {}
if kv_cache.dtype == torch.uint8:
assert kv_cache.shape[-1] in (584, 528), (
kernel = {
584: _rope_quant_insert_kernel,
528: _rope_quant_insert_mxfp8_kernel,
288: _rope_quant_insert_nvfp4_kernel,
}.get(kv_cache.shape[-1])
assert kernel is not None, (
f"unsupported paged KV record width {kv_cache.shape[-1]}"
)
kernel = (
_rope_quant_insert_mxfp8_kernel
if kv_cache.shape[-1] == 528
else _rope_quant_insert_kernel
)
kernel[(num_tokens,)](
latent,
positions,
@@ -402,6 +404,62 @@ def _rope_quant_insert_mxfp8_kernel(
tl.store(scales + tl.arange(0, 16), encoded.to(tl.uint8))
@triton.jit
def _rope_quant_insert_nvfp4_kernel(
latent,
positions,
cos_sin,
cache,
cache_slots,
COS_STRIDE: tl.constexpr,
CACHE_STRIDE: tl.constexpr,
CACHE_BLOCK: tl.constexpr,
COMPRESS_RATIO: tl.constexpr,
SANITIZE_CACHE_NANS: tl.constexpr,
):
"""V4.1 NVFP4 record: RoPE, then e2m1 with one e4m3 scale per 16 dims.
The scale is ``amax / 6`` (6 is e2m1's largest magnitude) clamped to the
e4m3 range, with no per-tensor scale on top.
"""
t = tl.program_id(0)
slot = tl.load(cache_slots + t)
if slot < 0:
return
position = tl.load(positions + t)
if (position + 1) % COMPRESS_RATIO != 0:
return
d = tl.arange(0, 512)
normed = tl.load(latent + t.to(tl.int64) * 512 + d).to(tl.float32)
# NoPE pairs load (cos, sin) = (1, 0), so the rotation is the identity there.
even, odd = tl.split(tl.reshape(normed, (256, 2)))
pair = tl.arange(0, 256) - 224
cs = cos_sin + (position // COMPRESS_RATIO * COMPRESS_RATIO) * COS_STRIDE
c = tl.load(cs + tl.maximum(pair, 0), pair >= 0, other=1.0).to(tl.float32)
s = tl.load(cs + 32 + tl.maximum(pair, 0), pair >= 0, other=0.0).to(tl.float32)
rotated = tl.interleave(even * c - odd * s, odd * c + even * s)
if SANITIZE_CACHE_NANS:
rotated = tl.where(rotated == rotated, rotated, 0.0)
tiles = tl.reshape(rotated, (32, 16))
amax = tl.max(tl.abs(tiles), 1)
# 2**-9 is the smallest normal e4m3 magnitude; 448 the largest.
scale = tl.clamp(amax * (1.0 / 6.0), 0.001953125, 448.0).to(tl.float8e4nv)
# Round-to-nearest division: Triton's default div.full misplaces values
# that land exactly on an e2m1 tie.
scaled = tl.math.div_rn(tiles, tl.reshape(scale.to(tl.float32), (32, 1)))
lo, hi = tl.split(tl.reshape(tl.reshape(scaled, (512,)), (256, 2)))
page = cache + (slot // CACHE_BLOCK).to(tl.int64) * CACHE_STRIDE
tl.store(
page + (slot % CACHE_BLOCK) * 256 + tl.arange(0, 256),
_fp32x2_to_fp4x2(lo, hi),
)
scales = page + CACHE_BLOCK * 256 + (slot % CACHE_BLOCK) * 32
tl.store(scales + tl.arange(0, 32), scale.to(tl.uint8, bitcast=True))
@triton.jit
def _rope_plain_insert_kernel(
latent,
@@ -0,0 +1,86 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Weight permutations for FlashMLA's mega-attention kernel.
The kernel reads Q with 16-element head-dim chunks interleaved across heads and
writes O with 32-element chunks interleaved across the 8 heads of a ``wo_a``
group. Every permutation here satisfies ``fused = standard[perm]``, and is
applied once to ``wq_b`` rows and ``wo_a`` columns at load time so the
surrounding GEMMs produce and consume the kernel's layouts directly -- no
per-step shuffle.
"""
import torch
HEAD_DIM = 512
Q_CHUNK = 16
O_CHUNK = 32
WV_GROUP_SIZE = 8
def q_fused_permutation(num_heads: int, head_dim: int = HEAD_DIM) -> torch.Tensor:
"""``fused[(d // 16) * (H * 16) + h * 16 + d % 16] = standard[h * D + d]``."""
h = torch.arange(num_heads).view(num_heads, 1)
d = torch.arange(head_dim).view(1, head_dim)
fused_index = (d // Q_CHUNK) * (num_heads * Q_CHUNK) + h * Q_CHUNK + d % Q_CHUNK
perm = torch.empty(num_heads * head_dim, dtype=torch.long)
perm[fused_index.reshape(-1)] = torch.arange(num_heads * head_dim)
return perm
def o_fused_permutation(
heads_per_group: int = WV_GROUP_SIZE, head_dim: int = HEAD_DIM
) -> torch.Tensor:
"""``fused[(c * G + h) * 32 + j] = standard[h * D + c * 32 + j]`` per group."""
h = torch.arange(heads_per_group).view(-1, 1, 1)
c = torch.arange(head_dim // O_CHUNK).view(1, -1, 1)
j = torch.arange(O_CHUNK).view(1, 1, -1)
fused_index = (c * heads_per_group + h) * O_CHUNK + j
perm = torch.empty(heads_per_group * head_dim, dtype=torch.long)
perm[fused_index.reshape(-1)] = torch.arange(heads_per_group * head_dim)
return perm
def o_fused_chunk_permutation(
heads_per_group: int = WV_GROUP_SIZE, head_dim: int = HEAD_DIM
) -> torch.Tensor:
"""Per-32-element-chunk form of :func:`o_fused_permutation`, for scales."""
return o_fused_permutation(heads_per_group, head_dim)[::O_CHUNK] // O_CHUNK
def permute_q_to_fused(q: torch.Tensor) -> torch.Tensor:
"""``[N, H, D]`` standard layout -> the same shape in the fused layout."""
n, h, d = q.shape
perm = q_fused_permutation(h, d).to(q.device)
return q.reshape(n, h * d)[:, perm].view(n, h, d)
def _bytes_view(t: torch.Tensor) -> torch.Tensor:
"""Byte view of a 1-byte-element tensor, so fp8/ue8m0 can be gathered."""
return t.view(torch.uint8) if t.element_size() == 1 else t
def permute_wq_b_(
weight: torch.Tensor, weight_scale: torch.Tensor, num_local_heads: int
) -> None:
"""Permute an MXFP8 ``wq_b`` shard's rows and per-row scales, in place."""
head_dim = weight.shape[0] // num_local_heads
perm = q_fused_permutation(num_local_heads, head_dim).to(weight.device)
for t in (weight, weight_scale):
b = _bytes_view(t)
b.copy_(b[perm])
def permute_wo_a_(
weight: torch.Tensor,
weight_scale: torch.Tensor,
heads_per_group: int = WV_GROUP_SIZE,
) -> None:
"""Permute an MXFP8 ``wo_a`` shard's input columns and scales, in place."""
head_dim = weight.shape[1] // heads_per_group
perm = o_fused_permutation(heads_per_group, head_dim).to(weight.device)
w = _bytes_view(weight)
w.copy_(w[:, perm])
chunk_perm = o_fused_chunk_permutation(heads_per_group, head_dim)
s = _bytes_view(weight_scale)
s.copy_(s[:, chunk_perm.to(weight.device)])
+14
View File
@@ -477,8 +477,22 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
for layer in self.model.layers:
layer.ffn.finalize_mega_moe_weights()
def _finalize_attn(self) -> None:
"""Run the attention backend's post-load weight step on the draft layers.
They are ordinary DeepseekV4DecoderLayers, so they take the engine-wide
attention backend and owe it whatever the target model owes it -- mega
attention permutes wq_b / wo_a here and refuses to run without it.
Idempotent, like the target's.
"""
for layer in self.model.layers:
finalize = getattr(layer.attn, "finalize_loaded_weights", None)
if finalize is not None:
finalize()
def process_weights_after_loading(self) -> None:
self._finalize_moe()
self._finalize_attn()
def _remap_dspark_name(self, name: str) -> str | None:
"""Map a checkpoint ``mtp.{i}.*`` name to this model's parameter path.
@@ -0,0 +1,497 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""DeepSeek V4.1 attention on FlashMLA's mega-attention kernel (SM100).
One kernel does Q RoPE, sparse attention, the inverse RoPE of the output and
its FP8 cast, and writes straight into the buffer ``wo_a`` consumes. So the
layer declares ``accepts_unnormed_unroped_query`` -- the kernel RoPEs Q itself,
leaving the fused KV insert only the zero-pad to the kernel's head count -- and
its ``_alloc_attn_out`` / ``_o_proj`` pair speaks QuantizedActivation instead
of bf16, since the output needs no rotation or quantization here.
``wq_b`` rows and ``wo_a`` columns are permuted once at load so the surrounding
GEMMs speak the kernel's chunk-interleaved layouts directly. A step's prefill
and decode segments write disjoint token ranges of one output buffer pair, so
a single ``wo_a`` einsum covers the whole step.
"""
from dataclasses import replace
from typing import TYPE_CHECKING, cast
import torch
from vllm.config import VllmConfig
from vllm.config.cache import CacheDType
from vllm.forward_context import get_forward_context
from vllm.model_executor.layers.fusion.quant_activation import QuantizedActivation
from vllm.model_executor.layers.quantization.utils.quant_utils import kMxfp8Dynamic
from vllm.models.deepseek_v41.common.ops import (
combine_topk_swa_indices,
compute_global_topk_indices_and_lens,
dequantize_and_gather_k_cache,
)
from vllm.models.deepseek_v41.common.ops.fused_layout import (
WV_GROUP_SIZE,
permute_wo_a_,
permute_wq_b_,
)
from vllm.models.deepseek_v41.nvidia.flashmla import DeepseekV4FlashMLAAttention
from vllm.models.deepseek_v41.sparse_mla import (
DeepseekV4FlashMLAMetadata,
FlashMLAMegaAttnBackend,
)
from vllm.platforms import current_platform
from vllm.utils.deep_gemm import fp8_einsum, get_tma_aligned_size
from vllm.utils.math_utils import round_up
from vllm.v1.attention.ops.flashmla import is_flashmla_sparse_supported
from vllm.v1.worker.workspace import current_workspace_manager
if TYPE_CHECKING:
from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata
_HEAD_DIM_V = 512
_ROPE_DIM = 64
# The kernel always emits per-32 scales, whatever num_per_channels says.
_QUANT_GROUP = 32
# Fixed kernel conventions: GPT-J (non-neox) RoPE, TMA-aligned col-major
# scales, round-to-nearest, packed ue8m0 -- i.e. what DeepGEMM's fp8_einsum
# consumes. Q norm is off: the V4.1 layer norms qr before wq_b.
_ROPE_ARGS = (False, _ROPE_DIM)
_SF_ARGS = (_QUANT_GROUP, True, True, True)
# Decode always goes through the mega kernel: there is no batch size at which
# falling back to the split-KV decode pays, so there is no minimum-token
# threshold and no fallback path to maintain. What decides whether this layer
# is worth selecting at all is TP, not the batch size.
#
# Measured on one GB300 with benchmarks/kernels/benchmark_dsv41_mega_attn.py,
# CUDA graphs on, counting every per-step op each path runs (64-head model,
# topk_swa=128, topk_extra=512, NVFP4 compressed cache), as mega / split-KV
# microseconds:
#
# s_q 1 32 128 256 512
# TP1 (64 live heads) 29/29 31/31 31/40 53/72 96/140
# TP2 (32 live) 29/29 31/30 35/35 61/63 110/119
# TP4 (16 live) 29/29 31/29 34/34 59/58 109/109
# TP8 (8 live) 29/29 31/29 33/33 59/57 107/104
#
# The kernel absorbs the inverse RoPE and the FP8 cast, whose cost scales with
# *live* heads, while its own cost scales with *padded* heads -- always 64,
# since FlashMLA's FP8 decode takes only h_q in {64, 128}. So it wins at TP1
# (up to 1.45x, and TP1 is also where the fused insert skips the Q pad
# entirely) and from TP2 down the two cancel to within a few percent, with a
# fixed ~2 us penalty at small s_q where the padded-head cost dominates.
# Measure before assuming a win at high TP -- and measure under CUDA graphs:
# in eager mode launch overhead swamps both paths and overstates the fused one
# several-fold.
def is_flashmla_mega_attn_supported() -> tuple[bool, str | None]:
"""Whether FlashMLA's fused mega-attention kernels are usable here."""
is_available, maybe_reason = is_flashmla_sparse_supported()
if not is_available:
return False, maybe_reason
if not current_platform.is_device_capability_family(100):
return False, "FlashMLA mega attention requires sm_10x GPUs."
if not hasattr(torch.ops._flashmla_C, "fused_norm_rope_attn_rope_cast_decode"):
return False, "vllm._flashmla_C was built without the mega attention ops."
if not hasattr(torch.ops._flashmla_C, "permute_q_b_proj"):
return False, "vllm._flashmla_C predates the mega attention weight permutes."
return True, None
def alloc_mega_attn_output(
num_tokens: int,
n_wv_group: int,
device: torch.device,
) -> QuantizedActivation:
"""Allocate the output buffer pair the mega-attention kernels write into.
One pair per forward step: the prefill and decode segments fill disjoint
token ranges, so a single ``fp8_einsum`` over all ``num_tokens`` can
consume the result instead of one call per segment.
The scale buffer is MN-major -- stride 1 along tokens, head-dim stride
``ceil4(num_tokens)`` -- which is both what the kernel requires of a
caller-provided buffer and what DeepGEMM expects for this N.
"""
d = WV_GROUP_SIZE * _HEAD_DIM_V
data = torch.empty(
(num_tokens, n_wv_group, d), dtype=torch.float8_e4m3fn, device=device
)
aligned = get_tma_aligned_size(num_tokens, torch.int32.itemsize)
scale = torch.empty(
(n_wv_group, d // (_QUANT_GROUP * 4), aligned),
dtype=torch.int32,
device=device,
).permute(2, 0, 1)[:num_tokens]
return QuantizedActivation(
data=data,
scale=scale,
orig_dtype=torch.bfloat16,
orig_shape=data.shape,
quant_key=kMxfp8Dynamic,
)
def _token_slice(out: QuantizedActivation, start: int, end: int) -> QuantizedActivation:
"""The ``[start, end)`` token slice of a mega-attention output buffer."""
data = out.data[start:end]
return replace(out, data=data, scale=out.scale[start:end], orig_shape=data.shape)
class DeepseekV4MegaAttnAttention(DeepseekV4FlashMLAAttention):
"""FlashMLA mega-attention layer for DeepSeek V4.1 (SM100)."""
backend_cls = FlashMLAMegaAttnBackend
@classmethod
def is_available_for(cls, vllm_config: VllmConfig) -> bool:
"""Whether this layer can serve the configured model on this device.
The kernel is SM100-only and wants ``WV_GROUP_SIZE`` heads per ``wo_a``
group, which stops holding once TP divides the head count far enough
(TP16 on a 64-head model leaves 4). ``__init__`` raises on both, so the
default-backend selector asks here first rather than turning an
unsupported topology into a startup crash.
"""
if not is_flashmla_mega_attn_supported()[0]:
return False
config = vllm_config.model_config.hf_config
config = getattr(config, "text_config", config)
n_heads = getattr(config, "num_attention_heads", 0) or 0
n_groups = getattr(config, "o_groups", 0) or 0
tp_size = vllm_config.parallel_config.tensor_parallel_size
if n_heads % tp_size or n_groups % tp_size:
return False
n_local_heads, n_local_groups = n_heads // tp_size, n_groups // tp_size
if not n_local_groups or n_local_heads % WV_GROUP_SIZE:
return False
return n_local_heads // n_local_groups == WV_GROUP_SIZE
def __init__(self, vllm_config: VllmConfig, *args, **kwargs) -> None:
super().__init__(vllm_config, *args, **kwargs)
if self.n_local_heads % WV_GROUP_SIZE:
raise ValueError(
f"{self.prefix}: mega attention needs the local head count "
f"({self.n_local_heads}) to be a multiple of {WV_GROUP_SIZE}."
)
if self.n_local_heads // self.n_local_groups != WV_GROUP_SIZE:
raise ValueError(
f"{self.prefix}: mega attention needs {WV_GROUP_SIZE} heads per "
"wo_a group."
)
self.n_wv_group = self.padded_heads // WV_GROUP_SIZE
self._fused_layouts_ready = False
# ---- interface contract ------------------------------------------------
@property
def accepts_unnormed_unroped_query(self) -> bool:
return True
@property
def packed_kv_cache_dtype(self) -> CacheDType:
# This kernel is the only one that reads an NVFP4 compressed cache, so
# it is what an unspecific --kv-cache-dtype resolves to here.
return "nvfp4_ds_mla"
def _alloc_attn_out(
self, num_tokens: int, hidden_states: torch.Tensor
) -> QuantizedActivation:
return alloc_mega_attn_output(num_tokens, self.n_wv_group, hidden_states.device)
def _o_proj(
self, attn_out: QuantizedActivation, positions: torch.Tensor
) -> torch.Tensor:
"""Grouped ``wo_a`` then ``wo_b`` over the kernel's quantized output.
The inverse RoPE and the FP8 cast happened inside the attention
kernel, and ``wo_a`` is permuted for its output layout, so there is
nothing to rotate or quantize here and ``positions`` is unused. The
scale already is DeepGEMM's packed-ue8m0 MN-major layout, so the
einsum consumes the kernel's output with no repacking.
"""
groups = self.n_local_groups
z = torch.empty(
(attn_out.data.shape[0], groups, self.o_lora_rank),
dtype=torch.bfloat16,
device=attn_out.data.device,
)
fp8_einsum(
"bhr,hdr->bhd",
(attn_out.data[:, :groups], attn_out.scale[:, :groups]),
(self.wo_a.weight, self.wo_a.weight_scale),
z,
recipe=self._einsum_recipe,
)
return self.wo_b(z.flatten(1))
# ---- weights -----------------------------------------------------------
def finalize_loaded_weights(self) -> None:
"""Permute wq_b rows / wo_a columns into the kernel's layouts.
Idempotent: a second post-load pass must not permute twice.
"""
if self._fused_layouts_ready:
return
permute_wq_b_(
self.wq_b.weight.data, self.wq_b.weight_scale.data, self.n_local_heads
)
permute_wo_a_(
self.wo_a.weight.data,
self.wo_a.weight_scale.data,
self.n_local_heads // self.n_local_groups,
)
self._fused_layouts_ready = True
# ---- forward -----------------------------------------------------------
def forward_mqa(
self,
q: torch.Tensor,
kv: torch.Tensor,
positions: torch.Tensor,
output: QuantizedActivation,
) -> None:
if not self._fused_layouts_ready:
raise RuntimeError(
f"{self.prefix}: wq_b / wo_a were never permuted for the mega "
"attention kernel; refusing to run with mismatched layouts."
)
attn_metadata = get_forward_context().attn_metadata
if attn_metadata is None:
# Warmup dummy run: reserve the prefill workspace, produce zeros.
self._reserve_prefill_workspace(q)
output.data.zero_()
output.scale.zero_()
return
assert isinstance(attn_metadata, dict)
# q was zero-padded to the kernel's head count by the fused KV insert.
assert q.shape[1] == self.padded_heads
flashmla_metadata = cast(
DeepseekV4FlashMLAMetadata | None,
attn_metadata.get(self.compressed_cache_prefix)
if self.compressed_cache_prefix is not None
else None,
)
swa_metadata = cast(
"DeepseekSparseSWAMetadata", attn_metadata[self.swa_cache_layer.prefix]
)
# The kernel takes int32 RoPE positions.
positions_int32 = positions.to(torch.int32)
num_decode_tokens = swa_metadata.num_decode_tokens
if swa_metadata.num_prefills > 0:
self._forward_prefill_mega(
q[num_decode_tokens:],
positions_int32[num_decode_tokens:],
flashmla_metadata,
swa_metadata,
output,
num_decode_tokens,
)
if swa_metadata.num_decodes > 0:
self._forward_decode_mega(
q[:num_decode_tokens],
positions_int32[:num_decode_tokens],
flashmla_metadata,
swa_metadata,
_token_slice(output, 0, num_decode_tokens),
)
def _reserve_prefill_workspace(self, q: torch.Tensor) -> None:
swa_only = self.compress_ratio == 0
n = 0 if swa_only else -(-self.max_model_len // self.compress_ratio)
m = n + self.window_size + self.max_num_batched_tokens
if swa_only:
top_k = 0
else:
assert self.topk_indices_buffer is not None
top_k = self.topk_indices_buffer.shape[-1]
combined_topk = round_up(top_k + self.window_size + self.max_image_tokens, 128)
current_workspace_manager().get_simultaneous(
((self.PREFILL_CHUNK_SIZE, m, q.shape[-1]), torch.bfloat16),
((self.max_num_batched_tokens, combined_topk), torch.int32),
((self.max_num_batched_tokens,), torch.int32),
)
def _decode_compressed_kv_and_topk(
self,
flashmla_metadata: DeepseekV4FlashMLAMetadata | None,
swa_metadata: "DeepseekSparseSWAMetadata",
) -> tuple[torch.Tensor | None, torch.Tensor | None, torch.Tensor | None]:
"""The compressed cache and its ``[n, topk]`` slot indices / lengths."""
if self.compress_ratio == 0:
return None, None, None
assert flashmla_metadata is not None
assert swa_metadata.is_valid_token is not None
assert self.topk_indices_buffer is not None
num_decode_tokens = swa_metadata.num_decode_tokens
indices, lens = compute_global_topk_indices_and_lens(
self.topk_indices_buffer[:num_decode_tokens],
swa_metadata.token_to_req_indices,
flashmla_metadata.block_table[: swa_metadata.num_decodes],
flashmla_metadata.block_size // self.compress_ratio,
swa_metadata.is_valid_token[:num_decode_tokens],
)
return self._compressed_kv_cache().unsqueeze(-2), indices, lens
def _forward_decode_mega(
self,
q: torch.Tensor,
positions_int32: torch.Tensor,
flashmla_metadata: DeepseekV4FlashMLAMetadata | None,
swa_metadata: "DeepseekSparseSWAMetadata",
out: QuantizedActivation,
) -> None:
"""Mega attention over the paged quantized caches, in place into ``out``.
The caches are ``[num_blocks, page, 1, bytes]``, the record taken from
``bytes`` (528 V4.1 fp8, 288 V4.1 NVFP4; NVFP4 only as the compressed
cache beside an fp8 SWA one), and the indices ``[s_q, topk]`` int32
slot ids (``block * page + offset``, ``-1`` invalid).
"""
extra_cache, extra_idx, extra_len = self._decode_compressed_kv_and_topk(
flashmla_metadata, swa_metadata
)
assert swa_metadata.decode_swa_indices is not None
torch.ops._flashmla_C.fused_norm_rope_attn_rope_cast_decode(
q,
self.swa_cache_layer.kv_cache.unsqueeze(-2),
swa_metadata.decode_swa_indices.view(q.shape[0], -1),
self.scale,
_HEAD_DIM_V,
self.attn_sink,
swa_metadata.decode_swa_lens,
extra_cache,
extra_idx,
extra_len,
False, # enable_q_norm
0.0, # rms_norm_eps
positions_int32,
*_ROPE_ARGS,
self.rotary_emb.cos_sin_cache,
self.n_wv_group,
*_SF_ARGS,
out.data,
out.scale,
)
def _forward_prefill_mega(
self,
q: torch.Tensor,
positions_int32: torch.Tensor,
flashmla_metadata: DeepseekV4FlashMLAMetadata | None,
swa_metadata: "DeepseekSparseSWAMetadata",
out: QuantizedActivation,
token_base: int,
) -> None:
swa_only = self.compress_ratio == 0
num_decodes = swa_metadata.num_decodes
num_decode_tokens = swa_metadata.num_decode_tokens
seq_lens = swa_metadata.prefill_seq_lens
gather_lens = swa_metadata.prefill_gather_lens
query_start_loc_cpu = swa_metadata.query_start_loc_cpu
query_start_loc = swa_metadata.query_start_loc
assert seq_lens is not None and gather_lens is not None
assert query_start_loc_cpu is not None and query_start_loc is not None
prefill_token_base = query_start_loc_cpu[num_decodes]
assert self.topk_indices_buffer is not None
topk_indices = self.topk_indices_buffer[num_decode_tokens:][
: swa_metadata.num_prefill_tokens
]
top_k = 0 if swa_only else topk_indices.shape[-1]
chunk_plan = swa_metadata.get_prefill_chunk_plan(
compress_ratio=self.compress_ratio,
prefill_chunk_size=self.PREFILL_CHUNK_SIZE,
has_compressed=not swa_only,
)
assert chunk_plan, "prefill chunk plan must be non-empty when num_prefills > 0"
workspace_manager = current_workspace_manager()
combined_topk = round_up(top_k + self.window_size + self.max_image_tokens, 128)
for chunk_start, chunk_end, chunk_n, chunk_m in chunk_plan:
chunk_size = chunk_end - chunk_start
kv_ws, idx_ws, lens_ws = workspace_manager.get_simultaneous(
((chunk_size, chunk_m, q.shape[-1]), torch.bfloat16),
((self.max_num_batched_tokens, combined_topk), torch.int32),
((self.max_num_batched_tokens,), torch.int32),
)
if not swa_only:
assert flashmla_metadata is not None
dequantize_and_gather_k_cache(
kv_ws[:chunk_size],
self._compressed_kv_cache(),
seq_lens=seq_lens[chunk_start:chunk_end] // self.compress_ratio,
gather_lens=None,
block_table=flashmla_metadata.block_table[num_decodes:][
chunk_start:chunk_end
],
block_size=flashmla_metadata.block_size // self.compress_ratio,
offset=0,
)
dequantize_and_gather_k_cache(
kv_ws[:chunk_size],
self.swa_cache_layer.kv_cache,
seq_lens=seq_lens[chunk_start:chunk_end],
gather_lens=gather_lens[chunk_start:chunk_end],
block_table=swa_metadata.block_table[num_decodes:][
chunk_start:chunk_end
],
block_size=swa_metadata.block_size,
offset=chunk_n,
)
first, last = num_decodes + chunk_start, num_decodes + chunk_end
qs = int(query_start_loc_cpu[first] - prefill_token_base)
qe = int(query_start_loc_cpu[last] - prefill_token_base)
combined_indices, combined_lens = combine_topk_swa_indices(
topk_indices[qs:qe],
query_start_loc[first : last + 1],
seq_lens[chunk_start:chunk_end],
gather_lens[chunk_start:chunk_end],
self.window_size,
self.compress_ratio,
top_k,
chunk_m,
chunk_n,
out=(idx_ws[: qe - qs], lens_ws[: qe - qs]),
left_visible=(
swa_metadata.prefill_left_visible[
num_decode_tokens + qs : num_decode_tokens + qe
]
if swa_metadata.prefill_left_visible is not None
else None
),
right_visible=(
swa_metadata.prefill_right_visible[
num_decode_tokens + qs : num_decode_tokens + qe
]
if swa_metadata.prefill_right_visible is not None
else None
),
max_image_tokens=self.max_image_tokens,
)
chunk_out = _token_slice(out, token_base + qs, token_base + qe)
# Mega attention over the gathered non-paged bf16 KV (RoPE already
# applied), in place into this chunk's token range. `indices`
# entries outside [0, s_kv) skip.
torch.ops._flashmla_C.fused_norm_rope_attn_rope_cast_fwd(
q[qs:qe],
kv_ws.view(-1, 1, q.shape[-1]),
combined_indices.unsqueeze(1),
self.scale,
_HEAD_DIM_V,
self.attn_sink,
combined_lens,
False, # enable_q_norm
0.0, # rms_norm_eps
positions_int32[qs:qe],
*_ROPE_ARGS,
self.rotary_emb.cos_sin_cache,
self.n_wv_group,
*_SF_ARGS,
chunk_out.data,
chunk_out.scale,
)
@@ -201,7 +201,8 @@ class DeepseekV4FlashInferMLAAttention(DeepseekV4Attention):
def get_padded_num_q_heads(cls, num_heads: int) -> int:
return _pad_to_supported_q_heads(num_heads)
def _o_proj(self, o: torch.Tensor, positions: torch.Tensor) -> torch.Tensor:
def _o_proj(self, attn_out: torch.Tensor, positions: torch.Tensor) -> torch.Tensor:
o = attn_out[:, : self.n_local_heads, :]
return deep_gemm_fp8_o_proj(
o,
positions,
@@ -562,7 +563,8 @@ class DeepseekV4FlashInferSM120Attention(DeepseekV4Attention):
def get_padded_num_q_heads(cls, num_heads: int) -> int:
return _pad_to_supported_q_heads(num_heads)
def _o_proj(self, o: torch.Tensor, positions: torch.Tensor) -> torch.Tensor:
def _o_proj(self, attn_out: torch.Tensor, positions: torch.Tensor) -> torch.Tensor:
o = attn_out[:, : self.n_local_heads, :]
return deep_gemm_fp8_o_proj(
o,
positions,
+2 -1
View File
@@ -58,7 +58,8 @@ class DeepseekV4FlashMLAAttention(DeepseekV4Attention):
self._o_proj_block_size
)
def _o_proj(self, o: torch.Tensor, positions: torch.Tensor) -> torch.Tensor:
def _o_proj(self, attn_out: torch.Tensor, positions: torch.Tensor) -> torch.Tensor:
o = attn_out[:, : self.n_local_heads, :]
return deep_gemm_fp8_o_proj(
o,
positions,
+28 -2
View File
@@ -64,6 +64,9 @@ from vllm.models.deepseek_v4.nvidia.model import (
make_deepseek_v4_expert_params_mapping,
)
from vllm.models.deepseek_v41.attention import DeepseekV4Attention
from vllm.models.deepseek_v41.nvidia.flash_mla_mega_attn import (
DeepseekV4MegaAttnAttention,
)
from vllm.models.deepseek_v41.nvidia.flashinfer_sparse import (
DeepseekV4FlashInferMLAAttention,
DeepseekV4FlashInferSM120Attention,
@@ -119,8 +122,9 @@ def _select_dsv4_attn_cls(vllm_config: VllmConfig) -> type[DeepseekV4Attention]:
The generic CUDA backend selector does not instantiate DSv4 layers directly,
so map generic sparse-MLA choices to the DSv4-specialized attention class.
Without an explicit backend, SM12 defaults to FlashInfer while the other
CUDA arches keep the FlashMLA path.
Without an explicit backend: SM12 takes FlashInfer, SM100 takes mega
attention where the topology allows it, and everything else keeps the
FlashMLA path.
"""
backend = vllm_config.attention_config.backend
device_capability = current_platform.get_device_capability()
@@ -140,6 +144,8 @@ def _select_dsv4_attn_cls(vllm_config: VllmConfig) -> type[DeepseekV4Attention]:
if device_capability is not None and device_capability.major == 12:
return DeepseekV4FlashInferSM120Attention
return DeepseekV4FlashInferMLAAttention
if backend is AttentionBackendEnum.FLASHMLA_MEGA_ATTN_DSV41:
return DeepseekV4MegaAttnAttention
if backend in (
AttentionBackendEnum.FLASHMLA_SPARSE,
AttentionBackendEnum.FLASHMLA_SPARSE_DSV4,
@@ -149,6 +155,14 @@ def _select_dsv4_attn_cls(vllm_config: VllmConfig) -> type[DeepseekV4Attention]:
if device_capability is not None and device_capability.major == 12:
return DeepseekV4FlashInferSM120Attention
# Mega attention is the SM100 default: it fuses Q RoPE, sparse attention,
# the output's inverse RoPE and its FP8 cast into one launch, and brings
# the 288 B NVFP4 compressed record -- the format the reference
# implementation itself stores. It declines topologies it cannot serve
# (non-SM100, TP that leaves fewer than WV_GROUP_SIZE heads per wo_a
# group, a build without the kernel), which then fall through to FlashMLA.
if DeepseekV4MegaAttnAttention.is_available_for(vllm_config):
return DeepseekV4MegaAttnAttention
return DeepseekV4FlashMLAAttention
@@ -936,6 +950,17 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
for layer in islice(self.layers, self.start_layer, self.end_layer):
layer.ffn.finalize_mega_moe_weights()
def finalize_mega_attn_weights(self) -> None:
"""Permute wq_b / wo_a into FlashMLA's mega-attention layouts.
A no-op for every other attention layer, and idempotent, so a second
post-load pass cannot permute twice.
"""
for layer in islice(self.layers, self.start_layer, self.end_layer):
finalize = getattr(layer.attn, "finalize_loaded_weights", None)
if finalize is not None:
finalize()
def finalize_mhc_broadcast_weights(self) -> None:
if not get_pp_group().is_first_rank or self.start_layer >= self.end_layer:
return
@@ -1190,6 +1215,7 @@ class DeepseekV41LLMForCausalLM(
def process_weights_after_loading(self) -> None:
self.model.finalize_mega_moe_weights()
self.model.finalize_mhc_broadcast_weights()
self.model.finalize_mega_attn_weights()
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
return self.model.get_expert_mapping()
+51
View File
@@ -229,3 +229,54 @@ class DeepseekV4FlashMLABackend(DeepseekV4SparseMLABackend):
@staticmethod
def get_builder_cls() -> type[DeepseekV4FlashMLAMetadataBuilder]:
return DeepseekV4FlashMLAMetadataBuilder
class FlashMLAMegaAttnBackend(DeepseekV4FlashMLABackend):
"""FlashMLA's mega-attention kernel: Q RoPE + sparse attention + inverse
RoPE + FP8 cast of the output, in one launch.
Same metadata and KV-cache geometry as ``FLASHMLA_SPARSE_DSV41`` -- what
differs is the attention layer's interface contract (it takes an unnormed,
unroped Q and returns an already-inverse-RoPE'd, quantized output) and the
extra ``nvfp4_ds_mla`` compressed-cache record only this kernel can read.
SM100 only; the kernel has no SM90 instantiation.
"""
supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = [
"auto",
"fp8_ds_mla",
"fp8", # alias for fp8_ds_mla
"nvfp4_ds_mla", # V4.1 fp8 SWA cache + NVFP4 compressed cache
]
@staticmethod
def get_name() -> str:
return "FLASHMLA_MEGA_ATTN_DSV41"
@staticmethod
def get_supported_kernel_block_sizes() -> list[int | MultipleOf]:
return [128]
@classmethod
def supports_compute_capability(cls, capability: DeviceCapability) -> bool:
return capability.major == 10
@classmethod
def supports_combination(
cls,
head_size: int,
dtype: torch.dtype,
kv_cache_dtype: CacheDType | None,
block_size: int | None,
use_mla: bool,
has_sink: bool,
use_sparse: bool,
use_mm_prefix: bool,
device_capability: DeviceCapability,
) -> str | None:
# Imported here: the layer module imports this backend class.
from vllm.models.deepseek_v41.nvidia.flash_mla_mega_attn import (
is_flashmla_mega_attn_supported,
)
return is_flashmla_mega_attn_supported()[1]
+4 -1
View File
@@ -112,7 +112,10 @@ class DeepseekV4SWACache(torch.nn.Module, AttentionLayerBase):
# fp8_ds_mla's UE8M0 paged layout rounds its page up to the decode
# kernel's TMA stride; contiguous bf16/fp8 cache uses the natural
# element-size page.
uses_fp8_ds_mla_layout = self.cache_config.cache_dtype == "fp8_ds_mla"
uses_fp8_ds_mla_layout = self.cache_config.cache_dtype in (
"fp8_ds_mla",
"nvfp4_ds_mla",
)
return SlidingWindowMLASpec(
block_size=self.block_size,
num_kv_heads=1,
+3
View File
@@ -114,6 +114,9 @@ class AttentionBackendEnum(Enum, metaclass=_AttentionBackendEnumMeta):
"vllm.models.deepseek_v41.nvidia.flashinfer_sparse."
"DeepseekV4FlashInferMLASparseBackend"
)
FLASHMLA_MEGA_ATTN_DSV41 = (
"vllm.models.deepseek_v41.sparse_mla.FlashMLAMegaAttnBackend"
)
B12X = "vllm.v1.attention.backends.b12x.B12xPagedAttentionBackend"
FLASH_ATTN_MLA = "vllm.v1.attention.backends.mla.flashattn_mla.FlashAttnMLABackend"
FLASH_ATTN_MLA_SPARSE = (