[Model][DSv4.1] FlashMLA mega attention and the NVFP4 compressed KV cache (#56935)
Signed-off-by: Yongye Zhu <yongye@inferact.ai> Signed-off-by: 云挚 <ningyunxiao.nyx@antgroup.com> Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Yongye Zhu <yongye@inferact.ai> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: Yunxiao Ning <73625538+foraxe@users.noreply.github.com> Co-authored-by: AI Assistant <noreply@openai.com>
This commit is contained in:
co-authored by
Yongye Zhu
Claude Opus 5
Yunxiao Ning
AI Assistant
parent
ceb87de065
commit
d6a1677d55
@@ -0,0 +1,272 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""FlashMLA mega attention vs the split-KV decode pipeline (DeepSeek V4.1).
|
||||
|
||||
Times the full set of per-layer ops each path runs in a decode step, captured
|
||||
in a CUDA graph -- eager timing is dominated by launch overhead and flatters
|
||||
whichever path launches fewer kernels.
|
||||
|
||||
mega: fused_qnorm_rope_kv_rope_quant_insert (Q pad + KV insert)
|
||||
+ mega kernel
|
||||
split-KV: fused_qnorm_rope_kv_rope_quant_insert (Q RoPE + pad + KV insert)
|
||||
+ flash_mla_with_kvcache + fused_inv_rope_fp8_quant
|
||||
|
||||
Both arms fuse their Q preparation into the KV insert, so that op is charged
|
||||
to both; the mega arm's Q side is only a zero-pad (nothing at all when the
|
||||
shard is already at the kernel head count) because its kernel does the Q RoPE,
|
||||
the inverse RoPE and the FP8 cast.
|
||||
|
||||
``--local-heads`` is the variable that decides the outcome: the mega kernel
|
||||
absorbs work proportional to the live head count while paying for the padded
|
||||
one, so it wins at 64 live heads and is a wash at 16 (TP4 on a 64-head model).
|
||||
|
||||
Both arms use the same KV records so the comparison is kernel-for-kernel:
|
||||
a V4.1 fp8 (528 B) sliding-window cache and a compressed cache that is either
|
||||
V4.1 fp8 or V4.1 NVFP4 (288 B) -- FlashMLA's SM100 sparse decode reads the
|
||||
NVFP4 compressed record in both the fused and unfused kernels.
|
||||
|
||||
Run: .venv/bin/python benchmarks/kernels/benchmark_dsv41_mega_attn.py
|
||||
"""
|
||||
|
||||
import argparse
|
||||
|
||||
import torch
|
||||
|
||||
import vllm.v1.attention.ops.flashmla as fm
|
||||
from vllm.models.deepseek_v4.common.ops.fused_inv_rope_fp8_quant import (
|
||||
fused_inv_rope_fp8_quant,
|
||||
)
|
||||
from vllm.models.deepseek_v41.common.ops import quantize_and_insert_k_cache
|
||||
from vllm.models.deepseek_v41.common.ops.fused_compress_quant_cache import (
|
||||
rope_quant_insert,
|
||||
)
|
||||
from vllm.models.deepseek_v41.common.ops.fused_layout import permute_q_to_fused
|
||||
from vllm.models.deepseek_v41.nvidia.flash_mla_mega_attn import (
|
||||
alloc_mega_attn_output,
|
||||
is_flashmla_mega_attn_supported,
|
||||
)
|
||||
from vllm.utils.math_utils import round_up
|
||||
|
||||
HEAD_DIM, ROPE_DIM = 512, 64
|
||||
V41_BYTES, V41_FP4_BYTES = 528, 288
|
||||
SWA_BLOCK, COMPRESSED_BLOCK = 32, 128
|
||||
|
||||
|
||||
def make_cos_sin_cache(max_pos, device):
|
||||
inv_freq = 1.0 / (
|
||||
10000 ** (torch.arange(0, ROPE_DIM, 2, device=device).float() / ROPE_DIM)
|
||||
)
|
||||
freqs = torch.outer(torch.arange(max_pos, device=device).float(), inv_freq)
|
||||
return torch.cat([freqs.cos(), freqs.sin()], -1)
|
||||
|
||||
|
||||
def empty_paged_cache(num_rows, block_size, bytes_per_token, device):
|
||||
"""A zeroed paged cache view [num_blocks, block_size, bytes_per_token]."""
|
||||
num_blocks = (num_rows + block_size - 1) // block_size + 1
|
||||
page = round_up(block_size * bytes_per_token, 512)
|
||||
backing = torch.zeros(num_blocks, page, dtype=torch.uint8, device=device)
|
||||
return backing, backing.as_strided(
|
||||
(num_blocks, block_size, bytes_per_token), (page, bytes_per_token, 1)
|
||||
)
|
||||
|
||||
|
||||
def fill_cache(view, bytes_per_token, cos_sin, device):
|
||||
n = view.shape[0] * view.shape[1]
|
||||
rows = torch.randn(n, HEAD_DIM, device=device, dtype=torch.bfloat16)
|
||||
slots = torch.arange(n, dtype=torch.int64, device=device)
|
||||
if bytes_per_token == V41_FP4_BYTES:
|
||||
# A page-aligned cache has more slots than the cos_sin table has rows;
|
||||
# wrap so the insert never indexes past it.
|
||||
positions = slots % cos_sin.shape[0]
|
||||
rope_quant_insert(rows, positions, cos_sin, view, slots, 1)
|
||||
else:
|
||||
quantize_and_insert_k_cache(
|
||||
rows,
|
||||
view.reshape(view.shape[0], -1),
|
||||
slots,
|
||||
block_size=view.shape[1],
|
||||
bytes_per_token=bytes_per_token,
|
||||
)
|
||||
|
||||
|
||||
def time_graph(fn, iters=100, warmup=5):
|
||||
"""Capture ``fn`` in a CUDA graph and time its replay."""
|
||||
s = torch.cuda.Stream()
|
||||
s.wait_stream(torch.cuda.current_stream())
|
||||
with torch.cuda.stream(s):
|
||||
for _ in range(3):
|
||||
fn()
|
||||
torch.cuda.current_stream().wait_stream(s)
|
||||
g = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(g):
|
||||
fn()
|
||||
for _ in range(warmup):
|
||||
g.replay()
|
||||
torch.accelerator.synchronize()
|
||||
start = torch.cuda.Event(enable_timing=True)
|
||||
end = torch.cuda.Event(enable_timing=True)
|
||||
start.record()
|
||||
for _ in range(iters):
|
||||
g.replay()
|
||||
end.record()
|
||||
torch.accelerator.synchronize()
|
||||
return start.elapsed_time(end) * 1000 / iters
|
||||
|
||||
|
||||
def bench_decode(s_q, device, topk_extra, extra_bytes, local_heads):
|
||||
padded_heads, scale = 64, HEAD_DIM**-0.5
|
||||
n_wv_group = padded_heads // 8
|
||||
cos_sin = make_cos_sin_cache(65536, device)
|
||||
positions = torch.randint(0, 65536, (s_q,), device=device, dtype=torch.int64)
|
||||
pos32 = positions.to(torch.int32)
|
||||
|
||||
# Per-step inputs: the wq_b output (local heads) and this step's KV row.
|
||||
q_local = torch.randn(
|
||||
s_q, local_heads, HEAD_DIM, device=device, dtype=torch.bfloat16
|
||||
)
|
||||
q_fused = permute_q_to_fused(q_local)
|
||||
kv = torch.randn(s_q, HEAD_DIM, device=device, dtype=torch.bfloat16)
|
||||
|
||||
n_swa = s_q * 128 + 128
|
||||
swa_backing, swa = empty_paged_cache(n_swa, SWA_BLOCK, V41_BYTES, device)
|
||||
fill_cache(swa, V41_BYTES, cos_sin, device)
|
||||
_, extra = empty_paged_cache(65536, COMPRESSED_BLOCK, extra_bytes, device)
|
||||
fill_cache(extra, extra_bytes, cos_sin, device)
|
||||
|
||||
slot_mapping = torch.arange(s_q, dtype=torch.int64, device=device)
|
||||
swa_idx = torch.randint(0, n_swa, (s_q, 128), device=device, dtype=torch.int32)
|
||||
swa_len = torch.full((s_q,), 128, device=device, dtype=torch.int32)
|
||||
ex_idx = torch.randint(
|
||||
0, 65536, (s_q, topk_extra), device=device, dtype=torch.int32
|
||||
)
|
||||
ex_len = torch.full((s_q,), topk_extra, device=device, dtype=torch.int32)
|
||||
sink = torch.zeros(padded_heads, device=device)
|
||||
sched = fm.FlashMLASchedMeta()
|
||||
out = alloc_mega_attn_output(s_q, n_wv_group, device)
|
||||
swa4 = swa.unsqueeze(-2)
|
||||
extra4 = extra.unsqueeze(-2)
|
||||
swa_2d = swa_backing
|
||||
|
||||
def mega():
|
||||
# One launch zero-pads the fused-layout Q and inserts the SWA KV.
|
||||
q_pad = torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
|
||||
q_fused,
|
||||
kv,
|
||||
swa_2d,
|
||||
slot_mapping,
|
||||
positions,
|
||||
cos_sin,
|
||||
0 if padded_heads == local_heads else padded_heads,
|
||||
1e-6,
|
||||
SWA_BLOCK,
|
||||
False, # apply_q_norm
|
||||
True, # kv_mxfp8
|
||||
False, # apply_q_rope: the mega kernel rotates Q itself
|
||||
True, # is_q_interleaved
|
||||
)
|
||||
torch.ops._flashmla_C.fused_norm_rope_attn_rope_cast_decode(
|
||||
q_fused if padded_heads == local_heads else q_pad,
|
||||
swa4,
|
||||
swa_idx,
|
||||
scale,
|
||||
HEAD_DIM,
|
||||
sink,
|
||||
swa_len,
|
||||
extra4,
|
||||
ex_idx,
|
||||
ex_len,
|
||||
False,
|
||||
0.0,
|
||||
pos32,
|
||||
False,
|
||||
64,
|
||||
cos_sin,
|
||||
n_wv_group,
|
||||
32,
|
||||
True,
|
||||
True,
|
||||
True,
|
||||
out.data,
|
||||
out.scale,
|
||||
)
|
||||
|
||||
def split_kv():
|
||||
q_pad = torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
|
||||
q_local,
|
||||
kv,
|
||||
swa_2d,
|
||||
slot_mapping,
|
||||
positions,
|
||||
cos_sin,
|
||||
padded_heads,
|
||||
1e-6,
|
||||
SWA_BLOCK,
|
||||
False,
|
||||
True,
|
||||
)
|
||||
o = fm.flash_mla_with_kvcache(
|
||||
q=q_pad.unsqueeze(1),
|
||||
k_cache=swa4,
|
||||
block_table=None,
|
||||
head_dim_v=HEAD_DIM,
|
||||
tile_scheduler_metadata=sched,
|
||||
cache_seqlens=None,
|
||||
is_fp8_kvcache=True,
|
||||
indices=swa_idx.view(s_q, 1, -1),
|
||||
topk_length=swa_len,
|
||||
softmax_scale=scale,
|
||||
attn_sink=sink,
|
||||
extra_k_cache=extra4,
|
||||
extra_indices_in_kvcache=ex_idx.view(s_q, 1, -1),
|
||||
extra_topk_length=ex_len,
|
||||
)[0]
|
||||
fused_inv_rope_fp8_quant(
|
||||
o.squeeze(1)[:, :local_heads],
|
||||
positions,
|
||||
cos_sin,
|
||||
n_groups=local_heads // 8,
|
||||
heads_per_group=8,
|
||||
quant_group_size=32,
|
||||
tma_aligned_scales=True,
|
||||
)
|
||||
|
||||
return time_graph(mega), time_graph(split_kv)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
"--decode-s-q",
|
||||
nargs="+",
|
||||
type=int,
|
||||
default=[1, 2, 4, 8, 16, 32, 64, 128, 256, 512],
|
||||
)
|
||||
parser.add_argument("--topk-extra", type=int, default=512)
|
||||
parser.add_argument("--local-heads", type=int, default=16)
|
||||
parser.add_argument(
|
||||
"--extra-bytes",
|
||||
type=int,
|
||||
choices=[V41_BYTES, V41_FP4_BYTES],
|
||||
default=V41_BYTES,
|
||||
help="compressed-cache record: 528 (V4.1 fp8) or 288 (V4.1 NVFP4)",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
ok, reason = is_flashmla_mega_attn_supported()
|
||||
if not ok:
|
||||
raise SystemExit(reason)
|
||||
device = torch.device("cuda")
|
||||
torch.manual_seed(0)
|
||||
print(
|
||||
f"decode, cudagraph (topk_swa=128, topk_extra={args.topk_extra}, "
|
||||
f"local_heads={args.local_heads}, extra={args.extra_bytes}B)"
|
||||
)
|
||||
print(f"{'s_q':>6} {'mega_us':>9} {'split_kv_us':>12} {'speedup':>8}")
|
||||
for s_q in args.decode_s_q:
|
||||
m, s = bench_decode(
|
||||
s_q, device, args.topk_extra, args.extra_bytes, args.local_heads
|
||||
)
|
||||
print(f"{s_q:>6d} {m:9.1f} {s:12.1f} {s / m:7.2f}x")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -3,7 +3,10 @@
|
||||
* SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
*
|
||||
* Horizontally-fused DeepseekV4-MLA kernel:
|
||||
* - Q side: optional per-head RMSNorm + GPT-J RoPE on last ROPE_DIM
|
||||
* - Q side: optional per-head RMSNorm (apply_q_norm) and GPT-J RoPE on the
|
||||
* last ROPE_DIM (apply_q_rope), plus the zero-pad to
|
||||
* q_head_padded, in the head-major layout or (is_q_interleaved)
|
||||
* FlashMLA's mega-attention chunk-interleaved one
|
||||
* - KV side: GPT-J RoPE on last ROPE_DIM + UE8M0 FP8 quant on NoPE + paged
|
||||
* cache insert
|
||||
*
|
||||
@@ -174,6 +177,30 @@ __device__ __forceinline__ float warpSum(float val) {
|
||||
return val;
|
||||
}
|
||||
|
||||
// Offset of the 16 elements a lane owns inside a Q tensor of `num_heads`
|
||||
// heads. The default layout is head-major: a head's 512 dims are contiguous.
|
||||
// `Q_INTERLEAVED` is FlashMLA's mega-attention layout, 32 chunks of
|
||||
// [num_heads, 16] per token, which changes the address only -- lane L still
|
||||
// owns dims [16L, 16L+16) of its head either way, so `dim_base` stays the
|
||||
// true dim offset and the norm and RoPE below are equally valid in both.
|
||||
// What it does change is where the padding heads land: at the tail of every
|
||||
// chunk rather than after the live heads.
|
||||
template <bool Q_INTERLEAVED>
|
||||
__device__ __forceinline__ int64_t qElemOffset(int const tokenIdx,
|
||||
int const slotIdx,
|
||||
int const laneId,
|
||||
int const dim_base,
|
||||
int const num_heads) {
|
||||
if constexpr (Q_INTERLEAVED) {
|
||||
return static_cast<int64_t>(tokenIdx) * num_heads * kHeadDim +
|
||||
static_cast<int64_t>(laneId) * num_heads * kElemsPerLane +
|
||||
static_cast<int64_t>(slotIdx) * kElemsPerLane;
|
||||
} else {
|
||||
return (static_cast<int64_t>(tokenIdx) * num_heads + slotIdx) * kHeadDim +
|
||||
dim_base;
|
||||
}
|
||||
}
|
||||
|
||||
// ────────────────────────────────────────────────────────────────────────────
|
||||
// Per-slot inner pipeline
|
||||
// ────────────────────────────────────────────────────────────────────────────
|
||||
@@ -192,10 +219,16 @@ __device__ __forceinline__ float warpSum(float val) {
|
||||
// + paged-cache
|
||||
// insert)
|
||||
//
|
||||
// `kv_mxfp8` is grid-uniform and only steers the KV branch, so it stays a
|
||||
// runtime argument rather than doubling every (head count, q-norm)
|
||||
// instantiation of a kernel whose hot path is the Q branch.
|
||||
template <typename scalar_t_in, int kNumHeadsQPadded, bool APPLY_Q_NORM>
|
||||
// `kv_mxfp8` and `apply_q_rope` are grid-uniform, so they stay runtime
|
||||
// arguments rather than multiplying every (head count, q-norm, q-layout)
|
||||
// instantiation of a kernel whose hot path is the Q branch. `Q_INTERLEAVED`
|
||||
// cannot: it picks the address arithmetic, which has to fold.
|
||||
//
|
||||
// The three Q knobs are independent. The mega-attention layer uses
|
||||
// Q_INTERLEAVED with both norm and RoPE off, because its attention kernel
|
||||
// applies those itself -- not because the layout forbids them.
|
||||
template <typename scalar_t_in, int kNumHeadsQPadded, bool APPLY_Q_NORM,
|
||||
bool Q_INTERLEAVED = false>
|
||||
__device__ __forceinline__ void processDeepseekV4Slot(
|
||||
uint4 v0, uint4 v1, int const tokenIdx, int const slotIdx,
|
||||
int const dim_base, int const laneId, int const num_heads_q,
|
||||
@@ -203,7 +236,7 @@ __device__ __forceinline__ void processDeepseekV4Slot(
|
||||
uint8_t* __restrict__ k_cache, int64_t const* __restrict__ slot_mapping,
|
||||
int64_t const* __restrict__ position_ids,
|
||||
float const* __restrict__ cos_sin_cache, int const cache_block_size,
|
||||
int const kv_block_stride, bool const kv_mxfp8) {
|
||||
int const kv_block_stride, bool const kv_mxfp8, bool const apply_q_rope) {
|
||||
using Converter = vllm::_typeConvert<scalar_t_in>;
|
||||
bool const isKV = (slotIdx == kNumHeadsQPadded);
|
||||
bool const isPadQ = !isKV && (slotIdx >= num_heads_q);
|
||||
@@ -213,10 +246,8 @@ __device__ __forceinline__ void processDeepseekV4Slot(
|
||||
// zero literal is correct. Matches the live-Q branch's vectorized store.
|
||||
if (isPadQ) {
|
||||
scalar_t_in* dst =
|
||||
q_out +
|
||||
(static_cast<int64_t>(tokenIdx) * kNumHeadsQPadded + slotIdx) *
|
||||
kHeadDim +
|
||||
dim_base;
|
||||
q_out + qElemOffset<Q_INTERLEAVED>(tokenIdx, slotIdx, laneId, dim_base,
|
||||
kNumHeadsQPadded);
|
||||
uint4 const zero4 = {0u, 0u, 0u, 0u};
|
||||
*reinterpret_cast<uint4*>(dst) = zero4;
|
||||
*reinterpret_cast<uint4*>(dst + 8) = zero4;
|
||||
@@ -264,7 +295,9 @@ __device__ __forceinline__ void processDeepseekV4Slot(
|
||||
|
||||
// ── GPT-J RoPE on dims [NOPE_DIM, HEAD_DIM) ─────────────────────────────
|
||||
// All math in fp32. cos_sin_cache is loaded as fp32 (its native storage).
|
||||
bool const is_rope_lane = dim_base >= kNopeDim;
|
||||
// KV is always rotated; Q only when the caller has not already had it
|
||||
// rotated elsewhere (the mega-attention kernel does its own).
|
||||
bool const is_rope_lane = dim_base >= kNopeDim && (isKV || apply_q_rope);
|
||||
if (is_rope_lane) {
|
||||
int64_t const pos = position_ids[tokenIdx];
|
||||
constexpr int kHalfRope = kRopeDim / 2;
|
||||
@@ -314,10 +347,8 @@ __device__ __forceinline__ void processDeepseekV4Slot(
|
||||
make_float2(elements[8 + 2 * i], elements[8 + 2 * i + 1]));
|
||||
}
|
||||
scalar_t_in* dst =
|
||||
q_out +
|
||||
(static_cast<int64_t>(tokenIdx) * kNumHeadsQPadded + slotIdx) *
|
||||
kHeadDim +
|
||||
dim_base;
|
||||
q_out + qElemOffset<Q_INTERLEAVED>(tokenIdx, slotIdx, laneId, dim_base,
|
||||
kNumHeadsQPadded);
|
||||
*reinterpret_cast<uint4*>(dst) = out0;
|
||||
*reinterpret_cast<uint4*>(dst + 8) = out1;
|
||||
} else {
|
||||
@@ -432,14 +463,17 @@ __device__ __forceinline__ void processDeepseekV4Slot(
|
||||
// `kNumHeadsQPadded` is a template parameter (compile-time constant) so the
|
||||
// divisions in the grid math and the KV-sentinel comparison fold to fast
|
||||
// constant operations. The launch wrapper dispatches the runtime value to
|
||||
// the matching instantiation.
|
||||
// the matching instantiation. `kNumHeadsQPadded == 0` leaves one slot per
|
||||
// token and is the KV-insert-only instantiation, for a Q that is already in
|
||||
// the shape its attention kernel wants.
|
||||
//
|
||||
// With DP padding, q/kv/position_ids can have more rows than slot_mapping.
|
||||
// The live-Q and pad-Q branches cover all `num_tokens_full` rows (downstream
|
||||
// attention uses them). The KV branch only inserts the first
|
||||
// `num_tokens_insert` tokens (= slot_mapping length) into the paged cache.
|
||||
//
|
||||
template <typename scalar_t_in, int kNumHeadsQPadded, bool APPLY_Q_NORM>
|
||||
template <typename scalar_t_in, int kNumHeadsQPadded, bool APPLY_Q_NORM,
|
||||
bool Q_INTERLEAVED>
|
||||
__global__ void fusedDeepseekV4QNormRopeKVRopeQuantInsertKernel(
|
||||
scalar_t_in const* __restrict__ q_in, // [N, num_heads_q, 512]
|
||||
scalar_t_in* __restrict__ q_out, // [N, kNumHeadsQPadded, 512]
|
||||
@@ -454,7 +488,8 @@ __global__ void fusedDeepseekV4QNormRopeKVRopeQuantInsertKernel(
|
||||
int const num_heads_q, // live Q heads (input layout)
|
||||
int const cache_block_size, // tokens per paged-cache block
|
||||
int const kv_block_stride, // bytes per paged-cache block
|
||||
bool const kv_mxfp8) { // V4.1 all-dims MXFP8 record
|
||||
bool const kv_mxfp8, // V4.1 all-dims MXFP8 record
|
||||
bool const apply_q_rope) { // rotate Q too, not just KV
|
||||
#if (!defined(__CUDA_ARCH__) || __CUDA_ARCH__ < 800) && !defined(USE_ROCM)
|
||||
// BF16 _typeConvert specialization is unavailable on pre-Ampere. The
|
||||
// DeepseekV4 kernel only runs with bf16 inputs in practice, so compile a
|
||||
@@ -497,20 +532,18 @@ __global__ void fusedDeepseekV4QNormRopeKVRopeQuantInsertKernel(
|
||||
if (isKV) {
|
||||
src_ptr = kv_in + static_cast<int64_t>(tokenIdx) * kHeadDim + dim_base;
|
||||
} else {
|
||||
int64_t const q_row_offset =
|
||||
(static_cast<int64_t>(tokenIdx) * num_heads_q + slotIdx) *
|
||||
kHeadDim +
|
||||
dim_base;
|
||||
src_ptr = q_in + q_row_offset;
|
||||
src_ptr = q_in + qElemOffset<Q_INTERLEAVED>(tokenIdx, slotIdx, laneId,
|
||||
dim_base, num_heads_q);
|
||||
}
|
||||
v0 = *reinterpret_cast<uint4 const*>(src_ptr);
|
||||
v1 = *reinterpret_cast<uint4 const*>(src_ptr + 8);
|
||||
}
|
||||
|
||||
processDeepseekV4Slot<scalar_t_in, kNumHeadsQPadded, APPLY_Q_NORM>(
|
||||
processDeepseekV4Slot<scalar_t_in, kNumHeadsQPadded, APPLY_Q_NORM,
|
||||
Q_INTERLEAVED>(
|
||||
v0, v1, tokenIdx, slotIdx, dim_base, laneId, num_heads_q, eps, q_out,
|
||||
k_cache, slot_mapping, position_ids, cos_sin_cache, cache_block_size,
|
||||
kv_block_stride, kv_mxfp8);
|
||||
kv_block_stride, kv_mxfp8, apply_q_rope);
|
||||
|
||||
#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)
|
||||
cudaTriggerProgrammaticLaunchCompletion();
|
||||
@@ -530,7 +563,8 @@ __global__ void fusedDeepseekV4QNormRopeKVRopeQuantInsertKernel(
|
||||
// Q branch (optional RMSNorm + RoPE, in place) head_slot == num_heads_q
|
||||
// KV branch (RoPE + UE8M0 quant + insert)
|
||||
//
|
||||
template <typename scalar_t_in, int kNumHeadsQPadded, bool APPLY_Q_NORM>
|
||||
template <typename scalar_t_in, int kNumHeadsQPadded, bool APPLY_Q_NORM,
|
||||
bool Q_INTERLEAVED>
|
||||
__global__ void fusedDeepseekV4QNormRopeKVRopeQuantInsertKernelReducedGrid(
|
||||
scalar_t_in const* __restrict__ q_in, scalar_t_in* __restrict__ q_out,
|
||||
scalar_t_in const* __restrict__ kv_in, uint8_t* __restrict__ k_cache,
|
||||
@@ -539,7 +573,7 @@ __global__ void fusedDeepseekV4QNormRopeKVRopeQuantInsertKernelReducedGrid(
|
||||
float const* __restrict__ cos_sin_cache, float const eps,
|
||||
int const num_tokens_full, int const num_tokens_insert,
|
||||
int const num_heads_q, int const cache_block_size,
|
||||
int const kv_block_stride, bool const kv_mxfp8) {
|
||||
int const kv_block_stride, bool const kv_mxfp8, bool const apply_q_rope) {
|
||||
#if (!defined(__CUDA_ARCH__) || __CUDA_ARCH__ < 800) && !defined(USE_ROCM)
|
||||
if constexpr (std::is_same_v<scalar_t_in, c10::BFloat16>) {
|
||||
return;
|
||||
@@ -569,11 +603,8 @@ __global__ void fusedDeepseekV4QNormRopeKVRopeQuantInsertKernelReducedGrid(
|
||||
if (s == kNumHeadsQPadded) {
|
||||
src = kv_in + static_cast<int64_t>(tokenIdx) * kHeadDim + dim_base;
|
||||
} else {
|
||||
src = q_in +
|
||||
(static_cast<int64_t>(tokenIdx) * num_heads_q +
|
||||
static_cast<int64_t>(s)) *
|
||||
kHeadDim +
|
||||
dim_base;
|
||||
src = q_in + qElemOffset<Q_INTERLEAVED>(tokenIdx, s, laneId, dim_base,
|
||||
num_heads_q);
|
||||
}
|
||||
va = *reinterpret_cast<uint4 const*>(src);
|
||||
vb = *reinterpret_cast<uint4 const*>(src + 8);
|
||||
@@ -594,10 +625,12 @@ __global__ void fusedDeepseekV4QNormRopeKVRopeQuantInsertKernelReducedGrid(
|
||||
load_slot(next_slot, v0_next, v1_next);
|
||||
}
|
||||
|
||||
processDeepseekV4Slot<scalar_t_in, kNumHeadsQPadded, APPLY_Q_NORM>(
|
||||
processDeepseekV4Slot<scalar_t_in, kNumHeadsQPadded, APPLY_Q_NORM,
|
||||
Q_INTERLEAVED>(
|
||||
v0_curr, v1_curr, tokenIdx, curr_slot, dim_base, laneId,
|
||||
num_heads_q, eps, q_out, k_cache, slot_mapping, position_ids,
|
||||
cos_sin_cache, cache_block_size, kv_block_stride, kv_mxfp8);
|
||||
cos_sin_cache, cache_block_size, kv_block_stride, kv_mxfp8,
|
||||
apply_q_rope);
|
||||
|
||||
// ── Buffer rotation: hand the prefetched LDGs to the next iter.
|
||||
v0_curr = v0_next;
|
||||
@@ -617,14 +650,15 @@ __global__ void fusedDeepseekV4QNormRopeKVRopeQuantInsertKernelReducedGrid(
|
||||
// ────────────────────────────────────────────────────────────────────────────
|
||||
// Launch wrapper
|
||||
// ────────────────────────────────────────────────────────────────────────────
|
||||
template <typename scalar_t_in, int kNumHeadsQPadded, bool APPLY_Q_NORM>
|
||||
template <typename scalar_t_in, int kNumHeadsQPadded, bool APPLY_Q_NORM,
|
||||
bool Q_INTERLEAVED>
|
||||
static void launchFusedDeepseekV4Templated(
|
||||
scalar_t_in const* q_in, scalar_t_in* q_out, scalar_t_in const* kv_in,
|
||||
uint8_t* k_cache, int64_t const* slot_mapping, int64_t const* position_ids,
|
||||
float const* cos_sin_cache, float const eps, int const num_tokens_full,
|
||||
int const num_tokens_insert, int const num_heads_q,
|
||||
int const cache_block_size, int const kv_block_stride, bool const kv_mxfp8,
|
||||
cudaStream_t stream) {
|
||||
bool const apply_q_rope, cudaStream_t stream) {
|
||||
constexpr int kBlockSize = 256;
|
||||
constexpr int kWarpsPerBlock = kBlockSize / 32;
|
||||
int64_t const total_warps =
|
||||
@@ -663,37 +697,40 @@ static void launchFusedDeepseekV4Templated(
|
||||
// grid instead. Only reachable above NUM_TOKEN_CUTOFF tokens in a single
|
||||
// insert, where it is worth ~2x.
|
||||
if (kNumHeadsQPadded == 0 || num_tokens_full < NUM_TOKEN_CUTOFF) {
|
||||
cudaLaunchKernelEx(&config,
|
||||
fusedDeepseekV4QNormRopeKVRopeQuantInsertKernel<
|
||||
scalar_t_in, kNumHeadsQPadded, APPLY_Q_NORM>,
|
||||
q_in, q_out, kv_in, k_cache, slot_mapping, position_ids,
|
||||
cos_sin_cache, eps, num_tokens_full, num_tokens_insert,
|
||||
num_heads_q, cache_block_size, kv_block_stride,
|
||||
kv_mxfp8);
|
||||
cudaLaunchKernelEx(
|
||||
&config,
|
||||
fusedDeepseekV4QNormRopeKVRopeQuantInsertKernel<
|
||||
scalar_t_in, kNumHeadsQPadded, APPLY_Q_NORM, Q_INTERLEAVED>,
|
||||
q_in, q_out, kv_in, k_cache, slot_mapping, position_ids, cos_sin_cache,
|
||||
eps, num_tokens_full, num_tokens_insert, num_heads_q, cache_block_size,
|
||||
kv_block_stride, kv_mxfp8, apply_q_rope);
|
||||
} else {
|
||||
config.gridDim = dim3(num_tokens_full);
|
||||
cudaLaunchKernelEx(
|
||||
&config,
|
||||
fusedDeepseekV4QNormRopeKVRopeQuantInsertKernelReducedGrid<
|
||||
scalar_t_in, kNumHeadsQPadded, APPLY_Q_NORM>,
|
||||
scalar_t_in, kNumHeadsQPadded, APPLY_Q_NORM, Q_INTERLEAVED>,
|
||||
q_in, q_out, kv_in, k_cache, slot_mapping, position_ids, cos_sin_cache,
|
||||
eps, num_tokens_full, num_tokens_insert, num_heads_q, cache_block_size,
|
||||
kv_block_stride, kv_mxfp8);
|
||||
kv_block_stride, kv_mxfp8, apply_q_rope);
|
||||
}
|
||||
#else
|
||||
// ROCm: use standard kernel launch syntax (no PDL/stream serialization)
|
||||
// clang-format off
|
||||
fusedDeepseekV4QNormRopeKVRopeQuantInsertKernel<
|
||||
scalar_t_in, kNumHeadsQPadded, APPLY_Q_NORM>
|
||||
scalar_t_in, kNumHeadsQPadded, APPLY_Q_NORM, Q_INTERLEAVED>
|
||||
<<<grid, kBlockSize, 0, stream>>>(
|
||||
q_in, q_out, kv_in, k_cache, slot_mapping, position_ids,
|
||||
cos_sin_cache, eps, num_tokens_full, num_tokens_insert, num_heads_q,
|
||||
cache_block_size, kv_block_stride, kv_mxfp8);
|
||||
cache_block_size, kv_block_stride, kv_mxfp8, apply_q_rope);
|
||||
#endif
|
||||
}
|
||||
|
||||
// Runtime dispatch into one of the precompiled `kNumHeadsQPadded`
|
||||
// instantiations. Supported padded head counts: 8, 16, 32, 64, 128.
|
||||
// instantiations. Supported padded head counts: 8, 16, 32, 64, 128, plus 0
|
||||
// for a KV-only launch that writes no Q at all. `apply_q_norm` and
|
||||
// `is_q_interleaved` are template parameters, `apply_q_rope` is not (see
|
||||
// processDeepseekV4Slot), so each head count carries four instantiations.
|
||||
template <typename scalar_t_in>
|
||||
void launchFusedDeepseekV4QNormRopeKVRopeQuantInsert(
|
||||
scalar_t_in const* q_in, scalar_t_in* q_out, scalar_t_in const* kv_in,
|
||||
@@ -702,22 +739,32 @@ void launchFusedDeepseekV4QNormRopeKVRopeQuantInsert(
|
||||
int const num_tokens_full, int const num_tokens_insert,
|
||||
int const num_heads_q, int const num_heads_q_padded,
|
||||
int const cache_block_size, int const kv_block_stride,
|
||||
bool const apply_q_norm, bool const kv_mxfp8, cudaStream_t stream) {
|
||||
#define DISPATCH(N) \
|
||||
case N: \
|
||||
if (apply_q_norm) { \
|
||||
launchFusedDeepseekV4Templated<scalar_t_in, N, true>( \
|
||||
q_in, q_out, kv_in, k_cache, slot_mapping, position_ids, \
|
||||
cos_sin_cache, eps, num_tokens_full, num_tokens_insert, \
|
||||
num_heads_q, cache_block_size, kv_block_stride, kv_mxfp8, stream); \
|
||||
} else { \
|
||||
launchFusedDeepseekV4Templated<scalar_t_in, N, false>( \
|
||||
q_in, q_out, kv_in, k_cache, slot_mapping, position_ids, \
|
||||
cos_sin_cache, eps, num_tokens_full, num_tokens_insert, \
|
||||
num_heads_q, cache_block_size, kv_block_stride, kv_mxfp8, stream); \
|
||||
} \
|
||||
bool const apply_q_norm, bool const kv_mxfp8, bool const apply_q_rope,
|
||||
bool const is_q_interleaved, cudaStream_t stream) {
|
||||
#define LAUNCH(N, NORM, INTERLEAVED) \
|
||||
launchFusedDeepseekV4Templated<scalar_t_in, N, NORM, INTERLEAVED>( \
|
||||
q_in, q_out, kv_in, k_cache, slot_mapping, position_ids, cos_sin_cache, \
|
||||
eps, num_tokens_full, num_tokens_insert, num_heads_q, cache_block_size, \
|
||||
kv_block_stride, kv_mxfp8, apply_q_rope, stream)
|
||||
#define DISPATCH(N) \
|
||||
case N: \
|
||||
if (is_q_interleaved) { \
|
||||
if (apply_q_norm) { \
|
||||
LAUNCH(N, true, true); \
|
||||
} else { \
|
||||
LAUNCH(N, false, true); \
|
||||
} \
|
||||
} else if (apply_q_norm) { \
|
||||
LAUNCH(N, true, false); \
|
||||
} else { \
|
||||
LAUNCH(N, false, false); \
|
||||
} \
|
||||
return;
|
||||
|
||||
if (num_heads_q_padded == 0) {
|
||||
LAUNCH(0, false, false);
|
||||
return;
|
||||
}
|
||||
switch (num_heads_q_padded) {
|
||||
DISPATCH(8)
|
||||
DISPATCH(16)
|
||||
@@ -729,9 +776,10 @@ void launchFusedDeepseekV4QNormRopeKVRopeQuantInsert(
|
||||
"fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert: "
|
||||
"unsupported num_heads_q_padded=",
|
||||
num_heads_q_padded,
|
||||
" (compiled instantiations: 8, 16, 32, 64, 128).");
|
||||
" (compiled instantiations: 0, 8, 16, 32, 64, 128).");
|
||||
}
|
||||
#undef DISPATCH
|
||||
#undef LAUNCH
|
||||
}
|
||||
|
||||
// ────────────────────────────────────────────────────────────────────────────
|
||||
@@ -1059,10 +1107,15 @@ void fused_deepseek_v4_kv_rope_insert(
|
||||
// Zero query heads schedules only the existing KV branch, preserving its
|
||||
// RoPE rounding and quantization without allocating any query tensors.
|
||||
if (packed) {
|
||||
vllm::deepseek_v4_fused_ops::launchFusedDeepseekV4Templated<scalar_t, 0, false>(
|
||||
// With zero query heads no Q slot is scheduled, so neither Q knob is
|
||||
// reachable: Q_INTERLEAVED picks addressing nothing uses, and apply_q_rope
|
||||
// gates a branch only a Q slot takes (KV is rotated unconditionally).
|
||||
vllm::deepseek_v4_fused_ops::launchFusedDeepseekV4Templated<scalar_t, 0,
|
||||
false, false>(
|
||||
nullptr, nullptr, input, cache, slots, positions, cos_sin, 0.0f,
|
||||
num_tokens, num_tokens, 0, static_cast<int>(cache_block_size),
|
||||
static_cast<int>(k_cache.stride(0)), kv_mxfp8, stream);
|
||||
static_cast<int>(k_cache.stride(0)), kv_mxfp8, /*apply_q_rope=*/true,
|
||||
stream);
|
||||
} else if (fp8) {
|
||||
vllm::deepseek_v4_fused_ops::launchFullCacheKernel<scalar_t, false, true>(
|
||||
nullptr, nullptr, 0, 0, input, cache, slots, positions, cos_sin,
|
||||
@@ -1085,8 +1138,9 @@ torch::stable::Tensor fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
|
||||
torch::stable::Tensor const& slot_mapping, // [N] int64
|
||||
torch::stable::Tensor const& position_ids, // [N] int64
|
||||
torch::stable::Tensor const& cos_sin_cache, // [max_pos, rope_dim] bf16
|
||||
int64_t q_head_padded, // padded Q head count for output
|
||||
double eps, int64_t cache_block_size, bool apply_q_norm, bool kv_mxfp8) {
|
||||
int64_t q_head_padded, // padded Q head count for output, 0 for no Q
|
||||
double eps, int64_t cache_block_size, bool apply_q_norm, bool kv_mxfp8,
|
||||
bool apply_q_rope, bool is_q_interleaved) {
|
||||
STD_TORCH_CHECK(q_in.device().is_cuda() && q_in.is_contiguous(),
|
||||
"q_in must be contiguous CUDA");
|
||||
STD_TORCH_CHECK(kv.device().is_cuda() && kv.is_contiguous(),
|
||||
@@ -1106,8 +1160,11 @@ torch::stable::Tensor fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
|
||||
STD_TORCH_CHECK(kv.dim() == 2 && kv.size(1) == 512, "kv shape [N, 512]");
|
||||
STD_TORCH_CHECK(q_in.scalar_type() == kv.scalar_type(),
|
||||
"q_in and kv dtype must match");
|
||||
STD_TORCH_CHECK(q_head_padded >= q_in.size(1),
|
||||
"q_head_padded must be >= q_in.size(1) (num_heads_q)");
|
||||
// q_head_padded == 0 asks for the KV insert alone: Q is left untouched and
|
||||
// an empty tensor comes back, for a caller whose Q is already in the shape
|
||||
// its attention kernel reads.
|
||||
STD_TORCH_CHECK(q_head_padded == 0 || q_head_padded >= q_in.size(1),
|
||||
"q_head_padded must be 0 or >= q_in.size(1) (num_heads_q)");
|
||||
STD_TORCH_CHECK(k_cache.scalar_type() == torch::headeronly::ScalarType::Byte,
|
||||
"k_cache must be uint8");
|
||||
STD_TORCH_CHECK(cos_sin_cache.dim() == 2 && cos_sin_cache.size(1) == 64,
|
||||
@@ -1138,8 +1195,12 @@ torch::stable::Tensor fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
|
||||
// Allocate the padded q output. The kernel writes every element (live
|
||||
// region gets optional RMSNorm+RoPE; pad region gets zeros), so `empty` is
|
||||
// safe.
|
||||
auto q_out = torch::stable::new_empty(
|
||||
q_in, {q_in.size(0), q_head_padded, q_in.size(2)}, q_in.scalar_type());
|
||||
auto q_out =
|
||||
q_head_padded == 0
|
||||
? torch::stable::new_empty(q_in, {int64_t{0}}, q_in.scalar_type())
|
||||
: torch::stable::new_empty(
|
||||
q_in, {q_in.size(0), q_head_padded, q_in.size(2)},
|
||||
q_in.scalar_type());
|
||||
|
||||
VLLM_STABLE_DISPATCH_HALF_TYPES(
|
||||
q_in.scalar_type(), "fused_deepseek_v4_qnorm_rope_kv_insert", [&] {
|
||||
@@ -1155,7 +1216,8 @@ torch::stable::Tensor fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
|
||||
cos_sin_cache.const_data_ptr<float>(), static_cast<float>(eps),
|
||||
num_tokens_full, num_tokens_insert, num_heads_q,
|
||||
num_heads_q_padded, cache_block_size_i, kv_block_stride,
|
||||
apply_q_norm, kv_mxfp8, stream);
|
||||
apply_q_norm, kv_mxfp8, apply_q_rope, is_q_interleaved,
|
||||
stream);
|
||||
});
|
||||
return q_out;
|
||||
}
|
||||
|
||||
@@ -271,7 +271,8 @@ torch::stable::Tensor fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
|
||||
torch::stable::Tensor& k_cache, torch::stable::Tensor const& slot_mapping,
|
||||
torch::stable::Tensor const& position_ids,
|
||||
torch::stable::Tensor const& cos_sin_cache, int64_t q_head_padded,
|
||||
double eps, int64_t cache_block_size, bool apply_q_norm, bool kv_mxfp8);
|
||||
double eps, int64_t cache_block_size, bool apply_q_norm, bool kv_mxfp8,
|
||||
bool apply_q_rope, bool is_q_interleaved);
|
||||
|
||||
void fused_deepseek_v4_qnorm_rope_kv_rope_full_cache_bf16_insert(
|
||||
torch::stable::Tensor& q, torch::stable::Tensor const& kv,
|
||||
|
||||
@@ -425,6 +425,12 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
|
||||
"bool is_neox, Tensor position_ids, "
|
||||
"int forced_token_heads_per_warp=-1) -> ()");
|
||||
|
||||
// q_head_padded is the padded Q head count of the returned tensor, or 0 to
|
||||
// do the KV insert alone and return an empty tensor. The Q knobs are
|
||||
// independent: apply_q_norm and apply_q_rope each drop that step for Q
|
||||
// alone (KV is always rotated), and is_q_interleaved reads and writes Q in
|
||||
// FlashMLA's mega-attention chunk-interleaved layout, which moves the
|
||||
// padding heads to the tail of every head-dim chunk.
|
||||
ops.def(
|
||||
"fused_deepseek_v4_kv_rope_insert("
|
||||
"Tensor kv, Tensor! k_cache, Tensor slot_mapping, Tensor position_ids, "
|
||||
@@ -436,7 +442,8 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
|
||||
"Tensor q_in, Tensor kv, Tensor! k_cache, "
|
||||
"Tensor slot_mapping, Tensor position_ids, Tensor cos_sin_cache, "
|
||||
"int q_head_padded, float eps, int cache_block_size, "
|
||||
"bool apply_q_norm=True, bool kv_mxfp8=False) -> Tensor");
|
||||
"bool apply_q_norm=True, bool kv_mxfp8=False, bool apply_q_rope=True, "
|
||||
"bool is_q_interleaved=False) -> Tensor");
|
||||
|
||||
// FlashInfer V4 full-cache variants: write Q in place (bf16) or to a separate
|
||||
// FP8 tensor, and KV into a contiguous 512-wide token-strided cache.
|
||||
|
||||
@@ -331,6 +331,19 @@ def test_v41_fused_save_compress_and_insert(
|
||||
torch.testing.assert_close(cache_backing, expected_cache, rtol=0, atol=0)
|
||||
|
||||
|
||||
def _rotate_rope_tail(
|
||||
latent_row: torch.Tensor, cos_sin_row: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
"""GPT-J RoPE of one latent row's last 64 dims, in fp32."""
|
||||
row = latent_row.float()
|
||||
c, s = cos_sin_row.chunk(2)
|
||||
rotated = row.clone()
|
||||
even, odd = row[448::2], row[449::2]
|
||||
rotated[448::2] = even * c - odd * s
|
||||
rotated[449::2] = odd * c + even * s
|
||||
return rotated
|
||||
|
||||
|
||||
def _mxfp8_record_reference(
|
||||
latent_row: torch.Tensor, cos_sin_row: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
@@ -339,12 +352,7 @@ def _mxfp8_record_reference(
|
||||
Mirrors FlashMLA's ``KVCacheLayout.V41_FP8Sparse`` quantizer: rotate the
|
||||
RoPE tail first, then scale every 32-dim tile, RoPE tiles included.
|
||||
"""
|
||||
row = latent_row.float()
|
||||
c, s = cos_sin_row.chunk(2)
|
||||
rotated = row.clone()
|
||||
even, odd = row[448::2], row[449::2]
|
||||
rotated[448::2] = even * c - odd * s
|
||||
rotated[449::2] = odd * c + even * s
|
||||
rotated = _rotate_rope_tail(latent_row, cos_sin_row)
|
||||
quantized, scales = _ue8m0_reference(rotated, 32, 448.0)
|
||||
return quantized.view(torch.uint8), (scales.log2() + 127).to(torch.uint8)
|
||||
|
||||
@@ -461,6 +469,125 @@ def test_v41_mxfp8_cache_round_trip():
|
||||
assert (error <= tolerance.repeat_interleave(32, dim=-1)).all()
|
||||
|
||||
|
||||
# e2m1 magnitudes indexed by the low 3 bits of a code; bit 3 is the sign.
|
||||
_E2M1_MAGNITUDES = [0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0]
|
||||
|
||||
|
||||
def _decode_nvfp4_row(packed: torch.Tensor, scales: torch.Tensor) -> torch.Tensor:
|
||||
"""Unpack one 256-byte e2m1 row and apply its 32 e4m3 tile scales."""
|
||||
mags = torch.tensor(_E2M1_MAGNITUDES, device=packed.device)
|
||||
codes = torch.empty(512, dtype=torch.uint8, device=packed.device)
|
||||
codes[0::2] = packed & 0xF # even element in the low nibble
|
||||
codes[1::2] = packed >> 4
|
||||
vals = mags[(codes & 7).long()] * torch.where(codes >= 8, -1.0, 1.0)
|
||||
return (
|
||||
vals.view(32, 16) * scales.view(torch.float8_e4m3fn).float().view(32, 1)
|
||||
).flatten()
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA only")
|
||||
@pytest.mark.parametrize("compress_ratio", [1, 2])
|
||||
def test_v41_rope_insert_nvfp4_record(compress_ratio: int):
|
||||
"""The 288-byte V4.1 NVFP4 record: 256 B of e2m1 pairs then 32 e4m3 scales.
|
||||
|
||||
Scales are byte-exact against the reference (``amax / 6`` clamped to the
|
||||
e4m3 range); values are checked as round-to-nearest on the e2m1 grid, whose
|
||||
coarsest step is 2 between 4 and 6, so half a step is ``1.0 * scale``.
|
||||
"""
|
||||
from vllm.models.deepseek_v41.common.ops.fused_compress_quant_cache import (
|
||||
rope_quant_insert,
|
||||
)
|
||||
|
||||
torch.manual_seed(13)
|
||||
device = "cuda"
|
||||
cache_block = 64
|
||||
cache_stride = math.ceil(cache_block * 288 / 512) * 512
|
||||
num_tokens = 12
|
||||
|
||||
positions = torch.arange(num_tokens, dtype=torch.int64, device=device)
|
||||
latent = torch.randn(num_tokens, 512, dtype=torch.bfloat16, device=device)
|
||||
angles = torch.randn(64, 32, device=device)
|
||||
cos_sin = torch.cat((angles.cos(), angles.sin()), dim=-1)
|
||||
|
||||
cache_backing = torch.full((2, cache_stride), 165, dtype=torch.uint8, device=device)
|
||||
cache = cache_backing.as_strided((2, cache_block, 288), (cache_stride, 288, 1))
|
||||
slots = torch.arange(num_tokens, dtype=torch.int64, device=device)
|
||||
slots[3] = -1 # covers the negative-slot skip
|
||||
|
||||
rope_quant_insert(latent, positions, cos_sin, cache, slots, compress_ratio)
|
||||
|
||||
untouched = torch.full_like(cache_backing, 165)
|
||||
for t in range(num_tokens):
|
||||
slot = slots[t].item()
|
||||
pos = positions[t].item()
|
||||
if slot < 0 or (pos + 1) % compress_ratio:
|
||||
continue
|
||||
page, row = divmod(slot, cache_block)
|
||||
cs = cos_sin[pos // compress_ratio * compress_ratio]
|
||||
rotated = _rotate_rope_tail(latent[t], cs)
|
||||
amax = rotated.view(32, 16).abs().amax(-1)
|
||||
scale = (amax / 6.0).clamp(2.0**-9, 448.0).to(torch.float8_e4m3fn)
|
||||
|
||||
got_scales = cache_backing[page, cache_block * 256 + row * 32 :][:32]
|
||||
torch.testing.assert_close(got_scales, scale.view(torch.uint8), rtol=0, atol=0)
|
||||
decoded = _decode_nvfp4_row(
|
||||
cache_backing[page, row * 256 : (row + 1) * 256], got_scales
|
||||
)
|
||||
step = scale.float().repeat_interleave(16)
|
||||
assert ((decoded - rotated).abs() <= step).all()
|
||||
untouched[page, row * 256 : (row + 1) * 256] = cache_backing[
|
||||
page, row * 256 : (row + 1) * 256
|
||||
]
|
||||
untouched[page, cache_block * 256 + row * 32 :][:32] = got_scales
|
||||
# Rows the kernel must not have touched, page padding included.
|
||||
torch.testing.assert_close(cache_backing, untouched, rtol=0, atol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA only")
|
||||
def test_v41_nvfp4_gather_matches_insert():
|
||||
"""The NVFP4 gather dequantizes exactly what the insert kernel wrote."""
|
||||
from vllm.models.deepseek_v41.common.ops import dequantize_and_gather_k_cache
|
||||
from vllm.models.deepseek_v41.common.ops.fused_compress_quant_cache import (
|
||||
rope_quant_insert,
|
||||
)
|
||||
|
||||
torch.manual_seed(17)
|
||||
device = "cuda"
|
||||
block_size = 64
|
||||
num_tokens = 70
|
||||
num_blocks = 4
|
||||
page_bytes = math.ceil(block_size * 288 / 512) * 512
|
||||
|
||||
positions = torch.arange(num_tokens, dtype=torch.int64, device=device)
|
||||
latent = torch.randn(num_tokens, 512, dtype=torch.bfloat16, device=device)
|
||||
angles = torch.randn(128, 32, device=device)
|
||||
cos_sin = torch.cat((angles.cos(), angles.sin()), dim=-1)
|
||||
backing = torch.zeros(num_blocks, page_bytes, dtype=torch.uint8, device=device)
|
||||
cache = backing.as_strided((num_blocks, block_size, 288), (page_bytes, 288, 1))
|
||||
slots = torch.arange(num_tokens, dtype=torch.int64, device=device)
|
||||
rope_quant_insert(latent, positions, cos_sin, cache, slots, 1)
|
||||
|
||||
out = torch.zeros(1, num_tokens, 512, dtype=torch.bfloat16, device=device)
|
||||
dequantize_and_gather_k_cache(
|
||||
out,
|
||||
cache,
|
||||
seq_lens=torch.tensor([num_tokens], dtype=torch.int32, device=device),
|
||||
gather_lens=None,
|
||||
block_table=torch.arange(num_blocks, dtype=torch.int32, device=device).view(
|
||||
1, -1
|
||||
),
|
||||
block_size=block_size,
|
||||
offset=0,
|
||||
)
|
||||
for t in (0, 1, block_size, num_tokens - 1):
|
||||
page, row = divmod(t, block_size)
|
||||
expected = _decode_nvfp4_row(
|
||||
backing[page, row * 256 : (row + 1) * 256],
|
||||
backing[page, block_size * 256 + row * 32 :][:32],
|
||||
)
|
||||
torch.testing.assert_close(out[0, t].float(), expected, rtol=0, atol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA only")
|
||||
@pytest.mark.parametrize("compress_ratio", [1, 2])
|
||||
@pytest.mark.parametrize("store_fp8", [False, True])
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Layout plumbing for DeepSeek V4.1 mega attention.
|
||||
|
||||
The kernel's Q and O layouts are produced by permuting ``wq_b`` rows and
|
||||
``wo_a`` columns once at load, so what has to hold is that the permuted GEMMs
|
||||
agree with a reference that permutes the activation instead.
|
||||
"""
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.models.deepseek_v41.common.ops.fused_layout import (
|
||||
WV_GROUP_SIZE,
|
||||
o_fused_chunk_permutation,
|
||||
o_fused_permutation,
|
||||
permute_q_to_fused,
|
||||
permute_wo_a_,
|
||||
permute_wq_b_,
|
||||
q_fused_permutation,
|
||||
)
|
||||
|
||||
HEAD_DIM = 512
|
||||
|
||||
|
||||
def test_permuted_wq_b_gemm_matches_permuted_activation():
|
||||
"""Permuting wq_b's rows makes the Q GEMM emit the fused layout directly."""
|
||||
torch.manual_seed(0)
|
||||
num_heads, q_lora_rank, num_tokens = 16, 64, 5
|
||||
weight = torch.randn(num_heads * HEAD_DIM, q_lora_rank)
|
||||
# One scale row per weight row, as an MXFP8 wq_b shard carries.
|
||||
scale = torch.randint(
|
||||
0, 256, (num_heads * HEAD_DIM, q_lora_rank // 32), dtype=torch.uint8
|
||||
)
|
||||
qr = torch.randn(num_tokens, q_lora_rank)
|
||||
orig_weight, orig_scale = weight.clone(), scale.clone()
|
||||
|
||||
standard = (qr @ weight.T).view(num_tokens, num_heads, HEAD_DIM)
|
||||
expected = permute_q_to_fused(standard)
|
||||
|
||||
permute_wq_b_(weight, scale, num_heads)
|
||||
fused = (qr @ weight.T).view(num_tokens, num_heads, HEAD_DIM)
|
||||
torch.testing.assert_close(fused, expected)
|
||||
# The scale has to follow its row, or dequant reads another row's scale.
|
||||
perm = q_fused_permutation(num_heads, HEAD_DIM)
|
||||
torch.testing.assert_close(weight, orig_weight[perm])
|
||||
torch.testing.assert_close(scale, orig_scale[perm])
|
||||
|
||||
|
||||
def test_permuted_wo_a_consumes_fused_output():
|
||||
"""Permuting wo_a's input columns lets it read the kernel's O layout."""
|
||||
torch.manual_seed(1)
|
||||
out_features, num_tokens = 32, 4
|
||||
in_features = WV_GROUP_SIZE * HEAD_DIM
|
||||
weight = torch.randn(out_features, in_features)
|
||||
scale = torch.randint(0, 256, (out_features, in_features // 32), dtype=torch.uint8)
|
||||
standard_o = torch.randn(num_tokens, in_features)
|
||||
orig_weight, orig_scale = weight.clone(), scale.clone()
|
||||
|
||||
expected = standard_o @ weight.T
|
||||
# The kernel emits the same values in the fused chunk order.
|
||||
perm = o_fused_permutation(WV_GROUP_SIZE, HEAD_DIM)
|
||||
fused_o = standard_o[:, perm]
|
||||
|
||||
permute_wo_a_(weight, scale, WV_GROUP_SIZE)
|
||||
torch.testing.assert_close(weight, orig_weight[:, perm])
|
||||
torch.testing.assert_close(
|
||||
scale, orig_scale[:, o_fused_chunk_permutation(WV_GROUP_SIZE, HEAD_DIM)]
|
||||
)
|
||||
# Unlike wq_b, this permutation sits inside the 4096-term reduction, so the
|
||||
# summation order changes and the result differs in the last few ulps.
|
||||
torch.testing.assert_close(fused_o @ weight.T, expected, rtol=1e-3, atol=1e-3)
|
||||
@@ -522,6 +522,189 @@ def test_kv_path_mxfp8_record_matches_reference(num_tokens: int, block_size: int
|
||||
)
|
||||
|
||||
|
||||
# ── Test 2d: mega-attention Q layout (is_q_interleaved) ──────────────────────
|
||||
|
||||
Q_CHUNK = 16
|
||||
NUM_Q_CHUNKS = HEAD_DIM // Q_CHUNK # 32
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [1, 17, 2048])
|
||||
@pytest.mark.parametrize("n_heads,q_head_padded", [(16, 64), (64, 128), (64, 0)])
|
||||
def test_q_interleaved_pads_without_rope(
|
||||
num_tokens: int, n_heads: int, q_head_padded: int
|
||||
):
|
||||
"""``is_q_interleaved`` with no norm or RoPE only zero-pads Q.
|
||||
|
||||
The mega-attention kernel norms and rotates Q itself, so every live
|
||||
element must survive bit-exact while the padding heads -- which interleave
|
||||
with the live ones inside each of the 32 head-dim chunks -- come back
|
||||
zero. ``q_head_padded=0`` drops the Q pass entirely and returns nothing,
|
||||
for a shard already at the kernel's head count.
|
||||
"""
|
||||
torch.manual_seed(5)
|
||||
device = "cuda"
|
||||
dtype = torch.bfloat16
|
||||
block_size = 64
|
||||
|
||||
q = torch.randn(num_tokens, n_heads, HEAD_DIM, dtype=dtype, device=device)
|
||||
kv = torch.randn(num_tokens, HEAD_DIM, dtype=dtype, device=device)
|
||||
positions = torch.arange(num_tokens, dtype=torch.int64, device=device)
|
||||
cos_sin_cache = make_cos_sin_cache(4096, ROPE_DIM, torch.float32, device)
|
||||
num_blocks = (num_tokens + block_size - 1) // block_size + 1
|
||||
slot_mapping = torch.arange(num_tokens, dtype=torch.int64, device=device)
|
||||
k_cache = torch.zeros(
|
||||
num_blocks, block_size * V41_HEAD_BYTES, dtype=torch.uint8, device=device
|
||||
)
|
||||
|
||||
q_out = torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
|
||||
q,
|
||||
kv,
|
||||
k_cache,
|
||||
slot_mapping,
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
q_head_padded,
|
||||
1e-6,
|
||||
block_size,
|
||||
False, # apply_q_norm
|
||||
True, # kv_mxfp8
|
||||
False, # apply_q_rope
|
||||
True, # is_q_interleaved
|
||||
)
|
||||
|
||||
# The KV half is unaffected by the Q mode: it must still be inserted.
|
||||
assert k_cache.any()
|
||||
|
||||
if q_head_padded == 0:
|
||||
assert q_out.numel() == 0
|
||||
return
|
||||
assert q_out.shape == (num_tokens, q_head_padded, HEAD_DIM)
|
||||
got = q_out.view(num_tokens, NUM_Q_CHUNKS, q_head_padded, Q_CHUNK)
|
||||
want = q.view(num_tokens, NUM_Q_CHUNKS, n_heads, Q_CHUNK)
|
||||
torch.testing.assert_close(got[:, :, :n_heads], want, rtol=0, atol=0)
|
||||
assert (got[:, :, n_heads:] == 0).all()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("n_heads", [8, 16, 32])
|
||||
def test_q_interleaved_survives_graph_replay(n_heads: int):
|
||||
"""A captured fused insert refills its padded Q buffer on every replay.
|
||||
|
||||
The op allocates ``q_out`` itself, so the tensor a capture hands back has
|
||||
to keep receiving the new padding when the graph is replayed over changed
|
||||
input -- that is what lets the layer prepare Q inside the captured region
|
||||
rather than in ``forward_mqa``.
|
||||
"""
|
||||
torch.manual_seed(6)
|
||||
device = "cuda"
|
||||
dtype = torch.bfloat16
|
||||
block_size, num_tokens, q_head_padded = 64, 7, 64
|
||||
|
||||
q = torch.randn(num_tokens, n_heads, HEAD_DIM, dtype=dtype, device=device)
|
||||
kv = torch.randn(num_tokens, HEAD_DIM, dtype=dtype, device=device)
|
||||
positions = torch.arange(num_tokens, dtype=torch.int64, device=device)
|
||||
cos_sin_cache = make_cos_sin_cache(4096, ROPE_DIM, torch.float32, device)
|
||||
num_blocks = (num_tokens + block_size - 1) // block_size + 1
|
||||
slot_mapping = torch.arange(num_tokens, dtype=torch.int64, device=device)
|
||||
k_cache = torch.zeros(
|
||||
num_blocks, block_size * V41_HEAD_BYTES, dtype=torch.uint8, device=device
|
||||
)
|
||||
|
||||
def prepare():
|
||||
return torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
|
||||
q,
|
||||
kv,
|
||||
k_cache,
|
||||
slot_mapping,
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
q_head_padded,
|
||||
1e-6,
|
||||
block_size,
|
||||
False, # apply_q_norm
|
||||
True, # kv_mxfp8
|
||||
False, # apply_q_rope
|
||||
True, # is_q_interleaved
|
||||
)
|
||||
|
||||
# Capture takes its warmup on a side stream.
|
||||
side = torch.cuda.Stream()
|
||||
side.wait_stream(torch.cuda.current_stream())
|
||||
with torch.cuda.stream(side):
|
||||
for _ in range(3):
|
||||
prepare()
|
||||
torch.cuda.current_stream().wait_stream(side)
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
q_out = prepare()
|
||||
|
||||
for factor in (0.25, -0.75):
|
||||
q.copy_(torch.randn_like(q) * factor)
|
||||
graph.replay()
|
||||
torch.accelerator.synchronize()
|
||||
got = q_out.view(num_tokens, NUM_Q_CHUNKS, q_head_padded, Q_CHUNK)
|
||||
want = q.view(num_tokens, NUM_Q_CHUNKS, n_heads, Q_CHUNK)
|
||||
torch.testing.assert_close(got[:, :, :n_heads], want, rtol=0, atol=0)
|
||||
assert (got[:, :, n_heads:] == 0).all()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("n_heads", [16, 64])
|
||||
@pytest.mark.parametrize("apply_q_norm", [False, True])
|
||||
@pytest.mark.parametrize("apply_q_rope", [False, True])
|
||||
def test_q_interleaved_is_orthogonal_to_norm_and_rope(
|
||||
n_heads: int, apply_q_norm: bool, apply_q_rope: bool
|
||||
):
|
||||
"""The Q layout composes with whatever norm and RoPE it is given.
|
||||
|
||||
A lane owns the same 512-dim slice of its head in either layout -- only
|
||||
the address differs -- so the head-major result permuted into the
|
||||
interleaved layout must equal running the interleaved path on a permuted
|
||||
input, bit for bit, for every norm/RoPE combination.
|
||||
"""
|
||||
from vllm.models.deepseek_v41.common.ops.fused_layout import permute_q_to_fused
|
||||
|
||||
torch.manual_seed(7)
|
||||
device = "cuda"
|
||||
dtype = torch.bfloat16
|
||||
block_size, num_tokens, q_head_padded = 64, 5, 64
|
||||
|
||||
q = torch.randn(num_tokens, n_heads, HEAD_DIM, dtype=dtype, device=device)
|
||||
kv = torch.randn(num_tokens, HEAD_DIM, dtype=dtype, device=device)
|
||||
positions = torch.arange(num_tokens, dtype=torch.int64, device=device)
|
||||
cos_sin_cache = make_cos_sin_cache(4096, ROPE_DIM, torch.float32, device)
|
||||
num_blocks = (num_tokens + block_size - 1) // block_size + 1
|
||||
slot_mapping = torch.arange(num_tokens, dtype=torch.int64, device=device)
|
||||
|
||||
def run(q_in, is_q_interleaved):
|
||||
cache = torch.zeros(
|
||||
num_blocks, block_size * V41_HEAD_BYTES, dtype=torch.uint8, device=device
|
||||
)
|
||||
out = torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
|
||||
q_in,
|
||||
kv,
|
||||
cache,
|
||||
slot_mapping,
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
q_head_padded,
|
||||
1e-6,
|
||||
block_size,
|
||||
apply_q_norm,
|
||||
True, # kv_mxfp8
|
||||
apply_q_rope,
|
||||
is_q_interleaved,
|
||||
)
|
||||
return out, cache
|
||||
|
||||
head_major, cache_major = run(q, False)
|
||||
interleaved, cache_inter = run(permute_q_to_fused(q), True)
|
||||
|
||||
torch.testing.assert_close(
|
||||
interleaved, permute_q_to_fused(head_major), rtol=0, atol=0
|
||||
)
|
||||
# The Q layout must not leak into the KV half.
|
||||
torch.testing.assert_close(cache_inter, cache_major, rtol=0, atol=0)
|
||||
|
||||
|
||||
# ── Test 2b: DP padding (slot_mapping shorter than q/kv) ─────────────────────
|
||||
|
||||
|
||||
|
||||
@@ -636,7 +636,8 @@ class DeepseekV41ROCMAiterMLAAttention(DeepseekV4Attention):
|
||||
transpose_scale=False,
|
||||
)
|
||||
|
||||
def _o_proj(self, o: torch.Tensor, positions: torch.Tensor) -> torch.Tensor:
|
||||
def _o_proj(self, attn_out: torch.Tensor, positions: torch.Tensor) -> torch.Tensor:
|
||||
o = attn_out[:, : self.n_local_heads, :]
|
||||
# ROCm BF16 reference wo_a path (inverse RoPE + einsum) + wo_b.
|
||||
z = rocm_inv_rope_einsum(
|
||||
self.rotary_emb,
|
||||
|
||||
@@ -44,6 +44,7 @@ from vllm.config import (
|
||||
VllmConfig,
|
||||
get_current_vllm_config,
|
||||
)
|
||||
from vllm.config.cache import CacheDType
|
||||
from vllm.distributed import get_tensor_model_parallel_world_size
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.logger import init_logger
|
||||
@@ -127,34 +128,40 @@ def _use_v41_mxfp8_kv_record() -> bool:
|
||||
|
||||
def _resolve_dsv4_kv_cache_dtype(
|
||||
use_fp8_ds_mla_layout: bool,
|
||||
kv_cache_dtype: str,
|
||||
kv_cache_dtype: CacheDType,
|
||||
cache_config: CacheConfig | None,
|
||||
) -> tuple[str, torch.dtype]:
|
||||
packed_kv_cache_dtype: CacheDType = "fp8_ds_mla",
|
||||
) -> tuple[CacheDType, torch.dtype]:
|
||||
"""Map ``(layout, --kv-cache-dtype)`` to ``(cache_dtype_str, torch_dtype)``.
|
||||
|
||||
Both layouts are paged; they differ in the per-token block format. The
|
||||
``fp8_ds_mla`` format is UE8M0 block-scaled fp8 packed as ``uint8`` (the
|
||||
canonical ``fp8_ds_mla`` string is written back onto ``cache_config`` so the
|
||||
page-size specs pick the 576B per-token slot). Plain-row backends store each
|
||||
token's KV row in its element dtype: bf16 or per-tensor FP8 E4M3.
|
||||
packed formats are ``uint8``-backed: ``fp8_ds_mla`` is UE8M0 block-scaled
|
||||
fp8 throughout, ``nvfp4_ds_mla`` keeps that sliding-window record and
|
||||
stores the compressed cache as NVFP4. An unspecific ``--kv-cache-dtype``
|
||||
(``auto`` / ``fp8``) resolves to ``packed_kv_cache_dtype``, the record this
|
||||
layer's kernel prefers, and the canonical string is written back onto
|
||||
``cache_config`` so the page-size specs pick the right per-token slot.
|
||||
Plain-row backends store each token's KV row in its element dtype: bf16 or
|
||||
per-tensor FP8 E4M3.
|
||||
"""
|
||||
if use_fp8_ds_mla_layout:
|
||||
# fp8_ds_mla block format: UE8M0 block-scaled fp8 packed as uint8.
|
||||
if kv_cache_dtype == "auto":
|
||||
kv_cache_dtype = "fp8"
|
||||
if not kv_cache_dtype.startswith("fp8"):
|
||||
if kv_cache_dtype in ("auto", "fp8"):
|
||||
kv_cache_dtype = packed_kv_cache_dtype
|
||||
elif not kv_cache_dtype.endswith("_ds_mla"):
|
||||
raise ValueError(
|
||||
"DeepseekV4 fp8_ds_mla layout only supports fp8 "
|
||||
f"kv-cache, got {kv_cache_dtype}. Please set "
|
||||
"`--kv-cache-dtype fp8` or select a backend that supports "
|
||||
"bfloat16 KV cache."
|
||||
"DeepseekV4 packed KV layouts only support fp8 kv-cache, got "
|
||||
f"{kv_cache_dtype}. Please set `--kv-cache-dtype fp8` or "
|
||||
"select a backend that supports bfloat16 KV cache."
|
||||
)
|
||||
if kv_cache_dtype != "fp8_ds_mla":
|
||||
if cache_config is not None:
|
||||
cache_config.cache_dtype = "fp8_ds_mla"
|
||||
kv_cache_dtype = "fp8_ds_mla"
|
||||
logger.info_once("Using DeepSeek's fp8_ds_mla KV cache format.")
|
||||
if cache_config is not None and cache_config.cache_dtype != kv_cache_dtype:
|
||||
cache_config.cache_dtype = kv_cache_dtype
|
||||
logger.info_once("Using DeepSeek's %s KV cache format.", kv_cache_dtype)
|
||||
return kv_cache_dtype, torch.uint8
|
||||
if kv_cache_dtype.endswith("_ds_mla"):
|
||||
raise ValueError(
|
||||
f"{kv_cache_dtype} is a packed FlashMLA DeepSeek V4.1 KV cache "
|
||||
"format; the selected backend stores plain KV rows."
|
||||
)
|
||||
|
||||
# Plain bf16 / per-tensor fp8 KV row (FlashInfer).
|
||||
if kv_cache_dtype.startswith("fp8"):
|
||||
@@ -189,6 +196,28 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
# path to pre-reserve that workspace.
|
||||
PREFILL_CHUNK_SIZE: ClassVar[int] = 4
|
||||
|
||||
# ---- attention-interface contract, declared by the platform subclass ----
|
||||
|
||||
@property
|
||||
def accepts_unnormed_unroped_query(self) -> bool:
|
||||
"""Whether ``forward_mqa``'s ``q`` is the raw ``wq_b`` output.
|
||||
|
||||
True when the attention kernel applies the Q norm and RoPE itself, and
|
||||
reads Q in its own chunk-interleaved layout, so the layer only
|
||||
zero-pads Q to ``padded_heads`` and inserts KV before calling it.
|
||||
"""
|
||||
return False
|
||||
|
||||
@property
|
||||
def packed_kv_cache_dtype(self) -> CacheDType:
|
||||
"""The packed KV record this layer's kernel prefers.
|
||||
|
||||
What an unspecific ``--kv-cache-dtype`` (``auto`` / ``fp8``) resolves
|
||||
to. Mega attention overrides it: its kernel is the one that can read
|
||||
an NVFP4 compressed cache.
|
||||
"""
|
||||
return "fp8_ds_mla"
|
||||
|
||||
@classmethod
|
||||
@abstractmethod
|
||||
def get_padded_num_q_heads(cls, num_heads: int) -> int:
|
||||
@@ -213,8 +242,19 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def _o_proj(self, o: torch.Tensor, positions: torch.Tensor) -> torch.Tensor:
|
||||
"""Inverse-RoPE + wo_a + wo_b output projection (platform-specific)."""
|
||||
def _o_proj(
|
||||
self,
|
||||
attn_out: "torch.Tensor | QuantizedActivation",
|
||||
positions: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""Project whatever ``_alloc_attn_out`` produced through wo_a and wo_b.
|
||||
|
||||
Takes the buffer whole, so each layer owns the shape it allocated: the
|
||||
bf16 layers slice off their padding heads and apply the inverse RoPE,
|
||||
while a layer whose attention kernel already did the inverse RoPE and
|
||||
the FP8 cast gets a QuantizedActivation and has only wo_a and wo_b
|
||||
left.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def _uses_fp8_ds_mla_layout(self) -> bool:
|
||||
@@ -459,11 +499,27 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
# Resolve the kv-cache dtype from this backend's block format. The same
|
||||
# resolution drives the SWA cache tensor dtype below.
|
||||
self.kv_cache_dtype, self.kv_cache_torch_dtype = _resolve_dsv4_kv_cache_dtype(
|
||||
self._uses_fp8_ds_mla_layout(), cache_config.cache_dtype, cache_config
|
||||
self._uses_fp8_ds_mla_layout(),
|
||||
cache_config.cache_dtype,
|
||||
cache_config,
|
||||
self.packed_kv_cache_dtype,
|
||||
)
|
||||
self.kv_mxfp8 = _use_v41_mxfp8_kv_record()
|
||||
self.kv_bytes_per_token = 528 if self.kv_mxfp8 else 584
|
||||
self.swa_bytes_per_token = 528 if self.kv_mxfp8 else 584
|
||||
# nvfp4_ds_mla keeps the MXFP8 sliding-window record and stores the
|
||||
# compressed cache as NVFP4 (256 B of e2m1 pairs + 32 e4m3 scales).
|
||||
self.compressed_bytes_per_token = (
|
||||
288 if self.kv_cache_dtype == "nvfp4_ds_mla" else self.swa_bytes_per_token
|
||||
)
|
||||
# One alignment for every page in the block: the block stride is their
|
||||
# sum, and 512 satisfies both TMA strides in play (512 for the V4.1
|
||||
# fp8 record, 256 for NVFP4).
|
||||
self.kv_page_alignment = 512 if self.kv_mxfp8 else 576
|
||||
if self.kv_cache_dtype == "nvfp4_ds_mla" and not self.kv_mxfp8:
|
||||
raise ValueError(
|
||||
"nvfp4_ds_mla needs the V4.1 KV records, which FlashMLA "
|
||||
"decodes only on SM100."
|
||||
)
|
||||
|
||||
self.swa_cache_layer = DeepseekV4SWACache(
|
||||
head_dim=self.head_dim,
|
||||
@@ -473,7 +529,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
cache_config=cache_config,
|
||||
backend_cls=self.swa_backend_cls,
|
||||
block_size=32,
|
||||
packed_bytes_per_token=self.kv_bytes_per_token,
|
||||
packed_bytes_per_token=self.swa_bytes_per_token,
|
||||
packed_page_alignment=self.kv_page_alignment,
|
||||
)
|
||||
|
||||
@@ -553,8 +609,12 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
block_size=self.swa_cache_layer.block_size,
|
||||
)
|
||||
|
||||
# Every backend that gathers a chunk's KV through
|
||||
# combine_topk_swa_indices needs its Triton kernel warmed, mega
|
||||
# attention included -- it calls it from _forward_prefill_mega.
|
||||
if self.backend_cls.get_name() in (
|
||||
"FLASHMLA_SPARSE_DSV41",
|
||||
"FLASHMLA_MEGA_ATTN_DSV41",
|
||||
"ROCM_FLASHMLA_SPARSE_DSV4",
|
||||
):
|
||||
from vllm.models.deepseek_v41.common.ops.cache_utils import (
|
||||
@@ -569,14 +629,10 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
hidden_states: torch.Tensor,
|
||||
llama_4_scaling: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
# Pre-allocate attention output with FlashMLA-padded head count.
|
||||
# The op writes into `o_padded`; we slice to n_local_heads after.
|
||||
num_tokens = hidden_states.shape[0]
|
||||
o_padded = torch.empty(
|
||||
(num_tokens, self.padded_heads, self.head_dim),
|
||||
dtype=hidden_states.dtype,
|
||||
device=hidden_states.device,
|
||||
)
|
||||
# The eager attention region writes into a caller-owned buffer
|
||||
# (breakable_cudagraph needs in-place outputs); its shape and how it is
|
||||
# projected afterwards follow the interface contract above.
|
||||
attn_out = self._alloc_attn_out(hidden_states.shape[0], hidden_states)
|
||||
|
||||
# Keep the attention input preparation in the captured graph. Only the
|
||||
# sparse indexer and MLA attention run in the eager break below.
|
||||
@@ -593,12 +649,19 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
kv_score,
|
||||
indexer_weights,
|
||||
positions,
|
||||
o_padded,
|
||||
attn_out,
|
||||
)
|
||||
o = o_padded[:, : self.n_local_heads, :]
|
||||
return self._o_proj(attn_out, positions)
|
||||
|
||||
# Inverse-RoPE + wo_a + wo_b output projection (platform-specific).
|
||||
return self._o_proj(o, positions)
|
||||
def _alloc_attn_out(
|
||||
self, num_tokens: int, hidden_states: torch.Tensor
|
||||
) -> "torch.Tensor | QuantizedActivation":
|
||||
"""The buffer ``forward_mqa`` fills, per the interface contract."""
|
||||
return torch.empty(
|
||||
(num_tokens, self.padded_heads, self.head_dim),
|
||||
dtype=hidden_states.dtype,
|
||||
device=hidden_states.device,
|
||||
)
|
||||
|
||||
@cached_property
|
||||
def _can_fuse_query_quant(self) -> bool:
|
||||
@@ -652,7 +715,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
kv_score: torch.Tensor,
|
||||
indexer_weights: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
o_padded: torch.Tensor,
|
||||
attn_out: "torch.Tensor | QuantizedActivation",
|
||||
) -> None:
|
||||
"""Wide eager region: the whole of ``_prepare_and_attn`` runs eagerly.
|
||||
|
||||
@@ -667,7 +730,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
kv_score,
|
||||
indexer_weights,
|
||||
positions,
|
||||
o_padded,
|
||||
attn_out,
|
||||
)
|
||||
|
||||
def _prepare_and_attn(
|
||||
@@ -679,7 +742,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
kv_score: torch.Tensor,
|
||||
indexer_weights: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
o_padded: torch.Tensor,
|
||||
attn_out: "torch.Tensor | QuantizedActivation",
|
||||
) -> None:
|
||||
"""Attention input preparation followed by the sparse indexer and MLA.
|
||||
|
||||
@@ -751,7 +814,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
q,
|
||||
kv,
|
||||
positions,
|
||||
o_padded,
|
||||
attn_out,
|
||||
)
|
||||
|
||||
def _fused_wqa_wkv_gemm(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
@@ -858,16 +921,28 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
dict[str, AttentionMetadata] | list[dict[str, AttentionMetadata]] | None
|
||||
),
|
||||
) -> torch.Tensor:
|
||||
"""Ready ``q`` for the attention kernel and publish this step's KV.
|
||||
|
||||
One launch does both. With ``accepts_unnormed_unroped_query`` the
|
||||
attention kernel norms and rotates Q itself and reads it in its own
|
||||
chunk-interleaved layout, so the Q half of the launch is a zero-pad to
|
||||
``padded_heads`` -- and nothing at all once the shard is that wide.
|
||||
"""
|
||||
if not isinstance(attn_metadata, dict):
|
||||
# Profile run: kernel doesn't fire; produce a padded tensor so
|
||||
# downstream FlashMLA gets the right shape.
|
||||
if self.n_local_heads < self.padded_heads:
|
||||
return F.pad(
|
||||
q,
|
||||
(0, 0, 0, self.padded_heads - self.n_local_heads),
|
||||
value=0.0,
|
||||
)
|
||||
return q
|
||||
if self.n_local_heads >= self.padded_heads:
|
||||
return q
|
||||
if self.accepts_unnormed_unroped_query:
|
||||
# Padding heads sit at the tail of every head-dim chunk in
|
||||
# that layout, so no head-major pad of `q` reproduces it --
|
||||
# and nothing reads it on a profile run.
|
||||
return q.new_zeros((q.shape[0], self.padded_heads, q.shape[2]))
|
||||
return F.pad(
|
||||
q,
|
||||
(0, 0, 0, self.padded_heads - self.n_local_heads),
|
||||
value=0.0,
|
||||
)
|
||||
|
||||
swa_metadata = cast(
|
||||
"DeepseekSparseSWAMetadata | None",
|
||||
@@ -886,23 +961,38 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
if cache_dtype == torch.uint8:
|
||||
# fp8_ds_mla UE8M0 paged path. Horizontally fused:
|
||||
# Q side: GPT-J RoPE, zero-filling the padding head slots; the
|
||||
# kernel allocates and returns the padded q tensor.
|
||||
# kernel allocates and returns the padded q tensor. An
|
||||
# interleaved Q skips the RoPE its attention kernel owns
|
||||
# and keeps only the pad, which q_head_padded=0 drops
|
||||
# too once the shard is already padded_heads wide.
|
||||
# KV side: GPT-J RoPE + UE8M0 FP8 quant + paged cache insert.
|
||||
swa_kv_cache_2d = swa_kv_cache.view(swa_kv_cache.shape[0], -1)
|
||||
return torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
|
||||
pad_to = (
|
||||
0
|
||||
if self.accepts_unnormed_unroped_query
|
||||
and self.n_local_heads == self.padded_heads
|
||||
else self.padded_heads
|
||||
)
|
||||
q_padded = torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
|
||||
q,
|
||||
kv,
|
||||
swa_kv_cache_2d,
|
||||
swa_metadata.slot_mapping,
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
self.padded_heads,
|
||||
pad_to,
|
||||
self.eps,
|
||||
swa_metadata.block_size,
|
||||
False,
|
||||
False, # apply_q_norm: qr is normed before wq_b
|
||||
self.kv_mxfp8,
|
||||
not self.accepts_unnormed_unroped_query, # apply_q_rope
|
||||
self.accepts_unnormed_unroped_query, # is_q_interleaved
|
||||
)
|
||||
return q if pad_to == 0 else q_padded
|
||||
|
||||
assert not self.accepts_unnormed_unroped_query, (
|
||||
"the chunk-interleaved Q layout only pairs with a packed KV record"
|
||||
)
|
||||
# Plain-row path: the [num_blocks, block_size, 512] cache stores the KV
|
||||
# row in its element dtype (no Q padding). bf16 rewrites q in place;
|
||||
# per-tensor fp8 writes a separately-allocated fp8 q and quantizes the
|
||||
@@ -959,7 +1049,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
# fp8_ds_mla is a UE8M0 block-scaled uint8 layout whose page rounds up
|
||||
# to the decode kernel's TMA stride; plain bf16 / per-tensor fp8 rows
|
||||
# use natural element-size pages.
|
||||
uses_fp8_ds_mla_layout = self.kv_cache_dtype == "fp8_ds_mla"
|
||||
uses_fp8_ds_mla_layout = self.kv_cache_dtype in ("fp8_ds_mla", "nvfp4_ds_mla")
|
||||
return MLAAttentionSpec(
|
||||
block_size=vllm_config.cache_config.block_size,
|
||||
num_kv_heads=1,
|
||||
@@ -972,7 +1062,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
kv_quant_mode=get_kv_quant_mode(self.kv_cache_dtype),
|
||||
# Packed record width; head_size stays semantic (512).
|
||||
state_content_bytes=(
|
||||
self.kv_bytes_per_token if uses_fp8_ds_mla_layout else None
|
||||
self.compressed_bytes_per_token if uses_fp8_ds_mla_layout else None
|
||||
),
|
||||
)
|
||||
|
||||
@@ -1016,7 +1106,13 @@ class DeepseekV4IndexerCache(torch.nn.Module, AttentionLayerBase):
|
||||
def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec:
|
||||
# head_dim already carries the fp8 scale padding
|
||||
# tokens_per_state=1 for V3.2, >1 for DeepseekV4; same cache layout.
|
||||
uses_fp8_ds_mla_layout = vllm_config.cache_config.cache_dtype == "fp8_ds_mla"
|
||||
# nvfp4_ds_mla is packed too: its compressed record is NVFP4 but the
|
||||
# sliding-window record stays the V4.1 MXFP8 one, so the indexer page
|
||||
# takes the same alignment either way.
|
||||
uses_fp8_ds_mla_layout = vllm_config.cache_config.cache_dtype in (
|
||||
"fp8_ds_mla",
|
||||
"nvfp4_ds_mla",
|
||||
)
|
||||
page_alignment = (
|
||||
576 if uses_fp8_ds_mla_layout and not _use_v41_mxfp8_kv_record() else 512
|
||||
)
|
||||
|
||||
@@ -43,10 +43,16 @@ from vllm.utils.math_utils import next_power_of_2
|
||||
# dims each plus a pad byte.
|
||||
# V4.1 (528 B): all 512 dims as fp8 e4m3 (RoPE included), then 16 UE8M0
|
||||
# scales of 32 dims each. FlashMLA's ``ModelType::V41``.
|
||||
# V4.1 NVFP4 (288 B): 512 e2m1 values packed two per byte (even element in
|
||||
# the low nibble), then 32 e4m3 scales of 16 dims each.
|
||||
# FlashMLA's ``ModelType::V41_FP4``, compressed cache only.
|
||||
V4_BYTES_PER_TOKEN = 584
|
||||
V41_BYTES_PER_TOKEN = 528
|
||||
V41_QUANT_BLOCK = 32
|
||||
V41_NUM_SCALES = 512 // V41_QUANT_BLOCK # 16
|
||||
V41_NVFP4_BYTES_PER_TOKEN = 288
|
||||
V41_NVFP4_QUANT_BLOCK = 16
|
||||
V41_NVFP4_NUM_SCALES = 512 // V41_NVFP4_QUANT_BLOCK # 32
|
||||
|
||||
|
||||
@triton.jit
|
||||
@@ -496,6 +502,73 @@ def _dequantize_and_gather_k_mxfp8_kernel(
|
||||
tl.store(output_row_ptr + d, dequant.to(tl.bfloat16))
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _dequantize_and_gather_k_nvfp4_kernel(
|
||||
out_ptr,
|
||||
out_stride0,
|
||||
out_stride1,
|
||||
k_cache_ptr,
|
||||
seq_lens_ptr,
|
||||
block_table_ptr,
|
||||
offset,
|
||||
gather_lens_ptr,
|
||||
max_blocks_per_seq: tl.constexpr,
|
||||
head_dim: tl.constexpr, # 512
|
||||
scale_dim: tl.constexpr, # 32
|
||||
quant_block: tl.constexpr, # 16
|
||||
cache_block_size: tl.constexpr,
|
||||
block_stride: tl.constexpr,
|
||||
):
|
||||
"""Gather and dequantize V4.1 NVFP4 rows into a bf16 workspace."""
|
||||
batch_idx = tl.program_id(0)
|
||||
worker_id = tl.program_id(1)
|
||||
num_workers = tl.num_programs(1)
|
||||
|
||||
seq_len = tl.load(seq_lens_ptr + batch_idx)
|
||||
if gather_lens_ptr is not None: # noqa: SIM108
|
||||
gather_len = tl.load(gather_lens_ptr + batch_idx)
|
||||
else:
|
||||
gather_len = seq_len
|
||||
start_pos = seq_len - gather_len
|
||||
|
||||
packed_bytes: tl.constexpr = head_dim // 2
|
||||
d = tl.arange(0, head_dim)
|
||||
for i in range(worker_id, gather_len, num_workers):
|
||||
pos = start_pos + i
|
||||
block_in_seq = pos // cache_block_size
|
||||
pos_in_block = pos % cache_block_size
|
||||
|
||||
physical_block_idx = tl.load(
|
||||
block_table_ptr + batch_idx * max_blocks_per_seq + block_in_seq
|
||||
)
|
||||
page = k_cache_ptr + physical_block_idx.to(tl.int64) * block_stride
|
||||
|
||||
packed = tl.load(
|
||||
page + pos_in_block * packed_bytes + tl.arange(0, packed_bytes)
|
||||
)
|
||||
# Even element in the low nibble, odd in the high nibble.
|
||||
codes = tl.interleave((packed & 0xF).to(tl.int32), (packed >> 4).to(tl.int32))
|
||||
# e2m1 magnitudes: 0, 0.5, 1, 1.5, 2, 3, 4, 6.
|
||||
mag_code = codes & 7
|
||||
e = (mag_code >> 1).to(tl.float32)
|
||||
m = (mag_code & 1).to(tl.float32)
|
||||
mag = tl.where(mag_code < 2, m * 0.5, (1.0 + m * 0.5) * tl.exp2(e - 1.0))
|
||||
vals = tl.where(codes >= 8, -mag, mag)
|
||||
|
||||
sf = tl.load(
|
||||
page
|
||||
+ cache_block_size * packed_bytes
|
||||
+ pos_in_block * scale_dim
|
||||
+ tl.arange(0, scale_dim)
|
||||
)
|
||||
scale = sf.to(tl.float8e4nv, bitcast=True).to(tl.float32)
|
||||
tiles = tl.reshape(vals, (scale_dim, quant_block))
|
||||
dequant = tl.reshape(tiles * tl.reshape(scale, (scale_dim, 1)), (head_dim,))
|
||||
|
||||
output_row_ptr = out_ptr + batch_idx * out_stride0 + (offset + i) * out_stride1
|
||||
tl.store(output_row_ptr + d, dequant.to(tl.bfloat16))
|
||||
|
||||
|
||||
def dequantize_and_gather_k_cache_triton(
|
||||
# [num_reqs, max_num_tokens, head_size]
|
||||
out: torch.Tensor,
|
||||
@@ -513,6 +586,25 @@ def dequantize_and_gather_k_cache_triton(
|
||||
) -> None:
|
||||
num_reqs = seq_lens.shape[0]
|
||||
NUM_WORKERS = 128
|
||||
if k_cache.shape[-1] == V41_NVFP4_BYTES_PER_TOKEN:
|
||||
_dequantize_and_gather_k_nvfp4_kernel[(num_reqs, NUM_WORKERS)](
|
||||
out,
|
||||
out.stride(0),
|
||||
out.stride(1),
|
||||
k_cache,
|
||||
seq_lens,
|
||||
block_table,
|
||||
offset,
|
||||
gather_lens,
|
||||
max_blocks_per_seq=block_table.shape[-1],
|
||||
head_dim=512,
|
||||
scale_dim=V41_NVFP4_NUM_SCALES,
|
||||
quant_block=V41_NVFP4_QUANT_BLOCK,
|
||||
cache_block_size=block_size,
|
||||
block_stride=k_cache.stride(0),
|
||||
)
|
||||
return
|
||||
|
||||
if k_cache.shape[-1] == V41_BYTES_PER_TOKEN:
|
||||
_dequantize_and_gather_k_mxfp8_kernel[(num_reqs, NUM_WORKERS)](
|
||||
out,
|
||||
@@ -581,14 +673,16 @@ def dequantize_and_gather_k_cache(
|
||||
) -> None:
|
||||
"""Dequantize and gather a paged DSv4 K cache.
|
||||
|
||||
The record is read off ``k_cache.shape[-1]``; see the module header.
|
||||
The record is read off ``k_cache.shape[-1]``; see the module header. Only
|
||||
the fp8 records have a CuteDSL gather, so NVFP4 always takes the Triton
|
||||
path.
|
||||
|
||||
``use_fnuz`` MUST match the encoder of the specific cache being read:
|
||||
``False`` for ``compressed_k_cache`` (Triton encoder is OCP everywhere),
|
||||
``current_platform.is_fp8_fnuz()`` for ``swa_k_cache`` (C++ encoder
|
||||
writes FNUZ on gfx942 and OCP on gfx950).
|
||||
"""
|
||||
if has_cutedsl():
|
||||
if has_cutedsl() and k_cache.shape[-1] != V41_NVFP4_BYTES_PER_TOKEN:
|
||||
# lazily import, otherwise some tests fail due to CUDA driver init failure.
|
||||
from vllm.models.deepseek_v4.nvidia.ops.dequant_gather_k_cutedsl import (
|
||||
_DEQUANT_GATHER_K_CACHE_CUTEDSL_KERNEL,
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.models.deepseek_v4.common.ops.fused_indexer_q import _fp32x2_to_fp4x2
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
@@ -232,13 +233,14 @@ def rope_quant_insert(
|
||||
|
||||
The BF16 latent supplies both NoPE quantization and RoPE input. It is read
|
||||
only for valid slots at group boundaries. The cache dtype selects the
|
||||
layout: ``uint8`` is the fp8_ds_mla paged layout, whose record the
|
||||
per-token byte width names -- 584 B for V4 (576 value bytes and eight
|
||||
segregated UE8M0 scale bytes, including one zero padding scale) or 528 B
|
||||
for V4.1 (512 MXFP8 value bytes covering the RoPE dims too, then 16 UE8M0
|
||||
scales of 32 dims each). ``bfloat16`` and ``float8_e4m3fn`` are the plain
|
||||
[448 NoPE | 64 RoPE] rows read by FlashInfer, the latter scaled by the
|
||||
per-tensor ``fp8_scale``.
|
||||
layout: ``uint8`` is a paged FlashMLA layout, whose record the per-token
|
||||
byte width names -- 584 B for V4 (576 value bytes and eight segregated
|
||||
UE8M0 scale bytes, including one zero padding scale), 528 B for V4.1
|
||||
(512 MXFP8 value bytes covering the RoPE dims too, then 16 UE8M0 scales of
|
||||
32 dims each), or 288 B for V4.1 NVFP4 (256 bytes of e2m1 pairs then 32
|
||||
e4m3 scales of 16 dims each), which only the compressed cache uses.
|
||||
``bfloat16`` and ``float8_e4m3fn`` are the plain [448 NoPE | 64 RoPE] rows
|
||||
read by FlashInfer, the latter scaled by the per-tensor ``fp8_scale``.
|
||||
"""
|
||||
assert compress_ratio in (1, 2)
|
||||
assert latent.shape[1] == 512 and latent.dtype == torch.bfloat16
|
||||
@@ -249,14 +251,14 @@ def rope_quant_insert(
|
||||
return
|
||||
launch_kwargs = {"launch_pdl": False} if current_platform.is_cuda() else {}
|
||||
if kv_cache.dtype == torch.uint8:
|
||||
assert kv_cache.shape[-1] in (584, 528), (
|
||||
kernel = {
|
||||
584: _rope_quant_insert_kernel,
|
||||
528: _rope_quant_insert_mxfp8_kernel,
|
||||
288: _rope_quant_insert_nvfp4_kernel,
|
||||
}.get(kv_cache.shape[-1])
|
||||
assert kernel is not None, (
|
||||
f"unsupported paged KV record width {kv_cache.shape[-1]}"
|
||||
)
|
||||
kernel = (
|
||||
_rope_quant_insert_mxfp8_kernel
|
||||
if kv_cache.shape[-1] == 528
|
||||
else _rope_quant_insert_kernel
|
||||
)
|
||||
kernel[(num_tokens,)](
|
||||
latent,
|
||||
positions,
|
||||
@@ -402,6 +404,62 @@ def _rope_quant_insert_mxfp8_kernel(
|
||||
tl.store(scales + tl.arange(0, 16), encoded.to(tl.uint8))
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _rope_quant_insert_nvfp4_kernel(
|
||||
latent,
|
||||
positions,
|
||||
cos_sin,
|
||||
cache,
|
||||
cache_slots,
|
||||
COS_STRIDE: tl.constexpr,
|
||||
CACHE_STRIDE: tl.constexpr,
|
||||
CACHE_BLOCK: tl.constexpr,
|
||||
COMPRESS_RATIO: tl.constexpr,
|
||||
SANITIZE_CACHE_NANS: tl.constexpr,
|
||||
):
|
||||
"""V4.1 NVFP4 record: RoPE, then e2m1 with one e4m3 scale per 16 dims.
|
||||
|
||||
The scale is ``amax / 6`` (6 is e2m1's largest magnitude) clamped to the
|
||||
e4m3 range, with no per-tensor scale on top.
|
||||
"""
|
||||
t = tl.program_id(0)
|
||||
slot = tl.load(cache_slots + t)
|
||||
if slot < 0:
|
||||
return
|
||||
position = tl.load(positions + t)
|
||||
if (position + 1) % COMPRESS_RATIO != 0:
|
||||
return
|
||||
d = tl.arange(0, 512)
|
||||
normed = tl.load(latent + t.to(tl.int64) * 512 + d).to(tl.float32)
|
||||
|
||||
# NoPE pairs load (cos, sin) = (1, 0), so the rotation is the identity there.
|
||||
even, odd = tl.split(tl.reshape(normed, (256, 2)))
|
||||
pair = tl.arange(0, 256) - 224
|
||||
cs = cos_sin + (position // COMPRESS_RATIO * COMPRESS_RATIO) * COS_STRIDE
|
||||
c = tl.load(cs + tl.maximum(pair, 0), pair >= 0, other=1.0).to(tl.float32)
|
||||
s = tl.load(cs + 32 + tl.maximum(pair, 0), pair >= 0, other=0.0).to(tl.float32)
|
||||
rotated = tl.interleave(even * c - odd * s, odd * c + even * s)
|
||||
if SANITIZE_CACHE_NANS:
|
||||
rotated = tl.where(rotated == rotated, rotated, 0.0)
|
||||
|
||||
tiles = tl.reshape(rotated, (32, 16))
|
||||
amax = tl.max(tl.abs(tiles), 1)
|
||||
# 2**-9 is the smallest normal e4m3 magnitude; 448 the largest.
|
||||
scale = tl.clamp(amax * (1.0 / 6.0), 0.001953125, 448.0).to(tl.float8e4nv)
|
||||
# Round-to-nearest division: Triton's default div.full misplaces values
|
||||
# that land exactly on an e2m1 tie.
|
||||
scaled = tl.math.div_rn(tiles, tl.reshape(scale.to(tl.float32), (32, 1)))
|
||||
lo, hi = tl.split(tl.reshape(tl.reshape(scaled, (512,)), (256, 2)))
|
||||
|
||||
page = cache + (slot // CACHE_BLOCK).to(tl.int64) * CACHE_STRIDE
|
||||
tl.store(
|
||||
page + (slot % CACHE_BLOCK) * 256 + tl.arange(0, 256),
|
||||
_fp32x2_to_fp4x2(lo, hi),
|
||||
)
|
||||
scales = page + CACHE_BLOCK * 256 + (slot % CACHE_BLOCK) * 32
|
||||
tl.store(scales + tl.arange(0, 32), scale.to(tl.uint8, bitcast=True))
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _rope_plain_insert_kernel(
|
||||
latent,
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Weight permutations for FlashMLA's mega-attention kernel.
|
||||
|
||||
The kernel reads Q with 16-element head-dim chunks interleaved across heads and
|
||||
writes O with 32-element chunks interleaved across the 8 heads of a ``wo_a``
|
||||
group. Every permutation here satisfies ``fused = standard[perm]``, and is
|
||||
applied once to ``wq_b`` rows and ``wo_a`` columns at load time so the
|
||||
surrounding GEMMs produce and consume the kernel's layouts directly -- no
|
||||
per-step shuffle.
|
||||
"""
|
||||
|
||||
import torch
|
||||
|
||||
HEAD_DIM = 512
|
||||
Q_CHUNK = 16
|
||||
O_CHUNK = 32
|
||||
WV_GROUP_SIZE = 8
|
||||
|
||||
|
||||
def q_fused_permutation(num_heads: int, head_dim: int = HEAD_DIM) -> torch.Tensor:
|
||||
"""``fused[(d // 16) * (H * 16) + h * 16 + d % 16] = standard[h * D + d]``."""
|
||||
h = torch.arange(num_heads).view(num_heads, 1)
|
||||
d = torch.arange(head_dim).view(1, head_dim)
|
||||
fused_index = (d // Q_CHUNK) * (num_heads * Q_CHUNK) + h * Q_CHUNK + d % Q_CHUNK
|
||||
perm = torch.empty(num_heads * head_dim, dtype=torch.long)
|
||||
perm[fused_index.reshape(-1)] = torch.arange(num_heads * head_dim)
|
||||
return perm
|
||||
|
||||
|
||||
def o_fused_permutation(
|
||||
heads_per_group: int = WV_GROUP_SIZE, head_dim: int = HEAD_DIM
|
||||
) -> torch.Tensor:
|
||||
"""``fused[(c * G + h) * 32 + j] = standard[h * D + c * 32 + j]`` per group."""
|
||||
h = torch.arange(heads_per_group).view(-1, 1, 1)
|
||||
c = torch.arange(head_dim // O_CHUNK).view(1, -1, 1)
|
||||
j = torch.arange(O_CHUNK).view(1, 1, -1)
|
||||
fused_index = (c * heads_per_group + h) * O_CHUNK + j
|
||||
perm = torch.empty(heads_per_group * head_dim, dtype=torch.long)
|
||||
perm[fused_index.reshape(-1)] = torch.arange(heads_per_group * head_dim)
|
||||
return perm
|
||||
|
||||
|
||||
def o_fused_chunk_permutation(
|
||||
heads_per_group: int = WV_GROUP_SIZE, head_dim: int = HEAD_DIM
|
||||
) -> torch.Tensor:
|
||||
"""Per-32-element-chunk form of :func:`o_fused_permutation`, for scales."""
|
||||
return o_fused_permutation(heads_per_group, head_dim)[::O_CHUNK] // O_CHUNK
|
||||
|
||||
|
||||
def permute_q_to_fused(q: torch.Tensor) -> torch.Tensor:
|
||||
"""``[N, H, D]`` standard layout -> the same shape in the fused layout."""
|
||||
n, h, d = q.shape
|
||||
perm = q_fused_permutation(h, d).to(q.device)
|
||||
return q.reshape(n, h * d)[:, perm].view(n, h, d)
|
||||
|
||||
|
||||
def _bytes_view(t: torch.Tensor) -> torch.Tensor:
|
||||
"""Byte view of a 1-byte-element tensor, so fp8/ue8m0 can be gathered."""
|
||||
return t.view(torch.uint8) if t.element_size() == 1 else t
|
||||
|
||||
|
||||
def permute_wq_b_(
|
||||
weight: torch.Tensor, weight_scale: torch.Tensor, num_local_heads: int
|
||||
) -> None:
|
||||
"""Permute an MXFP8 ``wq_b`` shard's rows and per-row scales, in place."""
|
||||
head_dim = weight.shape[0] // num_local_heads
|
||||
perm = q_fused_permutation(num_local_heads, head_dim).to(weight.device)
|
||||
for t in (weight, weight_scale):
|
||||
b = _bytes_view(t)
|
||||
b.copy_(b[perm])
|
||||
|
||||
|
||||
def permute_wo_a_(
|
||||
weight: torch.Tensor,
|
||||
weight_scale: torch.Tensor,
|
||||
heads_per_group: int = WV_GROUP_SIZE,
|
||||
) -> None:
|
||||
"""Permute an MXFP8 ``wo_a`` shard's input columns and scales, in place."""
|
||||
head_dim = weight.shape[1] // heads_per_group
|
||||
perm = o_fused_permutation(heads_per_group, head_dim).to(weight.device)
|
||||
w = _bytes_view(weight)
|
||||
w.copy_(w[:, perm])
|
||||
chunk_perm = o_fused_chunk_permutation(heads_per_group, head_dim)
|
||||
s = _bytes_view(weight_scale)
|
||||
s.copy_(s[:, chunk_perm.to(weight.device)])
|
||||
@@ -477,8 +477,22 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
|
||||
for layer in self.model.layers:
|
||||
layer.ffn.finalize_mega_moe_weights()
|
||||
|
||||
def _finalize_attn(self) -> None:
|
||||
"""Run the attention backend's post-load weight step on the draft layers.
|
||||
|
||||
They are ordinary DeepseekV4DecoderLayers, so they take the engine-wide
|
||||
attention backend and owe it whatever the target model owes it -- mega
|
||||
attention permutes wq_b / wo_a here and refuses to run without it.
|
||||
Idempotent, like the target's.
|
||||
"""
|
||||
for layer in self.model.layers:
|
||||
finalize = getattr(layer.attn, "finalize_loaded_weights", None)
|
||||
if finalize is not None:
|
||||
finalize()
|
||||
|
||||
def process_weights_after_loading(self) -> None:
|
||||
self._finalize_moe()
|
||||
self._finalize_attn()
|
||||
|
||||
def _remap_dspark_name(self, name: str) -> str | None:
|
||||
"""Map a checkpoint ``mtp.{i}.*`` name to this model's parameter path.
|
||||
|
||||
@@ -0,0 +1,497 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""DeepSeek V4.1 attention on FlashMLA's mega-attention kernel (SM100).
|
||||
|
||||
One kernel does Q RoPE, sparse attention, the inverse RoPE of the output and
|
||||
its FP8 cast, and writes straight into the buffer ``wo_a`` consumes. So the
|
||||
layer declares ``accepts_unnormed_unroped_query`` -- the kernel RoPEs Q itself,
|
||||
leaving the fused KV insert only the zero-pad to the kernel's head count -- and
|
||||
its ``_alloc_attn_out`` / ``_o_proj`` pair speaks QuantizedActivation instead
|
||||
of bf16, since the output needs no rotation or quantization here.
|
||||
|
||||
``wq_b`` rows and ``wo_a`` columns are permuted once at load so the surrounding
|
||||
GEMMs speak the kernel's chunk-interleaved layouts directly. A step's prefill
|
||||
and decode segments write disjoint token ranges of one output buffer pair, so
|
||||
a single ``wo_a`` einsum covers the whole step.
|
||||
"""
|
||||
|
||||
from dataclasses import replace
|
||||
from typing import TYPE_CHECKING, cast
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.cache import CacheDType
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.model_executor.layers.fusion.quant_activation import QuantizedActivation
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import kMxfp8Dynamic
|
||||
from vllm.models.deepseek_v41.common.ops import (
|
||||
combine_topk_swa_indices,
|
||||
compute_global_topk_indices_and_lens,
|
||||
dequantize_and_gather_k_cache,
|
||||
)
|
||||
from vllm.models.deepseek_v41.common.ops.fused_layout import (
|
||||
WV_GROUP_SIZE,
|
||||
permute_wo_a_,
|
||||
permute_wq_b_,
|
||||
)
|
||||
from vllm.models.deepseek_v41.nvidia.flashmla import DeepseekV4FlashMLAAttention
|
||||
from vllm.models.deepseek_v41.sparse_mla import (
|
||||
DeepseekV4FlashMLAMetadata,
|
||||
FlashMLAMegaAttnBackend,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.deep_gemm import fp8_einsum, get_tma_aligned_size
|
||||
from vllm.utils.math_utils import round_up
|
||||
from vllm.v1.attention.ops.flashmla import is_flashmla_sparse_supported
|
||||
from vllm.v1.worker.workspace import current_workspace_manager
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata
|
||||
|
||||
_HEAD_DIM_V = 512
|
||||
_ROPE_DIM = 64
|
||||
# The kernel always emits per-32 scales, whatever num_per_channels says.
|
||||
_QUANT_GROUP = 32
|
||||
# Fixed kernel conventions: GPT-J (non-neox) RoPE, TMA-aligned col-major
|
||||
# scales, round-to-nearest, packed ue8m0 -- i.e. what DeepGEMM's fp8_einsum
|
||||
# consumes. Q norm is off: the V4.1 layer norms qr before wq_b.
|
||||
_ROPE_ARGS = (False, _ROPE_DIM)
|
||||
_SF_ARGS = (_QUANT_GROUP, True, True, True)
|
||||
|
||||
# Decode always goes through the mega kernel: there is no batch size at which
|
||||
# falling back to the split-KV decode pays, so there is no minimum-token
|
||||
# threshold and no fallback path to maintain. What decides whether this layer
|
||||
# is worth selecting at all is TP, not the batch size.
|
||||
#
|
||||
# Measured on one GB300 with benchmarks/kernels/benchmark_dsv41_mega_attn.py,
|
||||
# CUDA graphs on, counting every per-step op each path runs (64-head model,
|
||||
# topk_swa=128, topk_extra=512, NVFP4 compressed cache), as mega / split-KV
|
||||
# microseconds:
|
||||
#
|
||||
# s_q 1 32 128 256 512
|
||||
# TP1 (64 live heads) 29/29 31/31 31/40 53/72 96/140
|
||||
# TP2 (32 live) 29/29 31/30 35/35 61/63 110/119
|
||||
# TP4 (16 live) 29/29 31/29 34/34 59/58 109/109
|
||||
# TP8 (8 live) 29/29 31/29 33/33 59/57 107/104
|
||||
#
|
||||
# The kernel absorbs the inverse RoPE and the FP8 cast, whose cost scales with
|
||||
# *live* heads, while its own cost scales with *padded* heads -- always 64,
|
||||
# since FlashMLA's FP8 decode takes only h_q in {64, 128}. So it wins at TP1
|
||||
# (up to 1.45x, and TP1 is also where the fused insert skips the Q pad
|
||||
# entirely) and from TP2 down the two cancel to within a few percent, with a
|
||||
# fixed ~2 us penalty at small s_q where the padded-head cost dominates.
|
||||
# Measure before assuming a win at high TP -- and measure under CUDA graphs:
|
||||
# in eager mode launch overhead swamps both paths and overstates the fused one
|
||||
# several-fold.
|
||||
|
||||
|
||||
def is_flashmla_mega_attn_supported() -> tuple[bool, str | None]:
|
||||
"""Whether FlashMLA's fused mega-attention kernels are usable here."""
|
||||
is_available, maybe_reason = is_flashmla_sparse_supported()
|
||||
if not is_available:
|
||||
return False, maybe_reason
|
||||
if not current_platform.is_device_capability_family(100):
|
||||
return False, "FlashMLA mega attention requires sm_10x GPUs."
|
||||
if not hasattr(torch.ops._flashmla_C, "fused_norm_rope_attn_rope_cast_decode"):
|
||||
return False, "vllm._flashmla_C was built without the mega attention ops."
|
||||
if not hasattr(torch.ops._flashmla_C, "permute_q_b_proj"):
|
||||
return False, "vllm._flashmla_C predates the mega attention weight permutes."
|
||||
return True, None
|
||||
|
||||
|
||||
def alloc_mega_attn_output(
|
||||
num_tokens: int,
|
||||
n_wv_group: int,
|
||||
device: torch.device,
|
||||
) -> QuantizedActivation:
|
||||
"""Allocate the output buffer pair the mega-attention kernels write into.
|
||||
|
||||
One pair per forward step: the prefill and decode segments fill disjoint
|
||||
token ranges, so a single ``fp8_einsum`` over all ``num_tokens`` can
|
||||
consume the result instead of one call per segment.
|
||||
|
||||
The scale buffer is MN-major -- stride 1 along tokens, head-dim stride
|
||||
``ceil4(num_tokens)`` -- which is both what the kernel requires of a
|
||||
caller-provided buffer and what DeepGEMM expects for this N.
|
||||
"""
|
||||
d = WV_GROUP_SIZE * _HEAD_DIM_V
|
||||
data = torch.empty(
|
||||
(num_tokens, n_wv_group, d), dtype=torch.float8_e4m3fn, device=device
|
||||
)
|
||||
aligned = get_tma_aligned_size(num_tokens, torch.int32.itemsize)
|
||||
scale = torch.empty(
|
||||
(n_wv_group, d // (_QUANT_GROUP * 4), aligned),
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
).permute(2, 0, 1)[:num_tokens]
|
||||
return QuantizedActivation(
|
||||
data=data,
|
||||
scale=scale,
|
||||
orig_dtype=torch.bfloat16,
|
||||
orig_shape=data.shape,
|
||||
quant_key=kMxfp8Dynamic,
|
||||
)
|
||||
|
||||
|
||||
def _token_slice(out: QuantizedActivation, start: int, end: int) -> QuantizedActivation:
|
||||
"""The ``[start, end)`` token slice of a mega-attention output buffer."""
|
||||
data = out.data[start:end]
|
||||
return replace(out, data=data, scale=out.scale[start:end], orig_shape=data.shape)
|
||||
|
||||
|
||||
class DeepseekV4MegaAttnAttention(DeepseekV4FlashMLAAttention):
|
||||
"""FlashMLA mega-attention layer for DeepSeek V4.1 (SM100)."""
|
||||
|
||||
backend_cls = FlashMLAMegaAttnBackend
|
||||
|
||||
@classmethod
|
||||
def is_available_for(cls, vllm_config: VllmConfig) -> bool:
|
||||
"""Whether this layer can serve the configured model on this device.
|
||||
|
||||
The kernel is SM100-only and wants ``WV_GROUP_SIZE`` heads per ``wo_a``
|
||||
group, which stops holding once TP divides the head count far enough
|
||||
(TP16 on a 64-head model leaves 4). ``__init__`` raises on both, so the
|
||||
default-backend selector asks here first rather than turning an
|
||||
unsupported topology into a startup crash.
|
||||
"""
|
||||
if not is_flashmla_mega_attn_supported()[0]:
|
||||
return False
|
||||
config = vllm_config.model_config.hf_config
|
||||
config = getattr(config, "text_config", config)
|
||||
n_heads = getattr(config, "num_attention_heads", 0) or 0
|
||||
n_groups = getattr(config, "o_groups", 0) or 0
|
||||
tp_size = vllm_config.parallel_config.tensor_parallel_size
|
||||
if n_heads % tp_size or n_groups % tp_size:
|
||||
return False
|
||||
n_local_heads, n_local_groups = n_heads // tp_size, n_groups // tp_size
|
||||
if not n_local_groups or n_local_heads % WV_GROUP_SIZE:
|
||||
return False
|
||||
return n_local_heads // n_local_groups == WV_GROUP_SIZE
|
||||
|
||||
def __init__(self, vllm_config: VllmConfig, *args, **kwargs) -> None:
|
||||
super().__init__(vllm_config, *args, **kwargs)
|
||||
if self.n_local_heads % WV_GROUP_SIZE:
|
||||
raise ValueError(
|
||||
f"{self.prefix}: mega attention needs the local head count "
|
||||
f"({self.n_local_heads}) to be a multiple of {WV_GROUP_SIZE}."
|
||||
)
|
||||
if self.n_local_heads // self.n_local_groups != WV_GROUP_SIZE:
|
||||
raise ValueError(
|
||||
f"{self.prefix}: mega attention needs {WV_GROUP_SIZE} heads per "
|
||||
"wo_a group."
|
||||
)
|
||||
self.n_wv_group = self.padded_heads // WV_GROUP_SIZE
|
||||
self._fused_layouts_ready = False
|
||||
|
||||
# ---- interface contract ------------------------------------------------
|
||||
|
||||
@property
|
||||
def accepts_unnormed_unroped_query(self) -> bool:
|
||||
return True
|
||||
|
||||
@property
|
||||
def packed_kv_cache_dtype(self) -> CacheDType:
|
||||
# This kernel is the only one that reads an NVFP4 compressed cache, so
|
||||
# it is what an unspecific --kv-cache-dtype resolves to here.
|
||||
return "nvfp4_ds_mla"
|
||||
|
||||
def _alloc_attn_out(
|
||||
self, num_tokens: int, hidden_states: torch.Tensor
|
||||
) -> QuantizedActivation:
|
||||
return alloc_mega_attn_output(num_tokens, self.n_wv_group, hidden_states.device)
|
||||
|
||||
def _o_proj(
|
||||
self, attn_out: QuantizedActivation, positions: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
"""Grouped ``wo_a`` then ``wo_b`` over the kernel's quantized output.
|
||||
|
||||
The inverse RoPE and the FP8 cast happened inside the attention
|
||||
kernel, and ``wo_a`` is permuted for its output layout, so there is
|
||||
nothing to rotate or quantize here and ``positions`` is unused. The
|
||||
scale already is DeepGEMM's packed-ue8m0 MN-major layout, so the
|
||||
einsum consumes the kernel's output with no repacking.
|
||||
"""
|
||||
groups = self.n_local_groups
|
||||
z = torch.empty(
|
||||
(attn_out.data.shape[0], groups, self.o_lora_rank),
|
||||
dtype=torch.bfloat16,
|
||||
device=attn_out.data.device,
|
||||
)
|
||||
fp8_einsum(
|
||||
"bhr,hdr->bhd",
|
||||
(attn_out.data[:, :groups], attn_out.scale[:, :groups]),
|
||||
(self.wo_a.weight, self.wo_a.weight_scale),
|
||||
z,
|
||||
recipe=self._einsum_recipe,
|
||||
)
|
||||
return self.wo_b(z.flatten(1))
|
||||
|
||||
# ---- weights -----------------------------------------------------------
|
||||
|
||||
def finalize_loaded_weights(self) -> None:
|
||||
"""Permute wq_b rows / wo_a columns into the kernel's layouts.
|
||||
|
||||
Idempotent: a second post-load pass must not permute twice.
|
||||
"""
|
||||
if self._fused_layouts_ready:
|
||||
return
|
||||
permute_wq_b_(
|
||||
self.wq_b.weight.data, self.wq_b.weight_scale.data, self.n_local_heads
|
||||
)
|
||||
permute_wo_a_(
|
||||
self.wo_a.weight.data,
|
||||
self.wo_a.weight_scale.data,
|
||||
self.n_local_heads // self.n_local_groups,
|
||||
)
|
||||
self._fused_layouts_ready = True
|
||||
|
||||
# ---- forward -----------------------------------------------------------
|
||||
|
||||
def forward_mqa(
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
kv: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
output: QuantizedActivation,
|
||||
) -> None:
|
||||
if not self._fused_layouts_ready:
|
||||
raise RuntimeError(
|
||||
f"{self.prefix}: wq_b / wo_a were never permuted for the mega "
|
||||
"attention kernel; refusing to run with mismatched layouts."
|
||||
)
|
||||
attn_metadata = get_forward_context().attn_metadata
|
||||
if attn_metadata is None:
|
||||
# Warmup dummy run: reserve the prefill workspace, produce zeros.
|
||||
self._reserve_prefill_workspace(q)
|
||||
output.data.zero_()
|
||||
output.scale.zero_()
|
||||
return
|
||||
|
||||
assert isinstance(attn_metadata, dict)
|
||||
# q was zero-padded to the kernel's head count by the fused KV insert.
|
||||
assert q.shape[1] == self.padded_heads
|
||||
flashmla_metadata = cast(
|
||||
DeepseekV4FlashMLAMetadata | None,
|
||||
attn_metadata.get(self.compressed_cache_prefix)
|
||||
if self.compressed_cache_prefix is not None
|
||||
else None,
|
||||
)
|
||||
swa_metadata = cast(
|
||||
"DeepseekSparseSWAMetadata", attn_metadata[self.swa_cache_layer.prefix]
|
||||
)
|
||||
# The kernel takes int32 RoPE positions.
|
||||
positions_int32 = positions.to(torch.int32)
|
||||
num_decode_tokens = swa_metadata.num_decode_tokens
|
||||
|
||||
if swa_metadata.num_prefills > 0:
|
||||
self._forward_prefill_mega(
|
||||
q[num_decode_tokens:],
|
||||
positions_int32[num_decode_tokens:],
|
||||
flashmla_metadata,
|
||||
swa_metadata,
|
||||
output,
|
||||
num_decode_tokens,
|
||||
)
|
||||
if swa_metadata.num_decodes > 0:
|
||||
self._forward_decode_mega(
|
||||
q[:num_decode_tokens],
|
||||
positions_int32[:num_decode_tokens],
|
||||
flashmla_metadata,
|
||||
swa_metadata,
|
||||
_token_slice(output, 0, num_decode_tokens),
|
||||
)
|
||||
|
||||
def _reserve_prefill_workspace(self, q: torch.Tensor) -> None:
|
||||
swa_only = self.compress_ratio == 0
|
||||
n = 0 if swa_only else -(-self.max_model_len // self.compress_ratio)
|
||||
m = n + self.window_size + self.max_num_batched_tokens
|
||||
if swa_only:
|
||||
top_k = 0
|
||||
else:
|
||||
assert self.topk_indices_buffer is not None
|
||||
top_k = self.topk_indices_buffer.shape[-1]
|
||||
combined_topk = round_up(top_k + self.window_size + self.max_image_tokens, 128)
|
||||
current_workspace_manager().get_simultaneous(
|
||||
((self.PREFILL_CHUNK_SIZE, m, q.shape[-1]), torch.bfloat16),
|
||||
((self.max_num_batched_tokens, combined_topk), torch.int32),
|
||||
((self.max_num_batched_tokens,), torch.int32),
|
||||
)
|
||||
|
||||
def _decode_compressed_kv_and_topk(
|
||||
self,
|
||||
flashmla_metadata: DeepseekV4FlashMLAMetadata | None,
|
||||
swa_metadata: "DeepseekSparseSWAMetadata",
|
||||
) -> tuple[torch.Tensor | None, torch.Tensor | None, torch.Tensor | None]:
|
||||
"""The compressed cache and its ``[n, topk]`` slot indices / lengths."""
|
||||
if self.compress_ratio == 0:
|
||||
return None, None, None
|
||||
assert flashmla_metadata is not None
|
||||
assert swa_metadata.is_valid_token is not None
|
||||
assert self.topk_indices_buffer is not None
|
||||
num_decode_tokens = swa_metadata.num_decode_tokens
|
||||
indices, lens = compute_global_topk_indices_and_lens(
|
||||
self.topk_indices_buffer[:num_decode_tokens],
|
||||
swa_metadata.token_to_req_indices,
|
||||
flashmla_metadata.block_table[: swa_metadata.num_decodes],
|
||||
flashmla_metadata.block_size // self.compress_ratio,
|
||||
swa_metadata.is_valid_token[:num_decode_tokens],
|
||||
)
|
||||
return self._compressed_kv_cache().unsqueeze(-2), indices, lens
|
||||
|
||||
def _forward_decode_mega(
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
positions_int32: torch.Tensor,
|
||||
flashmla_metadata: DeepseekV4FlashMLAMetadata | None,
|
||||
swa_metadata: "DeepseekSparseSWAMetadata",
|
||||
out: QuantizedActivation,
|
||||
) -> None:
|
||||
"""Mega attention over the paged quantized caches, in place into ``out``.
|
||||
|
||||
The caches are ``[num_blocks, page, 1, bytes]``, the record taken from
|
||||
``bytes`` (528 V4.1 fp8, 288 V4.1 NVFP4; NVFP4 only as the compressed
|
||||
cache beside an fp8 SWA one), and the indices ``[s_q, topk]`` int32
|
||||
slot ids (``block * page + offset``, ``-1`` invalid).
|
||||
"""
|
||||
extra_cache, extra_idx, extra_len = self._decode_compressed_kv_and_topk(
|
||||
flashmla_metadata, swa_metadata
|
||||
)
|
||||
assert swa_metadata.decode_swa_indices is not None
|
||||
torch.ops._flashmla_C.fused_norm_rope_attn_rope_cast_decode(
|
||||
q,
|
||||
self.swa_cache_layer.kv_cache.unsqueeze(-2),
|
||||
swa_metadata.decode_swa_indices.view(q.shape[0], -1),
|
||||
self.scale,
|
||||
_HEAD_DIM_V,
|
||||
self.attn_sink,
|
||||
swa_metadata.decode_swa_lens,
|
||||
extra_cache,
|
||||
extra_idx,
|
||||
extra_len,
|
||||
False, # enable_q_norm
|
||||
0.0, # rms_norm_eps
|
||||
positions_int32,
|
||||
*_ROPE_ARGS,
|
||||
self.rotary_emb.cos_sin_cache,
|
||||
self.n_wv_group,
|
||||
*_SF_ARGS,
|
||||
out.data,
|
||||
out.scale,
|
||||
)
|
||||
|
||||
def _forward_prefill_mega(
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
positions_int32: torch.Tensor,
|
||||
flashmla_metadata: DeepseekV4FlashMLAMetadata | None,
|
||||
swa_metadata: "DeepseekSparseSWAMetadata",
|
||||
out: QuantizedActivation,
|
||||
token_base: int,
|
||||
) -> None:
|
||||
swa_only = self.compress_ratio == 0
|
||||
num_decodes = swa_metadata.num_decodes
|
||||
num_decode_tokens = swa_metadata.num_decode_tokens
|
||||
seq_lens = swa_metadata.prefill_seq_lens
|
||||
gather_lens = swa_metadata.prefill_gather_lens
|
||||
query_start_loc_cpu = swa_metadata.query_start_loc_cpu
|
||||
query_start_loc = swa_metadata.query_start_loc
|
||||
assert seq_lens is not None and gather_lens is not None
|
||||
assert query_start_loc_cpu is not None and query_start_loc is not None
|
||||
prefill_token_base = query_start_loc_cpu[num_decodes]
|
||||
assert self.topk_indices_buffer is not None
|
||||
topk_indices = self.topk_indices_buffer[num_decode_tokens:][
|
||||
: swa_metadata.num_prefill_tokens
|
||||
]
|
||||
top_k = 0 if swa_only else topk_indices.shape[-1]
|
||||
chunk_plan = swa_metadata.get_prefill_chunk_plan(
|
||||
compress_ratio=self.compress_ratio,
|
||||
prefill_chunk_size=self.PREFILL_CHUNK_SIZE,
|
||||
has_compressed=not swa_only,
|
||||
)
|
||||
assert chunk_plan, "prefill chunk plan must be non-empty when num_prefills > 0"
|
||||
workspace_manager = current_workspace_manager()
|
||||
combined_topk = round_up(top_k + self.window_size + self.max_image_tokens, 128)
|
||||
for chunk_start, chunk_end, chunk_n, chunk_m in chunk_plan:
|
||||
chunk_size = chunk_end - chunk_start
|
||||
kv_ws, idx_ws, lens_ws = workspace_manager.get_simultaneous(
|
||||
((chunk_size, chunk_m, q.shape[-1]), torch.bfloat16),
|
||||
((self.max_num_batched_tokens, combined_topk), torch.int32),
|
||||
((self.max_num_batched_tokens,), torch.int32),
|
||||
)
|
||||
if not swa_only:
|
||||
assert flashmla_metadata is not None
|
||||
dequantize_and_gather_k_cache(
|
||||
kv_ws[:chunk_size],
|
||||
self._compressed_kv_cache(),
|
||||
seq_lens=seq_lens[chunk_start:chunk_end] // self.compress_ratio,
|
||||
gather_lens=None,
|
||||
block_table=flashmla_metadata.block_table[num_decodes:][
|
||||
chunk_start:chunk_end
|
||||
],
|
||||
block_size=flashmla_metadata.block_size // self.compress_ratio,
|
||||
offset=0,
|
||||
)
|
||||
dequantize_and_gather_k_cache(
|
||||
kv_ws[:chunk_size],
|
||||
self.swa_cache_layer.kv_cache,
|
||||
seq_lens=seq_lens[chunk_start:chunk_end],
|
||||
gather_lens=gather_lens[chunk_start:chunk_end],
|
||||
block_table=swa_metadata.block_table[num_decodes:][
|
||||
chunk_start:chunk_end
|
||||
],
|
||||
block_size=swa_metadata.block_size,
|
||||
offset=chunk_n,
|
||||
)
|
||||
first, last = num_decodes + chunk_start, num_decodes + chunk_end
|
||||
qs = int(query_start_loc_cpu[first] - prefill_token_base)
|
||||
qe = int(query_start_loc_cpu[last] - prefill_token_base)
|
||||
combined_indices, combined_lens = combine_topk_swa_indices(
|
||||
topk_indices[qs:qe],
|
||||
query_start_loc[first : last + 1],
|
||||
seq_lens[chunk_start:chunk_end],
|
||||
gather_lens[chunk_start:chunk_end],
|
||||
self.window_size,
|
||||
self.compress_ratio,
|
||||
top_k,
|
||||
chunk_m,
|
||||
chunk_n,
|
||||
out=(idx_ws[: qe - qs], lens_ws[: qe - qs]),
|
||||
left_visible=(
|
||||
swa_metadata.prefill_left_visible[
|
||||
num_decode_tokens + qs : num_decode_tokens + qe
|
||||
]
|
||||
if swa_metadata.prefill_left_visible is not None
|
||||
else None
|
||||
),
|
||||
right_visible=(
|
||||
swa_metadata.prefill_right_visible[
|
||||
num_decode_tokens + qs : num_decode_tokens + qe
|
||||
]
|
||||
if swa_metadata.prefill_right_visible is not None
|
||||
else None
|
||||
),
|
||||
max_image_tokens=self.max_image_tokens,
|
||||
)
|
||||
chunk_out = _token_slice(out, token_base + qs, token_base + qe)
|
||||
# Mega attention over the gathered non-paged bf16 KV (RoPE already
|
||||
# applied), in place into this chunk's token range. `indices`
|
||||
# entries outside [0, s_kv) skip.
|
||||
torch.ops._flashmla_C.fused_norm_rope_attn_rope_cast_fwd(
|
||||
q[qs:qe],
|
||||
kv_ws.view(-1, 1, q.shape[-1]),
|
||||
combined_indices.unsqueeze(1),
|
||||
self.scale,
|
||||
_HEAD_DIM_V,
|
||||
self.attn_sink,
|
||||
combined_lens,
|
||||
False, # enable_q_norm
|
||||
0.0, # rms_norm_eps
|
||||
positions_int32[qs:qe],
|
||||
*_ROPE_ARGS,
|
||||
self.rotary_emb.cos_sin_cache,
|
||||
self.n_wv_group,
|
||||
*_SF_ARGS,
|
||||
chunk_out.data,
|
||||
chunk_out.scale,
|
||||
)
|
||||
@@ -201,7 +201,8 @@ class DeepseekV4FlashInferMLAAttention(DeepseekV4Attention):
|
||||
def get_padded_num_q_heads(cls, num_heads: int) -> int:
|
||||
return _pad_to_supported_q_heads(num_heads)
|
||||
|
||||
def _o_proj(self, o: torch.Tensor, positions: torch.Tensor) -> torch.Tensor:
|
||||
def _o_proj(self, attn_out: torch.Tensor, positions: torch.Tensor) -> torch.Tensor:
|
||||
o = attn_out[:, : self.n_local_heads, :]
|
||||
return deep_gemm_fp8_o_proj(
|
||||
o,
|
||||
positions,
|
||||
@@ -562,7 +563,8 @@ class DeepseekV4FlashInferSM120Attention(DeepseekV4Attention):
|
||||
def get_padded_num_q_heads(cls, num_heads: int) -> int:
|
||||
return _pad_to_supported_q_heads(num_heads)
|
||||
|
||||
def _o_proj(self, o: torch.Tensor, positions: torch.Tensor) -> torch.Tensor:
|
||||
def _o_proj(self, attn_out: torch.Tensor, positions: torch.Tensor) -> torch.Tensor:
|
||||
o = attn_out[:, : self.n_local_heads, :]
|
||||
return deep_gemm_fp8_o_proj(
|
||||
o,
|
||||
positions,
|
||||
|
||||
@@ -58,7 +58,8 @@ class DeepseekV4FlashMLAAttention(DeepseekV4Attention):
|
||||
self._o_proj_block_size
|
||||
)
|
||||
|
||||
def _o_proj(self, o: torch.Tensor, positions: torch.Tensor) -> torch.Tensor:
|
||||
def _o_proj(self, attn_out: torch.Tensor, positions: torch.Tensor) -> torch.Tensor:
|
||||
o = attn_out[:, : self.n_local_heads, :]
|
||||
return deep_gemm_fp8_o_proj(
|
||||
o,
|
||||
positions,
|
||||
|
||||
@@ -64,6 +64,9 @@ from vllm.models.deepseek_v4.nvidia.model import (
|
||||
make_deepseek_v4_expert_params_mapping,
|
||||
)
|
||||
from vllm.models.deepseek_v41.attention import DeepseekV4Attention
|
||||
from vllm.models.deepseek_v41.nvidia.flash_mla_mega_attn import (
|
||||
DeepseekV4MegaAttnAttention,
|
||||
)
|
||||
from vllm.models.deepseek_v41.nvidia.flashinfer_sparse import (
|
||||
DeepseekV4FlashInferMLAAttention,
|
||||
DeepseekV4FlashInferSM120Attention,
|
||||
@@ -119,8 +122,9 @@ def _select_dsv4_attn_cls(vllm_config: VllmConfig) -> type[DeepseekV4Attention]:
|
||||
|
||||
The generic CUDA backend selector does not instantiate DSv4 layers directly,
|
||||
so map generic sparse-MLA choices to the DSv4-specialized attention class.
|
||||
Without an explicit backend, SM12 defaults to FlashInfer while the other
|
||||
CUDA arches keep the FlashMLA path.
|
||||
Without an explicit backend: SM12 takes FlashInfer, SM100 takes mega
|
||||
attention where the topology allows it, and everything else keeps the
|
||||
FlashMLA path.
|
||||
"""
|
||||
backend = vllm_config.attention_config.backend
|
||||
device_capability = current_platform.get_device_capability()
|
||||
@@ -140,6 +144,8 @@ def _select_dsv4_attn_cls(vllm_config: VllmConfig) -> type[DeepseekV4Attention]:
|
||||
if device_capability is not None and device_capability.major == 12:
|
||||
return DeepseekV4FlashInferSM120Attention
|
||||
return DeepseekV4FlashInferMLAAttention
|
||||
if backend is AttentionBackendEnum.FLASHMLA_MEGA_ATTN_DSV41:
|
||||
return DeepseekV4MegaAttnAttention
|
||||
if backend in (
|
||||
AttentionBackendEnum.FLASHMLA_SPARSE,
|
||||
AttentionBackendEnum.FLASHMLA_SPARSE_DSV4,
|
||||
@@ -149,6 +155,14 @@ def _select_dsv4_attn_cls(vllm_config: VllmConfig) -> type[DeepseekV4Attention]:
|
||||
|
||||
if device_capability is not None and device_capability.major == 12:
|
||||
return DeepseekV4FlashInferSM120Attention
|
||||
# Mega attention is the SM100 default: it fuses Q RoPE, sparse attention,
|
||||
# the output's inverse RoPE and its FP8 cast into one launch, and brings
|
||||
# the 288 B NVFP4 compressed record -- the format the reference
|
||||
# implementation itself stores. It declines topologies it cannot serve
|
||||
# (non-SM100, TP that leaves fewer than WV_GROUP_SIZE heads per wo_a
|
||||
# group, a build without the kernel), which then fall through to FlashMLA.
|
||||
if DeepseekV4MegaAttnAttention.is_available_for(vllm_config):
|
||||
return DeepseekV4MegaAttnAttention
|
||||
return DeepseekV4FlashMLAAttention
|
||||
|
||||
|
||||
@@ -936,6 +950,17 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
for layer in islice(self.layers, self.start_layer, self.end_layer):
|
||||
layer.ffn.finalize_mega_moe_weights()
|
||||
|
||||
def finalize_mega_attn_weights(self) -> None:
|
||||
"""Permute wq_b / wo_a into FlashMLA's mega-attention layouts.
|
||||
|
||||
A no-op for every other attention layer, and idempotent, so a second
|
||||
post-load pass cannot permute twice.
|
||||
"""
|
||||
for layer in islice(self.layers, self.start_layer, self.end_layer):
|
||||
finalize = getattr(layer.attn, "finalize_loaded_weights", None)
|
||||
if finalize is not None:
|
||||
finalize()
|
||||
|
||||
def finalize_mhc_broadcast_weights(self) -> None:
|
||||
if not get_pp_group().is_first_rank or self.start_layer >= self.end_layer:
|
||||
return
|
||||
@@ -1190,6 +1215,7 @@ class DeepseekV41LLMForCausalLM(
|
||||
def process_weights_after_loading(self) -> None:
|
||||
self.model.finalize_mega_moe_weights()
|
||||
self.model.finalize_mhc_broadcast_weights()
|
||||
self.model.finalize_mega_attn_weights()
|
||||
|
||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||
return self.model.get_expert_mapping()
|
||||
|
||||
@@ -229,3 +229,54 @@ class DeepseekV4FlashMLABackend(DeepseekV4SparseMLABackend):
|
||||
@staticmethod
|
||||
def get_builder_cls() -> type[DeepseekV4FlashMLAMetadataBuilder]:
|
||||
return DeepseekV4FlashMLAMetadataBuilder
|
||||
|
||||
|
||||
class FlashMLAMegaAttnBackend(DeepseekV4FlashMLABackend):
|
||||
"""FlashMLA's mega-attention kernel: Q RoPE + sparse attention + inverse
|
||||
RoPE + FP8 cast of the output, in one launch.
|
||||
|
||||
Same metadata and KV-cache geometry as ``FLASHMLA_SPARSE_DSV41`` -- what
|
||||
differs is the attention layer's interface contract (it takes an unnormed,
|
||||
unroped Q and returns an already-inverse-RoPE'd, quantized output) and the
|
||||
extra ``nvfp4_ds_mla`` compressed-cache record only this kernel can read.
|
||||
SM100 only; the kernel has no SM90 instantiation.
|
||||
"""
|
||||
|
||||
supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = [
|
||||
"auto",
|
||||
"fp8_ds_mla",
|
||||
"fp8", # alias for fp8_ds_mla
|
||||
"nvfp4_ds_mla", # V4.1 fp8 SWA cache + NVFP4 compressed cache
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def get_name() -> str:
|
||||
return "FLASHMLA_MEGA_ATTN_DSV41"
|
||||
|
||||
@staticmethod
|
||||
def get_supported_kernel_block_sizes() -> list[int | MultipleOf]:
|
||||
return [128]
|
||||
|
||||
@classmethod
|
||||
def supports_compute_capability(cls, capability: DeviceCapability) -> bool:
|
||||
return capability.major == 10
|
||||
|
||||
@classmethod
|
||||
def supports_combination(
|
||||
cls,
|
||||
head_size: int,
|
||||
dtype: torch.dtype,
|
||||
kv_cache_dtype: CacheDType | None,
|
||||
block_size: int | None,
|
||||
use_mla: bool,
|
||||
has_sink: bool,
|
||||
use_sparse: bool,
|
||||
use_mm_prefix: bool,
|
||||
device_capability: DeviceCapability,
|
||||
) -> str | None:
|
||||
# Imported here: the layer module imports this backend class.
|
||||
from vllm.models.deepseek_v41.nvidia.flash_mla_mega_attn import (
|
||||
is_flashmla_mega_attn_supported,
|
||||
)
|
||||
|
||||
return is_flashmla_mega_attn_supported()[1]
|
||||
|
||||
@@ -112,7 +112,10 @@ class DeepseekV4SWACache(torch.nn.Module, AttentionLayerBase):
|
||||
# fp8_ds_mla's UE8M0 paged layout rounds its page up to the decode
|
||||
# kernel's TMA stride; contiguous bf16/fp8 cache uses the natural
|
||||
# element-size page.
|
||||
uses_fp8_ds_mla_layout = self.cache_config.cache_dtype == "fp8_ds_mla"
|
||||
uses_fp8_ds_mla_layout = self.cache_config.cache_dtype in (
|
||||
"fp8_ds_mla",
|
||||
"nvfp4_ds_mla",
|
||||
)
|
||||
return SlidingWindowMLASpec(
|
||||
block_size=self.block_size,
|
||||
num_kv_heads=1,
|
||||
|
||||
@@ -114,6 +114,9 @@ class AttentionBackendEnum(Enum, metaclass=_AttentionBackendEnumMeta):
|
||||
"vllm.models.deepseek_v41.nvidia.flashinfer_sparse."
|
||||
"DeepseekV4FlashInferMLASparseBackend"
|
||||
)
|
||||
FLASHMLA_MEGA_ATTN_DSV41 = (
|
||||
"vllm.models.deepseek_v41.sparse_mla.FlashMLAMegaAttnBackend"
|
||||
)
|
||||
B12X = "vllm.v1.attention.backends.b12x.B12xPagedAttentionBackend"
|
||||
FLASH_ATTN_MLA = "vllm.v1.attention.backends.mla.flashattn_mla.FlashAttnMLABackend"
|
||||
FLASH_ATTN_MLA_SPARSE = (
|
||||
|
||||
Reference in New Issue
Block a user