diff --git a/benchmarks/kernels/benchmark_dsv41_mega_attn.py b/benchmarks/kernels/benchmark_dsv41_mega_attn.py new file mode 100644 index 0000000000..1b3b5d9c38 --- /dev/null +++ b/benchmarks/kernels/benchmark_dsv41_mega_attn.py @@ -0,0 +1,272 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""FlashMLA mega attention vs the split-KV decode pipeline (DeepSeek V4.1). + +Times the full set of per-layer ops each path runs in a decode step, captured +in a CUDA graph -- eager timing is dominated by launch overhead and flatters +whichever path launches fewer kernels. + + mega: fused_qnorm_rope_kv_rope_quant_insert (Q pad + KV insert) + + mega kernel + split-KV: fused_qnorm_rope_kv_rope_quant_insert (Q RoPE + pad + KV insert) + + flash_mla_with_kvcache + fused_inv_rope_fp8_quant + +Both arms fuse their Q preparation into the KV insert, so that op is charged +to both; the mega arm's Q side is only a zero-pad (nothing at all when the +shard is already at the kernel head count) because its kernel does the Q RoPE, +the inverse RoPE and the FP8 cast. + +``--local-heads`` is the variable that decides the outcome: the mega kernel +absorbs work proportional to the live head count while paying for the padded +one, so it wins at 64 live heads and is a wash at 16 (TP4 on a 64-head model). + +Both arms use the same KV records so the comparison is kernel-for-kernel: +a V4.1 fp8 (528 B) sliding-window cache and a compressed cache that is either +V4.1 fp8 or V4.1 NVFP4 (288 B) -- FlashMLA's SM100 sparse decode reads the +NVFP4 compressed record in both the fused and unfused kernels. + +Run: .venv/bin/python benchmarks/kernels/benchmark_dsv41_mega_attn.py +""" + +import argparse + +import torch + +import vllm.v1.attention.ops.flashmla as fm +from vllm.models.deepseek_v4.common.ops.fused_inv_rope_fp8_quant import ( + fused_inv_rope_fp8_quant, +) +from vllm.models.deepseek_v41.common.ops import quantize_and_insert_k_cache +from vllm.models.deepseek_v41.common.ops.fused_compress_quant_cache import ( + rope_quant_insert, +) +from vllm.models.deepseek_v41.common.ops.fused_layout import permute_q_to_fused +from vllm.models.deepseek_v41.nvidia.flash_mla_mega_attn import ( + alloc_mega_attn_output, + is_flashmla_mega_attn_supported, +) +from vllm.utils.math_utils import round_up + +HEAD_DIM, ROPE_DIM = 512, 64 +V41_BYTES, V41_FP4_BYTES = 528, 288 +SWA_BLOCK, COMPRESSED_BLOCK = 32, 128 + + +def make_cos_sin_cache(max_pos, device): + inv_freq = 1.0 / ( + 10000 ** (torch.arange(0, ROPE_DIM, 2, device=device).float() / ROPE_DIM) + ) + freqs = torch.outer(torch.arange(max_pos, device=device).float(), inv_freq) + return torch.cat([freqs.cos(), freqs.sin()], -1) + + +def empty_paged_cache(num_rows, block_size, bytes_per_token, device): + """A zeroed paged cache view [num_blocks, block_size, bytes_per_token].""" + num_blocks = (num_rows + block_size - 1) // block_size + 1 + page = round_up(block_size * bytes_per_token, 512) + backing = torch.zeros(num_blocks, page, dtype=torch.uint8, device=device) + return backing, backing.as_strided( + (num_blocks, block_size, bytes_per_token), (page, bytes_per_token, 1) + ) + + +def fill_cache(view, bytes_per_token, cos_sin, device): + n = view.shape[0] * view.shape[1] + rows = torch.randn(n, HEAD_DIM, device=device, dtype=torch.bfloat16) + slots = torch.arange(n, dtype=torch.int64, device=device) + if bytes_per_token == V41_FP4_BYTES: + # A page-aligned cache has more slots than the cos_sin table has rows; + # wrap so the insert never indexes past it. + positions = slots % cos_sin.shape[0] + rope_quant_insert(rows, positions, cos_sin, view, slots, 1) + else: + quantize_and_insert_k_cache( + rows, + view.reshape(view.shape[0], -1), + slots, + block_size=view.shape[1], + bytes_per_token=bytes_per_token, + ) + + +def time_graph(fn, iters=100, warmup=5): + """Capture ``fn`` in a CUDA graph and time its replay.""" + s = torch.cuda.Stream() + s.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(s): + for _ in range(3): + fn() + torch.cuda.current_stream().wait_stream(s) + g = torch.cuda.CUDAGraph() + with torch.cuda.graph(g): + fn() + for _ in range(warmup): + g.replay() + torch.accelerator.synchronize() + start = torch.cuda.Event(enable_timing=True) + end = torch.cuda.Event(enable_timing=True) + start.record() + for _ in range(iters): + g.replay() + end.record() + torch.accelerator.synchronize() + return start.elapsed_time(end) * 1000 / iters + + +def bench_decode(s_q, device, topk_extra, extra_bytes, local_heads): + padded_heads, scale = 64, HEAD_DIM**-0.5 + n_wv_group = padded_heads // 8 + cos_sin = make_cos_sin_cache(65536, device) + positions = torch.randint(0, 65536, (s_q,), device=device, dtype=torch.int64) + pos32 = positions.to(torch.int32) + + # Per-step inputs: the wq_b output (local heads) and this step's KV row. + q_local = torch.randn( + s_q, local_heads, HEAD_DIM, device=device, dtype=torch.bfloat16 + ) + q_fused = permute_q_to_fused(q_local) + kv = torch.randn(s_q, HEAD_DIM, device=device, dtype=torch.bfloat16) + + n_swa = s_q * 128 + 128 + swa_backing, swa = empty_paged_cache(n_swa, SWA_BLOCK, V41_BYTES, device) + fill_cache(swa, V41_BYTES, cos_sin, device) + _, extra = empty_paged_cache(65536, COMPRESSED_BLOCK, extra_bytes, device) + fill_cache(extra, extra_bytes, cos_sin, device) + + slot_mapping = torch.arange(s_q, dtype=torch.int64, device=device) + swa_idx = torch.randint(0, n_swa, (s_q, 128), device=device, dtype=torch.int32) + swa_len = torch.full((s_q,), 128, device=device, dtype=torch.int32) + ex_idx = torch.randint( + 0, 65536, (s_q, topk_extra), device=device, dtype=torch.int32 + ) + ex_len = torch.full((s_q,), topk_extra, device=device, dtype=torch.int32) + sink = torch.zeros(padded_heads, device=device) + sched = fm.FlashMLASchedMeta() + out = alloc_mega_attn_output(s_q, n_wv_group, device) + swa4 = swa.unsqueeze(-2) + extra4 = extra.unsqueeze(-2) + swa_2d = swa_backing + + def mega(): + # One launch zero-pads the fused-layout Q and inserts the SWA KV. + q_pad = torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert( + q_fused, + kv, + swa_2d, + slot_mapping, + positions, + cos_sin, + 0 if padded_heads == local_heads else padded_heads, + 1e-6, + SWA_BLOCK, + False, # apply_q_norm + True, # kv_mxfp8 + False, # apply_q_rope: the mega kernel rotates Q itself + True, # is_q_interleaved + ) + torch.ops._flashmla_C.fused_norm_rope_attn_rope_cast_decode( + q_fused if padded_heads == local_heads else q_pad, + swa4, + swa_idx, + scale, + HEAD_DIM, + sink, + swa_len, + extra4, + ex_idx, + ex_len, + False, + 0.0, + pos32, + False, + 64, + cos_sin, + n_wv_group, + 32, + True, + True, + True, + out.data, + out.scale, + ) + + def split_kv(): + q_pad = torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert( + q_local, + kv, + swa_2d, + slot_mapping, + positions, + cos_sin, + padded_heads, + 1e-6, + SWA_BLOCK, + False, + True, + ) + o = fm.flash_mla_with_kvcache( + q=q_pad.unsqueeze(1), + k_cache=swa4, + block_table=None, + head_dim_v=HEAD_DIM, + tile_scheduler_metadata=sched, + cache_seqlens=None, + is_fp8_kvcache=True, + indices=swa_idx.view(s_q, 1, -1), + topk_length=swa_len, + softmax_scale=scale, + attn_sink=sink, + extra_k_cache=extra4, + extra_indices_in_kvcache=ex_idx.view(s_q, 1, -1), + extra_topk_length=ex_len, + )[0] + fused_inv_rope_fp8_quant( + o.squeeze(1)[:, :local_heads], + positions, + cos_sin, + n_groups=local_heads // 8, + heads_per_group=8, + quant_group_size=32, + tma_aligned_scales=True, + ) + + return time_graph(mega), time_graph(split_kv) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--decode-s-q", + nargs="+", + type=int, + default=[1, 2, 4, 8, 16, 32, 64, 128, 256, 512], + ) + parser.add_argument("--topk-extra", type=int, default=512) + parser.add_argument("--local-heads", type=int, default=16) + parser.add_argument( + "--extra-bytes", + type=int, + choices=[V41_BYTES, V41_FP4_BYTES], + default=V41_BYTES, + help="compressed-cache record: 528 (V4.1 fp8) or 288 (V4.1 NVFP4)", + ) + args = parser.parse_args() + ok, reason = is_flashmla_mega_attn_supported() + if not ok: + raise SystemExit(reason) + device = torch.device("cuda") + torch.manual_seed(0) + print( + f"decode, cudagraph (topk_swa=128, topk_extra={args.topk_extra}, " + f"local_heads={args.local_heads}, extra={args.extra_bytes}B)" + ) + print(f"{'s_q':>6} {'mega_us':>9} {'split_kv_us':>12} {'speedup':>8}") + for s_q in args.decode_s_q: + m, s = bench_decode( + s_q, device, args.topk_extra, args.extra_bytes, args.local_heads + ) + print(f"{s_q:>6d} {m:9.1f} {s:12.1f} {s / m:7.2f}x") + + +if __name__ == "__main__": + main() diff --git a/csrc/libtorch_stable/fused_deepseek_v4_qnorm_rope_kv_insert_kernel.cu b/csrc/libtorch_stable/fused_deepseek_v4_qnorm_rope_kv_insert_kernel.cu index 344b82fbda..5a5caacce7 100644 --- a/csrc/libtorch_stable/fused_deepseek_v4_qnorm_rope_kv_insert_kernel.cu +++ b/csrc/libtorch_stable/fused_deepseek_v4_qnorm_rope_kv_insert_kernel.cu @@ -3,7 +3,10 @@ * SPDX-FileCopyrightText: Copyright contributors to the vLLM project * * Horizontally-fused DeepseekV4-MLA kernel: - * - Q side: optional per-head RMSNorm + GPT-J RoPE on last ROPE_DIM + * - Q side: optional per-head RMSNorm (apply_q_norm) and GPT-J RoPE on the + * last ROPE_DIM (apply_q_rope), plus the zero-pad to + * q_head_padded, in the head-major layout or (is_q_interleaved) + * FlashMLA's mega-attention chunk-interleaved one * - KV side: GPT-J RoPE on last ROPE_DIM + UE8M0 FP8 quant on NoPE + paged * cache insert * @@ -174,6 +177,30 @@ __device__ __forceinline__ float warpSum(float val) { return val; } +// Offset of the 16 elements a lane owns inside a Q tensor of `num_heads` +// heads. The default layout is head-major: a head's 512 dims are contiguous. +// `Q_INTERLEAVED` is FlashMLA's mega-attention layout, 32 chunks of +// [num_heads, 16] per token, which changes the address only -- lane L still +// owns dims [16L, 16L+16) of its head either way, so `dim_base` stays the +// true dim offset and the norm and RoPE below are equally valid in both. +// What it does change is where the padding heads land: at the tail of every +// chunk rather than after the live heads. +template +__device__ __forceinline__ int64_t qElemOffset(int const tokenIdx, + int const slotIdx, + int const laneId, + int const dim_base, + int const num_heads) { + if constexpr (Q_INTERLEAVED) { + return static_cast(tokenIdx) * num_heads * kHeadDim + + static_cast(laneId) * num_heads * kElemsPerLane + + static_cast(slotIdx) * kElemsPerLane; + } else { + return (static_cast(tokenIdx) * num_heads + slotIdx) * kHeadDim + + dim_base; + } +} + // ──────────────────────────────────────────────────────────────────────────── // Per-slot inner pipeline // ──────────────────────────────────────────────────────────────────────────── @@ -192,10 +219,16 @@ __device__ __forceinline__ float warpSum(float val) { // + paged-cache // insert) // -// `kv_mxfp8` is grid-uniform and only steers the KV branch, so it stays a -// runtime argument rather than doubling every (head count, q-norm) -// instantiation of a kernel whose hot path is the Q branch. -template +// `kv_mxfp8` and `apply_q_rope` are grid-uniform, so they stay runtime +// arguments rather than multiplying every (head count, q-norm, q-layout) +// instantiation of a kernel whose hot path is the Q branch. `Q_INTERLEAVED` +// cannot: it picks the address arithmetic, which has to fold. +// +// The three Q knobs are independent. The mega-attention layer uses +// Q_INTERLEAVED with both norm and RoPE off, because its attention kernel +// applies those itself -- not because the layout forbids them. +template __device__ __forceinline__ void processDeepseekV4Slot( uint4 v0, uint4 v1, int const tokenIdx, int const slotIdx, int const dim_base, int const laneId, int const num_heads_q, @@ -203,7 +236,7 @@ __device__ __forceinline__ void processDeepseekV4Slot( uint8_t* __restrict__ k_cache, int64_t const* __restrict__ slot_mapping, int64_t const* __restrict__ position_ids, float const* __restrict__ cos_sin_cache, int const cache_block_size, - int const kv_block_stride, bool const kv_mxfp8) { + int const kv_block_stride, bool const kv_mxfp8, bool const apply_q_rope) { using Converter = vllm::_typeConvert; bool const isKV = (slotIdx == kNumHeadsQPadded); bool const isPadQ = !isKV && (slotIdx >= num_heads_q); @@ -213,10 +246,8 @@ __device__ __forceinline__ void processDeepseekV4Slot( // zero literal is correct. Matches the live-Q branch's vectorized store. if (isPadQ) { scalar_t_in* dst = - q_out + - (static_cast(tokenIdx) * kNumHeadsQPadded + slotIdx) * - kHeadDim + - dim_base; + q_out + qElemOffset(tokenIdx, slotIdx, laneId, dim_base, + kNumHeadsQPadded); uint4 const zero4 = {0u, 0u, 0u, 0u}; *reinterpret_cast(dst) = zero4; *reinterpret_cast(dst + 8) = zero4; @@ -264,7 +295,9 @@ __device__ __forceinline__ void processDeepseekV4Slot( // ── GPT-J RoPE on dims [NOPE_DIM, HEAD_DIM) ───────────────────────────── // All math in fp32. cos_sin_cache is loaded as fp32 (its native storage). - bool const is_rope_lane = dim_base >= kNopeDim; + // KV is always rotated; Q only when the caller has not already had it + // rotated elsewhere (the mega-attention kernel does its own). + bool const is_rope_lane = dim_base >= kNopeDim && (isKV || apply_q_rope); if (is_rope_lane) { int64_t const pos = position_ids[tokenIdx]; constexpr int kHalfRope = kRopeDim / 2; @@ -314,10 +347,8 @@ __device__ __forceinline__ void processDeepseekV4Slot( make_float2(elements[8 + 2 * i], elements[8 + 2 * i + 1])); } scalar_t_in* dst = - q_out + - (static_cast(tokenIdx) * kNumHeadsQPadded + slotIdx) * - kHeadDim + - dim_base; + q_out + qElemOffset(tokenIdx, slotIdx, laneId, dim_base, + kNumHeadsQPadded); *reinterpret_cast(dst) = out0; *reinterpret_cast(dst + 8) = out1; } else { @@ -432,14 +463,17 @@ __device__ __forceinline__ void processDeepseekV4Slot( // `kNumHeadsQPadded` is a template parameter (compile-time constant) so the // divisions in the grid math and the KV-sentinel comparison fold to fast // constant operations. The launch wrapper dispatches the runtime value to -// the matching instantiation. +// the matching instantiation. `kNumHeadsQPadded == 0` leaves one slot per +// token and is the KV-insert-only instantiation, for a Q that is already in +// the shape its attention kernel wants. // // With DP padding, q/kv/position_ids can have more rows than slot_mapping. // The live-Q and pad-Q branches cover all `num_tokens_full` rows (downstream // attention uses them). The KV branch only inserts the first // `num_tokens_insert` tokens (= slot_mapping length) into the paged cache. // -template +template __global__ void fusedDeepseekV4QNormRopeKVRopeQuantInsertKernel( scalar_t_in const* __restrict__ q_in, // [N, num_heads_q, 512] scalar_t_in* __restrict__ q_out, // [N, kNumHeadsQPadded, 512] @@ -454,7 +488,8 @@ __global__ void fusedDeepseekV4QNormRopeKVRopeQuantInsertKernel( int const num_heads_q, // live Q heads (input layout) int const cache_block_size, // tokens per paged-cache block int const kv_block_stride, // bytes per paged-cache block - bool const kv_mxfp8) { // V4.1 all-dims MXFP8 record + bool const kv_mxfp8, // V4.1 all-dims MXFP8 record + bool const apply_q_rope) { // rotate Q too, not just KV #if (!defined(__CUDA_ARCH__) || __CUDA_ARCH__ < 800) && !defined(USE_ROCM) // BF16 _typeConvert specialization is unavailable on pre-Ampere. The // DeepseekV4 kernel only runs with bf16 inputs in practice, so compile a @@ -497,20 +532,18 @@ __global__ void fusedDeepseekV4QNormRopeKVRopeQuantInsertKernel( if (isKV) { src_ptr = kv_in + static_cast(tokenIdx) * kHeadDim + dim_base; } else { - int64_t const q_row_offset = - (static_cast(tokenIdx) * num_heads_q + slotIdx) * - kHeadDim + - dim_base; - src_ptr = q_in + q_row_offset; + src_ptr = q_in + qElemOffset(tokenIdx, slotIdx, laneId, + dim_base, num_heads_q); } v0 = *reinterpret_cast(src_ptr); v1 = *reinterpret_cast(src_ptr + 8); } - processDeepseekV4Slot( + processDeepseekV4Slot( v0, v1, tokenIdx, slotIdx, dim_base, laneId, num_heads_q, eps, q_out, k_cache, slot_mapping, position_ids, cos_sin_cache, cache_block_size, - kv_block_stride, kv_mxfp8); + kv_block_stride, kv_mxfp8, apply_q_rope); #if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900) cudaTriggerProgrammaticLaunchCompletion(); @@ -530,7 +563,8 @@ __global__ void fusedDeepseekV4QNormRopeKVRopeQuantInsertKernel( // Q branch (optional RMSNorm + RoPE, in place) head_slot == num_heads_q // KV branch (RoPE + UE8M0 quant + insert) // -template +template __global__ void fusedDeepseekV4QNormRopeKVRopeQuantInsertKernelReducedGrid( scalar_t_in const* __restrict__ q_in, scalar_t_in* __restrict__ q_out, scalar_t_in const* __restrict__ kv_in, uint8_t* __restrict__ k_cache, @@ -539,7 +573,7 @@ __global__ void fusedDeepseekV4QNormRopeKVRopeQuantInsertKernelReducedGrid( float const* __restrict__ cos_sin_cache, float const eps, int const num_tokens_full, int const num_tokens_insert, int const num_heads_q, int const cache_block_size, - int const kv_block_stride, bool const kv_mxfp8) { + int const kv_block_stride, bool const kv_mxfp8, bool const apply_q_rope) { #if (!defined(__CUDA_ARCH__) || __CUDA_ARCH__ < 800) && !defined(USE_ROCM) if constexpr (std::is_same_v) { return; @@ -569,11 +603,8 @@ __global__ void fusedDeepseekV4QNormRopeKVRopeQuantInsertKernelReducedGrid( if (s == kNumHeadsQPadded) { src = kv_in + static_cast(tokenIdx) * kHeadDim + dim_base; } else { - src = q_in + - (static_cast(tokenIdx) * num_heads_q + - static_cast(s)) * - kHeadDim + - dim_base; + src = q_in + qElemOffset(tokenIdx, s, laneId, dim_base, + num_heads_q); } va = *reinterpret_cast(src); vb = *reinterpret_cast(src + 8); @@ -594,10 +625,12 @@ __global__ void fusedDeepseekV4QNormRopeKVRopeQuantInsertKernelReducedGrid( load_slot(next_slot, v0_next, v1_next); } - processDeepseekV4Slot( + processDeepseekV4Slot( v0_curr, v1_curr, tokenIdx, curr_slot, dim_base, laneId, num_heads_q, eps, q_out, k_cache, slot_mapping, position_ids, - cos_sin_cache, cache_block_size, kv_block_stride, kv_mxfp8); + cos_sin_cache, cache_block_size, kv_block_stride, kv_mxfp8, + apply_q_rope); // ── Buffer rotation: hand the prefetched LDGs to the next iter. v0_curr = v0_next; @@ -617,14 +650,15 @@ __global__ void fusedDeepseekV4QNormRopeKVRopeQuantInsertKernelReducedGrid( // ──────────────────────────────────────────────────────────────────────────── // Launch wrapper // ──────────────────────────────────────────────────────────────────────────── -template +template static void launchFusedDeepseekV4Templated( scalar_t_in const* q_in, scalar_t_in* q_out, scalar_t_in const* kv_in, uint8_t* k_cache, int64_t const* slot_mapping, int64_t const* position_ids, float const* cos_sin_cache, float const eps, int const num_tokens_full, int const num_tokens_insert, int const num_heads_q, int const cache_block_size, int const kv_block_stride, bool const kv_mxfp8, - cudaStream_t stream) { + bool const apply_q_rope, cudaStream_t stream) { constexpr int kBlockSize = 256; constexpr int kWarpsPerBlock = kBlockSize / 32; int64_t const total_warps = @@ -663,37 +697,40 @@ static void launchFusedDeepseekV4Templated( // grid instead. Only reachable above NUM_TOKEN_CUTOFF tokens in a single // insert, where it is worth ~2x. if (kNumHeadsQPadded == 0 || num_tokens_full < NUM_TOKEN_CUTOFF) { - cudaLaunchKernelEx(&config, - fusedDeepseekV4QNormRopeKVRopeQuantInsertKernel< - scalar_t_in, kNumHeadsQPadded, APPLY_Q_NORM>, - q_in, q_out, kv_in, k_cache, slot_mapping, position_ids, - cos_sin_cache, eps, num_tokens_full, num_tokens_insert, - num_heads_q, cache_block_size, kv_block_stride, - kv_mxfp8); + cudaLaunchKernelEx( + &config, + fusedDeepseekV4QNormRopeKVRopeQuantInsertKernel< + scalar_t_in, kNumHeadsQPadded, APPLY_Q_NORM, Q_INTERLEAVED>, + q_in, q_out, kv_in, k_cache, slot_mapping, position_ids, cos_sin_cache, + eps, num_tokens_full, num_tokens_insert, num_heads_q, cache_block_size, + kv_block_stride, kv_mxfp8, apply_q_rope); } else { config.gridDim = dim3(num_tokens_full); cudaLaunchKernelEx( &config, fusedDeepseekV4QNormRopeKVRopeQuantInsertKernelReducedGrid< - scalar_t_in, kNumHeadsQPadded, APPLY_Q_NORM>, + scalar_t_in, kNumHeadsQPadded, APPLY_Q_NORM, Q_INTERLEAVED>, q_in, q_out, kv_in, k_cache, slot_mapping, position_ids, cos_sin_cache, eps, num_tokens_full, num_tokens_insert, num_heads_q, cache_block_size, - kv_block_stride, kv_mxfp8); + kv_block_stride, kv_mxfp8, apply_q_rope); } #else // ROCm: use standard kernel launch syntax (no PDL/stream serialization) // clang-format off fusedDeepseekV4QNormRopeKVRopeQuantInsertKernel< - scalar_t_in, kNumHeadsQPadded, APPLY_Q_NORM> + scalar_t_in, kNumHeadsQPadded, APPLY_Q_NORM, Q_INTERLEAVED> <<>>( q_in, q_out, kv_in, k_cache, slot_mapping, position_ids, cos_sin_cache, eps, num_tokens_full, num_tokens_insert, num_heads_q, - cache_block_size, kv_block_stride, kv_mxfp8); + cache_block_size, kv_block_stride, kv_mxfp8, apply_q_rope); #endif } // Runtime dispatch into one of the precompiled `kNumHeadsQPadded` -// instantiations. Supported padded head counts: 8, 16, 32, 64, 128. +// instantiations. Supported padded head counts: 8, 16, 32, 64, 128, plus 0 +// for a KV-only launch that writes no Q at all. `apply_q_norm` and +// `is_q_interleaved` are template parameters, `apply_q_rope` is not (see +// processDeepseekV4Slot), so each head count carries four instantiations. template void launchFusedDeepseekV4QNormRopeKVRopeQuantInsert( scalar_t_in const* q_in, scalar_t_in* q_out, scalar_t_in const* kv_in, @@ -702,22 +739,32 @@ void launchFusedDeepseekV4QNormRopeKVRopeQuantInsert( int const num_tokens_full, int const num_tokens_insert, int const num_heads_q, int const num_heads_q_padded, int const cache_block_size, int const kv_block_stride, - bool const apply_q_norm, bool const kv_mxfp8, cudaStream_t stream) { -#define DISPATCH(N) \ - case N: \ - if (apply_q_norm) { \ - launchFusedDeepseekV4Templated( \ - q_in, q_out, kv_in, k_cache, slot_mapping, position_ids, \ - cos_sin_cache, eps, num_tokens_full, num_tokens_insert, \ - num_heads_q, cache_block_size, kv_block_stride, kv_mxfp8, stream); \ - } else { \ - launchFusedDeepseekV4Templated( \ - q_in, q_out, kv_in, k_cache, slot_mapping, position_ids, \ - cos_sin_cache, eps, num_tokens_full, num_tokens_insert, \ - num_heads_q, cache_block_size, kv_block_stride, kv_mxfp8, stream); \ - } \ + bool const apply_q_norm, bool const kv_mxfp8, bool const apply_q_rope, + bool const is_q_interleaved, cudaStream_t stream) { +#define LAUNCH(N, NORM, INTERLEAVED) \ + launchFusedDeepseekV4Templated( \ + q_in, q_out, kv_in, k_cache, slot_mapping, position_ids, cos_sin_cache, \ + eps, num_tokens_full, num_tokens_insert, num_heads_q, cache_block_size, \ + kv_block_stride, kv_mxfp8, apply_q_rope, stream) +#define DISPATCH(N) \ + case N: \ + if (is_q_interleaved) { \ + if (apply_q_norm) { \ + LAUNCH(N, true, true); \ + } else { \ + LAUNCH(N, false, true); \ + } \ + } else if (apply_q_norm) { \ + LAUNCH(N, true, false); \ + } else { \ + LAUNCH(N, false, false); \ + } \ return; + if (num_heads_q_padded == 0) { + LAUNCH(0, false, false); + return; + } switch (num_heads_q_padded) { DISPATCH(8) DISPATCH(16) @@ -729,9 +776,10 @@ void launchFusedDeepseekV4QNormRopeKVRopeQuantInsert( "fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert: " "unsupported num_heads_q_padded=", num_heads_q_padded, - " (compiled instantiations: 8, 16, 32, 64, 128)."); + " (compiled instantiations: 0, 8, 16, 32, 64, 128)."); } #undef DISPATCH +#undef LAUNCH } // ──────────────────────────────────────────────────────────────────────────── @@ -1059,10 +1107,15 @@ void fused_deepseek_v4_kv_rope_insert( // Zero query heads schedules only the existing KV branch, preserving its // RoPE rounding and quantization without allocating any query tensors. if (packed) { - vllm::deepseek_v4_fused_ops::launchFusedDeepseekV4Templated( + // With zero query heads no Q slot is scheduled, so neither Q knob is + // reachable: Q_INTERLEAVED picks addressing nothing uses, and apply_q_rope + // gates a branch only a Q slot takes (KV is rotated unconditionally). + vllm::deepseek_v4_fused_ops::launchFusedDeepseekV4Templated( nullptr, nullptr, input, cache, slots, positions, cos_sin, 0.0f, num_tokens, num_tokens, 0, static_cast(cache_block_size), - static_cast(k_cache.stride(0)), kv_mxfp8, stream); + static_cast(k_cache.stride(0)), kv_mxfp8, /*apply_q_rope=*/true, + stream); } else if (fp8) { vllm::deepseek_v4_fused_ops::launchFullCacheKernel( nullptr, nullptr, 0, 0, input, cache, slots, positions, cos_sin, @@ -1085,8 +1138,9 @@ torch::stable::Tensor fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert( torch::stable::Tensor const& slot_mapping, // [N] int64 torch::stable::Tensor const& position_ids, // [N] int64 torch::stable::Tensor const& cos_sin_cache, // [max_pos, rope_dim] bf16 - int64_t q_head_padded, // padded Q head count for output - double eps, int64_t cache_block_size, bool apply_q_norm, bool kv_mxfp8) { + int64_t q_head_padded, // padded Q head count for output, 0 for no Q + double eps, int64_t cache_block_size, bool apply_q_norm, bool kv_mxfp8, + bool apply_q_rope, bool is_q_interleaved) { STD_TORCH_CHECK(q_in.device().is_cuda() && q_in.is_contiguous(), "q_in must be contiguous CUDA"); STD_TORCH_CHECK(kv.device().is_cuda() && kv.is_contiguous(), @@ -1106,8 +1160,11 @@ torch::stable::Tensor fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert( STD_TORCH_CHECK(kv.dim() == 2 && kv.size(1) == 512, "kv shape [N, 512]"); STD_TORCH_CHECK(q_in.scalar_type() == kv.scalar_type(), "q_in and kv dtype must match"); - STD_TORCH_CHECK(q_head_padded >= q_in.size(1), - "q_head_padded must be >= q_in.size(1) (num_heads_q)"); + // q_head_padded == 0 asks for the KV insert alone: Q is left untouched and + // an empty tensor comes back, for a caller whose Q is already in the shape + // its attention kernel reads. + STD_TORCH_CHECK(q_head_padded == 0 || q_head_padded >= q_in.size(1), + "q_head_padded must be 0 or >= q_in.size(1) (num_heads_q)"); STD_TORCH_CHECK(k_cache.scalar_type() == torch::headeronly::ScalarType::Byte, "k_cache must be uint8"); STD_TORCH_CHECK(cos_sin_cache.dim() == 2 && cos_sin_cache.size(1) == 64, @@ -1138,8 +1195,12 @@ torch::stable::Tensor fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert( // Allocate the padded q output. The kernel writes every element (live // region gets optional RMSNorm+RoPE; pad region gets zeros), so `empty` is // safe. - auto q_out = torch::stable::new_empty( - q_in, {q_in.size(0), q_head_padded, q_in.size(2)}, q_in.scalar_type()); + auto q_out = + q_head_padded == 0 + ? torch::stable::new_empty(q_in, {int64_t{0}}, q_in.scalar_type()) + : torch::stable::new_empty( + q_in, {q_in.size(0), q_head_padded, q_in.size(2)}, + q_in.scalar_type()); VLLM_STABLE_DISPATCH_HALF_TYPES( q_in.scalar_type(), "fused_deepseek_v4_qnorm_rope_kv_insert", [&] { @@ -1155,7 +1216,8 @@ torch::stable::Tensor fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert( cos_sin_cache.const_data_ptr(), static_cast(eps), num_tokens_full, num_tokens_insert, num_heads_q, num_heads_q_padded, cache_block_size_i, kv_block_stride, - apply_q_norm, kv_mxfp8, stream); + apply_q_norm, kv_mxfp8, apply_q_rope, is_q_interleaved, + stream); }); return q_out; } diff --git a/csrc/libtorch_stable/ops.h b/csrc/libtorch_stable/ops.h index bc08d90a48..6b17815a07 100644 --- a/csrc/libtorch_stable/ops.h +++ b/csrc/libtorch_stable/ops.h @@ -271,7 +271,8 @@ torch::stable::Tensor fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert( torch::stable::Tensor& k_cache, torch::stable::Tensor const& slot_mapping, torch::stable::Tensor const& position_ids, torch::stable::Tensor const& cos_sin_cache, int64_t q_head_padded, - double eps, int64_t cache_block_size, bool apply_q_norm, bool kv_mxfp8); + double eps, int64_t cache_block_size, bool apply_q_norm, bool kv_mxfp8, + bool apply_q_rope, bool is_q_interleaved); void fused_deepseek_v4_qnorm_rope_kv_rope_full_cache_bf16_insert( torch::stable::Tensor& q, torch::stable::Tensor const& kv, diff --git a/csrc/libtorch_stable/torch_bindings.cpp b/csrc/libtorch_stable/torch_bindings.cpp index 3918ab8498..a76852df89 100644 --- a/csrc/libtorch_stable/torch_bindings.cpp +++ b/csrc/libtorch_stable/torch_bindings.cpp @@ -425,6 +425,12 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) { "bool is_neox, Tensor position_ids, " "int forced_token_heads_per_warp=-1) -> ()"); + // q_head_padded is the padded Q head count of the returned tensor, or 0 to + // do the KV insert alone and return an empty tensor. The Q knobs are + // independent: apply_q_norm and apply_q_rope each drop that step for Q + // alone (KV is always rotated), and is_q_interleaved reads and writes Q in + // FlashMLA's mega-attention chunk-interleaved layout, which moves the + // padding heads to the tail of every head-dim chunk. ops.def( "fused_deepseek_v4_kv_rope_insert(" "Tensor kv, Tensor! k_cache, Tensor slot_mapping, Tensor position_ids, " @@ -436,7 +442,8 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) { "Tensor q_in, Tensor kv, Tensor! k_cache, " "Tensor slot_mapping, Tensor position_ids, Tensor cos_sin_cache, " "int q_head_padded, float eps, int cache_block_size, " - "bool apply_q_norm=True, bool kv_mxfp8=False) -> Tensor"); + "bool apply_q_norm=True, bool kv_mxfp8=False, bool apply_q_rope=True, " + "bool is_q_interleaved=False) -> Tensor"); // FlashInfer V4 full-cache variants: write Q in place (bf16) or to a separate // FP8 tensor, and KV into a contiguous 512-wide token-strided cache. diff --git a/tests/kernels/test_compressor_kv_cache.py b/tests/kernels/test_compressor_kv_cache.py index c1de94be92..73c2027a78 100644 --- a/tests/kernels/test_compressor_kv_cache.py +++ b/tests/kernels/test_compressor_kv_cache.py @@ -331,6 +331,19 @@ def test_v41_fused_save_compress_and_insert( torch.testing.assert_close(cache_backing, expected_cache, rtol=0, atol=0) +def _rotate_rope_tail( + latent_row: torch.Tensor, cos_sin_row: torch.Tensor +) -> torch.Tensor: + """GPT-J RoPE of one latent row's last 64 dims, in fp32.""" + row = latent_row.float() + c, s = cos_sin_row.chunk(2) + rotated = row.clone() + even, odd = row[448::2], row[449::2] + rotated[448::2] = even * c - odd * s + rotated[449::2] = odd * c + even * s + return rotated + + def _mxfp8_record_reference( latent_row: torch.Tensor, cos_sin_row: torch.Tensor ) -> tuple[torch.Tensor, torch.Tensor]: @@ -339,12 +352,7 @@ def _mxfp8_record_reference( Mirrors FlashMLA's ``KVCacheLayout.V41_FP8Sparse`` quantizer: rotate the RoPE tail first, then scale every 32-dim tile, RoPE tiles included. """ - row = latent_row.float() - c, s = cos_sin_row.chunk(2) - rotated = row.clone() - even, odd = row[448::2], row[449::2] - rotated[448::2] = even * c - odd * s - rotated[449::2] = odd * c + even * s + rotated = _rotate_rope_tail(latent_row, cos_sin_row) quantized, scales = _ue8m0_reference(rotated, 32, 448.0) return quantized.view(torch.uint8), (scales.log2() + 127).to(torch.uint8) @@ -461,6 +469,125 @@ def test_v41_mxfp8_cache_round_trip(): assert (error <= tolerance.repeat_interleave(32, dim=-1)).all() +# e2m1 magnitudes indexed by the low 3 bits of a code; bit 3 is the sign. +_E2M1_MAGNITUDES = [0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0] + + +def _decode_nvfp4_row(packed: torch.Tensor, scales: torch.Tensor) -> torch.Tensor: + """Unpack one 256-byte e2m1 row and apply its 32 e4m3 tile scales.""" + mags = torch.tensor(_E2M1_MAGNITUDES, device=packed.device) + codes = torch.empty(512, dtype=torch.uint8, device=packed.device) + codes[0::2] = packed & 0xF # even element in the low nibble + codes[1::2] = packed >> 4 + vals = mags[(codes & 7).long()] * torch.where(codes >= 8, -1.0, 1.0) + return ( + vals.view(32, 16) * scales.view(torch.float8_e4m3fn).float().view(32, 1) + ).flatten() + + +@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA only") +@pytest.mark.parametrize("compress_ratio", [1, 2]) +def test_v41_rope_insert_nvfp4_record(compress_ratio: int): + """The 288-byte V4.1 NVFP4 record: 256 B of e2m1 pairs then 32 e4m3 scales. + + Scales are byte-exact against the reference (``amax / 6`` clamped to the + e4m3 range); values are checked as round-to-nearest on the e2m1 grid, whose + coarsest step is 2 between 4 and 6, so half a step is ``1.0 * scale``. + """ + from vllm.models.deepseek_v41.common.ops.fused_compress_quant_cache import ( + rope_quant_insert, + ) + + torch.manual_seed(13) + device = "cuda" + cache_block = 64 + cache_stride = math.ceil(cache_block * 288 / 512) * 512 + num_tokens = 12 + + positions = torch.arange(num_tokens, dtype=torch.int64, device=device) + latent = torch.randn(num_tokens, 512, dtype=torch.bfloat16, device=device) + angles = torch.randn(64, 32, device=device) + cos_sin = torch.cat((angles.cos(), angles.sin()), dim=-1) + + cache_backing = torch.full((2, cache_stride), 165, dtype=torch.uint8, device=device) + cache = cache_backing.as_strided((2, cache_block, 288), (cache_stride, 288, 1)) + slots = torch.arange(num_tokens, dtype=torch.int64, device=device) + slots[3] = -1 # covers the negative-slot skip + + rope_quant_insert(latent, positions, cos_sin, cache, slots, compress_ratio) + + untouched = torch.full_like(cache_backing, 165) + for t in range(num_tokens): + slot = slots[t].item() + pos = positions[t].item() + if slot < 0 or (pos + 1) % compress_ratio: + continue + page, row = divmod(slot, cache_block) + cs = cos_sin[pos // compress_ratio * compress_ratio] + rotated = _rotate_rope_tail(latent[t], cs) + amax = rotated.view(32, 16).abs().amax(-1) + scale = (amax / 6.0).clamp(2.0**-9, 448.0).to(torch.float8_e4m3fn) + + got_scales = cache_backing[page, cache_block * 256 + row * 32 :][:32] + torch.testing.assert_close(got_scales, scale.view(torch.uint8), rtol=0, atol=0) + decoded = _decode_nvfp4_row( + cache_backing[page, row * 256 : (row + 1) * 256], got_scales + ) + step = scale.float().repeat_interleave(16) + assert ((decoded - rotated).abs() <= step).all() + untouched[page, row * 256 : (row + 1) * 256] = cache_backing[ + page, row * 256 : (row + 1) * 256 + ] + untouched[page, cache_block * 256 + row * 32 :][:32] = got_scales + # Rows the kernel must not have touched, page padding included. + torch.testing.assert_close(cache_backing, untouched, rtol=0, atol=0) + + +@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA only") +def test_v41_nvfp4_gather_matches_insert(): + """The NVFP4 gather dequantizes exactly what the insert kernel wrote.""" + from vllm.models.deepseek_v41.common.ops import dequantize_and_gather_k_cache + from vllm.models.deepseek_v41.common.ops.fused_compress_quant_cache import ( + rope_quant_insert, + ) + + torch.manual_seed(17) + device = "cuda" + block_size = 64 + num_tokens = 70 + num_blocks = 4 + page_bytes = math.ceil(block_size * 288 / 512) * 512 + + positions = torch.arange(num_tokens, dtype=torch.int64, device=device) + latent = torch.randn(num_tokens, 512, dtype=torch.bfloat16, device=device) + angles = torch.randn(128, 32, device=device) + cos_sin = torch.cat((angles.cos(), angles.sin()), dim=-1) + backing = torch.zeros(num_blocks, page_bytes, dtype=torch.uint8, device=device) + cache = backing.as_strided((num_blocks, block_size, 288), (page_bytes, 288, 1)) + slots = torch.arange(num_tokens, dtype=torch.int64, device=device) + rope_quant_insert(latent, positions, cos_sin, cache, slots, 1) + + out = torch.zeros(1, num_tokens, 512, dtype=torch.bfloat16, device=device) + dequantize_and_gather_k_cache( + out, + cache, + seq_lens=torch.tensor([num_tokens], dtype=torch.int32, device=device), + gather_lens=None, + block_table=torch.arange(num_blocks, dtype=torch.int32, device=device).view( + 1, -1 + ), + block_size=block_size, + offset=0, + ) + for t in (0, 1, block_size, num_tokens - 1): + page, row = divmod(t, block_size) + expected = _decode_nvfp4_row( + backing[page, row * 256 : (row + 1) * 256], + backing[page, block_size * 256 + row * 32 :][:32], + ) + torch.testing.assert_close(out[0, t].float(), expected, rtol=0, atol=0) + + @pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA only") @pytest.mark.parametrize("compress_ratio", [1, 2]) @pytest.mark.parametrize("store_fp8", [False, True]) diff --git a/tests/kernels/test_dsv41_mega_attn_layouts.py b/tests/kernels/test_dsv41_mega_attn_layouts.py new file mode 100644 index 0000000000..6f9ff56bd2 --- /dev/null +++ b/tests/kernels/test_dsv41_mega_attn_layouts.py @@ -0,0 +1,71 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Layout plumbing for DeepSeek V4.1 mega attention. + +The kernel's Q and O layouts are produced by permuting ``wq_b`` rows and +``wo_a`` columns once at load, so what has to hold is that the permuted GEMMs +agree with a reference that permutes the activation instead. +""" + +import torch + +from vllm.models.deepseek_v41.common.ops.fused_layout import ( + WV_GROUP_SIZE, + o_fused_chunk_permutation, + o_fused_permutation, + permute_q_to_fused, + permute_wo_a_, + permute_wq_b_, + q_fused_permutation, +) + +HEAD_DIM = 512 + + +def test_permuted_wq_b_gemm_matches_permuted_activation(): + """Permuting wq_b's rows makes the Q GEMM emit the fused layout directly.""" + torch.manual_seed(0) + num_heads, q_lora_rank, num_tokens = 16, 64, 5 + weight = torch.randn(num_heads * HEAD_DIM, q_lora_rank) + # One scale row per weight row, as an MXFP8 wq_b shard carries. + scale = torch.randint( + 0, 256, (num_heads * HEAD_DIM, q_lora_rank // 32), dtype=torch.uint8 + ) + qr = torch.randn(num_tokens, q_lora_rank) + orig_weight, orig_scale = weight.clone(), scale.clone() + + standard = (qr @ weight.T).view(num_tokens, num_heads, HEAD_DIM) + expected = permute_q_to_fused(standard) + + permute_wq_b_(weight, scale, num_heads) + fused = (qr @ weight.T).view(num_tokens, num_heads, HEAD_DIM) + torch.testing.assert_close(fused, expected) + # The scale has to follow its row, or dequant reads another row's scale. + perm = q_fused_permutation(num_heads, HEAD_DIM) + torch.testing.assert_close(weight, orig_weight[perm]) + torch.testing.assert_close(scale, orig_scale[perm]) + + +def test_permuted_wo_a_consumes_fused_output(): + """Permuting wo_a's input columns lets it read the kernel's O layout.""" + torch.manual_seed(1) + out_features, num_tokens = 32, 4 + in_features = WV_GROUP_SIZE * HEAD_DIM + weight = torch.randn(out_features, in_features) + scale = torch.randint(0, 256, (out_features, in_features // 32), dtype=torch.uint8) + standard_o = torch.randn(num_tokens, in_features) + orig_weight, orig_scale = weight.clone(), scale.clone() + + expected = standard_o @ weight.T + # The kernel emits the same values in the fused chunk order. + perm = o_fused_permutation(WV_GROUP_SIZE, HEAD_DIM) + fused_o = standard_o[:, perm] + + permute_wo_a_(weight, scale, WV_GROUP_SIZE) + torch.testing.assert_close(weight, orig_weight[:, perm]) + torch.testing.assert_close( + scale, orig_scale[:, o_fused_chunk_permutation(WV_GROUP_SIZE, HEAD_DIM)] + ) + # Unlike wq_b, this permutation sits inside the 4096-term reduction, so the + # summation order changes and the result differs in the last few ulps. + torch.testing.assert_close(fused_o @ weight.T, expected, rtol=1e-3, atol=1e-3) diff --git a/tests/kernels/test_fused_deepseek_v4_qnorm_rope_kv_insert.py b/tests/kernels/test_fused_deepseek_v4_qnorm_rope_kv_insert.py index 68eabfb94e..a0dcfc5bb7 100644 --- a/tests/kernels/test_fused_deepseek_v4_qnorm_rope_kv_insert.py +++ b/tests/kernels/test_fused_deepseek_v4_qnorm_rope_kv_insert.py @@ -522,6 +522,189 @@ def test_kv_path_mxfp8_record_matches_reference(num_tokens: int, block_size: int ) +# ── Test 2d: mega-attention Q layout (is_q_interleaved) ────────────────────── + +Q_CHUNK = 16 +NUM_Q_CHUNKS = HEAD_DIM // Q_CHUNK # 32 + + +@pytest.mark.parametrize("num_tokens", [1, 17, 2048]) +@pytest.mark.parametrize("n_heads,q_head_padded", [(16, 64), (64, 128), (64, 0)]) +def test_q_interleaved_pads_without_rope( + num_tokens: int, n_heads: int, q_head_padded: int +): + """``is_q_interleaved`` with no norm or RoPE only zero-pads Q. + + The mega-attention kernel norms and rotates Q itself, so every live + element must survive bit-exact while the padding heads -- which interleave + with the live ones inside each of the 32 head-dim chunks -- come back + zero. ``q_head_padded=0`` drops the Q pass entirely and returns nothing, + for a shard already at the kernel's head count. + """ + torch.manual_seed(5) + device = "cuda" + dtype = torch.bfloat16 + block_size = 64 + + q = torch.randn(num_tokens, n_heads, HEAD_DIM, dtype=dtype, device=device) + kv = torch.randn(num_tokens, HEAD_DIM, dtype=dtype, device=device) + positions = torch.arange(num_tokens, dtype=torch.int64, device=device) + cos_sin_cache = make_cos_sin_cache(4096, ROPE_DIM, torch.float32, device) + num_blocks = (num_tokens + block_size - 1) // block_size + 1 + slot_mapping = torch.arange(num_tokens, dtype=torch.int64, device=device) + k_cache = torch.zeros( + num_blocks, block_size * V41_HEAD_BYTES, dtype=torch.uint8, device=device + ) + + q_out = torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert( + q, + kv, + k_cache, + slot_mapping, + positions, + cos_sin_cache, + q_head_padded, + 1e-6, + block_size, + False, # apply_q_norm + True, # kv_mxfp8 + False, # apply_q_rope + True, # is_q_interleaved + ) + + # The KV half is unaffected by the Q mode: it must still be inserted. + assert k_cache.any() + + if q_head_padded == 0: + assert q_out.numel() == 0 + return + assert q_out.shape == (num_tokens, q_head_padded, HEAD_DIM) + got = q_out.view(num_tokens, NUM_Q_CHUNKS, q_head_padded, Q_CHUNK) + want = q.view(num_tokens, NUM_Q_CHUNKS, n_heads, Q_CHUNK) + torch.testing.assert_close(got[:, :, :n_heads], want, rtol=0, atol=0) + assert (got[:, :, n_heads:] == 0).all() + + +@pytest.mark.parametrize("n_heads", [8, 16, 32]) +def test_q_interleaved_survives_graph_replay(n_heads: int): + """A captured fused insert refills its padded Q buffer on every replay. + + The op allocates ``q_out`` itself, so the tensor a capture hands back has + to keep receiving the new padding when the graph is replayed over changed + input -- that is what lets the layer prepare Q inside the captured region + rather than in ``forward_mqa``. + """ + torch.manual_seed(6) + device = "cuda" + dtype = torch.bfloat16 + block_size, num_tokens, q_head_padded = 64, 7, 64 + + q = torch.randn(num_tokens, n_heads, HEAD_DIM, dtype=dtype, device=device) + kv = torch.randn(num_tokens, HEAD_DIM, dtype=dtype, device=device) + positions = torch.arange(num_tokens, dtype=torch.int64, device=device) + cos_sin_cache = make_cos_sin_cache(4096, ROPE_DIM, torch.float32, device) + num_blocks = (num_tokens + block_size - 1) // block_size + 1 + slot_mapping = torch.arange(num_tokens, dtype=torch.int64, device=device) + k_cache = torch.zeros( + num_blocks, block_size * V41_HEAD_BYTES, dtype=torch.uint8, device=device + ) + + def prepare(): + return torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert( + q, + kv, + k_cache, + slot_mapping, + positions, + cos_sin_cache, + q_head_padded, + 1e-6, + block_size, + False, # apply_q_norm + True, # kv_mxfp8 + False, # apply_q_rope + True, # is_q_interleaved + ) + + # Capture takes its warmup on a side stream. + side = torch.cuda.Stream() + side.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(side): + for _ in range(3): + prepare() + torch.cuda.current_stream().wait_stream(side) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + q_out = prepare() + + for factor in (0.25, -0.75): + q.copy_(torch.randn_like(q) * factor) + graph.replay() + torch.accelerator.synchronize() + got = q_out.view(num_tokens, NUM_Q_CHUNKS, q_head_padded, Q_CHUNK) + want = q.view(num_tokens, NUM_Q_CHUNKS, n_heads, Q_CHUNK) + torch.testing.assert_close(got[:, :, :n_heads], want, rtol=0, atol=0) + assert (got[:, :, n_heads:] == 0).all() + + +@pytest.mark.parametrize("n_heads", [16, 64]) +@pytest.mark.parametrize("apply_q_norm", [False, True]) +@pytest.mark.parametrize("apply_q_rope", [False, True]) +def test_q_interleaved_is_orthogonal_to_norm_and_rope( + n_heads: int, apply_q_norm: bool, apply_q_rope: bool +): + """The Q layout composes with whatever norm and RoPE it is given. + + A lane owns the same 512-dim slice of its head in either layout -- only + the address differs -- so the head-major result permuted into the + interleaved layout must equal running the interleaved path on a permuted + input, bit for bit, for every norm/RoPE combination. + """ + from vllm.models.deepseek_v41.common.ops.fused_layout import permute_q_to_fused + + torch.manual_seed(7) + device = "cuda" + dtype = torch.bfloat16 + block_size, num_tokens, q_head_padded = 64, 5, 64 + + q = torch.randn(num_tokens, n_heads, HEAD_DIM, dtype=dtype, device=device) + kv = torch.randn(num_tokens, HEAD_DIM, dtype=dtype, device=device) + positions = torch.arange(num_tokens, dtype=torch.int64, device=device) + cos_sin_cache = make_cos_sin_cache(4096, ROPE_DIM, torch.float32, device) + num_blocks = (num_tokens + block_size - 1) // block_size + 1 + slot_mapping = torch.arange(num_tokens, dtype=torch.int64, device=device) + + def run(q_in, is_q_interleaved): + cache = torch.zeros( + num_blocks, block_size * V41_HEAD_BYTES, dtype=torch.uint8, device=device + ) + out = torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert( + q_in, + kv, + cache, + slot_mapping, + positions, + cos_sin_cache, + q_head_padded, + 1e-6, + block_size, + apply_q_norm, + True, # kv_mxfp8 + apply_q_rope, + is_q_interleaved, + ) + return out, cache + + head_major, cache_major = run(q, False) + interleaved, cache_inter = run(permute_q_to_fused(q), True) + + torch.testing.assert_close( + interleaved, permute_q_to_fused(head_major), rtol=0, atol=0 + ) + # The Q layout must not leak into the KV half. + torch.testing.assert_close(cache_inter, cache_major, rtol=0, atol=0) + + # ── Test 2b: DP padding (slot_mapping shorter than q/kv) ───────────────────── diff --git a/vllm/models/deepseek_v41/amd/rocm.py b/vllm/models/deepseek_v41/amd/rocm.py index 802a59e340..b3d80f8d25 100644 --- a/vllm/models/deepseek_v41/amd/rocm.py +++ b/vllm/models/deepseek_v41/amd/rocm.py @@ -636,7 +636,8 @@ class DeepseekV41ROCMAiterMLAAttention(DeepseekV4Attention): transpose_scale=False, ) - def _o_proj(self, o: torch.Tensor, positions: torch.Tensor) -> torch.Tensor: + def _o_proj(self, attn_out: torch.Tensor, positions: torch.Tensor) -> torch.Tensor: + o = attn_out[:, : self.n_local_heads, :] # ROCm BF16 reference wo_a path (inverse RoPE + einsum) + wo_b. z = rocm_inv_rope_einsum( self.rotary_emb, diff --git a/vllm/models/deepseek_v41/attention.py b/vllm/models/deepseek_v41/attention.py index 134328dbe9..752b335eaf 100644 --- a/vllm/models/deepseek_v41/attention.py +++ b/vllm/models/deepseek_v41/attention.py @@ -44,6 +44,7 @@ from vllm.config import ( VllmConfig, get_current_vllm_config, ) +from vllm.config.cache import CacheDType from vllm.distributed import get_tensor_model_parallel_world_size from vllm.forward_context import get_forward_context from vllm.logger import init_logger @@ -127,34 +128,40 @@ def _use_v41_mxfp8_kv_record() -> bool: def _resolve_dsv4_kv_cache_dtype( use_fp8_ds_mla_layout: bool, - kv_cache_dtype: str, + kv_cache_dtype: CacheDType, cache_config: CacheConfig | None, -) -> tuple[str, torch.dtype]: + packed_kv_cache_dtype: CacheDType = "fp8_ds_mla", +) -> tuple[CacheDType, torch.dtype]: """Map ``(layout, --kv-cache-dtype)`` to ``(cache_dtype_str, torch_dtype)``. Both layouts are paged; they differ in the per-token block format. The - ``fp8_ds_mla`` format is UE8M0 block-scaled fp8 packed as ``uint8`` (the - canonical ``fp8_ds_mla`` string is written back onto ``cache_config`` so the - page-size specs pick the 576B per-token slot). Plain-row backends store each - token's KV row in its element dtype: bf16 or per-tensor FP8 E4M3. + packed formats are ``uint8``-backed: ``fp8_ds_mla`` is UE8M0 block-scaled + fp8 throughout, ``nvfp4_ds_mla`` keeps that sliding-window record and + stores the compressed cache as NVFP4. An unspecific ``--kv-cache-dtype`` + (``auto`` / ``fp8``) resolves to ``packed_kv_cache_dtype``, the record this + layer's kernel prefers, and the canonical string is written back onto + ``cache_config`` so the page-size specs pick the right per-token slot. + Plain-row backends store each token's KV row in its element dtype: bf16 or + per-tensor FP8 E4M3. """ if use_fp8_ds_mla_layout: - # fp8_ds_mla block format: UE8M0 block-scaled fp8 packed as uint8. - if kv_cache_dtype == "auto": - kv_cache_dtype = "fp8" - if not kv_cache_dtype.startswith("fp8"): + if kv_cache_dtype in ("auto", "fp8"): + kv_cache_dtype = packed_kv_cache_dtype + elif not kv_cache_dtype.endswith("_ds_mla"): raise ValueError( - "DeepseekV4 fp8_ds_mla layout only supports fp8 " - f"kv-cache, got {kv_cache_dtype}. Please set " - "`--kv-cache-dtype fp8` or select a backend that supports " - "bfloat16 KV cache." + "DeepseekV4 packed KV layouts only support fp8 kv-cache, got " + f"{kv_cache_dtype}. Please set `--kv-cache-dtype fp8` or " + "select a backend that supports bfloat16 KV cache." ) - if kv_cache_dtype != "fp8_ds_mla": - if cache_config is not None: - cache_config.cache_dtype = "fp8_ds_mla" - kv_cache_dtype = "fp8_ds_mla" - logger.info_once("Using DeepSeek's fp8_ds_mla KV cache format.") + if cache_config is not None and cache_config.cache_dtype != kv_cache_dtype: + cache_config.cache_dtype = kv_cache_dtype + logger.info_once("Using DeepSeek's %s KV cache format.", kv_cache_dtype) return kv_cache_dtype, torch.uint8 + if kv_cache_dtype.endswith("_ds_mla"): + raise ValueError( + f"{kv_cache_dtype} is a packed FlashMLA DeepSeek V4.1 KV cache " + "format; the selected backend stores plain KV rows." + ) # Plain bf16 / per-tensor fp8 KV row (FlashInfer). if kv_cache_dtype.startswith("fp8"): @@ -189,6 +196,28 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC): # path to pre-reserve that workspace. PREFILL_CHUNK_SIZE: ClassVar[int] = 4 + # ---- attention-interface contract, declared by the platform subclass ---- + + @property + def accepts_unnormed_unroped_query(self) -> bool: + """Whether ``forward_mqa``'s ``q`` is the raw ``wq_b`` output. + + True when the attention kernel applies the Q norm and RoPE itself, and + reads Q in its own chunk-interleaved layout, so the layer only + zero-pads Q to ``padded_heads`` and inserts KV before calling it. + """ + return False + + @property + def packed_kv_cache_dtype(self) -> CacheDType: + """The packed KV record this layer's kernel prefers. + + What an unspecific ``--kv-cache-dtype`` (``auto`` / ``fp8``) resolves + to. Mega attention overrides it: its kernel is the one that can read + an NVFP4 compressed cache. + """ + return "fp8_ds_mla" + @classmethod @abstractmethod def get_padded_num_q_heads(cls, num_heads: int) -> int: @@ -213,8 +242,19 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC): raise NotImplementedError @abstractmethod - def _o_proj(self, o: torch.Tensor, positions: torch.Tensor) -> torch.Tensor: - """Inverse-RoPE + wo_a + wo_b output projection (platform-specific).""" + def _o_proj( + self, + attn_out: "torch.Tensor | QuantizedActivation", + positions: torch.Tensor, + ) -> torch.Tensor: + """Project whatever ``_alloc_attn_out`` produced through wo_a and wo_b. + + Takes the buffer whole, so each layer owns the shape it allocated: the + bf16 layers slice off their padding heads and apply the inverse RoPE, + while a layer whose attention kernel already did the inverse RoPE and + the FP8 cast gets a QuantizedActivation and has only wo_a and wo_b + left. + """ raise NotImplementedError def _uses_fp8_ds_mla_layout(self) -> bool: @@ -459,11 +499,27 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC): # Resolve the kv-cache dtype from this backend's block format. The same # resolution drives the SWA cache tensor dtype below. self.kv_cache_dtype, self.kv_cache_torch_dtype = _resolve_dsv4_kv_cache_dtype( - self._uses_fp8_ds_mla_layout(), cache_config.cache_dtype, cache_config + self._uses_fp8_ds_mla_layout(), + cache_config.cache_dtype, + cache_config, + self.packed_kv_cache_dtype, ) self.kv_mxfp8 = _use_v41_mxfp8_kv_record() - self.kv_bytes_per_token = 528 if self.kv_mxfp8 else 584 + self.swa_bytes_per_token = 528 if self.kv_mxfp8 else 584 + # nvfp4_ds_mla keeps the MXFP8 sliding-window record and stores the + # compressed cache as NVFP4 (256 B of e2m1 pairs + 32 e4m3 scales). + self.compressed_bytes_per_token = ( + 288 if self.kv_cache_dtype == "nvfp4_ds_mla" else self.swa_bytes_per_token + ) + # One alignment for every page in the block: the block stride is their + # sum, and 512 satisfies both TMA strides in play (512 for the V4.1 + # fp8 record, 256 for NVFP4). self.kv_page_alignment = 512 if self.kv_mxfp8 else 576 + if self.kv_cache_dtype == "nvfp4_ds_mla" and not self.kv_mxfp8: + raise ValueError( + "nvfp4_ds_mla needs the V4.1 KV records, which FlashMLA " + "decodes only on SM100." + ) self.swa_cache_layer = DeepseekV4SWACache( head_dim=self.head_dim, @@ -473,7 +529,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC): cache_config=cache_config, backend_cls=self.swa_backend_cls, block_size=32, - packed_bytes_per_token=self.kv_bytes_per_token, + packed_bytes_per_token=self.swa_bytes_per_token, packed_page_alignment=self.kv_page_alignment, ) @@ -553,8 +609,12 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC): block_size=self.swa_cache_layer.block_size, ) + # Every backend that gathers a chunk's KV through + # combine_topk_swa_indices needs its Triton kernel warmed, mega + # attention included -- it calls it from _forward_prefill_mega. if self.backend_cls.get_name() in ( "FLASHMLA_SPARSE_DSV41", + "FLASHMLA_MEGA_ATTN_DSV41", "ROCM_FLASHMLA_SPARSE_DSV4", ): from vllm.models.deepseek_v41.common.ops.cache_utils import ( @@ -569,14 +629,10 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC): hidden_states: torch.Tensor, llama_4_scaling: torch.Tensor | None = None, ) -> torch.Tensor: - # Pre-allocate attention output with FlashMLA-padded head count. - # The op writes into `o_padded`; we slice to n_local_heads after. - num_tokens = hidden_states.shape[0] - o_padded = torch.empty( - (num_tokens, self.padded_heads, self.head_dim), - dtype=hidden_states.dtype, - device=hidden_states.device, - ) + # The eager attention region writes into a caller-owned buffer + # (breakable_cudagraph needs in-place outputs); its shape and how it is + # projected afterwards follow the interface contract above. + attn_out = self._alloc_attn_out(hidden_states.shape[0], hidden_states) # Keep the attention input preparation in the captured graph. Only the # sparse indexer and MLA attention run in the eager break below. @@ -593,12 +649,19 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC): kv_score, indexer_weights, positions, - o_padded, + attn_out, ) - o = o_padded[:, : self.n_local_heads, :] + return self._o_proj(attn_out, positions) - # Inverse-RoPE + wo_a + wo_b output projection (platform-specific). - return self._o_proj(o, positions) + def _alloc_attn_out( + self, num_tokens: int, hidden_states: torch.Tensor + ) -> "torch.Tensor | QuantizedActivation": + """The buffer ``forward_mqa`` fills, per the interface contract.""" + return torch.empty( + (num_tokens, self.padded_heads, self.head_dim), + dtype=hidden_states.dtype, + device=hidden_states.device, + ) @cached_property def _can_fuse_query_quant(self) -> bool: @@ -652,7 +715,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC): kv_score: torch.Tensor, indexer_weights: torch.Tensor, positions: torch.Tensor, - o_padded: torch.Tensor, + attn_out: "torch.Tensor | QuantizedActivation", ) -> None: """Wide eager region: the whole of ``_prepare_and_attn`` runs eagerly. @@ -667,7 +730,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC): kv_score, indexer_weights, positions, - o_padded, + attn_out, ) def _prepare_and_attn( @@ -679,7 +742,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC): kv_score: torch.Tensor, indexer_weights: torch.Tensor, positions: torch.Tensor, - o_padded: torch.Tensor, + attn_out: "torch.Tensor | QuantizedActivation", ) -> None: """Attention input preparation followed by the sparse indexer and MLA. @@ -751,7 +814,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC): q, kv, positions, - o_padded, + attn_out, ) def _fused_wqa_wkv_gemm(self, hidden_states: torch.Tensor) -> torch.Tensor: @@ -858,16 +921,28 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC): dict[str, AttentionMetadata] | list[dict[str, AttentionMetadata]] | None ), ) -> torch.Tensor: + """Ready ``q`` for the attention kernel and publish this step's KV. + + One launch does both. With ``accepts_unnormed_unroped_query`` the + attention kernel norms and rotates Q itself and reads it in its own + chunk-interleaved layout, so the Q half of the launch is a zero-pad to + ``padded_heads`` -- and nothing at all once the shard is that wide. + """ if not isinstance(attn_metadata, dict): # Profile run: kernel doesn't fire; produce a padded tensor so # downstream FlashMLA gets the right shape. - if self.n_local_heads < self.padded_heads: - return F.pad( - q, - (0, 0, 0, self.padded_heads - self.n_local_heads), - value=0.0, - ) - return q + if self.n_local_heads >= self.padded_heads: + return q + if self.accepts_unnormed_unroped_query: + # Padding heads sit at the tail of every head-dim chunk in + # that layout, so no head-major pad of `q` reproduces it -- + # and nothing reads it on a profile run. + return q.new_zeros((q.shape[0], self.padded_heads, q.shape[2])) + return F.pad( + q, + (0, 0, 0, self.padded_heads - self.n_local_heads), + value=0.0, + ) swa_metadata = cast( "DeepseekSparseSWAMetadata | None", @@ -886,23 +961,38 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC): if cache_dtype == torch.uint8: # fp8_ds_mla UE8M0 paged path. Horizontally fused: # Q side: GPT-J RoPE, zero-filling the padding head slots; the - # kernel allocates and returns the padded q tensor. + # kernel allocates and returns the padded q tensor. An + # interleaved Q skips the RoPE its attention kernel owns + # and keeps only the pad, which q_head_padded=0 drops + # too once the shard is already padded_heads wide. # KV side: GPT-J RoPE + UE8M0 FP8 quant + paged cache insert. swa_kv_cache_2d = swa_kv_cache.view(swa_kv_cache.shape[0], -1) - return torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert( + pad_to = ( + 0 + if self.accepts_unnormed_unroped_query + and self.n_local_heads == self.padded_heads + else self.padded_heads + ) + q_padded = torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert( q, kv, swa_kv_cache_2d, swa_metadata.slot_mapping, positions, cos_sin_cache, - self.padded_heads, + pad_to, self.eps, swa_metadata.block_size, - False, + False, # apply_q_norm: qr is normed before wq_b self.kv_mxfp8, + not self.accepts_unnormed_unroped_query, # apply_q_rope + self.accepts_unnormed_unroped_query, # is_q_interleaved ) + return q if pad_to == 0 else q_padded + assert not self.accepts_unnormed_unroped_query, ( + "the chunk-interleaved Q layout only pairs with a packed KV record" + ) # Plain-row path: the [num_blocks, block_size, 512] cache stores the KV # row in its element dtype (no Q padding). bf16 rewrites q in place; # per-tensor fp8 writes a separately-allocated fp8 q and quantizes the @@ -959,7 +1049,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC): # fp8_ds_mla is a UE8M0 block-scaled uint8 layout whose page rounds up # to the decode kernel's TMA stride; plain bf16 / per-tensor fp8 rows # use natural element-size pages. - uses_fp8_ds_mla_layout = self.kv_cache_dtype == "fp8_ds_mla" + uses_fp8_ds_mla_layout = self.kv_cache_dtype in ("fp8_ds_mla", "nvfp4_ds_mla") return MLAAttentionSpec( block_size=vllm_config.cache_config.block_size, num_kv_heads=1, @@ -972,7 +1062,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC): kv_quant_mode=get_kv_quant_mode(self.kv_cache_dtype), # Packed record width; head_size stays semantic (512). state_content_bytes=( - self.kv_bytes_per_token if uses_fp8_ds_mla_layout else None + self.compressed_bytes_per_token if uses_fp8_ds_mla_layout else None ), ) @@ -1016,7 +1106,13 @@ class DeepseekV4IndexerCache(torch.nn.Module, AttentionLayerBase): def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec: # head_dim already carries the fp8 scale padding # tokens_per_state=1 for V3.2, >1 for DeepseekV4; same cache layout. - uses_fp8_ds_mla_layout = vllm_config.cache_config.cache_dtype == "fp8_ds_mla" + # nvfp4_ds_mla is packed too: its compressed record is NVFP4 but the + # sliding-window record stays the V4.1 MXFP8 one, so the indexer page + # takes the same alignment either way. + uses_fp8_ds_mla_layout = vllm_config.cache_config.cache_dtype in ( + "fp8_ds_mla", + "nvfp4_ds_mla", + ) page_alignment = ( 576 if uses_fp8_ds_mla_layout and not _use_v41_mxfp8_kv_record() else 512 ) diff --git a/vllm/models/deepseek_v41/common/ops/cache_utils.py b/vllm/models/deepseek_v41/common/ops/cache_utils.py index 51743016bf..4a9da21c03 100644 --- a/vllm/models/deepseek_v41/common/ops/cache_utils.py +++ b/vllm/models/deepseek_v41/common/ops/cache_utils.py @@ -43,10 +43,16 @@ from vllm.utils.math_utils import next_power_of_2 # dims each plus a pad byte. # V4.1 (528 B): all 512 dims as fp8 e4m3 (RoPE included), then 16 UE8M0 # scales of 32 dims each. FlashMLA's ``ModelType::V41``. +# V4.1 NVFP4 (288 B): 512 e2m1 values packed two per byte (even element in +# the low nibble), then 32 e4m3 scales of 16 dims each. +# FlashMLA's ``ModelType::V41_FP4``, compressed cache only. V4_BYTES_PER_TOKEN = 584 V41_BYTES_PER_TOKEN = 528 V41_QUANT_BLOCK = 32 V41_NUM_SCALES = 512 // V41_QUANT_BLOCK # 16 +V41_NVFP4_BYTES_PER_TOKEN = 288 +V41_NVFP4_QUANT_BLOCK = 16 +V41_NVFP4_NUM_SCALES = 512 // V41_NVFP4_QUANT_BLOCK # 32 @triton.jit @@ -496,6 +502,73 @@ def _dequantize_and_gather_k_mxfp8_kernel( tl.store(output_row_ptr + d, dequant.to(tl.bfloat16)) +@triton.jit +def _dequantize_and_gather_k_nvfp4_kernel( + out_ptr, + out_stride0, + out_stride1, + k_cache_ptr, + seq_lens_ptr, + block_table_ptr, + offset, + gather_lens_ptr, + max_blocks_per_seq: tl.constexpr, + head_dim: tl.constexpr, # 512 + scale_dim: tl.constexpr, # 32 + quant_block: tl.constexpr, # 16 + cache_block_size: tl.constexpr, + block_stride: tl.constexpr, +): + """Gather and dequantize V4.1 NVFP4 rows into a bf16 workspace.""" + batch_idx = tl.program_id(0) + worker_id = tl.program_id(1) + num_workers = tl.num_programs(1) + + seq_len = tl.load(seq_lens_ptr + batch_idx) + if gather_lens_ptr is not None: # noqa: SIM108 + gather_len = tl.load(gather_lens_ptr + batch_idx) + else: + gather_len = seq_len + start_pos = seq_len - gather_len + + packed_bytes: tl.constexpr = head_dim // 2 + d = tl.arange(0, head_dim) + for i in range(worker_id, gather_len, num_workers): + pos = start_pos + i + block_in_seq = pos // cache_block_size + pos_in_block = pos % cache_block_size + + physical_block_idx = tl.load( + block_table_ptr + batch_idx * max_blocks_per_seq + block_in_seq + ) + page = k_cache_ptr + physical_block_idx.to(tl.int64) * block_stride + + packed = tl.load( + page + pos_in_block * packed_bytes + tl.arange(0, packed_bytes) + ) + # Even element in the low nibble, odd in the high nibble. + codes = tl.interleave((packed & 0xF).to(tl.int32), (packed >> 4).to(tl.int32)) + # e2m1 magnitudes: 0, 0.5, 1, 1.5, 2, 3, 4, 6. + mag_code = codes & 7 + e = (mag_code >> 1).to(tl.float32) + m = (mag_code & 1).to(tl.float32) + mag = tl.where(mag_code < 2, m * 0.5, (1.0 + m * 0.5) * tl.exp2(e - 1.0)) + vals = tl.where(codes >= 8, -mag, mag) + + sf = tl.load( + page + + cache_block_size * packed_bytes + + pos_in_block * scale_dim + + tl.arange(0, scale_dim) + ) + scale = sf.to(tl.float8e4nv, bitcast=True).to(tl.float32) + tiles = tl.reshape(vals, (scale_dim, quant_block)) + dequant = tl.reshape(tiles * tl.reshape(scale, (scale_dim, 1)), (head_dim,)) + + output_row_ptr = out_ptr + batch_idx * out_stride0 + (offset + i) * out_stride1 + tl.store(output_row_ptr + d, dequant.to(tl.bfloat16)) + + def dequantize_and_gather_k_cache_triton( # [num_reqs, max_num_tokens, head_size] out: torch.Tensor, @@ -513,6 +586,25 @@ def dequantize_and_gather_k_cache_triton( ) -> None: num_reqs = seq_lens.shape[0] NUM_WORKERS = 128 + if k_cache.shape[-1] == V41_NVFP4_BYTES_PER_TOKEN: + _dequantize_and_gather_k_nvfp4_kernel[(num_reqs, NUM_WORKERS)]( + out, + out.stride(0), + out.stride(1), + k_cache, + seq_lens, + block_table, + offset, + gather_lens, + max_blocks_per_seq=block_table.shape[-1], + head_dim=512, + scale_dim=V41_NVFP4_NUM_SCALES, + quant_block=V41_NVFP4_QUANT_BLOCK, + cache_block_size=block_size, + block_stride=k_cache.stride(0), + ) + return + if k_cache.shape[-1] == V41_BYTES_PER_TOKEN: _dequantize_and_gather_k_mxfp8_kernel[(num_reqs, NUM_WORKERS)]( out, @@ -581,14 +673,16 @@ def dequantize_and_gather_k_cache( ) -> None: """Dequantize and gather a paged DSv4 K cache. - The record is read off ``k_cache.shape[-1]``; see the module header. + The record is read off ``k_cache.shape[-1]``; see the module header. Only + the fp8 records have a CuteDSL gather, so NVFP4 always takes the Triton + path. ``use_fnuz`` MUST match the encoder of the specific cache being read: ``False`` for ``compressed_k_cache`` (Triton encoder is OCP everywhere), ``current_platform.is_fp8_fnuz()`` for ``swa_k_cache`` (C++ encoder writes FNUZ on gfx942 and OCP on gfx950). """ - if has_cutedsl(): + if has_cutedsl() and k_cache.shape[-1] != V41_NVFP4_BYTES_PER_TOKEN: # lazily import, otherwise some tests fail due to CUDA driver init failure. from vllm.models.deepseek_v4.nvidia.ops.dequant_gather_k_cutedsl import ( _DEQUANT_GATHER_K_CACHE_CUTEDSL_KERNEL, diff --git a/vllm/models/deepseek_v41/common/ops/fused_compress_quant_cache.py b/vllm/models/deepseek_v41/common/ops/fused_compress_quant_cache.py index 96f15849dd..0f7830cf58 100644 --- a/vllm/models/deepseek_v41/common/ops/fused_compress_quant_cache.py +++ b/vllm/models/deepseek_v41/common/ops/fused_compress_quant_cache.py @@ -4,6 +4,7 @@ import torch +from vllm.models.deepseek_v4.common.ops.fused_indexer_q import _fp32x2_to_fp4x2 from vllm.platforms import current_platform from vllm.triton_utils import tl, triton @@ -232,13 +233,14 @@ def rope_quant_insert( The BF16 latent supplies both NoPE quantization and RoPE input. It is read only for valid slots at group boundaries. The cache dtype selects the - layout: ``uint8`` is the fp8_ds_mla paged layout, whose record the - per-token byte width names -- 584 B for V4 (576 value bytes and eight - segregated UE8M0 scale bytes, including one zero padding scale) or 528 B - for V4.1 (512 MXFP8 value bytes covering the RoPE dims too, then 16 UE8M0 - scales of 32 dims each). ``bfloat16`` and ``float8_e4m3fn`` are the plain - [448 NoPE | 64 RoPE] rows read by FlashInfer, the latter scaled by the - per-tensor ``fp8_scale``. + layout: ``uint8`` is a paged FlashMLA layout, whose record the per-token + byte width names -- 584 B for V4 (576 value bytes and eight segregated + UE8M0 scale bytes, including one zero padding scale), 528 B for V4.1 + (512 MXFP8 value bytes covering the RoPE dims too, then 16 UE8M0 scales of + 32 dims each), or 288 B for V4.1 NVFP4 (256 bytes of e2m1 pairs then 32 + e4m3 scales of 16 dims each), which only the compressed cache uses. + ``bfloat16`` and ``float8_e4m3fn`` are the plain [448 NoPE | 64 RoPE] rows + read by FlashInfer, the latter scaled by the per-tensor ``fp8_scale``. """ assert compress_ratio in (1, 2) assert latent.shape[1] == 512 and latent.dtype == torch.bfloat16 @@ -249,14 +251,14 @@ def rope_quant_insert( return launch_kwargs = {"launch_pdl": False} if current_platform.is_cuda() else {} if kv_cache.dtype == torch.uint8: - assert kv_cache.shape[-1] in (584, 528), ( + kernel = { + 584: _rope_quant_insert_kernel, + 528: _rope_quant_insert_mxfp8_kernel, + 288: _rope_quant_insert_nvfp4_kernel, + }.get(kv_cache.shape[-1]) + assert kernel is not None, ( f"unsupported paged KV record width {kv_cache.shape[-1]}" ) - kernel = ( - _rope_quant_insert_mxfp8_kernel - if kv_cache.shape[-1] == 528 - else _rope_quant_insert_kernel - ) kernel[(num_tokens,)]( latent, positions, @@ -402,6 +404,62 @@ def _rope_quant_insert_mxfp8_kernel( tl.store(scales + tl.arange(0, 16), encoded.to(tl.uint8)) +@triton.jit +def _rope_quant_insert_nvfp4_kernel( + latent, + positions, + cos_sin, + cache, + cache_slots, + COS_STRIDE: tl.constexpr, + CACHE_STRIDE: tl.constexpr, + CACHE_BLOCK: tl.constexpr, + COMPRESS_RATIO: tl.constexpr, + SANITIZE_CACHE_NANS: tl.constexpr, +): + """V4.1 NVFP4 record: RoPE, then e2m1 with one e4m3 scale per 16 dims. + + The scale is ``amax / 6`` (6 is e2m1's largest magnitude) clamped to the + e4m3 range, with no per-tensor scale on top. + """ + t = tl.program_id(0) + slot = tl.load(cache_slots + t) + if slot < 0: + return + position = tl.load(positions + t) + if (position + 1) % COMPRESS_RATIO != 0: + return + d = tl.arange(0, 512) + normed = tl.load(latent + t.to(tl.int64) * 512 + d).to(tl.float32) + + # NoPE pairs load (cos, sin) = (1, 0), so the rotation is the identity there. + even, odd = tl.split(tl.reshape(normed, (256, 2))) + pair = tl.arange(0, 256) - 224 + cs = cos_sin + (position // COMPRESS_RATIO * COMPRESS_RATIO) * COS_STRIDE + c = tl.load(cs + tl.maximum(pair, 0), pair >= 0, other=1.0).to(tl.float32) + s = tl.load(cs + 32 + tl.maximum(pair, 0), pair >= 0, other=0.0).to(tl.float32) + rotated = tl.interleave(even * c - odd * s, odd * c + even * s) + if SANITIZE_CACHE_NANS: + rotated = tl.where(rotated == rotated, rotated, 0.0) + + tiles = tl.reshape(rotated, (32, 16)) + amax = tl.max(tl.abs(tiles), 1) + # 2**-9 is the smallest normal e4m3 magnitude; 448 the largest. + scale = tl.clamp(amax * (1.0 / 6.0), 0.001953125, 448.0).to(tl.float8e4nv) + # Round-to-nearest division: Triton's default div.full misplaces values + # that land exactly on an e2m1 tie. + scaled = tl.math.div_rn(tiles, tl.reshape(scale.to(tl.float32), (32, 1))) + lo, hi = tl.split(tl.reshape(tl.reshape(scaled, (512,)), (256, 2))) + + page = cache + (slot // CACHE_BLOCK).to(tl.int64) * CACHE_STRIDE + tl.store( + page + (slot % CACHE_BLOCK) * 256 + tl.arange(0, 256), + _fp32x2_to_fp4x2(lo, hi), + ) + scales = page + CACHE_BLOCK * 256 + (slot % CACHE_BLOCK) * 32 + tl.store(scales + tl.arange(0, 32), scale.to(tl.uint8, bitcast=True)) + + @triton.jit def _rope_plain_insert_kernel( latent, diff --git a/vllm/models/deepseek_v41/common/ops/fused_layout.py b/vllm/models/deepseek_v41/common/ops/fused_layout.py new file mode 100644 index 0000000000..071afde6bc --- /dev/null +++ b/vllm/models/deepseek_v41/common/ops/fused_layout.py @@ -0,0 +1,86 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Weight permutations for FlashMLA's mega-attention kernel. + +The kernel reads Q with 16-element head-dim chunks interleaved across heads and +writes O with 32-element chunks interleaved across the 8 heads of a ``wo_a`` +group. Every permutation here satisfies ``fused = standard[perm]``, and is +applied once to ``wq_b`` rows and ``wo_a`` columns at load time so the +surrounding GEMMs produce and consume the kernel's layouts directly -- no +per-step shuffle. +""" + +import torch + +HEAD_DIM = 512 +Q_CHUNK = 16 +O_CHUNK = 32 +WV_GROUP_SIZE = 8 + + +def q_fused_permutation(num_heads: int, head_dim: int = HEAD_DIM) -> torch.Tensor: + """``fused[(d // 16) * (H * 16) + h * 16 + d % 16] = standard[h * D + d]``.""" + h = torch.arange(num_heads).view(num_heads, 1) + d = torch.arange(head_dim).view(1, head_dim) + fused_index = (d // Q_CHUNK) * (num_heads * Q_CHUNK) + h * Q_CHUNK + d % Q_CHUNK + perm = torch.empty(num_heads * head_dim, dtype=torch.long) + perm[fused_index.reshape(-1)] = torch.arange(num_heads * head_dim) + return perm + + +def o_fused_permutation( + heads_per_group: int = WV_GROUP_SIZE, head_dim: int = HEAD_DIM +) -> torch.Tensor: + """``fused[(c * G + h) * 32 + j] = standard[h * D + c * 32 + j]`` per group.""" + h = torch.arange(heads_per_group).view(-1, 1, 1) + c = torch.arange(head_dim // O_CHUNK).view(1, -1, 1) + j = torch.arange(O_CHUNK).view(1, 1, -1) + fused_index = (c * heads_per_group + h) * O_CHUNK + j + perm = torch.empty(heads_per_group * head_dim, dtype=torch.long) + perm[fused_index.reshape(-1)] = torch.arange(heads_per_group * head_dim) + return perm + + +def o_fused_chunk_permutation( + heads_per_group: int = WV_GROUP_SIZE, head_dim: int = HEAD_DIM +) -> torch.Tensor: + """Per-32-element-chunk form of :func:`o_fused_permutation`, for scales.""" + return o_fused_permutation(heads_per_group, head_dim)[::O_CHUNK] // O_CHUNK + + +def permute_q_to_fused(q: torch.Tensor) -> torch.Tensor: + """``[N, H, D]`` standard layout -> the same shape in the fused layout.""" + n, h, d = q.shape + perm = q_fused_permutation(h, d).to(q.device) + return q.reshape(n, h * d)[:, perm].view(n, h, d) + + +def _bytes_view(t: torch.Tensor) -> torch.Tensor: + """Byte view of a 1-byte-element tensor, so fp8/ue8m0 can be gathered.""" + return t.view(torch.uint8) if t.element_size() == 1 else t + + +def permute_wq_b_( + weight: torch.Tensor, weight_scale: torch.Tensor, num_local_heads: int +) -> None: + """Permute an MXFP8 ``wq_b`` shard's rows and per-row scales, in place.""" + head_dim = weight.shape[0] // num_local_heads + perm = q_fused_permutation(num_local_heads, head_dim).to(weight.device) + for t in (weight, weight_scale): + b = _bytes_view(t) + b.copy_(b[perm]) + + +def permute_wo_a_( + weight: torch.Tensor, + weight_scale: torch.Tensor, + heads_per_group: int = WV_GROUP_SIZE, +) -> None: + """Permute an MXFP8 ``wo_a`` shard's input columns and scales, in place.""" + head_dim = weight.shape[1] // heads_per_group + perm = o_fused_permutation(heads_per_group, head_dim).to(weight.device) + w = _bytes_view(weight) + w.copy_(w[:, perm]) + chunk_perm = o_fused_chunk_permutation(heads_per_group, head_dim) + s = _bytes_view(weight_scale) + s.copy_(s[:, chunk_perm.to(weight.device)]) diff --git a/vllm/models/deepseek_v41/nvidia/dspark.py b/vllm/models/deepseek_v41/nvidia/dspark.py index 214681f3a9..9a07189bc3 100644 --- a/vllm/models/deepseek_v41/nvidia/dspark.py +++ b/vllm/models/deepseek_v41/nvidia/dspark.py @@ -477,8 +477,22 @@ class DSparkDeepseekV4ForCausalLM(nn.Module): for layer in self.model.layers: layer.ffn.finalize_mega_moe_weights() + def _finalize_attn(self) -> None: + """Run the attention backend's post-load weight step on the draft layers. + + They are ordinary DeepseekV4DecoderLayers, so they take the engine-wide + attention backend and owe it whatever the target model owes it -- mega + attention permutes wq_b / wo_a here and refuses to run without it. + Idempotent, like the target's. + """ + for layer in self.model.layers: + finalize = getattr(layer.attn, "finalize_loaded_weights", None) + if finalize is not None: + finalize() + def process_weights_after_loading(self) -> None: self._finalize_moe() + self._finalize_attn() def _remap_dspark_name(self, name: str) -> str | None: """Map a checkpoint ``mtp.{i}.*`` name to this model's parameter path. diff --git a/vllm/models/deepseek_v41/nvidia/flash_mla_mega_attn.py b/vllm/models/deepseek_v41/nvidia/flash_mla_mega_attn.py new file mode 100644 index 0000000000..233d2f80f9 --- /dev/null +++ b/vllm/models/deepseek_v41/nvidia/flash_mla_mega_attn.py @@ -0,0 +1,497 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""DeepSeek V4.1 attention on FlashMLA's mega-attention kernel (SM100). + +One kernel does Q RoPE, sparse attention, the inverse RoPE of the output and +its FP8 cast, and writes straight into the buffer ``wo_a`` consumes. So the +layer declares ``accepts_unnormed_unroped_query`` -- the kernel RoPEs Q itself, +leaving the fused KV insert only the zero-pad to the kernel's head count -- and +its ``_alloc_attn_out`` / ``_o_proj`` pair speaks QuantizedActivation instead +of bf16, since the output needs no rotation or quantization here. + +``wq_b`` rows and ``wo_a`` columns are permuted once at load so the surrounding +GEMMs speak the kernel's chunk-interleaved layouts directly. A step's prefill +and decode segments write disjoint token ranges of one output buffer pair, so +a single ``wo_a`` einsum covers the whole step. +""" + +from dataclasses import replace +from typing import TYPE_CHECKING, cast + +import torch + +from vllm.config import VllmConfig +from vllm.config.cache import CacheDType +from vllm.forward_context import get_forward_context +from vllm.model_executor.layers.fusion.quant_activation import QuantizedActivation +from vllm.model_executor.layers.quantization.utils.quant_utils import kMxfp8Dynamic +from vllm.models.deepseek_v41.common.ops import ( + combine_topk_swa_indices, + compute_global_topk_indices_and_lens, + dequantize_and_gather_k_cache, +) +from vllm.models.deepseek_v41.common.ops.fused_layout import ( + WV_GROUP_SIZE, + permute_wo_a_, + permute_wq_b_, +) +from vllm.models.deepseek_v41.nvidia.flashmla import DeepseekV4FlashMLAAttention +from vllm.models.deepseek_v41.sparse_mla import ( + DeepseekV4FlashMLAMetadata, + FlashMLAMegaAttnBackend, +) +from vllm.platforms import current_platform +from vllm.utils.deep_gemm import fp8_einsum, get_tma_aligned_size +from vllm.utils.math_utils import round_up +from vllm.v1.attention.ops.flashmla import is_flashmla_sparse_supported +from vllm.v1.worker.workspace import current_workspace_manager + +if TYPE_CHECKING: + from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata + +_HEAD_DIM_V = 512 +_ROPE_DIM = 64 +# The kernel always emits per-32 scales, whatever num_per_channels says. +_QUANT_GROUP = 32 +# Fixed kernel conventions: GPT-J (non-neox) RoPE, TMA-aligned col-major +# scales, round-to-nearest, packed ue8m0 -- i.e. what DeepGEMM's fp8_einsum +# consumes. Q norm is off: the V4.1 layer norms qr before wq_b. +_ROPE_ARGS = (False, _ROPE_DIM) +_SF_ARGS = (_QUANT_GROUP, True, True, True) + +# Decode always goes through the mega kernel: there is no batch size at which +# falling back to the split-KV decode pays, so there is no minimum-token +# threshold and no fallback path to maintain. What decides whether this layer +# is worth selecting at all is TP, not the batch size. +# +# Measured on one GB300 with benchmarks/kernels/benchmark_dsv41_mega_attn.py, +# CUDA graphs on, counting every per-step op each path runs (64-head model, +# topk_swa=128, topk_extra=512, NVFP4 compressed cache), as mega / split-KV +# microseconds: +# +# s_q 1 32 128 256 512 +# TP1 (64 live heads) 29/29 31/31 31/40 53/72 96/140 +# TP2 (32 live) 29/29 31/30 35/35 61/63 110/119 +# TP4 (16 live) 29/29 31/29 34/34 59/58 109/109 +# TP8 (8 live) 29/29 31/29 33/33 59/57 107/104 +# +# The kernel absorbs the inverse RoPE and the FP8 cast, whose cost scales with +# *live* heads, while its own cost scales with *padded* heads -- always 64, +# since FlashMLA's FP8 decode takes only h_q in {64, 128}. So it wins at TP1 +# (up to 1.45x, and TP1 is also where the fused insert skips the Q pad +# entirely) and from TP2 down the two cancel to within a few percent, with a +# fixed ~2 us penalty at small s_q where the padded-head cost dominates. +# Measure before assuming a win at high TP -- and measure under CUDA graphs: +# in eager mode launch overhead swamps both paths and overstates the fused one +# several-fold. + + +def is_flashmla_mega_attn_supported() -> tuple[bool, str | None]: + """Whether FlashMLA's fused mega-attention kernels are usable here.""" + is_available, maybe_reason = is_flashmla_sparse_supported() + if not is_available: + return False, maybe_reason + if not current_platform.is_device_capability_family(100): + return False, "FlashMLA mega attention requires sm_10x GPUs." + if not hasattr(torch.ops._flashmla_C, "fused_norm_rope_attn_rope_cast_decode"): + return False, "vllm._flashmla_C was built without the mega attention ops." + if not hasattr(torch.ops._flashmla_C, "permute_q_b_proj"): + return False, "vllm._flashmla_C predates the mega attention weight permutes." + return True, None + + +def alloc_mega_attn_output( + num_tokens: int, + n_wv_group: int, + device: torch.device, +) -> QuantizedActivation: + """Allocate the output buffer pair the mega-attention kernels write into. + + One pair per forward step: the prefill and decode segments fill disjoint + token ranges, so a single ``fp8_einsum`` over all ``num_tokens`` can + consume the result instead of one call per segment. + + The scale buffer is MN-major -- stride 1 along tokens, head-dim stride + ``ceil4(num_tokens)`` -- which is both what the kernel requires of a + caller-provided buffer and what DeepGEMM expects for this N. + """ + d = WV_GROUP_SIZE * _HEAD_DIM_V + data = torch.empty( + (num_tokens, n_wv_group, d), dtype=torch.float8_e4m3fn, device=device + ) + aligned = get_tma_aligned_size(num_tokens, torch.int32.itemsize) + scale = torch.empty( + (n_wv_group, d // (_QUANT_GROUP * 4), aligned), + dtype=torch.int32, + device=device, + ).permute(2, 0, 1)[:num_tokens] + return QuantizedActivation( + data=data, + scale=scale, + orig_dtype=torch.bfloat16, + orig_shape=data.shape, + quant_key=kMxfp8Dynamic, + ) + + +def _token_slice(out: QuantizedActivation, start: int, end: int) -> QuantizedActivation: + """The ``[start, end)`` token slice of a mega-attention output buffer.""" + data = out.data[start:end] + return replace(out, data=data, scale=out.scale[start:end], orig_shape=data.shape) + + +class DeepseekV4MegaAttnAttention(DeepseekV4FlashMLAAttention): + """FlashMLA mega-attention layer for DeepSeek V4.1 (SM100).""" + + backend_cls = FlashMLAMegaAttnBackend + + @classmethod + def is_available_for(cls, vllm_config: VllmConfig) -> bool: + """Whether this layer can serve the configured model on this device. + + The kernel is SM100-only and wants ``WV_GROUP_SIZE`` heads per ``wo_a`` + group, which stops holding once TP divides the head count far enough + (TP16 on a 64-head model leaves 4). ``__init__`` raises on both, so the + default-backend selector asks here first rather than turning an + unsupported topology into a startup crash. + """ + if not is_flashmla_mega_attn_supported()[0]: + return False + config = vllm_config.model_config.hf_config + config = getattr(config, "text_config", config) + n_heads = getattr(config, "num_attention_heads", 0) or 0 + n_groups = getattr(config, "o_groups", 0) or 0 + tp_size = vllm_config.parallel_config.tensor_parallel_size + if n_heads % tp_size or n_groups % tp_size: + return False + n_local_heads, n_local_groups = n_heads // tp_size, n_groups // tp_size + if not n_local_groups or n_local_heads % WV_GROUP_SIZE: + return False + return n_local_heads // n_local_groups == WV_GROUP_SIZE + + def __init__(self, vllm_config: VllmConfig, *args, **kwargs) -> None: + super().__init__(vllm_config, *args, **kwargs) + if self.n_local_heads % WV_GROUP_SIZE: + raise ValueError( + f"{self.prefix}: mega attention needs the local head count " + f"({self.n_local_heads}) to be a multiple of {WV_GROUP_SIZE}." + ) + if self.n_local_heads // self.n_local_groups != WV_GROUP_SIZE: + raise ValueError( + f"{self.prefix}: mega attention needs {WV_GROUP_SIZE} heads per " + "wo_a group." + ) + self.n_wv_group = self.padded_heads // WV_GROUP_SIZE + self._fused_layouts_ready = False + + # ---- interface contract ------------------------------------------------ + + @property + def accepts_unnormed_unroped_query(self) -> bool: + return True + + @property + def packed_kv_cache_dtype(self) -> CacheDType: + # This kernel is the only one that reads an NVFP4 compressed cache, so + # it is what an unspecific --kv-cache-dtype resolves to here. + return "nvfp4_ds_mla" + + def _alloc_attn_out( + self, num_tokens: int, hidden_states: torch.Tensor + ) -> QuantizedActivation: + return alloc_mega_attn_output(num_tokens, self.n_wv_group, hidden_states.device) + + def _o_proj( + self, attn_out: QuantizedActivation, positions: torch.Tensor + ) -> torch.Tensor: + """Grouped ``wo_a`` then ``wo_b`` over the kernel's quantized output. + + The inverse RoPE and the FP8 cast happened inside the attention + kernel, and ``wo_a`` is permuted for its output layout, so there is + nothing to rotate or quantize here and ``positions`` is unused. The + scale already is DeepGEMM's packed-ue8m0 MN-major layout, so the + einsum consumes the kernel's output with no repacking. + """ + groups = self.n_local_groups + z = torch.empty( + (attn_out.data.shape[0], groups, self.o_lora_rank), + dtype=torch.bfloat16, + device=attn_out.data.device, + ) + fp8_einsum( + "bhr,hdr->bhd", + (attn_out.data[:, :groups], attn_out.scale[:, :groups]), + (self.wo_a.weight, self.wo_a.weight_scale), + z, + recipe=self._einsum_recipe, + ) + return self.wo_b(z.flatten(1)) + + # ---- weights ----------------------------------------------------------- + + def finalize_loaded_weights(self) -> None: + """Permute wq_b rows / wo_a columns into the kernel's layouts. + + Idempotent: a second post-load pass must not permute twice. + """ + if self._fused_layouts_ready: + return + permute_wq_b_( + self.wq_b.weight.data, self.wq_b.weight_scale.data, self.n_local_heads + ) + permute_wo_a_( + self.wo_a.weight.data, + self.wo_a.weight_scale.data, + self.n_local_heads // self.n_local_groups, + ) + self._fused_layouts_ready = True + + # ---- forward ----------------------------------------------------------- + + def forward_mqa( + self, + q: torch.Tensor, + kv: torch.Tensor, + positions: torch.Tensor, + output: QuantizedActivation, + ) -> None: + if not self._fused_layouts_ready: + raise RuntimeError( + f"{self.prefix}: wq_b / wo_a were never permuted for the mega " + "attention kernel; refusing to run with mismatched layouts." + ) + attn_metadata = get_forward_context().attn_metadata + if attn_metadata is None: + # Warmup dummy run: reserve the prefill workspace, produce zeros. + self._reserve_prefill_workspace(q) + output.data.zero_() + output.scale.zero_() + return + + assert isinstance(attn_metadata, dict) + # q was zero-padded to the kernel's head count by the fused KV insert. + assert q.shape[1] == self.padded_heads + flashmla_metadata = cast( + DeepseekV4FlashMLAMetadata | None, + attn_metadata.get(self.compressed_cache_prefix) + if self.compressed_cache_prefix is not None + else None, + ) + swa_metadata = cast( + "DeepseekSparseSWAMetadata", attn_metadata[self.swa_cache_layer.prefix] + ) + # The kernel takes int32 RoPE positions. + positions_int32 = positions.to(torch.int32) + num_decode_tokens = swa_metadata.num_decode_tokens + + if swa_metadata.num_prefills > 0: + self._forward_prefill_mega( + q[num_decode_tokens:], + positions_int32[num_decode_tokens:], + flashmla_metadata, + swa_metadata, + output, + num_decode_tokens, + ) + if swa_metadata.num_decodes > 0: + self._forward_decode_mega( + q[:num_decode_tokens], + positions_int32[:num_decode_tokens], + flashmla_metadata, + swa_metadata, + _token_slice(output, 0, num_decode_tokens), + ) + + def _reserve_prefill_workspace(self, q: torch.Tensor) -> None: + swa_only = self.compress_ratio == 0 + n = 0 if swa_only else -(-self.max_model_len // self.compress_ratio) + m = n + self.window_size + self.max_num_batched_tokens + if swa_only: + top_k = 0 + else: + assert self.topk_indices_buffer is not None + top_k = self.topk_indices_buffer.shape[-1] + combined_topk = round_up(top_k + self.window_size + self.max_image_tokens, 128) + current_workspace_manager().get_simultaneous( + ((self.PREFILL_CHUNK_SIZE, m, q.shape[-1]), torch.bfloat16), + ((self.max_num_batched_tokens, combined_topk), torch.int32), + ((self.max_num_batched_tokens,), torch.int32), + ) + + def _decode_compressed_kv_and_topk( + self, + flashmla_metadata: DeepseekV4FlashMLAMetadata | None, + swa_metadata: "DeepseekSparseSWAMetadata", + ) -> tuple[torch.Tensor | None, torch.Tensor | None, torch.Tensor | None]: + """The compressed cache and its ``[n, topk]`` slot indices / lengths.""" + if self.compress_ratio == 0: + return None, None, None + assert flashmla_metadata is not None + assert swa_metadata.is_valid_token is not None + assert self.topk_indices_buffer is not None + num_decode_tokens = swa_metadata.num_decode_tokens + indices, lens = compute_global_topk_indices_and_lens( + self.topk_indices_buffer[:num_decode_tokens], + swa_metadata.token_to_req_indices, + flashmla_metadata.block_table[: swa_metadata.num_decodes], + flashmla_metadata.block_size // self.compress_ratio, + swa_metadata.is_valid_token[:num_decode_tokens], + ) + return self._compressed_kv_cache().unsqueeze(-2), indices, lens + + def _forward_decode_mega( + self, + q: torch.Tensor, + positions_int32: torch.Tensor, + flashmla_metadata: DeepseekV4FlashMLAMetadata | None, + swa_metadata: "DeepseekSparseSWAMetadata", + out: QuantizedActivation, + ) -> None: + """Mega attention over the paged quantized caches, in place into ``out``. + + The caches are ``[num_blocks, page, 1, bytes]``, the record taken from + ``bytes`` (528 V4.1 fp8, 288 V4.1 NVFP4; NVFP4 only as the compressed + cache beside an fp8 SWA one), and the indices ``[s_q, topk]`` int32 + slot ids (``block * page + offset``, ``-1`` invalid). + """ + extra_cache, extra_idx, extra_len = self._decode_compressed_kv_and_topk( + flashmla_metadata, swa_metadata + ) + assert swa_metadata.decode_swa_indices is not None + torch.ops._flashmla_C.fused_norm_rope_attn_rope_cast_decode( + q, + self.swa_cache_layer.kv_cache.unsqueeze(-2), + swa_metadata.decode_swa_indices.view(q.shape[0], -1), + self.scale, + _HEAD_DIM_V, + self.attn_sink, + swa_metadata.decode_swa_lens, + extra_cache, + extra_idx, + extra_len, + False, # enable_q_norm + 0.0, # rms_norm_eps + positions_int32, + *_ROPE_ARGS, + self.rotary_emb.cos_sin_cache, + self.n_wv_group, + *_SF_ARGS, + out.data, + out.scale, + ) + + def _forward_prefill_mega( + self, + q: torch.Tensor, + positions_int32: torch.Tensor, + flashmla_metadata: DeepseekV4FlashMLAMetadata | None, + swa_metadata: "DeepseekSparseSWAMetadata", + out: QuantizedActivation, + token_base: int, + ) -> None: + swa_only = self.compress_ratio == 0 + num_decodes = swa_metadata.num_decodes + num_decode_tokens = swa_metadata.num_decode_tokens + seq_lens = swa_metadata.prefill_seq_lens + gather_lens = swa_metadata.prefill_gather_lens + query_start_loc_cpu = swa_metadata.query_start_loc_cpu + query_start_loc = swa_metadata.query_start_loc + assert seq_lens is not None and gather_lens is not None + assert query_start_loc_cpu is not None and query_start_loc is not None + prefill_token_base = query_start_loc_cpu[num_decodes] + assert self.topk_indices_buffer is not None + topk_indices = self.topk_indices_buffer[num_decode_tokens:][ + : swa_metadata.num_prefill_tokens + ] + top_k = 0 if swa_only else topk_indices.shape[-1] + chunk_plan = swa_metadata.get_prefill_chunk_plan( + compress_ratio=self.compress_ratio, + prefill_chunk_size=self.PREFILL_CHUNK_SIZE, + has_compressed=not swa_only, + ) + assert chunk_plan, "prefill chunk plan must be non-empty when num_prefills > 0" + workspace_manager = current_workspace_manager() + combined_topk = round_up(top_k + self.window_size + self.max_image_tokens, 128) + for chunk_start, chunk_end, chunk_n, chunk_m in chunk_plan: + chunk_size = chunk_end - chunk_start + kv_ws, idx_ws, lens_ws = workspace_manager.get_simultaneous( + ((chunk_size, chunk_m, q.shape[-1]), torch.bfloat16), + ((self.max_num_batched_tokens, combined_topk), torch.int32), + ((self.max_num_batched_tokens,), torch.int32), + ) + if not swa_only: + assert flashmla_metadata is not None + dequantize_and_gather_k_cache( + kv_ws[:chunk_size], + self._compressed_kv_cache(), + seq_lens=seq_lens[chunk_start:chunk_end] // self.compress_ratio, + gather_lens=None, + block_table=flashmla_metadata.block_table[num_decodes:][ + chunk_start:chunk_end + ], + block_size=flashmla_metadata.block_size // self.compress_ratio, + offset=0, + ) + dequantize_and_gather_k_cache( + kv_ws[:chunk_size], + self.swa_cache_layer.kv_cache, + seq_lens=seq_lens[chunk_start:chunk_end], + gather_lens=gather_lens[chunk_start:chunk_end], + block_table=swa_metadata.block_table[num_decodes:][ + chunk_start:chunk_end + ], + block_size=swa_metadata.block_size, + offset=chunk_n, + ) + first, last = num_decodes + chunk_start, num_decodes + chunk_end + qs = int(query_start_loc_cpu[first] - prefill_token_base) + qe = int(query_start_loc_cpu[last] - prefill_token_base) + combined_indices, combined_lens = combine_topk_swa_indices( + topk_indices[qs:qe], + query_start_loc[first : last + 1], + seq_lens[chunk_start:chunk_end], + gather_lens[chunk_start:chunk_end], + self.window_size, + self.compress_ratio, + top_k, + chunk_m, + chunk_n, + out=(idx_ws[: qe - qs], lens_ws[: qe - qs]), + left_visible=( + swa_metadata.prefill_left_visible[ + num_decode_tokens + qs : num_decode_tokens + qe + ] + if swa_metadata.prefill_left_visible is not None + else None + ), + right_visible=( + swa_metadata.prefill_right_visible[ + num_decode_tokens + qs : num_decode_tokens + qe + ] + if swa_metadata.prefill_right_visible is not None + else None + ), + max_image_tokens=self.max_image_tokens, + ) + chunk_out = _token_slice(out, token_base + qs, token_base + qe) + # Mega attention over the gathered non-paged bf16 KV (RoPE already + # applied), in place into this chunk's token range. `indices` + # entries outside [0, s_kv) skip. + torch.ops._flashmla_C.fused_norm_rope_attn_rope_cast_fwd( + q[qs:qe], + kv_ws.view(-1, 1, q.shape[-1]), + combined_indices.unsqueeze(1), + self.scale, + _HEAD_DIM_V, + self.attn_sink, + combined_lens, + False, # enable_q_norm + 0.0, # rms_norm_eps + positions_int32[qs:qe], + *_ROPE_ARGS, + self.rotary_emb.cos_sin_cache, + self.n_wv_group, + *_SF_ARGS, + chunk_out.data, + chunk_out.scale, + ) diff --git a/vllm/models/deepseek_v41/nvidia/flashinfer_sparse.py b/vllm/models/deepseek_v41/nvidia/flashinfer_sparse.py index 7bed50bde7..0b43cfad19 100644 --- a/vllm/models/deepseek_v41/nvidia/flashinfer_sparse.py +++ b/vllm/models/deepseek_v41/nvidia/flashinfer_sparse.py @@ -201,7 +201,8 @@ class DeepseekV4FlashInferMLAAttention(DeepseekV4Attention): def get_padded_num_q_heads(cls, num_heads: int) -> int: return _pad_to_supported_q_heads(num_heads) - def _o_proj(self, o: torch.Tensor, positions: torch.Tensor) -> torch.Tensor: + def _o_proj(self, attn_out: torch.Tensor, positions: torch.Tensor) -> torch.Tensor: + o = attn_out[:, : self.n_local_heads, :] return deep_gemm_fp8_o_proj( o, positions, @@ -562,7 +563,8 @@ class DeepseekV4FlashInferSM120Attention(DeepseekV4Attention): def get_padded_num_q_heads(cls, num_heads: int) -> int: return _pad_to_supported_q_heads(num_heads) - def _o_proj(self, o: torch.Tensor, positions: torch.Tensor) -> torch.Tensor: + def _o_proj(self, attn_out: torch.Tensor, positions: torch.Tensor) -> torch.Tensor: + o = attn_out[:, : self.n_local_heads, :] return deep_gemm_fp8_o_proj( o, positions, diff --git a/vllm/models/deepseek_v41/nvidia/flashmla.py b/vllm/models/deepseek_v41/nvidia/flashmla.py index ec834aa80d..47a7b35c76 100644 --- a/vllm/models/deepseek_v41/nvidia/flashmla.py +++ b/vllm/models/deepseek_v41/nvidia/flashmla.py @@ -58,7 +58,8 @@ class DeepseekV4FlashMLAAttention(DeepseekV4Attention): self._o_proj_block_size ) - def _o_proj(self, o: torch.Tensor, positions: torch.Tensor) -> torch.Tensor: + def _o_proj(self, attn_out: torch.Tensor, positions: torch.Tensor) -> torch.Tensor: + o = attn_out[:, : self.n_local_heads, :] return deep_gemm_fp8_o_proj( o, positions, diff --git a/vllm/models/deepseek_v41/nvidia/model.py b/vllm/models/deepseek_v41/nvidia/model.py index 9c9719e977..21fa00e5c9 100644 --- a/vllm/models/deepseek_v41/nvidia/model.py +++ b/vllm/models/deepseek_v41/nvidia/model.py @@ -64,6 +64,9 @@ from vllm.models.deepseek_v4.nvidia.model import ( make_deepseek_v4_expert_params_mapping, ) from vllm.models.deepseek_v41.attention import DeepseekV4Attention +from vllm.models.deepseek_v41.nvidia.flash_mla_mega_attn import ( + DeepseekV4MegaAttnAttention, +) from vllm.models.deepseek_v41.nvidia.flashinfer_sparse import ( DeepseekV4FlashInferMLAAttention, DeepseekV4FlashInferSM120Attention, @@ -119,8 +122,9 @@ def _select_dsv4_attn_cls(vllm_config: VllmConfig) -> type[DeepseekV4Attention]: The generic CUDA backend selector does not instantiate DSv4 layers directly, so map generic sparse-MLA choices to the DSv4-specialized attention class. - Without an explicit backend, SM12 defaults to FlashInfer while the other - CUDA arches keep the FlashMLA path. + Without an explicit backend: SM12 takes FlashInfer, SM100 takes mega + attention where the topology allows it, and everything else keeps the + FlashMLA path. """ backend = vllm_config.attention_config.backend device_capability = current_platform.get_device_capability() @@ -140,6 +144,8 @@ def _select_dsv4_attn_cls(vllm_config: VllmConfig) -> type[DeepseekV4Attention]: if device_capability is not None and device_capability.major == 12: return DeepseekV4FlashInferSM120Attention return DeepseekV4FlashInferMLAAttention + if backend is AttentionBackendEnum.FLASHMLA_MEGA_ATTN_DSV41: + return DeepseekV4MegaAttnAttention if backend in ( AttentionBackendEnum.FLASHMLA_SPARSE, AttentionBackendEnum.FLASHMLA_SPARSE_DSV4, @@ -149,6 +155,14 @@ def _select_dsv4_attn_cls(vllm_config: VllmConfig) -> type[DeepseekV4Attention]: if device_capability is not None and device_capability.major == 12: return DeepseekV4FlashInferSM120Attention + # Mega attention is the SM100 default: it fuses Q RoPE, sparse attention, + # the output's inverse RoPE and its FP8 cast into one launch, and brings + # the 288 B NVFP4 compressed record -- the format the reference + # implementation itself stores. It declines topologies it cannot serve + # (non-SM100, TP that leaves fewer than WV_GROUP_SIZE heads per wo_a + # group, a build without the kernel), which then fall through to FlashMLA. + if DeepseekV4MegaAttnAttention.is_available_for(vllm_config): + return DeepseekV4MegaAttnAttention return DeepseekV4FlashMLAAttention @@ -936,6 +950,17 @@ class DeepseekV4Model(nn.Module, EagleModelMixin): for layer in islice(self.layers, self.start_layer, self.end_layer): layer.ffn.finalize_mega_moe_weights() + def finalize_mega_attn_weights(self) -> None: + """Permute wq_b / wo_a into FlashMLA's mega-attention layouts. + + A no-op for every other attention layer, and idempotent, so a second + post-load pass cannot permute twice. + """ + for layer in islice(self.layers, self.start_layer, self.end_layer): + finalize = getattr(layer.attn, "finalize_loaded_weights", None) + if finalize is not None: + finalize() + def finalize_mhc_broadcast_weights(self) -> None: if not get_pp_group().is_first_rank or self.start_layer >= self.end_layer: return @@ -1190,6 +1215,7 @@ class DeepseekV41LLMForCausalLM( def process_weights_after_loading(self) -> None: self.model.finalize_mega_moe_weights() self.model.finalize_mhc_broadcast_weights() + self.model.finalize_mega_attn_weights() def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: return self.model.get_expert_mapping() diff --git a/vllm/models/deepseek_v41/sparse_mla.py b/vllm/models/deepseek_v41/sparse_mla.py index 6a0f09f952..3d69499d2a 100644 --- a/vllm/models/deepseek_v41/sparse_mla.py +++ b/vllm/models/deepseek_v41/sparse_mla.py @@ -229,3 +229,54 @@ class DeepseekV4FlashMLABackend(DeepseekV4SparseMLABackend): @staticmethod def get_builder_cls() -> type[DeepseekV4FlashMLAMetadataBuilder]: return DeepseekV4FlashMLAMetadataBuilder + + +class FlashMLAMegaAttnBackend(DeepseekV4FlashMLABackend): + """FlashMLA's mega-attention kernel: Q RoPE + sparse attention + inverse + RoPE + FP8 cast of the output, in one launch. + + Same metadata and KV-cache geometry as ``FLASHMLA_SPARSE_DSV41`` -- what + differs is the attention layer's interface contract (it takes an unnormed, + unroped Q and returns an already-inverse-RoPE'd, quantized output) and the + extra ``nvfp4_ds_mla`` compressed-cache record only this kernel can read. + SM100 only; the kernel has no SM90 instantiation. + """ + + supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = [ + "auto", + "fp8_ds_mla", + "fp8", # alias for fp8_ds_mla + "nvfp4_ds_mla", # V4.1 fp8 SWA cache + NVFP4 compressed cache + ] + + @staticmethod + def get_name() -> str: + return "FLASHMLA_MEGA_ATTN_DSV41" + + @staticmethod + def get_supported_kernel_block_sizes() -> list[int | MultipleOf]: + return [128] + + @classmethod + def supports_compute_capability(cls, capability: DeviceCapability) -> bool: + return capability.major == 10 + + @classmethod + def supports_combination( + cls, + head_size: int, + dtype: torch.dtype, + kv_cache_dtype: CacheDType | None, + block_size: int | None, + use_mla: bool, + has_sink: bool, + use_sparse: bool, + use_mm_prefix: bool, + device_capability: DeviceCapability, + ) -> str | None: + # Imported here: the layer module imports this backend class. + from vllm.models.deepseek_v41.nvidia.flash_mla_mega_attn import ( + is_flashmla_mega_attn_supported, + ) + + return is_flashmla_mega_attn_supported()[1] diff --git a/vllm/v1/attention/backends/mla/sparse_swa.py b/vllm/v1/attention/backends/mla/sparse_swa.py index c4f3cd8b4b..700f328a37 100644 --- a/vllm/v1/attention/backends/mla/sparse_swa.py +++ b/vllm/v1/attention/backends/mla/sparse_swa.py @@ -112,7 +112,10 @@ class DeepseekV4SWACache(torch.nn.Module, AttentionLayerBase): # fp8_ds_mla's UE8M0 paged layout rounds its page up to the decode # kernel's TMA stride; contiguous bf16/fp8 cache uses the natural # element-size page. - uses_fp8_ds_mla_layout = self.cache_config.cache_dtype == "fp8_ds_mla" + uses_fp8_ds_mla_layout = self.cache_config.cache_dtype in ( + "fp8_ds_mla", + "nvfp4_ds_mla", + ) return SlidingWindowMLASpec( block_size=self.block_size, num_kv_heads=1, diff --git a/vllm/v1/attention/backends/registry.py b/vllm/v1/attention/backends/registry.py index 571566db1f..dcaaffaaa8 100644 --- a/vllm/v1/attention/backends/registry.py +++ b/vllm/v1/attention/backends/registry.py @@ -114,6 +114,9 @@ class AttentionBackendEnum(Enum, metaclass=_AttentionBackendEnumMeta): "vllm.models.deepseek_v41.nvidia.flashinfer_sparse." "DeepseekV4FlashInferMLASparseBackend" ) + FLASHMLA_MEGA_ATTN_DSV41 = ( + "vllm.models.deepseek_v41.sparse_mla.FlashMLAMegaAttnBackend" + ) B12X = "vllm.v1.attention.backends.b12x.B12xPagedAttentionBackend" FLASH_ATTN_MLA = "vllm.v1.attention.backends.mla.flashattn_mla.FlashAttnMLABackend" FLASH_ATTN_MLA_SPARSE = (