From be1cb9834b3d74114f4d426068d2bce57c9912a0 Mon Sep 17 00:00:00 2001 From: Yongye Zhu Date: Thu, 10 Sep 2026 00:35:59 -0700 Subject: [PATCH] [Kernel] Optional Q-norm in fused DSv4 MLA epilogue; group_size=32 for packed FP8 quant (#56215) Signed-off-by: Yongye Zhu Co-authored-by: Claude Opus 5 (1M context) --- ...deepseek_v4_qnorm_rope_kv_insert_kernel.cu | 134 +++++++++------- csrc/libtorch_stable/ops.h | 6 +- .../w8a8/fp8/per_token_group_quant.cu | 151 ++++++++++-------- csrc/libtorch_stable/torch_bindings.cpp | 7 +- .../test_per_token_group_quant.py | 5 + ..._fused_deepseek_v4_qnorm_rope_kv_insert.py | 112 +++++++++++-- 6 files changed, 267 insertions(+), 148 deletions(-) diff --git a/csrc/libtorch_stable/fused_deepseek_v4_qnorm_rope_kv_insert_kernel.cu b/csrc/libtorch_stable/fused_deepseek_v4_qnorm_rope_kv_insert_kernel.cu index 7bc435b8e0..06fe7108bd 100644 --- a/csrc/libtorch_stable/fused_deepseek_v4_qnorm_rope_kv_insert_kernel.cu +++ b/csrc/libtorch_stable/fused_deepseek_v4_qnorm_rope_kv_insert_kernel.cu @@ -3,7 +3,7 @@ * SPDX-FileCopyrightText: Copyright contributors to the vLLM project * * Horizontally-fused DeepseekV4-MLA kernel: - * - Q side: per-head RMSNorm (no weight) + GPT-J RoPE on last ROPE_DIM + * - Q side: optional per-head RMSNorm + GPT-J RoPE on last ROPE_DIM * - KV side: GPT-J RoPE on last ROPE_DIM + UE8M0 FP8 quant on NoPE + paged * cache insert * @@ -167,7 +167,8 @@ __device__ __forceinline__ float warpSum(float val) { // q_out stride fold to compile-time constants. // // Slot layout (per token): -// slot < num_heads_q → live-Q (RMSNorm + RoPE, +// slot < num_heads_q → live-Q (optional RMSNorm + +// RoPE, // read q_in → // write q_out) // num_heads_q <= slot < kNumHeadsQPadded → pad-Q (zero-fill q_out; @@ -175,7 +176,7 @@ __device__ __forceinline__ float warpSum(float val) { // slot == kNumHeadsQPadded → KV (RoPE + UE8M0 quant // + paged-cache // insert) -template +template __device__ __forceinline__ void processDeepseekV4Slot( uint4 v0, uint4 v1, int const tokenIdx, int const slotIdx, int const dim_base, int const laneId, int const num_heads_q, @@ -224,19 +225,21 @@ __device__ __forceinline__ void processDeepseekV4Slot( } } - // ── Q branch: RMSNorm (no weight) ─────────────────────────────────── - if (!isKV) { - float sumOfSquares = 0.0f; + // ── Q branch: optional RMSNorm (no weight) ────────────────────────── + if constexpr (APPLY_Q_NORM) { + if (!isKV) { + float sumOfSquares = 0.0f; #pragma unroll - for (int i = 0; i < kElemsPerLane; i++) { - sumOfSquares += elements[i] * elements[i]; - } - sumOfSquares = warpSum(sumOfSquares); - float const rms_rcp = - rsqrtf(sumOfSquares / static_cast(kHeadDim) + eps); + for (int i = 0; i < kElemsPerLane; i++) { + sumOfSquares += elements[i] * elements[i]; + } + sumOfSquares = warpSum(sumOfSquares); + float const rms_rcp = + rsqrtf(sumOfSquares / static_cast(kHeadDim) + eps); #pragma unroll - for (int i = 0; i < kElemsPerLane; i++) { - elements[i] = elements[i] * rms_rcp; + for (int i = 0; i < kElemsPerLane; i++) { + elements[i] = elements[i] * rms_rcp; + } } } @@ -386,7 +389,7 @@ __device__ __forceinline__ void processDeepseekV4Slot( // warps_per_block) Block: blockDim.x = 256 threads (8 warps per block) Each // warp handles one (token, head_slot) pair. // slot < num_heads_q → live-Q branch -// (RMSNorm + RoPE, +// (optional RMSNorm + RoPE, // read q_in → write q_out) // num_heads_q <= slot < kNumHeadsQPadded → pad-Q branch // (zero-fill q_out) @@ -404,7 +407,7 @@ __device__ __forceinline__ void processDeepseekV4Slot( // attention uses them). The KV branch only inserts the first // `num_tokens_insert` tokens (= slot_mapping length) into the paged cache. // -template +template __global__ void fusedDeepseekV4QNormRopeKVRopeQuantInsertKernel( scalar_t_in const* __restrict__ q_in, // [N, num_heads_q, 512] scalar_t_in* __restrict__ q_out, // [N, kNumHeadsQPadded, 512] @@ -471,7 +474,7 @@ __global__ void fusedDeepseekV4QNormRopeKVRopeQuantInsertKernel( v1 = *reinterpret_cast(src_ptr + 8); } - processDeepseekV4Slot( + processDeepseekV4Slot( v0, v1, tokenIdx, slotIdx, dim_base, laneId, num_heads_q, eps, q_out, k_cache, slot_mapping, position_ids, cos_sin_cache, cache_block_size, kv_block_stride); @@ -491,10 +494,10 @@ __global__ void fusedDeepseekV4QNormRopeKVRopeQuantInsertKernel( // Grid: 1D, gridDim.x = num_tokens_full // Block: blockDim.x = 256 threads (8 warps per block) Each // warp handles one token, iterating over each head. -// Q branch (RMSNorm + RoPE, in place) head_slot == num_heads_q +// Q branch (optional RMSNorm + RoPE, in place) head_slot == num_heads_q // KV branch (RoPE + UE8M0 quant + insert) // -template +template __global__ void fusedDeepseekV4QNormRopeKVRopeQuantInsertKernelReducedGrid( scalar_t_in const* __restrict__ q_in, scalar_t_in* __restrict__ q_out, scalar_t_in const* __restrict__ kv_in, uint8_t* __restrict__ k_cache, @@ -558,7 +561,7 @@ __global__ void fusedDeepseekV4QNormRopeKVRopeQuantInsertKernelReducedGrid( load_slot(next_slot, v0_next, v1_next); } - processDeepseekV4Slot( + processDeepseekV4Slot( v0_curr, v1_curr, tokenIdx, curr_slot, dim_base, laneId, num_heads_q, eps, q_out, k_cache, slot_mapping, position_ids, cos_sin_cache, cache_block_size, kv_block_stride); @@ -581,7 +584,7 @@ __global__ void fusedDeepseekV4QNormRopeKVRopeQuantInsertKernelReducedGrid( // ──────────────────────────────────────────────────────────────────────────── // Launch wrapper // ──────────────────────────────────────────────────────────────────────────── -template +template static void launchFusedDeepseekV4Templated( scalar_t_in const* q_in, scalar_t_in* q_out, scalar_t_in const* kv_in, uint8_t* k_cache, int64_t const* slot_mapping, int64_t const* position_ids, @@ -622,19 +625,18 @@ static void launchFusedDeepseekV4Templated( config.numAttrs = (sm_version >= 90) ? 1 : 0; if (num_tokens_full < NUM_TOKEN_CUTOFF) { - cudaLaunchKernelEx( - &config, - fusedDeepseekV4QNormRopeKVRopeQuantInsertKernel, - q_in, q_out, kv_in, k_cache, slot_mapping, position_ids, cos_sin_cache, - eps, num_tokens_full, num_tokens_insert, num_heads_q, cache_block_size, - kv_block_stride); + cudaLaunchKernelEx(&config, + fusedDeepseekV4QNormRopeKVRopeQuantInsertKernel< + scalar_t_in, kNumHeadsQPadded, APPLY_Q_NORM>, + q_in, q_out, kv_in, k_cache, slot_mapping, position_ids, + cos_sin_cache, eps, num_tokens_full, num_tokens_insert, + num_heads_q, cache_block_size, kv_block_stride); } else { config.gridDim = dim3(num_tokens_full); cudaLaunchKernelEx( &config, fusedDeepseekV4QNormRopeKVRopeQuantInsertKernelReducedGrid< - scalar_t_in, kNumHeadsQPadded>, + scalar_t_in, kNumHeadsQPadded, APPLY_Q_NORM>, q_in, q_out, kv_in, k_cache, slot_mapping, position_ids, cos_sin_cache, eps, num_tokens_full, num_tokens_insert, num_heads_q, cache_block_size, kv_block_stride); @@ -642,7 +644,8 @@ static void launchFusedDeepseekV4Templated( #else // ROCm: use standard kernel launch syntax (no PDL/stream serialization) // clang-format off - fusedDeepseekV4QNormRopeKVRopeQuantInsertKernel + fusedDeepseekV4QNormRopeKVRopeQuantInsertKernel< + scalar_t_in, kNumHeadsQPadded, APPLY_Q_NORM> <<>>( q_in, q_out, kv_in, k_cache, slot_mapping, position_ids, cos_sin_cache, eps, num_tokens_full, num_tokens_insert, num_heads_q, @@ -660,13 +663,20 @@ void launchFusedDeepseekV4QNormRopeKVRopeQuantInsert( int const num_tokens_full, int const num_tokens_insert, int const num_heads_q, int const num_heads_q_padded, int const cache_block_size, int const kv_block_stride, - cudaStream_t stream) { + bool const apply_q_norm, cudaStream_t stream) { #define DISPATCH(N) \ case N: \ - launchFusedDeepseekV4Templated( \ - q_in, q_out, kv_in, k_cache, slot_mapping, position_ids, \ - cos_sin_cache, eps, num_tokens_full, num_tokens_insert, num_heads_q, \ - cache_block_size, kv_block_stride, stream); \ + if (apply_q_norm) { \ + launchFusedDeepseekV4Templated( \ + q_in, q_out, kv_in, k_cache, slot_mapping, position_ids, \ + cos_sin_cache, eps, num_tokens_full, num_tokens_insert, \ + num_heads_q, cache_block_size, kv_block_stride, stream); \ + } else { \ + launchFusedDeepseekV4Templated( \ + q_in, q_out, kv_in, k_cache, slot_mapping, position_ids, \ + cos_sin_cache, eps, num_tokens_full, num_tokens_insert, \ + num_heads_q, cache_block_size, kv_block_stride, stream); \ + } \ return; switch (num_heads_q_padded) { @@ -700,7 +710,8 @@ void launchFusedDeepseekV4QNormRopeKVRopeQuantInsert( // Grid: 1D, gridDim.x = ceil(num_tokens_full * (num_heads_q + 1) / warps). // Each warp handles one (token, slot): slot < num_heads_q → Q, slot == // num_heads_q → KV. -template +template __global__ void fusedDeepseekV4FullCacheKernel( scalar_t_in* __restrict__ q_inout, // [N, H, 512], in place (bf16) uint8_t* __restrict__ q_fp8_out, // [N, H, 512] fp8, optional @@ -777,19 +788,21 @@ __global__ void fusedDeepseekV4FullCacheKernel( } } - // ── Q branch: RMSNorm (no weight) ───────────────────────────────────── - if (!isKV) { - float sumOfSquares = 0.0f; + // ── Q branch: optional RMSNorm (no weight) ──────────────────────────── + if constexpr (APPLY_Q_NORM) { + if (!isKV) { + float sumOfSquares = 0.0f; #pragma unroll - for (int i = 0; i < kElemsPerLane; i++) { - sumOfSquares += elements[i] * elements[i]; - } - sumOfSquares = warpSum(sumOfSquares); - float const rms_rcp = - rsqrtf(sumOfSquares / static_cast(kHeadDim) + eps); + for (int i = 0; i < kElemsPerLane; i++) { + sumOfSquares += elements[i] * elements[i]; + } + sumOfSquares = warpSum(sumOfSquares); + float const rms_rcp = + rsqrtf(sumOfSquares / static_cast(kHeadDim) + eps); #pragma unroll - for (int i = 0; i < kElemsPerLane; i++) { - elements[i] = elements[i] * rms_rcp; + for (int i = 0; i < kElemsPerLane; i++) { + elements[i] = elements[i] * rms_rcp; + } } } @@ -899,15 +912,19 @@ static void launchFullCacheKernel( float const* q_fp8_scale_inv, float const eps, int const num_tokens_full, int const num_tokens_insert, int const num_heads_q, int const cache_block_size, int64_t const kv_block_stride, - int64_t const kv_token_stride, char const* op_name, cudaStream_t stream) { + int64_t const kv_token_stride, bool const apply_q_norm, + char const* op_name, cudaStream_t stream) { constexpr int kBlockSize = 256; constexpr int kWarpsPerBlock = kBlockSize / 32; int64_t const total_warps = static_cast(num_tokens_full) * (num_heads_q + 1); int const grid = static_cast((total_warps + kWarpsPerBlock - 1) / kWarpsPerBlock); - auto* kernel = - fusedDeepseekV4FullCacheKernel; + auto* kernel = apply_q_norm + ? fusedDeepseekV4FullCacheKernel< + scalar_t_in, STORE_Q_FP8, STORE_KV_FP8, true> + : fusedDeepseekV4FullCacheKernel< + scalar_t_in, STORE_Q_FP8, STORE_KV_FP8, false>; #ifndef USE_ROCM static int const sm_version = getSMVersion(); STD_TORCH_CHECK(sm_version >= 80, op_name, @@ -950,7 +967,7 @@ torch::stable::Tensor fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert( torch::stable::Tensor const& position_ids, // [N] int64 torch::stable::Tensor const& cos_sin_cache, // [max_pos, rope_dim] bf16 int64_t q_head_padded, // padded Q head count for output - double eps, int64_t cache_block_size) { + double eps, int64_t cache_block_size, bool apply_q_norm) { STD_TORCH_CHECK(q_in.device().is_cuda() && q_in.is_contiguous(), "q_in must be contiguous CUDA"); STD_TORCH_CHECK(kv.device().is_cuda() && kv.is_contiguous(), @@ -981,7 +998,7 @@ torch::stable::Tensor fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert( "cos_sin_cache must be float32"); // With DP padding, slot_mapping can be shorter than q/kv/positions. - // Q-norm+RoPE runs on all q.size(0) rows (downstream attention uses them); + // Q processing runs on all q.size(0) rows (downstream attention uses them); // KV quant+insert runs only on the first slot_mapping.size(0) rows. int const num_tokens_full = static_cast(q_in.size(0)); int const num_tokens_insert = static_cast(slot_mapping.size(0)); @@ -1000,7 +1017,8 @@ torch::stable::Tensor fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert( const cudaStream_t stream = get_current_cuda_stream(q_in.get_device_index()); // Allocate the padded q output. The kernel writes every element (live - // region gets RMSNorm+RoPE; pad region gets zeros), so `empty` is safe. + // region gets optional RMSNorm+RoPE; pad region gets zeros), so `empty` is + // safe. auto q_out = torch::stable::new_empty( q_in, {q_in.size(0), q_head_padded, q_in.size(2)}, q_in.scalar_type()); @@ -1018,7 +1036,7 @@ torch::stable::Tensor fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert( cos_sin_cache.const_data_ptr(), static_cast(eps), num_tokens_full, num_tokens_insert, num_heads_q, num_heads_q_padded, cache_block_size_i, kv_block_stride, - stream); + apply_q_norm, stream); }); return q_out; } @@ -1033,7 +1051,7 @@ void fused_deepseek_v4_qnorm_rope_kv_rope_full_cache_bf16_insert( torch::stable::Tensor const& slot_mapping, // [num_tokens_insert] int64 torch::stable::Tensor const& position_ids, // [N] int64 torch::stable::Tensor const& cos_sin_cache, // [max_pos, 64] float32 - double eps, int64_t cache_block_size) { + double eps, int64_t cache_block_size, bool apply_q_norm) { using torch::headeronly::ScalarType; STD_TORCH_CHECK(q.device().is_cuda() && q.is_contiguous(), "q must be contiguous CUDA"); @@ -1091,7 +1109,7 @@ void fused_deepseek_v4_qnorm_rope_kv_rope_full_cache_bf16_insert( cos_sin_cache.const_data_ptr(), nullptr, nullptr, static_cast(eps), num_tokens_full, num_tokens_insert, num_heads_q, static_cast(cache_block_size), kv_block_stride, - kv_token_stride, + kv_token_stride, apply_q_norm, "fused_deepseek_v4_qnorm_rope_kv_rope_full_cache_bf16_insert", stream); }); @@ -1107,7 +1125,7 @@ void fused_deepseek_v4_qnorm_rope_kv_rope_full_cache_fp8_insert( torch::stable::Tensor const& cos_sin_cache, // [max_pos, 64] float32 torch::stable::Tensor const& fp8_scale, // scalar float32 (KV scale) torch::stable::Tensor const& q_fp8_scale_inv, // scalar float32 (1 / Q scale) - double eps, int64_t cache_block_size) { + double eps, int64_t cache_block_size, bool apply_q_norm) { using torch::headeronly::ScalarType; STD_TORCH_CHECK(q.device().is_cuda() && q.is_contiguous(), "q must be contiguous CUDA"); @@ -1181,7 +1199,7 @@ void fused_deepseek_v4_qnorm_rope_kv_rope_full_cache_fp8_insert( num_tokens_full, num_tokens_insert, num_heads_q, static_cast(cache_block_size), // fp8 cache: 1 byte/element -> stride already in bytes. - k_cache.stride(0), k_cache.stride(1), + k_cache.stride(0), k_cache.stride(1), apply_q_norm, "fused_deepseek_v4_qnorm_rope_kv_rope_full_cache_fp8_insert", stream); }); diff --git a/csrc/libtorch_stable/ops.h b/csrc/libtorch_stable/ops.h index acf665b07a..3ded65a6e2 100644 --- a/csrc/libtorch_stable/ops.h +++ b/csrc/libtorch_stable/ops.h @@ -264,14 +264,14 @@ torch::stable::Tensor fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert( torch::stable::Tensor& k_cache, torch::stable::Tensor const& slot_mapping, torch::stable::Tensor const& position_ids, torch::stable::Tensor const& cos_sin_cache, int64_t q_head_padded, - double eps, int64_t cache_block_size); + double eps, int64_t cache_block_size, bool apply_q_norm); void fused_deepseek_v4_qnorm_rope_kv_rope_full_cache_bf16_insert( torch::stable::Tensor& q, torch::stable::Tensor const& kv, torch::stable::Tensor& k_cache, torch::stable::Tensor const& slot_mapping, torch::stable::Tensor const& position_ids, torch::stable::Tensor const& cos_sin_cache, double eps, - int64_t cache_block_size); + int64_t cache_block_size, bool apply_q_norm); void fused_kimi_k3_mla_key_concat_kv_cache_insert( torch::stable::Tensor& q, torch::stable::Tensor const& k_nope, @@ -346,7 +346,7 @@ void fused_deepseek_v4_qnorm_rope_kv_rope_full_cache_fp8_insert( torch::stable::Tensor const& cos_sin_cache, torch::stable::Tensor const& fp8_scale, torch::stable::Tensor const& q_fp8_scale_inv, double eps, - int64_t cache_block_size); + int64_t cache_block_size, bool apply_q_norm); #ifndef USE_ROCM std::tuple diff --git a/csrc/libtorch_stable/quantization/w8a8/fp8/per_token_group_quant.cu b/csrc/libtorch_stable/quantization/w8a8/fp8/per_token_group_quant.cu index f4a0bd5442..06fa29eaba 100644 --- a/csrc/libtorch_stable/quantization/w8a8/fp8/per_token_group_quant.cu +++ b/csrc/libtorch_stable/quantization/w8a8/fp8/per_token_group_quant.cu @@ -295,8 +295,7 @@ void per_token_group_quant_8bit(const torch::stable::Tensor& input, // Loads two contiguous uint4s (16 B + 16 B = 32 B) per thread; on Blackwell // nvcc fuses these into a single 256-bit LDG.E.256. // -// Constraints: GROUP_SIZE % (THREADS_PER_GROUP * VEC_SIZE) == 0; for -// THREADS_PER_GROUP=8 and bf16/fp16 (VEC_SIZE=16), this means GROUP_SIZE=128. +// Each thread quantizes 16 bf16/fp16 values, using 2 or 8 threads per group. template __global__ void per_token_group_quant_8bit_packed_register_kernel( @@ -305,9 +304,9 @@ __global__ void per_token_group_quant_8bit_packed_register_kernel( const int groups_per_row, const int mn, const int output_q_mn_extent, const int tma_aligned_mn, const int64_t num_scale_elems, const float eps, const float min_8bit, const float max_8bit) { - static_assert(GROUP_SIZE == 128, "fast path supports GROUP_SIZE==128"); - constexpr int THREADS_PER_GROUP = 8; + static_assert(GROUP_SIZE == 32 || GROUP_SIZE == 128); constexpr int VEC_SIZE = 32 / sizeof(T); // 16 for bf16/fp16 + constexpr int THREADS_PER_GROUP = GROUP_SIZE / VEC_SIZE; static_assert(GROUP_SIZE == THREADS_PER_GROUP * VEC_SIZE, "GROUP_SIZE must equal THREADS_PER_GROUP * VEC_SIZE"); static_assert(32 % THREADS_PER_GROUP == 0, @@ -363,20 +362,20 @@ __global__ void per_token_group_quant_8bit_packed_register_kernel( } } - // 8-lane subgroup shuffle reduce (octet of the warp). The mask selects the - // 8 lanes within the warp that share a group. #ifdef USE_ROCM const int lane_in_wave = threadIdx.x % warpSize; - const unsigned long long mask = 0xFFull << (lane_in_wave & ~7); - local_absmax = fmaxf(local_absmax, __shfl_xor_sync(mask, local_absmax, 4, 8)); - local_absmax = fmaxf(local_absmax, __shfl_xor_sync(mask, local_absmax, 2, 8)); - local_absmax = fmaxf(local_absmax, __shfl_xor_sync(mask, local_absmax, 1, 8)); + const unsigned long long mask = ((1ull << THREADS_PER_GROUP) - 1) + << (lane_in_wave & ~(THREADS_PER_GROUP - 1)); #else - unsigned mask = 0xffu << (threadIdx.x & 24u); - local_absmax = fmaxf(local_absmax, __shfl_xor_sync(mask, local_absmax, 4)); - local_absmax = fmaxf(local_absmax, __shfl_xor_sync(mask, local_absmax, 2)); - local_absmax = fmaxf(local_absmax, __shfl_xor_sync(mask, local_absmax, 1)); + const unsigned mask = ((1u << THREADS_PER_GROUP) - 1) + << ((threadIdx.x % 32) & ~(THREADS_PER_GROUP - 1)); #endif +#pragma unroll + for (int offset = THREADS_PER_GROUP / 2; offset > 0; offset /= 2) { + local_absmax = + fmaxf(local_absmax, + __shfl_xor_sync(mask, local_absmax, offset, THREADS_PER_GROUP)); + } float y_s = local_absmax / max_8bit; y_s = fmaxf(y_s, 1e-10f); @@ -455,15 +454,15 @@ __global__ void per_token_group_quant_8bit_packed_register_kernel( } // Public entry point: register-resident packed quant kernel. -// Constraints: group_size == 128 and bf16/fp16 input. +// Constraints: group_size in {32, 128} and bf16/fp16 input. void per_token_group_quant_8bit_packed(const torch::stable::Tensor& input, torch::stable::Tensor& output_q, torch::stable::Tensor& output_s_packed, int64_t group_size, double eps, double min_8bit, double max_8bit) { - STD_TORCH_CHECK(group_size == 128, + STD_TORCH_CHECK(group_size == 32 || group_size == 128, "per_token_group_quant_8bit_packed only supports " - "group_size==128, got ", + "group_size in {32, 128}, got ", group_size, "."); const auto in_dtype = input.scalar_type(); STD_TORCH_CHECK( @@ -512,7 +511,7 @@ void per_token_group_quant_8bit_packed(const torch::stable::Tensor& input, input.get_device_index()); cudaStream_t stream = get_current_cuda_stream(); - constexpr int THREADS_PER_GROUP = 8; + const int threads_per_group = group_size / 16; const int64_t padded_groups_per_row = k_num_packed_sfk * 4; const int64_t num_scale_elems = mn + (k_num_packed_sfk - 1) * tma_aligned_mn; @@ -523,7 +522,7 @@ void per_token_group_quant_8bit_packed(const torch::stable::Tensor& input, const int ry = 16 / kx; const int64_t row_blocks = (tma_aligned_mn + ry - 1) / ry; const int64_t sf_k_blocks = padded_groups_per_row / kx; - const int num_threads = (kx * ry) * THREADS_PER_GROUP; + const int num_threads = (kx * ry) * threads_per_group; // CUDA caps grid.x at 2^31 - 1 and grid.y at 2^16 - 1 (65535). constexpr int64_t kMaxGridDimYZ = 65535; STD_TORCH_CHECK(row_blocks <= static_cast(INT32_MAX) && @@ -536,63 +535,72 @@ void per_token_group_quant_8bit_packed(const torch::stable::Tensor& input, // PDL (Programmatic Dependent Launch) is NVIDIA-only; ROCm/HIP has no // equivalent launch attribute, so fall back to a classic launch there. #ifndef USE_ROCM - #define LAUNCH_REG_KERNEL_INST(T, DST_DTYPE, KX, RY) \ - do { \ - cudaLaunchConfig_t config = {}; \ - config.gridDim = dim3(static_cast(row_blocks), \ - static_cast(sf_k_blocks)); \ - config.blockDim = dim3(num_threads); \ - config.dynamicSmemBytes = 0; \ - config.stream = stream; \ - cudaLaunchAttribute attrs[1]; \ - attrs[0].id = cudaLaunchAttributeProgrammaticStreamSerialization; \ - attrs[0].val.programmaticStreamSerializationAllowed = 1; \ - config.numAttrs = 1; \ - config.attrs = attrs; \ - cudaLaunchKernelEx( \ - &config, \ - per_token_group_quant_8bit_packed_register_kernel, \ - static_cast(input.data_ptr()), output_q.data_ptr(), \ - reinterpret_cast(output_s_packed.data_ptr()), \ - static_cast(padded_groups_per_row), \ - static_cast(groups_per_row), static_cast(mn), \ - static_cast(output_q_mn_extent), \ - static_cast(tma_aligned_mn), num_scale_elems, \ - static_cast(eps), static_cast(min_8bit), \ - static_cast(max_8bit)); \ + #define LAUNCH_REG_KERNEL_INST(T, DST_DTYPE, GROUP_SIZE, KX, RY) \ + do { \ + cudaLaunchConfig_t config = {}; \ + config.gridDim = dim3(static_cast(row_blocks), \ + static_cast(sf_k_blocks)); \ + config.blockDim = dim3(num_threads); \ + config.dynamicSmemBytes = 0; \ + config.stream = stream; \ + cudaLaunchAttribute attrs[1]; \ + attrs[0].id = cudaLaunchAttributeProgrammaticStreamSerialization; \ + attrs[0].val.programmaticStreamSerializationAllowed = 1; \ + config.numAttrs = 1; \ + config.attrs = attrs; \ + cudaLaunchKernelEx( \ + &config, \ + per_token_group_quant_8bit_packed_register_kernel< \ + T, DST_DTYPE, GROUP_SIZE, KX, RY>, \ + static_cast(input.data_ptr()), output_q.data_ptr(), \ + reinterpret_cast(output_s_packed.data_ptr()), \ + static_cast(padded_groups_per_row), \ + static_cast(groups_per_row), static_cast(mn), \ + static_cast(output_q_mn_extent), \ + static_cast(tma_aligned_mn), num_scale_elems, \ + static_cast(eps), static_cast(min_8bit), \ + static_cast(max_8bit)); \ } while (0) #else - #define LAUNCH_REG_KERNEL_INST(T, DST_DTYPE, KX, RY) \ - do { \ - dim3 grid(static_cast(row_blocks), \ - static_cast(sf_k_blocks)); \ - dim3 block(num_threads); \ - per_token_group_quant_8bit_packed_register_kernel \ - <<>>( \ - static_cast(input.data_ptr()), output_q.data_ptr(), \ - reinterpret_cast(output_s_packed.data_ptr()), \ - static_cast(padded_groups_per_row), \ - static_cast(groups_per_row), static_cast(mn), \ - static_cast(output_q_mn_extent), \ - static_cast(tma_aligned_mn), num_scale_elems, \ - static_cast(eps), static_cast(min_8bit), \ - static_cast(max_8bit)); \ + #define LAUNCH_REG_KERNEL_INST(T, DST_DTYPE, GROUP_SIZE, KX, RY) \ + do { \ + dim3 grid(static_cast(row_blocks), \ + static_cast(sf_k_blocks)); \ + dim3 block(num_threads); \ + per_token_group_quant_8bit_packed_register_kernel \ + <<>>( \ + static_cast(input.data_ptr()), output_q.data_ptr(), \ + reinterpret_cast(output_s_packed.data_ptr()), \ + static_cast(padded_groups_per_row), \ + static_cast(groups_per_row), static_cast(mn), \ + static_cast(output_q_mn_extent), \ + static_cast(tma_aligned_mn), num_scale_elems, \ + static_cast(eps), static_cast(min_8bit), \ + static_cast(max_8bit)); \ } while (0) #endif -#define LAUNCH_REG_KERNEL(T, DST_DTYPE) \ - do { \ - if (kx == 16) { \ - LAUNCH_REG_KERNEL_INST(T, DST_DTYPE, 16, 1); \ - } else if (kx == 8) { \ - LAUNCH_REG_KERNEL_INST(T, DST_DTYPE, 8, 2); \ - } else if (kx == 4) { \ - LAUNCH_REG_KERNEL_INST(T, DST_DTYPE, 4, 4); \ - } else { \ - STD_TORCH_CHECK(false, "Unsupported kx value ", kx); \ - } \ +#define LAUNCH_REG_KERNEL_GROUP_SIZE(T, DST_DTYPE, GROUP_SIZE) \ + do { \ + if (kx == 16) { \ + LAUNCH_REG_KERNEL_INST(T, DST_DTYPE, GROUP_SIZE, 16, 1); \ + } else if (kx == 8) { \ + LAUNCH_REG_KERNEL_INST(T, DST_DTYPE, GROUP_SIZE, 8, 2); \ + } else if (kx == 4) { \ + LAUNCH_REG_KERNEL_INST(T, DST_DTYPE, GROUP_SIZE, 4, 4); \ + } else { \ + STD_TORCH_CHECK(false, "Unsupported kx value ", kx); \ + } \ + } while (0) + +#define LAUNCH_REG_KERNEL(T, DST_DTYPE) \ + do { \ + if (group_size == 32) { \ + LAUNCH_REG_KERNEL_GROUP_SIZE(T, DST_DTYPE, 32); \ + } else { \ + LAUNCH_REG_KERNEL_GROUP_SIZE(T, DST_DTYPE, 128); \ + } \ } while (0) VLLM_STABLE_DISPATCH_HALF_TYPES( @@ -607,6 +615,7 @@ void per_token_group_quant_8bit_packed(const torch::stable::Tensor& input, })); #undef LAUNCH_REG_KERNEL +#undef LAUNCH_REG_KERNEL_GROUP_SIZE #undef LAUNCH_REG_KERNEL_INST } diff --git a/csrc/libtorch_stable/torch_bindings.cpp b/csrc/libtorch_stable/torch_bindings.cpp index 44b5191182..119f08e398 100644 --- a/csrc/libtorch_stable/torch_bindings.cpp +++ b/csrc/libtorch_stable/torch_bindings.cpp @@ -429,7 +429,8 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) { "fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(" "Tensor q_in, Tensor kv, Tensor! k_cache, " "Tensor slot_mapping, Tensor position_ids, Tensor cos_sin_cache, " - "int q_head_padded, float eps, int cache_block_size) -> Tensor"); + "int q_head_padded, float eps, int cache_block_size, " + "bool apply_q_norm=True) -> Tensor"); // FlashInfer V4 full-cache variants: write Q in place (bf16) or to a separate // FP8 tensor, and KV into a contiguous 512-wide token-strided cache. @@ -437,13 +438,13 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) { "fused_deepseek_v4_qnorm_rope_kv_rope_full_cache_bf16_insert(" "Tensor! q, Tensor kv, Tensor! k_cache, Tensor slot_mapping, " "Tensor position_ids, Tensor cos_sin_cache, float eps, " - "int cache_block_size) -> ()"); + "int cache_block_size, bool apply_q_norm=True) -> ()"); ops.def( "fused_deepseek_v4_qnorm_rope_kv_rope_full_cache_fp8_insert(" "Tensor q, Tensor kv, Tensor! q_fp8, Tensor! k_cache, " "Tensor slot_mapping, Tensor position_ids, Tensor cos_sin_cache, " "Tensor fp8_scale, Tensor q_fp8_scale_inv, float eps, " - "int cache_block_size) -> ()"); + "int cache_block_size, bool apply_q_norm=True) -> ()"); // Kimi-K3 MLA epilogues: optional RoPE followed by concat/cache insertion. ops.def( diff --git a/tests/kernels/quantization/test_per_token_group_quant.py b/tests/kernels/quantization/test_per_token_group_quant.py index 623c9b6f13..a2788d0583 100644 --- a/tests/kernels/quantization/test_per_token_group_quant.py +++ b/tests/kernels/quantization/test_per_token_group_quant.py @@ -80,6 +80,11 @@ def test_per_token_group_quant_fp8( # Larger shapes with padding (127, 7168, 128), (253, 640, 128), + (1, 32, 32), + (3, 64, 32), + (7, 160, 32), + (32, 512, 32), + (127, 4096, 32), ], ) @pytest.mark.parametrize("poisoned_scales", [False, True]) diff --git a/tests/kernels/test_fused_deepseek_v4_qnorm_rope_kv_insert.py b/tests/kernels/test_fused_deepseek_v4_qnorm_rope_kv_insert.py index ed163a0472..0e25884368 100644 --- a/tests/kernels/test_fused_deepseek_v4_qnorm_rope_kv_insert.py +++ b/tests/kernels/test_fused_deepseek_v4_qnorm_rope_kv_insert.py @@ -4,11 +4,11 @@ Standalone unit test for the horizontally-fused DeepseekV4-MLA kernel: fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert - - Q side: per-head RMSNorm (no weight) + GPT-J RoPE on last 64 dims + - Q side: optional per-head RMSNorm + GPT-J RoPE on last 64 dims - KV side: GPT-J RoPE on last 64 + UE8M0 FP8 quant + paged cache insert We compare against: - - PyTorch reference for RMSNorm + GPT-J RoPE on Q + - PyTorch references for RoPE with and without RMSNorm on Q - Existing Triton `quantize_and_insert_k_cache` + round-trip via `dequantize_and_gather_k_cache` for KV @@ -146,9 +146,18 @@ pytestmark = pytest.mark.skipif( def _call_fused( - q_in, q_head_padded, kv, k_cache, slot_mapping, positions, cos_sin_cache, eps, bs + q_in, + q_head_padded, + kv, + k_cache, + slot_mapping, + positions, + cos_sin_cache, + eps, + bs, + apply_q_norm=None, ): - return torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert( + args = ( q_in, kv, k_cache, @@ -159,6 +168,11 @@ def _call_fused( eps, bs, ) + if apply_q_norm is None: + return torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(*args) + return torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert( + *args, apply_q_norm + ) def _as_stored_fp8(t: torch.Tensor) -> torch.Tensor: @@ -269,6 +283,42 @@ def test_q_path_matches_reference(num_tokens: int, n_heads: int, padded_heads: i ) +@pytest.mark.parametrize("num_tokens", [4, 2048]) +def test_q_path_without_qnorm_matches_rope_only_reference(num_tokens: int): + """Disabling Q norm must apply RoPE directly to the projected query.""" + torch.manual_seed(7) + device = "cuda" + dtype = torch.bfloat16 + eps = 1e-6 + n_heads = 8 + padded_heads = 16 + block_size = 16 + + q = torch.randn(num_tokens, n_heads, HEAD_DIM, dtype=dtype, device=device) + positions = torch.arange(num_tokens, dtype=torch.int64, device=device) + cos_sin_cache = make_cos_sin_cache(4096, ROPE_DIM, torch.float32, device) + q_ref = apply_rope_gptj_last_k(q, positions, cos_sin_cache) + kv = torch.zeros(num_tokens, HEAD_DIM, dtype=dtype, device=device) + k_cache = torch.zeros(2, block_size * HEAD_BYTES, dtype=torch.uint8, device=device) + slot_mapping = torch.full((num_tokens,), -1, dtype=torch.int64, device=device) + + q_out = _call_fused( + q, + padded_heads, + kv, + k_cache, + slot_mapping, + positions, + cos_sin_cache, + eps, + block_size, + apply_q_norm=False, + ) + + torch.testing.assert_close(q_out[:, :n_heads], q_ref, rtol=0, atol=0) + assert q_out[:, n_heads:].count_nonzero().item() == 0 + + # ── Test 2: KV path round-trip byte/value parity ───────────────────────────── @@ -382,7 +432,7 @@ def test_kv_path_matches_reference(num_tokens: int, block_size: int): @pytest.mark.parametrize("block_size", [16, 64]) def test_kv_path_with_dp_padding(num_tokens: int, pad: int, block_size: int): """slot_mapping.size(0) < q.size(0): the kernel must skip padded - tokens in the KV branch while still running Q-norm+RoPE on all rows.""" + tokens in the KV branch while still processing Q on all rows.""" torch.manual_seed(3) device = "cuda" dtype = torch.bfloat16 @@ -517,8 +567,9 @@ def _call_full_cache_fp8_fused( q_fp8_scale_inv, eps, bs, + apply_q_norm=None, ): - torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_full_cache_fp8_insert( + args = ( q, kv, q_fp8, @@ -531,6 +582,11 @@ def _call_full_cache_fp8_fused( eps, bs, ) + op = torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_full_cache_fp8_insert + if apply_q_norm is None: + op(*args) + else: + op(*args, apply_q_norm) def _call_full_cache_bf16_fused( @@ -542,8 +598,9 @@ def _call_full_cache_bf16_fused( cos_sin_cache, eps, bs, + apply_q_norm=None, ): - torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_full_cache_bf16_insert( + args = ( q, kv, k_cache, @@ -553,6 +610,11 @@ def _call_full_cache_bf16_fused( eps, bs, ) + op = torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_full_cache_bf16_insert + if apply_q_norm is None: + op(*args) + else: + op(*args, apply_q_norm) def _fp8_full_cache_reference( @@ -567,8 +629,9 @@ def _fp8_full_cache_reference( block_size, fp8_scale, q_fp8_scale_inv, + apply_q_norm, ): - q_ref = rmsnorm_no_weight(q, eps) + q_ref = rmsnorm_no_weight(q, eps) if apply_q_norm else q.float() q_ref = apply_rope_gptj_last_k(q_ref, positions, cos_sin_cache) q_fp8.copy_( torch.clamp(q_ref.float() * q_fp8_scale_inv, -FP8_MAX, FP8_MAX).to( @@ -595,9 +658,10 @@ def _bf16_full_cache_reference( cos_sin_cache, eps, block_size, + apply_q_norm, ): - q_ref = rmsnorm_no_weight(q, eps) - # Kernel keeps RMSNorm+RoPE in fp32 and rounds to bf16 once at the store. + q_ref = rmsnorm_no_weight(q, eps) if apply_q_norm else q.float() + # Kernel keeps optional RMSNorm+RoPE in fp32 and rounds once at the store. q_ref = apply_rope_gptj_last_k(q_ref, positions, cos_sin_cache).to(q.dtype) kv_ref = apply_rope_gptj_last_k(kv, positions, cos_sin_cache) @@ -616,10 +680,19 @@ def _bf16_full_cache_reference( @pytest.mark.parametrize("num_tokens", [4, 17]) @pytest.mark.parametrize("n_heads", [8, 17]) @pytest.mark.parametrize("positions_dtype", [torch.int32, torch.int64]) +@pytest.mark.parametrize( + "apply_q_norm", + [ + pytest.param(None, id="qnorm_default"), + pytest.param(True, id="qnorm_explicit"), + pytest.param(False, id="rope_only"), + ], +) def test_full_cache_per_tensor_fp8_matches_reference( num_tokens: int, n_heads: int, positions_dtype: torch.dtype, + apply_q_norm: bool | None, ): torch.manual_seed(4) device = "cuda" @@ -662,6 +735,7 @@ def test_full_cache_per_tensor_fp8_matches_reference( block_size, fp8_scale, q_fp8_scale_inv, + apply_q_norm is not False, ) _call_full_cache_fp8_fused( q.clone(), @@ -675,11 +749,12 @@ def test_full_cache_per_tensor_fp8_matches_reference( q_fp8_scale_inv, eps, block_size, + apply_q_norm, ) - # Q is RMSNorm(no-weight)+RoPE in fp32 before fp8 quant; the RMSNorm - # reduction and RoPE rotation can land the kernel and the torch reference on - # opposite sides of an fp8 round-to-nearest tie, so allow <=1 fp8 ULP. + # Q uses optional RMSNorm followed by RoPE in fp32 before fp8 quant. The + # kernel and torch reference can land on opposite sides of an fp8 + # round-to-nearest tie, so allow <=1 fp8 ULP. q_fused = _as_stored_fp8(q_fp8_fused) q_max_ulp = int(fp8_ulp_distance(q_fused, q_fp8_ref).max().item()) assert q_max_ulp <= 1, f"Q fp8 differs by {q_max_ulp} ULP (>1)" @@ -709,10 +784,19 @@ def test_full_cache_per_tensor_fp8_matches_reference( @pytest.mark.parametrize("num_tokens", [4, 17]) @pytest.mark.parametrize("n_heads", [8, 17]) @pytest.mark.parametrize("positions_dtype", [torch.int32, torch.int64]) +@pytest.mark.parametrize( + "apply_q_norm", + [ + pytest.param(None, id="qnorm_default"), + pytest.param(True, id="qnorm_explicit"), + pytest.param(False, id="rope_only"), + ], +) def test_full_cache_bf16_matches_reference( num_tokens: int, n_heads: int, positions_dtype: torch.dtype, + apply_q_norm: bool | None, ): torch.manual_seed(5) device = "cuda" @@ -743,6 +827,7 @@ def test_full_cache_bf16_matches_reference( cos_sin_cache, eps, block_size, + apply_q_norm is not False, ) _call_full_cache_bf16_fused( q_fused, @@ -753,6 +838,7 @@ def test_full_cache_bf16_matches_reference( cos_sin_cache, eps, block_size, + apply_q_norm, ) torch.testing.assert_close(q_fused, q_ref, rtol=1e-2, atol=1e-2)