[Kernel] Optional Q-norm in fused DSv4 MLA epilogue; group_size=32 for packed FP8 quant (#56215)

Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Yongye Zhu
2026-09-10 00:35:59 -07:00
committed by GitHub
co-authored by Claude Opus 5
parent 9e25706560
commit be1cb9834b
6 changed files with 267 additions and 148 deletions
@@ -3,7 +3,7 @@
* SPDX-FileCopyrightText: Copyright contributors to the vLLM project
*
* Horizontally-fused DeepseekV4-MLA kernel:
* - Q side: per-head RMSNorm (no weight) + GPT-J RoPE on last ROPE_DIM
* - Q side: optional per-head RMSNorm + GPT-J RoPE on last ROPE_DIM
* - KV side: GPT-J RoPE on last ROPE_DIM + UE8M0 FP8 quant on NoPE + paged
* cache insert
*
@@ -167,7 +167,8 @@ __device__ __forceinline__ float warpSum(float val) {
// q_out stride fold to compile-time constants.
//
// Slot layout (per token):
// slot < num_heads_q → live-Q (RMSNorm + RoPE,
// slot < num_heads_q → live-Q (optional RMSNorm +
// RoPE,
// read q_in →
// write q_out)
// num_heads_q <= slot < kNumHeadsQPadded → pad-Q (zero-fill q_out;
@@ -175,7 +176,7 @@ __device__ __forceinline__ float warpSum(float val) {
// slot == kNumHeadsQPadded → KV (RoPE + UE8M0 quant
// + paged-cache
// insert)
template <typename scalar_t_in, int kNumHeadsQPadded>
template <typename scalar_t_in, int kNumHeadsQPadded, bool APPLY_Q_NORM>
__device__ __forceinline__ void processDeepseekV4Slot(
uint4 v0, uint4 v1, int const tokenIdx, int const slotIdx,
int const dim_base, int const laneId, int const num_heads_q,
@@ -224,19 +225,21 @@ __device__ __forceinline__ void processDeepseekV4Slot(
}
}
// ── Q branch: RMSNorm (no weight) ───────────────────────────────────
if (!isKV) {
float sumOfSquares = 0.0f;
// ── Q branch: optional RMSNorm (no weight) ──────────────────────────
if constexpr (APPLY_Q_NORM) {
if (!isKV) {
float sumOfSquares = 0.0f;
#pragma unroll
for (int i = 0; i < kElemsPerLane; i++) {
sumOfSquares += elements[i] * elements[i];
}
sumOfSquares = warpSum<float>(sumOfSquares);
float const rms_rcp =
rsqrtf(sumOfSquares / static_cast<float>(kHeadDim) + eps);
for (int i = 0; i < kElemsPerLane; i++) {
sumOfSquares += elements[i] * elements[i];
}
sumOfSquares = warpSum<float>(sumOfSquares);
float const rms_rcp =
rsqrtf(sumOfSquares / static_cast<float>(kHeadDim) + eps);
#pragma unroll
for (int i = 0; i < kElemsPerLane; i++) {
elements[i] = elements[i] * rms_rcp;
for (int i = 0; i < kElemsPerLane; i++) {
elements[i] = elements[i] * rms_rcp;
}
}
}
@@ -386,7 +389,7 @@ __device__ __forceinline__ void processDeepseekV4Slot(
// warps_per_block) Block: blockDim.x = 256 threads (8 warps per block) Each
// warp handles one (token, head_slot) pair.
// slot < num_heads_q → live-Q branch
// (RMSNorm + RoPE,
// (optional RMSNorm + RoPE,
// read q_in → write q_out)
// num_heads_q <= slot < kNumHeadsQPadded → pad-Q branch
// (zero-fill q_out)
@@ -404,7 +407,7 @@ __device__ __forceinline__ void processDeepseekV4Slot(
// attention uses them). The KV branch only inserts the first
// `num_tokens_insert` tokens (= slot_mapping length) into the paged cache.
//
template <typename scalar_t_in, int kNumHeadsQPadded>
template <typename scalar_t_in, int kNumHeadsQPadded, bool APPLY_Q_NORM>
__global__ void fusedDeepseekV4QNormRopeKVRopeQuantInsertKernel(
scalar_t_in const* __restrict__ q_in, // [N, num_heads_q, 512]
scalar_t_in* __restrict__ q_out, // [N, kNumHeadsQPadded, 512]
@@ -471,7 +474,7 @@ __global__ void fusedDeepseekV4QNormRopeKVRopeQuantInsertKernel(
v1 = *reinterpret_cast<uint4 const*>(src_ptr + 8);
}
processDeepseekV4Slot<scalar_t_in, kNumHeadsQPadded>(
processDeepseekV4Slot<scalar_t_in, kNumHeadsQPadded, APPLY_Q_NORM>(
v0, v1, tokenIdx, slotIdx, dim_base, laneId, num_heads_q, eps, q_out,
k_cache, slot_mapping, position_ids, cos_sin_cache, cache_block_size,
kv_block_stride);
@@ -491,10 +494,10 @@ __global__ void fusedDeepseekV4QNormRopeKVRopeQuantInsertKernel(
// Grid: 1D, gridDim.x = num_tokens_full
// Block: blockDim.x = 256 threads (8 warps per block) Each
// warp handles one token, iterating over each head.
// Q branch (RMSNorm + RoPE, in place) head_slot == num_heads_q
// Q branch (optional RMSNorm + RoPE, in place) head_slot == num_heads_q
// KV branch (RoPE + UE8M0 quant + insert)
//
template <typename scalar_t_in, int kNumHeadsQPadded>
template <typename scalar_t_in, int kNumHeadsQPadded, bool APPLY_Q_NORM>
__global__ void fusedDeepseekV4QNormRopeKVRopeQuantInsertKernelReducedGrid(
scalar_t_in const* __restrict__ q_in, scalar_t_in* __restrict__ q_out,
scalar_t_in const* __restrict__ kv_in, uint8_t* __restrict__ k_cache,
@@ -558,7 +561,7 @@ __global__ void fusedDeepseekV4QNormRopeKVRopeQuantInsertKernelReducedGrid(
load_slot(next_slot, v0_next, v1_next);
}
processDeepseekV4Slot<scalar_t_in, kNumHeadsQPadded>(
processDeepseekV4Slot<scalar_t_in, kNumHeadsQPadded, APPLY_Q_NORM>(
v0_curr, v1_curr, tokenIdx, curr_slot, dim_base, laneId,
num_heads_q, eps, q_out, k_cache, slot_mapping, position_ids,
cos_sin_cache, cache_block_size, kv_block_stride);
@@ -581,7 +584,7 @@ __global__ void fusedDeepseekV4QNormRopeKVRopeQuantInsertKernelReducedGrid(
// ────────────────────────────────────────────────────────────────────────────
// Launch wrapper
// ────────────────────────────────────────────────────────────────────────────
template <typename scalar_t_in, int kNumHeadsQPadded>
template <typename scalar_t_in, int kNumHeadsQPadded, bool APPLY_Q_NORM>
static void launchFusedDeepseekV4Templated(
scalar_t_in const* q_in, scalar_t_in* q_out, scalar_t_in const* kv_in,
uint8_t* k_cache, int64_t const* slot_mapping, int64_t const* position_ids,
@@ -622,19 +625,18 @@ static void launchFusedDeepseekV4Templated(
config.numAttrs = (sm_version >= 90) ? 1 : 0;
if (num_tokens_full < NUM_TOKEN_CUTOFF) {
cudaLaunchKernelEx(
&config,
fusedDeepseekV4QNormRopeKVRopeQuantInsertKernel<scalar_t_in,
kNumHeadsQPadded>,
q_in, q_out, kv_in, k_cache, slot_mapping, position_ids, cos_sin_cache,
eps, num_tokens_full, num_tokens_insert, num_heads_q, cache_block_size,
kv_block_stride);
cudaLaunchKernelEx(&config,
fusedDeepseekV4QNormRopeKVRopeQuantInsertKernel<
scalar_t_in, kNumHeadsQPadded, APPLY_Q_NORM>,
q_in, q_out, kv_in, k_cache, slot_mapping, position_ids,
cos_sin_cache, eps, num_tokens_full, num_tokens_insert,
num_heads_q, cache_block_size, kv_block_stride);
} else {
config.gridDim = dim3(num_tokens_full);
cudaLaunchKernelEx(
&config,
fusedDeepseekV4QNormRopeKVRopeQuantInsertKernelReducedGrid<
scalar_t_in, kNumHeadsQPadded>,
scalar_t_in, kNumHeadsQPadded, APPLY_Q_NORM>,
q_in, q_out, kv_in, k_cache, slot_mapping, position_ids, cos_sin_cache,
eps, num_tokens_full, num_tokens_insert, num_heads_q, cache_block_size,
kv_block_stride);
@@ -642,7 +644,8 @@ static void launchFusedDeepseekV4Templated(
#else
// ROCm: use standard kernel launch syntax (no PDL/stream serialization)
// clang-format off
fusedDeepseekV4QNormRopeKVRopeQuantInsertKernel<scalar_t_in, kNumHeadsQPadded>
fusedDeepseekV4QNormRopeKVRopeQuantInsertKernel<
scalar_t_in, kNumHeadsQPadded, APPLY_Q_NORM>
<<<grid, kBlockSize, 0, stream>>>(
q_in, q_out, kv_in, k_cache, slot_mapping, position_ids,
cos_sin_cache, eps, num_tokens_full, num_tokens_insert, num_heads_q,
@@ -660,13 +663,20 @@ void launchFusedDeepseekV4QNormRopeKVRopeQuantInsert(
int const num_tokens_full, int const num_tokens_insert,
int const num_heads_q, int const num_heads_q_padded,
int const cache_block_size, int const kv_block_stride,
cudaStream_t stream) {
bool const apply_q_norm, cudaStream_t stream) {
#define DISPATCH(N) \
case N: \
launchFusedDeepseekV4Templated<scalar_t_in, N>( \
q_in, q_out, kv_in, k_cache, slot_mapping, position_ids, \
cos_sin_cache, eps, num_tokens_full, num_tokens_insert, num_heads_q, \
cache_block_size, kv_block_stride, stream); \
if (apply_q_norm) { \
launchFusedDeepseekV4Templated<scalar_t_in, N, true>( \
q_in, q_out, kv_in, k_cache, slot_mapping, position_ids, \
cos_sin_cache, eps, num_tokens_full, num_tokens_insert, \
num_heads_q, cache_block_size, kv_block_stride, stream); \
} else { \
launchFusedDeepseekV4Templated<scalar_t_in, N, false>( \
q_in, q_out, kv_in, k_cache, slot_mapping, position_ids, \
cos_sin_cache, eps, num_tokens_full, num_tokens_insert, \
num_heads_q, cache_block_size, kv_block_stride, stream); \
} \
return;
switch (num_heads_q_padded) {
@@ -700,7 +710,8 @@ void launchFusedDeepseekV4QNormRopeKVRopeQuantInsert(
// Grid: 1D, gridDim.x = ceil(num_tokens_full * (num_heads_q + 1) / warps).
// Each warp handles one (token, slot): slot < num_heads_q → Q, slot ==
// num_heads_q → KV.
template <typename scalar_t_in, bool STORE_Q_FP8, bool STORE_KV_FP8>
template <typename scalar_t_in, bool STORE_Q_FP8, bool STORE_KV_FP8,
bool APPLY_Q_NORM>
__global__ void fusedDeepseekV4FullCacheKernel(
scalar_t_in* __restrict__ q_inout, // [N, H, 512], in place (bf16)
uint8_t* __restrict__ q_fp8_out, // [N, H, 512] fp8, optional
@@ -777,19 +788,21 @@ __global__ void fusedDeepseekV4FullCacheKernel(
}
}
// ── Q branch: RMSNorm (no weight) ─────────────────────────────────────
if (!isKV) {
float sumOfSquares = 0.0f;
// ── Q branch: optional RMSNorm (no weight) ────────────────────────────
if constexpr (APPLY_Q_NORM) {
if (!isKV) {
float sumOfSquares = 0.0f;
#pragma unroll
for (int i = 0; i < kElemsPerLane; i++) {
sumOfSquares += elements[i] * elements[i];
}
sumOfSquares = warpSum<float>(sumOfSquares);
float const rms_rcp =
rsqrtf(sumOfSquares / static_cast<float>(kHeadDim) + eps);
for (int i = 0; i < kElemsPerLane; i++) {
sumOfSquares += elements[i] * elements[i];
}
sumOfSquares = warpSum<float>(sumOfSquares);
float const rms_rcp =
rsqrtf(sumOfSquares / static_cast<float>(kHeadDim) + eps);
#pragma unroll
for (int i = 0; i < kElemsPerLane; i++) {
elements[i] = elements[i] * rms_rcp;
for (int i = 0; i < kElemsPerLane; i++) {
elements[i] = elements[i] * rms_rcp;
}
}
}
@@ -899,15 +912,19 @@ static void launchFullCacheKernel(
float const* q_fp8_scale_inv, float const eps, int const num_tokens_full,
int const num_tokens_insert, int const num_heads_q,
int const cache_block_size, int64_t const kv_block_stride,
int64_t const kv_token_stride, char const* op_name, cudaStream_t stream) {
int64_t const kv_token_stride, bool const apply_q_norm,
char const* op_name, cudaStream_t stream) {
constexpr int kBlockSize = 256;
constexpr int kWarpsPerBlock = kBlockSize / 32;
int64_t const total_warps =
static_cast<int64_t>(num_tokens_full) * (num_heads_q + 1);
int const grid =
static_cast<int>((total_warps + kWarpsPerBlock - 1) / kWarpsPerBlock);
auto* kernel =
fusedDeepseekV4FullCacheKernel<scalar_t_in, STORE_Q_FP8, STORE_KV_FP8>;
auto* kernel = apply_q_norm
? fusedDeepseekV4FullCacheKernel<
scalar_t_in, STORE_Q_FP8, STORE_KV_FP8, true>
: fusedDeepseekV4FullCacheKernel<
scalar_t_in, STORE_Q_FP8, STORE_KV_FP8, false>;
#ifndef USE_ROCM
static int const sm_version = getSMVersion();
STD_TORCH_CHECK(sm_version >= 80, op_name,
@@ -950,7 +967,7 @@ torch::stable::Tensor fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
torch::stable::Tensor const& position_ids, // [N] int64
torch::stable::Tensor const& cos_sin_cache, // [max_pos, rope_dim] bf16
int64_t q_head_padded, // padded Q head count for output
double eps, int64_t cache_block_size) {
double eps, int64_t cache_block_size, bool apply_q_norm) {
STD_TORCH_CHECK(q_in.device().is_cuda() && q_in.is_contiguous(),
"q_in must be contiguous CUDA");
STD_TORCH_CHECK(kv.device().is_cuda() && kv.is_contiguous(),
@@ -981,7 +998,7 @@ torch::stable::Tensor fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
"cos_sin_cache must be float32");
// With DP padding, slot_mapping can be shorter than q/kv/positions.
// Q-norm+RoPE runs on all q.size(0) rows (downstream attention uses them);
// Q processing runs on all q.size(0) rows (downstream attention uses them);
// KV quant+insert runs only on the first slot_mapping.size(0) rows.
int const num_tokens_full = static_cast<int>(q_in.size(0));
int const num_tokens_insert = static_cast<int>(slot_mapping.size(0));
@@ -1000,7 +1017,8 @@ torch::stable::Tensor fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
const cudaStream_t stream = get_current_cuda_stream(q_in.get_device_index());
// Allocate the padded q output. The kernel writes every element (live
// region gets RMSNorm+RoPE; pad region gets zeros), so `empty` is safe.
// region gets optional RMSNorm+RoPE; pad region gets zeros), so `empty` is
// safe.
auto q_out = torch::stable::new_empty(
q_in, {q_in.size(0), q_head_padded, q_in.size(2)}, q_in.scalar_type());
@@ -1018,7 +1036,7 @@ torch::stable::Tensor fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
cos_sin_cache.const_data_ptr<float>(), static_cast<float>(eps),
num_tokens_full, num_tokens_insert, num_heads_q,
num_heads_q_padded, cache_block_size_i, kv_block_stride,
stream);
apply_q_norm, stream);
});
return q_out;
}
@@ -1033,7 +1051,7 @@ void fused_deepseek_v4_qnorm_rope_kv_rope_full_cache_bf16_insert(
torch::stable::Tensor const& slot_mapping, // [num_tokens_insert] int64
torch::stable::Tensor const& position_ids, // [N] int64
torch::stable::Tensor const& cos_sin_cache, // [max_pos, 64] float32
double eps, int64_t cache_block_size) {
double eps, int64_t cache_block_size, bool apply_q_norm) {
using torch::headeronly::ScalarType;
STD_TORCH_CHECK(q.device().is_cuda() && q.is_contiguous(),
"q must be contiguous CUDA");
@@ -1091,7 +1109,7 @@ void fused_deepseek_v4_qnorm_rope_kv_rope_full_cache_bf16_insert(
cos_sin_cache.const_data_ptr<float>(), nullptr, nullptr,
static_cast<float>(eps), num_tokens_full, num_tokens_insert,
num_heads_q, static_cast<int>(cache_block_size), kv_block_stride,
kv_token_stride,
kv_token_stride, apply_q_norm,
"fused_deepseek_v4_qnorm_rope_kv_rope_full_cache_bf16_insert",
stream);
});
@@ -1107,7 +1125,7 @@ void fused_deepseek_v4_qnorm_rope_kv_rope_full_cache_fp8_insert(
torch::stable::Tensor const& cos_sin_cache, // [max_pos, 64] float32
torch::stable::Tensor const& fp8_scale, // scalar float32 (KV scale)
torch::stable::Tensor const& q_fp8_scale_inv, // scalar float32 (1 / Q scale)
double eps, int64_t cache_block_size) {
double eps, int64_t cache_block_size, bool apply_q_norm) {
using torch::headeronly::ScalarType;
STD_TORCH_CHECK(q.device().is_cuda() && q.is_contiguous(),
"q must be contiguous CUDA");
@@ -1181,7 +1199,7 @@ void fused_deepseek_v4_qnorm_rope_kv_rope_full_cache_fp8_insert(
num_tokens_full, num_tokens_insert, num_heads_q,
static_cast<int>(cache_block_size),
// fp8 cache: 1 byte/element -> stride already in bytes.
k_cache.stride(0), k_cache.stride(1),
k_cache.stride(0), k_cache.stride(1), apply_q_norm,
"fused_deepseek_v4_qnorm_rope_kv_rope_full_cache_fp8_insert",
stream);
});
+3 -3
View File
@@ -264,14 +264,14 @@ torch::stable::Tensor fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
torch::stable::Tensor& k_cache, torch::stable::Tensor const& slot_mapping,
torch::stable::Tensor const& position_ids,
torch::stable::Tensor const& cos_sin_cache, int64_t q_head_padded,
double eps, int64_t cache_block_size);
double eps, int64_t cache_block_size, bool apply_q_norm);
void fused_deepseek_v4_qnorm_rope_kv_rope_full_cache_bf16_insert(
torch::stable::Tensor& q, torch::stable::Tensor const& kv,
torch::stable::Tensor& k_cache, torch::stable::Tensor const& slot_mapping,
torch::stable::Tensor const& position_ids,
torch::stable::Tensor const& cos_sin_cache, double eps,
int64_t cache_block_size);
int64_t cache_block_size, bool apply_q_norm);
void fused_kimi_k3_mla_key_concat_kv_cache_insert(
torch::stable::Tensor& q, torch::stable::Tensor const& k_nope,
@@ -346,7 +346,7 @@ void fused_deepseek_v4_qnorm_rope_kv_rope_full_cache_fp8_insert(
torch::stable::Tensor const& cos_sin_cache,
torch::stable::Tensor const& fp8_scale,
torch::stable::Tensor const& q_fp8_scale_inv, double eps,
int64_t cache_block_size);
int64_t cache_block_size, bool apply_q_norm);
#ifndef USE_ROCM
std::tuple<torch::stable::Tensor, torch::stable::Tensor>
@@ -295,8 +295,7 @@ void per_token_group_quant_8bit(const torch::stable::Tensor& input,
// Loads two contiguous uint4s (16 B + 16 B = 32 B) per thread; on Blackwell
// nvcc fuses these into a single 256-bit LDG.E.256.
//
// Constraints: GROUP_SIZE % (THREADS_PER_GROUP * VEC_SIZE) == 0; for
// THREADS_PER_GROUP=8 and bf16/fp16 (VEC_SIZE=16), this means GROUP_SIZE=128.
// Each thread quantizes 16 bf16/fp16 values, using 2 or 8 threads per group.
template <typename T, typename DST_DTYPE, int GROUP_SIZE, int kGroupsPerBlockX,
int kRowsPerBlock>
__global__ void per_token_group_quant_8bit_packed_register_kernel(
@@ -305,9 +304,9 @@ __global__ void per_token_group_quant_8bit_packed_register_kernel(
const int groups_per_row, const int mn, const int output_q_mn_extent,
const int tma_aligned_mn, const int64_t num_scale_elems, const float eps,
const float min_8bit, const float max_8bit) {
static_assert(GROUP_SIZE == 128, "fast path supports GROUP_SIZE==128");
constexpr int THREADS_PER_GROUP = 8;
static_assert(GROUP_SIZE == 32 || GROUP_SIZE == 128);
constexpr int VEC_SIZE = 32 / sizeof(T); // 16 for bf16/fp16
constexpr int THREADS_PER_GROUP = GROUP_SIZE / VEC_SIZE;
static_assert(GROUP_SIZE == THREADS_PER_GROUP * VEC_SIZE,
"GROUP_SIZE must equal THREADS_PER_GROUP * VEC_SIZE");
static_assert(32 % THREADS_PER_GROUP == 0,
@@ -363,20 +362,20 @@ __global__ void per_token_group_quant_8bit_packed_register_kernel(
}
}
// 8-lane subgroup shuffle reduce (octet of the warp). The mask selects the
// 8 lanes within the warp that share a group.
#ifdef USE_ROCM
const int lane_in_wave = threadIdx.x % warpSize;
const unsigned long long mask = 0xFFull << (lane_in_wave & ~7);
local_absmax = fmaxf(local_absmax, __shfl_xor_sync(mask, local_absmax, 4, 8));
local_absmax = fmaxf(local_absmax, __shfl_xor_sync(mask, local_absmax, 2, 8));
local_absmax = fmaxf(local_absmax, __shfl_xor_sync(mask, local_absmax, 1, 8));
const unsigned long long mask = ((1ull << THREADS_PER_GROUP) - 1)
<< (lane_in_wave & ~(THREADS_PER_GROUP - 1));
#else
unsigned mask = 0xffu << (threadIdx.x & 24u);
local_absmax = fmaxf(local_absmax, __shfl_xor_sync(mask, local_absmax, 4));
local_absmax = fmaxf(local_absmax, __shfl_xor_sync(mask, local_absmax, 2));
local_absmax = fmaxf(local_absmax, __shfl_xor_sync(mask, local_absmax, 1));
const unsigned mask = ((1u << THREADS_PER_GROUP) - 1)
<< ((threadIdx.x % 32) & ~(THREADS_PER_GROUP - 1));
#endif
#pragma unroll
for (int offset = THREADS_PER_GROUP / 2; offset > 0; offset /= 2) {
local_absmax =
fmaxf(local_absmax,
__shfl_xor_sync(mask, local_absmax, offset, THREADS_PER_GROUP));
}
float y_s = local_absmax / max_8bit;
y_s = fmaxf(y_s, 1e-10f);
@@ -455,15 +454,15 @@ __global__ void per_token_group_quant_8bit_packed_register_kernel(
}
// Public entry point: register-resident packed quant kernel.
// Constraints: group_size == 128 and bf16/fp16 input.
// Constraints: group_size in {32, 128} and bf16/fp16 input.
void per_token_group_quant_8bit_packed(const torch::stable::Tensor& input,
torch::stable::Tensor& output_q,
torch::stable::Tensor& output_s_packed,
int64_t group_size, double eps,
double min_8bit, double max_8bit) {
STD_TORCH_CHECK(group_size == 128,
STD_TORCH_CHECK(group_size == 32 || group_size == 128,
"per_token_group_quant_8bit_packed only supports "
"group_size==128, got ",
"group_size in {32, 128}, got ",
group_size, ".");
const auto in_dtype = input.scalar_type();
STD_TORCH_CHECK(
@@ -512,7 +511,7 @@ void per_token_group_quant_8bit_packed(const torch::stable::Tensor& input,
input.get_device_index());
cudaStream_t stream = get_current_cuda_stream();
constexpr int THREADS_PER_GROUP = 8;
const int threads_per_group = group_size / 16;
const int64_t padded_groups_per_row = k_num_packed_sfk * 4;
const int64_t num_scale_elems = mn + (k_num_packed_sfk - 1) * tma_aligned_mn;
@@ -523,7 +522,7 @@ void per_token_group_quant_8bit_packed(const torch::stable::Tensor& input,
const int ry = 16 / kx;
const int64_t row_blocks = (tma_aligned_mn + ry - 1) / ry;
const int64_t sf_k_blocks = padded_groups_per_row / kx;
const int num_threads = (kx * ry) * THREADS_PER_GROUP;
const int num_threads = (kx * ry) * threads_per_group;
// CUDA caps grid.x at 2^31 - 1 and grid.y at 2^16 - 1 (65535).
constexpr int64_t kMaxGridDimYZ = 65535;
STD_TORCH_CHECK(row_blocks <= static_cast<int64_t>(INT32_MAX) &&
@@ -536,63 +535,72 @@ void per_token_group_quant_8bit_packed(const torch::stable::Tensor& input,
// PDL (Programmatic Dependent Launch) is NVIDIA-only; ROCm/HIP has no
// equivalent launch attribute, so fall back to a classic launch there.
#ifndef USE_ROCM
#define LAUNCH_REG_KERNEL_INST(T, DST_DTYPE, KX, RY) \
do { \
cudaLaunchConfig_t config = {}; \
config.gridDim = dim3(static_cast<unsigned int>(row_blocks), \
static_cast<unsigned int>(sf_k_blocks)); \
config.blockDim = dim3(num_threads); \
config.dynamicSmemBytes = 0; \
config.stream = stream; \
cudaLaunchAttribute attrs[1]; \
attrs[0].id = cudaLaunchAttributeProgrammaticStreamSerialization; \
attrs[0].val.programmaticStreamSerializationAllowed = 1; \
config.numAttrs = 1; \
config.attrs = attrs; \
cudaLaunchKernelEx( \
&config, \
per_token_group_quant_8bit_packed_register_kernel<T, DST_DTYPE, 128, \
KX, RY>, \
static_cast<const T*>(input.data_ptr()), output_q.data_ptr(), \
reinterpret_cast<unsigned int*>(output_s_packed.data_ptr()), \
static_cast<int>(padded_groups_per_row), \
static_cast<int>(groups_per_row), static_cast<int>(mn), \
static_cast<int>(output_q_mn_extent), \
static_cast<int>(tma_aligned_mn), num_scale_elems, \
static_cast<float>(eps), static_cast<float>(min_8bit), \
static_cast<float>(max_8bit)); \
#define LAUNCH_REG_KERNEL_INST(T, DST_DTYPE, GROUP_SIZE, KX, RY) \
do { \
cudaLaunchConfig_t config = {}; \
config.gridDim = dim3(static_cast<unsigned int>(row_blocks), \
static_cast<unsigned int>(sf_k_blocks)); \
config.blockDim = dim3(num_threads); \
config.dynamicSmemBytes = 0; \
config.stream = stream; \
cudaLaunchAttribute attrs[1]; \
attrs[0].id = cudaLaunchAttributeProgrammaticStreamSerialization; \
attrs[0].val.programmaticStreamSerializationAllowed = 1; \
config.numAttrs = 1; \
config.attrs = attrs; \
cudaLaunchKernelEx( \
&config, \
per_token_group_quant_8bit_packed_register_kernel< \
T, DST_DTYPE, GROUP_SIZE, KX, RY>, \
static_cast<const T*>(input.data_ptr()), output_q.data_ptr(), \
reinterpret_cast<unsigned int*>(output_s_packed.data_ptr()), \
static_cast<int>(padded_groups_per_row), \
static_cast<int>(groups_per_row), static_cast<int>(mn), \
static_cast<int>(output_q_mn_extent), \
static_cast<int>(tma_aligned_mn), num_scale_elems, \
static_cast<float>(eps), static_cast<float>(min_8bit), \
static_cast<float>(max_8bit)); \
} while (0)
#else
#define LAUNCH_REG_KERNEL_INST(T, DST_DTYPE, KX, RY) \
do { \
dim3 grid(static_cast<unsigned int>(row_blocks), \
static_cast<unsigned int>(sf_k_blocks)); \
dim3 block(num_threads); \
per_token_group_quant_8bit_packed_register_kernel<T, DST_DTYPE, 128, KX, \
RY> \
<<<grid, block, 0, stream>>>( \
static_cast<const T*>(input.data_ptr()), output_q.data_ptr(), \
reinterpret_cast<unsigned int*>(output_s_packed.data_ptr()), \
static_cast<int>(padded_groups_per_row), \
static_cast<int>(groups_per_row), static_cast<int>(mn), \
static_cast<int>(output_q_mn_extent), \
static_cast<int>(tma_aligned_mn), num_scale_elems, \
static_cast<float>(eps), static_cast<float>(min_8bit), \
static_cast<float>(max_8bit)); \
#define LAUNCH_REG_KERNEL_INST(T, DST_DTYPE, GROUP_SIZE, KX, RY) \
do { \
dim3 grid(static_cast<unsigned int>(row_blocks), \
static_cast<unsigned int>(sf_k_blocks)); \
dim3 block(num_threads); \
per_token_group_quant_8bit_packed_register_kernel<T, DST_DTYPE, \
GROUP_SIZE, KX, RY> \
<<<grid, block, 0, stream>>>( \
static_cast<const T*>(input.data_ptr()), output_q.data_ptr(), \
reinterpret_cast<unsigned int*>(output_s_packed.data_ptr()), \
static_cast<int>(padded_groups_per_row), \
static_cast<int>(groups_per_row), static_cast<int>(mn), \
static_cast<int>(output_q_mn_extent), \
static_cast<int>(tma_aligned_mn), num_scale_elems, \
static_cast<float>(eps), static_cast<float>(min_8bit), \
static_cast<float>(max_8bit)); \
} while (0)
#endif
#define LAUNCH_REG_KERNEL(T, DST_DTYPE) \
do { \
if (kx == 16) { \
LAUNCH_REG_KERNEL_INST(T, DST_DTYPE, 16, 1); \
} else if (kx == 8) { \
LAUNCH_REG_KERNEL_INST(T, DST_DTYPE, 8, 2); \
} else if (kx == 4) { \
LAUNCH_REG_KERNEL_INST(T, DST_DTYPE, 4, 4); \
} else { \
STD_TORCH_CHECK(false, "Unsupported kx value ", kx); \
} \
#define LAUNCH_REG_KERNEL_GROUP_SIZE(T, DST_DTYPE, GROUP_SIZE) \
do { \
if (kx == 16) { \
LAUNCH_REG_KERNEL_INST(T, DST_DTYPE, GROUP_SIZE, 16, 1); \
} else if (kx == 8) { \
LAUNCH_REG_KERNEL_INST(T, DST_DTYPE, GROUP_SIZE, 8, 2); \
} else if (kx == 4) { \
LAUNCH_REG_KERNEL_INST(T, DST_DTYPE, GROUP_SIZE, 4, 4); \
} else { \
STD_TORCH_CHECK(false, "Unsupported kx value ", kx); \
} \
} while (0)
#define LAUNCH_REG_KERNEL(T, DST_DTYPE) \
do { \
if (group_size == 32) { \
LAUNCH_REG_KERNEL_GROUP_SIZE(T, DST_DTYPE, 32); \
} else { \
LAUNCH_REG_KERNEL_GROUP_SIZE(T, DST_DTYPE, 128); \
} \
} while (0)
VLLM_STABLE_DISPATCH_HALF_TYPES(
@@ -607,6 +615,7 @@ void per_token_group_quant_8bit_packed(const torch::stable::Tensor& input,
}));
#undef LAUNCH_REG_KERNEL
#undef LAUNCH_REG_KERNEL_GROUP_SIZE
#undef LAUNCH_REG_KERNEL_INST
}
+4 -3
View File
@@ -429,7 +429,8 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
"fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert("
"Tensor q_in, Tensor kv, Tensor! k_cache, "
"Tensor slot_mapping, Tensor position_ids, Tensor cos_sin_cache, "
"int q_head_padded, float eps, int cache_block_size) -> Tensor");
"int q_head_padded, float eps, int cache_block_size, "
"bool apply_q_norm=True) -> Tensor");
// FlashInfer V4 full-cache variants: write Q in place (bf16) or to a separate
// FP8 tensor, and KV into a contiguous 512-wide token-strided cache.
@@ -437,13 +438,13 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
"fused_deepseek_v4_qnorm_rope_kv_rope_full_cache_bf16_insert("
"Tensor! q, Tensor kv, Tensor! k_cache, Tensor slot_mapping, "
"Tensor position_ids, Tensor cos_sin_cache, float eps, "
"int cache_block_size) -> ()");
"int cache_block_size, bool apply_q_norm=True) -> ()");
ops.def(
"fused_deepseek_v4_qnorm_rope_kv_rope_full_cache_fp8_insert("
"Tensor q, Tensor kv, Tensor! q_fp8, Tensor! k_cache, "
"Tensor slot_mapping, Tensor position_ids, Tensor cos_sin_cache, "
"Tensor fp8_scale, Tensor q_fp8_scale_inv, float eps, "
"int cache_block_size) -> ()");
"int cache_block_size, bool apply_q_norm=True) -> ()");
// Kimi-K3 MLA epilogues: optional RoPE followed by concat/cache insertion.
ops.def(