[DSV4] Fuse norm and router for low latency scenario (#41263)

Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Signed-off-by: jeejeelee <jeejeelee@verda-b300-05.datacrunch.io>
Co-authored-by: jeejeelee <jeejeelee@verda-b300-05.datacrunch.io>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Jee Jee Li
2026-05-14 05:11:02 -07:00
committed by GitHub
co-authored by jeejeelee mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 1ea9401364
commit 0a65d46628
11 changed files with 815 additions and 43 deletions
+30
View File
@@ -0,0 +1,30 @@
/*
* Fused RMSNorm + router GEMV for DeepSeek V4.
*
* Computes in a single kernel:
* normed_x[m,k] = x[m,k] * rsqrt(mean(x[m]^2) + eps) * norm_weight[k]
* router_logits[m,n] = sum_k(normed_x[m,k] * gate_weight[n,k])
*
* The GEMV body mirrors the algorithm in csrc/moe/dsv3_router_gemm_*.cu
* (warp butterfly + smem cross-warp reduction, fp32 accumulation, PDL on
* SM90+). Blocks 0..kNumTokens-1 each materialize one token's normed_x
* row to global memory using the algebraic identity
* logits[m,n] = rsqrt[m] * sum_k(x[m,k] * nw[k] * gw[n,k])
* which lets every block produce its column of logits before normed_x
* exists in gmem.
*
* Logits output is fp32 only — DeepSeek V4 router gate is hard-coded to
* fp32 (vllm/model_executor/models/deepseek_v4.py:749).
*/
#pragma once
#include <cuda_bf16.h>
#include <cuda_runtime.h>
#include "dsv3_router_gemm_utils.h"
template <typename T, int kNumTokens, int kNumExperts, int kHiddenDim>
void invokeNormRouterGemm(float* logits, __nv_bfloat16* normed_x, T const* x,
T const* norm_weight, T const* gate_weight, float eps,
cudaStream_t stream);
+130
View File
@@ -0,0 +1,130 @@
/*
* TORCH op entry for the fused RMSNorm + router GEMV kernel
* (DeepSeek V4 Pro). This op is DSV4-Pro-specific: the kernel is
* instantiated only for ``num_experts == 384`` and ``hidden_dim ==
* 7168``. Other configurations (e.g. DSV4-Flash with H=4096) must
* fall back to the unfused ``rms_norm`` + ``dsv3_router_gemm`` path.
*/
#include <ATen/ATen.h>
#include <ATen/cuda/CUDAContext.h>
#include <torch/all.h>
#include <cuda_bf16.h>
#include <cuda_runtime.h>
#include "core/registration.h"
#include "dsv4_norm_router_gemm.h"
namespace {
// DSV4-Pro hard-coded shape constants. Renamed from the earlier
// ``kKimiK2NumExperts`` to avoid the misleading impression that this
// kernel targets Kimi K2 — 384 happens to match Kimi K2's gate but the
// intent here is DSV4-Pro.
constexpr int kDsv4NumExperts = 384;
constexpr int kDsv4HiddenDim = 7168;
template <int kBegin, int kEnd>
struct LoopUnroller {
static void unroll(int num_tokens, float* logits, __nv_bfloat16* normed_x,
__nv_bfloat16 const* x, __nv_bfloat16 const* norm_weight,
__nv_bfloat16 const* gate_weight, float eps,
cudaStream_t stream) {
if (num_tokens == kBegin) {
invokeNormRouterGemm<__nv_bfloat16, kBegin, kDsv4NumExperts,
kDsv4HiddenDim>(logits, normed_x, x, norm_weight,
gate_weight, eps, stream);
} else {
LoopUnroller<kBegin + 1, kEnd>::unroll(num_tokens, logits, normed_x, x,
norm_weight, gate_weight, eps,
stream);
}
}
};
template <int kEnd>
struct LoopUnroller<kEnd, kEnd> {
static void unroll(int num_tokens, float* logits, __nv_bfloat16* normed_x,
__nv_bfloat16 const* x, __nv_bfloat16 const* norm_weight,
__nv_bfloat16 const* gate_weight, float eps,
cudaStream_t stream) {
if (num_tokens == kEnd) {
invokeNormRouterGemm<__nv_bfloat16, kEnd, kDsv4NumExperts,
kDsv4HiddenDim>(logits, normed_x, x, norm_weight,
gate_weight, eps, stream);
} else {
throw std::invalid_argument(
"Invalid num_tokens, only supports 1 to 16 for "
"dsv4_norm_router_gemm");
}
}
};
} // namespace
void dsv4_norm_router_gemm(at::Tensor& logits, // [num_tokens, E] fp32
at::Tensor& normed_x, // [num_tokens, H] bf16
at::Tensor const& x, // [num_tokens, H] bf16
at::Tensor const& norm_weight, // [H] bf16
at::Tensor const& gate_weight, // [E, H] bf16
double eps) {
TORCH_CHECK(x.dim() == 2 && norm_weight.dim() == 1 && gate_weight.dim() == 2,
"x must be 2D, norm_weight 1D, gate_weight 2D");
TORCH_CHECK(logits.dim() == 2 && normed_x.dim() == 2,
"logits and normed_x must be 2D");
int const num_tokens = x.size(0);
int const hidden_dim = x.size(1);
int const num_experts = gate_weight.size(0);
TORCH_CHECK(hidden_dim == kDsv4HiddenDim,
"Expected hidden_dim=", kDsv4HiddenDim,
" (DSV4-Pro), but got hidden_dim=", hidden_dim);
TORCH_CHECK(gate_weight.size(1) == hidden_dim,
"gate_weight.shape[1] must equal x.shape[1]");
TORCH_CHECK(norm_weight.size(0) == hidden_dim,
"norm_weight.shape[0] must equal x.shape[1]");
TORCH_CHECK(num_experts == kDsv4NumExperts,
"Expected num_experts=", kDsv4NumExperts,
" (DSV4-Pro), but got num_experts=", num_experts);
TORCH_CHECK(num_tokens >= 1 && num_tokens <= 16,
"num_tokens must be in [1, 16] for dsv4_norm_router_gemm");
TORCH_CHECK(x.dtype() == at::kBFloat16, "x must be bf16");
TORCH_CHECK(norm_weight.dtype() == at::kBFloat16, "norm_weight must be bf16");
TORCH_CHECK(gate_weight.dtype() == at::kBFloat16, "gate_weight must be bf16");
TORCH_CHECK(normed_x.dtype() == at::kBFloat16, "normed_x must be bf16");
TORCH_CHECK(logits.dtype() == at::kFloat,
"logits must be float32 (DSV4 router output is hard-coded fp32)");
TORCH_CHECK(normed_x.size(0) == num_tokens && normed_x.size(1) == hidden_dim,
"normed_x must be [num_tokens, hidden_dim]");
TORCH_CHECK(logits.size(0) == num_tokens && logits.size(1) == num_experts,
"logits must be [num_tokens, num_experts]");
TORCH_CHECK(x.is_contiguous() && norm_weight.is_contiguous() &&
gate_weight.is_contiguous() && normed_x.is_contiguous() &&
logits.is_contiguous(),
"all tensors must be contiguous");
auto const sm = getSMVersion();
TORCH_CHECK(sm >= 90 && sm <= 103,
"dsv4_norm_router_gemm requires SM_90 <= CUDA ARCH <= SM_103");
cudaStream_t const stream = at::cuda::getCurrentCUDAStream();
auto* logits_ptr = reinterpret_cast<float*>(logits.mutable_data_ptr());
auto* nx_ptr = reinterpret_cast<__nv_bfloat16*>(normed_x.mutable_data_ptr());
auto* x_ptr = reinterpret_cast<__nv_bfloat16 const*>(x.data_ptr());
auto* nw_ptr = reinterpret_cast<__nv_bfloat16 const*>(norm_weight.data_ptr());
auto* gw_ptr = reinterpret_cast<__nv_bfloat16 const*>(gate_weight.data_ptr());
float const eps_f = static_cast<float>(eps);
LoopUnroller<1, 16>::unroll(num_tokens, logits_ptr, nx_ptr, x_ptr, nw_ptr,
gw_ptr, eps_f, stream);
}
TORCH_LIBRARY_IMPL_EXPAND(TORCH_EXTENSION_NAME, CUDA, m) {
m.impl("dsv4_norm_router_gemm", &dsv4_norm_router_gemm);
}
+249
View File
@@ -0,0 +1,249 @@
/*
* Fused RMSNorm + router GEMV for DeepSeek V4 (logits are fp32; bf16
* output is unsupported because DSV4 hard-codes fp32 logits). See
* dsv4_norm_router_gemm.h for the math.
*
* The GEMV body mirrors csrc/moe/dsv3_router_gemm_float_out.cu (warp
* butterfly reduction + smem cross-warp reduction, fp32 accumulation,
* 128-thread block, PDL on SM90+). RMSNorm is folded into the same
* pass via the identity
* logits[m,n] = rsqrt[m] * sum_k(x[m,k] * nw[k] * gw[n,k])
* so x is read exactly once per block during the GEMV phase. Blocks
* 0..kNumTokens-1 each materialize one row of normed_x for downstream
* experts / shared_experts to consume.
*/
#include <ATen/ATen.h>
#include <ATen/cuda/CUDAContext.h>
#include <cuda_bf16.h>
#include <cuda_runtime.h>
#include "dsv4_norm_router_gemm.h"
namespace {
// Convert 8 bf16 values packed in uint4 into 8 floats. Mirrors the helper
// in dsv3_router_gemm_float_out.cu (kept local so the dsv3 file stays
// untouched).
template <int VPT>
__device__ __forceinline__ void bf16_uint4_to_float8(uint4 const& vec,
float* dst) {
__nv_bfloat16* bf16_ptr =
reinterpret_cast<__nv_bfloat16*>(const_cast<uint4*>(&vec));
#pragma unroll
for (int i = 0; i < VPT; i++) {
dst[i] = __bfloat162float(bf16_ptr[i]);
}
}
template <typename T, int kBlockSize, int VPT, int kNumTokens, int kNumExperts,
int kHiddenDim>
__global__ __launch_bounds__(128, 1) void norm_router_gemm_kernel(
float* __restrict__ logits, __nv_bfloat16* __restrict__ normed_x,
T const* __restrict__ x, T const* __restrict__ norm_weight,
T const* __restrict__ gate_weight, float eps) {
static_assert(kBlockSize == 128, "kernel assumes blockDim.x == 128");
static_assert(kHiddenDim % (VPT * kBlockSize) == 0,
"kHiddenDim must be a multiple of VPT * kBlockSize");
int const n_idx = blockIdx.x;
int const tid = threadIdx.x;
constexpr int kWarpSize = 32;
constexpr int kNumWarps = kBlockSize / kWarpSize;
constexpr int k_elems_per_iter = VPT * kBlockSize;
constexpr int k_iterations = kHiddenDim / k_elems_per_iter;
T const* gw_col = gate_weight + n_idx * kHiddenDim;
// Per-thread accumulators — fp32 throughout, matching dsv3 / layernorm.
float partial[kNumTokens] = {};
float ss[kNumTokens] = {};
// Cross-warp reduction scratch.
__shared__ float sm_partial[kNumTokens][kNumWarps];
__shared__ float sm_ss[kNumTokens][kNumWarps];
__shared__ float s_rsqrt[kNumTokens];
int k_bases[k_iterations];
#pragma unroll
for (int ki = 0; ki < k_iterations; ki++) {
k_bases[ki] = ki * k_elems_per_iter + tid * VPT;
}
#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900))
asm volatile("griddepcontrol.wait;");
#endif
// ---- Phase 1: single pass over x, accumulate partial GEMV and ss. ----
#pragma unroll
for (int ki = 0; ki < k_iterations; ki++) {
int const k_base = k_bases[ki];
uint4 nw_vec = *reinterpret_cast<uint4 const*>(norm_weight + k_base);
float nw_f[VPT];
bf16_uint4_to_float8<VPT>(nw_vec, nw_f);
uint4 b_vec = *reinterpret_cast<uint4 const*>(gw_col + k_base);
float b_f[VPT];
bf16_uint4_to_float8<VPT>(b_vec, b_f);
#pragma unroll
for (int m = 0; m < kNumTokens; m++) {
uint4 a_vec =
*reinterpret_cast<uint4 const*>(x + m * kHiddenDim + k_base);
float a_f[VPT];
bf16_uint4_to_float8<VPT>(a_vec, a_f);
#pragma unroll
for (int k = 0; k < VPT; k++) {
float a = a_f[k];
ss[m] += a * a;
partial[m] += a * nw_f[k] * b_f[k];
}
}
}
// ---- Phase 2: warp butterfly reduction for both ss[] and partial[]. ----
int const warpId = tid / kWarpSize;
int const laneId = tid % kWarpSize;
#pragma unroll
for (int m = 0; m < kNumTokens; m++) {
float p = partial[m];
float s = ss[m];
p += __shfl_xor_sync(0xffffffff, p, 16);
s += __shfl_xor_sync(0xffffffff, s, 16);
p += __shfl_xor_sync(0xffffffff, p, 8);
s += __shfl_xor_sync(0xffffffff, s, 8);
p += __shfl_xor_sync(0xffffffff, p, 4);
s += __shfl_xor_sync(0xffffffff, s, 4);
p += __shfl_xor_sync(0xffffffff, p, 2);
s += __shfl_xor_sync(0xffffffff, s, 2);
p += __shfl_xor_sync(0xffffffff, p, 1);
s += __shfl_xor_sync(0xffffffff, s, 1);
if (laneId == 0) {
sm_partial[m][warpId] = p;
sm_ss[m][warpId] = s;
}
}
__syncthreads();
// ---- Phase 3: tid 0 finalises the reduction, writes logits, stashes
// rsqrt[m] in smem for phase 4. ----
if (tid == 0) {
#pragma unroll
for (int m = 0; m < kNumTokens; m++) {
float p_sum = 0.0f;
float s_sum = 0.0f;
#pragma unroll
for (int w = 0; w < kNumWarps; w++) {
p_sum += sm_partial[m][w];
s_sum += sm_ss[m][w];
}
// Order matches layernorm_kernels.cu: rsqrtf(variance / H + eps).
// Use division (not multiply-by-reciprocal) to avoid an extra ULP
// mismatch with the reference RMSNorm.
float rs = rsqrtf(s_sum / static_cast<float>(kHiddenDim) + eps);
s_rsqrt[m] = rs;
logits[m * kNumExperts + n_idx] = p_sum * rs;
}
}
__syncthreads();
// ---- Phase 4: spread normed_x writes across blocks 0..kNumTokens-1.
// Each writer block handles exactly one token row,
// avoiding the long tail of block 0 doing all M rows.
// Every block has every token's rsqrt[] in s_rsqrt
// already (computed independently in phase 3), so no
// cross-block synchronization is required. ----
if (n_idx < kNumTokens) {
int const m_writer = n_idx;
float const rs = s_rsqrt[m_writer];
__nv_bfloat16 const* x_row = x + m_writer * kHiddenDim;
__nv_bfloat16* normed_row = normed_x + m_writer * kHiddenDim;
#pragma unroll
for (int ki = 0; ki < k_iterations; ki++) {
int const k_base = k_bases[ki];
uint4 nw_vec = *reinterpret_cast<uint4 const*>(norm_weight + k_base);
float nw_f[VPT];
bf16_uint4_to_float8<VPT>(nw_vec, nw_f);
uint4 a_vec = *reinterpret_cast<uint4 const*>(x_row + k_base);
float a_f[VPT];
bf16_uint4_to_float8<VPT>(a_vec, a_f);
uint4 normed_vec;
__nv_bfloat16* np = reinterpret_cast<__nv_bfloat16*>(&normed_vec);
#pragma unroll
for (int k = 0; k < VPT; k++) {
np[k] = __float2bfloat16(a_f[k] * rs * nw_f[k]);
}
*reinterpret_cast<uint4*>(normed_row + k_base) = normed_vec;
}
}
#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900))
asm volatile("griddepcontrol.launch_dependents;");
#endif
}
} // namespace
template <typename T, int kNumTokens, int kNumExperts, int kHiddenDim>
void invokeNormRouterGemm(float* logits, __nv_bfloat16* normed_x, T const* x,
T const* norm_weight, T const* gate_weight, float eps,
cudaStream_t stream) {
constexpr int VPT = 16 / sizeof(T);
constexpr int kBlockSize = 128;
cudaLaunchConfig_t config;
config.gridDim = kNumExperts;
config.blockDim = kBlockSize;
config.dynamicSmemBytes = 0;
config.stream = stream;
cudaLaunchAttribute attrs[1];
attrs[0].id = cudaLaunchAttributeProgrammaticStreamSerialization;
attrs[0].val.programmaticStreamSerializationAllowed = 1;
config.numAttrs = 1;
config.attrs = attrs;
cudaLaunchKernelEx(&config,
norm_router_gemm_kernel<T, kBlockSize, VPT, kNumTokens,
kNumExperts, kHiddenDim>,
logits, normed_x, x, norm_weight, gate_weight, eps);
}
// Template instantiations — DSV4-Pro is the only supported configuration:
// num_experts=384, hidden_dim=7168. Other shapes (e.g. DSV4-Flash with
// hidden_dim=4096) fall back to the unfused path on the Python side.
#define INSTANTIATE(M) \
template void invokeNormRouterGemm<__nv_bfloat16, M, 384, 7168>( \
float*, __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, \
__nv_bfloat16 const*, float, cudaStream_t);
INSTANTIATE(1)
INSTANTIATE(2)
INSTANTIATE(3)
INSTANTIATE(4)
INSTANTIATE(5)
INSTANTIATE(6)
INSTANTIATE(7)
INSTANTIATE(8)
INSTANTIATE(9)
INSTANTIATE(10)
INSTANTIATE(11)
INSTANTIATE(12)
INSTANTIATE(13)
INSTANTIATE(14)
INSTANTIATE(15)
INSTANTIATE(16)
#undef INSTANTIATE
+8
View File
@@ -75,4 +75,12 @@ void shuffle_rows(const torch::Tensor& input_tensor,
// Supports num_tokens in [1, 16], num_experts in {256, 384}, hidden_dim = 7168
void dsv3_router_gemm(torch::Tensor& output, const torch::Tensor& mat_a,
const torch::Tensor& mat_b);
// Fused RMSNorm + router GEMV for DeepSeek V4. Produces both:
// normed_x[m,k] = x[m,k] * rsqrt(mean(x[m]^2) + eps) * norm_weight[k]
// logits[m,n] = sum_k(normed_x[m,k] * gate_weight[n,k])
// in a single kernel launch. Same dim/dtype constraints as dsv3_router_gemm.
void dsv4_norm_router_gemm(at::Tensor& logits, at::Tensor& normed_x,
at::Tensor const& x, at::Tensor const& norm_weight,
at::Tensor const& gate_weight, double eps);
#endif
+6
View File
@@ -135,6 +135,12 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, m) {
// DeepSeek V3 optimized router GEMM for SM90+
m.def("dsv3_router_gemm(Tensor! output, Tensor mat_a, Tensor mat_b) -> ()");
// conditionally compiled so impl registration is in source file
// DeepSeek V4 fused RMSNorm + router GEMV for SM90+
m.def(
"dsv4_norm_router_gemm(Tensor! logits, Tensor! normed_x, Tensor x, "
"Tensor norm_weight, Tensor gate_weight, float eps) -> ()");
// conditionally compiled so impl registration is in source file
#endif
}