diff --git a/CMakeLists.txt b/CMakeLists.txt index 22a169b566..3198466566 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -727,6 +727,23 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP") "(requires SM90+ and CUDA >= 12.0).") endif() + # BF16 skinny GEMM (M<=32; weight-bandwidth-bound decode shapes). + # Requires SM90+. + cuda_archs_sm90plus(BF16_SKINNY_GEMM_ARCHS "${CUDA_ARCHS}") + if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 12.0 AND BF16_SKINNY_GEMM_ARCHS) + set(BF16_SKINNY_GEMM_SRCS + "csrc/libtorch_stable/bf16_skinny_gemm_entry.cu" + "csrc/libtorch_stable/bf16_skinny_gemm.cu") + set_gencode_flags_for_srcs( + SRCS "${BF16_SKINNY_GEMM_SRCS}" + CUDA_ARCHS "${BF16_SKINNY_GEMM_ARCHS}") + list(APPEND VLLM_STABLE_EXT_SRC "${BF16_SKINNY_GEMM_SRCS}") + message(STATUS "Building bf16_skinny_gemm for archs: ${BF16_SKINNY_GEMM_ARCHS}") + else() + message(STATUS "Not building bf16_skinny_gemm as no compatible archs found " + "(requires SM90+ and CUDA >= 12.0).") + endif() + # Only build AllSpark kernels if we are building for at least some compatible archs. cuda_archs_loose_intersection(ALLSPARK_ARCHS "8.0;8.6;8.7;8.9" "${CUDA_ARCHS}") if (ALLSPARK_ARCHS) diff --git a/csrc/libtorch_stable/bf16_skinny_gemm.cu b/csrc/libtorch_stable/bf16_skinny_gemm.cu new file mode 100644 index 0000000000..fef90d29d6 --- /dev/null +++ b/csrc/libtorch_stable/bf16_skinny_gemm.cu @@ -0,0 +1,262 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project +// +// Skinny GEMM: activation(bf16) x weight(bf16)^T -> bf16, for decode-time +// M <= 32 with a large reduction dim. Replaces cuBLAS splitK (GEMM + +// splitKreduce) with a single block-per-output-column kernel; these shapes +// are weight-bandwidth-bound, so one coalesced pass over the weight at +// fp32 accumulation is optimal. Adapted from fp32_router_gemm.cu. +// +// First user: the DeepSeek-V32/GLM-5.2 MTP eh_proj (K=2*hidden=12288, +// N=hidden/TP), whose cuBLAS splitK pick costs ~34.6us vs the ~19us +// bandwidth floor per replicated read (and ~4us once column-parallel). + +#include +#include + +// --------------------------------------------------------------------------- +// Load helpers (8 x bf16 = one uint4 load, converted to fp32) +// --------------------------------------------------------------------------- + +namespace skinny { + +constexpr int VPT = 8; // bf16 values per thread per load + +__device__ __forceinline__ void load_bf16x8(__nv_bfloat16 const* ptr, + float* dst) { + uint4 v = *reinterpret_cast(ptr); + __nv_bfloat16 const* p = reinterpret_cast<__nv_bfloat16 const*>(&v); +#pragma unroll + for (int i = 0; i < VPT; i++) dst[i] = __bfloat162float(p[i]); +} + +// Streaming variant for the weight: each row is read exactly once across the +// whole grid, so bypass L2 residency (evict-first). Measured -1.4us at M=1. +__device__ __forceinline__ void load_bf16x8_cs(__nv_bfloat16 const* ptr, + float* dst) { + uint4 v = __ldcs(reinterpret_cast(ptr)); + __nv_bfloat16 const* p = reinterpret_cast<__nv_bfloat16 const*>(&v); +#pragma unroll + for (int i = 0; i < VPT; i++) dst[i] = __bfloat162float(p[i]); +} + +// --------------------------------------------------------------------------- +// Kernel: each block computes kNPB output columns for all kNumTokens rows. +// grid = kN / kNPB, block = kBlockSize threads. K is reduced VPT elements +// per thread per iteration; fp32 accumulation, warp butterfly + smem +// finalize, bf16 store. +// --------------------------------------------------------------------------- + +template +__global__ __launch_bounds__(kBlockSize, 1) void bf16_skinny_gemm_kernel( + __nv_bfloat16* out, __nv_bfloat16 const* mat_a, __nv_bfloat16 const* mat_b, + int64_t out_stride) { + constexpr int k_elems_per_iter = VPT * kBlockSize; + constexpr int k_iterations = kK / k_elems_per_iter; + static_assert(kK % k_elems_per_iter == 0); + constexpr int kWarpSize = 32; + constexpr int kNumWarps = kBlockSize / kWarpSize; + + int const n_base = blockIdx.x * kNPB; + int const tid = threadIdx.x; + int const warpId = tid / kWarpSize; + int const laneId = tid % kWarpSize; + + float acc[kNumTokens][kNPB] = {}; + __shared__ float sm_reduction[kNumTokens][kNPB][kNumWarps]; + + // Register prefetch (kPF > 0): W does not depend on the predecessor, so + // the first kPF iterations' weight chunks are loaded raw BEFORE the + // dependency sync; with a PDL-releasing producer (fused_eh_norm fires + // gdc_launch_dependents early) these DRAM round trips overlap the norm. + // Pair-measured on B300 (norm+gemm in one graph, full 6144x12288): + // M=1 pf2 28.43us vs pf0 29.00us; deeper prefetch or M >= 2 regresses + // (register pressure), hence the per-M selection in the launcher. + uint4 w_pre[kPF > 0 ? kPF : 1][kNPB]; +#pragma unroll + for (int pf = 0; pf < kPF; pf++) { +#pragma unroll + for (int n = 0; n < kNPB; n++) { + w_pre[pf][n] = + *reinterpret_cast(mat_b + (size_t)(n_base + n) * kK + + pf * k_elems_per_iter + tid * VPT); + } + } + +#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900) + cudaGridDependencySynchronize(); +#endif + +#pragma unroll + for (int ki = 0; ki < k_iterations; ki++) { + int const k_base = ki * k_elems_per_iter + tid * VPT; + + float b_float[kNPB][VPT]; + if (ki < kPF) { +#pragma unroll + for (int n = 0; n < kNPB; n++) { + __nv_bfloat16 const* p = + reinterpret_cast<__nv_bfloat16 const*>(&w_pre[ki][n]); +#pragma unroll + for (int v = 0; v < VPT; v++) b_float[n][v] = __bfloat162float(p[v]); + } + } else { +#pragma unroll + for (int n = 0; n < kNPB; n++) { + load_bf16x8_cs(mat_b + (size_t)(n_base + n) * kK + k_base, b_float[n]); + } + } + +#pragma unroll + for (int m = 0; m < kNumTokens; m++) { + float a_float[VPT]; + load_bf16x8(mat_a + (size_t)m * kK + k_base, a_float); +#pragma unroll + for (int n = 0; n < kNPB; n++) { +#pragma unroll + for (int k = 0; k < VPT; k++) { + acc[m][n] += a_float[k] * b_float[n][k]; + } + } + } + } + + // Warp-level butterfly reduction +#pragma unroll + for (int m = 0; m < kNumTokens; m++) { +#pragma unroll + for (int n = 0; n < kNPB; n++) { + float sum = acc[m][n]; + sum += __shfl_xor_sync(0xffffffff, sum, 16); + sum += __shfl_xor_sync(0xffffffff, sum, 8); + sum += __shfl_xor_sync(0xffffffff, sum, 4); + sum += __shfl_xor_sync(0xffffffff, sum, 2); + sum += __shfl_xor_sync(0xffffffff, sum, 1); + if (laneId == 0) sm_reduction[m][n][warpId] = sum; + } + } + + __syncthreads(); + + // Parallel finalize: one thread per (m, n) output. + for (int idx = tid; idx < kNumTokens * kNPB; idx += kBlockSize) { + int const m = idx / kNPB; + int const n = idx % kNPB; + float final_sum = 0.0f; +#pragma unroll + for (int w = 0; w < kNumWarps; w++) final_sum += sm_reduction[m][n][w]; + out[(size_t)m * out_stride + n_base + n] = __float2bfloat16(final_sum); + } + + // Trigger after our stores: harmless hardening, not a guarantee — the + // trigger only permits dependent-launch scheduling and carries no memory + // visibility semantics, so a PDL consumer must still gridsync before + // reading our output. Every current consumer is a plain launch (full + // stream order); firing late just avoids an unnecessarily early launch + // window and matches fp32_router_gemm.cu. +#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900) + cudaTriggerProgrammaticLaunchCompletion(); +#endif +} + +} // namespace skinny + +// --------------------------------------------------------------------------- +// Launcher +// --------------------------------------------------------------------------- + +template +void invokeBf16SkinnyGemm(__nv_bfloat16* output, __nv_bfloat16 const* mat_a, + __nv_bfloat16 const* mat_b, int64_t out_stride, + cudaStream_t stream) { + static_assert(kN % kNPB == 0); + // Weight prefetch depth: only M=1 measured a win (see kernel comment). + constexpr int kPF = (kNumTokens == 1) ? 2 : 0; + cudaLaunchConfig_t config; + config.gridDim = kN / kNPB; + config.blockDim = kBlockSize; + config.dynamicSmemBytes = 0; + config.stream = stream; + cudaLaunchAttribute attrs[1]; + attrs[0].id = cudaLaunchAttributeProgrammaticStreamSerialization; + attrs[0].val.programmaticStreamSerializationAllowed = 1; + config.numAttrs = 1; + config.attrs = attrs; + cudaLaunchKernelEx(&config, + skinny::bf16_skinny_gemm_kernel, + output, mat_a, mat_b, out_stride); +} + +// --------------------------------------------------------------------------- +// Explicit instantiations. M = 1..32; (N, K) pairs: +// (768, 12288) eh_proj shard, TP8 +// (1536, 12288) eh_proj shard, TP4 +// (6144, 12288) eh_proj unsharded +// kNPB (B300 sweep, M=1): 6144 -> 2 (3072 blocks, 23.2us vs 25.3 at kNPB=8; +// narrow blocks minimize wave quantization); shards 768/1536 keep 4. +// --------------------------------------------------------------------------- + +#define INSTANTIATE(M, NPB, N, K) \ + template void invokeBf16SkinnyGemm<128, NPB, M, N, K>( \ + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, int64_t, \ + cudaStream_t); + +#define INSTANTIATE_ALL_M(NPB, N, K) \ + INSTANTIATE(1, NPB, N, K) \ + INSTANTIATE(2, NPB, N, K) \ + INSTANTIATE(3, NPB, N, K) \ + INSTANTIATE(4, NPB, N, K) \ + INSTANTIATE(5, NPB, N, K) \ + INSTANTIATE(6, NPB, N, K) \ + INSTANTIATE(7, NPB, N, K) \ + INSTANTIATE(8, NPB, N, K) \ + INSTANTIATE(9, NPB, N, K) \ + INSTANTIATE(10, NPB, N, K) \ + INSTANTIATE(11, NPB, N, K) \ + INSTANTIATE(12, NPB, N, K) \ + INSTANTIATE(13, NPB, N, K) \ + INSTANTIATE(14, NPB, N, K) \ + INSTANTIATE(15, NPB, N, K) \ + INSTANTIATE(16, NPB, N, K) \ + INSTANTIATE(17, NPB, N, K) \ + INSTANTIATE(18, NPB, N, K) \ + INSTANTIATE(19, NPB, N, K) \ + INSTANTIATE(20, NPB, N, K) \ + INSTANTIATE(21, NPB, N, K) \ + INSTANTIATE(22, NPB, N, K) \ + INSTANTIATE(23, NPB, N, K) \ + INSTANTIATE(24, NPB, N, K) \ + INSTANTIATE(25, NPB, N, K) \ + INSTANTIATE(26, NPB, N, K) \ + INSTANTIATE(27, NPB, N, K) \ + INSTANTIATE(28, NPB, N, K) \ + INSTANTIATE(29, NPB, N, K) \ + INSTANTIATE(30, NPB, N, K) \ + INSTANTIATE(31, NPB, N, K) \ + INSTANTIATE(32, NPB, N, K) + +INSTANTIATE_ALL_M(4, 768, 12288) +INSTANTIATE_ALL_M(4, 1536, 12288) +INSTANTIATE_ALL_M(2, 6144, 12288) +// LL-mode (M<=8 wiring guard) backbone shapes, B300 sweep vs cuBLAS: +// q_b_proj (2048, 2048): 1.67x/1.29x/1.15x at M=4/6/8 (NPB=4 within +// 0.1us of per-M best) +// shared-expert gate_up (512, 6144): 1.95x/1.58x/1.40x at M=4/6/8 +// cuBLAS keeps qkv_a (2624,6144) and o_proj (6144,2048) — already at +// 3.4-3.8 TB/s there; the GEMV loses on activation re-reads. +INSTANTIATE_ALL_M(4, 2048, 2048) +INSTANTIATE_ALL_M(4, 512, 6144) +// fused_qkv_a (2624, 6144), 32MB: skinny wins ONLY at M<=2 (B300: M=1 +// 6.99us vs cuBLAS 9.21 = 1.32x, M=2 1.24x; M>=4 cuBLAS holds at 3.5TB/s +// and every alternative loses — cublasLt top-8 3.6TB/s wall, DeepGEMM +// 0.71x, wmma+cp.async custom 0.30x pending a TMA rewrite). +INSTANTIATE_ALL_M(4, 2624, 6144) +// DSv3.2 (TP8) siblings of the GLM shapes above, same dual-chip matrix: +// fused_qkv_a (2112, 7168), 30MB: wins M<=2 (M=1 1.30-1.34x) +// MTP eh_proj (7168, 14336), 205MB: wins M<=2 (M=1 1.12-1.16x) +INSTANTIATE_ALL_M(4, 2112, 7168) +INSTANTIATE_ALL_M(2, 7168, 14336) + +#undef INSTANTIATE_ALL_M +#undef INSTANTIATE diff --git a/csrc/libtorch_stable/bf16_skinny_gemm_entry.cu b/csrc/libtorch_stable/bf16_skinny_gemm_entry.cu new file mode 100644 index 0000000000..60b9e8b60c --- /dev/null +++ b/csrc/libtorch_stable/bf16_skinny_gemm_entry.cu @@ -0,0 +1,170 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +#include +#include +#include + +#include "core/registration.h" +#include "libtorch_stable/torch_utils.h" + +#include +#include + +#include + +namespace { + +inline int getSMVersion() { + auto* props = get_device_prop(); + return props->major * 10 + props->minor; +} + +} // namespace + +static constexpr int SKINNY_MAX_TOKENS = 32; + +// Supported (N, K) pairs (must match the instantiations in +// bf16_skinny_gemm.cu): eh_proj shard TP8 / TP4 / unsharded. +static inline bool bf16_skinny_gemm_supported(int n, int k) { + if (k == 12288 && (n == 768 || n == 1536 || n == 6144)) return true; + // LL-mode backbone shapes (wire callers with an M <= 8 guard; the GEMV + // family loses to cuBLAS at larger M). + if (k == 2048 && n == 2048) return true; // q_b_proj (TP8) + if (k == 6144 && n == 2624) return true; // fused_qkv_a (wire M <= 2 only) + if (k == 7168 && n == 2112) return true; // DSv3.2 fused_qkv_a (M <= 2) + if (k == 14336 && n == 7168) return true; // DSv3.2 eh_proj (M <= 2) + if (k == 6144 && n == 512) return true; // shared-expert gate_up (TP8) + return false; +} + +// Forward declarations - template params must match bf16_skinny_gemm.cu +template +void invokeBf16SkinnyGemm(__nv_bfloat16* output, __nv_bfloat16 const* mat_a, + __nv_bfloat16 const* mat_b, int64_t out_stride, + cudaStream_t stream); + +template +struct SkinnyLoopUnroller { + static void unroll(int num_tokens, __nv_bfloat16* output, + __nv_bfloat16 const* mat_a, __nv_bfloat16 const* mat_b, + int64_t out_stride, cudaStream_t stream) { + if (num_tokens == kBegin) { + invokeBf16SkinnyGemm<128, kNPB, kBegin, kN, kK>(output, mat_a, mat_b, + out_stride, stream); + } else { + SkinnyLoopUnroller::unroll( + num_tokens, output, mat_a, mat_b, out_stride, stream); + } + } +}; + +template +struct SkinnyLoopUnroller { + static void unroll(int num_tokens, __nv_bfloat16* output, + __nv_bfloat16 const* mat_a, __nv_bfloat16 const* mat_b, + int64_t out_stride, cudaStream_t stream) { + if (num_tokens == kEnd) { + invokeBf16SkinnyGemm<128, kNPB, kEnd, kN, kK>(output, mat_a, mat_b, + out_stride, stream); + } else { + throw std::invalid_argument( + "bf16_skinny_gemm: num_tokens must be in [1, 32]"); + } + } +}; + +static void dispatchBf16SkinnyGemm(int n, int k, int num_tokens, + __nv_bfloat16* output, + __nv_bfloat16 const* mat_a, + __nv_bfloat16 const* mat_b, + int64_t out_stride, cudaStream_t stream) { + if (n == 768 && k == 12288) { + SkinnyLoopUnroller<4, 768, 12288, 1, SKINNY_MAX_TOKENS>::unroll( + num_tokens, output, mat_a, mat_b, out_stride, stream); + } else if (n == 1536 && k == 12288) { + SkinnyLoopUnroller<4, 1536, 12288, 1, SKINNY_MAX_TOKENS>::unroll( + num_tokens, output, mat_a, mat_b, out_stride, stream); + } else if (n == 6144 && k == 12288) { + SkinnyLoopUnroller<2, 6144, 12288, 1, SKINNY_MAX_TOKENS>::unroll( + num_tokens, output, mat_a, mat_b, out_stride, stream); + } else if (n == 2048 && k == 2048) { + SkinnyLoopUnroller<4, 2048, 2048, 1, SKINNY_MAX_TOKENS>::unroll( + num_tokens, output, mat_a, mat_b, out_stride, stream); + } else if (n == 2624 && k == 6144) { + SkinnyLoopUnroller<4, 2624, 6144, 1, SKINNY_MAX_TOKENS>::unroll( + num_tokens, output, mat_a, mat_b, out_stride, stream); + } else if (n == 2112 && k == 7168) { + SkinnyLoopUnroller<4, 2112, 7168, 1, SKINNY_MAX_TOKENS>::unroll( + num_tokens, output, mat_a, mat_b, out_stride, stream); + } else if (n == 7168 && k == 14336) { + SkinnyLoopUnroller<2, 7168, 14336, 1, SKINNY_MAX_TOKENS>::unroll( + num_tokens, output, mat_a, mat_b, out_stride, stream); + } else if (n == 512 && k == 6144) { + SkinnyLoopUnroller<4, 512, 6144, 1, SKINNY_MAX_TOKENS>::unroll( + num_tokens, output, mat_a, mat_b, out_stride, stream); + } else { + throw std::invalid_argument("bf16_skinny_gemm: unsupported (N, K) pair"); + } +} + +void bf16_skinny_gemm( + torch::stable::Tensor& output, // [num_tokens, N] bf16 + torch::stable::Tensor const& mat_a, // [num_tokens, K] bf16 + torch::stable::Tensor const& mat_b // [N, K] bf16 +) { + STD_TORCH_CHECK(output.dim() == 2 && mat_a.dim() == 2 && mat_b.dim() == 2); + STD_TORCH_CHECK(output.is_cuda() && mat_a.is_cuda() && mat_b.is_cuda(), + "bf16_skinny_gemm: all tensors must be CUDA tensors"); + STD_TORCH_CHECK(output.get_device_index() == mat_a.get_device_index() && + output.get_device_index() == mat_b.get_device_index(), + "bf16_skinny_gemm: all tensors must be on the same device"); + STD_TORCH_CHECK(mat_a.is_contiguous() && mat_b.is_contiguous(), + "bf16_skinny_gemm: inputs must be contiguous"); + // Output may be a column-slice view of a wider padded buffer: unit column + // stride, row stride >= N (rows must not overlap). + STD_TORCH_CHECK(output.stride(1) == 1, + "bf16_skinny_gemm: output columns must be contiguous"); + + const int num_tokens = mat_a.size(0); + const int n = mat_b.size(0); + const int k = mat_a.size(1); + + STD_TORCH_CHECK(output.size(0) == num_tokens && output.size(1) == n, + "bf16_skinny_gemm: output must be [num_tokens, N]"); + STD_TORCH_CHECK(mat_b.size(1) == k, + "bf16_skinny_gemm: mat_a and mat_b must share K"); + STD_TORCH_CHECK(bf16_skinny_gemm_supported(n, k), + "bf16_skinny_gemm: unsupported (N, K) pair"); + const int64_t out_stride = output.stride(0); + STD_TORCH_CHECK(num_tokens <= 1 || out_stride >= n, + "bf16_skinny_gemm: output rows overlap"); + STD_TORCH_CHECK(num_tokens >= 0 && num_tokens <= SKINNY_MAX_TOKENS, + "bf16_skinny_gemm: num_tokens must be in [0, 32]"); + STD_TORCH_CHECK( + mat_a.scalar_type() == torch::headeronly::ScalarType::BFloat16 && + mat_b.scalar_type() == torch::headeronly::ScalarType::BFloat16 && + output.scalar_type() == torch::headeronly::ScalarType::BFloat16, + "bf16_skinny_gemm: all tensors must be bfloat16"); + + // Empty batch (e.g. an empty rank at a DP/PP boundary): nothing to compute. + if (num_tokens == 0) { + return; + } + + const torch::stable::accelerator::DeviceGuard device_guard( + mat_a.get_device_index()); + STD_TORCH_CHECK(getSMVersion() >= 90, "bf16_skinny_gemm: requires SM90+"); + + auto stream = get_current_cuda_stream(mat_a.get_device_index()); + dispatchBf16SkinnyGemm( + n, k, num_tokens, + reinterpret_cast<__nv_bfloat16*>(output.mutable_data_ptr()), + reinterpret_cast<__nv_bfloat16 const*>(mat_a.data_ptr()), + reinterpret_cast<__nv_bfloat16 const*>(mat_b.data_ptr()), out_stride, + stream); +} + +STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, m) { + m.impl("bf16_skinny_gemm", TORCH_BOX(&bf16_skinny_gemm)); +} diff --git a/csrc/libtorch_stable/dsv3_fused_a_gemm.cu b/csrc/libtorch_stable/dsv3_fused_a_gemm.cu index 585004c047..e1ba2a8d0e 100644 --- a/csrc/libtorch_stable/dsv3_fused_a_gemm.cu +++ b/csrc/libtorch_stable/dsv3_fused_a_gemm.cu @@ -391,7 +391,7 @@ struct MmaComputer { static constexpr int n_iter_cnt = (tile_n + 7) / 8; // Possible to have non-1 n_iter_cnt for ab_swap m16 case. - static_assert(m_iter_cnt == 1); + static_assert(m_iter_cnt == 1 || m_iter_cnt == 2); static_assert(n_iter_cnt == 1 || n_iter_cnt == 2); __device__ MmaComputer(bf16_t* gmem_c_local_, bf16_t* smem_a_, @@ -416,13 +416,18 @@ struct MmaComputer { public: __device__ void prepare() { #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900 + // Fragment addressing is per 16-row ldmatrix tile; m_iter selects the + // 16-row half within tile_m. #pragma unroll - for (int i = 0; i < k_phase_cnt; i++) { - int linear_idx = (lane_idx % 16) + (lane_idx / 16) * 128 + i * 256; - int m_idx = linear_idx % tile_m; - int k_idx = linear_idx / tile_m + warp_k_offset_in_tile_k; - k_idx = apply_swizzle_343_on_elem_row_col(m_idx, k_idx); - a_smem_offsets[0][i] = m_idx * tile_k + k_idx; + for (int m = 0; m < m_iter_cnt; m++) { + #pragma unroll + for (int i = 0; i < k_phase_cnt; i++) { + int linear_idx = (lane_idx % 16) + (lane_idx / 16) * 128 + i * 256; + int m_idx = linear_idx % 16 + m * 16; + int k_idx = linear_idx / 16 + warp_k_offset_in_tile_k; + k_idx = apply_swizzle_343_on_elem_row_col(m_idx, k_idx); + a_smem_offsets[m][i] = m_idx * tile_k + k_idx; + } } #pragma unroll for (int n_iter_idx = 0; n_iter_idx < n_iter_cnt; n_iter_idx++) { @@ -446,11 +451,14 @@ struct MmaComputer { wait_barrier(smem_barrier + 0 + stage_idx * 2, phase_bit); #pragma unroll - for (int i = 0; i < k_phase_cnt; i++) { - int smem_offset = a_smem_offsets[0][i]; - bf16_t* smem_ptr_this_iter = - smem_a + stage_idx * tile_m * tile_k + smem_offset; - ldsm_x4(smem_ptr_this_iter, reinterpret_cast(a_reg[0][i])); + for (int m = 0; m < m_iter_cnt; m++) { + #pragma unroll + for (int i = 0; i < k_phase_cnt; i++) { + int smem_offset = a_smem_offsets[m][i]; + bf16_t* smem_ptr_this_iter = + smem_a + stage_idx * tile_m * tile_k + smem_offset; + ldsm_x4(smem_ptr_this_iter, reinterpret_cast(a_reg[m][i])); + } } #pragma unroll @@ -469,9 +477,12 @@ struct MmaComputer { for (int k_iter_idx = 0; k_iter_idx < k_phase_cnt; k_iter_idx++) { #pragma unroll for (int n_iter_idx = 0; n_iter_idx < n_iter_cnt; n_iter_idx++) { - hmma_16_8_16_f32acc_bf16ab( - acc_reg[0][n_iter_idx], a_reg[0][k_iter_idx], - b_reg[n_iter_idx][k_iter_idx], acc_reg[0][n_iter_idx]); + #pragma unroll + for (int m = 0; m < m_iter_cnt; m++) { + hmma_16_8_16_f32acc_bf16ab( + acc_reg[m][n_iter_idx], a_reg[m][k_iter_idx], + b_reg[n_iter_idx][k_iter_idx], acc_reg[m][n_iter_idx]); + } } } ::arrive_barrier(smem_barrier + 1 + stage_idx * 2); @@ -486,14 +497,14 @@ struct MmaComputer { #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900 asm volatile("bar.sync %0, %1;" : : "r"(1), "r"(thread_cnt)); // reorganize the acc_reg - constexpr int thread_m = 2; + constexpr int thread_m = 2 * m_iter_cnt; constexpr int thread_n = 2 * n_iter_cnt; constexpr int cta_mma_n = n_iter_cnt * 8; float acc_reg_reorg[thread_m][thread_n]; for (int i = 0; i < thread_m; i++) { for (int j = 0; j < thread_n; j++) { - acc_reg_reorg[i][j] = acc_reg[0][j / 2][(j % 2) + (i * 2)]; + acc_reg_reorg[i][j] = acc_reg[i / 2][j / 2][(j % 2) + (i % 2) * 2]; } } @@ -514,7 +525,8 @@ struct MmaComputer { for (int m_idx_thread = 0; m_idx_thread < thread_m; m_idx_thread++) { #pragma unroll for (int n_idx_thread = 0; n_idx_thread < thread_n; n_idx_thread++) { - int m_idx = (lane_idx / 4) + m_idx_thread * 8; + int m_idx = + (lane_idx / 4) + (m_idx_thread % 2) * 8 + (m_idx_thread / 2) * 16; int n_idx = ((lane_idx % 4) * 2) + (n_idx_thread % 2) + (n_idx_thread / 2) * 8; smem_c[cosize_smem_c * warp_idx + smem_c_index_func(m_idx, n_idx)] = @@ -587,7 +599,7 @@ __global__ __launch_bounds__(256, 1) void fused_a_gemm_kernel( static_assert( tile_k == 128 || tile_k == 256 || tile_k == 512 || tile_k == 1024); // tile_k must be larger than 64 since 4 warp splitK. - static_assert(tile_m == 16); + static_assert(tile_m == 16 || tile_m == 32); constexpr int g2s_vec_bytes = 16; constexpr int a_elem_bytes = 2; constexpr int b_elem_bytes = 2; @@ -647,7 +659,7 @@ __global__ __launch_bounds__(256, 1) void fused_a_gemm_kernel( #endif } -template +template void invokeFusedAGemm(T* output, T const* mat_a, T const* mat_b, int num_tokens, cudaStream_t const stream) { constexpr int gemm_m = kHdOut; // 2112 @@ -655,7 +667,7 @@ void invokeFusedAGemm(T* output, T const* mat_a, T const* mat_b, int num_tokens, constexpr int gemm_k = kHdIn; // 7168 constexpr int batch_size = 1; std::swap(mat_a, mat_b); - constexpr int tile_m = 16; + constexpr int tile_m = kTileM; constexpr int tile_n = kTileN; // 8 or 16 constexpr int tile_k = std::max(256, 1024 / tile_n); // 256 constexpr int max_stage_cnt = @@ -702,6 +714,44 @@ template void invokeFusedAGemm<__nv_bfloat16, 7168, 2112, 16>( __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, int num_tokens, cudaStream_t); +// GLM-5.2 fused_qkv_a (K=6144 -> N=2624). tile_m=32 so the grid is +// 2624/32 = 82 CTAs, a single wave on B300 (148 SMs); tile_m=16's 164 CTAs +// straddle two waves and drop to 2.9 TB/s vs 3.9-4.0 here. Beats cuBLAS +// at every decode M: 1.11-1.15x for M<=8 (tile_n=8), 1.12x at M=16 +// (tile_n=16). The M<=2 dispatch still belongs to bf16_skinny_gemm +// (4.5 TB/s). +template void invokeFusedAGemm<__nv_bfloat16, 6144, 2624, 8, 32>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, int num_tokens, + cudaStream_t); + +template void invokeFusedAGemm<__nv_bfloat16, 6144, 2624, 16, 32>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, int num_tokens, + cudaStream_t); + +// GLM-5.2 q_b_proj TP8 (K=2048 -> N=2048). tile_m=16 keeps 128 CTAs (already +// a single wave) and measures ahead of tile_m=32 here. Beats cuBLAS +// 1.79-1.87x at M=3..8 (tile_n=8) and 1.19-1.30x at M=9..16 (tile_n=16); +// M<=2 belongs to bf16_skinny_gemm, M>=20 to cuBLAS. +template void invokeFusedAGemm<__nv_bfloat16, 2048, 2048, 8>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, int num_tokens, + cudaStream_t); + +template void invokeFusedAGemm<__nv_bfloat16, 2048, 2048, 16>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, int num_tokens, + cudaStream_t); + +// DSv3.2 q_b_proj TP8 (K=1536 -> N=3072). tile_m=32 keeps 96 CTAs (single +// wave; tile_m=16's 192 straddle two). Beats cuBLAS 1.6-1.8x at M=1..8 and +// 1.05-1.18x at M=9..16 on B300/B200; the whole 1..16 range dispatches here +// (no skinny tier: K=1536 does not fit the GEMV's 128x8 K-step). +template void invokeFusedAGemm<__nv_bfloat16, 1536, 3072, 8, 32>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, int num_tokens, + cudaStream_t); + +template void invokeFusedAGemm<__nv_bfloat16, 1536, 3072, 16, 32>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, int num_tokens, + cudaStream_t); + void dsv3_fused_a_gemm(torch::stable::Tensor& output, torch::stable::Tensor const& mat_a, torch::stable::Tensor const& mat_b) { @@ -710,12 +760,15 @@ void dsv3_fused_a_gemm(torch::stable::Tensor& output, int const hd_in = mat_a.size(1); int const hd_out = mat_b.size(1); - constexpr int kHdIn = 7168; - constexpr int kHdOut = 2112; + bool const is_dsv3 = hd_in == 7168 && hd_out == 2112; + bool const is_glm = hd_in == 6144 && hd_out == 2624; + bool const is_glm_qb = hd_in == 2048 && hd_out == 2048; + bool const is_ds_qb = hd_in == 1536 && hd_out == 3072; STD_TORCH_CHECK(num_tokens >= 1 && num_tokens <= 16, "required 1 <= mat_a.shape[0] <= 16"); - STD_TORCH_CHECK(hd_in == kHdIn, "required mat_a.shape[1] == 7168"); - STD_TORCH_CHECK(hd_out == kHdOut, "required mat_b.shape[1] == 2112"); + STD_TORCH_CHECK(is_dsv3 || is_glm || is_glm_qb || is_ds_qb, + "supported (hd_in, hd_out): (7168, 2112), (6144, 2624), " + "(2048, 2048), (1536, 3072)"); STD_TORCH_CHECK(output.size(0) == num_tokens, "required output.shape[0] == mat_a.shape[0]"); STD_TORCH_CHECK(output.size(1) == hd_out, @@ -738,18 +791,41 @@ void dsv3_fused_a_gemm(torch::stable::Tensor& output, STD_TORCH_CHECK(getSMVersion() >= 90, "required CUDA ARCH >= SM_90"); auto stream = get_current_cuda_stream(mat_a.get_device_index()); - if (num_tokens <= 8) { - invokeFusedAGemm<__nv_bfloat16, kHdIn, kHdOut, 8>( - reinterpret_cast<__nv_bfloat16*>(output.mutable_data_ptr()), - reinterpret_cast<__nv_bfloat16 const*>(mat_a.data_ptr()), - reinterpret_cast<__nv_bfloat16 const*>(mat_b.data_ptr()), num_tokens, - stream); + auto* out = reinterpret_cast<__nv_bfloat16*>(output.mutable_data_ptr()); + auto* a = reinterpret_cast<__nv_bfloat16 const*>(mat_a.data_ptr()); + auto* b = reinterpret_cast<__nv_bfloat16 const*>(mat_b.data_ptr()); + if (is_dsv3) { + if (num_tokens <= 8) { + invokeFusedAGemm<__nv_bfloat16, 7168, 2112, 8>(out, a, b, num_tokens, + stream); + } else { + invokeFusedAGemm<__nv_bfloat16, 7168, 2112, 16>(out, a, b, num_tokens, + stream); + } + } else if (is_glm) { + if (num_tokens <= 8) { + invokeFusedAGemm<__nv_bfloat16, 6144, 2624, 8, 32>(out, a, b, num_tokens, + stream); + } else { + invokeFusedAGemm<__nv_bfloat16, 6144, 2624, 16, 32>(out, a, b, num_tokens, + stream); + } + } else if (is_glm_qb) { + if (num_tokens <= 8) { + invokeFusedAGemm<__nv_bfloat16, 2048, 2048, 8>(out, a, b, num_tokens, + stream); + } else { + invokeFusedAGemm<__nv_bfloat16, 2048, 2048, 16>(out, a, b, num_tokens, + stream); + } } else { - invokeFusedAGemm<__nv_bfloat16, kHdIn, kHdOut, 16>( - reinterpret_cast<__nv_bfloat16*>(output.mutable_data_ptr()), - reinterpret_cast<__nv_bfloat16 const*>(mat_a.data_ptr()), - reinterpret_cast<__nv_bfloat16 const*>(mat_b.data_ptr()), num_tokens, - stream); + if (num_tokens <= 8) { + invokeFusedAGemm<__nv_bfloat16, 1536, 3072, 8, 32>(out, a, b, num_tokens, + stream); + } else { + invokeFusedAGemm<__nv_bfloat16, 1536, 3072, 16, 32>(out, a, b, num_tokens, + stream); + } } } diff --git a/csrc/libtorch_stable/quantization/fp4/nvfp4_quant_kernels.cu b/csrc/libtorch_stable/quantization/fp4/nvfp4_quant_kernels.cu index f7c965dbc1..f10c58eab7 100644 --- a/csrc/libtorch_stable/quantization/fp4/nvfp4_quant_kernels.cu +++ b/csrc/libtorch_stable/quantization/fp4/nvfp4_quant_kernels.cu @@ -49,6 +49,11 @@ __global__ void __launch_bounds__(512, VLLM_BLOCKS_PER_SM(512)) static_assert(sizeof(PackedVec) == sizeof(Type) * CVT_FP4_ELTS_PER_THREAD, "Vec size is not matched."); +#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)) + cudaGridDependencySynchronize(); + cudaTriggerProgrammaticLaunchCompletion(); +#endif + // Precompute SF layout parameter (constant for entire kernel). int32_t const numKTiles = (outputCols + 63) / 64; @@ -123,6 +128,11 @@ __global__ void __launch_bounds__(512, VLLM_BLOCKS_PER_SM(512)) static_assert(sizeof(PackedVec) == sizeof(Type) * CVT_FP4_ELTS_PER_THREAD, "Vec size is not matched."); +#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)) + cudaGridDependencySynchronize(); + cudaTriggerProgrammaticLaunchCompletion(); +#endif + int32_t const colIdx = blockDim.x * blockIdx.y + threadIdx.x; int elem_idx = colIdx * CVT_FP4_ELTS_PER_THREAD; @@ -201,6 +211,8 @@ void scaled_fp4_quant_sm1xxa(torch::stable::Tensor const& output, const torch::stable::accelerator::DeviceGuard device_guard( input.get_device_index()); auto stream = get_current_cuda_stream(input.get_device_index()); + auto* device_props = get_device_prop(); + int const sm_version = device_props->major * 10 + device_props->minor; int output_sf_n_unpadded = int(output_n / CVT_FP4_SF_VEC_SIZE); @@ -224,10 +236,21 @@ void scaled_fp4_quant_sm1xxa(torch::stable::Tensor const& output, input.scalar_type(), "nvfp4_quant_kernel", [&] { using cuda_type = vllm::CUDATypeConverter::Type; auto input_ptr = static_cast(input.data_ptr()); - vllm::cvt_fp16_to_fp4<<>>( - m, n, output_n, num_padded_cols, input_ptr, input_sf_ptr, - reinterpret_cast(output_ptr), - reinterpret_cast(sf_out)); + cudaLaunchConfig_t config = {}; + config.gridDim = grid; + config.blockDim = block; + config.dynamicSmemBytes = 0; + config.stream = stream; + cudaLaunchAttribute attrs[1]; + attrs[0].id = cudaLaunchAttributeProgrammaticStreamSerialization; + attrs[0].val.programmaticStreamSerializationAllowed = 1; + config.numAttrs = (sm_version >= 90) ? 1 : 0; + config.attrs = attrs; + cudaLaunchKernelEx(&config, vllm::cvt_fp16_to_fp4, + m, n, output_n, num_padded_cols, input_ptr, + input_sf_ptr, + reinterpret_cast(output_ptr), + reinterpret_cast(sf_out)); }); } else { int num_packed_cols = output_n / CVT_FP4_ELTS_PER_THREAD; @@ -240,12 +263,21 @@ void scaled_fp4_quant_sm1xxa(torch::stable::Tensor const& output, input.scalar_type(), "nvfp4_quant_kernel", [&] { using cuda_type = vllm::CUDATypeConverter::Type; auto input_ptr = static_cast(input.data_ptr()); - vllm::cvt_fp16_to_fp4_sf_major - <<>>( - m, n, output_n, output_sf_n_unpadded, num_packed_cols, - input_ptr, input_sf_ptr, - reinterpret_cast(output_ptr), - reinterpret_cast(sf_out)); + cudaLaunchConfig_t config = {}; + config.gridDim = grid; + config.blockDim = block; + config.dynamicSmemBytes = 0; + config.stream = stream; + cudaLaunchAttribute attrs[1]; + attrs[0].id = cudaLaunchAttributeProgrammaticStreamSerialization; + attrs[0].val.programmaticStreamSerializationAllowed = 1; + config.numAttrs = (sm_version >= 90) ? 1 : 0; + config.attrs = attrs; + cudaLaunchKernelEx( + &config, vllm::cvt_fp16_to_fp4_sf_major, m, n, + output_n, output_sf_n_unpadded, num_packed_cols, input_ptr, + input_sf_ptr, reinterpret_cast(output_ptr), + reinterpret_cast(sf_out)); }); } } diff --git a/csrc/libtorch_stable/torch_bindings.cpp b/csrc/libtorch_stable/torch_bindings.cpp index 0a475d02c6..0144118062 100644 --- a/csrc/libtorch_stable/torch_bindings.cpp +++ b/csrc/libtorch_stable/torch_bindings.cpp @@ -330,6 +330,10 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) { // conditionally compiled so impl registration is in source file ops.def("fp32_router_gemm(Tensor! output, Tensor mat_a, Tensor mat_b) -> ()"); + // BF16 skinny GEMM (M<=32, weight-BW-bound shapes, e.g. MTP eh_proj). + // conditionally compiled so impl registration is in source file + ops.def("bf16_skinny_gemm(Tensor! output, Tensor mat_a, Tensor mat_b) -> ()"); + // reorder weight for AllSpark Ampere W8A16 Fused Gemm kernel ops.def( "rearrange_kn_weight_as_n32k16_order(Tensor b_qweight, Tensor b_scales, " diff --git a/recipes/glm5.2-ll-b300-tp8-mtp5.md b/recipes/glm5.2-ll-b300-tp8-mtp5.md new file mode 100644 index 0000000000..a0deca9bd9 --- /dev/null +++ b/recipes/glm5.2-ll-b300-tp8-mtp5.md @@ -0,0 +1,41 @@ +# GLM-5.2-NVFP4 — Low-Latency (LL) serving recipe · 8×B300 · TP8 · MTP=5 + +Single-node 8×B300 (sm_100/103), pure tensor parallel, NVFP4 weights, fp8 KV cache, +MTP=5 speculative decode, model runner v2. + +Validated on `glm5.2-LL` @ `8d407aee1`. + +## Versions + +| Component | Version | +| --- | --- | +| vLLM | `glm5.2-LL` @ `8d407aee1` | +| flashinfer-python | `0.6.15` | +| Model | `nvidia/GLM-5.2-NVFP4` (`modelopt_fp4`) | + +## Build (once) + +```bash +pip install 'flashinfer-python==0.6.15' + +CUDA_HOME=/usr/local/cuda-13.0 TORCH_CUDA_ARCH_LIST="10.0 10.3" MAX_JOBS=96 \ +VLLM_USE_PRECOMPILED=0 VLLM_USE_PRECOMPILED_RUST=0 \ +pip install -e . --no-deps --no-build-isolation +``` + +## Server + +```bash +export VLLM_USE_V2_MODEL_RUNNER=1 # model runner v2 +export VLLM_DEEP_GEMM_WARMUP=skip + +vllm serve nvidia/GLM-5.2-NVFP4 \ + --trust-remote-code \ + --tensor-parallel-size 8 \ + --quantization modelopt_fp4 --kv-cache-dtype fp8_e4m3 \ + --max-model-len 32768 --max-num-batched-tokens 16384 --max-num-seqs 256 \ + --no-enable-prefix-caching --gpu-memory-utilization 0.85 \ + --speculative-config '{"method":"mtp","num_speculative_tokens":5}' \ + --kernel-config '{"ir_op_priority":{"rms_norm":["vllm_c","native"],"fused_add_rms_norm":["vllm_c","native"]}}' \ + --host 0.0.0.0 --port 8000 +``` diff --git a/tests/kernels/test_bf16_skinny_gemm.py b/tests/kernels/test_bf16_skinny_gemm.py new file mode 100644 index 0000000000..f49e4b4db0 --- /dev/null +++ b/tests/kernels/test_bf16_skinny_gemm.py @@ -0,0 +1,70 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Numerical correctness for the bf16 skinny GEMM (decode-M GEMV) kernel. + +Covers every supported (N, K) shape across M = 0/1/2/8/16/32 (M=32 exercises +the tile_m=32 MMA path), plus strided (column-slice) output. M=0 must return +an empty result without launching the kernel (empty ranks at DP/PP +boundaries).""" + +import pytest +import torch + +from vllm import _custom_ops as ops +from vllm.platforms import current_platform + +# (N, K) pairs must match bf16_skinny_gemm_supported() in the entry .cu. +SHAPES = [ + (768, 12288), + (1536, 12288), + (6144, 12288), + (2048, 2048), + (2624, 6144), + (2112, 7168), + (7168, 14336), + (512, 6144), +] +MS = [0, 1, 2, 8, 16, 32] + +pytestmark = pytest.mark.skipif( + not current_platform.is_cuda() or not current_platform.has_device_capability(90), + reason="bf16_skinny_gemm requires CUDA SM90+", +) + + +def _rel_err(out: torch.Tensor, ref: torch.Tensor) -> float: + return (out.float() - ref).norm().item() / ref.norm().clamp_min(1e-6).item() + + +@pytest.mark.parametrize("n,k", SHAPES) +@pytest.mark.parametrize("m", MS) +def test_bf16_skinny_gemm_matches_reference(n: int, k: int, m: int): + torch.manual_seed(0) + x = torch.randn(m, k, dtype=torch.bfloat16, device="cuda") + w = torch.randn(n, k, dtype=torch.bfloat16, device="cuda") + + out = ops.bf16_skinny_gemm(x, w) + assert out.shape == (m, n) + assert out.dtype == torch.bfloat16 + + if m == 0: # empty batch: nothing to check beyond the empty shape + return + ref = x.float() @ w.float().t() + rel = _rel_err(out, ref) + assert rel < 2e-2, f"rel err {rel:.4f} for (m,n,k)=({m},{n},{k})" + + +@pytest.mark.parametrize("n,k", [(2048, 2048), (512, 6144)]) +def test_bf16_skinny_gemm_strided_output(n: int, k: int): + """Output is a column-slice of a wider padded buffer (row stride > N).""" + torch.manual_seed(0) + m = 8 + x = torch.randn(m, k, dtype=torch.bfloat16, device="cuda") + w = torch.randn(n, k, dtype=torch.bfloat16, device="cuda") + + wide = torch.empty(m, n + 64, dtype=torch.bfloat16, device="cuda") + view = wide[:, :n] + torch.ops._C.bf16_skinny_gemm(view, x, w) + + ref = x.float() @ w.float().t() + assert _rel_err(view, ref) < 2e-2 diff --git a/tests/kernels/test_fused_deepseek_v32_norm_rope.py b/tests/kernels/test_fused_deepseek_v32_norm_rope.py index a27e67b624..f2302a896b 100644 --- a/tests/kernels/test_fused_deepseek_v32_norm_rope.py +++ b/tests/kernels/test_fused_deepseek_v32_norm_rope.py @@ -49,6 +49,32 @@ pytestmark = pytest.mark.skipif( ) +def test_platform_capability_queries_are_constant_during_compile(monkeypatch): + monkeypatch.setattr( + K.current_platform, "has_device_capability", lambda capability: True + ) + monkeypatch.setattr(K, "has_cutedsl", lambda: True) + monkeypatch.setattr(K.current_platform, "is_arch_support_pdl", lambda: True) + + def capability_branches(x: torch.Tensor) -> torch.Tensor: + if K._can_use_fused_q_cutedsl() and K._is_arch_support_pdl(): + return x + 1 + return x - 1 + + compiled = torch.compile(capability_branches, backend="eager", fullgraph=True) + x = torch.zeros(1, device="cuda") + torch.testing.assert_close(compiled(x), torch.ones_like(x)) + + +def test_pdl_is_disabled_before_blackwell(monkeypatch): + monkeypatch.setattr( + K.current_platform, "has_device_capability", lambda capability: False + ) + monkeypatch.setattr(K.current_platform, "is_arch_support_pdl", lambda: True) + + assert not K._is_arch_support_pdl() + + # ── reference helpers ──────────────────────────────────────────────────────── @@ -292,6 +318,57 @@ def test_fused_norm_rope_no_indexer(num_tokens: int): assert (topk == 7).all(), "topk buffer should be untouched on shared layer" +def test_fused_norm_rope_profile_without_cache_compiles(): + """The cache-free profiling path must still compile and produce Q.""" + torch.manual_seed(2) + dev = "cuda" + num_tokens = 4 + pos = torch.arange(num_tokens, device=dev, dtype=torch.int64) + q_c = torch.randn(num_tokens, Q_LORA, device=dev, dtype=torch.bfloat16) + kv_c = torch.randn(num_tokens, KV_LORA, device=dev, dtype=torch.bfloat16) + k_pe = torch.randn(num_tokens, ROPE_DIM, device=dev, dtype=torch.bfloat16) + qw = torch.randn(Q_LORA, device=dev, dtype=torch.bfloat16) + kvw = torch.randn(KV_LORA, device=dev, dtype=torch.bfloat16) + index_k = torch.randn(num_tokens, INDEX_HEAD_DIM, device=dev, dtype=torch.bfloat16) + index_w = torch.randn(INDEX_HEAD_DIM, device=dev, dtype=torch.float32) + index_b = torch.randn(INDEX_HEAD_DIM, device=dev, dtype=torch.float32) + cos_sin = make_cos_sin(64, ROPE_DIM, dev) + topk = torch.empty(num_tokens, 2048, device=dev, dtype=torch.int32) + + def profile_run( + q_c: torch.Tensor, + kv_c: torch.Tensor, + k_pe: torch.Tensor, + index_k: torch.Tensor, + ) -> torch.Tensor: + return K.fused_norm_rope( + pos, + q_c, + qw, + EPS, + kv_c, + kvw, + EPS, + k_pe, + cos_sin, + index_k, + index_w, + index_b, + EPS, + cos_sin, + topk, + slot_mapping=None, + indexer_k_cache=None, + mla_kv_cache=None, + has_indexer=True, + index_rope_interleave=False, + ) + + compiled = torch.compile(profile_run, fullgraph=True) + q_out = compiled(q_c, kv_c, k_pe, index_k) + assert_bf16(q_out, rms_norm(q_c, qw), "q_c rmsnorm (profiling)") + + @pytest.mark.parametrize("num_tokens", [1, 4, 17, 512]) def test_fused_norm_rope_ds_mla(num_tokens: int): """fp8_ds_mla MLA cache layout (FlashMLA sparse, bf16-query path; SM90/SM100). diff --git a/tests/models/registry.py b/tests/models/registry.py index d614659b1d..4362e134c7 100644 --- a/tests/models/registry.py +++ b/tests/models/registry.py @@ -1604,6 +1604,10 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = { speculative_model="luccafong/deepseek_mtp_draft_random", trust_remote_code=True, ), + "DeepseekV32MTPModel": _HfExamplesInfo( + "deepseek-ai/DeepSeek-V3.2-Exp", + speculative_model="deepseek-ai/DeepSeek-V3.2-Exp", + ), "DeepSeekV4MTPModel": _HfExamplesInfo( "deepseek-ai/DeepSeek-V4-Flash", speculative_model="deepseek-ai/DeepSeek-V4-Flash", diff --git a/vllm/_custom_ops.py b/vllm/_custom_ops.py index aa75c50a51..018fc1ac2b 100644 --- a/vllm/_custom_ops.py +++ b/vllm/_custom_ops.py @@ -2371,6 +2371,20 @@ def fp32_router_gemm( return output +def bf16_skinny_gemm( + x: torch.Tensor, + weight: torch.Tensor, +) -> torch.Tensor: + """Skinny bf16 GEMM (M<=32): x [M, K] @ weight [N, K]^T -> [M, N]. + + Single block-per-column kernel with fp32 accumulation; replaces cuBLAS + splitK for weight-bandwidth-bound decode shapes (e.g. MTP eh_proj). + """ + output = torch.empty((x.shape[0], weight.shape[0]), dtype=x.dtype, device=x.device) + torch.ops._C.bf16_skinny_gemm(output, x, weight) + return output + + if hasattr(torch.ops, "_C") and hasattr(torch.ops._C, "fp32_router_gemm"): @register_fake("_C::fp32_router_gemm") @@ -2382,6 +2396,17 @@ if hasattr(torch.ops, "_C") and hasattr(torch.ops._C, "fp32_router_gemm"): return +if hasattr(torch.ops, "_C") and hasattr(torch.ops._C, "bf16_skinny_gemm"): + + @register_fake("_C::bf16_skinny_gemm") + def bf16_skinny_gemm_fake( + output: torch.Tensor, + mat_a: torch.Tensor, + mat_b: torch.Tensor, + ) -> None: + return + + def topk_softmax( topk_weights: torch.Tensor, topk_ids: torch.Tensor, diff --git a/vllm/compilation/passes/fusion/allreduce_rms_fusion.py b/vllm/compilation/passes/fusion/allreduce_rms_fusion.py index 4716e7012f..64ff381178 100644 --- a/vllm/compilation/passes/fusion/allreduce_rms_fusion.py +++ b/vllm/compilation/passes/fusion/allreduce_rms_fusion.py @@ -202,11 +202,7 @@ if flashinfer_comm is not None: element_size = allreduce_in.element_size() current_tensor_size = num_tokens * hidden_size * element_size max_tensor_size = max_token_num * hidden_size * element_size - assert current_tensor_size <= max_tensor_size, ( - f"Current tensor size {current_tensor_size} is larger than " - f"max token num {max_token_num} * hidden size {hidden_size} * " - f"element size {element_size}" - ) + oversize = current_tensor_size > max_tensor_size curr_device = current_platform.get_device_capability() device_capability = curr_device.to_int() if curr_device is not None else None @@ -219,6 +215,47 @@ if flashinfer_comm is not None: get_workspace_fn = ( get_fi_ar_quant_workspace if is_quant_pattern else get_fi_ar_workspace ) + + orig_norm_out = norm_out + + def unfused_fallback() -> None: + # Fused-path equivalent honoring the op's output-buffer contract: + # with norm_out given, normed -> norm_out and the new residual -> + # the allreduce_in buffer; otherwise normed -> allreduce_in and + # the new residual -> the residual buffer. Runs fine during CUDA + # graph capture (the branch is resolved per captured shape). + # + # weight_bias mirrors the fused kernel's gamma offset (Gemma + # patterns pass 1.0 for rms_norm(x, weight + 1)); fold it into + # the gamma in fp32 like the reference before the plain rms_norm. + gamma = ( + rms_gamma + if weight_bias == 0.0 + else (rms_gamma.float() + weight_bias).to(rms_gamma.dtype) + ) + reduced = tensor_model_parallel_all_reduce(allreduce_in) + new_residual = reduced + residual + if orig_norm_out is None: + torch.ops._C.rms_norm(allreduce_in, new_residual, gamma, rms_eps) + residual.copy_(new_residual) + else: + torch.ops._C.rms_norm(orig_norm_out, new_residual, gamma, rms_eps) + allreduce_in.copy_(new_residual) + + if oversize: + assert not is_quant_pattern, ( + "flashinfer fused allreduce: input exceeds the workspace budget " + "and the unfused fallback does not cover quant patterns" + ) + logger.warning_once( + "fused allreduce+rmsnorm: %d tokens exceeds the workspace " + "budget (max_token_num=%d); using the unfused fallback for " + "this shape.", + num_tokens, + max_token_num, + ) + unfused_fallback() + return workspace = get_workspace_fn( world_size=world_size, rank=get_tensor_model_parallel_rank(), @@ -252,36 +289,75 @@ if flashinfer_comm is not None: if workspace.backend in ("trtllm", "mnnvl"): layout_code = flashinfer_comm.QuantizationSFLayout.SWIZZLED_128x4 - flashinfer_comm.allreduce_fusion( - input=allreduce_in, - workspace=workspace, - pattern=pattern_code, - launch_with_pdl=launch_with_pdl, - output=None, - residual_out=residual_out, - norm_out=norm_out, - quant_out=quant_out, - scale_out=scale_out, - residual_in=residual, - rms_gamma=rms_gamma, - rms_eps=rms_eps, - scale_factor=scale_factor, - layout_code=layout_code, - use_oneshot=use_oneshot, - fp32_acc=fp32_acc, - weight_bias=weight_bias, - # The one-shot Lamport all-reduce signals PDL completion before its - # output buffer is committed when trigger_completion_at_end is - # False, so the next PDL-launched kernel can read the uninitialized - # Lamport buffer and produce NaN. This only fires for - # num_tokens <= PDL_ADVANCE_LAUNCH_TOKENS (the batch=1 / spec-decode - # shapes, where the one-shot path is always selected). Complete at - # the end for the one-shot path; the two-shot path is synchronized - # and keeps the early completion. Related one-shot instability in - # the same kernel: flashinfer-ai/flashinfer#1223. - trigger_completion_at_end=(use_oneshot is True) - or num_tokens > PDL_ADVANCE_LAUNCH_TOKENS, - ) + # vllm's one-shot size table is tuned for the trtllm backend. For + # mnnvl, flashinfer caps the workspace's one-shot region at its own + # MNNVL_ONE_SHOT_THRESHOLD (~4MB, e.g. 44 tokens at hidden 6144 bf16 + # TP8); forcing use_oneshot=True past that raises ValueError instead + # of using two-shot, so every decode capture between the one-shot cap + # and the fusion threshold silently lost fusion. Let flashinfer's + # AUTO heuristic pick the strategy (it matches its own workspace + # sizing), and keep trigger_completion_at_end=True since AUTO may + # select one-shot (see the Lamport NaN note below). + mnnvl_auto = workspace.backend == "mnnvl" + if mnnvl_auto: + use_oneshot = None + + try: + flashinfer_comm.allreduce_fusion( + input=allreduce_in, + workspace=workspace, + pattern=pattern_code, + launch_with_pdl=launch_with_pdl, + output=None, + residual_out=residual_out, + norm_out=norm_out, + quant_out=quant_out, + scale_out=scale_out, + residual_in=residual, + rms_gamma=rms_gamma, + rms_eps=rms_eps, + scale_factor=scale_factor, + layout_code=layout_code, + use_oneshot=use_oneshot, + fp32_acc=fp32_acc, + weight_bias=weight_bias, + # The one-shot Lamport all-reduce signals PDL completion before + # its output buffer is committed when trigger_completion_at_end + # is False, so the next PDL-launched kernel can read the + # uninitialized Lamport buffer and produce NaN. This only fires + # for num_tokens <= PDL_ADVANCE_LAUNCH_TOKENS (the batch=1 / + # spec-decode shapes, where the one-shot path is always + # selected). Complete at the end for the one-shot path; the + # two-shot path is synchronized and keeps the early completion. + # Related one-shot instability in the same kernel: + # flashinfer-ai/flashinfer#1223. + trigger_completion_at_end=True + if mnnvl_auto + else (use_oneshot or num_tokens > PDL_ADVANCE_LAUNCH_TOKENS), + ) + except ValueError as e: + # The globally cached workspace can be smaller than this call's + # budget (it is created once by whichever caller runs first, and + # the mnnvl workspace's is_buffer_size_sufficient() checks trtllm + # metadata, so vllm-side accounting cannot see the real capacity). + # flashinfer raises before launching anything, so falling back + # mid-capture is safe. Observed with spec-decode draft graphs + # whose token count (max_num_reqs * (1 + num_spec)) exceeds the + # workspace sized for the fusion threshold. + assert not is_quant_pattern, ( + "flashinfer fused allreduce failed and the unfused fallback " + f"does not cover quant patterns: {e}" + ) + logger.warning_once( + "flashinfer fused allreduce+rmsnorm unavailable for this " + "shape (num_tokens=%d, max_token_num=%d, world_size=%d): %s. " + "Using the unfused fallback.", + num_tokens, + max_token_num, + world_size, + e, + ) + unfused_fallback() def call_trtllm_fused_allreduce_norm_fake( allreduce_in: torch.Tensor, diff --git a/vllm/config/speculative.py b/vllm/config/speculative.py index 8ba55a2ec9..bd536800cc 100644 --- a/vllm/config/speculative.py +++ b/vllm/config/speculative.py @@ -330,6 +330,7 @@ class SpeculativeConfig: @staticmethod def hf_config_override(hf_config: PretrainedConfig) -> PretrainedConfig: initial_architecture = hf_config.architectures[0] + use_sparse_mtp = hf_config.model_type == "glm_moe_dsa" if hf_config.model_type in ( "deepseek_v3", "deepseek_v32", @@ -339,7 +340,12 @@ class SpeculativeConfig: if hf_config.model_type == "deepseek_mtp": n_predict = getattr(hf_config, "num_nextn_predict_layers", None) hf_config.update( - {"n_predict": n_predict, "architectures": ["DeepSeekMTPModel"]} + { + "n_predict": n_predict, + "architectures": [ + "DeepseekV32MTPModel" if use_sparse_mtp else "DeepSeekMTPModel" + ], + } ) if hf_config.model_type == "deepseek_v4": hf_config.model_type = "deepseek_mtp" diff --git a/vllm/cute_utils/__init__.py b/vllm/cute_utils/__init__.py index ca445284bf..d5e536db90 100644 --- a/vllm/cute_utils/__init__.py +++ b/vllm/cute_utils/__init__.py @@ -33,6 +33,11 @@ EVICT_NORMAL = Int64(0x1000000000000000) EVICT_FIRST = Int64(0x12F0000000000000) EVICT_LAST = Int64(0x14F0000000000000) +TORCH_TO_CUTE_DTYPE = { + torch.bfloat16: BFloat16, + torch.float32: Float32, +} + @dsl_user_op def recast_val(x, dtype, *, loc=None, ip=None): diff --git a/vllm/cute_utils/cvt.py b/vllm/cute_utils/cvt.py index 4707f3f742..8bb4ad9519 100644 --- a/vllm/cute_utils/cvt.py +++ b/vllm/cute_utils/cvt.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from cutlass import Constexpr, Float32, Uint32, cute +from cutlass import BFloat16, Constexpr, Float32, Uint16, Uint32, cute from cutlass._mlir import ir from cutlass._mlir.dialects import llvm, vector from cutlass.cutlass_dsl import T, dsl_user_op @@ -38,6 +38,11 @@ def bf16x2_to_fp32x2(data, *, loc=None, ip=None) -> tuple[Float32, Float32]: ) elif isinstance(data, (cute.Tensor, cute.TensorSSA)): + # recast to 32-bit registers if needed + if data.element_type == BFloat16: + data = cute.recast_tensor(data, Uint32) + assert data.element_type == Uint32 + # NOTE: the output is always 1D size = cute.size(data.shape) out = cute.make_rmem_tensor(size * 2, Float32) @@ -84,6 +89,22 @@ def fp8x4_to_bf16x4(x: Uint32, *, loc=None, ip=None) -> cute.TensorSSA: return cute.TensorSSA(vec, 2, Uint32) +@dsl_user_op +def fp32x2_to_fp8x2(a0: Float32, a1: Float32, *, loc=None, ip=None) -> Uint16: + out = llvm.inline_asm( + T.i16(), + [ + a0.ir_value(loc=loc, ip=ip), + a1.ir_value(loc=loc, ip=ip), + ], + "cvt.rn.satfinite.e4m3x2.f32 $0, $2, $1;", + "=h,f,f", + has_side_effects=False, + is_align_stack=False, + ) + return Uint16(out) + + @dsl_user_op def fp8x4_to_fp16x4(x: Uint32, *, loc=None, ip=None) -> cute.TensorSSA: out = llvm.inline_asm( diff --git a/vllm/model_executor/layers/fused_allreduce_gemma_rms_norm.py b/vllm/model_executor/layers/fused_allreduce_gemma_rms_norm.py index e49e135b26..509befbcc1 100644 --- a/vllm/model_executor/layers/fused_allreduce_gemma_rms_norm.py +++ b/vllm/model_executor/layers/fused_allreduce_gemma_rms_norm.py @@ -43,7 +43,7 @@ try: if flashinfer_comm is not None else None ) -except ImportError: +except (ImportError, AttributeError): flashinfer_trtllm_fused_allreduce_norm = None # type: ignore[assignment] get_fi_ar_workspace = None # type: ignore[assignment] _AR_RESIDUAL_RMS_NORM = None @@ -52,12 +52,23 @@ except ImportError: _FI_SUPPORTED_DTYPES = (torch.bfloat16, torch.float16) +@torch.compiler.assume_constant_result +def _fi_ar_max_size_mb() -> dict[int, float]: + """Flashinfer all-reduce fusion size table for the current device. + + Device capability is constant; marking the result constant keeps this out + of the traced graph. Otherwise the per-forward call reaches + ``current_platform.get_device_capability()`` inside ``torch.compile`` and + graph-breaks ("can't handle functions not implemented in python").""" + from vllm.config.compilation import PassConfig + + return PassConfig.default_fi_allreduce_fusion_max_size_mb() + + def _max_token_num(tp_size: int, hidden_size: int, dtype: torch.dtype) -> int | None: """Workspace token budget for flashinfer fused all-reduce, or None if the current world size / device is unsupported. Mirrors ``FlashInferAllReduce``.""" - from vllm.config.compilation import PassConfig - - max_size_mb = PassConfig.default_fi_allreduce_fusion_max_size_mb().get(tp_size) + max_size_mb = _fi_ar_max_size_mb().get(tp_size) if not max_size_mb: return None element_size = torch.tensor([], dtype=dtype).element_size() diff --git a/vllm/model_executor/models/registry.py b/vllm/model_executor/models/registry.py index d28aac974b..b17a41834d 100644 --- a/vllm/model_executor/models/registry.py +++ b/vllm/model_executor/models/registry.py @@ -113,7 +113,7 @@ _TEXT_GENERATION_MODELS = { "Glm4ForCausalLM": ("glm4", "Glm4ForCausalLM"), "Glm4MoeForCausalLM": ("glm4_moe", "Glm4MoeForCausalLM"), "Glm4MoeLiteForCausalLM": ("glm4_moe_lite", "Glm4MoeLiteForCausalLM"), - "GlmMoeDsaForCausalLM": ("deepseek_v2", "GlmMoeDsaForCausalLM"), + "GlmMoeDsaForCausalLM": ("vllm.models.deepseek_v32", "GlmMoeDsaForCausalLM"), "GptOssForCausalLM": ("gpt_oss", "GptOssForCausalLM"), "GPT2LMHeadModel": ("gpt2", "GPT2LMHeadModel"), "GPTJForCausalLM": ("gpt_j", "GPTJForCausalLM"), @@ -628,6 +628,7 @@ _SPECULATIVE_DECODING_MODELS = { "Eagle3DeepseekV3ForCausalLM": ("deepseek_eagle3", "Eagle3DeepseekV2ForCausalLM"), "EagleDeepSeekMTPModel": ("deepseek_eagle", "EagleDeepseekV3ForCausalLM"), "DeepSeekMTPModel": ("deepseek_mtp", "DeepSeekMTP"), + "DeepseekV32MTPModel": ("vllm.models.deepseek_v32", "DeepseekV32MTP"), "DeepSeekV4MTPModel": ("vllm.models.deepseek_v4", "DeepSeekV4MTP"), "MiniMaxM3MTP": ("vllm.models.minimax_m3", "MiniMaxM3MTP"), "BailingMoeV25MTPModel": ("bailing_moe_mtp", "BailingMoeV25MTPModel"), diff --git a/vllm/models/deepseek_v32/__init__.py b/vllm/models/deepseek_v32/__init__.py index 1b0aa64262..6bd447bab2 100644 --- a/vllm/models/deepseek_v32/__init__.py +++ b/vllm/models/deepseek_v32/__init__.py @@ -6,17 +6,31 @@ DeepSeek V3.2 introduced the DeepSeek Sparse Attention (DSA) architecture: MLA + a "lightning indexer" that selects the top-k tokens for a sparse MLA attend. The same model code serves any DSA checkpoint, including GLM-5.2 (``glm_moe_dsa``), which reuses this architecture. + +The optimized kernels under ``nvidia/`` target the Blackwell (SM100) family. +Every other platform — ROCm, XPU, pre-SM100 CUDA (e.g. H100), CPU — falls back +to the generic ``deepseek_v2`` implementation, which already handles the DSA +(index_topk) architecture and is ``torch.compile``-friendly there. This matches +main's behavior on those platforms (no hard failure). """ from vllm.platforms import current_platform -if current_platform.is_rocm() or current_platform.is_xpu(): - raise NotImplementedError("deepseek_v32 currently supports NVIDIA SM100 only.") +if current_platform.is_cuda() and current_platform.is_device_capability_family(100): + from .nvidia.model import DeepseekV32ForCausalLM + from .nvidia.mtp import DeepseekV32MTP -from .nvidia.model import DeepseekV32ForCausalLM -from .nvidia.mtp import DeepseekV32MTP + # GLM-5.2 (glm_moe_dsa) reuses the same optimized DSA module on SM100. + GlmMoeDsaForCausalLM = DeepseekV32ForCausalLM +else: + from vllm.model_executor.models.deepseek_mtp import DeepSeekMTP as DeepseekV32MTP + from vllm.model_executor.models.deepseek_v2 import ( + DeepseekV3ForCausalLM as DeepseekV32ForCausalLM, + ) + from vllm.model_executor.models.deepseek_v2 import GlmMoeDsaForCausalLM __all__ = [ "DeepseekV32ForCausalLM", "DeepseekV32MTP", + "GlmMoeDsaForCausalLM", ] diff --git a/vllm/models/deepseek_v32/nvidia/attention.py b/vllm/models/deepseek_v32/nvidia/attention.py index dcf955ad59..8e60e87189 100644 --- a/vllm/models/deepseek_v32/nvidia/attention.py +++ b/vllm/models/deepseek_v32/nvidia/attention.py @@ -4,6 +4,7 @@ import torch import torch.nn as nn from transformers import DeepseekV2Config, DeepseekV3Config +import vllm._custom_ops as ops from vllm.compilation.breakable_cudagraph import eager_break_during_capture from vllm.config import CacheConfig, VllmConfig from vllm.distributed import get_tensor_model_parallel_world_size @@ -21,17 +22,15 @@ from vllm.model_executor.layers.quantization.utils.fp8_utils import ( per_token_group_quant_fp8, ) from vllm.model_executor.layers.rotary_embedding import get_rope -from vllm.model_executor.layers.sparse_attn_indexer import ( - SparseAttnIndexer, - sparse_attn_indexer, -) +from vllm.model_executor.layers.sparse_attn_indexer import SparseAttnIndexer from vllm.model_executor.models.deepseek_v2 import ( DeepSeekV2FusedQkvAProjLinear, DeepseekV32IndexerCache, yarn_get_mscale, ) from vllm.model_executor.models.utils import extract_layer_index -from vllm.utils.torch_utils import is_quantized_kv_cache +from vllm.platforms import current_platform +from vllm.utils.torch_utils import _encode_layer_name, is_quantized_kv_cache from .kernels import fused_norm_rope, fused_q @@ -171,6 +170,11 @@ class DeepseekV32Attention(MLAAttention): ) -> None: quant_config = vllm_config.quant_config cache_config = vllm_config.cache_config + if cache_config is not None and cache_config.cache_dtype == "auto": + # This implementation requires an FP8 sparse cache. Start with the + # generic FP8 format; MLAAttention will canonicalize it for the + # selected sparse backend. + cache_config.cache_dtype = "fp8" hidden_size = config.hidden_size qk_nope_head_dim = config.qk_nope_head_dim @@ -331,6 +335,37 @@ class DeepseekV32Attention(MLAAttention): rope_parameters=config.rope_parameters, is_neox_style=False, ) + # Decode-M GEMM dispatch for the bf16 A/B projections, measured on + # B300 and B200 (graph replay + rotating weights vs fp64): + # bf16_skinny_gemm for M <= skinny_max, dsv3_fused_a_gemm up to + # M = 16, cuBLAS above. Blackwell-only: the boundaries and the + # fused_a tile choices are tied to the 148/152-SM single-wave + # geometry. DSv3.2 q_b has no skinny tier (K=1536 does not fit the + # GEMV's 128x8 K-step; fused_a still wins 1.6-1.8x there). + # Quantized linear methods may not register .weight until + # process_weights_after_loading (e.g. compressed-tensors NVFP4 + # registers weight_packed at init) — probe with getattr. + # (N, K) -> skinny_max + qkv_a_dispatch = {(2624, 6144): 2, (2112, 7168): 2} + q_b_dispatch = {(2048, 2048): 2, (3072, 1536): 0} + ok = ( + current_platform.is_device_capability_family(100) + and hasattr(torch.ops._C, "bf16_skinny_gemm") + and hasattr(torch.ops._C, "dsv3_fused_a_gemm") + ) + qkv_a_weight = getattr(self.fused_qkv_a_proj, "weight", None) + q_b_weight = getattr(self.q_b_proj, "weight", None) + self._qkv_a_skinny_max = ( + qkv_a_dispatch.get(tuple(qkv_a_weight.shape)) + if ok and qkv_a_weight is not None and qkv_a_weight.dtype == torch.bfloat16 + else None + ) + self._q_b_skinny_max = ( + q_b_dispatch.get(tuple(q_b_weight.shape)) + if ok and q_b_weight is not None and q_b_weight.dtype == torch.bfloat16 + else None + ) + # Lightning indexer uses its own RoPE; interleave maps to non-NeoX. self.indexer_rope_emb = get_rope( qk_rope_head_dim, @@ -339,13 +374,32 @@ class DeepseekV32Attention(MLAAttention): is_neox_style=not getattr(config, "indexer_rope_interleave", False), ) + def _decode_m_gemm( + self, x: torch.Tensor, weight: torch.Tensor, skinny_max: int + ) -> torch.Tensor: + """x @ weight.T for decode M <= 16.""" + if x.shape[0] == 0: + return torch.empty((0, weight.shape[0]), dtype=x.dtype, device=x.device) + if x.shape[0] <= skinny_max: + return ops.bf16_skinny_gemm(x, weight) + out = torch.empty((x.shape[0], weight.shape[0]), dtype=x.dtype, device=x.device) + ops.dsv3_fused_a_gemm(out, x, weight.t()) + return out + def forward( # type: ignore[override] self, positions: torch.Tensor, hidden_states: torch.Tensor, ) -> torch.Tensor: # Captured: A-projections (+ indexer A-GEMM on indexer layers). - qkv_lora = self.fused_qkv_a_proj(hidden_states)[0] + if self._qkv_a_skinny_max is not None and hidden_states.shape[0] <= 16: + qkv_lora = self._decode_m_gemm( + hidden_states, + self.fused_qkv_a_proj.weight, + self._qkv_a_skinny_max, + ) + else: + qkv_lora = self.fused_qkv_a_proj(hidden_states)[0] q_c, kv_c, k_pe = qkv_lora.split( [self.q_lora_rank, self.kv_lora_rank, self.qk_rope_head_dim], dim=-1 ) @@ -398,7 +452,9 @@ class DeepseekV32Attention(MLAAttention): assert isinstance(slot_mapping, dict) mla_slot = slot_mapping.get(self.layer_name) - if self.indexer is not None: + # skip_topk (index_share_for_mtp_iteration reuse mode): don't run the + # indexer this step; keep the top-k already in the shared buffer. + if self.indexer is not None and not self.skip_topk: has_indexer = True indexer_k_norm_w = self.indexer.k_norm.weight indexer_k_norm_bias = self.indexer.k_norm.bias @@ -451,12 +507,16 @@ class DeepseekV32Attention(MLAAttention): index_rope_interleave=self._index_rope_interleave, ) - q = self.q_b_proj(q_c)[0].view(-1, self.num_local_heads, self.qk_head_dim) + if self._q_b_skinny_max is not None and q_c.shape[0] <= 16: + q = self._decode_m_gemm(q_c, self.q_b_proj.weight, self._q_b_skinny_max) + else: + q = self.q_b_proj(q_c)[0] + q = q.view(-1, self.num_local_heads, self.qk_head_dim) q_nope, q_pe = q.split([self.qk_nope_head_dim, self.qk_rope_head_dim], dim=-1) q_nope = q_nope.transpose(0, 1) ql_nope = torch.bmm(q_nope, self.W_UK_T).transpose(0, 1) - if self.indexer is not None: + if self.indexer is not None and not self.skip_topk: index_q = self.indexer.wq_b(q_c)[0] index_q = index_q.view(-1, self.indexer.n_head, self.indexer.head_dim) else: @@ -478,10 +538,10 @@ class DeepseekV32Attention(MLAAttention): quantize_mqa=self._fp8_query, ) - if self.indexer is not None: - sparse_attn_indexer( + if self.indexer is not None and not self.skip_topk: + torch.ops.vllm.sparse_attn_indexer( q_c, - self.indexer.k_cache.prefix, + _encode_layer_name(self.indexer.k_cache.prefix), self.indexer.k_cache.kv_cache, index_q_fp8, None, # q_scale folded into weights on the fp8 path diff --git a/vllm/models/deepseek_v32/nvidia/kernels.py b/vllm/models/deepseek_v32/nvidia/kernels.py index ef7b14ef1c..c931ac7cb4 100644 --- a/vllm/models/deepseek_v32/nvidia/kernels.py +++ b/vllm/models/deepseek_v32/nvidia/kernels.py @@ -2,13 +2,80 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import torch +from vllm.platforms import current_platform from vllm.triton_utils import tl, triton +from vllm.utils.import_utils import has_cutedsl +from vllm.utils.torch_utils import direct_register_custom_op # Cache of tiny 1-element dummy tensors (per device, dtype) reused by the # has_indexer=False path so the indexer args don't allocate every call. _DUMMY_CACHE: dict[tuple, torch.Tensor] = {} +@torch.compiler.assume_constant_result +def _can_use_fused_q_cutedsl() -> bool: + return current_platform.has_device_capability(100) and has_cutedsl() + + +@torch.compiler.assume_constant_result +def _is_arch_support_pdl() -> bool: + return ( + current_platform.has_device_capability(100) + and current_platform.is_arch_support_pdl() + ) + + +def _fused_q_cutedsl_impl( + positions: torch.Tensor, + q_pe: torch.Tensor, + rope_cache: torch.Tensor, + ql_nope: torch.Tensor, + q_scale: torch.Tensor, + mqa_output: torch.Tensor, + idx_q: torch.Tensor, + idx_rope_cache: torch.Tensor, + idx_weights: torch.Tensor, + idx_weights_softmax_scale: float, + idx_weights_head_scale: float, + idx_q_fp8: torch.Tensor, + idx_weights_out: torch.Tensor, + has_indexer: bool, + index_rope_interleave: bool, +) -> None: + from .ops.fused_q_cutedsl import fused_q_cutedsl + + fused_q_cutedsl( + positions, + q_pe, + rope_cache, + ql_nope, + q_scale, + mqa_output, + idx_q, + idx_rope_cache, + idx_weights, + idx_weights_softmax_scale, + idx_weights_head_scale, + idx_q_fp8, + idx_weights_out, + has_indexer=has_indexer, + index_rope_interleave=index_rope_interleave, + ) + + +def _fused_q_cutedsl_fake(*args, **kwargs) -> None: + pass + + +direct_register_custom_op( + op_name="fused_q_cutedsl", + op_func=_fused_q_cutedsl_impl, + mutates_args=["mqa_output", "idx_q_fp8", "idx_weights_out"], + fake_impl=_fused_q_cutedsl_fake, + dispatch_key="CUDA", +) + + def _dummy(shape: tuple, dtype: torch.dtype, device: torch.device) -> torch.Tensor: key = (shape, dtype, device) t = _DUMMY_CACHE.get(key) @@ -147,9 +214,14 @@ def _fused_norm_rope_kernel( TOPK_BLOCK_SIZE: tl.constexpr, HAS_INDEXER: tl.constexpr, INDEX_ROPE_INTERLEAVE: tl.constexpr, + USE_PDL: tl.constexpr, ): pid = tl.program_id(0) tok_idx = tl.program_id(1) + if USE_PDL: + tl.extra.cuda.gdc_wait() + tl.extra.cuda.gdc_launch_dependents() + if pid == 3: if not HAS_INDEXER: # Shared layer: reuse the previous indexer layer's top-k; do not @@ -166,6 +238,17 @@ def _fused_norm_rope_kernel( ) return + if pid == 2: + # Q RMS norm does not depend on cache availability. In particular, + # profiling uses a negative dummy slot mapping but still consumes Q. + q_block = tl.arange(0, Q_BLOCK_SIZE) + q_mask = q_block < Q_DIM + q_c = tl.load(q_c_ptr + tok_idx * q_c_stride + q_block, mask=q_mask, other=0.0) + q_c_rms_w = tl.load(q_rms_norm_w_ptr + q_block, mask=q_mask) + q_c = _rms_norm(q_c, q_c_rms_w, q_rms_eps, Q_DIM) + tl.store(q_c_out_ptr + tok_idx * q_c_out_stride + q_block, q_c, mask=q_mask) + return + if slot_mapping_ptr is None: # Memory profiling run. return @@ -174,15 +257,7 @@ def _fused_norm_rope_kernel( # Padding return - if pid == 2: - # Q RMS norm - q_block = tl.arange(0, Q_BLOCK_SIZE) - q_mask = q_block < Q_DIM - q_c = tl.load(q_c_ptr + tok_idx * q_c_stride + q_block, mask=q_mask, other=0.0) - q_c_rms_w = tl.load(q_rms_norm_w_ptr + q_block, mask=q_mask) - q_c = _rms_norm(q_c, q_c_rms_w, q_rms_eps, Q_DIM) - tl.store(q_c_out_ptr + tok_idx * q_c_out_stride + q_block, q_c, mask=q_mask) - elif pid == 1: + if pid == 1: # KV RMS Norm + KV RoPE + MLA concat_and_cache. # Merged so the normed kv_c and RoPE'd k_pe can be written # to the MLA KV cache directly without a separate kernel. @@ -211,9 +286,6 @@ def _fused_norm_rope_kernel( r2 = x2 * cos + x1 * sin # MLA concat_and_cache: write [kv_c_normed, k_pe_roped] to cache. - if mla_cache_entry_stride == 0: - return - mla_block_size = mla_cache_block_stride // mla_cache_entry_stride mla_block_idx = slot_idx // mla_block_size mla_block_off = slot_idx % mla_block_size @@ -363,7 +435,7 @@ def _fused_norm_rope_kernel( ) -def fused_norm_rope( +def _fused_norm_rope_impl( positions: torch.Tensor, q_c: torch.Tensor, q_rms_norm_w: torch.Tensor, @@ -465,12 +537,17 @@ def fused_norm_rope( # Dummy values — pid 2 will skip the MLA cache write because # slot_mapping is all -1. mla_kv_cache = torch.empty(0, dtype=torch.bfloat16, device=device) - mla_block_stride = 0 - mla_entry_stride = 0 + # Torch's Triton HOP mutation analysis compiles every program-id + # branch even though the dummy slot mapping returns before cache + # access. Keep the unused strides nonzero so that analysis can lower + # the block-size calculation safely. + mla_block_stride = 1 + mla_entry_stride = 1 mla_k_scale = torch.ones(1, dtype=torch.float32, device=device) if q_c_out is None: q_c_out = torch.empty_like(q_c) + use_pdl = _is_arch_support_pdl() _fused_norm_rope_kernel[(4, num_tokens)]( positions, # Q RMS norm @@ -529,6 +606,157 @@ def fused_norm_rope( TOPK_BLOCK_SIZE=1024, HAS_INDEXER=has_indexer, INDEX_ROPE_INTERLEAVE=index_rope_interleave, + USE_PDL=use_pdl, + launch_pdl=use_pdl, + ) + return q_c_out + + +def _fused_norm_rope_op( + positions: torch.Tensor, + q_c: torch.Tensor, + q_rms_norm_w: torch.Tensor, + q_rms_eps: float, + kv_c: torch.Tensor, + kv_rms_norm_w: torch.Tensor, + kv_rms_eps: float, + k_pe: torch.Tensor, + k_rope_cos_sin_cache: torch.Tensor, + index_k: torch.Tensor | None, + index_k_layer_norm_w: torch.Tensor | None, + index_k_layer_norm_bias: torch.Tensor | None, + index_k_layer_norm_eps: float, + index_k_rope_cos_sin_cache: torch.Tensor | None, + topk_indices_buffer: torch.Tensor, + slot_mapping: torch.Tensor | None, + indexer_k_cache: torch.Tensor | None, + mla_kv_cache: torch.Tensor | None, + mla_kv_cache_dtype: str, + mla_k_scale: torch.Tensor | None, + has_indexer: bool, + index_rope_interleave: bool, + q_c_out: torch.Tensor, +) -> None: + _fused_norm_rope_impl( + positions, + q_c, + q_rms_norm_w, + q_rms_eps, + kv_c, + kv_rms_norm_w, + kv_rms_eps, + k_pe, + k_rope_cos_sin_cache, + index_k, + index_k_layer_norm_w, + index_k_layer_norm_bias, + index_k_layer_norm_eps, + index_k_rope_cos_sin_cache, + topk_indices_buffer, + slot_mapping=slot_mapping, + indexer_k_cache=indexer_k_cache, + mla_kv_cache=mla_kv_cache, + mla_kv_cache_dtype=mla_kv_cache_dtype, + mla_k_scale=mla_k_scale, + has_indexer=has_indexer, + index_rope_interleave=index_rope_interleave, + q_c_out=q_c_out, + ) + + +def _fused_norm_rope_fake( + positions: torch.Tensor, + q_c: torch.Tensor, + q_rms_norm_w: torch.Tensor, + q_rms_eps: float, + kv_c: torch.Tensor, + kv_rms_norm_w: torch.Tensor, + kv_rms_eps: float, + k_pe: torch.Tensor, + k_rope_cos_sin_cache: torch.Tensor, + index_k: torch.Tensor | None, + index_k_layer_norm_w: torch.Tensor | None, + index_k_layer_norm_bias: torch.Tensor | None, + index_k_layer_norm_eps: float, + index_k_rope_cos_sin_cache: torch.Tensor | None, + topk_indices_buffer: torch.Tensor, + slot_mapping: torch.Tensor | None, + indexer_k_cache: torch.Tensor | None, + mla_kv_cache: torch.Tensor | None, + mla_kv_cache_dtype: str, + mla_k_scale: torch.Tensor | None, + has_indexer: bool, + index_rope_interleave: bool, + q_c_out: torch.Tensor, +) -> None: + return + + +direct_register_custom_op( + op_name="fused_norm_rope_deepseek_v32", + op_func=_fused_norm_rope_op, + mutates_args=[ + "topk_indices_buffer", + "indexer_k_cache", + "mla_kv_cache", + "q_c_out", + ], + fake_impl=_fused_norm_rope_fake, + dispatch_key="CUDA", +) + + +def fused_norm_rope( + positions: torch.Tensor, + q_c: torch.Tensor, + q_rms_norm_w: torch.Tensor, + q_rms_eps: float, + kv_c: torch.Tensor, + kv_rms_norm_w: torch.Tensor, + kv_rms_eps: float, + k_pe: torch.Tensor, + k_rope_cos_sin_cache: torch.Tensor, + index_k: torch.Tensor | None, + index_k_layer_norm_w: torch.Tensor | None, + index_k_layer_norm_bias: torch.Tensor | None, + index_k_layer_norm_eps: float, + index_k_rope_cos_sin_cache: torch.Tensor | None, + topk_indices_buffer: torch.Tensor, + slot_mapping: torch.Tensor | None = None, + indexer_k_cache: torch.Tensor | None = None, + mla_kv_cache: torch.Tensor | None = None, + mla_kv_cache_dtype: str = "auto", + mla_k_scale: torch.Tensor | None = None, + has_indexer: bool = True, + index_rope_interleave: bool = False, + q_c_out: torch.Tensor | None = None, +) -> torch.Tensor: + if q_c_out is None: + q_c_out = torch.empty_like(q_c) + torch.ops.vllm.fused_norm_rope_deepseek_v32( + positions, + q_c, + q_rms_norm_w, + q_rms_eps, + kv_c, + kv_rms_norm_w, + kv_rms_eps, + k_pe, + k_rope_cos_sin_cache, + index_k, + index_k_layer_norm_w, + index_k_layer_norm_bias, + index_k_layer_norm_eps, + index_k_rope_cos_sin_cache, + topk_indices_buffer, + slot_mapping, + indexer_k_cache, + mla_kv_cache, + mla_kv_cache_dtype, + mla_k_scale, + has_indexer, + index_rope_interleave, + q_c_out, ) return q_c_out @@ -582,10 +810,14 @@ def _fused_q_kernel( HAS_INDEXER: tl.constexpr, INDEX_ROPE_INTERLEAVE: tl.constexpr, QUANTIZE_MQA: tl.constexpr, + USE_PDL: tl.constexpr, ): pid = tl.program_id(0) tok_idx = tl.program_id(1) head_idx = tl.program_id(2) + if USE_PDL: + tl.extra.cuda.gdc_wait() + tl.extra.cuda.gdc_launch_dependents() if pid == 2: # ql_nope quantize + pack into the front of mqa_q_fp8. On the bf16 @@ -773,10 +1005,12 @@ def fused_q( ``(ql_nope, q_pe)`` tuple the backend expects. """ assert positions.ndim == 1 + assert positions.dtype == torch.int64 assert q_pe.ndim == 3 assert q_pe_cos_sin_cache.ndim == 2 assert ql_nope.ndim == 3 assert ql_nope.shape[:2] == q_pe.shape[:2] + assert q_scale.dtype == torch.float32 and q_scale.numel() == 1 num_tokens = positions.shape[0] num_q_heads = q_pe.shape[1] @@ -794,7 +1028,18 @@ def fused_q( assert index_weights is not None num_index_q_heads = index_q.shape[1] index_q_head_dim = index_q.shape[2] - grid_heads = max(mqa_grid_heads, num_index_q_heads) + use_cutedsl = False + if _can_use_fused_q_cutedsl(): + from .ops.fused_q_cutedsl import is_fused_q_cutedsl_supported + + use_cutedsl = is_fused_q_cutedsl_supported( + q_pe, + index_q, + ql_nope, + has_indexer=has_indexer, + quantize_mqa=quantize_mqa, + ) + grid_heads = max(mqa_grid_heads, num_index_q_heads if has_indexer else 1) if quantize_mqa: # fp8 path: pack [ql_nope; q_pe] into a single fp8 tensor. mqa_q_fp8 = torch.empty( @@ -815,6 +1060,27 @@ def fused_q( index_q_fp8 = torch.empty_like(index_q, dtype=torch.float8_e4m3fn) index_weights_out = torch.empty_like(index_weights, dtype=torch.float32) + if use_cutedsl: + torch.ops.vllm.fused_q_cutedsl( + positions, + q_pe, + q_pe_cos_sin_cache, + ql_nope, + q_scale, + mqa_q, + index_q, + index_q_cos_sin_cache, + index_weights, + index_weights_softmax_scale, + index_weights_head_scale, + index_q_fp8, + index_weights_out, + has_indexer=has_indexer, + index_rope_interleave=index_rope_interleave, + ) + return index_q_fp8, index_weights_out, mqa_q + + use_pdl = _is_arch_support_pdl() _fused_q_kernel[(3, num_tokens, grid_heads)]( positions, q_pe, @@ -856,6 +1122,8 @@ def fused_q( HAS_INDEXER=has_indexer, INDEX_ROPE_INTERLEAVE=index_rope_interleave, QUANTIZE_MQA=quantize_mqa, + USE_PDL=use_pdl, + launch_pdl=use_pdl, # num_warps=1 is optimal here: each program is a single 128-element # rope+quant, so the kernel is program-count/occupancy bound, not # per-program compute bound (swept 1/2/4/8 — 1 wins or ties everywhere). diff --git a/vllm/models/deepseek_v32/nvidia/model.py b/vllm/models/deepseek_v32/nvidia/model.py index 353aedc8ce..90a6a2cf45 100644 --- a/vllm/models/deepseek_v32/nvidia/model.py +++ b/vllm/models/deepseek_v32/nvidia/model.py @@ -6,6 +6,7 @@ from itertools import islice import torch +from vllm.compilation.decorators import support_torch_compile from vllm.config import VllmConfig from vllm.distributed import ( get_pp_group, @@ -39,6 +40,7 @@ from vllm.model_executor.models.utils import ( sequence_parallel_chunk, ) from vllm.sequence import IntermediateTensors +from vllm.v1.attention.backends.mla.sparse_utils import register_phys_shadow from .attention import DeepseekV32Attention from .fused_ops import fused_allreduce_rms_norm @@ -177,6 +179,7 @@ class DeepseekV32DecoderLayer(torch.nn.Module): return hidden_states, residual +@support_torch_compile class DeepseekV32Model(torch.nn.Module): fall_back_to_pt_during_load = False @@ -200,6 +203,7 @@ class DeepseekV32Model(torch.nn.Module): dtype=torch.int32, device=self.device, ) + register_phys_shadow(topk_indices_buffer) if get_pp_group().is_first_rank: self.embed_tokens = VocabParallelEmbedding( diff --git a/vllm/models/deepseek_v32/nvidia/mtp.py b/vllm/models/deepseek_v32/nvidia/mtp.py index c8f2fcc5ff..bafbf383ca 100644 --- a/vllm/models/deepseek_v32/nvidia/mtp.py +++ b/vllm/models/deepseek_v32/nvidia/mtp.py @@ -6,9 +6,9 @@ from collections.abc import Callable, Iterable import torch import torch.nn as nn +import vllm._custom_ops as ops from vllm._aiter_ops import rocm_aiter_ops from vllm.config import VllmConfig -from vllm.distributed import tensor_model_parallel_all_reduce from vllm.model_executor.layers.fused_moe import ( fused_moe_make_expert_params_mapping, ) @@ -40,7 +40,12 @@ from vllm.model_executor.models.utils import ( ) from vllm.platforms import current_platform from vllm.sequence import IntermediateTensors +from vllm.v1.attention.backends.mla.sparse_utils import ( + phys_shadow, + register_phys_shadow, +) +from .fused_ops import fused_allreduce_rms_norm from .kernels import fused_eh_norm from .model import DeepseekV32DecoderLayer @@ -56,6 +61,19 @@ class DeepseekV32MultiTokenPredictorLayer(nn.Module): self.enorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) self.hnorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) self.eh_proj = nn.Linear(config.hidden_size * 2, config.hidden_size, bias=False) + # bf16 skinny GEMM for the eh_proj, B300+B200 measured: GLM-5.2 + # (6144, 12288) wins at M <= 3, DSv3.2 (7168, 14336) at M <= 2; + # cuBLAS holds above (the 151/205MB weights stream at ~5.7TB/s). + eh_dispatch = {(6144, 12288): 3, (7168, 14336): 2} + eh_weight = getattr(self.eh_proj, "weight", None) + self._eh_skinny_max = ( + eh_dispatch.get(tuple(eh_weight.shape), 0) + if current_platform.is_device_capability_family(100) + and hasattr(torch.ops._C, "bf16_skinny_gemm") + and eh_weight is not None + and eh_weight.dtype == torch.bfloat16 + else 0 + ) topk_indices_buffer = torch.empty( vllm_config.scheduler_config.max_num_batched_tokens, @@ -63,6 +81,7 @@ class DeepseekV32MultiTokenPredictorLayer(nn.Module): dtype=torch.int32, device=current_platform.device_type, ) + register_phys_shadow(topk_indices_buffer) self.shared_head = SharedHead( config=config, prefix=prefix, quant_config=quant_config ) @@ -91,7 +110,13 @@ class DeepseekV32MultiTokenPredictorLayer(nn.Module): self.hnorm.weight, self.enorm.variance_epsilon, ) - hidden_states = self.eh_proj(eh_input) + if ( + eh_input.shape[0] <= self._eh_skinny_max + and eh_input.dtype == torch.bfloat16 + ): + hidden_states = ops.bf16_skinny_gemm(eh_input, self.eh_proj.weight) + else: + hidden_states = self.eh_proj(eh_input) hidden_states, residual = self.mtp_block( positions=positions, hidden_states=hidden_states, residual=None ) @@ -101,9 +126,6 @@ class DeepseekV32MultiTokenPredictorLayer(nn.Module): positions.shape[0], is_sequence_parallel=self.mtp_block.use_sequence_parallel_moe, ) - if not self.mtp_block.use_sequence_parallel_moe: - # Without sequence parallelism, the MoE output is left un-reduced. - hidden_states = tensor_model_parallel_all_reduce(hidden_states) # Recycle the POST-final-norm hidden into the next draft step. The # residual-add is fused into the final RMSNorm so it is computed # exactly once, and the result is returned for both tuple positions: @@ -114,7 +136,14 @@ class DeepseekV32MultiTokenPredictorLayer(nn.Module): # is understood by both the V2 speculator (isinstance-tuple check) and # the legacy proposer (model_returns_tuple is True for the # DeepSeekMTPModel architecture). - hidden_states, _ = self.shared_head.norm(hidden_states, residual) + if self.mtp_block.use_sequence_parallel_moe: + hidden_states, _ = self.shared_head.norm(hidden_states, residual) + else: + # The MoE output is left un-reduced; fuse its all-reduce into the + # final norm, as the main model does at layer boundaries. + hidden_states, _ = fused_allreduce_rms_norm( + hidden_states, residual, self.shared_head.norm + ) return hidden_states, hidden_states @@ -157,6 +186,15 @@ class DeepseekV32MultiTokenPredictor(nn.Module): if self_attn is not None and hasattr(self_attn, "topk_indices_buffer"): topk_indices_buffer = self_attn.topk_indices_buffer topk_indices_buffer[:num_slots] = topk_indices_buffer[slot_ids] + # The physical-index shadow mirrors this buffer row-wise + # (shadow[j] == convert(logical[j])); re-arrange it the same + # way or skip_topk draft steps read the step-0 multi-token + # layout's rows for the wrong tokens/requests. + shadow = phys_shadow(topk_indices_buffer) + if shadow is not None: + phys, seq = shadow + phys[:num_slots] = phys[slot_ids] + seq[:num_slots] = seq[slot_ids] def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: return self.embed_tokens(input_ids) @@ -192,6 +230,19 @@ class DeepseekV32MultiTokenPredictor(nn.Module): # second RMSNorm. return self.logits_processor(mtp_layer.shared_head.head, hidden_states) + def get_top_tokens( + self, + hidden_states: torch.Tensor, + spec_step_idx: int = 0, + ) -> torch.Tensor: + # Greedy draft token ids via vocab-parallel local argmax: skips the + # full-vocab logits all-gather (use_local_argmax_reduction). + current_step_idx = spec_step_idx % self.num_mtp_layers + mtp_layer = self.layers[str(self.mtp_start_layer_idx + current_step_idx)] + return self.logits_processor.get_top_tokens( + mtp_layer.shared_head.head, hidden_states + ) + class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts): def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): @@ -240,6 +291,25 @@ class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts): ) -> torch.Tensor | None: return self.model.compute_logits(hidden_states, spec_step_idx) + def get_top_tokens( + self, + hidden_states: torch.Tensor, + spec_step_idx: int | None = None, + ) -> torch.Tensor: + # The V2 speculator calls this without spec_step_idx; that is only + # correct while a single MTP layer is cycled for every draft step + # (num_nextn_predict_layers == 1, as in GLM-5.2/DSV3.2). Guard the + # assumption so a future multi-layer MTP fails loudly instead of + # silently using step 0's head for every step. + if spec_step_idx is None: + assert self.model.num_mtp_layers == 1, ( + "get_top_tokens called without spec_step_idx on a " + "multi-layer MTP; thread the draft step index through " + "the speculator." + ) + spec_step_idx = 0 + return self.model.get_top_tokens(hidden_states, spec_step_idx) + def _rewrite_spec_layer_name(self, spec_layer: int, name: str) -> str: spec_layer_weight_names = [ "embed_tokens", diff --git a/vllm/models/deepseek_v32/nvidia/ops/__init__.py b/vllm/models/deepseek_v32/nvidia/ops/__init__.py new file mode 100644 index 0000000000..208f01a7cb --- /dev/null +++ b/vllm/models/deepseek_v32/nvidia/ops/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project diff --git a/vllm/models/deepseek_v32/nvidia/ops/fused_q_cutedsl.py b/vllm/models/deepseek_v32/nvidia/ops/fused_q_cutedsl.py new file mode 100644 index 0000000000..b98147a34a --- /dev/null +++ b/vllm/models/deepseek_v32/nvidia/ops/fused_q_cutedsl.py @@ -0,0 +1,532 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from functools import cache + +import cutlass +import cutlass.cute as cute +import torch +from cuda.bindings.driver import CUstream +from cutlass import BFloat16, Float8E4M3FN, Float32, Int64, Uint8, Uint16, Uint32 + +from vllm.cute_utils import TORCH_TO_CUTE_DTYPE, cvt + + +def _make_fake_tensor(dtype, shape, divisibility): + stride = tuple( + 1 if i == len(shape) - 1 else cute.sym_int64(divisibility=divisibility) + for i in range(len(shape)) + ) + return cute.runtime.make_fake_tensor( + dtype, + shape, + stride, + assumed_align=divisibility * dtype.width // 8, + ) + + +def is_fused_q_cutedsl_supported( + q_pe: torch.Tensor, + index_q: torch.Tensor | None, + ql_nope: torch.Tensor, + *, + has_indexer: bool, + quantize_mqa: bool, +) -> bool: + if not ( + quantize_mqa + and q_pe.dtype == ql_nope.dtype == torch.bfloat16 + and q_pe.shape[-1] == 64 + and ql_nope.shape[-1] == 512 + ): + return False + return not has_indexer or ( + index_q is not None + and index_q.dtype == torch.bfloat16 + and index_q.shape[1] % 16 == 0 + and index_q.shape[-1] == 128 + ) + + +def fused_q_cutedsl( + positions: torch.Tensor, + q_pe: torch.Tensor, + rope_cache: torch.Tensor, + ql_nope: torch.Tensor, + q_scale: torch.Tensor, + mqa_output: torch.Tensor, + idx_q: torch.Tensor, + idx_rope_cache: torch.Tensor, + idx_weights: torch.Tensor, + idx_weights_softmax_scale: float, + idx_weights_head_scale: float, + idx_q_fp8: torch.Tensor, + idx_weights_out: torch.Tensor, + has_indexer: bool = True, + index_rope_interleave: bool = True, +) -> None: + _, num_heads, rope_dim = q_pe.shape + _, _, nope_dim = ql_nope.shape + _, num_idx_heads, idx_dim = idx_q.shape + + if has_indexer: + idx_rope_type = TORCH_TO_CUTE_DTYPE[idx_rope_cache.dtype] + idx_weights_type = TORCH_TO_CUTE_DTYPE[idx_weights.dtype] + else: + idx_dim = num_idx_heads = 0 + idx_q = idx_rope_cache = idx_q_fp8 = None + idx_weights = idx_weights_out = None + idx_rope_type = idx_weights_type = None + + rope_type = TORCH_TO_CUTE_DTYPE[rope_cache.dtype] + compiled = FusedQKernel.compile( + rope_dim, + nope_dim, + num_heads, + rope_type, + idx_dim, + num_idx_heads, + idx_rope_type, + idx_weights_type, + index_rope_interleave, + ) + compiled( + positions, + q_pe, + rope_cache, + ql_nope, + q_scale.view(1), + mqa_output, + idx_q, + idx_rope_cache, + idx_weights, + idx_q_fp8, + idx_weights_out, + float(idx_weights_softmax_scale * idx_weights_head_scale), + ) + + +class FusedQKernel: + def __init__( + self, + rope_dim: int, + nope_dim: int, + num_heads: int, + idx_dim: int, + num_idx_heads: int, + index_rope_interleave: bool, + ) -> None: + assert rope_dim == 64 + assert nope_dim == 512 + assert idx_dim in (128, 0) + + self.rope_dim = rope_dim + self.nope_dim = nope_dim + self.num_heads = num_heads + self.idx_dim = idx_dim + self.num_idx_heads = num_idx_heads + self.index_rope_interleave = index_rope_interleave + + # mqa: rope_dim=64, nope_dim=512, num_heads=64/TP + # indexer: rope_dim=64, nope_dim=64, num_heads=32 + + self.num_warps = 4 + assert num_heads % self.num_warps == 0 + assert num_idx_heads % (4 * self.num_warps) == 0 + self.num_ctas_per_tok = num_heads // self.num_warps + self.num_ctas_per_idx_tok = num_idx_heads // (4 * self.num_warps) + + @cute.jit + def __call__( + self, + positions: cute.Tensor, + q_pe: cute.Tensor, + rope_cache: cute.Tensor, + ql_nope: cute.Tensor, + q_scale: cute.Tensor, + mqa_output: cute.Tensor, + idx_q: cute.Tensor, + idx_rope_cache: cute.Tensor, + idx_weights: cute.Tensor, + idx_q_fp8: cute.Tensor, + idx_weights_out: cute.Tensor, + weight_scale: Float32, + stream: CUstream, + ): + num_tokens = positions.shape[0] + if cutlass.const_expr(self.idx_dim == 0): + grid = (num_tokens, self.num_ctas_per_tok, 1) + else: + num_mqa_ctas = num_tokens * self.num_ctas_per_tok + num_idx_ctas = num_tokens * self.num_ctas_per_idx_tok + grid = (num_mqa_ctas + num_idx_ctas, 1, 1) + + self.kernel( + positions, + q_pe, + rope_cache, + ql_nope, + q_scale, + mqa_output, + idx_q, + idx_rope_cache, + idx_weights, + idx_q_fp8, + idx_weights_out, + weight_scale, + ).launch( + grid=grid, + block=(self.num_warps * 32, 1, 1), + stream=stream, + use_pdl=True, + ) + + @cute.kernel + def kernel( + self, + positions: cute.Tensor, + q_pe: cute.Tensor, + q_pe_rope_cache: cute.Tensor, + ql_nope: cute.Tensor, + q_scale: cute.Tensor, + mqa_output: cute.Tensor, + idx_q: cute.Tensor, + idx_q_rope_cache: cute.Tensor, + idx_weights: cute.Tensor, + idx_q_fp8: cute.Tensor, + idx_weights_out: cute.Tensor, + weight_scale: Float32, + ): + if cutlass.const_expr(self.idx_dim == 0): + token_id, group_id, _ = cute.arch.block_idx() + self.mqa( + positions, + q_pe, + q_pe_rope_cache, + ql_nope, + q_scale, + mqa_output, + token_id, + group_id, + ) + else: + # CTA-specialization + bid, _, _ = cute.arch.block_idx() + num_mqa_ctas = positions.shape[0] * self.num_ctas_per_tok + if bid < num_mqa_ctas: + self.mqa( + positions, + q_pe, + q_pe_rope_cache, + ql_nope, + q_scale, + mqa_output, + bid // self.num_ctas_per_tok, + bid % self.num_ctas_per_tok, + ) + else: + bid -= num_mqa_ctas + self.indexer( + positions, + idx_q, + idx_q_rope_cache, + idx_weights, + idx_q_fp8, + idx_weights_out, + weight_scale, + bid // self.num_ctas_per_idx_tok, + bid % self.num_ctas_per_idx_tok, + ) + + @cute.jit + def mqa( + self, + positions: cute.Tensor, + q_pe: cute.Tensor, + q_pe_rope_cache: cute.Tensor, + ql_nope: cute.Tensor, + q_scale: cute.Tensor, + mqa_output: cute.Tensor, + token_id, + group_id, + ): + tid, _, _ = cute.arch.thread_idx() + warp_id = cute.arch.make_warp_uniform(tid // 32) + lane_id = tid % 32 + head_id = group_id * self.num_warps + warp_id + + cute.arch.griddepcontrol_wait() + + pos = positions[token_id] + inv_scale = 1.0 / q_scale[0] + + cp_op = cute.nvgpu.CopyUniversalOp() + cp_32B = cute.make_copy_atom(cp_op, Uint32, num_bits_per_copy=256) + cp_16B = cute.make_copy_atom(cp_op, Uint32, num_bits_per_copy=128) + cp_4B = cute.make_copy_atom(cp_op, Uint32, num_bits_per_copy=32) + cp_2B = cute.make_copy_atom(cp_op, Uint8, num_bits_per_copy=16) + + ##### issue all loads asap ##### + rQ_nope_bf16 = cute.make_rmem_tensor(16, BFloat16) + rQ_rope_bf16 = cute.make_rmem_tensor(2, BFloat16) + + src_ql_nope = cute.local_tile( + ql_nope[token_id, head_id, None], (16,), (lane_id,) + ) + src_q_rope = cute.local_tile(q_pe[token_id, head_id, None], (2,), (lane_id,)) + cute.copy(cp_32B, src_ql_nope, rQ_nope_bf16) + cute.copy(cp_4B, src_q_rope, rQ_rope_bf16) + + rCos_raw = q_pe_rope_cache[pos, 0 + lane_id] + rSin_raw = q_pe_rope_cache[pos, 32 + lane_id] + + ##### process NoPE ##### + rQ_nope_f32 = cvt.bf16x2_to_fp32x2(rQ_nope_bf16).load() * inv_scale + rQ_nope_f8 = cute.make_rmem_tensor(16, Float8E4M3FN) + rQ_nope_f8.store(rQ_nope_f32.to(Float8E4M3FN)) + dst_Q_nope = cute.local_tile( + mqa_output[token_id, head_id, None], (16,), (lane_id,) + ) + cute.copy(cp_16B, rQ_nope_f8, dst_Q_nope) + + ##### process RoPE ###### + rQ_rope_f32 = cvt.bf16x2_to_fp32x2(rQ_rope_bf16) + rCos = rCos_raw.to(Float32) + rSin = rSin_raw.to(Float32) + r0 = (rQ_rope_f32[0] * rCos - rQ_rope_f32[1] * rSin) * inv_scale + r1 = (rQ_rope_f32[1] * rCos + rQ_rope_f32[0] * rSin) * inv_scale + + cute.arch.griddepcontrol_launch_dependents() + + # TensorSSA fp32->fp8 cvt has a bug. rely on direct PTX + rQ_rope_f8 = cute.make_rmem_tensor(2, Float8E4M3FN) + cute.recast_tensor(rQ_rope_f8, Uint16)[0] = cvt.fp32x2_to_fp8x2(r0, r1) + dst_Q_rope = cute.local_tile( + mqa_output[token_id, head_id, None], (2,), (256 + lane_id,) + ) + cute.copy(cp_2B, rQ_rope_f8, dst_Q_rope) + + @cute.jit + def indexer( + self, + positions: cute.Tensor, + idx_q: cute.Tensor, + idx_q_rope_cache: cute.Tensor, + idx_weights: cute.Tensor, + idx_q_fp8: cute.Tensor, + idx_weights_out: cute.Tensor, + weight_scale: Float32, + token_id, + group_id, + ): + tid, _, _ = cute.arch.thread_idx() + subwarp_id = tid // 8 + sublane_id = tid % 8 + head_id = group_id * (4 * self.num_warps) + subwarp_id + cute.arch.griddepcontrol_wait() + + pos = positions[token_id] + + cp_op = cute.nvgpu.CopyUniversalOp() + cp_16B = cute.make_copy_atom(cp_op, Uint32, num_bits_per_copy=128) + cp_8B = cute.make_copy_atom(cp_op, Uint32, num_bits_per_copy=64) + cp_4B = cute.make_copy_atom(cp_op, Uint32, num_bits_per_copy=32) + + ##### issue all loads first ##### + rQ_rope_bf16 = cute.make_rmem_tensor(8, BFloat16) + if cutlass.const_expr(self.index_rope_interleave): + src_idx_q_rope = cute.local_tile( + idx_q[token_id, head_id, None], (8,), (sublane_id,) + ) + cute.copy(cp_16B, src_idx_q_rope, rQ_rope_bf16) + else: + src_idx_q_rope = cute.zipped_divide( + idx_q[token_id, head_id, None], (4,) + ) # (4,32) + cute.copy( + cp_8B, + src_idx_q_rope[None, 0 + sublane_id], + cute.local_tile(rQ_rope_bf16, (4,), (0,)), + ) + cute.copy( + cp_8B, + src_idx_q_rope[None, 8 + sublane_id], + cute.local_tile(rQ_rope_bf16, (4,), (1,)), + ) + + rQ_nope_bf16 = cute.make_rmem_tensor(8, BFloat16) + src_idx_q_nope = cute.local_tile( + idx_q[token_id, head_id, None], (8,), (8 + sublane_id,) + ) + cute.copy(cp_16B, src_idx_q_nope, rQ_nope_bf16) + + rCos_raw = cute.make_rmem_tensor(4, idx_q_rope_cache.element_type) + rSin_raw = cute.make_rmem_tensor(4, idx_q_rope_cache.element_type) + rope_cache_view = cute.zipped_divide(idx_q_rope_cache[pos, None], (4,)) + + if cutlass.const_expr(idx_q_rope_cache.element_type == Float32): + cute.copy(cp_16B, rope_cache_view[None, sublane_id], rCos_raw) + cute.copy(cp_16B, rope_cache_view[None, 8 + sublane_id], rSin_raw) + elif cutlass.const_expr(idx_q_rope_cache.element_type == BFloat16): + cute.copy(cp_8B, rope_cache_view[None, sublane_id], rCos_raw) + cute.copy(cp_8B, rope_cache_view[None, 8 + sublane_id], rSin_raw) + + # unpack to FP32 + rQ_rope_f32 = cvt.bf16x2_to_fp32x2(rQ_rope_bf16) + rQ_nope_f32 = cvt.bf16x2_to_fp32x2(rQ_nope_bf16) + if cutlass.const_expr(idx_q_rope_cache.element_type == Float32): + rCos = rCos_raw + rSin = rSin_raw + elif cutlass.const_expr(idx_q_rope_cache.element_type == BFloat16): + rCos = cvt.bf16x2_to_fp32x2(rCos_raw) + rSin = cvt.bf16x2_to_fp32x2(rSin_raw) + + # apply rope + for i in cutlass.range_constexpr(4): + if cutlass.const_expr(self.index_rope_interleave): + r0 = rQ_rope_f32[i * 2 + 0] * rCos[i] - rQ_rope_f32[i * 2 + 1] * rSin[i] + r1 = rQ_rope_f32[i * 2 + 1] * rCos[i] + rQ_rope_f32[i * 2 + 0] * rSin[i] + rQ_rope_f32[i * 2 + 0] = r0 + rQ_rope_f32[i * 2 + 1] = r1 + else: + r0 = rQ_rope_f32[0 + i] * rCos[i] - rQ_rope_f32[4 + i] * rSin[i] + r1 = rQ_rope_f32[4 + i] * rCos[i] + rQ_rope_f32[0 + i] * rSin[i] + rQ_rope_f32[0 + i] = r0 + rQ_rope_f32[4 + i] = r1 + + # amax + amax = Float32(1e-4) + for i in cutlass.range_constexpr(8): + amax = cute.arch.fmax(amax, cute.math.absf(rQ_rope_f32[i])) + amax = cute.arch.fmax(amax, cute.math.absf(rQ_nope_f32[i])) + + # warp reduction among 8 lanes + for i in cutlass.range_constexpr(3): + other = cute.arch.shuffle_sync_bfly(amax, 1 << i) + amax = cute.arch.fmax(amax, other) + + # compute scale from amax + # exp2(ceil(log2(scale))) via bit manipulation + scale = amax * (1.0 / 448.0) + bits = scale.bitcast(Uint32) + exp_bits = (bits + Uint32(0x007FFFFF)) & Uint32(0x7F800000) + scale = exp_bits.bitcast(Float32) + inv_scale = (Uint32(0x7F000000) - exp_bits).bitcast(Float32) + + for i in cutlass.range_constexpr(8): + rQ_nope_f32[i] *= inv_scale + rQ_rope_f32[i] *= inv_scale + + cute.arch.griddepcontrol_launch_dependents() + + # quantize and store + rQ_nope_f8 = cute.make_rmem_tensor(8, Float8E4M3FN) + rQ_nope_f8.store(rQ_nope_f32.load().to(Float8E4M3FN)) + dst_idx_q_nope = cute.local_tile( + idx_q_fp8[token_id, head_id, None], (8,), (8 + sublane_id,) + ) + cute.copy(cp_8B, rQ_nope_f8, dst_idx_q_nope) + + rQ_rope_f8 = cute.make_rmem_tensor(8, Float8E4M3FN) + rQ_rope_f8.store(rQ_rope_f32.load().to(Float8E4M3FN)) + if cutlass.const_expr(self.index_rope_interleave): + dst_idx_q_rope = cute.local_tile( + idx_q_fp8[token_id, head_id, None], (8,), (sublane_id,) + ) + cute.copy(cp_8B, rQ_rope_f8, dst_idx_q_rope) + else: + dst_idx_q_rope = cute.zipped_divide( + idx_q_fp8[token_id, head_id, None], (4,) + ) # (4,32) + cute.copy( + cp_4B, + cute.local_tile(rQ_rope_f8, (4,), (0,)), + dst_idx_q_rope[None, 0 + sublane_id], + ) + cute.copy( + cp_4B, + cute.local_tile(rQ_rope_f8, (4,), (1,)), + dst_idx_q_rope[None, 8 + sublane_id], + ) + + # scale indexer weights + if sublane_id == 0: + w = idx_weights[token_id, head_id].to(Float32) + idx_weights_out[token_id, head_id] = w * scale * weight_scale + + @cache + @staticmethod + def compile( + rope_dim: int, + nope_dim: int, + num_heads: int, + rope_type: type[cutlass.Numeric], + idx_dim: int, + num_idx_heads: int, + idx_rope_type: type[cutlass.Numeric] | None, + idx_weights_type: type[cutlass.Numeric] | None, + index_rope_interleave: bool, + ): + num_tokens = cute.sym_int() + max_pos = cute.sym_int() + + positions = _make_fake_tensor(Int64, (num_tokens,), divisibility=1) + q_pe = _make_fake_tensor( + BFloat16, (num_tokens, num_heads, rope_dim), divisibility=16 + ) + rope_cache = _make_fake_tensor(rope_type, (max_pos, rope_dim), divisibility=8) + ql_nope = _make_fake_tensor( + BFloat16, (num_tokens, num_heads, nope_dim), divisibility=16 + ) + q_scale = _make_fake_tensor(Float32, (1,), divisibility=4) + mqa_output = _make_fake_tensor( + Float8E4M3FN, + (num_tokens, num_heads, nope_dim + rope_dim), + divisibility=16, + ) + + if idx_rope_type is not None: + index_q = _make_fake_tensor( + BFloat16, (num_tokens, num_idx_heads, idx_dim), divisibility=16 + ) + index_rope_cache = _make_fake_tensor( + idx_rope_type, (max_pos, rope_dim), divisibility=8 + ) + index_weights = _make_fake_tensor( + idx_weights_type, (num_tokens, num_idx_heads), divisibility=8 + ) + index_q_fp8 = _make_fake_tensor( + Float8E4M3FN, (num_tokens, num_idx_heads, idx_dim), divisibility=16 + ) + index_weights_out = _make_fake_tensor( + Float32, (num_tokens, num_idx_heads), divisibility=4 + ) + else: + index_q = index_rope_cache = index_q_fp8 = None + index_weights = index_weights_out = None + + kernel = FusedQKernel( + rope_dim, + nope_dim, + num_heads, + idx_dim, + num_idx_heads, + index_rope_interleave, + ) + stream = cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True) + return cute.compile( + kernel, + positions, + q_pe, + rope_cache, + ql_nope, + q_scale, + mqa_output, + index_q, + index_rope_cache, + index_weights, + index_q_fp8, + index_weights_out, + Float32(0.0), + stream, + options="--enable-tvm-ffi", + ) diff --git a/vllm/v1/attention/backends/mla/flashinfer_mla_sparse.py b/vllm/v1/attention/backends/mla/flashinfer_mla_sparse.py index a6b9f4abb0..36df138a4e 100644 --- a/vllm/v1/attention/backends/mla/flashinfer_mla_sparse.py +++ b/vllm/v1/attention/backends/mla/flashinfer_mla_sparse.py @@ -28,6 +28,7 @@ from vllm.v1.attention.backend import ( MultipleOf, ) from vllm.v1.attention.backends.mla.sparse_utils import ( + phys_shadow, triton_convert_req_index_to_global_index, triton_filter_and_convert_dcp_index, ) @@ -404,6 +405,36 @@ class FlashInferMLASparseImpl(SparseMLACommonImpl[FlashInferMLASparseMetadata]): NUM_TOPK_TOKENS=topk_indices.shape[1], return_valid_counts=True, ) + elif (shadow := phys_shadow(self.topk_indices_buffer)) is not None: + # skip_topk layers reuse the previous fresh layer's PHYSICAL + # indices: the logical top-k indices are shared (one buffer for + # the whole forward) and block_table/req_id are per-step constant, + # so re-converting them per layer produces identical output. This + # covers both the backbone (index_topk_freq > 1) and MTP draft + # steps 1+ (index_share_for_mtp_iteration sets skip_topk; the + # draft compact re-arranges the shadow together with the logical + # buffer). The fresh/skip split is fixed per captured graph, so + # the branch is cudagraph-safe. + phys_buf, seq_buf = shadow + wrote_fresh = getattr(layer, "indexer", None) is not None and not getattr( + layer, "skip_topk", False + ) + if wrote_fresh: + topk_indices_physical, seq_lens = ( + triton_convert_req_index_to_global_index( + attn_metadata.req_id_per_token[:num_actual_toks], + attn_metadata.block_table, + topk_indices, + BLOCK_SIZE=attn_metadata.block_size, + NUM_TOPK_TOKENS=topk_indices.shape[1], + return_valid_counts=True, + out=phys_buf[:num_actual_toks], + valid_counts_out=seq_buf[:num_actual_toks], + ) + ) + else: + topk_indices_physical = phys_buf[:num_actual_toks] + seq_lens = seq_buf[:num_actual_toks] else: topk_indices_physical, seq_lens = triton_convert_req_index_to_global_index( attn_metadata.req_id_per_token[:num_actual_toks], diff --git a/vllm/v1/attention/backends/mla/sparse_utils.py b/vllm/v1/attention/backends/mla/sparse_utils.py index 681ab7cfd9..3cc141e292 100644 --- a/vllm/v1/attention/backends/mla/sparse_utils.py +++ b/vllm/v1/attention/backends/mla/sparse_utils.py @@ -128,6 +128,8 @@ def triton_convert_req_index_to_global_index( prefill_workspace_request_ids: torch.Tensor | None = None, prefill_workspace_starts: torch.Tensor | None = None, return_valid_counts: bool = False, + out: torch.Tensor | None = None, + valid_counts_out: torch.Tensor | None = None, ) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]: """ out[token_id, indice_id] = @@ -178,14 +180,20 @@ def triton_convert_req_index_to_global_index( req_id_c = req_id.contiguous() block_table_c = block_table.contiguous() token_indices_c = token_indices.contiguous() - out = torch.empty_like(token_indices_c) + # `out`/`valid_counts_out` allow writing into a stable pre-allocated buffer + # (used by the shared-physical-index cache to stay cudagraph-safe). + out = torch.empty_like(token_indices_c) if out is None else out # Allocate valid count buffer if needed (must be zero-initialized for atomics) valid_counts: torch.Tensor | None = None if return_valid_counts: - valid_counts = torch.zeros( - num_tokens, dtype=torch.int32, device=token_indices.device - ) + if valid_counts_out is None: + valid_counts = torch.zeros( + num_tokens, dtype=torch.int32, device=token_indices.device + ) + else: + valid_counts = valid_counts_out + valid_counts.zero_() # Strides in elements bt_stride0, bt_stride1 = block_table_c.stride() @@ -341,3 +349,47 @@ def triton_filter_and_convert_dcp_index( assert valid_counts is not None return out, valid_counts return out + + +# --- Physical-index shadow (DSA index_topk_freq > 1) ------------------------- +# GLM-5.2 uses index_topk_freq=4: only ~1/4 of layers write a fresh top-k into +# the single shared topk_indices_buffer; the rest reuse it. Since block_table +# and req_id_per_token are constant within a decode step, the physical indices +# (block_table lookup of the logical top-k) are identical across a freq-group. +# Fresh layers convert once into a STABLE shadow of the logical buffer and +# skip layers read it -- eliminating ~3/4 of the per-layer convert kernels. +# +# Invariant: shadow[j] == convert(logical[j]) row-wise. Whoever re-arranges +# the logical buffer (e.g. the MTP draft compact between the multi-token +# step-0 layout and the single-token steps-1+ layout) must apply the same +# gather to the shadow. +# +# Shadows are registered ONLY at buffer creation time (model init, never +# inside cudagraph capture) via register_phys_shadow; every other access is +# the read-only phys_shadow lookup, so addresses are stable across graph +# replays and no allocation can happen during capture. +import weakref as _weakref # noqa: E402 + +_PHYS_SHADOWS: dict[int, tuple[torch.Tensor, torch.Tensor]] = {} + + +def register_phys_shadow(topk_buf: torch.Tensor) -> None: + """Allocate the (physical-index, valid-count) shadow for a logical top-k + buffer. Call once where the buffer is created. The finalizer drops the + entry with the owning buffer, so a recycled id can never alias a stale + shadow.""" + key = id(topk_buf) + if key not in _PHYS_SHADOWS: + _PHYS_SHADOWS[key] = ( + torch.empty_like(topk_buf), + torch.empty(topk_buf.shape[0], dtype=torch.int32, device=topk_buf.device), + ) + _weakref.finalize(topk_buf, _PHYS_SHADOWS.pop, key, None) + + +def phys_shadow( + topk_buf: torch.Tensor, +) -> tuple[torch.Tensor, torch.Tensor] | None: + """The registered shadow for this buffer, or None if it was never + registered. Never allocates; capture-safe.""" + return _PHYS_SHADOWS.get(id(topk_buf)) diff --git a/vllm/v1/spec_decode/llm_base_proposer.py b/vllm/v1/spec_decode/llm_base_proposer.py index 6ee442af51..bb1c59a8c2 100644 --- a/vllm/v1/spec_decode/llm_base_proposer.py +++ b/vllm/v1/spec_decode/llm_base_proposer.py @@ -1009,8 +1009,9 @@ class SpecDecodeBaseProposer: # DeepSeek-family MTP (deepseek_mtp.py) recycles the post-final- # norm hidden, so its forward returns (logit_hidden, # recycle_hidden). Other MTP families return a single tensor. - return "DeepSeekMTPModel" in ( - self.draft_model_config.hf_config.architectures or [] + return bool( + {"DeepSeekMTPModel", "DeepseekV32MTPModel"} + & set(self.draft_model_config.hf_config.architectures or []) ) return self.method not in ("mtp", "draft_model", "dflash") diff --git a/vllm/v1/worker/gpu/spec_decode/autoregressive/speculator.py b/vllm/v1/worker/gpu/spec_decode/autoregressive/speculator.py index b890574971..9a14d132c5 100644 --- a/vllm/v1/worker/gpu/spec_decode/autoregressive/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/autoregressive/speculator.py @@ -48,6 +48,18 @@ class AutoRegressiveSpeculator(DraftModelSpeculator): self.prefill_cudagraph_manager: SpeculatorCudaGraphManager | None = None self.decode_cudagraph_manager: SpeculatorCudaGraphManager | None = None + # Lifecycle hooks for model-specific optimizations. Subclasses override + # the ones they need. These fire in both `capture` and `propose` so that + # any state they toggle (e.g. attention flags baked into a CUDA graph) is + # identical at capture time and replay time. + def on_prefill_begin(self, num_reqs: int, num_tokens: int) -> None: ... + + def on_prefill_end(self, num_reqs: int, num_tokens: int) -> None: ... + + def on_multi_step_decode_begin(self, num_reqs: int) -> None: ... + + def on_multi_step_decode_end(self, num_reqs: int) -> None: ... + @property def advance_draft_positions(self) -> bool: """ @@ -98,6 +110,9 @@ class AutoRegressiveSpeculator(DraftModelSpeculator): assert self.prefill_cudagraph_manager is not None if self.prefill_cudagraph_manager.use_breakable_cg: self.prefill_cudagraph_manager.init_breakable_cg_runner(self.model) + + self.on_prefill_begin(self.max_num_reqs, self.max_num_tokens) + self.prefill_cudagraph_manager.capture( self._prefill, self.model_state, @@ -108,9 +123,13 @@ class AutoRegressiveSpeculator(DraftModelSpeculator): progress_bar_desc="Capturing prefill CUDA graphs", ) + self.on_prefill_end(self.max_num_reqs, self.max_num_tokens) + if self.num_speculative_steps == 1: return + self.on_multi_step_decode_begin(self.max_num_reqs) + # Capture the decode draft generation routine (model forward + # sample + update_draft_inputs) for a single # step. @@ -125,6 +144,8 @@ class AutoRegressiveSpeculator(DraftModelSpeculator): progress_bar_desc="Capturing decode CUDA graphs", ) + self.on_multi_step_decode_end(self.max_num_reqs) + @torch.inference_mode() def propose( self, @@ -153,7 +174,8 @@ class AutoRegressiveSpeculator(DraftModelSpeculator): mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None, is_profile: bool = False, ) -> torch.Tensor: - num_tokens = input_batch.num_tokens_after_padding + num_tokens = input_batch.num_tokens + num_tokens_padded = input_batch.num_tokens_after_padding num_reqs = input_batch.num_reqs max_query_len = input_batch.num_scheduled_tokens.max() max_seq_len = input_batch.seq_lens_cpu_upper_bound[:num_reqs].max().item() @@ -174,7 +196,7 @@ class AutoRegressiveSpeculator(DraftModelSpeculator): ) else: hidden_states = last_hidden_states - self.hidden_states[:num_tokens].copy_(hidden_states) + self.hidden_states[:num_tokens_padded].copy_(hidden_states) self._copy_request_inputs( num_reqs, @@ -202,20 +224,22 @@ class AutoRegressiveSpeculator(DraftModelSpeculator): num_reqs, # Use the actual number of tokens without padding added by # the target model during FULL cudagraph. - input_batch.num_tokens, + num_tokens, max_query_len, ) prefill_batch_desc, num_tokens_across_dp = dispatch_cg_and_sync_dp( self.prefill_cudagraph_manager, num_reqs, - num_tokens, + num_tokens_padded, uniform_token_count, dp_size=self.dp_size, dp_rank=self.dp_rank, need_eager=is_profile, ) - self._prepare_eplb_forward(input_batch.num_tokens) + self._prepare_eplb_forward(num_tokens) + + self.on_prefill_begin(num_reqs, num_tokens) if prefill_batch_desc.cg_mode == CUDAGraphMode.FULL: # Replay the full graph for draft prefill. @@ -235,6 +259,8 @@ class AutoRegressiveSpeculator(DraftModelSpeculator): mm_inputs=mm_inputs, ) + self.on_prefill_end(num_reqs, num_tokens) + if self.num_speculative_steps == 1: # Early exit. return self.draft_tokens[:num_reqs, :1] @@ -262,6 +288,8 @@ class AutoRegressiveSpeculator(DraftModelSpeculator): need_eager=is_profile, ) + self.on_multi_step_decode_begin(num_reqs) + # Generate the remaining num_speculative_steps - 1 draft tokens. self._multi_step_decode( num_reqs, @@ -271,6 +299,8 @@ class AutoRegressiveSpeculator(DraftModelSpeculator): input_batch.seq_lens_cpu_upper_bound, ) + self.on_multi_step_decode_end(num_reqs) + return self.draft_tokens[:num_reqs] @torch.inference_mode() diff --git a/vllm/v1/worker/gpu/spec_decode/mtp/speculator.py b/vllm/v1/worker/gpu/spec_decode/mtp/speculator.py index 4b9354f23e..5f1cebc519 100644 --- a/vllm/v1/worker/gpu/spec_decode/mtp/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/mtp/speculator.py @@ -1,8 +1,10 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import torch import torch.nn as nn +from vllm.config import VllmConfig from vllm.v1.worker.gpu.spec_decode.autoregressive.speculator import ( AutoRegressiveSpeculator, ) @@ -10,9 +12,46 @@ from vllm.v1.worker.gpu.spec_decode.eagle.utils import load_eagle_model class MTPSpeculator(AutoRegressiveSpeculator): + def __init__(self, vllm_config: VllmConfig, device: torch.device): + super().__init__(vllm_config, device) + + spec_config = vllm_config.speculative_config + draft_hf_config = ( + spec_config.draft_model_config.hf_config + if spec_config is not None + else None + ) + # Detect index_share_for_mtp_iteration. When True, the proposer + # toggles skip_topk so step 0 computes MTP's own indices and + # steps 1+ reuse them. + self.share_mtp_topk_indices = getattr( + draft_hf_config, "index_share_for_mtp_iteration", False + ) + def load_draft_model( self, target_model: nn.Module, target_attn_layer_names: set[str], ) -> nn.Module: - return load_eagle_model(target_model, self.vllm_config) + draft_model = load_eagle_model(target_model, self.vllm_config) + self.share_mtp_topk_indices = self.share_mtp_topk_indices and hasattr( + draft_model.model, "set_skip_topk" + ) + return draft_model + + def on_prefill_end(self, num_reqs: int, num_tokens: int) -> None: + # Step 0 (prefill) wrote topk indices for every query token in the + # multi-token batch. Compact them down to each request's last token so + # steps 1+ can reuse them from the shared buffer. + if self.share_mtp_topk_indices and self.num_speculative_steps > 1: + self.model.model.compact_topk_indices(self.last_token_indices[:num_reqs]) + + def on_multi_step_decode_begin(self, num_reqs: int) -> None: + # Switch to reuse mode so draft steps 1+ skip the indexer op and read + # the indices that step 0 wrote into the shared buffer. + if self.share_mtp_topk_indices: + self.model.model.set_skip_topk(True) + + def on_multi_step_decode_end(self, num_reqs: int) -> None: + if self.share_mtp_topk_indices: + self.model.model.set_skip_topk(False)