From 213f681f8117b9026ca8189583d2855d70104401 Mon Sep 17 00:00:00 2001 From: Woosuk Kwon Date: Fri, 24 Jul 2026 16:04:59 -0700 Subject: [PATCH] Revert "[Perf][GLM-5.2] Blackwell decode optimizations" (#49768) --- CMakeLists.txt | 17 - csrc/libtorch_stable/bf16_skinny_gemm.cu | 262 --------- .../libtorch_stable/bf16_skinny_gemm_entry.cu | 170 ------ csrc/libtorch_stable/dsv3_fused_a_gemm.cu | 148 ++--- .../quantization/fp4/nvfp4_quant_kernels.cu | 52 +- csrc/libtorch_stable/torch_bindings.cpp | 4 - recipes/glm5.2-ll-b300-tp8-mtp5.md | 41 -- tests/kernels/test_bf16_skinny_gemm.py | 70 --- .../test_fused_deepseek_v32_norm_rope.py | 77 --- tests/models/registry.py | 4 - vllm/_custom_ops.py | 25 - .../passes/fusion/allreduce_rms_fusion.py | 146 ++--- vllm/config/speculative.py | 8 +- vllm/cute_utils/__init__.py | 5 - vllm/cute_utils/cvt.py | 23 +- .../layers/fused_allreduce_gemma_rms_norm.py | 19 +- vllm/model_executor/models/registry.py | 3 +- vllm/models/deepseek_v32/__init__.py | 22 +- vllm/models/deepseek_v32/nvidia/attention.py | 84 +-- vllm/models/deepseek_v32/nvidia/kernels.py | 300 +--------- vllm/models/deepseek_v32/nvidia/model.py | 4 - vllm/models/deepseek_v32/nvidia/mtp.py | 82 +-- .../deepseek_v32/nvidia/ops/__init__.py | 2 - .../nvidia/ops/fused_q_cutedsl.py | 532 ------------------ .../backends/mla/flashinfer_mla_sparse.py | 31 - .../v1/attention/backends/mla/sparse_utils.py | 60 +- vllm/v1/spec_decode/llm_base_proposer.py | 5 +- .../spec_decode/autoregressive/speculator.py | 40 +- .../worker/gpu/spec_decode/mtp/speculator.py | 41 +- 29 files changed, 138 insertions(+), 2139 deletions(-) delete mode 100644 csrc/libtorch_stable/bf16_skinny_gemm.cu delete mode 100644 csrc/libtorch_stable/bf16_skinny_gemm_entry.cu delete mode 100644 recipes/glm5.2-ll-b300-tp8-mtp5.md delete mode 100644 tests/kernels/test_bf16_skinny_gemm.py delete mode 100644 vllm/models/deepseek_v32/nvidia/ops/__init__.py delete mode 100644 vllm/models/deepseek_v32/nvidia/ops/fused_q_cutedsl.py diff --git a/CMakeLists.txt b/CMakeLists.txt index d91113bd1c..9bd73f18e7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -732,23 +732,6 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP") "(requires SM90+ and CUDA >= 12.0).") endif() - # BF16 skinny GEMM (M<=32; weight-bandwidth-bound decode shapes). - # Requires SM90+. - cuda_archs_sm90plus(BF16_SKINNY_GEMM_ARCHS "${CUDA_ARCHS}") - if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 12.0 AND BF16_SKINNY_GEMM_ARCHS) - set(BF16_SKINNY_GEMM_SRCS - "csrc/libtorch_stable/bf16_skinny_gemm_entry.cu" - "csrc/libtorch_stable/bf16_skinny_gemm.cu") - set_gencode_flags_for_srcs( - SRCS "${BF16_SKINNY_GEMM_SRCS}" - CUDA_ARCHS "${BF16_SKINNY_GEMM_ARCHS}") - list(APPEND VLLM_STABLE_EXT_SRC "${BF16_SKINNY_GEMM_SRCS}") - message(STATUS "Building bf16_skinny_gemm for archs: ${BF16_SKINNY_GEMM_ARCHS}") - else() - message(STATUS "Not building bf16_skinny_gemm as no compatible archs found " - "(requires SM90+ and CUDA >= 12.0).") - endif() - # Only build AllSpark kernels if we are building for at least some compatible archs. cuda_archs_loose_intersection(ALLSPARK_ARCHS "8.0;8.6;8.7;8.9" "${CUDA_ARCHS}") if (ALLSPARK_ARCHS) diff --git a/csrc/libtorch_stable/bf16_skinny_gemm.cu b/csrc/libtorch_stable/bf16_skinny_gemm.cu deleted file mode 100644 index fef90d29d6..0000000000 --- a/csrc/libtorch_stable/bf16_skinny_gemm.cu +++ /dev/null @@ -1,262 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright contributors to the vLLM project -// -// Skinny GEMM: activation(bf16) x weight(bf16)^T -> bf16, for decode-time -// M <= 32 with a large reduction dim. Replaces cuBLAS splitK (GEMM + -// splitKreduce) with a single block-per-output-column kernel; these shapes -// are weight-bandwidth-bound, so one coalesced pass over the weight at -// fp32 accumulation is optimal. Adapted from fp32_router_gemm.cu. -// -// First user: the DeepSeek-V32/GLM-5.2 MTP eh_proj (K=2*hidden=12288, -// N=hidden/TP), whose cuBLAS splitK pick costs ~34.6us vs the ~19us -// bandwidth floor per replicated read (and ~4us once column-parallel). - -#include -#include - -// --------------------------------------------------------------------------- -// Load helpers (8 x bf16 = one uint4 load, converted to fp32) -// --------------------------------------------------------------------------- - -namespace skinny { - -constexpr int VPT = 8; // bf16 values per thread per load - -__device__ __forceinline__ void load_bf16x8(__nv_bfloat16 const* ptr, - float* dst) { - uint4 v = *reinterpret_cast(ptr); - __nv_bfloat16 const* p = reinterpret_cast<__nv_bfloat16 const*>(&v); -#pragma unroll - for (int i = 0; i < VPT; i++) dst[i] = __bfloat162float(p[i]); -} - -// Streaming variant for the weight: each row is read exactly once across the -// whole grid, so bypass L2 residency (evict-first). Measured -1.4us at M=1. -__device__ __forceinline__ void load_bf16x8_cs(__nv_bfloat16 const* ptr, - float* dst) { - uint4 v = __ldcs(reinterpret_cast(ptr)); - __nv_bfloat16 const* p = reinterpret_cast<__nv_bfloat16 const*>(&v); -#pragma unroll - for (int i = 0; i < VPT; i++) dst[i] = __bfloat162float(p[i]); -} - -// --------------------------------------------------------------------------- -// Kernel: each block computes kNPB output columns for all kNumTokens rows. -// grid = kN / kNPB, block = kBlockSize threads. K is reduced VPT elements -// per thread per iteration; fp32 accumulation, warp butterfly + smem -// finalize, bf16 store. -// --------------------------------------------------------------------------- - -template -__global__ __launch_bounds__(kBlockSize, 1) void bf16_skinny_gemm_kernel( - __nv_bfloat16* out, __nv_bfloat16 const* mat_a, __nv_bfloat16 const* mat_b, - int64_t out_stride) { - constexpr int k_elems_per_iter = VPT * kBlockSize; - constexpr int k_iterations = kK / k_elems_per_iter; - static_assert(kK % k_elems_per_iter == 0); - constexpr int kWarpSize = 32; - constexpr int kNumWarps = kBlockSize / kWarpSize; - - int const n_base = blockIdx.x * kNPB; - int const tid = threadIdx.x; - int const warpId = tid / kWarpSize; - int const laneId = tid % kWarpSize; - - float acc[kNumTokens][kNPB] = {}; - __shared__ float sm_reduction[kNumTokens][kNPB][kNumWarps]; - - // Register prefetch (kPF > 0): W does not depend on the predecessor, so - // the first kPF iterations' weight chunks are loaded raw BEFORE the - // dependency sync; with a PDL-releasing producer (fused_eh_norm fires - // gdc_launch_dependents early) these DRAM round trips overlap the norm. - // Pair-measured on B300 (norm+gemm in one graph, full 6144x12288): - // M=1 pf2 28.43us vs pf0 29.00us; deeper prefetch or M >= 2 regresses - // (register pressure), hence the per-M selection in the launcher. - uint4 w_pre[kPF > 0 ? kPF : 1][kNPB]; -#pragma unroll - for (int pf = 0; pf < kPF; pf++) { -#pragma unroll - for (int n = 0; n < kNPB; n++) { - w_pre[pf][n] = - *reinterpret_cast(mat_b + (size_t)(n_base + n) * kK + - pf * k_elems_per_iter + tid * VPT); - } - } - -#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900) - cudaGridDependencySynchronize(); -#endif - -#pragma unroll - for (int ki = 0; ki < k_iterations; ki++) { - int const k_base = ki * k_elems_per_iter + tid * VPT; - - float b_float[kNPB][VPT]; - if (ki < kPF) { -#pragma unroll - for (int n = 0; n < kNPB; n++) { - __nv_bfloat16 const* p = - reinterpret_cast<__nv_bfloat16 const*>(&w_pre[ki][n]); -#pragma unroll - for (int v = 0; v < VPT; v++) b_float[n][v] = __bfloat162float(p[v]); - } - } else { -#pragma unroll - for (int n = 0; n < kNPB; n++) { - load_bf16x8_cs(mat_b + (size_t)(n_base + n) * kK + k_base, b_float[n]); - } - } - -#pragma unroll - for (int m = 0; m < kNumTokens; m++) { - float a_float[VPT]; - load_bf16x8(mat_a + (size_t)m * kK + k_base, a_float); -#pragma unroll - for (int n = 0; n < kNPB; n++) { -#pragma unroll - for (int k = 0; k < VPT; k++) { - acc[m][n] += a_float[k] * b_float[n][k]; - } - } - } - } - - // Warp-level butterfly reduction -#pragma unroll - for (int m = 0; m < kNumTokens; m++) { -#pragma unroll - for (int n = 0; n < kNPB; n++) { - float sum = acc[m][n]; - sum += __shfl_xor_sync(0xffffffff, sum, 16); - sum += __shfl_xor_sync(0xffffffff, sum, 8); - sum += __shfl_xor_sync(0xffffffff, sum, 4); - sum += __shfl_xor_sync(0xffffffff, sum, 2); - sum += __shfl_xor_sync(0xffffffff, sum, 1); - if (laneId == 0) sm_reduction[m][n][warpId] = sum; - } - } - - __syncthreads(); - - // Parallel finalize: one thread per (m, n) output. - for (int idx = tid; idx < kNumTokens * kNPB; idx += kBlockSize) { - int const m = idx / kNPB; - int const n = idx % kNPB; - float final_sum = 0.0f; -#pragma unroll - for (int w = 0; w < kNumWarps; w++) final_sum += sm_reduction[m][n][w]; - out[(size_t)m * out_stride + n_base + n] = __float2bfloat16(final_sum); - } - - // Trigger after our stores: harmless hardening, not a guarantee — the - // trigger only permits dependent-launch scheduling and carries no memory - // visibility semantics, so a PDL consumer must still gridsync before - // reading our output. Every current consumer is a plain launch (full - // stream order); firing late just avoids an unnecessarily early launch - // window and matches fp32_router_gemm.cu. -#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900) - cudaTriggerProgrammaticLaunchCompletion(); -#endif -} - -} // namespace skinny - -// --------------------------------------------------------------------------- -// Launcher -// --------------------------------------------------------------------------- - -template -void invokeBf16SkinnyGemm(__nv_bfloat16* output, __nv_bfloat16 const* mat_a, - __nv_bfloat16 const* mat_b, int64_t out_stride, - cudaStream_t stream) { - static_assert(kN % kNPB == 0); - // Weight prefetch depth: only M=1 measured a win (see kernel comment). - constexpr int kPF = (kNumTokens == 1) ? 2 : 0; - cudaLaunchConfig_t config; - config.gridDim = kN / kNPB; - config.blockDim = kBlockSize; - config.dynamicSmemBytes = 0; - config.stream = stream; - cudaLaunchAttribute attrs[1]; - attrs[0].id = cudaLaunchAttributeProgrammaticStreamSerialization; - attrs[0].val.programmaticStreamSerializationAllowed = 1; - config.numAttrs = 1; - config.attrs = attrs; - cudaLaunchKernelEx(&config, - skinny::bf16_skinny_gemm_kernel, - output, mat_a, mat_b, out_stride); -} - -// --------------------------------------------------------------------------- -// Explicit instantiations. M = 1..32; (N, K) pairs: -// (768, 12288) eh_proj shard, TP8 -// (1536, 12288) eh_proj shard, TP4 -// (6144, 12288) eh_proj unsharded -// kNPB (B300 sweep, M=1): 6144 -> 2 (3072 blocks, 23.2us vs 25.3 at kNPB=8; -// narrow blocks minimize wave quantization); shards 768/1536 keep 4. -// --------------------------------------------------------------------------- - -#define INSTANTIATE(M, NPB, N, K) \ - template void invokeBf16SkinnyGemm<128, NPB, M, N, K>( \ - __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, int64_t, \ - cudaStream_t); - -#define INSTANTIATE_ALL_M(NPB, N, K) \ - INSTANTIATE(1, NPB, N, K) \ - INSTANTIATE(2, NPB, N, K) \ - INSTANTIATE(3, NPB, N, K) \ - INSTANTIATE(4, NPB, N, K) \ - INSTANTIATE(5, NPB, N, K) \ - INSTANTIATE(6, NPB, N, K) \ - INSTANTIATE(7, NPB, N, K) \ - INSTANTIATE(8, NPB, N, K) \ - INSTANTIATE(9, NPB, N, K) \ - INSTANTIATE(10, NPB, N, K) \ - INSTANTIATE(11, NPB, N, K) \ - INSTANTIATE(12, NPB, N, K) \ - INSTANTIATE(13, NPB, N, K) \ - INSTANTIATE(14, NPB, N, K) \ - INSTANTIATE(15, NPB, N, K) \ - INSTANTIATE(16, NPB, N, K) \ - INSTANTIATE(17, NPB, N, K) \ - INSTANTIATE(18, NPB, N, K) \ - INSTANTIATE(19, NPB, N, K) \ - INSTANTIATE(20, NPB, N, K) \ - INSTANTIATE(21, NPB, N, K) \ - INSTANTIATE(22, NPB, N, K) \ - INSTANTIATE(23, NPB, N, K) \ - INSTANTIATE(24, NPB, N, K) \ - INSTANTIATE(25, NPB, N, K) \ - INSTANTIATE(26, NPB, N, K) \ - INSTANTIATE(27, NPB, N, K) \ - INSTANTIATE(28, NPB, N, K) \ - INSTANTIATE(29, NPB, N, K) \ - INSTANTIATE(30, NPB, N, K) \ - INSTANTIATE(31, NPB, N, K) \ - INSTANTIATE(32, NPB, N, K) - -INSTANTIATE_ALL_M(4, 768, 12288) -INSTANTIATE_ALL_M(4, 1536, 12288) -INSTANTIATE_ALL_M(2, 6144, 12288) -// LL-mode (M<=8 wiring guard) backbone shapes, B300 sweep vs cuBLAS: -// q_b_proj (2048, 2048): 1.67x/1.29x/1.15x at M=4/6/8 (NPB=4 within -// 0.1us of per-M best) -// shared-expert gate_up (512, 6144): 1.95x/1.58x/1.40x at M=4/6/8 -// cuBLAS keeps qkv_a (2624,6144) and o_proj (6144,2048) — already at -// 3.4-3.8 TB/s there; the GEMV loses on activation re-reads. -INSTANTIATE_ALL_M(4, 2048, 2048) -INSTANTIATE_ALL_M(4, 512, 6144) -// fused_qkv_a (2624, 6144), 32MB: skinny wins ONLY at M<=2 (B300: M=1 -// 6.99us vs cuBLAS 9.21 = 1.32x, M=2 1.24x; M>=4 cuBLAS holds at 3.5TB/s -// and every alternative loses — cublasLt top-8 3.6TB/s wall, DeepGEMM -// 0.71x, wmma+cp.async custom 0.30x pending a TMA rewrite). -INSTANTIATE_ALL_M(4, 2624, 6144) -// DSv3.2 (TP8) siblings of the GLM shapes above, same dual-chip matrix: -// fused_qkv_a (2112, 7168), 30MB: wins M<=2 (M=1 1.30-1.34x) -// MTP eh_proj (7168, 14336), 205MB: wins M<=2 (M=1 1.12-1.16x) -INSTANTIATE_ALL_M(4, 2112, 7168) -INSTANTIATE_ALL_M(2, 7168, 14336) - -#undef INSTANTIATE_ALL_M -#undef INSTANTIATE diff --git a/csrc/libtorch_stable/bf16_skinny_gemm_entry.cu b/csrc/libtorch_stable/bf16_skinny_gemm_entry.cu deleted file mode 100644 index 60b9e8b60c..0000000000 --- a/csrc/libtorch_stable/bf16_skinny_gemm_entry.cu +++ /dev/null @@ -1,170 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -#include -#include -#include - -#include "core/registration.h" -#include "libtorch_stable/torch_utils.h" - -#include -#include - -#include - -namespace { - -inline int getSMVersion() { - auto* props = get_device_prop(); - return props->major * 10 + props->minor; -} - -} // namespace - -static constexpr int SKINNY_MAX_TOKENS = 32; - -// Supported (N, K) pairs (must match the instantiations in -// bf16_skinny_gemm.cu): eh_proj shard TP8 / TP4 / unsharded. -static inline bool bf16_skinny_gemm_supported(int n, int k) { - if (k == 12288 && (n == 768 || n == 1536 || n == 6144)) return true; - // LL-mode backbone shapes (wire callers with an M <= 8 guard; the GEMV - // family loses to cuBLAS at larger M). - if (k == 2048 && n == 2048) return true; // q_b_proj (TP8) - if (k == 6144 && n == 2624) return true; // fused_qkv_a (wire M <= 2 only) - if (k == 7168 && n == 2112) return true; // DSv3.2 fused_qkv_a (M <= 2) - if (k == 14336 && n == 7168) return true; // DSv3.2 eh_proj (M <= 2) - if (k == 6144 && n == 512) return true; // shared-expert gate_up (TP8) - return false; -} - -// Forward declarations - template params must match bf16_skinny_gemm.cu -template -void invokeBf16SkinnyGemm(__nv_bfloat16* output, __nv_bfloat16 const* mat_a, - __nv_bfloat16 const* mat_b, int64_t out_stride, - cudaStream_t stream); - -template -struct SkinnyLoopUnroller { - static void unroll(int num_tokens, __nv_bfloat16* output, - __nv_bfloat16 const* mat_a, __nv_bfloat16 const* mat_b, - int64_t out_stride, cudaStream_t stream) { - if (num_tokens == kBegin) { - invokeBf16SkinnyGemm<128, kNPB, kBegin, kN, kK>(output, mat_a, mat_b, - out_stride, stream); - } else { - SkinnyLoopUnroller::unroll( - num_tokens, output, mat_a, mat_b, out_stride, stream); - } - } -}; - -template -struct SkinnyLoopUnroller { - static void unroll(int num_tokens, __nv_bfloat16* output, - __nv_bfloat16 const* mat_a, __nv_bfloat16 const* mat_b, - int64_t out_stride, cudaStream_t stream) { - if (num_tokens == kEnd) { - invokeBf16SkinnyGemm<128, kNPB, kEnd, kN, kK>(output, mat_a, mat_b, - out_stride, stream); - } else { - throw std::invalid_argument( - "bf16_skinny_gemm: num_tokens must be in [1, 32]"); - } - } -}; - -static void dispatchBf16SkinnyGemm(int n, int k, int num_tokens, - __nv_bfloat16* output, - __nv_bfloat16 const* mat_a, - __nv_bfloat16 const* mat_b, - int64_t out_stride, cudaStream_t stream) { - if (n == 768 && k == 12288) { - SkinnyLoopUnroller<4, 768, 12288, 1, SKINNY_MAX_TOKENS>::unroll( - num_tokens, output, mat_a, mat_b, out_stride, stream); - } else if (n == 1536 && k == 12288) { - SkinnyLoopUnroller<4, 1536, 12288, 1, SKINNY_MAX_TOKENS>::unroll( - num_tokens, output, mat_a, mat_b, out_stride, stream); - } else if (n == 6144 && k == 12288) { - SkinnyLoopUnroller<2, 6144, 12288, 1, SKINNY_MAX_TOKENS>::unroll( - num_tokens, output, mat_a, mat_b, out_stride, stream); - } else if (n == 2048 && k == 2048) { - SkinnyLoopUnroller<4, 2048, 2048, 1, SKINNY_MAX_TOKENS>::unroll( - num_tokens, output, mat_a, mat_b, out_stride, stream); - } else if (n == 2624 && k == 6144) { - SkinnyLoopUnroller<4, 2624, 6144, 1, SKINNY_MAX_TOKENS>::unroll( - num_tokens, output, mat_a, mat_b, out_stride, stream); - } else if (n == 2112 && k == 7168) { - SkinnyLoopUnroller<4, 2112, 7168, 1, SKINNY_MAX_TOKENS>::unroll( - num_tokens, output, mat_a, mat_b, out_stride, stream); - } else if (n == 7168 && k == 14336) { - SkinnyLoopUnroller<2, 7168, 14336, 1, SKINNY_MAX_TOKENS>::unroll( - num_tokens, output, mat_a, mat_b, out_stride, stream); - } else if (n == 512 && k == 6144) { - SkinnyLoopUnroller<4, 512, 6144, 1, SKINNY_MAX_TOKENS>::unroll( - num_tokens, output, mat_a, mat_b, out_stride, stream); - } else { - throw std::invalid_argument("bf16_skinny_gemm: unsupported (N, K) pair"); - } -} - -void bf16_skinny_gemm( - torch::stable::Tensor& output, // [num_tokens, N] bf16 - torch::stable::Tensor const& mat_a, // [num_tokens, K] bf16 - torch::stable::Tensor const& mat_b // [N, K] bf16 -) { - STD_TORCH_CHECK(output.dim() == 2 && mat_a.dim() == 2 && mat_b.dim() == 2); - STD_TORCH_CHECK(output.is_cuda() && mat_a.is_cuda() && mat_b.is_cuda(), - "bf16_skinny_gemm: all tensors must be CUDA tensors"); - STD_TORCH_CHECK(output.get_device_index() == mat_a.get_device_index() && - output.get_device_index() == mat_b.get_device_index(), - "bf16_skinny_gemm: all tensors must be on the same device"); - STD_TORCH_CHECK(mat_a.is_contiguous() && mat_b.is_contiguous(), - "bf16_skinny_gemm: inputs must be contiguous"); - // Output may be a column-slice view of a wider padded buffer: unit column - // stride, row stride >= N (rows must not overlap). - STD_TORCH_CHECK(output.stride(1) == 1, - "bf16_skinny_gemm: output columns must be contiguous"); - - const int num_tokens = mat_a.size(0); - const int n = mat_b.size(0); - const int k = mat_a.size(1); - - STD_TORCH_CHECK(output.size(0) == num_tokens && output.size(1) == n, - "bf16_skinny_gemm: output must be [num_tokens, N]"); - STD_TORCH_CHECK(mat_b.size(1) == k, - "bf16_skinny_gemm: mat_a and mat_b must share K"); - STD_TORCH_CHECK(bf16_skinny_gemm_supported(n, k), - "bf16_skinny_gemm: unsupported (N, K) pair"); - const int64_t out_stride = output.stride(0); - STD_TORCH_CHECK(num_tokens <= 1 || out_stride >= n, - "bf16_skinny_gemm: output rows overlap"); - STD_TORCH_CHECK(num_tokens >= 0 && num_tokens <= SKINNY_MAX_TOKENS, - "bf16_skinny_gemm: num_tokens must be in [0, 32]"); - STD_TORCH_CHECK( - mat_a.scalar_type() == torch::headeronly::ScalarType::BFloat16 && - mat_b.scalar_type() == torch::headeronly::ScalarType::BFloat16 && - output.scalar_type() == torch::headeronly::ScalarType::BFloat16, - "bf16_skinny_gemm: all tensors must be bfloat16"); - - // Empty batch (e.g. an empty rank at a DP/PP boundary): nothing to compute. - if (num_tokens == 0) { - return; - } - - const torch::stable::accelerator::DeviceGuard device_guard( - mat_a.get_device_index()); - STD_TORCH_CHECK(getSMVersion() >= 90, "bf16_skinny_gemm: requires SM90+"); - - auto stream = get_current_cuda_stream(mat_a.get_device_index()); - dispatchBf16SkinnyGemm( - n, k, num_tokens, - reinterpret_cast<__nv_bfloat16*>(output.mutable_data_ptr()), - reinterpret_cast<__nv_bfloat16 const*>(mat_a.data_ptr()), - reinterpret_cast<__nv_bfloat16 const*>(mat_b.data_ptr()), out_stride, - stream); -} - -STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, m) { - m.impl("bf16_skinny_gemm", TORCH_BOX(&bf16_skinny_gemm)); -} diff --git a/csrc/libtorch_stable/dsv3_fused_a_gemm.cu b/csrc/libtorch_stable/dsv3_fused_a_gemm.cu index e1ba2a8d0e..585004c047 100644 --- a/csrc/libtorch_stable/dsv3_fused_a_gemm.cu +++ b/csrc/libtorch_stable/dsv3_fused_a_gemm.cu @@ -391,7 +391,7 @@ struct MmaComputer { static constexpr int n_iter_cnt = (tile_n + 7) / 8; // Possible to have non-1 n_iter_cnt for ab_swap m16 case. - static_assert(m_iter_cnt == 1 || m_iter_cnt == 2); + static_assert(m_iter_cnt == 1); static_assert(n_iter_cnt == 1 || n_iter_cnt == 2); __device__ MmaComputer(bf16_t* gmem_c_local_, bf16_t* smem_a_, @@ -416,18 +416,13 @@ struct MmaComputer { public: __device__ void prepare() { #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900 - // Fragment addressing is per 16-row ldmatrix tile; m_iter selects the - // 16-row half within tile_m. #pragma unroll - for (int m = 0; m < m_iter_cnt; m++) { - #pragma unroll - for (int i = 0; i < k_phase_cnt; i++) { - int linear_idx = (lane_idx % 16) + (lane_idx / 16) * 128 + i * 256; - int m_idx = linear_idx % 16 + m * 16; - int k_idx = linear_idx / 16 + warp_k_offset_in_tile_k; - k_idx = apply_swizzle_343_on_elem_row_col(m_idx, k_idx); - a_smem_offsets[m][i] = m_idx * tile_k + k_idx; - } + for (int i = 0; i < k_phase_cnt; i++) { + int linear_idx = (lane_idx % 16) + (lane_idx / 16) * 128 + i * 256; + int m_idx = linear_idx % tile_m; + int k_idx = linear_idx / tile_m + warp_k_offset_in_tile_k; + k_idx = apply_swizzle_343_on_elem_row_col(m_idx, k_idx); + a_smem_offsets[0][i] = m_idx * tile_k + k_idx; } #pragma unroll for (int n_iter_idx = 0; n_iter_idx < n_iter_cnt; n_iter_idx++) { @@ -451,14 +446,11 @@ struct MmaComputer { wait_barrier(smem_barrier + 0 + stage_idx * 2, phase_bit); #pragma unroll - for (int m = 0; m < m_iter_cnt; m++) { - #pragma unroll - for (int i = 0; i < k_phase_cnt; i++) { - int smem_offset = a_smem_offsets[m][i]; - bf16_t* smem_ptr_this_iter = - smem_a + stage_idx * tile_m * tile_k + smem_offset; - ldsm_x4(smem_ptr_this_iter, reinterpret_cast(a_reg[m][i])); - } + for (int i = 0; i < k_phase_cnt; i++) { + int smem_offset = a_smem_offsets[0][i]; + bf16_t* smem_ptr_this_iter = + smem_a + stage_idx * tile_m * tile_k + smem_offset; + ldsm_x4(smem_ptr_this_iter, reinterpret_cast(a_reg[0][i])); } #pragma unroll @@ -477,12 +469,9 @@ struct MmaComputer { for (int k_iter_idx = 0; k_iter_idx < k_phase_cnt; k_iter_idx++) { #pragma unroll for (int n_iter_idx = 0; n_iter_idx < n_iter_cnt; n_iter_idx++) { - #pragma unroll - for (int m = 0; m < m_iter_cnt; m++) { - hmma_16_8_16_f32acc_bf16ab( - acc_reg[m][n_iter_idx], a_reg[m][k_iter_idx], - b_reg[n_iter_idx][k_iter_idx], acc_reg[m][n_iter_idx]); - } + hmma_16_8_16_f32acc_bf16ab( + acc_reg[0][n_iter_idx], a_reg[0][k_iter_idx], + b_reg[n_iter_idx][k_iter_idx], acc_reg[0][n_iter_idx]); } } ::arrive_barrier(smem_barrier + 1 + stage_idx * 2); @@ -497,14 +486,14 @@ struct MmaComputer { #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900 asm volatile("bar.sync %0, %1;" : : "r"(1), "r"(thread_cnt)); // reorganize the acc_reg - constexpr int thread_m = 2 * m_iter_cnt; + constexpr int thread_m = 2; constexpr int thread_n = 2 * n_iter_cnt; constexpr int cta_mma_n = n_iter_cnt * 8; float acc_reg_reorg[thread_m][thread_n]; for (int i = 0; i < thread_m; i++) { for (int j = 0; j < thread_n; j++) { - acc_reg_reorg[i][j] = acc_reg[i / 2][j / 2][(j % 2) + (i % 2) * 2]; + acc_reg_reorg[i][j] = acc_reg[0][j / 2][(j % 2) + (i * 2)]; } } @@ -525,8 +514,7 @@ struct MmaComputer { for (int m_idx_thread = 0; m_idx_thread < thread_m; m_idx_thread++) { #pragma unroll for (int n_idx_thread = 0; n_idx_thread < thread_n; n_idx_thread++) { - int m_idx = - (lane_idx / 4) + (m_idx_thread % 2) * 8 + (m_idx_thread / 2) * 16; + int m_idx = (lane_idx / 4) + m_idx_thread * 8; int n_idx = ((lane_idx % 4) * 2) + (n_idx_thread % 2) + (n_idx_thread / 2) * 8; smem_c[cosize_smem_c * warp_idx + smem_c_index_func(m_idx, n_idx)] = @@ -599,7 +587,7 @@ __global__ __launch_bounds__(256, 1) void fused_a_gemm_kernel( static_assert( tile_k == 128 || tile_k == 256 || tile_k == 512 || tile_k == 1024); // tile_k must be larger than 64 since 4 warp splitK. - static_assert(tile_m == 16 || tile_m == 32); + static_assert(tile_m == 16); constexpr int g2s_vec_bytes = 16; constexpr int a_elem_bytes = 2; constexpr int b_elem_bytes = 2; @@ -659,7 +647,7 @@ __global__ __launch_bounds__(256, 1) void fused_a_gemm_kernel( #endif } -template +template void invokeFusedAGemm(T* output, T const* mat_a, T const* mat_b, int num_tokens, cudaStream_t const stream) { constexpr int gemm_m = kHdOut; // 2112 @@ -667,7 +655,7 @@ void invokeFusedAGemm(T* output, T const* mat_a, T const* mat_b, int num_tokens, constexpr int gemm_k = kHdIn; // 7168 constexpr int batch_size = 1; std::swap(mat_a, mat_b); - constexpr int tile_m = kTileM; + constexpr int tile_m = 16; constexpr int tile_n = kTileN; // 8 or 16 constexpr int tile_k = std::max(256, 1024 / tile_n); // 256 constexpr int max_stage_cnt = @@ -714,44 +702,6 @@ template void invokeFusedAGemm<__nv_bfloat16, 7168, 2112, 16>( __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, int num_tokens, cudaStream_t); -// GLM-5.2 fused_qkv_a (K=6144 -> N=2624). tile_m=32 so the grid is -// 2624/32 = 82 CTAs, a single wave on B300 (148 SMs); tile_m=16's 164 CTAs -// straddle two waves and drop to 2.9 TB/s vs 3.9-4.0 here. Beats cuBLAS -// at every decode M: 1.11-1.15x for M<=8 (tile_n=8), 1.12x at M=16 -// (tile_n=16). The M<=2 dispatch still belongs to bf16_skinny_gemm -// (4.5 TB/s). -template void invokeFusedAGemm<__nv_bfloat16, 6144, 2624, 8, 32>( - __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, int num_tokens, - cudaStream_t); - -template void invokeFusedAGemm<__nv_bfloat16, 6144, 2624, 16, 32>( - __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, int num_tokens, - cudaStream_t); - -// GLM-5.2 q_b_proj TP8 (K=2048 -> N=2048). tile_m=16 keeps 128 CTAs (already -// a single wave) and measures ahead of tile_m=32 here. Beats cuBLAS -// 1.79-1.87x at M=3..8 (tile_n=8) and 1.19-1.30x at M=9..16 (tile_n=16); -// M<=2 belongs to bf16_skinny_gemm, M>=20 to cuBLAS. -template void invokeFusedAGemm<__nv_bfloat16, 2048, 2048, 8>( - __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, int num_tokens, - cudaStream_t); - -template void invokeFusedAGemm<__nv_bfloat16, 2048, 2048, 16>( - __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, int num_tokens, - cudaStream_t); - -// DSv3.2 q_b_proj TP8 (K=1536 -> N=3072). tile_m=32 keeps 96 CTAs (single -// wave; tile_m=16's 192 straddle two). Beats cuBLAS 1.6-1.8x at M=1..8 and -// 1.05-1.18x at M=9..16 on B300/B200; the whole 1..16 range dispatches here -// (no skinny tier: K=1536 does not fit the GEMV's 128x8 K-step). -template void invokeFusedAGemm<__nv_bfloat16, 1536, 3072, 8, 32>( - __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, int num_tokens, - cudaStream_t); - -template void invokeFusedAGemm<__nv_bfloat16, 1536, 3072, 16, 32>( - __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, int num_tokens, - cudaStream_t); - void dsv3_fused_a_gemm(torch::stable::Tensor& output, torch::stable::Tensor const& mat_a, torch::stable::Tensor const& mat_b) { @@ -760,15 +710,12 @@ void dsv3_fused_a_gemm(torch::stable::Tensor& output, int const hd_in = mat_a.size(1); int const hd_out = mat_b.size(1); - bool const is_dsv3 = hd_in == 7168 && hd_out == 2112; - bool const is_glm = hd_in == 6144 && hd_out == 2624; - bool const is_glm_qb = hd_in == 2048 && hd_out == 2048; - bool const is_ds_qb = hd_in == 1536 && hd_out == 3072; + constexpr int kHdIn = 7168; + constexpr int kHdOut = 2112; STD_TORCH_CHECK(num_tokens >= 1 && num_tokens <= 16, "required 1 <= mat_a.shape[0] <= 16"); - STD_TORCH_CHECK(is_dsv3 || is_glm || is_glm_qb || is_ds_qb, - "supported (hd_in, hd_out): (7168, 2112), (6144, 2624), " - "(2048, 2048), (1536, 3072)"); + STD_TORCH_CHECK(hd_in == kHdIn, "required mat_a.shape[1] == 7168"); + STD_TORCH_CHECK(hd_out == kHdOut, "required mat_b.shape[1] == 2112"); STD_TORCH_CHECK(output.size(0) == num_tokens, "required output.shape[0] == mat_a.shape[0]"); STD_TORCH_CHECK(output.size(1) == hd_out, @@ -791,41 +738,18 @@ void dsv3_fused_a_gemm(torch::stable::Tensor& output, STD_TORCH_CHECK(getSMVersion() >= 90, "required CUDA ARCH >= SM_90"); auto stream = get_current_cuda_stream(mat_a.get_device_index()); - auto* out = reinterpret_cast<__nv_bfloat16*>(output.mutable_data_ptr()); - auto* a = reinterpret_cast<__nv_bfloat16 const*>(mat_a.data_ptr()); - auto* b = reinterpret_cast<__nv_bfloat16 const*>(mat_b.data_ptr()); - if (is_dsv3) { - if (num_tokens <= 8) { - invokeFusedAGemm<__nv_bfloat16, 7168, 2112, 8>(out, a, b, num_tokens, - stream); - } else { - invokeFusedAGemm<__nv_bfloat16, 7168, 2112, 16>(out, a, b, num_tokens, - stream); - } - } else if (is_glm) { - if (num_tokens <= 8) { - invokeFusedAGemm<__nv_bfloat16, 6144, 2624, 8, 32>(out, a, b, num_tokens, - stream); - } else { - invokeFusedAGemm<__nv_bfloat16, 6144, 2624, 16, 32>(out, a, b, num_tokens, - stream); - } - } else if (is_glm_qb) { - if (num_tokens <= 8) { - invokeFusedAGemm<__nv_bfloat16, 2048, 2048, 8>(out, a, b, num_tokens, - stream); - } else { - invokeFusedAGemm<__nv_bfloat16, 2048, 2048, 16>(out, a, b, num_tokens, - stream); - } + if (num_tokens <= 8) { + invokeFusedAGemm<__nv_bfloat16, kHdIn, kHdOut, 8>( + reinterpret_cast<__nv_bfloat16*>(output.mutable_data_ptr()), + reinterpret_cast<__nv_bfloat16 const*>(mat_a.data_ptr()), + reinterpret_cast<__nv_bfloat16 const*>(mat_b.data_ptr()), num_tokens, + stream); } else { - if (num_tokens <= 8) { - invokeFusedAGemm<__nv_bfloat16, 1536, 3072, 8, 32>(out, a, b, num_tokens, - stream); - } else { - invokeFusedAGemm<__nv_bfloat16, 1536, 3072, 16, 32>(out, a, b, num_tokens, - stream); - } + invokeFusedAGemm<__nv_bfloat16, kHdIn, kHdOut, 16>( + reinterpret_cast<__nv_bfloat16*>(output.mutable_data_ptr()), + reinterpret_cast<__nv_bfloat16 const*>(mat_a.data_ptr()), + reinterpret_cast<__nv_bfloat16 const*>(mat_b.data_ptr()), num_tokens, + stream); } } diff --git a/csrc/libtorch_stable/quantization/fp4/nvfp4_quant_kernels.cu b/csrc/libtorch_stable/quantization/fp4/nvfp4_quant_kernels.cu index f10c58eab7..f7c965dbc1 100644 --- a/csrc/libtorch_stable/quantization/fp4/nvfp4_quant_kernels.cu +++ b/csrc/libtorch_stable/quantization/fp4/nvfp4_quant_kernels.cu @@ -49,11 +49,6 @@ __global__ void __launch_bounds__(512, VLLM_BLOCKS_PER_SM(512)) static_assert(sizeof(PackedVec) == sizeof(Type) * CVT_FP4_ELTS_PER_THREAD, "Vec size is not matched."); -#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)) - cudaGridDependencySynchronize(); - cudaTriggerProgrammaticLaunchCompletion(); -#endif - // Precompute SF layout parameter (constant for entire kernel). int32_t const numKTiles = (outputCols + 63) / 64; @@ -128,11 +123,6 @@ __global__ void __launch_bounds__(512, VLLM_BLOCKS_PER_SM(512)) static_assert(sizeof(PackedVec) == sizeof(Type) * CVT_FP4_ELTS_PER_THREAD, "Vec size is not matched."); -#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)) - cudaGridDependencySynchronize(); - cudaTriggerProgrammaticLaunchCompletion(); -#endif - int32_t const colIdx = blockDim.x * blockIdx.y + threadIdx.x; int elem_idx = colIdx * CVT_FP4_ELTS_PER_THREAD; @@ -211,8 +201,6 @@ void scaled_fp4_quant_sm1xxa(torch::stable::Tensor const& output, const torch::stable::accelerator::DeviceGuard device_guard( input.get_device_index()); auto stream = get_current_cuda_stream(input.get_device_index()); - auto* device_props = get_device_prop(); - int const sm_version = device_props->major * 10 + device_props->minor; int output_sf_n_unpadded = int(output_n / CVT_FP4_SF_VEC_SIZE); @@ -236,21 +224,10 @@ void scaled_fp4_quant_sm1xxa(torch::stable::Tensor const& output, input.scalar_type(), "nvfp4_quant_kernel", [&] { using cuda_type = vllm::CUDATypeConverter::Type; auto input_ptr = static_cast(input.data_ptr()); - cudaLaunchConfig_t config = {}; - config.gridDim = grid; - config.blockDim = block; - config.dynamicSmemBytes = 0; - config.stream = stream; - cudaLaunchAttribute attrs[1]; - attrs[0].id = cudaLaunchAttributeProgrammaticStreamSerialization; - attrs[0].val.programmaticStreamSerializationAllowed = 1; - config.numAttrs = (sm_version >= 90) ? 1 : 0; - config.attrs = attrs; - cudaLaunchKernelEx(&config, vllm::cvt_fp16_to_fp4, - m, n, output_n, num_padded_cols, input_ptr, - input_sf_ptr, - reinterpret_cast(output_ptr), - reinterpret_cast(sf_out)); + vllm::cvt_fp16_to_fp4<<>>( + m, n, output_n, num_padded_cols, input_ptr, input_sf_ptr, + reinterpret_cast(output_ptr), + reinterpret_cast(sf_out)); }); } else { int num_packed_cols = output_n / CVT_FP4_ELTS_PER_THREAD; @@ -263,21 +240,12 @@ void scaled_fp4_quant_sm1xxa(torch::stable::Tensor const& output, input.scalar_type(), "nvfp4_quant_kernel", [&] { using cuda_type = vllm::CUDATypeConverter::Type; auto input_ptr = static_cast(input.data_ptr()); - cudaLaunchConfig_t config = {}; - config.gridDim = grid; - config.blockDim = block; - config.dynamicSmemBytes = 0; - config.stream = stream; - cudaLaunchAttribute attrs[1]; - attrs[0].id = cudaLaunchAttributeProgrammaticStreamSerialization; - attrs[0].val.programmaticStreamSerializationAllowed = 1; - config.numAttrs = (sm_version >= 90) ? 1 : 0; - config.attrs = attrs; - cudaLaunchKernelEx( - &config, vllm::cvt_fp16_to_fp4_sf_major, m, n, - output_n, output_sf_n_unpadded, num_packed_cols, input_ptr, - input_sf_ptr, reinterpret_cast(output_ptr), - reinterpret_cast(sf_out)); + vllm::cvt_fp16_to_fp4_sf_major + <<>>( + m, n, output_n, output_sf_n_unpadded, num_packed_cols, + input_ptr, input_sf_ptr, + reinterpret_cast(output_ptr), + reinterpret_cast(sf_out)); }); } } diff --git a/csrc/libtorch_stable/torch_bindings.cpp b/csrc/libtorch_stable/torch_bindings.cpp index 0144118062..0a475d02c6 100644 --- a/csrc/libtorch_stable/torch_bindings.cpp +++ b/csrc/libtorch_stable/torch_bindings.cpp @@ -330,10 +330,6 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) { // conditionally compiled so impl registration is in source file ops.def("fp32_router_gemm(Tensor! output, Tensor mat_a, Tensor mat_b) -> ()"); - // BF16 skinny GEMM (M<=32, weight-BW-bound shapes, e.g. MTP eh_proj). - // conditionally compiled so impl registration is in source file - ops.def("bf16_skinny_gemm(Tensor! output, Tensor mat_a, Tensor mat_b) -> ()"); - // reorder weight for AllSpark Ampere W8A16 Fused Gemm kernel ops.def( "rearrange_kn_weight_as_n32k16_order(Tensor b_qweight, Tensor b_scales, " diff --git a/recipes/glm5.2-ll-b300-tp8-mtp5.md b/recipes/glm5.2-ll-b300-tp8-mtp5.md deleted file mode 100644 index a0deca9bd9..0000000000 --- a/recipes/glm5.2-ll-b300-tp8-mtp5.md +++ /dev/null @@ -1,41 +0,0 @@ -# GLM-5.2-NVFP4 — Low-Latency (LL) serving recipe · 8×B300 · TP8 · MTP=5 - -Single-node 8×B300 (sm_100/103), pure tensor parallel, NVFP4 weights, fp8 KV cache, -MTP=5 speculative decode, model runner v2. - -Validated on `glm5.2-LL` @ `8d407aee1`. - -## Versions - -| Component | Version | -| --- | --- | -| vLLM | `glm5.2-LL` @ `8d407aee1` | -| flashinfer-python | `0.6.15` | -| Model | `nvidia/GLM-5.2-NVFP4` (`modelopt_fp4`) | - -## Build (once) - -```bash -pip install 'flashinfer-python==0.6.15' - -CUDA_HOME=/usr/local/cuda-13.0 TORCH_CUDA_ARCH_LIST="10.0 10.3" MAX_JOBS=96 \ -VLLM_USE_PRECOMPILED=0 VLLM_USE_PRECOMPILED_RUST=0 \ -pip install -e . --no-deps --no-build-isolation -``` - -## Server - -```bash -export VLLM_USE_V2_MODEL_RUNNER=1 # model runner v2 -export VLLM_DEEP_GEMM_WARMUP=skip - -vllm serve nvidia/GLM-5.2-NVFP4 \ - --trust-remote-code \ - --tensor-parallel-size 8 \ - --quantization modelopt_fp4 --kv-cache-dtype fp8_e4m3 \ - --max-model-len 32768 --max-num-batched-tokens 16384 --max-num-seqs 256 \ - --no-enable-prefix-caching --gpu-memory-utilization 0.85 \ - --speculative-config '{"method":"mtp","num_speculative_tokens":5}' \ - --kernel-config '{"ir_op_priority":{"rms_norm":["vllm_c","native"],"fused_add_rms_norm":["vllm_c","native"]}}' \ - --host 0.0.0.0 --port 8000 -``` diff --git a/tests/kernels/test_bf16_skinny_gemm.py b/tests/kernels/test_bf16_skinny_gemm.py deleted file mode 100644 index f49e4b4db0..0000000000 --- a/tests/kernels/test_bf16_skinny_gemm.py +++ /dev/null @@ -1,70 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Numerical correctness for the bf16 skinny GEMM (decode-M GEMV) kernel. - -Covers every supported (N, K) shape across M = 0/1/2/8/16/32 (M=32 exercises -the tile_m=32 MMA path), plus strided (column-slice) output. M=0 must return -an empty result without launching the kernel (empty ranks at DP/PP -boundaries).""" - -import pytest -import torch - -from vllm import _custom_ops as ops -from vllm.platforms import current_platform - -# (N, K) pairs must match bf16_skinny_gemm_supported() in the entry .cu. -SHAPES = [ - (768, 12288), - (1536, 12288), - (6144, 12288), - (2048, 2048), - (2624, 6144), - (2112, 7168), - (7168, 14336), - (512, 6144), -] -MS = [0, 1, 2, 8, 16, 32] - -pytestmark = pytest.mark.skipif( - not current_platform.is_cuda() or not current_platform.has_device_capability(90), - reason="bf16_skinny_gemm requires CUDA SM90+", -) - - -def _rel_err(out: torch.Tensor, ref: torch.Tensor) -> float: - return (out.float() - ref).norm().item() / ref.norm().clamp_min(1e-6).item() - - -@pytest.mark.parametrize("n,k", SHAPES) -@pytest.mark.parametrize("m", MS) -def test_bf16_skinny_gemm_matches_reference(n: int, k: int, m: int): - torch.manual_seed(0) - x = torch.randn(m, k, dtype=torch.bfloat16, device="cuda") - w = torch.randn(n, k, dtype=torch.bfloat16, device="cuda") - - out = ops.bf16_skinny_gemm(x, w) - assert out.shape == (m, n) - assert out.dtype == torch.bfloat16 - - if m == 0: # empty batch: nothing to check beyond the empty shape - return - ref = x.float() @ w.float().t() - rel = _rel_err(out, ref) - assert rel < 2e-2, f"rel err {rel:.4f} for (m,n,k)=({m},{n},{k})" - - -@pytest.mark.parametrize("n,k", [(2048, 2048), (512, 6144)]) -def test_bf16_skinny_gemm_strided_output(n: int, k: int): - """Output is a column-slice of a wider padded buffer (row stride > N).""" - torch.manual_seed(0) - m = 8 - x = torch.randn(m, k, dtype=torch.bfloat16, device="cuda") - w = torch.randn(n, k, dtype=torch.bfloat16, device="cuda") - - wide = torch.empty(m, n + 64, dtype=torch.bfloat16, device="cuda") - view = wide[:, :n] - torch.ops._C.bf16_skinny_gemm(view, x, w) - - ref = x.float() @ w.float().t() - assert _rel_err(view, ref) < 2e-2 diff --git a/tests/kernels/test_fused_deepseek_v32_norm_rope.py b/tests/kernels/test_fused_deepseek_v32_norm_rope.py index f2302a896b..a27e67b624 100644 --- a/tests/kernels/test_fused_deepseek_v32_norm_rope.py +++ b/tests/kernels/test_fused_deepseek_v32_norm_rope.py @@ -49,32 +49,6 @@ pytestmark = pytest.mark.skipif( ) -def test_platform_capability_queries_are_constant_during_compile(monkeypatch): - monkeypatch.setattr( - K.current_platform, "has_device_capability", lambda capability: True - ) - monkeypatch.setattr(K, "has_cutedsl", lambda: True) - monkeypatch.setattr(K.current_platform, "is_arch_support_pdl", lambda: True) - - def capability_branches(x: torch.Tensor) -> torch.Tensor: - if K._can_use_fused_q_cutedsl() and K._is_arch_support_pdl(): - return x + 1 - return x - 1 - - compiled = torch.compile(capability_branches, backend="eager", fullgraph=True) - x = torch.zeros(1, device="cuda") - torch.testing.assert_close(compiled(x), torch.ones_like(x)) - - -def test_pdl_is_disabled_before_blackwell(monkeypatch): - monkeypatch.setattr( - K.current_platform, "has_device_capability", lambda capability: False - ) - monkeypatch.setattr(K.current_platform, "is_arch_support_pdl", lambda: True) - - assert not K._is_arch_support_pdl() - - # ── reference helpers ──────────────────────────────────────────────────────── @@ -318,57 +292,6 @@ def test_fused_norm_rope_no_indexer(num_tokens: int): assert (topk == 7).all(), "topk buffer should be untouched on shared layer" -def test_fused_norm_rope_profile_without_cache_compiles(): - """The cache-free profiling path must still compile and produce Q.""" - torch.manual_seed(2) - dev = "cuda" - num_tokens = 4 - pos = torch.arange(num_tokens, device=dev, dtype=torch.int64) - q_c = torch.randn(num_tokens, Q_LORA, device=dev, dtype=torch.bfloat16) - kv_c = torch.randn(num_tokens, KV_LORA, device=dev, dtype=torch.bfloat16) - k_pe = torch.randn(num_tokens, ROPE_DIM, device=dev, dtype=torch.bfloat16) - qw = torch.randn(Q_LORA, device=dev, dtype=torch.bfloat16) - kvw = torch.randn(KV_LORA, device=dev, dtype=torch.bfloat16) - index_k = torch.randn(num_tokens, INDEX_HEAD_DIM, device=dev, dtype=torch.bfloat16) - index_w = torch.randn(INDEX_HEAD_DIM, device=dev, dtype=torch.float32) - index_b = torch.randn(INDEX_HEAD_DIM, device=dev, dtype=torch.float32) - cos_sin = make_cos_sin(64, ROPE_DIM, dev) - topk = torch.empty(num_tokens, 2048, device=dev, dtype=torch.int32) - - def profile_run( - q_c: torch.Tensor, - kv_c: torch.Tensor, - k_pe: torch.Tensor, - index_k: torch.Tensor, - ) -> torch.Tensor: - return K.fused_norm_rope( - pos, - q_c, - qw, - EPS, - kv_c, - kvw, - EPS, - k_pe, - cos_sin, - index_k, - index_w, - index_b, - EPS, - cos_sin, - topk, - slot_mapping=None, - indexer_k_cache=None, - mla_kv_cache=None, - has_indexer=True, - index_rope_interleave=False, - ) - - compiled = torch.compile(profile_run, fullgraph=True) - q_out = compiled(q_c, kv_c, k_pe, index_k) - assert_bf16(q_out, rms_norm(q_c, qw), "q_c rmsnorm (profiling)") - - @pytest.mark.parametrize("num_tokens", [1, 4, 17, 512]) def test_fused_norm_rope_ds_mla(num_tokens: int): """fp8_ds_mla MLA cache layout (FlashMLA sparse, bf16-query path; SM90/SM100). diff --git a/tests/models/registry.py b/tests/models/registry.py index 65cdc84dec..afbac216c9 100644 --- a/tests/models/registry.py +++ b/tests/models/registry.py @@ -1600,10 +1600,6 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = { speculative_model="luccafong/deepseek_mtp_draft_random", trust_remote_code=True, ), - "DeepseekV32MTPModel": _HfExamplesInfo( - "deepseek-ai/DeepSeek-V3.2-Exp", - speculative_model="deepseek-ai/DeepSeek-V3.2-Exp", - ), "DeepSeekV4MTPModel": _HfExamplesInfo( "deepseek-ai/DeepSeek-V4-Flash", speculative_model="deepseek-ai/DeepSeek-V4-Flash", diff --git a/vllm/_custom_ops.py b/vllm/_custom_ops.py index 018fc1ac2b..aa75c50a51 100644 --- a/vllm/_custom_ops.py +++ b/vllm/_custom_ops.py @@ -2371,20 +2371,6 @@ def fp32_router_gemm( return output -def bf16_skinny_gemm( - x: torch.Tensor, - weight: torch.Tensor, -) -> torch.Tensor: - """Skinny bf16 GEMM (M<=32): x [M, K] @ weight [N, K]^T -> [M, N]. - - Single block-per-column kernel with fp32 accumulation; replaces cuBLAS - splitK for weight-bandwidth-bound decode shapes (e.g. MTP eh_proj). - """ - output = torch.empty((x.shape[0], weight.shape[0]), dtype=x.dtype, device=x.device) - torch.ops._C.bf16_skinny_gemm(output, x, weight) - return output - - if hasattr(torch.ops, "_C") and hasattr(torch.ops._C, "fp32_router_gemm"): @register_fake("_C::fp32_router_gemm") @@ -2396,17 +2382,6 @@ if hasattr(torch.ops, "_C") and hasattr(torch.ops._C, "fp32_router_gemm"): return -if hasattr(torch.ops, "_C") and hasattr(torch.ops._C, "bf16_skinny_gemm"): - - @register_fake("_C::bf16_skinny_gemm") - def bf16_skinny_gemm_fake( - output: torch.Tensor, - mat_a: torch.Tensor, - mat_b: torch.Tensor, - ) -> None: - return - - def topk_softmax( topk_weights: torch.Tensor, topk_ids: torch.Tensor, diff --git a/vllm/compilation/passes/fusion/allreduce_rms_fusion.py b/vllm/compilation/passes/fusion/allreduce_rms_fusion.py index 64ff381178..4716e7012f 100644 --- a/vllm/compilation/passes/fusion/allreduce_rms_fusion.py +++ b/vllm/compilation/passes/fusion/allreduce_rms_fusion.py @@ -202,7 +202,11 @@ if flashinfer_comm is not None: element_size = allreduce_in.element_size() current_tensor_size = num_tokens * hidden_size * element_size max_tensor_size = max_token_num * hidden_size * element_size - oversize = current_tensor_size > max_tensor_size + assert current_tensor_size <= max_tensor_size, ( + f"Current tensor size {current_tensor_size} is larger than " + f"max token num {max_token_num} * hidden size {hidden_size} * " + f"element size {element_size}" + ) curr_device = current_platform.get_device_capability() device_capability = curr_device.to_int() if curr_device is not None else None @@ -215,47 +219,6 @@ if flashinfer_comm is not None: get_workspace_fn = ( get_fi_ar_quant_workspace if is_quant_pattern else get_fi_ar_workspace ) - - orig_norm_out = norm_out - - def unfused_fallback() -> None: - # Fused-path equivalent honoring the op's output-buffer contract: - # with norm_out given, normed -> norm_out and the new residual -> - # the allreduce_in buffer; otherwise normed -> allreduce_in and - # the new residual -> the residual buffer. Runs fine during CUDA - # graph capture (the branch is resolved per captured shape). - # - # weight_bias mirrors the fused kernel's gamma offset (Gemma - # patterns pass 1.0 for rms_norm(x, weight + 1)); fold it into - # the gamma in fp32 like the reference before the plain rms_norm. - gamma = ( - rms_gamma - if weight_bias == 0.0 - else (rms_gamma.float() + weight_bias).to(rms_gamma.dtype) - ) - reduced = tensor_model_parallel_all_reduce(allreduce_in) - new_residual = reduced + residual - if orig_norm_out is None: - torch.ops._C.rms_norm(allreduce_in, new_residual, gamma, rms_eps) - residual.copy_(new_residual) - else: - torch.ops._C.rms_norm(orig_norm_out, new_residual, gamma, rms_eps) - allreduce_in.copy_(new_residual) - - if oversize: - assert not is_quant_pattern, ( - "flashinfer fused allreduce: input exceeds the workspace budget " - "and the unfused fallback does not cover quant patterns" - ) - logger.warning_once( - "fused allreduce+rmsnorm: %d tokens exceeds the workspace " - "budget (max_token_num=%d); using the unfused fallback for " - "this shape.", - num_tokens, - max_token_num, - ) - unfused_fallback() - return workspace = get_workspace_fn( world_size=world_size, rank=get_tensor_model_parallel_rank(), @@ -289,75 +252,36 @@ if flashinfer_comm is not None: if workspace.backend in ("trtllm", "mnnvl"): layout_code = flashinfer_comm.QuantizationSFLayout.SWIZZLED_128x4 - # vllm's one-shot size table is tuned for the trtllm backend. For - # mnnvl, flashinfer caps the workspace's one-shot region at its own - # MNNVL_ONE_SHOT_THRESHOLD (~4MB, e.g. 44 tokens at hidden 6144 bf16 - # TP8); forcing use_oneshot=True past that raises ValueError instead - # of using two-shot, so every decode capture between the one-shot cap - # and the fusion threshold silently lost fusion. Let flashinfer's - # AUTO heuristic pick the strategy (it matches its own workspace - # sizing), and keep trigger_completion_at_end=True since AUTO may - # select one-shot (see the Lamport NaN note below). - mnnvl_auto = workspace.backend == "mnnvl" - if mnnvl_auto: - use_oneshot = None - - try: - flashinfer_comm.allreduce_fusion( - input=allreduce_in, - workspace=workspace, - pattern=pattern_code, - launch_with_pdl=launch_with_pdl, - output=None, - residual_out=residual_out, - norm_out=norm_out, - quant_out=quant_out, - scale_out=scale_out, - residual_in=residual, - rms_gamma=rms_gamma, - rms_eps=rms_eps, - scale_factor=scale_factor, - layout_code=layout_code, - use_oneshot=use_oneshot, - fp32_acc=fp32_acc, - weight_bias=weight_bias, - # The one-shot Lamport all-reduce signals PDL completion before - # its output buffer is committed when trigger_completion_at_end - # is False, so the next PDL-launched kernel can read the - # uninitialized Lamport buffer and produce NaN. This only fires - # for num_tokens <= PDL_ADVANCE_LAUNCH_TOKENS (the batch=1 / - # spec-decode shapes, where the one-shot path is always - # selected). Complete at the end for the one-shot path; the - # two-shot path is synchronized and keeps the early completion. - # Related one-shot instability in the same kernel: - # flashinfer-ai/flashinfer#1223. - trigger_completion_at_end=True - if mnnvl_auto - else (use_oneshot or num_tokens > PDL_ADVANCE_LAUNCH_TOKENS), - ) - except ValueError as e: - # The globally cached workspace can be smaller than this call's - # budget (it is created once by whichever caller runs first, and - # the mnnvl workspace's is_buffer_size_sufficient() checks trtllm - # metadata, so vllm-side accounting cannot see the real capacity). - # flashinfer raises before launching anything, so falling back - # mid-capture is safe. Observed with spec-decode draft graphs - # whose token count (max_num_reqs * (1 + num_spec)) exceeds the - # workspace sized for the fusion threshold. - assert not is_quant_pattern, ( - "flashinfer fused allreduce failed and the unfused fallback " - f"does not cover quant patterns: {e}" - ) - logger.warning_once( - "flashinfer fused allreduce+rmsnorm unavailable for this " - "shape (num_tokens=%d, max_token_num=%d, world_size=%d): %s. " - "Using the unfused fallback.", - num_tokens, - max_token_num, - world_size, - e, - ) - unfused_fallback() + flashinfer_comm.allreduce_fusion( + input=allreduce_in, + workspace=workspace, + pattern=pattern_code, + launch_with_pdl=launch_with_pdl, + output=None, + residual_out=residual_out, + norm_out=norm_out, + quant_out=quant_out, + scale_out=scale_out, + residual_in=residual, + rms_gamma=rms_gamma, + rms_eps=rms_eps, + scale_factor=scale_factor, + layout_code=layout_code, + use_oneshot=use_oneshot, + fp32_acc=fp32_acc, + weight_bias=weight_bias, + # The one-shot Lamport all-reduce signals PDL completion before its + # output buffer is committed when trigger_completion_at_end is + # False, so the next PDL-launched kernel can read the uninitialized + # Lamport buffer and produce NaN. This only fires for + # num_tokens <= PDL_ADVANCE_LAUNCH_TOKENS (the batch=1 / spec-decode + # shapes, where the one-shot path is always selected). Complete at + # the end for the one-shot path; the two-shot path is synchronized + # and keeps the early completion. Related one-shot instability in + # the same kernel: flashinfer-ai/flashinfer#1223. + trigger_completion_at_end=(use_oneshot is True) + or num_tokens > PDL_ADVANCE_LAUNCH_TOKENS, + ) def call_trtllm_fused_allreduce_norm_fake( allreduce_in: torch.Tensor, diff --git a/vllm/config/speculative.py b/vllm/config/speculative.py index bd536800cc..8ba55a2ec9 100644 --- a/vllm/config/speculative.py +++ b/vllm/config/speculative.py @@ -330,7 +330,6 @@ class SpeculativeConfig: @staticmethod def hf_config_override(hf_config: PretrainedConfig) -> PretrainedConfig: initial_architecture = hf_config.architectures[0] - use_sparse_mtp = hf_config.model_type == "glm_moe_dsa" if hf_config.model_type in ( "deepseek_v3", "deepseek_v32", @@ -340,12 +339,7 @@ class SpeculativeConfig: if hf_config.model_type == "deepseek_mtp": n_predict = getattr(hf_config, "num_nextn_predict_layers", None) hf_config.update( - { - "n_predict": n_predict, - "architectures": [ - "DeepseekV32MTPModel" if use_sparse_mtp else "DeepSeekMTPModel" - ], - } + {"n_predict": n_predict, "architectures": ["DeepSeekMTPModel"]} ) if hf_config.model_type == "deepseek_v4": hf_config.model_type = "deepseek_mtp" diff --git a/vllm/cute_utils/__init__.py b/vllm/cute_utils/__init__.py index d5e536db90..ca445284bf 100644 --- a/vllm/cute_utils/__init__.py +++ b/vllm/cute_utils/__init__.py @@ -33,11 +33,6 @@ EVICT_NORMAL = Int64(0x1000000000000000) EVICT_FIRST = Int64(0x12F0000000000000) EVICT_LAST = Int64(0x14F0000000000000) -TORCH_TO_CUTE_DTYPE = { - torch.bfloat16: BFloat16, - torch.float32: Float32, -} - @dsl_user_op def recast_val(x, dtype, *, loc=None, ip=None): diff --git a/vllm/cute_utils/cvt.py b/vllm/cute_utils/cvt.py index 8bb4ad9519..4707f3f742 100644 --- a/vllm/cute_utils/cvt.py +++ b/vllm/cute_utils/cvt.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from cutlass import BFloat16, Constexpr, Float32, Uint16, Uint32, cute +from cutlass import Constexpr, Float32, Uint32, cute from cutlass._mlir import ir from cutlass._mlir.dialects import llvm, vector from cutlass.cutlass_dsl import T, dsl_user_op @@ -38,11 +38,6 @@ def bf16x2_to_fp32x2(data, *, loc=None, ip=None) -> tuple[Float32, Float32]: ) elif isinstance(data, (cute.Tensor, cute.TensorSSA)): - # recast to 32-bit registers if needed - if data.element_type == BFloat16: - data = cute.recast_tensor(data, Uint32) - assert data.element_type == Uint32 - # NOTE: the output is always 1D size = cute.size(data.shape) out = cute.make_rmem_tensor(size * 2, Float32) @@ -89,22 +84,6 @@ def fp8x4_to_bf16x4(x: Uint32, *, loc=None, ip=None) -> cute.TensorSSA: return cute.TensorSSA(vec, 2, Uint32) -@dsl_user_op -def fp32x2_to_fp8x2(a0: Float32, a1: Float32, *, loc=None, ip=None) -> Uint16: - out = llvm.inline_asm( - T.i16(), - [ - a0.ir_value(loc=loc, ip=ip), - a1.ir_value(loc=loc, ip=ip), - ], - "cvt.rn.satfinite.e4m3x2.f32 $0, $2, $1;", - "=h,f,f", - has_side_effects=False, - is_align_stack=False, - ) - return Uint16(out) - - @dsl_user_op def fp8x4_to_fp16x4(x: Uint32, *, loc=None, ip=None) -> cute.TensorSSA: out = llvm.inline_asm( diff --git a/vllm/model_executor/layers/fused_allreduce_gemma_rms_norm.py b/vllm/model_executor/layers/fused_allreduce_gemma_rms_norm.py index 509befbcc1..e49e135b26 100644 --- a/vllm/model_executor/layers/fused_allreduce_gemma_rms_norm.py +++ b/vllm/model_executor/layers/fused_allreduce_gemma_rms_norm.py @@ -43,7 +43,7 @@ try: if flashinfer_comm is not None else None ) -except (ImportError, AttributeError): +except ImportError: flashinfer_trtllm_fused_allreduce_norm = None # type: ignore[assignment] get_fi_ar_workspace = None # type: ignore[assignment] _AR_RESIDUAL_RMS_NORM = None @@ -52,23 +52,12 @@ except (ImportError, AttributeError): _FI_SUPPORTED_DTYPES = (torch.bfloat16, torch.float16) -@torch.compiler.assume_constant_result -def _fi_ar_max_size_mb() -> dict[int, float]: - """Flashinfer all-reduce fusion size table for the current device. - - Device capability is constant; marking the result constant keeps this out - of the traced graph. Otherwise the per-forward call reaches - ``current_platform.get_device_capability()`` inside ``torch.compile`` and - graph-breaks ("can't handle functions not implemented in python").""" - from vllm.config.compilation import PassConfig - - return PassConfig.default_fi_allreduce_fusion_max_size_mb() - - def _max_token_num(tp_size: int, hidden_size: int, dtype: torch.dtype) -> int | None: """Workspace token budget for flashinfer fused all-reduce, or None if the current world size / device is unsupported. Mirrors ``FlashInferAllReduce``.""" - max_size_mb = _fi_ar_max_size_mb().get(tp_size) + from vllm.config.compilation import PassConfig + + max_size_mb = PassConfig.default_fi_allreduce_fusion_max_size_mb().get(tp_size) if not max_size_mb: return None element_size = torch.tensor([], dtype=dtype).element_size() diff --git a/vllm/model_executor/models/registry.py b/vllm/model_executor/models/registry.py index 947e0d043d..cc2688e37f 100644 --- a/vllm/model_executor/models/registry.py +++ b/vllm/model_executor/models/registry.py @@ -114,7 +114,7 @@ _TEXT_GENERATION_MODELS = { "Glm4ForCausalLM": ("glm4", "Glm4ForCausalLM"), "Glm4MoeForCausalLM": ("glm4_moe", "Glm4MoeForCausalLM"), "Glm4MoeLiteForCausalLM": ("glm4_moe_lite", "Glm4MoeLiteForCausalLM"), - "GlmMoeDsaForCausalLM": ("vllm.models.deepseek_v32", "GlmMoeDsaForCausalLM"), + "GlmMoeDsaForCausalLM": ("deepseek_v2", "GlmMoeDsaForCausalLM"), "GptOssForCausalLM": ("gpt_oss", "GptOssForCausalLM"), "GPT2LMHeadModel": ("gpt2", "GPT2LMHeadModel"), "GPTJForCausalLM": ("gpt_j", "GPTJForCausalLM"), @@ -628,7 +628,6 @@ _SPECULATIVE_DECODING_MODELS = { "Eagle3DeepseekV3ForCausalLM": ("deepseek_eagle3", "Eagle3DeepseekV2ForCausalLM"), "EagleDeepSeekMTPModel": ("deepseek_eagle", "EagleDeepseekV3ForCausalLM"), "DeepSeekMTPModel": ("deepseek_mtp", "DeepSeekMTP"), - "DeepseekV32MTPModel": ("vllm.models.deepseek_v32", "DeepseekV32MTP"), "DeepSeekV4MTPModel": ("vllm.models.deepseek_v4", "DeepSeekV4MTP"), "MiniMaxM3MTP": ("vllm.models.minimax_m3", "MiniMaxM3MTP"), "BailingMoeV25MTPModel": ("bailing_moe_mtp", "BailingMoeV25MTPModel"), diff --git a/vllm/models/deepseek_v32/__init__.py b/vllm/models/deepseek_v32/__init__.py index 6bd447bab2..1b0aa64262 100644 --- a/vllm/models/deepseek_v32/__init__.py +++ b/vllm/models/deepseek_v32/__init__.py @@ -6,31 +6,17 @@ DeepSeek V3.2 introduced the DeepSeek Sparse Attention (DSA) architecture: MLA + a "lightning indexer" that selects the top-k tokens for a sparse MLA attend. The same model code serves any DSA checkpoint, including GLM-5.2 (``glm_moe_dsa``), which reuses this architecture. - -The optimized kernels under ``nvidia/`` target the Blackwell (SM100) family. -Every other platform — ROCm, XPU, pre-SM100 CUDA (e.g. H100), CPU — falls back -to the generic ``deepseek_v2`` implementation, which already handles the DSA -(index_topk) architecture and is ``torch.compile``-friendly there. This matches -main's behavior on those platforms (no hard failure). """ from vllm.platforms import current_platform -if current_platform.is_cuda() and current_platform.is_device_capability_family(100): - from .nvidia.model import DeepseekV32ForCausalLM - from .nvidia.mtp import DeepseekV32MTP +if current_platform.is_rocm() or current_platform.is_xpu(): + raise NotImplementedError("deepseek_v32 currently supports NVIDIA SM100 only.") - # GLM-5.2 (glm_moe_dsa) reuses the same optimized DSA module on SM100. - GlmMoeDsaForCausalLM = DeepseekV32ForCausalLM -else: - from vllm.model_executor.models.deepseek_mtp import DeepSeekMTP as DeepseekV32MTP - from vllm.model_executor.models.deepseek_v2 import ( - DeepseekV3ForCausalLM as DeepseekV32ForCausalLM, - ) - from vllm.model_executor.models.deepseek_v2 import GlmMoeDsaForCausalLM +from .nvidia.model import DeepseekV32ForCausalLM +from .nvidia.mtp import DeepseekV32MTP __all__ = [ "DeepseekV32ForCausalLM", "DeepseekV32MTP", - "GlmMoeDsaForCausalLM", ] diff --git a/vllm/models/deepseek_v32/nvidia/attention.py b/vllm/models/deepseek_v32/nvidia/attention.py index 8e60e87189..dcf955ad59 100644 --- a/vllm/models/deepseek_v32/nvidia/attention.py +++ b/vllm/models/deepseek_v32/nvidia/attention.py @@ -4,7 +4,6 @@ import torch import torch.nn as nn from transformers import DeepseekV2Config, DeepseekV3Config -import vllm._custom_ops as ops from vllm.compilation.breakable_cudagraph import eager_break_during_capture from vllm.config import CacheConfig, VllmConfig from vllm.distributed import get_tensor_model_parallel_world_size @@ -22,15 +21,17 @@ from vllm.model_executor.layers.quantization.utils.fp8_utils import ( per_token_group_quant_fp8, ) from vllm.model_executor.layers.rotary_embedding import get_rope -from vllm.model_executor.layers.sparse_attn_indexer import SparseAttnIndexer +from vllm.model_executor.layers.sparse_attn_indexer import ( + SparseAttnIndexer, + sparse_attn_indexer, +) from vllm.model_executor.models.deepseek_v2 import ( DeepSeekV2FusedQkvAProjLinear, DeepseekV32IndexerCache, yarn_get_mscale, ) from vllm.model_executor.models.utils import extract_layer_index -from vllm.platforms import current_platform -from vllm.utils.torch_utils import _encode_layer_name, is_quantized_kv_cache +from vllm.utils.torch_utils import is_quantized_kv_cache from .kernels import fused_norm_rope, fused_q @@ -170,11 +171,6 @@ class DeepseekV32Attention(MLAAttention): ) -> None: quant_config = vllm_config.quant_config cache_config = vllm_config.cache_config - if cache_config is not None and cache_config.cache_dtype == "auto": - # This implementation requires an FP8 sparse cache. Start with the - # generic FP8 format; MLAAttention will canonicalize it for the - # selected sparse backend. - cache_config.cache_dtype = "fp8" hidden_size = config.hidden_size qk_nope_head_dim = config.qk_nope_head_dim @@ -335,37 +331,6 @@ class DeepseekV32Attention(MLAAttention): rope_parameters=config.rope_parameters, is_neox_style=False, ) - # Decode-M GEMM dispatch for the bf16 A/B projections, measured on - # B300 and B200 (graph replay + rotating weights vs fp64): - # bf16_skinny_gemm for M <= skinny_max, dsv3_fused_a_gemm up to - # M = 16, cuBLAS above. Blackwell-only: the boundaries and the - # fused_a tile choices are tied to the 148/152-SM single-wave - # geometry. DSv3.2 q_b has no skinny tier (K=1536 does not fit the - # GEMV's 128x8 K-step; fused_a still wins 1.6-1.8x there). - # Quantized linear methods may not register .weight until - # process_weights_after_loading (e.g. compressed-tensors NVFP4 - # registers weight_packed at init) — probe with getattr. - # (N, K) -> skinny_max - qkv_a_dispatch = {(2624, 6144): 2, (2112, 7168): 2} - q_b_dispatch = {(2048, 2048): 2, (3072, 1536): 0} - ok = ( - current_platform.is_device_capability_family(100) - and hasattr(torch.ops._C, "bf16_skinny_gemm") - and hasattr(torch.ops._C, "dsv3_fused_a_gemm") - ) - qkv_a_weight = getattr(self.fused_qkv_a_proj, "weight", None) - q_b_weight = getattr(self.q_b_proj, "weight", None) - self._qkv_a_skinny_max = ( - qkv_a_dispatch.get(tuple(qkv_a_weight.shape)) - if ok and qkv_a_weight is not None and qkv_a_weight.dtype == torch.bfloat16 - else None - ) - self._q_b_skinny_max = ( - q_b_dispatch.get(tuple(q_b_weight.shape)) - if ok and q_b_weight is not None and q_b_weight.dtype == torch.bfloat16 - else None - ) - # Lightning indexer uses its own RoPE; interleave maps to non-NeoX. self.indexer_rope_emb = get_rope( qk_rope_head_dim, @@ -374,32 +339,13 @@ class DeepseekV32Attention(MLAAttention): is_neox_style=not getattr(config, "indexer_rope_interleave", False), ) - def _decode_m_gemm( - self, x: torch.Tensor, weight: torch.Tensor, skinny_max: int - ) -> torch.Tensor: - """x @ weight.T for decode M <= 16.""" - if x.shape[0] == 0: - return torch.empty((0, weight.shape[0]), dtype=x.dtype, device=x.device) - if x.shape[0] <= skinny_max: - return ops.bf16_skinny_gemm(x, weight) - out = torch.empty((x.shape[0], weight.shape[0]), dtype=x.dtype, device=x.device) - ops.dsv3_fused_a_gemm(out, x, weight.t()) - return out - def forward( # type: ignore[override] self, positions: torch.Tensor, hidden_states: torch.Tensor, ) -> torch.Tensor: # Captured: A-projections (+ indexer A-GEMM on indexer layers). - if self._qkv_a_skinny_max is not None and hidden_states.shape[0] <= 16: - qkv_lora = self._decode_m_gemm( - hidden_states, - self.fused_qkv_a_proj.weight, - self._qkv_a_skinny_max, - ) - else: - qkv_lora = self.fused_qkv_a_proj(hidden_states)[0] + qkv_lora = self.fused_qkv_a_proj(hidden_states)[0] q_c, kv_c, k_pe = qkv_lora.split( [self.q_lora_rank, self.kv_lora_rank, self.qk_rope_head_dim], dim=-1 ) @@ -452,9 +398,7 @@ class DeepseekV32Attention(MLAAttention): assert isinstance(slot_mapping, dict) mla_slot = slot_mapping.get(self.layer_name) - # skip_topk (index_share_for_mtp_iteration reuse mode): don't run the - # indexer this step; keep the top-k already in the shared buffer. - if self.indexer is not None and not self.skip_topk: + if self.indexer is not None: has_indexer = True indexer_k_norm_w = self.indexer.k_norm.weight indexer_k_norm_bias = self.indexer.k_norm.bias @@ -507,16 +451,12 @@ class DeepseekV32Attention(MLAAttention): index_rope_interleave=self._index_rope_interleave, ) - if self._q_b_skinny_max is not None and q_c.shape[0] <= 16: - q = self._decode_m_gemm(q_c, self.q_b_proj.weight, self._q_b_skinny_max) - else: - q = self.q_b_proj(q_c)[0] - q = q.view(-1, self.num_local_heads, self.qk_head_dim) + q = self.q_b_proj(q_c)[0].view(-1, self.num_local_heads, self.qk_head_dim) q_nope, q_pe = q.split([self.qk_nope_head_dim, self.qk_rope_head_dim], dim=-1) q_nope = q_nope.transpose(0, 1) ql_nope = torch.bmm(q_nope, self.W_UK_T).transpose(0, 1) - if self.indexer is not None and not self.skip_topk: + if self.indexer is not None: index_q = self.indexer.wq_b(q_c)[0] index_q = index_q.view(-1, self.indexer.n_head, self.indexer.head_dim) else: @@ -538,10 +478,10 @@ class DeepseekV32Attention(MLAAttention): quantize_mqa=self._fp8_query, ) - if self.indexer is not None and not self.skip_topk: - torch.ops.vllm.sparse_attn_indexer( + if self.indexer is not None: + sparse_attn_indexer( q_c, - _encode_layer_name(self.indexer.k_cache.prefix), + self.indexer.k_cache.prefix, self.indexer.k_cache.kv_cache, index_q_fp8, None, # q_scale folded into weights on the fp8 path diff --git a/vllm/models/deepseek_v32/nvidia/kernels.py b/vllm/models/deepseek_v32/nvidia/kernels.py index c931ac7cb4..ef7b14ef1c 100644 --- a/vllm/models/deepseek_v32/nvidia/kernels.py +++ b/vllm/models/deepseek_v32/nvidia/kernels.py @@ -2,80 +2,13 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import torch -from vllm.platforms import current_platform from vllm.triton_utils import tl, triton -from vllm.utils.import_utils import has_cutedsl -from vllm.utils.torch_utils import direct_register_custom_op # Cache of tiny 1-element dummy tensors (per device, dtype) reused by the # has_indexer=False path so the indexer args don't allocate every call. _DUMMY_CACHE: dict[tuple, torch.Tensor] = {} -@torch.compiler.assume_constant_result -def _can_use_fused_q_cutedsl() -> bool: - return current_platform.has_device_capability(100) and has_cutedsl() - - -@torch.compiler.assume_constant_result -def _is_arch_support_pdl() -> bool: - return ( - current_platform.has_device_capability(100) - and current_platform.is_arch_support_pdl() - ) - - -def _fused_q_cutedsl_impl( - positions: torch.Tensor, - q_pe: torch.Tensor, - rope_cache: torch.Tensor, - ql_nope: torch.Tensor, - q_scale: torch.Tensor, - mqa_output: torch.Tensor, - idx_q: torch.Tensor, - idx_rope_cache: torch.Tensor, - idx_weights: torch.Tensor, - idx_weights_softmax_scale: float, - idx_weights_head_scale: float, - idx_q_fp8: torch.Tensor, - idx_weights_out: torch.Tensor, - has_indexer: bool, - index_rope_interleave: bool, -) -> None: - from .ops.fused_q_cutedsl import fused_q_cutedsl - - fused_q_cutedsl( - positions, - q_pe, - rope_cache, - ql_nope, - q_scale, - mqa_output, - idx_q, - idx_rope_cache, - idx_weights, - idx_weights_softmax_scale, - idx_weights_head_scale, - idx_q_fp8, - idx_weights_out, - has_indexer=has_indexer, - index_rope_interleave=index_rope_interleave, - ) - - -def _fused_q_cutedsl_fake(*args, **kwargs) -> None: - pass - - -direct_register_custom_op( - op_name="fused_q_cutedsl", - op_func=_fused_q_cutedsl_impl, - mutates_args=["mqa_output", "idx_q_fp8", "idx_weights_out"], - fake_impl=_fused_q_cutedsl_fake, - dispatch_key="CUDA", -) - - def _dummy(shape: tuple, dtype: torch.dtype, device: torch.device) -> torch.Tensor: key = (shape, dtype, device) t = _DUMMY_CACHE.get(key) @@ -214,14 +147,9 @@ def _fused_norm_rope_kernel( TOPK_BLOCK_SIZE: tl.constexpr, HAS_INDEXER: tl.constexpr, INDEX_ROPE_INTERLEAVE: tl.constexpr, - USE_PDL: tl.constexpr, ): pid = tl.program_id(0) tok_idx = tl.program_id(1) - if USE_PDL: - tl.extra.cuda.gdc_wait() - tl.extra.cuda.gdc_launch_dependents() - if pid == 3: if not HAS_INDEXER: # Shared layer: reuse the previous indexer layer's top-k; do not @@ -238,17 +166,6 @@ def _fused_norm_rope_kernel( ) return - if pid == 2: - # Q RMS norm does not depend on cache availability. In particular, - # profiling uses a negative dummy slot mapping but still consumes Q. - q_block = tl.arange(0, Q_BLOCK_SIZE) - q_mask = q_block < Q_DIM - q_c = tl.load(q_c_ptr + tok_idx * q_c_stride + q_block, mask=q_mask, other=0.0) - q_c_rms_w = tl.load(q_rms_norm_w_ptr + q_block, mask=q_mask) - q_c = _rms_norm(q_c, q_c_rms_w, q_rms_eps, Q_DIM) - tl.store(q_c_out_ptr + tok_idx * q_c_out_stride + q_block, q_c, mask=q_mask) - return - if slot_mapping_ptr is None: # Memory profiling run. return @@ -257,7 +174,15 @@ def _fused_norm_rope_kernel( # Padding return - if pid == 1: + if pid == 2: + # Q RMS norm + q_block = tl.arange(0, Q_BLOCK_SIZE) + q_mask = q_block < Q_DIM + q_c = tl.load(q_c_ptr + tok_idx * q_c_stride + q_block, mask=q_mask, other=0.0) + q_c_rms_w = tl.load(q_rms_norm_w_ptr + q_block, mask=q_mask) + q_c = _rms_norm(q_c, q_c_rms_w, q_rms_eps, Q_DIM) + tl.store(q_c_out_ptr + tok_idx * q_c_out_stride + q_block, q_c, mask=q_mask) + elif pid == 1: # KV RMS Norm + KV RoPE + MLA concat_and_cache. # Merged so the normed kv_c and RoPE'd k_pe can be written # to the MLA KV cache directly without a separate kernel. @@ -286,6 +211,9 @@ def _fused_norm_rope_kernel( r2 = x2 * cos + x1 * sin # MLA concat_and_cache: write [kv_c_normed, k_pe_roped] to cache. + if mla_cache_entry_stride == 0: + return + mla_block_size = mla_cache_block_stride // mla_cache_entry_stride mla_block_idx = slot_idx // mla_block_size mla_block_off = slot_idx % mla_block_size @@ -435,7 +363,7 @@ def _fused_norm_rope_kernel( ) -def _fused_norm_rope_impl( +def fused_norm_rope( positions: torch.Tensor, q_c: torch.Tensor, q_rms_norm_w: torch.Tensor, @@ -537,17 +465,12 @@ def _fused_norm_rope_impl( # Dummy values — pid 2 will skip the MLA cache write because # slot_mapping is all -1. mla_kv_cache = torch.empty(0, dtype=torch.bfloat16, device=device) - # Torch's Triton HOP mutation analysis compiles every program-id - # branch even though the dummy slot mapping returns before cache - # access. Keep the unused strides nonzero so that analysis can lower - # the block-size calculation safely. - mla_block_stride = 1 - mla_entry_stride = 1 + mla_block_stride = 0 + mla_entry_stride = 0 mla_k_scale = torch.ones(1, dtype=torch.float32, device=device) if q_c_out is None: q_c_out = torch.empty_like(q_c) - use_pdl = _is_arch_support_pdl() _fused_norm_rope_kernel[(4, num_tokens)]( positions, # Q RMS norm @@ -606,157 +529,6 @@ def _fused_norm_rope_impl( TOPK_BLOCK_SIZE=1024, HAS_INDEXER=has_indexer, INDEX_ROPE_INTERLEAVE=index_rope_interleave, - USE_PDL=use_pdl, - launch_pdl=use_pdl, - ) - return q_c_out - - -def _fused_norm_rope_op( - positions: torch.Tensor, - q_c: torch.Tensor, - q_rms_norm_w: torch.Tensor, - q_rms_eps: float, - kv_c: torch.Tensor, - kv_rms_norm_w: torch.Tensor, - kv_rms_eps: float, - k_pe: torch.Tensor, - k_rope_cos_sin_cache: torch.Tensor, - index_k: torch.Tensor | None, - index_k_layer_norm_w: torch.Tensor | None, - index_k_layer_norm_bias: torch.Tensor | None, - index_k_layer_norm_eps: float, - index_k_rope_cos_sin_cache: torch.Tensor | None, - topk_indices_buffer: torch.Tensor, - slot_mapping: torch.Tensor | None, - indexer_k_cache: torch.Tensor | None, - mla_kv_cache: torch.Tensor | None, - mla_kv_cache_dtype: str, - mla_k_scale: torch.Tensor | None, - has_indexer: bool, - index_rope_interleave: bool, - q_c_out: torch.Tensor, -) -> None: - _fused_norm_rope_impl( - positions, - q_c, - q_rms_norm_w, - q_rms_eps, - kv_c, - kv_rms_norm_w, - kv_rms_eps, - k_pe, - k_rope_cos_sin_cache, - index_k, - index_k_layer_norm_w, - index_k_layer_norm_bias, - index_k_layer_norm_eps, - index_k_rope_cos_sin_cache, - topk_indices_buffer, - slot_mapping=slot_mapping, - indexer_k_cache=indexer_k_cache, - mla_kv_cache=mla_kv_cache, - mla_kv_cache_dtype=mla_kv_cache_dtype, - mla_k_scale=mla_k_scale, - has_indexer=has_indexer, - index_rope_interleave=index_rope_interleave, - q_c_out=q_c_out, - ) - - -def _fused_norm_rope_fake( - positions: torch.Tensor, - q_c: torch.Tensor, - q_rms_norm_w: torch.Tensor, - q_rms_eps: float, - kv_c: torch.Tensor, - kv_rms_norm_w: torch.Tensor, - kv_rms_eps: float, - k_pe: torch.Tensor, - k_rope_cos_sin_cache: torch.Tensor, - index_k: torch.Tensor | None, - index_k_layer_norm_w: torch.Tensor | None, - index_k_layer_norm_bias: torch.Tensor | None, - index_k_layer_norm_eps: float, - index_k_rope_cos_sin_cache: torch.Tensor | None, - topk_indices_buffer: torch.Tensor, - slot_mapping: torch.Tensor | None, - indexer_k_cache: torch.Tensor | None, - mla_kv_cache: torch.Tensor | None, - mla_kv_cache_dtype: str, - mla_k_scale: torch.Tensor | None, - has_indexer: bool, - index_rope_interleave: bool, - q_c_out: torch.Tensor, -) -> None: - return - - -direct_register_custom_op( - op_name="fused_norm_rope_deepseek_v32", - op_func=_fused_norm_rope_op, - mutates_args=[ - "topk_indices_buffer", - "indexer_k_cache", - "mla_kv_cache", - "q_c_out", - ], - fake_impl=_fused_norm_rope_fake, - dispatch_key="CUDA", -) - - -def fused_norm_rope( - positions: torch.Tensor, - q_c: torch.Tensor, - q_rms_norm_w: torch.Tensor, - q_rms_eps: float, - kv_c: torch.Tensor, - kv_rms_norm_w: torch.Tensor, - kv_rms_eps: float, - k_pe: torch.Tensor, - k_rope_cos_sin_cache: torch.Tensor, - index_k: torch.Tensor | None, - index_k_layer_norm_w: torch.Tensor | None, - index_k_layer_norm_bias: torch.Tensor | None, - index_k_layer_norm_eps: float, - index_k_rope_cos_sin_cache: torch.Tensor | None, - topk_indices_buffer: torch.Tensor, - slot_mapping: torch.Tensor | None = None, - indexer_k_cache: torch.Tensor | None = None, - mla_kv_cache: torch.Tensor | None = None, - mla_kv_cache_dtype: str = "auto", - mla_k_scale: torch.Tensor | None = None, - has_indexer: bool = True, - index_rope_interleave: bool = False, - q_c_out: torch.Tensor | None = None, -) -> torch.Tensor: - if q_c_out is None: - q_c_out = torch.empty_like(q_c) - torch.ops.vllm.fused_norm_rope_deepseek_v32( - positions, - q_c, - q_rms_norm_w, - q_rms_eps, - kv_c, - kv_rms_norm_w, - kv_rms_eps, - k_pe, - k_rope_cos_sin_cache, - index_k, - index_k_layer_norm_w, - index_k_layer_norm_bias, - index_k_layer_norm_eps, - index_k_rope_cos_sin_cache, - topk_indices_buffer, - slot_mapping, - indexer_k_cache, - mla_kv_cache, - mla_kv_cache_dtype, - mla_k_scale, - has_indexer, - index_rope_interleave, - q_c_out, ) return q_c_out @@ -810,14 +582,10 @@ def _fused_q_kernel( HAS_INDEXER: tl.constexpr, INDEX_ROPE_INTERLEAVE: tl.constexpr, QUANTIZE_MQA: tl.constexpr, - USE_PDL: tl.constexpr, ): pid = tl.program_id(0) tok_idx = tl.program_id(1) head_idx = tl.program_id(2) - if USE_PDL: - tl.extra.cuda.gdc_wait() - tl.extra.cuda.gdc_launch_dependents() if pid == 2: # ql_nope quantize + pack into the front of mqa_q_fp8. On the bf16 @@ -1005,12 +773,10 @@ def fused_q( ``(ql_nope, q_pe)`` tuple the backend expects. """ assert positions.ndim == 1 - assert positions.dtype == torch.int64 assert q_pe.ndim == 3 assert q_pe_cos_sin_cache.ndim == 2 assert ql_nope.ndim == 3 assert ql_nope.shape[:2] == q_pe.shape[:2] - assert q_scale.dtype == torch.float32 and q_scale.numel() == 1 num_tokens = positions.shape[0] num_q_heads = q_pe.shape[1] @@ -1028,18 +794,7 @@ def fused_q( assert index_weights is not None num_index_q_heads = index_q.shape[1] index_q_head_dim = index_q.shape[2] - use_cutedsl = False - if _can_use_fused_q_cutedsl(): - from .ops.fused_q_cutedsl import is_fused_q_cutedsl_supported - - use_cutedsl = is_fused_q_cutedsl_supported( - q_pe, - index_q, - ql_nope, - has_indexer=has_indexer, - quantize_mqa=quantize_mqa, - ) - grid_heads = max(mqa_grid_heads, num_index_q_heads if has_indexer else 1) + grid_heads = max(mqa_grid_heads, num_index_q_heads) if quantize_mqa: # fp8 path: pack [ql_nope; q_pe] into a single fp8 tensor. mqa_q_fp8 = torch.empty( @@ -1060,27 +815,6 @@ def fused_q( index_q_fp8 = torch.empty_like(index_q, dtype=torch.float8_e4m3fn) index_weights_out = torch.empty_like(index_weights, dtype=torch.float32) - if use_cutedsl: - torch.ops.vllm.fused_q_cutedsl( - positions, - q_pe, - q_pe_cos_sin_cache, - ql_nope, - q_scale, - mqa_q, - index_q, - index_q_cos_sin_cache, - index_weights, - index_weights_softmax_scale, - index_weights_head_scale, - index_q_fp8, - index_weights_out, - has_indexer=has_indexer, - index_rope_interleave=index_rope_interleave, - ) - return index_q_fp8, index_weights_out, mqa_q - - use_pdl = _is_arch_support_pdl() _fused_q_kernel[(3, num_tokens, grid_heads)]( positions, q_pe, @@ -1122,8 +856,6 @@ def fused_q( HAS_INDEXER=has_indexer, INDEX_ROPE_INTERLEAVE=index_rope_interleave, QUANTIZE_MQA=quantize_mqa, - USE_PDL=use_pdl, - launch_pdl=use_pdl, # num_warps=1 is optimal here: each program is a single 128-element # rope+quant, so the kernel is program-count/occupancy bound, not # per-program compute bound (swept 1/2/4/8 — 1 wins or ties everywhere). diff --git a/vllm/models/deepseek_v32/nvidia/model.py b/vllm/models/deepseek_v32/nvidia/model.py index 90a6a2cf45..353aedc8ce 100644 --- a/vllm/models/deepseek_v32/nvidia/model.py +++ b/vllm/models/deepseek_v32/nvidia/model.py @@ -6,7 +6,6 @@ from itertools import islice import torch -from vllm.compilation.decorators import support_torch_compile from vllm.config import VllmConfig from vllm.distributed import ( get_pp_group, @@ -40,7 +39,6 @@ from vllm.model_executor.models.utils import ( sequence_parallel_chunk, ) from vllm.sequence import IntermediateTensors -from vllm.v1.attention.backends.mla.sparse_utils import register_phys_shadow from .attention import DeepseekV32Attention from .fused_ops import fused_allreduce_rms_norm @@ -179,7 +177,6 @@ class DeepseekV32DecoderLayer(torch.nn.Module): return hidden_states, residual -@support_torch_compile class DeepseekV32Model(torch.nn.Module): fall_back_to_pt_during_load = False @@ -203,7 +200,6 @@ class DeepseekV32Model(torch.nn.Module): dtype=torch.int32, device=self.device, ) - register_phys_shadow(topk_indices_buffer) if get_pp_group().is_first_rank: self.embed_tokens = VocabParallelEmbedding( diff --git a/vllm/models/deepseek_v32/nvidia/mtp.py b/vllm/models/deepseek_v32/nvidia/mtp.py index bafbf383ca..c8f2fcc5ff 100644 --- a/vllm/models/deepseek_v32/nvidia/mtp.py +++ b/vllm/models/deepseek_v32/nvidia/mtp.py @@ -6,9 +6,9 @@ from collections.abc import Callable, Iterable import torch import torch.nn as nn -import vllm._custom_ops as ops from vllm._aiter_ops import rocm_aiter_ops from vllm.config import VllmConfig +from vllm.distributed import tensor_model_parallel_all_reduce from vllm.model_executor.layers.fused_moe import ( fused_moe_make_expert_params_mapping, ) @@ -40,12 +40,7 @@ from vllm.model_executor.models.utils import ( ) from vllm.platforms import current_platform from vllm.sequence import IntermediateTensors -from vllm.v1.attention.backends.mla.sparse_utils import ( - phys_shadow, - register_phys_shadow, -) -from .fused_ops import fused_allreduce_rms_norm from .kernels import fused_eh_norm from .model import DeepseekV32DecoderLayer @@ -61,19 +56,6 @@ class DeepseekV32MultiTokenPredictorLayer(nn.Module): self.enorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) self.hnorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) self.eh_proj = nn.Linear(config.hidden_size * 2, config.hidden_size, bias=False) - # bf16 skinny GEMM for the eh_proj, B300+B200 measured: GLM-5.2 - # (6144, 12288) wins at M <= 3, DSv3.2 (7168, 14336) at M <= 2; - # cuBLAS holds above (the 151/205MB weights stream at ~5.7TB/s). - eh_dispatch = {(6144, 12288): 3, (7168, 14336): 2} - eh_weight = getattr(self.eh_proj, "weight", None) - self._eh_skinny_max = ( - eh_dispatch.get(tuple(eh_weight.shape), 0) - if current_platform.is_device_capability_family(100) - and hasattr(torch.ops._C, "bf16_skinny_gemm") - and eh_weight is not None - and eh_weight.dtype == torch.bfloat16 - else 0 - ) topk_indices_buffer = torch.empty( vllm_config.scheduler_config.max_num_batched_tokens, @@ -81,7 +63,6 @@ class DeepseekV32MultiTokenPredictorLayer(nn.Module): dtype=torch.int32, device=current_platform.device_type, ) - register_phys_shadow(topk_indices_buffer) self.shared_head = SharedHead( config=config, prefix=prefix, quant_config=quant_config ) @@ -110,13 +91,7 @@ class DeepseekV32MultiTokenPredictorLayer(nn.Module): self.hnorm.weight, self.enorm.variance_epsilon, ) - if ( - eh_input.shape[0] <= self._eh_skinny_max - and eh_input.dtype == torch.bfloat16 - ): - hidden_states = ops.bf16_skinny_gemm(eh_input, self.eh_proj.weight) - else: - hidden_states = self.eh_proj(eh_input) + hidden_states = self.eh_proj(eh_input) hidden_states, residual = self.mtp_block( positions=positions, hidden_states=hidden_states, residual=None ) @@ -126,6 +101,9 @@ class DeepseekV32MultiTokenPredictorLayer(nn.Module): positions.shape[0], is_sequence_parallel=self.mtp_block.use_sequence_parallel_moe, ) + if not self.mtp_block.use_sequence_parallel_moe: + # Without sequence parallelism, the MoE output is left un-reduced. + hidden_states = tensor_model_parallel_all_reduce(hidden_states) # Recycle the POST-final-norm hidden into the next draft step. The # residual-add is fused into the final RMSNorm so it is computed # exactly once, and the result is returned for both tuple positions: @@ -136,14 +114,7 @@ class DeepseekV32MultiTokenPredictorLayer(nn.Module): # is understood by both the V2 speculator (isinstance-tuple check) and # the legacy proposer (model_returns_tuple is True for the # DeepSeekMTPModel architecture). - if self.mtp_block.use_sequence_parallel_moe: - hidden_states, _ = self.shared_head.norm(hidden_states, residual) - else: - # The MoE output is left un-reduced; fuse its all-reduce into the - # final norm, as the main model does at layer boundaries. - hidden_states, _ = fused_allreduce_rms_norm( - hidden_states, residual, self.shared_head.norm - ) + hidden_states, _ = self.shared_head.norm(hidden_states, residual) return hidden_states, hidden_states @@ -186,15 +157,6 @@ class DeepseekV32MultiTokenPredictor(nn.Module): if self_attn is not None and hasattr(self_attn, "topk_indices_buffer"): topk_indices_buffer = self_attn.topk_indices_buffer topk_indices_buffer[:num_slots] = topk_indices_buffer[slot_ids] - # The physical-index shadow mirrors this buffer row-wise - # (shadow[j] == convert(logical[j])); re-arrange it the same - # way or skip_topk draft steps read the step-0 multi-token - # layout's rows for the wrong tokens/requests. - shadow = phys_shadow(topk_indices_buffer) - if shadow is not None: - phys, seq = shadow - phys[:num_slots] = phys[slot_ids] - seq[:num_slots] = seq[slot_ids] def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: return self.embed_tokens(input_ids) @@ -230,19 +192,6 @@ class DeepseekV32MultiTokenPredictor(nn.Module): # second RMSNorm. return self.logits_processor(mtp_layer.shared_head.head, hidden_states) - def get_top_tokens( - self, - hidden_states: torch.Tensor, - spec_step_idx: int = 0, - ) -> torch.Tensor: - # Greedy draft token ids via vocab-parallel local argmax: skips the - # full-vocab logits all-gather (use_local_argmax_reduction). - current_step_idx = spec_step_idx % self.num_mtp_layers - mtp_layer = self.layers[str(self.mtp_start_layer_idx + current_step_idx)] - return self.logits_processor.get_top_tokens( - mtp_layer.shared_head.head, hidden_states - ) - class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts): def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): @@ -291,25 +240,6 @@ class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts): ) -> torch.Tensor | None: return self.model.compute_logits(hidden_states, spec_step_idx) - def get_top_tokens( - self, - hidden_states: torch.Tensor, - spec_step_idx: int | None = None, - ) -> torch.Tensor: - # The V2 speculator calls this without spec_step_idx; that is only - # correct while a single MTP layer is cycled for every draft step - # (num_nextn_predict_layers == 1, as in GLM-5.2/DSV3.2). Guard the - # assumption so a future multi-layer MTP fails loudly instead of - # silently using step 0's head for every step. - if spec_step_idx is None: - assert self.model.num_mtp_layers == 1, ( - "get_top_tokens called without spec_step_idx on a " - "multi-layer MTP; thread the draft step index through " - "the speculator." - ) - spec_step_idx = 0 - return self.model.get_top_tokens(hidden_states, spec_step_idx) - def _rewrite_spec_layer_name(self, spec_layer: int, name: str) -> str: spec_layer_weight_names = [ "embed_tokens", diff --git a/vllm/models/deepseek_v32/nvidia/ops/__init__.py b/vllm/models/deepseek_v32/nvidia/ops/__init__.py deleted file mode 100644 index 208f01a7cb..0000000000 --- a/vllm/models/deepseek_v32/nvidia/ops/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project diff --git a/vllm/models/deepseek_v32/nvidia/ops/fused_q_cutedsl.py b/vllm/models/deepseek_v32/nvidia/ops/fused_q_cutedsl.py deleted file mode 100644 index b98147a34a..0000000000 --- a/vllm/models/deepseek_v32/nvidia/ops/fused_q_cutedsl.py +++ /dev/null @@ -1,532 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from functools import cache - -import cutlass -import cutlass.cute as cute -import torch -from cuda.bindings.driver import CUstream -from cutlass import BFloat16, Float8E4M3FN, Float32, Int64, Uint8, Uint16, Uint32 - -from vllm.cute_utils import TORCH_TO_CUTE_DTYPE, cvt - - -def _make_fake_tensor(dtype, shape, divisibility): - stride = tuple( - 1 if i == len(shape) - 1 else cute.sym_int64(divisibility=divisibility) - for i in range(len(shape)) - ) - return cute.runtime.make_fake_tensor( - dtype, - shape, - stride, - assumed_align=divisibility * dtype.width // 8, - ) - - -def is_fused_q_cutedsl_supported( - q_pe: torch.Tensor, - index_q: torch.Tensor | None, - ql_nope: torch.Tensor, - *, - has_indexer: bool, - quantize_mqa: bool, -) -> bool: - if not ( - quantize_mqa - and q_pe.dtype == ql_nope.dtype == torch.bfloat16 - and q_pe.shape[-1] == 64 - and ql_nope.shape[-1] == 512 - ): - return False - return not has_indexer or ( - index_q is not None - and index_q.dtype == torch.bfloat16 - and index_q.shape[1] % 16 == 0 - and index_q.shape[-1] == 128 - ) - - -def fused_q_cutedsl( - positions: torch.Tensor, - q_pe: torch.Tensor, - rope_cache: torch.Tensor, - ql_nope: torch.Tensor, - q_scale: torch.Tensor, - mqa_output: torch.Tensor, - idx_q: torch.Tensor, - idx_rope_cache: torch.Tensor, - idx_weights: torch.Tensor, - idx_weights_softmax_scale: float, - idx_weights_head_scale: float, - idx_q_fp8: torch.Tensor, - idx_weights_out: torch.Tensor, - has_indexer: bool = True, - index_rope_interleave: bool = True, -) -> None: - _, num_heads, rope_dim = q_pe.shape - _, _, nope_dim = ql_nope.shape - _, num_idx_heads, idx_dim = idx_q.shape - - if has_indexer: - idx_rope_type = TORCH_TO_CUTE_DTYPE[idx_rope_cache.dtype] - idx_weights_type = TORCH_TO_CUTE_DTYPE[idx_weights.dtype] - else: - idx_dim = num_idx_heads = 0 - idx_q = idx_rope_cache = idx_q_fp8 = None - idx_weights = idx_weights_out = None - idx_rope_type = idx_weights_type = None - - rope_type = TORCH_TO_CUTE_DTYPE[rope_cache.dtype] - compiled = FusedQKernel.compile( - rope_dim, - nope_dim, - num_heads, - rope_type, - idx_dim, - num_idx_heads, - idx_rope_type, - idx_weights_type, - index_rope_interleave, - ) - compiled( - positions, - q_pe, - rope_cache, - ql_nope, - q_scale.view(1), - mqa_output, - idx_q, - idx_rope_cache, - idx_weights, - idx_q_fp8, - idx_weights_out, - float(idx_weights_softmax_scale * idx_weights_head_scale), - ) - - -class FusedQKernel: - def __init__( - self, - rope_dim: int, - nope_dim: int, - num_heads: int, - idx_dim: int, - num_idx_heads: int, - index_rope_interleave: bool, - ) -> None: - assert rope_dim == 64 - assert nope_dim == 512 - assert idx_dim in (128, 0) - - self.rope_dim = rope_dim - self.nope_dim = nope_dim - self.num_heads = num_heads - self.idx_dim = idx_dim - self.num_idx_heads = num_idx_heads - self.index_rope_interleave = index_rope_interleave - - # mqa: rope_dim=64, nope_dim=512, num_heads=64/TP - # indexer: rope_dim=64, nope_dim=64, num_heads=32 - - self.num_warps = 4 - assert num_heads % self.num_warps == 0 - assert num_idx_heads % (4 * self.num_warps) == 0 - self.num_ctas_per_tok = num_heads // self.num_warps - self.num_ctas_per_idx_tok = num_idx_heads // (4 * self.num_warps) - - @cute.jit - def __call__( - self, - positions: cute.Tensor, - q_pe: cute.Tensor, - rope_cache: cute.Tensor, - ql_nope: cute.Tensor, - q_scale: cute.Tensor, - mqa_output: cute.Tensor, - idx_q: cute.Tensor, - idx_rope_cache: cute.Tensor, - idx_weights: cute.Tensor, - idx_q_fp8: cute.Tensor, - idx_weights_out: cute.Tensor, - weight_scale: Float32, - stream: CUstream, - ): - num_tokens = positions.shape[0] - if cutlass.const_expr(self.idx_dim == 0): - grid = (num_tokens, self.num_ctas_per_tok, 1) - else: - num_mqa_ctas = num_tokens * self.num_ctas_per_tok - num_idx_ctas = num_tokens * self.num_ctas_per_idx_tok - grid = (num_mqa_ctas + num_idx_ctas, 1, 1) - - self.kernel( - positions, - q_pe, - rope_cache, - ql_nope, - q_scale, - mqa_output, - idx_q, - idx_rope_cache, - idx_weights, - idx_q_fp8, - idx_weights_out, - weight_scale, - ).launch( - grid=grid, - block=(self.num_warps * 32, 1, 1), - stream=stream, - use_pdl=True, - ) - - @cute.kernel - def kernel( - self, - positions: cute.Tensor, - q_pe: cute.Tensor, - q_pe_rope_cache: cute.Tensor, - ql_nope: cute.Tensor, - q_scale: cute.Tensor, - mqa_output: cute.Tensor, - idx_q: cute.Tensor, - idx_q_rope_cache: cute.Tensor, - idx_weights: cute.Tensor, - idx_q_fp8: cute.Tensor, - idx_weights_out: cute.Tensor, - weight_scale: Float32, - ): - if cutlass.const_expr(self.idx_dim == 0): - token_id, group_id, _ = cute.arch.block_idx() - self.mqa( - positions, - q_pe, - q_pe_rope_cache, - ql_nope, - q_scale, - mqa_output, - token_id, - group_id, - ) - else: - # CTA-specialization - bid, _, _ = cute.arch.block_idx() - num_mqa_ctas = positions.shape[0] * self.num_ctas_per_tok - if bid < num_mqa_ctas: - self.mqa( - positions, - q_pe, - q_pe_rope_cache, - ql_nope, - q_scale, - mqa_output, - bid // self.num_ctas_per_tok, - bid % self.num_ctas_per_tok, - ) - else: - bid -= num_mqa_ctas - self.indexer( - positions, - idx_q, - idx_q_rope_cache, - idx_weights, - idx_q_fp8, - idx_weights_out, - weight_scale, - bid // self.num_ctas_per_idx_tok, - bid % self.num_ctas_per_idx_tok, - ) - - @cute.jit - def mqa( - self, - positions: cute.Tensor, - q_pe: cute.Tensor, - q_pe_rope_cache: cute.Tensor, - ql_nope: cute.Tensor, - q_scale: cute.Tensor, - mqa_output: cute.Tensor, - token_id, - group_id, - ): - tid, _, _ = cute.arch.thread_idx() - warp_id = cute.arch.make_warp_uniform(tid // 32) - lane_id = tid % 32 - head_id = group_id * self.num_warps + warp_id - - cute.arch.griddepcontrol_wait() - - pos = positions[token_id] - inv_scale = 1.0 / q_scale[0] - - cp_op = cute.nvgpu.CopyUniversalOp() - cp_32B = cute.make_copy_atom(cp_op, Uint32, num_bits_per_copy=256) - cp_16B = cute.make_copy_atom(cp_op, Uint32, num_bits_per_copy=128) - cp_4B = cute.make_copy_atom(cp_op, Uint32, num_bits_per_copy=32) - cp_2B = cute.make_copy_atom(cp_op, Uint8, num_bits_per_copy=16) - - ##### issue all loads asap ##### - rQ_nope_bf16 = cute.make_rmem_tensor(16, BFloat16) - rQ_rope_bf16 = cute.make_rmem_tensor(2, BFloat16) - - src_ql_nope = cute.local_tile( - ql_nope[token_id, head_id, None], (16,), (lane_id,) - ) - src_q_rope = cute.local_tile(q_pe[token_id, head_id, None], (2,), (lane_id,)) - cute.copy(cp_32B, src_ql_nope, rQ_nope_bf16) - cute.copy(cp_4B, src_q_rope, rQ_rope_bf16) - - rCos_raw = q_pe_rope_cache[pos, 0 + lane_id] - rSin_raw = q_pe_rope_cache[pos, 32 + lane_id] - - ##### process NoPE ##### - rQ_nope_f32 = cvt.bf16x2_to_fp32x2(rQ_nope_bf16).load() * inv_scale - rQ_nope_f8 = cute.make_rmem_tensor(16, Float8E4M3FN) - rQ_nope_f8.store(rQ_nope_f32.to(Float8E4M3FN)) - dst_Q_nope = cute.local_tile( - mqa_output[token_id, head_id, None], (16,), (lane_id,) - ) - cute.copy(cp_16B, rQ_nope_f8, dst_Q_nope) - - ##### process RoPE ###### - rQ_rope_f32 = cvt.bf16x2_to_fp32x2(rQ_rope_bf16) - rCos = rCos_raw.to(Float32) - rSin = rSin_raw.to(Float32) - r0 = (rQ_rope_f32[0] * rCos - rQ_rope_f32[1] * rSin) * inv_scale - r1 = (rQ_rope_f32[1] * rCos + rQ_rope_f32[0] * rSin) * inv_scale - - cute.arch.griddepcontrol_launch_dependents() - - # TensorSSA fp32->fp8 cvt has a bug. rely on direct PTX - rQ_rope_f8 = cute.make_rmem_tensor(2, Float8E4M3FN) - cute.recast_tensor(rQ_rope_f8, Uint16)[0] = cvt.fp32x2_to_fp8x2(r0, r1) - dst_Q_rope = cute.local_tile( - mqa_output[token_id, head_id, None], (2,), (256 + lane_id,) - ) - cute.copy(cp_2B, rQ_rope_f8, dst_Q_rope) - - @cute.jit - def indexer( - self, - positions: cute.Tensor, - idx_q: cute.Tensor, - idx_q_rope_cache: cute.Tensor, - idx_weights: cute.Tensor, - idx_q_fp8: cute.Tensor, - idx_weights_out: cute.Tensor, - weight_scale: Float32, - token_id, - group_id, - ): - tid, _, _ = cute.arch.thread_idx() - subwarp_id = tid // 8 - sublane_id = tid % 8 - head_id = group_id * (4 * self.num_warps) + subwarp_id - cute.arch.griddepcontrol_wait() - - pos = positions[token_id] - - cp_op = cute.nvgpu.CopyUniversalOp() - cp_16B = cute.make_copy_atom(cp_op, Uint32, num_bits_per_copy=128) - cp_8B = cute.make_copy_atom(cp_op, Uint32, num_bits_per_copy=64) - cp_4B = cute.make_copy_atom(cp_op, Uint32, num_bits_per_copy=32) - - ##### issue all loads first ##### - rQ_rope_bf16 = cute.make_rmem_tensor(8, BFloat16) - if cutlass.const_expr(self.index_rope_interleave): - src_idx_q_rope = cute.local_tile( - idx_q[token_id, head_id, None], (8,), (sublane_id,) - ) - cute.copy(cp_16B, src_idx_q_rope, rQ_rope_bf16) - else: - src_idx_q_rope = cute.zipped_divide( - idx_q[token_id, head_id, None], (4,) - ) # (4,32) - cute.copy( - cp_8B, - src_idx_q_rope[None, 0 + sublane_id], - cute.local_tile(rQ_rope_bf16, (4,), (0,)), - ) - cute.copy( - cp_8B, - src_idx_q_rope[None, 8 + sublane_id], - cute.local_tile(rQ_rope_bf16, (4,), (1,)), - ) - - rQ_nope_bf16 = cute.make_rmem_tensor(8, BFloat16) - src_idx_q_nope = cute.local_tile( - idx_q[token_id, head_id, None], (8,), (8 + sublane_id,) - ) - cute.copy(cp_16B, src_idx_q_nope, rQ_nope_bf16) - - rCos_raw = cute.make_rmem_tensor(4, idx_q_rope_cache.element_type) - rSin_raw = cute.make_rmem_tensor(4, idx_q_rope_cache.element_type) - rope_cache_view = cute.zipped_divide(idx_q_rope_cache[pos, None], (4,)) - - if cutlass.const_expr(idx_q_rope_cache.element_type == Float32): - cute.copy(cp_16B, rope_cache_view[None, sublane_id], rCos_raw) - cute.copy(cp_16B, rope_cache_view[None, 8 + sublane_id], rSin_raw) - elif cutlass.const_expr(idx_q_rope_cache.element_type == BFloat16): - cute.copy(cp_8B, rope_cache_view[None, sublane_id], rCos_raw) - cute.copy(cp_8B, rope_cache_view[None, 8 + sublane_id], rSin_raw) - - # unpack to FP32 - rQ_rope_f32 = cvt.bf16x2_to_fp32x2(rQ_rope_bf16) - rQ_nope_f32 = cvt.bf16x2_to_fp32x2(rQ_nope_bf16) - if cutlass.const_expr(idx_q_rope_cache.element_type == Float32): - rCos = rCos_raw - rSin = rSin_raw - elif cutlass.const_expr(idx_q_rope_cache.element_type == BFloat16): - rCos = cvt.bf16x2_to_fp32x2(rCos_raw) - rSin = cvt.bf16x2_to_fp32x2(rSin_raw) - - # apply rope - for i in cutlass.range_constexpr(4): - if cutlass.const_expr(self.index_rope_interleave): - r0 = rQ_rope_f32[i * 2 + 0] * rCos[i] - rQ_rope_f32[i * 2 + 1] * rSin[i] - r1 = rQ_rope_f32[i * 2 + 1] * rCos[i] + rQ_rope_f32[i * 2 + 0] * rSin[i] - rQ_rope_f32[i * 2 + 0] = r0 - rQ_rope_f32[i * 2 + 1] = r1 - else: - r0 = rQ_rope_f32[0 + i] * rCos[i] - rQ_rope_f32[4 + i] * rSin[i] - r1 = rQ_rope_f32[4 + i] * rCos[i] + rQ_rope_f32[0 + i] * rSin[i] - rQ_rope_f32[0 + i] = r0 - rQ_rope_f32[4 + i] = r1 - - # amax - amax = Float32(1e-4) - for i in cutlass.range_constexpr(8): - amax = cute.arch.fmax(amax, cute.math.absf(rQ_rope_f32[i])) - amax = cute.arch.fmax(amax, cute.math.absf(rQ_nope_f32[i])) - - # warp reduction among 8 lanes - for i in cutlass.range_constexpr(3): - other = cute.arch.shuffle_sync_bfly(amax, 1 << i) - amax = cute.arch.fmax(amax, other) - - # compute scale from amax - # exp2(ceil(log2(scale))) via bit manipulation - scale = amax * (1.0 / 448.0) - bits = scale.bitcast(Uint32) - exp_bits = (bits + Uint32(0x007FFFFF)) & Uint32(0x7F800000) - scale = exp_bits.bitcast(Float32) - inv_scale = (Uint32(0x7F000000) - exp_bits).bitcast(Float32) - - for i in cutlass.range_constexpr(8): - rQ_nope_f32[i] *= inv_scale - rQ_rope_f32[i] *= inv_scale - - cute.arch.griddepcontrol_launch_dependents() - - # quantize and store - rQ_nope_f8 = cute.make_rmem_tensor(8, Float8E4M3FN) - rQ_nope_f8.store(rQ_nope_f32.load().to(Float8E4M3FN)) - dst_idx_q_nope = cute.local_tile( - idx_q_fp8[token_id, head_id, None], (8,), (8 + sublane_id,) - ) - cute.copy(cp_8B, rQ_nope_f8, dst_idx_q_nope) - - rQ_rope_f8 = cute.make_rmem_tensor(8, Float8E4M3FN) - rQ_rope_f8.store(rQ_rope_f32.load().to(Float8E4M3FN)) - if cutlass.const_expr(self.index_rope_interleave): - dst_idx_q_rope = cute.local_tile( - idx_q_fp8[token_id, head_id, None], (8,), (sublane_id,) - ) - cute.copy(cp_8B, rQ_rope_f8, dst_idx_q_rope) - else: - dst_idx_q_rope = cute.zipped_divide( - idx_q_fp8[token_id, head_id, None], (4,) - ) # (4,32) - cute.copy( - cp_4B, - cute.local_tile(rQ_rope_f8, (4,), (0,)), - dst_idx_q_rope[None, 0 + sublane_id], - ) - cute.copy( - cp_4B, - cute.local_tile(rQ_rope_f8, (4,), (1,)), - dst_idx_q_rope[None, 8 + sublane_id], - ) - - # scale indexer weights - if sublane_id == 0: - w = idx_weights[token_id, head_id].to(Float32) - idx_weights_out[token_id, head_id] = w * scale * weight_scale - - @cache - @staticmethod - def compile( - rope_dim: int, - nope_dim: int, - num_heads: int, - rope_type: type[cutlass.Numeric], - idx_dim: int, - num_idx_heads: int, - idx_rope_type: type[cutlass.Numeric] | None, - idx_weights_type: type[cutlass.Numeric] | None, - index_rope_interleave: bool, - ): - num_tokens = cute.sym_int() - max_pos = cute.sym_int() - - positions = _make_fake_tensor(Int64, (num_tokens,), divisibility=1) - q_pe = _make_fake_tensor( - BFloat16, (num_tokens, num_heads, rope_dim), divisibility=16 - ) - rope_cache = _make_fake_tensor(rope_type, (max_pos, rope_dim), divisibility=8) - ql_nope = _make_fake_tensor( - BFloat16, (num_tokens, num_heads, nope_dim), divisibility=16 - ) - q_scale = _make_fake_tensor(Float32, (1,), divisibility=4) - mqa_output = _make_fake_tensor( - Float8E4M3FN, - (num_tokens, num_heads, nope_dim + rope_dim), - divisibility=16, - ) - - if idx_rope_type is not None: - index_q = _make_fake_tensor( - BFloat16, (num_tokens, num_idx_heads, idx_dim), divisibility=16 - ) - index_rope_cache = _make_fake_tensor( - idx_rope_type, (max_pos, rope_dim), divisibility=8 - ) - index_weights = _make_fake_tensor( - idx_weights_type, (num_tokens, num_idx_heads), divisibility=8 - ) - index_q_fp8 = _make_fake_tensor( - Float8E4M3FN, (num_tokens, num_idx_heads, idx_dim), divisibility=16 - ) - index_weights_out = _make_fake_tensor( - Float32, (num_tokens, num_idx_heads), divisibility=4 - ) - else: - index_q = index_rope_cache = index_q_fp8 = None - index_weights = index_weights_out = None - - kernel = FusedQKernel( - rope_dim, - nope_dim, - num_heads, - idx_dim, - num_idx_heads, - index_rope_interleave, - ) - stream = cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True) - return cute.compile( - kernel, - positions, - q_pe, - rope_cache, - ql_nope, - q_scale, - mqa_output, - index_q, - index_rope_cache, - index_weights, - index_q_fp8, - index_weights_out, - Float32(0.0), - stream, - options="--enable-tvm-ffi", - ) diff --git a/vllm/v1/attention/backends/mla/flashinfer_mla_sparse.py b/vllm/v1/attention/backends/mla/flashinfer_mla_sparse.py index 36df138a4e..a6b9f4abb0 100644 --- a/vllm/v1/attention/backends/mla/flashinfer_mla_sparse.py +++ b/vllm/v1/attention/backends/mla/flashinfer_mla_sparse.py @@ -28,7 +28,6 @@ from vllm.v1.attention.backend import ( MultipleOf, ) from vllm.v1.attention.backends.mla.sparse_utils import ( - phys_shadow, triton_convert_req_index_to_global_index, triton_filter_and_convert_dcp_index, ) @@ -405,36 +404,6 @@ class FlashInferMLASparseImpl(SparseMLACommonImpl[FlashInferMLASparseMetadata]): NUM_TOPK_TOKENS=topk_indices.shape[1], return_valid_counts=True, ) - elif (shadow := phys_shadow(self.topk_indices_buffer)) is not None: - # skip_topk layers reuse the previous fresh layer's PHYSICAL - # indices: the logical top-k indices are shared (one buffer for - # the whole forward) and block_table/req_id are per-step constant, - # so re-converting them per layer produces identical output. This - # covers both the backbone (index_topk_freq > 1) and MTP draft - # steps 1+ (index_share_for_mtp_iteration sets skip_topk; the - # draft compact re-arranges the shadow together with the logical - # buffer). The fresh/skip split is fixed per captured graph, so - # the branch is cudagraph-safe. - phys_buf, seq_buf = shadow - wrote_fresh = getattr(layer, "indexer", None) is not None and not getattr( - layer, "skip_topk", False - ) - if wrote_fresh: - topk_indices_physical, seq_lens = ( - triton_convert_req_index_to_global_index( - attn_metadata.req_id_per_token[:num_actual_toks], - attn_metadata.block_table, - topk_indices, - BLOCK_SIZE=attn_metadata.block_size, - NUM_TOPK_TOKENS=topk_indices.shape[1], - return_valid_counts=True, - out=phys_buf[:num_actual_toks], - valid_counts_out=seq_buf[:num_actual_toks], - ) - ) - else: - topk_indices_physical = phys_buf[:num_actual_toks] - seq_lens = seq_buf[:num_actual_toks] else: topk_indices_physical, seq_lens = triton_convert_req_index_to_global_index( attn_metadata.req_id_per_token[:num_actual_toks], diff --git a/vllm/v1/attention/backends/mla/sparse_utils.py b/vllm/v1/attention/backends/mla/sparse_utils.py index 3cc141e292..681ab7cfd9 100644 --- a/vllm/v1/attention/backends/mla/sparse_utils.py +++ b/vllm/v1/attention/backends/mla/sparse_utils.py @@ -128,8 +128,6 @@ def triton_convert_req_index_to_global_index( prefill_workspace_request_ids: torch.Tensor | None = None, prefill_workspace_starts: torch.Tensor | None = None, return_valid_counts: bool = False, - out: torch.Tensor | None = None, - valid_counts_out: torch.Tensor | None = None, ) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]: """ out[token_id, indice_id] = @@ -180,20 +178,14 @@ def triton_convert_req_index_to_global_index( req_id_c = req_id.contiguous() block_table_c = block_table.contiguous() token_indices_c = token_indices.contiguous() - # `out`/`valid_counts_out` allow writing into a stable pre-allocated buffer - # (used by the shared-physical-index cache to stay cudagraph-safe). - out = torch.empty_like(token_indices_c) if out is None else out + out = torch.empty_like(token_indices_c) # Allocate valid count buffer if needed (must be zero-initialized for atomics) valid_counts: torch.Tensor | None = None if return_valid_counts: - if valid_counts_out is None: - valid_counts = torch.zeros( - num_tokens, dtype=torch.int32, device=token_indices.device - ) - else: - valid_counts = valid_counts_out - valid_counts.zero_() + valid_counts = torch.zeros( + num_tokens, dtype=torch.int32, device=token_indices.device + ) # Strides in elements bt_stride0, bt_stride1 = block_table_c.stride() @@ -349,47 +341,3 @@ def triton_filter_and_convert_dcp_index( assert valid_counts is not None return out, valid_counts return out - - -# --- Physical-index shadow (DSA index_topk_freq > 1) ------------------------- -# GLM-5.2 uses index_topk_freq=4: only ~1/4 of layers write a fresh top-k into -# the single shared topk_indices_buffer; the rest reuse it. Since block_table -# and req_id_per_token are constant within a decode step, the physical indices -# (block_table lookup of the logical top-k) are identical across a freq-group. -# Fresh layers convert once into a STABLE shadow of the logical buffer and -# skip layers read it -- eliminating ~3/4 of the per-layer convert kernels. -# -# Invariant: shadow[j] == convert(logical[j]) row-wise. Whoever re-arranges -# the logical buffer (e.g. the MTP draft compact between the multi-token -# step-0 layout and the single-token steps-1+ layout) must apply the same -# gather to the shadow. -# -# Shadows are registered ONLY at buffer creation time (model init, never -# inside cudagraph capture) via register_phys_shadow; every other access is -# the read-only phys_shadow lookup, so addresses are stable across graph -# replays and no allocation can happen during capture. -import weakref as _weakref # noqa: E402 - -_PHYS_SHADOWS: dict[int, tuple[torch.Tensor, torch.Tensor]] = {} - - -def register_phys_shadow(topk_buf: torch.Tensor) -> None: - """Allocate the (physical-index, valid-count) shadow for a logical top-k - buffer. Call once where the buffer is created. The finalizer drops the - entry with the owning buffer, so a recycled id can never alias a stale - shadow.""" - key = id(topk_buf) - if key not in _PHYS_SHADOWS: - _PHYS_SHADOWS[key] = ( - torch.empty_like(topk_buf), - torch.empty(topk_buf.shape[0], dtype=torch.int32, device=topk_buf.device), - ) - _weakref.finalize(topk_buf, _PHYS_SHADOWS.pop, key, None) - - -def phys_shadow( - topk_buf: torch.Tensor, -) -> tuple[torch.Tensor, torch.Tensor] | None: - """The registered shadow for this buffer, or None if it was never - registered. Never allocates; capture-safe.""" - return _PHYS_SHADOWS.get(id(topk_buf)) diff --git a/vllm/v1/spec_decode/llm_base_proposer.py b/vllm/v1/spec_decode/llm_base_proposer.py index bb1c59a8c2..6ee442af51 100644 --- a/vllm/v1/spec_decode/llm_base_proposer.py +++ b/vllm/v1/spec_decode/llm_base_proposer.py @@ -1009,9 +1009,8 @@ class SpecDecodeBaseProposer: # DeepSeek-family MTP (deepseek_mtp.py) recycles the post-final- # norm hidden, so its forward returns (logit_hidden, # recycle_hidden). Other MTP families return a single tensor. - return bool( - {"DeepSeekMTPModel", "DeepseekV32MTPModel"} - & set(self.draft_model_config.hf_config.architectures or []) + return "DeepSeekMTPModel" in ( + self.draft_model_config.hf_config.architectures or [] ) return self.method not in ("mtp", "draft_model", "dflash") diff --git a/vllm/v1/worker/gpu/spec_decode/autoregressive/speculator.py b/vllm/v1/worker/gpu/spec_decode/autoregressive/speculator.py index 9a14d132c5..b890574971 100644 --- a/vllm/v1/worker/gpu/spec_decode/autoregressive/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/autoregressive/speculator.py @@ -48,18 +48,6 @@ class AutoRegressiveSpeculator(DraftModelSpeculator): self.prefill_cudagraph_manager: SpeculatorCudaGraphManager | None = None self.decode_cudagraph_manager: SpeculatorCudaGraphManager | None = None - # Lifecycle hooks for model-specific optimizations. Subclasses override - # the ones they need. These fire in both `capture` and `propose` so that - # any state they toggle (e.g. attention flags baked into a CUDA graph) is - # identical at capture time and replay time. - def on_prefill_begin(self, num_reqs: int, num_tokens: int) -> None: ... - - def on_prefill_end(self, num_reqs: int, num_tokens: int) -> None: ... - - def on_multi_step_decode_begin(self, num_reqs: int) -> None: ... - - def on_multi_step_decode_end(self, num_reqs: int) -> None: ... - @property def advance_draft_positions(self) -> bool: """ @@ -110,9 +98,6 @@ class AutoRegressiveSpeculator(DraftModelSpeculator): assert self.prefill_cudagraph_manager is not None if self.prefill_cudagraph_manager.use_breakable_cg: self.prefill_cudagraph_manager.init_breakable_cg_runner(self.model) - - self.on_prefill_begin(self.max_num_reqs, self.max_num_tokens) - self.prefill_cudagraph_manager.capture( self._prefill, self.model_state, @@ -123,13 +108,9 @@ class AutoRegressiveSpeculator(DraftModelSpeculator): progress_bar_desc="Capturing prefill CUDA graphs", ) - self.on_prefill_end(self.max_num_reqs, self.max_num_tokens) - if self.num_speculative_steps == 1: return - self.on_multi_step_decode_begin(self.max_num_reqs) - # Capture the decode draft generation routine (model forward + # sample + update_draft_inputs) for a single # step. @@ -144,8 +125,6 @@ class AutoRegressiveSpeculator(DraftModelSpeculator): progress_bar_desc="Capturing decode CUDA graphs", ) - self.on_multi_step_decode_end(self.max_num_reqs) - @torch.inference_mode() def propose( self, @@ -174,8 +153,7 @@ class AutoRegressiveSpeculator(DraftModelSpeculator): mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None, is_profile: bool = False, ) -> torch.Tensor: - num_tokens = input_batch.num_tokens - num_tokens_padded = input_batch.num_tokens_after_padding + num_tokens = input_batch.num_tokens_after_padding num_reqs = input_batch.num_reqs max_query_len = input_batch.num_scheduled_tokens.max() max_seq_len = input_batch.seq_lens_cpu_upper_bound[:num_reqs].max().item() @@ -196,7 +174,7 @@ class AutoRegressiveSpeculator(DraftModelSpeculator): ) else: hidden_states = last_hidden_states - self.hidden_states[:num_tokens_padded].copy_(hidden_states) + self.hidden_states[:num_tokens].copy_(hidden_states) self._copy_request_inputs( num_reqs, @@ -224,22 +202,20 @@ class AutoRegressiveSpeculator(DraftModelSpeculator): num_reqs, # Use the actual number of tokens without padding added by # the target model during FULL cudagraph. - num_tokens, + input_batch.num_tokens, max_query_len, ) prefill_batch_desc, num_tokens_across_dp = dispatch_cg_and_sync_dp( self.prefill_cudagraph_manager, num_reqs, - num_tokens_padded, + num_tokens, uniform_token_count, dp_size=self.dp_size, dp_rank=self.dp_rank, need_eager=is_profile, ) - self._prepare_eplb_forward(num_tokens) - - self.on_prefill_begin(num_reqs, num_tokens) + self._prepare_eplb_forward(input_batch.num_tokens) if prefill_batch_desc.cg_mode == CUDAGraphMode.FULL: # Replay the full graph for draft prefill. @@ -259,8 +235,6 @@ class AutoRegressiveSpeculator(DraftModelSpeculator): mm_inputs=mm_inputs, ) - self.on_prefill_end(num_reqs, num_tokens) - if self.num_speculative_steps == 1: # Early exit. return self.draft_tokens[:num_reqs, :1] @@ -288,8 +262,6 @@ class AutoRegressiveSpeculator(DraftModelSpeculator): need_eager=is_profile, ) - self.on_multi_step_decode_begin(num_reqs) - # Generate the remaining num_speculative_steps - 1 draft tokens. self._multi_step_decode( num_reqs, @@ -299,8 +271,6 @@ class AutoRegressiveSpeculator(DraftModelSpeculator): input_batch.seq_lens_cpu_upper_bound, ) - self.on_multi_step_decode_end(num_reqs) - return self.draft_tokens[:num_reqs] @torch.inference_mode() diff --git a/vllm/v1/worker/gpu/spec_decode/mtp/speculator.py b/vllm/v1/worker/gpu/spec_decode/mtp/speculator.py index 5f1cebc519..4b9354f23e 100644 --- a/vllm/v1/worker/gpu/spec_decode/mtp/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/mtp/speculator.py @@ -1,10 +1,8 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -import torch import torch.nn as nn -from vllm.config import VllmConfig from vllm.v1.worker.gpu.spec_decode.autoregressive.speculator import ( AutoRegressiveSpeculator, ) @@ -12,46 +10,9 @@ from vllm.v1.worker.gpu.spec_decode.eagle.utils import load_eagle_model class MTPSpeculator(AutoRegressiveSpeculator): - def __init__(self, vllm_config: VllmConfig, device: torch.device): - super().__init__(vllm_config, device) - - spec_config = vllm_config.speculative_config - draft_hf_config = ( - spec_config.draft_model_config.hf_config - if spec_config is not None - else None - ) - # Detect index_share_for_mtp_iteration. When True, the proposer - # toggles skip_topk so step 0 computes MTP's own indices and - # steps 1+ reuse them. - self.share_mtp_topk_indices = getattr( - draft_hf_config, "index_share_for_mtp_iteration", False - ) - def load_draft_model( self, target_model: nn.Module, target_attn_layer_names: set[str], ) -> nn.Module: - draft_model = load_eagle_model(target_model, self.vllm_config) - self.share_mtp_topk_indices = self.share_mtp_topk_indices and hasattr( - draft_model.model, "set_skip_topk" - ) - return draft_model - - def on_prefill_end(self, num_reqs: int, num_tokens: int) -> None: - # Step 0 (prefill) wrote topk indices for every query token in the - # multi-token batch. Compact them down to each request's last token so - # steps 1+ can reuse them from the shared buffer. - if self.share_mtp_topk_indices and self.num_speculative_steps > 1: - self.model.model.compact_topk_indices(self.last_token_indices[:num_reqs]) - - def on_multi_step_decode_begin(self, num_reqs: int) -> None: - # Switch to reuse mode so draft steps 1+ skip the indexer op and read - # the indices that step 0 wrote into the shared buffer. - if self.share_mtp_topk_indices: - self.model.model.set_skip_topk(True) - - def on_multi_step_decode_end(self, num_reqs: int) -> None: - if self.share_mtp_topk_indices: - self.model.model.set_skip_topk(False) + return load_eagle_model(target_model, self.vllm_config)