Revert "[Perf][GLM-5.2] Blackwell decode optimizations" (#49768)

This commit is contained in:
Woosuk Kwon
2026-07-24 16:04:59 -07:00
committed by GitHub
parent 33c4f3551c
commit 213f681f81
29 changed files with 138 additions and 2139 deletions
-17
View File
@@ -732,23 +732,6 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP")
"(requires SM90+ and CUDA >= 12.0).")
endif()
# BF16 skinny GEMM (M<=32; weight-bandwidth-bound decode shapes).
# Requires SM90+.
cuda_archs_sm90plus(BF16_SKINNY_GEMM_ARCHS "${CUDA_ARCHS}")
if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 12.0 AND BF16_SKINNY_GEMM_ARCHS)
set(BF16_SKINNY_GEMM_SRCS
"csrc/libtorch_stable/bf16_skinny_gemm_entry.cu"
"csrc/libtorch_stable/bf16_skinny_gemm.cu")
set_gencode_flags_for_srcs(
SRCS "${BF16_SKINNY_GEMM_SRCS}"
CUDA_ARCHS "${BF16_SKINNY_GEMM_ARCHS}")
list(APPEND VLLM_STABLE_EXT_SRC "${BF16_SKINNY_GEMM_SRCS}")
message(STATUS "Building bf16_skinny_gemm for archs: ${BF16_SKINNY_GEMM_ARCHS}")
else()
message(STATUS "Not building bf16_skinny_gemm as no compatible archs found "
"(requires SM90+ and CUDA >= 12.0).")
endif()
# Only build AllSpark kernels if we are building for at least some compatible archs.
cuda_archs_loose_intersection(ALLSPARK_ARCHS "8.0;8.6;8.7;8.9" "${CUDA_ARCHS}")
if (ALLSPARK_ARCHS)
-262
View File
@@ -1,262 +0,0 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
//
// Skinny GEMM: activation(bf16) x weight(bf16)^T -> bf16, for decode-time
// M <= 32 with a large reduction dim. Replaces cuBLAS splitK (GEMM +
// splitKreduce) with a single block-per-output-column kernel; these shapes
// are weight-bandwidth-bound, so one coalesced pass over the weight at
// fp32 accumulation is optimal. Adapted from fp32_router_gemm.cu.
//
// First user: the DeepSeek-V32/GLM-5.2 MTP eh_proj (K=2*hidden=12288,
// N=hidden/TP), whose cuBLAS splitK pick costs ~34.6us vs the ~19us
// bandwidth floor per replicated read (and ~4us once column-parallel).
#include <cuda_bf16.h>
#include <cuda_runtime.h>
// ---------------------------------------------------------------------------
// Load helpers (8 x bf16 = one uint4 load, converted to fp32)
// ---------------------------------------------------------------------------
namespace skinny {
constexpr int VPT = 8; // bf16 values per thread per load
__device__ __forceinline__ void load_bf16x8(__nv_bfloat16 const* ptr,
float* dst) {
uint4 v = *reinterpret_cast<uint4 const*>(ptr);
__nv_bfloat16 const* p = reinterpret_cast<__nv_bfloat16 const*>(&v);
#pragma unroll
for (int i = 0; i < VPT; i++) dst[i] = __bfloat162float(p[i]);
}
// Streaming variant for the weight: each row is read exactly once across the
// whole grid, so bypass L2 residency (evict-first). Measured -1.4us at M=1.
__device__ __forceinline__ void load_bf16x8_cs(__nv_bfloat16 const* ptr,
float* dst) {
uint4 v = __ldcs(reinterpret_cast<uint4 const*>(ptr));
__nv_bfloat16 const* p = reinterpret_cast<__nv_bfloat16 const*>(&v);
#pragma unroll
for (int i = 0; i < VPT; i++) dst[i] = __bfloat162float(p[i]);
}
// ---------------------------------------------------------------------------
// Kernel: each block computes kNPB output columns for all kNumTokens rows.
// grid = kN / kNPB, block = kBlockSize threads. K is reduced VPT elements
// per thread per iteration; fp32 accumulation, warp butterfly + smem
// finalize, bf16 store.
// ---------------------------------------------------------------------------
template <int kBlockSize, int kNumTokens, int kNPB, int kPF, int kN, int kK>
__global__ __launch_bounds__(kBlockSize, 1) void bf16_skinny_gemm_kernel(
__nv_bfloat16* out, __nv_bfloat16 const* mat_a, __nv_bfloat16 const* mat_b,
int64_t out_stride) {
constexpr int k_elems_per_iter = VPT * kBlockSize;
constexpr int k_iterations = kK / k_elems_per_iter;
static_assert(kK % k_elems_per_iter == 0);
constexpr int kWarpSize = 32;
constexpr int kNumWarps = kBlockSize / kWarpSize;
int const n_base = blockIdx.x * kNPB;
int const tid = threadIdx.x;
int const warpId = tid / kWarpSize;
int const laneId = tid % kWarpSize;
float acc[kNumTokens][kNPB] = {};
__shared__ float sm_reduction[kNumTokens][kNPB][kNumWarps];
// Register prefetch (kPF > 0): W does not depend on the predecessor, so
// the first kPF iterations' weight chunks are loaded raw BEFORE the
// dependency sync; with a PDL-releasing producer (fused_eh_norm fires
// gdc_launch_dependents early) these DRAM round trips overlap the norm.
// Pair-measured on B300 (norm+gemm in one graph, full 6144x12288):
// M=1 pf2 28.43us vs pf0 29.00us; deeper prefetch or M >= 2 regresses
// (register pressure), hence the per-M selection in the launcher.
uint4 w_pre[kPF > 0 ? kPF : 1][kNPB];
#pragma unroll
for (int pf = 0; pf < kPF; pf++) {
#pragma unroll
for (int n = 0; n < kNPB; n++) {
w_pre[pf][n] =
*reinterpret_cast<uint4 const*>(mat_b + (size_t)(n_base + n) * kK +
pf * k_elems_per_iter + tid * VPT);
}
}
#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)
cudaGridDependencySynchronize();
#endif
#pragma unroll
for (int ki = 0; ki < k_iterations; ki++) {
int const k_base = ki * k_elems_per_iter + tid * VPT;
float b_float[kNPB][VPT];
if (ki < kPF) {
#pragma unroll
for (int n = 0; n < kNPB; n++) {
__nv_bfloat16 const* p =
reinterpret_cast<__nv_bfloat16 const*>(&w_pre[ki][n]);
#pragma unroll
for (int v = 0; v < VPT; v++) b_float[n][v] = __bfloat162float(p[v]);
}
} else {
#pragma unroll
for (int n = 0; n < kNPB; n++) {
load_bf16x8_cs(mat_b + (size_t)(n_base + n) * kK + k_base, b_float[n]);
}
}
#pragma unroll
for (int m = 0; m < kNumTokens; m++) {
float a_float[VPT];
load_bf16x8(mat_a + (size_t)m * kK + k_base, a_float);
#pragma unroll
for (int n = 0; n < kNPB; n++) {
#pragma unroll
for (int k = 0; k < VPT; k++) {
acc[m][n] += a_float[k] * b_float[n][k];
}
}
}
}
// Warp-level butterfly reduction
#pragma unroll
for (int m = 0; m < kNumTokens; m++) {
#pragma unroll
for (int n = 0; n < kNPB; n++) {
float sum = acc[m][n];
sum += __shfl_xor_sync(0xffffffff, sum, 16);
sum += __shfl_xor_sync(0xffffffff, sum, 8);
sum += __shfl_xor_sync(0xffffffff, sum, 4);
sum += __shfl_xor_sync(0xffffffff, sum, 2);
sum += __shfl_xor_sync(0xffffffff, sum, 1);
if (laneId == 0) sm_reduction[m][n][warpId] = sum;
}
}
__syncthreads();
// Parallel finalize: one thread per (m, n) output.
for (int idx = tid; idx < kNumTokens * kNPB; idx += kBlockSize) {
int const m = idx / kNPB;
int const n = idx % kNPB;
float final_sum = 0.0f;
#pragma unroll
for (int w = 0; w < kNumWarps; w++) final_sum += sm_reduction[m][n][w];
out[(size_t)m * out_stride + n_base + n] = __float2bfloat16(final_sum);
}
// Trigger after our stores: harmless hardening, not a guarantee — the
// trigger only permits dependent-launch scheduling and carries no memory
// visibility semantics, so a PDL consumer must still gridsync before
// reading our output. Every current consumer is a plain launch (full
// stream order); firing late just avoids an unnecessarily early launch
// window and matches fp32_router_gemm.cu.
#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)
cudaTriggerProgrammaticLaunchCompletion();
#endif
}
} // namespace skinny
// ---------------------------------------------------------------------------
// Launcher
// ---------------------------------------------------------------------------
template <int kBlockSize, int kNPB, int kNumTokens, int kN, int kK>
void invokeBf16SkinnyGemm(__nv_bfloat16* output, __nv_bfloat16 const* mat_a,
__nv_bfloat16 const* mat_b, int64_t out_stride,
cudaStream_t stream) {
static_assert(kN % kNPB == 0);
// Weight prefetch depth: only M=1 measured a win (see kernel comment).
constexpr int kPF = (kNumTokens == 1) ? 2 : 0;
cudaLaunchConfig_t config;
config.gridDim = kN / kNPB;
config.blockDim = kBlockSize;
config.dynamicSmemBytes = 0;
config.stream = stream;
cudaLaunchAttribute attrs[1];
attrs[0].id = cudaLaunchAttributeProgrammaticStreamSerialization;
attrs[0].val.programmaticStreamSerializationAllowed = 1;
config.numAttrs = 1;
config.attrs = attrs;
cudaLaunchKernelEx(&config,
skinny::bf16_skinny_gemm_kernel<kBlockSize, kNumTokens,
kNPB, kPF, kN, kK>,
output, mat_a, mat_b, out_stride);
}
// ---------------------------------------------------------------------------
// Explicit instantiations. M = 1..32; (N, K) pairs:
// (768, 12288) eh_proj shard, TP8
// (1536, 12288) eh_proj shard, TP4
// (6144, 12288) eh_proj unsharded
// kNPB (B300 sweep, M=1): 6144 -> 2 (3072 blocks, 23.2us vs 25.3 at kNPB=8;
// narrow blocks minimize wave quantization); shards 768/1536 keep 4.
// ---------------------------------------------------------------------------
#define INSTANTIATE(M, NPB, N, K) \
template void invokeBf16SkinnyGemm<128, NPB, M, N, K>( \
__nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, int64_t, \
cudaStream_t);
#define INSTANTIATE_ALL_M(NPB, N, K) \
INSTANTIATE(1, NPB, N, K) \
INSTANTIATE(2, NPB, N, K) \
INSTANTIATE(3, NPB, N, K) \
INSTANTIATE(4, NPB, N, K) \
INSTANTIATE(5, NPB, N, K) \
INSTANTIATE(6, NPB, N, K) \
INSTANTIATE(7, NPB, N, K) \
INSTANTIATE(8, NPB, N, K) \
INSTANTIATE(9, NPB, N, K) \
INSTANTIATE(10, NPB, N, K) \
INSTANTIATE(11, NPB, N, K) \
INSTANTIATE(12, NPB, N, K) \
INSTANTIATE(13, NPB, N, K) \
INSTANTIATE(14, NPB, N, K) \
INSTANTIATE(15, NPB, N, K) \
INSTANTIATE(16, NPB, N, K) \
INSTANTIATE(17, NPB, N, K) \
INSTANTIATE(18, NPB, N, K) \
INSTANTIATE(19, NPB, N, K) \
INSTANTIATE(20, NPB, N, K) \
INSTANTIATE(21, NPB, N, K) \
INSTANTIATE(22, NPB, N, K) \
INSTANTIATE(23, NPB, N, K) \
INSTANTIATE(24, NPB, N, K) \
INSTANTIATE(25, NPB, N, K) \
INSTANTIATE(26, NPB, N, K) \
INSTANTIATE(27, NPB, N, K) \
INSTANTIATE(28, NPB, N, K) \
INSTANTIATE(29, NPB, N, K) \
INSTANTIATE(30, NPB, N, K) \
INSTANTIATE(31, NPB, N, K) \
INSTANTIATE(32, NPB, N, K)
INSTANTIATE_ALL_M(4, 768, 12288)
INSTANTIATE_ALL_M(4, 1536, 12288)
INSTANTIATE_ALL_M(2, 6144, 12288)
// LL-mode (M<=8 wiring guard) backbone shapes, B300 sweep vs cuBLAS:
// q_b_proj (2048, 2048): 1.67x/1.29x/1.15x at M=4/6/8 (NPB=4 within
// 0.1us of per-M best)
// shared-expert gate_up (512, 6144): 1.95x/1.58x/1.40x at M=4/6/8
// cuBLAS keeps qkv_a (2624,6144) and o_proj (6144,2048) — already at
// 3.4-3.8 TB/s there; the GEMV loses on activation re-reads.
INSTANTIATE_ALL_M(4, 2048, 2048)
INSTANTIATE_ALL_M(4, 512, 6144)
// fused_qkv_a (2624, 6144), 32MB: skinny wins ONLY at M<=2 (B300: M=1
// 6.99us vs cuBLAS 9.21 = 1.32x, M=2 1.24x; M>=4 cuBLAS holds at 3.5TB/s
// and every alternative loses — cublasLt top-8 3.6TB/s wall, DeepGEMM
// 0.71x, wmma+cp.async custom 0.30x pending a TMA rewrite).
INSTANTIATE_ALL_M(4, 2624, 6144)
// DSv3.2 (TP8) siblings of the GLM shapes above, same dual-chip matrix:
// fused_qkv_a (2112, 7168), 30MB: wins M<=2 (M=1 1.30-1.34x)
// MTP eh_proj (7168, 14336), 205MB: wins M<=2 (M=1 1.12-1.16x)
INSTANTIATE_ALL_M(4, 2112, 7168)
INSTANTIATE_ALL_M(2, 7168, 14336)
#undef INSTANTIATE_ALL_M
#undef INSTANTIATE
@@ -1,170 +0,0 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
#include <torch/csrc/stable/library.h>
#include <torch/csrc/stable/tensor.h>
#include <torch/headeronly/core/ScalarType.h>
#include "core/registration.h"
#include "libtorch_stable/torch_utils.h"
#include <cuda_bf16.h>
#include <cuda_runtime.h>
#include <stdexcept>
namespace {
inline int getSMVersion() {
auto* props = get_device_prop();
return props->major * 10 + props->minor;
}
} // namespace
static constexpr int SKINNY_MAX_TOKENS = 32;
// Supported (N, K) pairs (must match the instantiations in
// bf16_skinny_gemm.cu): eh_proj shard TP8 / TP4 / unsharded.
static inline bool bf16_skinny_gemm_supported(int n, int k) {
if (k == 12288 && (n == 768 || n == 1536 || n == 6144)) return true;
// LL-mode backbone shapes (wire callers with an M <= 8 guard; the GEMV
// family loses to cuBLAS at larger M).
if (k == 2048 && n == 2048) return true; // q_b_proj (TP8)
if (k == 6144 && n == 2624) return true; // fused_qkv_a (wire M <= 2 only)
if (k == 7168 && n == 2112) return true; // DSv3.2 fused_qkv_a (M <= 2)
if (k == 14336 && n == 7168) return true; // DSv3.2 eh_proj (M <= 2)
if (k == 6144 && n == 512) return true; // shared-expert gate_up (TP8)
return false;
}
// Forward declarations - template params must match bf16_skinny_gemm.cu
template <int kBlockSize, int kNPB, int kNumTokens, int kN, int kK>
void invokeBf16SkinnyGemm(__nv_bfloat16* output, __nv_bfloat16 const* mat_a,
__nv_bfloat16 const* mat_b, int64_t out_stride,
cudaStream_t stream);
template <int kNPB, int kN, int kK, int kBegin, int kEnd>
struct SkinnyLoopUnroller {
static void unroll(int num_tokens, __nv_bfloat16* output,
__nv_bfloat16 const* mat_a, __nv_bfloat16 const* mat_b,
int64_t out_stride, cudaStream_t stream) {
if (num_tokens == kBegin) {
invokeBf16SkinnyGemm<128, kNPB, kBegin, kN, kK>(output, mat_a, mat_b,
out_stride, stream);
} else {
SkinnyLoopUnroller<kNPB, kN, kK, kBegin + 1, kEnd>::unroll(
num_tokens, output, mat_a, mat_b, out_stride, stream);
}
}
};
template <int kNPB, int kN, int kK, int kEnd>
struct SkinnyLoopUnroller<kNPB, kN, kK, kEnd, kEnd> {
static void unroll(int num_tokens, __nv_bfloat16* output,
__nv_bfloat16 const* mat_a, __nv_bfloat16 const* mat_b,
int64_t out_stride, cudaStream_t stream) {
if (num_tokens == kEnd) {
invokeBf16SkinnyGemm<128, kNPB, kEnd, kN, kK>(output, mat_a, mat_b,
out_stride, stream);
} else {
throw std::invalid_argument(
"bf16_skinny_gemm: num_tokens must be in [1, 32]");
}
}
};
static void dispatchBf16SkinnyGemm(int n, int k, int num_tokens,
__nv_bfloat16* output,
__nv_bfloat16 const* mat_a,
__nv_bfloat16 const* mat_b,
int64_t out_stride, cudaStream_t stream) {
if (n == 768 && k == 12288) {
SkinnyLoopUnroller<4, 768, 12288, 1, SKINNY_MAX_TOKENS>::unroll(
num_tokens, output, mat_a, mat_b, out_stride, stream);
} else if (n == 1536 && k == 12288) {
SkinnyLoopUnroller<4, 1536, 12288, 1, SKINNY_MAX_TOKENS>::unroll(
num_tokens, output, mat_a, mat_b, out_stride, stream);
} else if (n == 6144 && k == 12288) {
SkinnyLoopUnroller<2, 6144, 12288, 1, SKINNY_MAX_TOKENS>::unroll(
num_tokens, output, mat_a, mat_b, out_stride, stream);
} else if (n == 2048 && k == 2048) {
SkinnyLoopUnroller<4, 2048, 2048, 1, SKINNY_MAX_TOKENS>::unroll(
num_tokens, output, mat_a, mat_b, out_stride, stream);
} else if (n == 2624 && k == 6144) {
SkinnyLoopUnroller<4, 2624, 6144, 1, SKINNY_MAX_TOKENS>::unroll(
num_tokens, output, mat_a, mat_b, out_stride, stream);
} else if (n == 2112 && k == 7168) {
SkinnyLoopUnroller<4, 2112, 7168, 1, SKINNY_MAX_TOKENS>::unroll(
num_tokens, output, mat_a, mat_b, out_stride, stream);
} else if (n == 7168 && k == 14336) {
SkinnyLoopUnroller<2, 7168, 14336, 1, SKINNY_MAX_TOKENS>::unroll(
num_tokens, output, mat_a, mat_b, out_stride, stream);
} else if (n == 512 && k == 6144) {
SkinnyLoopUnroller<4, 512, 6144, 1, SKINNY_MAX_TOKENS>::unroll(
num_tokens, output, mat_a, mat_b, out_stride, stream);
} else {
throw std::invalid_argument("bf16_skinny_gemm: unsupported (N, K) pair");
}
}
void bf16_skinny_gemm(
torch::stable::Tensor& output, // [num_tokens, N] bf16
torch::stable::Tensor const& mat_a, // [num_tokens, K] bf16
torch::stable::Tensor const& mat_b // [N, K] bf16
) {
STD_TORCH_CHECK(output.dim() == 2 && mat_a.dim() == 2 && mat_b.dim() == 2);
STD_TORCH_CHECK(output.is_cuda() && mat_a.is_cuda() && mat_b.is_cuda(),
"bf16_skinny_gemm: all tensors must be CUDA tensors");
STD_TORCH_CHECK(output.get_device_index() == mat_a.get_device_index() &&
output.get_device_index() == mat_b.get_device_index(),
"bf16_skinny_gemm: all tensors must be on the same device");
STD_TORCH_CHECK(mat_a.is_contiguous() && mat_b.is_contiguous(),
"bf16_skinny_gemm: inputs must be contiguous");
// Output may be a column-slice view of a wider padded buffer: unit column
// stride, row stride >= N (rows must not overlap).
STD_TORCH_CHECK(output.stride(1) == 1,
"bf16_skinny_gemm: output columns must be contiguous");
const int num_tokens = mat_a.size(0);
const int n = mat_b.size(0);
const int k = mat_a.size(1);
STD_TORCH_CHECK(output.size(0) == num_tokens && output.size(1) == n,
"bf16_skinny_gemm: output must be [num_tokens, N]");
STD_TORCH_CHECK(mat_b.size(1) == k,
"bf16_skinny_gemm: mat_a and mat_b must share K");
STD_TORCH_CHECK(bf16_skinny_gemm_supported(n, k),
"bf16_skinny_gemm: unsupported (N, K) pair");
const int64_t out_stride = output.stride(0);
STD_TORCH_CHECK(num_tokens <= 1 || out_stride >= n,
"bf16_skinny_gemm: output rows overlap");
STD_TORCH_CHECK(num_tokens >= 0 && num_tokens <= SKINNY_MAX_TOKENS,
"bf16_skinny_gemm: num_tokens must be in [0, 32]");
STD_TORCH_CHECK(
mat_a.scalar_type() == torch::headeronly::ScalarType::BFloat16 &&
mat_b.scalar_type() == torch::headeronly::ScalarType::BFloat16 &&
output.scalar_type() == torch::headeronly::ScalarType::BFloat16,
"bf16_skinny_gemm: all tensors must be bfloat16");
// Empty batch (e.g. an empty rank at a DP/PP boundary): nothing to compute.
if (num_tokens == 0) {
return;
}
const torch::stable::accelerator::DeviceGuard device_guard(
mat_a.get_device_index());
STD_TORCH_CHECK(getSMVersion() >= 90, "bf16_skinny_gemm: requires SM90+");
auto stream = get_current_cuda_stream(mat_a.get_device_index());
dispatchBf16SkinnyGemm(
n, k, num_tokens,
reinterpret_cast<__nv_bfloat16*>(output.mutable_data_ptr()),
reinterpret_cast<__nv_bfloat16 const*>(mat_a.data_ptr()),
reinterpret_cast<__nv_bfloat16 const*>(mat_b.data_ptr()), out_stride,
stream);
}
STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, m) {
m.impl("bf16_skinny_gemm", TORCH_BOX(&bf16_skinny_gemm));
}
+36 -112
View File
@@ -391,7 +391,7 @@ struct MmaComputer {
static constexpr int n_iter_cnt =
(tile_n + 7) /
8; // Possible to have non-1 n_iter_cnt for ab_swap m16 case.
static_assert(m_iter_cnt == 1 || m_iter_cnt == 2);
static_assert(m_iter_cnt == 1);
static_assert(n_iter_cnt == 1 || n_iter_cnt == 2);
__device__ MmaComputer(bf16_t* gmem_c_local_, bf16_t* smem_a_,
@@ -416,18 +416,13 @@ struct MmaComputer {
public:
__device__ void prepare() {
#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900
// Fragment addressing is per 16-row ldmatrix tile; m_iter selects the
// 16-row half within tile_m.
#pragma unroll
for (int m = 0; m < m_iter_cnt; m++) {
#pragma unroll
for (int i = 0; i < k_phase_cnt; i++) {
int linear_idx = (lane_idx % 16) + (lane_idx / 16) * 128 + i * 256;
int m_idx = linear_idx % 16 + m * 16;
int k_idx = linear_idx / 16 + warp_k_offset_in_tile_k;
k_idx = apply_swizzle_343_on_elem_row_col<bf16_t>(m_idx, k_idx);
a_smem_offsets[m][i] = m_idx * tile_k + k_idx;
}
for (int i = 0; i < k_phase_cnt; i++) {
int linear_idx = (lane_idx % 16) + (lane_idx / 16) * 128 + i * 256;
int m_idx = linear_idx % tile_m;
int k_idx = linear_idx / tile_m + warp_k_offset_in_tile_k;
k_idx = apply_swizzle_343_on_elem_row_col<bf16_t>(m_idx, k_idx);
a_smem_offsets[0][i] = m_idx * tile_k + k_idx;
}
#pragma unroll
for (int n_iter_idx = 0; n_iter_idx < n_iter_cnt; n_iter_idx++) {
@@ -451,14 +446,11 @@ struct MmaComputer {
wait_barrier(smem_barrier + 0 + stage_idx * 2, phase_bit);
#pragma unroll
for (int m = 0; m < m_iter_cnt; m++) {
#pragma unroll
for (int i = 0; i < k_phase_cnt; i++) {
int smem_offset = a_smem_offsets[m][i];
bf16_t* smem_ptr_this_iter =
smem_a + stage_idx * tile_m * tile_k + smem_offset;
ldsm_x4(smem_ptr_this_iter, reinterpret_cast<uint32_t*>(a_reg[m][i]));
}
for (int i = 0; i < k_phase_cnt; i++) {
int smem_offset = a_smem_offsets[0][i];
bf16_t* smem_ptr_this_iter =
smem_a + stage_idx * tile_m * tile_k + smem_offset;
ldsm_x4(smem_ptr_this_iter, reinterpret_cast<uint32_t*>(a_reg[0][i]));
}
#pragma unroll
@@ -477,12 +469,9 @@ struct MmaComputer {
for (int k_iter_idx = 0; k_iter_idx < k_phase_cnt; k_iter_idx++) {
#pragma unroll
for (int n_iter_idx = 0; n_iter_idx < n_iter_cnt; n_iter_idx++) {
#pragma unroll
for (int m = 0; m < m_iter_cnt; m++) {
hmma_16_8_16_f32acc_bf16ab(
acc_reg[m][n_iter_idx], a_reg[m][k_iter_idx],
b_reg[n_iter_idx][k_iter_idx], acc_reg[m][n_iter_idx]);
}
hmma_16_8_16_f32acc_bf16ab(
acc_reg[0][n_iter_idx], a_reg[0][k_iter_idx],
b_reg[n_iter_idx][k_iter_idx], acc_reg[0][n_iter_idx]);
}
}
::arrive_barrier(smem_barrier + 1 + stage_idx * 2);
@@ -497,14 +486,14 @@ struct MmaComputer {
#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900
asm volatile("bar.sync %0, %1;" : : "r"(1), "r"(thread_cnt));
// reorganize the acc_reg
constexpr int thread_m = 2 * m_iter_cnt;
constexpr int thread_m = 2;
constexpr int thread_n = 2 * n_iter_cnt;
constexpr int cta_mma_n = n_iter_cnt * 8;
float acc_reg_reorg[thread_m][thread_n];
for (int i = 0; i < thread_m; i++) {
for (int j = 0; j < thread_n; j++) {
acc_reg_reorg[i][j] = acc_reg[i / 2][j / 2][(j % 2) + (i % 2) * 2];
acc_reg_reorg[i][j] = acc_reg[0][j / 2][(j % 2) + (i * 2)];
}
}
@@ -525,8 +514,7 @@ struct MmaComputer {
for (int m_idx_thread = 0; m_idx_thread < thread_m; m_idx_thread++) {
#pragma unroll
for (int n_idx_thread = 0; n_idx_thread < thread_n; n_idx_thread++) {
int m_idx =
(lane_idx / 4) + (m_idx_thread % 2) * 8 + (m_idx_thread / 2) * 16;
int m_idx = (lane_idx / 4) + m_idx_thread * 8;
int n_idx =
((lane_idx % 4) * 2) + (n_idx_thread % 2) + (n_idx_thread / 2) * 8;
smem_c[cosize_smem_c * warp_idx + smem_c_index_func(m_idx, n_idx)] =
@@ -599,7 +587,7 @@ __global__ __launch_bounds__(256, 1) void fused_a_gemm_kernel(
static_assert(
tile_k == 128 || tile_k == 256 || tile_k == 512 ||
tile_k == 1024); // tile_k must be larger than 64 since 4 warp splitK.
static_assert(tile_m == 16 || tile_m == 32);
static_assert(tile_m == 16);
constexpr int g2s_vec_bytes = 16;
constexpr int a_elem_bytes = 2;
constexpr int b_elem_bytes = 2;
@@ -659,7 +647,7 @@ __global__ __launch_bounds__(256, 1) void fused_a_gemm_kernel(
#endif
}
template <typename T, int kHdIn, int kHdOut, int kTileN, int kTileM = 16>
template <typename T, int kHdIn, int kHdOut, int kTileN>
void invokeFusedAGemm(T* output, T const* mat_a, T const* mat_b, int num_tokens,
cudaStream_t const stream) {
constexpr int gemm_m = kHdOut; // 2112
@@ -667,7 +655,7 @@ void invokeFusedAGemm(T* output, T const* mat_a, T const* mat_b, int num_tokens,
constexpr int gemm_k = kHdIn; // 7168
constexpr int batch_size = 1;
std::swap(mat_a, mat_b);
constexpr int tile_m = kTileM;
constexpr int tile_m = 16;
constexpr int tile_n = kTileN; // 8 or 16
constexpr int tile_k = std::max(256, 1024 / tile_n); // 256
constexpr int max_stage_cnt =
@@ -714,44 +702,6 @@ template void invokeFusedAGemm<__nv_bfloat16, 7168, 2112, 16>(
__nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, int num_tokens,
cudaStream_t);
// GLM-5.2 fused_qkv_a (K=6144 -> N=2624). tile_m=32 so the grid is
// 2624/32 = 82 CTAs, a single wave on B300 (148 SMs); tile_m=16's 164 CTAs
// straddle two waves and drop to 2.9 TB/s vs 3.9-4.0 here. Beats cuBLAS
// at every decode M: 1.11-1.15x for M<=8 (tile_n=8), 1.12x at M=16
// (tile_n=16). The M<=2 dispatch still belongs to bf16_skinny_gemm
// (4.5 TB/s).
template void invokeFusedAGemm<__nv_bfloat16, 6144, 2624, 8, 32>(
__nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, int num_tokens,
cudaStream_t);
template void invokeFusedAGemm<__nv_bfloat16, 6144, 2624, 16, 32>(
__nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, int num_tokens,
cudaStream_t);
// GLM-5.2 q_b_proj TP8 (K=2048 -> N=2048). tile_m=16 keeps 128 CTAs (already
// a single wave) and measures ahead of tile_m=32 here. Beats cuBLAS
// 1.79-1.87x at M=3..8 (tile_n=8) and 1.19-1.30x at M=9..16 (tile_n=16);
// M<=2 belongs to bf16_skinny_gemm, M>=20 to cuBLAS.
template void invokeFusedAGemm<__nv_bfloat16, 2048, 2048, 8>(
__nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, int num_tokens,
cudaStream_t);
template void invokeFusedAGemm<__nv_bfloat16, 2048, 2048, 16>(
__nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, int num_tokens,
cudaStream_t);
// DSv3.2 q_b_proj TP8 (K=1536 -> N=3072). tile_m=32 keeps 96 CTAs (single
// wave; tile_m=16's 192 straddle two). Beats cuBLAS 1.6-1.8x at M=1..8 and
// 1.05-1.18x at M=9..16 on B300/B200; the whole 1..16 range dispatches here
// (no skinny tier: K=1536 does not fit the GEMV's 128x8 K-step).
template void invokeFusedAGemm<__nv_bfloat16, 1536, 3072, 8, 32>(
__nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, int num_tokens,
cudaStream_t);
template void invokeFusedAGemm<__nv_bfloat16, 1536, 3072, 16, 32>(
__nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, int num_tokens,
cudaStream_t);
void dsv3_fused_a_gemm(torch::stable::Tensor& output,
torch::stable::Tensor const& mat_a,
torch::stable::Tensor const& mat_b) {
@@ -760,15 +710,12 @@ void dsv3_fused_a_gemm(torch::stable::Tensor& output,
int const hd_in = mat_a.size(1);
int const hd_out = mat_b.size(1);
bool const is_dsv3 = hd_in == 7168 && hd_out == 2112;
bool const is_glm = hd_in == 6144 && hd_out == 2624;
bool const is_glm_qb = hd_in == 2048 && hd_out == 2048;
bool const is_ds_qb = hd_in == 1536 && hd_out == 3072;
constexpr int kHdIn = 7168;
constexpr int kHdOut = 2112;
STD_TORCH_CHECK(num_tokens >= 1 && num_tokens <= 16,
"required 1 <= mat_a.shape[0] <= 16");
STD_TORCH_CHECK(is_dsv3 || is_glm || is_glm_qb || is_ds_qb,
"supported (hd_in, hd_out): (7168, 2112), (6144, 2624), "
"(2048, 2048), (1536, 3072)");
STD_TORCH_CHECK(hd_in == kHdIn, "required mat_a.shape[1] == 7168");
STD_TORCH_CHECK(hd_out == kHdOut, "required mat_b.shape[1] == 2112");
STD_TORCH_CHECK(output.size(0) == num_tokens,
"required output.shape[0] == mat_a.shape[0]");
STD_TORCH_CHECK(output.size(1) == hd_out,
@@ -791,41 +738,18 @@ void dsv3_fused_a_gemm(torch::stable::Tensor& output,
STD_TORCH_CHECK(getSMVersion() >= 90, "required CUDA ARCH >= SM_90");
auto stream = get_current_cuda_stream(mat_a.get_device_index());
auto* out = reinterpret_cast<__nv_bfloat16*>(output.mutable_data_ptr());
auto* a = reinterpret_cast<__nv_bfloat16 const*>(mat_a.data_ptr());
auto* b = reinterpret_cast<__nv_bfloat16 const*>(mat_b.data_ptr());
if (is_dsv3) {
if (num_tokens <= 8) {
invokeFusedAGemm<__nv_bfloat16, 7168, 2112, 8>(out, a, b, num_tokens,
stream);
} else {
invokeFusedAGemm<__nv_bfloat16, 7168, 2112, 16>(out, a, b, num_tokens,
stream);
}
} else if (is_glm) {
if (num_tokens <= 8) {
invokeFusedAGemm<__nv_bfloat16, 6144, 2624, 8, 32>(out, a, b, num_tokens,
stream);
} else {
invokeFusedAGemm<__nv_bfloat16, 6144, 2624, 16, 32>(out, a, b, num_tokens,
stream);
}
} else if (is_glm_qb) {
if (num_tokens <= 8) {
invokeFusedAGemm<__nv_bfloat16, 2048, 2048, 8>(out, a, b, num_tokens,
stream);
} else {
invokeFusedAGemm<__nv_bfloat16, 2048, 2048, 16>(out, a, b, num_tokens,
stream);
}
if (num_tokens <= 8) {
invokeFusedAGemm<__nv_bfloat16, kHdIn, kHdOut, 8>(
reinterpret_cast<__nv_bfloat16*>(output.mutable_data_ptr()),
reinterpret_cast<__nv_bfloat16 const*>(mat_a.data_ptr()),
reinterpret_cast<__nv_bfloat16 const*>(mat_b.data_ptr()), num_tokens,
stream);
} else {
if (num_tokens <= 8) {
invokeFusedAGemm<__nv_bfloat16, 1536, 3072, 8, 32>(out, a, b, num_tokens,
stream);
} else {
invokeFusedAGemm<__nv_bfloat16, 1536, 3072, 16, 32>(out, a, b, num_tokens,
stream);
}
invokeFusedAGemm<__nv_bfloat16, kHdIn, kHdOut, 16>(
reinterpret_cast<__nv_bfloat16*>(output.mutable_data_ptr()),
reinterpret_cast<__nv_bfloat16 const*>(mat_a.data_ptr()),
reinterpret_cast<__nv_bfloat16 const*>(mat_b.data_ptr()), num_tokens,
stream);
}
}
@@ -49,11 +49,6 @@ __global__ void __launch_bounds__(512, VLLM_BLOCKS_PER_SM(512))
static_assert(sizeof(PackedVec) == sizeof(Type) * CVT_FP4_ELTS_PER_THREAD,
"Vec size is not matched.");
#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900))
cudaGridDependencySynchronize();
cudaTriggerProgrammaticLaunchCompletion();
#endif
// Precompute SF layout parameter (constant for entire kernel).
int32_t const numKTiles = (outputCols + 63) / 64;
@@ -128,11 +123,6 @@ __global__ void __launch_bounds__(512, VLLM_BLOCKS_PER_SM(512))
static_assert(sizeof(PackedVec) == sizeof(Type) * CVT_FP4_ELTS_PER_THREAD,
"Vec size is not matched.");
#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900))
cudaGridDependencySynchronize();
cudaTriggerProgrammaticLaunchCompletion();
#endif
int32_t const colIdx = blockDim.x * blockIdx.y + threadIdx.x;
int elem_idx = colIdx * CVT_FP4_ELTS_PER_THREAD;
@@ -211,8 +201,6 @@ void scaled_fp4_quant_sm1xxa(torch::stable::Tensor const& output,
const torch::stable::accelerator::DeviceGuard device_guard(
input.get_device_index());
auto stream = get_current_cuda_stream(input.get_device_index());
auto* device_props = get_device_prop();
int const sm_version = device_props->major * 10 + device_props->minor;
int output_sf_n_unpadded = int(output_n / CVT_FP4_SF_VEC_SIZE);
@@ -236,21 +224,10 @@ void scaled_fp4_quant_sm1xxa(torch::stable::Tensor const& output,
input.scalar_type(), "nvfp4_quant_kernel", [&] {
using cuda_type = vllm::CUDATypeConverter<scalar_t>::Type;
auto input_ptr = static_cast<cuda_type const*>(input.data_ptr());
cudaLaunchConfig_t config = {};
config.gridDim = grid;
config.blockDim = block;
config.dynamicSmemBytes = 0;
config.stream = stream;
cudaLaunchAttribute attrs[1];
attrs[0].id = cudaLaunchAttributeProgrammaticStreamSerialization;
attrs[0].val.programmaticStreamSerializationAllowed = 1;
config.numAttrs = (sm_version >= 90) ? 1 : 0;
config.attrs = attrs;
cudaLaunchKernelEx(&config, vllm::cvt_fp16_to_fp4<cuda_type, false>,
m, n, output_n, num_padded_cols, input_ptr,
input_sf_ptr,
reinterpret_cast<uint32_t*>(output_ptr),
reinterpret_cast<uint32_t*>(sf_out));
vllm::cvt_fp16_to_fp4<cuda_type, false><<<grid, block, 0, stream>>>(
m, n, output_n, num_padded_cols, input_ptr, input_sf_ptr,
reinterpret_cast<uint32_t*>(output_ptr),
reinterpret_cast<uint32_t*>(sf_out));
});
} else {
int num_packed_cols = output_n / CVT_FP4_ELTS_PER_THREAD;
@@ -263,21 +240,12 @@ void scaled_fp4_quant_sm1xxa(torch::stable::Tensor const& output,
input.scalar_type(), "nvfp4_quant_kernel", [&] {
using cuda_type = vllm::CUDATypeConverter<scalar_t>::Type;
auto input_ptr = static_cast<cuda_type const*>(input.data_ptr());
cudaLaunchConfig_t config = {};
config.gridDim = grid;
config.blockDim = block;
config.dynamicSmemBytes = 0;
config.stream = stream;
cudaLaunchAttribute attrs[1];
attrs[0].id = cudaLaunchAttributeProgrammaticStreamSerialization;
attrs[0].val.programmaticStreamSerializationAllowed = 1;
config.numAttrs = (sm_version >= 90) ? 1 : 0;
config.attrs = attrs;
cudaLaunchKernelEx(
&config, vllm::cvt_fp16_to_fp4_sf_major<cuda_type, false>, m, n,
output_n, output_sf_n_unpadded, num_packed_cols, input_ptr,
input_sf_ptr, reinterpret_cast<uint32_t*>(output_ptr),
reinterpret_cast<uint32_t*>(sf_out));
vllm::cvt_fp16_to_fp4_sf_major<cuda_type, false>
<<<grid, block, 0, stream>>>(
m, n, output_n, output_sf_n_unpadded, num_packed_cols,
input_ptr, input_sf_ptr,
reinterpret_cast<uint32_t*>(output_ptr),
reinterpret_cast<uint32_t*>(sf_out));
});
}
}
-4
View File
@@ -330,10 +330,6 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
// conditionally compiled so impl registration is in source file
ops.def("fp32_router_gemm(Tensor! output, Tensor mat_a, Tensor mat_b) -> ()");
// BF16 skinny GEMM (M<=32, weight-BW-bound shapes, e.g. MTP eh_proj).
// conditionally compiled so impl registration is in source file
ops.def("bf16_skinny_gemm(Tensor! output, Tensor mat_a, Tensor mat_b) -> ()");
// reorder weight for AllSpark Ampere W8A16 Fused Gemm kernel
ops.def(
"rearrange_kn_weight_as_n32k16_order(Tensor b_qweight, Tensor b_scales, "
-41
View File
@@ -1,41 +0,0 @@
# GLM-5.2-NVFP4 — Low-Latency (LL) serving recipe · 8×B300 · TP8 · MTP=5
Single-node 8×B300 (sm_100/103), pure tensor parallel, NVFP4 weights, fp8 KV cache,
MTP=5 speculative decode, model runner v2.
Validated on `glm5.2-LL` @ `8d407aee1`.
## Versions
| Component | Version |
| --- | --- |
| vLLM | `glm5.2-LL` @ `8d407aee1` |
| flashinfer-python | `0.6.15` |
| Model | `nvidia/GLM-5.2-NVFP4` (`modelopt_fp4`) |
## Build (once)
```bash
pip install 'flashinfer-python==0.6.15'
CUDA_HOME=/usr/local/cuda-13.0 TORCH_CUDA_ARCH_LIST="10.0 10.3" MAX_JOBS=96 \
VLLM_USE_PRECOMPILED=0 VLLM_USE_PRECOMPILED_RUST=0 \
pip install -e . --no-deps --no-build-isolation
```
## Server
```bash
export VLLM_USE_V2_MODEL_RUNNER=1 # model runner v2
export VLLM_DEEP_GEMM_WARMUP=skip
vllm serve nvidia/GLM-5.2-NVFP4 \
--trust-remote-code \
--tensor-parallel-size 8 \
--quantization modelopt_fp4 --kv-cache-dtype fp8_e4m3 \
--max-model-len 32768 --max-num-batched-tokens 16384 --max-num-seqs 256 \
--no-enable-prefix-caching --gpu-memory-utilization 0.85 \
--speculative-config '{"method":"mtp","num_speculative_tokens":5}' \
--kernel-config '{"ir_op_priority":{"rms_norm":["vllm_c","native"],"fused_add_rms_norm":["vllm_c","native"]}}' \
--host 0.0.0.0 --port 8000
```
-70
View File
@@ -1,70 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Numerical correctness for the bf16 skinny GEMM (decode-M GEMV) kernel.
Covers every supported (N, K) shape across M = 0/1/2/8/16/32 (M=32 exercises
the tile_m=32 MMA path), plus strided (column-slice) output. M=0 must return
an empty result without launching the kernel (empty ranks at DP/PP
boundaries)."""
import pytest
import torch
from vllm import _custom_ops as ops
from vllm.platforms import current_platform
# (N, K) pairs must match bf16_skinny_gemm_supported() in the entry .cu.
SHAPES = [
(768, 12288),
(1536, 12288),
(6144, 12288),
(2048, 2048),
(2624, 6144),
(2112, 7168),
(7168, 14336),
(512, 6144),
]
MS = [0, 1, 2, 8, 16, 32]
pytestmark = pytest.mark.skipif(
not current_platform.is_cuda() or not current_platform.has_device_capability(90),
reason="bf16_skinny_gemm requires CUDA SM90+",
)
def _rel_err(out: torch.Tensor, ref: torch.Tensor) -> float:
return (out.float() - ref).norm().item() / ref.norm().clamp_min(1e-6).item()
@pytest.mark.parametrize("n,k", SHAPES)
@pytest.mark.parametrize("m", MS)
def test_bf16_skinny_gemm_matches_reference(n: int, k: int, m: int):
torch.manual_seed(0)
x = torch.randn(m, k, dtype=torch.bfloat16, device="cuda")
w = torch.randn(n, k, dtype=torch.bfloat16, device="cuda")
out = ops.bf16_skinny_gemm(x, w)
assert out.shape == (m, n)
assert out.dtype == torch.bfloat16
if m == 0: # empty batch: nothing to check beyond the empty shape
return
ref = x.float() @ w.float().t()
rel = _rel_err(out, ref)
assert rel < 2e-2, f"rel err {rel:.4f} for (m,n,k)=({m},{n},{k})"
@pytest.mark.parametrize("n,k", [(2048, 2048), (512, 6144)])
def test_bf16_skinny_gemm_strided_output(n: int, k: int):
"""Output is a column-slice of a wider padded buffer (row stride > N)."""
torch.manual_seed(0)
m = 8
x = torch.randn(m, k, dtype=torch.bfloat16, device="cuda")
w = torch.randn(n, k, dtype=torch.bfloat16, device="cuda")
wide = torch.empty(m, n + 64, dtype=torch.bfloat16, device="cuda")
view = wide[:, :n]
torch.ops._C.bf16_skinny_gemm(view, x, w)
ref = x.float() @ w.float().t()
assert _rel_err(view, ref) < 2e-2
@@ -49,32 +49,6 @@ pytestmark = pytest.mark.skipif(
)
def test_platform_capability_queries_are_constant_during_compile(monkeypatch):
monkeypatch.setattr(
K.current_platform, "has_device_capability", lambda capability: True
)
monkeypatch.setattr(K, "has_cutedsl", lambda: True)
monkeypatch.setattr(K.current_platform, "is_arch_support_pdl", lambda: True)
def capability_branches(x: torch.Tensor) -> torch.Tensor:
if K._can_use_fused_q_cutedsl() and K._is_arch_support_pdl():
return x + 1
return x - 1
compiled = torch.compile(capability_branches, backend="eager", fullgraph=True)
x = torch.zeros(1, device="cuda")
torch.testing.assert_close(compiled(x), torch.ones_like(x))
def test_pdl_is_disabled_before_blackwell(monkeypatch):
monkeypatch.setattr(
K.current_platform, "has_device_capability", lambda capability: False
)
monkeypatch.setattr(K.current_platform, "is_arch_support_pdl", lambda: True)
assert not K._is_arch_support_pdl()
# ── reference helpers ────────────────────────────────────────────────────────
@@ -318,57 +292,6 @@ def test_fused_norm_rope_no_indexer(num_tokens: int):
assert (topk == 7).all(), "topk buffer should be untouched on shared layer"
def test_fused_norm_rope_profile_without_cache_compiles():
"""The cache-free profiling path must still compile and produce Q."""
torch.manual_seed(2)
dev = "cuda"
num_tokens = 4
pos = torch.arange(num_tokens, device=dev, dtype=torch.int64)
q_c = torch.randn(num_tokens, Q_LORA, device=dev, dtype=torch.bfloat16)
kv_c = torch.randn(num_tokens, KV_LORA, device=dev, dtype=torch.bfloat16)
k_pe = torch.randn(num_tokens, ROPE_DIM, device=dev, dtype=torch.bfloat16)
qw = torch.randn(Q_LORA, device=dev, dtype=torch.bfloat16)
kvw = torch.randn(KV_LORA, device=dev, dtype=torch.bfloat16)
index_k = torch.randn(num_tokens, INDEX_HEAD_DIM, device=dev, dtype=torch.bfloat16)
index_w = torch.randn(INDEX_HEAD_DIM, device=dev, dtype=torch.float32)
index_b = torch.randn(INDEX_HEAD_DIM, device=dev, dtype=torch.float32)
cos_sin = make_cos_sin(64, ROPE_DIM, dev)
topk = torch.empty(num_tokens, 2048, device=dev, dtype=torch.int32)
def profile_run(
q_c: torch.Tensor,
kv_c: torch.Tensor,
k_pe: torch.Tensor,
index_k: torch.Tensor,
) -> torch.Tensor:
return K.fused_norm_rope(
pos,
q_c,
qw,
EPS,
kv_c,
kvw,
EPS,
k_pe,
cos_sin,
index_k,
index_w,
index_b,
EPS,
cos_sin,
topk,
slot_mapping=None,
indexer_k_cache=None,
mla_kv_cache=None,
has_indexer=True,
index_rope_interleave=False,
)
compiled = torch.compile(profile_run, fullgraph=True)
q_out = compiled(q_c, kv_c, k_pe, index_k)
assert_bf16(q_out, rms_norm(q_c, qw), "q_c rmsnorm (profiling)")
@pytest.mark.parametrize("num_tokens", [1, 4, 17, 512])
def test_fused_norm_rope_ds_mla(num_tokens: int):
"""fp8_ds_mla MLA cache layout (FlashMLA sparse, bf16-query path; SM90/SM100).
-4
View File
@@ -1600,10 +1600,6 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
speculative_model="luccafong/deepseek_mtp_draft_random",
trust_remote_code=True,
),
"DeepseekV32MTPModel": _HfExamplesInfo(
"deepseek-ai/DeepSeek-V3.2-Exp",
speculative_model="deepseek-ai/DeepSeek-V3.2-Exp",
),
"DeepSeekV4MTPModel": _HfExamplesInfo(
"deepseek-ai/DeepSeek-V4-Flash",
speculative_model="deepseek-ai/DeepSeek-V4-Flash",
-25
View File
@@ -2371,20 +2371,6 @@ def fp32_router_gemm(
return output
def bf16_skinny_gemm(
x: torch.Tensor,
weight: torch.Tensor,
) -> torch.Tensor:
"""Skinny bf16 GEMM (M<=32): x [M, K] @ weight [N, K]^T -> [M, N].
Single block-per-column kernel with fp32 accumulation; replaces cuBLAS
splitK for weight-bandwidth-bound decode shapes (e.g. MTP eh_proj).
"""
output = torch.empty((x.shape[0], weight.shape[0]), dtype=x.dtype, device=x.device)
torch.ops._C.bf16_skinny_gemm(output, x, weight)
return output
if hasattr(torch.ops, "_C") and hasattr(torch.ops._C, "fp32_router_gemm"):
@register_fake("_C::fp32_router_gemm")
@@ -2396,17 +2382,6 @@ if hasattr(torch.ops, "_C") and hasattr(torch.ops._C, "fp32_router_gemm"):
return
if hasattr(torch.ops, "_C") and hasattr(torch.ops._C, "bf16_skinny_gemm"):
@register_fake("_C::bf16_skinny_gemm")
def bf16_skinny_gemm_fake(
output: torch.Tensor,
mat_a: torch.Tensor,
mat_b: torch.Tensor,
) -> None:
return
def topk_softmax(
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
@@ -202,7 +202,11 @@ if flashinfer_comm is not None:
element_size = allreduce_in.element_size()
current_tensor_size = num_tokens * hidden_size * element_size
max_tensor_size = max_token_num * hidden_size * element_size
oversize = current_tensor_size > max_tensor_size
assert current_tensor_size <= max_tensor_size, (
f"Current tensor size {current_tensor_size} is larger than "
f"max token num {max_token_num} * hidden size {hidden_size} * "
f"element size {element_size}"
)
curr_device = current_platform.get_device_capability()
device_capability = curr_device.to_int() if curr_device is not None else None
@@ -215,47 +219,6 @@ if flashinfer_comm is not None:
get_workspace_fn = (
get_fi_ar_quant_workspace if is_quant_pattern else get_fi_ar_workspace
)
orig_norm_out = norm_out
def unfused_fallback() -> None:
# Fused-path equivalent honoring the op's output-buffer contract:
# with norm_out given, normed -> norm_out and the new residual ->
# the allreduce_in buffer; otherwise normed -> allreduce_in and
# the new residual -> the residual buffer. Runs fine during CUDA
# graph capture (the branch is resolved per captured shape).
#
# weight_bias mirrors the fused kernel's gamma offset (Gemma
# patterns pass 1.0 for rms_norm(x, weight + 1)); fold it into
# the gamma in fp32 like the reference before the plain rms_norm.
gamma = (
rms_gamma
if weight_bias == 0.0
else (rms_gamma.float() + weight_bias).to(rms_gamma.dtype)
)
reduced = tensor_model_parallel_all_reduce(allreduce_in)
new_residual = reduced + residual
if orig_norm_out is None:
torch.ops._C.rms_norm(allreduce_in, new_residual, gamma, rms_eps)
residual.copy_(new_residual)
else:
torch.ops._C.rms_norm(orig_norm_out, new_residual, gamma, rms_eps)
allreduce_in.copy_(new_residual)
if oversize:
assert not is_quant_pattern, (
"flashinfer fused allreduce: input exceeds the workspace budget "
"and the unfused fallback does not cover quant patterns"
)
logger.warning_once(
"fused allreduce+rmsnorm: %d tokens exceeds the workspace "
"budget (max_token_num=%d); using the unfused fallback for "
"this shape.",
num_tokens,
max_token_num,
)
unfused_fallback()
return
workspace = get_workspace_fn(
world_size=world_size,
rank=get_tensor_model_parallel_rank(),
@@ -289,75 +252,36 @@ if flashinfer_comm is not None:
if workspace.backend in ("trtllm", "mnnvl"):
layout_code = flashinfer_comm.QuantizationSFLayout.SWIZZLED_128x4
# vllm's one-shot size table is tuned for the trtllm backend. For
# mnnvl, flashinfer caps the workspace's one-shot region at its own
# MNNVL_ONE_SHOT_THRESHOLD (~4MB, e.g. 44 tokens at hidden 6144 bf16
# TP8); forcing use_oneshot=True past that raises ValueError instead
# of using two-shot, so every decode capture between the one-shot cap
# and the fusion threshold silently lost fusion. Let flashinfer's
# AUTO heuristic pick the strategy (it matches its own workspace
# sizing), and keep trigger_completion_at_end=True since AUTO may
# select one-shot (see the Lamport NaN note below).
mnnvl_auto = workspace.backend == "mnnvl"
if mnnvl_auto:
use_oneshot = None
try:
flashinfer_comm.allreduce_fusion(
input=allreduce_in,
workspace=workspace,
pattern=pattern_code,
launch_with_pdl=launch_with_pdl,
output=None,
residual_out=residual_out,
norm_out=norm_out,
quant_out=quant_out,
scale_out=scale_out,
residual_in=residual,
rms_gamma=rms_gamma,
rms_eps=rms_eps,
scale_factor=scale_factor,
layout_code=layout_code,
use_oneshot=use_oneshot,
fp32_acc=fp32_acc,
weight_bias=weight_bias,
# The one-shot Lamport all-reduce signals PDL completion before
# its output buffer is committed when trigger_completion_at_end
# is False, so the next PDL-launched kernel can read the
# uninitialized Lamport buffer and produce NaN. This only fires
# for num_tokens <= PDL_ADVANCE_LAUNCH_TOKENS (the batch=1 /
# spec-decode shapes, where the one-shot path is always
# selected). Complete at the end for the one-shot path; the
# two-shot path is synchronized and keeps the early completion.
# Related one-shot instability in the same kernel:
# flashinfer-ai/flashinfer#1223.
trigger_completion_at_end=True
if mnnvl_auto
else (use_oneshot or num_tokens > PDL_ADVANCE_LAUNCH_TOKENS),
)
except ValueError as e:
# The globally cached workspace can be smaller than this call's
# budget (it is created once by whichever caller runs first, and
# the mnnvl workspace's is_buffer_size_sufficient() checks trtllm
# metadata, so vllm-side accounting cannot see the real capacity).
# flashinfer raises before launching anything, so falling back
# mid-capture is safe. Observed with spec-decode draft graphs
# whose token count (max_num_reqs * (1 + num_spec)) exceeds the
# workspace sized for the fusion threshold.
assert not is_quant_pattern, (
"flashinfer fused allreduce failed and the unfused fallback "
f"does not cover quant patterns: {e}"
)
logger.warning_once(
"flashinfer fused allreduce+rmsnorm unavailable for this "
"shape (num_tokens=%d, max_token_num=%d, world_size=%d): %s. "
"Using the unfused fallback.",
num_tokens,
max_token_num,
world_size,
e,
)
unfused_fallback()
flashinfer_comm.allreduce_fusion(
input=allreduce_in,
workspace=workspace,
pattern=pattern_code,
launch_with_pdl=launch_with_pdl,
output=None,
residual_out=residual_out,
norm_out=norm_out,
quant_out=quant_out,
scale_out=scale_out,
residual_in=residual,
rms_gamma=rms_gamma,
rms_eps=rms_eps,
scale_factor=scale_factor,
layout_code=layout_code,
use_oneshot=use_oneshot,
fp32_acc=fp32_acc,
weight_bias=weight_bias,
# The one-shot Lamport all-reduce signals PDL completion before its
# output buffer is committed when trigger_completion_at_end is
# False, so the next PDL-launched kernel can read the uninitialized
# Lamport buffer and produce NaN. This only fires for
# num_tokens <= PDL_ADVANCE_LAUNCH_TOKENS (the batch=1 / spec-decode
# shapes, where the one-shot path is always selected). Complete at
# the end for the one-shot path; the two-shot path is synchronized
# and keeps the early completion. Related one-shot instability in
# the same kernel: flashinfer-ai/flashinfer#1223.
trigger_completion_at_end=(use_oneshot is True)
or num_tokens > PDL_ADVANCE_LAUNCH_TOKENS,
)
def call_trtllm_fused_allreduce_norm_fake(
allreduce_in: torch.Tensor,
+1 -7
View File
@@ -330,7 +330,6 @@ class SpeculativeConfig:
@staticmethod
def hf_config_override(hf_config: PretrainedConfig) -> PretrainedConfig:
initial_architecture = hf_config.architectures[0]
use_sparse_mtp = hf_config.model_type == "glm_moe_dsa"
if hf_config.model_type in (
"deepseek_v3",
"deepseek_v32",
@@ -340,12 +339,7 @@ class SpeculativeConfig:
if hf_config.model_type == "deepseek_mtp":
n_predict = getattr(hf_config, "num_nextn_predict_layers", None)
hf_config.update(
{
"n_predict": n_predict,
"architectures": [
"DeepseekV32MTPModel" if use_sparse_mtp else "DeepSeekMTPModel"
],
}
{"n_predict": n_predict, "architectures": ["DeepSeekMTPModel"]}
)
if hf_config.model_type == "deepseek_v4":
hf_config.model_type = "deepseek_mtp"
-5
View File
@@ -33,11 +33,6 @@ EVICT_NORMAL = Int64(0x1000000000000000)
EVICT_FIRST = Int64(0x12F0000000000000)
EVICT_LAST = Int64(0x14F0000000000000)
TORCH_TO_CUTE_DTYPE = {
torch.bfloat16: BFloat16,
torch.float32: Float32,
}
@dsl_user_op
def recast_val(x, dtype, *, loc=None, ip=None):
+1 -22
View File
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from cutlass import BFloat16, Constexpr, Float32, Uint16, Uint32, cute
from cutlass import Constexpr, Float32, Uint32, cute
from cutlass._mlir import ir
from cutlass._mlir.dialects import llvm, vector
from cutlass.cutlass_dsl import T, dsl_user_op
@@ -38,11 +38,6 @@ def bf16x2_to_fp32x2(data, *, loc=None, ip=None) -> tuple[Float32, Float32]:
)
elif isinstance(data, (cute.Tensor, cute.TensorSSA)):
# recast to 32-bit registers if needed
if data.element_type == BFloat16:
data = cute.recast_tensor(data, Uint32)
assert data.element_type == Uint32
# NOTE: the output is always 1D
size = cute.size(data.shape)
out = cute.make_rmem_tensor(size * 2, Float32)
@@ -89,22 +84,6 @@ def fp8x4_to_bf16x4(x: Uint32, *, loc=None, ip=None) -> cute.TensorSSA:
return cute.TensorSSA(vec, 2, Uint32)
@dsl_user_op
def fp32x2_to_fp8x2(a0: Float32, a1: Float32, *, loc=None, ip=None) -> Uint16:
out = llvm.inline_asm(
T.i16(),
[
a0.ir_value(loc=loc, ip=ip),
a1.ir_value(loc=loc, ip=ip),
],
"cvt.rn.satfinite.e4m3x2.f32 $0, $2, $1;",
"=h,f,f",
has_side_effects=False,
is_align_stack=False,
)
return Uint16(out)
@dsl_user_op
def fp8x4_to_fp16x4(x: Uint32, *, loc=None, ip=None) -> cute.TensorSSA:
out = llvm.inline_asm(
@@ -43,7 +43,7 @@ try:
if flashinfer_comm is not None
else None
)
except (ImportError, AttributeError):
except ImportError:
flashinfer_trtllm_fused_allreduce_norm = None # type: ignore[assignment]
get_fi_ar_workspace = None # type: ignore[assignment]
_AR_RESIDUAL_RMS_NORM = None
@@ -52,23 +52,12 @@ except (ImportError, AttributeError):
_FI_SUPPORTED_DTYPES = (torch.bfloat16, torch.float16)
@torch.compiler.assume_constant_result
def _fi_ar_max_size_mb() -> dict[int, float]:
"""Flashinfer all-reduce fusion size table for the current device.
Device capability is constant; marking the result constant keeps this out
of the traced graph. Otherwise the per-forward call reaches
``current_platform.get_device_capability()`` inside ``torch.compile`` and
graph-breaks ("can't handle functions not implemented in python")."""
from vllm.config.compilation import PassConfig
return PassConfig.default_fi_allreduce_fusion_max_size_mb()
def _max_token_num(tp_size: int, hidden_size: int, dtype: torch.dtype) -> int | None:
"""Workspace token budget for flashinfer fused all-reduce, or None if the
current world size / device is unsupported. Mirrors ``FlashInferAllReduce``."""
max_size_mb = _fi_ar_max_size_mb().get(tp_size)
from vllm.config.compilation import PassConfig
max_size_mb = PassConfig.default_fi_allreduce_fusion_max_size_mb().get(tp_size)
if not max_size_mb:
return None
element_size = torch.tensor([], dtype=dtype).element_size()
+1 -2
View File
@@ -114,7 +114,7 @@ _TEXT_GENERATION_MODELS = {
"Glm4ForCausalLM": ("glm4", "Glm4ForCausalLM"),
"Glm4MoeForCausalLM": ("glm4_moe", "Glm4MoeForCausalLM"),
"Glm4MoeLiteForCausalLM": ("glm4_moe_lite", "Glm4MoeLiteForCausalLM"),
"GlmMoeDsaForCausalLM": ("vllm.models.deepseek_v32", "GlmMoeDsaForCausalLM"),
"GlmMoeDsaForCausalLM": ("deepseek_v2", "GlmMoeDsaForCausalLM"),
"GptOssForCausalLM": ("gpt_oss", "GptOssForCausalLM"),
"GPT2LMHeadModel": ("gpt2", "GPT2LMHeadModel"),
"GPTJForCausalLM": ("gpt_j", "GPTJForCausalLM"),
@@ -628,7 +628,6 @@ _SPECULATIVE_DECODING_MODELS = {
"Eagle3DeepseekV3ForCausalLM": ("deepseek_eagle3", "Eagle3DeepseekV2ForCausalLM"),
"EagleDeepSeekMTPModel": ("deepseek_eagle", "EagleDeepseekV3ForCausalLM"),
"DeepSeekMTPModel": ("deepseek_mtp", "DeepSeekMTP"),
"DeepseekV32MTPModel": ("vllm.models.deepseek_v32", "DeepseekV32MTP"),
"DeepSeekV4MTPModel": ("vllm.models.deepseek_v4", "DeepSeekV4MTP"),
"MiniMaxM3MTP": ("vllm.models.minimax_m3", "MiniMaxM3MTP"),
"BailingMoeV25MTPModel": ("bailing_moe_mtp", "BailingMoeV25MTPModel"),
+4 -18
View File
@@ -6,31 +6,17 @@ DeepSeek V3.2 introduced the DeepSeek Sparse Attention (DSA) architecture:
MLA + a "lightning indexer" that selects the top-k tokens for a sparse MLA
attend. The same model code serves any DSA checkpoint, including GLM-5.2
(``glm_moe_dsa``), which reuses this architecture.
The optimized kernels under ``nvidia/`` target the Blackwell (SM100) family.
Every other platform — ROCm, XPU, pre-SM100 CUDA (e.g. H100), CPU — falls back
to the generic ``deepseek_v2`` implementation, which already handles the DSA
(index_topk) architecture and is ``torch.compile``-friendly there. This matches
main's behavior on those platforms (no hard failure).
"""
from vllm.platforms import current_platform
if current_platform.is_cuda() and current_platform.is_device_capability_family(100):
from .nvidia.model import DeepseekV32ForCausalLM
from .nvidia.mtp import DeepseekV32MTP
if current_platform.is_rocm() or current_platform.is_xpu():
raise NotImplementedError("deepseek_v32 currently supports NVIDIA SM100 only.")
# GLM-5.2 (glm_moe_dsa) reuses the same optimized DSA module on SM100.
GlmMoeDsaForCausalLM = DeepseekV32ForCausalLM
else:
from vllm.model_executor.models.deepseek_mtp import DeepSeekMTP as DeepseekV32MTP
from vllm.model_executor.models.deepseek_v2 import (
DeepseekV3ForCausalLM as DeepseekV32ForCausalLM,
)
from vllm.model_executor.models.deepseek_v2 import GlmMoeDsaForCausalLM
from .nvidia.model import DeepseekV32ForCausalLM
from .nvidia.mtp import DeepseekV32MTP
__all__ = [
"DeepseekV32ForCausalLM",
"DeepseekV32MTP",
"GlmMoeDsaForCausalLM",
]
+12 -72
View File
@@ -4,7 +4,6 @@ import torch
import torch.nn as nn
from transformers import DeepseekV2Config, DeepseekV3Config
import vllm._custom_ops as ops
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
from vllm.config import CacheConfig, VllmConfig
from vllm.distributed import get_tensor_model_parallel_world_size
@@ -22,15 +21,17 @@ from vllm.model_executor.layers.quantization.utils.fp8_utils import (
per_token_group_quant_fp8,
)
from vllm.model_executor.layers.rotary_embedding import get_rope
from vllm.model_executor.layers.sparse_attn_indexer import SparseAttnIndexer
from vllm.model_executor.layers.sparse_attn_indexer import (
SparseAttnIndexer,
sparse_attn_indexer,
)
from vllm.model_executor.models.deepseek_v2 import (
DeepSeekV2FusedQkvAProjLinear,
DeepseekV32IndexerCache,
yarn_get_mscale,
)
from vllm.model_executor.models.utils import extract_layer_index
from vllm.platforms import current_platform
from vllm.utils.torch_utils import _encode_layer_name, is_quantized_kv_cache
from vllm.utils.torch_utils import is_quantized_kv_cache
from .kernels import fused_norm_rope, fused_q
@@ -170,11 +171,6 @@ class DeepseekV32Attention(MLAAttention):
) -> None:
quant_config = vllm_config.quant_config
cache_config = vllm_config.cache_config
if cache_config is not None and cache_config.cache_dtype == "auto":
# This implementation requires an FP8 sparse cache. Start with the
# generic FP8 format; MLAAttention will canonicalize it for the
# selected sparse backend.
cache_config.cache_dtype = "fp8"
hidden_size = config.hidden_size
qk_nope_head_dim = config.qk_nope_head_dim
@@ -335,37 +331,6 @@ class DeepseekV32Attention(MLAAttention):
rope_parameters=config.rope_parameters,
is_neox_style=False,
)
# Decode-M GEMM dispatch for the bf16 A/B projections, measured on
# B300 and B200 (graph replay + rotating weights vs fp64):
# bf16_skinny_gemm for M <= skinny_max, dsv3_fused_a_gemm up to
# M = 16, cuBLAS above. Blackwell-only: the boundaries and the
# fused_a tile choices are tied to the 148/152-SM single-wave
# geometry. DSv3.2 q_b has no skinny tier (K=1536 does not fit the
# GEMV's 128x8 K-step; fused_a still wins 1.6-1.8x there).
# Quantized linear methods may not register .weight until
# process_weights_after_loading (e.g. compressed-tensors NVFP4
# registers weight_packed at init) — probe with getattr.
# (N, K) -> skinny_max
qkv_a_dispatch = {(2624, 6144): 2, (2112, 7168): 2}
q_b_dispatch = {(2048, 2048): 2, (3072, 1536): 0}
ok = (
current_platform.is_device_capability_family(100)
and hasattr(torch.ops._C, "bf16_skinny_gemm")
and hasattr(torch.ops._C, "dsv3_fused_a_gemm")
)
qkv_a_weight = getattr(self.fused_qkv_a_proj, "weight", None)
q_b_weight = getattr(self.q_b_proj, "weight", None)
self._qkv_a_skinny_max = (
qkv_a_dispatch.get(tuple(qkv_a_weight.shape))
if ok and qkv_a_weight is not None and qkv_a_weight.dtype == torch.bfloat16
else None
)
self._q_b_skinny_max = (
q_b_dispatch.get(tuple(q_b_weight.shape))
if ok and q_b_weight is not None and q_b_weight.dtype == torch.bfloat16
else None
)
# Lightning indexer uses its own RoPE; interleave maps to non-NeoX.
self.indexer_rope_emb = get_rope(
qk_rope_head_dim,
@@ -374,32 +339,13 @@ class DeepseekV32Attention(MLAAttention):
is_neox_style=not getattr(config, "indexer_rope_interleave", False),
)
def _decode_m_gemm(
self, x: torch.Tensor, weight: torch.Tensor, skinny_max: int
) -> torch.Tensor:
"""x @ weight.T for decode M <= 16."""
if x.shape[0] == 0:
return torch.empty((0, weight.shape[0]), dtype=x.dtype, device=x.device)
if x.shape[0] <= skinny_max:
return ops.bf16_skinny_gemm(x, weight)
out = torch.empty((x.shape[0], weight.shape[0]), dtype=x.dtype, device=x.device)
ops.dsv3_fused_a_gemm(out, x, weight.t())
return out
def forward( # type: ignore[override]
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
) -> torch.Tensor:
# Captured: A-projections (+ indexer A-GEMM on indexer layers).
if self._qkv_a_skinny_max is not None and hidden_states.shape[0] <= 16:
qkv_lora = self._decode_m_gemm(
hidden_states,
self.fused_qkv_a_proj.weight,
self._qkv_a_skinny_max,
)
else:
qkv_lora = self.fused_qkv_a_proj(hidden_states)[0]
qkv_lora = self.fused_qkv_a_proj(hidden_states)[0]
q_c, kv_c, k_pe = qkv_lora.split(
[self.q_lora_rank, self.kv_lora_rank, self.qk_rope_head_dim], dim=-1
)
@@ -452,9 +398,7 @@ class DeepseekV32Attention(MLAAttention):
assert isinstance(slot_mapping, dict)
mla_slot = slot_mapping.get(self.layer_name)
# skip_topk (index_share_for_mtp_iteration reuse mode): don't run the
# indexer this step; keep the top-k already in the shared buffer.
if self.indexer is not None and not self.skip_topk:
if self.indexer is not None:
has_indexer = True
indexer_k_norm_w = self.indexer.k_norm.weight
indexer_k_norm_bias = self.indexer.k_norm.bias
@@ -507,16 +451,12 @@ class DeepseekV32Attention(MLAAttention):
index_rope_interleave=self._index_rope_interleave,
)
if self._q_b_skinny_max is not None and q_c.shape[0] <= 16:
q = self._decode_m_gemm(q_c, self.q_b_proj.weight, self._q_b_skinny_max)
else:
q = self.q_b_proj(q_c)[0]
q = q.view(-1, self.num_local_heads, self.qk_head_dim)
q = self.q_b_proj(q_c)[0].view(-1, self.num_local_heads, self.qk_head_dim)
q_nope, q_pe = q.split([self.qk_nope_head_dim, self.qk_rope_head_dim], dim=-1)
q_nope = q_nope.transpose(0, 1)
ql_nope = torch.bmm(q_nope, self.W_UK_T).transpose(0, 1)
if self.indexer is not None and not self.skip_topk:
if self.indexer is not None:
index_q = self.indexer.wq_b(q_c)[0]
index_q = index_q.view(-1, self.indexer.n_head, self.indexer.head_dim)
else:
@@ -538,10 +478,10 @@ class DeepseekV32Attention(MLAAttention):
quantize_mqa=self._fp8_query,
)
if self.indexer is not None and not self.skip_topk:
torch.ops.vllm.sparse_attn_indexer(
if self.indexer is not None:
sparse_attn_indexer(
q_c,
_encode_layer_name(self.indexer.k_cache.prefix),
self.indexer.k_cache.prefix,
self.indexer.k_cache.kv_cache,
index_q_fp8,
None, # q_scale folded into weights on the fp8 path
+16 -284
View File
@@ -2,80 +2,13 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
from vllm.utils.import_utils import has_cutedsl
from vllm.utils.torch_utils import direct_register_custom_op
# Cache of tiny 1-element dummy tensors (per device, dtype) reused by the
# has_indexer=False path so the indexer args don't allocate every call.
_DUMMY_CACHE: dict[tuple, torch.Tensor] = {}
@torch.compiler.assume_constant_result
def _can_use_fused_q_cutedsl() -> bool:
return current_platform.has_device_capability(100) and has_cutedsl()
@torch.compiler.assume_constant_result
def _is_arch_support_pdl() -> bool:
return (
current_platform.has_device_capability(100)
and current_platform.is_arch_support_pdl()
)
def _fused_q_cutedsl_impl(
positions: torch.Tensor,
q_pe: torch.Tensor,
rope_cache: torch.Tensor,
ql_nope: torch.Tensor,
q_scale: torch.Tensor,
mqa_output: torch.Tensor,
idx_q: torch.Tensor,
idx_rope_cache: torch.Tensor,
idx_weights: torch.Tensor,
idx_weights_softmax_scale: float,
idx_weights_head_scale: float,
idx_q_fp8: torch.Tensor,
idx_weights_out: torch.Tensor,
has_indexer: bool,
index_rope_interleave: bool,
) -> None:
from .ops.fused_q_cutedsl import fused_q_cutedsl
fused_q_cutedsl(
positions,
q_pe,
rope_cache,
ql_nope,
q_scale,
mqa_output,
idx_q,
idx_rope_cache,
idx_weights,
idx_weights_softmax_scale,
idx_weights_head_scale,
idx_q_fp8,
idx_weights_out,
has_indexer=has_indexer,
index_rope_interleave=index_rope_interleave,
)
def _fused_q_cutedsl_fake(*args, **kwargs) -> None:
pass
direct_register_custom_op(
op_name="fused_q_cutedsl",
op_func=_fused_q_cutedsl_impl,
mutates_args=["mqa_output", "idx_q_fp8", "idx_weights_out"],
fake_impl=_fused_q_cutedsl_fake,
dispatch_key="CUDA",
)
def _dummy(shape: tuple, dtype: torch.dtype, device: torch.device) -> torch.Tensor:
key = (shape, dtype, device)
t = _DUMMY_CACHE.get(key)
@@ -214,14 +147,9 @@ def _fused_norm_rope_kernel(
TOPK_BLOCK_SIZE: tl.constexpr,
HAS_INDEXER: tl.constexpr,
INDEX_ROPE_INTERLEAVE: tl.constexpr,
USE_PDL: tl.constexpr,
):
pid = tl.program_id(0)
tok_idx = tl.program_id(1)
if USE_PDL:
tl.extra.cuda.gdc_wait()
tl.extra.cuda.gdc_launch_dependents()
if pid == 3:
if not HAS_INDEXER:
# Shared layer: reuse the previous indexer layer's top-k; do not
@@ -238,17 +166,6 @@ def _fused_norm_rope_kernel(
)
return
if pid == 2:
# Q RMS norm does not depend on cache availability. In particular,
# profiling uses a negative dummy slot mapping but still consumes Q.
q_block = tl.arange(0, Q_BLOCK_SIZE)
q_mask = q_block < Q_DIM
q_c = tl.load(q_c_ptr + tok_idx * q_c_stride + q_block, mask=q_mask, other=0.0)
q_c_rms_w = tl.load(q_rms_norm_w_ptr + q_block, mask=q_mask)
q_c = _rms_norm(q_c, q_c_rms_w, q_rms_eps, Q_DIM)
tl.store(q_c_out_ptr + tok_idx * q_c_out_stride + q_block, q_c, mask=q_mask)
return
if slot_mapping_ptr is None:
# Memory profiling run.
return
@@ -257,7 +174,15 @@ def _fused_norm_rope_kernel(
# Padding
return
if pid == 1:
if pid == 2:
# Q RMS norm
q_block = tl.arange(0, Q_BLOCK_SIZE)
q_mask = q_block < Q_DIM
q_c = tl.load(q_c_ptr + tok_idx * q_c_stride + q_block, mask=q_mask, other=0.0)
q_c_rms_w = tl.load(q_rms_norm_w_ptr + q_block, mask=q_mask)
q_c = _rms_norm(q_c, q_c_rms_w, q_rms_eps, Q_DIM)
tl.store(q_c_out_ptr + tok_idx * q_c_out_stride + q_block, q_c, mask=q_mask)
elif pid == 1:
# KV RMS Norm + KV RoPE + MLA concat_and_cache.
# Merged so the normed kv_c and RoPE'd k_pe can be written
# to the MLA KV cache directly without a separate kernel.
@@ -286,6 +211,9 @@ def _fused_norm_rope_kernel(
r2 = x2 * cos + x1 * sin
# MLA concat_and_cache: write [kv_c_normed, k_pe_roped] to cache.
if mla_cache_entry_stride == 0:
return
mla_block_size = mla_cache_block_stride // mla_cache_entry_stride
mla_block_idx = slot_idx // mla_block_size
mla_block_off = slot_idx % mla_block_size
@@ -435,7 +363,7 @@ def _fused_norm_rope_kernel(
)
def _fused_norm_rope_impl(
def fused_norm_rope(
positions: torch.Tensor,
q_c: torch.Tensor,
q_rms_norm_w: torch.Tensor,
@@ -537,17 +465,12 @@ def _fused_norm_rope_impl(
# Dummy values — pid 2 will skip the MLA cache write because
# slot_mapping is all -1.
mla_kv_cache = torch.empty(0, dtype=torch.bfloat16, device=device)
# Torch's Triton HOP mutation analysis compiles every program-id
# branch even though the dummy slot mapping returns before cache
# access. Keep the unused strides nonzero so that analysis can lower
# the block-size calculation safely.
mla_block_stride = 1
mla_entry_stride = 1
mla_block_stride = 0
mla_entry_stride = 0
mla_k_scale = torch.ones(1, dtype=torch.float32, device=device)
if q_c_out is None:
q_c_out = torch.empty_like(q_c)
use_pdl = _is_arch_support_pdl()
_fused_norm_rope_kernel[(4, num_tokens)](
positions,
# Q RMS norm
@@ -606,157 +529,6 @@ def _fused_norm_rope_impl(
TOPK_BLOCK_SIZE=1024,
HAS_INDEXER=has_indexer,
INDEX_ROPE_INTERLEAVE=index_rope_interleave,
USE_PDL=use_pdl,
launch_pdl=use_pdl,
)
return q_c_out
def _fused_norm_rope_op(
positions: torch.Tensor,
q_c: torch.Tensor,
q_rms_norm_w: torch.Tensor,
q_rms_eps: float,
kv_c: torch.Tensor,
kv_rms_norm_w: torch.Tensor,
kv_rms_eps: float,
k_pe: torch.Tensor,
k_rope_cos_sin_cache: torch.Tensor,
index_k: torch.Tensor | None,
index_k_layer_norm_w: torch.Tensor | None,
index_k_layer_norm_bias: torch.Tensor | None,
index_k_layer_norm_eps: float,
index_k_rope_cos_sin_cache: torch.Tensor | None,
topk_indices_buffer: torch.Tensor,
slot_mapping: torch.Tensor | None,
indexer_k_cache: torch.Tensor | None,
mla_kv_cache: torch.Tensor | None,
mla_kv_cache_dtype: str,
mla_k_scale: torch.Tensor | None,
has_indexer: bool,
index_rope_interleave: bool,
q_c_out: torch.Tensor,
) -> None:
_fused_norm_rope_impl(
positions,
q_c,
q_rms_norm_w,
q_rms_eps,
kv_c,
kv_rms_norm_w,
kv_rms_eps,
k_pe,
k_rope_cos_sin_cache,
index_k,
index_k_layer_norm_w,
index_k_layer_norm_bias,
index_k_layer_norm_eps,
index_k_rope_cos_sin_cache,
topk_indices_buffer,
slot_mapping=slot_mapping,
indexer_k_cache=indexer_k_cache,
mla_kv_cache=mla_kv_cache,
mla_kv_cache_dtype=mla_kv_cache_dtype,
mla_k_scale=mla_k_scale,
has_indexer=has_indexer,
index_rope_interleave=index_rope_interleave,
q_c_out=q_c_out,
)
def _fused_norm_rope_fake(
positions: torch.Tensor,
q_c: torch.Tensor,
q_rms_norm_w: torch.Tensor,
q_rms_eps: float,
kv_c: torch.Tensor,
kv_rms_norm_w: torch.Tensor,
kv_rms_eps: float,
k_pe: torch.Tensor,
k_rope_cos_sin_cache: torch.Tensor,
index_k: torch.Tensor | None,
index_k_layer_norm_w: torch.Tensor | None,
index_k_layer_norm_bias: torch.Tensor | None,
index_k_layer_norm_eps: float,
index_k_rope_cos_sin_cache: torch.Tensor | None,
topk_indices_buffer: torch.Tensor,
slot_mapping: torch.Tensor | None,
indexer_k_cache: torch.Tensor | None,
mla_kv_cache: torch.Tensor | None,
mla_kv_cache_dtype: str,
mla_k_scale: torch.Tensor | None,
has_indexer: bool,
index_rope_interleave: bool,
q_c_out: torch.Tensor,
) -> None:
return
direct_register_custom_op(
op_name="fused_norm_rope_deepseek_v32",
op_func=_fused_norm_rope_op,
mutates_args=[
"topk_indices_buffer",
"indexer_k_cache",
"mla_kv_cache",
"q_c_out",
],
fake_impl=_fused_norm_rope_fake,
dispatch_key="CUDA",
)
def fused_norm_rope(
positions: torch.Tensor,
q_c: torch.Tensor,
q_rms_norm_w: torch.Tensor,
q_rms_eps: float,
kv_c: torch.Tensor,
kv_rms_norm_w: torch.Tensor,
kv_rms_eps: float,
k_pe: torch.Tensor,
k_rope_cos_sin_cache: torch.Tensor,
index_k: torch.Tensor | None,
index_k_layer_norm_w: torch.Tensor | None,
index_k_layer_norm_bias: torch.Tensor | None,
index_k_layer_norm_eps: float,
index_k_rope_cos_sin_cache: torch.Tensor | None,
topk_indices_buffer: torch.Tensor,
slot_mapping: torch.Tensor | None = None,
indexer_k_cache: torch.Tensor | None = None,
mla_kv_cache: torch.Tensor | None = None,
mla_kv_cache_dtype: str = "auto",
mla_k_scale: torch.Tensor | None = None,
has_indexer: bool = True,
index_rope_interleave: bool = False,
q_c_out: torch.Tensor | None = None,
) -> torch.Tensor:
if q_c_out is None:
q_c_out = torch.empty_like(q_c)
torch.ops.vllm.fused_norm_rope_deepseek_v32(
positions,
q_c,
q_rms_norm_w,
q_rms_eps,
kv_c,
kv_rms_norm_w,
kv_rms_eps,
k_pe,
k_rope_cos_sin_cache,
index_k,
index_k_layer_norm_w,
index_k_layer_norm_bias,
index_k_layer_norm_eps,
index_k_rope_cos_sin_cache,
topk_indices_buffer,
slot_mapping,
indexer_k_cache,
mla_kv_cache,
mla_kv_cache_dtype,
mla_k_scale,
has_indexer,
index_rope_interleave,
q_c_out,
)
return q_c_out
@@ -810,14 +582,10 @@ def _fused_q_kernel(
HAS_INDEXER: tl.constexpr,
INDEX_ROPE_INTERLEAVE: tl.constexpr,
QUANTIZE_MQA: tl.constexpr,
USE_PDL: tl.constexpr,
):
pid = tl.program_id(0)
tok_idx = tl.program_id(1)
head_idx = tl.program_id(2)
if USE_PDL:
tl.extra.cuda.gdc_wait()
tl.extra.cuda.gdc_launch_dependents()
if pid == 2:
# ql_nope quantize + pack into the front of mqa_q_fp8. On the bf16
@@ -1005,12 +773,10 @@ def fused_q(
``(ql_nope, q_pe)`` tuple the backend expects.
"""
assert positions.ndim == 1
assert positions.dtype == torch.int64
assert q_pe.ndim == 3
assert q_pe_cos_sin_cache.ndim == 2
assert ql_nope.ndim == 3
assert ql_nope.shape[:2] == q_pe.shape[:2]
assert q_scale.dtype == torch.float32 and q_scale.numel() == 1
num_tokens = positions.shape[0]
num_q_heads = q_pe.shape[1]
@@ -1028,18 +794,7 @@ def fused_q(
assert index_weights is not None
num_index_q_heads = index_q.shape[1]
index_q_head_dim = index_q.shape[2]
use_cutedsl = False
if _can_use_fused_q_cutedsl():
from .ops.fused_q_cutedsl import is_fused_q_cutedsl_supported
use_cutedsl = is_fused_q_cutedsl_supported(
q_pe,
index_q,
ql_nope,
has_indexer=has_indexer,
quantize_mqa=quantize_mqa,
)
grid_heads = max(mqa_grid_heads, num_index_q_heads if has_indexer else 1)
grid_heads = max(mqa_grid_heads, num_index_q_heads)
if quantize_mqa:
# fp8 path: pack [ql_nope; q_pe] into a single fp8 tensor.
mqa_q_fp8 = torch.empty(
@@ -1060,27 +815,6 @@ def fused_q(
index_q_fp8 = torch.empty_like(index_q, dtype=torch.float8_e4m3fn)
index_weights_out = torch.empty_like(index_weights, dtype=torch.float32)
if use_cutedsl:
torch.ops.vllm.fused_q_cutedsl(
positions,
q_pe,
q_pe_cos_sin_cache,
ql_nope,
q_scale,
mqa_q,
index_q,
index_q_cos_sin_cache,
index_weights,
index_weights_softmax_scale,
index_weights_head_scale,
index_q_fp8,
index_weights_out,
has_indexer=has_indexer,
index_rope_interleave=index_rope_interleave,
)
return index_q_fp8, index_weights_out, mqa_q
use_pdl = _is_arch_support_pdl()
_fused_q_kernel[(3, num_tokens, grid_heads)](
positions,
q_pe,
@@ -1122,8 +856,6 @@ def fused_q(
HAS_INDEXER=has_indexer,
INDEX_ROPE_INTERLEAVE=index_rope_interleave,
QUANTIZE_MQA=quantize_mqa,
USE_PDL=use_pdl,
launch_pdl=use_pdl,
# num_warps=1 is optimal here: each program is a single 128-element
# rope+quant, so the kernel is program-count/occupancy bound, not
# per-program compute bound (swept 1/2/4/8 — 1 wins or ties everywhere).
-4
View File
@@ -6,7 +6,6 @@ from itertools import islice
import torch
from vllm.compilation.decorators import support_torch_compile
from vllm.config import VllmConfig
from vllm.distributed import (
get_pp_group,
@@ -40,7 +39,6 @@ from vllm.model_executor.models.utils import (
sequence_parallel_chunk,
)
from vllm.sequence import IntermediateTensors
from vllm.v1.attention.backends.mla.sparse_utils import register_phys_shadow
from .attention import DeepseekV32Attention
from .fused_ops import fused_allreduce_rms_norm
@@ -179,7 +177,6 @@ class DeepseekV32DecoderLayer(torch.nn.Module):
return hidden_states, residual
@support_torch_compile
class DeepseekV32Model(torch.nn.Module):
fall_back_to_pt_during_load = False
@@ -203,7 +200,6 @@ class DeepseekV32Model(torch.nn.Module):
dtype=torch.int32,
device=self.device,
)
register_phys_shadow(topk_indices_buffer)
if get_pp_group().is_first_rank:
self.embed_tokens = VocabParallelEmbedding(
+6 -76
View File
@@ -6,9 +6,9 @@ from collections.abc import Callable, Iterable
import torch
import torch.nn as nn
import vllm._custom_ops as ops
from vllm._aiter_ops import rocm_aiter_ops
from vllm.config import VllmConfig
from vllm.distributed import tensor_model_parallel_all_reduce
from vllm.model_executor.layers.fused_moe import (
fused_moe_make_expert_params_mapping,
)
@@ -40,12 +40,7 @@ from vllm.model_executor.models.utils import (
)
from vllm.platforms import current_platform
from vllm.sequence import IntermediateTensors
from vllm.v1.attention.backends.mla.sparse_utils import (
phys_shadow,
register_phys_shadow,
)
from .fused_ops import fused_allreduce_rms_norm
from .kernels import fused_eh_norm
from .model import DeepseekV32DecoderLayer
@@ -61,19 +56,6 @@ class DeepseekV32MultiTokenPredictorLayer(nn.Module):
self.enorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.hnorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.eh_proj = nn.Linear(config.hidden_size * 2, config.hidden_size, bias=False)
# bf16 skinny GEMM for the eh_proj, B300+B200 measured: GLM-5.2
# (6144, 12288) wins at M <= 3, DSv3.2 (7168, 14336) at M <= 2;
# cuBLAS holds above (the 151/205MB weights stream at ~5.7TB/s).
eh_dispatch = {(6144, 12288): 3, (7168, 14336): 2}
eh_weight = getattr(self.eh_proj, "weight", None)
self._eh_skinny_max = (
eh_dispatch.get(tuple(eh_weight.shape), 0)
if current_platform.is_device_capability_family(100)
and hasattr(torch.ops._C, "bf16_skinny_gemm")
and eh_weight is not None
and eh_weight.dtype == torch.bfloat16
else 0
)
topk_indices_buffer = torch.empty(
vllm_config.scheduler_config.max_num_batched_tokens,
@@ -81,7 +63,6 @@ class DeepseekV32MultiTokenPredictorLayer(nn.Module):
dtype=torch.int32,
device=current_platform.device_type,
)
register_phys_shadow(topk_indices_buffer)
self.shared_head = SharedHead(
config=config, prefix=prefix, quant_config=quant_config
)
@@ -110,13 +91,7 @@ class DeepseekV32MultiTokenPredictorLayer(nn.Module):
self.hnorm.weight,
self.enorm.variance_epsilon,
)
if (
eh_input.shape[0] <= self._eh_skinny_max
and eh_input.dtype == torch.bfloat16
):
hidden_states = ops.bf16_skinny_gemm(eh_input, self.eh_proj.weight)
else:
hidden_states = self.eh_proj(eh_input)
hidden_states = self.eh_proj(eh_input)
hidden_states, residual = self.mtp_block(
positions=positions, hidden_states=hidden_states, residual=None
)
@@ -126,6 +101,9 @@ class DeepseekV32MultiTokenPredictorLayer(nn.Module):
positions.shape[0],
is_sequence_parallel=self.mtp_block.use_sequence_parallel_moe,
)
if not self.mtp_block.use_sequence_parallel_moe:
# Without sequence parallelism, the MoE output is left un-reduced.
hidden_states = tensor_model_parallel_all_reduce(hidden_states)
# Recycle the POST-final-norm hidden into the next draft step. The
# residual-add is fused into the final RMSNorm so it is computed
# exactly once, and the result is returned for both tuple positions:
@@ -136,14 +114,7 @@ class DeepseekV32MultiTokenPredictorLayer(nn.Module):
# is understood by both the V2 speculator (isinstance-tuple check) and
# the legacy proposer (model_returns_tuple is True for the
# DeepSeekMTPModel architecture).
if self.mtp_block.use_sequence_parallel_moe:
hidden_states, _ = self.shared_head.norm(hidden_states, residual)
else:
# The MoE output is left un-reduced; fuse its all-reduce into the
# final norm, as the main model does at layer boundaries.
hidden_states, _ = fused_allreduce_rms_norm(
hidden_states, residual, self.shared_head.norm
)
hidden_states, _ = self.shared_head.norm(hidden_states, residual)
return hidden_states, hidden_states
@@ -186,15 +157,6 @@ class DeepseekV32MultiTokenPredictor(nn.Module):
if self_attn is not None and hasattr(self_attn, "topk_indices_buffer"):
topk_indices_buffer = self_attn.topk_indices_buffer
topk_indices_buffer[:num_slots] = topk_indices_buffer[slot_ids]
# The physical-index shadow mirrors this buffer row-wise
# (shadow[j] == convert(logical[j])); re-arrange it the same
# way or skip_topk draft steps read the step-0 multi-token
# layout's rows for the wrong tokens/requests.
shadow = phys_shadow(topk_indices_buffer)
if shadow is not None:
phys, seq = shadow
phys[:num_slots] = phys[slot_ids]
seq[:num_slots] = seq[slot_ids]
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.embed_tokens(input_ids)
@@ -230,19 +192,6 @@ class DeepseekV32MultiTokenPredictor(nn.Module):
# second RMSNorm.
return self.logits_processor(mtp_layer.shared_head.head, hidden_states)
def get_top_tokens(
self,
hidden_states: torch.Tensor,
spec_step_idx: int = 0,
) -> torch.Tensor:
# Greedy draft token ids via vocab-parallel local argmax: skips the
# full-vocab logits all-gather (use_local_argmax_reduction).
current_step_idx = spec_step_idx % self.num_mtp_layers
mtp_layer = self.layers[str(self.mtp_start_layer_idx + current_step_idx)]
return self.logits_processor.get_top_tokens(
mtp_layer.shared_head.head, hidden_states
)
class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
@@ -291,25 +240,6 @@ class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts):
) -> torch.Tensor | None:
return self.model.compute_logits(hidden_states, spec_step_idx)
def get_top_tokens(
self,
hidden_states: torch.Tensor,
spec_step_idx: int | None = None,
) -> torch.Tensor:
# The V2 speculator calls this without spec_step_idx; that is only
# correct while a single MTP layer is cycled for every draft step
# (num_nextn_predict_layers == 1, as in GLM-5.2/DSV3.2). Guard the
# assumption so a future multi-layer MTP fails loudly instead of
# silently using step 0's head for every step.
if spec_step_idx is None:
assert self.model.num_mtp_layers == 1, (
"get_top_tokens called without spec_step_idx on a "
"multi-layer MTP; thread the draft step index through "
"the speculator."
)
spec_step_idx = 0
return self.model.get_top_tokens(hidden_states, spec_step_idx)
def _rewrite_spec_layer_name(self, spec_layer: int, name: str) -> str:
spec_layer_weight_names = [
"embed_tokens",
@@ -1,2 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
@@ -1,532 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from functools import cache
import cutlass
import cutlass.cute as cute
import torch
from cuda.bindings.driver import CUstream
from cutlass import BFloat16, Float8E4M3FN, Float32, Int64, Uint8, Uint16, Uint32
from vllm.cute_utils import TORCH_TO_CUTE_DTYPE, cvt
def _make_fake_tensor(dtype, shape, divisibility):
stride = tuple(
1 if i == len(shape) - 1 else cute.sym_int64(divisibility=divisibility)
for i in range(len(shape))
)
return cute.runtime.make_fake_tensor(
dtype,
shape,
stride,
assumed_align=divisibility * dtype.width // 8,
)
def is_fused_q_cutedsl_supported(
q_pe: torch.Tensor,
index_q: torch.Tensor | None,
ql_nope: torch.Tensor,
*,
has_indexer: bool,
quantize_mqa: bool,
) -> bool:
if not (
quantize_mqa
and q_pe.dtype == ql_nope.dtype == torch.bfloat16
and q_pe.shape[-1] == 64
and ql_nope.shape[-1] == 512
):
return False
return not has_indexer or (
index_q is not None
and index_q.dtype == torch.bfloat16
and index_q.shape[1] % 16 == 0
and index_q.shape[-1] == 128
)
def fused_q_cutedsl(
positions: torch.Tensor,
q_pe: torch.Tensor,
rope_cache: torch.Tensor,
ql_nope: torch.Tensor,
q_scale: torch.Tensor,
mqa_output: torch.Tensor,
idx_q: torch.Tensor,
idx_rope_cache: torch.Tensor,
idx_weights: torch.Tensor,
idx_weights_softmax_scale: float,
idx_weights_head_scale: float,
idx_q_fp8: torch.Tensor,
idx_weights_out: torch.Tensor,
has_indexer: bool = True,
index_rope_interleave: bool = True,
) -> None:
_, num_heads, rope_dim = q_pe.shape
_, _, nope_dim = ql_nope.shape
_, num_idx_heads, idx_dim = idx_q.shape
if has_indexer:
idx_rope_type = TORCH_TO_CUTE_DTYPE[idx_rope_cache.dtype]
idx_weights_type = TORCH_TO_CUTE_DTYPE[idx_weights.dtype]
else:
idx_dim = num_idx_heads = 0
idx_q = idx_rope_cache = idx_q_fp8 = None
idx_weights = idx_weights_out = None
idx_rope_type = idx_weights_type = None
rope_type = TORCH_TO_CUTE_DTYPE[rope_cache.dtype]
compiled = FusedQKernel.compile(
rope_dim,
nope_dim,
num_heads,
rope_type,
idx_dim,
num_idx_heads,
idx_rope_type,
idx_weights_type,
index_rope_interleave,
)
compiled(
positions,
q_pe,
rope_cache,
ql_nope,
q_scale.view(1),
mqa_output,
idx_q,
idx_rope_cache,
idx_weights,
idx_q_fp8,
idx_weights_out,
float(idx_weights_softmax_scale * idx_weights_head_scale),
)
class FusedQKernel:
def __init__(
self,
rope_dim: int,
nope_dim: int,
num_heads: int,
idx_dim: int,
num_idx_heads: int,
index_rope_interleave: bool,
) -> None:
assert rope_dim == 64
assert nope_dim == 512
assert idx_dim in (128, 0)
self.rope_dim = rope_dim
self.nope_dim = nope_dim
self.num_heads = num_heads
self.idx_dim = idx_dim
self.num_idx_heads = num_idx_heads
self.index_rope_interleave = index_rope_interleave
# mqa: rope_dim=64, nope_dim=512, num_heads=64/TP
# indexer: rope_dim=64, nope_dim=64, num_heads=32
self.num_warps = 4
assert num_heads % self.num_warps == 0
assert num_idx_heads % (4 * self.num_warps) == 0
self.num_ctas_per_tok = num_heads // self.num_warps
self.num_ctas_per_idx_tok = num_idx_heads // (4 * self.num_warps)
@cute.jit
def __call__(
self,
positions: cute.Tensor,
q_pe: cute.Tensor,
rope_cache: cute.Tensor,
ql_nope: cute.Tensor,
q_scale: cute.Tensor,
mqa_output: cute.Tensor,
idx_q: cute.Tensor,
idx_rope_cache: cute.Tensor,
idx_weights: cute.Tensor,
idx_q_fp8: cute.Tensor,
idx_weights_out: cute.Tensor,
weight_scale: Float32,
stream: CUstream,
):
num_tokens = positions.shape[0]
if cutlass.const_expr(self.idx_dim == 0):
grid = (num_tokens, self.num_ctas_per_tok, 1)
else:
num_mqa_ctas = num_tokens * self.num_ctas_per_tok
num_idx_ctas = num_tokens * self.num_ctas_per_idx_tok
grid = (num_mqa_ctas + num_idx_ctas, 1, 1)
self.kernel(
positions,
q_pe,
rope_cache,
ql_nope,
q_scale,
mqa_output,
idx_q,
idx_rope_cache,
idx_weights,
idx_q_fp8,
idx_weights_out,
weight_scale,
).launch(
grid=grid,
block=(self.num_warps * 32, 1, 1),
stream=stream,
use_pdl=True,
)
@cute.kernel
def kernel(
self,
positions: cute.Tensor,
q_pe: cute.Tensor,
q_pe_rope_cache: cute.Tensor,
ql_nope: cute.Tensor,
q_scale: cute.Tensor,
mqa_output: cute.Tensor,
idx_q: cute.Tensor,
idx_q_rope_cache: cute.Tensor,
idx_weights: cute.Tensor,
idx_q_fp8: cute.Tensor,
idx_weights_out: cute.Tensor,
weight_scale: Float32,
):
if cutlass.const_expr(self.idx_dim == 0):
token_id, group_id, _ = cute.arch.block_idx()
self.mqa(
positions,
q_pe,
q_pe_rope_cache,
ql_nope,
q_scale,
mqa_output,
token_id,
group_id,
)
else:
# CTA-specialization
bid, _, _ = cute.arch.block_idx()
num_mqa_ctas = positions.shape[0] * self.num_ctas_per_tok
if bid < num_mqa_ctas:
self.mqa(
positions,
q_pe,
q_pe_rope_cache,
ql_nope,
q_scale,
mqa_output,
bid // self.num_ctas_per_tok,
bid % self.num_ctas_per_tok,
)
else:
bid -= num_mqa_ctas
self.indexer(
positions,
idx_q,
idx_q_rope_cache,
idx_weights,
idx_q_fp8,
idx_weights_out,
weight_scale,
bid // self.num_ctas_per_idx_tok,
bid % self.num_ctas_per_idx_tok,
)
@cute.jit
def mqa(
self,
positions: cute.Tensor,
q_pe: cute.Tensor,
q_pe_rope_cache: cute.Tensor,
ql_nope: cute.Tensor,
q_scale: cute.Tensor,
mqa_output: cute.Tensor,
token_id,
group_id,
):
tid, _, _ = cute.arch.thread_idx()
warp_id = cute.arch.make_warp_uniform(tid // 32)
lane_id = tid % 32
head_id = group_id * self.num_warps + warp_id
cute.arch.griddepcontrol_wait()
pos = positions[token_id]
inv_scale = 1.0 / q_scale[0]
cp_op = cute.nvgpu.CopyUniversalOp()
cp_32B = cute.make_copy_atom(cp_op, Uint32, num_bits_per_copy=256)
cp_16B = cute.make_copy_atom(cp_op, Uint32, num_bits_per_copy=128)
cp_4B = cute.make_copy_atom(cp_op, Uint32, num_bits_per_copy=32)
cp_2B = cute.make_copy_atom(cp_op, Uint8, num_bits_per_copy=16)
##### issue all loads asap #####
rQ_nope_bf16 = cute.make_rmem_tensor(16, BFloat16)
rQ_rope_bf16 = cute.make_rmem_tensor(2, BFloat16)
src_ql_nope = cute.local_tile(
ql_nope[token_id, head_id, None], (16,), (lane_id,)
)
src_q_rope = cute.local_tile(q_pe[token_id, head_id, None], (2,), (lane_id,))
cute.copy(cp_32B, src_ql_nope, rQ_nope_bf16)
cute.copy(cp_4B, src_q_rope, rQ_rope_bf16)
rCos_raw = q_pe_rope_cache[pos, 0 + lane_id]
rSin_raw = q_pe_rope_cache[pos, 32 + lane_id]
##### process NoPE #####
rQ_nope_f32 = cvt.bf16x2_to_fp32x2(rQ_nope_bf16).load() * inv_scale
rQ_nope_f8 = cute.make_rmem_tensor(16, Float8E4M3FN)
rQ_nope_f8.store(rQ_nope_f32.to(Float8E4M3FN))
dst_Q_nope = cute.local_tile(
mqa_output[token_id, head_id, None], (16,), (lane_id,)
)
cute.copy(cp_16B, rQ_nope_f8, dst_Q_nope)
##### process RoPE ######
rQ_rope_f32 = cvt.bf16x2_to_fp32x2(rQ_rope_bf16)
rCos = rCos_raw.to(Float32)
rSin = rSin_raw.to(Float32)
r0 = (rQ_rope_f32[0] * rCos - rQ_rope_f32[1] * rSin) * inv_scale
r1 = (rQ_rope_f32[1] * rCos + rQ_rope_f32[0] * rSin) * inv_scale
cute.arch.griddepcontrol_launch_dependents()
# TensorSSA fp32->fp8 cvt has a bug. rely on direct PTX
rQ_rope_f8 = cute.make_rmem_tensor(2, Float8E4M3FN)
cute.recast_tensor(rQ_rope_f8, Uint16)[0] = cvt.fp32x2_to_fp8x2(r0, r1)
dst_Q_rope = cute.local_tile(
mqa_output[token_id, head_id, None], (2,), (256 + lane_id,)
)
cute.copy(cp_2B, rQ_rope_f8, dst_Q_rope)
@cute.jit
def indexer(
self,
positions: cute.Tensor,
idx_q: cute.Tensor,
idx_q_rope_cache: cute.Tensor,
idx_weights: cute.Tensor,
idx_q_fp8: cute.Tensor,
idx_weights_out: cute.Tensor,
weight_scale: Float32,
token_id,
group_id,
):
tid, _, _ = cute.arch.thread_idx()
subwarp_id = tid // 8
sublane_id = tid % 8
head_id = group_id * (4 * self.num_warps) + subwarp_id
cute.arch.griddepcontrol_wait()
pos = positions[token_id]
cp_op = cute.nvgpu.CopyUniversalOp()
cp_16B = cute.make_copy_atom(cp_op, Uint32, num_bits_per_copy=128)
cp_8B = cute.make_copy_atom(cp_op, Uint32, num_bits_per_copy=64)
cp_4B = cute.make_copy_atom(cp_op, Uint32, num_bits_per_copy=32)
##### issue all loads first #####
rQ_rope_bf16 = cute.make_rmem_tensor(8, BFloat16)
if cutlass.const_expr(self.index_rope_interleave):
src_idx_q_rope = cute.local_tile(
idx_q[token_id, head_id, None], (8,), (sublane_id,)
)
cute.copy(cp_16B, src_idx_q_rope, rQ_rope_bf16)
else:
src_idx_q_rope = cute.zipped_divide(
idx_q[token_id, head_id, None], (4,)
) # (4,32)
cute.copy(
cp_8B,
src_idx_q_rope[None, 0 + sublane_id],
cute.local_tile(rQ_rope_bf16, (4,), (0,)),
)
cute.copy(
cp_8B,
src_idx_q_rope[None, 8 + sublane_id],
cute.local_tile(rQ_rope_bf16, (4,), (1,)),
)
rQ_nope_bf16 = cute.make_rmem_tensor(8, BFloat16)
src_idx_q_nope = cute.local_tile(
idx_q[token_id, head_id, None], (8,), (8 + sublane_id,)
)
cute.copy(cp_16B, src_idx_q_nope, rQ_nope_bf16)
rCos_raw = cute.make_rmem_tensor(4, idx_q_rope_cache.element_type)
rSin_raw = cute.make_rmem_tensor(4, idx_q_rope_cache.element_type)
rope_cache_view = cute.zipped_divide(idx_q_rope_cache[pos, None], (4,))
if cutlass.const_expr(idx_q_rope_cache.element_type == Float32):
cute.copy(cp_16B, rope_cache_view[None, sublane_id], rCos_raw)
cute.copy(cp_16B, rope_cache_view[None, 8 + sublane_id], rSin_raw)
elif cutlass.const_expr(idx_q_rope_cache.element_type == BFloat16):
cute.copy(cp_8B, rope_cache_view[None, sublane_id], rCos_raw)
cute.copy(cp_8B, rope_cache_view[None, 8 + sublane_id], rSin_raw)
# unpack to FP32
rQ_rope_f32 = cvt.bf16x2_to_fp32x2(rQ_rope_bf16)
rQ_nope_f32 = cvt.bf16x2_to_fp32x2(rQ_nope_bf16)
if cutlass.const_expr(idx_q_rope_cache.element_type == Float32):
rCos = rCos_raw
rSin = rSin_raw
elif cutlass.const_expr(idx_q_rope_cache.element_type == BFloat16):
rCos = cvt.bf16x2_to_fp32x2(rCos_raw)
rSin = cvt.bf16x2_to_fp32x2(rSin_raw)
# apply rope
for i in cutlass.range_constexpr(4):
if cutlass.const_expr(self.index_rope_interleave):
r0 = rQ_rope_f32[i * 2 + 0] * rCos[i] - rQ_rope_f32[i * 2 + 1] * rSin[i]
r1 = rQ_rope_f32[i * 2 + 1] * rCos[i] + rQ_rope_f32[i * 2 + 0] * rSin[i]
rQ_rope_f32[i * 2 + 0] = r0
rQ_rope_f32[i * 2 + 1] = r1
else:
r0 = rQ_rope_f32[0 + i] * rCos[i] - rQ_rope_f32[4 + i] * rSin[i]
r1 = rQ_rope_f32[4 + i] * rCos[i] + rQ_rope_f32[0 + i] * rSin[i]
rQ_rope_f32[0 + i] = r0
rQ_rope_f32[4 + i] = r1
# amax
amax = Float32(1e-4)
for i in cutlass.range_constexpr(8):
amax = cute.arch.fmax(amax, cute.math.absf(rQ_rope_f32[i]))
amax = cute.arch.fmax(amax, cute.math.absf(rQ_nope_f32[i]))
# warp reduction among 8 lanes
for i in cutlass.range_constexpr(3):
other = cute.arch.shuffle_sync_bfly(amax, 1 << i)
amax = cute.arch.fmax(amax, other)
# compute scale from amax
# exp2(ceil(log2(scale))) via bit manipulation
scale = amax * (1.0 / 448.0)
bits = scale.bitcast(Uint32)
exp_bits = (bits + Uint32(0x007FFFFF)) & Uint32(0x7F800000)
scale = exp_bits.bitcast(Float32)
inv_scale = (Uint32(0x7F000000) - exp_bits).bitcast(Float32)
for i in cutlass.range_constexpr(8):
rQ_nope_f32[i] *= inv_scale
rQ_rope_f32[i] *= inv_scale
cute.arch.griddepcontrol_launch_dependents()
# quantize and store
rQ_nope_f8 = cute.make_rmem_tensor(8, Float8E4M3FN)
rQ_nope_f8.store(rQ_nope_f32.load().to(Float8E4M3FN))
dst_idx_q_nope = cute.local_tile(
idx_q_fp8[token_id, head_id, None], (8,), (8 + sublane_id,)
)
cute.copy(cp_8B, rQ_nope_f8, dst_idx_q_nope)
rQ_rope_f8 = cute.make_rmem_tensor(8, Float8E4M3FN)
rQ_rope_f8.store(rQ_rope_f32.load().to(Float8E4M3FN))
if cutlass.const_expr(self.index_rope_interleave):
dst_idx_q_rope = cute.local_tile(
idx_q_fp8[token_id, head_id, None], (8,), (sublane_id,)
)
cute.copy(cp_8B, rQ_rope_f8, dst_idx_q_rope)
else:
dst_idx_q_rope = cute.zipped_divide(
idx_q_fp8[token_id, head_id, None], (4,)
) # (4,32)
cute.copy(
cp_4B,
cute.local_tile(rQ_rope_f8, (4,), (0,)),
dst_idx_q_rope[None, 0 + sublane_id],
)
cute.copy(
cp_4B,
cute.local_tile(rQ_rope_f8, (4,), (1,)),
dst_idx_q_rope[None, 8 + sublane_id],
)
# scale indexer weights
if sublane_id == 0:
w = idx_weights[token_id, head_id].to(Float32)
idx_weights_out[token_id, head_id] = w * scale * weight_scale
@cache
@staticmethod
def compile(
rope_dim: int,
nope_dim: int,
num_heads: int,
rope_type: type[cutlass.Numeric],
idx_dim: int,
num_idx_heads: int,
idx_rope_type: type[cutlass.Numeric] | None,
idx_weights_type: type[cutlass.Numeric] | None,
index_rope_interleave: bool,
):
num_tokens = cute.sym_int()
max_pos = cute.sym_int()
positions = _make_fake_tensor(Int64, (num_tokens,), divisibility=1)
q_pe = _make_fake_tensor(
BFloat16, (num_tokens, num_heads, rope_dim), divisibility=16
)
rope_cache = _make_fake_tensor(rope_type, (max_pos, rope_dim), divisibility=8)
ql_nope = _make_fake_tensor(
BFloat16, (num_tokens, num_heads, nope_dim), divisibility=16
)
q_scale = _make_fake_tensor(Float32, (1,), divisibility=4)
mqa_output = _make_fake_tensor(
Float8E4M3FN,
(num_tokens, num_heads, nope_dim + rope_dim),
divisibility=16,
)
if idx_rope_type is not None:
index_q = _make_fake_tensor(
BFloat16, (num_tokens, num_idx_heads, idx_dim), divisibility=16
)
index_rope_cache = _make_fake_tensor(
idx_rope_type, (max_pos, rope_dim), divisibility=8
)
index_weights = _make_fake_tensor(
idx_weights_type, (num_tokens, num_idx_heads), divisibility=8
)
index_q_fp8 = _make_fake_tensor(
Float8E4M3FN, (num_tokens, num_idx_heads, idx_dim), divisibility=16
)
index_weights_out = _make_fake_tensor(
Float32, (num_tokens, num_idx_heads), divisibility=4
)
else:
index_q = index_rope_cache = index_q_fp8 = None
index_weights = index_weights_out = None
kernel = FusedQKernel(
rope_dim,
nope_dim,
num_heads,
idx_dim,
num_idx_heads,
index_rope_interleave,
)
stream = cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True)
return cute.compile(
kernel,
positions,
q_pe,
rope_cache,
ql_nope,
q_scale,
mqa_output,
index_q,
index_rope_cache,
index_weights,
index_q_fp8,
index_weights_out,
Float32(0.0),
stream,
options="--enable-tvm-ffi",
)
@@ -28,7 +28,6 @@ from vllm.v1.attention.backend import (
MultipleOf,
)
from vllm.v1.attention.backends.mla.sparse_utils import (
phys_shadow,
triton_convert_req_index_to_global_index,
triton_filter_and_convert_dcp_index,
)
@@ -405,36 +404,6 @@ class FlashInferMLASparseImpl(SparseMLACommonImpl[FlashInferMLASparseMetadata]):
NUM_TOPK_TOKENS=topk_indices.shape[1],
return_valid_counts=True,
)
elif (shadow := phys_shadow(self.topk_indices_buffer)) is not None:
# skip_topk layers reuse the previous fresh layer's PHYSICAL
# indices: the logical top-k indices are shared (one buffer for
# the whole forward) and block_table/req_id are per-step constant,
# so re-converting them per layer produces identical output. This
# covers both the backbone (index_topk_freq > 1) and MTP draft
# steps 1+ (index_share_for_mtp_iteration sets skip_topk; the
# draft compact re-arranges the shadow together with the logical
# buffer). The fresh/skip split is fixed per captured graph, so
# the branch is cudagraph-safe.
phys_buf, seq_buf = shadow
wrote_fresh = getattr(layer, "indexer", None) is not None and not getattr(
layer, "skip_topk", False
)
if wrote_fresh:
topk_indices_physical, seq_lens = (
triton_convert_req_index_to_global_index(
attn_metadata.req_id_per_token[:num_actual_toks],
attn_metadata.block_table,
topk_indices,
BLOCK_SIZE=attn_metadata.block_size,
NUM_TOPK_TOKENS=topk_indices.shape[1],
return_valid_counts=True,
out=phys_buf[:num_actual_toks],
valid_counts_out=seq_buf[:num_actual_toks],
)
)
else:
topk_indices_physical = phys_buf[:num_actual_toks]
seq_lens = seq_buf[:num_actual_toks]
else:
topk_indices_physical, seq_lens = triton_convert_req_index_to_global_index(
attn_metadata.req_id_per_token[:num_actual_toks],
+4 -56
View File
@@ -128,8 +128,6 @@ def triton_convert_req_index_to_global_index(
prefill_workspace_request_ids: torch.Tensor | None = None,
prefill_workspace_starts: torch.Tensor | None = None,
return_valid_counts: bool = False,
out: torch.Tensor | None = None,
valid_counts_out: torch.Tensor | None = None,
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
"""
out[token_id, indice_id] =
@@ -180,20 +178,14 @@ def triton_convert_req_index_to_global_index(
req_id_c = req_id.contiguous()
block_table_c = block_table.contiguous()
token_indices_c = token_indices.contiguous()
# `out`/`valid_counts_out` allow writing into a stable pre-allocated buffer
# (used by the shared-physical-index cache to stay cudagraph-safe).
out = torch.empty_like(token_indices_c) if out is None else out
out = torch.empty_like(token_indices_c)
# Allocate valid count buffer if needed (must be zero-initialized for atomics)
valid_counts: torch.Tensor | None = None
if return_valid_counts:
if valid_counts_out is None:
valid_counts = torch.zeros(
num_tokens, dtype=torch.int32, device=token_indices.device
)
else:
valid_counts = valid_counts_out
valid_counts.zero_()
valid_counts = torch.zeros(
num_tokens, dtype=torch.int32, device=token_indices.device
)
# Strides in elements
bt_stride0, bt_stride1 = block_table_c.stride()
@@ -349,47 +341,3 @@ def triton_filter_and_convert_dcp_index(
assert valid_counts is not None
return out, valid_counts
return out
# --- Physical-index shadow (DSA index_topk_freq > 1) -------------------------
# GLM-5.2 uses index_topk_freq=4: only ~1/4 of layers write a fresh top-k into
# the single shared topk_indices_buffer; the rest reuse it. Since block_table
# and req_id_per_token are constant within a decode step, the physical indices
# (block_table lookup of the logical top-k) are identical across a freq-group.
# Fresh layers convert once into a STABLE shadow of the logical buffer and
# skip layers read it -- eliminating ~3/4 of the per-layer convert kernels.
#
# Invariant: shadow[j] == convert(logical[j]) row-wise. Whoever re-arranges
# the logical buffer (e.g. the MTP draft compact between the multi-token
# step-0 layout and the single-token steps-1+ layout) must apply the same
# gather to the shadow.
#
# Shadows are registered ONLY at buffer creation time (model init, never
# inside cudagraph capture) via register_phys_shadow; every other access is
# the read-only phys_shadow lookup, so addresses are stable across graph
# replays and no allocation can happen during capture.
import weakref as _weakref # noqa: E402
_PHYS_SHADOWS: dict[int, tuple[torch.Tensor, torch.Tensor]] = {}
def register_phys_shadow(topk_buf: torch.Tensor) -> None:
"""Allocate the (physical-index, valid-count) shadow for a logical top-k
buffer. Call once where the buffer is created. The finalizer drops the
entry with the owning buffer, so a recycled id can never alias a stale
shadow."""
key = id(topk_buf)
if key not in _PHYS_SHADOWS:
_PHYS_SHADOWS[key] = (
torch.empty_like(topk_buf),
torch.empty(topk_buf.shape[0], dtype=torch.int32, device=topk_buf.device),
)
_weakref.finalize(topk_buf, _PHYS_SHADOWS.pop, key, None)
def phys_shadow(
topk_buf: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor] | None:
"""The registered shadow for this buffer, or None if it was never
registered. Never allocates; capture-safe."""
return _PHYS_SHADOWS.get(id(topk_buf))
+2 -3
View File
@@ -1009,9 +1009,8 @@ class SpecDecodeBaseProposer:
# DeepSeek-family MTP (deepseek_mtp.py) recycles the post-final-
# norm hidden, so its forward returns (logit_hidden,
# recycle_hidden). Other MTP families return a single tensor.
return bool(
{"DeepSeekMTPModel", "DeepseekV32MTPModel"}
& set(self.draft_model_config.hf_config.architectures or [])
return "DeepSeekMTPModel" in (
self.draft_model_config.hf_config.architectures or []
)
return self.method not in ("mtp", "draft_model", "dflash")
@@ -48,18 +48,6 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
self.prefill_cudagraph_manager: SpeculatorCudaGraphManager | None = None
self.decode_cudagraph_manager: SpeculatorCudaGraphManager | None = None
# Lifecycle hooks for model-specific optimizations. Subclasses override
# the ones they need. These fire in both `capture` and `propose` so that
# any state they toggle (e.g. attention flags baked into a CUDA graph) is
# identical at capture time and replay time.
def on_prefill_begin(self, num_reqs: int, num_tokens: int) -> None: ...
def on_prefill_end(self, num_reqs: int, num_tokens: int) -> None: ...
def on_multi_step_decode_begin(self, num_reqs: int) -> None: ...
def on_multi_step_decode_end(self, num_reqs: int) -> None: ...
@property
def advance_draft_positions(self) -> bool:
"""
@@ -110,9 +98,6 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
assert self.prefill_cudagraph_manager is not None
if self.prefill_cudagraph_manager.use_breakable_cg:
self.prefill_cudagraph_manager.init_breakable_cg_runner(self.model)
self.on_prefill_begin(self.max_num_reqs, self.max_num_tokens)
self.prefill_cudagraph_manager.capture(
self._prefill,
self.model_state,
@@ -123,13 +108,9 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
progress_bar_desc="Capturing prefill CUDA graphs",
)
self.on_prefill_end(self.max_num_reqs, self.max_num_tokens)
if self.num_speculative_steps == 1:
return
self.on_multi_step_decode_begin(self.max_num_reqs)
# Capture the decode draft generation routine (model forward +
# sample + update_draft_inputs) for a single
# step.
@@ -144,8 +125,6 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
progress_bar_desc="Capturing decode CUDA graphs",
)
self.on_multi_step_decode_end(self.max_num_reqs)
@torch.inference_mode()
def propose(
self,
@@ -174,8 +153,7 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None,
is_profile: bool = False,
) -> torch.Tensor:
num_tokens = input_batch.num_tokens
num_tokens_padded = input_batch.num_tokens_after_padding
num_tokens = input_batch.num_tokens_after_padding
num_reqs = input_batch.num_reqs
max_query_len = input_batch.num_scheduled_tokens.max()
max_seq_len = input_batch.seq_lens_cpu_upper_bound[:num_reqs].max().item()
@@ -196,7 +174,7 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
)
else:
hidden_states = last_hidden_states
self.hidden_states[:num_tokens_padded].copy_(hidden_states)
self.hidden_states[:num_tokens].copy_(hidden_states)
self._copy_request_inputs(
num_reqs,
@@ -224,22 +202,20 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
num_reqs,
# Use the actual number of tokens without padding added by
# the target model during FULL cudagraph.
num_tokens,
input_batch.num_tokens,
max_query_len,
)
prefill_batch_desc, num_tokens_across_dp = dispatch_cg_and_sync_dp(
self.prefill_cudagraph_manager,
num_reqs,
num_tokens_padded,
num_tokens,
uniform_token_count,
dp_size=self.dp_size,
dp_rank=self.dp_rank,
need_eager=is_profile,
)
self._prepare_eplb_forward(num_tokens)
self.on_prefill_begin(num_reqs, num_tokens)
self._prepare_eplb_forward(input_batch.num_tokens)
if prefill_batch_desc.cg_mode == CUDAGraphMode.FULL:
# Replay the full graph for draft prefill.
@@ -259,8 +235,6 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
mm_inputs=mm_inputs,
)
self.on_prefill_end(num_reqs, num_tokens)
if self.num_speculative_steps == 1:
# Early exit.
return self.draft_tokens[:num_reqs, :1]
@@ -288,8 +262,6 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
need_eager=is_profile,
)
self.on_multi_step_decode_begin(num_reqs)
# Generate the remaining num_speculative_steps - 1 draft tokens.
self._multi_step_decode(
num_reqs,
@@ -299,8 +271,6 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
input_batch.seq_lens_cpu_upper_bound,
)
self.on_multi_step_decode_end(num_reqs)
return self.draft_tokens[:num_reqs]
@torch.inference_mode()
@@ -1,10 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
import torch.nn as nn
from vllm.config import VllmConfig
from vllm.v1.worker.gpu.spec_decode.autoregressive.speculator import (
AutoRegressiveSpeculator,
)
@@ -12,46 +10,9 @@ from vllm.v1.worker.gpu.spec_decode.eagle.utils import load_eagle_model
class MTPSpeculator(AutoRegressiveSpeculator):
def __init__(self, vllm_config: VllmConfig, device: torch.device):
super().__init__(vllm_config, device)
spec_config = vllm_config.speculative_config
draft_hf_config = (
spec_config.draft_model_config.hf_config
if spec_config is not None
else None
)
# Detect index_share_for_mtp_iteration. When True, the proposer
# toggles skip_topk so step 0 computes MTP's own indices and
# steps 1+ reuse them.
self.share_mtp_topk_indices = getattr(
draft_hf_config, "index_share_for_mtp_iteration", False
)
def load_draft_model(
self,
target_model: nn.Module,
target_attn_layer_names: set[str],
) -> nn.Module:
draft_model = load_eagle_model(target_model, self.vllm_config)
self.share_mtp_topk_indices = self.share_mtp_topk_indices and hasattr(
draft_model.model, "set_skip_topk"
)
return draft_model
def on_prefill_end(self, num_reqs: int, num_tokens: int) -> None:
# Step 0 (prefill) wrote topk indices for every query token in the
# multi-token batch. Compact them down to each request's last token so
# steps 1+ can reuse them from the shared buffer.
if self.share_mtp_topk_indices and self.num_speculative_steps > 1:
self.model.model.compact_topk_indices(self.last_token_indices[:num_reqs])
def on_multi_step_decode_begin(self, num_reqs: int) -> None:
# Switch to reuse mode so draft steps 1+ skip the indexer op and read
# the indices that step 0 wrote into the shared buffer.
if self.share_mtp_topk_indices:
self.model.model.set_skip_topk(True)
def on_multi_step_decode_end(self, num_reqs: int) -> None:
if self.share_mtp_topk_indices:
self.model.model.set_skip_topk(False)
return load_eagle_model(target_model, self.vllm_config)