Revert "[Perf][GLM-5.2] Blackwell decode optimizations" (#49768)
This commit is contained in:
@@ -732,23 +732,6 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP")
|
||||
"(requires SM90+ and CUDA >= 12.0).")
|
||||
endif()
|
||||
|
||||
# BF16 skinny GEMM (M<=32; weight-bandwidth-bound decode shapes).
|
||||
# Requires SM90+.
|
||||
cuda_archs_sm90plus(BF16_SKINNY_GEMM_ARCHS "${CUDA_ARCHS}")
|
||||
if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 12.0 AND BF16_SKINNY_GEMM_ARCHS)
|
||||
set(BF16_SKINNY_GEMM_SRCS
|
||||
"csrc/libtorch_stable/bf16_skinny_gemm_entry.cu"
|
||||
"csrc/libtorch_stable/bf16_skinny_gemm.cu")
|
||||
set_gencode_flags_for_srcs(
|
||||
SRCS "${BF16_SKINNY_GEMM_SRCS}"
|
||||
CUDA_ARCHS "${BF16_SKINNY_GEMM_ARCHS}")
|
||||
list(APPEND VLLM_STABLE_EXT_SRC "${BF16_SKINNY_GEMM_SRCS}")
|
||||
message(STATUS "Building bf16_skinny_gemm for archs: ${BF16_SKINNY_GEMM_ARCHS}")
|
||||
else()
|
||||
message(STATUS "Not building bf16_skinny_gemm as no compatible archs found "
|
||||
"(requires SM90+ and CUDA >= 12.0).")
|
||||
endif()
|
||||
|
||||
# Only build AllSpark kernels if we are building for at least some compatible archs.
|
||||
cuda_archs_loose_intersection(ALLSPARK_ARCHS "8.0;8.6;8.7;8.9" "${CUDA_ARCHS}")
|
||||
if (ALLSPARK_ARCHS)
|
||||
|
||||
@@ -1,262 +0,0 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
//
|
||||
// Skinny GEMM: activation(bf16) x weight(bf16)^T -> bf16, for decode-time
|
||||
// M <= 32 with a large reduction dim. Replaces cuBLAS splitK (GEMM +
|
||||
// splitKreduce) with a single block-per-output-column kernel; these shapes
|
||||
// are weight-bandwidth-bound, so one coalesced pass over the weight at
|
||||
// fp32 accumulation is optimal. Adapted from fp32_router_gemm.cu.
|
||||
//
|
||||
// First user: the DeepSeek-V32/GLM-5.2 MTP eh_proj (K=2*hidden=12288,
|
||||
// N=hidden/TP), whose cuBLAS splitK pick costs ~34.6us vs the ~19us
|
||||
// bandwidth floor per replicated read (and ~4us once column-parallel).
|
||||
|
||||
#include <cuda_bf16.h>
|
||||
#include <cuda_runtime.h>
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Load helpers (8 x bf16 = one uint4 load, converted to fp32)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
namespace skinny {
|
||||
|
||||
constexpr int VPT = 8; // bf16 values per thread per load
|
||||
|
||||
__device__ __forceinline__ void load_bf16x8(__nv_bfloat16 const* ptr,
|
||||
float* dst) {
|
||||
uint4 v = *reinterpret_cast<uint4 const*>(ptr);
|
||||
__nv_bfloat16 const* p = reinterpret_cast<__nv_bfloat16 const*>(&v);
|
||||
#pragma unroll
|
||||
for (int i = 0; i < VPT; i++) dst[i] = __bfloat162float(p[i]);
|
||||
}
|
||||
|
||||
// Streaming variant for the weight: each row is read exactly once across the
|
||||
// whole grid, so bypass L2 residency (evict-first). Measured -1.4us at M=1.
|
||||
__device__ __forceinline__ void load_bf16x8_cs(__nv_bfloat16 const* ptr,
|
||||
float* dst) {
|
||||
uint4 v = __ldcs(reinterpret_cast<uint4 const*>(ptr));
|
||||
__nv_bfloat16 const* p = reinterpret_cast<__nv_bfloat16 const*>(&v);
|
||||
#pragma unroll
|
||||
for (int i = 0; i < VPT; i++) dst[i] = __bfloat162float(p[i]);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Kernel: each block computes kNPB output columns for all kNumTokens rows.
|
||||
// grid = kN / kNPB, block = kBlockSize threads. K is reduced VPT elements
|
||||
// per thread per iteration; fp32 accumulation, warp butterfly + smem
|
||||
// finalize, bf16 store.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
template <int kBlockSize, int kNumTokens, int kNPB, int kPF, int kN, int kK>
|
||||
__global__ __launch_bounds__(kBlockSize, 1) void bf16_skinny_gemm_kernel(
|
||||
__nv_bfloat16* out, __nv_bfloat16 const* mat_a, __nv_bfloat16 const* mat_b,
|
||||
int64_t out_stride) {
|
||||
constexpr int k_elems_per_iter = VPT * kBlockSize;
|
||||
constexpr int k_iterations = kK / k_elems_per_iter;
|
||||
static_assert(kK % k_elems_per_iter == 0);
|
||||
constexpr int kWarpSize = 32;
|
||||
constexpr int kNumWarps = kBlockSize / kWarpSize;
|
||||
|
||||
int const n_base = blockIdx.x * kNPB;
|
||||
int const tid = threadIdx.x;
|
||||
int const warpId = tid / kWarpSize;
|
||||
int const laneId = tid % kWarpSize;
|
||||
|
||||
float acc[kNumTokens][kNPB] = {};
|
||||
__shared__ float sm_reduction[kNumTokens][kNPB][kNumWarps];
|
||||
|
||||
// Register prefetch (kPF > 0): W does not depend on the predecessor, so
|
||||
// the first kPF iterations' weight chunks are loaded raw BEFORE the
|
||||
// dependency sync; with a PDL-releasing producer (fused_eh_norm fires
|
||||
// gdc_launch_dependents early) these DRAM round trips overlap the norm.
|
||||
// Pair-measured on B300 (norm+gemm in one graph, full 6144x12288):
|
||||
// M=1 pf2 28.43us vs pf0 29.00us; deeper prefetch or M >= 2 regresses
|
||||
// (register pressure), hence the per-M selection in the launcher.
|
||||
uint4 w_pre[kPF > 0 ? kPF : 1][kNPB];
|
||||
#pragma unroll
|
||||
for (int pf = 0; pf < kPF; pf++) {
|
||||
#pragma unroll
|
||||
for (int n = 0; n < kNPB; n++) {
|
||||
w_pre[pf][n] =
|
||||
*reinterpret_cast<uint4 const*>(mat_b + (size_t)(n_base + n) * kK +
|
||||
pf * k_elems_per_iter + tid * VPT);
|
||||
}
|
||||
}
|
||||
|
||||
#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)
|
||||
cudaGridDependencySynchronize();
|
||||
#endif
|
||||
|
||||
#pragma unroll
|
||||
for (int ki = 0; ki < k_iterations; ki++) {
|
||||
int const k_base = ki * k_elems_per_iter + tid * VPT;
|
||||
|
||||
float b_float[kNPB][VPT];
|
||||
if (ki < kPF) {
|
||||
#pragma unroll
|
||||
for (int n = 0; n < kNPB; n++) {
|
||||
__nv_bfloat16 const* p =
|
||||
reinterpret_cast<__nv_bfloat16 const*>(&w_pre[ki][n]);
|
||||
#pragma unroll
|
||||
for (int v = 0; v < VPT; v++) b_float[n][v] = __bfloat162float(p[v]);
|
||||
}
|
||||
} else {
|
||||
#pragma unroll
|
||||
for (int n = 0; n < kNPB; n++) {
|
||||
load_bf16x8_cs(mat_b + (size_t)(n_base + n) * kK + k_base, b_float[n]);
|
||||
}
|
||||
}
|
||||
|
||||
#pragma unroll
|
||||
for (int m = 0; m < kNumTokens; m++) {
|
||||
float a_float[VPT];
|
||||
load_bf16x8(mat_a + (size_t)m * kK + k_base, a_float);
|
||||
#pragma unroll
|
||||
for (int n = 0; n < kNPB; n++) {
|
||||
#pragma unroll
|
||||
for (int k = 0; k < VPT; k++) {
|
||||
acc[m][n] += a_float[k] * b_float[n][k];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Warp-level butterfly reduction
|
||||
#pragma unroll
|
||||
for (int m = 0; m < kNumTokens; m++) {
|
||||
#pragma unroll
|
||||
for (int n = 0; n < kNPB; n++) {
|
||||
float sum = acc[m][n];
|
||||
sum += __shfl_xor_sync(0xffffffff, sum, 16);
|
||||
sum += __shfl_xor_sync(0xffffffff, sum, 8);
|
||||
sum += __shfl_xor_sync(0xffffffff, sum, 4);
|
||||
sum += __shfl_xor_sync(0xffffffff, sum, 2);
|
||||
sum += __shfl_xor_sync(0xffffffff, sum, 1);
|
||||
if (laneId == 0) sm_reduction[m][n][warpId] = sum;
|
||||
}
|
||||
}
|
||||
|
||||
__syncthreads();
|
||||
|
||||
// Parallel finalize: one thread per (m, n) output.
|
||||
for (int idx = tid; idx < kNumTokens * kNPB; idx += kBlockSize) {
|
||||
int const m = idx / kNPB;
|
||||
int const n = idx % kNPB;
|
||||
float final_sum = 0.0f;
|
||||
#pragma unroll
|
||||
for (int w = 0; w < kNumWarps; w++) final_sum += sm_reduction[m][n][w];
|
||||
out[(size_t)m * out_stride + n_base + n] = __float2bfloat16(final_sum);
|
||||
}
|
||||
|
||||
// Trigger after our stores: harmless hardening, not a guarantee — the
|
||||
// trigger only permits dependent-launch scheduling and carries no memory
|
||||
// visibility semantics, so a PDL consumer must still gridsync before
|
||||
// reading our output. Every current consumer is a plain launch (full
|
||||
// stream order); firing late just avoids an unnecessarily early launch
|
||||
// window and matches fp32_router_gemm.cu.
|
||||
#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)
|
||||
cudaTriggerProgrammaticLaunchCompletion();
|
||||
#endif
|
||||
}
|
||||
|
||||
} // namespace skinny
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Launcher
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
template <int kBlockSize, int kNPB, int kNumTokens, int kN, int kK>
|
||||
void invokeBf16SkinnyGemm(__nv_bfloat16* output, __nv_bfloat16 const* mat_a,
|
||||
__nv_bfloat16 const* mat_b, int64_t out_stride,
|
||||
cudaStream_t stream) {
|
||||
static_assert(kN % kNPB == 0);
|
||||
// Weight prefetch depth: only M=1 measured a win (see kernel comment).
|
||||
constexpr int kPF = (kNumTokens == 1) ? 2 : 0;
|
||||
cudaLaunchConfig_t config;
|
||||
config.gridDim = kN / kNPB;
|
||||
config.blockDim = kBlockSize;
|
||||
config.dynamicSmemBytes = 0;
|
||||
config.stream = stream;
|
||||
cudaLaunchAttribute attrs[1];
|
||||
attrs[0].id = cudaLaunchAttributeProgrammaticStreamSerialization;
|
||||
attrs[0].val.programmaticStreamSerializationAllowed = 1;
|
||||
config.numAttrs = 1;
|
||||
config.attrs = attrs;
|
||||
cudaLaunchKernelEx(&config,
|
||||
skinny::bf16_skinny_gemm_kernel<kBlockSize, kNumTokens,
|
||||
kNPB, kPF, kN, kK>,
|
||||
output, mat_a, mat_b, out_stride);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Explicit instantiations. M = 1..32; (N, K) pairs:
|
||||
// (768, 12288) eh_proj shard, TP8
|
||||
// (1536, 12288) eh_proj shard, TP4
|
||||
// (6144, 12288) eh_proj unsharded
|
||||
// kNPB (B300 sweep, M=1): 6144 -> 2 (3072 blocks, 23.2us vs 25.3 at kNPB=8;
|
||||
// narrow blocks minimize wave quantization); shards 768/1536 keep 4.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#define INSTANTIATE(M, NPB, N, K) \
|
||||
template void invokeBf16SkinnyGemm<128, NPB, M, N, K>( \
|
||||
__nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, int64_t, \
|
||||
cudaStream_t);
|
||||
|
||||
#define INSTANTIATE_ALL_M(NPB, N, K) \
|
||||
INSTANTIATE(1, NPB, N, K) \
|
||||
INSTANTIATE(2, NPB, N, K) \
|
||||
INSTANTIATE(3, NPB, N, K) \
|
||||
INSTANTIATE(4, NPB, N, K) \
|
||||
INSTANTIATE(5, NPB, N, K) \
|
||||
INSTANTIATE(6, NPB, N, K) \
|
||||
INSTANTIATE(7, NPB, N, K) \
|
||||
INSTANTIATE(8, NPB, N, K) \
|
||||
INSTANTIATE(9, NPB, N, K) \
|
||||
INSTANTIATE(10, NPB, N, K) \
|
||||
INSTANTIATE(11, NPB, N, K) \
|
||||
INSTANTIATE(12, NPB, N, K) \
|
||||
INSTANTIATE(13, NPB, N, K) \
|
||||
INSTANTIATE(14, NPB, N, K) \
|
||||
INSTANTIATE(15, NPB, N, K) \
|
||||
INSTANTIATE(16, NPB, N, K) \
|
||||
INSTANTIATE(17, NPB, N, K) \
|
||||
INSTANTIATE(18, NPB, N, K) \
|
||||
INSTANTIATE(19, NPB, N, K) \
|
||||
INSTANTIATE(20, NPB, N, K) \
|
||||
INSTANTIATE(21, NPB, N, K) \
|
||||
INSTANTIATE(22, NPB, N, K) \
|
||||
INSTANTIATE(23, NPB, N, K) \
|
||||
INSTANTIATE(24, NPB, N, K) \
|
||||
INSTANTIATE(25, NPB, N, K) \
|
||||
INSTANTIATE(26, NPB, N, K) \
|
||||
INSTANTIATE(27, NPB, N, K) \
|
||||
INSTANTIATE(28, NPB, N, K) \
|
||||
INSTANTIATE(29, NPB, N, K) \
|
||||
INSTANTIATE(30, NPB, N, K) \
|
||||
INSTANTIATE(31, NPB, N, K) \
|
||||
INSTANTIATE(32, NPB, N, K)
|
||||
|
||||
INSTANTIATE_ALL_M(4, 768, 12288)
|
||||
INSTANTIATE_ALL_M(4, 1536, 12288)
|
||||
INSTANTIATE_ALL_M(2, 6144, 12288)
|
||||
// LL-mode (M<=8 wiring guard) backbone shapes, B300 sweep vs cuBLAS:
|
||||
// q_b_proj (2048, 2048): 1.67x/1.29x/1.15x at M=4/6/8 (NPB=4 within
|
||||
// 0.1us of per-M best)
|
||||
// shared-expert gate_up (512, 6144): 1.95x/1.58x/1.40x at M=4/6/8
|
||||
// cuBLAS keeps qkv_a (2624,6144) and o_proj (6144,2048) — already at
|
||||
// 3.4-3.8 TB/s there; the GEMV loses on activation re-reads.
|
||||
INSTANTIATE_ALL_M(4, 2048, 2048)
|
||||
INSTANTIATE_ALL_M(4, 512, 6144)
|
||||
// fused_qkv_a (2624, 6144), 32MB: skinny wins ONLY at M<=2 (B300: M=1
|
||||
// 6.99us vs cuBLAS 9.21 = 1.32x, M=2 1.24x; M>=4 cuBLAS holds at 3.5TB/s
|
||||
// and every alternative loses — cublasLt top-8 3.6TB/s wall, DeepGEMM
|
||||
// 0.71x, wmma+cp.async custom 0.30x pending a TMA rewrite).
|
||||
INSTANTIATE_ALL_M(4, 2624, 6144)
|
||||
// DSv3.2 (TP8) siblings of the GLM shapes above, same dual-chip matrix:
|
||||
// fused_qkv_a (2112, 7168), 30MB: wins M<=2 (M=1 1.30-1.34x)
|
||||
// MTP eh_proj (7168, 14336), 205MB: wins M<=2 (M=1 1.12-1.16x)
|
||||
INSTANTIATE_ALL_M(4, 2112, 7168)
|
||||
INSTANTIATE_ALL_M(2, 7168, 14336)
|
||||
|
||||
#undef INSTANTIATE_ALL_M
|
||||
#undef INSTANTIATE
|
||||
@@ -1,170 +0,0 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
#include <torch/csrc/stable/library.h>
|
||||
#include <torch/csrc/stable/tensor.h>
|
||||
#include <torch/headeronly/core/ScalarType.h>
|
||||
|
||||
#include "core/registration.h"
|
||||
#include "libtorch_stable/torch_utils.h"
|
||||
|
||||
#include <cuda_bf16.h>
|
||||
#include <cuda_runtime.h>
|
||||
|
||||
#include <stdexcept>
|
||||
|
||||
namespace {
|
||||
|
||||
inline int getSMVersion() {
|
||||
auto* props = get_device_prop();
|
||||
return props->major * 10 + props->minor;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
static constexpr int SKINNY_MAX_TOKENS = 32;
|
||||
|
||||
// Supported (N, K) pairs (must match the instantiations in
|
||||
// bf16_skinny_gemm.cu): eh_proj shard TP8 / TP4 / unsharded.
|
||||
static inline bool bf16_skinny_gemm_supported(int n, int k) {
|
||||
if (k == 12288 && (n == 768 || n == 1536 || n == 6144)) return true;
|
||||
// LL-mode backbone shapes (wire callers with an M <= 8 guard; the GEMV
|
||||
// family loses to cuBLAS at larger M).
|
||||
if (k == 2048 && n == 2048) return true; // q_b_proj (TP8)
|
||||
if (k == 6144 && n == 2624) return true; // fused_qkv_a (wire M <= 2 only)
|
||||
if (k == 7168 && n == 2112) return true; // DSv3.2 fused_qkv_a (M <= 2)
|
||||
if (k == 14336 && n == 7168) return true; // DSv3.2 eh_proj (M <= 2)
|
||||
if (k == 6144 && n == 512) return true; // shared-expert gate_up (TP8)
|
||||
return false;
|
||||
}
|
||||
|
||||
// Forward declarations - template params must match bf16_skinny_gemm.cu
|
||||
template <int kBlockSize, int kNPB, int kNumTokens, int kN, int kK>
|
||||
void invokeBf16SkinnyGemm(__nv_bfloat16* output, __nv_bfloat16 const* mat_a,
|
||||
__nv_bfloat16 const* mat_b, int64_t out_stride,
|
||||
cudaStream_t stream);
|
||||
|
||||
template <int kNPB, int kN, int kK, int kBegin, int kEnd>
|
||||
struct SkinnyLoopUnroller {
|
||||
static void unroll(int num_tokens, __nv_bfloat16* output,
|
||||
__nv_bfloat16 const* mat_a, __nv_bfloat16 const* mat_b,
|
||||
int64_t out_stride, cudaStream_t stream) {
|
||||
if (num_tokens == kBegin) {
|
||||
invokeBf16SkinnyGemm<128, kNPB, kBegin, kN, kK>(output, mat_a, mat_b,
|
||||
out_stride, stream);
|
||||
} else {
|
||||
SkinnyLoopUnroller<kNPB, kN, kK, kBegin + 1, kEnd>::unroll(
|
||||
num_tokens, output, mat_a, mat_b, out_stride, stream);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
template <int kNPB, int kN, int kK, int kEnd>
|
||||
struct SkinnyLoopUnroller<kNPB, kN, kK, kEnd, kEnd> {
|
||||
static void unroll(int num_tokens, __nv_bfloat16* output,
|
||||
__nv_bfloat16 const* mat_a, __nv_bfloat16 const* mat_b,
|
||||
int64_t out_stride, cudaStream_t stream) {
|
||||
if (num_tokens == kEnd) {
|
||||
invokeBf16SkinnyGemm<128, kNPB, kEnd, kN, kK>(output, mat_a, mat_b,
|
||||
out_stride, stream);
|
||||
} else {
|
||||
throw std::invalid_argument(
|
||||
"bf16_skinny_gemm: num_tokens must be in [1, 32]");
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
static void dispatchBf16SkinnyGemm(int n, int k, int num_tokens,
|
||||
__nv_bfloat16* output,
|
||||
__nv_bfloat16 const* mat_a,
|
||||
__nv_bfloat16 const* mat_b,
|
||||
int64_t out_stride, cudaStream_t stream) {
|
||||
if (n == 768 && k == 12288) {
|
||||
SkinnyLoopUnroller<4, 768, 12288, 1, SKINNY_MAX_TOKENS>::unroll(
|
||||
num_tokens, output, mat_a, mat_b, out_stride, stream);
|
||||
} else if (n == 1536 && k == 12288) {
|
||||
SkinnyLoopUnroller<4, 1536, 12288, 1, SKINNY_MAX_TOKENS>::unroll(
|
||||
num_tokens, output, mat_a, mat_b, out_stride, stream);
|
||||
} else if (n == 6144 && k == 12288) {
|
||||
SkinnyLoopUnroller<2, 6144, 12288, 1, SKINNY_MAX_TOKENS>::unroll(
|
||||
num_tokens, output, mat_a, mat_b, out_stride, stream);
|
||||
} else if (n == 2048 && k == 2048) {
|
||||
SkinnyLoopUnroller<4, 2048, 2048, 1, SKINNY_MAX_TOKENS>::unroll(
|
||||
num_tokens, output, mat_a, mat_b, out_stride, stream);
|
||||
} else if (n == 2624 && k == 6144) {
|
||||
SkinnyLoopUnroller<4, 2624, 6144, 1, SKINNY_MAX_TOKENS>::unroll(
|
||||
num_tokens, output, mat_a, mat_b, out_stride, stream);
|
||||
} else if (n == 2112 && k == 7168) {
|
||||
SkinnyLoopUnroller<4, 2112, 7168, 1, SKINNY_MAX_TOKENS>::unroll(
|
||||
num_tokens, output, mat_a, mat_b, out_stride, stream);
|
||||
} else if (n == 7168 && k == 14336) {
|
||||
SkinnyLoopUnroller<2, 7168, 14336, 1, SKINNY_MAX_TOKENS>::unroll(
|
||||
num_tokens, output, mat_a, mat_b, out_stride, stream);
|
||||
} else if (n == 512 && k == 6144) {
|
||||
SkinnyLoopUnroller<4, 512, 6144, 1, SKINNY_MAX_TOKENS>::unroll(
|
||||
num_tokens, output, mat_a, mat_b, out_stride, stream);
|
||||
} else {
|
||||
throw std::invalid_argument("bf16_skinny_gemm: unsupported (N, K) pair");
|
||||
}
|
||||
}
|
||||
|
||||
void bf16_skinny_gemm(
|
||||
torch::stable::Tensor& output, // [num_tokens, N] bf16
|
||||
torch::stable::Tensor const& mat_a, // [num_tokens, K] bf16
|
||||
torch::stable::Tensor const& mat_b // [N, K] bf16
|
||||
) {
|
||||
STD_TORCH_CHECK(output.dim() == 2 && mat_a.dim() == 2 && mat_b.dim() == 2);
|
||||
STD_TORCH_CHECK(output.is_cuda() && mat_a.is_cuda() && mat_b.is_cuda(),
|
||||
"bf16_skinny_gemm: all tensors must be CUDA tensors");
|
||||
STD_TORCH_CHECK(output.get_device_index() == mat_a.get_device_index() &&
|
||||
output.get_device_index() == mat_b.get_device_index(),
|
||||
"bf16_skinny_gemm: all tensors must be on the same device");
|
||||
STD_TORCH_CHECK(mat_a.is_contiguous() && mat_b.is_contiguous(),
|
||||
"bf16_skinny_gemm: inputs must be contiguous");
|
||||
// Output may be a column-slice view of a wider padded buffer: unit column
|
||||
// stride, row stride >= N (rows must not overlap).
|
||||
STD_TORCH_CHECK(output.stride(1) == 1,
|
||||
"bf16_skinny_gemm: output columns must be contiguous");
|
||||
|
||||
const int num_tokens = mat_a.size(0);
|
||||
const int n = mat_b.size(0);
|
||||
const int k = mat_a.size(1);
|
||||
|
||||
STD_TORCH_CHECK(output.size(0) == num_tokens && output.size(1) == n,
|
||||
"bf16_skinny_gemm: output must be [num_tokens, N]");
|
||||
STD_TORCH_CHECK(mat_b.size(1) == k,
|
||||
"bf16_skinny_gemm: mat_a and mat_b must share K");
|
||||
STD_TORCH_CHECK(bf16_skinny_gemm_supported(n, k),
|
||||
"bf16_skinny_gemm: unsupported (N, K) pair");
|
||||
const int64_t out_stride = output.stride(0);
|
||||
STD_TORCH_CHECK(num_tokens <= 1 || out_stride >= n,
|
||||
"bf16_skinny_gemm: output rows overlap");
|
||||
STD_TORCH_CHECK(num_tokens >= 0 && num_tokens <= SKINNY_MAX_TOKENS,
|
||||
"bf16_skinny_gemm: num_tokens must be in [0, 32]");
|
||||
STD_TORCH_CHECK(
|
||||
mat_a.scalar_type() == torch::headeronly::ScalarType::BFloat16 &&
|
||||
mat_b.scalar_type() == torch::headeronly::ScalarType::BFloat16 &&
|
||||
output.scalar_type() == torch::headeronly::ScalarType::BFloat16,
|
||||
"bf16_skinny_gemm: all tensors must be bfloat16");
|
||||
|
||||
// Empty batch (e.g. an empty rank at a DP/PP boundary): nothing to compute.
|
||||
if (num_tokens == 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(
|
||||
mat_a.get_device_index());
|
||||
STD_TORCH_CHECK(getSMVersion() >= 90, "bf16_skinny_gemm: requires SM90+");
|
||||
|
||||
auto stream = get_current_cuda_stream(mat_a.get_device_index());
|
||||
dispatchBf16SkinnyGemm(
|
||||
n, k, num_tokens,
|
||||
reinterpret_cast<__nv_bfloat16*>(output.mutable_data_ptr()),
|
||||
reinterpret_cast<__nv_bfloat16 const*>(mat_a.data_ptr()),
|
||||
reinterpret_cast<__nv_bfloat16 const*>(mat_b.data_ptr()), out_stride,
|
||||
stream);
|
||||
}
|
||||
|
||||
STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, m) {
|
||||
m.impl("bf16_skinny_gemm", TORCH_BOX(&bf16_skinny_gemm));
|
||||
}
|
||||
@@ -391,7 +391,7 @@ struct MmaComputer {
|
||||
static constexpr int n_iter_cnt =
|
||||
(tile_n + 7) /
|
||||
8; // Possible to have non-1 n_iter_cnt for ab_swap m16 case.
|
||||
static_assert(m_iter_cnt == 1 || m_iter_cnt == 2);
|
||||
static_assert(m_iter_cnt == 1);
|
||||
static_assert(n_iter_cnt == 1 || n_iter_cnt == 2);
|
||||
|
||||
__device__ MmaComputer(bf16_t* gmem_c_local_, bf16_t* smem_a_,
|
||||
@@ -416,18 +416,13 @@ struct MmaComputer {
|
||||
public:
|
||||
__device__ void prepare() {
|
||||
#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900
|
||||
// Fragment addressing is per 16-row ldmatrix tile; m_iter selects the
|
||||
// 16-row half within tile_m.
|
||||
#pragma unroll
|
||||
for (int m = 0; m < m_iter_cnt; m++) {
|
||||
#pragma unroll
|
||||
for (int i = 0; i < k_phase_cnt; i++) {
|
||||
int linear_idx = (lane_idx % 16) + (lane_idx / 16) * 128 + i * 256;
|
||||
int m_idx = linear_idx % 16 + m * 16;
|
||||
int k_idx = linear_idx / 16 + warp_k_offset_in_tile_k;
|
||||
k_idx = apply_swizzle_343_on_elem_row_col<bf16_t>(m_idx, k_idx);
|
||||
a_smem_offsets[m][i] = m_idx * tile_k + k_idx;
|
||||
}
|
||||
for (int i = 0; i < k_phase_cnt; i++) {
|
||||
int linear_idx = (lane_idx % 16) + (lane_idx / 16) * 128 + i * 256;
|
||||
int m_idx = linear_idx % tile_m;
|
||||
int k_idx = linear_idx / tile_m + warp_k_offset_in_tile_k;
|
||||
k_idx = apply_swizzle_343_on_elem_row_col<bf16_t>(m_idx, k_idx);
|
||||
a_smem_offsets[0][i] = m_idx * tile_k + k_idx;
|
||||
}
|
||||
#pragma unroll
|
||||
for (int n_iter_idx = 0; n_iter_idx < n_iter_cnt; n_iter_idx++) {
|
||||
@@ -451,14 +446,11 @@ struct MmaComputer {
|
||||
wait_barrier(smem_barrier + 0 + stage_idx * 2, phase_bit);
|
||||
|
||||
#pragma unroll
|
||||
for (int m = 0; m < m_iter_cnt; m++) {
|
||||
#pragma unroll
|
||||
for (int i = 0; i < k_phase_cnt; i++) {
|
||||
int smem_offset = a_smem_offsets[m][i];
|
||||
bf16_t* smem_ptr_this_iter =
|
||||
smem_a + stage_idx * tile_m * tile_k + smem_offset;
|
||||
ldsm_x4(smem_ptr_this_iter, reinterpret_cast<uint32_t*>(a_reg[m][i]));
|
||||
}
|
||||
for (int i = 0; i < k_phase_cnt; i++) {
|
||||
int smem_offset = a_smem_offsets[0][i];
|
||||
bf16_t* smem_ptr_this_iter =
|
||||
smem_a + stage_idx * tile_m * tile_k + smem_offset;
|
||||
ldsm_x4(smem_ptr_this_iter, reinterpret_cast<uint32_t*>(a_reg[0][i]));
|
||||
}
|
||||
|
||||
#pragma unroll
|
||||
@@ -477,12 +469,9 @@ struct MmaComputer {
|
||||
for (int k_iter_idx = 0; k_iter_idx < k_phase_cnt; k_iter_idx++) {
|
||||
#pragma unroll
|
||||
for (int n_iter_idx = 0; n_iter_idx < n_iter_cnt; n_iter_idx++) {
|
||||
#pragma unroll
|
||||
for (int m = 0; m < m_iter_cnt; m++) {
|
||||
hmma_16_8_16_f32acc_bf16ab(
|
||||
acc_reg[m][n_iter_idx], a_reg[m][k_iter_idx],
|
||||
b_reg[n_iter_idx][k_iter_idx], acc_reg[m][n_iter_idx]);
|
||||
}
|
||||
hmma_16_8_16_f32acc_bf16ab(
|
||||
acc_reg[0][n_iter_idx], a_reg[0][k_iter_idx],
|
||||
b_reg[n_iter_idx][k_iter_idx], acc_reg[0][n_iter_idx]);
|
||||
}
|
||||
}
|
||||
::arrive_barrier(smem_barrier + 1 + stage_idx * 2);
|
||||
@@ -497,14 +486,14 @@ struct MmaComputer {
|
||||
#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900
|
||||
asm volatile("bar.sync %0, %1;" : : "r"(1), "r"(thread_cnt));
|
||||
// reorganize the acc_reg
|
||||
constexpr int thread_m = 2 * m_iter_cnt;
|
||||
constexpr int thread_m = 2;
|
||||
constexpr int thread_n = 2 * n_iter_cnt;
|
||||
constexpr int cta_mma_n = n_iter_cnt * 8;
|
||||
float acc_reg_reorg[thread_m][thread_n];
|
||||
|
||||
for (int i = 0; i < thread_m; i++) {
|
||||
for (int j = 0; j < thread_n; j++) {
|
||||
acc_reg_reorg[i][j] = acc_reg[i / 2][j / 2][(j % 2) + (i % 2) * 2];
|
||||
acc_reg_reorg[i][j] = acc_reg[0][j / 2][(j % 2) + (i * 2)];
|
||||
}
|
||||
}
|
||||
|
||||
@@ -525,8 +514,7 @@ struct MmaComputer {
|
||||
for (int m_idx_thread = 0; m_idx_thread < thread_m; m_idx_thread++) {
|
||||
#pragma unroll
|
||||
for (int n_idx_thread = 0; n_idx_thread < thread_n; n_idx_thread++) {
|
||||
int m_idx =
|
||||
(lane_idx / 4) + (m_idx_thread % 2) * 8 + (m_idx_thread / 2) * 16;
|
||||
int m_idx = (lane_idx / 4) + m_idx_thread * 8;
|
||||
int n_idx =
|
||||
((lane_idx % 4) * 2) + (n_idx_thread % 2) + (n_idx_thread / 2) * 8;
|
||||
smem_c[cosize_smem_c * warp_idx + smem_c_index_func(m_idx, n_idx)] =
|
||||
@@ -599,7 +587,7 @@ __global__ __launch_bounds__(256, 1) void fused_a_gemm_kernel(
|
||||
static_assert(
|
||||
tile_k == 128 || tile_k == 256 || tile_k == 512 ||
|
||||
tile_k == 1024); // tile_k must be larger than 64 since 4 warp splitK.
|
||||
static_assert(tile_m == 16 || tile_m == 32);
|
||||
static_assert(tile_m == 16);
|
||||
constexpr int g2s_vec_bytes = 16;
|
||||
constexpr int a_elem_bytes = 2;
|
||||
constexpr int b_elem_bytes = 2;
|
||||
@@ -659,7 +647,7 @@ __global__ __launch_bounds__(256, 1) void fused_a_gemm_kernel(
|
||||
#endif
|
||||
}
|
||||
|
||||
template <typename T, int kHdIn, int kHdOut, int kTileN, int kTileM = 16>
|
||||
template <typename T, int kHdIn, int kHdOut, int kTileN>
|
||||
void invokeFusedAGemm(T* output, T const* mat_a, T const* mat_b, int num_tokens,
|
||||
cudaStream_t const stream) {
|
||||
constexpr int gemm_m = kHdOut; // 2112
|
||||
@@ -667,7 +655,7 @@ void invokeFusedAGemm(T* output, T const* mat_a, T const* mat_b, int num_tokens,
|
||||
constexpr int gemm_k = kHdIn; // 7168
|
||||
constexpr int batch_size = 1;
|
||||
std::swap(mat_a, mat_b);
|
||||
constexpr int tile_m = kTileM;
|
||||
constexpr int tile_m = 16;
|
||||
constexpr int tile_n = kTileN; // 8 or 16
|
||||
constexpr int tile_k = std::max(256, 1024 / tile_n); // 256
|
||||
constexpr int max_stage_cnt =
|
||||
@@ -714,44 +702,6 @@ template void invokeFusedAGemm<__nv_bfloat16, 7168, 2112, 16>(
|
||||
__nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, int num_tokens,
|
||||
cudaStream_t);
|
||||
|
||||
// GLM-5.2 fused_qkv_a (K=6144 -> N=2624). tile_m=32 so the grid is
|
||||
// 2624/32 = 82 CTAs, a single wave on B300 (148 SMs); tile_m=16's 164 CTAs
|
||||
// straddle two waves and drop to 2.9 TB/s vs 3.9-4.0 here. Beats cuBLAS
|
||||
// at every decode M: 1.11-1.15x for M<=8 (tile_n=8), 1.12x at M=16
|
||||
// (tile_n=16). The M<=2 dispatch still belongs to bf16_skinny_gemm
|
||||
// (4.5 TB/s).
|
||||
template void invokeFusedAGemm<__nv_bfloat16, 6144, 2624, 8, 32>(
|
||||
__nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, int num_tokens,
|
||||
cudaStream_t);
|
||||
|
||||
template void invokeFusedAGemm<__nv_bfloat16, 6144, 2624, 16, 32>(
|
||||
__nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, int num_tokens,
|
||||
cudaStream_t);
|
||||
|
||||
// GLM-5.2 q_b_proj TP8 (K=2048 -> N=2048). tile_m=16 keeps 128 CTAs (already
|
||||
// a single wave) and measures ahead of tile_m=32 here. Beats cuBLAS
|
||||
// 1.79-1.87x at M=3..8 (tile_n=8) and 1.19-1.30x at M=9..16 (tile_n=16);
|
||||
// M<=2 belongs to bf16_skinny_gemm, M>=20 to cuBLAS.
|
||||
template void invokeFusedAGemm<__nv_bfloat16, 2048, 2048, 8>(
|
||||
__nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, int num_tokens,
|
||||
cudaStream_t);
|
||||
|
||||
template void invokeFusedAGemm<__nv_bfloat16, 2048, 2048, 16>(
|
||||
__nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, int num_tokens,
|
||||
cudaStream_t);
|
||||
|
||||
// DSv3.2 q_b_proj TP8 (K=1536 -> N=3072). tile_m=32 keeps 96 CTAs (single
|
||||
// wave; tile_m=16's 192 straddle two). Beats cuBLAS 1.6-1.8x at M=1..8 and
|
||||
// 1.05-1.18x at M=9..16 on B300/B200; the whole 1..16 range dispatches here
|
||||
// (no skinny tier: K=1536 does not fit the GEMV's 128x8 K-step).
|
||||
template void invokeFusedAGemm<__nv_bfloat16, 1536, 3072, 8, 32>(
|
||||
__nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, int num_tokens,
|
||||
cudaStream_t);
|
||||
|
||||
template void invokeFusedAGemm<__nv_bfloat16, 1536, 3072, 16, 32>(
|
||||
__nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, int num_tokens,
|
||||
cudaStream_t);
|
||||
|
||||
void dsv3_fused_a_gemm(torch::stable::Tensor& output,
|
||||
torch::stable::Tensor const& mat_a,
|
||||
torch::stable::Tensor const& mat_b) {
|
||||
@@ -760,15 +710,12 @@ void dsv3_fused_a_gemm(torch::stable::Tensor& output,
|
||||
int const hd_in = mat_a.size(1);
|
||||
int const hd_out = mat_b.size(1);
|
||||
|
||||
bool const is_dsv3 = hd_in == 7168 && hd_out == 2112;
|
||||
bool const is_glm = hd_in == 6144 && hd_out == 2624;
|
||||
bool const is_glm_qb = hd_in == 2048 && hd_out == 2048;
|
||||
bool const is_ds_qb = hd_in == 1536 && hd_out == 3072;
|
||||
constexpr int kHdIn = 7168;
|
||||
constexpr int kHdOut = 2112;
|
||||
STD_TORCH_CHECK(num_tokens >= 1 && num_tokens <= 16,
|
||||
"required 1 <= mat_a.shape[0] <= 16");
|
||||
STD_TORCH_CHECK(is_dsv3 || is_glm || is_glm_qb || is_ds_qb,
|
||||
"supported (hd_in, hd_out): (7168, 2112), (6144, 2624), "
|
||||
"(2048, 2048), (1536, 3072)");
|
||||
STD_TORCH_CHECK(hd_in == kHdIn, "required mat_a.shape[1] == 7168");
|
||||
STD_TORCH_CHECK(hd_out == kHdOut, "required mat_b.shape[1] == 2112");
|
||||
STD_TORCH_CHECK(output.size(0) == num_tokens,
|
||||
"required output.shape[0] == mat_a.shape[0]");
|
||||
STD_TORCH_CHECK(output.size(1) == hd_out,
|
||||
@@ -791,41 +738,18 @@ void dsv3_fused_a_gemm(torch::stable::Tensor& output,
|
||||
STD_TORCH_CHECK(getSMVersion() >= 90, "required CUDA ARCH >= SM_90");
|
||||
|
||||
auto stream = get_current_cuda_stream(mat_a.get_device_index());
|
||||
auto* out = reinterpret_cast<__nv_bfloat16*>(output.mutable_data_ptr());
|
||||
auto* a = reinterpret_cast<__nv_bfloat16 const*>(mat_a.data_ptr());
|
||||
auto* b = reinterpret_cast<__nv_bfloat16 const*>(mat_b.data_ptr());
|
||||
if (is_dsv3) {
|
||||
if (num_tokens <= 8) {
|
||||
invokeFusedAGemm<__nv_bfloat16, 7168, 2112, 8>(out, a, b, num_tokens,
|
||||
stream);
|
||||
} else {
|
||||
invokeFusedAGemm<__nv_bfloat16, 7168, 2112, 16>(out, a, b, num_tokens,
|
||||
stream);
|
||||
}
|
||||
} else if (is_glm) {
|
||||
if (num_tokens <= 8) {
|
||||
invokeFusedAGemm<__nv_bfloat16, 6144, 2624, 8, 32>(out, a, b, num_tokens,
|
||||
stream);
|
||||
} else {
|
||||
invokeFusedAGemm<__nv_bfloat16, 6144, 2624, 16, 32>(out, a, b, num_tokens,
|
||||
stream);
|
||||
}
|
||||
} else if (is_glm_qb) {
|
||||
if (num_tokens <= 8) {
|
||||
invokeFusedAGemm<__nv_bfloat16, 2048, 2048, 8>(out, a, b, num_tokens,
|
||||
stream);
|
||||
} else {
|
||||
invokeFusedAGemm<__nv_bfloat16, 2048, 2048, 16>(out, a, b, num_tokens,
|
||||
stream);
|
||||
}
|
||||
if (num_tokens <= 8) {
|
||||
invokeFusedAGemm<__nv_bfloat16, kHdIn, kHdOut, 8>(
|
||||
reinterpret_cast<__nv_bfloat16*>(output.mutable_data_ptr()),
|
||||
reinterpret_cast<__nv_bfloat16 const*>(mat_a.data_ptr()),
|
||||
reinterpret_cast<__nv_bfloat16 const*>(mat_b.data_ptr()), num_tokens,
|
||||
stream);
|
||||
} else {
|
||||
if (num_tokens <= 8) {
|
||||
invokeFusedAGemm<__nv_bfloat16, 1536, 3072, 8, 32>(out, a, b, num_tokens,
|
||||
stream);
|
||||
} else {
|
||||
invokeFusedAGemm<__nv_bfloat16, 1536, 3072, 16, 32>(out, a, b, num_tokens,
|
||||
stream);
|
||||
}
|
||||
invokeFusedAGemm<__nv_bfloat16, kHdIn, kHdOut, 16>(
|
||||
reinterpret_cast<__nv_bfloat16*>(output.mutable_data_ptr()),
|
||||
reinterpret_cast<__nv_bfloat16 const*>(mat_a.data_ptr()),
|
||||
reinterpret_cast<__nv_bfloat16 const*>(mat_b.data_ptr()), num_tokens,
|
||||
stream);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -49,11 +49,6 @@ __global__ void __launch_bounds__(512, VLLM_BLOCKS_PER_SM(512))
|
||||
static_assert(sizeof(PackedVec) == sizeof(Type) * CVT_FP4_ELTS_PER_THREAD,
|
||||
"Vec size is not matched.");
|
||||
|
||||
#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900))
|
||||
cudaGridDependencySynchronize();
|
||||
cudaTriggerProgrammaticLaunchCompletion();
|
||||
#endif
|
||||
|
||||
// Precompute SF layout parameter (constant for entire kernel).
|
||||
int32_t const numKTiles = (outputCols + 63) / 64;
|
||||
|
||||
@@ -128,11 +123,6 @@ __global__ void __launch_bounds__(512, VLLM_BLOCKS_PER_SM(512))
|
||||
static_assert(sizeof(PackedVec) == sizeof(Type) * CVT_FP4_ELTS_PER_THREAD,
|
||||
"Vec size is not matched.");
|
||||
|
||||
#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900))
|
||||
cudaGridDependencySynchronize();
|
||||
cudaTriggerProgrammaticLaunchCompletion();
|
||||
#endif
|
||||
|
||||
int32_t const colIdx = blockDim.x * blockIdx.y + threadIdx.x;
|
||||
int elem_idx = colIdx * CVT_FP4_ELTS_PER_THREAD;
|
||||
|
||||
@@ -211,8 +201,6 @@ void scaled_fp4_quant_sm1xxa(torch::stable::Tensor const& output,
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(
|
||||
input.get_device_index());
|
||||
auto stream = get_current_cuda_stream(input.get_device_index());
|
||||
auto* device_props = get_device_prop();
|
||||
int const sm_version = device_props->major * 10 + device_props->minor;
|
||||
|
||||
int output_sf_n_unpadded = int(output_n / CVT_FP4_SF_VEC_SIZE);
|
||||
|
||||
@@ -236,21 +224,10 @@ void scaled_fp4_quant_sm1xxa(torch::stable::Tensor const& output,
|
||||
input.scalar_type(), "nvfp4_quant_kernel", [&] {
|
||||
using cuda_type = vllm::CUDATypeConverter<scalar_t>::Type;
|
||||
auto input_ptr = static_cast<cuda_type const*>(input.data_ptr());
|
||||
cudaLaunchConfig_t config = {};
|
||||
config.gridDim = grid;
|
||||
config.blockDim = block;
|
||||
config.dynamicSmemBytes = 0;
|
||||
config.stream = stream;
|
||||
cudaLaunchAttribute attrs[1];
|
||||
attrs[0].id = cudaLaunchAttributeProgrammaticStreamSerialization;
|
||||
attrs[0].val.programmaticStreamSerializationAllowed = 1;
|
||||
config.numAttrs = (sm_version >= 90) ? 1 : 0;
|
||||
config.attrs = attrs;
|
||||
cudaLaunchKernelEx(&config, vllm::cvt_fp16_to_fp4<cuda_type, false>,
|
||||
m, n, output_n, num_padded_cols, input_ptr,
|
||||
input_sf_ptr,
|
||||
reinterpret_cast<uint32_t*>(output_ptr),
|
||||
reinterpret_cast<uint32_t*>(sf_out));
|
||||
vllm::cvt_fp16_to_fp4<cuda_type, false><<<grid, block, 0, stream>>>(
|
||||
m, n, output_n, num_padded_cols, input_ptr, input_sf_ptr,
|
||||
reinterpret_cast<uint32_t*>(output_ptr),
|
||||
reinterpret_cast<uint32_t*>(sf_out));
|
||||
});
|
||||
} else {
|
||||
int num_packed_cols = output_n / CVT_FP4_ELTS_PER_THREAD;
|
||||
@@ -263,21 +240,12 @@ void scaled_fp4_quant_sm1xxa(torch::stable::Tensor const& output,
|
||||
input.scalar_type(), "nvfp4_quant_kernel", [&] {
|
||||
using cuda_type = vllm::CUDATypeConverter<scalar_t>::Type;
|
||||
auto input_ptr = static_cast<cuda_type const*>(input.data_ptr());
|
||||
cudaLaunchConfig_t config = {};
|
||||
config.gridDim = grid;
|
||||
config.blockDim = block;
|
||||
config.dynamicSmemBytes = 0;
|
||||
config.stream = stream;
|
||||
cudaLaunchAttribute attrs[1];
|
||||
attrs[0].id = cudaLaunchAttributeProgrammaticStreamSerialization;
|
||||
attrs[0].val.programmaticStreamSerializationAllowed = 1;
|
||||
config.numAttrs = (sm_version >= 90) ? 1 : 0;
|
||||
config.attrs = attrs;
|
||||
cudaLaunchKernelEx(
|
||||
&config, vllm::cvt_fp16_to_fp4_sf_major<cuda_type, false>, m, n,
|
||||
output_n, output_sf_n_unpadded, num_packed_cols, input_ptr,
|
||||
input_sf_ptr, reinterpret_cast<uint32_t*>(output_ptr),
|
||||
reinterpret_cast<uint32_t*>(sf_out));
|
||||
vllm::cvt_fp16_to_fp4_sf_major<cuda_type, false>
|
||||
<<<grid, block, 0, stream>>>(
|
||||
m, n, output_n, output_sf_n_unpadded, num_packed_cols,
|
||||
input_ptr, input_sf_ptr,
|
||||
reinterpret_cast<uint32_t*>(output_ptr),
|
||||
reinterpret_cast<uint32_t*>(sf_out));
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -330,10 +330,6 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
|
||||
// conditionally compiled so impl registration is in source file
|
||||
ops.def("fp32_router_gemm(Tensor! output, Tensor mat_a, Tensor mat_b) -> ()");
|
||||
|
||||
// BF16 skinny GEMM (M<=32, weight-BW-bound shapes, e.g. MTP eh_proj).
|
||||
// conditionally compiled so impl registration is in source file
|
||||
ops.def("bf16_skinny_gemm(Tensor! output, Tensor mat_a, Tensor mat_b) -> ()");
|
||||
|
||||
// reorder weight for AllSpark Ampere W8A16 Fused Gemm kernel
|
||||
ops.def(
|
||||
"rearrange_kn_weight_as_n32k16_order(Tensor b_qweight, Tensor b_scales, "
|
||||
|
||||
@@ -1,41 +0,0 @@
|
||||
# GLM-5.2-NVFP4 — Low-Latency (LL) serving recipe · 8×B300 · TP8 · MTP=5
|
||||
|
||||
Single-node 8×B300 (sm_100/103), pure tensor parallel, NVFP4 weights, fp8 KV cache,
|
||||
MTP=5 speculative decode, model runner v2.
|
||||
|
||||
Validated on `glm5.2-LL` @ `8d407aee1`.
|
||||
|
||||
## Versions
|
||||
|
||||
| Component | Version |
|
||||
| --- | --- |
|
||||
| vLLM | `glm5.2-LL` @ `8d407aee1` |
|
||||
| flashinfer-python | `0.6.15` |
|
||||
| Model | `nvidia/GLM-5.2-NVFP4` (`modelopt_fp4`) |
|
||||
|
||||
## Build (once)
|
||||
|
||||
```bash
|
||||
pip install 'flashinfer-python==0.6.15'
|
||||
|
||||
CUDA_HOME=/usr/local/cuda-13.0 TORCH_CUDA_ARCH_LIST="10.0 10.3" MAX_JOBS=96 \
|
||||
VLLM_USE_PRECOMPILED=0 VLLM_USE_PRECOMPILED_RUST=0 \
|
||||
pip install -e . --no-deps --no-build-isolation
|
||||
```
|
||||
|
||||
## Server
|
||||
|
||||
```bash
|
||||
export VLLM_USE_V2_MODEL_RUNNER=1 # model runner v2
|
||||
export VLLM_DEEP_GEMM_WARMUP=skip
|
||||
|
||||
vllm serve nvidia/GLM-5.2-NVFP4 \
|
||||
--trust-remote-code \
|
||||
--tensor-parallel-size 8 \
|
||||
--quantization modelopt_fp4 --kv-cache-dtype fp8_e4m3 \
|
||||
--max-model-len 32768 --max-num-batched-tokens 16384 --max-num-seqs 256 \
|
||||
--no-enable-prefix-caching --gpu-memory-utilization 0.85 \
|
||||
--speculative-config '{"method":"mtp","num_speculative_tokens":5}' \
|
||||
--kernel-config '{"ir_op_priority":{"rms_norm":["vllm_c","native"],"fused_add_rms_norm":["vllm_c","native"]}}' \
|
||||
--host 0.0.0.0 --port 8000
|
||||
```
|
||||
@@ -1,70 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Numerical correctness for the bf16 skinny GEMM (decode-M GEMV) kernel.
|
||||
|
||||
Covers every supported (N, K) shape across M = 0/1/2/8/16/32 (M=32 exercises
|
||||
the tile_m=32 MMA path), plus strided (column-slice) output. M=0 must return
|
||||
an empty result without launching the kernel (empty ranks at DP/PP
|
||||
boundaries)."""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
# (N, K) pairs must match bf16_skinny_gemm_supported() in the entry .cu.
|
||||
SHAPES = [
|
||||
(768, 12288),
|
||||
(1536, 12288),
|
||||
(6144, 12288),
|
||||
(2048, 2048),
|
||||
(2624, 6144),
|
||||
(2112, 7168),
|
||||
(7168, 14336),
|
||||
(512, 6144),
|
||||
]
|
||||
MS = [0, 1, 2, 8, 16, 32]
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_cuda() or not current_platform.has_device_capability(90),
|
||||
reason="bf16_skinny_gemm requires CUDA SM90+",
|
||||
)
|
||||
|
||||
|
||||
def _rel_err(out: torch.Tensor, ref: torch.Tensor) -> float:
|
||||
return (out.float() - ref).norm().item() / ref.norm().clamp_min(1e-6).item()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("n,k", SHAPES)
|
||||
@pytest.mark.parametrize("m", MS)
|
||||
def test_bf16_skinny_gemm_matches_reference(n: int, k: int, m: int):
|
||||
torch.manual_seed(0)
|
||||
x = torch.randn(m, k, dtype=torch.bfloat16, device="cuda")
|
||||
w = torch.randn(n, k, dtype=torch.bfloat16, device="cuda")
|
||||
|
||||
out = ops.bf16_skinny_gemm(x, w)
|
||||
assert out.shape == (m, n)
|
||||
assert out.dtype == torch.bfloat16
|
||||
|
||||
if m == 0: # empty batch: nothing to check beyond the empty shape
|
||||
return
|
||||
ref = x.float() @ w.float().t()
|
||||
rel = _rel_err(out, ref)
|
||||
assert rel < 2e-2, f"rel err {rel:.4f} for (m,n,k)=({m},{n},{k})"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("n,k", [(2048, 2048), (512, 6144)])
|
||||
def test_bf16_skinny_gemm_strided_output(n: int, k: int):
|
||||
"""Output is a column-slice of a wider padded buffer (row stride > N)."""
|
||||
torch.manual_seed(0)
|
||||
m = 8
|
||||
x = torch.randn(m, k, dtype=torch.bfloat16, device="cuda")
|
||||
w = torch.randn(n, k, dtype=torch.bfloat16, device="cuda")
|
||||
|
||||
wide = torch.empty(m, n + 64, dtype=torch.bfloat16, device="cuda")
|
||||
view = wide[:, :n]
|
||||
torch.ops._C.bf16_skinny_gemm(view, x, w)
|
||||
|
||||
ref = x.float() @ w.float().t()
|
||||
assert _rel_err(view, ref) < 2e-2
|
||||
@@ -49,32 +49,6 @@ pytestmark = pytest.mark.skipif(
|
||||
)
|
||||
|
||||
|
||||
def test_platform_capability_queries_are_constant_during_compile(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
K.current_platform, "has_device_capability", lambda capability: True
|
||||
)
|
||||
monkeypatch.setattr(K, "has_cutedsl", lambda: True)
|
||||
monkeypatch.setattr(K.current_platform, "is_arch_support_pdl", lambda: True)
|
||||
|
||||
def capability_branches(x: torch.Tensor) -> torch.Tensor:
|
||||
if K._can_use_fused_q_cutedsl() and K._is_arch_support_pdl():
|
||||
return x + 1
|
||||
return x - 1
|
||||
|
||||
compiled = torch.compile(capability_branches, backend="eager", fullgraph=True)
|
||||
x = torch.zeros(1, device="cuda")
|
||||
torch.testing.assert_close(compiled(x), torch.ones_like(x))
|
||||
|
||||
|
||||
def test_pdl_is_disabled_before_blackwell(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
K.current_platform, "has_device_capability", lambda capability: False
|
||||
)
|
||||
monkeypatch.setattr(K.current_platform, "is_arch_support_pdl", lambda: True)
|
||||
|
||||
assert not K._is_arch_support_pdl()
|
||||
|
||||
|
||||
# ── reference helpers ────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@@ -318,57 +292,6 @@ def test_fused_norm_rope_no_indexer(num_tokens: int):
|
||||
assert (topk == 7).all(), "topk buffer should be untouched on shared layer"
|
||||
|
||||
|
||||
def test_fused_norm_rope_profile_without_cache_compiles():
|
||||
"""The cache-free profiling path must still compile and produce Q."""
|
||||
torch.manual_seed(2)
|
||||
dev = "cuda"
|
||||
num_tokens = 4
|
||||
pos = torch.arange(num_tokens, device=dev, dtype=torch.int64)
|
||||
q_c = torch.randn(num_tokens, Q_LORA, device=dev, dtype=torch.bfloat16)
|
||||
kv_c = torch.randn(num_tokens, KV_LORA, device=dev, dtype=torch.bfloat16)
|
||||
k_pe = torch.randn(num_tokens, ROPE_DIM, device=dev, dtype=torch.bfloat16)
|
||||
qw = torch.randn(Q_LORA, device=dev, dtype=torch.bfloat16)
|
||||
kvw = torch.randn(KV_LORA, device=dev, dtype=torch.bfloat16)
|
||||
index_k = torch.randn(num_tokens, INDEX_HEAD_DIM, device=dev, dtype=torch.bfloat16)
|
||||
index_w = torch.randn(INDEX_HEAD_DIM, device=dev, dtype=torch.float32)
|
||||
index_b = torch.randn(INDEX_HEAD_DIM, device=dev, dtype=torch.float32)
|
||||
cos_sin = make_cos_sin(64, ROPE_DIM, dev)
|
||||
topk = torch.empty(num_tokens, 2048, device=dev, dtype=torch.int32)
|
||||
|
||||
def profile_run(
|
||||
q_c: torch.Tensor,
|
||||
kv_c: torch.Tensor,
|
||||
k_pe: torch.Tensor,
|
||||
index_k: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
return K.fused_norm_rope(
|
||||
pos,
|
||||
q_c,
|
||||
qw,
|
||||
EPS,
|
||||
kv_c,
|
||||
kvw,
|
||||
EPS,
|
||||
k_pe,
|
||||
cos_sin,
|
||||
index_k,
|
||||
index_w,
|
||||
index_b,
|
||||
EPS,
|
||||
cos_sin,
|
||||
topk,
|
||||
slot_mapping=None,
|
||||
indexer_k_cache=None,
|
||||
mla_kv_cache=None,
|
||||
has_indexer=True,
|
||||
index_rope_interleave=False,
|
||||
)
|
||||
|
||||
compiled = torch.compile(profile_run, fullgraph=True)
|
||||
q_out = compiled(q_c, kv_c, k_pe, index_k)
|
||||
assert_bf16(q_out, rms_norm(q_c, qw), "q_c rmsnorm (profiling)")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [1, 4, 17, 512])
|
||||
def test_fused_norm_rope_ds_mla(num_tokens: int):
|
||||
"""fp8_ds_mla MLA cache layout (FlashMLA sparse, bf16-query path; SM90/SM100).
|
||||
|
||||
@@ -1600,10 +1600,6 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
|
||||
speculative_model="luccafong/deepseek_mtp_draft_random",
|
||||
trust_remote_code=True,
|
||||
),
|
||||
"DeepseekV32MTPModel": _HfExamplesInfo(
|
||||
"deepseek-ai/DeepSeek-V3.2-Exp",
|
||||
speculative_model="deepseek-ai/DeepSeek-V3.2-Exp",
|
||||
),
|
||||
"DeepSeekV4MTPModel": _HfExamplesInfo(
|
||||
"deepseek-ai/DeepSeek-V4-Flash",
|
||||
speculative_model="deepseek-ai/DeepSeek-V4-Flash",
|
||||
|
||||
@@ -2371,20 +2371,6 @@ def fp32_router_gemm(
|
||||
return output
|
||||
|
||||
|
||||
def bf16_skinny_gemm(
|
||||
x: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""Skinny bf16 GEMM (M<=32): x [M, K] @ weight [N, K]^T -> [M, N].
|
||||
|
||||
Single block-per-column kernel with fp32 accumulation; replaces cuBLAS
|
||||
splitK for weight-bandwidth-bound decode shapes (e.g. MTP eh_proj).
|
||||
"""
|
||||
output = torch.empty((x.shape[0], weight.shape[0]), dtype=x.dtype, device=x.device)
|
||||
torch.ops._C.bf16_skinny_gemm(output, x, weight)
|
||||
return output
|
||||
|
||||
|
||||
if hasattr(torch.ops, "_C") and hasattr(torch.ops._C, "fp32_router_gemm"):
|
||||
|
||||
@register_fake("_C::fp32_router_gemm")
|
||||
@@ -2396,17 +2382,6 @@ if hasattr(torch.ops, "_C") and hasattr(torch.ops._C, "fp32_router_gemm"):
|
||||
return
|
||||
|
||||
|
||||
if hasattr(torch.ops, "_C") and hasattr(torch.ops._C, "bf16_skinny_gemm"):
|
||||
|
||||
@register_fake("_C::bf16_skinny_gemm")
|
||||
def bf16_skinny_gemm_fake(
|
||||
output: torch.Tensor,
|
||||
mat_a: torch.Tensor,
|
||||
mat_b: torch.Tensor,
|
||||
) -> None:
|
||||
return
|
||||
|
||||
|
||||
def topk_softmax(
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
|
||||
@@ -202,7 +202,11 @@ if flashinfer_comm is not None:
|
||||
element_size = allreduce_in.element_size()
|
||||
current_tensor_size = num_tokens * hidden_size * element_size
|
||||
max_tensor_size = max_token_num * hidden_size * element_size
|
||||
oversize = current_tensor_size > max_tensor_size
|
||||
assert current_tensor_size <= max_tensor_size, (
|
||||
f"Current tensor size {current_tensor_size} is larger than "
|
||||
f"max token num {max_token_num} * hidden size {hidden_size} * "
|
||||
f"element size {element_size}"
|
||||
)
|
||||
curr_device = current_platform.get_device_capability()
|
||||
device_capability = curr_device.to_int() if curr_device is not None else None
|
||||
|
||||
@@ -215,47 +219,6 @@ if flashinfer_comm is not None:
|
||||
get_workspace_fn = (
|
||||
get_fi_ar_quant_workspace if is_quant_pattern else get_fi_ar_workspace
|
||||
)
|
||||
|
||||
orig_norm_out = norm_out
|
||||
|
||||
def unfused_fallback() -> None:
|
||||
# Fused-path equivalent honoring the op's output-buffer contract:
|
||||
# with norm_out given, normed -> norm_out and the new residual ->
|
||||
# the allreduce_in buffer; otherwise normed -> allreduce_in and
|
||||
# the new residual -> the residual buffer. Runs fine during CUDA
|
||||
# graph capture (the branch is resolved per captured shape).
|
||||
#
|
||||
# weight_bias mirrors the fused kernel's gamma offset (Gemma
|
||||
# patterns pass 1.0 for rms_norm(x, weight + 1)); fold it into
|
||||
# the gamma in fp32 like the reference before the plain rms_norm.
|
||||
gamma = (
|
||||
rms_gamma
|
||||
if weight_bias == 0.0
|
||||
else (rms_gamma.float() + weight_bias).to(rms_gamma.dtype)
|
||||
)
|
||||
reduced = tensor_model_parallel_all_reduce(allreduce_in)
|
||||
new_residual = reduced + residual
|
||||
if orig_norm_out is None:
|
||||
torch.ops._C.rms_norm(allreduce_in, new_residual, gamma, rms_eps)
|
||||
residual.copy_(new_residual)
|
||||
else:
|
||||
torch.ops._C.rms_norm(orig_norm_out, new_residual, gamma, rms_eps)
|
||||
allreduce_in.copy_(new_residual)
|
||||
|
||||
if oversize:
|
||||
assert not is_quant_pattern, (
|
||||
"flashinfer fused allreduce: input exceeds the workspace budget "
|
||||
"and the unfused fallback does not cover quant patterns"
|
||||
)
|
||||
logger.warning_once(
|
||||
"fused allreduce+rmsnorm: %d tokens exceeds the workspace "
|
||||
"budget (max_token_num=%d); using the unfused fallback for "
|
||||
"this shape.",
|
||||
num_tokens,
|
||||
max_token_num,
|
||||
)
|
||||
unfused_fallback()
|
||||
return
|
||||
workspace = get_workspace_fn(
|
||||
world_size=world_size,
|
||||
rank=get_tensor_model_parallel_rank(),
|
||||
@@ -289,75 +252,36 @@ if flashinfer_comm is not None:
|
||||
if workspace.backend in ("trtllm", "mnnvl"):
|
||||
layout_code = flashinfer_comm.QuantizationSFLayout.SWIZZLED_128x4
|
||||
|
||||
# vllm's one-shot size table is tuned for the trtllm backend. For
|
||||
# mnnvl, flashinfer caps the workspace's one-shot region at its own
|
||||
# MNNVL_ONE_SHOT_THRESHOLD (~4MB, e.g. 44 tokens at hidden 6144 bf16
|
||||
# TP8); forcing use_oneshot=True past that raises ValueError instead
|
||||
# of using two-shot, so every decode capture between the one-shot cap
|
||||
# and the fusion threshold silently lost fusion. Let flashinfer's
|
||||
# AUTO heuristic pick the strategy (it matches its own workspace
|
||||
# sizing), and keep trigger_completion_at_end=True since AUTO may
|
||||
# select one-shot (see the Lamport NaN note below).
|
||||
mnnvl_auto = workspace.backend == "mnnvl"
|
||||
if mnnvl_auto:
|
||||
use_oneshot = None
|
||||
|
||||
try:
|
||||
flashinfer_comm.allreduce_fusion(
|
||||
input=allreduce_in,
|
||||
workspace=workspace,
|
||||
pattern=pattern_code,
|
||||
launch_with_pdl=launch_with_pdl,
|
||||
output=None,
|
||||
residual_out=residual_out,
|
||||
norm_out=norm_out,
|
||||
quant_out=quant_out,
|
||||
scale_out=scale_out,
|
||||
residual_in=residual,
|
||||
rms_gamma=rms_gamma,
|
||||
rms_eps=rms_eps,
|
||||
scale_factor=scale_factor,
|
||||
layout_code=layout_code,
|
||||
use_oneshot=use_oneshot,
|
||||
fp32_acc=fp32_acc,
|
||||
weight_bias=weight_bias,
|
||||
# The one-shot Lamport all-reduce signals PDL completion before
|
||||
# its output buffer is committed when trigger_completion_at_end
|
||||
# is False, so the next PDL-launched kernel can read the
|
||||
# uninitialized Lamport buffer and produce NaN. This only fires
|
||||
# for num_tokens <= PDL_ADVANCE_LAUNCH_TOKENS (the batch=1 /
|
||||
# spec-decode shapes, where the one-shot path is always
|
||||
# selected). Complete at the end for the one-shot path; the
|
||||
# two-shot path is synchronized and keeps the early completion.
|
||||
# Related one-shot instability in the same kernel:
|
||||
# flashinfer-ai/flashinfer#1223.
|
||||
trigger_completion_at_end=True
|
||||
if mnnvl_auto
|
||||
else (use_oneshot or num_tokens > PDL_ADVANCE_LAUNCH_TOKENS),
|
||||
)
|
||||
except ValueError as e:
|
||||
# The globally cached workspace can be smaller than this call's
|
||||
# budget (it is created once by whichever caller runs first, and
|
||||
# the mnnvl workspace's is_buffer_size_sufficient() checks trtllm
|
||||
# metadata, so vllm-side accounting cannot see the real capacity).
|
||||
# flashinfer raises before launching anything, so falling back
|
||||
# mid-capture is safe. Observed with spec-decode draft graphs
|
||||
# whose token count (max_num_reqs * (1 + num_spec)) exceeds the
|
||||
# workspace sized for the fusion threshold.
|
||||
assert not is_quant_pattern, (
|
||||
"flashinfer fused allreduce failed and the unfused fallback "
|
||||
f"does not cover quant patterns: {e}"
|
||||
)
|
||||
logger.warning_once(
|
||||
"flashinfer fused allreduce+rmsnorm unavailable for this "
|
||||
"shape (num_tokens=%d, max_token_num=%d, world_size=%d): %s. "
|
||||
"Using the unfused fallback.",
|
||||
num_tokens,
|
||||
max_token_num,
|
||||
world_size,
|
||||
e,
|
||||
)
|
||||
unfused_fallback()
|
||||
flashinfer_comm.allreduce_fusion(
|
||||
input=allreduce_in,
|
||||
workspace=workspace,
|
||||
pattern=pattern_code,
|
||||
launch_with_pdl=launch_with_pdl,
|
||||
output=None,
|
||||
residual_out=residual_out,
|
||||
norm_out=norm_out,
|
||||
quant_out=quant_out,
|
||||
scale_out=scale_out,
|
||||
residual_in=residual,
|
||||
rms_gamma=rms_gamma,
|
||||
rms_eps=rms_eps,
|
||||
scale_factor=scale_factor,
|
||||
layout_code=layout_code,
|
||||
use_oneshot=use_oneshot,
|
||||
fp32_acc=fp32_acc,
|
||||
weight_bias=weight_bias,
|
||||
# The one-shot Lamport all-reduce signals PDL completion before its
|
||||
# output buffer is committed when trigger_completion_at_end is
|
||||
# False, so the next PDL-launched kernel can read the uninitialized
|
||||
# Lamport buffer and produce NaN. This only fires for
|
||||
# num_tokens <= PDL_ADVANCE_LAUNCH_TOKENS (the batch=1 / spec-decode
|
||||
# shapes, where the one-shot path is always selected). Complete at
|
||||
# the end for the one-shot path; the two-shot path is synchronized
|
||||
# and keeps the early completion. Related one-shot instability in
|
||||
# the same kernel: flashinfer-ai/flashinfer#1223.
|
||||
trigger_completion_at_end=(use_oneshot is True)
|
||||
or num_tokens > PDL_ADVANCE_LAUNCH_TOKENS,
|
||||
)
|
||||
|
||||
def call_trtllm_fused_allreduce_norm_fake(
|
||||
allreduce_in: torch.Tensor,
|
||||
|
||||
@@ -330,7 +330,6 @@ class SpeculativeConfig:
|
||||
@staticmethod
|
||||
def hf_config_override(hf_config: PretrainedConfig) -> PretrainedConfig:
|
||||
initial_architecture = hf_config.architectures[0]
|
||||
use_sparse_mtp = hf_config.model_type == "glm_moe_dsa"
|
||||
if hf_config.model_type in (
|
||||
"deepseek_v3",
|
||||
"deepseek_v32",
|
||||
@@ -340,12 +339,7 @@ class SpeculativeConfig:
|
||||
if hf_config.model_type == "deepseek_mtp":
|
||||
n_predict = getattr(hf_config, "num_nextn_predict_layers", None)
|
||||
hf_config.update(
|
||||
{
|
||||
"n_predict": n_predict,
|
||||
"architectures": [
|
||||
"DeepseekV32MTPModel" if use_sparse_mtp else "DeepSeekMTPModel"
|
||||
],
|
||||
}
|
||||
{"n_predict": n_predict, "architectures": ["DeepSeekMTPModel"]}
|
||||
)
|
||||
if hf_config.model_type == "deepseek_v4":
|
||||
hf_config.model_type = "deepseek_mtp"
|
||||
|
||||
@@ -33,11 +33,6 @@ EVICT_NORMAL = Int64(0x1000000000000000)
|
||||
EVICT_FIRST = Int64(0x12F0000000000000)
|
||||
EVICT_LAST = Int64(0x14F0000000000000)
|
||||
|
||||
TORCH_TO_CUTE_DTYPE = {
|
||||
torch.bfloat16: BFloat16,
|
||||
torch.float32: Float32,
|
||||
}
|
||||
|
||||
|
||||
@dsl_user_op
|
||||
def recast_val(x, dtype, *, loc=None, ip=None):
|
||||
|
||||
+1
-22
@@ -1,6 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from cutlass import BFloat16, Constexpr, Float32, Uint16, Uint32, cute
|
||||
from cutlass import Constexpr, Float32, Uint32, cute
|
||||
from cutlass._mlir import ir
|
||||
from cutlass._mlir.dialects import llvm, vector
|
||||
from cutlass.cutlass_dsl import T, dsl_user_op
|
||||
@@ -38,11 +38,6 @@ def bf16x2_to_fp32x2(data, *, loc=None, ip=None) -> tuple[Float32, Float32]:
|
||||
)
|
||||
|
||||
elif isinstance(data, (cute.Tensor, cute.TensorSSA)):
|
||||
# recast to 32-bit registers if needed
|
||||
if data.element_type == BFloat16:
|
||||
data = cute.recast_tensor(data, Uint32)
|
||||
assert data.element_type == Uint32
|
||||
|
||||
# NOTE: the output is always 1D
|
||||
size = cute.size(data.shape)
|
||||
out = cute.make_rmem_tensor(size * 2, Float32)
|
||||
@@ -89,22 +84,6 @@ def fp8x4_to_bf16x4(x: Uint32, *, loc=None, ip=None) -> cute.TensorSSA:
|
||||
return cute.TensorSSA(vec, 2, Uint32)
|
||||
|
||||
|
||||
@dsl_user_op
|
||||
def fp32x2_to_fp8x2(a0: Float32, a1: Float32, *, loc=None, ip=None) -> Uint16:
|
||||
out = llvm.inline_asm(
|
||||
T.i16(),
|
||||
[
|
||||
a0.ir_value(loc=loc, ip=ip),
|
||||
a1.ir_value(loc=loc, ip=ip),
|
||||
],
|
||||
"cvt.rn.satfinite.e4m3x2.f32 $0, $2, $1;",
|
||||
"=h,f,f",
|
||||
has_side_effects=False,
|
||||
is_align_stack=False,
|
||||
)
|
||||
return Uint16(out)
|
||||
|
||||
|
||||
@dsl_user_op
|
||||
def fp8x4_to_fp16x4(x: Uint32, *, loc=None, ip=None) -> cute.TensorSSA:
|
||||
out = llvm.inline_asm(
|
||||
|
||||
@@ -43,7 +43,7 @@ try:
|
||||
if flashinfer_comm is not None
|
||||
else None
|
||||
)
|
||||
except (ImportError, AttributeError):
|
||||
except ImportError:
|
||||
flashinfer_trtllm_fused_allreduce_norm = None # type: ignore[assignment]
|
||||
get_fi_ar_workspace = None # type: ignore[assignment]
|
||||
_AR_RESIDUAL_RMS_NORM = None
|
||||
@@ -52,23 +52,12 @@ except (ImportError, AttributeError):
|
||||
_FI_SUPPORTED_DTYPES = (torch.bfloat16, torch.float16)
|
||||
|
||||
|
||||
@torch.compiler.assume_constant_result
|
||||
def _fi_ar_max_size_mb() -> dict[int, float]:
|
||||
"""Flashinfer all-reduce fusion size table for the current device.
|
||||
|
||||
Device capability is constant; marking the result constant keeps this out
|
||||
of the traced graph. Otherwise the per-forward call reaches
|
||||
``current_platform.get_device_capability()`` inside ``torch.compile`` and
|
||||
graph-breaks ("can't handle functions not implemented in python")."""
|
||||
from vllm.config.compilation import PassConfig
|
||||
|
||||
return PassConfig.default_fi_allreduce_fusion_max_size_mb()
|
||||
|
||||
|
||||
def _max_token_num(tp_size: int, hidden_size: int, dtype: torch.dtype) -> int | None:
|
||||
"""Workspace token budget for flashinfer fused all-reduce, or None if the
|
||||
current world size / device is unsupported. Mirrors ``FlashInferAllReduce``."""
|
||||
max_size_mb = _fi_ar_max_size_mb().get(tp_size)
|
||||
from vllm.config.compilation import PassConfig
|
||||
|
||||
max_size_mb = PassConfig.default_fi_allreduce_fusion_max_size_mb().get(tp_size)
|
||||
if not max_size_mb:
|
||||
return None
|
||||
element_size = torch.tensor([], dtype=dtype).element_size()
|
||||
|
||||
@@ -114,7 +114,7 @@ _TEXT_GENERATION_MODELS = {
|
||||
"Glm4ForCausalLM": ("glm4", "Glm4ForCausalLM"),
|
||||
"Glm4MoeForCausalLM": ("glm4_moe", "Glm4MoeForCausalLM"),
|
||||
"Glm4MoeLiteForCausalLM": ("glm4_moe_lite", "Glm4MoeLiteForCausalLM"),
|
||||
"GlmMoeDsaForCausalLM": ("vllm.models.deepseek_v32", "GlmMoeDsaForCausalLM"),
|
||||
"GlmMoeDsaForCausalLM": ("deepseek_v2", "GlmMoeDsaForCausalLM"),
|
||||
"GptOssForCausalLM": ("gpt_oss", "GptOssForCausalLM"),
|
||||
"GPT2LMHeadModel": ("gpt2", "GPT2LMHeadModel"),
|
||||
"GPTJForCausalLM": ("gpt_j", "GPTJForCausalLM"),
|
||||
@@ -628,7 +628,6 @@ _SPECULATIVE_DECODING_MODELS = {
|
||||
"Eagle3DeepseekV3ForCausalLM": ("deepseek_eagle3", "Eagle3DeepseekV2ForCausalLM"),
|
||||
"EagleDeepSeekMTPModel": ("deepseek_eagle", "EagleDeepseekV3ForCausalLM"),
|
||||
"DeepSeekMTPModel": ("deepseek_mtp", "DeepSeekMTP"),
|
||||
"DeepseekV32MTPModel": ("vllm.models.deepseek_v32", "DeepseekV32MTP"),
|
||||
"DeepSeekV4MTPModel": ("vllm.models.deepseek_v4", "DeepSeekV4MTP"),
|
||||
"MiniMaxM3MTP": ("vllm.models.minimax_m3", "MiniMaxM3MTP"),
|
||||
"BailingMoeV25MTPModel": ("bailing_moe_mtp", "BailingMoeV25MTPModel"),
|
||||
|
||||
@@ -6,31 +6,17 @@ DeepSeek V3.2 introduced the DeepSeek Sparse Attention (DSA) architecture:
|
||||
MLA + a "lightning indexer" that selects the top-k tokens for a sparse MLA
|
||||
attend. The same model code serves any DSA checkpoint, including GLM-5.2
|
||||
(``glm_moe_dsa``), which reuses this architecture.
|
||||
|
||||
The optimized kernels under ``nvidia/`` target the Blackwell (SM100) family.
|
||||
Every other platform — ROCm, XPU, pre-SM100 CUDA (e.g. H100), CPU — falls back
|
||||
to the generic ``deepseek_v2`` implementation, which already handles the DSA
|
||||
(index_topk) architecture and is ``torch.compile``-friendly there. This matches
|
||||
main's behavior on those platforms (no hard failure).
|
||||
"""
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if current_platform.is_cuda() and current_platform.is_device_capability_family(100):
|
||||
from .nvidia.model import DeepseekV32ForCausalLM
|
||||
from .nvidia.mtp import DeepseekV32MTP
|
||||
if current_platform.is_rocm() or current_platform.is_xpu():
|
||||
raise NotImplementedError("deepseek_v32 currently supports NVIDIA SM100 only.")
|
||||
|
||||
# GLM-5.2 (glm_moe_dsa) reuses the same optimized DSA module on SM100.
|
||||
GlmMoeDsaForCausalLM = DeepseekV32ForCausalLM
|
||||
else:
|
||||
from vllm.model_executor.models.deepseek_mtp import DeepSeekMTP as DeepseekV32MTP
|
||||
from vllm.model_executor.models.deepseek_v2 import (
|
||||
DeepseekV3ForCausalLM as DeepseekV32ForCausalLM,
|
||||
)
|
||||
from vllm.model_executor.models.deepseek_v2 import GlmMoeDsaForCausalLM
|
||||
from .nvidia.model import DeepseekV32ForCausalLM
|
||||
from .nvidia.mtp import DeepseekV32MTP
|
||||
|
||||
__all__ = [
|
||||
"DeepseekV32ForCausalLM",
|
||||
"DeepseekV32MTP",
|
||||
"GlmMoeDsaForCausalLM",
|
||||
]
|
||||
|
||||
@@ -4,7 +4,6 @@ import torch
|
||||
import torch.nn as nn
|
||||
from transformers import DeepseekV2Config, DeepseekV3Config
|
||||
|
||||
import vllm._custom_ops as ops
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.config import CacheConfig, VllmConfig
|
||||
from vllm.distributed import get_tensor_model_parallel_world_size
|
||||
@@ -22,15 +21,17 @@ from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
per_token_group_quant_fp8,
|
||||
)
|
||||
from vllm.model_executor.layers.rotary_embedding import get_rope
|
||||
from vllm.model_executor.layers.sparse_attn_indexer import SparseAttnIndexer
|
||||
from vllm.model_executor.layers.sparse_attn_indexer import (
|
||||
SparseAttnIndexer,
|
||||
sparse_attn_indexer,
|
||||
)
|
||||
from vllm.model_executor.models.deepseek_v2 import (
|
||||
DeepSeekV2FusedQkvAProjLinear,
|
||||
DeepseekV32IndexerCache,
|
||||
yarn_get_mscale,
|
||||
)
|
||||
from vllm.model_executor.models.utils import extract_layer_index
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import _encode_layer_name, is_quantized_kv_cache
|
||||
from vllm.utils.torch_utils import is_quantized_kv_cache
|
||||
|
||||
from .kernels import fused_norm_rope, fused_q
|
||||
|
||||
@@ -170,11 +171,6 @@ class DeepseekV32Attention(MLAAttention):
|
||||
) -> None:
|
||||
quant_config = vllm_config.quant_config
|
||||
cache_config = vllm_config.cache_config
|
||||
if cache_config is not None and cache_config.cache_dtype == "auto":
|
||||
# This implementation requires an FP8 sparse cache. Start with the
|
||||
# generic FP8 format; MLAAttention will canonicalize it for the
|
||||
# selected sparse backend.
|
||||
cache_config.cache_dtype = "fp8"
|
||||
|
||||
hidden_size = config.hidden_size
|
||||
qk_nope_head_dim = config.qk_nope_head_dim
|
||||
@@ -335,37 +331,6 @@ class DeepseekV32Attention(MLAAttention):
|
||||
rope_parameters=config.rope_parameters,
|
||||
is_neox_style=False,
|
||||
)
|
||||
# Decode-M GEMM dispatch for the bf16 A/B projections, measured on
|
||||
# B300 and B200 (graph replay + rotating weights vs fp64):
|
||||
# bf16_skinny_gemm for M <= skinny_max, dsv3_fused_a_gemm up to
|
||||
# M = 16, cuBLAS above. Blackwell-only: the boundaries and the
|
||||
# fused_a tile choices are tied to the 148/152-SM single-wave
|
||||
# geometry. DSv3.2 q_b has no skinny tier (K=1536 does not fit the
|
||||
# GEMV's 128x8 K-step; fused_a still wins 1.6-1.8x there).
|
||||
# Quantized linear methods may not register .weight until
|
||||
# process_weights_after_loading (e.g. compressed-tensors NVFP4
|
||||
# registers weight_packed at init) — probe with getattr.
|
||||
# (N, K) -> skinny_max
|
||||
qkv_a_dispatch = {(2624, 6144): 2, (2112, 7168): 2}
|
||||
q_b_dispatch = {(2048, 2048): 2, (3072, 1536): 0}
|
||||
ok = (
|
||||
current_platform.is_device_capability_family(100)
|
||||
and hasattr(torch.ops._C, "bf16_skinny_gemm")
|
||||
and hasattr(torch.ops._C, "dsv3_fused_a_gemm")
|
||||
)
|
||||
qkv_a_weight = getattr(self.fused_qkv_a_proj, "weight", None)
|
||||
q_b_weight = getattr(self.q_b_proj, "weight", None)
|
||||
self._qkv_a_skinny_max = (
|
||||
qkv_a_dispatch.get(tuple(qkv_a_weight.shape))
|
||||
if ok and qkv_a_weight is not None and qkv_a_weight.dtype == torch.bfloat16
|
||||
else None
|
||||
)
|
||||
self._q_b_skinny_max = (
|
||||
q_b_dispatch.get(tuple(q_b_weight.shape))
|
||||
if ok and q_b_weight is not None and q_b_weight.dtype == torch.bfloat16
|
||||
else None
|
||||
)
|
||||
|
||||
# Lightning indexer uses its own RoPE; interleave maps to non-NeoX.
|
||||
self.indexer_rope_emb = get_rope(
|
||||
qk_rope_head_dim,
|
||||
@@ -374,32 +339,13 @@ class DeepseekV32Attention(MLAAttention):
|
||||
is_neox_style=not getattr(config, "indexer_rope_interleave", False),
|
||||
)
|
||||
|
||||
def _decode_m_gemm(
|
||||
self, x: torch.Tensor, weight: torch.Tensor, skinny_max: int
|
||||
) -> torch.Tensor:
|
||||
"""x @ weight.T for decode M <= 16."""
|
||||
if x.shape[0] == 0:
|
||||
return torch.empty((0, weight.shape[0]), dtype=x.dtype, device=x.device)
|
||||
if x.shape[0] <= skinny_max:
|
||||
return ops.bf16_skinny_gemm(x, weight)
|
||||
out = torch.empty((x.shape[0], weight.shape[0]), dtype=x.dtype, device=x.device)
|
||||
ops.dsv3_fused_a_gemm(out, x, weight.t())
|
||||
return out
|
||||
|
||||
def forward( # type: ignore[override]
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
# Captured: A-projections (+ indexer A-GEMM on indexer layers).
|
||||
if self._qkv_a_skinny_max is not None and hidden_states.shape[0] <= 16:
|
||||
qkv_lora = self._decode_m_gemm(
|
||||
hidden_states,
|
||||
self.fused_qkv_a_proj.weight,
|
||||
self._qkv_a_skinny_max,
|
||||
)
|
||||
else:
|
||||
qkv_lora = self.fused_qkv_a_proj(hidden_states)[0]
|
||||
qkv_lora = self.fused_qkv_a_proj(hidden_states)[0]
|
||||
q_c, kv_c, k_pe = qkv_lora.split(
|
||||
[self.q_lora_rank, self.kv_lora_rank, self.qk_rope_head_dim], dim=-1
|
||||
)
|
||||
@@ -452,9 +398,7 @@ class DeepseekV32Attention(MLAAttention):
|
||||
assert isinstance(slot_mapping, dict)
|
||||
mla_slot = slot_mapping.get(self.layer_name)
|
||||
|
||||
# skip_topk (index_share_for_mtp_iteration reuse mode): don't run the
|
||||
# indexer this step; keep the top-k already in the shared buffer.
|
||||
if self.indexer is not None and not self.skip_topk:
|
||||
if self.indexer is not None:
|
||||
has_indexer = True
|
||||
indexer_k_norm_w = self.indexer.k_norm.weight
|
||||
indexer_k_norm_bias = self.indexer.k_norm.bias
|
||||
@@ -507,16 +451,12 @@ class DeepseekV32Attention(MLAAttention):
|
||||
index_rope_interleave=self._index_rope_interleave,
|
||||
)
|
||||
|
||||
if self._q_b_skinny_max is not None and q_c.shape[0] <= 16:
|
||||
q = self._decode_m_gemm(q_c, self.q_b_proj.weight, self._q_b_skinny_max)
|
||||
else:
|
||||
q = self.q_b_proj(q_c)[0]
|
||||
q = q.view(-1, self.num_local_heads, self.qk_head_dim)
|
||||
q = self.q_b_proj(q_c)[0].view(-1, self.num_local_heads, self.qk_head_dim)
|
||||
q_nope, q_pe = q.split([self.qk_nope_head_dim, self.qk_rope_head_dim], dim=-1)
|
||||
q_nope = q_nope.transpose(0, 1)
|
||||
ql_nope = torch.bmm(q_nope, self.W_UK_T).transpose(0, 1)
|
||||
|
||||
if self.indexer is not None and not self.skip_topk:
|
||||
if self.indexer is not None:
|
||||
index_q = self.indexer.wq_b(q_c)[0]
|
||||
index_q = index_q.view(-1, self.indexer.n_head, self.indexer.head_dim)
|
||||
else:
|
||||
@@ -538,10 +478,10 @@ class DeepseekV32Attention(MLAAttention):
|
||||
quantize_mqa=self._fp8_query,
|
||||
)
|
||||
|
||||
if self.indexer is not None and not self.skip_topk:
|
||||
torch.ops.vllm.sparse_attn_indexer(
|
||||
if self.indexer is not None:
|
||||
sparse_attn_indexer(
|
||||
q_c,
|
||||
_encode_layer_name(self.indexer.k_cache.prefix),
|
||||
self.indexer.k_cache.prefix,
|
||||
self.indexer.k_cache.kv_cache,
|
||||
index_q_fp8,
|
||||
None, # q_scale folded into weights on the fp8 path
|
||||
|
||||
@@ -2,80 +2,13 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.import_utils import has_cutedsl
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
|
||||
# Cache of tiny 1-element dummy tensors (per device, dtype) reused by the
|
||||
# has_indexer=False path so the indexer args don't allocate every call.
|
||||
_DUMMY_CACHE: dict[tuple, torch.Tensor] = {}
|
||||
|
||||
|
||||
@torch.compiler.assume_constant_result
|
||||
def _can_use_fused_q_cutedsl() -> bool:
|
||||
return current_platform.has_device_capability(100) and has_cutedsl()
|
||||
|
||||
|
||||
@torch.compiler.assume_constant_result
|
||||
def _is_arch_support_pdl() -> bool:
|
||||
return (
|
||||
current_platform.has_device_capability(100)
|
||||
and current_platform.is_arch_support_pdl()
|
||||
)
|
||||
|
||||
|
||||
def _fused_q_cutedsl_impl(
|
||||
positions: torch.Tensor,
|
||||
q_pe: torch.Tensor,
|
||||
rope_cache: torch.Tensor,
|
||||
ql_nope: torch.Tensor,
|
||||
q_scale: torch.Tensor,
|
||||
mqa_output: torch.Tensor,
|
||||
idx_q: torch.Tensor,
|
||||
idx_rope_cache: torch.Tensor,
|
||||
idx_weights: torch.Tensor,
|
||||
idx_weights_softmax_scale: float,
|
||||
idx_weights_head_scale: float,
|
||||
idx_q_fp8: torch.Tensor,
|
||||
idx_weights_out: torch.Tensor,
|
||||
has_indexer: bool,
|
||||
index_rope_interleave: bool,
|
||||
) -> None:
|
||||
from .ops.fused_q_cutedsl import fused_q_cutedsl
|
||||
|
||||
fused_q_cutedsl(
|
||||
positions,
|
||||
q_pe,
|
||||
rope_cache,
|
||||
ql_nope,
|
||||
q_scale,
|
||||
mqa_output,
|
||||
idx_q,
|
||||
idx_rope_cache,
|
||||
idx_weights,
|
||||
idx_weights_softmax_scale,
|
||||
idx_weights_head_scale,
|
||||
idx_q_fp8,
|
||||
idx_weights_out,
|
||||
has_indexer=has_indexer,
|
||||
index_rope_interleave=index_rope_interleave,
|
||||
)
|
||||
|
||||
|
||||
def _fused_q_cutedsl_fake(*args, **kwargs) -> None:
|
||||
pass
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="fused_q_cutedsl",
|
||||
op_func=_fused_q_cutedsl_impl,
|
||||
mutates_args=["mqa_output", "idx_q_fp8", "idx_weights_out"],
|
||||
fake_impl=_fused_q_cutedsl_fake,
|
||||
dispatch_key="CUDA",
|
||||
)
|
||||
|
||||
|
||||
def _dummy(shape: tuple, dtype: torch.dtype, device: torch.device) -> torch.Tensor:
|
||||
key = (shape, dtype, device)
|
||||
t = _DUMMY_CACHE.get(key)
|
||||
@@ -214,14 +147,9 @@ def _fused_norm_rope_kernel(
|
||||
TOPK_BLOCK_SIZE: tl.constexpr,
|
||||
HAS_INDEXER: tl.constexpr,
|
||||
INDEX_ROPE_INTERLEAVE: tl.constexpr,
|
||||
USE_PDL: tl.constexpr,
|
||||
):
|
||||
pid = tl.program_id(0)
|
||||
tok_idx = tl.program_id(1)
|
||||
if USE_PDL:
|
||||
tl.extra.cuda.gdc_wait()
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
|
||||
if pid == 3:
|
||||
if not HAS_INDEXER:
|
||||
# Shared layer: reuse the previous indexer layer's top-k; do not
|
||||
@@ -238,17 +166,6 @@ def _fused_norm_rope_kernel(
|
||||
)
|
||||
return
|
||||
|
||||
if pid == 2:
|
||||
# Q RMS norm does not depend on cache availability. In particular,
|
||||
# profiling uses a negative dummy slot mapping but still consumes Q.
|
||||
q_block = tl.arange(0, Q_BLOCK_SIZE)
|
||||
q_mask = q_block < Q_DIM
|
||||
q_c = tl.load(q_c_ptr + tok_idx * q_c_stride + q_block, mask=q_mask, other=0.0)
|
||||
q_c_rms_w = tl.load(q_rms_norm_w_ptr + q_block, mask=q_mask)
|
||||
q_c = _rms_norm(q_c, q_c_rms_w, q_rms_eps, Q_DIM)
|
||||
tl.store(q_c_out_ptr + tok_idx * q_c_out_stride + q_block, q_c, mask=q_mask)
|
||||
return
|
||||
|
||||
if slot_mapping_ptr is None:
|
||||
# Memory profiling run.
|
||||
return
|
||||
@@ -257,7 +174,15 @@ def _fused_norm_rope_kernel(
|
||||
# Padding
|
||||
return
|
||||
|
||||
if pid == 1:
|
||||
if pid == 2:
|
||||
# Q RMS norm
|
||||
q_block = tl.arange(0, Q_BLOCK_SIZE)
|
||||
q_mask = q_block < Q_DIM
|
||||
q_c = tl.load(q_c_ptr + tok_idx * q_c_stride + q_block, mask=q_mask, other=0.0)
|
||||
q_c_rms_w = tl.load(q_rms_norm_w_ptr + q_block, mask=q_mask)
|
||||
q_c = _rms_norm(q_c, q_c_rms_w, q_rms_eps, Q_DIM)
|
||||
tl.store(q_c_out_ptr + tok_idx * q_c_out_stride + q_block, q_c, mask=q_mask)
|
||||
elif pid == 1:
|
||||
# KV RMS Norm + KV RoPE + MLA concat_and_cache.
|
||||
# Merged so the normed kv_c and RoPE'd k_pe can be written
|
||||
# to the MLA KV cache directly without a separate kernel.
|
||||
@@ -286,6 +211,9 @@ def _fused_norm_rope_kernel(
|
||||
r2 = x2 * cos + x1 * sin
|
||||
|
||||
# MLA concat_and_cache: write [kv_c_normed, k_pe_roped] to cache.
|
||||
if mla_cache_entry_stride == 0:
|
||||
return
|
||||
|
||||
mla_block_size = mla_cache_block_stride // mla_cache_entry_stride
|
||||
mla_block_idx = slot_idx // mla_block_size
|
||||
mla_block_off = slot_idx % mla_block_size
|
||||
@@ -435,7 +363,7 @@ def _fused_norm_rope_kernel(
|
||||
)
|
||||
|
||||
|
||||
def _fused_norm_rope_impl(
|
||||
def fused_norm_rope(
|
||||
positions: torch.Tensor,
|
||||
q_c: torch.Tensor,
|
||||
q_rms_norm_w: torch.Tensor,
|
||||
@@ -537,17 +465,12 @@ def _fused_norm_rope_impl(
|
||||
# Dummy values — pid 2 will skip the MLA cache write because
|
||||
# slot_mapping is all -1.
|
||||
mla_kv_cache = torch.empty(0, dtype=torch.bfloat16, device=device)
|
||||
# Torch's Triton HOP mutation analysis compiles every program-id
|
||||
# branch even though the dummy slot mapping returns before cache
|
||||
# access. Keep the unused strides nonzero so that analysis can lower
|
||||
# the block-size calculation safely.
|
||||
mla_block_stride = 1
|
||||
mla_entry_stride = 1
|
||||
mla_block_stride = 0
|
||||
mla_entry_stride = 0
|
||||
mla_k_scale = torch.ones(1, dtype=torch.float32, device=device)
|
||||
|
||||
if q_c_out is None:
|
||||
q_c_out = torch.empty_like(q_c)
|
||||
use_pdl = _is_arch_support_pdl()
|
||||
_fused_norm_rope_kernel[(4, num_tokens)](
|
||||
positions,
|
||||
# Q RMS norm
|
||||
@@ -606,157 +529,6 @@ def _fused_norm_rope_impl(
|
||||
TOPK_BLOCK_SIZE=1024,
|
||||
HAS_INDEXER=has_indexer,
|
||||
INDEX_ROPE_INTERLEAVE=index_rope_interleave,
|
||||
USE_PDL=use_pdl,
|
||||
launch_pdl=use_pdl,
|
||||
)
|
||||
return q_c_out
|
||||
|
||||
|
||||
def _fused_norm_rope_op(
|
||||
positions: torch.Tensor,
|
||||
q_c: torch.Tensor,
|
||||
q_rms_norm_w: torch.Tensor,
|
||||
q_rms_eps: float,
|
||||
kv_c: torch.Tensor,
|
||||
kv_rms_norm_w: torch.Tensor,
|
||||
kv_rms_eps: float,
|
||||
k_pe: torch.Tensor,
|
||||
k_rope_cos_sin_cache: torch.Tensor,
|
||||
index_k: torch.Tensor | None,
|
||||
index_k_layer_norm_w: torch.Tensor | None,
|
||||
index_k_layer_norm_bias: torch.Tensor | None,
|
||||
index_k_layer_norm_eps: float,
|
||||
index_k_rope_cos_sin_cache: torch.Tensor | None,
|
||||
topk_indices_buffer: torch.Tensor,
|
||||
slot_mapping: torch.Tensor | None,
|
||||
indexer_k_cache: torch.Tensor | None,
|
||||
mla_kv_cache: torch.Tensor | None,
|
||||
mla_kv_cache_dtype: str,
|
||||
mla_k_scale: torch.Tensor | None,
|
||||
has_indexer: bool,
|
||||
index_rope_interleave: bool,
|
||||
q_c_out: torch.Tensor,
|
||||
) -> None:
|
||||
_fused_norm_rope_impl(
|
||||
positions,
|
||||
q_c,
|
||||
q_rms_norm_w,
|
||||
q_rms_eps,
|
||||
kv_c,
|
||||
kv_rms_norm_w,
|
||||
kv_rms_eps,
|
||||
k_pe,
|
||||
k_rope_cos_sin_cache,
|
||||
index_k,
|
||||
index_k_layer_norm_w,
|
||||
index_k_layer_norm_bias,
|
||||
index_k_layer_norm_eps,
|
||||
index_k_rope_cos_sin_cache,
|
||||
topk_indices_buffer,
|
||||
slot_mapping=slot_mapping,
|
||||
indexer_k_cache=indexer_k_cache,
|
||||
mla_kv_cache=mla_kv_cache,
|
||||
mla_kv_cache_dtype=mla_kv_cache_dtype,
|
||||
mla_k_scale=mla_k_scale,
|
||||
has_indexer=has_indexer,
|
||||
index_rope_interleave=index_rope_interleave,
|
||||
q_c_out=q_c_out,
|
||||
)
|
||||
|
||||
|
||||
def _fused_norm_rope_fake(
|
||||
positions: torch.Tensor,
|
||||
q_c: torch.Tensor,
|
||||
q_rms_norm_w: torch.Tensor,
|
||||
q_rms_eps: float,
|
||||
kv_c: torch.Tensor,
|
||||
kv_rms_norm_w: torch.Tensor,
|
||||
kv_rms_eps: float,
|
||||
k_pe: torch.Tensor,
|
||||
k_rope_cos_sin_cache: torch.Tensor,
|
||||
index_k: torch.Tensor | None,
|
||||
index_k_layer_norm_w: torch.Tensor | None,
|
||||
index_k_layer_norm_bias: torch.Tensor | None,
|
||||
index_k_layer_norm_eps: float,
|
||||
index_k_rope_cos_sin_cache: torch.Tensor | None,
|
||||
topk_indices_buffer: torch.Tensor,
|
||||
slot_mapping: torch.Tensor | None,
|
||||
indexer_k_cache: torch.Tensor | None,
|
||||
mla_kv_cache: torch.Tensor | None,
|
||||
mla_kv_cache_dtype: str,
|
||||
mla_k_scale: torch.Tensor | None,
|
||||
has_indexer: bool,
|
||||
index_rope_interleave: bool,
|
||||
q_c_out: torch.Tensor,
|
||||
) -> None:
|
||||
return
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="fused_norm_rope_deepseek_v32",
|
||||
op_func=_fused_norm_rope_op,
|
||||
mutates_args=[
|
||||
"topk_indices_buffer",
|
||||
"indexer_k_cache",
|
||||
"mla_kv_cache",
|
||||
"q_c_out",
|
||||
],
|
||||
fake_impl=_fused_norm_rope_fake,
|
||||
dispatch_key="CUDA",
|
||||
)
|
||||
|
||||
|
||||
def fused_norm_rope(
|
||||
positions: torch.Tensor,
|
||||
q_c: torch.Tensor,
|
||||
q_rms_norm_w: torch.Tensor,
|
||||
q_rms_eps: float,
|
||||
kv_c: torch.Tensor,
|
||||
kv_rms_norm_w: torch.Tensor,
|
||||
kv_rms_eps: float,
|
||||
k_pe: torch.Tensor,
|
||||
k_rope_cos_sin_cache: torch.Tensor,
|
||||
index_k: torch.Tensor | None,
|
||||
index_k_layer_norm_w: torch.Tensor | None,
|
||||
index_k_layer_norm_bias: torch.Tensor | None,
|
||||
index_k_layer_norm_eps: float,
|
||||
index_k_rope_cos_sin_cache: torch.Tensor | None,
|
||||
topk_indices_buffer: torch.Tensor,
|
||||
slot_mapping: torch.Tensor | None = None,
|
||||
indexer_k_cache: torch.Tensor | None = None,
|
||||
mla_kv_cache: torch.Tensor | None = None,
|
||||
mla_kv_cache_dtype: str = "auto",
|
||||
mla_k_scale: torch.Tensor | None = None,
|
||||
has_indexer: bool = True,
|
||||
index_rope_interleave: bool = False,
|
||||
q_c_out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
if q_c_out is None:
|
||||
q_c_out = torch.empty_like(q_c)
|
||||
torch.ops.vllm.fused_norm_rope_deepseek_v32(
|
||||
positions,
|
||||
q_c,
|
||||
q_rms_norm_w,
|
||||
q_rms_eps,
|
||||
kv_c,
|
||||
kv_rms_norm_w,
|
||||
kv_rms_eps,
|
||||
k_pe,
|
||||
k_rope_cos_sin_cache,
|
||||
index_k,
|
||||
index_k_layer_norm_w,
|
||||
index_k_layer_norm_bias,
|
||||
index_k_layer_norm_eps,
|
||||
index_k_rope_cos_sin_cache,
|
||||
topk_indices_buffer,
|
||||
slot_mapping,
|
||||
indexer_k_cache,
|
||||
mla_kv_cache,
|
||||
mla_kv_cache_dtype,
|
||||
mla_k_scale,
|
||||
has_indexer,
|
||||
index_rope_interleave,
|
||||
q_c_out,
|
||||
)
|
||||
return q_c_out
|
||||
|
||||
@@ -810,14 +582,10 @@ def _fused_q_kernel(
|
||||
HAS_INDEXER: tl.constexpr,
|
||||
INDEX_ROPE_INTERLEAVE: tl.constexpr,
|
||||
QUANTIZE_MQA: tl.constexpr,
|
||||
USE_PDL: tl.constexpr,
|
||||
):
|
||||
pid = tl.program_id(0)
|
||||
tok_idx = tl.program_id(1)
|
||||
head_idx = tl.program_id(2)
|
||||
if USE_PDL:
|
||||
tl.extra.cuda.gdc_wait()
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
|
||||
if pid == 2:
|
||||
# ql_nope quantize + pack into the front of mqa_q_fp8. On the bf16
|
||||
@@ -1005,12 +773,10 @@ def fused_q(
|
||||
``(ql_nope, q_pe)`` tuple the backend expects.
|
||||
"""
|
||||
assert positions.ndim == 1
|
||||
assert positions.dtype == torch.int64
|
||||
assert q_pe.ndim == 3
|
||||
assert q_pe_cos_sin_cache.ndim == 2
|
||||
assert ql_nope.ndim == 3
|
||||
assert ql_nope.shape[:2] == q_pe.shape[:2]
|
||||
assert q_scale.dtype == torch.float32 and q_scale.numel() == 1
|
||||
|
||||
num_tokens = positions.shape[0]
|
||||
num_q_heads = q_pe.shape[1]
|
||||
@@ -1028,18 +794,7 @@ def fused_q(
|
||||
assert index_weights is not None
|
||||
num_index_q_heads = index_q.shape[1]
|
||||
index_q_head_dim = index_q.shape[2]
|
||||
use_cutedsl = False
|
||||
if _can_use_fused_q_cutedsl():
|
||||
from .ops.fused_q_cutedsl import is_fused_q_cutedsl_supported
|
||||
|
||||
use_cutedsl = is_fused_q_cutedsl_supported(
|
||||
q_pe,
|
||||
index_q,
|
||||
ql_nope,
|
||||
has_indexer=has_indexer,
|
||||
quantize_mqa=quantize_mqa,
|
||||
)
|
||||
grid_heads = max(mqa_grid_heads, num_index_q_heads if has_indexer else 1)
|
||||
grid_heads = max(mqa_grid_heads, num_index_q_heads)
|
||||
if quantize_mqa:
|
||||
# fp8 path: pack [ql_nope; q_pe] into a single fp8 tensor.
|
||||
mqa_q_fp8 = torch.empty(
|
||||
@@ -1060,27 +815,6 @@ def fused_q(
|
||||
|
||||
index_q_fp8 = torch.empty_like(index_q, dtype=torch.float8_e4m3fn)
|
||||
index_weights_out = torch.empty_like(index_weights, dtype=torch.float32)
|
||||
if use_cutedsl:
|
||||
torch.ops.vllm.fused_q_cutedsl(
|
||||
positions,
|
||||
q_pe,
|
||||
q_pe_cos_sin_cache,
|
||||
ql_nope,
|
||||
q_scale,
|
||||
mqa_q,
|
||||
index_q,
|
||||
index_q_cos_sin_cache,
|
||||
index_weights,
|
||||
index_weights_softmax_scale,
|
||||
index_weights_head_scale,
|
||||
index_q_fp8,
|
||||
index_weights_out,
|
||||
has_indexer=has_indexer,
|
||||
index_rope_interleave=index_rope_interleave,
|
||||
)
|
||||
return index_q_fp8, index_weights_out, mqa_q
|
||||
|
||||
use_pdl = _is_arch_support_pdl()
|
||||
_fused_q_kernel[(3, num_tokens, grid_heads)](
|
||||
positions,
|
||||
q_pe,
|
||||
@@ -1122,8 +856,6 @@ def fused_q(
|
||||
HAS_INDEXER=has_indexer,
|
||||
INDEX_ROPE_INTERLEAVE=index_rope_interleave,
|
||||
QUANTIZE_MQA=quantize_mqa,
|
||||
USE_PDL=use_pdl,
|
||||
launch_pdl=use_pdl,
|
||||
# num_warps=1 is optimal here: each program is a single 128-element
|
||||
# rope+quant, so the kernel is program-count/occupancy bound, not
|
||||
# per-program compute bound (swept 1/2/4/8 — 1 wins or ties everywhere).
|
||||
|
||||
@@ -6,7 +6,6 @@ from itertools import islice
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed import (
|
||||
get_pp_group,
|
||||
@@ -40,7 +39,6 @@ from vllm.model_executor.models.utils import (
|
||||
sequence_parallel_chunk,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.v1.attention.backends.mla.sparse_utils import register_phys_shadow
|
||||
|
||||
from .attention import DeepseekV32Attention
|
||||
from .fused_ops import fused_allreduce_rms_norm
|
||||
@@ -179,7 +177,6 @@ class DeepseekV32DecoderLayer(torch.nn.Module):
|
||||
return hidden_states, residual
|
||||
|
||||
|
||||
@support_torch_compile
|
||||
class DeepseekV32Model(torch.nn.Module):
|
||||
fall_back_to_pt_during_load = False
|
||||
|
||||
@@ -203,7 +200,6 @@ class DeepseekV32Model(torch.nn.Module):
|
||||
dtype=torch.int32,
|
||||
device=self.device,
|
||||
)
|
||||
register_phys_shadow(topk_indices_buffer)
|
||||
|
||||
if get_pp_group().is_first_rank:
|
||||
self.embed_tokens = VocabParallelEmbedding(
|
||||
|
||||
@@ -6,9 +6,9 @@ from collections.abc import Callable, Iterable
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
import vllm._custom_ops as ops
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed import tensor_model_parallel_all_reduce
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
@@ -40,12 +40,7 @@ from vllm.model_executor.models.utils import (
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.v1.attention.backends.mla.sparse_utils import (
|
||||
phys_shadow,
|
||||
register_phys_shadow,
|
||||
)
|
||||
|
||||
from .fused_ops import fused_allreduce_rms_norm
|
||||
from .kernels import fused_eh_norm
|
||||
from .model import DeepseekV32DecoderLayer
|
||||
|
||||
@@ -61,19 +56,6 @@ class DeepseekV32MultiTokenPredictorLayer(nn.Module):
|
||||
self.enorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.hnorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.eh_proj = nn.Linear(config.hidden_size * 2, config.hidden_size, bias=False)
|
||||
# bf16 skinny GEMM for the eh_proj, B300+B200 measured: GLM-5.2
|
||||
# (6144, 12288) wins at M <= 3, DSv3.2 (7168, 14336) at M <= 2;
|
||||
# cuBLAS holds above (the 151/205MB weights stream at ~5.7TB/s).
|
||||
eh_dispatch = {(6144, 12288): 3, (7168, 14336): 2}
|
||||
eh_weight = getattr(self.eh_proj, "weight", None)
|
||||
self._eh_skinny_max = (
|
||||
eh_dispatch.get(tuple(eh_weight.shape), 0)
|
||||
if current_platform.is_device_capability_family(100)
|
||||
and hasattr(torch.ops._C, "bf16_skinny_gemm")
|
||||
and eh_weight is not None
|
||||
and eh_weight.dtype == torch.bfloat16
|
||||
else 0
|
||||
)
|
||||
|
||||
topk_indices_buffer = torch.empty(
|
||||
vllm_config.scheduler_config.max_num_batched_tokens,
|
||||
@@ -81,7 +63,6 @@ class DeepseekV32MultiTokenPredictorLayer(nn.Module):
|
||||
dtype=torch.int32,
|
||||
device=current_platform.device_type,
|
||||
)
|
||||
register_phys_shadow(topk_indices_buffer)
|
||||
self.shared_head = SharedHead(
|
||||
config=config, prefix=prefix, quant_config=quant_config
|
||||
)
|
||||
@@ -110,13 +91,7 @@ class DeepseekV32MultiTokenPredictorLayer(nn.Module):
|
||||
self.hnorm.weight,
|
||||
self.enorm.variance_epsilon,
|
||||
)
|
||||
if (
|
||||
eh_input.shape[0] <= self._eh_skinny_max
|
||||
and eh_input.dtype == torch.bfloat16
|
||||
):
|
||||
hidden_states = ops.bf16_skinny_gemm(eh_input, self.eh_proj.weight)
|
||||
else:
|
||||
hidden_states = self.eh_proj(eh_input)
|
||||
hidden_states = self.eh_proj(eh_input)
|
||||
hidden_states, residual = self.mtp_block(
|
||||
positions=positions, hidden_states=hidden_states, residual=None
|
||||
)
|
||||
@@ -126,6 +101,9 @@ class DeepseekV32MultiTokenPredictorLayer(nn.Module):
|
||||
positions.shape[0],
|
||||
is_sequence_parallel=self.mtp_block.use_sequence_parallel_moe,
|
||||
)
|
||||
if not self.mtp_block.use_sequence_parallel_moe:
|
||||
# Without sequence parallelism, the MoE output is left un-reduced.
|
||||
hidden_states = tensor_model_parallel_all_reduce(hidden_states)
|
||||
# Recycle the POST-final-norm hidden into the next draft step. The
|
||||
# residual-add is fused into the final RMSNorm so it is computed
|
||||
# exactly once, and the result is returned for both tuple positions:
|
||||
@@ -136,14 +114,7 @@ class DeepseekV32MultiTokenPredictorLayer(nn.Module):
|
||||
# is understood by both the V2 speculator (isinstance-tuple check) and
|
||||
# the legacy proposer (model_returns_tuple is True for the
|
||||
# DeepSeekMTPModel architecture).
|
||||
if self.mtp_block.use_sequence_parallel_moe:
|
||||
hidden_states, _ = self.shared_head.norm(hidden_states, residual)
|
||||
else:
|
||||
# The MoE output is left un-reduced; fuse its all-reduce into the
|
||||
# final norm, as the main model does at layer boundaries.
|
||||
hidden_states, _ = fused_allreduce_rms_norm(
|
||||
hidden_states, residual, self.shared_head.norm
|
||||
)
|
||||
hidden_states, _ = self.shared_head.norm(hidden_states, residual)
|
||||
return hidden_states, hidden_states
|
||||
|
||||
|
||||
@@ -186,15 +157,6 @@ class DeepseekV32MultiTokenPredictor(nn.Module):
|
||||
if self_attn is not None and hasattr(self_attn, "topk_indices_buffer"):
|
||||
topk_indices_buffer = self_attn.topk_indices_buffer
|
||||
topk_indices_buffer[:num_slots] = topk_indices_buffer[slot_ids]
|
||||
# The physical-index shadow mirrors this buffer row-wise
|
||||
# (shadow[j] == convert(logical[j])); re-arrange it the same
|
||||
# way or skip_topk draft steps read the step-0 multi-token
|
||||
# layout's rows for the wrong tokens/requests.
|
||||
shadow = phys_shadow(topk_indices_buffer)
|
||||
if shadow is not None:
|
||||
phys, seq = shadow
|
||||
phys[:num_slots] = phys[slot_ids]
|
||||
seq[:num_slots] = seq[slot_ids]
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.embed_tokens(input_ids)
|
||||
@@ -230,19 +192,6 @@ class DeepseekV32MultiTokenPredictor(nn.Module):
|
||||
# second RMSNorm.
|
||||
return self.logits_processor(mtp_layer.shared_head.head, hidden_states)
|
||||
|
||||
def get_top_tokens(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
spec_step_idx: int = 0,
|
||||
) -> torch.Tensor:
|
||||
# Greedy draft token ids via vocab-parallel local argmax: skips the
|
||||
# full-vocab logits all-gather (use_local_argmax_reduction).
|
||||
current_step_idx = spec_step_idx % self.num_mtp_layers
|
||||
mtp_layer = self.layers[str(self.mtp_start_layer_idx + current_step_idx)]
|
||||
return self.logits_processor.get_top_tokens(
|
||||
mtp_layer.shared_head.head, hidden_states
|
||||
)
|
||||
|
||||
|
||||
class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
@@ -291,25 +240,6 @@ class DeepseekV32MTP(nn.Module, DeepseekV2MixtureOfExperts):
|
||||
) -> torch.Tensor | None:
|
||||
return self.model.compute_logits(hidden_states, spec_step_idx)
|
||||
|
||||
def get_top_tokens(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
spec_step_idx: int | None = None,
|
||||
) -> torch.Tensor:
|
||||
# The V2 speculator calls this without spec_step_idx; that is only
|
||||
# correct while a single MTP layer is cycled for every draft step
|
||||
# (num_nextn_predict_layers == 1, as in GLM-5.2/DSV3.2). Guard the
|
||||
# assumption so a future multi-layer MTP fails loudly instead of
|
||||
# silently using step 0's head for every step.
|
||||
if spec_step_idx is None:
|
||||
assert self.model.num_mtp_layers == 1, (
|
||||
"get_top_tokens called without spec_step_idx on a "
|
||||
"multi-layer MTP; thread the draft step index through "
|
||||
"the speculator."
|
||||
)
|
||||
spec_step_idx = 0
|
||||
return self.model.get_top_tokens(hidden_states, spec_step_idx)
|
||||
|
||||
def _rewrite_spec_layer_name(self, spec_layer: int, name: str) -> str:
|
||||
spec_layer_weight_names = [
|
||||
"embed_tokens",
|
||||
|
||||
@@ -1,2 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -1,532 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from functools import cache
|
||||
|
||||
import cutlass
|
||||
import cutlass.cute as cute
|
||||
import torch
|
||||
from cuda.bindings.driver import CUstream
|
||||
from cutlass import BFloat16, Float8E4M3FN, Float32, Int64, Uint8, Uint16, Uint32
|
||||
|
||||
from vllm.cute_utils import TORCH_TO_CUTE_DTYPE, cvt
|
||||
|
||||
|
||||
def _make_fake_tensor(dtype, shape, divisibility):
|
||||
stride = tuple(
|
||||
1 if i == len(shape) - 1 else cute.sym_int64(divisibility=divisibility)
|
||||
for i in range(len(shape))
|
||||
)
|
||||
return cute.runtime.make_fake_tensor(
|
||||
dtype,
|
||||
shape,
|
||||
stride,
|
||||
assumed_align=divisibility * dtype.width // 8,
|
||||
)
|
||||
|
||||
|
||||
def is_fused_q_cutedsl_supported(
|
||||
q_pe: torch.Tensor,
|
||||
index_q: torch.Tensor | None,
|
||||
ql_nope: torch.Tensor,
|
||||
*,
|
||||
has_indexer: bool,
|
||||
quantize_mqa: bool,
|
||||
) -> bool:
|
||||
if not (
|
||||
quantize_mqa
|
||||
and q_pe.dtype == ql_nope.dtype == torch.bfloat16
|
||||
and q_pe.shape[-1] == 64
|
||||
and ql_nope.shape[-1] == 512
|
||||
):
|
||||
return False
|
||||
return not has_indexer or (
|
||||
index_q is not None
|
||||
and index_q.dtype == torch.bfloat16
|
||||
and index_q.shape[1] % 16 == 0
|
||||
and index_q.shape[-1] == 128
|
||||
)
|
||||
|
||||
|
||||
def fused_q_cutedsl(
|
||||
positions: torch.Tensor,
|
||||
q_pe: torch.Tensor,
|
||||
rope_cache: torch.Tensor,
|
||||
ql_nope: torch.Tensor,
|
||||
q_scale: torch.Tensor,
|
||||
mqa_output: torch.Tensor,
|
||||
idx_q: torch.Tensor,
|
||||
idx_rope_cache: torch.Tensor,
|
||||
idx_weights: torch.Tensor,
|
||||
idx_weights_softmax_scale: float,
|
||||
idx_weights_head_scale: float,
|
||||
idx_q_fp8: torch.Tensor,
|
||||
idx_weights_out: torch.Tensor,
|
||||
has_indexer: bool = True,
|
||||
index_rope_interleave: bool = True,
|
||||
) -> None:
|
||||
_, num_heads, rope_dim = q_pe.shape
|
||||
_, _, nope_dim = ql_nope.shape
|
||||
_, num_idx_heads, idx_dim = idx_q.shape
|
||||
|
||||
if has_indexer:
|
||||
idx_rope_type = TORCH_TO_CUTE_DTYPE[idx_rope_cache.dtype]
|
||||
idx_weights_type = TORCH_TO_CUTE_DTYPE[idx_weights.dtype]
|
||||
else:
|
||||
idx_dim = num_idx_heads = 0
|
||||
idx_q = idx_rope_cache = idx_q_fp8 = None
|
||||
idx_weights = idx_weights_out = None
|
||||
idx_rope_type = idx_weights_type = None
|
||||
|
||||
rope_type = TORCH_TO_CUTE_DTYPE[rope_cache.dtype]
|
||||
compiled = FusedQKernel.compile(
|
||||
rope_dim,
|
||||
nope_dim,
|
||||
num_heads,
|
||||
rope_type,
|
||||
idx_dim,
|
||||
num_idx_heads,
|
||||
idx_rope_type,
|
||||
idx_weights_type,
|
||||
index_rope_interleave,
|
||||
)
|
||||
compiled(
|
||||
positions,
|
||||
q_pe,
|
||||
rope_cache,
|
||||
ql_nope,
|
||||
q_scale.view(1),
|
||||
mqa_output,
|
||||
idx_q,
|
||||
idx_rope_cache,
|
||||
idx_weights,
|
||||
idx_q_fp8,
|
||||
idx_weights_out,
|
||||
float(idx_weights_softmax_scale * idx_weights_head_scale),
|
||||
)
|
||||
|
||||
|
||||
class FusedQKernel:
|
||||
def __init__(
|
||||
self,
|
||||
rope_dim: int,
|
||||
nope_dim: int,
|
||||
num_heads: int,
|
||||
idx_dim: int,
|
||||
num_idx_heads: int,
|
||||
index_rope_interleave: bool,
|
||||
) -> None:
|
||||
assert rope_dim == 64
|
||||
assert nope_dim == 512
|
||||
assert idx_dim in (128, 0)
|
||||
|
||||
self.rope_dim = rope_dim
|
||||
self.nope_dim = nope_dim
|
||||
self.num_heads = num_heads
|
||||
self.idx_dim = idx_dim
|
||||
self.num_idx_heads = num_idx_heads
|
||||
self.index_rope_interleave = index_rope_interleave
|
||||
|
||||
# mqa: rope_dim=64, nope_dim=512, num_heads=64/TP
|
||||
# indexer: rope_dim=64, nope_dim=64, num_heads=32
|
||||
|
||||
self.num_warps = 4
|
||||
assert num_heads % self.num_warps == 0
|
||||
assert num_idx_heads % (4 * self.num_warps) == 0
|
||||
self.num_ctas_per_tok = num_heads // self.num_warps
|
||||
self.num_ctas_per_idx_tok = num_idx_heads // (4 * self.num_warps)
|
||||
|
||||
@cute.jit
|
||||
def __call__(
|
||||
self,
|
||||
positions: cute.Tensor,
|
||||
q_pe: cute.Tensor,
|
||||
rope_cache: cute.Tensor,
|
||||
ql_nope: cute.Tensor,
|
||||
q_scale: cute.Tensor,
|
||||
mqa_output: cute.Tensor,
|
||||
idx_q: cute.Tensor,
|
||||
idx_rope_cache: cute.Tensor,
|
||||
idx_weights: cute.Tensor,
|
||||
idx_q_fp8: cute.Tensor,
|
||||
idx_weights_out: cute.Tensor,
|
||||
weight_scale: Float32,
|
||||
stream: CUstream,
|
||||
):
|
||||
num_tokens = positions.shape[0]
|
||||
if cutlass.const_expr(self.idx_dim == 0):
|
||||
grid = (num_tokens, self.num_ctas_per_tok, 1)
|
||||
else:
|
||||
num_mqa_ctas = num_tokens * self.num_ctas_per_tok
|
||||
num_idx_ctas = num_tokens * self.num_ctas_per_idx_tok
|
||||
grid = (num_mqa_ctas + num_idx_ctas, 1, 1)
|
||||
|
||||
self.kernel(
|
||||
positions,
|
||||
q_pe,
|
||||
rope_cache,
|
||||
ql_nope,
|
||||
q_scale,
|
||||
mqa_output,
|
||||
idx_q,
|
||||
idx_rope_cache,
|
||||
idx_weights,
|
||||
idx_q_fp8,
|
||||
idx_weights_out,
|
||||
weight_scale,
|
||||
).launch(
|
||||
grid=grid,
|
||||
block=(self.num_warps * 32, 1, 1),
|
||||
stream=stream,
|
||||
use_pdl=True,
|
||||
)
|
||||
|
||||
@cute.kernel
|
||||
def kernel(
|
||||
self,
|
||||
positions: cute.Tensor,
|
||||
q_pe: cute.Tensor,
|
||||
q_pe_rope_cache: cute.Tensor,
|
||||
ql_nope: cute.Tensor,
|
||||
q_scale: cute.Tensor,
|
||||
mqa_output: cute.Tensor,
|
||||
idx_q: cute.Tensor,
|
||||
idx_q_rope_cache: cute.Tensor,
|
||||
idx_weights: cute.Tensor,
|
||||
idx_q_fp8: cute.Tensor,
|
||||
idx_weights_out: cute.Tensor,
|
||||
weight_scale: Float32,
|
||||
):
|
||||
if cutlass.const_expr(self.idx_dim == 0):
|
||||
token_id, group_id, _ = cute.arch.block_idx()
|
||||
self.mqa(
|
||||
positions,
|
||||
q_pe,
|
||||
q_pe_rope_cache,
|
||||
ql_nope,
|
||||
q_scale,
|
||||
mqa_output,
|
||||
token_id,
|
||||
group_id,
|
||||
)
|
||||
else:
|
||||
# CTA-specialization
|
||||
bid, _, _ = cute.arch.block_idx()
|
||||
num_mqa_ctas = positions.shape[0] * self.num_ctas_per_tok
|
||||
if bid < num_mqa_ctas:
|
||||
self.mqa(
|
||||
positions,
|
||||
q_pe,
|
||||
q_pe_rope_cache,
|
||||
ql_nope,
|
||||
q_scale,
|
||||
mqa_output,
|
||||
bid // self.num_ctas_per_tok,
|
||||
bid % self.num_ctas_per_tok,
|
||||
)
|
||||
else:
|
||||
bid -= num_mqa_ctas
|
||||
self.indexer(
|
||||
positions,
|
||||
idx_q,
|
||||
idx_q_rope_cache,
|
||||
idx_weights,
|
||||
idx_q_fp8,
|
||||
idx_weights_out,
|
||||
weight_scale,
|
||||
bid // self.num_ctas_per_idx_tok,
|
||||
bid % self.num_ctas_per_idx_tok,
|
||||
)
|
||||
|
||||
@cute.jit
|
||||
def mqa(
|
||||
self,
|
||||
positions: cute.Tensor,
|
||||
q_pe: cute.Tensor,
|
||||
q_pe_rope_cache: cute.Tensor,
|
||||
ql_nope: cute.Tensor,
|
||||
q_scale: cute.Tensor,
|
||||
mqa_output: cute.Tensor,
|
||||
token_id,
|
||||
group_id,
|
||||
):
|
||||
tid, _, _ = cute.arch.thread_idx()
|
||||
warp_id = cute.arch.make_warp_uniform(tid // 32)
|
||||
lane_id = tid % 32
|
||||
head_id = group_id * self.num_warps + warp_id
|
||||
|
||||
cute.arch.griddepcontrol_wait()
|
||||
|
||||
pos = positions[token_id]
|
||||
inv_scale = 1.0 / q_scale[0]
|
||||
|
||||
cp_op = cute.nvgpu.CopyUniversalOp()
|
||||
cp_32B = cute.make_copy_atom(cp_op, Uint32, num_bits_per_copy=256)
|
||||
cp_16B = cute.make_copy_atom(cp_op, Uint32, num_bits_per_copy=128)
|
||||
cp_4B = cute.make_copy_atom(cp_op, Uint32, num_bits_per_copy=32)
|
||||
cp_2B = cute.make_copy_atom(cp_op, Uint8, num_bits_per_copy=16)
|
||||
|
||||
##### issue all loads asap #####
|
||||
rQ_nope_bf16 = cute.make_rmem_tensor(16, BFloat16)
|
||||
rQ_rope_bf16 = cute.make_rmem_tensor(2, BFloat16)
|
||||
|
||||
src_ql_nope = cute.local_tile(
|
||||
ql_nope[token_id, head_id, None], (16,), (lane_id,)
|
||||
)
|
||||
src_q_rope = cute.local_tile(q_pe[token_id, head_id, None], (2,), (lane_id,))
|
||||
cute.copy(cp_32B, src_ql_nope, rQ_nope_bf16)
|
||||
cute.copy(cp_4B, src_q_rope, rQ_rope_bf16)
|
||||
|
||||
rCos_raw = q_pe_rope_cache[pos, 0 + lane_id]
|
||||
rSin_raw = q_pe_rope_cache[pos, 32 + lane_id]
|
||||
|
||||
##### process NoPE #####
|
||||
rQ_nope_f32 = cvt.bf16x2_to_fp32x2(rQ_nope_bf16).load() * inv_scale
|
||||
rQ_nope_f8 = cute.make_rmem_tensor(16, Float8E4M3FN)
|
||||
rQ_nope_f8.store(rQ_nope_f32.to(Float8E4M3FN))
|
||||
dst_Q_nope = cute.local_tile(
|
||||
mqa_output[token_id, head_id, None], (16,), (lane_id,)
|
||||
)
|
||||
cute.copy(cp_16B, rQ_nope_f8, dst_Q_nope)
|
||||
|
||||
##### process RoPE ######
|
||||
rQ_rope_f32 = cvt.bf16x2_to_fp32x2(rQ_rope_bf16)
|
||||
rCos = rCos_raw.to(Float32)
|
||||
rSin = rSin_raw.to(Float32)
|
||||
r0 = (rQ_rope_f32[0] * rCos - rQ_rope_f32[1] * rSin) * inv_scale
|
||||
r1 = (rQ_rope_f32[1] * rCos + rQ_rope_f32[0] * rSin) * inv_scale
|
||||
|
||||
cute.arch.griddepcontrol_launch_dependents()
|
||||
|
||||
# TensorSSA fp32->fp8 cvt has a bug. rely on direct PTX
|
||||
rQ_rope_f8 = cute.make_rmem_tensor(2, Float8E4M3FN)
|
||||
cute.recast_tensor(rQ_rope_f8, Uint16)[0] = cvt.fp32x2_to_fp8x2(r0, r1)
|
||||
dst_Q_rope = cute.local_tile(
|
||||
mqa_output[token_id, head_id, None], (2,), (256 + lane_id,)
|
||||
)
|
||||
cute.copy(cp_2B, rQ_rope_f8, dst_Q_rope)
|
||||
|
||||
@cute.jit
|
||||
def indexer(
|
||||
self,
|
||||
positions: cute.Tensor,
|
||||
idx_q: cute.Tensor,
|
||||
idx_q_rope_cache: cute.Tensor,
|
||||
idx_weights: cute.Tensor,
|
||||
idx_q_fp8: cute.Tensor,
|
||||
idx_weights_out: cute.Tensor,
|
||||
weight_scale: Float32,
|
||||
token_id,
|
||||
group_id,
|
||||
):
|
||||
tid, _, _ = cute.arch.thread_idx()
|
||||
subwarp_id = tid // 8
|
||||
sublane_id = tid % 8
|
||||
head_id = group_id * (4 * self.num_warps) + subwarp_id
|
||||
cute.arch.griddepcontrol_wait()
|
||||
|
||||
pos = positions[token_id]
|
||||
|
||||
cp_op = cute.nvgpu.CopyUniversalOp()
|
||||
cp_16B = cute.make_copy_atom(cp_op, Uint32, num_bits_per_copy=128)
|
||||
cp_8B = cute.make_copy_atom(cp_op, Uint32, num_bits_per_copy=64)
|
||||
cp_4B = cute.make_copy_atom(cp_op, Uint32, num_bits_per_copy=32)
|
||||
|
||||
##### issue all loads first #####
|
||||
rQ_rope_bf16 = cute.make_rmem_tensor(8, BFloat16)
|
||||
if cutlass.const_expr(self.index_rope_interleave):
|
||||
src_idx_q_rope = cute.local_tile(
|
||||
idx_q[token_id, head_id, None], (8,), (sublane_id,)
|
||||
)
|
||||
cute.copy(cp_16B, src_idx_q_rope, rQ_rope_bf16)
|
||||
else:
|
||||
src_idx_q_rope = cute.zipped_divide(
|
||||
idx_q[token_id, head_id, None], (4,)
|
||||
) # (4,32)
|
||||
cute.copy(
|
||||
cp_8B,
|
||||
src_idx_q_rope[None, 0 + sublane_id],
|
||||
cute.local_tile(rQ_rope_bf16, (4,), (0,)),
|
||||
)
|
||||
cute.copy(
|
||||
cp_8B,
|
||||
src_idx_q_rope[None, 8 + sublane_id],
|
||||
cute.local_tile(rQ_rope_bf16, (4,), (1,)),
|
||||
)
|
||||
|
||||
rQ_nope_bf16 = cute.make_rmem_tensor(8, BFloat16)
|
||||
src_idx_q_nope = cute.local_tile(
|
||||
idx_q[token_id, head_id, None], (8,), (8 + sublane_id,)
|
||||
)
|
||||
cute.copy(cp_16B, src_idx_q_nope, rQ_nope_bf16)
|
||||
|
||||
rCos_raw = cute.make_rmem_tensor(4, idx_q_rope_cache.element_type)
|
||||
rSin_raw = cute.make_rmem_tensor(4, idx_q_rope_cache.element_type)
|
||||
rope_cache_view = cute.zipped_divide(idx_q_rope_cache[pos, None], (4,))
|
||||
|
||||
if cutlass.const_expr(idx_q_rope_cache.element_type == Float32):
|
||||
cute.copy(cp_16B, rope_cache_view[None, sublane_id], rCos_raw)
|
||||
cute.copy(cp_16B, rope_cache_view[None, 8 + sublane_id], rSin_raw)
|
||||
elif cutlass.const_expr(idx_q_rope_cache.element_type == BFloat16):
|
||||
cute.copy(cp_8B, rope_cache_view[None, sublane_id], rCos_raw)
|
||||
cute.copy(cp_8B, rope_cache_view[None, 8 + sublane_id], rSin_raw)
|
||||
|
||||
# unpack to FP32
|
||||
rQ_rope_f32 = cvt.bf16x2_to_fp32x2(rQ_rope_bf16)
|
||||
rQ_nope_f32 = cvt.bf16x2_to_fp32x2(rQ_nope_bf16)
|
||||
if cutlass.const_expr(idx_q_rope_cache.element_type == Float32):
|
||||
rCos = rCos_raw
|
||||
rSin = rSin_raw
|
||||
elif cutlass.const_expr(idx_q_rope_cache.element_type == BFloat16):
|
||||
rCos = cvt.bf16x2_to_fp32x2(rCos_raw)
|
||||
rSin = cvt.bf16x2_to_fp32x2(rSin_raw)
|
||||
|
||||
# apply rope
|
||||
for i in cutlass.range_constexpr(4):
|
||||
if cutlass.const_expr(self.index_rope_interleave):
|
||||
r0 = rQ_rope_f32[i * 2 + 0] * rCos[i] - rQ_rope_f32[i * 2 + 1] * rSin[i]
|
||||
r1 = rQ_rope_f32[i * 2 + 1] * rCos[i] + rQ_rope_f32[i * 2 + 0] * rSin[i]
|
||||
rQ_rope_f32[i * 2 + 0] = r0
|
||||
rQ_rope_f32[i * 2 + 1] = r1
|
||||
else:
|
||||
r0 = rQ_rope_f32[0 + i] * rCos[i] - rQ_rope_f32[4 + i] * rSin[i]
|
||||
r1 = rQ_rope_f32[4 + i] * rCos[i] + rQ_rope_f32[0 + i] * rSin[i]
|
||||
rQ_rope_f32[0 + i] = r0
|
||||
rQ_rope_f32[4 + i] = r1
|
||||
|
||||
# amax
|
||||
amax = Float32(1e-4)
|
||||
for i in cutlass.range_constexpr(8):
|
||||
amax = cute.arch.fmax(amax, cute.math.absf(rQ_rope_f32[i]))
|
||||
amax = cute.arch.fmax(amax, cute.math.absf(rQ_nope_f32[i]))
|
||||
|
||||
# warp reduction among 8 lanes
|
||||
for i in cutlass.range_constexpr(3):
|
||||
other = cute.arch.shuffle_sync_bfly(amax, 1 << i)
|
||||
amax = cute.arch.fmax(amax, other)
|
||||
|
||||
# compute scale from amax
|
||||
# exp2(ceil(log2(scale))) via bit manipulation
|
||||
scale = amax * (1.0 / 448.0)
|
||||
bits = scale.bitcast(Uint32)
|
||||
exp_bits = (bits + Uint32(0x007FFFFF)) & Uint32(0x7F800000)
|
||||
scale = exp_bits.bitcast(Float32)
|
||||
inv_scale = (Uint32(0x7F000000) - exp_bits).bitcast(Float32)
|
||||
|
||||
for i in cutlass.range_constexpr(8):
|
||||
rQ_nope_f32[i] *= inv_scale
|
||||
rQ_rope_f32[i] *= inv_scale
|
||||
|
||||
cute.arch.griddepcontrol_launch_dependents()
|
||||
|
||||
# quantize and store
|
||||
rQ_nope_f8 = cute.make_rmem_tensor(8, Float8E4M3FN)
|
||||
rQ_nope_f8.store(rQ_nope_f32.load().to(Float8E4M3FN))
|
||||
dst_idx_q_nope = cute.local_tile(
|
||||
idx_q_fp8[token_id, head_id, None], (8,), (8 + sublane_id,)
|
||||
)
|
||||
cute.copy(cp_8B, rQ_nope_f8, dst_idx_q_nope)
|
||||
|
||||
rQ_rope_f8 = cute.make_rmem_tensor(8, Float8E4M3FN)
|
||||
rQ_rope_f8.store(rQ_rope_f32.load().to(Float8E4M3FN))
|
||||
if cutlass.const_expr(self.index_rope_interleave):
|
||||
dst_idx_q_rope = cute.local_tile(
|
||||
idx_q_fp8[token_id, head_id, None], (8,), (sublane_id,)
|
||||
)
|
||||
cute.copy(cp_8B, rQ_rope_f8, dst_idx_q_rope)
|
||||
else:
|
||||
dst_idx_q_rope = cute.zipped_divide(
|
||||
idx_q_fp8[token_id, head_id, None], (4,)
|
||||
) # (4,32)
|
||||
cute.copy(
|
||||
cp_4B,
|
||||
cute.local_tile(rQ_rope_f8, (4,), (0,)),
|
||||
dst_idx_q_rope[None, 0 + sublane_id],
|
||||
)
|
||||
cute.copy(
|
||||
cp_4B,
|
||||
cute.local_tile(rQ_rope_f8, (4,), (1,)),
|
||||
dst_idx_q_rope[None, 8 + sublane_id],
|
||||
)
|
||||
|
||||
# scale indexer weights
|
||||
if sublane_id == 0:
|
||||
w = idx_weights[token_id, head_id].to(Float32)
|
||||
idx_weights_out[token_id, head_id] = w * scale * weight_scale
|
||||
|
||||
@cache
|
||||
@staticmethod
|
||||
def compile(
|
||||
rope_dim: int,
|
||||
nope_dim: int,
|
||||
num_heads: int,
|
||||
rope_type: type[cutlass.Numeric],
|
||||
idx_dim: int,
|
||||
num_idx_heads: int,
|
||||
idx_rope_type: type[cutlass.Numeric] | None,
|
||||
idx_weights_type: type[cutlass.Numeric] | None,
|
||||
index_rope_interleave: bool,
|
||||
):
|
||||
num_tokens = cute.sym_int()
|
||||
max_pos = cute.sym_int()
|
||||
|
||||
positions = _make_fake_tensor(Int64, (num_tokens,), divisibility=1)
|
||||
q_pe = _make_fake_tensor(
|
||||
BFloat16, (num_tokens, num_heads, rope_dim), divisibility=16
|
||||
)
|
||||
rope_cache = _make_fake_tensor(rope_type, (max_pos, rope_dim), divisibility=8)
|
||||
ql_nope = _make_fake_tensor(
|
||||
BFloat16, (num_tokens, num_heads, nope_dim), divisibility=16
|
||||
)
|
||||
q_scale = _make_fake_tensor(Float32, (1,), divisibility=4)
|
||||
mqa_output = _make_fake_tensor(
|
||||
Float8E4M3FN,
|
||||
(num_tokens, num_heads, nope_dim + rope_dim),
|
||||
divisibility=16,
|
||||
)
|
||||
|
||||
if idx_rope_type is not None:
|
||||
index_q = _make_fake_tensor(
|
||||
BFloat16, (num_tokens, num_idx_heads, idx_dim), divisibility=16
|
||||
)
|
||||
index_rope_cache = _make_fake_tensor(
|
||||
idx_rope_type, (max_pos, rope_dim), divisibility=8
|
||||
)
|
||||
index_weights = _make_fake_tensor(
|
||||
idx_weights_type, (num_tokens, num_idx_heads), divisibility=8
|
||||
)
|
||||
index_q_fp8 = _make_fake_tensor(
|
||||
Float8E4M3FN, (num_tokens, num_idx_heads, idx_dim), divisibility=16
|
||||
)
|
||||
index_weights_out = _make_fake_tensor(
|
||||
Float32, (num_tokens, num_idx_heads), divisibility=4
|
||||
)
|
||||
else:
|
||||
index_q = index_rope_cache = index_q_fp8 = None
|
||||
index_weights = index_weights_out = None
|
||||
|
||||
kernel = FusedQKernel(
|
||||
rope_dim,
|
||||
nope_dim,
|
||||
num_heads,
|
||||
idx_dim,
|
||||
num_idx_heads,
|
||||
index_rope_interleave,
|
||||
)
|
||||
stream = cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True)
|
||||
return cute.compile(
|
||||
kernel,
|
||||
positions,
|
||||
q_pe,
|
||||
rope_cache,
|
||||
ql_nope,
|
||||
q_scale,
|
||||
mqa_output,
|
||||
index_q,
|
||||
index_rope_cache,
|
||||
index_weights,
|
||||
index_q_fp8,
|
||||
index_weights_out,
|
||||
Float32(0.0),
|
||||
stream,
|
||||
options="--enable-tvm-ffi",
|
||||
)
|
||||
@@ -28,7 +28,6 @@ from vllm.v1.attention.backend import (
|
||||
MultipleOf,
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.sparse_utils import (
|
||||
phys_shadow,
|
||||
triton_convert_req_index_to_global_index,
|
||||
triton_filter_and_convert_dcp_index,
|
||||
)
|
||||
@@ -405,36 +404,6 @@ class FlashInferMLASparseImpl(SparseMLACommonImpl[FlashInferMLASparseMetadata]):
|
||||
NUM_TOPK_TOKENS=topk_indices.shape[1],
|
||||
return_valid_counts=True,
|
||||
)
|
||||
elif (shadow := phys_shadow(self.topk_indices_buffer)) is not None:
|
||||
# skip_topk layers reuse the previous fresh layer's PHYSICAL
|
||||
# indices: the logical top-k indices are shared (one buffer for
|
||||
# the whole forward) and block_table/req_id are per-step constant,
|
||||
# so re-converting them per layer produces identical output. This
|
||||
# covers both the backbone (index_topk_freq > 1) and MTP draft
|
||||
# steps 1+ (index_share_for_mtp_iteration sets skip_topk; the
|
||||
# draft compact re-arranges the shadow together with the logical
|
||||
# buffer). The fresh/skip split is fixed per captured graph, so
|
||||
# the branch is cudagraph-safe.
|
||||
phys_buf, seq_buf = shadow
|
||||
wrote_fresh = getattr(layer, "indexer", None) is not None and not getattr(
|
||||
layer, "skip_topk", False
|
||||
)
|
||||
if wrote_fresh:
|
||||
topk_indices_physical, seq_lens = (
|
||||
triton_convert_req_index_to_global_index(
|
||||
attn_metadata.req_id_per_token[:num_actual_toks],
|
||||
attn_metadata.block_table,
|
||||
topk_indices,
|
||||
BLOCK_SIZE=attn_metadata.block_size,
|
||||
NUM_TOPK_TOKENS=topk_indices.shape[1],
|
||||
return_valid_counts=True,
|
||||
out=phys_buf[:num_actual_toks],
|
||||
valid_counts_out=seq_buf[:num_actual_toks],
|
||||
)
|
||||
)
|
||||
else:
|
||||
topk_indices_physical = phys_buf[:num_actual_toks]
|
||||
seq_lens = seq_buf[:num_actual_toks]
|
||||
else:
|
||||
topk_indices_physical, seq_lens = triton_convert_req_index_to_global_index(
|
||||
attn_metadata.req_id_per_token[:num_actual_toks],
|
||||
|
||||
@@ -128,8 +128,6 @@ def triton_convert_req_index_to_global_index(
|
||||
prefill_workspace_request_ids: torch.Tensor | None = None,
|
||||
prefill_workspace_starts: torch.Tensor | None = None,
|
||||
return_valid_counts: bool = False,
|
||||
out: torch.Tensor | None = None,
|
||||
valid_counts_out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
out[token_id, indice_id] =
|
||||
@@ -180,20 +178,14 @@ def triton_convert_req_index_to_global_index(
|
||||
req_id_c = req_id.contiguous()
|
||||
block_table_c = block_table.contiguous()
|
||||
token_indices_c = token_indices.contiguous()
|
||||
# `out`/`valid_counts_out` allow writing into a stable pre-allocated buffer
|
||||
# (used by the shared-physical-index cache to stay cudagraph-safe).
|
||||
out = torch.empty_like(token_indices_c) if out is None else out
|
||||
out = torch.empty_like(token_indices_c)
|
||||
|
||||
# Allocate valid count buffer if needed (must be zero-initialized for atomics)
|
||||
valid_counts: torch.Tensor | None = None
|
||||
if return_valid_counts:
|
||||
if valid_counts_out is None:
|
||||
valid_counts = torch.zeros(
|
||||
num_tokens, dtype=torch.int32, device=token_indices.device
|
||||
)
|
||||
else:
|
||||
valid_counts = valid_counts_out
|
||||
valid_counts.zero_()
|
||||
valid_counts = torch.zeros(
|
||||
num_tokens, dtype=torch.int32, device=token_indices.device
|
||||
)
|
||||
|
||||
# Strides in elements
|
||||
bt_stride0, bt_stride1 = block_table_c.stride()
|
||||
@@ -349,47 +341,3 @@ def triton_filter_and_convert_dcp_index(
|
||||
assert valid_counts is not None
|
||||
return out, valid_counts
|
||||
return out
|
||||
|
||||
|
||||
# --- Physical-index shadow (DSA index_topk_freq > 1) -------------------------
|
||||
# GLM-5.2 uses index_topk_freq=4: only ~1/4 of layers write a fresh top-k into
|
||||
# the single shared topk_indices_buffer; the rest reuse it. Since block_table
|
||||
# and req_id_per_token are constant within a decode step, the physical indices
|
||||
# (block_table lookup of the logical top-k) are identical across a freq-group.
|
||||
# Fresh layers convert once into a STABLE shadow of the logical buffer and
|
||||
# skip layers read it -- eliminating ~3/4 of the per-layer convert kernels.
|
||||
#
|
||||
# Invariant: shadow[j] == convert(logical[j]) row-wise. Whoever re-arranges
|
||||
# the logical buffer (e.g. the MTP draft compact between the multi-token
|
||||
# step-0 layout and the single-token steps-1+ layout) must apply the same
|
||||
# gather to the shadow.
|
||||
#
|
||||
# Shadows are registered ONLY at buffer creation time (model init, never
|
||||
# inside cudagraph capture) via register_phys_shadow; every other access is
|
||||
# the read-only phys_shadow lookup, so addresses are stable across graph
|
||||
# replays and no allocation can happen during capture.
|
||||
import weakref as _weakref # noqa: E402
|
||||
|
||||
_PHYS_SHADOWS: dict[int, tuple[torch.Tensor, torch.Tensor]] = {}
|
||||
|
||||
|
||||
def register_phys_shadow(topk_buf: torch.Tensor) -> None:
|
||||
"""Allocate the (physical-index, valid-count) shadow for a logical top-k
|
||||
buffer. Call once where the buffer is created. The finalizer drops the
|
||||
entry with the owning buffer, so a recycled id can never alias a stale
|
||||
shadow."""
|
||||
key = id(topk_buf)
|
||||
if key not in _PHYS_SHADOWS:
|
||||
_PHYS_SHADOWS[key] = (
|
||||
torch.empty_like(topk_buf),
|
||||
torch.empty(topk_buf.shape[0], dtype=torch.int32, device=topk_buf.device),
|
||||
)
|
||||
_weakref.finalize(topk_buf, _PHYS_SHADOWS.pop, key, None)
|
||||
|
||||
|
||||
def phys_shadow(
|
||||
topk_buf: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor] | None:
|
||||
"""The registered shadow for this buffer, or None if it was never
|
||||
registered. Never allocates; capture-safe."""
|
||||
return _PHYS_SHADOWS.get(id(topk_buf))
|
||||
|
||||
@@ -1009,9 +1009,8 @@ class SpecDecodeBaseProposer:
|
||||
# DeepSeek-family MTP (deepseek_mtp.py) recycles the post-final-
|
||||
# norm hidden, so its forward returns (logit_hidden,
|
||||
# recycle_hidden). Other MTP families return a single tensor.
|
||||
return bool(
|
||||
{"DeepSeekMTPModel", "DeepseekV32MTPModel"}
|
||||
& set(self.draft_model_config.hf_config.architectures or [])
|
||||
return "DeepSeekMTPModel" in (
|
||||
self.draft_model_config.hf_config.architectures or []
|
||||
)
|
||||
return self.method not in ("mtp", "draft_model", "dflash")
|
||||
|
||||
|
||||
@@ -48,18 +48,6 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
|
||||
self.prefill_cudagraph_manager: SpeculatorCudaGraphManager | None = None
|
||||
self.decode_cudagraph_manager: SpeculatorCudaGraphManager | None = None
|
||||
|
||||
# Lifecycle hooks for model-specific optimizations. Subclasses override
|
||||
# the ones they need. These fire in both `capture` and `propose` so that
|
||||
# any state they toggle (e.g. attention flags baked into a CUDA graph) is
|
||||
# identical at capture time and replay time.
|
||||
def on_prefill_begin(self, num_reqs: int, num_tokens: int) -> None: ...
|
||||
|
||||
def on_prefill_end(self, num_reqs: int, num_tokens: int) -> None: ...
|
||||
|
||||
def on_multi_step_decode_begin(self, num_reqs: int) -> None: ...
|
||||
|
||||
def on_multi_step_decode_end(self, num_reqs: int) -> None: ...
|
||||
|
||||
@property
|
||||
def advance_draft_positions(self) -> bool:
|
||||
"""
|
||||
@@ -110,9 +98,6 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
|
||||
assert self.prefill_cudagraph_manager is not None
|
||||
if self.prefill_cudagraph_manager.use_breakable_cg:
|
||||
self.prefill_cudagraph_manager.init_breakable_cg_runner(self.model)
|
||||
|
||||
self.on_prefill_begin(self.max_num_reqs, self.max_num_tokens)
|
||||
|
||||
self.prefill_cudagraph_manager.capture(
|
||||
self._prefill,
|
||||
self.model_state,
|
||||
@@ -123,13 +108,9 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
|
||||
progress_bar_desc="Capturing prefill CUDA graphs",
|
||||
)
|
||||
|
||||
self.on_prefill_end(self.max_num_reqs, self.max_num_tokens)
|
||||
|
||||
if self.num_speculative_steps == 1:
|
||||
return
|
||||
|
||||
self.on_multi_step_decode_begin(self.max_num_reqs)
|
||||
|
||||
# Capture the decode draft generation routine (model forward +
|
||||
# sample + update_draft_inputs) for a single
|
||||
# step.
|
||||
@@ -144,8 +125,6 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
|
||||
progress_bar_desc="Capturing decode CUDA graphs",
|
||||
)
|
||||
|
||||
self.on_multi_step_decode_end(self.max_num_reqs)
|
||||
|
||||
@torch.inference_mode()
|
||||
def propose(
|
||||
self,
|
||||
@@ -174,8 +153,7 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
|
||||
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None,
|
||||
is_profile: bool = False,
|
||||
) -> torch.Tensor:
|
||||
num_tokens = input_batch.num_tokens
|
||||
num_tokens_padded = input_batch.num_tokens_after_padding
|
||||
num_tokens = input_batch.num_tokens_after_padding
|
||||
num_reqs = input_batch.num_reqs
|
||||
max_query_len = input_batch.num_scheduled_tokens.max()
|
||||
max_seq_len = input_batch.seq_lens_cpu_upper_bound[:num_reqs].max().item()
|
||||
@@ -196,7 +174,7 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
|
||||
)
|
||||
else:
|
||||
hidden_states = last_hidden_states
|
||||
self.hidden_states[:num_tokens_padded].copy_(hidden_states)
|
||||
self.hidden_states[:num_tokens].copy_(hidden_states)
|
||||
|
||||
self._copy_request_inputs(
|
||||
num_reqs,
|
||||
@@ -224,22 +202,20 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
|
||||
num_reqs,
|
||||
# Use the actual number of tokens without padding added by
|
||||
# the target model during FULL cudagraph.
|
||||
num_tokens,
|
||||
input_batch.num_tokens,
|
||||
max_query_len,
|
||||
)
|
||||
prefill_batch_desc, num_tokens_across_dp = dispatch_cg_and_sync_dp(
|
||||
self.prefill_cudagraph_manager,
|
||||
num_reqs,
|
||||
num_tokens_padded,
|
||||
num_tokens,
|
||||
uniform_token_count,
|
||||
dp_size=self.dp_size,
|
||||
dp_rank=self.dp_rank,
|
||||
need_eager=is_profile,
|
||||
)
|
||||
|
||||
self._prepare_eplb_forward(num_tokens)
|
||||
|
||||
self.on_prefill_begin(num_reqs, num_tokens)
|
||||
self._prepare_eplb_forward(input_batch.num_tokens)
|
||||
|
||||
if prefill_batch_desc.cg_mode == CUDAGraphMode.FULL:
|
||||
# Replay the full graph for draft prefill.
|
||||
@@ -259,8 +235,6 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
|
||||
mm_inputs=mm_inputs,
|
||||
)
|
||||
|
||||
self.on_prefill_end(num_reqs, num_tokens)
|
||||
|
||||
if self.num_speculative_steps == 1:
|
||||
# Early exit.
|
||||
return self.draft_tokens[:num_reqs, :1]
|
||||
@@ -288,8 +262,6 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
|
||||
need_eager=is_profile,
|
||||
)
|
||||
|
||||
self.on_multi_step_decode_begin(num_reqs)
|
||||
|
||||
# Generate the remaining num_speculative_steps - 1 draft tokens.
|
||||
self._multi_step_decode(
|
||||
num_reqs,
|
||||
@@ -299,8 +271,6 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
|
||||
input_batch.seq_lens_cpu_upper_bound,
|
||||
)
|
||||
|
||||
self.on_multi_step_decode_end(num_reqs)
|
||||
|
||||
return self.draft_tokens[:num_reqs]
|
||||
|
||||
@torch.inference_mode()
|
||||
|
||||
@@ -1,10 +1,8 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.v1.worker.gpu.spec_decode.autoregressive.speculator import (
|
||||
AutoRegressiveSpeculator,
|
||||
)
|
||||
@@ -12,46 +10,9 @@ from vllm.v1.worker.gpu.spec_decode.eagle.utils import load_eagle_model
|
||||
|
||||
|
||||
class MTPSpeculator(AutoRegressiveSpeculator):
|
||||
def __init__(self, vllm_config: VllmConfig, device: torch.device):
|
||||
super().__init__(vllm_config, device)
|
||||
|
||||
spec_config = vllm_config.speculative_config
|
||||
draft_hf_config = (
|
||||
spec_config.draft_model_config.hf_config
|
||||
if spec_config is not None
|
||||
else None
|
||||
)
|
||||
# Detect index_share_for_mtp_iteration. When True, the proposer
|
||||
# toggles skip_topk so step 0 computes MTP's own indices and
|
||||
# steps 1+ reuse them.
|
||||
self.share_mtp_topk_indices = getattr(
|
||||
draft_hf_config, "index_share_for_mtp_iteration", False
|
||||
)
|
||||
|
||||
def load_draft_model(
|
||||
self,
|
||||
target_model: nn.Module,
|
||||
target_attn_layer_names: set[str],
|
||||
) -> nn.Module:
|
||||
draft_model = load_eagle_model(target_model, self.vllm_config)
|
||||
self.share_mtp_topk_indices = self.share_mtp_topk_indices and hasattr(
|
||||
draft_model.model, "set_skip_topk"
|
||||
)
|
||||
return draft_model
|
||||
|
||||
def on_prefill_end(self, num_reqs: int, num_tokens: int) -> None:
|
||||
# Step 0 (prefill) wrote topk indices for every query token in the
|
||||
# multi-token batch. Compact them down to each request's last token so
|
||||
# steps 1+ can reuse them from the shared buffer.
|
||||
if self.share_mtp_topk_indices and self.num_speculative_steps > 1:
|
||||
self.model.model.compact_topk_indices(self.last_token_indices[:num_reqs])
|
||||
|
||||
def on_multi_step_decode_begin(self, num_reqs: int) -> None:
|
||||
# Switch to reuse mode so draft steps 1+ skip the indexer op and read
|
||||
# the indices that step 0 wrote into the shared buffer.
|
||||
if self.share_mtp_topk_indices:
|
||||
self.model.model.set_skip_topk(True)
|
||||
|
||||
def on_multi_step_decode_end(self, num_reqs: int) -> None:
|
||||
if self.share_mtp_topk_indices:
|
||||
self.model.model.set_skip_topk(False)
|
||||
return load_eagle_model(target_model, self.vllm_config)
|
||||
|
||||
Reference in New Issue
Block a user