Merge branch 'main' into patches
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
+21
-9
@@ -273,7 +273,7 @@ __global__ void reshape_and_cache_kernel(
|
||||
const int64_t* __restrict__ slot_mapping, // [num_tokens]
|
||||
const int key_stride, const int value_stride, const int num_heads,
|
||||
const int head_size, const int block_size, const int x,
|
||||
const float* k_scale, const float* v_scale) {
|
||||
const float* k_scale, const float* v_scale, const bool interleaved_v) {
|
||||
const int64_t token_idx = blockIdx.x;
|
||||
const int64_t slot_idx = slot_mapping[token_idx];
|
||||
if (slot_idx < 0) {
|
||||
@@ -301,10 +301,20 @@ __global__ void reshape_and_cache_kernel(
|
||||
key_cache + block_idx * num_heads * h_block_count * block_size * x +
|
||||
head_idx * h_block_count * block_size * x + h_block * block_size * x +
|
||||
block_offset * x;
|
||||
const int64_t tgt_value_start =
|
||||
block_idx * num_heads * h_block_count * x * block_size +
|
||||
head_idx * h_block_count * x * block_size + h_block * x * block_size +
|
||||
block_offset;
|
||||
// V cache target: standard [.., head_size, block_size], or the interleaved
|
||||
// [.., block_size/x, head_size, x] the fused AITER writer/reader use.
|
||||
const int64_t v_head_base = block_idx * num_heads * head_size * block_size +
|
||||
head_idx * head_size * block_size;
|
||||
int64_t tgt_value_start;
|
||||
int v_dim_stride;
|
||||
if (interleaved_v) {
|
||||
tgt_value_start = v_head_base + (block_offset / x) * (head_size * x) +
|
||||
(h_block * x) * x + (block_offset % x);
|
||||
v_dim_stride = x;
|
||||
} else {
|
||||
tgt_value_start = v_head_base + h_block * x * block_size + block_offset;
|
||||
v_dim_stride = block_size;
|
||||
}
|
||||
|
||||
constexpr int VEC_SIZE = (sizeof(scalar_t) == 2) ? 8 : 4;
|
||||
float k_scale_val = (kv_dt == Fp8KVCacheDataType::kAuto) ? 0.f : *k_scale;
|
||||
@@ -318,7 +328,7 @@ __global__ void reshape_and_cache_kernel(
|
||||
cache_t* __restrict__ value_dst = value_cache + tgt_value_start;
|
||||
#pragma unroll
|
||||
for (int i = 0; i < x; i++) {
|
||||
v_op(value_dst[i * block_size], value_src[i]);
|
||||
v_op(value_dst[i * v_dim_stride], value_src[i]);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -775,7 +785,7 @@ __global__ void cp_gather_indexer_k_quant_cache_kernel(
|
||||
slot_mapping.const_data_ptr<int64_t>(), key_stride, value_stride, \
|
||||
num_heads, head_size, block_size, x, \
|
||||
reinterpret_cast<const float*>(k_scale.data_ptr()), \
|
||||
reinterpret_cast<const float*>(v_scale.data_ptr()));
|
||||
reinterpret_cast<const float*>(v_scale.data_ptr()), interleaved_v);
|
||||
|
||||
void reshape_and_cache(
|
||||
torch::stable::Tensor& key, // [num_tokens, num_heads, head_size]
|
||||
@@ -783,10 +793,12 @@ void reshape_and_cache(
|
||||
torch::stable::Tensor&
|
||||
key_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
|
||||
torch::stable::Tensor&
|
||||
value_cache, // [num_blocks, num_heads, head_size, block_size]
|
||||
value_cache, // [num_blocks, num_heads, head_size, block_size], or the
|
||||
// interleaved [.., block_size/x, head_size, x] when
|
||||
// interleaved_v is set
|
||||
torch::stable::Tensor& slot_mapping, // [num_tokens]
|
||||
const std::string& kv_cache_dtype, torch::stable::Tensor& k_scale,
|
||||
torch::stable::Tensor& v_scale) {
|
||||
torch::stable::Tensor& v_scale, bool interleaved_v) {
|
||||
int num_tokens = slot_mapping.size(0);
|
||||
int num_heads = key.size(1);
|
||||
int head_size = key.size(2);
|
||||
|
||||
@@ -32,9 +32,10 @@
|
||||
* [0, bs*576): token data, 448 fp8 + 128 bf16 each
|
||||
* [bs*576, bs*576 + bs*8): UE8M0 scales, 7 real + 1 pad per token
|
||||
* V4.1 (528 B/token, kv_mxfp8=true) -- RoPE is quantized as well, so the
|
||||
* whole row is MXFP8 (fp8 e4m3 with one UE8M0 scale per 32 dims)
|
||||
* [0, bs*512): token data, 512 fp8 each
|
||||
* [bs*512, bs*512 + bs*16): UE8M0 scales, 16 per token
|
||||
* whole row is MXFP8 (fp8 e4m3 with one UE8M0 scale per 32 dims), and each
|
||||
* token's scales follow its own data
|
||||
* [i*528, i*528 + 512): token i data, 512 fp8
|
||||
* [i*528 + 512, (i+1)*528): token i UE8M0 scales, 16
|
||||
*/
|
||||
|
||||
#include "torch_utils.h"
|
||||
@@ -104,6 +105,8 @@ constexpr int kTokenDataBytes = kNopeDim + kRopeDim * 2; // 448 + 128 = 576
|
||||
constexpr int kMxfp8QuantBlock = 32;
|
||||
constexpr int kMxfp8NumQuantBlocks = kHeadDim / kMxfp8QuantBlock; // 16
|
||||
constexpr int kMxfp8TokenDataBytes = kHeadDim; // 512
|
||||
constexpr int kMxfp8BytesPerToken =
|
||||
kMxfp8TokenDataBytes + kMxfp8NumQuantBlocks; // 528
|
||||
// FNUZ on gfx942 / OCP elsewhere. FNUZ uses 224.0 (not the dtype's raw
|
||||
// 240.0) to match the rest of vLLM's FNUZ pipeline.
|
||||
#if defined(USE_ROCM) && defined(__gfx942__)
|
||||
@@ -356,20 +359,20 @@ __device__ __forceinline__ void processDeepseekV4Slot(
|
||||
// V4.1 (kv_mxfp8) quantizes them too.
|
||||
int64_t const slot_id = slot_mapping[tokenIdx];
|
||||
if (slot_id >= 0) {
|
||||
int const token_data_bytes =
|
||||
kv_mxfp8 ? kMxfp8TokenDataBytes : kTokenDataBytes;
|
||||
int const scale_bytes =
|
||||
kv_mxfp8 ? kMxfp8NumQuantBlocks : kScaleBytesPerToken;
|
||||
int64_t const block_idx = slot_id / cache_block_size;
|
||||
int64_t const pos_in_block = slot_id % cache_block_size;
|
||||
uint8_t* block_base =
|
||||
k_cache + block_idx * static_cast<int64_t>(kv_block_stride);
|
||||
uint8_t* token_fp8_ptr = block_base + pos_in_block * token_data_bytes;
|
||||
uint8_t* token_fp8_ptr =
|
||||
block_base +
|
||||
pos_in_block * (kv_mxfp8 ? kMxfp8BytesPerToken : kTokenDataBytes);
|
||||
uint8_t* token_bf16_ptr = token_fp8_ptr + kNopeDim;
|
||||
uint8_t* token_scale_ptr =
|
||||
block_base +
|
||||
static_cast<int64_t>(cache_block_size) * token_data_bytes +
|
||||
pos_in_block * scale_bytes;
|
||||
kv_mxfp8
|
||||
? token_fp8_ptr + kMxfp8TokenDataBytes
|
||||
: block_base +
|
||||
static_cast<int64_t>(cache_block_size) * kTokenDataBytes +
|
||||
pos_in_block * kScaleBytesPerToken;
|
||||
|
||||
#pragma unroll
|
||||
for (int i = 0; i < kElemsPerLane; i++) {
|
||||
@@ -790,7 +793,7 @@ void launchFusedDeepseekV4QNormRopeKVRopeQuantInsert(
|
||||
// backend. Differences from the legacy path:
|
||||
// * No Q head padding — output Q layout matches the input num_heads_q.
|
||||
// * KV is written as a *contiguous* 512-wide row per token (token-strided),
|
||||
// not the legacy UE8M0 paged layout with a separate scale tail.
|
||||
// not the V4 paged record with a separate scale tail.
|
||||
// * Q/KV are stored either as bf16 or as per-tensor E4M3 FP8 (one global
|
||||
// scale), selected by the STORE_Q_FP8 / STORE_KV_FP8 template flags.
|
||||
//
|
||||
|
||||
+1
-1
@@ -650,7 +650,7 @@ void reshape_and_cache(torch::stable::Tensor& key, torch::stable::Tensor& value,
|
||||
torch::stable::Tensor& slot_mapping,
|
||||
const std::string& kv_cache_dtype,
|
||||
torch::stable::Tensor& k_scale,
|
||||
torch::stable::Tensor& v_scale);
|
||||
torch::stable::Tensor& v_scale, bool interleaved_v);
|
||||
|
||||
void reshape_and_cache_flash(
|
||||
torch::stable::Tensor& key, torch::stable::Tensor& value,
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
|
||||
#include "core/utils.hpp"
|
||||
#include "core/cuda_compat.h"
|
||||
#include "cuda_vec_utils.cuh"
|
||||
#include "dispatch_utils.h"
|
||||
#include "quantization/w8a8/fp8/common.cuh"
|
||||
|
||||
@@ -43,7 +44,7 @@ template <typename scalar_t, scalar_t (*ACT_FN)(const scalar_t&),
|
||||
__global__ void act_and_mul_quant_kernel(
|
||||
fp8_type* __restrict__ out, // [..., d]
|
||||
const scalar_t* __restrict__ input, // [..., 2, d]
|
||||
const float* scale, const int d) {
|
||||
const float* scale, const int d, const bool vectorize) {
|
||||
const int32_t blocks_per_token = gridDim.y;
|
||||
|
||||
const int32_t elems_per_128bit_load = (128 / 8) / sizeof(scalar_t);
|
||||
@@ -64,9 +65,12 @@ __global__ void act_and_mul_quant_kernel(
|
||||
const scalar_t* __restrict__ y_ptr = input + token_idx * 2 * d + d;
|
||||
fp8_type* __restrict__ out_ptr = out + token_idx * d;
|
||||
|
||||
// 128-bit vectorized code
|
||||
const int32_t vec_loop_end =
|
||||
round_to_previous_multiple_of(elems_per_128bit_load, block_end);
|
||||
// 128-bit vectorized code, when d and the data pointers allow 16-byte
|
||||
// loads and 8-byte stores on every row; otherwise the cleanup loop below
|
||||
// handles the whole block.
|
||||
const int32_t vec_loop_end = vectorize ? round_to_previous_multiple_of(
|
||||
block_end, elems_per_128bit_load)
|
||||
: block_start;
|
||||
const int32_t vec_end_idx = vec_loop_end / elems_per_128bit_load;
|
||||
const int32_t vec_start_idx = block_start / elems_per_128bit_load;
|
||||
|
||||
@@ -559,26 +563,30 @@ __global__ void silu_mul_fp8_quant_deep_gemm_kernel(
|
||||
} // namespace vllm
|
||||
|
||||
// Launch activation, gating, and quantize kernel.
|
||||
#define LAUNCH_ACTIVATION_GATE_KERNEL(KERNEL) \
|
||||
int d = input.size(-1) / 2; \
|
||||
int64_t num_tokens = input.numel() / input.size(-1); \
|
||||
dim3 grid(num_tokens, num_tokens > 16 ? num_tokens > 32 ? 1 : 2 : 4); \
|
||||
dim3 block(std::min(d, 512)); \
|
||||
const torch::stable::accelerator::DeviceGuard device_guard( \
|
||||
input.get_device_index()); \
|
||||
const cudaStream_t stream = \
|
||||
get_current_cuda_stream(input.get_device_index()); \
|
||||
VLLM_STABLE_DISPATCH_FLOATING_TYPES( \
|
||||
input.scalar_type(), "act_and_mul_kernel", [&] { \
|
||||
VLLM_STABLE_DISPATCH_FP8_TYPES( \
|
||||
out.scalar_type(), "act_and_mul_quant_kernel_fp8_type", [&] { \
|
||||
vllm::act_and_mul_quant_kernel<scalar_t, KERNEL<scalar_t>, \
|
||||
fp8_t> \
|
||||
<<<grid, block, 0, stream>>>( \
|
||||
out.mutable_data_ptr<fp8_t>(), \
|
||||
input.const_data_ptr<scalar_t>(), \
|
||||
scale.const_data_ptr<float>(), d); \
|
||||
}); \
|
||||
#define LAUNCH_ACTIVATION_GATE_KERNEL(KERNEL) \
|
||||
int d = input.size(-1) / 2; \
|
||||
int64_t num_tokens = input.numel() / input.size(-1); \
|
||||
dim3 grid(num_tokens, num_tokens > 16 ? num_tokens > 32 ? 1 : 2 : 4); \
|
||||
dim3 block(std::min(d, 512)); \
|
||||
const torch::stable::accelerator::DeviceGuard device_guard( \
|
||||
input.get_device_index()); \
|
||||
const cudaStream_t stream = \
|
||||
get_current_cuda_stream(input.get_device_index()); \
|
||||
VLLM_STABLE_DISPATCH_FLOATING_TYPES( \
|
||||
input.scalar_type(), "act_and_mul_kernel", [&] { \
|
||||
VLLM_STABLE_DISPATCH_FP8_TYPES( \
|
||||
out.scalar_type(), "act_and_mul_quant_kernel_fp8_type", [&] { \
|
||||
const scalar_t* in_ptr = input.const_data_ptr<scalar_t>(); \
|
||||
fp8_t* out_ptr = out.mutable_data_ptr<fp8_t>(); \
|
||||
const bool vectorize = d % (16 / sizeof(scalar_t)) == 0 && \
|
||||
vllm::is_16byte_aligned(in_ptr) && \
|
||||
vllm::is_16byte_aligned(out_ptr); \
|
||||
vllm::act_and_mul_quant_kernel<scalar_t, KERNEL<scalar_t>, \
|
||||
fp8_t> \
|
||||
<<<grid, block, 0, stream>>>(out_ptr, in_ptr, \
|
||||
scale.const_data_ptr<float>(), \
|
||||
d, vectorize); \
|
||||
}); \
|
||||
});
|
||||
|
||||
void silu_and_mul_quant(torch::stable::Tensor& out, // [..., d]
|
||||
|
||||
+154
-67
@@ -355,7 +355,8 @@ __device__ float warpReduceMax(float val) {
|
||||
// clang-format off
|
||||
template <typename scalar_t, typename cache_t,
|
||||
vllm::Fp8KVCacheDataType KV_DTYPE, typename OUTT, int BLOCK_SIZE,
|
||||
int HEAD_SIZE, int NUM_THREADS, bool ALIBI_ENABLED, int GQA_RATIO, MFMAType MFMA_TYPE>
|
||||
int HEAD_SIZE, int NUM_THREADS, bool ALIBI_ENABLED, int GQA_RATIO, MFMAType MFMA_TYPE,
|
||||
bool USE_INTERLEAVED_V_CACHE>
|
||||
__global__
|
||||
__launch_bounds__(NUM_THREADS, 4) void paged_attention_ll4mi_QKV_mfma16_kernel(
|
||||
const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
|
||||
@@ -593,26 +594,59 @@ __launch_bounds__(NUM_THREADS, 4) void paged_attention_ll4mi_QKV_mfma16_kernel(
|
||||
|
||||
_B16x8 Vlocal[VTLOOP][VHELOOP][VTLANELOOP]; // this could be B8x16 too
|
||||
|
||||
const cache_t* v_ptr = v_cache + wg_start_kv_head_idx * kv_head_stride +
|
||||
((rowid * VTOKENS_PER_LANE) % BLOCK_SIZE);
|
||||
// KX_V = number of contiguous token elements per interleaved chunk
|
||||
// (e.g. 8 for bf16, 16 for fp8). This is also the 'x' dimension size.
|
||||
constexpr int KX_V = 16 / sizeof(cache_t);
|
||||
|
||||
// v fetches are 16head elems across lanes x 16 tokens per lane
|
||||
for (int vhe_depth = 0; vhe_depth < VHELOOP; vhe_depth++) {
|
||||
const int vhead_elem = vhe_depth * NWARPS * 16 + warpid * 16 + lane16id;
|
||||
const cache_t* v_ptr2 = v_ptr + vhead_elem * BLOCK_SIZE;
|
||||
|
||||
for (int vtoken_depth = 0; vtoken_depth < VTLOOP; vtoken_depth++) {
|
||||
for (int vfetch_depth = 0; vfetch_depth < VTLANELOOP; vfetch_depth++) {
|
||||
if constexpr (USE_INTERLEAVED_V_CACHE) {
|
||||
// Interleaved V-cache layout: [num_blocks, num_heads, block_size/x,
|
||||
// head_dim, x] Address for v[block][head][token t][dim d]:
|
||||
// chunk = t / x, sub = t % x
|
||||
// addr = base + chunk * head_dim * x + d * x + sub
|
||||
// Each 16B fetch reads KX_V contiguous elements within a single chunk, so
|
||||
// a lane's token offset must land on a chunk boundary. This holds only
|
||||
// when BLOCK_SIZE is a multiple of KX_V.
|
||||
static_assert(BLOCK_SIZE % KX_V == 0,
|
||||
"Interleaved V-cache read requires BLOCK_SIZE % KX_V == 0");
|
||||
const cache_t* v_ptr_head = v_cache + wg_start_kv_head_idx * kv_head_stride;
|
||||
for (int vhe_depth = 0; vhe_depth < VHELOOP; vhe_depth++) {
|
||||
const int vhead_elem = vhe_depth * NWARPS * 16 + warpid * 16 + lane16id;
|
||||
for (int vtoken_depth = 0; vtoken_depth < VTLOOP; vtoken_depth++) {
|
||||
const int vblock_depth = 0;
|
||||
const int64_t vblock_number = static_cast<int64_t>(
|
||||
vphysical_block_number[vtoken_depth][vblock_depth]);
|
||||
const cache_t* v_ptr3 = v_ptr2 + (vblock_number * kv_block_stride);
|
||||
|
||||
const cache_t* v_fetch_ptr =
|
||||
v_ptr3 + vfetch_depth * CONTIGUOUS_KV_ELEMS_16B_LOAD;
|
||||
const _B16x8* v_fetch_ptr_16B =
|
||||
reinterpret_cast<const _B16x8*>(v_fetch_ptr);
|
||||
Vlocal[vtoken_depth][vhe_depth][vfetch_depth] = *v_fetch_ptr_16B;
|
||||
const cache_t* v_block_ptr =
|
||||
v_ptr_head + vblock_number * kv_block_stride;
|
||||
for (int vfetch_depth = 0; vfetch_depth < VTLANELOOP; vfetch_depth++) {
|
||||
const int bo = ((rowid * VTOKENS_PER_LANE) % BLOCK_SIZE) +
|
||||
vfetch_depth * CONTIGUOUS_KV_ELEMS_16B_LOAD;
|
||||
const int bo_chunk = bo / KX_V; // which chunk of x tokens
|
||||
const cache_t* v_fetch_ptr =
|
||||
v_block_ptr + (bo_chunk * HEAD_SIZE + vhead_elem) * KX_V +
|
||||
(bo % KX_V);
|
||||
Vlocal[vtoken_depth][vhe_depth][vfetch_depth] =
|
||||
*reinterpret_cast<const _B16x8*>(v_fetch_ptr);
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
const cache_t* v_ptr = v_cache + wg_start_kv_head_idx * kv_head_stride +
|
||||
((rowid * VTOKENS_PER_LANE) % BLOCK_SIZE);
|
||||
for (int vhe_depth = 0; vhe_depth < VHELOOP; vhe_depth++) {
|
||||
const int vhead_elem = vhe_depth * NWARPS * 16 + warpid * 16 + lane16id;
|
||||
const cache_t* v_ptr2 = v_ptr + vhead_elem * BLOCK_SIZE;
|
||||
for (int vtoken_depth = 0; vtoken_depth < VTLOOP; vtoken_depth++) {
|
||||
for (int vfetch_depth = 0; vfetch_depth < VTLANELOOP; vfetch_depth++) {
|
||||
const int vblock_depth = 0;
|
||||
const int64_t vblock_number = static_cast<int64_t>(
|
||||
vphysical_block_number[vtoken_depth][vblock_depth]);
|
||||
const cache_t* v_ptr3 = v_ptr2 + (vblock_number * kv_block_stride);
|
||||
const cache_t* v_fetch_ptr =
|
||||
v_ptr3 + vfetch_depth * CONTIGUOUS_KV_ELEMS_16B_LOAD;
|
||||
const _B16x8* v_fetch_ptr_16B =
|
||||
reinterpret_cast<const _B16x8*>(v_fetch_ptr);
|
||||
Vlocal[vtoken_depth][vhe_depth][vfetch_depth] = *v_fetch_ptr_16B;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -996,7 +1030,8 @@ __launch_bounds__(NUM_THREADS, 4) void paged_attention_ll4mi_QKV_mfma16_kernel(
|
||||
template <typename scalar_t, typename cache_t,
|
||||
vllm::Fp8KVCacheDataType KV_DTYPE, typename OUTT, int BLOCK_SIZE,
|
||||
int HEAD_SIZE, int NUM_THREADS, bool ALIBI_ENABLED,
|
||||
int GQA_RATIO>
|
||||
int GQA_RATIO,
|
||||
bool USE_INTERLEAVED_V_CACHE>
|
||||
__global__
|
||||
__launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_mfma4_kernel(
|
||||
const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
|
||||
@@ -1177,25 +1212,41 @@ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_mfma4_kernel(
|
||||
}
|
||||
}
|
||||
|
||||
// KX_V = number of contiguous token elements per interleaved chunk
|
||||
constexpr int KX_V = 16 / sizeof(cache_t);
|
||||
const cache_t* v_ptr = v_cache + wg_start_kv_head_idx * kv_head_stride;
|
||||
// fetch vcache in kv cache auto case
|
||||
// fetch vcache in kv cache auto case (bf16/fp16)
|
||||
if constexpr (KV_DTYPE == vllm::Fp8KVCacheDataType::kAuto) {
|
||||
const _B16x8* v_ptrh8 = reinterpret_cast<const _B16x8*>(v_ptr);
|
||||
// iterate over each v block
|
||||
for (int b = 0; b < VBLOCKS; b++) {
|
||||
// int32 physical_block_number leads to overflow when multiplied with
|
||||
// kv_block_stride
|
||||
const int64_t vphysical_block_number =
|
||||
static_cast<int64_t>(vphysical_blocks[b]);
|
||||
const _B16x8* v_ptrh8b =
|
||||
v_ptrh8 + (vphysical_block_number * kv_block_stride) / 8;
|
||||
// iterate over each head elem (within head_size)
|
||||
for (int h = 0; h < VHELOOP; h++) {
|
||||
const int head_size_elem = h * WARP_SIZE + laneid;
|
||||
const _B16x8* v_ptrh8be = v_ptrh8b + head_size_elem * BLOCK_SIZE / 8;
|
||||
// iterate over all velems within block
|
||||
for (int d = 0; d < BLOCK_SIZE / 8; d++) {
|
||||
Vlocal[h][b * BLOCK_SIZE / 8 + d] = v_ptrh8be[d];
|
||||
if constexpr (USE_INTERLEAVED_V_CACHE) {
|
||||
// Interleaved layout: [num_blocks, num_heads, block_size/x,
|
||||
// head_dim, x] Iterate over chunks of x tokens; within each chunk,
|
||||
// x values for the same head_size_elem are contiguous (16-byte
|
||||
// aligned).
|
||||
for (int bo_chunk = 0; bo_chunk < BLOCK_SIZE / KX_V; bo_chunk++) {
|
||||
const _B16x8* v_chunk =
|
||||
v_ptrh8b + (bo_chunk * HEAD_SIZE + head_size_elem) * KX_V / 8;
|
||||
for (int xd = 0; xd < KX_V / 8; xd++) {
|
||||
Vlocal[h][b * BLOCK_SIZE / 8 + bo_chunk * (KX_V / 8) + xd] =
|
||||
v_chunk[xd];
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Standard PagedAttention layout: [num_blocks, num_heads, head_dim,
|
||||
// block_size] All block_size tokens for one head_size_elem are
|
||||
// contiguous.
|
||||
const _B16x8* v_ptrh8be =
|
||||
v_ptrh8b + head_size_elem * BLOCK_SIZE / 8;
|
||||
for (int d = 0; d < BLOCK_SIZE / 8; d++) {
|
||||
Vlocal[h][b * BLOCK_SIZE / 8 + d] = v_ptrh8be[d];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1203,21 +1254,29 @@ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_mfma4_kernel(
|
||||
// fetch vcache in fp8 case
|
||||
else { // if constexpr (KV_DTYPE != vllm::Fp8KVCacheDataType::kAuto)
|
||||
const _B8x8* v_ptrh8 = reinterpret_cast<const _B8x8*>(v_ptr);
|
||||
// iterate over each v block
|
||||
for (int b = 0; b < VBLOCKS; b++) {
|
||||
// int32 physical_block_number leads to overflow when multiplied with
|
||||
// kv_block_stride
|
||||
const int64_t vphysical_block_number =
|
||||
static_cast<int64_t>(vphysical_blocks[b]);
|
||||
const _B8x8* v_ptrh8b =
|
||||
v_ptrh8 + (vphysical_block_number * kv_block_stride) / 8;
|
||||
// iterate over each head elem (within head_size)
|
||||
for (int h = 0; h < VHELOOP; h++) {
|
||||
const int head_size_elem = h * WARP_SIZE + laneid;
|
||||
const _B8x8* v_ptrh8be = v_ptrh8b + head_size_elem * BLOCK_SIZE / 8;
|
||||
// iterate over all velems within block
|
||||
for (int d = 0; d < BLOCK_SIZE / 8; d++) {
|
||||
Vlocalb8[h][b * BLOCK_SIZE / 8 + d] = v_ptrh8be[d];
|
||||
if constexpr (USE_INTERLEAVED_V_CACHE) {
|
||||
// Same interleaved addressing as bf16 above, but with fp8
|
||||
// elements (KX_V=16, using _B8x8 loads instead of _B16x8).
|
||||
for (int bo_chunk = 0; bo_chunk < BLOCK_SIZE / KX_V; bo_chunk++) {
|
||||
const _B8x8* v_chunk =
|
||||
v_ptrh8b + (bo_chunk * HEAD_SIZE + head_size_elem) * KX_V / 8;
|
||||
for (int xd = 0; xd < KX_V / 8; xd++) {
|
||||
Vlocalb8[h][b * BLOCK_SIZE / 8 + bo_chunk * (KX_V / 8) + xd] =
|
||||
v_chunk[xd];
|
||||
}
|
||||
}
|
||||
} else {
|
||||
const _B8x8* v_ptrh8be = v_ptrh8b + head_size_elem * BLOCK_SIZE / 8;
|
||||
for (int d = 0; d < BLOCK_SIZE / 8; d++) {
|
||||
Vlocalb8[h][b * BLOCK_SIZE / 8 + d] = v_ptrh8be[d];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1831,7 +1890,8 @@ __device__ __forceinline__ _B16x8 from_floatx8(const floatx8& inp) {
|
||||
template <typename scalar_t, typename cache_t,
|
||||
vllm::Fp8KVCacheDataType KV_DTYPE, typename OUTT, int BLOCK_SIZE,
|
||||
int HEAD_SIZE, int NUM_THREADS, bool ALIBI_ENABLED, int GQA_RATIO,
|
||||
MFMAType MFMA_TYPE>
|
||||
MFMAType MFMA_TYPE,
|
||||
bool USE_INTERLEAVED_V_CACHE>
|
||||
__global__
|
||||
__launch_bounds__(NUM_THREADS, 3) void paged_attention_ll4mi_QKV_mfma16_kernel(
|
||||
const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
|
||||
@@ -2285,8 +2345,8 @@ __launch_bounds__(NUM_THREADS, 3) void paged_attention_ll4mi_QKV_mfma16_kernel(
|
||||
|
||||
template <typename scalar_t, typename cache_t,
|
||||
vllm::Fp8KVCacheDataType KV_DTYPE, typename OUTT, int BLOCK_SIZE,
|
||||
int HEAD_SIZE, int NUM_THREADS, bool ALIBI_ENABLED,
|
||||
int GQA_RATIO>
|
||||
int HEAD_SIZE, int NUM_THREADS, bool ALIBI_ENABLED, int GQA_RATIO,
|
||||
bool USE_INTERLEAVED_V_CACHE>
|
||||
__global__
|
||||
__launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_mfma4_kernel(
|
||||
const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
|
||||
@@ -2611,7 +2671,8 @@ __device__ __forceinline__ _B16x8 from_floatx8(const floatx8& inp) {
|
||||
template <typename scalar_t, typename cache_t,
|
||||
vllm::Fp8KVCacheDataType KV_DTYPE, typename OUTT, int BLOCK_SIZE,
|
||||
int HEAD_SIZE, int NUM_THREADS, bool ALIBI_ENABLED, int GQA_RATIO,
|
||||
MFMAType MFMA_TYPE>
|
||||
MFMAType MFMA_TYPE,
|
||||
bool USE_INTERLEAVED_V_CACHE>
|
||||
__global__
|
||||
__launch_bounds__(NUM_THREADS, 3) void paged_attention_ll4mi_QKV_mfma16_kernel(
|
||||
const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
|
||||
@@ -3030,8 +3091,8 @@ __launch_bounds__(NUM_THREADS, 3) void paged_attention_ll4mi_QKV_mfma16_kernel(
|
||||
|
||||
template <typename scalar_t, typename cache_t,
|
||||
vllm::Fp8KVCacheDataType KV_DTYPE, typename OUTT, int BLOCK_SIZE,
|
||||
int HEAD_SIZE, int NUM_THREADS, bool ALIBI_ENABLED,
|
||||
int GQA_RATIO>
|
||||
int HEAD_SIZE, int NUM_THREADS, bool ALIBI_ENABLED, int GQA_RATIO,
|
||||
bool USE_INTERLEAVED_V_CACHE>
|
||||
__global__
|
||||
__launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_mfma4_kernel(
|
||||
const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
|
||||
@@ -3261,7 +3322,8 @@ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_reduce_kernel(
|
||||
template <typename scalar_t, typename cache_t,
|
||||
vllm::Fp8KVCacheDataType KV_DTYPE, typename OUTT, int BLOCK_SIZE,
|
||||
int HEAD_SIZE, int NUM_THREADS, bool ALIBI_ENABLED,
|
||||
int GQA_RATIO, MFMAType MFMA_TYPE>
|
||||
int GQA_RATIO, MFMAType MFMA_TYPE,
|
||||
bool USE_INTERLEAVED_V_CACHE>
|
||||
__global__
|
||||
__launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_mfma16_kernel(
|
||||
const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
|
||||
@@ -3288,7 +3350,8 @@ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_mfma16_kernel(
|
||||
template <typename scalar_t, typename cache_t,
|
||||
vllm::Fp8KVCacheDataType KV_DTYPE, typename OUTT, int BLOCK_SIZE,
|
||||
int HEAD_SIZE, int NUM_THREADS, bool ALIBI_ENABLED,
|
||||
int GQA_RATIO>
|
||||
int GQA_RATIO,
|
||||
bool USE_INTERLEAVED_V_CACHE>
|
||||
__global__
|
||||
__launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_mfma4_kernel(
|
||||
const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
|
||||
@@ -3330,10 +3393,11 @@ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_reduce_kernel(
|
||||
|
||||
#endif
|
||||
|
||||
// clang-format off
|
||||
#define LAUNCH_CUSTOM_ATTENTION_MFMA16(GQA_RATIO) \
|
||||
paged_attention_ll4mi_QKV_mfma16_kernel<T, KVT, KV_DTYPE, OUTT, BLOCK_SIZE, \
|
||||
HEAD_SIZE, NTHR, ALIBI_ENABLED, \
|
||||
GQA_RATIO, MFMA_TYPE> \
|
||||
paged_attention_ll4mi_QKV_mfma16_kernel< \
|
||||
T, KVT, KV_DTYPE, OUTT, BLOCK_SIZE, HEAD_SIZE, NTHR, ALIBI_ENABLED, \
|
||||
GQA_RATIO, MFMA_TYPE, USE_INTERLEAVED_V_CACHE> \
|
||||
<<<grid, block, 0, stream>>>( \
|
||||
query_ptr, key_cache_ptr, value_cache_ptr, num_kv_heads, scale, \
|
||||
block_tables_ptr, seq_lens_ptr, query_start_loc_ptr, \
|
||||
@@ -3342,15 +3406,16 @@ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_reduce_kernel(
|
||||
max_ctx_blocks, k_scale_ptr, v_scale_ptr);
|
||||
|
||||
#define LAUNCH_CUSTOM_ATTENTION_MFMA4(GQA_RATIO) \
|
||||
paged_attention_ll4mi_QKV_mfma4_kernel<T, KVT, KV_DTYPE, OUTT, BLOCK_SIZE, \
|
||||
HEAD_SIZE, NTHR, ALIBI_ENABLED, \
|
||||
GQA_RATIO> \
|
||||
paged_attention_ll4mi_QKV_mfma4_kernel< \
|
||||
T, KVT, KV_DTYPE, OUTT, BLOCK_SIZE, HEAD_SIZE, NTHR, ALIBI_ENABLED, \
|
||||
GQA_RATIO, USE_INTERLEAVED_V_CACHE> \
|
||||
<<<grid, block, 0, stream>>>( \
|
||||
query_ptr, key_cache_ptr, value_cache_ptr, num_kv_heads, scale, \
|
||||
block_tables_ptr, seq_lens_ptr, query_start_loc_ptr, \
|
||||
max_num_blocks_per_seq, alibi_slopes_ptr, q_stride, kv_block_stride, \
|
||||
kv_head_stride, exp_sums_ptr, max_logits_ptr, tmp_out_ptr, out_ptr, \
|
||||
max_ctx_blocks, k_scale_ptr, v_scale_ptr);
|
||||
// clang-format on
|
||||
|
||||
#define LAUNCH_CUSTOM_REDUCTION(NPAR_LOOPS) \
|
||||
paged_attention_ll4mi_reduce_kernel<T, OUTT, HEAD_SIZE, HEAD_SIZE, \
|
||||
@@ -3361,7 +3426,7 @@ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_reduce_kernel(
|
||||
|
||||
template <typename T, typename KVT, vllm::Fp8KVCacheDataType KV_DTYPE,
|
||||
int BLOCK_SIZE, int HEAD_SIZE, typename OUTT, int PARTITION_SIZE_OLD,
|
||||
bool ALIBI_ENABLED, MFMAType MFMA_TYPE>
|
||||
bool ALIBI_ENABLED, MFMAType MFMA_TYPE, bool USE_INTERLEAVED_V_CACHE>
|
||||
void paged_attention_custom_launcher(
|
||||
torch::Tensor& out, torch::Tensor& exp_sums, torch::Tensor& max_logits,
|
||||
torch::Tensor& tmp_out, torch::Tensor& query, torch::Tensor& key_cache,
|
||||
@@ -3517,7 +3582,7 @@ void paged_attention_custom_launcher(
|
||||
|
||||
template <typename T, typename KVT, vllm::Fp8KVCacheDataType KV_DTYPE,
|
||||
int BLOCK_SIZE, int HEAD_SIZE, typename OUTT, int PARTITION_SIZE_OLD,
|
||||
bool ALIBI_ENABLED, MFMAType MFMA_TYPE>
|
||||
bool ALIBI_ENABLED, MFMAType MFMA_TYPE, bool USE_INTERLEAVED_V_CACHE>
|
||||
void paged_attention_custom_launcher_navi(
|
||||
torch::Tensor& out, torch::Tensor& exp_sums, torch::Tensor& max_logits,
|
||||
torch::Tensor& tmp_out, torch::Tensor& query, torch::Tensor& key_cache,
|
||||
@@ -3688,21 +3753,42 @@ void paged_attention_custom_launcher_navi(
|
||||
}
|
||||
}
|
||||
|
||||
#define CALL_CUSTOM_LAUNCHER(T, KVT, KV_DTYPE, BLK_SIZE, HEAD_SIZE, OUTT, \
|
||||
PSIZE, ALIBI_ENABLED, MFMA_TYPE) \
|
||||
if (!is_navi) { \
|
||||
paged_attention_custom_launcher<T, KVT, KV_DTYPE, BLK_SIZE, HEAD_SIZE, \
|
||||
OUTT, PSIZE, ALIBI_ENABLED, MFMA_TYPE>( \
|
||||
out, exp_sums, max_logits, tmp_out, query, key_cache, value_cache, \
|
||||
num_kv_heads, scale, block_tables, seq_lens, query_start_loc, \
|
||||
max_seq_len, alibi_slopes, k_scale, v_scale, fp8_out_scale); \
|
||||
} else { \
|
||||
paged_attention_custom_launcher_navi<T, KVT, KV_DTYPE, BLK_SIZE, \
|
||||
HEAD_SIZE, OUTT, PSIZE, \
|
||||
ALIBI_ENABLED, MFMA_TYPE>( \
|
||||
out, exp_sums, max_logits, tmp_out, query, key_cache, value_cache, \
|
||||
num_kv_heads, scale, block_tables, seq_lens, query_start_loc, \
|
||||
max_seq_len, alibi_slopes, k_scale, v_scale); \
|
||||
// USE_INTERLEAVED_V_CACHE gates the V-cache read path at compile time.
|
||||
// Currently only the GFX9 kernels (MI300x/MI325x/MI350x/MI355x) have the
|
||||
// actual interleaved V-fetch logic implemented. The GFX11 (RDNA 3) and
|
||||
// GFX12 (RDNA 4) mfma16 kernels do NOT support interleaved V-cache addressing.
|
||||
// The template parameter is carried on all variants for compilation
|
||||
// compatibility; a TORCH_CHECK prevents dispatch to NAVI with interleaved V.
|
||||
// TODO: implement interleaved V-cache support in the NAVI (GFX11/GFX12)
|
||||
// kernels if shuffle KV cache layout is needed on RDNA GPUs.
|
||||
#define CALL_CUSTOM_LAUNCHER(T, KVT, KV_DTYPE, BLK_SIZE, HEAD_SIZE, OUTT, \
|
||||
PSIZE, ALIBI_ENABLED, MFMA_TYPE) \
|
||||
if (use_interleaved_v_cache) { \
|
||||
TORCH_CHECK(!is_navi, \
|
||||
"Reading interleaved V-cache layout from paged decode " \
|
||||
"attention not supported on NAVI GPUs"); \
|
||||
paged_attention_custom_launcher<T, KVT, KV_DTYPE, BLK_SIZE, HEAD_SIZE, \
|
||||
OUTT, PSIZE, ALIBI_ENABLED, MFMA_TYPE, \
|
||||
true>( \
|
||||
out, exp_sums, max_logits, tmp_out, query, key_cache, value_cache, \
|
||||
num_kv_heads, scale, block_tables, seq_lens, query_start_loc, \
|
||||
max_seq_len, alibi_slopes, k_scale, v_scale, fp8_out_scale); \
|
||||
} else { \
|
||||
if (!is_navi) { \
|
||||
paged_attention_custom_launcher<T, KVT, KV_DTYPE, BLK_SIZE, HEAD_SIZE, \
|
||||
OUTT, PSIZE, ALIBI_ENABLED, MFMA_TYPE, \
|
||||
false>( \
|
||||
out, exp_sums, max_logits, tmp_out, query, key_cache, value_cache, \
|
||||
num_kv_heads, scale, block_tables, seq_lens, query_start_loc, \
|
||||
max_seq_len, alibi_slopes, k_scale, v_scale, fp8_out_scale); \
|
||||
} else { \
|
||||
paged_attention_custom_launcher_navi<T, KVT, KV_DTYPE, BLK_SIZE, \
|
||||
HEAD_SIZE, OUTT, PSIZE, \
|
||||
ALIBI_ENABLED, MFMA_TYPE, false>( \
|
||||
out, exp_sums, max_logits, tmp_out, query, key_cache, value_cache, \
|
||||
num_kv_heads, scale, block_tables, seq_lens, query_start_loc, \
|
||||
max_seq_len, alibi_slopes, k_scale, v_scale); \
|
||||
} \
|
||||
}
|
||||
|
||||
#define CALL_CUSTOM_LAUNCHER_ALIBI(T, KVT, KV_DTYPE, BLK_SIZE, HEAD_SIZE, \
|
||||
@@ -3799,7 +3885,8 @@ void paged_attention(
|
||||
const std::string& kv_cache_dtype, torch::Tensor& k_scale,
|
||||
torch::Tensor& v_scale,
|
||||
const std::optional<torch::Tensor>& fp8_out_scale,
|
||||
const std::string& mfma_type) {
|
||||
const std::string& mfma_type,
|
||||
bool use_interleaved_v_cache) {
|
||||
// clang-format on
|
||||
bool is_navi = is_navi_gpu();
|
||||
const int head_size = query.size(2);
|
||||
|
||||
+1
-1
@@ -50,4 +50,4 @@ void paged_attention(
|
||||
int64_t max_seq_len, const std::optional<torch::Tensor>& alibi_slopes,
|
||||
const std::string& kv_cache_dtype, torch::Tensor& k_scale,
|
||||
torch::Tensor& v_scale, const std::optional<torch::Tensor>& fp8_out_scale,
|
||||
const std::string& mfma_type);
|
||||
const std::string& mfma_type, bool use_interleaved_v_cache);
|
||||
|
||||
@@ -92,7 +92,8 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, rocm_ops) {
|
||||
" str kv_cache_dtype,"
|
||||
" Tensor k_scale, Tensor v_scale,"
|
||||
" Tensor? fp8_out_scale,"
|
||||
" str mfma_type) -> ()");
|
||||
" str mfma_type,"
|
||||
" bool use_interleaved_v_cache=False) -> ()");
|
||||
rocm_ops.impl("paged_attention", torch::kCUDA, &paged_attention);
|
||||
}
|
||||
|
||||
|
||||
@@ -955,7 +955,8 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C_cache_ops, ops) {
|
||||
" Tensor! key_cache, Tensor! value_cache,"
|
||||
" Tensor slot_mapping,"
|
||||
" str kv_cache_dtype,"
|
||||
" Tensor k_scale, Tensor v_scale) -> ()");
|
||||
" Tensor k_scale, Tensor v_scale,"
|
||||
" bool interleaved_v=False) -> ()");
|
||||
|
||||
// Reshape the key and value tensors and cache them.
|
||||
ops.def(
|
||||
|
||||
Reference in New Issue
Block a user