[Model][Hardware][AMD][Kernel] Part 2/2 -> Enable e2e QK Norm + RoPE + KV Cache runtime fusion for Qwen3-30B-A3B on ROCM_ATTN (#42755)
Signed-off-by: Jack Hu <Jack.Hu@amd.com> Signed-off-by: Jack Hu <jingxujaster@hotmail.com> Signed-off-by: Jack Hu <jack.hu@amd.com> Co-authored-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com>
This commit is contained in:
+154
-67
@@ -355,7 +355,8 @@ __device__ float warpReduceMax(float val) {
|
||||
// clang-format off
|
||||
template <typename scalar_t, typename cache_t,
|
||||
vllm::Fp8KVCacheDataType KV_DTYPE, typename OUTT, int BLOCK_SIZE,
|
||||
int HEAD_SIZE, int NUM_THREADS, bool ALIBI_ENABLED, int GQA_RATIO, MFMAType MFMA_TYPE>
|
||||
int HEAD_SIZE, int NUM_THREADS, bool ALIBI_ENABLED, int GQA_RATIO, MFMAType MFMA_TYPE,
|
||||
bool USE_INTERLEAVED_V_CACHE>
|
||||
__global__
|
||||
__launch_bounds__(NUM_THREADS, 4) void paged_attention_ll4mi_QKV_mfma16_kernel(
|
||||
const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
|
||||
@@ -593,26 +594,59 @@ __launch_bounds__(NUM_THREADS, 4) void paged_attention_ll4mi_QKV_mfma16_kernel(
|
||||
|
||||
_B16x8 Vlocal[VTLOOP][VHELOOP][VTLANELOOP]; // this could be B8x16 too
|
||||
|
||||
const cache_t* v_ptr = v_cache + wg_start_kv_head_idx * kv_head_stride +
|
||||
((rowid * VTOKENS_PER_LANE) % BLOCK_SIZE);
|
||||
// KX_V = number of contiguous token elements per interleaved chunk
|
||||
// (e.g. 8 for bf16, 16 for fp8). This is also the 'x' dimension size.
|
||||
constexpr int KX_V = 16 / sizeof(cache_t);
|
||||
|
||||
// v fetches are 16head elems across lanes x 16 tokens per lane
|
||||
for (int vhe_depth = 0; vhe_depth < VHELOOP; vhe_depth++) {
|
||||
const int vhead_elem = vhe_depth * NWARPS * 16 + warpid * 16 + lane16id;
|
||||
const cache_t* v_ptr2 = v_ptr + vhead_elem * BLOCK_SIZE;
|
||||
|
||||
for (int vtoken_depth = 0; vtoken_depth < VTLOOP; vtoken_depth++) {
|
||||
for (int vfetch_depth = 0; vfetch_depth < VTLANELOOP; vfetch_depth++) {
|
||||
if constexpr (USE_INTERLEAVED_V_CACHE) {
|
||||
// Interleaved V-cache layout: [num_blocks, num_heads, block_size/x,
|
||||
// head_dim, x] Address for v[block][head][token t][dim d]:
|
||||
// chunk = t / x, sub = t % x
|
||||
// addr = base + chunk * head_dim * x + d * x + sub
|
||||
// Each 16B fetch reads KX_V contiguous elements within a single chunk, so
|
||||
// a lane's token offset must land on a chunk boundary. This holds only
|
||||
// when BLOCK_SIZE is a multiple of KX_V.
|
||||
static_assert(BLOCK_SIZE % KX_V == 0,
|
||||
"Interleaved V-cache read requires BLOCK_SIZE % KX_V == 0");
|
||||
const cache_t* v_ptr_head = v_cache + wg_start_kv_head_idx * kv_head_stride;
|
||||
for (int vhe_depth = 0; vhe_depth < VHELOOP; vhe_depth++) {
|
||||
const int vhead_elem = vhe_depth * NWARPS * 16 + warpid * 16 + lane16id;
|
||||
for (int vtoken_depth = 0; vtoken_depth < VTLOOP; vtoken_depth++) {
|
||||
const int vblock_depth = 0;
|
||||
const int64_t vblock_number = static_cast<int64_t>(
|
||||
vphysical_block_number[vtoken_depth][vblock_depth]);
|
||||
const cache_t* v_ptr3 = v_ptr2 + (vblock_number * kv_block_stride);
|
||||
|
||||
const cache_t* v_fetch_ptr =
|
||||
v_ptr3 + vfetch_depth * CONTIGUOUS_KV_ELEMS_16B_LOAD;
|
||||
const _B16x8* v_fetch_ptr_16B =
|
||||
reinterpret_cast<const _B16x8*>(v_fetch_ptr);
|
||||
Vlocal[vtoken_depth][vhe_depth][vfetch_depth] = *v_fetch_ptr_16B;
|
||||
const cache_t* v_block_ptr =
|
||||
v_ptr_head + vblock_number * kv_block_stride;
|
||||
for (int vfetch_depth = 0; vfetch_depth < VTLANELOOP; vfetch_depth++) {
|
||||
const int bo = ((rowid * VTOKENS_PER_LANE) % BLOCK_SIZE) +
|
||||
vfetch_depth * CONTIGUOUS_KV_ELEMS_16B_LOAD;
|
||||
const int bo_chunk = bo / KX_V; // which chunk of x tokens
|
||||
const cache_t* v_fetch_ptr =
|
||||
v_block_ptr + (bo_chunk * HEAD_SIZE + vhead_elem) * KX_V +
|
||||
(bo % KX_V);
|
||||
Vlocal[vtoken_depth][vhe_depth][vfetch_depth] =
|
||||
*reinterpret_cast<const _B16x8*>(v_fetch_ptr);
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
const cache_t* v_ptr = v_cache + wg_start_kv_head_idx * kv_head_stride +
|
||||
((rowid * VTOKENS_PER_LANE) % BLOCK_SIZE);
|
||||
for (int vhe_depth = 0; vhe_depth < VHELOOP; vhe_depth++) {
|
||||
const int vhead_elem = vhe_depth * NWARPS * 16 + warpid * 16 + lane16id;
|
||||
const cache_t* v_ptr2 = v_ptr + vhead_elem * BLOCK_SIZE;
|
||||
for (int vtoken_depth = 0; vtoken_depth < VTLOOP; vtoken_depth++) {
|
||||
for (int vfetch_depth = 0; vfetch_depth < VTLANELOOP; vfetch_depth++) {
|
||||
const int vblock_depth = 0;
|
||||
const int64_t vblock_number = static_cast<int64_t>(
|
||||
vphysical_block_number[vtoken_depth][vblock_depth]);
|
||||
const cache_t* v_ptr3 = v_ptr2 + (vblock_number * kv_block_stride);
|
||||
const cache_t* v_fetch_ptr =
|
||||
v_ptr3 + vfetch_depth * CONTIGUOUS_KV_ELEMS_16B_LOAD;
|
||||
const _B16x8* v_fetch_ptr_16B =
|
||||
reinterpret_cast<const _B16x8*>(v_fetch_ptr);
|
||||
Vlocal[vtoken_depth][vhe_depth][vfetch_depth] = *v_fetch_ptr_16B;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -996,7 +1030,8 @@ __launch_bounds__(NUM_THREADS, 4) void paged_attention_ll4mi_QKV_mfma16_kernel(
|
||||
template <typename scalar_t, typename cache_t,
|
||||
vllm::Fp8KVCacheDataType KV_DTYPE, typename OUTT, int BLOCK_SIZE,
|
||||
int HEAD_SIZE, int NUM_THREADS, bool ALIBI_ENABLED,
|
||||
int GQA_RATIO>
|
||||
int GQA_RATIO,
|
||||
bool USE_INTERLEAVED_V_CACHE>
|
||||
__global__
|
||||
__launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_mfma4_kernel(
|
||||
const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
|
||||
@@ -1177,25 +1212,41 @@ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_mfma4_kernel(
|
||||
}
|
||||
}
|
||||
|
||||
// KX_V = number of contiguous token elements per interleaved chunk
|
||||
constexpr int KX_V = 16 / sizeof(cache_t);
|
||||
const cache_t* v_ptr = v_cache + wg_start_kv_head_idx * kv_head_stride;
|
||||
// fetch vcache in kv cache auto case
|
||||
// fetch vcache in kv cache auto case (bf16/fp16)
|
||||
if constexpr (KV_DTYPE == vllm::Fp8KVCacheDataType::kAuto) {
|
||||
const _B16x8* v_ptrh8 = reinterpret_cast<const _B16x8*>(v_ptr);
|
||||
// iterate over each v block
|
||||
for (int b = 0; b < VBLOCKS; b++) {
|
||||
// int32 physical_block_number leads to overflow when multiplied with
|
||||
// kv_block_stride
|
||||
const int64_t vphysical_block_number =
|
||||
static_cast<int64_t>(vphysical_blocks[b]);
|
||||
const _B16x8* v_ptrh8b =
|
||||
v_ptrh8 + (vphysical_block_number * kv_block_stride) / 8;
|
||||
// iterate over each head elem (within head_size)
|
||||
for (int h = 0; h < VHELOOP; h++) {
|
||||
const int head_size_elem = h * WARP_SIZE + laneid;
|
||||
const _B16x8* v_ptrh8be = v_ptrh8b + head_size_elem * BLOCK_SIZE / 8;
|
||||
// iterate over all velems within block
|
||||
for (int d = 0; d < BLOCK_SIZE / 8; d++) {
|
||||
Vlocal[h][b * BLOCK_SIZE / 8 + d] = v_ptrh8be[d];
|
||||
if constexpr (USE_INTERLEAVED_V_CACHE) {
|
||||
// Interleaved layout: [num_blocks, num_heads, block_size/x,
|
||||
// head_dim, x] Iterate over chunks of x tokens; within each chunk,
|
||||
// x values for the same head_size_elem are contiguous (16-byte
|
||||
// aligned).
|
||||
for (int bo_chunk = 0; bo_chunk < BLOCK_SIZE / KX_V; bo_chunk++) {
|
||||
const _B16x8* v_chunk =
|
||||
v_ptrh8b + (bo_chunk * HEAD_SIZE + head_size_elem) * KX_V / 8;
|
||||
for (int xd = 0; xd < KX_V / 8; xd++) {
|
||||
Vlocal[h][b * BLOCK_SIZE / 8 + bo_chunk * (KX_V / 8) + xd] =
|
||||
v_chunk[xd];
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Standard PagedAttention layout: [num_blocks, num_heads, head_dim,
|
||||
// block_size] All block_size tokens for one head_size_elem are
|
||||
// contiguous.
|
||||
const _B16x8* v_ptrh8be =
|
||||
v_ptrh8b + head_size_elem * BLOCK_SIZE / 8;
|
||||
for (int d = 0; d < BLOCK_SIZE / 8; d++) {
|
||||
Vlocal[h][b * BLOCK_SIZE / 8 + d] = v_ptrh8be[d];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1203,21 +1254,29 @@ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_mfma4_kernel(
|
||||
// fetch vcache in fp8 case
|
||||
else { // if constexpr (KV_DTYPE != vllm::Fp8KVCacheDataType::kAuto)
|
||||
const _B8x8* v_ptrh8 = reinterpret_cast<const _B8x8*>(v_ptr);
|
||||
// iterate over each v block
|
||||
for (int b = 0; b < VBLOCKS; b++) {
|
||||
// int32 physical_block_number leads to overflow when multiplied with
|
||||
// kv_block_stride
|
||||
const int64_t vphysical_block_number =
|
||||
static_cast<int64_t>(vphysical_blocks[b]);
|
||||
const _B8x8* v_ptrh8b =
|
||||
v_ptrh8 + (vphysical_block_number * kv_block_stride) / 8;
|
||||
// iterate over each head elem (within head_size)
|
||||
for (int h = 0; h < VHELOOP; h++) {
|
||||
const int head_size_elem = h * WARP_SIZE + laneid;
|
||||
const _B8x8* v_ptrh8be = v_ptrh8b + head_size_elem * BLOCK_SIZE / 8;
|
||||
// iterate over all velems within block
|
||||
for (int d = 0; d < BLOCK_SIZE / 8; d++) {
|
||||
Vlocalb8[h][b * BLOCK_SIZE / 8 + d] = v_ptrh8be[d];
|
||||
if constexpr (USE_INTERLEAVED_V_CACHE) {
|
||||
// Same interleaved addressing as bf16 above, but with fp8
|
||||
// elements (KX_V=16, using _B8x8 loads instead of _B16x8).
|
||||
for (int bo_chunk = 0; bo_chunk < BLOCK_SIZE / KX_V; bo_chunk++) {
|
||||
const _B8x8* v_chunk =
|
||||
v_ptrh8b + (bo_chunk * HEAD_SIZE + head_size_elem) * KX_V / 8;
|
||||
for (int xd = 0; xd < KX_V / 8; xd++) {
|
||||
Vlocalb8[h][b * BLOCK_SIZE / 8 + bo_chunk * (KX_V / 8) + xd] =
|
||||
v_chunk[xd];
|
||||
}
|
||||
}
|
||||
} else {
|
||||
const _B8x8* v_ptrh8be = v_ptrh8b + head_size_elem * BLOCK_SIZE / 8;
|
||||
for (int d = 0; d < BLOCK_SIZE / 8; d++) {
|
||||
Vlocalb8[h][b * BLOCK_SIZE / 8 + d] = v_ptrh8be[d];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1831,7 +1890,8 @@ __device__ __forceinline__ _B16x8 from_floatx8(const floatx8& inp) {
|
||||
template <typename scalar_t, typename cache_t,
|
||||
vllm::Fp8KVCacheDataType KV_DTYPE, typename OUTT, int BLOCK_SIZE,
|
||||
int HEAD_SIZE, int NUM_THREADS, bool ALIBI_ENABLED, int GQA_RATIO,
|
||||
MFMAType MFMA_TYPE>
|
||||
MFMAType MFMA_TYPE,
|
||||
bool USE_INTERLEAVED_V_CACHE>
|
||||
__global__
|
||||
__launch_bounds__(NUM_THREADS, 3) void paged_attention_ll4mi_QKV_mfma16_kernel(
|
||||
const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
|
||||
@@ -2285,8 +2345,8 @@ __launch_bounds__(NUM_THREADS, 3) void paged_attention_ll4mi_QKV_mfma16_kernel(
|
||||
|
||||
template <typename scalar_t, typename cache_t,
|
||||
vllm::Fp8KVCacheDataType KV_DTYPE, typename OUTT, int BLOCK_SIZE,
|
||||
int HEAD_SIZE, int NUM_THREADS, bool ALIBI_ENABLED,
|
||||
int GQA_RATIO>
|
||||
int HEAD_SIZE, int NUM_THREADS, bool ALIBI_ENABLED, int GQA_RATIO,
|
||||
bool USE_INTERLEAVED_V_CACHE>
|
||||
__global__
|
||||
__launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_mfma4_kernel(
|
||||
const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
|
||||
@@ -2611,7 +2671,8 @@ __device__ __forceinline__ _B16x8 from_floatx8(const floatx8& inp) {
|
||||
template <typename scalar_t, typename cache_t,
|
||||
vllm::Fp8KVCacheDataType KV_DTYPE, typename OUTT, int BLOCK_SIZE,
|
||||
int HEAD_SIZE, int NUM_THREADS, bool ALIBI_ENABLED, int GQA_RATIO,
|
||||
MFMAType MFMA_TYPE>
|
||||
MFMAType MFMA_TYPE,
|
||||
bool USE_INTERLEAVED_V_CACHE>
|
||||
__global__
|
||||
__launch_bounds__(NUM_THREADS, 3) void paged_attention_ll4mi_QKV_mfma16_kernel(
|
||||
const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
|
||||
@@ -3030,8 +3091,8 @@ __launch_bounds__(NUM_THREADS, 3) void paged_attention_ll4mi_QKV_mfma16_kernel(
|
||||
|
||||
template <typename scalar_t, typename cache_t,
|
||||
vllm::Fp8KVCacheDataType KV_DTYPE, typename OUTT, int BLOCK_SIZE,
|
||||
int HEAD_SIZE, int NUM_THREADS, bool ALIBI_ENABLED,
|
||||
int GQA_RATIO>
|
||||
int HEAD_SIZE, int NUM_THREADS, bool ALIBI_ENABLED, int GQA_RATIO,
|
||||
bool USE_INTERLEAVED_V_CACHE>
|
||||
__global__
|
||||
__launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_mfma4_kernel(
|
||||
const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
|
||||
@@ -3261,7 +3322,8 @@ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_reduce_kernel(
|
||||
template <typename scalar_t, typename cache_t,
|
||||
vllm::Fp8KVCacheDataType KV_DTYPE, typename OUTT, int BLOCK_SIZE,
|
||||
int HEAD_SIZE, int NUM_THREADS, bool ALIBI_ENABLED,
|
||||
int GQA_RATIO, MFMAType MFMA_TYPE>
|
||||
int GQA_RATIO, MFMAType MFMA_TYPE,
|
||||
bool USE_INTERLEAVED_V_CACHE>
|
||||
__global__
|
||||
__launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_mfma16_kernel(
|
||||
const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
|
||||
@@ -3288,7 +3350,8 @@ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_mfma16_kernel(
|
||||
template <typename scalar_t, typename cache_t,
|
||||
vllm::Fp8KVCacheDataType KV_DTYPE, typename OUTT, int BLOCK_SIZE,
|
||||
int HEAD_SIZE, int NUM_THREADS, bool ALIBI_ENABLED,
|
||||
int GQA_RATIO>
|
||||
int GQA_RATIO,
|
||||
bool USE_INTERLEAVED_V_CACHE>
|
||||
__global__
|
||||
__launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_mfma4_kernel(
|
||||
const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
|
||||
@@ -3330,10 +3393,11 @@ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_reduce_kernel(
|
||||
|
||||
#endif
|
||||
|
||||
// clang-format off
|
||||
#define LAUNCH_CUSTOM_ATTENTION_MFMA16(GQA_RATIO) \
|
||||
paged_attention_ll4mi_QKV_mfma16_kernel<T, KVT, KV_DTYPE, OUTT, BLOCK_SIZE, \
|
||||
HEAD_SIZE, NTHR, ALIBI_ENABLED, \
|
||||
GQA_RATIO, MFMA_TYPE> \
|
||||
paged_attention_ll4mi_QKV_mfma16_kernel< \
|
||||
T, KVT, KV_DTYPE, OUTT, BLOCK_SIZE, HEAD_SIZE, NTHR, ALIBI_ENABLED, \
|
||||
GQA_RATIO, MFMA_TYPE, USE_INTERLEAVED_V_CACHE> \
|
||||
<<<grid, block, 0, stream>>>( \
|
||||
query_ptr, key_cache_ptr, value_cache_ptr, num_kv_heads, scale, \
|
||||
block_tables_ptr, seq_lens_ptr, query_start_loc_ptr, \
|
||||
@@ -3342,15 +3406,16 @@ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_reduce_kernel(
|
||||
max_ctx_blocks, k_scale_ptr, v_scale_ptr);
|
||||
|
||||
#define LAUNCH_CUSTOM_ATTENTION_MFMA4(GQA_RATIO) \
|
||||
paged_attention_ll4mi_QKV_mfma4_kernel<T, KVT, KV_DTYPE, OUTT, BLOCK_SIZE, \
|
||||
HEAD_SIZE, NTHR, ALIBI_ENABLED, \
|
||||
GQA_RATIO> \
|
||||
paged_attention_ll4mi_QKV_mfma4_kernel< \
|
||||
T, KVT, KV_DTYPE, OUTT, BLOCK_SIZE, HEAD_SIZE, NTHR, ALIBI_ENABLED, \
|
||||
GQA_RATIO, USE_INTERLEAVED_V_CACHE> \
|
||||
<<<grid, block, 0, stream>>>( \
|
||||
query_ptr, key_cache_ptr, value_cache_ptr, num_kv_heads, scale, \
|
||||
block_tables_ptr, seq_lens_ptr, query_start_loc_ptr, \
|
||||
max_num_blocks_per_seq, alibi_slopes_ptr, q_stride, kv_block_stride, \
|
||||
kv_head_stride, exp_sums_ptr, max_logits_ptr, tmp_out_ptr, out_ptr, \
|
||||
max_ctx_blocks, k_scale_ptr, v_scale_ptr);
|
||||
// clang-format on
|
||||
|
||||
#define LAUNCH_CUSTOM_REDUCTION(NPAR_LOOPS) \
|
||||
paged_attention_ll4mi_reduce_kernel<T, OUTT, HEAD_SIZE, HEAD_SIZE, \
|
||||
@@ -3361,7 +3426,7 @@ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_reduce_kernel(
|
||||
|
||||
template <typename T, typename KVT, vllm::Fp8KVCacheDataType KV_DTYPE,
|
||||
int BLOCK_SIZE, int HEAD_SIZE, typename OUTT, int PARTITION_SIZE_OLD,
|
||||
bool ALIBI_ENABLED, MFMAType MFMA_TYPE>
|
||||
bool ALIBI_ENABLED, MFMAType MFMA_TYPE, bool USE_INTERLEAVED_V_CACHE>
|
||||
void paged_attention_custom_launcher(
|
||||
torch::Tensor& out, torch::Tensor& exp_sums, torch::Tensor& max_logits,
|
||||
torch::Tensor& tmp_out, torch::Tensor& query, torch::Tensor& key_cache,
|
||||
@@ -3517,7 +3582,7 @@ void paged_attention_custom_launcher(
|
||||
|
||||
template <typename T, typename KVT, vllm::Fp8KVCacheDataType KV_DTYPE,
|
||||
int BLOCK_SIZE, int HEAD_SIZE, typename OUTT, int PARTITION_SIZE_OLD,
|
||||
bool ALIBI_ENABLED, MFMAType MFMA_TYPE>
|
||||
bool ALIBI_ENABLED, MFMAType MFMA_TYPE, bool USE_INTERLEAVED_V_CACHE>
|
||||
void paged_attention_custom_launcher_navi(
|
||||
torch::Tensor& out, torch::Tensor& exp_sums, torch::Tensor& max_logits,
|
||||
torch::Tensor& tmp_out, torch::Tensor& query, torch::Tensor& key_cache,
|
||||
@@ -3688,21 +3753,42 @@ void paged_attention_custom_launcher_navi(
|
||||
}
|
||||
}
|
||||
|
||||
#define CALL_CUSTOM_LAUNCHER(T, KVT, KV_DTYPE, BLK_SIZE, HEAD_SIZE, OUTT, \
|
||||
PSIZE, ALIBI_ENABLED, MFMA_TYPE) \
|
||||
if (!is_navi) { \
|
||||
paged_attention_custom_launcher<T, KVT, KV_DTYPE, BLK_SIZE, HEAD_SIZE, \
|
||||
OUTT, PSIZE, ALIBI_ENABLED, MFMA_TYPE>( \
|
||||
out, exp_sums, max_logits, tmp_out, query, key_cache, value_cache, \
|
||||
num_kv_heads, scale, block_tables, seq_lens, query_start_loc, \
|
||||
max_seq_len, alibi_slopes, k_scale, v_scale, fp8_out_scale); \
|
||||
} else { \
|
||||
paged_attention_custom_launcher_navi<T, KVT, KV_DTYPE, BLK_SIZE, \
|
||||
HEAD_SIZE, OUTT, PSIZE, \
|
||||
ALIBI_ENABLED, MFMA_TYPE>( \
|
||||
out, exp_sums, max_logits, tmp_out, query, key_cache, value_cache, \
|
||||
num_kv_heads, scale, block_tables, seq_lens, query_start_loc, \
|
||||
max_seq_len, alibi_slopes, k_scale, v_scale); \
|
||||
// USE_INTERLEAVED_V_CACHE gates the V-cache read path at compile time.
|
||||
// Currently only the GFX9 kernels (MI300x/MI325x/MI350x/MI355x) have the
|
||||
// actual interleaved V-fetch logic implemented. The GFX11 (RDNA 3) and
|
||||
// GFX12 (RDNA 4) mfma16 kernels do NOT support interleaved V-cache addressing.
|
||||
// The template parameter is carried on all variants for compilation
|
||||
// compatibility; a TORCH_CHECK prevents dispatch to NAVI with interleaved V.
|
||||
// TODO: implement interleaved V-cache support in the NAVI (GFX11/GFX12)
|
||||
// kernels if shuffle KV cache layout is needed on RDNA GPUs.
|
||||
#define CALL_CUSTOM_LAUNCHER(T, KVT, KV_DTYPE, BLK_SIZE, HEAD_SIZE, OUTT, \
|
||||
PSIZE, ALIBI_ENABLED, MFMA_TYPE) \
|
||||
if (use_interleaved_v_cache) { \
|
||||
TORCH_CHECK(!is_navi, \
|
||||
"Reading interleaved V-cache layout from paged decode " \
|
||||
"attention not supported on NAVI GPUs"); \
|
||||
paged_attention_custom_launcher<T, KVT, KV_DTYPE, BLK_SIZE, HEAD_SIZE, \
|
||||
OUTT, PSIZE, ALIBI_ENABLED, MFMA_TYPE, \
|
||||
true>( \
|
||||
out, exp_sums, max_logits, tmp_out, query, key_cache, value_cache, \
|
||||
num_kv_heads, scale, block_tables, seq_lens, query_start_loc, \
|
||||
max_seq_len, alibi_slopes, k_scale, v_scale, fp8_out_scale); \
|
||||
} else { \
|
||||
if (!is_navi) { \
|
||||
paged_attention_custom_launcher<T, KVT, KV_DTYPE, BLK_SIZE, HEAD_SIZE, \
|
||||
OUTT, PSIZE, ALIBI_ENABLED, MFMA_TYPE, \
|
||||
false>( \
|
||||
out, exp_sums, max_logits, tmp_out, query, key_cache, value_cache, \
|
||||
num_kv_heads, scale, block_tables, seq_lens, query_start_loc, \
|
||||
max_seq_len, alibi_slopes, k_scale, v_scale, fp8_out_scale); \
|
||||
} else { \
|
||||
paged_attention_custom_launcher_navi<T, KVT, KV_DTYPE, BLK_SIZE, \
|
||||
HEAD_SIZE, OUTT, PSIZE, \
|
||||
ALIBI_ENABLED, MFMA_TYPE, false>( \
|
||||
out, exp_sums, max_logits, tmp_out, query, key_cache, value_cache, \
|
||||
num_kv_heads, scale, block_tables, seq_lens, query_start_loc, \
|
||||
max_seq_len, alibi_slopes, k_scale, v_scale); \
|
||||
} \
|
||||
}
|
||||
|
||||
#define CALL_CUSTOM_LAUNCHER_ALIBI(T, KVT, KV_DTYPE, BLK_SIZE, HEAD_SIZE, \
|
||||
@@ -3799,7 +3885,8 @@ void paged_attention(
|
||||
const std::string& kv_cache_dtype, torch::Tensor& k_scale,
|
||||
torch::Tensor& v_scale,
|
||||
const std::optional<torch::Tensor>& fp8_out_scale,
|
||||
const std::string& mfma_type) {
|
||||
const std::string& mfma_type,
|
||||
bool use_interleaved_v_cache) {
|
||||
// clang-format on
|
||||
bool is_navi = is_navi_gpu();
|
||||
const int head_size = query.size(2);
|
||||
|
||||
+1
-1
@@ -50,4 +50,4 @@ void paged_attention(
|
||||
int64_t max_seq_len, const std::optional<torch::Tensor>& alibi_slopes,
|
||||
const std::string& kv_cache_dtype, torch::Tensor& k_scale,
|
||||
torch::Tensor& v_scale, const std::optional<torch::Tensor>& fp8_out_scale,
|
||||
const std::string& mfma_type);
|
||||
const std::string& mfma_type, bool use_interleaved_v_cache);
|
||||
|
||||
@@ -92,7 +92,8 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, rocm_ops) {
|
||||
" str kv_cache_dtype,"
|
||||
" Tensor k_scale, Tensor v_scale,"
|
||||
" Tensor? fp8_out_scale,"
|
||||
" str mfma_type) -> ()");
|
||||
" str mfma_type,"
|
||||
" bool use_interleaved_v_cache=False) -> ()");
|
||||
rocm_ops.impl("paged_attention", torch::kCUDA, &paged_attention);
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user