[Bugfix][ROCm] Mask paged attention V cache padding (#53856)

Signed-off-by: aoshen02 <aoshen@inferact.ai>
This commit is contained in:
aoshen02
2026-09-08 14:39:23 +08:00
committed by GitHub
parent 782f36cd0c
commit 755541443f
2 changed files with 222 additions and 41 deletions
+196 -31
View File
@@ -59,6 +59,37 @@ enum class MFMAType {
Fp4 = 2,
};
// Score masking is insufficient for unwritten V padding: 0 * NaN is NaN.
template <typename cache_t, typename Vec>
__device__ __forceinline__ Vec mask_v_cache_padding(Vec values,
const int first_token_idx,
const int seq_len) {
constexpr int NUM_ELEMS = sizeof(Vec) / sizeof(cache_t);
if (first_token_idx + NUM_ELEMS <= seq_len) {
return values;
}
if (first_token_idx >= seq_len) {
return Vec{};
}
constexpr int WORD_BYTES = sizeof(uint64_t);
constexpr int NUM_WORDS = sizeof(Vec) / WORD_BYTES;
static_assert(sizeof(Vec) % WORD_BYTES == 0);
const int valid_bytes = (seq_len - first_token_idx) * sizeof(cache_t);
uint64_t* words = reinterpret_cast<uint64_t*>(&values);
#pragma unroll
for (int i = 0; i < NUM_WORDS; i++) {
const int word_valid_bytes = valid_bytes - i * WORD_BYTES;
const uint64_t word_mask =
word_valid_bytes >= WORD_BYTES
? UINT64_MAX
: (word_valid_bytes <= 0
? 0
: (uint64_t{1} << (word_valid_bytes * 8)) - 1);
words[i] &= word_mask;
}
return values;
}
#if defined(__HIP__GFX9__)
#define GCN_MFMA_INSTR1 __builtin_amdgcn_mfma_f32_16x16x4f32
@@ -82,6 +113,30 @@ typedef struct _B16x8 {
_B16x4 xy[2];
} _B16x8;
__device__ __forceinline__ _B16x8
mask_16b_v_cache_boundary(_B16x8 values, const int valid_bytes) {
// The caller guarantees 0 < valid_bytes < 16.
uint64_t* words = reinterpret_cast<uint64_t*>(&values);
if (valid_bytes <= 8) {
const uint64_t low_mask =
valid_bytes == 8 ? UINT64_MAX : (uint64_t{1} << (valid_bytes * 8)) - 1;
words[0] &= low_mask;
words[1] = 0;
} else {
const uint64_t high_mask = (uint64_t{1} << ((valid_bytes - 8) * 8)) - 1;
words[1] &= high_mask;
}
return values;
}
__device__ __forceinline__ uint2
mask_8b_v_cache_boundary(uint2 values, const int valid_bytes) {
// The caller guarantees 0 < valid_bytes < 8.
uint64_t* word = reinterpret_cast<uint64_t*>(&values);
*word &= (uint64_t{1} << (valid_bytes * 8)) - 1;
return values;
}
using _B8x8 = uint2;
using _B8x4 = int32_t; // used in builtins
using bit8_t = uint8_t;
@@ -318,7 +373,7 @@ template <typename scalar_t, typename cache_t,
vllm::Fp8KVCacheDataType KV_DTYPE, typename OUTT, int BLOCK_SIZE,
int HEAD_SIZE, int NUM_THREADS, bool ALIBI_ENABLED, int GQA_RATIO, MFMAType MFMA_TYPE>
__global__
__launch_bounds__(NUM_THREADS, 5) void paged_attention_ll4mi_QKV_mfma16_kernel(
__launch_bounds__(NUM_THREADS, 4) void paged_attention_ll4mi_QKV_mfma16_kernel(
const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
const cache_t* __restrict__ k_cache, // [num_blocks, num_kv_heads, head_size/x, block_size, x]
const cache_t* __restrict__ v_cache, // [num_blocks, num_kv_heads, head_size, block_size]
@@ -789,29 +844,72 @@ __launch_bounds__(NUM_THREADS, 5) void paged_attention_ll4mi_QKV_mfma16_kernel(
_B16x4 outelems[VHELOOP];
// Softmax V mfma
// v layout: 16he across lanes x 16 tokens per lane
for (int vhe_depth = 0; vhe_depth < VHELOOP; vhe_depth++) {
floatx4 tmp_out = {0};
for (int vtoken_depth = 0; vtoken_depth < VTLOOP; vtoken_depth++) {
if constexpr (KV_DTYPE == vllm::Fp8KVCacheDataType::kAuto) {
if constexpr (KV_DTYPE == vllm::Fp8KVCacheDataType::kAuto) {
for (int vhe_depth = 0; vhe_depth < VHELOOP; vhe_depth++) {
floatx4 tmp_out = {0};
for (int vtoken_depth = 0; vtoken_depth < VTLOOP; vtoken_depth++) {
for (int vfetch_depth = 0; vfetch_depth < VTLANELOOP; vfetch_depth++) {
if (__builtin_expect(partition_start_token_idx + T_PAR_SIZE > seq_len,
0)) {
const int vglobal_fetch_start =
partition_start_token_idx +
vtoken_depth * VTOKENS_PER_LANE * ROWS_PER_WARP +
rowid * VTOKENS_PER_LANE +
vfetch_depth * CONTIGUOUS_KV_ELEMS_16B_LOAD;
if (vglobal_fetch_start >= seq_len) {
continue;
}
if (vglobal_fetch_start + CONTIGUOUS_KV_ELEMS_16B_LOAD > seq_len) {
Vlocal[vtoken_depth][vhe_depth][vfetch_depth] =
mask_16b_v_cache_boundary(
Vlocal[vtoken_depth][vhe_depth][vfetch_depth],
(seq_len - vglobal_fetch_start) * sizeof(cache_t));
}
}
for (int i = 0; i < ELEMS8_ELEMS4_RATIO; i++) {
const int offset = rowid * VTLANELOOP * ELEMS8_ELEMS4_RATIO +
vfetch_depth * ELEMS8_ELEMS4_RATIO + i;
const int offset1 = offset % ROWS_PER_WARP;
const int offset2 = offset / ROWS_PER_WARP;
// output format is 16 qheads across 16 lanes, 16 head elems spread
// across 4 rows
tmp_out = gcn_mfma16x16x16_instr<scalar_t, 0, 0, 0>(
Vlocal[vtoken_depth][vhe_depth][vfetch_depth].xy[i],
shared_logits[vtoken_depth][offset2][lane16id][offset1],
tmp_out);
}
}
// KV cache fp8
} else {
}
outelems[vhe_depth] = from_floatx4<scalar_t>(tmp_out);
}
} else {
const int valid_vtoken_depths =
min(VTLOOP, DIVIDE_ROUND_UP(seq_len - partition_start_token_idx,
VTOKENS_PER_LANE * ROWS_PER_WARP));
for (int vhe_depth = 0; vhe_depth < VHELOOP; vhe_depth++) {
floatx4 tmp_out = {0};
for (int vtoken_depth = 0; vtoken_depth < VTLOOP; vtoken_depth++) {
for (int vfetch_depth = 0; vfetch_depth < VTLANELOOP; vfetch_depth++) {
_B16x8 Vtmp = Vlocal[vtoken_depth][vhe_depth][vfetch_depth];
if (__builtin_expect(partition_start_token_idx + T_PAR_SIZE > seq_len,
0)) {
if (vtoken_depth >= valid_vtoken_depths) {
continue;
}
if (vtoken_depth + 1 == valid_vtoken_depths) {
const int vglobal_fetch_start =
partition_start_token_idx +
vtoken_depth * VTOKENS_PER_LANE * ROWS_PER_WARP +
rowid * VTOKENS_PER_LANE +
vfetch_depth * CONTIGUOUS_KV_ELEMS_16B_LOAD;
if (vglobal_fetch_start >= seq_len) {
continue;
}
if (vglobal_fetch_start + CONTIGUOUS_KV_ELEMS_16B_LOAD >
seq_len) {
Vtmp = mask_16b_v_cache_boundary(Vtmp,
seq_len - vglobal_fetch_start);
}
}
}
// reinterpret V format as 16 elements of 8bits
_B8x16 Vtmp8x16 = *reinterpret_cast<_B8x16*>(&Vtmp);
for (int j = 0; j < ELEMS16_ELEMS8_RATIO; j++) {
@@ -856,12 +954,9 @@ __launch_bounds__(NUM_THREADS, 5) void paged_attention_ll4mi_QKV_mfma16_kernel(
}
}
}
}
// apply post Softmax V mfma v_scale
if constexpr (KV_DTYPE != vllm::Fp8KVCacheDataType::kAuto) {
tmp_out *= *v_scale;
outelems[vhe_depth] = from_floatx4<scalar_t>(tmp_out);
}
outelems[vhe_depth] = from_floatx4<scalar_t>(tmp_out);
}
__syncthreads();
@@ -1334,15 +1429,37 @@ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_mfma4_kernel(
}
}
} else { // warp in context
#define SV_mfma_value(x, value) \
for (int qh = 0; qh < QHLOOP; qh++) { \
acc[qh] = gcn_mfma4x4x4_instr<scalar_t, 4, 2 * x, 0>( \
logits[qh], value.xy[0], acc[qh]); \
acc[qh] = gcn_mfma4x4x4_instr<scalar_t, 4, 2 * x + 1, 0>( \
logits[qh], value.xy[1], acc[qh]); \
}
#define SV_mfma(x) \
if constexpr (KV_DTYPE != vllm::Fp8KVCacheDataType::kAuto) { \
Vlocal[vh][x] = convert_b8x8_custom<scalar_t>(Vlocalb8[vh][x]); \
} \
for (int qh = 0; qh < QHLOOP; qh++) { \
acc[qh] = gcn_mfma4x4x4_instr<scalar_t, 4, 2 * x, 0>( \
logits[qh], Vlocal[vh][x].xy[0], acc[qh]); \
acc[qh] = gcn_mfma4x4x4_instr<scalar_t, 4, 2 * x + 1, 0>( \
logits[qh], Vlocal[vh][x].xy[1], acc[qh]); \
SV_mfma_value(x, Vlocal[vh][x])
#define SV_mfma_tail(x) \
if (x * 8 < valid_warp_tokens) { \
_B16x8 Vtmp; \
if constexpr (KV_DTYPE == vllm::Fp8KVCacheDataType::kAuto) { \
Vtmp = Vlocal[vh][x]; \
if ((x + 1) * 8 > valid_warp_tokens) { \
Vtmp = mask_16b_v_cache_boundary( \
Vtmp, (valid_warp_tokens - x * 8) * sizeof(cache_t)); \
} \
} else { \
_B8x8 Vtmp8 = Vlocalb8[vh][x]; \
if ((x + 1) * 8 > valid_warp_tokens) { \
Vtmp8 = mask_8b_v_cache_boundary(Vtmp8, valid_warp_tokens - x * 8); \
} \
Vtmp = convert_b8x8_custom<scalar_t>(Vtmp8); \
} \
SV_mfma_value(x, Vtmp) \
}
for (int vh = 0; vh < VHELOOP; vh++) {
@@ -1350,18 +1467,28 @@ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_mfma4_kernel(
for (int qh = 0; qh < QHLOOP; qh++) {
acc[qh] = {0};
}
// SoftMax-V calculation
// logits -> token dimension is distributed across lanes
// Vlocal -> token dimension is depthwise within lane
// uses mfma instruction block broadcast for logits
SV_mfma(0);
SV_mfma(1);
SV_mfma(2);
SV_mfma(3);
SV_mfma(4);
SV_mfma(5);
SV_mfma(6);
SV_mfma(7);
// SoftMax-V calculation. Skip fully invalid V vectors in the partial
// tail warp, and only mask the one vector crossing seq_len.
if (__builtin_expect(warp_start_token_idx + WARP_SIZE > seq_len, 0)) {
const int valid_warp_tokens = seq_len - warp_start_token_idx;
SV_mfma_tail(0);
SV_mfma_tail(1);
SV_mfma_tail(2);
SV_mfma_tail(3);
SV_mfma_tail(4);
SV_mfma_tail(5);
SV_mfma_tail(6);
SV_mfma_tail(7);
} else {
SV_mfma(0);
SV_mfma(1);
SV_mfma(2);
SV_mfma(3);
SV_mfma(4);
SV_mfma(5);
SV_mfma(6);
SV_mfma(7);
}
for (int qh = 0; qh < QHLOOP; qh++) {
if constexpr (KV_DTYPE != vllm::Fp8KVCacheDataType::kAuto) {
@@ -1373,6 +1500,8 @@ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_mfma4_kernel(
}
#undef SV_mfma
#undef SV_mfma_tail
#undef SV_mfma_value
} // warp in context
__syncthreads();
@@ -1963,6 +2092,24 @@ __launch_bounds__(NUM_THREADS, 3) void paged_attention_ll4mi_QKV_mfma16_kernel(
}
}
if (partition_start_token_idx + T_PAR_SIZE > seq_len) {
for (int vhe_depth = 0; vhe_depth < VHELOOP; vhe_depth++) {
for (int vtoken_depth = 0; vtoken_depth < VTLOOP; vtoken_depth++) {
for (int vfetch_depth = 0; vfetch_depth < VTLANELOOP; vfetch_depth++) {
const int vglobal_fetch_start =
partition_start_token_idx +
vtoken_depth * VTOKENS_PER_LANE * ROWS_PER_WARP +
vfetch_depth * CONTIGUOUS_KV_ELEMS_16B_LOAD;
Vlocal[vtoken_depth][vhe_depth][vfetch_depth / 2]
.xy[vfetch_depth % 2] = mask_v_cache_padding<cache_t>(
Vlocal[vtoken_depth][vhe_depth][vfetch_depth / 2]
.xy[vfetch_depth % 2],
vglobal_fetch_start, seq_len);
}
}
}
}
floatx8 dout[TLOOP];
// qk wmma
for (int token_depth = 0; token_depth < TLOOP; token_depth++) {
@@ -2741,6 +2888,24 @@ __launch_bounds__(NUM_THREADS, 3) void paged_attention_ll4mi_QKV_mfma16_kernel(
}
}
if (partition_start_token_idx + T_PAR_SIZE > seq_len) {
for (int vhe_depth = 0; vhe_depth < VHELOOP; vhe_depth++) {
for (int vtoken_depth = 0; vtoken_depth < VTLOOP; vtoken_depth++) {
for (int vfetch_depth = 0; vfetch_depth < VTLANELOOP; vfetch_depth++) {
const int vglobal_fetch_start =
partition_start_token_idx +
vtoken_depth * VTOKENS_PER_LANE * ROWS_PER_WARP +
rowid * VTOKENS_PER_LANE +
vfetch_depth * CONTIGUOUS_KV_ELEMS_16B_LOAD;
Vlocal[vtoken_depth][vhe_depth][vfetch_depth] =
mask_v_cache_padding<cache_t>(
Vlocal[vtoken_depth][vhe_depth][vfetch_depth],
vglobal_fetch_start, seq_len);
}
}
}
}
floatx8 dout[TLOOP];
// qk wmma
for (int token_depth = 0; token_depth < TLOOP; token_depth++) {
+26 -10
View File
@@ -153,18 +153,22 @@ def test_paged_attention(
if use_alibi:
alibi_slopes = torch.randn(num_query_heads, dtype=torch.float)
seq_lens = [random.randint(1, MAX_SEQ_LEN) for _ in range(num_seqs)]
seq_lens[-1] = MAX_SEQ_LEN
max_seq_len = max(seq_lens)
seq_lens = torch.tensor(seq_lens, dtype=torch.int)
seq_lens_lst = [random.randint(1, MAX_SEQ_LEN) for _ in range(num_seqs)]
seq_lens_lst[-1] = MAX_SEQ_LEN
max_seq_len = max(seq_lens_lst)
seq_lens = torch.tensor(seq_lens_lst, dtype=torch.int)
# Create the block tables.
max_num_blocks_per_seq = (max_seq_len + block_size - 1) // block_size
block_tables_lst: list[list[int]] = []
for _ in range(num_seqs):
for seq_idx, seq_len in enumerate(seq_lens_lst):
block_table = [
random.randint(0, NUM_BLOCKS - 1) for _ in range(max_num_blocks_per_seq)
random.randint(num_seqs, NUM_BLOCKS - 1)
for _ in range(max_num_blocks_per_seq)
]
# Keep the last block private to this sequence so its padding can be
# poisoned without corrupting valid tokens from another sequence.
block_table[(seq_len - 1) // block_size] = seq_idx
block_tables_lst.append(block_table)
block_tables = torch.tensor(block_tables_lst, dtype=torch.int)
@@ -183,8 +187,20 @@ def test_paged_attention(
)
key_cache, value_cache = key_caches[0], value_caches[0]
# Using default kv_scale
k_scale = v_scale = torch.tensor(1.0, dtype=torch.float32, device=device)
k_scale_value = 0.5 if kv_cache_dtype == "fp8" else 1.0
v_scale_value = 0.25 if kv_cache_dtype == "fp8" else 1.0
k_scale = torch.tensor(k_scale_value, dtype=torch.float32, device=device)
v_scale = torch.tensor(v_scale_value, dtype=torch.float32, device=device)
if kv_cache_dtype == "fp8":
padding_nan = 0x80 if current_platform.is_fp8_fnuz() else 0x7F
else:
padding_nan = torch.nan
for seq_idx, seq_len in enumerate(seq_lens_lst):
padding_start = seq_len % block_size
if padding_start:
last_block_idx = block_tables_lst[seq_idx][seq_len // block_size]
value_cache[last_block_idx, :, :, padding_start:] = padding_nan
# Call the paged attention kernel.
output = torch.empty_like(query)
@@ -256,14 +272,14 @@ def test_paged_attention(
dequantized_key_cache = torch.empty(
size=key_cache_shape, dtype=dtype, device=device
)
ops.convert_fp8(dequantized_key_cache, key_cache)
ops.convert_fp8(dequantized_key_cache, key_cache, k_scale_value)
key_cache = dequantized_key_cache
value_cache_shape = value_cache.shape
dequantized_value_cache = torch.empty(
size=value_cache_shape, dtype=dtype, device=device
)
ops.convert_fp8(dequantized_value_cache, value_cache)
ops.convert_fp8(dequantized_value_cache, value_cache, v_scale_value)
value_cache = dequantized_value_cache
ref_output = torch.empty_like(query)