[Bugfix][ROCm] Mask paged attention V cache padding (#53856)
Signed-off-by: aoshen02 <aoshen@inferact.ai>
This commit is contained in:
+196
-31
@@ -59,6 +59,37 @@ enum class MFMAType {
|
||||
Fp4 = 2,
|
||||
};
|
||||
|
||||
// Score masking is insufficient for unwritten V padding: 0 * NaN is NaN.
|
||||
template <typename cache_t, typename Vec>
|
||||
__device__ __forceinline__ Vec mask_v_cache_padding(Vec values,
|
||||
const int first_token_idx,
|
||||
const int seq_len) {
|
||||
constexpr int NUM_ELEMS = sizeof(Vec) / sizeof(cache_t);
|
||||
if (first_token_idx + NUM_ELEMS <= seq_len) {
|
||||
return values;
|
||||
}
|
||||
if (first_token_idx >= seq_len) {
|
||||
return Vec{};
|
||||
}
|
||||
constexpr int WORD_BYTES = sizeof(uint64_t);
|
||||
constexpr int NUM_WORDS = sizeof(Vec) / WORD_BYTES;
|
||||
static_assert(sizeof(Vec) % WORD_BYTES == 0);
|
||||
const int valid_bytes = (seq_len - first_token_idx) * sizeof(cache_t);
|
||||
uint64_t* words = reinterpret_cast<uint64_t*>(&values);
|
||||
#pragma unroll
|
||||
for (int i = 0; i < NUM_WORDS; i++) {
|
||||
const int word_valid_bytes = valid_bytes - i * WORD_BYTES;
|
||||
const uint64_t word_mask =
|
||||
word_valid_bytes >= WORD_BYTES
|
||||
? UINT64_MAX
|
||||
: (word_valid_bytes <= 0
|
||||
? 0
|
||||
: (uint64_t{1} << (word_valid_bytes * 8)) - 1);
|
||||
words[i] &= word_mask;
|
||||
}
|
||||
return values;
|
||||
}
|
||||
|
||||
#if defined(__HIP__GFX9__)
|
||||
|
||||
#define GCN_MFMA_INSTR1 __builtin_amdgcn_mfma_f32_16x16x4f32
|
||||
@@ -82,6 +113,30 @@ typedef struct _B16x8 {
|
||||
_B16x4 xy[2];
|
||||
} _B16x8;
|
||||
|
||||
__device__ __forceinline__ _B16x8
|
||||
mask_16b_v_cache_boundary(_B16x8 values, const int valid_bytes) {
|
||||
// The caller guarantees 0 < valid_bytes < 16.
|
||||
uint64_t* words = reinterpret_cast<uint64_t*>(&values);
|
||||
if (valid_bytes <= 8) {
|
||||
const uint64_t low_mask =
|
||||
valid_bytes == 8 ? UINT64_MAX : (uint64_t{1} << (valid_bytes * 8)) - 1;
|
||||
words[0] &= low_mask;
|
||||
words[1] = 0;
|
||||
} else {
|
||||
const uint64_t high_mask = (uint64_t{1} << ((valid_bytes - 8) * 8)) - 1;
|
||||
words[1] &= high_mask;
|
||||
}
|
||||
return values;
|
||||
}
|
||||
|
||||
__device__ __forceinline__ uint2
|
||||
mask_8b_v_cache_boundary(uint2 values, const int valid_bytes) {
|
||||
// The caller guarantees 0 < valid_bytes < 8.
|
||||
uint64_t* word = reinterpret_cast<uint64_t*>(&values);
|
||||
*word &= (uint64_t{1} << (valid_bytes * 8)) - 1;
|
||||
return values;
|
||||
}
|
||||
|
||||
using _B8x8 = uint2;
|
||||
using _B8x4 = int32_t; // used in builtins
|
||||
using bit8_t = uint8_t;
|
||||
@@ -318,7 +373,7 @@ template <typename scalar_t, typename cache_t,
|
||||
vllm::Fp8KVCacheDataType KV_DTYPE, typename OUTT, int BLOCK_SIZE,
|
||||
int HEAD_SIZE, int NUM_THREADS, bool ALIBI_ENABLED, int GQA_RATIO, MFMAType MFMA_TYPE>
|
||||
__global__
|
||||
__launch_bounds__(NUM_THREADS, 5) void paged_attention_ll4mi_QKV_mfma16_kernel(
|
||||
__launch_bounds__(NUM_THREADS, 4) void paged_attention_ll4mi_QKV_mfma16_kernel(
|
||||
const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
|
||||
const cache_t* __restrict__ k_cache, // [num_blocks, num_kv_heads, head_size/x, block_size, x]
|
||||
const cache_t* __restrict__ v_cache, // [num_blocks, num_kv_heads, head_size, block_size]
|
||||
@@ -789,29 +844,72 @@ __launch_bounds__(NUM_THREADS, 5) void paged_attention_ll4mi_QKV_mfma16_kernel(
|
||||
_B16x4 outelems[VHELOOP];
|
||||
// Softmax V mfma
|
||||
// v layout: 16he across lanes x 16 tokens per lane
|
||||
for (int vhe_depth = 0; vhe_depth < VHELOOP; vhe_depth++) {
|
||||
floatx4 tmp_out = {0};
|
||||
|
||||
for (int vtoken_depth = 0; vtoken_depth < VTLOOP; vtoken_depth++) {
|
||||
if constexpr (KV_DTYPE == vllm::Fp8KVCacheDataType::kAuto) {
|
||||
if constexpr (KV_DTYPE == vllm::Fp8KVCacheDataType::kAuto) {
|
||||
for (int vhe_depth = 0; vhe_depth < VHELOOP; vhe_depth++) {
|
||||
floatx4 tmp_out = {0};
|
||||
for (int vtoken_depth = 0; vtoken_depth < VTLOOP; vtoken_depth++) {
|
||||
for (int vfetch_depth = 0; vfetch_depth < VTLANELOOP; vfetch_depth++) {
|
||||
if (__builtin_expect(partition_start_token_idx + T_PAR_SIZE > seq_len,
|
||||
0)) {
|
||||
const int vglobal_fetch_start =
|
||||
partition_start_token_idx +
|
||||
vtoken_depth * VTOKENS_PER_LANE * ROWS_PER_WARP +
|
||||
rowid * VTOKENS_PER_LANE +
|
||||
vfetch_depth * CONTIGUOUS_KV_ELEMS_16B_LOAD;
|
||||
if (vglobal_fetch_start >= seq_len) {
|
||||
continue;
|
||||
}
|
||||
if (vglobal_fetch_start + CONTIGUOUS_KV_ELEMS_16B_LOAD > seq_len) {
|
||||
Vlocal[vtoken_depth][vhe_depth][vfetch_depth] =
|
||||
mask_16b_v_cache_boundary(
|
||||
Vlocal[vtoken_depth][vhe_depth][vfetch_depth],
|
||||
(seq_len - vglobal_fetch_start) * sizeof(cache_t));
|
||||
}
|
||||
}
|
||||
for (int i = 0; i < ELEMS8_ELEMS4_RATIO; i++) {
|
||||
const int offset = rowid * VTLANELOOP * ELEMS8_ELEMS4_RATIO +
|
||||
vfetch_depth * ELEMS8_ELEMS4_RATIO + i;
|
||||
const int offset1 = offset % ROWS_PER_WARP;
|
||||
const int offset2 = offset / ROWS_PER_WARP;
|
||||
// output format is 16 qheads across 16 lanes, 16 head elems spread
|
||||
// across 4 rows
|
||||
tmp_out = gcn_mfma16x16x16_instr<scalar_t, 0, 0, 0>(
|
||||
Vlocal[vtoken_depth][vhe_depth][vfetch_depth].xy[i],
|
||||
shared_logits[vtoken_depth][offset2][lane16id][offset1],
|
||||
tmp_out);
|
||||
}
|
||||
}
|
||||
// KV cache fp8
|
||||
} else {
|
||||
}
|
||||
outelems[vhe_depth] = from_floatx4<scalar_t>(tmp_out);
|
||||
}
|
||||
} else {
|
||||
const int valid_vtoken_depths =
|
||||
min(VTLOOP, DIVIDE_ROUND_UP(seq_len - partition_start_token_idx,
|
||||
VTOKENS_PER_LANE * ROWS_PER_WARP));
|
||||
for (int vhe_depth = 0; vhe_depth < VHELOOP; vhe_depth++) {
|
||||
floatx4 tmp_out = {0};
|
||||
for (int vtoken_depth = 0; vtoken_depth < VTLOOP; vtoken_depth++) {
|
||||
for (int vfetch_depth = 0; vfetch_depth < VTLANELOOP; vfetch_depth++) {
|
||||
_B16x8 Vtmp = Vlocal[vtoken_depth][vhe_depth][vfetch_depth];
|
||||
if (__builtin_expect(partition_start_token_idx + T_PAR_SIZE > seq_len,
|
||||
0)) {
|
||||
if (vtoken_depth >= valid_vtoken_depths) {
|
||||
continue;
|
||||
}
|
||||
if (vtoken_depth + 1 == valid_vtoken_depths) {
|
||||
const int vglobal_fetch_start =
|
||||
partition_start_token_idx +
|
||||
vtoken_depth * VTOKENS_PER_LANE * ROWS_PER_WARP +
|
||||
rowid * VTOKENS_PER_LANE +
|
||||
vfetch_depth * CONTIGUOUS_KV_ELEMS_16B_LOAD;
|
||||
if (vglobal_fetch_start >= seq_len) {
|
||||
continue;
|
||||
}
|
||||
if (vglobal_fetch_start + CONTIGUOUS_KV_ELEMS_16B_LOAD >
|
||||
seq_len) {
|
||||
Vtmp = mask_16b_v_cache_boundary(Vtmp,
|
||||
seq_len - vglobal_fetch_start);
|
||||
}
|
||||
}
|
||||
}
|
||||
// reinterpret V format as 16 elements of 8bits
|
||||
_B8x16 Vtmp8x16 = *reinterpret_cast<_B8x16*>(&Vtmp);
|
||||
for (int j = 0; j < ELEMS16_ELEMS8_RATIO; j++) {
|
||||
@@ -856,12 +954,9 @@ __launch_bounds__(NUM_THREADS, 5) void paged_attention_ll4mi_QKV_mfma16_kernel(
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// apply post Softmax V mfma v_scale
|
||||
if constexpr (KV_DTYPE != vllm::Fp8KVCacheDataType::kAuto) {
|
||||
tmp_out *= *v_scale;
|
||||
outelems[vhe_depth] = from_floatx4<scalar_t>(tmp_out);
|
||||
}
|
||||
outelems[vhe_depth] = from_floatx4<scalar_t>(tmp_out);
|
||||
}
|
||||
|
||||
__syncthreads();
|
||||
@@ -1334,15 +1429,37 @@ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_mfma4_kernel(
|
||||
}
|
||||
}
|
||||
} else { // warp in context
|
||||
#define SV_mfma_value(x, value) \
|
||||
for (int qh = 0; qh < QHLOOP; qh++) { \
|
||||
acc[qh] = gcn_mfma4x4x4_instr<scalar_t, 4, 2 * x, 0>( \
|
||||
logits[qh], value.xy[0], acc[qh]); \
|
||||
acc[qh] = gcn_mfma4x4x4_instr<scalar_t, 4, 2 * x + 1, 0>( \
|
||||
logits[qh], value.xy[1], acc[qh]); \
|
||||
}
|
||||
|
||||
#define SV_mfma(x) \
|
||||
if constexpr (KV_DTYPE != vllm::Fp8KVCacheDataType::kAuto) { \
|
||||
Vlocal[vh][x] = convert_b8x8_custom<scalar_t>(Vlocalb8[vh][x]); \
|
||||
} \
|
||||
for (int qh = 0; qh < QHLOOP; qh++) { \
|
||||
acc[qh] = gcn_mfma4x4x4_instr<scalar_t, 4, 2 * x, 0>( \
|
||||
logits[qh], Vlocal[vh][x].xy[0], acc[qh]); \
|
||||
acc[qh] = gcn_mfma4x4x4_instr<scalar_t, 4, 2 * x + 1, 0>( \
|
||||
logits[qh], Vlocal[vh][x].xy[1], acc[qh]); \
|
||||
SV_mfma_value(x, Vlocal[vh][x])
|
||||
|
||||
#define SV_mfma_tail(x) \
|
||||
if (x * 8 < valid_warp_tokens) { \
|
||||
_B16x8 Vtmp; \
|
||||
if constexpr (KV_DTYPE == vllm::Fp8KVCacheDataType::kAuto) { \
|
||||
Vtmp = Vlocal[vh][x]; \
|
||||
if ((x + 1) * 8 > valid_warp_tokens) { \
|
||||
Vtmp = mask_16b_v_cache_boundary( \
|
||||
Vtmp, (valid_warp_tokens - x * 8) * sizeof(cache_t)); \
|
||||
} \
|
||||
} else { \
|
||||
_B8x8 Vtmp8 = Vlocalb8[vh][x]; \
|
||||
if ((x + 1) * 8 > valid_warp_tokens) { \
|
||||
Vtmp8 = mask_8b_v_cache_boundary(Vtmp8, valid_warp_tokens - x * 8); \
|
||||
} \
|
||||
Vtmp = convert_b8x8_custom<scalar_t>(Vtmp8); \
|
||||
} \
|
||||
SV_mfma_value(x, Vtmp) \
|
||||
}
|
||||
|
||||
for (int vh = 0; vh < VHELOOP; vh++) {
|
||||
@@ -1350,18 +1467,28 @@ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_mfma4_kernel(
|
||||
for (int qh = 0; qh < QHLOOP; qh++) {
|
||||
acc[qh] = {0};
|
||||
}
|
||||
// SoftMax-V calculation
|
||||
// logits -> token dimension is distributed across lanes
|
||||
// Vlocal -> token dimension is depthwise within lane
|
||||
// uses mfma instruction block broadcast for logits
|
||||
SV_mfma(0);
|
||||
SV_mfma(1);
|
||||
SV_mfma(2);
|
||||
SV_mfma(3);
|
||||
SV_mfma(4);
|
||||
SV_mfma(5);
|
||||
SV_mfma(6);
|
||||
SV_mfma(7);
|
||||
// SoftMax-V calculation. Skip fully invalid V vectors in the partial
|
||||
// tail warp, and only mask the one vector crossing seq_len.
|
||||
if (__builtin_expect(warp_start_token_idx + WARP_SIZE > seq_len, 0)) {
|
||||
const int valid_warp_tokens = seq_len - warp_start_token_idx;
|
||||
SV_mfma_tail(0);
|
||||
SV_mfma_tail(1);
|
||||
SV_mfma_tail(2);
|
||||
SV_mfma_tail(3);
|
||||
SV_mfma_tail(4);
|
||||
SV_mfma_tail(5);
|
||||
SV_mfma_tail(6);
|
||||
SV_mfma_tail(7);
|
||||
} else {
|
||||
SV_mfma(0);
|
||||
SV_mfma(1);
|
||||
SV_mfma(2);
|
||||
SV_mfma(3);
|
||||
SV_mfma(4);
|
||||
SV_mfma(5);
|
||||
SV_mfma(6);
|
||||
SV_mfma(7);
|
||||
}
|
||||
|
||||
for (int qh = 0; qh < QHLOOP; qh++) {
|
||||
if constexpr (KV_DTYPE != vllm::Fp8KVCacheDataType::kAuto) {
|
||||
@@ -1373,6 +1500,8 @@ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_mfma4_kernel(
|
||||
}
|
||||
|
||||
#undef SV_mfma
|
||||
#undef SV_mfma_tail
|
||||
#undef SV_mfma_value
|
||||
} // warp in context
|
||||
|
||||
__syncthreads();
|
||||
@@ -1963,6 +2092,24 @@ __launch_bounds__(NUM_THREADS, 3) void paged_attention_ll4mi_QKV_mfma16_kernel(
|
||||
}
|
||||
}
|
||||
|
||||
if (partition_start_token_idx + T_PAR_SIZE > seq_len) {
|
||||
for (int vhe_depth = 0; vhe_depth < VHELOOP; vhe_depth++) {
|
||||
for (int vtoken_depth = 0; vtoken_depth < VTLOOP; vtoken_depth++) {
|
||||
for (int vfetch_depth = 0; vfetch_depth < VTLANELOOP; vfetch_depth++) {
|
||||
const int vglobal_fetch_start =
|
||||
partition_start_token_idx +
|
||||
vtoken_depth * VTOKENS_PER_LANE * ROWS_PER_WARP +
|
||||
vfetch_depth * CONTIGUOUS_KV_ELEMS_16B_LOAD;
|
||||
Vlocal[vtoken_depth][vhe_depth][vfetch_depth / 2]
|
||||
.xy[vfetch_depth % 2] = mask_v_cache_padding<cache_t>(
|
||||
Vlocal[vtoken_depth][vhe_depth][vfetch_depth / 2]
|
||||
.xy[vfetch_depth % 2],
|
||||
vglobal_fetch_start, seq_len);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
floatx8 dout[TLOOP];
|
||||
// qk wmma
|
||||
for (int token_depth = 0; token_depth < TLOOP; token_depth++) {
|
||||
@@ -2741,6 +2888,24 @@ __launch_bounds__(NUM_THREADS, 3) void paged_attention_ll4mi_QKV_mfma16_kernel(
|
||||
}
|
||||
}
|
||||
|
||||
if (partition_start_token_idx + T_PAR_SIZE > seq_len) {
|
||||
for (int vhe_depth = 0; vhe_depth < VHELOOP; vhe_depth++) {
|
||||
for (int vtoken_depth = 0; vtoken_depth < VTLOOP; vtoken_depth++) {
|
||||
for (int vfetch_depth = 0; vfetch_depth < VTLANELOOP; vfetch_depth++) {
|
||||
const int vglobal_fetch_start =
|
||||
partition_start_token_idx +
|
||||
vtoken_depth * VTOKENS_PER_LANE * ROWS_PER_WARP +
|
||||
rowid * VTOKENS_PER_LANE +
|
||||
vfetch_depth * CONTIGUOUS_KV_ELEMS_16B_LOAD;
|
||||
Vlocal[vtoken_depth][vhe_depth][vfetch_depth] =
|
||||
mask_v_cache_padding<cache_t>(
|
||||
Vlocal[vtoken_depth][vhe_depth][vfetch_depth],
|
||||
vglobal_fetch_start, seq_len);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
floatx8 dout[TLOOP];
|
||||
// qk wmma
|
||||
for (int token_depth = 0; token_depth < TLOOP; token_depth++) {
|
||||
|
||||
@@ -153,18 +153,22 @@ def test_paged_attention(
|
||||
if use_alibi:
|
||||
alibi_slopes = torch.randn(num_query_heads, dtype=torch.float)
|
||||
|
||||
seq_lens = [random.randint(1, MAX_SEQ_LEN) for _ in range(num_seqs)]
|
||||
seq_lens[-1] = MAX_SEQ_LEN
|
||||
max_seq_len = max(seq_lens)
|
||||
seq_lens = torch.tensor(seq_lens, dtype=torch.int)
|
||||
seq_lens_lst = [random.randint(1, MAX_SEQ_LEN) for _ in range(num_seqs)]
|
||||
seq_lens_lst[-1] = MAX_SEQ_LEN
|
||||
max_seq_len = max(seq_lens_lst)
|
||||
seq_lens = torch.tensor(seq_lens_lst, dtype=torch.int)
|
||||
|
||||
# Create the block tables.
|
||||
max_num_blocks_per_seq = (max_seq_len + block_size - 1) // block_size
|
||||
block_tables_lst: list[list[int]] = []
|
||||
for _ in range(num_seqs):
|
||||
for seq_idx, seq_len in enumerate(seq_lens_lst):
|
||||
block_table = [
|
||||
random.randint(0, NUM_BLOCKS - 1) for _ in range(max_num_blocks_per_seq)
|
||||
random.randint(num_seqs, NUM_BLOCKS - 1)
|
||||
for _ in range(max_num_blocks_per_seq)
|
||||
]
|
||||
# Keep the last block private to this sequence so its padding can be
|
||||
# poisoned without corrupting valid tokens from another sequence.
|
||||
block_table[(seq_len - 1) // block_size] = seq_idx
|
||||
block_tables_lst.append(block_table)
|
||||
|
||||
block_tables = torch.tensor(block_tables_lst, dtype=torch.int)
|
||||
@@ -183,8 +187,20 @@ def test_paged_attention(
|
||||
)
|
||||
key_cache, value_cache = key_caches[0], value_caches[0]
|
||||
|
||||
# Using default kv_scale
|
||||
k_scale = v_scale = torch.tensor(1.0, dtype=torch.float32, device=device)
|
||||
k_scale_value = 0.5 if kv_cache_dtype == "fp8" else 1.0
|
||||
v_scale_value = 0.25 if kv_cache_dtype == "fp8" else 1.0
|
||||
k_scale = torch.tensor(k_scale_value, dtype=torch.float32, device=device)
|
||||
v_scale = torch.tensor(v_scale_value, dtype=torch.float32, device=device)
|
||||
|
||||
if kv_cache_dtype == "fp8":
|
||||
padding_nan = 0x80 if current_platform.is_fp8_fnuz() else 0x7F
|
||||
else:
|
||||
padding_nan = torch.nan
|
||||
for seq_idx, seq_len in enumerate(seq_lens_lst):
|
||||
padding_start = seq_len % block_size
|
||||
if padding_start:
|
||||
last_block_idx = block_tables_lst[seq_idx][seq_len // block_size]
|
||||
value_cache[last_block_idx, :, :, padding_start:] = padding_nan
|
||||
|
||||
# Call the paged attention kernel.
|
||||
output = torch.empty_like(query)
|
||||
@@ -256,14 +272,14 @@ def test_paged_attention(
|
||||
dequantized_key_cache = torch.empty(
|
||||
size=key_cache_shape, dtype=dtype, device=device
|
||||
)
|
||||
ops.convert_fp8(dequantized_key_cache, key_cache)
|
||||
ops.convert_fp8(dequantized_key_cache, key_cache, k_scale_value)
|
||||
key_cache = dequantized_key_cache
|
||||
|
||||
value_cache_shape = value_cache.shape
|
||||
dequantized_value_cache = torch.empty(
|
||||
size=value_cache_shape, dtype=dtype, device=device
|
||||
)
|
||||
ops.convert_fp8(dequantized_value_cache, value_cache)
|
||||
ops.convert_fp8(dequantized_value_cache, value_cache, v_scale_value)
|
||||
value_cache = dequantized_value_cache
|
||||
|
||||
ref_output = torch.empty_like(query)
|
||||
|
||||
Reference in New Issue
Block a user