[Bugfix][ROCm] Mask paged attention V cache padding (#53856)

Signed-off-by: aoshen02 <aoshen@inferact.ai>
This commit is contained in:
aoshen02
2026-09-08 14:39:23 +08:00
committed by GitHub
parent 782f36cd0c
commit 755541443f
2 changed files with 222 additions and 41 deletions
+26 -10
View File
@@ -153,18 +153,22 @@ def test_paged_attention(
if use_alibi:
alibi_slopes = torch.randn(num_query_heads, dtype=torch.float)
seq_lens = [random.randint(1, MAX_SEQ_LEN) for _ in range(num_seqs)]
seq_lens[-1] = MAX_SEQ_LEN
max_seq_len = max(seq_lens)
seq_lens = torch.tensor(seq_lens, dtype=torch.int)
seq_lens_lst = [random.randint(1, MAX_SEQ_LEN) for _ in range(num_seqs)]
seq_lens_lst[-1] = MAX_SEQ_LEN
max_seq_len = max(seq_lens_lst)
seq_lens = torch.tensor(seq_lens_lst, dtype=torch.int)
# Create the block tables.
max_num_blocks_per_seq = (max_seq_len + block_size - 1) // block_size
block_tables_lst: list[list[int]] = []
for _ in range(num_seqs):
for seq_idx, seq_len in enumerate(seq_lens_lst):
block_table = [
random.randint(0, NUM_BLOCKS - 1) for _ in range(max_num_blocks_per_seq)
random.randint(num_seqs, NUM_BLOCKS - 1)
for _ in range(max_num_blocks_per_seq)
]
# Keep the last block private to this sequence so its padding can be
# poisoned without corrupting valid tokens from another sequence.
block_table[(seq_len - 1) // block_size] = seq_idx
block_tables_lst.append(block_table)
block_tables = torch.tensor(block_tables_lst, dtype=torch.int)
@@ -183,8 +187,20 @@ def test_paged_attention(
)
key_cache, value_cache = key_caches[0], value_caches[0]
# Using default kv_scale
k_scale = v_scale = torch.tensor(1.0, dtype=torch.float32, device=device)
k_scale_value = 0.5 if kv_cache_dtype == "fp8" else 1.0
v_scale_value = 0.25 if kv_cache_dtype == "fp8" else 1.0
k_scale = torch.tensor(k_scale_value, dtype=torch.float32, device=device)
v_scale = torch.tensor(v_scale_value, dtype=torch.float32, device=device)
if kv_cache_dtype == "fp8":
padding_nan = 0x80 if current_platform.is_fp8_fnuz() else 0x7F
else:
padding_nan = torch.nan
for seq_idx, seq_len in enumerate(seq_lens_lst):
padding_start = seq_len % block_size
if padding_start:
last_block_idx = block_tables_lst[seq_idx][seq_len // block_size]
value_cache[last_block_idx, :, :, padding_start:] = padding_nan
# Call the paged attention kernel.
output = torch.empty_like(query)
@@ -256,14 +272,14 @@ def test_paged_attention(
dequantized_key_cache = torch.empty(
size=key_cache_shape, dtype=dtype, device=device
)
ops.convert_fp8(dequantized_key_cache, key_cache)
ops.convert_fp8(dequantized_key_cache, key_cache, k_scale_value)
key_cache = dequantized_key_cache
value_cache_shape = value_cache.shape
dequantized_value_cache = torch.empty(
size=value_cache_shape, dtype=dtype, device=device
)
ops.convert_fp8(dequantized_value_cache, value_cache)
ops.convert_fp8(dequantized_value_cache, value_cache, v_scale_value)
value_cache = dequantized_value_cache
ref_output = torch.empty_like(query)