[ROCm][Bugfix] Ignore descales for unquantized AITER caches (#56726)

Signed-off-by: donglu <donglu@cohere.com>
Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
dongluw
2026-09-17 01:29:50 +00:00
committed by GitHub
co-authored by Andreas Karatzas
parent ed769b4a45
commit 19b6ff62f2
2 changed files with 85 additions and 22 deletions
+59 -7
View File
@@ -243,14 +243,19 @@ def create_and_prepopulate_kv_cache(
class MockAttentionLayer:
"""A mock attention layer for testing."""
def __init__(self, device: torch.device):
def __init__(
self,
device: torch.device,
k_scale: float = 1.0,
v_scale: float = 1.0,
):
self._q_scale = torch.tensor(1.0, device=device)
self._k_scale = torch.tensor(1.0, device=device)
self._v_scale = torch.tensor(1.0, device=device)
self._k_scale = torch.tensor(k_scale, device=device)
self._v_scale = torch.tensor(v_scale, device=device)
# Add float versions for flashinfer
self._q_scale_float = 1.0
self._k_scale_float = 1.0
self._v_scale_float = 1.0
self._k_scale_float = k_scale
self._v_scale_float = v_scale
def _clone_kv_cache_in_layout(
@@ -285,6 +290,8 @@ def run_attention_backend(
kv_cache_dtype: str = "auto",
sinks: torch.Tensor | None = None,
use_cuda_graph: bool = False,
layer_k_scale: float = 1.0,
layer_v_scale: float = 1.0,
) -> torch.Tensor:
"""Run attention computation using the specified backend's AttentionImpl."""
use_direct_block_mask = is_torch_equal_or_newer("2.9.0.dev0")
@@ -356,7 +363,7 @@ def run_attention_backend(
)
# Create mock layer and output buffer
mock_layer = MockAttentionLayer(device)
mock_layer = MockAttentionLayer(device, layer_k_scale, layer_v_scale)
output = torch.empty_like(query)
if is_quantized_kv_cache(kv_cache_dtype) and impl.supports_quant_query_input:
@@ -386,9 +393,11 @@ def run_attention_backend(
)
graph.replay()
else:
output = impl.forward(
backend_output = impl.forward(
mock_layer, query, key, value, kv_cache, attn_metadata, output=output
)
if backend_output is not None:
output = backend_output
return output
@@ -413,6 +422,8 @@ def _test_backend_correctness(
model_dtype: torch.dtype | None = None,
max_num_seqs: int | None = None,
max_num_batched_tokens: int | None = None,
layer_k_scale: float = 1.0,
layer_v_scale: float = 1.0,
):
"""Test that all backends produce similar outputs to a reference implementation
using FlexAttention or an explicit attention-sink reference.
@@ -687,6 +698,8 @@ def _test_backend_correctness(
kv_cache_dtype=kv_cache_dtype,
sinks=sinks,
use_cuda_graph=use_cuda_graph,
layer_k_scale=layer_k_scale,
layer_v_scale=layer_v_scale,
)
# Check shape and dtype consistency
@@ -1317,6 +1330,45 @@ def test_sliding_window_backend_correctness(
)
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm-specific test")
def test_rocm_aiter_fa_unquantized_cache_ignores_kv_scales(
default_vllm_config,
):
"""Stale scale tensors must not affect an unquantized KV cache."""
from vllm._aiter_ops import is_aiter_found_and_supported
if not is_aiter_found_and_supported():
pytest.skip("AITER is required")
def sliding_window_mask_mod(
b: torch.Tensor,
h: torch.Tensor,
q_idx: torch.Tensor,
kv_idx: torch.Tensor,
*,
context_len: int,
sliding_window: int,
):
causal_mask = q_idx + context_len >= kv_idx
window_mask = q_idx + context_len - kv_idx < sliding_window
return causal_mask & window_mask
model = "microsoft/Phi-tiny-MoE-instruct"
batch_spec = BATCH_SPECS["small_decode"]
model_config = ModelConfig(model=model, max_model_len=max(batch_spec.seq_lens))
sliding_window = model_config.get_sliding_window()
assert sliding_window is not None
_test_backend_correctness(
batch_spec,
model,
[AttentionBackendEnum.ROCM_AITER_FA],
partial(sliding_window_mask_mod, sliding_window=sliding_window),
layer_k_scale=0.25,
layer_v_scale=0.5,
)
@pytest.mark.parametrize(
"batch_spec_name",
[
+26 -15
View File
@@ -989,6 +989,20 @@ class AiterFlashAttentionImpl(AttentionImpl):
"cross-attention."
)
def _get_kv_cache_descales(
self,
layer: AttentionLayer,
num_decodes: int,
) -> tuple[torch.Tensor | None, torch.Tensor | None]:
# AITER treats non-null descales as an instruction to dequantize.
if not is_quantized_kv_cache(self.kv_cache_dtype):
return None, None
descale_shape = (num_decodes, self.num_kv_heads)
return (
layer._k_scale.expand(descale_shape),
layer._v_scale.expand(descale_shape),
)
def extend_for_sliding_window(
self,
attn_metadata: AiterFlashAttentionMetadata,
@@ -1367,6 +1381,9 @@ class AiterFlashAttentionImpl(AttentionImpl):
or decode_max_query_len > 1
or self.sinks is not None
):
k_descale, v_descale = self._get_kv_cache_descales(
layer, num_decodes
)
assert not rocm_aiter_ops.is_shuffle_kv_cache_enabled(), (
"Shuffle KV cache layout is not supported with sliding "
"window, sinks, or speculative decoding (multi-token decode)."
@@ -1376,7 +1393,6 @@ class AiterFlashAttentionImpl(AttentionImpl):
flash_attn_with_kvcache,
)
descale_shape = (num_decodes, key_cache.shape[2])
decode_query = query[:num_decode_tokens].reshape(
num_decodes,
decode_max_query_len,
@@ -1393,8 +1409,8 @@ class AiterFlashAttentionImpl(AttentionImpl):
window_size=self.sliding_window,
softcap=self.logits_soft_cap,
q_descale=None,
k_descale=layer._k_scale.expand(descale_shape),
v_descale=layer._v_scale.expand(descale_shape),
k_descale=k_descale,
v_descale=v_descale,
page_table=attn_metadata.block_table[:num_decodes],
)
output[:num_decode_tokens].copy_(
@@ -1412,10 +1428,6 @@ class AiterFlashAttentionImpl(AttentionImpl):
unified_attention,
)
descale_shape = (
num_decodes,
key_cache.shape[2],
)
unified_attention(
q=query[:num_decode_tokens],
k=key_cache,
@@ -1434,8 +1446,8 @@ class AiterFlashAttentionImpl(AttentionImpl):
block_table=attn_metadata.block_table[:num_decodes],
softcap=self.logits_soft_cap,
q_descale=None,
k_descale=layer._k_scale.expand(descale_shape),
v_descale=layer._v_scale.expand(descale_shape),
k_descale=k_descale,
v_descale=v_descale,
sinks=self.sinks,
)
return
@@ -1449,6 +1461,9 @@ class AiterFlashAttentionImpl(AttentionImpl):
use_unified_attention = self.head_size < _MIN_HEAD_SIZE_FOR_LL4MI
if use_unified_attention:
k_descale, v_descale = self._get_kv_cache_descales(
layer, num_decodes
)
assert not rocm_aiter_ops.is_shuffle_kv_cache_enabled(), (
"unified_attention fallback with shuffle layout "
"is not supported yet."
@@ -1460,10 +1475,6 @@ class AiterFlashAttentionImpl(AttentionImpl):
decode_cu_seqlens_q = attn_metadata.query_start_loc[
: num_decodes + 1
]
descale_shape = (
num_decodes,
key_cache.shape[2],
)
unified_attention(
q=query[:num_decode_tokens],
k=key_cache,
@@ -1480,8 +1491,8 @@ class AiterFlashAttentionImpl(AttentionImpl):
block_table=attn_metadata.block_table[:num_decodes],
softcap=self.logits_soft_cap,
q_descale=None,
k_descale=layer._k_scale.expand(descale_shape),
v_descale=layer._v_scale.expand(descale_shape),
k_descale=k_descale,
v_descale=v_descale,
)
elif rocm_aiter_ops.is_shuffle_kv_cache_enabled():
_, num_heads, head_size = query.shape