diff --git a/tests/v1/attention/test_attention_backends.py b/tests/v1/attention/test_attention_backends.py index 66a1c5c088..b02c85f682 100644 --- a/tests/v1/attention/test_attention_backends.py +++ b/tests/v1/attention/test_attention_backends.py @@ -243,14 +243,19 @@ def create_and_prepopulate_kv_cache( class MockAttentionLayer: """A mock attention layer for testing.""" - def __init__(self, device: torch.device): + def __init__( + self, + device: torch.device, + k_scale: float = 1.0, + v_scale: float = 1.0, + ): self._q_scale = torch.tensor(1.0, device=device) - self._k_scale = torch.tensor(1.0, device=device) - self._v_scale = torch.tensor(1.0, device=device) + self._k_scale = torch.tensor(k_scale, device=device) + self._v_scale = torch.tensor(v_scale, device=device) # Add float versions for flashinfer self._q_scale_float = 1.0 - self._k_scale_float = 1.0 - self._v_scale_float = 1.0 + self._k_scale_float = k_scale + self._v_scale_float = v_scale def _clone_kv_cache_in_layout( @@ -285,6 +290,8 @@ def run_attention_backend( kv_cache_dtype: str = "auto", sinks: torch.Tensor | None = None, use_cuda_graph: bool = False, + layer_k_scale: float = 1.0, + layer_v_scale: float = 1.0, ) -> torch.Tensor: """Run attention computation using the specified backend's AttentionImpl.""" use_direct_block_mask = is_torch_equal_or_newer("2.9.0.dev0") @@ -356,7 +363,7 @@ def run_attention_backend( ) # Create mock layer and output buffer - mock_layer = MockAttentionLayer(device) + mock_layer = MockAttentionLayer(device, layer_k_scale, layer_v_scale) output = torch.empty_like(query) if is_quantized_kv_cache(kv_cache_dtype) and impl.supports_quant_query_input: @@ -386,9 +393,11 @@ def run_attention_backend( ) graph.replay() else: - output = impl.forward( + backend_output = impl.forward( mock_layer, query, key, value, kv_cache, attn_metadata, output=output ) + if backend_output is not None: + output = backend_output return output @@ -413,6 +422,8 @@ def _test_backend_correctness( model_dtype: torch.dtype | None = None, max_num_seqs: int | None = None, max_num_batched_tokens: int | None = None, + layer_k_scale: float = 1.0, + layer_v_scale: float = 1.0, ): """Test that all backends produce similar outputs to a reference implementation using FlexAttention or an explicit attention-sink reference. @@ -687,6 +698,8 @@ def _test_backend_correctness( kv_cache_dtype=kv_cache_dtype, sinks=sinks, use_cuda_graph=use_cuda_graph, + layer_k_scale=layer_k_scale, + layer_v_scale=layer_v_scale, ) # Check shape and dtype consistency @@ -1317,6 +1330,45 @@ def test_sliding_window_backend_correctness( ) +@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm-specific test") +def test_rocm_aiter_fa_unquantized_cache_ignores_kv_scales( + default_vllm_config, +): + """Stale scale tensors must not affect an unquantized KV cache.""" + from vllm._aiter_ops import is_aiter_found_and_supported + + if not is_aiter_found_and_supported(): + pytest.skip("AITER is required") + + def sliding_window_mask_mod( + b: torch.Tensor, + h: torch.Tensor, + q_idx: torch.Tensor, + kv_idx: torch.Tensor, + *, + context_len: int, + sliding_window: int, + ): + causal_mask = q_idx + context_len >= kv_idx + window_mask = q_idx + context_len - kv_idx < sliding_window + return causal_mask & window_mask + + model = "microsoft/Phi-tiny-MoE-instruct" + batch_spec = BATCH_SPECS["small_decode"] + model_config = ModelConfig(model=model, max_model_len=max(batch_spec.seq_lens)) + sliding_window = model_config.get_sliding_window() + assert sliding_window is not None + + _test_backend_correctness( + batch_spec, + model, + [AttentionBackendEnum.ROCM_AITER_FA], + partial(sliding_window_mask_mod, sliding_window=sliding_window), + layer_k_scale=0.25, + layer_v_scale=0.5, + ) + + @pytest.mark.parametrize( "batch_spec_name", [ diff --git a/vllm/v1/attention/backends/rocm_aiter_fa.py b/vllm/v1/attention/backends/rocm_aiter_fa.py index 1615f883bc..4ce8b23d31 100644 --- a/vllm/v1/attention/backends/rocm_aiter_fa.py +++ b/vllm/v1/attention/backends/rocm_aiter_fa.py @@ -989,6 +989,20 @@ class AiterFlashAttentionImpl(AttentionImpl): "cross-attention." ) + def _get_kv_cache_descales( + self, + layer: AttentionLayer, + num_decodes: int, + ) -> tuple[torch.Tensor | None, torch.Tensor | None]: + # AITER treats non-null descales as an instruction to dequantize. + if not is_quantized_kv_cache(self.kv_cache_dtype): + return None, None + descale_shape = (num_decodes, self.num_kv_heads) + return ( + layer._k_scale.expand(descale_shape), + layer._v_scale.expand(descale_shape), + ) + def extend_for_sliding_window( self, attn_metadata: AiterFlashAttentionMetadata, @@ -1367,6 +1381,9 @@ class AiterFlashAttentionImpl(AttentionImpl): or decode_max_query_len > 1 or self.sinks is not None ): + k_descale, v_descale = self._get_kv_cache_descales( + layer, num_decodes + ) assert not rocm_aiter_ops.is_shuffle_kv_cache_enabled(), ( "Shuffle KV cache layout is not supported with sliding " "window, sinks, or speculative decoding (multi-token decode)." @@ -1376,7 +1393,6 @@ class AiterFlashAttentionImpl(AttentionImpl): flash_attn_with_kvcache, ) - descale_shape = (num_decodes, key_cache.shape[2]) decode_query = query[:num_decode_tokens].reshape( num_decodes, decode_max_query_len, @@ -1393,8 +1409,8 @@ class AiterFlashAttentionImpl(AttentionImpl): window_size=self.sliding_window, softcap=self.logits_soft_cap, q_descale=None, - k_descale=layer._k_scale.expand(descale_shape), - v_descale=layer._v_scale.expand(descale_shape), + k_descale=k_descale, + v_descale=v_descale, page_table=attn_metadata.block_table[:num_decodes], ) output[:num_decode_tokens].copy_( @@ -1412,10 +1428,6 @@ class AiterFlashAttentionImpl(AttentionImpl): unified_attention, ) - descale_shape = ( - num_decodes, - key_cache.shape[2], - ) unified_attention( q=query[:num_decode_tokens], k=key_cache, @@ -1434,8 +1446,8 @@ class AiterFlashAttentionImpl(AttentionImpl): block_table=attn_metadata.block_table[:num_decodes], softcap=self.logits_soft_cap, q_descale=None, - k_descale=layer._k_scale.expand(descale_shape), - v_descale=layer._v_scale.expand(descale_shape), + k_descale=k_descale, + v_descale=v_descale, sinks=self.sinks, ) return @@ -1449,6 +1461,9 @@ class AiterFlashAttentionImpl(AttentionImpl): use_unified_attention = self.head_size < _MIN_HEAD_SIZE_FOR_LL4MI if use_unified_attention: + k_descale, v_descale = self._get_kv_cache_descales( + layer, num_decodes + ) assert not rocm_aiter_ops.is_shuffle_kv_cache_enabled(), ( "unified_attention fallback with shuffle layout " "is not supported yet." @@ -1460,10 +1475,6 @@ class AiterFlashAttentionImpl(AttentionImpl): decode_cu_seqlens_q = attn_metadata.query_start_loc[ : num_decodes + 1 ] - descale_shape = ( - num_decodes, - key_cache.shape[2], - ) unified_attention( q=query[:num_decode_tokens], k=key_cache, @@ -1480,8 +1491,8 @@ class AiterFlashAttentionImpl(AttentionImpl): block_table=attn_metadata.block_table[:num_decodes], softcap=self.logits_soft_cap, q_descale=None, - k_descale=layer._k_scale.expand(descale_shape), - v_descale=layer._v_scale.expand(descale_shape), + k_descale=k_descale, + v_descale=v_descale, ) elif rocm_aiter_ops.is_shuffle_kv_cache_enabled(): _, num_heads, head_size = query.shape