From dcd544486b7f4672b3c2e60ea289f19d86cf355c Mon Sep 17 00:00:00 2001 From: Andreas Karatzas Date: Wed, 9 Sep 2026 16:51:34 -0500 Subject: [PATCH] [ROCm][Bugfix] Support shared KV prefill in AITER attention (#55887) Signed-off-by: Andreas Karatzas --- tests/kernels/attention/test_rocm_aiter_fa.py | 148 ++++++++++++++++++ vllm/v1/attention/backends/rocm_aiter_fa.py | 74 ++++++++- 2 files changed, 220 insertions(+), 2 deletions(-) diff --git a/tests/kernels/attention/test_rocm_aiter_fa.py b/tests/kernels/attention/test_rocm_aiter_fa.py index 163dfbd35b..dd2870ccf1 100644 --- a/tests/kernels/attention/test_rocm_aiter_fa.py +++ b/tests/kernels/attention/test_rocm_aiter_fa.py @@ -687,6 +687,154 @@ def test_aiter_fa_sliding_window_matches_reference(): torch.testing.assert_close(output, ref, atol=atol, rtol=rtol) +@pytest.mark.skipif(not on_mi3xx(), reason="MI300/MI350 ROCm only") +@pytest.mark.parametrize( + "shuffle,sliding_window", [(False, None), (False, 16), (True, None)] +) +@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"]) +@pytest.mark.parametrize("batch", ["prefill", "extend", "mixed"]) +def test_aiter_fa_shared_kv_matches_reference( + monkeypatch, shuffle, sliding_window, kv_cache_dtype, batch +): + """Shared layers read cached K/V for prefill and extend without modifying it.""" + from tests.v1.attention.utils import ( + BatchSpec, + create_common_attn_metadata, + create_standard_kv_cache_spec, + create_vllm_config, + ) + from vllm._aiter_ops import rocm_aiter_ops + from vllm.config import set_current_vllm_config + from vllm.model_executor.layers.attention import Attention + from vllm.v1.attention.backends.rocm_aiter_fa import ( + AiterFlashAttentionBackend, + AiterFlashAttentionMetadataBuilder, + ) + + _assert_aiter_supported() + monkeypatch.setattr(rocm_aiter_ops, "_SHUFFLE_KV_CACHE_ENABLED", shuffle) + set_random_seed(42) + batch_spec = { + "prefill": BatchSpec(seq_lens=[32, 24], query_lens=[32, 24]), + "extend": BatchSpec(seq_lens=[80, 56], query_lens=[32, 24]), + "mixed": BatchSpec(seq_lens=[48, 80, 24], query_lens=[1, 32, 24]), + }[batch] + config = create_vllm_config( + model_name="Qwen/Qwen3-0.6B", + dtype="bfloat16", + max_num_batched_tokens=1024, + ) + config.cache_config.cache_dtype = kv_cache_dtype + num_heads = config.model_config.get_num_attention_heads(config.parallel_config) + num_kv_heads = config.model_config.get_num_kv_heads(config.parallel_config) + head_size = config.model_config.get_head_size() + scale = head_size**-0.5 + with set_current_vllm_config(config): + target = Attention( + num_heads, + head_size, + scale, + num_kv_heads, + cache_config=config.cache_config, + per_layer_sliding_window=sliding_window, + prefix="target", + attn_backend=AiterFlashAttentionBackend, + ) + layer = Attention( + num_heads, + head_size, + scale, + num_kv_heads, + cache_config=config.cache_config, + per_layer_sliding_window=sliding_window, + kv_sharing_target_layer_name="target", + prefix="shared", + attn_backend=AiterFlashAttentionBackend, + ) + builder = AiterFlashAttentionMetadataBuilder( + create_standard_kv_cache_spec(config), + ["target", "shared"], + config, + torch.device("cuda"), + ) + common = create_common_attn_metadata( + batch_spec, BLOCK_SIZE, torch.device("cuda"), max_block_idx=32 + ) + cache_dtype = ( + current_platform.fp8_dtype() if kv_cache_dtype == "fp8" else torch.bfloat16 + ) + kv_cache = ( + torch.empty( + 2, 32, BLOCK_SIZE, num_kv_heads * head_size, dtype=cache_dtype + ).transpose(0, 1) + if shuffle + else torch.empty( + 32, num_kv_heads, BLOCK_SIZE, 2 * head_size, dtype=cache_dtype + ) + ) + target.kv_cache = kv_cache + layer.kv_cache = kv_cache + k_scale = 0.5 if kv_cache_dtype == "fp8" and not shuffle else 1.0 + v_scale = 0.25 if kv_cache_dtype == "fp8" and not shuffle else 1.0 + for attn_layer in (target, layer): + attn_layer._k_scale.fill_(k_scale) + attn_layer._v_scale.fill_(v_scale) + key = torch.randn( + 32 * BLOCK_SIZE, num_kv_heads, head_size, dtype=torch.bfloat16 + ) + value = torch.randn_like(key) + target.impl.do_kv_cache_update( + target, + key, + value, + kv_cache, + torch.arange(32 * BLOCK_SIZE, dtype=torch.int64), + ) + key_cache = ((key / k_scale).to(cache_dtype).to(key.dtype) * k_scale).reshape( + 32, BLOCK_SIZE, num_kv_heads, head_size + ) + value_cache = ( + (value / v_scale).to(cache_dtype).to(value.dtype) * v_scale + ).reshape(32, BLOCK_SIZE, num_kv_heads, head_size) + with torch.device("cpu"): + metadata = builder.build(0, common) + cache_before = kv_cache.view(torch.uint8).clone() + query = torch.randn( + batch_spec.compute_num_tokens(), + num_heads, + head_size, + dtype=torch.bfloat16, + ) + output = torch.empty_like(query) + layer.impl.forward(layer, query, None, None, kv_cache, metadata, output) + expected = ref_paged_attn( + query, + key_cache.contiguous(), + value_cache.contiguous(), + batch_spec.query_lens, + batch_spec.seq_lens, + common.block_table_tensor, + scale, + sliding_window, + ) + num_decode_tokens = metadata.num_decode_tokens + if kv_cache_dtype == "fp8" and num_decode_tokens: + # Use the established FP8 decode tolerance; gathered prefill and + # extend tokens must still satisfy the tighter BF16 tolerance. + torch.testing.assert_close( + output[:num_decode_tokens], + expected[:num_decode_tokens], + atol=6e-2, + rtol=1e-1, + ) + output = output[num_decode_tokens:] + expected = expected[num_decode_tokens:] + torch.testing.assert_close(output, expected, atol=2e-2, rtol=2e-2) + torch.testing.assert_close( + kv_cache.view(torch.uint8), cache_before, atol=0, rtol=0 + ) + + # Decode path test -------------------------------------------------------- @pytest.mark.skipif(not on_mi3xx(), reason="MI300/MI350 ROCm only") @pytest.mark.parametrize( diff --git a/vllm/v1/attention/backends/rocm_aiter_fa.py b/vllm/v1/attention/backends/rocm_aiter_fa.py index cbae01519b..062aa82285 100644 --- a/vllm/v1/attention/backends/rocm_aiter_fa.py +++ b/vllm/v1/attention/backends/rocm_aiter_fa.py @@ -403,6 +403,14 @@ class AiterFlashAttentionChunkPrefillMetadata: chunk_context_metadata: AiterChunkContextMetadata +@dataclass +class AiterKVSharingMetadata: + workspace: torch.Tensor + query_start_loc: torch.Tensor + token_to_batch: torch.Tensor + seq_starts: torch.Tensor + + @dataclass class AiterFlashAttentionMetadata: # NOTE(sang): Definition of context_len, query_len, and seq_len. @@ -439,6 +447,7 @@ class AiterFlashAttentionMetadata: # since we might integrate per token quant for kv cache in the future. k_scale: dict[str, torch.Tensor] | None v_scale: dict[str, torch.Tensor] | None + kv_sharing_metadata: AiterKVSharingMetadata | None = None class AiterFlashAttentionMetadataBuilder( @@ -471,6 +480,7 @@ class AiterFlashAttentionMetadataBuilder( self._init_reorder_batch_threshold(1, supports_spec_as_decode=True) sliding_window_configs: set[tuple[int, int] | None] = set() + kv_sharing_shape = None layers = get_layers_from_vllm_config(self.vllm_config, Attention) for name, layer in layers.items(): if name not in layer_names: @@ -480,6 +490,8 @@ class AiterFlashAttentionMetadataBuilder( "with Aiter Flash Attention Impl." ) sliding_window_configs.add(layer.impl.sliding_window) + if layer.kv_sharing_target_layer_name is not None: + kv_sharing_shape = (layer.impl.num_kv_heads, layer.impl.head_size) while len(sliding_window_configs) > 0: sliding_window_config = sliding_window_configs.pop() @@ -495,6 +507,19 @@ class AiterFlashAttentionMetadataBuilder( device=device, ) self.scale = torch.tensor([1.0], dtype=torch.float, device=self.device) + self.kv_sharing_workspace = ( + torch.empty( + ( + 2, + vllm_config.scheduler_config.max_num_batched_tokens, + *kv_sharing_shape, + ), + dtype=self.model_config.dtype, + device=device, + ) + if kv_sharing_shape is not None + else None + ) def build_for_cudagraph_capture( self, common_attn_metadata: CommonAttentionMetadata @@ -554,6 +579,23 @@ class AiterFlashAttentionMetadataBuilder( query_lens_cpu = query_start_loc_cpu[1:] - query_start_loc_cpu[:-1] + kv_sharing_metadata = None + if self.kv_sharing_workspace is not None and seq_lens is not None: + query_lens = query_lens_cpu[num_decodes:] + token_to_batch = torch.repeat_interleave( + torch.arange(len(query_lens), dtype=torch.int32, device="cpu"), + query_lens, + ) + query_start_loc = common_attn_metadata.query_start_loc[num_decodes:] + kv_sharing_metadata = AiterKVSharingMetadata( + workspace=self.kv_sharing_workspace, + query_start_loc=query_start_loc - query_start_loc[0], + token_to_batch=async_tensor_h2d(token_to_batch, self.device), + seq_starts=async_tensor_h2d( + seq_lens[num_decodes:] - query_lens, self.device + ), + ) + decode_metadata = None if num_decodes > 0: decode_max_query_len = query_lens_cpu[:num_decodes].max().item() @@ -719,6 +761,7 @@ class AiterFlashAttentionMetadataBuilder( use_cascade=use_cascade, k_scale=self.scale, v_scale=self.scale, + kv_sharing_metadata=kv_sharing_metadata, ) return attn_metadata @@ -1135,8 +1178,8 @@ class AiterFlashAttentionImpl(AttentionImpl): self, layer: torch.nn.Module, query: torch.Tensor, - key: torch.Tensor, - value: torch.Tensor, + key: torch.Tensor | None, + value: torch.Tensor | None, kv_cache: torch.Tensor, attn_metadata: AiterFlashAttentionMetadata, output: torch.Tensor, @@ -1199,10 +1242,36 @@ class AiterFlashAttentionImpl(AttentionImpl): num_decode_tokens = attn_metadata.num_decode_tokens num_extend_tokens = attn_metadata.num_extend_tokens + if self.kv_sharing_target_layer_name is not None and ( + num_prefills > 0 or num_extends > 0 + ): + # Shared layers project only Q. Fetch the new K/V tokens from the + # target cache for the direct prefill and extend suffix kernels. + shared = attn_metadata.kv_sharing_metadata + assert shared is not None + key, value = shared.workspace[:, :num_actual_tokens].unbind(0) + cp_mha_gather_cache( + key_cache=key_cache, + value_cache=value_cache, + key=key[num_decode_tokens:], + value=value[num_decode_tokens:], + block_tables=attn_metadata.block_table[num_decodes:], + k_scales=layer._k_scale, + v_scales=layer._v_scale, + cu_seqlens_kv=shared.query_start_loc, + token_to_batch=shared.token_to_batch, + seq_starts=shared.seq_starts, + dequant=is_quantized_kv_cache(self.kv_cache_dtype), + kv_cache_layout="SHUFFLE" + if rocm_aiter_ops.is_shuffle_kv_cache_enabled() + else "NHD", + total_tokens=num_actual_tokens - num_decode_tokens, + ) if not attn_metadata.use_cascade: # calculate for pure prefills if num_prefills > 0: assert attn_metadata.prefill_metadata is not None + assert key is not None and value is not None prefill_query = query[num_decode_tokens + num_extend_tokens :] prefill_key = key[num_decode_tokens + num_extend_tokens :] @@ -1229,6 +1298,7 @@ class AiterFlashAttentionImpl(AttentionImpl): # calculate for extends if num_extends > 0: assert attn_metadata.extend_metadata is not None + assert key is not None and value is not None extend_tokens_slice = slice( num_decode_tokens, num_decode_tokens + num_extend_tokens )