[ROCm][Bugfix] Support shared KV prefill in AITER attention (#55887)

Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com>
This commit is contained in:
Andreas Karatzas
2026-09-09 14:51:34 -07:00
committed by GitHub
parent 26fec6d183
commit dcd544486b
2 changed files with 220 additions and 2 deletions
@@ -687,6 +687,154 @@ def test_aiter_fa_sliding_window_matches_reference():
torch.testing.assert_close(output, ref, atol=atol, rtol=rtol) torch.testing.assert_close(output, ref, atol=atol, rtol=rtol)
@pytest.mark.skipif(not on_mi3xx(), reason="MI300/MI350 ROCm only")
@pytest.mark.parametrize(
"shuffle,sliding_window", [(False, None), (False, 16), (True, None)]
)
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
@pytest.mark.parametrize("batch", ["prefill", "extend", "mixed"])
def test_aiter_fa_shared_kv_matches_reference(
monkeypatch, shuffle, sliding_window, kv_cache_dtype, batch
):
"""Shared layers read cached K/V for prefill and extend without modifying it."""
from tests.v1.attention.utils import (
BatchSpec,
create_common_attn_metadata,
create_standard_kv_cache_spec,
create_vllm_config,
)
from vllm._aiter_ops import rocm_aiter_ops
from vllm.config import set_current_vllm_config
from vllm.model_executor.layers.attention import Attention
from vllm.v1.attention.backends.rocm_aiter_fa import (
AiterFlashAttentionBackend,
AiterFlashAttentionMetadataBuilder,
)
_assert_aiter_supported()
monkeypatch.setattr(rocm_aiter_ops, "_SHUFFLE_KV_CACHE_ENABLED", shuffle)
set_random_seed(42)
batch_spec = {
"prefill": BatchSpec(seq_lens=[32, 24], query_lens=[32, 24]),
"extend": BatchSpec(seq_lens=[80, 56], query_lens=[32, 24]),
"mixed": BatchSpec(seq_lens=[48, 80, 24], query_lens=[1, 32, 24]),
}[batch]
config = create_vllm_config(
model_name="Qwen/Qwen3-0.6B",
dtype="bfloat16",
max_num_batched_tokens=1024,
)
config.cache_config.cache_dtype = kv_cache_dtype
num_heads = config.model_config.get_num_attention_heads(config.parallel_config)
num_kv_heads = config.model_config.get_num_kv_heads(config.parallel_config)
head_size = config.model_config.get_head_size()
scale = head_size**-0.5
with set_current_vllm_config(config):
target = Attention(
num_heads,
head_size,
scale,
num_kv_heads,
cache_config=config.cache_config,
per_layer_sliding_window=sliding_window,
prefix="target",
attn_backend=AiterFlashAttentionBackend,
)
layer = Attention(
num_heads,
head_size,
scale,
num_kv_heads,
cache_config=config.cache_config,
per_layer_sliding_window=sliding_window,
kv_sharing_target_layer_name="target",
prefix="shared",
attn_backend=AiterFlashAttentionBackend,
)
builder = AiterFlashAttentionMetadataBuilder(
create_standard_kv_cache_spec(config),
["target", "shared"],
config,
torch.device("cuda"),
)
common = create_common_attn_metadata(
batch_spec, BLOCK_SIZE, torch.device("cuda"), max_block_idx=32
)
cache_dtype = (
current_platform.fp8_dtype() if kv_cache_dtype == "fp8" else torch.bfloat16
)
kv_cache = (
torch.empty(
2, 32, BLOCK_SIZE, num_kv_heads * head_size, dtype=cache_dtype
).transpose(0, 1)
if shuffle
else torch.empty(
32, num_kv_heads, BLOCK_SIZE, 2 * head_size, dtype=cache_dtype
)
)
target.kv_cache = kv_cache
layer.kv_cache = kv_cache
k_scale = 0.5 if kv_cache_dtype == "fp8" and not shuffle else 1.0
v_scale = 0.25 if kv_cache_dtype == "fp8" and not shuffle else 1.0
for attn_layer in (target, layer):
attn_layer._k_scale.fill_(k_scale)
attn_layer._v_scale.fill_(v_scale)
key = torch.randn(
32 * BLOCK_SIZE, num_kv_heads, head_size, dtype=torch.bfloat16
)
value = torch.randn_like(key)
target.impl.do_kv_cache_update(
target,
key,
value,
kv_cache,
torch.arange(32 * BLOCK_SIZE, dtype=torch.int64),
)
key_cache = ((key / k_scale).to(cache_dtype).to(key.dtype) * k_scale).reshape(
32, BLOCK_SIZE, num_kv_heads, head_size
)
value_cache = (
(value / v_scale).to(cache_dtype).to(value.dtype) * v_scale
).reshape(32, BLOCK_SIZE, num_kv_heads, head_size)
with torch.device("cpu"):
metadata = builder.build(0, common)
cache_before = kv_cache.view(torch.uint8).clone()
query = torch.randn(
batch_spec.compute_num_tokens(),
num_heads,
head_size,
dtype=torch.bfloat16,
)
output = torch.empty_like(query)
layer.impl.forward(layer, query, None, None, kv_cache, metadata, output)
expected = ref_paged_attn(
query,
key_cache.contiguous(),
value_cache.contiguous(),
batch_spec.query_lens,
batch_spec.seq_lens,
common.block_table_tensor,
scale,
sliding_window,
)
num_decode_tokens = metadata.num_decode_tokens
if kv_cache_dtype == "fp8" and num_decode_tokens:
# Use the established FP8 decode tolerance; gathered prefill and
# extend tokens must still satisfy the tighter BF16 tolerance.
torch.testing.assert_close(
output[:num_decode_tokens],
expected[:num_decode_tokens],
atol=6e-2,
rtol=1e-1,
)
output = output[num_decode_tokens:]
expected = expected[num_decode_tokens:]
torch.testing.assert_close(output, expected, atol=2e-2, rtol=2e-2)
torch.testing.assert_close(
kv_cache.view(torch.uint8), cache_before, atol=0, rtol=0
)
# Decode path test -------------------------------------------------------- # Decode path test --------------------------------------------------------
@pytest.mark.skipif(not on_mi3xx(), reason="MI300/MI350 ROCm only") @pytest.mark.skipif(not on_mi3xx(), reason="MI300/MI350 ROCm only")
@pytest.mark.parametrize( @pytest.mark.parametrize(
+72 -2
View File
@@ -403,6 +403,14 @@ class AiterFlashAttentionChunkPrefillMetadata:
chunk_context_metadata: AiterChunkContextMetadata chunk_context_metadata: AiterChunkContextMetadata
@dataclass
class AiterKVSharingMetadata:
workspace: torch.Tensor
query_start_loc: torch.Tensor
token_to_batch: torch.Tensor
seq_starts: torch.Tensor
@dataclass @dataclass
class AiterFlashAttentionMetadata: class AiterFlashAttentionMetadata:
# NOTE(sang): Definition of context_len, query_len, and seq_len. # NOTE(sang): Definition of context_len, query_len, and seq_len.
@@ -439,6 +447,7 @@ class AiterFlashAttentionMetadata:
# since we might integrate per token quant for kv cache in the future. # since we might integrate per token quant for kv cache in the future.
k_scale: dict[str, torch.Tensor] | None k_scale: dict[str, torch.Tensor] | None
v_scale: dict[str, torch.Tensor] | None v_scale: dict[str, torch.Tensor] | None
kv_sharing_metadata: AiterKVSharingMetadata | None = None
class AiterFlashAttentionMetadataBuilder( class AiterFlashAttentionMetadataBuilder(
@@ -471,6 +480,7 @@ class AiterFlashAttentionMetadataBuilder(
self._init_reorder_batch_threshold(1, supports_spec_as_decode=True) self._init_reorder_batch_threshold(1, supports_spec_as_decode=True)
sliding_window_configs: set[tuple[int, int] | None] = set() sliding_window_configs: set[tuple[int, int] | None] = set()
kv_sharing_shape = None
layers = get_layers_from_vllm_config(self.vllm_config, Attention) layers = get_layers_from_vllm_config(self.vllm_config, Attention)
for name, layer in layers.items(): for name, layer in layers.items():
if name not in layer_names: if name not in layer_names:
@@ -480,6 +490,8 @@ class AiterFlashAttentionMetadataBuilder(
"with Aiter Flash Attention Impl." "with Aiter Flash Attention Impl."
) )
sliding_window_configs.add(layer.impl.sliding_window) sliding_window_configs.add(layer.impl.sliding_window)
if layer.kv_sharing_target_layer_name is not None:
kv_sharing_shape = (layer.impl.num_kv_heads, layer.impl.head_size)
while len(sliding_window_configs) > 0: while len(sliding_window_configs) > 0:
sliding_window_config = sliding_window_configs.pop() sliding_window_config = sliding_window_configs.pop()
@@ -495,6 +507,19 @@ class AiterFlashAttentionMetadataBuilder(
device=device, device=device,
) )
self.scale = torch.tensor([1.0], dtype=torch.float, device=self.device) self.scale = torch.tensor([1.0], dtype=torch.float, device=self.device)
self.kv_sharing_workspace = (
torch.empty(
(
2,
vllm_config.scheduler_config.max_num_batched_tokens,
*kv_sharing_shape,
),
dtype=self.model_config.dtype,
device=device,
)
if kv_sharing_shape is not None
else None
)
def build_for_cudagraph_capture( def build_for_cudagraph_capture(
self, common_attn_metadata: CommonAttentionMetadata self, common_attn_metadata: CommonAttentionMetadata
@@ -554,6 +579,23 @@ class AiterFlashAttentionMetadataBuilder(
query_lens_cpu = query_start_loc_cpu[1:] - query_start_loc_cpu[:-1] query_lens_cpu = query_start_loc_cpu[1:] - query_start_loc_cpu[:-1]
kv_sharing_metadata = None
if self.kv_sharing_workspace is not None and seq_lens is not None:
query_lens = query_lens_cpu[num_decodes:]
token_to_batch = torch.repeat_interleave(
torch.arange(len(query_lens), dtype=torch.int32, device="cpu"),
query_lens,
)
query_start_loc = common_attn_metadata.query_start_loc[num_decodes:]
kv_sharing_metadata = AiterKVSharingMetadata(
workspace=self.kv_sharing_workspace,
query_start_loc=query_start_loc - query_start_loc[0],
token_to_batch=async_tensor_h2d(token_to_batch, self.device),
seq_starts=async_tensor_h2d(
seq_lens[num_decodes:] - query_lens, self.device
),
)
decode_metadata = None decode_metadata = None
if num_decodes > 0: if num_decodes > 0:
decode_max_query_len = query_lens_cpu[:num_decodes].max().item() decode_max_query_len = query_lens_cpu[:num_decodes].max().item()
@@ -719,6 +761,7 @@ class AiterFlashAttentionMetadataBuilder(
use_cascade=use_cascade, use_cascade=use_cascade,
k_scale=self.scale, k_scale=self.scale,
v_scale=self.scale, v_scale=self.scale,
kv_sharing_metadata=kv_sharing_metadata,
) )
return attn_metadata return attn_metadata
@@ -1135,8 +1178,8 @@ class AiterFlashAttentionImpl(AttentionImpl):
self, self,
layer: torch.nn.Module, layer: torch.nn.Module,
query: torch.Tensor, query: torch.Tensor,
key: torch.Tensor, key: torch.Tensor | None,
value: torch.Tensor, value: torch.Tensor | None,
kv_cache: torch.Tensor, kv_cache: torch.Tensor,
attn_metadata: AiterFlashAttentionMetadata, attn_metadata: AiterFlashAttentionMetadata,
output: torch.Tensor, output: torch.Tensor,
@@ -1199,10 +1242,36 @@ class AiterFlashAttentionImpl(AttentionImpl):
num_decode_tokens = attn_metadata.num_decode_tokens num_decode_tokens = attn_metadata.num_decode_tokens
num_extend_tokens = attn_metadata.num_extend_tokens num_extend_tokens = attn_metadata.num_extend_tokens
if self.kv_sharing_target_layer_name is not None and (
num_prefills > 0 or num_extends > 0
):
# Shared layers project only Q. Fetch the new K/V tokens from the
# target cache for the direct prefill and extend suffix kernels.
shared = attn_metadata.kv_sharing_metadata
assert shared is not None
key, value = shared.workspace[:, :num_actual_tokens].unbind(0)
cp_mha_gather_cache(
key_cache=key_cache,
value_cache=value_cache,
key=key[num_decode_tokens:],
value=value[num_decode_tokens:],
block_tables=attn_metadata.block_table[num_decodes:],
k_scales=layer._k_scale,
v_scales=layer._v_scale,
cu_seqlens_kv=shared.query_start_loc,
token_to_batch=shared.token_to_batch,
seq_starts=shared.seq_starts,
dequant=is_quantized_kv_cache(self.kv_cache_dtype),
kv_cache_layout="SHUFFLE"
if rocm_aiter_ops.is_shuffle_kv_cache_enabled()
else "NHD",
total_tokens=num_actual_tokens - num_decode_tokens,
)
if not attn_metadata.use_cascade: if not attn_metadata.use_cascade:
# calculate for pure prefills # calculate for pure prefills
if num_prefills > 0: if num_prefills > 0:
assert attn_metadata.prefill_metadata is not None assert attn_metadata.prefill_metadata is not None
assert key is not None and value is not None
prefill_query = query[num_decode_tokens + num_extend_tokens :] prefill_query = query[num_decode_tokens + num_extend_tokens :]
prefill_key = key[num_decode_tokens + num_extend_tokens :] prefill_key = key[num_decode_tokens + num_extend_tokens :]
@@ -1229,6 +1298,7 @@ class AiterFlashAttentionImpl(AttentionImpl):
# calculate for extends # calculate for extends
if num_extends > 0: if num_extends > 0:
assert attn_metadata.extend_metadata is not None assert attn_metadata.extend_metadata is not None
assert key is not None and value is not None
extend_tokens_slice = slice( extend_tokens_slice = slice(
num_decode_tokens, num_decode_tokens + num_extend_tokens num_decode_tokens, num_decode_tokens + num_extend_tokens
) )