[ROCm][Bugfix] Support shared KV prefill in AITER attention (#55887)
Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com>
This commit is contained in:
@@ -687,6 +687,154 @@ def test_aiter_fa_sliding_window_matches_reference():
|
|||||||
torch.testing.assert_close(output, ref, atol=atol, rtol=rtol)
|
torch.testing.assert_close(output, ref, atol=atol, rtol=rtol)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.skipif(not on_mi3xx(), reason="MI300/MI350 ROCm only")
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"shuffle,sliding_window", [(False, None), (False, 16), (True, None)]
|
||||||
|
)
|
||||||
|
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
|
||||||
|
@pytest.mark.parametrize("batch", ["prefill", "extend", "mixed"])
|
||||||
|
def test_aiter_fa_shared_kv_matches_reference(
|
||||||
|
monkeypatch, shuffle, sliding_window, kv_cache_dtype, batch
|
||||||
|
):
|
||||||
|
"""Shared layers read cached K/V for prefill and extend without modifying it."""
|
||||||
|
from tests.v1.attention.utils import (
|
||||||
|
BatchSpec,
|
||||||
|
create_common_attn_metadata,
|
||||||
|
create_standard_kv_cache_spec,
|
||||||
|
create_vllm_config,
|
||||||
|
)
|
||||||
|
from vllm._aiter_ops import rocm_aiter_ops
|
||||||
|
from vllm.config import set_current_vllm_config
|
||||||
|
from vllm.model_executor.layers.attention import Attention
|
||||||
|
from vllm.v1.attention.backends.rocm_aiter_fa import (
|
||||||
|
AiterFlashAttentionBackend,
|
||||||
|
AiterFlashAttentionMetadataBuilder,
|
||||||
|
)
|
||||||
|
|
||||||
|
_assert_aiter_supported()
|
||||||
|
monkeypatch.setattr(rocm_aiter_ops, "_SHUFFLE_KV_CACHE_ENABLED", shuffle)
|
||||||
|
set_random_seed(42)
|
||||||
|
batch_spec = {
|
||||||
|
"prefill": BatchSpec(seq_lens=[32, 24], query_lens=[32, 24]),
|
||||||
|
"extend": BatchSpec(seq_lens=[80, 56], query_lens=[32, 24]),
|
||||||
|
"mixed": BatchSpec(seq_lens=[48, 80, 24], query_lens=[1, 32, 24]),
|
||||||
|
}[batch]
|
||||||
|
config = create_vllm_config(
|
||||||
|
model_name="Qwen/Qwen3-0.6B",
|
||||||
|
dtype="bfloat16",
|
||||||
|
max_num_batched_tokens=1024,
|
||||||
|
)
|
||||||
|
config.cache_config.cache_dtype = kv_cache_dtype
|
||||||
|
num_heads = config.model_config.get_num_attention_heads(config.parallel_config)
|
||||||
|
num_kv_heads = config.model_config.get_num_kv_heads(config.parallel_config)
|
||||||
|
head_size = config.model_config.get_head_size()
|
||||||
|
scale = head_size**-0.5
|
||||||
|
with set_current_vllm_config(config):
|
||||||
|
target = Attention(
|
||||||
|
num_heads,
|
||||||
|
head_size,
|
||||||
|
scale,
|
||||||
|
num_kv_heads,
|
||||||
|
cache_config=config.cache_config,
|
||||||
|
per_layer_sliding_window=sliding_window,
|
||||||
|
prefix="target",
|
||||||
|
attn_backend=AiterFlashAttentionBackend,
|
||||||
|
)
|
||||||
|
layer = Attention(
|
||||||
|
num_heads,
|
||||||
|
head_size,
|
||||||
|
scale,
|
||||||
|
num_kv_heads,
|
||||||
|
cache_config=config.cache_config,
|
||||||
|
per_layer_sliding_window=sliding_window,
|
||||||
|
kv_sharing_target_layer_name="target",
|
||||||
|
prefix="shared",
|
||||||
|
attn_backend=AiterFlashAttentionBackend,
|
||||||
|
)
|
||||||
|
builder = AiterFlashAttentionMetadataBuilder(
|
||||||
|
create_standard_kv_cache_spec(config),
|
||||||
|
["target", "shared"],
|
||||||
|
config,
|
||||||
|
torch.device("cuda"),
|
||||||
|
)
|
||||||
|
common = create_common_attn_metadata(
|
||||||
|
batch_spec, BLOCK_SIZE, torch.device("cuda"), max_block_idx=32
|
||||||
|
)
|
||||||
|
cache_dtype = (
|
||||||
|
current_platform.fp8_dtype() if kv_cache_dtype == "fp8" else torch.bfloat16
|
||||||
|
)
|
||||||
|
kv_cache = (
|
||||||
|
torch.empty(
|
||||||
|
2, 32, BLOCK_SIZE, num_kv_heads * head_size, dtype=cache_dtype
|
||||||
|
).transpose(0, 1)
|
||||||
|
if shuffle
|
||||||
|
else torch.empty(
|
||||||
|
32, num_kv_heads, BLOCK_SIZE, 2 * head_size, dtype=cache_dtype
|
||||||
|
)
|
||||||
|
)
|
||||||
|
target.kv_cache = kv_cache
|
||||||
|
layer.kv_cache = kv_cache
|
||||||
|
k_scale = 0.5 if kv_cache_dtype == "fp8" and not shuffle else 1.0
|
||||||
|
v_scale = 0.25 if kv_cache_dtype == "fp8" and not shuffle else 1.0
|
||||||
|
for attn_layer in (target, layer):
|
||||||
|
attn_layer._k_scale.fill_(k_scale)
|
||||||
|
attn_layer._v_scale.fill_(v_scale)
|
||||||
|
key = torch.randn(
|
||||||
|
32 * BLOCK_SIZE, num_kv_heads, head_size, dtype=torch.bfloat16
|
||||||
|
)
|
||||||
|
value = torch.randn_like(key)
|
||||||
|
target.impl.do_kv_cache_update(
|
||||||
|
target,
|
||||||
|
key,
|
||||||
|
value,
|
||||||
|
kv_cache,
|
||||||
|
torch.arange(32 * BLOCK_SIZE, dtype=torch.int64),
|
||||||
|
)
|
||||||
|
key_cache = ((key / k_scale).to(cache_dtype).to(key.dtype) * k_scale).reshape(
|
||||||
|
32, BLOCK_SIZE, num_kv_heads, head_size
|
||||||
|
)
|
||||||
|
value_cache = (
|
||||||
|
(value / v_scale).to(cache_dtype).to(value.dtype) * v_scale
|
||||||
|
).reshape(32, BLOCK_SIZE, num_kv_heads, head_size)
|
||||||
|
with torch.device("cpu"):
|
||||||
|
metadata = builder.build(0, common)
|
||||||
|
cache_before = kv_cache.view(torch.uint8).clone()
|
||||||
|
query = torch.randn(
|
||||||
|
batch_spec.compute_num_tokens(),
|
||||||
|
num_heads,
|
||||||
|
head_size,
|
||||||
|
dtype=torch.bfloat16,
|
||||||
|
)
|
||||||
|
output = torch.empty_like(query)
|
||||||
|
layer.impl.forward(layer, query, None, None, kv_cache, metadata, output)
|
||||||
|
expected = ref_paged_attn(
|
||||||
|
query,
|
||||||
|
key_cache.contiguous(),
|
||||||
|
value_cache.contiguous(),
|
||||||
|
batch_spec.query_lens,
|
||||||
|
batch_spec.seq_lens,
|
||||||
|
common.block_table_tensor,
|
||||||
|
scale,
|
||||||
|
sliding_window,
|
||||||
|
)
|
||||||
|
num_decode_tokens = metadata.num_decode_tokens
|
||||||
|
if kv_cache_dtype == "fp8" and num_decode_tokens:
|
||||||
|
# Use the established FP8 decode tolerance; gathered prefill and
|
||||||
|
# extend tokens must still satisfy the tighter BF16 tolerance.
|
||||||
|
torch.testing.assert_close(
|
||||||
|
output[:num_decode_tokens],
|
||||||
|
expected[:num_decode_tokens],
|
||||||
|
atol=6e-2,
|
||||||
|
rtol=1e-1,
|
||||||
|
)
|
||||||
|
output = output[num_decode_tokens:]
|
||||||
|
expected = expected[num_decode_tokens:]
|
||||||
|
torch.testing.assert_close(output, expected, atol=2e-2, rtol=2e-2)
|
||||||
|
torch.testing.assert_close(
|
||||||
|
kv_cache.view(torch.uint8), cache_before, atol=0, rtol=0
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
# Decode path test --------------------------------------------------------
|
# Decode path test --------------------------------------------------------
|
||||||
@pytest.mark.skipif(not on_mi3xx(), reason="MI300/MI350 ROCm only")
|
@pytest.mark.skipif(not on_mi3xx(), reason="MI300/MI350 ROCm only")
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
|
|||||||
@@ -403,6 +403,14 @@ class AiterFlashAttentionChunkPrefillMetadata:
|
|||||||
chunk_context_metadata: AiterChunkContextMetadata
|
chunk_context_metadata: AiterChunkContextMetadata
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class AiterKVSharingMetadata:
|
||||||
|
workspace: torch.Tensor
|
||||||
|
query_start_loc: torch.Tensor
|
||||||
|
token_to_batch: torch.Tensor
|
||||||
|
seq_starts: torch.Tensor
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class AiterFlashAttentionMetadata:
|
class AiterFlashAttentionMetadata:
|
||||||
# NOTE(sang): Definition of context_len, query_len, and seq_len.
|
# NOTE(sang): Definition of context_len, query_len, and seq_len.
|
||||||
@@ -439,6 +447,7 @@ class AiterFlashAttentionMetadata:
|
|||||||
# since we might integrate per token quant for kv cache in the future.
|
# since we might integrate per token quant for kv cache in the future.
|
||||||
k_scale: dict[str, torch.Tensor] | None
|
k_scale: dict[str, torch.Tensor] | None
|
||||||
v_scale: dict[str, torch.Tensor] | None
|
v_scale: dict[str, torch.Tensor] | None
|
||||||
|
kv_sharing_metadata: AiterKVSharingMetadata | None = None
|
||||||
|
|
||||||
|
|
||||||
class AiterFlashAttentionMetadataBuilder(
|
class AiterFlashAttentionMetadataBuilder(
|
||||||
@@ -471,6 +480,7 @@ class AiterFlashAttentionMetadataBuilder(
|
|||||||
self._init_reorder_batch_threshold(1, supports_spec_as_decode=True)
|
self._init_reorder_batch_threshold(1, supports_spec_as_decode=True)
|
||||||
|
|
||||||
sliding_window_configs: set[tuple[int, int] | None] = set()
|
sliding_window_configs: set[tuple[int, int] | None] = set()
|
||||||
|
kv_sharing_shape = None
|
||||||
layers = get_layers_from_vllm_config(self.vllm_config, Attention)
|
layers = get_layers_from_vllm_config(self.vllm_config, Attention)
|
||||||
for name, layer in layers.items():
|
for name, layer in layers.items():
|
||||||
if name not in layer_names:
|
if name not in layer_names:
|
||||||
@@ -480,6 +490,8 @@ class AiterFlashAttentionMetadataBuilder(
|
|||||||
"with Aiter Flash Attention Impl."
|
"with Aiter Flash Attention Impl."
|
||||||
)
|
)
|
||||||
sliding_window_configs.add(layer.impl.sliding_window)
|
sliding_window_configs.add(layer.impl.sliding_window)
|
||||||
|
if layer.kv_sharing_target_layer_name is not None:
|
||||||
|
kv_sharing_shape = (layer.impl.num_kv_heads, layer.impl.head_size)
|
||||||
|
|
||||||
while len(sliding_window_configs) > 0:
|
while len(sliding_window_configs) > 0:
|
||||||
sliding_window_config = sliding_window_configs.pop()
|
sliding_window_config = sliding_window_configs.pop()
|
||||||
@@ -495,6 +507,19 @@ class AiterFlashAttentionMetadataBuilder(
|
|||||||
device=device,
|
device=device,
|
||||||
)
|
)
|
||||||
self.scale = torch.tensor([1.0], dtype=torch.float, device=self.device)
|
self.scale = torch.tensor([1.0], dtype=torch.float, device=self.device)
|
||||||
|
self.kv_sharing_workspace = (
|
||||||
|
torch.empty(
|
||||||
|
(
|
||||||
|
2,
|
||||||
|
vllm_config.scheduler_config.max_num_batched_tokens,
|
||||||
|
*kv_sharing_shape,
|
||||||
|
),
|
||||||
|
dtype=self.model_config.dtype,
|
||||||
|
device=device,
|
||||||
|
)
|
||||||
|
if kv_sharing_shape is not None
|
||||||
|
else None
|
||||||
|
)
|
||||||
|
|
||||||
def build_for_cudagraph_capture(
|
def build_for_cudagraph_capture(
|
||||||
self, common_attn_metadata: CommonAttentionMetadata
|
self, common_attn_metadata: CommonAttentionMetadata
|
||||||
@@ -554,6 +579,23 @@ class AiterFlashAttentionMetadataBuilder(
|
|||||||
|
|
||||||
query_lens_cpu = query_start_loc_cpu[1:] - query_start_loc_cpu[:-1]
|
query_lens_cpu = query_start_loc_cpu[1:] - query_start_loc_cpu[:-1]
|
||||||
|
|
||||||
|
kv_sharing_metadata = None
|
||||||
|
if self.kv_sharing_workspace is not None and seq_lens is not None:
|
||||||
|
query_lens = query_lens_cpu[num_decodes:]
|
||||||
|
token_to_batch = torch.repeat_interleave(
|
||||||
|
torch.arange(len(query_lens), dtype=torch.int32, device="cpu"),
|
||||||
|
query_lens,
|
||||||
|
)
|
||||||
|
query_start_loc = common_attn_metadata.query_start_loc[num_decodes:]
|
||||||
|
kv_sharing_metadata = AiterKVSharingMetadata(
|
||||||
|
workspace=self.kv_sharing_workspace,
|
||||||
|
query_start_loc=query_start_loc - query_start_loc[0],
|
||||||
|
token_to_batch=async_tensor_h2d(token_to_batch, self.device),
|
||||||
|
seq_starts=async_tensor_h2d(
|
||||||
|
seq_lens[num_decodes:] - query_lens, self.device
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
decode_metadata = None
|
decode_metadata = None
|
||||||
if num_decodes > 0:
|
if num_decodes > 0:
|
||||||
decode_max_query_len = query_lens_cpu[:num_decodes].max().item()
|
decode_max_query_len = query_lens_cpu[:num_decodes].max().item()
|
||||||
@@ -719,6 +761,7 @@ class AiterFlashAttentionMetadataBuilder(
|
|||||||
use_cascade=use_cascade,
|
use_cascade=use_cascade,
|
||||||
k_scale=self.scale,
|
k_scale=self.scale,
|
||||||
v_scale=self.scale,
|
v_scale=self.scale,
|
||||||
|
kv_sharing_metadata=kv_sharing_metadata,
|
||||||
)
|
)
|
||||||
return attn_metadata
|
return attn_metadata
|
||||||
|
|
||||||
@@ -1135,8 +1178,8 @@ class AiterFlashAttentionImpl(AttentionImpl):
|
|||||||
self,
|
self,
|
||||||
layer: torch.nn.Module,
|
layer: torch.nn.Module,
|
||||||
query: torch.Tensor,
|
query: torch.Tensor,
|
||||||
key: torch.Tensor,
|
key: torch.Tensor | None,
|
||||||
value: torch.Tensor,
|
value: torch.Tensor | None,
|
||||||
kv_cache: torch.Tensor,
|
kv_cache: torch.Tensor,
|
||||||
attn_metadata: AiterFlashAttentionMetadata,
|
attn_metadata: AiterFlashAttentionMetadata,
|
||||||
output: torch.Tensor,
|
output: torch.Tensor,
|
||||||
@@ -1199,10 +1242,36 @@ class AiterFlashAttentionImpl(AttentionImpl):
|
|||||||
|
|
||||||
num_decode_tokens = attn_metadata.num_decode_tokens
|
num_decode_tokens = attn_metadata.num_decode_tokens
|
||||||
num_extend_tokens = attn_metadata.num_extend_tokens
|
num_extend_tokens = attn_metadata.num_extend_tokens
|
||||||
|
if self.kv_sharing_target_layer_name is not None and (
|
||||||
|
num_prefills > 0 or num_extends > 0
|
||||||
|
):
|
||||||
|
# Shared layers project only Q. Fetch the new K/V tokens from the
|
||||||
|
# target cache for the direct prefill and extend suffix kernels.
|
||||||
|
shared = attn_metadata.kv_sharing_metadata
|
||||||
|
assert shared is not None
|
||||||
|
key, value = shared.workspace[:, :num_actual_tokens].unbind(0)
|
||||||
|
cp_mha_gather_cache(
|
||||||
|
key_cache=key_cache,
|
||||||
|
value_cache=value_cache,
|
||||||
|
key=key[num_decode_tokens:],
|
||||||
|
value=value[num_decode_tokens:],
|
||||||
|
block_tables=attn_metadata.block_table[num_decodes:],
|
||||||
|
k_scales=layer._k_scale,
|
||||||
|
v_scales=layer._v_scale,
|
||||||
|
cu_seqlens_kv=shared.query_start_loc,
|
||||||
|
token_to_batch=shared.token_to_batch,
|
||||||
|
seq_starts=shared.seq_starts,
|
||||||
|
dequant=is_quantized_kv_cache(self.kv_cache_dtype),
|
||||||
|
kv_cache_layout="SHUFFLE"
|
||||||
|
if rocm_aiter_ops.is_shuffle_kv_cache_enabled()
|
||||||
|
else "NHD",
|
||||||
|
total_tokens=num_actual_tokens - num_decode_tokens,
|
||||||
|
)
|
||||||
if not attn_metadata.use_cascade:
|
if not attn_metadata.use_cascade:
|
||||||
# calculate for pure prefills
|
# calculate for pure prefills
|
||||||
if num_prefills > 0:
|
if num_prefills > 0:
|
||||||
assert attn_metadata.prefill_metadata is not None
|
assert attn_metadata.prefill_metadata is not None
|
||||||
|
assert key is not None and value is not None
|
||||||
|
|
||||||
prefill_query = query[num_decode_tokens + num_extend_tokens :]
|
prefill_query = query[num_decode_tokens + num_extend_tokens :]
|
||||||
prefill_key = key[num_decode_tokens + num_extend_tokens :]
|
prefill_key = key[num_decode_tokens + num_extend_tokens :]
|
||||||
@@ -1229,6 +1298,7 @@ class AiterFlashAttentionImpl(AttentionImpl):
|
|||||||
# calculate for extends
|
# calculate for extends
|
||||||
if num_extends > 0:
|
if num_extends > 0:
|
||||||
assert attn_metadata.extend_metadata is not None
|
assert attn_metadata.extend_metadata is not None
|
||||||
|
assert key is not None and value is not None
|
||||||
extend_tokens_slice = slice(
|
extend_tokens_slice = slice(
|
||||||
num_decode_tokens, num_decode_tokens + num_extend_tokens
|
num_decode_tokens, num_decode_tokens + num_extend_tokens
|
||||||
)
|
)
|
||||||
|
|||||||
Reference in New Issue
Block a user