[Bugfix][ROCm] Fix ROCM_AITER_FA & ROCM_AITER_UNIFIED_ATTN QK-Norm+RoPE+KVCache fusion for the packed KV-cache [BLOCKS, HEADS, BLOCK_SIZE, 2*HEAD_DIM] layout (#49373)
Signed-off-by: Jack Hu <Jack.Hu@amd.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
co-authored by
Andreas Karatzas
parent
7e85d3a42c
commit
34c1cd20a5
@@ -420,10 +420,9 @@ _FUSION_CONFIGS = [
|
|||||||
@pytest.mark.parametrize("use_shuffle_kv_layout", ["1", "0"])
|
@pytest.mark.parametrize("use_shuffle_kv_layout", ["1", "0"])
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
"kv_stride_order",
|
"kv_stride_order",
|
||||||
[
|
# FA/unified use the 4D packed cache (num_blocks, num_kv_heads, block_size,
|
||||||
pytest.param((0, 1, 2, 3, 4), id="block_first"),
|
# 2*head_size); no separate K/V dim to permute.
|
||||||
pytest.param((1, 0, 2, 3, 4), id="kv_first"),
|
[pytest.param((0, 1, 2, 3), id="packed_4d")],
|
||||||
],
|
|
||||||
)
|
)
|
||||||
@pytest.mark.parametrize("enable_aiter_triton_rope", [True, False])
|
@pytest.mark.parametrize("enable_aiter_triton_rope", [True, False])
|
||||||
@pytest.mark.parametrize("block_size", [16, 32, 64])
|
@pytest.mark.parametrize("block_size", [16, 32, 64])
|
||||||
@@ -458,6 +457,14 @@ def test_qk_norm_rope_kvcache_fusion(
|
|||||||
and use_shuffle_kv_layout == "1"
|
and use_shuffle_kv_layout == "1"
|
||||||
):
|
):
|
||||||
pytest.skip("ROCM_AITER_UNIFIED_ATTN is NHD-only; shuffle env is ignored")
|
pytest.skip("ROCM_AITER_UNIFIED_ATTN is NHD-only; shuffle env is ignored")
|
||||||
|
if (
|
||||||
|
attn_backend == AttentionBackendEnum.ROCM_AITER_FA
|
||||||
|
and use_shuffle_kv_layout == "1"
|
||||||
|
):
|
||||||
|
pytest.skip(
|
||||||
|
"ROCM_AITER_FA gates the qk_norm+rope+kvcache fusion off under shuffle "
|
||||||
|
"layout (defers to reshape_and_cache_shuffle_triton), so nothing fuses"
|
||||||
|
)
|
||||||
_run_qk_norm_rope_kvcache_fusion_test(
|
_run_qk_norm_rope_kvcache_fusion_test(
|
||||||
attn_backend=attn_backend,
|
attn_backend=attn_backend,
|
||||||
enable_aiter_triton_rope=enable_aiter_triton_rope,
|
enable_aiter_triton_rope=enable_aiter_triton_rope,
|
||||||
|
|||||||
@@ -1065,9 +1065,8 @@ class AiterFlashAttentionImpl(AttentionImpl):
|
|||||||
# Whenever making a change in this method, please benchmark the
|
# Whenever making a change in this method, please benchmark the
|
||||||
# performance to make sure it does not introduce any overhead.
|
# performance to make sure it does not introduce any overhead.
|
||||||
num_actual_tokens = attn_metadata.num_actual_tokens
|
num_actual_tokens = attn_metadata.num_actual_tokens
|
||||||
# (B, H, N, 2*hs) -> ((B, N, H, hs), (B, N, H, hs))
|
|
||||||
key_cache, value_cache = kv_cache.transpose(1, 2).split(self.head_size, dim=-1)
|
|
||||||
|
|
||||||
|
key_cache, value_cache = self._split_kv_cache(kv_cache)
|
||||||
if is_quantized_kv_cache(self.kv_cache_dtype):
|
if is_quantized_kv_cache(self.kv_cache_dtype):
|
||||||
key_cache = key_cache.view(current_platform.fp8_dtype())
|
key_cache = key_cache.view(current_platform.fp8_dtype())
|
||||||
value_cache = value_cache.view(current_platform.fp8_dtype())
|
value_cache = value_cache.view(current_platform.fp8_dtype())
|
||||||
@@ -1385,6 +1384,12 @@ class AiterFlashAttentionImpl(AttentionImpl):
|
|||||||
|
|
||||||
return output
|
return output
|
||||||
|
|
||||||
|
def _split_kv_cache(
|
||||||
|
self, kv_cache: torch.Tensor
|
||||||
|
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||||
|
# (B, H, N, 2*hs) -> ((B, N, H, hs), (B, N, H, hs))
|
||||||
|
return kv_cache.transpose(1, 2).split(self.head_size, dim=-1)
|
||||||
|
|
||||||
def do_kv_cache_update(
|
def do_kv_cache_update(
|
||||||
self,
|
self,
|
||||||
layer: AttentionLayer,
|
layer: AttentionLayer,
|
||||||
@@ -1393,8 +1398,7 @@ class AiterFlashAttentionImpl(AttentionImpl):
|
|||||||
kv_cache: torch.Tensor,
|
kv_cache: torch.Tensor,
|
||||||
slot_mapping: torch.Tensor,
|
slot_mapping: torch.Tensor,
|
||||||
):
|
):
|
||||||
# (B, H, N, 2*hs) -> ((B, N, H, hs), (B, N, H, hs))
|
key_cache, value_cache = self._split_kv_cache(kv_cache)
|
||||||
key_cache, value_cache = kv_cache.transpose(1, 2).split(self.head_size, dim=-1)
|
|
||||||
|
|
||||||
# key and value may be None in the case of cross attention. They are
|
# key and value may be None in the case of cross attention. They are
|
||||||
# calculated once based on the output from the encoder and then cached
|
# calculated once based on the output from the encoder and then cached
|
||||||
@@ -1449,7 +1453,12 @@ class AiterFlashAttentionImpl(AttentionImpl):
|
|||||||
)
|
)
|
||||||
|
|
||||||
def fused_qk_norm_rope_kvcache_supported(self):
|
def fused_qk_norm_rope_kvcache_supported(self):
|
||||||
return rocm_aiter_ops.is_enabled()
|
# Only fuse when shuffle layout is off; the shuffle write path uses a
|
||||||
|
# dedicated cache update, mirroring fused_rope_kvcache_supported.
|
||||||
|
return (
|
||||||
|
rocm_aiter_ops.is_enabled()
|
||||||
|
and not rocm_aiter_ops.is_shuffle_kv_cache_enabled()
|
||||||
|
)
|
||||||
|
|
||||||
def do_qk_norm_rope_kvcache_update(
|
def do_qk_norm_rope_kvcache_update(
|
||||||
self,
|
self,
|
||||||
@@ -1466,7 +1475,7 @@ class AiterFlashAttentionImpl(AttentionImpl):
|
|||||||
kv_cache: torch.Tensor,
|
kv_cache: torch.Tensor,
|
||||||
layer_slot_mapping: torch.Tensor,
|
layer_slot_mapping: torch.Tensor,
|
||||||
):
|
):
|
||||||
key_cache, value_cache = kv_cache.unbind(1)
|
key_cache, value_cache = self._split_kv_cache(kv_cache)
|
||||||
rocm_aiter_ops.do_qk_norm_rope_kvcache_update(
|
rocm_aiter_ops.do_qk_norm_rope_kvcache_update(
|
||||||
qkv=qkv,
|
qkv=qkv,
|
||||||
q_weight=q_weight,
|
q_weight=q_weight,
|
||||||
@@ -1502,7 +1511,7 @@ class AiterFlashAttentionImpl(AttentionImpl):
|
|||||||
layer_slot_mapping: torch.Tensor,
|
layer_slot_mapping: torch.Tensor,
|
||||||
):
|
):
|
||||||
# (B, H, N, 2*hs) -> ((B, N, H, hs), (B, N, H, hs))
|
# (B, H, N, 2*hs) -> ((B, N, H, hs), (B, N, H, hs))
|
||||||
key_cache, value_cache = kv_cache.transpose(1, 2).split(self.head_size, dim=-1)
|
key_cache, value_cache = self._split_kv_cache(kv_cache)
|
||||||
flash_layout = True
|
flash_layout = True
|
||||||
|
|
||||||
is_fp8_kv_cache = is_quantized_kv_cache(self.kv_cache_dtype)
|
is_fp8_kv_cache = is_quantized_kv_cache(self.kv_cache_dtype)
|
||||||
|
|||||||
@@ -329,9 +329,6 @@ class RocmAiterUnifiedAttentionImpl(RocmAttentionImpl):
|
|||||||
kv_cache: torch.Tensor,
|
kv_cache: torch.Tensor,
|
||||||
layer_slot_mapping: torch.Tensor,
|
layer_slot_mapping: torch.Tensor,
|
||||||
):
|
):
|
||||||
# _split_kv_cache picks the unbind dim per layout (incl. the K/V-first
|
|
||||||
# encoder-decoder path). unified reads NHD, so never write the shuffle
|
|
||||||
# layout here.
|
|
||||||
key_cache, value_cache = self._split_kv_cache(kv_cache)
|
key_cache, value_cache = self._split_kv_cache(kv_cache)
|
||||||
rocm_aiter_ops.do_qk_norm_rope_kvcache_update(
|
rocm_aiter_ops.do_qk_norm_rope_kvcache_update(
|
||||||
qkv=qkv,
|
qkv=qkv,
|
||||||
|
|||||||
Reference in New Issue
Block a user