[Bugfix][ROCm] Preserve AITER unified-attention metadata during graph replay (#53821)
Signed-off-by: andyluo7 <andy.luo@amd.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
@@ -34,6 +34,46 @@ def mock_get_cdna_version():
|
|||||||
yield
|
yield
|
||||||
|
|
||||||
|
|
||||||
|
def test_aiter_unified_attention_uses_dedicated_metadata_builder():
|
||||||
|
from vllm.v1.attention.backends.rocm_aiter_unified_attn import (
|
||||||
|
RocmAiterUnifiedAttentionBackend,
|
||||||
|
RocmAiterUnifiedAttentionMetadataBuilder,
|
||||||
|
)
|
||||||
|
from vllm.v1.attention.backends.rocm_attn import (
|
||||||
|
RocmAttentionBackend,
|
||||||
|
RocmAttentionMetadataBuilder,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert RocmAttentionBackend.get_builder_cls() is RocmAttentionMetadataBuilder
|
||||||
|
assert (
|
||||||
|
RocmAiterUnifiedAttentionBackend.get_builder_cls()
|
||||||
|
is RocmAiterUnifiedAttentionMetadataBuilder
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_aiter_unified_attention_capture_preserves_query_start_locations():
|
||||||
|
from vllm.v1.attention.backends.rocm_aiter_unified_attn import (
|
||||||
|
RocmAiterUnifiedAttentionMetadataBuilder,
|
||||||
|
)
|
||||||
|
|
||||||
|
builder = object.__new__(RocmAiterUnifiedAttentionMetadataBuilder)
|
||||||
|
metadata = MagicMock()
|
||||||
|
metadata.seq_lens = torch.tensor([1048576, 524288], dtype=torch.int32)
|
||||||
|
builder.build = MagicMock(return_value=metadata)
|
||||||
|
common = MagicMock()
|
||||||
|
expected_query_start_loc = torch.tensor([0, 2, 5], dtype=torch.int32)
|
||||||
|
common.query_start_loc = expected_query_start_loc.clone()
|
||||||
|
metadata.query_start_loc = common.query_start_loc
|
||||||
|
|
||||||
|
actual = builder.build_for_cudagraph_capture(common)
|
||||||
|
|
||||||
|
builder.build.assert_called_once_with(0, common)
|
||||||
|
assert actual is metadata
|
||||||
|
assert torch.equal(actual.seq_lens, torch.ones_like(actual.seq_lens))
|
||||||
|
assert actual.query_start_loc is common.query_start_loc
|
||||||
|
assert torch.equal(actual.query_start_loc, expected_query_start_loc)
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
"env_vars, selected_backend, expected_backend_path",
|
"env_vars, selected_backend, expected_backend_path",
|
||||||
[
|
[
|
||||||
|
|||||||
@@ -15,7 +15,12 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
|||||||
kFp8StaticTensorSym,
|
kFp8StaticTensorSym,
|
||||||
)
|
)
|
||||||
from vllm.utils.torch_utils import is_quantized_kv_cache
|
from vllm.utils.torch_utils import is_quantized_kv_cache
|
||||||
from vllm.v1.attention.backend import AttentionLayer, AttentionType, MultipleOf
|
from vllm.v1.attention.backend import (
|
||||||
|
AttentionLayer,
|
||||||
|
AttentionType,
|
||||||
|
CommonAttentionMetadata,
|
||||||
|
MultipleOf,
|
||||||
|
)
|
||||||
from vllm.v1.attention.backends.rocm_attn import (
|
from vllm.v1.attention.backends.rocm_attn import (
|
||||||
RocmAttentionBackend,
|
RocmAttentionBackend,
|
||||||
RocmAttentionImpl,
|
RocmAttentionImpl,
|
||||||
@@ -27,6 +32,17 @@ from vllm.v1.kv_cache_interface import AttentionSpec, KVCacheLayout
|
|||||||
logger = init_logger(__name__)
|
logger = init_logger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class RocmAiterUnifiedAttentionMetadataBuilder(RocmAttentionMetadataBuilder):
|
||||||
|
def build_for_cudagraph_capture(
|
||||||
|
self, common_attn_metadata: CommonAttentionMetadata
|
||||||
|
) -> RocmAttentionMetadata:
|
||||||
|
attn_metadata = self.build(0, common_attn_metadata)
|
||||||
|
# Avoid capturing the real maximum sequence length while preserving
|
||||||
|
# query_start_loc, which unified attention consumes during replay.
|
||||||
|
attn_metadata.seq_lens.fill_(1)
|
||||||
|
return attn_metadata
|
||||||
|
|
||||||
|
|
||||||
class RocmAiterUnifiedAttentionBackend(RocmAttentionBackend):
|
class RocmAiterUnifiedAttentionBackend(RocmAttentionBackend):
|
||||||
supported_dtypes: ClassVar[list[torch.dtype]] = [torch.float16, torch.bfloat16]
|
supported_dtypes: ClassVar[list[torch.dtype]] = [torch.float16, torch.bfloat16]
|
||||||
supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = [
|
supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = [
|
||||||
@@ -101,8 +117,8 @@ class RocmAiterUnifiedAttentionBackend(RocmAttentionBackend):
|
|||||||
return False
|
return False
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def get_builder_cls() -> type["RocmAttentionMetadataBuilder"]:
|
def get_builder_cls() -> type["RocmAiterUnifiedAttentionMetadataBuilder"]:
|
||||||
return RocmAttentionMetadataBuilder
|
return RocmAiterUnifiedAttentionMetadataBuilder
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def supports_attn_type(cls, attn_type: str) -> bool:
|
def supports_attn_type(cls, attn_type: str) -> bool:
|
||||||
|
|||||||
Reference in New Issue
Block a user