[Bugfix][ROCm] Preserve AITER unified-attention metadata during graph replay (#53821)
Signed-off-by: andyluo7 <andy.luo@amd.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
@@ -15,7 +15,12 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kFp8StaticTensorSym,
|
||||
)
|
||||
from vllm.utils.torch_utils import is_quantized_kv_cache
|
||||
from vllm.v1.attention.backend import AttentionLayer, AttentionType, MultipleOf
|
||||
from vllm.v1.attention.backend import (
|
||||
AttentionLayer,
|
||||
AttentionType,
|
||||
CommonAttentionMetadata,
|
||||
MultipleOf,
|
||||
)
|
||||
from vllm.v1.attention.backends.rocm_attn import (
|
||||
RocmAttentionBackend,
|
||||
RocmAttentionImpl,
|
||||
@@ -27,6 +32,17 @@ from vllm.v1.kv_cache_interface import AttentionSpec, KVCacheLayout
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class RocmAiterUnifiedAttentionMetadataBuilder(RocmAttentionMetadataBuilder):
|
||||
def build_for_cudagraph_capture(
|
||||
self, common_attn_metadata: CommonAttentionMetadata
|
||||
) -> RocmAttentionMetadata:
|
||||
attn_metadata = self.build(0, common_attn_metadata)
|
||||
# Avoid capturing the real maximum sequence length while preserving
|
||||
# query_start_loc, which unified attention consumes during replay.
|
||||
attn_metadata.seq_lens.fill_(1)
|
||||
return attn_metadata
|
||||
|
||||
|
||||
class RocmAiterUnifiedAttentionBackend(RocmAttentionBackend):
|
||||
supported_dtypes: ClassVar[list[torch.dtype]] = [torch.float16, torch.bfloat16]
|
||||
supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = [
|
||||
@@ -101,8 +117,8 @@ class RocmAiterUnifiedAttentionBackend(RocmAttentionBackend):
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def get_builder_cls() -> type["RocmAttentionMetadataBuilder"]:
|
||||
return RocmAttentionMetadataBuilder
|
||||
def get_builder_cls() -> type["RocmAiterUnifiedAttentionMetadataBuilder"]:
|
||||
return RocmAiterUnifiedAttentionMetadataBuilder
|
||||
|
||||
@classmethod
|
||||
def supports_attn_type(cls, attn_type: str) -> bool:
|
||||
|
||||
Reference in New Issue
Block a user