[Bugfix] Avoid import AttentionMetadata explicitly in Mllama (#10593)
Signed-off-by: Isotr0py <2037008807@qq.com>
This commit is contained in:
@@ -32,9 +32,8 @@ from transformers.models.mllama.processing_mllama import (
|
||||
|
||||
import vllm.distributed.parallel_state as ps
|
||||
from vllm.attention import Attention, AttentionMetadata, AttentionType
|
||||
from vllm.attention.backends.flash_attn import FlashAttentionMetadata
|
||||
from vllm.attention.backends.xformers import XFormersMetadata
|
||||
from vllm.attention.ops.paged_attn import PagedAttention
|
||||
from vllm.attention.selector import _Backend
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed import get_tensor_model_parallel_world_size
|
||||
from vllm.inputs import (INPUT_REGISTRY, DummyData, EncoderDecoderInputs,
|
||||
@@ -828,7 +827,8 @@ class MllamaTextCrossAttention(nn.Module):
|
||||
) -> torch.Tensor:
|
||||
# Skip writing kv-cache for the initial profiling run.
|
||||
if len(kv_cache.shape) > 1:
|
||||
if isinstance(attn_metadata, FlashAttentionMetadata):
|
||||
if self.attn.backend in (_Backend.FLASH_ATTN,
|
||||
_Backend.FLASH_ATTN_VLLM_V1):
|
||||
cached_k = torch.cat([k[s:e] for s, e in kv_range_for_decode])
|
||||
cached_v = torch.cat([v[s:e] for s, e in kv_range_for_decode])
|
||||
torch.ops._C_cache_ops.reshape_and_cache_flash(
|
||||
@@ -842,7 +842,7 @@ class MllamaTextCrossAttention(nn.Module):
|
||||
1.0,
|
||||
1.0,
|
||||
)
|
||||
elif isinstance(attn_metadata, XFormersMetadata):
|
||||
elif self.attn.backend in (_Backend.XFORMERS, _Backend.TORCH_SDPA):
|
||||
key_cache, value_cache = PagedAttention.split_kv_cache(
|
||||
kv_cache, self.num_local_key_value_heads, self.head_dim)
|
||||
cached_k = torch.cat([k[s:e] for s, e in kv_range_for_decode])
|
||||
@@ -852,9 +852,9 @@ class MllamaTextCrossAttention(nn.Module):
|
||||
attn_metadata.cross_slot_mapping, "auto", 1.0, 1.0)
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Unsupported AttentionMetadata {type(attn_metadata)} "
|
||||
f"class found. Expected the AttentionMetadata to "
|
||||
f"be either XFormersMetadata or FlashAttentionMetadata.")
|
||||
f"Unsupported Attention backend {self.attn.backend} "
|
||||
"enum found. Expected the Attention backend to be "
|
||||
"FLASH_ATTN, FLASH_ATTN_VLLM_V1, XFORMERS or TORCH_SDPA.")
|
||||
|
||||
# We have to call torch.sdpa for prefill when using a
|
||||
# custom cross-attention mask. Because the mask is not a
|
||||
|
||||
Reference in New Issue
Block a user